gheim 0.1.1 → 0.1.3

package/dist/cjs/openai/index.cjs

@@ -276,8 +276,10 @@ class LocalDetector {
   chunkTokens;
   chunkOverlapTokens;
   batchSize;
+  actualDevice = null;
   _pipeline = null;
   _loadPromise = null;
+  _onProgress;
   constructor(opts = {}) {
     this.model = opts.model ?? DEFAULT_MODEL2;
     this.device = opts.device ?? "auto";
@@ -286,11 +288,13 @@ class LocalDetector {
     this.chunkTokens = opts.chunkTokens ?? DEFAULT_CHUNK_TOKENS;
     this.chunkOverlapTokens = opts.chunkOverlapTokens ?? DEFAULT_OVERLAP_TOKENS;
     this.batchSize = opts.batchSize ?? DEFAULT_BATCH_SIZE;
+    this._onProgress = opts.onProgress;
     if (this.chunkOverlapTokens >= this.chunkTokens) {
       throw new Error(`chunkOverlapTokens (${this.chunkOverlapTokens}) must be smaller than chunkTokens (${this.chunkTokens})`);
     }
     if (opts.pipelineInstance) {
       this._pipeline = opts.pipelineInstance;
+      this.actualDevice = this.device === "auto" ? null : this.device;
     }
   }
   async load() {
@@ -299,24 +303,70 @@ class LocalDetector {
     if (this._loadPromise)
       return this._loadPromise;
     this._loadPromise = (async () => {
-      let mod;
-      try {
-        mod = await import("@huggingface/transformers");
-      } catch (e) {
-        throw new Error("LocalDetector requires `@huggingface/transformers`. Install with: bun add @huggingface/transformers");
+      const mod = await loadTransformersJs();
+      const sequence = await this._planDeviceSequence();
+      let lastErr;
+      for (let i = 0;i < sequence.length; i++) {
+        const dev = sequence[i] ?? "wasm";
+        try {
+          const pipe = await mod.pipeline("token-classification", this.model, {
+            device: dev,
+            dtype: this.dtype,
+            progress_callback: this._onProgress ? (raw) => this._emitProgress(raw, dev) : undefined
+          });
+          this._pipeline = pipe;
+          this.actualDevice = dev;
+          return pipe;
+        } catch (e) {
+          lastErr = e;
+          if (i < sequence.length - 1)
+            continue;
+          throw e;
+        }
       }
-      const pipe = await mod.pipeline("token-classification", this.model, {
-        device: this.device,
-        dtype: this.dtype
-      });
-      this._pipeline = pipe;
-      return pipe;
+      throw lastErr ?? new Error("LocalDetector.load(): no candidate backends");
     })();
     return this._loadPromise;
   }
+  async _planDeviceSequence() {
+    if (this.device !== "auto")
+      return [this.device];
+    const seq = [];
+    const nav = typeof navigator !== "undefined" ? navigator : null;
+    if (nav?.gpu?.requestAdapter) {
+      try {
+        const adapter = await nav.gpu.requestAdapter();
+        if (adapter)
+          seq.push("webgpu");
+      } catch {}
+    }
+    seq.push("wasm");
+    return seq;
+  }
+  _emitProgress(raw, device) {
+    if (!this._onProgress)
+      return;
+    const event = {
+      phase: raw.status ?? "progress",
+      device
+    };
+    if (raw.file)
+      event.file = raw.file;
+    if (raw.status === "progress") {
+      if (raw.loaded != null)
+        event.loadedBytes = raw.loaded;
+      if (raw.total != null)
+        event.totalBytes = raw.total;
+      if (raw.progress != null) {
+        const norm = raw.progress > 1 ? raw.progress / 100 : raw.progress;
+        event.fraction = Math.max(0, Math.min(1, norm));
+      }
+    }
+    this._onProgress(event);
+  }
   async detect(text, opts) {
     if (opts?.model && opts.model !== this.model) {
-      throw new Error(`LocalDetector was constructed with model='${this.model}'; per-call model override ('${opts.model}') not supported.`);
+      throw new Error(`LocalDetector was constructed with model='${this.model}'; ` + `per-call model override ('${opts.model}') not supported.`);
     }
     if (!text)
       return [];
@@ -325,7 +375,7 @@ class LocalDetector {
   }
   async detectBatch(texts, opts) {
     if (opts?.model && opts.model !== this.model) {
-      throw new Error(`LocalDetector was constructed with model='${this.model}'; per-call model override ('${opts.model}') not supported.`);
+      throw new Error(`LocalDetector was constructed with model='${this.model}'; ` + `per-call model override ('${opts.model}') not supported.`);
     }
     if (texts.length === 0)
       return [];
@@ -486,102 +536,225 @@ function chunkByChars(text, chunkChars, overlapChars) {
 function normalizeWord(word) {
   return word.replace(/##/g, "").trim();
 }
-// src/detectors/index.ts
-function defaultDetector() {
-  const hasKey = typeof process !== "undefined" && Boolean(process.env?.GHEIM_API_KEY);
-  return hasKey ? new RemoteDetector : new LocalDetector;
-}
-
-// src/normalizers.ts
-var import_node_module = require("node:module");
-var _require = null;
-var _requireResolved = false;
-function _getRequire() {
-  if (_requireResolved)
-    return _require;
-  _requireResolved = true;
+async function loadTransformersJs() {
   try {
-    _require = import_node_module.createRequire("file:///home/runner/work/gheim/gheim/packages/gheim-js/src/normalizers.ts");
+    return await import("@huggingface/transformers");
   } catch {
-    _require = null;
+    throw new Error("LocalDetector requires `@huggingface/transformers`. Install with: bun add @huggingface/transformers");
   }
-  return _require;
 }
-function loadSync(name, errMsg) {
-  const req = _getRequire();
-  if (!req)
-    throw new Error(errMsg);
-  try {
-    return req(name);
-  } catch {
-    throw new Error(errMsg);
-  }
-}
-function loadPhoneLib() {
-  return loadSync("libphonenumber-js", "the e164 normalizer requires libphonenumber-js. " + "Install with: bun add libphonenumber-js");
-}
-function e164(opts = {}) {
-  const lib = loadPhoneLib();
-  return (s) => {
-    try {
-      const parsed = lib.parsePhoneNumberFromString(s, opts.region);
-      if (!parsed || !parsed.isValid())
-        return null;
-      return parsed.format("E.164");
-    } catch {
-      return null;
+// src/detectors/composite.ts
+function luhnOk(s) {
+  const digits = [];
+  for (const c of s) {
+    if (c >= "0" && c <= "9")
+      digits.push(Number(c));
+  }
+  if (digits.length < 13 || digits.length > 19)
+    return false;
+  let total = 0;
+  for (let i = 0;i < digits.length; i++) {
+    let d = digits[digits.length - 1 - i] ?? 0;
+    if (i % 2 === 1) {
+      d *= 2;
+      if (d > 9)
+        d -= 9;
     }
-  };
+    total += d;
+  }
+  return total % 10 === 0;
 }
-function loadChrono(opts) {
-  const mod = loadSync("chrono-node", "the isoDate normalizer requires chrono-node. " + "Install with: bun add chrono-node");
-  if (opts.locale) {
-    const localeParser = mod[opts.locale];
-    if (localeParser)
-      return localeParser;
-  }
-  return mod;
-}
-function isoDate(opts = {}) {
-  const parser = loadChrono(opts);
-  return (s) => {
-    const d = parser.parseDate(s);
-    if (!d)
-      return null;
-    return d.toISOString().slice(0, 10);
-  };
+function ibanChOk(raw) {
+  const s = raw.replace(/[\s-]/g, "");
+  if (s.length !== 21 || !s.startsWith("CH") || !/^\d+$/.test(s.slice(2))) {
+    return false;
+  }
+  const rearranged = s.slice(4) + s.slice(0, 4);
+  let numeric = "";
+  for (const c of rearranged) {
+    numeric += c >= "A" && c <= "Z" ? String(c.charCodeAt(0) - 55) : c;
+  }
+  let rem = 0;
+  for (const ch of numeric) {
+    rem = (rem * 10 + Number(ch)) % 97;
+  }
+  return rem === 1;
 }
-function germanTransliteration() {
-  const table = {
-    ä: "ae",
-    Ä: "ae",
-    ö: "oe",
-    Ö: "oe",
-    ü: "ue",
-    Ü: "ue",
-    ß: "ss"
-  };
-  return (s) => {
-    let out = "";
-    for (const ch of s.normalize("NFKC")) {
-      out += table[ch] ?? ch;
+function ahvOk(s) {
+  const digits = [];
+  for (const c of s) {
+    if (c >= "0" && c <= "9")
+      digits.push(Number(c));
+  }
+  if (digits.length !== 13)
+    return false;
+  if (digits[0] !== 7 || digits[1] !== 5 || digits[2] !== 6)
+    return false;
+  let total = 0;
+  for (let i = 0;i < 12; i++) {
+    total += (digits[i] ?? 0) * (i % 2 === 0 ? 1 : 3);
+  }
+  const check = (10 - total % 10) % 10;
+  return digits[12] === check;
+}
+function vatCheOk(s) {
+  const digits = [];
+  for (const c of s) {
+    if (c >= "0" && c <= "9")
+      digits.push(Number(c));
+  }
+  if (digits.length < 9)
+    return false;
+  const weights = [5, 4, 3, 2, 7, 6, 5, 4];
+  let total = 0;
+  for (let i = 0;i < 8; i++) {
+    total += (digits[i] ?? 0) * (weights[i] ?? 0);
+  }
+  let expected = 11 - total % 11;
+  if (expected === 11)
+    expected = 0;
+  else if (expected === 10)
+    return false;
+  return digits[8] === expected;
+}
+var REGEXES = [
+  {
+    label: "private_email",
+    pattern: /\b[\w._%+\-]+@[\w.\-]+\.[A-Za-z]{2,}\b/g
+  },
+  {
+    label: "private_url",
+    pattern: /https?:\/\/[^\s)>\]]+/g
+  },
+  {
+    label: "private_phone",
+    pattern: /(?:\+41|0041|0)\s?(?:\(0\)\s?)?\d{2}[\s\-/.]?\d{3}[\s\-/.]?\d{2}[\s\-/.]?\d{2}\b/g
+  },
+  {
+    label: "account_number",
+    pattern: /\bCH\d{2}(?:[\s-]?\d){17}\b/g,
+    validator: ibanChOk
+  },
+  {
+    label: "account_number",
+    pattern: /\b756\.\d{4}\.\d{4}\.\d{2}\b/g,
+    validator: ahvOk
+  },
+  {
+    label: "account_number",
+    pattern: /\b756\d{10}\b/g,
+    validator: ahvOk
+  },
+  {
+    label: "account_number",
+    pattern: /\bCHE-\d{3}\.\d{3}\.\d{3}(?:\s+(?:MWST|TVA|IVA))?\b/g,
+    validator: vatCheOk
+  },
+  {
+    label: "account_number",
+    pattern: /\b(?:\d[\s-]?){15,16}\d\b/g,
+    validator: luhnOk
+  },
+  {
+    label: "private_date",
+    pattern: /\b(?:\d{1,2}[./]\d{1,2}[./]\d{4}|\d{4}[./-]\d{1,2}[./-]\d{1,2})\b/g
+  },
+  {
+    label: "secret",
+    pattern: /\bsk-(?:proj-)?[A-Za-z0-9_\-]{20,}\b/g
+  },
+  {
+    label: "secret",
+    pattern: /\bghp_[A-Za-z0-9]{36}\b/g
+  },
+  {
+    label: "secret",
+    pattern: /\bAKIA[A-Z0-9]{16}\b/g
+  }
+];
+var URL_TRAIL_TRIM = '.,;:!?")>]';
+function trimUrl(text, start, end) {
+  while (end > start && URL_TRAIL_TRIM.includes(text[end - 1] ?? ""))
+    end--;
+  return [start, end];
+}
+function findRegexSpans(text) {
+  const candidates = [];
+  for (const { label, pattern, validator } of REGEXES) {
+    pattern.lastIndex = 0;
+    let m;
+    while ((m = pattern.exec(text)) !== null) {
+      let start = m.index;
+      let end = m.index + m[0].length;
+      if (label === "private_url")
+        [start, end] = trimUrl(text, start, end);
+      const value = text.slice(start, end);
+      if (!value.trim())
+        continue;
+      if (validator && !validator(value))
+        continue;
+      candidates.push({
+        start,
+        end,
+        label,
+        text: value,
+        score: 1
+      });
     }
-    return out.toLowerCase().split(/\s+/).filter(Boolean).join(" ");
-  };
+  }
+  candidates.sort((a, b) => a.start !== b.start ? a.start - b.start : b.end - b.start - (a.end - a.start));
+  const out = [];
+  let lastEnd = -1;
+  for (const sp of candidates) {
+    if (sp.start < lastEnd)
+      continue;
+    out.push(sp);
+    lastEnd = sp.end;
+  }
+  return out;
 }
-var BUILTINS = {
-  e164: () => e164(),
-  iso_date: () => isoDate(),
-  german: () => germanTransliteration()
-};
-function resolveNormalizer(spec) {
-  if (typeof spec === "function")
-    return spec;
-  const factory = BUILTINS[spec];
-  if (!factory) {
-    throw new Error(`unknown built-in normalizer: ${JSON.stringify(spec)}. ` + `Available: ${Object.keys(BUILTINS).join(", ")}, or pass a callable.`);
+function maskSpans(text, spans) {
+  if (spans.length === 0)
+    return text;
+  const chars = [...text];
+  for (const sp of spans) {
+    for (let i = sp.start;i < sp.end && i < chars.length; i++) {
+      chars[i] = " ";
+    }
   }
-  return factory();
+  return chars.join("");
+}
+
+class CompositeDetector {
+  model;
+  constructor(opts) {
+    if (!opts || !opts.model) {
+      throw new Error("CompositeDetector requires { model: Detector } — pass any object with " + "a .detect(text) method (e.g. new LocalDetector({...})).");
+    }
+    this.model = opts.model;
+  }
+  async detect(text, opts) {
+    if (!text)
+      return [];
+    const regexSpans = findRegexSpans(text);
+    const masked = maskSpans(text, regexSpans);
+    const modelSpans = await this.model.detect(masked, opts);
+    const out = [...regexSpans];
+    const regexRanges = regexSpans.map((s) => [s.start, s.end]);
+    for (const sp of modelSpans) {
+      const overlaps = regexRanges.some(([rs, re]) => sp.start < re && rs < sp.end);
+      if (overlaps)
+        continue;
+      out.push(sp);
+    }
+    out.sort((a, b) => a.start !== b.start ? a.start - b.start : b.end - b.start - (a.end - a.start));
+    return out;
+  }
+}
+// src/detectors/index.ts
+function defaultDetector() {
+  const hasKey = typeof process !== "undefined" && Boolean(process.env?.GHEIM_API_KEY);
+  return hasKey ? new RemoteDetector : new LocalDetector;
 }
 
 // src/core/session.ts
@@ -596,8 +769,8 @@ class Session {
   resolvedNormalizers = {};
   constructor(opts = {}) {
     if (opts.normalizers) {
-      for (const [label, spec] of Object.entries(opts.normalizers)) {
-        this.resolvedNormalizers[label] = resolveNormalizer(spec);
+      for (const [label, fn] of Object.entries(opts.normalizers)) {
+        this.resolvedNormalizers[label] = fn;
       }
     }
   }
@@ -621,8 +794,11 @@ class Session {
     return sentinel;
   }
   applySpans(text, spans) {
+    return this.applySpansDetailed(text, spans).redacted;
+  }
+  applySpansDetailed(text, spans) {
     if (spans.length === 0)
-      return text;
+      return { redacted: text, merged: [] };
     const sorted = [...spans].sort((a, b) => a.start - b.start || b.end - b.start - (a.end - a.start));
     const merged = mergeAdjacent(text, sorted);
     const out = [];
@@ -638,7 +814,7 @@ class Session {
       cursor = end;
     }
     out.push(text.slice(cursor));
-    return out.join("");
+    return { redacted: out.join(""), merged };
   }
   restore(text) {
     if (Object.keys(this.mapping).length === 0)
@@ -871,6 +1047,16 @@ function restoreChatResponse(response, session) {
     if (msg && typeof msg.content === "string") {
       msg.content = session.restore(msg.content);
     }
+    if (msg && typeof msg.reasoning === "string") {
+      msg.reasoning = session.restore(msg.reasoning);
+    }
+    if (msg && Array.isArray(msg.reasoning_details)) {
+      for (const r of msg.reasoning_details) {
+        if (r && typeof r.text === "string") {
+          r.text = session.restore(r.text);
+        }
+      }
+    }
     for (const tc of msg?.tool_calls ?? []) {
       const fn = tc.function;
       if (fn && typeof fn.arguments === "string") {
@@ -885,6 +1071,16 @@ function patchChatChunk(chunk, buf) {
     if (typeof delta.content === "string") {
       delta.content = buf.feed(delta.content);
     }
+    if (typeof delta.reasoning === "string") {
+      delta.reasoning = buf.feed(delta.reasoning);
+    }
+    if (Array.isArray(delta.reasoning_details)) {
+      for (const r of delta.reasoning_details) {
+        if (r && typeof r.text === "string") {
+          r.text = buf.feed(r.text);
+        }
+      }
+    }
   }
 }
 function makeChatTailChunk(template, content) {
@@ -1211,9 +1407,20 @@ class OpenAI {
   gheimStrict;
   _clientPromise;
   constructor(opts = {}) {
-    const detector = opts.gheimDetector ?? defaultDetector();
-    this.gheimStrict = opts.gheimStrict ?? true;
-    this._clientPromise = opts.openaiClient ? Promise.resolve(opts.openaiClient) : loadOpenAI(opts.clientOptions);
+    const {
+      gheimDetector,
+      gheimStrict,
+      openaiClient,
+      clientOptions,
+      ...passthrough
+    } = opts;
+    const detector = gheimDetector ?? defaultDetector();
+    this.gheimStrict = gheimStrict ?? true;
+    const innerOpts = {
+      ...clientOptions ?? {},
+      ...passthrough
+    };
+    this._clientPromise = openaiClient ? Promise.resolve(openaiClient) : loadOpenAI(innerOpts);
     this._clientPromise.then((c) => {
       this.raw = c;
     });