gheim 0.1.0 → 0.1.2

package/dist/cjs/index.cjs

@@ -41,6 +41,7 @@ var exports_src = {};
 __export(exports_src, {
   renderSentinel: () => renderSentinel,
   parseSentinel: () => parseSentinel,
+  mergeAdjacent: () => mergeAdjacent,
   labelTag: () => labelTag,
   isPossibleSentinelPrefix: () => isPossibleSentinelPrefix,
   defaultDetector: () => defaultDetector,
@@ -96,98 +97,6 @@ function isPossibleSentinelPrefix(buffer) {
   return PARTIAL_SENTINEL_RE.test(buffer);
 }
 
-// src/normalizers.ts
-var import_node_module = require("node:module");
-var _require = null;
-var _requireResolved = false;
-function _getRequire() {
-  if (_requireResolved)
-    return _require;
-  _requireResolved = true;
-  try {
-    _require = import_node_module.createRequire("file:///home/joelbarmettler/projects/gheim/packages/gheim-js/src/normalizers.ts");
-  } catch {
-    _require = null;
-  }
-  return _require;
-}
-function loadSync(name, errMsg) {
-  const req = _getRequire();
-  if (!req)
-    throw new Error(errMsg);
-  try {
-    return req(name);
-  } catch {
-    throw new Error(errMsg);
-  }
-}
-function loadPhoneLib() {
-  return loadSync("libphonenumber-js", "the e164 normalizer requires libphonenumber-js. " + "Install with: bun add libphonenumber-js");
-}
-function e164(opts = {}) {
-  const lib = loadPhoneLib();
-  return (s) => {
-    try {
-      const parsed = lib.parsePhoneNumberFromString(s, opts.region);
-      if (!parsed || !parsed.isValid())
-        return null;
-      return parsed.format("E.164");
-    } catch {
-      return null;
-    }
-  };
-}
-function loadChrono(opts) {
-  const mod = loadSync("chrono-node", "the isoDate normalizer requires chrono-node. " + "Install with: bun add chrono-node");
-  if (opts.locale) {
-    const localeParser = mod[opts.locale];
-    if (localeParser)
-      return localeParser;
-  }
-  return mod;
-}
-function isoDate(opts = {}) {
-  const parser = loadChrono(opts);
-  return (s) => {
-    const d = parser.parseDate(s);
-    if (!d)
-      return null;
-    return d.toISOString().slice(0, 10);
-  };
-}
-function germanTransliteration() {
-  const table = {
-    ä: "ae",
-    Ä: "ae",
-    ö: "oe",
-    Ö: "oe",
-    ü: "ue",
-    Ü: "ue",
-    ß: "ss"
-  };
-  return (s) => {
-    let out = "";
-    for (const ch of s.normalize("NFKC")) {
-      out += table[ch] ?? ch;
-    }
-    return out.toLowerCase().split(/\s+/).filter(Boolean).join(" ");
-  };
-}
-var BUILTINS = {
-  e164: () => e164(),
-  iso_date: () => isoDate(),
-  german: () => germanTransliteration()
-};
-function resolveNormalizer(spec) {
-  if (typeof spec === "function")
-    return spec;
-  const factory = BUILTINS[spec];
-  if (!factory) {
-    throw new Error(`unknown built-in normalizer: ${JSON.stringify(spec)}. ` + `Available: ${Object.keys(BUILTINS).join(", ")}, or pass a callable.`);
-  }
-  return factory();
-}
-
 // src/core/session.ts
 function normalizeSurface(s) {
   return s.normalize("NFKC").toLowerCase().split(/\s+/).filter(Boolean).join(" ");
@@ -200,8 +109,8 @@ class Session {
   resolvedNormalizers = {};
   constructor(opts = {}) {
     if (opts.normalizers) {
-      for (const [label, spec] of Object.entries(opts.normalizers)) {
-        this.resolvedNormalizers[label] = resolveNormalizer(spec);
+      for (const [label, fn] of Object.entries(opts.normalizers)) {
+        this.resolvedNormalizers[label] = fn;
       }
     }
   }
@@ -225,8 +134,11 @@ class Session {
     return sentinel;
   }
   applySpans(text, spans) {
+    return this.applySpansDetailed(text, spans).redacted;
+  }
+  applySpansDetailed(text, spans) {
     if (spans.length === 0)
-      return text;
+      return { redacted: text, merged: [] };
     const sorted = [...spans].sort((a, b) => a.start - b.start || b.end - b.start - (a.end - a.start));
     const merged = mergeAdjacent(text, sorted);
     const out = [];
@@ -242,7 +154,7 @@ class Session {
       cursor = end;
     }
     out.push(text.slice(cursor));
-    return out.join("");
+    return { redacted: out.join(""), merged };
   }
   restore(text) {
     if (Object.keys(this.mapping).length === 0)
@@ -440,8 +352,10 @@ class LocalDetector {
   chunkTokens;
   chunkOverlapTokens;
   batchSize;
+  actualDevice = null;
   _pipeline = null;
   _loadPromise = null;
+  _onProgress;
   constructor(opts = {}) {
     this.model = opts.model ?? DEFAULT_MODEL2;
     this.device = opts.device ?? "auto";
@@ -450,11 +364,13 @@ class LocalDetector {
     this.chunkTokens = opts.chunkTokens ?? DEFAULT_CHUNK_TOKENS;
     this.chunkOverlapTokens = opts.chunkOverlapTokens ?? DEFAULT_OVERLAP_TOKENS;
     this.batchSize = opts.batchSize ?? DEFAULT_BATCH_SIZE;
+    this._onProgress = opts.onProgress;
     if (this.chunkOverlapTokens >= this.chunkTokens) {
       throw new Error(`chunkOverlapTokens (${this.chunkOverlapTokens}) must be smaller than chunkTokens (${this.chunkTokens})`);
     }
     if (opts.pipelineInstance) {
       this._pipeline = opts.pipelineInstance;
+      this.actualDevice = this.device === "auto" ? null : this.device;
     }
   }
   async load() {
@@ -463,24 +379,70 @@ class LocalDetector {
     if (this._loadPromise)
       return this._loadPromise;
     this._loadPromise = (async () => {
-      let mod;
-      try {
-        mod = await import("@huggingface/transformers");
-      } catch (e) {
-        throw new Error("LocalDetector requires `@huggingface/transformers`. Install with: bun add @huggingface/transformers");
+      const mod = await loadTransformersJs();
+      const sequence = await this._planDeviceSequence();
+      let lastErr;
+      for (let i = 0;i < sequence.length; i++) {
+        const dev = sequence[i] ?? "wasm";
+        try {
+          const pipe = await mod.pipeline("token-classification", this.model, {
+            device: dev,
+            dtype: this.dtype,
+            progress_callback: this._onProgress ? (raw) => this._emitProgress(raw, dev) : undefined
+          });
+          this._pipeline = pipe;
+          this.actualDevice = dev;
+          return pipe;
+        } catch (e) {
+          lastErr = e;
+          if (i < sequence.length - 1)
+            continue;
+          throw e;
+        }
       }
-      const pipe = await mod.pipeline("token-classification", this.model, {
-        device: this.device,
-        dtype: this.dtype
-      });
-      this._pipeline = pipe;
-      return pipe;
+      throw lastErr ?? new Error("LocalDetector.load(): no candidate backends");
     })();
     return this._loadPromise;
   }
+  async _planDeviceSequence() {
+    if (this.device !== "auto")
+      return [this.device];
+    const seq = [];
+    const nav = typeof navigator !== "undefined" ? navigator : null;
+    if (nav?.gpu?.requestAdapter) {
+      try {
+        const adapter = await nav.gpu.requestAdapter();
+        if (adapter)
+          seq.push("webgpu");
+      } catch {}
+    }
+    seq.push("wasm");
+    return seq;
+  }
+  _emitProgress(raw, device) {
+    if (!this._onProgress)
+      return;
+    const event = {
+      phase: raw.status ?? "progress",
+      device
+    };
+    if (raw.file)
+      event.file = raw.file;
+    if (raw.status === "progress") {
+      if (raw.loaded != null)
+        event.loadedBytes = raw.loaded;
+      if (raw.total != null)
+        event.totalBytes = raw.total;
+      if (raw.progress != null) {
+        const norm = raw.progress > 1 ? raw.progress / 100 : raw.progress;
+        event.fraction = Math.max(0, Math.min(1, norm));
+      }
+    }
+    this._onProgress(event);
+  }
   async detect(text, opts) {
     if (opts?.model && opts.model !== this.model) {
-      throw new Error(`LocalDetector was constructed with model='${this.model}'; per-call model override ('${opts.model}') not supported.`);
+      throw new Error(`LocalDetector was constructed with model='${this.model}'; ` + `per-call model override ('${opts.model}') not supported.`);
     }
     if (!text)
       return [];
@@ -489,7 +451,7 @@ class LocalDetector {
   }
   async detectBatch(texts, opts) {
     if (opts?.model && opts.model !== this.model) {
-      throw new Error(`LocalDetector was constructed with model='${this.model}'; per-call model override ('${opts.model}') not supported.`);
+      throw new Error(`LocalDetector was constructed with model='${this.model}'; ` + `per-call model override ('${opts.model}') not supported.`);
     }
     if (texts.length === 0)
       return [];
@@ -650,6 +612,13 @@ function chunkByChars(text, chunkChars, overlapChars) {
 function normalizeWord(word) {
   return word.replace(/##/g, "").trim();
 }
+async function loadTransformersJs() {
+  try {
+    return await import("@huggingface/transformers");
+  } catch {
+    throw new Error("LocalDetector requires `@huggingface/transformers`. Install with: bun add @huggingface/transformers");
+  }
+}
 // src/detectors/index.ts
 function defaultDetector() {
   const hasKey = typeof process !== "undefined" && Boolean(process.env?.GHEIM_API_KEY);
@@ -704,4 +673,4 @@ async function* deanonymizeStream(chunks, session) {
 }
 
 // src/index.ts
-var VERSION = "0.1.0";
+var VERSION = "0.1.2";

package/dist/cjs/openai/index.cjs

@@ -276,8 +276,10 @@ class LocalDetector {
   chunkTokens;
   chunkOverlapTokens;
   batchSize;
+  actualDevice = null;
   _pipeline = null;
   _loadPromise = null;
+  _onProgress;
   constructor(opts = {}) {
     this.model = opts.model ?? DEFAULT_MODEL2;
     this.device = opts.device ?? "auto";
@@ -286,11 +288,13 @@ class LocalDetector {
     this.chunkTokens = opts.chunkTokens ?? DEFAULT_CHUNK_TOKENS;
     this.chunkOverlapTokens = opts.chunkOverlapTokens ?? DEFAULT_OVERLAP_TOKENS;
     this.batchSize = opts.batchSize ?? DEFAULT_BATCH_SIZE;
+    this._onProgress = opts.onProgress;
     if (this.chunkOverlapTokens >= this.chunkTokens) {
       throw new Error(`chunkOverlapTokens (${this.chunkOverlapTokens}) must be smaller than chunkTokens (${this.chunkTokens})`);
     }
     if (opts.pipelineInstance) {
       this._pipeline = opts.pipelineInstance;
+      this.actualDevice = this.device === "auto" ? null : this.device;
     }
   }
   async load() {
@@ -299,24 +303,70 @@ class LocalDetector {
     if (this._loadPromise)
       return this._loadPromise;
     this._loadPromise = (async () => {
-      let mod;
-      try {
-        mod = await import("@huggingface/transformers");
-      } catch (e) {
-        throw new Error("LocalDetector requires `@huggingface/transformers`. Install with: bun add @huggingface/transformers");
+      const mod = await loadTransformersJs();
+      const sequence = await this._planDeviceSequence();
+      let lastErr;
+      for (let i = 0;i < sequence.length; i++) {
+        const dev = sequence[i] ?? "wasm";
+        try {
+          const pipe = await mod.pipeline("token-classification", this.model, {
+            device: dev,
+            dtype: this.dtype,
+            progress_callback: this._onProgress ? (raw) => this._emitProgress(raw, dev) : undefined
+          });
+          this._pipeline = pipe;
+          this.actualDevice = dev;
+          return pipe;
+        } catch (e) {
+          lastErr = e;
+          if (i < sequence.length - 1)
+            continue;
+          throw e;
+        }
       }
-      const pipe = await mod.pipeline("token-classification", this.model, {
-        device: this.device,
-        dtype: this.dtype
-      });
-      this._pipeline = pipe;
-      return pipe;
+      throw lastErr ?? new Error("LocalDetector.load(): no candidate backends");
     })();
     return this._loadPromise;
   }
+  async _planDeviceSequence() {
+    if (this.device !== "auto")
+      return [this.device];
+    const seq = [];
+    const nav = typeof navigator !== "undefined" ? navigator : null;
+    if (nav?.gpu?.requestAdapter) {
+      try {
+        const adapter = await nav.gpu.requestAdapter();
+        if (adapter)
+          seq.push("webgpu");
+      } catch {}
+    }
+    seq.push("wasm");
+    return seq;
+  }
+  _emitProgress(raw, device) {
+    if (!this._onProgress)
+      return;
+    const event = {
+      phase: raw.status ?? "progress",
+      device
+    };
+    if (raw.file)
+      event.file = raw.file;
+    if (raw.status === "progress") {
+      if (raw.loaded != null)
+        event.loadedBytes = raw.loaded;
+      if (raw.total != null)
+        event.totalBytes = raw.total;
+      if (raw.progress != null) {
+        const norm = raw.progress > 1 ? raw.progress / 100 : raw.progress;
+        event.fraction = Math.max(0, Math.min(1, norm));
+      }
+    }
+    this._onProgress(event);
+  }
   async detect(text, opts) {
     if (opts?.model && opts.model !== this.model) {
-      throw new Error(`LocalDetector was constructed with model='${this.model}'; per-call model override ('${opts.model}') not supported.`);
+      throw new Error(`LocalDetector was constructed with model='${this.model}'; ` + `per-call model override ('${opts.model}') not supported.`);
     }
     if (!text)
       return [];
@@ -325,7 +375,7 @@ class LocalDetector {
   }
   async detectBatch(texts, opts) {
     if (opts?.model && opts.model !== this.model) {
-      throw new Error(`LocalDetector was constructed with model='${this.model}'; per-call model override ('${opts.model}') not supported.`);
+      throw new Error(`LocalDetector was constructed with model='${this.model}'; ` + `per-call model override ('${opts.model}') not supported.`);
     }
     if (texts.length === 0)
       return [];
@@ -486,102 +536,17 @@ function chunkByChars(text, chunkChars, overlapChars) {
 function normalizeWord(word) {
   return word.replace(/##/g, "").trim();
 }
-// src/detectors/index.ts
-function defaultDetector() {
-  const hasKey = typeof process !== "undefined" && Boolean(process.env?.GHEIM_API_KEY);
-  return hasKey ? new RemoteDetector : new LocalDetector;
-}
-
-// src/normalizers.ts
-var import_node_module = require("node:module");
-var _require = null;
-var _requireResolved = false;
-function _getRequire() {
-  if (_requireResolved)
-    return _require;
-  _requireResolved = true;
-  try {
-    _require = import_node_module.createRequire("file:///home/joelbarmettler/projects/gheim/packages/gheim-js/src/normalizers.ts");
-  } catch {
-    _require = null;
-  }
-  return _require;
-}
-function loadSync(name, errMsg) {
-  const req = _getRequire();
-  if (!req)
-    throw new Error(errMsg);
+async function loadTransformersJs() {
   try {
-    return req(name);
+    return await import("@huggingface/transformers");
   } catch {
-    throw new Error(errMsg);
+    throw new Error("LocalDetector requires `@huggingface/transformers`. Install with: bun add @huggingface/transformers");
   }
 }
-function loadPhoneLib() {
-  return loadSync("libphonenumber-js", "the e164 normalizer requires libphonenumber-js. " + "Install with: bun add libphonenumber-js");
-}
-function e164(opts = {}) {
-  const lib = loadPhoneLib();
-  return (s) => {
-    try {
-      const parsed = lib.parsePhoneNumberFromString(s, opts.region);
-      if (!parsed || !parsed.isValid())
-        return null;
-      return parsed.format("E.164");
-    } catch {
-      return null;
-    }
-  };
-}
-function loadChrono(opts) {
-  const mod = loadSync("chrono-node", "the isoDate normalizer requires chrono-node. " + "Install with: bun add chrono-node");
-  if (opts.locale) {
-    const localeParser = mod[opts.locale];
-    if (localeParser)
-      return localeParser;
-  }
-  return mod;
-}
-function isoDate(opts = {}) {
-  const parser = loadChrono(opts);
-  return (s) => {
-    const d = parser.parseDate(s);
-    if (!d)
-      return null;
-    return d.toISOString().slice(0, 10);
-  };
-}
-function germanTransliteration() {
-  const table = {
-    ä: "ae",
-    Ä: "ae",
-    ö: "oe",
-    Ö: "oe",
-    ü: "ue",
-    Ü: "ue",
-    ß: "ss"
-  };
-  return (s) => {
-    let out = "";
-    for (const ch of s.normalize("NFKC")) {
-      out += table[ch] ?? ch;
-    }
-    return out.toLowerCase().split(/\s+/).filter(Boolean).join(" ");
-  };
-}
-var BUILTINS = {
-  e164: () => e164(),
-  iso_date: () => isoDate(),
-  german: () => germanTransliteration()
-};
-function resolveNormalizer(spec) {
-  if (typeof spec === "function")
-    return spec;
-  const factory = BUILTINS[spec];
-  if (!factory) {
-    throw new Error(`unknown built-in normalizer: ${JSON.stringify(spec)}. ` + `Available: ${Object.keys(BUILTINS).join(", ")}, or pass a callable.`);
-  }
-  return factory();
+// src/detectors/index.ts
+function defaultDetector() {
+  const hasKey = typeof process !== "undefined" && Boolean(process.env?.GHEIM_API_KEY);
+  return hasKey ? new RemoteDetector : new LocalDetector;
 }
 
 // src/core/session.ts
@@ -596,8 +561,8 @@ class Session {
   resolvedNormalizers = {};
   constructor(opts = {}) {
     if (opts.normalizers) {
-      for (const [label, spec] of Object.entries(opts.normalizers)) {
-        this.resolvedNormalizers[label] = resolveNormalizer(spec);
+      for (const [label, fn] of Object.entries(opts.normalizers)) {
+        this.resolvedNormalizers[label] = fn;
       }
     }
   }
@@ -621,8 +586,11 @@ class Session {
     return sentinel;
   }
   applySpans(text, spans) {
+    return this.applySpansDetailed(text, spans).redacted;
+  }
+  applySpansDetailed(text, spans) {
     if (spans.length === 0)
-      return text;
+      return { redacted: text, merged: [] };
     const sorted = [...spans].sort((a, b) => a.start - b.start || b.end - b.start - (a.end - a.start));
     const merged = mergeAdjacent(text, sorted);
     const out = [];
@@ -638,7 +606,7 @@ class Session {
       cursor = end;
     }
     out.push(text.slice(cursor));
-    return out.join("");
+    return { redacted: out.join(""), merged };
   }
   restore(text) {
     if (Object.keys(this.mapping).length === 0)
@@ -871,6 +839,16 @@ function restoreChatResponse(response, session) {
     if (msg && typeof msg.content === "string") {
       msg.content = session.restore(msg.content);
     }
+    if (msg && typeof msg.reasoning === "string") {
+      msg.reasoning = session.restore(msg.reasoning);
+    }
+    if (msg && Array.isArray(msg.reasoning_details)) {
+      for (const r of msg.reasoning_details) {
+        if (r && typeof r.text === "string") {
+          r.text = session.restore(r.text);
+        }
+      }
+    }
     for (const tc of msg?.tool_calls ?? []) {
       const fn = tc.function;
       if (fn && typeof fn.arguments === "string") {
@@ -885,6 +863,16 @@ function patchChatChunk(chunk, buf) {
     if (typeof delta.content === "string") {
       delta.content = buf.feed(delta.content);
     }
+    if (typeof delta.reasoning === "string") {
+      delta.reasoning = buf.feed(delta.reasoning);
+    }
+    if (Array.isArray(delta.reasoning_details)) {
+      for (const r of delta.reasoning_details) {
+        if (r && typeof r.text === "string") {
+          r.text = buf.feed(r.text);
+        }
+      }
+    }
   }
 }
 function makeChatTailChunk(template, content) {
@@ -1211,9 +1199,20 @@ class OpenAI {
   gheimStrict;
   _clientPromise;
   constructor(opts = {}) {
-    const detector = opts.gheimDetector ?? defaultDetector();
-    this.gheimStrict = opts.gheimStrict ?? true;
-    this._clientPromise = opts.openaiClient ? Promise.resolve(opts.openaiClient) : loadOpenAI(opts.clientOptions);
+    const {
+      gheimDetector,
+      gheimStrict,
+      openaiClient,
+      clientOptions,
+      ...passthrough
+    } = opts;
+    const detector = gheimDetector ?? defaultDetector();
+    this.gheimStrict = gheimStrict ?? true;
+    const innerOpts = {
+      ...clientOptions ?? {},
+      ...passthrough
+    };
+    this._clientPromise = openaiClient ? Promise.resolve(openaiClient) : loadOpenAI(innerOpts);
     this._clientPromise.then((c) => {
       this.raw = c;
     });