gheim 0.1.8 → 0.1.10

package/dist/cjs/openai/index.cjs

@@ -39,13 +39,13 @@ var __export = (target, all) => {
 // src/openai/index.ts
 var exports_openai = {};
 __export(exports_openai, {
-  deanonymizeOpenAIStream: () => deanonymizeOpenAIStream,
-  deanonymizeOpenAIResponse: () => deanonymizeOpenAIResponse,
-  anonymizeOpenAIMessages: () => anonymizeOpenAIMessages,
-  STRICT_REASONS: () => STRICT_REASONS,
-  OpenAI: () => OpenAI,
+  AsyncOpenAI: () => AsyncOpenAI,
   GheimStrictError: () => GheimStrictError,
-  AsyncOpenAI: () => AsyncOpenAI
+  OpenAI: () => OpenAI,
+  STRICT_REASONS: () => STRICT_REASONS,
+  anonymizeOpenAIMessages: () => anonymizeOpenAIMessages,
+  deanonymizeOpenAIResponse: () => deanonymizeOpenAIResponse,
+  deanonymizeOpenAIStream: () => deanonymizeOpenAIStream
 });
 module.exports = __toCommonJS(exports_openai);
 
@@ -207,10 +207,40 @@ function trimSpan(text, start, end) {
     return null;
   return { start, end };
 }
+var WORD_CHAR = /[\p{L}\p{N}_]/u;
+var isWordChar = (ch) => ch !== undefined && WORD_CHAR.test(ch);
+function snapToWords(text, spans) {
+  const snapped = spans.map((sp) => {
+    let { start, end } = sp;
+    while (start > 0 && isWordChar(text[start - 1]) && isWordChar(text[start]))
+      start -= 1;
+    while (end < text.length && isWordChar(text[end - 1]) && isWordChar(text[end]))
+      end += 1;
+    return start === sp.start && end === sp.end ? sp : { ...sp, start, end, text: text.slice(start, end) };
+  });
+  snapped.sort((a, b) => a.start - b.start || a.end - b.end);
+  const merged = [];
+  const lastByLabel = new Map;
+  for (const sp of snapped) {
+    const i = lastByLabel.get(sp.label);
+    const prev = i === undefined ? undefined : merged[i];
+    if (i !== undefined && prev !== undefined && sp.start <= prev.end) {
+      const end = Math.max(prev.end, sp.end);
+      const wPrev = prev.end - prev.start;
+      const wSp = sp.end - sp.start;
+      const score = ((prev.score ?? 1) * wPrev + (sp.score ?? 1) * wSp) / (wPrev + wSp);
+      merged[i] = { ...prev, end, text: text.slice(prev.start, end), score };
+    } else {
+      lastByLabel.set(sp.label, merged.length);
+      merged.push(sp);
+    }
+  }
+  return merged.sort((a, b) => a.start - b.start || a.end - b.end);
+}
 
 // src/detectors/remote.ts
 var DEFAULT_BASE_URL = "https://api.gheim.ch";
-var DEFAULT_MODEL = "openai/privacy-filter";
+var DEFAULT_MODEL = "joelbarmettler/gheim-ch-560m";
 
 class RemoteDetector {
   baseUrl;
@@ -455,10 +485,7 @@ class LocalDetector {
         });
       }
     }
-    for (const spans of out) {
-      spans.sort((a, b) => a.start - b.start || a.end - b.end);
-    }
-    return out;
+    return out.map((spans, i) => snapToWords(texts[i] ?? "", spans));
   }
   async chunkText(text, tokenizer) {
     if (tokenizer) {
@@ -735,21 +762,136 @@ function maskSpans(text, spans) {
   return chars.join("");
 }
 
+class CustomPattern {
+  label;
+  group;
+  name;
+  regex;
+  validator;
+  constructor(opts) {
+    if (!/^[A-Za-z][A-Za-z0-9_]*$/.test(opts.label)) {
+      throw new Error(`CustomPattern label ${JSON.stringify(opts.label)} must start with a letter and contain ` + "only letters, digits and underscores (it becomes a <TAG_1> placeholder).");
+    }
+    this.label = opts.label;
+    this.name = opts.name ?? opts.label;
+    this.group = opts.group ?? 0;
+    this.validator = opts.validator;
+    const source = typeof opts.regex === "string" ? opts.regex : opts.regex.source;
+    const base = typeof opts.regex === "string" ? opts.flags ?? "i" : opts.regex.flags;
+    const flags = [...new Set([...base, "g", "d"])].join("");
+    this.regex = new RegExp(source, flags);
+    const groups = (new RegExp(`${source}|`, flags.replace(/[gd]/g, "")).exec("") ?? []).length - 1;
+    if (!Number.isInteger(this.group) || this.group < 0 || this.group > groups) {
+      throw new Error(`CustomPattern ${JSON.stringify(this.name)}: group ${this.group} does not exist ` + `(the pattern has ${groups}).`);
+    }
+  }
+  find(text) {
+    const out = [];
+    this.regex.lastIndex = 0;
+    let m;
+    while ((m = this.regex.exec(text)) !== null) {
+      if (m[0] === "") {
+        this.regex.lastIndex++;
+        continue;
+      }
+      const range = m.indices[this.group];
+      if (!range || range[0] === range[1])
+        continue;
+      const [start, end] = range;
+      const value = text.slice(start, end);
+      if (this.validator && !this.validator(value))
+        continue;
+      out.push({ start, end, label: this.label, text: value, score: 1 });
+    }
+    return out;
+  }
+}
+var NAME_PART = /[A-ZÀ-Ý][\p{L}\p{N}_'’-]{3,}/gu;
+var HONORIFICS = new Set([
+  "frau",
+  "herr",
+  "herrn",
+  "madame",
+  "monsieur",
+  "signora",
+  "signor",
+  "signore",
+  "dott",
+  "dottor",
+  "dottoressa",
+  "doktor",
+  "docteur",
+  "prof",
+  "professor",
+  "mister",
+  "miss",
+  "mrs"
+]);
+function escapeRegExp(s) {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+function propagateNames(text, spans, surnames) {
+  const names = new Set;
+  for (const sp of spans) {
+    if (sp.label !== "private_person")
+      continue;
+    const value = text.slice(sp.start, sp.end).trim();
+    if (value.length >= 4)
+      names.add(value);
+    if (surnames) {
+      for (const part of value.match(NAME_PART) ?? []) {
+        if (!HONORIFICS.has(part.toLowerCase()))
+          names.add(part);
+      }
+    }
+  }
+  let result = [...spans];
+  for (const name of [...names].sort((a, b) => b.length - a.length)) {
+    const re = new RegExp(`(?<![\\p{L}\\p{N}_])${escapeRegExp(name)}(?![\\p{L}\\p{N}_])`, "gu");
+    let m;
+    while ((m = re.exec(text)) !== null) {
+      const start = m.index;
+      const end = start + m[0].length;
+      const hits = result.filter((sp) => start < sp.end && sp.start < end);
+      if (hits.some((sp) => sp.start <= start && end <= sp.end))
+        continue;
+      if (hits.some((sp) => sp.start < start || end < sp.end))
+        continue;
+      result = result.filter((sp) => !hits.includes(sp));
+      result.push({ start, end, label: "private_person", text: m[0], score: 1 });
+    }
+  }
+  return result;
+}
+
 class CompositeDetector {
   model;
+  patterns;
+  propagateNames;
+  propagateSurnames;
   constructor(opts) {
     if (!opts || !opts.model) {
       throw new Error("CompositeDetector requires { model: Detector } — pass any object with " + "a .detect(text) method (e.g. new LocalDetector({...})).");
     }
     this.model = opts.model;
+    this.patterns = opts.patterns ?? [];
+    this.propagateNames = opts.propagateNames ?? true;
+    this.propagateSurnames = opts.propagateSurnames ?? false;
   }
   async detect(text, opts) {
     if (!text)
       return [];
     const regexSpans = findRegexSpans(text);
+    const extra = this.patterns.flatMap((p) => p.find(text));
+    extra.sort((a, b) => a.start !== b.start ? a.start - b.start : b.end - b.start - (a.end - a.start));
+    for (const sp of extra) {
+      if (!regexSpans.some((o) => sp.start < o.end && o.start < sp.end))
+        regexSpans.push(sp);
+    }
+    regexSpans.sort((a, b) => a.start - b.start);
     const masked = maskSpans(text, regexSpans);
     const modelSpans = await this.model.detect(masked, opts);
-    const out = [...regexSpans];
+    let out = [...regexSpans];
     const regexRanges = regexSpans.map((s) => [s.start, s.end]);
     for (const sp of modelSpans) {
       const overlaps = regexRanges.some(([rs, re]) => sp.start < re && rs < sp.end);
@@ -757,6 +899,8 @@ class CompositeDetector {
         continue;
       out.push(sp);
     }
+    if (this.propagateNames)
+      out = propagateNames(text, out, this.propagateSurnames);
     out.sort((a, b) => a.start !== b.start ? a.start - b.start : b.end - b.start - (a.end - a.start));
     return out;
   }
@@ -846,7 +990,7 @@ class CalibratedDetector {
     if (!text)
       return [];
     await this.load();
-    return this._detectChunked(text);
+    return snapToWords(text, await this._detectChunked(text));
   }
   async _detectChunked(text) {
     if (text.length <= 1500) {
@@ -913,15 +1057,21 @@ class CalibratedDetector {
           maxIdx = i;
         }
       }
-      tokenLabels.push({ id, label: this._id2label[maxIdx] ?? "O" });
+      let denom = 0;
+      for (let i = 0;i < row.length; i++)
+        denom += Math.exp(row[i] - maxVal);
+      tokenLabels.push({ id, label: this._id2label[maxIdx] ?? "O", prob: 1 / denom });
     }
     const spans = [];
     let cursor = 0;
     let groupIds = [];
+    let groupProbs = [];
     let groupCat = null;
     const flush = () => {
       if (groupCat === null || groupIds.length === 0)
         return;
+      const score = groupProbs.reduce((a, b) => a + b, 0) / groupProbs.length;
+      groupProbs = [];
       const surface = normalizeDecoded(tokenizer.decode(groupIds, { skip_special_tokens: true }));
       groupIds = [];
       const cat = groupCat;
@@ -943,17 +1093,19 @@ class CalibratedDetector {
         end: e + charOffset,
         label: cat,
         text: text.slice(s, e),
-        score: 1
+        score
       });
     };
-    for (const { id, label } of tokenLabels) {
+    for (const { id, label, prob } of tokenLabels) {
       const cat = label === "O" ? null : label.split("-", 2)[1] ?? null;
       if (cat === groupCat && cat !== null) {
         groupIds.push(id);
+        groupProbs.push(prob);
       } else {
         flush();
         if (cat !== null) {
           groupIds.push(id);
+          groupProbs.push(prob);
           groupCat = cat;
         }
       }
@@ -1039,7 +1191,7 @@ function readLogits(tensor) {
 // src/detectors/index.ts
 function defaultDetector() {
   const hasKey = typeof process !== "undefined" && Boolean(process.env?.GHEIM_API_KEY);
-  return hasKey ? new RemoteDetector : new CalibratedDetector;
+  return hasKey ? new RemoteDetector : new CompositeDetector({ model: new CalibratedDetector });
 }
 
 // src/core/session.ts

package/dist/esm/index.js

@@ -1,11 +1,3 @@
-var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require : typeof Proxy !== "undefined" ? new Proxy(x, {
-  get: (a, b) => (typeof require !== "undefined" ? require : a)[b]
-}) : x)(function(x) {
-  if (typeof require !== "undefined")
-    return require.apply(this, arguments);
-  throw Error('Dynamic require of "' + x + '" is not supported');
-});
-
 // src/core/sentinels.ts
 var LABEL_TO_TAG = {
   private_person: "PERSON",
@@ -264,10 +256,40 @@ function trimSpan(text, start, end) {
     return null;
   return { start, end };
 }
+var WORD_CHAR = /[\p{L}\p{N}_]/u;
+var isWordChar = (ch) => ch !== undefined && WORD_CHAR.test(ch);
+function snapToWords(text, spans) {
+  const snapped = spans.map((sp) => {
+    let { start, end } = sp;
+    while (start > 0 && isWordChar(text[start - 1]) && isWordChar(text[start]))
+      start -= 1;
+    while (end < text.length && isWordChar(text[end - 1]) && isWordChar(text[end]))
+      end += 1;
+    return start === sp.start && end === sp.end ? sp : { ...sp, start, end, text: text.slice(start, end) };
+  });
+  snapped.sort((a, b) => a.start - b.start || a.end - b.end);
+  const merged = [];
+  const lastByLabel = new Map;
+  for (const sp of snapped) {
+    const i = lastByLabel.get(sp.label);
+    const prev = i === undefined ? undefined : merged[i];
+    if (i !== undefined && prev !== undefined && sp.start <= prev.end) {
+      const end = Math.max(prev.end, sp.end);
+      const wPrev = prev.end - prev.start;
+      const wSp = sp.end - sp.start;
+      const score = ((prev.score ?? 1) * wPrev + (sp.score ?? 1) * wSp) / (wPrev + wSp);
+      merged[i] = { ...prev, end, text: text.slice(prev.start, end), score };
+    } else {
+      lastByLabel.set(sp.label, merged.length);
+      merged.push(sp);
+    }
+  }
+  return merged.sort((a, b) => a.start - b.start || a.end - b.end);
+}
 
 // src/detectors/remote.ts
 var DEFAULT_BASE_URL = "https://api.gheim.ch";
-var DEFAULT_MODEL = "openai/privacy-filter";
+var DEFAULT_MODEL = "joelbarmettler/gheim-ch-560m";
 
 class RemoteDetector {
   baseUrl;
@@ -512,10 +534,7 @@ class LocalDetector {
         });
       }
     }
-    for (const spans of out) {
-      spans.sort((a, b) => a.start - b.start || a.end - b.end);
-    }
-    return out;
+    return out.map((spans, i) => snapToWords(texts[i] ?? "", spans));
   }
   async chunkText(text, tokenizer) {
     if (tokenizer) {
@@ -792,21 +811,136 @@ function maskSpans(text, spans) {
   return chars.join("");
 }
 
+class CustomPattern {
+  label;
+  group;
+  name;
+  regex;
+  validator;
+  constructor(opts) {
+    if (!/^[A-Za-z][A-Za-z0-9_]*$/.test(opts.label)) {
+      throw new Error(`CustomPattern label ${JSON.stringify(opts.label)} must start with a letter and contain ` + "only letters, digits and underscores (it becomes a <TAG_1> placeholder).");
+    }
+    this.label = opts.label;
+    this.name = opts.name ?? opts.label;
+    this.group = opts.group ?? 0;
+    this.validator = opts.validator;
+    const source = typeof opts.regex === "string" ? opts.regex : opts.regex.source;
+    const base = typeof opts.regex === "string" ? opts.flags ?? "i" : opts.regex.flags;
+    const flags = [...new Set([...base, "g", "d"])].join("");
+    this.regex = new RegExp(source, flags);
+    const groups = (new RegExp(`${source}|`, flags.replace(/[gd]/g, "")).exec("") ?? []).length - 1;
+    if (!Number.isInteger(this.group) || this.group < 0 || this.group > groups) {
+      throw new Error(`CustomPattern ${JSON.stringify(this.name)}: group ${this.group} does not exist ` + `(the pattern has ${groups}).`);
+    }
+  }
+  find(text) {
+    const out = [];
+    this.regex.lastIndex = 0;
+    let m;
+    while ((m = this.regex.exec(text)) !== null) {
+      if (m[0] === "") {
+        this.regex.lastIndex++;
+        continue;
+      }
+      const range = m.indices[this.group];
+      if (!range || range[0] === range[1])
+        continue;
+      const [start, end] = range;
+      const value = text.slice(start, end);
+      if (this.validator && !this.validator(value))
+        continue;
+      out.push({ start, end, label: this.label, text: value, score: 1 });
+    }
+    return out;
+  }
+}
+var NAME_PART = /[A-ZÀ-Ý][\p{L}\p{N}_'’-]{3,}/gu;
+var HONORIFICS = new Set([
+  "frau",
+  "herr",
+  "herrn",
+  "madame",
+  "monsieur",
+  "signora",
+  "signor",
+  "signore",
+  "dott",
+  "dottor",
+  "dottoressa",
+  "doktor",
+  "docteur",
+  "prof",
+  "professor",
+  "mister",
+  "miss",
+  "mrs"
+]);
+function escapeRegExp(s) {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+function propagateNames(text, spans, surnames) {
+  const names = new Set;
+  for (const sp of spans) {
+    if (sp.label !== "private_person")
+      continue;
+    const value = text.slice(sp.start, sp.end).trim();
+    if (value.length >= 4)
+      names.add(value);
+    if (surnames) {
+      for (const part of value.match(NAME_PART) ?? []) {
+        if (!HONORIFICS.has(part.toLowerCase()))
+          names.add(part);
+      }
+    }
+  }
+  let result = [...spans];
+  for (const name of [...names].sort((a, b) => b.length - a.length)) {
+    const re = new RegExp(`(?<![\\p{L}\\p{N}_])${escapeRegExp(name)}(?![\\p{L}\\p{N}_])`, "gu");
+    let m;
+    while ((m = re.exec(text)) !== null) {
+      const start = m.index;
+      const end = start + m[0].length;
+      const hits = result.filter((sp) => start < sp.end && sp.start < end);
+      if (hits.some((sp) => sp.start <= start && end <= sp.end))
+        continue;
+      if (hits.some((sp) => sp.start < start || end < sp.end))
+        continue;
+      result = result.filter((sp) => !hits.includes(sp));
+      result.push({ start, end, label: "private_person", text: m[0], score: 1 });
+    }
+  }
+  return result;
+}
+
 class CompositeDetector {
   model;
+  patterns;
+  propagateNames;
+  propagateSurnames;
   constructor(opts) {
     if (!opts || !opts.model) {
       throw new Error("CompositeDetector requires { model: Detector } — pass any object with " + "a .detect(text) method (e.g. new LocalDetector({...})).");
     }
     this.model = opts.model;
+    this.patterns = opts.patterns ?? [];
+    this.propagateNames = opts.propagateNames ?? true;
+    this.propagateSurnames = opts.propagateSurnames ?? false;
   }
   async detect(text, opts) {
     if (!text)
       return [];
     const regexSpans = findRegexSpans(text);
+    const extra = this.patterns.flatMap((p) => p.find(text));
+    extra.sort((a, b) => a.start !== b.start ? a.start - b.start : b.end - b.start - (a.end - a.start));
+    for (const sp of extra) {
+      if (!regexSpans.some((o) => sp.start < o.end && o.start < sp.end))
+        regexSpans.push(sp);
+    }
+    regexSpans.sort((a, b) => a.start - b.start);
     const masked = maskSpans(text, regexSpans);
     const modelSpans = await this.model.detect(masked, opts);
-    const out = [...regexSpans];
+    let out = [...regexSpans];
     const regexRanges = regexSpans.map((s) => [s.start, s.end]);
     for (const sp of modelSpans) {
       const overlaps = regexRanges.some(([rs, re]) => sp.start < re && rs < sp.end);
@@ -814,6 +948,8 @@ class CompositeDetector {
         continue;
       out.push(sp);
     }
+    if (this.propagateNames)
+      out = propagateNames(text, out, this.propagateSurnames);
     out.sort((a, b) => a.start !== b.start ? a.start - b.start : b.end - b.start - (a.end - a.start));
     return out;
   }
@@ -903,7 +1039,7 @@ class CalibratedDetector {
     if (!text)
       return [];
     await this.load();
-    return this._detectChunked(text);
+    return snapToWords(text, await this._detectChunked(text));
   }
   async _detectChunked(text) {
     if (text.length <= 1500) {
@@ -970,15 +1106,21 @@ class CalibratedDetector {
           maxIdx = i;
         }
       }
-      tokenLabels.push({ id, label: this._id2label[maxIdx] ?? "O" });
+      let denom = 0;
+      for (let i = 0;i < row.length; i++)
+        denom += Math.exp(row[i] - maxVal);
+      tokenLabels.push({ id, label: this._id2label[maxIdx] ?? "O", prob: 1 / denom });
     }
     const spans = [];
     let cursor = 0;
     let groupIds = [];
+    let groupProbs = [];
     let groupCat = null;
     const flush = () => {
       if (groupCat === null || groupIds.length === 0)
         return;
+      const score = groupProbs.reduce((a, b) => a + b, 0) / groupProbs.length;
+      groupProbs = [];
       const surface = normalizeDecoded(tokenizer.decode(groupIds, { skip_special_tokens: true }));
       groupIds = [];
       const cat = groupCat;
@@ -1000,17 +1142,19 @@ class CalibratedDetector {
         end: e + charOffset,
         label: cat,
         text: text.slice(s, e),
-        score: 1
+        score
       });
     };
-    for (const { id, label } of tokenLabels) {
+    for (const { id, label, prob } of tokenLabels) {
       const cat = label === "O" ? null : label.split("-", 2)[1] ?? null;
       if (cat === groupCat && cat !== null) {
         groupIds.push(id);
+        groupProbs.push(prob);
       } else {
         flush();
         if (cat !== null) {
           groupIds.push(id);
+          groupProbs.push(prob);
           groupCat = cat;
         }
       }
@@ -1096,7 +1240,7 @@ function readLogits(tensor) {
 // src/detectors/index.ts
 function defaultDetector() {
   const hasKey = typeof process !== "undefined" && Boolean(process.env?.GHEIM_API_KEY);
-  return hasKey ? new RemoteDetector : new CalibratedDetector;
+  return hasKey ? new RemoteDetector : new CompositeDetector({ model: new CalibratedDetector });
 }
 // src/plain.ts
 function resolve(session, detector) {
@@ -1149,25 +1293,26 @@ async function* deanonymizeStream(chunks, session) {
 // src/index.ts
 var VERSION = "0.1.6";
 export {
-  renderSentinel,
-  parseSentinel,
-  mergeAdjacent,
-  labelTag,
-  isPossibleSentinelPrefix,
-  defaultDetector,
-  deanonymizeText,
-  deanonymizeStream,
-  anonymizeText,
-  anonymizeMessages,
-  VERSION,
-  StreamDeanonymizer,
-  Session,
-  SENTINEL_RE,
-  RemoteDetector,
-  DEFAULT_MODEL as REMOTE_DEFAULT_MODEL,
-  LocalDetector,
-  LABEL_TO_TAG,
-  DEFAULT_BASE_URL,
+  CalibratedDetector,
   CompositeDetector,
-  CalibratedDetector
+  CustomPattern,
+  DEFAULT_BASE_URL,
+  LABEL_TO_TAG,
+  LocalDetector,
+  DEFAULT_MODEL as REMOTE_DEFAULT_MODEL,
+  RemoteDetector,
+  SENTINEL_RE,
+  Session,
+  StreamDeanonymizer,
+  VERSION,
+  anonymizeMessages,
+  anonymizeText,
+  deanonymizeStream,
+  deanonymizeText,
+  defaultDetector,
+  isPossibleSentinelPrefix,
+  labelTag,
+  mergeAdjacent,
+  parseSentinel,
+  renderSentinel
 };