gheim 0.1.8 → 0.1.10

package/README.md

@@ -194,7 +194,8 @@ const remote = new RemoteDetector({
   apiKey: "...",
 });
 
-// Picks remote when GHEIM_API_KEY is set in the environment, else local.
+// Picks remote when GHEIM_API_KEY is set in the environment, else the composite
+// detector (checksum regex + calibrated local model, see below).
 const auto = defaultDetector();
 ```
 
@@ -221,11 +222,100 @@ dist/types/**/*.d.ts
 ## Composite detector (recommended for production)
 
 For categories where structure is verifiable by checksum (CH-IBAN, AHV,
-VAT-CHE, credit cards, common token formats), the Python sibling
-package ships a composite detector that pairs the regex catalogue with
-the model. The JavaScript package implements the model side; combine
-with your own regex layer or proxy through `gheim-server`, which
-applies the composite detector internally.
+VAT-CHE, credit cards, common token formats) the `CompositeDetector` applies a
+regex catalogue first, masks the matched spans, then runs the model on the
+remainder. `defaultDetector()` returns it (over `CalibratedDetector`) unless
+`GHEIM_API_KEY` is set; use `new CalibratedDetector()` for the model alone.
+
+```ts
+import { CalibratedDetector, CompositeDetector } from "gheim";
+
+const detector = new CompositeDetector({ model: new CalibratedDetector() });
+```
+
+`gheim-server` runs the model only, with no regex layer. To add the regex front-end
+and the name handling below to a server, wrap a `RemoteDetector`:
+`new CompositeDetector({ model: new RemoteDetector({ baseUrl }) })`.
+
+### Long inputs
+
+Inputs longer than the model's 512-token limit are split into overlapping
+windows (`chunkTokens: 510`, `chunkOverlapTokens: 32`) and the spans merged.
+Smaller windows catch more on dense documents but produce more false positives:
+on a Swiss test set of mixed letters and long email threads, 128-token windows
+found noticeably more entities than 510 before repeated-name handling was added,
+at about 40 extra false positives; with it, the gain was small and the false
+positives remained. The default stays at 510; lower it only after measuring on
+your own documents.
+
+### Repeated names
+
+A token classifier often tags a name once and skips a later occurrence of the
+same string, so a name caught in a greeting leaks again in a signature. The
+composite therefore masks every other occurrence of a string the model found as
+a person (`propagateNames`, on by default). A longer propagated occurrence
+replaces shorter model fragments it contains (`Käthi` inside `Zbinden Käthi`).
+`propagateSurnames: true` also propagates the individual capitalised parts of
+each detected name, honorifics such as "Frau" excluded: higher recall, more
+false positives. `propagateNames: false` turns it off.
+
+### Whole words
+
+A token classifier sometimes tags only part of a word (`gano` of `Lugano`, `Zbind` +
+`en` of `Zbinden`), and a window boundary can cut a word in half. `LocalDetector` and
+`CalibratedDetector` widen such a span to the whole word and merge same-label spans
+that overlap or touch, so one entity gets one placeholder and nothing readable is
+left behind. A span is widened only when the characters on both sides of its edge
+are word characters; edges at spaces or punctuation (`Gay` in `Gay-Crosier`) are
+left alone.
+
+### Confidence
+
+`CalibratedDetector` sets each span's `score` to the mean token probability of
+its tokens, computed from the same logits the label decision used. Clear cases
+score near 1.0; ambiguous ones score visibly lower. Regex spans score 1.0.
+
+### Your own patterns
+
+Formats that only your deployment has (insurance card numbers, hospital record
+numbers, internal case IDs, ...) are not built in. Pass them as `CustomPattern`
+rules; they run in the regex front-end, are masked before the model sees the
+text, and win over a model span they overlap.
+
+```ts
+import { CalibratedDetector, CompositeDetector, CustomPattern } from "gheim";
+
+const detector = new CompositeDetector({
+  model: new CalibratedDetector(),
+  patterns: [
+    // whole match is the PII
+    new CustomPattern({ label: "insurance_card", regex: /\b80756(?:[ .\-]?\d){15}\b/ }),
+    // hospital / case numbers after a label word (DE/FR/IT/EN); reported as
+    // account_number, only the digits (group 1) are masked
+    new CustomPattern({
+      label: "account_number",
+      regex:
+        /\b(?:PID|FID|IPP|MRN|Patienten-?(?:nummer|nr\.?|id)|Patienten ?nr\.?|Fall-?(?:nummer|nr\.?)|Fall|KG-?Nr\.?|Patient ?ID|case|N° ?(?:patient|de séjour|dossier|IPP)|N\. ?(?:paziente|caso)|numero ?(?:paziente|caso))[ ]?[:#]?[ ]?(\d{6,11})\b/i,
+      group: 1,
+    }),
+    // a label word anchors the match; only group 1 (the ID) is masked.
+    // The `i` flag covers the label words; the validator keeps the ID letter upper case.
+    new CustomPattern({
+      label: "id_document",
+      regex: /\b(?:pass|ausweis|id-?karte)(?:[ \-]?(?:nr\.?|nummer))?[ ]?[:#]?[ ]?([A-Za-z]\d{7})\b/i,
+      group: 1,
+      validator: (id) => /^[A-Z]/.test(id),
+    }),
+  ],
+});
+```
+
+`label` becomes the placeholder tag (`insurance_card` -> `<INSURANCE_CARD_1>`), so
+it must start with a letter and contain only letters, digits and underscores;
+reuse a built-in label such as `account_number` to share its tag. A string
+`regex` is compiled with `flags` (default `"i"`); a `RegExp` keeps its own.
+`validator` receives the PII text and can reject a match, for example on a
+checksum.
 
 ## Consulting
 

package/dist/cjs/index.cjs

@@ -39,27 +39,28 @@ var __export = (target, all) => {
 // src/index.ts
 var exports_src = {};
 __export(exports_src, {
-  renderSentinel: () => renderSentinel,
-  parseSentinel: () => parseSentinel,
-  mergeAdjacent: () => mergeAdjacent,
-  labelTag: () => labelTag,
-  isPossibleSentinelPrefix: () => isPossibleSentinelPrefix,
-  defaultDetector: () => defaultDetector,
-  deanonymizeText: () => deanonymizeText,
-  deanonymizeStream: () => deanonymizeStream,
-  anonymizeText: () => anonymizeText,
-  anonymizeMessages: () => anonymizeMessages,
-  VERSION: () => VERSION,
-  StreamDeanonymizer: () => StreamDeanonymizer,
-  Session: () => Session,
-  SENTINEL_RE: () => SENTINEL_RE,
-  RemoteDetector: () => RemoteDetector,
-  REMOTE_DEFAULT_MODEL: () => DEFAULT_MODEL,
-  LocalDetector: () => LocalDetector,
-  LABEL_TO_TAG: () => LABEL_TO_TAG,
-  DEFAULT_BASE_URL: () => DEFAULT_BASE_URL,
+  CalibratedDetector: () => CalibratedDetector,
   CompositeDetector: () => CompositeDetector,
-  CalibratedDetector: () => CalibratedDetector
+  CustomPattern: () => CustomPattern,
+  DEFAULT_BASE_URL: () => DEFAULT_BASE_URL,
+  LABEL_TO_TAG: () => LABEL_TO_TAG,
+  LocalDetector: () => LocalDetector,
+  REMOTE_DEFAULT_MODEL: () => DEFAULT_MODEL,
+  RemoteDetector: () => RemoteDetector,
+  SENTINEL_RE: () => SENTINEL_RE,
+  Session: () => Session,
+  StreamDeanonymizer: () => StreamDeanonymizer,
+  VERSION: () => VERSION,
+  anonymizeMessages: () => anonymizeMessages,
+  anonymizeText: () => anonymizeText,
+  deanonymizeStream: () => deanonymizeStream,
+  deanonymizeText: () => deanonymizeText,
+  defaultDetector: () => defaultDetector,
+  isPossibleSentinelPrefix: () => isPossibleSentinelPrefix,
+  labelTag: () => labelTag,
+  mergeAdjacent: () => mergeAdjacent,
+  parseSentinel: () => parseSentinel,
+  renderSentinel: () => renderSentinel
 });
 module.exports = __toCommonJS(exports_src);
 
@@ -321,10 +322,40 @@ function trimSpan(text, start, end) {
     return null;
   return { start, end };
 }
+var WORD_CHAR = /[\p{L}\p{N}_]/u;
+var isWordChar = (ch) => ch !== undefined && WORD_CHAR.test(ch);
+function snapToWords(text, spans) {
+  const snapped = spans.map((sp) => {
+    let { start, end } = sp;
+    while (start > 0 && isWordChar(text[start - 1]) && isWordChar(text[start]))
+      start -= 1;
+    while (end < text.length && isWordChar(text[end - 1]) && isWordChar(text[end]))
+      end += 1;
+    return start === sp.start && end === sp.end ? sp : { ...sp, start, end, text: text.slice(start, end) };
+  });
+  snapped.sort((a, b) => a.start - b.start || a.end - b.end);
+  const merged = [];
+  const lastByLabel = new Map;
+  for (const sp of snapped) {
+    const i = lastByLabel.get(sp.label);
+    const prev = i === undefined ? undefined : merged[i];
+    if (i !== undefined && prev !== undefined && sp.start <= prev.end) {
+      const end = Math.max(prev.end, sp.end);
+      const wPrev = prev.end - prev.start;
+      const wSp = sp.end - sp.start;
+      const score = ((prev.score ?? 1) * wPrev + (sp.score ?? 1) * wSp) / (wPrev + wSp);
+      merged[i] = { ...prev, end, text: text.slice(prev.start, end), score };
+    } else {
+      lastByLabel.set(sp.label, merged.length);
+      merged.push(sp);
+    }
+  }
+  return merged.sort((a, b) => a.start - b.start || a.end - b.end);
+}
 
 // src/detectors/remote.ts
 var DEFAULT_BASE_URL = "https://api.gheim.ch";
-var DEFAULT_MODEL = "openai/privacy-filter";
+var DEFAULT_MODEL = "joelbarmettler/gheim-ch-560m";
 
 class RemoteDetector {
   baseUrl;
@@ -569,10 +600,7 @@ class LocalDetector {
         });
       }
     }
-    for (const spans of out) {
-      spans.sort((a, b) => a.start - b.start || a.end - b.end);
-    }
-    return out;
+    return out.map((spans, i) => snapToWords(texts[i] ?? "", spans));
   }
   async chunkText(text, tokenizer) {
     if (tokenizer) {
@@ -849,21 +877,136 @@ function maskSpans(text, spans) {
   return chars.join("");
 }
 
+class CustomPattern {
+  label;
+  group;
+  name;
+  regex;
+  validator;
+  constructor(opts) {
+    if (!/^[A-Za-z][A-Za-z0-9_]*$/.test(opts.label)) {
+      throw new Error(`CustomPattern label ${JSON.stringify(opts.label)} must start with a letter and contain ` + "only letters, digits and underscores (it becomes a <TAG_1> placeholder).");
+    }
+    this.label = opts.label;
+    this.name = opts.name ?? opts.label;
+    this.group = opts.group ?? 0;
+    this.validator = opts.validator;
+    const source = typeof opts.regex === "string" ? opts.regex : opts.regex.source;
+    const base = typeof opts.regex === "string" ? opts.flags ?? "i" : opts.regex.flags;
+    const flags = [...new Set([...base, "g", "d"])].join("");
+    this.regex = new RegExp(source, flags);
+    const groups = (new RegExp(`${source}|`, flags.replace(/[gd]/g, "")).exec("") ?? []).length - 1;
+    if (!Number.isInteger(this.group) || this.group < 0 || this.group > groups) {
+      throw new Error(`CustomPattern ${JSON.stringify(this.name)}: group ${this.group} does not exist ` + `(the pattern has ${groups}).`);
+    }
+  }
+  find(text) {
+    const out = [];
+    this.regex.lastIndex = 0;
+    let m;
+    while ((m = this.regex.exec(text)) !== null) {
+      if (m[0] === "") {
+        this.regex.lastIndex++;
+        continue;
+      }
+      const range = m.indices[this.group];
+      if (!range || range[0] === range[1])
+        continue;
+      const [start, end] = range;
+      const value = text.slice(start, end);
+      if (this.validator && !this.validator(value))
+        continue;
+      out.push({ start, end, label: this.label, text: value, score: 1 });
+    }
+    return out;
+  }
+}
+var NAME_PART = /[A-ZÀ-Ý][\p{L}\p{N}_'’-]{3,}/gu;
+var HONORIFICS = new Set([
+  "frau",
+  "herr",
+  "herrn",
+  "madame",
+  "monsieur",
+  "signora",
+  "signor",
+  "signore",
+  "dott",
+  "dottor",
+  "dottoressa",
+  "doktor",
+  "docteur",
+  "prof",
+  "professor",
+  "mister",
+  "miss",
+  "mrs"
+]);
+function escapeRegExp(s) {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+function propagateNames(text, spans, surnames) {
+  const names = new Set;
+  for (const sp of spans) {
+    if (sp.label !== "private_person")
+      continue;
+    const value = text.slice(sp.start, sp.end).trim();
+    if (value.length >= 4)
+      names.add(value);
+    if (surnames) {
+      for (const part of value.match(NAME_PART) ?? []) {
+        if (!HONORIFICS.has(part.toLowerCase()))
+          names.add(part);
+      }
+    }
+  }
+  let result = [...spans];
+  for (const name of [...names].sort((a, b) => b.length - a.length)) {
+    const re = new RegExp(`(?<![\\p{L}\\p{N}_])${escapeRegExp(name)}(?![\\p{L}\\p{N}_])`, "gu");
+    let m;
+    while ((m = re.exec(text)) !== null) {
+      const start = m.index;
+      const end = start + m[0].length;
+      const hits = result.filter((sp) => start < sp.end && sp.start < end);
+      if (hits.some((sp) => sp.start <= start && end <= sp.end))
+        continue;
+      if (hits.some((sp) => sp.start < start || end < sp.end))
+        continue;
+      result = result.filter((sp) => !hits.includes(sp));
+      result.push({ start, end, label: "private_person", text: m[0], score: 1 });
+    }
+  }
+  return result;
+}
+
 class CompositeDetector {
   model;
+  patterns;
+  propagateNames;
+  propagateSurnames;
   constructor(opts) {
     if (!opts || !opts.model) {
       throw new Error("CompositeDetector requires { model: Detector } — pass any object with " + "a .detect(text) method (e.g. new LocalDetector({...})).");
     }
     this.model = opts.model;
+    this.patterns = opts.patterns ?? [];
+    this.propagateNames = opts.propagateNames ?? true;
+    this.propagateSurnames = opts.propagateSurnames ?? false;
   }
   async detect(text, opts) {
     if (!text)
       return [];
     const regexSpans = findRegexSpans(text);
+    const extra = this.patterns.flatMap((p) => p.find(text));
+    extra.sort((a, b) => a.start !== b.start ? a.start - b.start : b.end - b.start - (a.end - a.start));
+    for (const sp of extra) {
+      if (!regexSpans.some((o) => sp.start < o.end && o.start < sp.end))
+        regexSpans.push(sp);
+    }
+    regexSpans.sort((a, b) => a.start - b.start);
     const masked = maskSpans(text, regexSpans);
     const modelSpans = await this.model.detect(masked, opts);
-    const out = [...regexSpans];
+    let out = [...regexSpans];
     const regexRanges = regexSpans.map((s) => [s.start, s.end]);
     for (const sp of modelSpans) {
       const overlaps = regexRanges.some(([rs, re]) => sp.start < re && rs < sp.end);
@@ -871,6 +1014,8 @@ class CompositeDetector {
         continue;
       out.push(sp);
     }
+    if (this.propagateNames)
+      out = propagateNames(text, out, this.propagateSurnames);
     out.sort((a, b) => a.start !== b.start ? a.start - b.start : b.end - b.start - (a.end - a.start));
     return out;
   }
@@ -960,7 +1105,7 @@ class CalibratedDetector {
     if (!text)
       return [];
     await this.load();
-    return this._detectChunked(text);
+    return snapToWords(text, await this._detectChunked(text));
   }
   async _detectChunked(text) {
     if (text.length <= 1500) {
@@ -1027,15 +1172,21 @@ class CalibratedDetector {
           maxIdx = i;
         }
       }
-      tokenLabels.push({ id, label: this._id2label[maxIdx] ?? "O" });
+      let denom = 0;
+      for (let i = 0;i < row.length; i++)
+        denom += Math.exp(row[i] - maxVal);
+      tokenLabels.push({ id, label: this._id2label[maxIdx] ?? "O", prob: 1 / denom });
     }
     const spans = [];
     let cursor = 0;
     let groupIds = [];
+    let groupProbs = [];
     let groupCat = null;
     const flush = () => {
       if (groupCat === null || groupIds.length === 0)
         return;
+      const score = groupProbs.reduce((a, b) => a + b, 0) / groupProbs.length;
+      groupProbs = [];
       const surface = normalizeDecoded(tokenizer.decode(groupIds, { skip_special_tokens: true }));
       groupIds = [];
       const cat = groupCat;
@@ -1057,17 +1208,19 @@ class CalibratedDetector {
         end: e + charOffset,
         label: cat,
         text: text.slice(s, e),
-        score: 1
+        score
       });
     };
-    for (const { id, label } of tokenLabels) {
+    for (const { id, label, prob } of tokenLabels) {
       const cat = label === "O" ? null : label.split("-", 2)[1] ?? null;
       if (cat === groupCat && cat !== null) {
         groupIds.push(id);
+        groupProbs.push(prob);
       } else {
         flush();
         if (cat !== null) {
           groupIds.push(id);
+          groupProbs.push(prob);
           groupCat = cat;
         }
       }
@@ -1153,7 +1306,7 @@ function readLogits(tensor) {
 // src/detectors/index.ts
 function defaultDetector() {
   const hasKey = typeof process !== "undefined" && Boolean(process.env?.GHEIM_API_KEY);
-  return hasKey ? new RemoteDetector : new CalibratedDetector;
+  return hasKey ? new RemoteDetector : new CompositeDetector({ model: new CalibratedDetector });
 }
 // src/plain.ts
 function resolve(session, detector) {