gheim 0.1.8 → 0.1.9

package/dist/esm/index.js

@@ -1,11 +1,3 @@
-var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require : typeof Proxy !== "undefined" ? new Proxy(x, {
-  get: (a, b) => (typeof require !== "undefined" ? require : a)[b]
-}) : x)(function(x) {
-  if (typeof require !== "undefined")
-    return require.apply(this, arguments);
-  throw Error('Dynamic require of "' + x + '" is not supported');
-});
-
 // src/core/sentinels.ts
 var LABEL_TO_TAG = {
   private_person: "PERSON",
@@ -264,6 +256,36 @@ function trimSpan(text, start, end) {
     return null;
   return { start, end };
 }
+var WORD_CHAR = /[\p{L}\p{N}_]/u;
+var isWordChar = (ch) => ch !== undefined && WORD_CHAR.test(ch);
+function snapToWords(text, spans) {
+  const snapped = spans.map((sp) => {
+    let { start, end } = sp;
+    while (start > 0 && isWordChar(text[start - 1]) && isWordChar(text[start]))
+      start -= 1;
+    while (end < text.length && isWordChar(text[end - 1]) && isWordChar(text[end]))
+      end += 1;
+    return start === sp.start && end === sp.end ? sp : { ...sp, start, end, text: text.slice(start, end) };
+  });
+  snapped.sort((a, b) => a.start - b.start || a.end - b.end);
+  const merged = [];
+  const lastByLabel = new Map;
+  for (const sp of snapped) {
+    const i = lastByLabel.get(sp.label);
+    const prev = i === undefined ? undefined : merged[i];
+    if (i !== undefined && prev !== undefined && sp.start <= prev.end) {
+      const end = Math.max(prev.end, sp.end);
+      const wPrev = prev.end - prev.start;
+      const wSp = sp.end - sp.start;
+      const score = ((prev.score ?? 1) * wPrev + (sp.score ?? 1) * wSp) / (wPrev + wSp);
+      merged[i] = { ...prev, end, text: text.slice(prev.start, end), score };
+    } else {
+      lastByLabel.set(sp.label, merged.length);
+      merged.push(sp);
+    }
+  }
+  return merged.sort((a, b) => a.start - b.start || a.end - b.end);
+}
 
 // src/detectors/remote.ts
 var DEFAULT_BASE_URL = "https://api.gheim.ch";
@@ -512,10 +534,7 @@ class LocalDetector {
         });
       }
     }
-    for (const spans of out) {
-      spans.sort((a, b) => a.start - b.start || a.end - b.end);
-    }
-    return out;
+    return out.map((spans, i) => snapToWords(texts[i] ?? "", spans));
   }
   async chunkText(text, tokenizer) {
     if (tokenizer) {
@@ -792,21 +811,136 @@ function maskSpans(text, spans) {
   return chars.join("");
 }
 
+class CustomPattern {
+  label;
+  group;
+  name;
+  regex;
+  validator;
+  constructor(opts) {
+    if (!/^[A-Za-z][A-Za-z0-9_]*$/.test(opts.label)) {
+      throw new Error(`CustomPattern label ${JSON.stringify(opts.label)} must start with a letter and contain ` + "only letters, digits and underscores (it becomes a <TAG_1> placeholder).");
+    }
+    this.label = opts.label;
+    this.name = opts.name ?? opts.label;
+    this.group = opts.group ?? 0;
+    this.validator = opts.validator;
+    const source = typeof opts.regex === "string" ? opts.regex : opts.regex.source;
+    const base = typeof opts.regex === "string" ? opts.flags ?? "i" : opts.regex.flags;
+    const flags = [...new Set([...base, "g", "d"])].join("");
+    this.regex = new RegExp(source, flags);
+    const groups = (new RegExp(`${source}|`, flags.replace(/[gd]/g, "")).exec("") ?? []).length - 1;
+    if (!Number.isInteger(this.group) || this.group < 0 || this.group > groups) {
+      throw new Error(`CustomPattern ${JSON.stringify(this.name)}: group ${this.group} does not exist ` + `(the pattern has ${groups}).`);
+    }
+  }
+  find(text) {
+    const out = [];
+    this.regex.lastIndex = 0;
+    let m;
+    while ((m = this.regex.exec(text)) !== null) {
+      if (m[0] === "") {
+        this.regex.lastIndex++;
+        continue;
+      }
+      const range = m.indices[this.group];
+      if (!range || range[0] === range[1])
+        continue;
+      const [start, end] = range;
+      const value = text.slice(start, end);
+      if (this.validator && !this.validator(value))
+        continue;
+      out.push({ start, end, label: this.label, text: value, score: 1 });
+    }
+    return out;
+  }
+}
+var NAME_PART = /[A-ZÀ-Ý][\p{L}\p{N}_'’-]{3,}/gu;
+var HONORIFICS = new Set([
+  "frau",
+  "herr",
+  "herrn",
+  "madame",
+  "monsieur",
+  "signora",
+  "signor",
+  "signore",
+  "dott",
+  "dottor",
+  "dottoressa",
+  "doktor",
+  "docteur",
+  "prof",
+  "professor",
+  "mister",
+  "miss",
+  "mrs"
+]);
+function escapeRegExp(s) {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+function propagateNames(text, spans, surnames) {
+  const names = new Set;
+  for (const sp of spans) {
+    if (sp.label !== "private_person")
+      continue;
+    const value = text.slice(sp.start, sp.end).trim();
+    if (value.length >= 4)
+      names.add(value);
+    if (surnames) {
+      for (const part of value.match(NAME_PART) ?? []) {
+        if (!HONORIFICS.has(part.toLowerCase()))
+          names.add(part);
+      }
+    }
+  }
+  let result = [...spans];
+  for (const name of [...names].sort((a, b) => b.length - a.length)) {
+    const re = new RegExp(`(?<![\\p{L}\\p{N}_])${escapeRegExp(name)}(?![\\p{L}\\p{N}_])`, "gu");
+    let m;
+    while ((m = re.exec(text)) !== null) {
+      const start = m.index;
+      const end = start + m[0].length;
+      const hits = result.filter((sp) => start < sp.end && sp.start < end);
+      if (hits.some((sp) => sp.start <= start && end <= sp.end))
+        continue;
+      if (hits.some((sp) => sp.start < start || end < sp.end))
+        continue;
+      result = result.filter((sp) => !hits.includes(sp));
+      result.push({ start, end, label: "private_person", text: m[0], score: 1 });
+    }
+  }
+  return result;
+}
+
 class CompositeDetector {
   model;
+  patterns;
+  propagateNames;
+  propagateSurnames;
   constructor(opts) {
     if (!opts || !opts.model) {
       throw new Error("CompositeDetector requires { model: Detector } — pass any object with " + "a .detect(text) method (e.g. new LocalDetector({...})).");
     }
     this.model = opts.model;
+    this.patterns = opts.patterns ?? [];
+    this.propagateNames = opts.propagateNames ?? true;
+    this.propagateSurnames = opts.propagateSurnames ?? false;
   }
   async detect(text, opts) {
     if (!text)
       return [];
     const regexSpans = findRegexSpans(text);
+    const extra = this.patterns.flatMap((p) => p.find(text));
+    extra.sort((a, b) => a.start !== b.start ? a.start - b.start : b.end - b.start - (a.end - a.start));
+    for (const sp of extra) {
+      if (!regexSpans.some((o) => sp.start < o.end && o.start < sp.end))
+        regexSpans.push(sp);
+    }
+    regexSpans.sort((a, b) => a.start - b.start);
     const masked = maskSpans(text, regexSpans);
     const modelSpans = await this.model.detect(masked, opts);
-    const out = [...regexSpans];
+    let out = [...regexSpans];
     const regexRanges = regexSpans.map((s) => [s.start, s.end]);
     for (const sp of modelSpans) {
       const overlaps = regexRanges.some(([rs, re]) => sp.start < re && rs < sp.end);
@@ -814,6 +948,8 @@ class CompositeDetector {
         continue;
       out.push(sp);
     }
+    if (this.propagateNames)
+      out = propagateNames(text, out, this.propagateSurnames);
     out.sort((a, b) => a.start !== b.start ? a.start - b.start : b.end - b.start - (a.end - a.start));
     return out;
   }
@@ -903,7 +1039,7 @@ class CalibratedDetector {
     if (!text)
       return [];
     await this.load();
-    return this._detectChunked(text);
+    return snapToWords(text, await this._detectChunked(text));
   }
   async _detectChunked(text) {
     if (text.length <= 1500) {
@@ -970,15 +1106,21 @@ class CalibratedDetector {
           maxIdx = i;
         }
       }
-      tokenLabels.push({ id, label: this._id2label[maxIdx] ?? "O" });
+      let denom = 0;
+      for (let i = 0;i < row.length; i++)
+        denom += Math.exp(row[i] - maxVal);
+      tokenLabels.push({ id, label: this._id2label[maxIdx] ?? "O", prob: 1 / denom });
     }
     const spans = [];
     let cursor = 0;
     let groupIds = [];
+    let groupProbs = [];
     let groupCat = null;
     const flush = () => {
       if (groupCat === null || groupIds.length === 0)
         return;
+      const score = groupProbs.reduce((a, b) => a + b, 0) / groupProbs.length;
+      groupProbs = [];
       const surface = normalizeDecoded(tokenizer.decode(groupIds, { skip_special_tokens: true }));
       groupIds = [];
       const cat = groupCat;
@@ -1000,17 +1142,19 @@ class CalibratedDetector {
         end: e + charOffset,
         label: cat,
         text: text.slice(s, e),
-        score: 1
+        score
       });
     };
-    for (const { id, label } of tokenLabels) {
+    for (const { id, label, prob } of tokenLabels) {
       const cat = label === "O" ? null : label.split("-", 2)[1] ?? null;
       if (cat === groupCat && cat !== null) {
         groupIds.push(id);
+        groupProbs.push(prob);
       } else {
         flush();
         if (cat !== null) {
           groupIds.push(id);
+          groupProbs.push(prob);
           groupCat = cat;
         }
       }
@@ -1096,7 +1240,7 @@ function readLogits(tensor) {
 // src/detectors/index.ts
 function defaultDetector() {
   const hasKey = typeof process !== "undefined" && Boolean(process.env?.GHEIM_API_KEY);
-  return hasKey ? new RemoteDetector : new CalibratedDetector;
+  return hasKey ? new RemoteDetector : new CompositeDetector({ model: new CalibratedDetector });
 }
 // src/plain.ts
 function resolve(session, detector) {
@@ -1149,25 +1293,26 @@ async function* deanonymizeStream(chunks, session) {
 // src/index.ts
 var VERSION = "0.1.6";
 export {
-  renderSentinel,
-  parseSentinel,
-  mergeAdjacent,
-  labelTag,
-  isPossibleSentinelPrefix,
-  defaultDetector,
-  deanonymizeText,
-  deanonymizeStream,
-  anonymizeText,
-  anonymizeMessages,
-  VERSION,
-  StreamDeanonymizer,
-  Session,
-  SENTINEL_RE,
-  RemoteDetector,
-  DEFAULT_MODEL as REMOTE_DEFAULT_MODEL,
-  LocalDetector,
-  LABEL_TO_TAG,
-  DEFAULT_BASE_URL,
+  CalibratedDetector,
   CompositeDetector,
-  CalibratedDetector
+  CustomPattern,
+  DEFAULT_BASE_URL,
+  LABEL_TO_TAG,
+  LocalDetector,
+  DEFAULT_MODEL as REMOTE_DEFAULT_MODEL,
+  RemoteDetector,
+  SENTINEL_RE,
+  Session,
+  StreamDeanonymizer,
+  VERSION,
+  anonymizeMessages,
+  anonymizeText,
+  deanonymizeStream,
+  deanonymizeText,
+  defaultDetector,
+  isPossibleSentinelPrefix,
+  labelTag,
+  mergeAdjacent,
+  parseSentinel,
+  renderSentinel
 };

package/dist/esm/openai/index.js

@@ -1,11 +1,3 @@
-var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require : typeof Proxy !== "undefined" ? new Proxy(x, {
-  get: (a, b) => (typeof require !== "undefined" ? require : a)[b]
-}) : x)(function(x) {
-  if (typeof require !== "undefined")
-    return require.apply(this, arguments);
-  throw Error('Dynamic require of "' + x + '" is not supported');
-});
-
 // src/core/sentinels.ts
 var LABEL_TO_TAG = {
   private_person: "PERSON",
@@ -164,6 +156,36 @@ function trimSpan(text, start, end) {
     return null;
   return { start, end };
 }
+var WORD_CHAR = /[\p{L}\p{N}_]/u;
+var isWordChar = (ch) => ch !== undefined && WORD_CHAR.test(ch);
+function snapToWords(text, spans) {
+  const snapped = spans.map((sp) => {
+    let { start, end } = sp;
+    while (start > 0 && isWordChar(text[start - 1]) && isWordChar(text[start]))
+      start -= 1;
+    while (end < text.length && isWordChar(text[end - 1]) && isWordChar(text[end]))
+      end += 1;
+    return start === sp.start && end === sp.end ? sp : { ...sp, start, end, text: text.slice(start, end) };
+  });
+  snapped.sort((a, b) => a.start - b.start || a.end - b.end);
+  const merged = [];
+  const lastByLabel = new Map;
+  for (const sp of snapped) {
+    const i = lastByLabel.get(sp.label);
+    const prev = i === undefined ? undefined : merged[i];
+    if (i !== undefined && prev !== undefined && sp.start <= prev.end) {
+      const end = Math.max(prev.end, sp.end);
+      const wPrev = prev.end - prev.start;
+      const wSp = sp.end - sp.start;
+      const score = ((prev.score ?? 1) * wPrev + (sp.score ?? 1) * wSp) / (wPrev + wSp);
+      merged[i] = { ...prev, end, text: text.slice(prev.start, end), score };
+    } else {
+      lastByLabel.set(sp.label, merged.length);
+      merged.push(sp);
+    }
+  }
+  return merged.sort((a, b) => a.start - b.start || a.end - b.end);
+}
 
 // src/detectors/remote.ts
 var DEFAULT_BASE_URL = "https://api.gheim.ch";
@@ -412,10 +434,7 @@ class LocalDetector {
         });
       }
     }
-    for (const spans of out) {
-      spans.sort((a, b) => a.start - b.start || a.end - b.end);
-    }
-    return out;
+    return out.map((spans, i) => snapToWords(texts[i] ?? "", spans));
   }
   async chunkText(text, tokenizer) {
     if (tokenizer) {
@@ -692,21 +711,136 @@ function maskSpans(text, spans) {
   return chars.join("");
 }
 
+class CustomPattern {
+  label;
+  group;
+  name;
+  regex;
+  validator;
+  constructor(opts) {
+    if (!/^[A-Za-z][A-Za-z0-9_]*$/.test(opts.label)) {
+      throw new Error(`CustomPattern label ${JSON.stringify(opts.label)} must start with a letter and contain ` + "only letters, digits and underscores (it becomes a <TAG_1> placeholder).");
+    }
+    this.label = opts.label;
+    this.name = opts.name ?? opts.label;
+    this.group = opts.group ?? 0;
+    this.validator = opts.validator;
+    const source = typeof opts.regex === "string" ? opts.regex : opts.regex.source;
+    const base = typeof opts.regex === "string" ? opts.flags ?? "i" : opts.regex.flags;
+    const flags = [...new Set([...base, "g", "d"])].join("");
+    this.regex = new RegExp(source, flags);
+    const groups = (new RegExp(`${source}|`, flags.replace(/[gd]/g, "")).exec("") ?? []).length - 1;
+    if (!Number.isInteger(this.group) || this.group < 0 || this.group > groups) {
+      throw new Error(`CustomPattern ${JSON.stringify(this.name)}: group ${this.group} does not exist ` + `(the pattern has ${groups}).`);
+    }
+  }
+  find(text) {
+    const out = [];
+    this.regex.lastIndex = 0;
+    let m;
+    while ((m = this.regex.exec(text)) !== null) {
+      if (m[0] === "") {
+        this.regex.lastIndex++;
+        continue;
+      }
+      const range = m.indices[this.group];
+      if (!range || range[0] === range[1])
+        continue;
+      const [start, end] = range;
+      const value = text.slice(start, end);
+      if (this.validator && !this.validator(value))
+        continue;
+      out.push({ start, end, label: this.label, text: value, score: 1 });
+    }
+    return out;
+  }
+}
+var NAME_PART = /[A-ZÀ-Ý][\p{L}\p{N}_'’-]{3,}/gu;
+var HONORIFICS = new Set([
+  "frau",
+  "herr",
+  "herrn",
+  "madame",
+  "monsieur",
+  "signora",
+  "signor",
+  "signore",
+  "dott",
+  "dottor",
+  "dottoressa",
+  "doktor",
+  "docteur",
+  "prof",
+  "professor",
+  "mister",
+  "miss",
+  "mrs"
+]);
+function escapeRegExp(s) {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+function propagateNames(text, spans, surnames) {
+  const names = new Set;
+  for (const sp of spans) {
+    if (sp.label !== "private_person")
+      continue;
+    const value = text.slice(sp.start, sp.end).trim();
+    if (value.length >= 4)
+      names.add(value);
+    if (surnames) {
+      for (const part of value.match(NAME_PART) ?? []) {
+        if (!HONORIFICS.has(part.toLowerCase()))
+          names.add(part);
+      }
+    }
+  }
+  let result = [...spans];
+  for (const name of [...names].sort((a, b) => b.length - a.length)) {
+    const re = new RegExp(`(?<![\\p{L}\\p{N}_])${escapeRegExp(name)}(?![\\p{L}\\p{N}_])`, "gu");
+    let m;
+    while ((m = re.exec(text)) !== null) {
+      const start = m.index;
+      const end = start + m[0].length;
+      const hits = result.filter((sp) => start < sp.end && sp.start < end);
+      if (hits.some((sp) => sp.start <= start && end <= sp.end))
+        continue;
+      if (hits.some((sp) => sp.start < start || end < sp.end))
+        continue;
+      result = result.filter((sp) => !hits.includes(sp));
+      result.push({ start, end, label: "private_person", text: m[0], score: 1 });
+    }
+  }
+  return result;
+}
+
 class CompositeDetector {
   model;
+  patterns;
+  propagateNames;
+  propagateSurnames;
   constructor(opts) {
     if (!opts || !opts.model) {
       throw new Error("CompositeDetector requires { model: Detector } — pass any object with " + "a .detect(text) method (e.g. new LocalDetector({...})).");
     }
     this.model = opts.model;
+    this.patterns = opts.patterns ?? [];
+    this.propagateNames = opts.propagateNames ?? true;
+    this.propagateSurnames = opts.propagateSurnames ?? false;
   }
   async detect(text, opts) {
     if (!text)
       return [];
     const regexSpans = findRegexSpans(text);
+    const extra = this.patterns.flatMap((p) => p.find(text));
+    extra.sort((a, b) => a.start !== b.start ? a.start - b.start : b.end - b.start - (a.end - a.start));
+    for (const sp of extra) {
+      if (!regexSpans.some((o) => sp.start < o.end && o.start < sp.end))
+        regexSpans.push(sp);
+    }
+    regexSpans.sort((a, b) => a.start - b.start);
     const masked = maskSpans(text, regexSpans);
     const modelSpans = await this.model.detect(masked, opts);
-    const out = [...regexSpans];
+    let out = [...regexSpans];
     const regexRanges = regexSpans.map((s) => [s.start, s.end]);
     for (const sp of modelSpans) {
       const overlaps = regexRanges.some(([rs, re]) => sp.start < re && rs < sp.end);
@@ -714,6 +848,8 @@ class CompositeDetector {
         continue;
       out.push(sp);
     }
+    if (this.propagateNames)
+      out = propagateNames(text, out, this.propagateSurnames);
     out.sort((a, b) => a.start !== b.start ? a.start - b.start : b.end - b.start - (a.end - a.start));
     return out;
   }
@@ -803,7 +939,7 @@ class CalibratedDetector {
     if (!text)
       return [];
     await this.load();
-    return this._detectChunked(text);
+    return snapToWords(text, await this._detectChunked(text));
   }
   async _detectChunked(text) {
     if (text.length <= 1500) {
@@ -870,15 +1006,21 @@ class CalibratedDetector {
           maxIdx = i;
         }
       }
-      tokenLabels.push({ id, label: this._id2label[maxIdx] ?? "O" });
+      let denom = 0;
+      for (let i = 0;i < row.length; i++)
+        denom += Math.exp(row[i] - maxVal);
+      tokenLabels.push({ id, label: this._id2label[maxIdx] ?? "O", prob: 1 / denom });
     }
     const spans = [];
     let cursor = 0;
     let groupIds = [];
+    let groupProbs = [];
     let groupCat = null;
     const flush = () => {
       if (groupCat === null || groupIds.length === 0)
         return;
+      const score = groupProbs.reduce((a, b) => a + b, 0) / groupProbs.length;
+      groupProbs = [];
       const surface = normalizeDecoded(tokenizer.decode(groupIds, { skip_special_tokens: true }));
       groupIds = [];
       const cat = groupCat;
@@ -900,17 +1042,19 @@ class CalibratedDetector {
         end: e + charOffset,
         label: cat,
         text: text.slice(s, e),
-        score: 1
+        score
       });
     };
-    for (const { id, label } of tokenLabels) {
+    for (const { id, label, prob } of tokenLabels) {
       const cat = label === "O" ? null : label.split("-", 2)[1] ?? null;
       if (cat === groupCat && cat !== null) {
         groupIds.push(id);
+        groupProbs.push(prob);
       } else {
         flush();
         if (cat !== null) {
           groupIds.push(id);
+          groupProbs.push(prob);
           groupCat = cat;
         }
       }
@@ -996,7 +1140,7 @@ function readLogits(tensor) {
 // src/detectors/index.ts
 function defaultDetector() {
   const hasKey = typeof process !== "undefined" && Boolean(process.env?.GHEIM_API_KEY);
-  return hasKey ? new RemoteDetector : new CalibratedDetector;
+  return hasKey ? new RemoteDetector : new CompositeDetector({ model: new CalibratedDetector });
 }
 
 // src/core/session.ts
@@ -1775,11 +1919,11 @@ async function* deanonymizeOpenAIStream(stream, session) {
   }
 }
 export {
-  deanonymizeOpenAIStream,
-  deanonymizeOpenAIResponse,
-  anonymizeOpenAIMessages,
-  STRICT_REASONS,
-  OpenAI,
+  AsyncOpenAI,
   GheimStrictError,
-  AsyncOpenAI
+  OpenAI,
+  STRICT_REASONS,
+  anonymizeOpenAIMessages,
+  deanonymizeOpenAIResponse,
+  deanonymizeOpenAIStream
 };

package/dist/types/detectors/base.d.ts

@@ -25,4 +25,17 @@ export interface Detector {
         model?: string;
     }): Promise<Span[]>;
 }
+/**
+ * Widen spans that start or end inside a word to the whole word, then merge
+ * same-label spans that overlap or touch.
+ *
+ * A token classifier sometimes tags only part of a word (`gano` of `Lugano`,
+ * `Zbind` + `en` of `Zbinden`), and a window boundary can cut a word in half.
+ * Masking a fragment leaves the rest of the word readable and splits one entity
+ * into two placeholders. A span is widened only when the characters on both
+ * sides of its edge are word characters, so edges at spaces or punctuation
+ * (`Gay` in `Gay-Crosier`) are left alone. The merged span's score is the
+ * length-weighted mean of its parts.
+ */
+export declare function snapToWords(text: string, spans: Span[]): Span[];
 //# sourceMappingURL=base.d.ts.map

package/dist/types/detectors/base.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"base.d.ts","sourceRoot":"","sources":["../../../src/detectors/base.ts"],"names":[],"mappings":"AAAA,gEAAgE;AAEhE;;;;;GAKG;AACH,wBAAgB,QAAQ,CACtB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,GACV;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAKvC;AAED,MAAM,WAAW,IAAI;IACnB,6CAA6C;IAC7C,KAAK,EAAE,MAAM,CAAC;IACd,yDAAyD;IACzD,KAAK,EAAE,MAAM,CAAC;IACd,iBAAiB;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,sDAAsD;IACtD,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,QAAQ;IACvB,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;CAClE"}
+{"version":3,"file":"base.d.ts","sourceRoot":"","sources":["../../../src/detectors/base.ts"],"names":[],"mappings":"AAAA,gEAAgE;AAEhE;;;;;GAKG;AACH,wBAAgB,QAAQ,CACtB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,GACV;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAKvC;AAED,MAAM,WAAW,IAAI;IACnB,6CAA6C;IAC7C,KAAK,EAAE,MAAM,CAAC;IACd,yDAAyD;IACzD,KAAK,EAAE,MAAM,CAAC;IACd,iBAAiB;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,sDAAsD;IACtD,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,QAAQ;IACvB,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;CAClE;AAKD;;;;;;;;;;;GAWG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,IAAI,EAAE,CA0B/D"}

package/dist/types/detectors/calibrated.d.ts

@@ -1,31 +1,3 @@
-/**
- * Calibrated detector — subtracts a constant `oBias` from the O-class
- * logit before argmax. Originally introduced to recover the v1 model's
- * "single-category mode" pathology (Müller + IBAN co-occurrence flipped
- * the person prediction to O); the v3 / gheim-ch-560m training pipeline
- * largely resolved that pathology, but the calibration still buys a
- * small precision/recall improvement and remains the default.
- *
- * Sweep on gheim-ch-560m (see eval/calibration_sweep.json):
- * `oBias=0.5` is the Pareto-clean default — probe perfect-rate is flat
- * at 91.5% across bias 0.0 and 0.5, and test char-F1 lifts by 0.27pp.
- * Higher biases (1.0–1.5) trade test F1 for marginal probe gains.
- *
- * A separate sweep on the int8 ONNX path via `transformers.js`
- * (eval/calibration_sweep_q8.json) showed a much larger calibration
- * effect (+1.4pp probe, +0.43pp char-F1 at 0.5). That gap is not a
- * property of the int8 quantisation — Python `onnxruntime` on the same
- * int8 file gives 91.0% probe perfect-rate, essentially fp32-equivalent
- * — but of a runtime divergence inside `onnxruntime-web` / WASM that
- * the calibration partially papers over. The reliable fix is to use the
- * `model_fp16.onnx` variant when WebGPU is available (which is what
- * LocalDetector's default `dtype: "auto"` does); int8 should be treated
- * as a low-RAM fallback. See eval/q8_quality_report.md for the
- * diagnostic.
- *
- * `defaultDetector()` returns this with oBias=0.5, so users get the
- * better behavior automatically.
- */
 import type { Detector, Span } from "./base.ts";
 import type { LocalDetectorLoadEvent } from "./local.ts";
 /** Constructor options for {@link CalibratedDetector}. */

package/dist/types/detectors/calibrated.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"calibrated.d.ts","sourceRoot":"","sources":["../../../src/detectors/calibrated.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,OAAO,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAMzD,0DAA0D;AAC1D,MAAM,WAAW,yBAAyB;IACxC,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;;;;;OASG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,iEAAiE;IACjE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yDAAyD;IACzD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wDAAwD;IACxD,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,8EAA8E;IAC9E,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,4EAA4E;IAC5E,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,sBAAsB,KAAK,IAAI,CAAC;CACtD;AA4CD,uDAAuD;AACvD,qBAAa,kBAAmB,YAAW,QAAQ;IACjD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,oEAAoE;IACpE,YAAY,EAAE,MAAM,GAAG,IAAI,CAAQ;IACnC;sEACkE;IAClE,WAAW,EAAE,MAAM,GAAG,IAAI,CAAQ;IAClC,OAAO,CAAC,MAAM,CAA8B;IAC5C,OAAO,CAAC,UAAU,CAAkC;IACpD,OAAO,CAAC,SAAS,CAA8B;IAC/C,OAAO,CAAC,IAAI,CAAK;IACjB,OAAO,CAAC,YAAY,CAA8B;IAClD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAA0C;gBAE3D,IAAI,GAAE,yBAA8B;IAiB1C,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAoDrB,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;YAYxD,cAAc;YA0Bd,aAAa;IA6F3B,OAAO,CAAC,aAAa;CAiBtB"}
+{"version":3,"file":"calibrated.d.ts","sourceRoot":"","sources":["../../../src/detectors/calibrated.ts"],"names":[],"mappings":"AA6BA,OAAO,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAMzD,0DAA0D;AAC1D,MAAM,WAAW,yBAAyB;IACxC,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;;;;;OASG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,iEAAiE;IACjE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yDAAyD;IACzD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wDAAwD;IACxD,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,8EAA8E;IAC9E,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,4EAA4E;IAC5E,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,sBAAsB,KAAK,IAAI,CAAC;CACtD;AA4CD,uDAAuD;AACvD,qBAAa,kBAAmB,YAAW,QAAQ;IACjD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,oEAAoE;IACpE,YAAY,EAAE,MAAM,GAAG,IAAI,CAAQ;IACnC;sEACkE;IAClE,WAAW,EAAE,MAAM,GAAG,IAAI,CAAQ;IAClC,OAAO,CAAC,MAAM,CAA8B;IAC5C,OAAO,CAAC,UAAU,CAAkC;IACpD,OAAO,CAAC,SAAS,CAA8B;IAC/C,OAAO,CAAC,IAAI,CAAK;IACjB,OAAO,CAAC,YAAY,CAA8B;IAClD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAA0C;gBAE3D,IAAI,GAAE,yBAA8B;IAiB1C,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAoDrB,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;YAYxD,cAAc;YA0Bd,aAAa;IAuG3B,OAAO,CAAC,aAAa;CAiBtB"}