gheim 0.1.0

package/dist/cjs/index.cjs

@@ -0,0 +1,707 @@
+var __defProp = Object.defineProperty;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+function __accessProp(key) {
+  return this[key];
+}
+var __toCommonJS = (from) => {
+  var entry = (__moduleCache ??= new WeakMap).get(from), desc;
+  if (entry)
+    return entry;
+  entry = __defProp({}, "__esModule", { value: true });
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (var key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(entry, key))
+        __defProp(entry, key, {
+          get: __accessProp.bind(from, key),
+          enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable
+        });
+  }
+  __moduleCache.set(from, entry);
+  return entry;
+};
+var __moduleCache;
+var __returnValue = (v) => v;
+function __exportSetter(name, newValue) {
+  this[name] = __returnValue.bind(null, newValue);
+}
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, {
+      get: all[name],
+      enumerable: true,
+      configurable: true,
+      set: __exportSetter.bind(all, name)
+    });
+};
+
+// src/index.ts
+var exports_src = {};
+__export(exports_src, {
+  renderSentinel: () => renderSentinel,
+  parseSentinel: () => parseSentinel,
+  labelTag: () => labelTag,
+  isPossibleSentinelPrefix: () => isPossibleSentinelPrefix,
+  defaultDetector: () => defaultDetector,
+  deanonymizeText: () => deanonymizeText,
+  deanonymizeStream: () => deanonymizeStream,
+  anonymizeText: () => anonymizeText,
+  anonymizeMessages: () => anonymizeMessages,
+  VERSION: () => VERSION,
+  StreamDeanonymizer: () => StreamDeanonymizer,
+  Session: () => Session,
+  SENTINEL_RE: () => SENTINEL_RE,
+  RemoteDetector: () => RemoteDetector,
+  REMOTE_DEFAULT_MODEL: () => DEFAULT_MODEL,
+  LocalDetector: () => LocalDetector,
+  LABEL_TO_TAG: () => LABEL_TO_TAG,
+  DEFAULT_BASE_URL: () => DEFAULT_BASE_URL
+});
+module.exports = __toCommonJS(exports_src);
+
+// src/core/sentinels.ts
+var LABEL_TO_TAG = {
+  private_person: "PERSON",
+  private_email: "EMAIL",
+  private_phone: "PHONE",
+  private_address: "ADDRESS",
+  private_url: "URL",
+  private_date: "DATE",
+  account_number: "ACCOUNT",
+  secret: "SECRET"
+};
+var TAG_RE = /[A-Z][A-Z0-9_]*/;
+var INDEX_RE = /\d+/;
+var SENTINEL_RE = new RegExp(`<(?<tag>${TAG_RE.source})_(?<index>${INDEX_RE.source})>`, "g");
+var SENTINEL_FULL_RE = new RegExp(`^<(?<tag>${TAG_RE.source})_(?<index>${INDEX_RE.source})>$`);
+var PARTIAL_SENTINEL_RE = new RegExp(`^<(${TAG_RE.source})?(_(${INDEX_RE.source})?)?>?$`);
+function labelTag(label) {
+  return LABEL_TO_TAG[label] ?? label.toUpperCase();
+}
+function renderSentinel(s) {
+  return `<${s.tag}_${s.index}>`;
+}
+function parseSentinel(s) {
+  const m = SENTINEL_FULL_RE.exec(s);
+  if (!m || !m.groups)
+    return null;
+  return { tag: m.groups.tag, index: Number.parseInt(m.groups.index, 10) };
+}
+function isPossibleSentinelPrefix(buffer) {
+  if (!buffer || buffer[0] !== "<")
+    return false;
+  if (buffer.endsWith(">"))
+    return false;
+  return PARTIAL_SENTINEL_RE.test(buffer);
+}
+
+// src/normalizers.ts
+var import_node_module = require("node:module");
+var _require = null;
+var _requireResolved = false;
+function _getRequire() {
+  if (_requireResolved)
+    return _require;
+  _requireResolved = true;
+  try {
+    _require = import_node_module.createRequire("file:///home/joelbarmettler/projects/gheim/packages/gheim-js/src/normalizers.ts");
+  } catch {
+    _require = null;
+  }
+  return _require;
+}
+function loadSync(name, errMsg) {
+  const req = _getRequire();
+  if (!req)
+    throw new Error(errMsg);
+  try {
+    return req(name);
+  } catch {
+    throw new Error(errMsg);
+  }
+}
+function loadPhoneLib() {
+  return loadSync("libphonenumber-js", "the e164 normalizer requires libphonenumber-js. " + "Install with: bun add libphonenumber-js");
+}
+function e164(opts = {}) {
+  const lib = loadPhoneLib();
+  return (s) => {
+    try {
+      const parsed = lib.parsePhoneNumberFromString(s, opts.region);
+      if (!parsed || !parsed.isValid())
+        return null;
+      return parsed.format("E.164");
+    } catch {
+      return null;
+    }
+  };
+}
+function loadChrono(opts) {
+  const mod = loadSync("chrono-node", "the isoDate normalizer requires chrono-node. " + "Install with: bun add chrono-node");
+  if (opts.locale) {
+    const localeParser = mod[opts.locale];
+    if (localeParser)
+      return localeParser;
+  }
+  return mod;
+}
+function isoDate(opts = {}) {
+  const parser = loadChrono(opts);
+  return (s) => {
+    const d = parser.parseDate(s);
+    if (!d)
+      return null;
+    return d.toISOString().slice(0, 10);
+  };
+}
+function germanTransliteration() {
+  const table = {
+    ä: "ae",
+    Ä: "ae",
+    ö: "oe",
+    Ö: "oe",
+    ü: "ue",
+    Ü: "ue",
+    ß: "ss"
+  };
+  return (s) => {
+    let out = "";
+    for (const ch of s.normalize("NFKC")) {
+      out += table[ch] ?? ch;
+    }
+    return out.toLowerCase().split(/\s+/).filter(Boolean).join(" ");
+  };
+}
+var BUILTINS = {
+  e164: () => e164(),
+  iso_date: () => isoDate(),
+  german: () => germanTransliteration()
+};
+function resolveNormalizer(spec) {
+  if (typeof spec === "function")
+    return spec;
+  const factory = BUILTINS[spec];
+  if (!factory) {
+    throw new Error(`unknown built-in normalizer: ${JSON.stringify(spec)}. ` + `Available: ${Object.keys(BUILTINS).join(", ")}, or pass a callable.`);
+  }
+  return factory();
+}
+
+// src/core/session.ts
+function normalizeSurface(s) {
+  return s.normalize("NFKC").toLowerCase().split(/\s+/).filter(Boolean).join(" ");
+}
+
+class Session {
+  mapping = {};
+  coalesce = new Map;
+  counters = new Map;
+  resolvedNormalizers = {};
+  constructor(opts = {}) {
+    if (opts.normalizers) {
+      for (const [label, spec] of Object.entries(opts.normalizers)) {
+        this.resolvedNormalizers[label] = resolveNormalizer(spec);
+      }
+    }
+  }
+  allocate(label, surface) {
+    const tag = labelTag(label);
+    const norm = this.resolvedNormalizers[label];
+    let normalized = null;
+    if (norm)
+      normalized = norm(surface);
+    if (normalized === null)
+      normalized = normalizeSurface(surface);
+    const key = `${tag}\x00${normalized}`;
+    const existing = this.coalesce.get(key);
+    if (existing)
+      return existing;
+    const next = (this.counters.get(tag) ?? 0) + 1;
+    this.counters.set(tag, next);
+    const sentinel = renderSentinel({ tag, index: next });
+    this.coalesce.set(key, sentinel);
+    this.mapping[sentinel] = surface;
+    return sentinel;
+  }
+  applySpans(text, spans) {
+    if (spans.length === 0)
+      return text;
+    const sorted = [...spans].sort((a, b) => a.start - b.start || b.end - b.start - (a.end - a.start));
+    const merged = mergeAdjacent(text, sorted);
+    const out = [];
+    let cursor = 0;
+    for (const { start, end, label } of merged) {
+      if (start < cursor)
+        continue;
+      if (start < 0 || end > text.length || start >= end)
+        continue;
+      const surface = text.slice(start, end);
+      out.push(text.slice(cursor, start));
+      out.push(this.allocate(label, surface));
+      cursor = end;
+    }
+    out.push(text.slice(cursor));
+    return out.join("");
+  }
+  restore(text) {
+    if (Object.keys(this.mapping).length === 0)
+      return text;
+    return text.replace(SENTINEL_RE, (match) => this.mapping[match] ?? match);
+  }
+  toJSON() {
+    return { mapping: { ...this.mapping } };
+  }
+  static fromJSON(raw, opts = {}) {
+    const s = new Session(opts);
+    s.mapping = { ...raw.mapping };
+    for (const [sentinel, surface] of Object.entries(s.mapping)) {
+      const parsed = parseSentinel(sentinel);
+      if (!parsed)
+        continue;
+      let normalized = null;
+      for (const [label, fn] of Object.entries(s.resolvedNormalizers)) {
+        if (labelTag(label) === parsed.tag) {
+          normalized = fn(surface);
+          if (normalized !== null)
+            break;
+        }
+      }
+      if (normalized === null)
+        normalized = normalizeSurface(surface);
+      s.coalesce.set(`${parsed.tag}\x00${normalized}`, sentinel);
+      s.counters.set(parsed.tag, Math.max(s.counters.get(parsed.tag) ?? 0, parsed.index));
+    }
+    return s;
+  }
+}
+var MERGE_GAP = 1;
+function mergeAdjacent(text, spans) {
+  const out = [];
+  for (const span of spans) {
+    if (span.start < 0 || span.end > text.length || span.start >= span.end)
+      continue;
+    const last = out[out.length - 1];
+    if (!last) {
+      out.push({ start: span.start, end: span.end, label: span.label });
+      continue;
+    }
+    if (span.start < last.end) {
+      continue;
+    }
+    const gap = text.slice(last.end, span.start);
+    if (span.label === last.label && gap.length <= MERGE_GAP && gap.trim() === "") {
+      last.end = span.end;
+    } else {
+      out.push({ start: span.start, end: span.end, label: span.label });
+    }
+  }
+  return out;
+}
+// src/core/stream.ts
+var SENTINEL_AT = /<([A-Z][A-Z0-9_]*)_(\d+)>/y;
+var MAX_SENTINEL_LEN = 64;
+
+class StreamDeanonymizer {
+  mapping;
+  hold = "";
+  constructor(mapping) {
+    this.mapping = mapping;
+  }
+  feed(chunk) {
+    if (!chunk)
+      return "";
+    const buffer = this.hold + chunk;
+    const out = [];
+    let i = 0;
+    const n = buffer.length;
+    while (i < n) {
+      if (buffer[i] !== "<") {
+        const lt = buffer.indexOf("<", i);
+        if (lt === -1) {
+          out.push(buffer.slice(i));
+          i = n;
+          break;
+        }
+        out.push(buffer.slice(i, lt));
+        i = lt;
+      }
+      SENTINEL_AT.lastIndex = i;
+      const m = SENTINEL_AT.exec(buffer);
+      if (m && m.index === i) {
+        const sentinel = m[0];
+        out.push(this.mapping[sentinel] ?? sentinel);
+        i = SENTINEL_AT.lastIndex;
+        continue;
+      }
+      const tail = buffer.slice(i);
+      if (tail.length < MAX_SENTINEL_LEN && isPossibleSentinelPrefix(tail)) {
+        this.hold = tail;
+        return out.join("");
+      }
+      out.push("<");
+      i += 1;
+    }
+    this.hold = "";
+    return out.join("");
+  }
+  flush() {
+    const held = this.hold;
+    this.hold = "";
+    return held;
+  }
+  async* transform(chunks) {
+    for await (const chunk of chunks) {
+      const out = this.feed(chunk);
+      if (out)
+        yield out;
+    }
+    const tail = this.flush();
+    if (tail)
+      yield tail;
+  }
+}
+// src/detectors/base.ts
+function trimSpan(text, start, end) {
+  while (start < end && /\s/.test(text[start] ?? ""))
+    start += 1;
+  while (end > start && /\s/.test(text[end - 1] ?? ""))
+    end -= 1;
+  if (start >= end)
+    return null;
+  return { start, end };
+}
+
+// src/detectors/remote.ts
+var DEFAULT_BASE_URL = "https://api.gheim.ch";
+var DEFAULT_MODEL = "openai/privacy-filter";
+
+class RemoteDetector {
+  baseUrl;
+  apiKey;
+  model;
+  trimWhitespace;
+  fetchImpl;
+  constructor(opts = {}) {
+    const envBase = typeof process !== "undefined" ? process.env?.GHEIM_BASE_URL : undefined;
+    const envKey = typeof process !== "undefined" ? process.env?.GHEIM_API_KEY : undefined;
+    this.baseUrl = (opts.baseUrl ?? envBase ?? DEFAULT_BASE_URL).replace(/\/+$/, "");
+    this.apiKey = opts.apiKey ?? envKey;
+    this.model = opts.model ?? DEFAULT_MODEL;
+    this.trimWhitespace = opts.trimWhitespace ?? true;
+    this.fetchImpl = opts.fetch ?? fetch;
+  }
+  async detect(text, opts) {
+    if (!text)
+      return [];
+    const headers = { "Content-Type": "application/json" };
+    if (this.apiKey)
+      headers.Authorization = `Bearer ${this.apiKey}`;
+    const resp = await this.fetchImpl(`${this.baseUrl}/v1/detect`, {
+      method: "POST",
+      headers,
+      body: JSON.stringify({ model: opts?.model ?? this.model, text })
+    });
+    if (!resp.ok) {
+      throw new Error(`gheim-server ${resp.status}: ${await resp.text()}`);
+    }
+    const data = await resp.json();
+    const out = [];
+    for (const item of data.spans ?? []) {
+      let { start, end } = item;
+      if (this.trimWhitespace) {
+        const trimmed = trimSpan(text, start, end);
+        if (!trimmed)
+          continue;
+        ({ start, end } = trimmed);
+      }
+      out.push({
+        label: item.label,
+        start,
+        end,
+        text: text.slice(start, end),
+        score: item.score
+      });
+    }
+    return out;
+  }
+}
+// src/detectors/local.ts
+var DEFAULT_MODEL2 = "joelbarmettler/gheim-ch-560m";
+var DEFAULT_CHUNK_TOKENS = 510;
+var DEFAULT_OVERLAP_TOKENS = 32;
+var DEFAULT_BATCH_SIZE = 8;
+
+class LocalDetector {
+  model;
+  device;
+  dtype;
+  aggregationStrategy;
+  chunkTokens;
+  chunkOverlapTokens;
+  batchSize;
+  _pipeline = null;
+  _loadPromise = null;
+  constructor(opts = {}) {
+    this.model = opts.model ?? DEFAULT_MODEL2;
+    this.device = opts.device ?? "auto";
+    this.dtype = opts.dtype ?? "fp32";
+    this.aggregationStrategy = opts.aggregationStrategy ?? "simple";
+    this.chunkTokens = opts.chunkTokens ?? DEFAULT_CHUNK_TOKENS;
+    this.chunkOverlapTokens = opts.chunkOverlapTokens ?? DEFAULT_OVERLAP_TOKENS;
+    this.batchSize = opts.batchSize ?? DEFAULT_BATCH_SIZE;
+    if (this.chunkOverlapTokens >= this.chunkTokens) {
+      throw new Error(`chunkOverlapTokens (${this.chunkOverlapTokens}) must be smaller than chunkTokens (${this.chunkTokens})`);
+    }
+    if (opts.pipelineInstance) {
+      this._pipeline = opts.pipelineInstance;
+    }
+  }
+  async load() {
+    if (this._pipeline)
+      return this._pipeline;
+    if (this._loadPromise)
+      return this._loadPromise;
+    this._loadPromise = (async () => {
+      let mod;
+      try {
+        mod = await import("@huggingface/transformers");
+      } catch (e) {
+        throw new Error("LocalDetector requires `@huggingface/transformers`. Install with: bun add @huggingface/transformers");
+      }
+      const pipe = await mod.pipeline("token-classification", this.model, {
+        device: this.device,
+        dtype: this.dtype
+      });
+      this._pipeline = pipe;
+      return pipe;
+    })();
+    return this._loadPromise;
+  }
+  async detect(text, opts) {
+    if (opts?.model && opts.model !== this.model) {
+      throw new Error(`LocalDetector was constructed with model='${this.model}'; per-call model override ('${opts.model}') not supported.`);
+    }
+    if (!text)
+      return [];
+    const results = await this.detectBatch([text]);
+    return results[0] ?? [];
+  }
+  async detectBatch(texts, opts) {
+    if (opts?.model && opts.model !== this.model) {
+      throw new Error(`LocalDetector was constructed with model='${this.model}'; per-call model override ('${opts.model}') not supported.`);
+    }
+    if (texts.length === 0)
+      return [];
+    const pipe = await this.load();
+    const tokenizer = pipe.tokenizer;
+    const plan = [];
+    for (let i = 0;i < texts.length; i++) {
+      const text = texts[i] ?? "";
+      if (!text)
+        continue;
+      const chunks = await this.chunkText(text, tokenizer);
+      for (const c of chunks)
+        plan.push({ inputIdx: i, charOffset: c.offset, text: c.text });
+    }
+    const out = texts.map(() => []);
+    if (plan.length === 0)
+      return out;
+    const batchInput = plan.map((p) => p.text);
+    const raw = await pipe(batchInput, {
+      aggregation_strategy: this.aggregationStrategy,
+      batch_size: this.batchSize
+    });
+    const seen = texts.map(() => new Set);
+    for (let p = 0;p < plan.length; p++) {
+      const planEntry = plan[p];
+      if (!planEntry)
+        continue;
+      const { inputIdx, charOffset, text: chunkText } = planEntry;
+      const fullText = texts[inputIdx] ?? "";
+      const inputSeen = seen[inputIdx];
+      const inputOut = out[inputIdx];
+      if (!inputSeen || !inputOut)
+        continue;
+      const items = raw[p] ?? [];
+      let cursor = 0;
+      for (const item of items) {
+        const label = String(item.entity_group ?? item.entity ?? "unknown");
+        const score = typeof item.score === "number" ? item.score : 1;
+        let localStart;
+        let localEnd;
+        if (item.start === undefined || item.end === undefined) {
+          const surface = normalizeWord(item.word ?? "");
+          if (!surface)
+            continue;
+          const idx = chunkText.indexOf(surface, cursor);
+          if (idx === -1)
+            continue;
+          localStart = idx;
+          localEnd = idx + surface.length;
+          cursor = localEnd;
+        } else {
+          localStart = item.start;
+          localEnd = item.end;
+        }
+        const start = localStart + charOffset;
+        const end = localEnd + charOffset;
+        const key = `${start}|${end}|${label}`;
+        if (inputSeen.has(key))
+          continue;
+        inputSeen.add(key);
+        inputOut.push({
+          label,
+          start,
+          end,
+          text: fullText.slice(start, end),
+          score
+        });
+      }
+    }
+    for (const spans of out) {
+      spans.sort((a, b) => a.start - b.start || a.end - b.end);
+    }
+    return out;
+  }
+  async chunkText(text, tokenizer) {
+    if (tokenizer) {
+      const tokenChunks = await chunkByTokens(text, tokenizer, this.chunkTokens, this.chunkOverlapTokens);
+      if (tokenChunks)
+        return tokenChunks;
+    }
+    return chunkByChars(text, 1500, 200);
+  }
+}
+async function chunkByTokens(text, tokenizer, chunkTokens, overlapTokens) {
+  let enc;
+  try {
+    enc = await tokenizer(text, {
+      return_offsets_mapping: true,
+      add_special_tokens: false,
+      truncation: false
+    });
+  } catch {
+    return null;
+  }
+  const offsets = readOffsetMapping(enc);
+  if (!offsets)
+    return null;
+  const real = offsets.filter(([a, b]) => !(a === 0 && b === 0));
+  if (real.length === 0)
+    return [{ offset: 0, text }];
+  if (real.length <= chunkTokens)
+    return [{ offset: 0, text }];
+  const stride = Math.max(1, chunkTokens - overlapTokens);
+  const out = [];
+  for (let s = 0;s < real.length; s += stride) {
+    const e = Math.min(s + chunkTokens, real.length);
+    const startPair = real[s];
+    const endPair = real[e - 1];
+    if (!startPair || !endPair)
+      break;
+    const charStart = startPair[0];
+    const charEnd = endPair[1];
+    out.push({ offset: charStart, text: text.slice(charStart, charEnd) });
+    if (e >= real.length)
+      break;
+  }
+  return out;
+}
+function readOffsetMapping(enc) {
+  const om = enc.offset_mapping;
+  if (!om)
+    return null;
+  if (typeof om.tolist === "function") {
+    const list = om.tolist();
+    if (!Array.isArray(list) || list.length === 0)
+      return [];
+    const inner = list[0];
+    if (Array.isArray(inner) && Array.isArray(inner[0])) {
+      return inner;
+    }
+    return list;
+  }
+  if (om.data && om.dims && om.dims.length >= 2) {
+    const dims = om.dims;
+    const pairs = dims[dims.length - 2] ?? 0;
+    const data = om.data;
+    const out = [];
+    for (let i = 0;i < pairs; i++) {
+      out.push([Number(data[i * 2]), Number(data[i * 2 + 1])]);
+    }
+    return out;
+  }
+  return null;
+}
+function chunkByChars(text, chunkChars, overlapChars) {
+  if (text.length <= chunkChars)
+    return [{ offset: 0, text }];
+  const stride = Math.max(1, chunkChars - overlapChars);
+  const out = [];
+  for (let s = 0;s < text.length; s += stride) {
+    const e = Math.min(s + chunkChars, text.length);
+    out.push({ offset: s, text: text.slice(s, e) });
+    if (e >= text.length)
+      break;
+  }
+  return out;
+}
+function normalizeWord(word) {
+  return word.replace(/##/g, "").trim();
+}
+// src/detectors/index.ts
+function defaultDetector() {
+  const hasKey = typeof process !== "undefined" && Boolean(process.env?.GHEIM_API_KEY);
+  return hasKey ? new RemoteDetector : new LocalDetector;
+}
+// src/plain.ts
+function resolve(session, detector) {
+  const det = detector ?? session.detector;
+  if (!det) {
+    throw new Error("No detector — pass one to anonymizeText or attach to the session.");
+  }
+  return det;
+}
+async function anonymizeText(text, session, opts) {
+  const det = resolve(session, opts?.detector);
+  const spans = await det.detect(text, { model: opts?.model });
+  return session.applySpans(text, spans);
+}
+function deanonymizeText(text, session) {
+  return session.restore(text);
+}
+async function anonymizeMessages(messages, session, opts) {
+  const det = resolve(session, opts?.detector);
+  const out = [];
+  for (const msg of messages) {
+    const content = msg.content;
+    if (typeof content === "string") {
+      const redacted = await anonymizeText(content, session, { detector: det, model: opts?.model });
+      out.push({ ...msg, content: redacted });
+    } else if (Array.isArray(content)) {
+      const parts = await Promise.all(content.map(async (p) => {
+        const part = p;
+        if (part && part.type === "text" && typeof part.text === "string") {
+          const redacted = await anonymizeText(part.text, session, {
+            detector: det,
+            model: opts?.model
+          });
+          return { ...part, text: redacted };
+        }
+        return p;
+      }));
+      out.push({ ...msg, content: parts });
+    } else {
+      out.push({ ...msg });
+    }
+  }
+  return out;
+}
+async function* deanonymizeStream(chunks, session) {
+  const buf = new StreamDeanonymizer(session.mapping);
+  yield* buf.transform(chunks);
+}
+
+// src/index.ts
+var VERSION = "0.1.0";