gheim 0.1.5 → 0.1.6

package/dist/cjs/index.cjs

@@ -157,8 +157,8 @@ class Session {
     }
     out.push(text.slice(cursor));
     let redacted = out.join("");
-    redacted = recoverDuplicateOccurrences(redacted, this.mapping);
-    return { redacted, merged };
+    const recovered = recoverDuplicateOccurrences(text, redacted, merged, this.mapping);
+    return { redacted: recovered.redacted, merged: recovered.merged };
   }
   restore(text) {
     if (Object.keys(this.mapping).length === 0)
@@ -217,14 +217,36 @@ function mergeAdjacent(text, spans) {
 function escapeRegex(s) {
   return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
 }
-function recoverDuplicateOccurrences(redacted, mapping) {
+function recoverDuplicateOccurrences(text, redacted, merged, mapping) {
+  const surfaceToLabel = new Map;
+  for (const m of merged)
+    surfaceToLabel.set(text.slice(m.start, m.end), m.label);
+  const covered = merged.map((m) => [m.start, m.end]);
+  const overlaps = (s, e) => covered.some(([cs, ce]) => s < ce && e > cs);
   const entries = Object.entries(mapping).filter(([, surface]) => surface && surface.length > 0).sort((a, b) => b[1].length - a[1].length);
   let result = redacted;
+  const additional = [];
   for (const [sentinel, surface] of entries) {
-    const pattern = new RegExp(`(?<![\\p{L}\\p{N}_])${escapeRegex(surface)}(?![\\p{L}\\p{N}_])`, "gu");
-    result = result.replace(pattern, sentinel);
+    const boundaryPattern = `(?<![\\p{L}\\p{N}_])${escapeRegex(surface)}(?![\\p{L}\\p{N}_])`;
+    result = result.replace(new RegExp(boundaryPattern, "gu"), sentinel);
+    const label = surfaceToLabel.get(surface);
+    if (!label)
+      continue;
+    const origRe = new RegExp(boundaryPattern, "gu");
+    let match;
+    while ((match = origRe.exec(text)) !== null) {
+      const start = match.index;
+      const end = start + surface.length;
+      if (overlaps(start, end))
+        continue;
+      additional.push({ start, end, label });
+      covered.push([start, end]);
+    }
   }
-  return result;
+  if (additional.length === 0)
+    return { redacted: result, merged };
+  const augmented = [...merged, ...additional].sort((a, b) => a.start - b.start);
+  return { redacted: result, merged: augmented };
 }
 // src/core/stream.ts
 var SENTINEL_AT = /<([A-Z][A-Z0-9_]*)_(\d+)>/y;
@@ -1168,4 +1190,4 @@ async function* deanonymizeStream(chunks, session) {
 }
 
 // src/index.ts
-var VERSION = "0.1.5";
+var VERSION = "0.1.6";

package/dist/cjs/openai/index.cjs

@@ -1086,8 +1086,8 @@ class Session {
     }
     out.push(text.slice(cursor));
     let redacted = out.join("");
-    redacted = recoverDuplicateOccurrences(redacted, this.mapping);
-    return { redacted, merged };
+    const recovered = recoverDuplicateOccurrences(text, redacted, merged, this.mapping);
+    return { redacted: recovered.redacted, merged: recovered.merged };
   }
   restore(text) {
     if (Object.keys(this.mapping).length === 0)
@@ -1146,14 +1146,36 @@ function mergeAdjacent(text, spans) {
 function escapeRegex(s) {
   return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
 }
-function recoverDuplicateOccurrences(redacted, mapping) {
+function recoverDuplicateOccurrences(text, redacted, merged, mapping) {
+  const surfaceToLabel = new Map;
+  for (const m of merged)
+    surfaceToLabel.set(text.slice(m.start, m.end), m.label);
+  const covered = merged.map((m) => [m.start, m.end]);
+  const overlaps = (s, e) => covered.some(([cs, ce]) => s < ce && e > cs);
   const entries = Object.entries(mapping).filter(([, surface]) => surface && surface.length > 0).sort((a, b) => b[1].length - a[1].length);
   let result = redacted;
+  const additional = [];
   for (const [sentinel, surface] of entries) {
-    const pattern = new RegExp(`(?<![\\p{L}\\p{N}_])${escapeRegex(surface)}(?![\\p{L}\\p{N}_])`, "gu");
-    result = result.replace(pattern, sentinel);
+    const boundaryPattern = `(?<![\\p{L}\\p{N}_])${escapeRegex(surface)}(?![\\p{L}\\p{N}_])`;
+    result = result.replace(new RegExp(boundaryPattern, "gu"), sentinel);
+    const label = surfaceToLabel.get(surface);
+    if (!label)
+      continue;
+    const origRe = new RegExp(boundaryPattern, "gu");
+    let match;
+    while ((match = origRe.exec(text)) !== null) {
+      const start = match.index;
+      const end = start + surface.length;
+      if (overlaps(start, end))
+        continue;
+      additional.push({ start, end, label });
+      covered.push([start, end]);
+    }
   }
-  return result;
+  if (additional.length === 0)
+    return { redacted: result, merged };
+  const augmented = [...merged, ...additional].sort((a, b) => a.start - b.start);
+  return { redacted: result, merged: augmented };
 }
 
 // src/openai/_base.ts

package/dist/esm/index.js

@@ -100,8 +100,8 @@ class Session {
     }
     out.push(text.slice(cursor));
     let redacted = out.join("");
-    redacted = recoverDuplicateOccurrences(redacted, this.mapping);
-    return { redacted, merged };
+    const recovered = recoverDuplicateOccurrences(text, redacted, merged, this.mapping);
+    return { redacted: recovered.redacted, merged: recovered.merged };
   }
   restore(text) {
     if (Object.keys(this.mapping).length === 0)
@@ -160,14 +160,36 @@ function mergeAdjacent(text, spans) {
 function escapeRegex(s) {
   return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
 }
-function recoverDuplicateOccurrences(redacted, mapping) {
+function recoverDuplicateOccurrences(text, redacted, merged, mapping) {
+  const surfaceToLabel = new Map;
+  for (const m of merged)
+    surfaceToLabel.set(text.slice(m.start, m.end), m.label);
+  const covered = merged.map((m) => [m.start, m.end]);
+  const overlaps = (s, e) => covered.some(([cs, ce]) => s < ce && e > cs);
   const entries = Object.entries(mapping).filter(([, surface]) => surface && surface.length > 0).sort((a, b) => b[1].length - a[1].length);
   let result = redacted;
+  const additional = [];
   for (const [sentinel, surface] of entries) {
-    const pattern = new RegExp(`(?<![\\p{L}\\p{N}_])${escapeRegex(surface)}(?![\\p{L}\\p{N}_])`, "gu");
-    result = result.replace(pattern, sentinel);
+    const boundaryPattern = `(?<![\\p{L}\\p{N}_])${escapeRegex(surface)}(?![\\p{L}\\p{N}_])`;
+    result = result.replace(new RegExp(boundaryPattern, "gu"), sentinel);
+    const label = surfaceToLabel.get(surface);
+    if (!label)
+      continue;
+    const origRe = new RegExp(boundaryPattern, "gu");
+    let match;
+    while ((match = origRe.exec(text)) !== null) {
+      const start = match.index;
+      const end = start + surface.length;
+      if (overlaps(start, end))
+        continue;
+      additional.push({ start, end, label });
+      covered.push([start, end]);
+    }
   }
-  return result;
+  if (additional.length === 0)
+    return { redacted: result, merged };
+  const augmented = [...merged, ...additional].sort((a, b) => a.start - b.start);
+  return { redacted: result, merged: augmented };
 }
 // src/core/stream.ts
 var SENTINEL_AT = /<([A-Z][A-Z0-9_]*)_(\d+)>/y;
@@ -1111,7 +1133,7 @@ async function* deanonymizeStream(chunks, session) {
 }
 
 // src/index.ts
-var VERSION = "0.1.5";
+var VERSION = "0.1.6";
 export {
   renderSentinel,
   parseSentinel,

package/dist/esm/openai/index.js

@@ -1043,8 +1043,8 @@ class Session {
     }
     out.push(text.slice(cursor));
     let redacted = out.join("");
-    redacted = recoverDuplicateOccurrences(redacted, this.mapping);
-    return { redacted, merged };
+    const recovered = recoverDuplicateOccurrences(text, redacted, merged, this.mapping);
+    return { redacted: recovered.redacted, merged: recovered.merged };
   }
   restore(text) {
     if (Object.keys(this.mapping).length === 0)
@@ -1103,14 +1103,36 @@ function mergeAdjacent(text, spans) {
 function escapeRegex(s) {
   return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
 }
-function recoverDuplicateOccurrences(redacted, mapping) {
+function recoverDuplicateOccurrences(text, redacted, merged, mapping) {
+  const surfaceToLabel = new Map;
+  for (const m of merged)
+    surfaceToLabel.set(text.slice(m.start, m.end), m.label);
+  const covered = merged.map((m) => [m.start, m.end]);
+  const overlaps = (s, e) => covered.some(([cs, ce]) => s < ce && e > cs);
   const entries = Object.entries(mapping).filter(([, surface]) => surface && surface.length > 0).sort((a, b) => b[1].length - a[1].length);
   let result = redacted;
+  const additional = [];
   for (const [sentinel, surface] of entries) {
-    const pattern = new RegExp(`(?<![\\p{L}\\p{N}_])${escapeRegex(surface)}(?![\\p{L}\\p{N}_])`, "gu");
-    result = result.replace(pattern, sentinel);
+    const boundaryPattern = `(?<![\\p{L}\\p{N}_])${escapeRegex(surface)}(?![\\p{L}\\p{N}_])`;
+    result = result.replace(new RegExp(boundaryPattern, "gu"), sentinel);
+    const label = surfaceToLabel.get(surface);
+    if (!label)
+      continue;
+    const origRe = new RegExp(boundaryPattern, "gu");
+    let match;
+    while ((match = origRe.exec(text)) !== null) {
+      const start = match.index;
+      const end = start + surface.length;
+      if (overlaps(start, end))
+        continue;
+      additional.push({ start, end, label });
+      covered.push([start, end]);
+    }
   }
-  return result;
+  if (additional.length === 0)
+    return { redacted: result, merged };
+  const augmented = [...merged, ...additional].sort((a, b) => a.start - b.start);
+  return { redacted: result, merged: augmented };
 }
 
 // src/openai/_base.ts

package/dist/types/core/session.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../../src/core/session.ts"],"names":[],"mappings":"AAAA,kDAAkD;AAClD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AAGjD;;;gEAGgE;AAChE,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;AAkBtD,uCAAuC;AACvC,MAAM,WAAW,cAAc;IAC7B;;;;;;;;;;OAUG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;CAC1C;AAED,qBAAa,OAAO;IAClB,8DAA8D;IAC9D,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAM;IACrC,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,mBAAmB,CAAkC;gBAEjD,IAAI,GAAE,cAAmB;IAQrC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM;IAiBhD,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GAAG,MAAM;IAIvE;;;;;;;OAOG;IACH,kBAAkB,CAChB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GACpC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,UAAU,EAAE,CAAA;KAAE;IA+B7C,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAK7B,MAAM,IAAI;QAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE;IAI7C,MAAM,CAAC,QAAQ,CACb,GAAG,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,EACxC,IAAI,GAAE,cAAmB,GACxB,OAAO;CAqBX;AAID,8EAA8E;AAC9E,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAC3B,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GACpC,UAAU,EAAE,CAqBd"}
+{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../../src/core/session.ts"],"names":[],"mappings":"AAAA,kDAAkD;AAClD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AAGjD;;;gEAGgE;AAChE,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;AAkBtD,uCAAuC;AACvC,MAAM,WAAW,cAAc;IAC7B;;;;;;;;;;OAUG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;CAC1C;AAED,qBAAa,OAAO;IAClB,8DAA8D;IAC9D,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAM;IACrC,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,mBAAmB,CAAkC;gBAEjD,IAAI,GAAE,cAAmB;IAQrC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM;IAiBhD,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GAAG,MAAM;IAIvE;;;;;;;OAOG;IACH,kBAAkB,CAChB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GACpC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,UAAU,EAAE,CAAA;KAAE;IA8B7C,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAK7B,MAAM,IAAI;QAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE;IAI7C,MAAM,CAAC,QAAQ,CACb,GAAG,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,EACxC,IAAI,GAAE,cAAmB,GACxB,OAAO;CAqBX;AAID,8EAA8E;AAC9E,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAC3B,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GACpC,UAAU,EAAE,CAqBd"}

package/dist/types/index.d.ts

@@ -17,7 +17,7 @@ export { DEFAULT_BASE_URL, REMOTE_DEFAULT_MODEL, RemoteDetector, LocalDetector,
 export type { RemoteDetectorOptions, LocalDetectorOptions, LocalDetectorLoadEvent, CompositeDetectorOptions, CalibratedDetectorOptions, } from "./detectors/index.ts";
 export { anonymizeMessages, anonymizeText, deanonymizeStream, deanonymizeText, } from "./plain.ts";
 export type { ChatMessage } from "./plain.ts";
-export declare const VERSION = "0.1.5";
+export declare const VERSION = "0.1.6";
 export { mergeAdjacent } from "./core/session.ts";
 export type { MergedSpan } from "./core/session.ts";
 //# sourceMappingURL=index.d.ts.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gheim",
-  "version": "0.1.5",
+  "version": "0.1.6",
   "description": "PII round-trip for LLM APIs: anonymize before the request, de-anonymize the stream on the way back.",
   "license": "Apache-2.0",
   "author": "Joel Barmettler <joel.barmettler@gmail.com>",