gheim 0.1.3 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/esm/index.js CHANGED
@@ -99,7 +99,9 @@ class Session {
99
99
  cursor = end;
100
100
  }
101
101
  out.push(text.slice(cursor));
102
- return { redacted: out.join(""), merged };
102
+ let redacted = out.join("");
103
+ redacted = recoverDuplicateOccurrences(redacted, this.mapping);
104
+ return { redacted, merged };
103
105
  }
104
106
  restore(text) {
105
107
  if (Object.keys(this.mapping).length === 0)
@@ -155,6 +157,18 @@ function mergeAdjacent(text, spans) {
155
157
  }
156
158
  return out;
157
159
  }
160
+ function escapeRegex(s) {
161
+ return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
162
+ }
163
+ function recoverDuplicateOccurrences(redacted, mapping) {
164
+ const entries = Object.entries(mapping).filter(([, surface]) => surface && surface.length > 0).sort((a, b) => b[1].length - a[1].length);
165
+ let result = redacted;
166
+ for (const [sentinel, surface] of entries) {
167
+ const pattern = new RegExp(`(?<![\\p{L}\\p{N}_])${escapeRegex(surface)}(?![\\p{L}\\p{N}_])`, "gu");
168
+ result = result.replace(pattern, sentinel);
169
+ }
170
+ return result;
171
+ }
158
172
  // src/core/stream.ts
159
173
  var SENTINEL_AT = /<([A-Z][A-Z0-9_]*)_(\d+)>/y;
160
174
  var MAX_SENTINEL_LEN = 64;
@@ -772,10 +786,281 @@ class CompositeDetector {
772
786
  return out;
773
787
  }
774
788
  }
789
+ // src/detectors/calibrated.ts
790
+ var DEFAULT_MODEL3 = "joelbarmettler/gheim-ch-560m";
791
+ var DEFAULT_CHUNK_TOKENS2 = 510;
792
+ var DEFAULT_OVERLAP_TOKENS2 = 32;
793
+ function trimSpan2(text, start, end) {
794
+ while (start < end && /\s/.test(text[start] ?? ""))
795
+ start++;
796
+ while (end > start && /\s/.test(text[end - 1] ?? ""))
797
+ end--;
798
+ return [start, end];
799
+ }
800
+
801
+ class CalibratedDetector {
802
+ model;
803
+ oBias;
804
+ device;
805
+ dtype;
806
+ chunkTokens;
807
+ chunkOverlapTokens;
808
+ trimWhitespace;
809
+ actualDevice = null;
810
+ _model = null;
811
+ _tokenizer = null;
812
+ _id2label = {};
813
+ _oId = 0;
814
+ _loadPromise = null;
815
+ _onProgress;
816
+ constructor(opts = {}) {
817
+ this.model = opts.model ?? DEFAULT_MODEL3;
818
+ this.oBias = opts.oBias ?? 0.5;
819
+ this.device = opts.device ?? "auto";
820
+ this.dtype = opts.dtype ?? "fp32";
821
+ this.chunkTokens = opts.chunkTokens ?? DEFAULT_CHUNK_TOKENS2;
822
+ this.chunkOverlapTokens = opts.chunkOverlapTokens ?? DEFAULT_OVERLAP_TOKENS2;
823
+ this.trimWhitespace = opts.trimWhitespace ?? true;
824
+ this._onProgress = opts.onProgress;
825
+ if (this.chunkOverlapTokens >= this.chunkTokens) {
826
+ throw new Error(`chunkOverlapTokens (${this.chunkOverlapTokens}) must be smaller than ` + `chunkTokens (${this.chunkTokens})`);
827
+ }
828
+ }
829
+ async load() {
830
+ if (this._model)
831
+ return;
832
+ if (this._loadPromise)
833
+ return this._loadPromise;
834
+ this._loadPromise = (async () => {
835
+ const mod = await loadTransformersJs2();
836
+ let preferredDevice = this.device;
837
+ if (this.device === "auto") {
838
+ const nav = typeof navigator !== "undefined" ? navigator : null;
839
+ preferredDevice = "wasm";
840
+ if (nav?.gpu?.requestAdapter) {
841
+ try {
842
+ const adapter = await nav.gpu.requestAdapter();
843
+ if (adapter)
844
+ preferredDevice = "webgpu";
845
+ } catch {}
846
+ }
847
+ }
848
+ const tokenizer = await mod.AutoTokenizer.from_pretrained(this.model);
849
+ const model = await mod.AutoModelForTokenClassification.from_pretrained(this.model, {
850
+ device: preferredDevice,
851
+ dtype: this.dtype,
852
+ progress_callback: this._onProgress ? (raw) => this._emitProgress(raw, preferredDevice) : undefined
853
+ });
854
+ this._tokenizer = tokenizer;
855
+ this._model = model;
856
+ this._id2label = model.config.id2label;
857
+ const label2id = model.config.label2id;
858
+ this._oId = label2id?.["O"] ?? 0;
859
+ this.actualDevice = preferredDevice;
860
+ })();
861
+ return this._loadPromise;
862
+ }
863
+ async detect(text, opts) {
864
+ if (opts?.model && opts.model !== this.model) {
865
+ throw new Error(`CalibratedDetector was constructed with model='${this.model}'; ` + `per-call model override ('${opts.model}') not supported.`);
866
+ }
867
+ if (!text)
868
+ return [];
869
+ await this.load();
870
+ return this._detectChunked(text);
871
+ }
872
+ async _detectChunked(text) {
873
+ if (text.length <= 1500) {
874
+ return await this._detectWindow(text, 0);
875
+ }
876
+ const stride = 1500 - 200;
877
+ const seen = new Set;
878
+ const out = [];
879
+ for (let s = 0;s < text.length; s += stride) {
880
+ const e = Math.min(s + 1500, text.length);
881
+ const sub = text.slice(s, e);
882
+ const subSpans = await this._detectWindow(sub, s);
883
+ for (const sp of subSpans) {
884
+ const key = `${sp.start}|${sp.end}|${sp.label}`;
885
+ if (seen.has(key))
886
+ continue;
887
+ seen.add(key);
888
+ out.push(sp);
889
+ }
890
+ if (e >= text.length)
891
+ break;
892
+ }
893
+ out.sort((a, b) => a.start - b.start || a.end - b.end);
894
+ return out;
895
+ }
896
+ async _detectWindow(text, charOffset) {
897
+ const tokenizer = this._tokenizer;
898
+ const model = this._model;
899
+ const enc = await tokenizer([text], {
900
+ padding: true,
901
+ truncation: true,
902
+ max_length: this.chunkTokens + 2
903
+ });
904
+ const inputs = {};
905
+ if (enc.input_ids)
906
+ inputs["input_ids"] = enc.input_ids;
907
+ if (enc.attention_mask)
908
+ inputs["attention_mask"] = enc.attention_mask;
909
+ const out = await model(inputs);
910
+ const logits = readLogits(out.logits);
911
+ if (this.oBias !== 0) {
912
+ for (let t = 0;t < logits.length; t++) {
913
+ const row = logits[t];
914
+ if (row === undefined)
915
+ continue;
916
+ const cur = row[this._oId];
917
+ if (cur !== undefined) {
918
+ row[this._oId] = cur - this.oBias;
919
+ }
920
+ }
921
+ }
922
+ const inputIds = readInputIds(enc.input_ids);
923
+ const tokenLabels = [];
924
+ for (let t = 0;t < logits.length; t++) {
925
+ const row = logits[t];
926
+ const id = inputIds[t];
927
+ if (!row || id === undefined)
928
+ continue;
929
+ let maxIdx = 0;
930
+ let maxVal = row[0];
931
+ for (let i = 1;i < row.length; i++) {
932
+ if (row[i] > maxVal) {
933
+ maxVal = row[i];
934
+ maxIdx = i;
935
+ }
936
+ }
937
+ tokenLabels.push({ id, label: this._id2label[maxIdx] ?? "O" });
938
+ }
939
+ const spans = [];
940
+ let cursor = 0;
941
+ let groupIds = [];
942
+ let groupCat = null;
943
+ const flush = () => {
944
+ if (groupCat === null || groupIds.length === 0)
945
+ return;
946
+ const surface = normalizeDecoded(tokenizer.decode(groupIds, { skip_special_tokens: true }));
947
+ groupIds = [];
948
+ const cat = groupCat;
949
+ groupCat = null;
950
+ if (!surface)
951
+ return;
952
+ const idx = text.indexOf(surface, cursor);
953
+ if (idx === -1)
954
+ return;
955
+ let s = idx;
956
+ let e = idx + surface.length;
957
+ cursor = e;
958
+ if (this.trimWhitespace)
959
+ [s, e] = trimSpan2(text, s, e);
960
+ if (s >= e)
961
+ return;
962
+ spans.push({
963
+ start: s + charOffset,
964
+ end: e + charOffset,
965
+ label: cat,
966
+ text: text.slice(s, e),
967
+ score: 1
968
+ });
969
+ };
970
+ for (const { id, label } of tokenLabels) {
971
+ const cat = label === "O" ? null : label.split("-", 2)[1] ?? null;
972
+ if (cat === groupCat && cat !== null) {
973
+ groupIds.push(id);
974
+ } else {
975
+ flush();
976
+ if (cat !== null) {
977
+ groupIds.push(id);
978
+ groupCat = cat;
979
+ }
980
+ }
981
+ }
982
+ flush();
983
+ return spans;
984
+ }
985
+ _emitProgress(raw, device) {
986
+ if (!this._onProgress)
987
+ return;
988
+ const event = {
989
+ phase: raw.status ?? "progress",
990
+ device
991
+ };
992
+ if (raw.file)
993
+ event.file = raw.file;
994
+ if (raw.status === "progress") {
995
+ if (raw.loaded != null)
996
+ event.loadedBytes = raw.loaded;
997
+ if (raw.total != null)
998
+ event.totalBytes = raw.total;
999
+ if (raw.progress != null) {
1000
+ const norm = raw.progress > 1 ? raw.progress / 100 : raw.progress;
1001
+ event.fraction = Math.max(0, Math.min(1, norm));
1002
+ }
1003
+ }
1004
+ this._onProgress(event);
1005
+ }
1006
+ }
1007
+ async function loadTransformersJs2() {
1008
+ try {
1009
+ return await import("@huggingface/transformers");
1010
+ } catch {
1011
+ throw new Error("CalibratedDetector requires `@huggingface/transformers`. Install with: bun add @huggingface/transformers");
1012
+ }
1013
+ }
1014
+ function readInputIds(tensor) {
1015
+ if (!tensor)
1016
+ return [];
1017
+ if (typeof tensor.tolist === "function") {
1018
+ const list = tensor.tolist();
1019
+ if (Array.isArray(list) && Array.isArray(list[0])) {
1020
+ return list[0].map((x) => Number(x));
1021
+ }
1022
+ if (Array.isArray(list))
1023
+ return list.map((x) => Number(x));
1024
+ }
1025
+ const dims = tensor.dims;
1026
+ const seq = dims[dims.length - 1] ?? 0;
1027
+ const data = tensor.data;
1028
+ const out = [];
1029
+ for (let i = 0;i < seq; i++)
1030
+ out.push(Number(data[i]));
1031
+ return out;
1032
+ }
1033
+ function normalizeDecoded(s) {
1034
+ return s.replace(/##/g, "").trim();
1035
+ }
1036
+ function readLogits(tensor) {
1037
+ if (typeof tensor.tolist === "function") {
1038
+ const list = tensor.tolist();
1039
+ if (Array.isArray(list) && Array.isArray(list[0]) && Array.isArray(list[0][0])) {
1040
+ return list[0];
1041
+ }
1042
+ if (Array.isArray(list) && Array.isArray(list[0])) {
1043
+ return list;
1044
+ }
1045
+ }
1046
+ const dims = tensor.dims;
1047
+ const seq = dims[dims.length - 2] ?? 0;
1048
+ const cls = dims[dims.length - 1] ?? 0;
1049
+ const data = tensor.data;
1050
+ const out = [];
1051
+ for (let t = 0;t < seq; t++) {
1052
+ const row = [];
1053
+ for (let c = 0;c < cls; c++) {
1054
+ row.push(Number(data[t * cls + c]));
1055
+ }
1056
+ out.push(row);
1057
+ }
1058
+ return out;
1059
+ }
775
1060
  // src/detectors/index.ts
776
1061
  function defaultDetector() {
777
1062
  const hasKey = typeof process !== "undefined" && Boolean(process.env?.GHEIM_API_KEY);
778
- return hasKey ? new RemoteDetector : new LocalDetector;
1063
+ return hasKey ? new RemoteDetector : new CalibratedDetector;
779
1064
  }
780
1065
  // src/plain.ts
781
1066
  function resolve(session, detector) {
@@ -826,7 +1111,7 @@ async function* deanonymizeStream(chunks, session) {
826
1111
  }
827
1112
 
828
1113
  // src/index.ts
829
- var VERSION = "0.1.3";
1114
+ var VERSION = "0.1.5";
830
1115
  export {
831
1116
  renderSentinel,
832
1117
  parseSentinel,
@@ -847,5 +1132,6 @@ export {
847
1132
  LocalDetector,
848
1133
  LABEL_TO_TAG,
849
1134
  DEFAULT_BASE_URL,
850
- CompositeDetector
1135
+ CompositeDetector,
1136
+ CalibratedDetector
851
1137
  };
@@ -708,10 +708,281 @@ class CompositeDetector {
708
708
  return out;
709
709
  }
710
710
  }
711
+ // src/detectors/calibrated.ts
712
+ var DEFAULT_MODEL3 = "joelbarmettler/gheim-ch-560m";
713
+ var DEFAULT_CHUNK_TOKENS2 = 510;
714
+ var DEFAULT_OVERLAP_TOKENS2 = 32;
715
+ function trimSpan2(text, start, end) {
716
+ while (start < end && /\s/.test(text[start] ?? ""))
717
+ start++;
718
+ while (end > start && /\s/.test(text[end - 1] ?? ""))
719
+ end--;
720
+ return [start, end];
721
+ }
722
+
723
+ class CalibratedDetector {
724
+ model;
725
+ oBias;
726
+ device;
727
+ dtype;
728
+ chunkTokens;
729
+ chunkOverlapTokens;
730
+ trimWhitespace;
731
+ actualDevice = null;
732
+ _model = null;
733
+ _tokenizer = null;
734
+ _id2label = {};
735
+ _oId = 0;
736
+ _loadPromise = null;
737
+ _onProgress;
738
+ constructor(opts = {}) {
739
+ this.model = opts.model ?? DEFAULT_MODEL3;
740
+ this.oBias = opts.oBias ?? 0.5;
741
+ this.device = opts.device ?? "auto";
742
+ this.dtype = opts.dtype ?? "fp32";
743
+ this.chunkTokens = opts.chunkTokens ?? DEFAULT_CHUNK_TOKENS2;
744
+ this.chunkOverlapTokens = opts.chunkOverlapTokens ?? DEFAULT_OVERLAP_TOKENS2;
745
+ this.trimWhitespace = opts.trimWhitespace ?? true;
746
+ this._onProgress = opts.onProgress;
747
+ if (this.chunkOverlapTokens >= this.chunkTokens) {
748
+ throw new Error(`chunkOverlapTokens (${this.chunkOverlapTokens}) must be smaller than ` + `chunkTokens (${this.chunkTokens})`);
749
+ }
750
+ }
751
+ async load() {
752
+ if (this._model)
753
+ return;
754
+ if (this._loadPromise)
755
+ return this._loadPromise;
756
+ this._loadPromise = (async () => {
757
+ const mod = await loadTransformersJs2();
758
+ let preferredDevice = this.device;
759
+ if (this.device === "auto") {
760
+ const nav = typeof navigator !== "undefined" ? navigator : null;
761
+ preferredDevice = "wasm";
762
+ if (nav?.gpu?.requestAdapter) {
763
+ try {
764
+ const adapter = await nav.gpu.requestAdapter();
765
+ if (adapter)
766
+ preferredDevice = "webgpu";
767
+ } catch {}
768
+ }
769
+ }
770
+ const tokenizer = await mod.AutoTokenizer.from_pretrained(this.model);
771
+ const model = await mod.AutoModelForTokenClassification.from_pretrained(this.model, {
772
+ device: preferredDevice,
773
+ dtype: this.dtype,
774
+ progress_callback: this._onProgress ? (raw) => this._emitProgress(raw, preferredDevice) : undefined
775
+ });
776
+ this._tokenizer = tokenizer;
777
+ this._model = model;
778
+ this._id2label = model.config.id2label;
779
+ const label2id = model.config.label2id;
780
+ this._oId = label2id?.["O"] ?? 0;
781
+ this.actualDevice = preferredDevice;
782
+ })();
783
+ return this._loadPromise;
784
+ }
785
+ async detect(text, opts) {
786
+ if (opts?.model && opts.model !== this.model) {
787
+ throw new Error(`CalibratedDetector was constructed with model='${this.model}'; ` + `per-call model override ('${opts.model}') not supported.`);
788
+ }
789
+ if (!text)
790
+ return [];
791
+ await this.load();
792
+ return this._detectChunked(text);
793
+ }
794
+ async _detectChunked(text) {
795
+ if (text.length <= 1500) {
796
+ return await this._detectWindow(text, 0);
797
+ }
798
+ const stride = 1500 - 200;
799
+ const seen = new Set;
800
+ const out = [];
801
+ for (let s = 0;s < text.length; s += stride) {
802
+ const e = Math.min(s + 1500, text.length);
803
+ const sub = text.slice(s, e);
804
+ const subSpans = await this._detectWindow(sub, s);
805
+ for (const sp of subSpans) {
806
+ const key = `${sp.start}|${sp.end}|${sp.label}`;
807
+ if (seen.has(key))
808
+ continue;
809
+ seen.add(key);
810
+ out.push(sp);
811
+ }
812
+ if (e >= text.length)
813
+ break;
814
+ }
815
+ out.sort((a, b) => a.start - b.start || a.end - b.end);
816
+ return out;
817
+ }
818
+ async _detectWindow(text, charOffset) {
819
+ const tokenizer = this._tokenizer;
820
+ const model = this._model;
821
+ const enc = await tokenizer([text], {
822
+ padding: true,
823
+ truncation: true,
824
+ max_length: this.chunkTokens + 2
825
+ });
826
+ const inputs = {};
827
+ if (enc.input_ids)
828
+ inputs["input_ids"] = enc.input_ids;
829
+ if (enc.attention_mask)
830
+ inputs["attention_mask"] = enc.attention_mask;
831
+ const out = await model(inputs);
832
+ const logits = readLogits(out.logits);
833
+ if (this.oBias !== 0) {
834
+ for (let t = 0;t < logits.length; t++) {
835
+ const row = logits[t];
836
+ if (row === undefined)
837
+ continue;
838
+ const cur = row[this._oId];
839
+ if (cur !== undefined) {
840
+ row[this._oId] = cur - this.oBias;
841
+ }
842
+ }
843
+ }
844
+ const inputIds = readInputIds(enc.input_ids);
845
+ const tokenLabels = [];
846
+ for (let t = 0;t < logits.length; t++) {
847
+ const row = logits[t];
848
+ const id = inputIds[t];
849
+ if (!row || id === undefined)
850
+ continue;
851
+ let maxIdx = 0;
852
+ let maxVal = row[0];
853
+ for (let i = 1;i < row.length; i++) {
854
+ if (row[i] > maxVal) {
855
+ maxVal = row[i];
856
+ maxIdx = i;
857
+ }
858
+ }
859
+ tokenLabels.push({ id, label: this._id2label[maxIdx] ?? "O" });
860
+ }
861
+ const spans = [];
862
+ let cursor = 0;
863
+ let groupIds = [];
864
+ let groupCat = null;
865
+ const flush = () => {
866
+ if (groupCat === null || groupIds.length === 0)
867
+ return;
868
+ const surface = normalizeDecoded(tokenizer.decode(groupIds, { skip_special_tokens: true }));
869
+ groupIds = [];
870
+ const cat = groupCat;
871
+ groupCat = null;
872
+ if (!surface)
873
+ return;
874
+ const idx = text.indexOf(surface, cursor);
875
+ if (idx === -1)
876
+ return;
877
+ let s = idx;
878
+ let e = idx + surface.length;
879
+ cursor = e;
880
+ if (this.trimWhitespace)
881
+ [s, e] = trimSpan2(text, s, e);
882
+ if (s >= e)
883
+ return;
884
+ spans.push({
885
+ start: s + charOffset,
886
+ end: e + charOffset,
887
+ label: cat,
888
+ text: text.slice(s, e),
889
+ score: 1
890
+ });
891
+ };
892
+ for (const { id, label } of tokenLabels) {
893
+ const cat = label === "O" ? null : label.split("-", 2)[1] ?? null;
894
+ if (cat === groupCat && cat !== null) {
895
+ groupIds.push(id);
896
+ } else {
897
+ flush();
898
+ if (cat !== null) {
899
+ groupIds.push(id);
900
+ groupCat = cat;
901
+ }
902
+ }
903
+ }
904
+ flush();
905
+ return spans;
906
+ }
907
+ _emitProgress(raw, device) {
908
+ if (!this._onProgress)
909
+ return;
910
+ const event = {
911
+ phase: raw.status ?? "progress",
912
+ device
913
+ };
914
+ if (raw.file)
915
+ event.file = raw.file;
916
+ if (raw.status === "progress") {
917
+ if (raw.loaded != null)
918
+ event.loadedBytes = raw.loaded;
919
+ if (raw.total != null)
920
+ event.totalBytes = raw.total;
921
+ if (raw.progress != null) {
922
+ const norm = raw.progress > 1 ? raw.progress / 100 : raw.progress;
923
+ event.fraction = Math.max(0, Math.min(1, norm));
924
+ }
925
+ }
926
+ this._onProgress(event);
927
+ }
928
+ }
929
+ async function loadTransformersJs2() {
930
+ try {
931
+ return await import("@huggingface/transformers");
932
+ } catch {
933
+ throw new Error("CalibratedDetector requires `@huggingface/transformers`. Install with: bun add @huggingface/transformers");
934
+ }
935
+ }
936
+ function readInputIds(tensor) {
937
+ if (!tensor)
938
+ return [];
939
+ if (typeof tensor.tolist === "function") {
940
+ const list = tensor.tolist();
941
+ if (Array.isArray(list) && Array.isArray(list[0])) {
942
+ return list[0].map((x) => Number(x));
943
+ }
944
+ if (Array.isArray(list))
945
+ return list.map((x) => Number(x));
946
+ }
947
+ const dims = tensor.dims;
948
+ const seq = dims[dims.length - 1] ?? 0;
949
+ const data = tensor.data;
950
+ const out = [];
951
+ for (let i = 0;i < seq; i++)
952
+ out.push(Number(data[i]));
953
+ return out;
954
+ }
955
+ function normalizeDecoded(s) {
956
+ return s.replace(/##/g, "").trim();
957
+ }
958
+ function readLogits(tensor) {
959
+ if (typeof tensor.tolist === "function") {
960
+ const list = tensor.tolist();
961
+ if (Array.isArray(list) && Array.isArray(list[0]) && Array.isArray(list[0][0])) {
962
+ return list[0];
963
+ }
964
+ if (Array.isArray(list) && Array.isArray(list[0])) {
965
+ return list;
966
+ }
967
+ }
968
+ const dims = tensor.dims;
969
+ const seq = dims[dims.length - 2] ?? 0;
970
+ const cls = dims[dims.length - 1] ?? 0;
971
+ const data = tensor.data;
972
+ const out = [];
973
+ for (let t = 0;t < seq; t++) {
974
+ const row = [];
975
+ for (let c = 0;c < cls; c++) {
976
+ row.push(Number(data[t * cls + c]));
977
+ }
978
+ out.push(row);
979
+ }
980
+ return out;
981
+ }
711
982
  // src/detectors/index.ts
712
983
  function defaultDetector() {
713
984
  const hasKey = typeof process !== "undefined" && Boolean(process.env?.GHEIM_API_KEY);
714
- return hasKey ? new RemoteDetector : new LocalDetector;
985
+ return hasKey ? new RemoteDetector : new CalibratedDetector;
715
986
  }
716
987
 
717
988
  // src/core/session.ts
@@ -771,7 +1042,9 @@ class Session {
771
1042
  cursor = end;
772
1043
  }
773
1044
  out.push(text.slice(cursor));
774
- return { redacted: out.join(""), merged };
1045
+ let redacted = out.join("");
1046
+ redacted = recoverDuplicateOccurrences(redacted, this.mapping);
1047
+ return { redacted, merged };
775
1048
  }
776
1049
  restore(text) {
777
1050
  if (Object.keys(this.mapping).length === 0)
@@ -827,6 +1100,18 @@ function mergeAdjacent(text, spans) {
827
1100
  }
828
1101
  return out;
829
1102
  }
1103
+ function escapeRegex(s) {
1104
+ return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
1105
+ }
1106
+ function recoverDuplicateOccurrences(redacted, mapping) {
1107
+ const entries = Object.entries(mapping).filter(([, surface]) => surface && surface.length > 0).sort((a, b) => b[1].length - a[1].length);
1108
+ let result = redacted;
1109
+ for (const [sentinel, surface] of entries) {
1110
+ const pattern = new RegExp(`(?<![\\p{L}\\p{N}_])${escapeRegex(surface)}(?![\\p{L}\\p{N}_])`, "gu");
1111
+ result = result.replace(pattern, sentinel);
1112
+ }
1113
+ return result;
1114
+ }
830
1115
 
831
1116
  // src/openai/_base.ts
832
1117
  function resolveCallSession(body, defaultDet) {
@@ -1 +1 @@
1
- {"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../../src/core/session.ts"],"names":[],"mappings":"AAAA,kDAAkD;AAClD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AAGjD;;;gEAGgE;AAChE,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;AAkBtD,uCAAuC;AACvC,MAAM,WAAW,cAAc;IAC7B;;;;;;;;;;OAUG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;CAC1C;AAED,qBAAa,OAAO;IAClB,8DAA8D;IAC9D,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAM;IACrC,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,mBAAmB,CAAkC;gBAEjD,IAAI,GAAE,cAAmB;IAQrC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM;IAiBhD,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GAAG,MAAM;IAIvE;;;;;;;OAOG;IACH,kBAAkB,CAChB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GACpC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,UAAU,EAAE,CAAA;KAAE;IAoB7C,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAK7B,MAAM,IAAI;QAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE;IAI7C,MAAM,CAAC,QAAQ,CACb,GAAG,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,EACxC,IAAI,GAAE,cAAmB,GACxB,OAAO;CAqBX;AAID,8EAA8E;AAC9E,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAC3B,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GACpC,UAAU,EAAE,CAqBd"}
1
+ {"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../../src/core/session.ts"],"names":[],"mappings":"AAAA,kDAAkD;AAClD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AAGjD;;;gEAGgE;AAChE,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;AAkBtD,uCAAuC;AACvC,MAAM,WAAW,cAAc;IAC7B;;;;;;;;;;OAUG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;CAC1C;AAED,qBAAa,OAAO;IAClB,8DAA8D;IAC9D,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAM;IACrC,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,mBAAmB,CAAkC;gBAEjD,IAAI,GAAE,cAAmB;IAQrC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM;IAiBhD,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GAAG,MAAM;IAIvE;;;;;;;OAOG;IACH,kBAAkB,CAChB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GACpC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,UAAU,EAAE,CAAA;KAAE;IA+B7C,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAK7B,MAAM,IAAI;QAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE;IAI7C,MAAM,CAAC,QAAQ,CACb,GAAG,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,EACxC,IAAI,GAAE,cAAmB,GACxB,OAAO;CAqBX;AAID,8EAA8E;AAC9E,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAC3B,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GACpC,UAAU,EAAE,CAqBd"}