gheim 0.1.4 → 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/esm/index.js CHANGED
@@ -100,8 +100,8 @@ class Session {
100
100
  }
101
101
  out.push(text.slice(cursor));
102
102
  let redacted = out.join("");
103
- redacted = recoverDuplicateOccurrences(redacted, this.mapping);
104
- return { redacted, merged };
103
+ const recovered = recoverDuplicateOccurrences(text, redacted, merged, this.mapping);
104
+ return { redacted: recovered.redacted, merged: recovered.merged };
105
105
  }
106
106
  restore(text) {
107
107
  if (Object.keys(this.mapping).length === 0)
@@ -160,14 +160,36 @@ function mergeAdjacent(text, spans) {
160
160
  function escapeRegex(s) {
161
161
  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
162
162
  }
163
- function recoverDuplicateOccurrences(redacted, mapping) {
163
+ function recoverDuplicateOccurrences(text, redacted, merged, mapping) {
164
+ const surfaceToLabel = new Map;
165
+ for (const m of merged)
166
+ surfaceToLabel.set(text.slice(m.start, m.end), m.label);
167
+ const covered = merged.map((m) => [m.start, m.end]);
168
+ const overlaps = (s, e) => covered.some(([cs, ce]) => s < ce && e > cs);
164
169
  const entries = Object.entries(mapping).filter(([, surface]) => surface && surface.length > 0).sort((a, b) => b[1].length - a[1].length);
165
170
  let result = redacted;
171
+ const additional = [];
166
172
  for (const [sentinel, surface] of entries) {
167
- const pattern = new RegExp(`(?<![\\p{L}\\p{N}_])${escapeRegex(surface)}(?![\\p{L}\\p{N}_])`, "gu");
168
- result = result.replace(pattern, sentinel);
173
+ const boundaryPattern = `(?<![\\p{L}\\p{N}_])${escapeRegex(surface)}(?![\\p{L}\\p{N}_])`;
174
+ result = result.replace(new RegExp(boundaryPattern, "gu"), sentinel);
175
+ const label = surfaceToLabel.get(surface);
176
+ if (!label)
177
+ continue;
178
+ const origRe = new RegExp(boundaryPattern, "gu");
179
+ let match;
180
+ while ((match = origRe.exec(text)) !== null) {
181
+ const start = match.index;
182
+ const end = start + surface.length;
183
+ if (overlaps(start, end))
184
+ continue;
185
+ additional.push({ start, end, label });
186
+ covered.push([start, end]);
187
+ }
169
188
  }
170
- return result;
189
+ if (additional.length === 0)
190
+ return { redacted: result, merged };
191
+ const augmented = [...merged, ...additional].sort((a, b) => a.start - b.start);
192
+ return { redacted: result, merged: augmented };
171
193
  }
172
194
  // src/core/stream.ts
173
195
  var SENTINEL_AT = /<([A-Z][A-Z0-9_]*)_(\d+)>/y;
@@ -786,10 +808,281 @@ class CompositeDetector {
786
808
  return out;
787
809
  }
788
810
  }
811
+ // src/detectors/calibrated.ts
812
+ var DEFAULT_MODEL3 = "joelbarmettler/gheim-ch-560m";
813
+ var DEFAULT_CHUNK_TOKENS2 = 510;
814
+ var DEFAULT_OVERLAP_TOKENS2 = 32;
815
+ function trimSpan2(text, start, end) {
816
+ while (start < end && /\s/.test(text[start] ?? ""))
817
+ start++;
818
+ while (end > start && /\s/.test(text[end - 1] ?? ""))
819
+ end--;
820
+ return [start, end];
821
+ }
822
+
823
+ class CalibratedDetector {
824
+ model;
825
+ oBias;
826
+ device;
827
+ dtype;
828
+ chunkTokens;
829
+ chunkOverlapTokens;
830
+ trimWhitespace;
831
+ actualDevice = null;
832
+ _model = null;
833
+ _tokenizer = null;
834
+ _id2label = {};
835
+ _oId = 0;
836
+ _loadPromise = null;
837
+ _onProgress;
838
+ constructor(opts = {}) {
839
+ this.model = opts.model ?? DEFAULT_MODEL3;
840
+ this.oBias = opts.oBias ?? 0.5;
841
+ this.device = opts.device ?? "auto";
842
+ this.dtype = opts.dtype ?? "fp32";
843
+ this.chunkTokens = opts.chunkTokens ?? DEFAULT_CHUNK_TOKENS2;
844
+ this.chunkOverlapTokens = opts.chunkOverlapTokens ?? DEFAULT_OVERLAP_TOKENS2;
845
+ this.trimWhitespace = opts.trimWhitespace ?? true;
846
+ this._onProgress = opts.onProgress;
847
+ if (this.chunkOverlapTokens >= this.chunkTokens) {
848
+ throw new Error(`chunkOverlapTokens (${this.chunkOverlapTokens}) must be smaller than ` + `chunkTokens (${this.chunkTokens})`);
849
+ }
850
+ }
851
+ async load() {
852
+ if (this._model)
853
+ return;
854
+ if (this._loadPromise)
855
+ return this._loadPromise;
856
+ this._loadPromise = (async () => {
857
+ const mod = await loadTransformersJs2();
858
+ let preferredDevice = this.device;
859
+ if (this.device === "auto") {
860
+ const nav = typeof navigator !== "undefined" ? navigator : null;
861
+ preferredDevice = "wasm";
862
+ if (nav?.gpu?.requestAdapter) {
863
+ try {
864
+ const adapter = await nav.gpu.requestAdapter();
865
+ if (adapter)
866
+ preferredDevice = "webgpu";
867
+ } catch {}
868
+ }
869
+ }
870
+ const tokenizer = await mod.AutoTokenizer.from_pretrained(this.model);
871
+ const model = await mod.AutoModelForTokenClassification.from_pretrained(this.model, {
872
+ device: preferredDevice,
873
+ dtype: this.dtype,
874
+ progress_callback: this._onProgress ? (raw) => this._emitProgress(raw, preferredDevice) : undefined
875
+ });
876
+ this._tokenizer = tokenizer;
877
+ this._model = model;
878
+ this._id2label = model.config.id2label;
879
+ const label2id = model.config.label2id;
880
+ this._oId = label2id?.["O"] ?? 0;
881
+ this.actualDevice = preferredDevice;
882
+ })();
883
+ return this._loadPromise;
884
+ }
885
+ async detect(text, opts) {
886
+ if (opts?.model && opts.model !== this.model) {
887
+ throw new Error(`CalibratedDetector was constructed with model='${this.model}'; ` + `per-call model override ('${opts.model}') not supported.`);
888
+ }
889
+ if (!text)
890
+ return [];
891
+ await this.load();
892
+ return this._detectChunked(text);
893
+ }
894
+ async _detectChunked(text) {
895
+ if (text.length <= 1500) {
896
+ return await this._detectWindow(text, 0);
897
+ }
898
+ const stride = 1500 - 200;
899
+ const seen = new Set;
900
+ const out = [];
901
+ for (let s = 0;s < text.length; s += stride) {
902
+ const e = Math.min(s + 1500, text.length);
903
+ const sub = text.slice(s, e);
904
+ const subSpans = await this._detectWindow(sub, s);
905
+ for (const sp of subSpans) {
906
+ const key = `${sp.start}|${sp.end}|${sp.label}`;
907
+ if (seen.has(key))
908
+ continue;
909
+ seen.add(key);
910
+ out.push(sp);
911
+ }
912
+ if (e >= text.length)
913
+ break;
914
+ }
915
+ out.sort((a, b) => a.start - b.start || a.end - b.end);
916
+ return out;
917
+ }
918
+ async _detectWindow(text, charOffset) {
919
+ const tokenizer = this._tokenizer;
920
+ const model = this._model;
921
+ const enc = await tokenizer([text], {
922
+ padding: true,
923
+ truncation: true,
924
+ max_length: this.chunkTokens + 2
925
+ });
926
+ const inputs = {};
927
+ if (enc.input_ids)
928
+ inputs["input_ids"] = enc.input_ids;
929
+ if (enc.attention_mask)
930
+ inputs["attention_mask"] = enc.attention_mask;
931
+ const out = await model(inputs);
932
+ const logits = readLogits(out.logits);
933
+ if (this.oBias !== 0) {
934
+ for (let t = 0;t < logits.length; t++) {
935
+ const row = logits[t];
936
+ if (row === undefined)
937
+ continue;
938
+ const cur = row[this._oId];
939
+ if (cur !== undefined) {
940
+ row[this._oId] = cur - this.oBias;
941
+ }
942
+ }
943
+ }
944
+ const inputIds = readInputIds(enc.input_ids);
945
+ const tokenLabels = [];
946
+ for (let t = 0;t < logits.length; t++) {
947
+ const row = logits[t];
948
+ const id = inputIds[t];
949
+ if (!row || id === undefined)
950
+ continue;
951
+ let maxIdx = 0;
952
+ let maxVal = row[0];
953
+ for (let i = 1;i < row.length; i++) {
954
+ if (row[i] > maxVal) {
955
+ maxVal = row[i];
956
+ maxIdx = i;
957
+ }
958
+ }
959
+ tokenLabels.push({ id, label: this._id2label[maxIdx] ?? "O" });
960
+ }
961
+ const spans = [];
962
+ let cursor = 0;
963
+ let groupIds = [];
964
+ let groupCat = null;
965
+ const flush = () => {
966
+ if (groupCat === null || groupIds.length === 0)
967
+ return;
968
+ const surface = normalizeDecoded(tokenizer.decode(groupIds, { skip_special_tokens: true }));
969
+ groupIds = [];
970
+ const cat = groupCat;
971
+ groupCat = null;
972
+ if (!surface)
973
+ return;
974
+ const idx = text.indexOf(surface, cursor);
975
+ if (idx === -1)
976
+ return;
977
+ let s = idx;
978
+ let e = idx + surface.length;
979
+ cursor = e;
980
+ if (this.trimWhitespace)
981
+ [s, e] = trimSpan2(text, s, e);
982
+ if (s >= e)
983
+ return;
984
+ spans.push({
985
+ start: s + charOffset,
986
+ end: e + charOffset,
987
+ label: cat,
988
+ text: text.slice(s, e),
989
+ score: 1
990
+ });
991
+ };
992
+ for (const { id, label } of tokenLabels) {
993
+ const cat = label === "O" ? null : label.split("-", 2)[1] ?? null;
994
+ if (cat === groupCat && cat !== null) {
995
+ groupIds.push(id);
996
+ } else {
997
+ flush();
998
+ if (cat !== null) {
999
+ groupIds.push(id);
1000
+ groupCat = cat;
1001
+ }
1002
+ }
1003
+ }
1004
+ flush();
1005
+ return spans;
1006
+ }
1007
+ _emitProgress(raw, device) {
1008
+ if (!this._onProgress)
1009
+ return;
1010
+ const event = {
1011
+ phase: raw.status ?? "progress",
1012
+ device
1013
+ };
1014
+ if (raw.file)
1015
+ event.file = raw.file;
1016
+ if (raw.status === "progress") {
1017
+ if (raw.loaded != null)
1018
+ event.loadedBytes = raw.loaded;
1019
+ if (raw.total != null)
1020
+ event.totalBytes = raw.total;
1021
+ if (raw.progress != null) {
1022
+ const norm = raw.progress > 1 ? raw.progress / 100 : raw.progress;
1023
+ event.fraction = Math.max(0, Math.min(1, norm));
1024
+ }
1025
+ }
1026
+ this._onProgress(event);
1027
+ }
1028
+ }
1029
+ async function loadTransformersJs2() {
1030
+ try {
1031
+ return await import("@huggingface/transformers");
1032
+ } catch {
1033
+ throw new Error("CalibratedDetector requires `@huggingface/transformers`. Install with: bun add @huggingface/transformers");
1034
+ }
1035
+ }
1036
+ function readInputIds(tensor) {
1037
+ if (!tensor)
1038
+ return [];
1039
+ if (typeof tensor.tolist === "function") {
1040
+ const list = tensor.tolist();
1041
+ if (Array.isArray(list) && Array.isArray(list[0])) {
1042
+ return list[0].map((x) => Number(x));
1043
+ }
1044
+ if (Array.isArray(list))
1045
+ return list.map((x) => Number(x));
1046
+ }
1047
+ const dims = tensor.dims;
1048
+ const seq = dims[dims.length - 1] ?? 0;
1049
+ const data = tensor.data;
1050
+ const out = [];
1051
+ for (let i = 0;i < seq; i++)
1052
+ out.push(Number(data[i]));
1053
+ return out;
1054
+ }
1055
+ function normalizeDecoded(s) {
1056
+ return s.replace(/##/g, "").trim();
1057
+ }
1058
+ function readLogits(tensor) {
1059
+ if (typeof tensor.tolist === "function") {
1060
+ const list = tensor.tolist();
1061
+ if (Array.isArray(list) && Array.isArray(list[0]) && Array.isArray(list[0][0])) {
1062
+ return list[0];
1063
+ }
1064
+ if (Array.isArray(list) && Array.isArray(list[0])) {
1065
+ return list;
1066
+ }
1067
+ }
1068
+ const dims = tensor.dims;
1069
+ const seq = dims[dims.length - 2] ?? 0;
1070
+ const cls = dims[dims.length - 1] ?? 0;
1071
+ const data = tensor.data;
1072
+ const out = [];
1073
+ for (let t = 0;t < seq; t++) {
1074
+ const row = [];
1075
+ for (let c = 0;c < cls; c++) {
1076
+ row.push(Number(data[t * cls + c]));
1077
+ }
1078
+ out.push(row);
1079
+ }
1080
+ return out;
1081
+ }
789
1082
  // src/detectors/index.ts
790
1083
  function defaultDetector() {
791
1084
  const hasKey = typeof process !== "undefined" && Boolean(process.env?.GHEIM_API_KEY);
792
- return hasKey ? new RemoteDetector : new LocalDetector;
1085
+ return hasKey ? new RemoteDetector : new CalibratedDetector;
793
1086
  }
794
1087
  // src/plain.ts
795
1088
  function resolve(session, detector) {
@@ -840,7 +1133,7 @@ async function* deanonymizeStream(chunks, session) {
840
1133
  }
841
1134
 
842
1135
  // src/index.ts
843
- var VERSION = "0.1.4";
1136
+ var VERSION = "0.1.6";
844
1137
  export {
845
1138
  renderSentinel,
846
1139
  parseSentinel,
@@ -861,5 +1154,6 @@ export {
861
1154
  LocalDetector,
862
1155
  LABEL_TO_TAG,
863
1156
  DEFAULT_BASE_URL,
864
- CompositeDetector
1157
+ CompositeDetector,
1158
+ CalibratedDetector
865
1159
  };
@@ -708,10 +708,281 @@ class CompositeDetector {
708
708
  return out;
709
709
  }
710
710
  }
711
+ // src/detectors/calibrated.ts
712
+ var DEFAULT_MODEL3 = "joelbarmettler/gheim-ch-560m";
713
+ var DEFAULT_CHUNK_TOKENS2 = 510;
714
+ var DEFAULT_OVERLAP_TOKENS2 = 32;
715
+ function trimSpan2(text, start, end) {
716
+ while (start < end && /\s/.test(text[start] ?? ""))
717
+ start++;
718
+ while (end > start && /\s/.test(text[end - 1] ?? ""))
719
+ end--;
720
+ return [start, end];
721
+ }
722
+
723
+ class CalibratedDetector {
724
+ model;
725
+ oBias;
726
+ device;
727
+ dtype;
728
+ chunkTokens;
729
+ chunkOverlapTokens;
730
+ trimWhitespace;
731
+ actualDevice = null;
732
+ _model = null;
733
+ _tokenizer = null;
734
+ _id2label = {};
735
+ _oId = 0;
736
+ _loadPromise = null;
737
+ _onProgress;
738
+ constructor(opts = {}) {
739
+ this.model = opts.model ?? DEFAULT_MODEL3;
740
+ this.oBias = opts.oBias ?? 0.5;
741
+ this.device = opts.device ?? "auto";
742
+ this.dtype = opts.dtype ?? "fp32";
743
+ this.chunkTokens = opts.chunkTokens ?? DEFAULT_CHUNK_TOKENS2;
744
+ this.chunkOverlapTokens = opts.chunkOverlapTokens ?? DEFAULT_OVERLAP_TOKENS2;
745
+ this.trimWhitespace = opts.trimWhitespace ?? true;
746
+ this._onProgress = opts.onProgress;
747
+ if (this.chunkOverlapTokens >= this.chunkTokens) {
748
+ throw new Error(`chunkOverlapTokens (${this.chunkOverlapTokens}) must be smaller than ` + `chunkTokens (${this.chunkTokens})`);
749
+ }
750
+ }
751
+ async load() {
752
+ if (this._model)
753
+ return;
754
+ if (this._loadPromise)
755
+ return this._loadPromise;
756
+ this._loadPromise = (async () => {
757
+ const mod = await loadTransformersJs2();
758
+ let preferredDevice = this.device;
759
+ if (this.device === "auto") {
760
+ const nav = typeof navigator !== "undefined" ? navigator : null;
761
+ preferredDevice = "wasm";
762
+ if (nav?.gpu?.requestAdapter) {
763
+ try {
764
+ const adapter = await nav.gpu.requestAdapter();
765
+ if (adapter)
766
+ preferredDevice = "webgpu";
767
+ } catch {}
768
+ }
769
+ }
770
+ const tokenizer = await mod.AutoTokenizer.from_pretrained(this.model);
771
+ const model = await mod.AutoModelForTokenClassification.from_pretrained(this.model, {
772
+ device: preferredDevice,
773
+ dtype: this.dtype,
774
+ progress_callback: this._onProgress ? (raw) => this._emitProgress(raw, preferredDevice) : undefined
775
+ });
776
+ this._tokenizer = tokenizer;
777
+ this._model = model;
778
+ this._id2label = model.config.id2label;
779
+ const label2id = model.config.label2id;
780
+ this._oId = label2id?.["O"] ?? 0;
781
+ this.actualDevice = preferredDevice;
782
+ })();
783
+ return this._loadPromise;
784
+ }
785
+ async detect(text, opts) {
786
+ if (opts?.model && opts.model !== this.model) {
787
+ throw new Error(`CalibratedDetector was constructed with model='${this.model}'; ` + `per-call model override ('${opts.model}') not supported.`);
788
+ }
789
+ if (!text)
790
+ return [];
791
+ await this.load();
792
+ return this._detectChunked(text);
793
+ }
794
+ async _detectChunked(text) {
795
+ if (text.length <= 1500) {
796
+ return await this._detectWindow(text, 0);
797
+ }
798
+ const stride = 1500 - 200;
799
+ const seen = new Set;
800
+ const out = [];
801
+ for (let s = 0;s < text.length; s += stride) {
802
+ const e = Math.min(s + 1500, text.length);
803
+ const sub = text.slice(s, e);
804
+ const subSpans = await this._detectWindow(sub, s);
805
+ for (const sp of subSpans) {
806
+ const key = `${sp.start}|${sp.end}|${sp.label}`;
807
+ if (seen.has(key))
808
+ continue;
809
+ seen.add(key);
810
+ out.push(sp);
811
+ }
812
+ if (e >= text.length)
813
+ break;
814
+ }
815
+ out.sort((a, b) => a.start - b.start || a.end - b.end);
816
+ return out;
817
+ }
818
+ async _detectWindow(text, charOffset) {
819
+ const tokenizer = this._tokenizer;
820
+ const model = this._model;
821
+ const enc = await tokenizer([text], {
822
+ padding: true,
823
+ truncation: true,
824
+ max_length: this.chunkTokens + 2
825
+ });
826
+ const inputs = {};
827
+ if (enc.input_ids)
828
+ inputs["input_ids"] = enc.input_ids;
829
+ if (enc.attention_mask)
830
+ inputs["attention_mask"] = enc.attention_mask;
831
+ const out = await model(inputs);
832
+ const logits = readLogits(out.logits);
833
+ if (this.oBias !== 0) {
834
+ for (let t = 0;t < logits.length; t++) {
835
+ const row = logits[t];
836
+ if (row === undefined)
837
+ continue;
838
+ const cur = row[this._oId];
839
+ if (cur !== undefined) {
840
+ row[this._oId] = cur - this.oBias;
841
+ }
842
+ }
843
+ }
844
+ const inputIds = readInputIds(enc.input_ids);
845
+ const tokenLabels = [];
846
+ for (let t = 0;t < logits.length; t++) {
847
+ const row = logits[t];
848
+ const id = inputIds[t];
849
+ if (!row || id === undefined)
850
+ continue;
851
+ let maxIdx = 0;
852
+ let maxVal = row[0];
853
+ for (let i = 1;i < row.length; i++) {
854
+ if (row[i] > maxVal) {
855
+ maxVal = row[i];
856
+ maxIdx = i;
857
+ }
858
+ }
859
+ tokenLabels.push({ id, label: this._id2label[maxIdx] ?? "O" });
860
+ }
861
+ const spans = [];
862
+ let cursor = 0;
863
+ let groupIds = [];
864
+ let groupCat = null;
865
+ const flush = () => {
866
+ if (groupCat === null || groupIds.length === 0)
867
+ return;
868
+ const surface = normalizeDecoded(tokenizer.decode(groupIds, { skip_special_tokens: true }));
869
+ groupIds = [];
870
+ const cat = groupCat;
871
+ groupCat = null;
872
+ if (!surface)
873
+ return;
874
+ const idx = text.indexOf(surface, cursor);
875
+ if (idx === -1)
876
+ return;
877
+ let s = idx;
878
+ let e = idx + surface.length;
879
+ cursor = e;
880
+ if (this.trimWhitespace)
881
+ [s, e] = trimSpan2(text, s, e);
882
+ if (s >= e)
883
+ return;
884
+ spans.push({
885
+ start: s + charOffset,
886
+ end: e + charOffset,
887
+ label: cat,
888
+ text: text.slice(s, e),
889
+ score: 1
890
+ });
891
+ };
892
+ for (const { id, label } of tokenLabels) {
893
+ const cat = label === "O" ? null : label.split("-", 2)[1] ?? null;
894
+ if (cat === groupCat && cat !== null) {
895
+ groupIds.push(id);
896
+ } else {
897
+ flush();
898
+ if (cat !== null) {
899
+ groupIds.push(id);
900
+ groupCat = cat;
901
+ }
902
+ }
903
+ }
904
+ flush();
905
+ return spans;
906
+ }
907
+ _emitProgress(raw, device) {
908
+ if (!this._onProgress)
909
+ return;
910
+ const event = {
911
+ phase: raw.status ?? "progress",
912
+ device
913
+ };
914
+ if (raw.file)
915
+ event.file = raw.file;
916
+ if (raw.status === "progress") {
917
+ if (raw.loaded != null)
918
+ event.loadedBytes = raw.loaded;
919
+ if (raw.total != null)
920
+ event.totalBytes = raw.total;
921
+ if (raw.progress != null) {
922
+ const norm = raw.progress > 1 ? raw.progress / 100 : raw.progress;
923
+ event.fraction = Math.max(0, Math.min(1, norm));
924
+ }
925
+ }
926
+ this._onProgress(event);
927
+ }
928
+ }
929
+ async function loadTransformersJs2() {
930
+ try {
931
+ return await import("@huggingface/transformers");
932
+ } catch {
933
+ throw new Error("CalibratedDetector requires `@huggingface/transformers`. Install with: bun add @huggingface/transformers");
934
+ }
935
+ }
936
+ function readInputIds(tensor) {
937
+ if (!tensor)
938
+ return [];
939
+ if (typeof tensor.tolist === "function") {
940
+ const list = tensor.tolist();
941
+ if (Array.isArray(list) && Array.isArray(list[0])) {
942
+ return list[0].map((x) => Number(x));
943
+ }
944
+ if (Array.isArray(list))
945
+ return list.map((x) => Number(x));
946
+ }
947
+ const dims = tensor.dims;
948
+ const seq = dims[dims.length - 1] ?? 0;
949
+ const data = tensor.data;
950
+ const out = [];
951
+ for (let i = 0;i < seq; i++)
952
+ out.push(Number(data[i]));
953
+ return out;
954
+ }
955
+ function normalizeDecoded(s) {
956
+ return s.replace(/##/g, "").trim();
957
+ }
958
+ function readLogits(tensor) {
959
+ if (typeof tensor.tolist === "function") {
960
+ const list = tensor.tolist();
961
+ if (Array.isArray(list) && Array.isArray(list[0]) && Array.isArray(list[0][0])) {
962
+ return list[0];
963
+ }
964
+ if (Array.isArray(list) && Array.isArray(list[0])) {
965
+ return list;
966
+ }
967
+ }
968
+ const dims = tensor.dims;
969
+ const seq = dims[dims.length - 2] ?? 0;
970
+ const cls = dims[dims.length - 1] ?? 0;
971
+ const data = tensor.data;
972
+ const out = [];
973
+ for (let t = 0;t < seq; t++) {
974
+ const row = [];
975
+ for (let c = 0;c < cls; c++) {
976
+ row.push(Number(data[t * cls + c]));
977
+ }
978
+ out.push(row);
979
+ }
980
+ return out;
981
+ }
711
982
  // src/detectors/index.ts
712
983
  function defaultDetector() {
713
984
  const hasKey = typeof process !== "undefined" && Boolean(process.env?.GHEIM_API_KEY);
714
- return hasKey ? new RemoteDetector : new LocalDetector;
985
+ return hasKey ? new RemoteDetector : new CalibratedDetector;
715
986
  }
716
987
 
717
988
  // src/core/session.ts
@@ -772,8 +1043,8 @@ class Session {
772
1043
  }
773
1044
  out.push(text.slice(cursor));
774
1045
  let redacted = out.join("");
775
- redacted = recoverDuplicateOccurrences(redacted, this.mapping);
776
- return { redacted, merged };
1046
+ const recovered = recoverDuplicateOccurrences(text, redacted, merged, this.mapping);
1047
+ return { redacted: recovered.redacted, merged: recovered.merged };
777
1048
  }
778
1049
  restore(text) {
779
1050
  if (Object.keys(this.mapping).length === 0)
@@ -832,14 +1103,36 @@ function mergeAdjacent(text, spans) {
832
1103
  function escapeRegex(s) {
833
1104
  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
834
1105
  }
835
- function recoverDuplicateOccurrences(redacted, mapping) {
1106
+ function recoverDuplicateOccurrences(text, redacted, merged, mapping) {
1107
+ const surfaceToLabel = new Map;
1108
+ for (const m of merged)
1109
+ surfaceToLabel.set(text.slice(m.start, m.end), m.label);
1110
+ const covered = merged.map((m) => [m.start, m.end]);
1111
+ const overlaps = (s, e) => covered.some(([cs, ce]) => s < ce && e > cs);
836
1112
  const entries = Object.entries(mapping).filter(([, surface]) => surface && surface.length > 0).sort((a, b) => b[1].length - a[1].length);
837
1113
  let result = redacted;
1114
+ const additional = [];
838
1115
  for (const [sentinel, surface] of entries) {
839
- const pattern = new RegExp(`(?<![\\p{L}\\p{N}_])${escapeRegex(surface)}(?![\\p{L}\\p{N}_])`, "gu");
840
- result = result.replace(pattern, sentinel);
1116
+ const boundaryPattern = `(?<![\\p{L}\\p{N}_])${escapeRegex(surface)}(?![\\p{L}\\p{N}_])`;
1117
+ result = result.replace(new RegExp(boundaryPattern, "gu"), sentinel);
1118
+ const label = surfaceToLabel.get(surface);
1119
+ if (!label)
1120
+ continue;
1121
+ const origRe = new RegExp(boundaryPattern, "gu");
1122
+ let match;
1123
+ while ((match = origRe.exec(text)) !== null) {
1124
+ const start = match.index;
1125
+ const end = start + surface.length;
1126
+ if (overlaps(start, end))
1127
+ continue;
1128
+ additional.push({ start, end, label });
1129
+ covered.push([start, end]);
1130
+ }
841
1131
  }
842
- return result;
1132
+ if (additional.length === 0)
1133
+ return { redacted: result, merged };
1134
+ const augmented = [...merged, ...additional].sort((a, b) => a.start - b.start);
1135
+ return { redacted: result, merged: augmented };
843
1136
  }
844
1137
 
845
1138
  // src/openai/_base.ts
@@ -1 +1 @@
1
- {"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../../src/core/session.ts"],"names":[],"mappings":"AAAA,kDAAkD;AAClD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AAGjD;;;gEAGgE;AAChE,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;AAkBtD,uCAAuC;AACvC,MAAM,WAAW,cAAc;IAC7B;;;;;;;;;;OAUG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;CAC1C;AAED,qBAAa,OAAO;IAClB,8DAA8D;IAC9D,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAM;IACrC,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,mBAAmB,CAAkC;gBAEjD,IAAI,GAAE,cAAmB;IAQrC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM;IAiBhD,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GAAG,MAAM;IAIvE;;;;;;;OAOG;IACH,kBAAkB,CAChB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GACpC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,UAAU,EAAE,CAAA;KAAE;IA+B7C,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAK7B,MAAM,IAAI;QAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE;IAI7C,MAAM,CAAC,QAAQ,CACb,GAAG,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,EACxC,IAAI,GAAE,cAAmB,GACxB,OAAO;CAqBX;AAID,8EAA8E;AAC9E,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAC3B,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GACpC,UAAU,EAAE,CAqBd"}
1
+ {"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../../src/core/session.ts"],"names":[],"mappings":"AAAA,kDAAkD;AAClD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AAGjD;;;gEAGgE;AAChE,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;AAkBtD,uCAAuC;AACvC,MAAM,WAAW,cAAc;IAC7B;;;;;;;;;;OAUG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;CAC1C;AAED,qBAAa,OAAO;IAClB,8DAA8D;IAC9D,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAM;IACrC,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,mBAAmB,CAAkC;gBAEjD,IAAI,GAAE,cAAmB;IAQrC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM;IAiBhD,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GAAG,MAAM;IAIvE;;;;;;;OAOG;IACH,kBAAkB,CAChB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GACpC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,UAAU,EAAE,CAAA;KAAE;IA8B7C,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAK7B,MAAM,IAAI;QAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE;IAI7C,MAAM,CAAC,QAAQ,CACb,GAAG,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,EACxC,IAAI,GAAE,cAAmB,GACxB,OAAO;CAqBX;AAID,8EAA8E;AAC9E,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAC3B,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GACpC,UAAU,EAAE,CAqBd"}