gheim 0.1.4 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/index.cjs +303 -9
- package/dist/cjs/openai/index.cjs +300 -7
- package/dist/esm/index.js +303 -9
- package/dist/esm/openai/index.js +300 -7
- package/dist/types/core/session.d.ts.map +1 -1
- package/dist/types/detectors/calibrated.d.ts +78 -0
- package/dist/types/detectors/calibrated.d.ts.map +1 -0
- package/dist/types/detectors/index.d.ts +8 -1
- package/dist/types/detectors/index.d.ts.map +1 -1
- package/dist/types/index.d.ts +3 -3
- package/dist/types/index.d.ts.map +1 -1
- package/package.json +1 -1
package/dist/esm/index.js
CHANGED
|
@@ -100,8 +100,8 @@ class Session {
|
|
|
100
100
|
}
|
|
101
101
|
out.push(text.slice(cursor));
|
|
102
102
|
let redacted = out.join("");
|
|
103
|
-
|
|
104
|
-
return { redacted, merged };
|
|
103
|
+
const recovered = recoverDuplicateOccurrences(text, redacted, merged, this.mapping);
|
|
104
|
+
return { redacted: recovered.redacted, merged: recovered.merged };
|
|
105
105
|
}
|
|
106
106
|
restore(text) {
|
|
107
107
|
if (Object.keys(this.mapping).length === 0)
|
|
@@ -160,14 +160,36 @@ function mergeAdjacent(text, spans) {
|
|
|
160
160
|
function escapeRegex(s) {
|
|
161
161
|
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
162
162
|
}
|
|
163
|
-
function recoverDuplicateOccurrences(redacted, mapping) {
|
|
163
|
+
function recoverDuplicateOccurrences(text, redacted, merged, mapping) {
|
|
164
|
+
const surfaceToLabel = new Map;
|
|
165
|
+
for (const m of merged)
|
|
166
|
+
surfaceToLabel.set(text.slice(m.start, m.end), m.label);
|
|
167
|
+
const covered = merged.map((m) => [m.start, m.end]);
|
|
168
|
+
const overlaps = (s, e) => covered.some(([cs, ce]) => s < ce && e > cs);
|
|
164
169
|
const entries = Object.entries(mapping).filter(([, surface]) => surface && surface.length > 0).sort((a, b) => b[1].length - a[1].length);
|
|
165
170
|
let result = redacted;
|
|
171
|
+
const additional = [];
|
|
166
172
|
for (const [sentinel, surface] of entries) {
|
|
167
|
-
const
|
|
168
|
-
result = result.replace(
|
|
173
|
+
const boundaryPattern = `(?<![\\p{L}\\p{N}_])${escapeRegex(surface)}(?![\\p{L}\\p{N}_])`;
|
|
174
|
+
result = result.replace(new RegExp(boundaryPattern, "gu"), sentinel);
|
|
175
|
+
const label = surfaceToLabel.get(surface);
|
|
176
|
+
if (!label)
|
|
177
|
+
continue;
|
|
178
|
+
const origRe = new RegExp(boundaryPattern, "gu");
|
|
179
|
+
let match;
|
|
180
|
+
while ((match = origRe.exec(text)) !== null) {
|
|
181
|
+
const start = match.index;
|
|
182
|
+
const end = start + surface.length;
|
|
183
|
+
if (overlaps(start, end))
|
|
184
|
+
continue;
|
|
185
|
+
additional.push({ start, end, label });
|
|
186
|
+
covered.push([start, end]);
|
|
187
|
+
}
|
|
169
188
|
}
|
|
170
|
-
|
|
189
|
+
if (additional.length === 0)
|
|
190
|
+
return { redacted: result, merged };
|
|
191
|
+
const augmented = [...merged, ...additional].sort((a, b) => a.start - b.start);
|
|
192
|
+
return { redacted: result, merged: augmented };
|
|
171
193
|
}
|
|
172
194
|
// src/core/stream.ts
|
|
173
195
|
var SENTINEL_AT = /<([A-Z][A-Z0-9_]*)_(\d+)>/y;
|
|
@@ -786,10 +808,281 @@ class CompositeDetector {
|
|
|
786
808
|
return out;
|
|
787
809
|
}
|
|
788
810
|
}
|
|
811
|
+
// src/detectors/calibrated.ts
|
|
812
|
+
var DEFAULT_MODEL3 = "joelbarmettler/gheim-ch-560m";
|
|
813
|
+
var DEFAULT_CHUNK_TOKENS2 = 510;
|
|
814
|
+
var DEFAULT_OVERLAP_TOKENS2 = 32;
|
|
815
|
+
function trimSpan2(text, start, end) {
|
|
816
|
+
while (start < end && /\s/.test(text[start] ?? ""))
|
|
817
|
+
start++;
|
|
818
|
+
while (end > start && /\s/.test(text[end - 1] ?? ""))
|
|
819
|
+
end--;
|
|
820
|
+
return [start, end];
|
|
821
|
+
}
|
|
822
|
+
|
|
823
|
+
class CalibratedDetector {
|
|
824
|
+
model;
|
|
825
|
+
oBias;
|
|
826
|
+
device;
|
|
827
|
+
dtype;
|
|
828
|
+
chunkTokens;
|
|
829
|
+
chunkOverlapTokens;
|
|
830
|
+
trimWhitespace;
|
|
831
|
+
actualDevice = null;
|
|
832
|
+
_model = null;
|
|
833
|
+
_tokenizer = null;
|
|
834
|
+
_id2label = {};
|
|
835
|
+
_oId = 0;
|
|
836
|
+
_loadPromise = null;
|
|
837
|
+
_onProgress;
|
|
838
|
+
constructor(opts = {}) {
|
|
839
|
+
this.model = opts.model ?? DEFAULT_MODEL3;
|
|
840
|
+
this.oBias = opts.oBias ?? 0.5;
|
|
841
|
+
this.device = opts.device ?? "auto";
|
|
842
|
+
this.dtype = opts.dtype ?? "fp32";
|
|
843
|
+
this.chunkTokens = opts.chunkTokens ?? DEFAULT_CHUNK_TOKENS2;
|
|
844
|
+
this.chunkOverlapTokens = opts.chunkOverlapTokens ?? DEFAULT_OVERLAP_TOKENS2;
|
|
845
|
+
this.trimWhitespace = opts.trimWhitespace ?? true;
|
|
846
|
+
this._onProgress = opts.onProgress;
|
|
847
|
+
if (this.chunkOverlapTokens >= this.chunkTokens) {
|
|
848
|
+
throw new Error(`chunkOverlapTokens (${this.chunkOverlapTokens}) must be smaller than ` + `chunkTokens (${this.chunkTokens})`);
|
|
849
|
+
}
|
|
850
|
+
}
|
|
851
|
+
async load() {
|
|
852
|
+
if (this._model)
|
|
853
|
+
return;
|
|
854
|
+
if (this._loadPromise)
|
|
855
|
+
return this._loadPromise;
|
|
856
|
+
this._loadPromise = (async () => {
|
|
857
|
+
const mod = await loadTransformersJs2();
|
|
858
|
+
let preferredDevice = this.device;
|
|
859
|
+
if (this.device === "auto") {
|
|
860
|
+
const nav = typeof navigator !== "undefined" ? navigator : null;
|
|
861
|
+
preferredDevice = "wasm";
|
|
862
|
+
if (nav?.gpu?.requestAdapter) {
|
|
863
|
+
try {
|
|
864
|
+
const adapter = await nav.gpu.requestAdapter();
|
|
865
|
+
if (adapter)
|
|
866
|
+
preferredDevice = "webgpu";
|
|
867
|
+
} catch {}
|
|
868
|
+
}
|
|
869
|
+
}
|
|
870
|
+
const tokenizer = await mod.AutoTokenizer.from_pretrained(this.model);
|
|
871
|
+
const model = await mod.AutoModelForTokenClassification.from_pretrained(this.model, {
|
|
872
|
+
device: preferredDevice,
|
|
873
|
+
dtype: this.dtype,
|
|
874
|
+
progress_callback: this._onProgress ? (raw) => this._emitProgress(raw, preferredDevice) : undefined
|
|
875
|
+
});
|
|
876
|
+
this._tokenizer = tokenizer;
|
|
877
|
+
this._model = model;
|
|
878
|
+
this._id2label = model.config.id2label;
|
|
879
|
+
const label2id = model.config.label2id;
|
|
880
|
+
this._oId = label2id?.["O"] ?? 0;
|
|
881
|
+
this.actualDevice = preferredDevice;
|
|
882
|
+
})();
|
|
883
|
+
return this._loadPromise;
|
|
884
|
+
}
|
|
885
|
+
async detect(text, opts) {
|
|
886
|
+
if (opts?.model && opts.model !== this.model) {
|
|
887
|
+
throw new Error(`CalibratedDetector was constructed with model='${this.model}'; ` + `per-call model override ('${opts.model}') not supported.`);
|
|
888
|
+
}
|
|
889
|
+
if (!text)
|
|
890
|
+
return [];
|
|
891
|
+
await this.load();
|
|
892
|
+
return this._detectChunked(text);
|
|
893
|
+
}
|
|
894
|
+
async _detectChunked(text) {
|
|
895
|
+
if (text.length <= 1500) {
|
|
896
|
+
return await this._detectWindow(text, 0);
|
|
897
|
+
}
|
|
898
|
+
const stride = 1500 - 200;
|
|
899
|
+
const seen = new Set;
|
|
900
|
+
const out = [];
|
|
901
|
+
for (let s = 0;s < text.length; s += stride) {
|
|
902
|
+
const e = Math.min(s + 1500, text.length);
|
|
903
|
+
const sub = text.slice(s, e);
|
|
904
|
+
const subSpans = await this._detectWindow(sub, s);
|
|
905
|
+
for (const sp of subSpans) {
|
|
906
|
+
const key = `${sp.start}|${sp.end}|${sp.label}`;
|
|
907
|
+
if (seen.has(key))
|
|
908
|
+
continue;
|
|
909
|
+
seen.add(key);
|
|
910
|
+
out.push(sp);
|
|
911
|
+
}
|
|
912
|
+
if (e >= text.length)
|
|
913
|
+
break;
|
|
914
|
+
}
|
|
915
|
+
out.sort((a, b) => a.start - b.start || a.end - b.end);
|
|
916
|
+
return out;
|
|
917
|
+
}
|
|
918
|
+
async _detectWindow(text, charOffset) {
|
|
919
|
+
const tokenizer = this._tokenizer;
|
|
920
|
+
const model = this._model;
|
|
921
|
+
const enc = await tokenizer([text], {
|
|
922
|
+
padding: true,
|
|
923
|
+
truncation: true,
|
|
924
|
+
max_length: this.chunkTokens + 2
|
|
925
|
+
});
|
|
926
|
+
const inputs = {};
|
|
927
|
+
if (enc.input_ids)
|
|
928
|
+
inputs["input_ids"] = enc.input_ids;
|
|
929
|
+
if (enc.attention_mask)
|
|
930
|
+
inputs["attention_mask"] = enc.attention_mask;
|
|
931
|
+
const out = await model(inputs);
|
|
932
|
+
const logits = readLogits(out.logits);
|
|
933
|
+
if (this.oBias !== 0) {
|
|
934
|
+
for (let t = 0;t < logits.length; t++) {
|
|
935
|
+
const row = logits[t];
|
|
936
|
+
if (row === undefined)
|
|
937
|
+
continue;
|
|
938
|
+
const cur = row[this._oId];
|
|
939
|
+
if (cur !== undefined) {
|
|
940
|
+
row[this._oId] = cur - this.oBias;
|
|
941
|
+
}
|
|
942
|
+
}
|
|
943
|
+
}
|
|
944
|
+
const inputIds = readInputIds(enc.input_ids);
|
|
945
|
+
const tokenLabels = [];
|
|
946
|
+
for (let t = 0;t < logits.length; t++) {
|
|
947
|
+
const row = logits[t];
|
|
948
|
+
const id = inputIds[t];
|
|
949
|
+
if (!row || id === undefined)
|
|
950
|
+
continue;
|
|
951
|
+
let maxIdx = 0;
|
|
952
|
+
let maxVal = row[0];
|
|
953
|
+
for (let i = 1;i < row.length; i++) {
|
|
954
|
+
if (row[i] > maxVal) {
|
|
955
|
+
maxVal = row[i];
|
|
956
|
+
maxIdx = i;
|
|
957
|
+
}
|
|
958
|
+
}
|
|
959
|
+
tokenLabels.push({ id, label: this._id2label[maxIdx] ?? "O" });
|
|
960
|
+
}
|
|
961
|
+
const spans = [];
|
|
962
|
+
let cursor = 0;
|
|
963
|
+
let groupIds = [];
|
|
964
|
+
let groupCat = null;
|
|
965
|
+
const flush = () => {
|
|
966
|
+
if (groupCat === null || groupIds.length === 0)
|
|
967
|
+
return;
|
|
968
|
+
const surface = normalizeDecoded(tokenizer.decode(groupIds, { skip_special_tokens: true }));
|
|
969
|
+
groupIds = [];
|
|
970
|
+
const cat = groupCat;
|
|
971
|
+
groupCat = null;
|
|
972
|
+
if (!surface)
|
|
973
|
+
return;
|
|
974
|
+
const idx = text.indexOf(surface, cursor);
|
|
975
|
+
if (idx === -1)
|
|
976
|
+
return;
|
|
977
|
+
let s = idx;
|
|
978
|
+
let e = idx + surface.length;
|
|
979
|
+
cursor = e;
|
|
980
|
+
if (this.trimWhitespace)
|
|
981
|
+
[s, e] = trimSpan2(text, s, e);
|
|
982
|
+
if (s >= e)
|
|
983
|
+
return;
|
|
984
|
+
spans.push({
|
|
985
|
+
start: s + charOffset,
|
|
986
|
+
end: e + charOffset,
|
|
987
|
+
label: cat,
|
|
988
|
+
text: text.slice(s, e),
|
|
989
|
+
score: 1
|
|
990
|
+
});
|
|
991
|
+
};
|
|
992
|
+
for (const { id, label } of tokenLabels) {
|
|
993
|
+
const cat = label === "O" ? null : label.split("-", 2)[1] ?? null;
|
|
994
|
+
if (cat === groupCat && cat !== null) {
|
|
995
|
+
groupIds.push(id);
|
|
996
|
+
} else {
|
|
997
|
+
flush();
|
|
998
|
+
if (cat !== null) {
|
|
999
|
+
groupIds.push(id);
|
|
1000
|
+
groupCat = cat;
|
|
1001
|
+
}
|
|
1002
|
+
}
|
|
1003
|
+
}
|
|
1004
|
+
flush();
|
|
1005
|
+
return spans;
|
|
1006
|
+
}
|
|
1007
|
+
_emitProgress(raw, device) {
|
|
1008
|
+
if (!this._onProgress)
|
|
1009
|
+
return;
|
|
1010
|
+
const event = {
|
|
1011
|
+
phase: raw.status ?? "progress",
|
|
1012
|
+
device
|
|
1013
|
+
};
|
|
1014
|
+
if (raw.file)
|
|
1015
|
+
event.file = raw.file;
|
|
1016
|
+
if (raw.status === "progress") {
|
|
1017
|
+
if (raw.loaded != null)
|
|
1018
|
+
event.loadedBytes = raw.loaded;
|
|
1019
|
+
if (raw.total != null)
|
|
1020
|
+
event.totalBytes = raw.total;
|
|
1021
|
+
if (raw.progress != null) {
|
|
1022
|
+
const norm = raw.progress > 1 ? raw.progress / 100 : raw.progress;
|
|
1023
|
+
event.fraction = Math.max(0, Math.min(1, norm));
|
|
1024
|
+
}
|
|
1025
|
+
}
|
|
1026
|
+
this._onProgress(event);
|
|
1027
|
+
}
|
|
1028
|
+
}
|
|
1029
|
+
async function loadTransformersJs2() {
|
|
1030
|
+
try {
|
|
1031
|
+
return await import("@huggingface/transformers");
|
|
1032
|
+
} catch {
|
|
1033
|
+
throw new Error("CalibratedDetector requires `@huggingface/transformers`. Install with: bun add @huggingface/transformers");
|
|
1034
|
+
}
|
|
1035
|
+
}
|
|
1036
|
+
function readInputIds(tensor) {
|
|
1037
|
+
if (!tensor)
|
|
1038
|
+
return [];
|
|
1039
|
+
if (typeof tensor.tolist === "function") {
|
|
1040
|
+
const list = tensor.tolist();
|
|
1041
|
+
if (Array.isArray(list) && Array.isArray(list[0])) {
|
|
1042
|
+
return list[0].map((x) => Number(x));
|
|
1043
|
+
}
|
|
1044
|
+
if (Array.isArray(list))
|
|
1045
|
+
return list.map((x) => Number(x));
|
|
1046
|
+
}
|
|
1047
|
+
const dims = tensor.dims;
|
|
1048
|
+
const seq = dims[dims.length - 1] ?? 0;
|
|
1049
|
+
const data = tensor.data;
|
|
1050
|
+
const out = [];
|
|
1051
|
+
for (let i = 0;i < seq; i++)
|
|
1052
|
+
out.push(Number(data[i]));
|
|
1053
|
+
return out;
|
|
1054
|
+
}
|
|
1055
|
+
function normalizeDecoded(s) {
|
|
1056
|
+
return s.replace(/##/g, "").trim();
|
|
1057
|
+
}
|
|
1058
|
+
function readLogits(tensor) {
|
|
1059
|
+
if (typeof tensor.tolist === "function") {
|
|
1060
|
+
const list = tensor.tolist();
|
|
1061
|
+
if (Array.isArray(list) && Array.isArray(list[0]) && Array.isArray(list[0][0])) {
|
|
1062
|
+
return list[0];
|
|
1063
|
+
}
|
|
1064
|
+
if (Array.isArray(list) && Array.isArray(list[0])) {
|
|
1065
|
+
return list;
|
|
1066
|
+
}
|
|
1067
|
+
}
|
|
1068
|
+
const dims = tensor.dims;
|
|
1069
|
+
const seq = dims[dims.length - 2] ?? 0;
|
|
1070
|
+
const cls = dims[dims.length - 1] ?? 0;
|
|
1071
|
+
const data = tensor.data;
|
|
1072
|
+
const out = [];
|
|
1073
|
+
for (let t = 0;t < seq; t++) {
|
|
1074
|
+
const row = [];
|
|
1075
|
+
for (let c = 0;c < cls; c++) {
|
|
1076
|
+
row.push(Number(data[t * cls + c]));
|
|
1077
|
+
}
|
|
1078
|
+
out.push(row);
|
|
1079
|
+
}
|
|
1080
|
+
return out;
|
|
1081
|
+
}
|
|
789
1082
|
// src/detectors/index.ts
|
|
790
1083
|
function defaultDetector() {
|
|
791
1084
|
const hasKey = typeof process !== "undefined" && Boolean(process.env?.GHEIM_API_KEY);
|
|
792
|
-
return hasKey ? new RemoteDetector : new
|
|
1085
|
+
return hasKey ? new RemoteDetector : new CalibratedDetector;
|
|
793
1086
|
}
|
|
794
1087
|
// src/plain.ts
|
|
795
1088
|
function resolve(session, detector) {
|
|
@@ -840,7 +1133,7 @@ async function* deanonymizeStream(chunks, session) {
|
|
|
840
1133
|
}
|
|
841
1134
|
|
|
842
1135
|
// src/index.ts
|
|
843
|
-
var VERSION = "0.1.
|
|
1136
|
+
var VERSION = "0.1.6";
|
|
844
1137
|
export {
|
|
845
1138
|
renderSentinel,
|
|
846
1139
|
parseSentinel,
|
|
@@ -861,5 +1154,6 @@ export {
|
|
|
861
1154
|
LocalDetector,
|
|
862
1155
|
LABEL_TO_TAG,
|
|
863
1156
|
DEFAULT_BASE_URL,
|
|
864
|
-
CompositeDetector
|
|
1157
|
+
CompositeDetector,
|
|
1158
|
+
CalibratedDetector
|
|
865
1159
|
};
|
package/dist/esm/openai/index.js
CHANGED
|
@@ -708,10 +708,281 @@ class CompositeDetector {
|
|
|
708
708
|
return out;
|
|
709
709
|
}
|
|
710
710
|
}
|
|
711
|
+
// src/detectors/calibrated.ts
|
|
712
|
+
var DEFAULT_MODEL3 = "joelbarmettler/gheim-ch-560m";
|
|
713
|
+
var DEFAULT_CHUNK_TOKENS2 = 510;
|
|
714
|
+
var DEFAULT_OVERLAP_TOKENS2 = 32;
|
|
715
|
+
function trimSpan2(text, start, end) {
|
|
716
|
+
while (start < end && /\s/.test(text[start] ?? ""))
|
|
717
|
+
start++;
|
|
718
|
+
while (end > start && /\s/.test(text[end - 1] ?? ""))
|
|
719
|
+
end--;
|
|
720
|
+
return [start, end];
|
|
721
|
+
}
|
|
722
|
+
|
|
723
|
+
class CalibratedDetector {
|
|
724
|
+
model;
|
|
725
|
+
oBias;
|
|
726
|
+
device;
|
|
727
|
+
dtype;
|
|
728
|
+
chunkTokens;
|
|
729
|
+
chunkOverlapTokens;
|
|
730
|
+
trimWhitespace;
|
|
731
|
+
actualDevice = null;
|
|
732
|
+
_model = null;
|
|
733
|
+
_tokenizer = null;
|
|
734
|
+
_id2label = {};
|
|
735
|
+
_oId = 0;
|
|
736
|
+
_loadPromise = null;
|
|
737
|
+
_onProgress;
|
|
738
|
+
constructor(opts = {}) {
|
|
739
|
+
this.model = opts.model ?? DEFAULT_MODEL3;
|
|
740
|
+
this.oBias = opts.oBias ?? 0.5;
|
|
741
|
+
this.device = opts.device ?? "auto";
|
|
742
|
+
this.dtype = opts.dtype ?? "fp32";
|
|
743
|
+
this.chunkTokens = opts.chunkTokens ?? DEFAULT_CHUNK_TOKENS2;
|
|
744
|
+
this.chunkOverlapTokens = opts.chunkOverlapTokens ?? DEFAULT_OVERLAP_TOKENS2;
|
|
745
|
+
this.trimWhitespace = opts.trimWhitespace ?? true;
|
|
746
|
+
this._onProgress = opts.onProgress;
|
|
747
|
+
if (this.chunkOverlapTokens >= this.chunkTokens) {
|
|
748
|
+
throw new Error(`chunkOverlapTokens (${this.chunkOverlapTokens}) must be smaller than ` + `chunkTokens (${this.chunkTokens})`);
|
|
749
|
+
}
|
|
750
|
+
}
|
|
751
|
+
async load() {
|
|
752
|
+
if (this._model)
|
|
753
|
+
return;
|
|
754
|
+
if (this._loadPromise)
|
|
755
|
+
return this._loadPromise;
|
|
756
|
+
this._loadPromise = (async () => {
|
|
757
|
+
const mod = await loadTransformersJs2();
|
|
758
|
+
let preferredDevice = this.device;
|
|
759
|
+
if (this.device === "auto") {
|
|
760
|
+
const nav = typeof navigator !== "undefined" ? navigator : null;
|
|
761
|
+
preferredDevice = "wasm";
|
|
762
|
+
if (nav?.gpu?.requestAdapter) {
|
|
763
|
+
try {
|
|
764
|
+
const adapter = await nav.gpu.requestAdapter();
|
|
765
|
+
if (adapter)
|
|
766
|
+
preferredDevice = "webgpu";
|
|
767
|
+
} catch {}
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
const tokenizer = await mod.AutoTokenizer.from_pretrained(this.model);
|
|
771
|
+
const model = await mod.AutoModelForTokenClassification.from_pretrained(this.model, {
|
|
772
|
+
device: preferredDevice,
|
|
773
|
+
dtype: this.dtype,
|
|
774
|
+
progress_callback: this._onProgress ? (raw) => this._emitProgress(raw, preferredDevice) : undefined
|
|
775
|
+
});
|
|
776
|
+
this._tokenizer = tokenizer;
|
|
777
|
+
this._model = model;
|
|
778
|
+
this._id2label = model.config.id2label;
|
|
779
|
+
const label2id = model.config.label2id;
|
|
780
|
+
this._oId = label2id?.["O"] ?? 0;
|
|
781
|
+
this.actualDevice = preferredDevice;
|
|
782
|
+
})();
|
|
783
|
+
return this._loadPromise;
|
|
784
|
+
}
|
|
785
|
+
async detect(text, opts) {
|
|
786
|
+
if (opts?.model && opts.model !== this.model) {
|
|
787
|
+
throw new Error(`CalibratedDetector was constructed with model='${this.model}'; ` + `per-call model override ('${opts.model}') not supported.`);
|
|
788
|
+
}
|
|
789
|
+
if (!text)
|
|
790
|
+
return [];
|
|
791
|
+
await this.load();
|
|
792
|
+
return this._detectChunked(text);
|
|
793
|
+
}
|
|
794
|
+
async _detectChunked(text) {
|
|
795
|
+
if (text.length <= 1500) {
|
|
796
|
+
return await this._detectWindow(text, 0);
|
|
797
|
+
}
|
|
798
|
+
const stride = 1500 - 200;
|
|
799
|
+
const seen = new Set;
|
|
800
|
+
const out = [];
|
|
801
|
+
for (let s = 0;s < text.length; s += stride) {
|
|
802
|
+
const e = Math.min(s + 1500, text.length);
|
|
803
|
+
const sub = text.slice(s, e);
|
|
804
|
+
const subSpans = await this._detectWindow(sub, s);
|
|
805
|
+
for (const sp of subSpans) {
|
|
806
|
+
const key = `${sp.start}|${sp.end}|${sp.label}`;
|
|
807
|
+
if (seen.has(key))
|
|
808
|
+
continue;
|
|
809
|
+
seen.add(key);
|
|
810
|
+
out.push(sp);
|
|
811
|
+
}
|
|
812
|
+
if (e >= text.length)
|
|
813
|
+
break;
|
|
814
|
+
}
|
|
815
|
+
out.sort((a, b) => a.start - b.start || a.end - b.end);
|
|
816
|
+
return out;
|
|
817
|
+
}
|
|
818
|
+
async _detectWindow(text, charOffset) {
|
|
819
|
+
const tokenizer = this._tokenizer;
|
|
820
|
+
const model = this._model;
|
|
821
|
+
const enc = await tokenizer([text], {
|
|
822
|
+
padding: true,
|
|
823
|
+
truncation: true,
|
|
824
|
+
max_length: this.chunkTokens + 2
|
|
825
|
+
});
|
|
826
|
+
const inputs = {};
|
|
827
|
+
if (enc.input_ids)
|
|
828
|
+
inputs["input_ids"] = enc.input_ids;
|
|
829
|
+
if (enc.attention_mask)
|
|
830
|
+
inputs["attention_mask"] = enc.attention_mask;
|
|
831
|
+
const out = await model(inputs);
|
|
832
|
+
const logits = readLogits(out.logits);
|
|
833
|
+
if (this.oBias !== 0) {
|
|
834
|
+
for (let t = 0;t < logits.length; t++) {
|
|
835
|
+
const row = logits[t];
|
|
836
|
+
if (row === undefined)
|
|
837
|
+
continue;
|
|
838
|
+
const cur = row[this._oId];
|
|
839
|
+
if (cur !== undefined) {
|
|
840
|
+
row[this._oId] = cur - this.oBias;
|
|
841
|
+
}
|
|
842
|
+
}
|
|
843
|
+
}
|
|
844
|
+
const inputIds = readInputIds(enc.input_ids);
|
|
845
|
+
const tokenLabels = [];
|
|
846
|
+
for (let t = 0;t < logits.length; t++) {
|
|
847
|
+
const row = logits[t];
|
|
848
|
+
const id = inputIds[t];
|
|
849
|
+
if (!row || id === undefined)
|
|
850
|
+
continue;
|
|
851
|
+
let maxIdx = 0;
|
|
852
|
+
let maxVal = row[0];
|
|
853
|
+
for (let i = 1;i < row.length; i++) {
|
|
854
|
+
if (row[i] > maxVal) {
|
|
855
|
+
maxVal = row[i];
|
|
856
|
+
maxIdx = i;
|
|
857
|
+
}
|
|
858
|
+
}
|
|
859
|
+
tokenLabels.push({ id, label: this._id2label[maxIdx] ?? "O" });
|
|
860
|
+
}
|
|
861
|
+
const spans = [];
|
|
862
|
+
let cursor = 0;
|
|
863
|
+
let groupIds = [];
|
|
864
|
+
let groupCat = null;
|
|
865
|
+
const flush = () => {
|
|
866
|
+
if (groupCat === null || groupIds.length === 0)
|
|
867
|
+
return;
|
|
868
|
+
const surface = normalizeDecoded(tokenizer.decode(groupIds, { skip_special_tokens: true }));
|
|
869
|
+
groupIds = [];
|
|
870
|
+
const cat = groupCat;
|
|
871
|
+
groupCat = null;
|
|
872
|
+
if (!surface)
|
|
873
|
+
return;
|
|
874
|
+
const idx = text.indexOf(surface, cursor);
|
|
875
|
+
if (idx === -1)
|
|
876
|
+
return;
|
|
877
|
+
let s = idx;
|
|
878
|
+
let e = idx + surface.length;
|
|
879
|
+
cursor = e;
|
|
880
|
+
if (this.trimWhitespace)
|
|
881
|
+
[s, e] = trimSpan2(text, s, e);
|
|
882
|
+
if (s >= e)
|
|
883
|
+
return;
|
|
884
|
+
spans.push({
|
|
885
|
+
start: s + charOffset,
|
|
886
|
+
end: e + charOffset,
|
|
887
|
+
label: cat,
|
|
888
|
+
text: text.slice(s, e),
|
|
889
|
+
score: 1
|
|
890
|
+
});
|
|
891
|
+
};
|
|
892
|
+
for (const { id, label } of tokenLabels) {
|
|
893
|
+
const cat = label === "O" ? null : label.split("-", 2)[1] ?? null;
|
|
894
|
+
if (cat === groupCat && cat !== null) {
|
|
895
|
+
groupIds.push(id);
|
|
896
|
+
} else {
|
|
897
|
+
flush();
|
|
898
|
+
if (cat !== null) {
|
|
899
|
+
groupIds.push(id);
|
|
900
|
+
groupCat = cat;
|
|
901
|
+
}
|
|
902
|
+
}
|
|
903
|
+
}
|
|
904
|
+
flush();
|
|
905
|
+
return spans;
|
|
906
|
+
}
|
|
907
|
+
_emitProgress(raw, device) {
|
|
908
|
+
if (!this._onProgress)
|
|
909
|
+
return;
|
|
910
|
+
const event = {
|
|
911
|
+
phase: raw.status ?? "progress",
|
|
912
|
+
device
|
|
913
|
+
};
|
|
914
|
+
if (raw.file)
|
|
915
|
+
event.file = raw.file;
|
|
916
|
+
if (raw.status === "progress") {
|
|
917
|
+
if (raw.loaded != null)
|
|
918
|
+
event.loadedBytes = raw.loaded;
|
|
919
|
+
if (raw.total != null)
|
|
920
|
+
event.totalBytes = raw.total;
|
|
921
|
+
if (raw.progress != null) {
|
|
922
|
+
const norm = raw.progress > 1 ? raw.progress / 100 : raw.progress;
|
|
923
|
+
event.fraction = Math.max(0, Math.min(1, norm));
|
|
924
|
+
}
|
|
925
|
+
}
|
|
926
|
+
this._onProgress(event);
|
|
927
|
+
}
|
|
928
|
+
}
|
|
929
|
+
async function loadTransformersJs2() {
|
|
930
|
+
try {
|
|
931
|
+
return await import("@huggingface/transformers");
|
|
932
|
+
} catch {
|
|
933
|
+
throw new Error("CalibratedDetector requires `@huggingface/transformers`. Install with: bun add @huggingface/transformers");
|
|
934
|
+
}
|
|
935
|
+
}
|
|
936
|
+
function readInputIds(tensor) {
|
|
937
|
+
if (!tensor)
|
|
938
|
+
return [];
|
|
939
|
+
if (typeof tensor.tolist === "function") {
|
|
940
|
+
const list = tensor.tolist();
|
|
941
|
+
if (Array.isArray(list) && Array.isArray(list[0])) {
|
|
942
|
+
return list[0].map((x) => Number(x));
|
|
943
|
+
}
|
|
944
|
+
if (Array.isArray(list))
|
|
945
|
+
return list.map((x) => Number(x));
|
|
946
|
+
}
|
|
947
|
+
const dims = tensor.dims;
|
|
948
|
+
const seq = dims[dims.length - 1] ?? 0;
|
|
949
|
+
const data = tensor.data;
|
|
950
|
+
const out = [];
|
|
951
|
+
for (let i = 0;i < seq; i++)
|
|
952
|
+
out.push(Number(data[i]));
|
|
953
|
+
return out;
|
|
954
|
+
}
|
|
955
|
+
function normalizeDecoded(s) {
|
|
956
|
+
return s.replace(/##/g, "").trim();
|
|
957
|
+
}
|
|
958
|
+
function readLogits(tensor) {
|
|
959
|
+
if (typeof tensor.tolist === "function") {
|
|
960
|
+
const list = tensor.tolist();
|
|
961
|
+
if (Array.isArray(list) && Array.isArray(list[0]) && Array.isArray(list[0][0])) {
|
|
962
|
+
return list[0];
|
|
963
|
+
}
|
|
964
|
+
if (Array.isArray(list) && Array.isArray(list[0])) {
|
|
965
|
+
return list;
|
|
966
|
+
}
|
|
967
|
+
}
|
|
968
|
+
const dims = tensor.dims;
|
|
969
|
+
const seq = dims[dims.length - 2] ?? 0;
|
|
970
|
+
const cls = dims[dims.length - 1] ?? 0;
|
|
971
|
+
const data = tensor.data;
|
|
972
|
+
const out = [];
|
|
973
|
+
for (let t = 0;t < seq; t++) {
|
|
974
|
+
const row = [];
|
|
975
|
+
for (let c = 0;c < cls; c++) {
|
|
976
|
+
row.push(Number(data[t * cls + c]));
|
|
977
|
+
}
|
|
978
|
+
out.push(row);
|
|
979
|
+
}
|
|
980
|
+
return out;
|
|
981
|
+
}
|
|
711
982
|
// src/detectors/index.ts
|
|
712
983
|
function defaultDetector() {
|
|
713
984
|
const hasKey = typeof process !== "undefined" && Boolean(process.env?.GHEIM_API_KEY);
|
|
714
|
-
return hasKey ? new RemoteDetector : new
|
|
985
|
+
return hasKey ? new RemoteDetector : new CalibratedDetector;
|
|
715
986
|
}
|
|
716
987
|
|
|
717
988
|
// src/core/session.ts
|
|
@@ -772,8 +1043,8 @@ class Session {
|
|
|
772
1043
|
}
|
|
773
1044
|
out.push(text.slice(cursor));
|
|
774
1045
|
let redacted = out.join("");
|
|
775
|
-
|
|
776
|
-
return { redacted, merged };
|
|
1046
|
+
const recovered = recoverDuplicateOccurrences(text, redacted, merged, this.mapping);
|
|
1047
|
+
return { redacted: recovered.redacted, merged: recovered.merged };
|
|
777
1048
|
}
|
|
778
1049
|
restore(text) {
|
|
779
1050
|
if (Object.keys(this.mapping).length === 0)
|
|
@@ -832,14 +1103,36 @@ function mergeAdjacent(text, spans) {
|
|
|
832
1103
|
function escapeRegex(s) {
|
|
833
1104
|
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
834
1105
|
}
|
|
835
|
-
function recoverDuplicateOccurrences(redacted, mapping) {
|
|
1106
|
+
function recoverDuplicateOccurrences(text, redacted, merged, mapping) {
|
|
1107
|
+
const surfaceToLabel = new Map;
|
|
1108
|
+
for (const m of merged)
|
|
1109
|
+
surfaceToLabel.set(text.slice(m.start, m.end), m.label);
|
|
1110
|
+
const covered = merged.map((m) => [m.start, m.end]);
|
|
1111
|
+
const overlaps = (s, e) => covered.some(([cs, ce]) => s < ce && e > cs);
|
|
836
1112
|
const entries = Object.entries(mapping).filter(([, surface]) => surface && surface.length > 0).sort((a, b) => b[1].length - a[1].length);
|
|
837
1113
|
let result = redacted;
|
|
1114
|
+
const additional = [];
|
|
838
1115
|
for (const [sentinel, surface] of entries) {
|
|
839
|
-
const
|
|
840
|
-
result = result.replace(
|
|
1116
|
+
const boundaryPattern = `(?<![\\p{L}\\p{N}_])${escapeRegex(surface)}(?![\\p{L}\\p{N}_])`;
|
|
1117
|
+
result = result.replace(new RegExp(boundaryPattern, "gu"), sentinel);
|
|
1118
|
+
const label = surfaceToLabel.get(surface);
|
|
1119
|
+
if (!label)
|
|
1120
|
+
continue;
|
|
1121
|
+
const origRe = new RegExp(boundaryPattern, "gu");
|
|
1122
|
+
let match;
|
|
1123
|
+
while ((match = origRe.exec(text)) !== null) {
|
|
1124
|
+
const start = match.index;
|
|
1125
|
+
const end = start + surface.length;
|
|
1126
|
+
if (overlaps(start, end))
|
|
1127
|
+
continue;
|
|
1128
|
+
additional.push({ start, end, label });
|
|
1129
|
+
covered.push([start, end]);
|
|
1130
|
+
}
|
|
841
1131
|
}
|
|
842
|
-
|
|
1132
|
+
if (additional.length === 0)
|
|
1133
|
+
return { redacted: result, merged };
|
|
1134
|
+
const augmented = [...merged, ...additional].sort((a, b) => a.start - b.start);
|
|
1135
|
+
return { redacted: result, merged: augmented };
|
|
843
1136
|
}
|
|
844
1137
|
|
|
845
1138
|
// src/openai/_base.ts
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../../src/core/session.ts"],"names":[],"mappings":"AAAA,kDAAkD;AAClD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AAGjD;;;gEAGgE;AAChE,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;AAkBtD,uCAAuC;AACvC,MAAM,WAAW,cAAc;IAC7B;;;;;;;;;;OAUG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;CAC1C;AAED,qBAAa,OAAO;IAClB,8DAA8D;IAC9D,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAM;IACrC,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,mBAAmB,CAAkC;gBAEjD,IAAI,GAAE,cAAmB;IAQrC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM;IAiBhD,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GAAG,MAAM;IAIvE;;;;;;;OAOG;IACH,kBAAkB,CAChB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GACpC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,UAAU,EAAE,CAAA;KAAE;
|
|
1
|
+
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../../src/core/session.ts"],"names":[],"mappings":"AAAA,kDAAkD;AAClD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AAGjD;;;gEAGgE;AAChE,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;AAkBtD,uCAAuC;AACvC,MAAM,WAAW,cAAc;IAC7B;;;;;;;;;;OAUG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;CAC1C;AAED,qBAAa,OAAO;IAClB,8DAA8D;IAC9D,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAM;IACrC,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,mBAAmB,CAAkC;gBAEjD,IAAI,GAAE,cAAmB;IAQrC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM;IAiBhD,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GAAG,MAAM;IAIvE;;;;;;;OAOG;IACH,kBAAkB,CAChB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GACpC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,UAAU,EAAE,CAAA;KAAE;IA8B7C,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAK7B,MAAM,IAAI;QAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE;IAI7C,MAAM,CAAC,QAAQ,CACb,GAAG,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,EACxC,IAAI,GAAE,cAAmB,GACxB,OAAO;CAqBX;AAID,8EAA8E;AAC9E,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAC3B,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GACpC,UAAU,EAAE,CAqBd"}
|