gheim 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/index.cjs +290 -4
- package/dist/cjs/openai/index.cjs +287 -2
- package/dist/esm/index.js +290 -4
- package/dist/esm/openai/index.js +287 -2
- package/dist/types/core/session.d.ts.map +1 -1
- package/dist/types/detectors/calibrated.d.ts +78 -0
- package/dist/types/detectors/calibrated.d.ts.map +1 -0
- package/dist/types/detectors/index.d.ts +8 -1
- package/dist/types/detectors/index.d.ts.map +1 -1
- package/dist/types/index.d.ts +3 -3
- package/dist/types/index.d.ts.map +1 -1
- package/package.json +1 -1
package/dist/esm/index.js
CHANGED
|
@@ -99,7 +99,9 @@ class Session {
|
|
|
99
99
|
cursor = end;
|
|
100
100
|
}
|
|
101
101
|
out.push(text.slice(cursor));
|
|
102
|
-
|
|
102
|
+
let redacted = out.join("");
|
|
103
|
+
redacted = recoverDuplicateOccurrences(redacted, this.mapping);
|
|
104
|
+
return { redacted, merged };
|
|
103
105
|
}
|
|
104
106
|
restore(text) {
|
|
105
107
|
if (Object.keys(this.mapping).length === 0)
|
|
@@ -155,6 +157,18 @@ function mergeAdjacent(text, spans) {
|
|
|
155
157
|
}
|
|
156
158
|
return out;
|
|
157
159
|
}
|
|
160
|
+
function escapeRegex(s) {
|
|
161
|
+
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
162
|
+
}
|
|
163
|
+
function recoverDuplicateOccurrences(redacted, mapping) {
|
|
164
|
+
const entries = Object.entries(mapping).filter(([, surface]) => surface && surface.length > 0).sort((a, b) => b[1].length - a[1].length);
|
|
165
|
+
let result = redacted;
|
|
166
|
+
for (const [sentinel, surface] of entries) {
|
|
167
|
+
const pattern = new RegExp(`(?<![\\p{L}\\p{N}_])${escapeRegex(surface)}(?![\\p{L}\\p{N}_])`, "gu");
|
|
168
|
+
result = result.replace(pattern, sentinel);
|
|
169
|
+
}
|
|
170
|
+
return result;
|
|
171
|
+
}
|
|
158
172
|
// src/core/stream.ts
|
|
159
173
|
var SENTINEL_AT = /<([A-Z][A-Z0-9_]*)_(\d+)>/y;
|
|
160
174
|
var MAX_SENTINEL_LEN = 64;
|
|
@@ -772,10 +786,281 @@ class CompositeDetector {
|
|
|
772
786
|
return out;
|
|
773
787
|
}
|
|
774
788
|
}
|
|
789
|
+
// src/detectors/calibrated.ts
|
|
790
|
+
var DEFAULT_MODEL3 = "joelbarmettler/gheim-ch-560m";
|
|
791
|
+
var DEFAULT_CHUNK_TOKENS2 = 510;
|
|
792
|
+
var DEFAULT_OVERLAP_TOKENS2 = 32;
|
|
793
|
+
function trimSpan2(text, start, end) {
|
|
794
|
+
while (start < end && /\s/.test(text[start] ?? ""))
|
|
795
|
+
start++;
|
|
796
|
+
while (end > start && /\s/.test(text[end - 1] ?? ""))
|
|
797
|
+
end--;
|
|
798
|
+
return [start, end];
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
class CalibratedDetector {
|
|
802
|
+
model;
|
|
803
|
+
oBias;
|
|
804
|
+
device;
|
|
805
|
+
dtype;
|
|
806
|
+
chunkTokens;
|
|
807
|
+
chunkOverlapTokens;
|
|
808
|
+
trimWhitespace;
|
|
809
|
+
actualDevice = null;
|
|
810
|
+
_model = null;
|
|
811
|
+
_tokenizer = null;
|
|
812
|
+
_id2label = {};
|
|
813
|
+
_oId = 0;
|
|
814
|
+
_loadPromise = null;
|
|
815
|
+
_onProgress;
|
|
816
|
+
constructor(opts = {}) {
|
|
817
|
+
this.model = opts.model ?? DEFAULT_MODEL3;
|
|
818
|
+
this.oBias = opts.oBias ?? 0.5;
|
|
819
|
+
this.device = opts.device ?? "auto";
|
|
820
|
+
this.dtype = opts.dtype ?? "fp32";
|
|
821
|
+
this.chunkTokens = opts.chunkTokens ?? DEFAULT_CHUNK_TOKENS2;
|
|
822
|
+
this.chunkOverlapTokens = opts.chunkOverlapTokens ?? DEFAULT_OVERLAP_TOKENS2;
|
|
823
|
+
this.trimWhitespace = opts.trimWhitespace ?? true;
|
|
824
|
+
this._onProgress = opts.onProgress;
|
|
825
|
+
if (this.chunkOverlapTokens >= this.chunkTokens) {
|
|
826
|
+
throw new Error(`chunkOverlapTokens (${this.chunkOverlapTokens}) must be smaller than ` + `chunkTokens (${this.chunkTokens})`);
|
|
827
|
+
}
|
|
828
|
+
}
|
|
829
|
+
async load() {
|
|
830
|
+
if (this._model)
|
|
831
|
+
return;
|
|
832
|
+
if (this._loadPromise)
|
|
833
|
+
return this._loadPromise;
|
|
834
|
+
this._loadPromise = (async () => {
|
|
835
|
+
const mod = await loadTransformersJs2();
|
|
836
|
+
let preferredDevice = this.device;
|
|
837
|
+
if (this.device === "auto") {
|
|
838
|
+
const nav = typeof navigator !== "undefined" ? navigator : null;
|
|
839
|
+
preferredDevice = "wasm";
|
|
840
|
+
if (nav?.gpu?.requestAdapter) {
|
|
841
|
+
try {
|
|
842
|
+
const adapter = await nav.gpu.requestAdapter();
|
|
843
|
+
if (adapter)
|
|
844
|
+
preferredDevice = "webgpu";
|
|
845
|
+
} catch {}
|
|
846
|
+
}
|
|
847
|
+
}
|
|
848
|
+
const tokenizer = await mod.AutoTokenizer.from_pretrained(this.model);
|
|
849
|
+
const model = await mod.AutoModelForTokenClassification.from_pretrained(this.model, {
|
|
850
|
+
device: preferredDevice,
|
|
851
|
+
dtype: this.dtype,
|
|
852
|
+
progress_callback: this._onProgress ? (raw) => this._emitProgress(raw, preferredDevice) : undefined
|
|
853
|
+
});
|
|
854
|
+
this._tokenizer = tokenizer;
|
|
855
|
+
this._model = model;
|
|
856
|
+
this._id2label = model.config.id2label;
|
|
857
|
+
const label2id = model.config.label2id;
|
|
858
|
+
this._oId = label2id?.["O"] ?? 0;
|
|
859
|
+
this.actualDevice = preferredDevice;
|
|
860
|
+
})();
|
|
861
|
+
return this._loadPromise;
|
|
862
|
+
}
|
|
863
|
+
async detect(text, opts) {
|
|
864
|
+
if (opts?.model && opts.model !== this.model) {
|
|
865
|
+
throw new Error(`CalibratedDetector was constructed with model='${this.model}'; ` + `per-call model override ('${opts.model}') not supported.`);
|
|
866
|
+
}
|
|
867
|
+
if (!text)
|
|
868
|
+
return [];
|
|
869
|
+
await this.load();
|
|
870
|
+
return this._detectChunked(text);
|
|
871
|
+
}
|
|
872
|
+
async _detectChunked(text) {
|
|
873
|
+
if (text.length <= 1500) {
|
|
874
|
+
return await this._detectWindow(text, 0);
|
|
875
|
+
}
|
|
876
|
+
const stride = 1500 - 200;
|
|
877
|
+
const seen = new Set;
|
|
878
|
+
const out = [];
|
|
879
|
+
for (let s = 0;s < text.length; s += stride) {
|
|
880
|
+
const e = Math.min(s + 1500, text.length);
|
|
881
|
+
const sub = text.slice(s, e);
|
|
882
|
+
const subSpans = await this._detectWindow(sub, s);
|
|
883
|
+
for (const sp of subSpans) {
|
|
884
|
+
const key = `${sp.start}|${sp.end}|${sp.label}`;
|
|
885
|
+
if (seen.has(key))
|
|
886
|
+
continue;
|
|
887
|
+
seen.add(key);
|
|
888
|
+
out.push(sp);
|
|
889
|
+
}
|
|
890
|
+
if (e >= text.length)
|
|
891
|
+
break;
|
|
892
|
+
}
|
|
893
|
+
out.sort((a, b) => a.start - b.start || a.end - b.end);
|
|
894
|
+
return out;
|
|
895
|
+
}
|
|
896
|
+
async _detectWindow(text, charOffset) {
|
|
897
|
+
const tokenizer = this._tokenizer;
|
|
898
|
+
const model = this._model;
|
|
899
|
+
const enc = await tokenizer([text], {
|
|
900
|
+
padding: true,
|
|
901
|
+
truncation: true,
|
|
902
|
+
max_length: this.chunkTokens + 2
|
|
903
|
+
});
|
|
904
|
+
const inputs = {};
|
|
905
|
+
if (enc.input_ids)
|
|
906
|
+
inputs["input_ids"] = enc.input_ids;
|
|
907
|
+
if (enc.attention_mask)
|
|
908
|
+
inputs["attention_mask"] = enc.attention_mask;
|
|
909
|
+
const out = await model(inputs);
|
|
910
|
+
const logits = readLogits(out.logits);
|
|
911
|
+
if (this.oBias !== 0) {
|
|
912
|
+
for (let t = 0;t < logits.length; t++) {
|
|
913
|
+
const row = logits[t];
|
|
914
|
+
if (row === undefined)
|
|
915
|
+
continue;
|
|
916
|
+
const cur = row[this._oId];
|
|
917
|
+
if (cur !== undefined) {
|
|
918
|
+
row[this._oId] = cur - this.oBias;
|
|
919
|
+
}
|
|
920
|
+
}
|
|
921
|
+
}
|
|
922
|
+
const inputIds = readInputIds(enc.input_ids);
|
|
923
|
+
const tokenLabels = [];
|
|
924
|
+
for (let t = 0;t < logits.length; t++) {
|
|
925
|
+
const row = logits[t];
|
|
926
|
+
const id = inputIds[t];
|
|
927
|
+
if (!row || id === undefined)
|
|
928
|
+
continue;
|
|
929
|
+
let maxIdx = 0;
|
|
930
|
+
let maxVal = row[0];
|
|
931
|
+
for (let i = 1;i < row.length; i++) {
|
|
932
|
+
if (row[i] > maxVal) {
|
|
933
|
+
maxVal = row[i];
|
|
934
|
+
maxIdx = i;
|
|
935
|
+
}
|
|
936
|
+
}
|
|
937
|
+
tokenLabels.push({ id, label: this._id2label[maxIdx] ?? "O" });
|
|
938
|
+
}
|
|
939
|
+
const spans = [];
|
|
940
|
+
let cursor = 0;
|
|
941
|
+
let groupIds = [];
|
|
942
|
+
let groupCat = null;
|
|
943
|
+
const flush = () => {
|
|
944
|
+
if (groupCat === null || groupIds.length === 0)
|
|
945
|
+
return;
|
|
946
|
+
const surface = normalizeDecoded(tokenizer.decode(groupIds, { skip_special_tokens: true }));
|
|
947
|
+
groupIds = [];
|
|
948
|
+
const cat = groupCat;
|
|
949
|
+
groupCat = null;
|
|
950
|
+
if (!surface)
|
|
951
|
+
return;
|
|
952
|
+
const idx = text.indexOf(surface, cursor);
|
|
953
|
+
if (idx === -1)
|
|
954
|
+
return;
|
|
955
|
+
let s = idx;
|
|
956
|
+
let e = idx + surface.length;
|
|
957
|
+
cursor = e;
|
|
958
|
+
if (this.trimWhitespace)
|
|
959
|
+
[s, e] = trimSpan2(text, s, e);
|
|
960
|
+
if (s >= e)
|
|
961
|
+
return;
|
|
962
|
+
spans.push({
|
|
963
|
+
start: s + charOffset,
|
|
964
|
+
end: e + charOffset,
|
|
965
|
+
label: cat,
|
|
966
|
+
text: text.slice(s, e),
|
|
967
|
+
score: 1
|
|
968
|
+
});
|
|
969
|
+
};
|
|
970
|
+
for (const { id, label } of tokenLabels) {
|
|
971
|
+
const cat = label === "O" ? null : label.split("-", 2)[1] ?? null;
|
|
972
|
+
if (cat === groupCat && cat !== null) {
|
|
973
|
+
groupIds.push(id);
|
|
974
|
+
} else {
|
|
975
|
+
flush();
|
|
976
|
+
if (cat !== null) {
|
|
977
|
+
groupIds.push(id);
|
|
978
|
+
groupCat = cat;
|
|
979
|
+
}
|
|
980
|
+
}
|
|
981
|
+
}
|
|
982
|
+
flush();
|
|
983
|
+
return spans;
|
|
984
|
+
}
|
|
985
|
+
_emitProgress(raw, device) {
|
|
986
|
+
if (!this._onProgress)
|
|
987
|
+
return;
|
|
988
|
+
const event = {
|
|
989
|
+
phase: raw.status ?? "progress",
|
|
990
|
+
device
|
|
991
|
+
};
|
|
992
|
+
if (raw.file)
|
|
993
|
+
event.file = raw.file;
|
|
994
|
+
if (raw.status === "progress") {
|
|
995
|
+
if (raw.loaded != null)
|
|
996
|
+
event.loadedBytes = raw.loaded;
|
|
997
|
+
if (raw.total != null)
|
|
998
|
+
event.totalBytes = raw.total;
|
|
999
|
+
if (raw.progress != null) {
|
|
1000
|
+
const norm = raw.progress > 1 ? raw.progress / 100 : raw.progress;
|
|
1001
|
+
event.fraction = Math.max(0, Math.min(1, norm));
|
|
1002
|
+
}
|
|
1003
|
+
}
|
|
1004
|
+
this._onProgress(event);
|
|
1005
|
+
}
|
|
1006
|
+
}
|
|
1007
|
+
async function loadTransformersJs2() {
|
|
1008
|
+
try {
|
|
1009
|
+
return await import("@huggingface/transformers");
|
|
1010
|
+
} catch {
|
|
1011
|
+
throw new Error("CalibratedDetector requires `@huggingface/transformers`. Install with: bun add @huggingface/transformers");
|
|
1012
|
+
}
|
|
1013
|
+
}
|
|
1014
|
+
function readInputIds(tensor) {
|
|
1015
|
+
if (!tensor)
|
|
1016
|
+
return [];
|
|
1017
|
+
if (typeof tensor.tolist === "function") {
|
|
1018
|
+
const list = tensor.tolist();
|
|
1019
|
+
if (Array.isArray(list) && Array.isArray(list[0])) {
|
|
1020
|
+
return list[0].map((x) => Number(x));
|
|
1021
|
+
}
|
|
1022
|
+
if (Array.isArray(list))
|
|
1023
|
+
return list.map((x) => Number(x));
|
|
1024
|
+
}
|
|
1025
|
+
const dims = tensor.dims;
|
|
1026
|
+
const seq = dims[dims.length - 1] ?? 0;
|
|
1027
|
+
const data = tensor.data;
|
|
1028
|
+
const out = [];
|
|
1029
|
+
for (let i = 0;i < seq; i++)
|
|
1030
|
+
out.push(Number(data[i]));
|
|
1031
|
+
return out;
|
|
1032
|
+
}
|
|
1033
|
+
function normalizeDecoded(s) {
|
|
1034
|
+
return s.replace(/##/g, "").trim();
|
|
1035
|
+
}
|
|
1036
|
+
function readLogits(tensor) {
|
|
1037
|
+
if (typeof tensor.tolist === "function") {
|
|
1038
|
+
const list = tensor.tolist();
|
|
1039
|
+
if (Array.isArray(list) && Array.isArray(list[0]) && Array.isArray(list[0][0])) {
|
|
1040
|
+
return list[0];
|
|
1041
|
+
}
|
|
1042
|
+
if (Array.isArray(list) && Array.isArray(list[0])) {
|
|
1043
|
+
return list;
|
|
1044
|
+
}
|
|
1045
|
+
}
|
|
1046
|
+
const dims = tensor.dims;
|
|
1047
|
+
const seq = dims[dims.length - 2] ?? 0;
|
|
1048
|
+
const cls = dims[dims.length - 1] ?? 0;
|
|
1049
|
+
const data = tensor.data;
|
|
1050
|
+
const out = [];
|
|
1051
|
+
for (let t = 0;t < seq; t++) {
|
|
1052
|
+
const row = [];
|
|
1053
|
+
for (let c = 0;c < cls; c++) {
|
|
1054
|
+
row.push(Number(data[t * cls + c]));
|
|
1055
|
+
}
|
|
1056
|
+
out.push(row);
|
|
1057
|
+
}
|
|
1058
|
+
return out;
|
|
1059
|
+
}
|
|
775
1060
|
// src/detectors/index.ts
|
|
776
1061
|
function defaultDetector() {
|
|
777
1062
|
const hasKey = typeof process !== "undefined" && Boolean(process.env?.GHEIM_API_KEY);
|
|
778
|
-
return hasKey ? new RemoteDetector : new
|
|
1063
|
+
return hasKey ? new RemoteDetector : new CalibratedDetector;
|
|
779
1064
|
}
|
|
780
1065
|
// src/plain.ts
|
|
781
1066
|
function resolve(session, detector) {
|
|
@@ -826,7 +1111,7 @@ async function* deanonymizeStream(chunks, session) {
|
|
|
826
1111
|
}
|
|
827
1112
|
|
|
828
1113
|
// src/index.ts
|
|
829
|
-
var VERSION = "0.1.
|
|
1114
|
+
var VERSION = "0.1.5";
|
|
830
1115
|
export {
|
|
831
1116
|
renderSentinel,
|
|
832
1117
|
parseSentinel,
|
|
@@ -847,5 +1132,6 @@ export {
|
|
|
847
1132
|
LocalDetector,
|
|
848
1133
|
LABEL_TO_TAG,
|
|
849
1134
|
DEFAULT_BASE_URL,
|
|
850
|
-
CompositeDetector
|
|
1135
|
+
CompositeDetector,
|
|
1136
|
+
CalibratedDetector
|
|
851
1137
|
};
|
package/dist/esm/openai/index.js
CHANGED
|
@@ -708,10 +708,281 @@ class CompositeDetector {
|
|
|
708
708
|
return out;
|
|
709
709
|
}
|
|
710
710
|
}
|
|
711
|
+
// src/detectors/calibrated.ts
|
|
712
|
+
var DEFAULT_MODEL3 = "joelbarmettler/gheim-ch-560m";
|
|
713
|
+
var DEFAULT_CHUNK_TOKENS2 = 510;
|
|
714
|
+
var DEFAULT_OVERLAP_TOKENS2 = 32;
|
|
715
|
+
function trimSpan2(text, start, end) {
|
|
716
|
+
while (start < end && /\s/.test(text[start] ?? ""))
|
|
717
|
+
start++;
|
|
718
|
+
while (end > start && /\s/.test(text[end - 1] ?? ""))
|
|
719
|
+
end--;
|
|
720
|
+
return [start, end];
|
|
721
|
+
}
|
|
722
|
+
|
|
723
|
+
class CalibratedDetector {
|
|
724
|
+
model;
|
|
725
|
+
oBias;
|
|
726
|
+
device;
|
|
727
|
+
dtype;
|
|
728
|
+
chunkTokens;
|
|
729
|
+
chunkOverlapTokens;
|
|
730
|
+
trimWhitespace;
|
|
731
|
+
actualDevice = null;
|
|
732
|
+
_model = null;
|
|
733
|
+
_tokenizer = null;
|
|
734
|
+
_id2label = {};
|
|
735
|
+
_oId = 0;
|
|
736
|
+
_loadPromise = null;
|
|
737
|
+
_onProgress;
|
|
738
|
+
constructor(opts = {}) {
|
|
739
|
+
this.model = opts.model ?? DEFAULT_MODEL3;
|
|
740
|
+
this.oBias = opts.oBias ?? 0.5;
|
|
741
|
+
this.device = opts.device ?? "auto";
|
|
742
|
+
this.dtype = opts.dtype ?? "fp32";
|
|
743
|
+
this.chunkTokens = opts.chunkTokens ?? DEFAULT_CHUNK_TOKENS2;
|
|
744
|
+
this.chunkOverlapTokens = opts.chunkOverlapTokens ?? DEFAULT_OVERLAP_TOKENS2;
|
|
745
|
+
this.trimWhitespace = opts.trimWhitespace ?? true;
|
|
746
|
+
this._onProgress = opts.onProgress;
|
|
747
|
+
if (this.chunkOverlapTokens >= this.chunkTokens) {
|
|
748
|
+
throw new Error(`chunkOverlapTokens (${this.chunkOverlapTokens}) must be smaller than ` + `chunkTokens (${this.chunkTokens})`);
|
|
749
|
+
}
|
|
750
|
+
}
|
|
751
|
+
async load() {
|
|
752
|
+
if (this._model)
|
|
753
|
+
return;
|
|
754
|
+
if (this._loadPromise)
|
|
755
|
+
return this._loadPromise;
|
|
756
|
+
this._loadPromise = (async () => {
|
|
757
|
+
const mod = await loadTransformersJs2();
|
|
758
|
+
let preferredDevice = this.device;
|
|
759
|
+
if (this.device === "auto") {
|
|
760
|
+
const nav = typeof navigator !== "undefined" ? navigator : null;
|
|
761
|
+
preferredDevice = "wasm";
|
|
762
|
+
if (nav?.gpu?.requestAdapter) {
|
|
763
|
+
try {
|
|
764
|
+
const adapter = await nav.gpu.requestAdapter();
|
|
765
|
+
if (adapter)
|
|
766
|
+
preferredDevice = "webgpu";
|
|
767
|
+
} catch {}
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
const tokenizer = await mod.AutoTokenizer.from_pretrained(this.model);
|
|
771
|
+
const model = await mod.AutoModelForTokenClassification.from_pretrained(this.model, {
|
|
772
|
+
device: preferredDevice,
|
|
773
|
+
dtype: this.dtype,
|
|
774
|
+
progress_callback: this._onProgress ? (raw) => this._emitProgress(raw, preferredDevice) : undefined
|
|
775
|
+
});
|
|
776
|
+
this._tokenizer = tokenizer;
|
|
777
|
+
this._model = model;
|
|
778
|
+
this._id2label = model.config.id2label;
|
|
779
|
+
const label2id = model.config.label2id;
|
|
780
|
+
this._oId = label2id?.["O"] ?? 0;
|
|
781
|
+
this.actualDevice = preferredDevice;
|
|
782
|
+
})();
|
|
783
|
+
return this._loadPromise;
|
|
784
|
+
}
|
|
785
|
+
async detect(text, opts) {
|
|
786
|
+
if (opts?.model && opts.model !== this.model) {
|
|
787
|
+
throw new Error(`CalibratedDetector was constructed with model='${this.model}'; ` + `per-call model override ('${opts.model}') not supported.`);
|
|
788
|
+
}
|
|
789
|
+
if (!text)
|
|
790
|
+
return [];
|
|
791
|
+
await this.load();
|
|
792
|
+
return this._detectChunked(text);
|
|
793
|
+
}
|
|
794
|
+
async _detectChunked(text) {
|
|
795
|
+
if (text.length <= 1500) {
|
|
796
|
+
return await this._detectWindow(text, 0);
|
|
797
|
+
}
|
|
798
|
+
const stride = 1500 - 200;
|
|
799
|
+
const seen = new Set;
|
|
800
|
+
const out = [];
|
|
801
|
+
for (let s = 0;s < text.length; s += stride) {
|
|
802
|
+
const e = Math.min(s + 1500, text.length);
|
|
803
|
+
const sub = text.slice(s, e);
|
|
804
|
+
const subSpans = await this._detectWindow(sub, s);
|
|
805
|
+
for (const sp of subSpans) {
|
|
806
|
+
const key = `${sp.start}|${sp.end}|${sp.label}`;
|
|
807
|
+
if (seen.has(key))
|
|
808
|
+
continue;
|
|
809
|
+
seen.add(key);
|
|
810
|
+
out.push(sp);
|
|
811
|
+
}
|
|
812
|
+
if (e >= text.length)
|
|
813
|
+
break;
|
|
814
|
+
}
|
|
815
|
+
out.sort((a, b) => a.start - b.start || a.end - b.end);
|
|
816
|
+
return out;
|
|
817
|
+
}
|
|
818
|
+
async _detectWindow(text, charOffset) {
|
|
819
|
+
const tokenizer = this._tokenizer;
|
|
820
|
+
const model = this._model;
|
|
821
|
+
const enc = await tokenizer([text], {
|
|
822
|
+
padding: true,
|
|
823
|
+
truncation: true,
|
|
824
|
+
max_length: this.chunkTokens + 2
|
|
825
|
+
});
|
|
826
|
+
const inputs = {};
|
|
827
|
+
if (enc.input_ids)
|
|
828
|
+
inputs["input_ids"] = enc.input_ids;
|
|
829
|
+
if (enc.attention_mask)
|
|
830
|
+
inputs["attention_mask"] = enc.attention_mask;
|
|
831
|
+
const out = await model(inputs);
|
|
832
|
+
const logits = readLogits(out.logits);
|
|
833
|
+
if (this.oBias !== 0) {
|
|
834
|
+
for (let t = 0;t < logits.length; t++) {
|
|
835
|
+
const row = logits[t];
|
|
836
|
+
if (row === undefined)
|
|
837
|
+
continue;
|
|
838
|
+
const cur = row[this._oId];
|
|
839
|
+
if (cur !== undefined) {
|
|
840
|
+
row[this._oId] = cur - this.oBias;
|
|
841
|
+
}
|
|
842
|
+
}
|
|
843
|
+
}
|
|
844
|
+
const inputIds = readInputIds(enc.input_ids);
|
|
845
|
+
const tokenLabels = [];
|
|
846
|
+
for (let t = 0;t < logits.length; t++) {
|
|
847
|
+
const row = logits[t];
|
|
848
|
+
const id = inputIds[t];
|
|
849
|
+
if (!row || id === undefined)
|
|
850
|
+
continue;
|
|
851
|
+
let maxIdx = 0;
|
|
852
|
+
let maxVal = row[0];
|
|
853
|
+
for (let i = 1;i < row.length; i++) {
|
|
854
|
+
if (row[i] > maxVal) {
|
|
855
|
+
maxVal = row[i];
|
|
856
|
+
maxIdx = i;
|
|
857
|
+
}
|
|
858
|
+
}
|
|
859
|
+
tokenLabels.push({ id, label: this._id2label[maxIdx] ?? "O" });
|
|
860
|
+
}
|
|
861
|
+
const spans = [];
|
|
862
|
+
let cursor = 0;
|
|
863
|
+
let groupIds = [];
|
|
864
|
+
let groupCat = null;
|
|
865
|
+
const flush = () => {
|
|
866
|
+
if (groupCat === null || groupIds.length === 0)
|
|
867
|
+
return;
|
|
868
|
+
const surface = normalizeDecoded(tokenizer.decode(groupIds, { skip_special_tokens: true }));
|
|
869
|
+
groupIds = [];
|
|
870
|
+
const cat = groupCat;
|
|
871
|
+
groupCat = null;
|
|
872
|
+
if (!surface)
|
|
873
|
+
return;
|
|
874
|
+
const idx = text.indexOf(surface, cursor);
|
|
875
|
+
if (idx === -1)
|
|
876
|
+
return;
|
|
877
|
+
let s = idx;
|
|
878
|
+
let e = idx + surface.length;
|
|
879
|
+
cursor = e;
|
|
880
|
+
if (this.trimWhitespace)
|
|
881
|
+
[s, e] = trimSpan2(text, s, e);
|
|
882
|
+
if (s >= e)
|
|
883
|
+
return;
|
|
884
|
+
spans.push({
|
|
885
|
+
start: s + charOffset,
|
|
886
|
+
end: e + charOffset,
|
|
887
|
+
label: cat,
|
|
888
|
+
text: text.slice(s, e),
|
|
889
|
+
score: 1
|
|
890
|
+
});
|
|
891
|
+
};
|
|
892
|
+
for (const { id, label } of tokenLabels) {
|
|
893
|
+
const cat = label === "O" ? null : label.split("-", 2)[1] ?? null;
|
|
894
|
+
if (cat === groupCat && cat !== null) {
|
|
895
|
+
groupIds.push(id);
|
|
896
|
+
} else {
|
|
897
|
+
flush();
|
|
898
|
+
if (cat !== null) {
|
|
899
|
+
groupIds.push(id);
|
|
900
|
+
groupCat = cat;
|
|
901
|
+
}
|
|
902
|
+
}
|
|
903
|
+
}
|
|
904
|
+
flush();
|
|
905
|
+
return spans;
|
|
906
|
+
}
|
|
907
|
+
_emitProgress(raw, device) {
|
|
908
|
+
if (!this._onProgress)
|
|
909
|
+
return;
|
|
910
|
+
const event = {
|
|
911
|
+
phase: raw.status ?? "progress",
|
|
912
|
+
device
|
|
913
|
+
};
|
|
914
|
+
if (raw.file)
|
|
915
|
+
event.file = raw.file;
|
|
916
|
+
if (raw.status === "progress") {
|
|
917
|
+
if (raw.loaded != null)
|
|
918
|
+
event.loadedBytes = raw.loaded;
|
|
919
|
+
if (raw.total != null)
|
|
920
|
+
event.totalBytes = raw.total;
|
|
921
|
+
if (raw.progress != null) {
|
|
922
|
+
const norm = raw.progress > 1 ? raw.progress / 100 : raw.progress;
|
|
923
|
+
event.fraction = Math.max(0, Math.min(1, norm));
|
|
924
|
+
}
|
|
925
|
+
}
|
|
926
|
+
this._onProgress(event);
|
|
927
|
+
}
|
|
928
|
+
}
|
|
929
|
+
async function loadTransformersJs2() {
|
|
930
|
+
try {
|
|
931
|
+
return await import("@huggingface/transformers");
|
|
932
|
+
} catch {
|
|
933
|
+
throw new Error("CalibratedDetector requires `@huggingface/transformers`. Install with: bun add @huggingface/transformers");
|
|
934
|
+
}
|
|
935
|
+
}
|
|
936
|
+
function readInputIds(tensor) {
|
|
937
|
+
if (!tensor)
|
|
938
|
+
return [];
|
|
939
|
+
if (typeof tensor.tolist === "function") {
|
|
940
|
+
const list = tensor.tolist();
|
|
941
|
+
if (Array.isArray(list) && Array.isArray(list[0])) {
|
|
942
|
+
return list[0].map((x) => Number(x));
|
|
943
|
+
}
|
|
944
|
+
if (Array.isArray(list))
|
|
945
|
+
return list.map((x) => Number(x));
|
|
946
|
+
}
|
|
947
|
+
const dims = tensor.dims;
|
|
948
|
+
const seq = dims[dims.length - 1] ?? 0;
|
|
949
|
+
const data = tensor.data;
|
|
950
|
+
const out = [];
|
|
951
|
+
for (let i = 0;i < seq; i++)
|
|
952
|
+
out.push(Number(data[i]));
|
|
953
|
+
return out;
|
|
954
|
+
}
|
|
955
|
+
function normalizeDecoded(s) {
|
|
956
|
+
return s.replace(/##/g, "").trim();
|
|
957
|
+
}
|
|
958
|
+
function readLogits(tensor) {
|
|
959
|
+
if (typeof tensor.tolist === "function") {
|
|
960
|
+
const list = tensor.tolist();
|
|
961
|
+
if (Array.isArray(list) && Array.isArray(list[0]) && Array.isArray(list[0][0])) {
|
|
962
|
+
return list[0];
|
|
963
|
+
}
|
|
964
|
+
if (Array.isArray(list) && Array.isArray(list[0])) {
|
|
965
|
+
return list;
|
|
966
|
+
}
|
|
967
|
+
}
|
|
968
|
+
const dims = tensor.dims;
|
|
969
|
+
const seq = dims[dims.length - 2] ?? 0;
|
|
970
|
+
const cls = dims[dims.length - 1] ?? 0;
|
|
971
|
+
const data = tensor.data;
|
|
972
|
+
const out = [];
|
|
973
|
+
for (let t = 0;t < seq; t++) {
|
|
974
|
+
const row = [];
|
|
975
|
+
for (let c = 0;c < cls; c++) {
|
|
976
|
+
row.push(Number(data[t * cls + c]));
|
|
977
|
+
}
|
|
978
|
+
out.push(row);
|
|
979
|
+
}
|
|
980
|
+
return out;
|
|
981
|
+
}
|
|
711
982
|
// src/detectors/index.ts
|
|
712
983
|
function defaultDetector() {
|
|
713
984
|
const hasKey = typeof process !== "undefined" && Boolean(process.env?.GHEIM_API_KEY);
|
|
714
|
-
return hasKey ? new RemoteDetector : new
|
|
985
|
+
return hasKey ? new RemoteDetector : new CalibratedDetector;
|
|
715
986
|
}
|
|
716
987
|
|
|
717
988
|
// src/core/session.ts
|
|
@@ -771,7 +1042,9 @@ class Session {
|
|
|
771
1042
|
cursor = end;
|
|
772
1043
|
}
|
|
773
1044
|
out.push(text.slice(cursor));
|
|
774
|
-
|
|
1045
|
+
let redacted = out.join("");
|
|
1046
|
+
redacted = recoverDuplicateOccurrences(redacted, this.mapping);
|
|
1047
|
+
return { redacted, merged };
|
|
775
1048
|
}
|
|
776
1049
|
restore(text) {
|
|
777
1050
|
if (Object.keys(this.mapping).length === 0)
|
|
@@ -827,6 +1100,18 @@ function mergeAdjacent(text, spans) {
|
|
|
827
1100
|
}
|
|
828
1101
|
return out;
|
|
829
1102
|
}
|
|
1103
|
+
function escapeRegex(s) {
|
|
1104
|
+
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
1105
|
+
}
|
|
1106
|
+
function recoverDuplicateOccurrences(redacted, mapping) {
|
|
1107
|
+
const entries = Object.entries(mapping).filter(([, surface]) => surface && surface.length > 0).sort((a, b) => b[1].length - a[1].length);
|
|
1108
|
+
let result = redacted;
|
|
1109
|
+
for (const [sentinel, surface] of entries) {
|
|
1110
|
+
const pattern = new RegExp(`(?<![\\p{L}\\p{N}_])${escapeRegex(surface)}(?![\\p{L}\\p{N}_])`, "gu");
|
|
1111
|
+
result = result.replace(pattern, sentinel);
|
|
1112
|
+
}
|
|
1113
|
+
return result;
|
|
1114
|
+
}
|
|
830
1115
|
|
|
831
1116
|
// src/openai/_base.ts
|
|
832
1117
|
function resolveCallSession(body, defaultDet) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../../src/core/session.ts"],"names":[],"mappings":"AAAA,kDAAkD;AAClD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AAGjD;;;gEAGgE;AAChE,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;AAkBtD,uCAAuC;AACvC,MAAM,WAAW,cAAc;IAC7B;;;;;;;;;;OAUG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;CAC1C;AAED,qBAAa,OAAO;IAClB,8DAA8D;IAC9D,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAM;IACrC,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,mBAAmB,CAAkC;gBAEjD,IAAI,GAAE,cAAmB;IAQrC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM;IAiBhD,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GAAG,MAAM;IAIvE;;;;;;;OAOG;IACH,kBAAkB,CAChB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GACpC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,UAAU,EAAE,CAAA;KAAE;
|
|
1
|
+
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../../src/core/session.ts"],"names":[],"mappings":"AAAA,kDAAkD;AAClD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AAGjD;;;gEAGgE;AAChE,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;AAkBtD,uCAAuC;AACvC,MAAM,WAAW,cAAc;IAC7B;;;;;;;;;;OAUG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;CAC1C;AAED,qBAAa,OAAO;IAClB,8DAA8D;IAC9D,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAM;IACrC,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,mBAAmB,CAAkC;gBAEjD,IAAI,GAAE,cAAmB;IAQrC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM;IAiBhD,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GAAG,MAAM;IAIvE;;;;;;;OAOG;IACH,kBAAkB,CAChB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GACpC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,UAAU,EAAE,CAAA;KAAE;IA+B7C,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAK7B,MAAM,IAAI;QAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE;IAI7C,MAAM,CAAC,QAAQ,CACb,GAAG,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,EACxC,IAAI,GAAE,cAAmB,GACxB,OAAO;CAqBX;AAID,8EAA8E;AAC9E,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAC3B,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GACpC,UAAU,EAAE,CAqBd"}
|