gheim 0.1.8 → 0.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +96 -6
- package/dist/cjs/index.cjs +184 -31
- package/dist/cjs/openai/index.cjs +169 -17
- package/dist/esm/index.js +184 -39
- package/dist/esm/openai/index.js +169 -25
- package/dist/types/detectors/base.d.ts +13 -0
- package/dist/types/detectors/base.d.ts.map +1 -1
- package/dist/types/detectors/calibrated.d.ts +0 -28
- package/dist/types/detectors/calibrated.d.ts.map +1 -1
- package/dist/types/detectors/composite.d.ts +50 -0
- package/dist/types/detectors/composite.d.ts.map +1 -1
- package/dist/types/detectors/index.d.ts +10 -6
- package/dist/types/detectors/index.d.ts.map +1 -1
- package/dist/types/detectors/local.d.ts +0 -14
- package/dist/types/detectors/local.d.ts.map +1 -1
- package/dist/types/detectors/remote.d.ts +1 -1
- package/dist/types/detectors/remote.d.ts.map +1 -1
- package/dist/types/index.d.ts +2 -2
- package/dist/types/index.d.ts.map +1 -1
- package/package.json +1 -1
package/dist/esm/openai/index.js
CHANGED
|
@@ -1,11 +1,3 @@
|
|
|
1
|
-
var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require : typeof Proxy !== "undefined" ? new Proxy(x, {
|
|
2
|
-
get: (a, b) => (typeof require !== "undefined" ? require : a)[b]
|
|
3
|
-
}) : x)(function(x) {
|
|
4
|
-
if (typeof require !== "undefined")
|
|
5
|
-
return require.apply(this, arguments);
|
|
6
|
-
throw Error('Dynamic require of "' + x + '" is not supported');
|
|
7
|
-
});
|
|
8
|
-
|
|
9
1
|
// src/core/sentinels.ts
|
|
10
2
|
var LABEL_TO_TAG = {
|
|
11
3
|
private_person: "PERSON",
|
|
@@ -164,10 +156,40 @@ function trimSpan(text, start, end) {
|
|
|
164
156
|
return null;
|
|
165
157
|
return { start, end };
|
|
166
158
|
}
|
|
159
|
+
var WORD_CHAR = /[\p{L}\p{N}_]/u;
|
|
160
|
+
var isWordChar = (ch) => ch !== undefined && WORD_CHAR.test(ch);
|
|
161
|
+
function snapToWords(text, spans) {
|
|
162
|
+
const snapped = spans.map((sp) => {
|
|
163
|
+
let { start, end } = sp;
|
|
164
|
+
while (start > 0 && isWordChar(text[start - 1]) && isWordChar(text[start]))
|
|
165
|
+
start -= 1;
|
|
166
|
+
while (end < text.length && isWordChar(text[end - 1]) && isWordChar(text[end]))
|
|
167
|
+
end += 1;
|
|
168
|
+
return start === sp.start && end === sp.end ? sp : { ...sp, start, end, text: text.slice(start, end) };
|
|
169
|
+
});
|
|
170
|
+
snapped.sort((a, b) => a.start - b.start || a.end - b.end);
|
|
171
|
+
const merged = [];
|
|
172
|
+
const lastByLabel = new Map;
|
|
173
|
+
for (const sp of snapped) {
|
|
174
|
+
const i = lastByLabel.get(sp.label);
|
|
175
|
+
const prev = i === undefined ? undefined : merged[i];
|
|
176
|
+
if (i !== undefined && prev !== undefined && sp.start <= prev.end) {
|
|
177
|
+
const end = Math.max(prev.end, sp.end);
|
|
178
|
+
const wPrev = prev.end - prev.start;
|
|
179
|
+
const wSp = sp.end - sp.start;
|
|
180
|
+
const score = ((prev.score ?? 1) * wPrev + (sp.score ?? 1) * wSp) / (wPrev + wSp);
|
|
181
|
+
merged[i] = { ...prev, end, text: text.slice(prev.start, end), score };
|
|
182
|
+
} else {
|
|
183
|
+
lastByLabel.set(sp.label, merged.length);
|
|
184
|
+
merged.push(sp);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
return merged.sort((a, b) => a.start - b.start || a.end - b.end);
|
|
188
|
+
}
|
|
167
189
|
|
|
168
190
|
// src/detectors/remote.ts
|
|
169
191
|
var DEFAULT_BASE_URL = "https://api.gheim.ch";
|
|
170
|
-
var DEFAULT_MODEL = "
|
|
192
|
+
var DEFAULT_MODEL = "joelbarmettler/gheim-ch-560m";
|
|
171
193
|
|
|
172
194
|
class RemoteDetector {
|
|
173
195
|
baseUrl;
|
|
@@ -412,10 +434,7 @@ class LocalDetector {
|
|
|
412
434
|
});
|
|
413
435
|
}
|
|
414
436
|
}
|
|
415
|
-
|
|
416
|
-
spans.sort((a, b) => a.start - b.start || a.end - b.end);
|
|
417
|
-
}
|
|
418
|
-
return out;
|
|
437
|
+
return out.map((spans, i) => snapToWords(texts[i] ?? "", spans));
|
|
419
438
|
}
|
|
420
439
|
async chunkText(text, tokenizer) {
|
|
421
440
|
if (tokenizer) {
|
|
@@ -692,21 +711,136 @@ function maskSpans(text, spans) {
|
|
|
692
711
|
return chars.join("");
|
|
693
712
|
}
|
|
694
713
|
|
|
714
|
+
class CustomPattern {
|
|
715
|
+
label;
|
|
716
|
+
group;
|
|
717
|
+
name;
|
|
718
|
+
regex;
|
|
719
|
+
validator;
|
|
720
|
+
constructor(opts) {
|
|
721
|
+
if (!/^[A-Za-z][A-Za-z0-9_]*$/.test(opts.label)) {
|
|
722
|
+
throw new Error(`CustomPattern label ${JSON.stringify(opts.label)} must start with a letter and contain ` + "only letters, digits and underscores (it becomes a <TAG_1> placeholder).");
|
|
723
|
+
}
|
|
724
|
+
this.label = opts.label;
|
|
725
|
+
this.name = opts.name ?? opts.label;
|
|
726
|
+
this.group = opts.group ?? 0;
|
|
727
|
+
this.validator = opts.validator;
|
|
728
|
+
const source = typeof opts.regex === "string" ? opts.regex : opts.regex.source;
|
|
729
|
+
const base = typeof opts.regex === "string" ? opts.flags ?? "i" : opts.regex.flags;
|
|
730
|
+
const flags = [...new Set([...base, "g", "d"])].join("");
|
|
731
|
+
this.regex = new RegExp(source, flags);
|
|
732
|
+
const groups = (new RegExp(`${source}|`, flags.replace(/[gd]/g, "")).exec("") ?? []).length - 1;
|
|
733
|
+
if (!Number.isInteger(this.group) || this.group < 0 || this.group > groups) {
|
|
734
|
+
throw new Error(`CustomPattern ${JSON.stringify(this.name)}: group ${this.group} does not exist ` + `(the pattern has ${groups}).`);
|
|
735
|
+
}
|
|
736
|
+
}
|
|
737
|
+
find(text) {
|
|
738
|
+
const out = [];
|
|
739
|
+
this.regex.lastIndex = 0;
|
|
740
|
+
let m;
|
|
741
|
+
while ((m = this.regex.exec(text)) !== null) {
|
|
742
|
+
if (m[0] === "") {
|
|
743
|
+
this.regex.lastIndex++;
|
|
744
|
+
continue;
|
|
745
|
+
}
|
|
746
|
+
const range = m.indices[this.group];
|
|
747
|
+
if (!range || range[0] === range[1])
|
|
748
|
+
continue;
|
|
749
|
+
const [start, end] = range;
|
|
750
|
+
const value = text.slice(start, end);
|
|
751
|
+
if (this.validator && !this.validator(value))
|
|
752
|
+
continue;
|
|
753
|
+
out.push({ start, end, label: this.label, text: value, score: 1 });
|
|
754
|
+
}
|
|
755
|
+
return out;
|
|
756
|
+
}
|
|
757
|
+
}
|
|
758
|
+
var NAME_PART = /[A-ZÀ-Ý][\p{L}\p{N}_'’-]{3,}/gu;
|
|
759
|
+
var HONORIFICS = new Set([
|
|
760
|
+
"frau",
|
|
761
|
+
"herr",
|
|
762
|
+
"herrn",
|
|
763
|
+
"madame",
|
|
764
|
+
"monsieur",
|
|
765
|
+
"signora",
|
|
766
|
+
"signor",
|
|
767
|
+
"signore",
|
|
768
|
+
"dott",
|
|
769
|
+
"dottor",
|
|
770
|
+
"dottoressa",
|
|
771
|
+
"doktor",
|
|
772
|
+
"docteur",
|
|
773
|
+
"prof",
|
|
774
|
+
"professor",
|
|
775
|
+
"mister",
|
|
776
|
+
"miss",
|
|
777
|
+
"mrs"
|
|
778
|
+
]);
|
|
779
|
+
function escapeRegExp(s) {
|
|
780
|
+
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
781
|
+
}
|
|
782
|
+
function propagateNames(text, spans, surnames) {
|
|
783
|
+
const names = new Set;
|
|
784
|
+
for (const sp of spans) {
|
|
785
|
+
if (sp.label !== "private_person")
|
|
786
|
+
continue;
|
|
787
|
+
const value = text.slice(sp.start, sp.end).trim();
|
|
788
|
+
if (value.length >= 4)
|
|
789
|
+
names.add(value);
|
|
790
|
+
if (surnames) {
|
|
791
|
+
for (const part of value.match(NAME_PART) ?? []) {
|
|
792
|
+
if (!HONORIFICS.has(part.toLowerCase()))
|
|
793
|
+
names.add(part);
|
|
794
|
+
}
|
|
795
|
+
}
|
|
796
|
+
}
|
|
797
|
+
let result = [...spans];
|
|
798
|
+
for (const name of [...names].sort((a, b) => b.length - a.length)) {
|
|
799
|
+
const re = new RegExp(`(?<![\\p{L}\\p{N}_])${escapeRegExp(name)}(?![\\p{L}\\p{N}_])`, "gu");
|
|
800
|
+
let m;
|
|
801
|
+
while ((m = re.exec(text)) !== null) {
|
|
802
|
+
const start = m.index;
|
|
803
|
+
const end = start + m[0].length;
|
|
804
|
+
const hits = result.filter((sp) => start < sp.end && sp.start < end);
|
|
805
|
+
if (hits.some((sp) => sp.start <= start && end <= sp.end))
|
|
806
|
+
continue;
|
|
807
|
+
if (hits.some((sp) => sp.start < start || end < sp.end))
|
|
808
|
+
continue;
|
|
809
|
+
result = result.filter((sp) => !hits.includes(sp));
|
|
810
|
+
result.push({ start, end, label: "private_person", text: m[0], score: 1 });
|
|
811
|
+
}
|
|
812
|
+
}
|
|
813
|
+
return result;
|
|
814
|
+
}
|
|
815
|
+
|
|
695
816
|
class CompositeDetector {
|
|
696
817
|
model;
|
|
818
|
+
patterns;
|
|
819
|
+
propagateNames;
|
|
820
|
+
propagateSurnames;
|
|
697
821
|
constructor(opts) {
|
|
698
822
|
if (!opts || !opts.model) {
|
|
699
823
|
throw new Error("CompositeDetector requires { model: Detector } — pass any object with " + "a .detect(text) method (e.g. new LocalDetector({...})).");
|
|
700
824
|
}
|
|
701
825
|
this.model = opts.model;
|
|
826
|
+
this.patterns = opts.patterns ?? [];
|
|
827
|
+
this.propagateNames = opts.propagateNames ?? true;
|
|
828
|
+
this.propagateSurnames = opts.propagateSurnames ?? false;
|
|
702
829
|
}
|
|
703
830
|
async detect(text, opts) {
|
|
704
831
|
if (!text)
|
|
705
832
|
return [];
|
|
706
833
|
const regexSpans = findRegexSpans(text);
|
|
834
|
+
const extra = this.patterns.flatMap((p) => p.find(text));
|
|
835
|
+
extra.sort((a, b) => a.start !== b.start ? a.start - b.start : b.end - b.start - (a.end - a.start));
|
|
836
|
+
for (const sp of extra) {
|
|
837
|
+
if (!regexSpans.some((o) => sp.start < o.end && o.start < sp.end))
|
|
838
|
+
regexSpans.push(sp);
|
|
839
|
+
}
|
|
840
|
+
regexSpans.sort((a, b) => a.start - b.start);
|
|
707
841
|
const masked = maskSpans(text, regexSpans);
|
|
708
842
|
const modelSpans = await this.model.detect(masked, opts);
|
|
709
|
-
|
|
843
|
+
let out = [...regexSpans];
|
|
710
844
|
const regexRanges = regexSpans.map((s) => [s.start, s.end]);
|
|
711
845
|
for (const sp of modelSpans) {
|
|
712
846
|
const overlaps = regexRanges.some(([rs, re]) => sp.start < re && rs < sp.end);
|
|
@@ -714,6 +848,8 @@ class CompositeDetector {
|
|
|
714
848
|
continue;
|
|
715
849
|
out.push(sp);
|
|
716
850
|
}
|
|
851
|
+
if (this.propagateNames)
|
|
852
|
+
out = propagateNames(text, out, this.propagateSurnames);
|
|
717
853
|
out.sort((a, b) => a.start !== b.start ? a.start - b.start : b.end - b.start - (a.end - a.start));
|
|
718
854
|
return out;
|
|
719
855
|
}
|
|
@@ -803,7 +939,7 @@ class CalibratedDetector {
|
|
|
803
939
|
if (!text)
|
|
804
940
|
return [];
|
|
805
941
|
await this.load();
|
|
806
|
-
return this._detectChunked(text);
|
|
942
|
+
return snapToWords(text, await this._detectChunked(text));
|
|
807
943
|
}
|
|
808
944
|
async _detectChunked(text) {
|
|
809
945
|
if (text.length <= 1500) {
|
|
@@ -870,15 +1006,21 @@ class CalibratedDetector {
|
|
|
870
1006
|
maxIdx = i;
|
|
871
1007
|
}
|
|
872
1008
|
}
|
|
873
|
-
|
|
1009
|
+
let denom = 0;
|
|
1010
|
+
for (let i = 0;i < row.length; i++)
|
|
1011
|
+
denom += Math.exp(row[i] - maxVal);
|
|
1012
|
+
tokenLabels.push({ id, label: this._id2label[maxIdx] ?? "O", prob: 1 / denom });
|
|
874
1013
|
}
|
|
875
1014
|
const spans = [];
|
|
876
1015
|
let cursor = 0;
|
|
877
1016
|
let groupIds = [];
|
|
1017
|
+
let groupProbs = [];
|
|
878
1018
|
let groupCat = null;
|
|
879
1019
|
const flush = () => {
|
|
880
1020
|
if (groupCat === null || groupIds.length === 0)
|
|
881
1021
|
return;
|
|
1022
|
+
const score = groupProbs.reduce((a, b) => a + b, 0) / groupProbs.length;
|
|
1023
|
+
groupProbs = [];
|
|
882
1024
|
const surface = normalizeDecoded(tokenizer.decode(groupIds, { skip_special_tokens: true }));
|
|
883
1025
|
groupIds = [];
|
|
884
1026
|
const cat = groupCat;
|
|
@@ -900,17 +1042,19 @@ class CalibratedDetector {
|
|
|
900
1042
|
end: e + charOffset,
|
|
901
1043
|
label: cat,
|
|
902
1044
|
text: text.slice(s, e),
|
|
903
|
-
score
|
|
1045
|
+
score
|
|
904
1046
|
});
|
|
905
1047
|
};
|
|
906
|
-
for (const { id, label } of tokenLabels) {
|
|
1048
|
+
for (const { id, label, prob } of tokenLabels) {
|
|
907
1049
|
const cat = label === "O" ? null : label.split("-", 2)[1] ?? null;
|
|
908
1050
|
if (cat === groupCat && cat !== null) {
|
|
909
1051
|
groupIds.push(id);
|
|
1052
|
+
groupProbs.push(prob);
|
|
910
1053
|
} else {
|
|
911
1054
|
flush();
|
|
912
1055
|
if (cat !== null) {
|
|
913
1056
|
groupIds.push(id);
|
|
1057
|
+
groupProbs.push(prob);
|
|
914
1058
|
groupCat = cat;
|
|
915
1059
|
}
|
|
916
1060
|
}
|
|
@@ -996,7 +1140,7 @@ function readLogits(tensor) {
|
|
|
996
1140
|
// src/detectors/index.ts
|
|
997
1141
|
function defaultDetector() {
|
|
998
1142
|
const hasKey = typeof process !== "undefined" && Boolean(process.env?.GHEIM_API_KEY);
|
|
999
|
-
return hasKey ? new RemoteDetector : new CalibratedDetector;
|
|
1143
|
+
return hasKey ? new RemoteDetector : new CompositeDetector({ model: new CalibratedDetector });
|
|
1000
1144
|
}
|
|
1001
1145
|
|
|
1002
1146
|
// src/core/session.ts
|
|
@@ -1775,11 +1919,11 @@ async function* deanonymizeOpenAIStream(stream, session) {
|
|
|
1775
1919
|
}
|
|
1776
1920
|
}
|
|
1777
1921
|
export {
|
|
1778
|
-
|
|
1779
|
-
deanonymizeOpenAIResponse,
|
|
1780
|
-
anonymizeOpenAIMessages,
|
|
1781
|
-
STRICT_REASONS,
|
|
1782
|
-
OpenAI,
|
|
1922
|
+
AsyncOpenAI,
|
|
1783
1923
|
GheimStrictError,
|
|
1784
|
-
|
|
1924
|
+
OpenAI,
|
|
1925
|
+
STRICT_REASONS,
|
|
1926
|
+
anonymizeOpenAIMessages,
|
|
1927
|
+
deanonymizeOpenAIResponse,
|
|
1928
|
+
deanonymizeOpenAIStream
|
|
1785
1929
|
};
|
|
@@ -25,4 +25,17 @@ export interface Detector {
|
|
|
25
25
|
model?: string;
|
|
26
26
|
}): Promise<Span[]>;
|
|
27
27
|
}
|
|
28
|
+
/**
|
|
29
|
+
* Widen spans that start or end inside a word to the whole word, then merge
|
|
30
|
+
* same-label spans that overlap or touch.
|
|
31
|
+
*
|
|
32
|
+
* A token classifier sometimes tags only part of a word (`gano` of `Lugano`,
|
|
33
|
+
* `Zbind` + `en` of `Zbinden`), and a window boundary can cut a word in half.
|
|
34
|
+
* Masking a fragment leaves the rest of the word readable and splits one entity
|
|
35
|
+
* into two placeholders. A span is widened only when the characters on both
|
|
36
|
+
* sides of its edge are word characters, so edges at spaces or punctuation
|
|
37
|
+
* (`Gay` in `Gay-Crosier`) are left alone. The merged span's score is the
|
|
38
|
+
* length-weighted mean of its parts.
|
|
39
|
+
*/
|
|
40
|
+
export declare function snapToWords(text: string, spans: Span[]): Span[];
|
|
28
41
|
//# sourceMappingURL=base.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"base.d.ts","sourceRoot":"","sources":["../../../src/detectors/base.ts"],"names":[],"mappings":"AAAA,gEAAgE;AAEhE;;;;;GAKG;AACH,wBAAgB,QAAQ,CACtB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,GACV;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAKvC;AAED,MAAM,WAAW,IAAI;IACnB,6CAA6C;IAC7C,KAAK,EAAE,MAAM,CAAC;IACd,yDAAyD;IACzD,KAAK,EAAE,MAAM,CAAC;IACd,iBAAiB;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,sDAAsD;IACtD,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,QAAQ;IACvB,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;CAClE"}
|
|
1
|
+
{"version":3,"file":"base.d.ts","sourceRoot":"","sources":["../../../src/detectors/base.ts"],"names":[],"mappings":"AAAA,gEAAgE;AAEhE;;;;;GAKG;AACH,wBAAgB,QAAQ,CACtB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,GACV;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAKvC;AAED,MAAM,WAAW,IAAI;IACnB,6CAA6C;IAC7C,KAAK,EAAE,MAAM,CAAC;IACd,yDAAyD;IACzD,KAAK,EAAE,MAAM,CAAC;IACd,iBAAiB;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,sDAAsD;IACtD,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,QAAQ;IACvB,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;CAClE;AAKD;;;;;;;;;;;GAWG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,IAAI,EAAE,CA0B/D"}
|
|
@@ -1,31 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Calibrated detector — subtracts a constant `oBias` from the O-class
|
|
3
|
-
* logit before argmax. Originally introduced to recover the v1 model's
|
|
4
|
-
* "single-category mode" pathology (Müller + IBAN co-occurrence flipped
|
|
5
|
-
* the person prediction to O); the v3 / gheim-ch-560m training pipeline
|
|
6
|
-
* largely resolved that pathology, but the calibration still buys a
|
|
7
|
-
* small precision/recall improvement and remains the default.
|
|
8
|
-
*
|
|
9
|
-
* Sweep on gheim-ch-560m (see eval/calibration_sweep.json):
|
|
10
|
-
* `oBias=0.5` is the Pareto-clean default — probe perfect-rate is flat
|
|
11
|
-
* at 91.5% across bias 0.0 and 0.5, and test char-F1 lifts by 0.27pp.
|
|
12
|
-
* Higher biases (1.0–1.5) trade test F1 for marginal probe gains.
|
|
13
|
-
*
|
|
14
|
-
* A separate sweep on the int8 ONNX path via `transformers.js`
|
|
15
|
-
* (eval/calibration_sweep_q8.json) showed a much larger calibration
|
|
16
|
-
* effect (+1.4pp probe, +0.43pp char-F1 at 0.5). That gap is not a
|
|
17
|
-
* property of the int8 quantisation — Python `onnxruntime` on the same
|
|
18
|
-
* int8 file gives 91.0% probe perfect-rate, essentially fp32-equivalent
|
|
19
|
-
* — but of a runtime divergence inside `onnxruntime-web` / WASM that
|
|
20
|
-
* the calibration partially papers over. The reliable fix is to use the
|
|
21
|
-
* `model_fp16.onnx` variant when WebGPU is available (which is what
|
|
22
|
-
* LocalDetector's default `dtype: "auto"` does); int8 should be treated
|
|
23
|
-
* as a low-RAM fallback. See eval/q8_quality_report.md for the
|
|
24
|
-
* diagnostic.
|
|
25
|
-
*
|
|
26
|
-
* `defaultDetector()` returns this with oBias=0.5, so users get the
|
|
27
|
-
* better behavior automatically.
|
|
28
|
-
*/
|
|
29
1
|
import type { Detector, Span } from "./base.ts";
|
|
30
2
|
import type { LocalDetectorLoadEvent } from "./local.ts";
|
|
31
3
|
/** Constructor options for {@link CalibratedDetector}. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"calibrated.d.ts","sourceRoot":"","sources":["../../../src/detectors/calibrated.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"calibrated.d.ts","sourceRoot":"","sources":["../../../src/detectors/calibrated.ts"],"names":[],"mappings":"AA6BA,OAAO,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAMzD,0DAA0D;AAC1D,MAAM,WAAW,yBAAyB;IACxC,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;;;;;OASG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,iEAAiE;IACjE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yDAAyD;IACzD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wDAAwD;IACxD,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,8EAA8E;IAC9E,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,4EAA4E;IAC5E,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,sBAAsB,KAAK,IAAI,CAAC;CACtD;AA4CD,uDAAuD;AACvD,qBAAa,kBAAmB,YAAW,QAAQ;IACjD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,oEAAoE;IACpE,YAAY,EAAE,MAAM,GAAG,IAAI,CAAQ;IACnC;sEACkE;IAClE,WAAW,EAAE,MAAM,GAAG,IAAI,CAAQ;IAClC,OAAO,CAAC,MAAM,CAA8B;IAC5C,OAAO,CAAC,UAAU,CAAkC;IACpD,OAAO,CAAC,SAAS,CAA8B;IAC/C,OAAO,CAAC,IAAI,CAAK;IACjB,OAAO,CAAC,YAAY,CAA8B;IAClD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAA0C;gBAE3D,IAAI,GAAE,yBAA8B;IAiB1C,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAoDrB,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;YAYxD,cAAc;YA0Bd,aAAa;IAuG3B,OAAO,CAAC,aAAa;CAiBtB"}
|
|
@@ -29,11 +29,58 @@
|
|
|
29
29
|
* detector.
|
|
30
30
|
*/
|
|
31
31
|
import type { Detector, Span } from "./base.ts";
|
|
32
|
+
/** Options for {@link CustomPattern}. */
|
|
33
|
+
export interface CustomPatternOptions {
|
|
34
|
+
/** Label of the spans this rule produces. It becomes the placeholder tag
|
|
35
|
+
* (`insurance_card` -> `<INSURANCE_CARD_1>`), so it must start with a
|
|
36
|
+
* letter and contain only letters, digits and underscores. Reuse a
|
|
37
|
+
* built-in label such as `account_number` to share its tag. */
|
|
38
|
+
label: string;
|
|
39
|
+
/** Pattern to search for. A string is compiled with `flags`; a RegExp
|
|
40
|
+
* keeps its own flags. */
|
|
41
|
+
regex: string | RegExp;
|
|
42
|
+
/** Capture group that is the PII itself (0 = whole match). Lets a label
|
|
43
|
+
* word anchor the match without being masked (`Pass-Nr. (X1234567)`). */
|
|
44
|
+
group?: number;
|
|
45
|
+
/** Flags for a string pattern. Default `"i"`. `g` and `d` are always added. */
|
|
46
|
+
flags?: string;
|
|
47
|
+
/** Receives the PII text; return false to reject the match (a checksum,
|
|
48
|
+
* or a case check, since JS has no scoped `(?i:...)` flag). */
|
|
49
|
+
validator?: (value: string) => boolean;
|
|
50
|
+
/** Optional name for error messages. */
|
|
51
|
+
name?: string;
|
|
52
|
+
}
|
|
53
|
+
/** A user-supplied regex rule for formats that are specific to one
|
|
54
|
+
* deployment (insurance card numbers, hospital record numbers, internal
|
|
55
|
+
* case IDs, ...). Nothing in the library is domain-specific: the caller
|
|
56
|
+
* passes the rules it needs to {@link CompositeDetector}.
|
|
57
|
+
*
|
|
58
|
+
* Matches run in the regex front-end next to the built-in rules: the text
|
|
59
|
+
* is masked before the model sees it, and a match wins over any model span
|
|
60
|
+
* it overlaps. */
|
|
61
|
+
export declare class CustomPattern {
|
|
62
|
+
readonly label: string;
|
|
63
|
+
readonly group: number;
|
|
64
|
+
readonly name: string;
|
|
65
|
+
private readonly regex;
|
|
66
|
+
private readonly validator;
|
|
67
|
+
constructor(opts: CustomPatternOptions);
|
|
68
|
+
/** Every non-empty, validated match of the PII group in `text`. */
|
|
69
|
+
find(text: string): Span[];
|
|
70
|
+
}
|
|
32
71
|
/** Options for {@link CompositeDetector}. */
|
|
33
72
|
export interface CompositeDetectorOptions {
|
|
34
73
|
/** Inner detector — usually a {@link LocalDetector}. Anything with
|
|
35
74
|
* a `detect(text) → Promise<Span[]>` method works. */
|
|
36
75
|
model: Detector;
|
|
76
|
+
/** Extra regex rules for formats specific to your deployment. */
|
|
77
|
+
patterns?: CustomPattern[];
|
|
78
|
+
/** Mask every other occurrence of a string the model found as a person,
|
|
79
|
+
* so a name caught once is not leaked by a later occurrence. Default true. */
|
|
80
|
+
propagateNames?: boolean;
|
|
81
|
+
/** With `propagateNames`: also propagate the individual capitalised parts
|
|
82
|
+
* of each detected name. Higher recall, more false positives. Default false. */
|
|
83
|
+
propagateSurnames?: boolean;
|
|
37
84
|
}
|
|
38
85
|
/** Regex front-end (with checksum validation) + model on the remainder.
|
|
39
86
|
*
|
|
@@ -48,6 +95,9 @@ export interface CompositeDetectorOptions {
|
|
|
48
95
|
*/
|
|
49
96
|
export declare class CompositeDetector implements Detector {
|
|
50
97
|
readonly model: Detector;
|
|
98
|
+
readonly patterns: readonly CustomPattern[];
|
|
99
|
+
readonly propagateNames: boolean;
|
|
100
|
+
readonly propagateSurnames: boolean;
|
|
51
101
|
constructor(opts: CompositeDetectorOptions);
|
|
52
102
|
/** Two-pass detection: regex first (with checksum), then model on
|
|
53
103
|
* the masked remainder. Returns a single overlap-resolved span list. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"composite.d.ts","sourceRoot":"","sources":["../../../src/detectors/composite.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,OAAO,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAiNhD,6CAA6C;AAC7C,MAAM,WAAW,wBAAwB;IACvC;2DACuD;IACvD,KAAK,EAAE,QAAQ,CAAC;
|
|
1
|
+
{"version":3,"file":"composite.d.ts","sourceRoot":"","sources":["../../../src/detectors/composite.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,OAAO,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAiNhD,yCAAyC;AACzC,MAAM,WAAW,oBAAoB;IACnC;;;oEAGgE;IAChE,KAAK,EAAE,MAAM,CAAC;IACd;+BAC2B;IAC3B,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC;IACvB;8EAC0E;IAC1E,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+EAA+E;IAC/E,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;oEACgE;IAChE,SAAS,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC;IACvC,wCAAwC;IACxC,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;mBAOmB;AACnB,qBAAa,aAAa;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAA2C;gBAEzD,IAAI,EAAE,oBAAoB;IA0BtC,mEAAmE;IACnE,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,EAAE;CAoB3B;AAwDD,6CAA6C;AAC7C,MAAM,WAAW,wBAAwB;IACvC;2DACuD;IACvD,KAAK,EAAE,QAAQ,CAAC;IAChB,iEAAiE;IACjE,QAAQ,CAAC,EAAE,aAAa,EAAE,CAAC;IAC3B;mFAC+E;IAC/E,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB;qFACiF;IACjF,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAC7B;AAED;;;;;;;;;;GAUG;AACH,qBAAa,iBAAkB,YAAW,QAAQ;IAChD,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC;IACzB,QAAQ,CAAC,QAAQ,EAAE,SAAS,aAAa,EAAE,CAAC;IAC5C,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,QAAQ,CAAC,iBAAiB,EAAE,OAAO,CAAC;gBAExB,IAAI,EAAE,wBAAwB;IAa1C;6EACyE;IACnE,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;CA8BvE"}
|
|
@@ -3,16 +3,20 @@ export { DEFAULT_BASE_URL, DEFAULT_MODEL as REMOTE_DEFAULT_MODEL, RemoteDetector
|
|
|
3
3
|
export type { RemoteDetectorOptions } from "./remote.ts";
|
|
4
4
|
export { LocalDetector } from "./local.ts";
|
|
5
5
|
export type { LocalDetectorOptions, LocalDetectorLoadEvent } from "./local.ts";
|
|
6
|
-
export { CompositeDetector } from "./composite.ts";
|
|
7
|
-
export type { CompositeDetectorOptions } from "./composite.ts";
|
|
6
|
+
export { CompositeDetector, CustomPattern } from "./composite.ts";
|
|
7
|
+
export type { CompositeDetectorOptions, CustomPatternOptions } from "./composite.ts";
|
|
8
8
|
export { CalibratedDetector } from "./calibrated.ts";
|
|
9
9
|
export type { CalibratedDetectorOptions } from "./calibrated.ts";
|
|
10
10
|
import type { Detector } from "./base.ts";
|
|
11
11
|
/** Pick a reasonable default: remote when `GHEIM_API_KEY` is set in env,
|
|
12
|
-
* else
|
|
12
|
+
* else the composite detector (checksum regex front-end + calibrated local
|
|
13
|
+
* model).
|
|
13
14
|
*
|
|
14
|
-
* The calibrated
|
|
15
|
-
* +0.27pp test char-F1 with no probe cost (91.5% perfect at
|
|
16
|
-
* and 0.5). See eval/calibration_sweep.json.
|
|
15
|
+
* The calibrated model alone (`oBias=0.5`) is Pareto-clean on
|
|
16
|
+
* gheim-ch-560m: +0.27pp test char-F1 with no probe cost (91.5% perfect at
|
|
17
|
+
* both 0.0 and 0.5). See eval/calibration_sweep.json. The composite adds
|
|
18
|
+
* what a token classifier cannot verify: checksummed IBAN / AHV / VAT /
|
|
19
|
+
* card numbers, phone numbers, and repeated occurrences of a name it found
|
|
20
|
+
* once. Use `new CalibratedDetector()` for the model alone. */
|
|
17
21
|
export declare function defaultDetector(): Detector;
|
|
18
22
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/detectors/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,EACL,gBAAgB,EAChB,aAAa,IAAI,oBAAoB,EACrC,cAAc,GACf,MAAM,aAAa,CAAC;AACrB,YAAY,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,YAAY,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAC/E,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/detectors/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,EACL,gBAAgB,EAChB,aAAa,IAAI,oBAAoB,EACrC,cAAc,GACf,MAAM,aAAa,CAAC;AACrB,YAAY,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,YAAY,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAC/E,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAClE,YAAY,EAAE,wBAAwB,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AACrF,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACrD,YAAY,EAAE,yBAAyB,EAAE,MAAM,iBAAiB,CAAC;AAEjE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAK1C;;;;;;;;;gEASgE;AAChE,wBAAgB,eAAe,IAAI,QAAQ,CAM1C"}
|
|
@@ -1,17 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Local detector — runs the privacy-filter model in-process via @huggingface/transformers.
|
|
3
|
-
*
|
|
4
|
-
* Works in Node and the browser (with `device: 'webgpu'` for the latter).
|
|
5
|
-
* `@huggingface/transformers` is an *optional peer dependency* — only needed
|
|
6
|
-
* when this detector is actually instantiated.
|
|
7
|
-
*
|
|
8
|
-
* Long inputs (more tokens than the model's positional limit, typically
|
|
9
|
-
* 512 for BERT/XLM-R/roberta encoders) are split into overlapping sliding
|
|
10
|
-
* windows; spans from each window are reprojected to the original character
|
|
11
|
-
* offsets and deduplicated. Without this, anything past ~512 tokens would
|
|
12
|
-
* be silently truncated by the underlying pipeline. Multiple inputs and
|
|
13
|
-
* within-input chunks are batched together for one forward pass.
|
|
14
|
-
*/
|
|
15
1
|
import type { Detector, Span } from "./base.ts";
|
|
16
2
|
export declare const DEFAULT_MODEL = "joelbarmettler/gheim-ch-560m";
|
|
17
3
|
/** transformers.js token-classification options used by this detector. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"local.d.ts","sourceRoot":"","sources":["../../../src/detectors/local.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"local.d.ts","sourceRoot":"","sources":["../../../src/detectors/local.ts"],"names":[],"mappings":"AAeA,OAAO,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEhD,eAAO,MAAM,aAAa,iCAAiC,CAAC;AAQ5D,0EAA0E;AAC1E,MAAM,WAAW,oBAAoB;IACnC,mEAAmE;IACnE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;;;;;OASG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;;;;OAQG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wDAAwD;IACxD,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,+EAA+E;IAC/E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,6DAA6D;IAC7D,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;;;OAKG;IACH,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,sBAAsB,KAAK,IAAI,CAAC;CACtD;AAED,uFAAuF;AACvF,MAAM,WAAW,sBAAsB;IACrC,yBAAyB;IACzB,KAAK,EAAE,UAAU,GAAG,UAAU,GAAG,UAAU,GAAG,OAAO,GAAG,MAAM,CAAC;IAC/D,6CAA6C;IAC7C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,6DAA6D;IAC7D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,4DAA4D;IAC5D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;wEACoE;IACpE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;+EAE2E;IAC3E,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,UAAU,aAAa;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,KAAK,gBAAgB,GAAG,CACtB,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,EACvB,IAAI,CAAC,EAAE;IAAE,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAAE,KAC1D,OAAO,CAAC,aAAa,EAAE,GAAG,aAAa,EAAE,EAAE,CAAC,CAAC;AAwBlD,qBAAa,aAAc,YAAW,QAAQ;IAC5C,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,kFAAkF;IAClF,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B;;0CAEsC;IACtC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAQ;IACnC;;;gDAG4C;IAC5C,WAAW,EAAE,MAAM,GAAG,IAAI,CAAQ;IAClC,OAAO,CAAC,SAAS,CAAiC;IAClD,OAAO,CAAC,YAAY,CAA0C;IAC9D,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAA0C;gBAE3D,IAAI,GAAE,oBAAyB;IAuB3C;;;;gFAI4E;IAC5E,OAAO,CAAC,aAAa;IAKrB;;;;;OAKG;IACG,IAAI,IAAI,OAAO,CAAC,gBAAgB,CAAC;IAoCvC;;;;;;;;OAQG;YACW,mBAAmB;IAsBjC,mFAAmF;IACnF,OAAO,CAAC,aAAa;IAqBf,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAYtE;;;;;OAKG;IACG,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;YAsFlE,SAAS;CAkBxB"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/** Remote detector — calls a gheim-server (or compatible) over HTTP. */
|
|
2
2
|
import { type Detector, type Span } from "./base.ts";
|
|
3
3
|
export declare const DEFAULT_BASE_URL = "https://api.gheim.ch";
|
|
4
|
-
export declare const DEFAULT_MODEL = "
|
|
4
|
+
export declare const DEFAULT_MODEL = "joelbarmettler/gheim-ch-560m";
|
|
5
5
|
export interface RemoteDetectorOptions {
|
|
6
6
|
baseUrl?: string;
|
|
7
7
|
apiKey?: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"remote.d.ts","sourceRoot":"","sources":["../../../src/detectors/remote.ts"],"names":[],"mappings":"AAAA,wEAAwE;AACxE,OAAO,EAAE,KAAK,QAAQ,EAAE,KAAK,IAAI,EAAY,MAAM,WAAW,CAAC;AAE/D,eAAO,MAAM,gBAAgB,yBAAyB,CAAC;AACvD,eAAO,MAAM,aAAa,
|
|
1
|
+
{"version":3,"file":"remote.d.ts","sourceRoot":"","sources":["../../../src/detectors/remote.ts"],"names":[],"mappings":"AAAA,wEAAwE;AACxE,OAAO,EAAE,KAAK,QAAQ,EAAE,KAAK,IAAI,EAAY,MAAM,WAAW,CAAC;AAE/D,eAAO,MAAM,gBAAgB,yBAAyB,CAAC;AACvD,eAAO,MAAM,aAAa,iCAAiC,CAAC;AAE5D,MAAM,WAAW,qBAAqB;IACpC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wEAAwE;IACxE,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACtB;AAED,qBAAa,cAAe,YAAW,QAAQ;IAC7C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAU;IACzC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAe;gBAE7B,IAAI,GAAE,qBAA0B;IAUtC,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;CA+BvE"}
|
package/dist/types/index.d.ts
CHANGED
|
@@ -13,8 +13,8 @@ export { LABEL_TO_TAG, SENTINEL_RE, isPossibleSentinelPrefix, labelTag, parseSen
|
|
|
13
13
|
export type { Sentinel } from "./core/sentinels.ts";
|
|
14
14
|
export { StreamDeanonymizer } from "./core/stream.ts";
|
|
15
15
|
export type { Detector, Span } from "./detectors/base.ts";
|
|
16
|
-
export { DEFAULT_BASE_URL, REMOTE_DEFAULT_MODEL, RemoteDetector, LocalDetector, CompositeDetector, CalibratedDetector, defaultDetector, } from "./detectors/index.ts";
|
|
17
|
-
export type { RemoteDetectorOptions, LocalDetectorOptions, LocalDetectorLoadEvent, CompositeDetectorOptions, CalibratedDetectorOptions, } from "./detectors/index.ts";
|
|
16
|
+
export { DEFAULT_BASE_URL, REMOTE_DEFAULT_MODEL, RemoteDetector, LocalDetector, CompositeDetector, CustomPattern, CalibratedDetector, defaultDetector, } from "./detectors/index.ts";
|
|
17
|
+
export type { RemoteDetectorOptions, LocalDetectorOptions, LocalDetectorLoadEvent, CompositeDetectorOptions, CustomPatternOptions, CalibratedDetectorOptions, } from "./detectors/index.ts";
|
|
18
18
|
export { anonymizeMessages, anonymizeText, deanonymizeStream, deanonymizeText, } from "./plain.ts";
|
|
19
19
|
export type { ChatMessage } from "./plain.ts";
|
|
20
20
|
export declare const VERSION = "0.1.6";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,EACL,YAAY,EACZ,WAAW,EACX,wBAAwB,EACxB,QAAQ,EACR,aAAa,EACb,cAAc,GACf,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,YAAY,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EACL,gBAAgB,EAChB,oBAAoB,EACpB,cAAc,EACd,aAAa,EACb,iBAAiB,EACjB,kBAAkB,EAClB,eAAe,GAChB,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EACV,qBAAqB,EACrB,oBAAoB,EACpB,sBAAsB,EACtB,wBAAwB,EACxB,yBAAyB,GAC1B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,iBAAiB,EACjB,aAAa,EACb,iBAAiB,EACjB,eAAe,GAChB,MAAM,YAAY,CAAC;AACpB,YAAY,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAK9C,eAAO,MAAM,OAAO,UAAU,CAAC;AAE/B,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,YAAY,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,EACL,YAAY,EACZ,WAAW,EACX,wBAAwB,EACxB,QAAQ,EACR,aAAa,EACb,cAAc,GACf,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,YAAY,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EACL,gBAAgB,EAChB,oBAAoB,EACpB,cAAc,EACd,aAAa,EACb,iBAAiB,EACjB,aAAa,EACb,kBAAkB,EAClB,eAAe,GAChB,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EACV,qBAAqB,EACrB,oBAAoB,EACpB,sBAAsB,EACtB,wBAAwB,EACxB,oBAAoB,EACpB,yBAAyB,GAC1B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,iBAAiB,EACjB,aAAa,EACb,iBAAiB,EACjB,eAAe,GAChB,MAAM,YAAY,CAAC;AACpB,YAAY,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAK9C,eAAO,MAAM,OAAO,UAAU,CAAC;AAE/B,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,YAAY,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "gheim",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.10",
|
|
4
4
|
"description": "PII round-trip for LLM APIs: anonymize before the request, de-anonymize the stream on the way back.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Joel Barmettler <joel.barmettler@gmail.com>",
|