gheim 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -19
- package/dist/cjs/index.cjs +226 -3
- package/dist/cjs/openai/index.cjs +223 -1
- package/dist/esm/index.js +226 -3
- package/dist/esm/openai/index.js +223 -1
- package/dist/types/core/session.d.ts.map +1 -1
- package/dist/types/detectors/composite.d.ts +58 -0
- package/dist/types/detectors/composite.d.ts.map +1 -0
- package/dist/types/detectors/index.d.ts +2 -0
- package/dist/types/detectors/index.d.ts.map +1 -1
- package/dist/types/index.d.ts +3 -3
- package/dist/types/index.d.ts.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -38,33 +38,38 @@ The core package itself has no runtime dependencies.
|
|
|
38
38
|
|
|
39
39
|
`LocalDetector` loads a token-classification model via
|
|
40
40
|
[`@huggingface/transformers`](https://www.npmjs.com/package/@huggingface/transformers)
|
|
41
|
-
(transformers.js).
|
|
42
|
-
|
|
43
|
-
|
|
41
|
+
(transformers.js). The package's default model is
|
|
42
|
+
[`joelbarmettler/gheim-ch-560m`](https://huggingface.co/joelbarmettler/gheim-ch-560m)
|
|
43
|
+
— a 560M xlm-roberta-large fine-tune optimised for Swiss-market PII
|
|
44
|
+
(strict-span F1 0.916 on Swiss text, see
|
|
45
|
+
[MODEL_CARD.md](https://github.com/joelbarmettlerUZH/gheim/blob/main/MODEL_CARD.md)).
|
|
46
|
+
Any HuggingFace token-classification model that emits the same 33-class
|
|
47
|
+
BIOES schema can be substituted via `model:`.
|
|
44
48
|
|
|
45
49
|
| Model | Best for | Parameters | Notes |
|
|
46
50
|
|---|---|---:|---|
|
|
47
|
-
| [`joelbarmettler/gheim-ch-560m`](https://huggingface.co/joelbarmettler/gheim-ch-560m) | Swiss-market text (de_CH, fr_CH, it_CH, rm, en) with CH-format account numbers (IBAN, AHV, VAT-CHE) | 560M | Apache 2.0. Test F1
|
|
51
|
+
| [`joelbarmettler/gheim-ch-560m`](https://huggingface.co/joelbarmettler/gheim-ch-560m) **(default)** | Swiss-market text (de_CH, fr_CH, it_CH, rm, en) with CH-format account numbers (IBAN, AHV, VAT-CHE) | 560M | Apache 2.0. Test F1 0.916. Hub repo ships `onnx/model_quantized.onnx` only — load with `dtype: "q8"`. |
|
|
48
52
|
| [`openai/privacy-filter`](https://huggingface.co/openai/privacy-filter) | English-first or general use, long-context (up to 128k tokens) | 1.4B (50M active, MoE) | Apache 2.0. Wider language coverage, larger weights. |
|
|
49
53
|
|
|
50
54
|
```ts
|
|
51
55
|
import { LocalDetector } from "gheim";
|
|
52
56
|
|
|
53
|
-
//
|
|
54
|
-
|
|
57
|
+
// Default — Swiss-tuned, q8 ONNX. `device: "auto"` probes WebGPU in
|
|
58
|
+
// the browser and falls back to WASM; in Node it's WASM directly.
|
|
59
|
+
const det = new LocalDetector({ dtype: "q8" });
|
|
55
60
|
|
|
56
61
|
// Alternative for English or general use:
|
|
57
62
|
const detEn = new LocalDetector({ model: "openai/privacy-filter" });
|
|
58
63
|
```
|
|
59
64
|
|
|
60
|
-
The package default `model` is `openai/privacy-filter`. Pass `model`
|
|
61
|
-
explicitly to opt into the Swiss model.
|
|
62
|
-
|
|
63
65
|
## Drop-in OpenAI client
|
|
64
66
|
|
|
65
67
|
```ts
|
|
66
68
|
import { OpenAI } from "gheim/openai";
|
|
67
69
|
|
|
70
|
+
// Same constructor shape as `openai`. `apiKey`, `baseURL`, etc. work
|
|
71
|
+
// at the top level (forwarded to the inner client). gheim-specific
|
|
72
|
+
// keys: `gheimDetector`, `gheimStrict`, `openaiClient`, `clientOptions`.
|
|
68
73
|
const client = new OpenAI();
|
|
69
74
|
const r = await client.chat.completions.create({
|
|
70
75
|
model: "gpt-4o",
|
|
@@ -74,6 +79,17 @@ const r = await client.chat.completions.create({
|
|
|
74
79
|
// OpenAI only ever saw "<PERSON_1>".
|
|
75
80
|
```
|
|
76
81
|
|
|
82
|
+
Custom endpoint or key (e.g. OpenRouter, local vLLM):
|
|
83
|
+
|
|
84
|
+
```ts
|
|
85
|
+
import { OpenAI } from "gheim/openai";
|
|
86
|
+
|
|
87
|
+
const client = new OpenAI({
|
|
88
|
+
apiKey: process.env.OPENROUTER_API_KEY,
|
|
89
|
+
baseURL: "https://openrouter.ai/api/v1",
|
|
90
|
+
});
|
|
91
|
+
```
|
|
92
|
+
|
|
77
93
|
Streaming:
|
|
78
94
|
|
|
79
95
|
```ts
|
|
@@ -104,9 +120,9 @@ All non-chat OpenAI resources are on `client.raw` once `await client.ready()` re
|
|
|
104
120
|
import { Session, LocalDetector, anonymizeText, deanonymizeText } from "gheim";
|
|
105
121
|
|
|
106
122
|
const session = new Session();
|
|
107
|
-
(session as any).detector = new LocalDetector({
|
|
108
|
-
|
|
109
|
-
|
|
123
|
+
(session as any).detector = new LocalDetector({ dtype: "q8" });
|
|
124
|
+
// ^ defaults to joelbarmettler/gheim-ch-560m
|
|
125
|
+
|
|
110
126
|
const clean = await anonymizeText("Hi, my name is Joel", session);
|
|
111
127
|
// ... call any LLM with clean ...
|
|
112
128
|
const final = deanonymizeText(responseText, session);
|
|
@@ -155,12 +171,18 @@ client.raw.beta.assistants.create(...); // always works regardless of strict mo
|
|
|
155
171
|
```ts
|
|
156
172
|
import { LocalDetector, RemoteDetector, defaultDetector } from "gheim";
|
|
157
173
|
|
|
158
|
-
// Local: via @huggingface/transformers, on Node and
|
|
174
|
+
// Local: via @huggingface/transformers, on Node, Bun, and browsers.
|
|
175
|
+
// `device: "auto"` (default) probes navigator.gpu in the browser and
|
|
176
|
+
// falls back to WASM if WebGPU isn't usable; in Node it's WASM only.
|
|
159
177
|
const local = new LocalDetector({
|
|
160
|
-
model: "joelbarmettler/gheim-ch-560m",
|
|
161
178
|
device: "auto",
|
|
162
|
-
dtype: "
|
|
179
|
+
dtype: "q8",
|
|
180
|
+
onProgress: (e) => {
|
|
181
|
+
if (e.fraction != null) console.log(`loading: ${(e.fraction * 100) | 0}%`);
|
|
182
|
+
},
|
|
163
183
|
});
|
|
184
|
+
await local.load();
|
|
185
|
+
console.log("backend in use:", local.actualDevice); // "webgpu" or "wasm"
|
|
164
186
|
|
|
165
187
|
// Remote: against your own gheim-server or api.gheim.ch.
|
|
166
188
|
const remote = new RemoteDetector({
|
|
@@ -174,10 +196,9 @@ const auto = defaultDetector();
|
|
|
174
196
|
|
|
175
197
|
## Runtime support
|
|
176
198
|
|
|
177
|
-
Tested on Node 18
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
considered best-effort.
|
|
199
|
+
Tested on Node 18+, Bun 1.1+, and modern browsers (Chrome 113+, Edge,
|
|
200
|
+
Safari 17+, Firefox via flag). Browser builds use WebGPU when available
|
|
201
|
+
and fall back to WebAssembly automatically.
|
|
181
202
|
|
|
182
203
|
For Node servers, installing `onnxruntime-node` alongside
|
|
183
204
|
`@huggingface/transformers` enables the native ONNX backend, which is
|
package/dist/cjs/index.cjs
CHANGED
|
@@ -57,7 +57,8 @@ __export(exports_src, {
|
|
|
57
57
|
REMOTE_DEFAULT_MODEL: () => DEFAULT_MODEL,
|
|
58
58
|
LocalDetector: () => LocalDetector,
|
|
59
59
|
LABEL_TO_TAG: () => LABEL_TO_TAG,
|
|
60
|
-
DEFAULT_BASE_URL: () => DEFAULT_BASE_URL
|
|
60
|
+
DEFAULT_BASE_URL: () => DEFAULT_BASE_URL,
|
|
61
|
+
CompositeDetector: () => CompositeDetector
|
|
61
62
|
});
|
|
62
63
|
module.exports = __toCommonJS(exports_src);
|
|
63
64
|
|
|
@@ -154,7 +155,9 @@ class Session {
|
|
|
154
155
|
cursor = end;
|
|
155
156
|
}
|
|
156
157
|
out.push(text.slice(cursor));
|
|
157
|
-
|
|
158
|
+
let redacted = out.join("");
|
|
159
|
+
redacted = recoverDuplicateOccurrences(redacted, this.mapping);
|
|
160
|
+
return { redacted, merged };
|
|
158
161
|
}
|
|
159
162
|
restore(text) {
|
|
160
163
|
if (Object.keys(this.mapping).length === 0)
|
|
@@ -210,6 +213,18 @@ function mergeAdjacent(text, spans) {
|
|
|
210
213
|
}
|
|
211
214
|
return out;
|
|
212
215
|
}
|
|
216
|
+
function escapeRegex(s) {
|
|
217
|
+
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
218
|
+
}
|
|
219
|
+
function recoverDuplicateOccurrences(redacted, mapping) {
|
|
220
|
+
const entries = Object.entries(mapping).filter(([, surface]) => surface && surface.length > 0).sort((a, b) => b[1].length - a[1].length);
|
|
221
|
+
let result = redacted;
|
|
222
|
+
for (const [sentinel, surface] of entries) {
|
|
223
|
+
const pattern = new RegExp(`(?<![\\p{L}\\p{N}_])${escapeRegex(surface)}(?![\\p{L}\\p{N}_])`, "gu");
|
|
224
|
+
result = result.replace(pattern, sentinel);
|
|
225
|
+
}
|
|
226
|
+
return result;
|
|
227
|
+
}
|
|
213
228
|
// src/core/stream.ts
|
|
214
229
|
var SENTINEL_AT = /<([A-Z][A-Z0-9_]*)_(\d+)>/y;
|
|
215
230
|
var MAX_SENTINEL_LEN = 64;
|
|
@@ -619,6 +634,214 @@ async function loadTransformersJs() {
|
|
|
619
634
|
throw new Error("LocalDetector requires `@huggingface/transformers`. Install with: bun add @huggingface/transformers");
|
|
620
635
|
}
|
|
621
636
|
}
|
|
637
|
+
// src/detectors/composite.ts
|
|
638
|
+
function luhnOk(s) {
|
|
639
|
+
const digits = [];
|
|
640
|
+
for (const c of s) {
|
|
641
|
+
if (c >= "0" && c <= "9")
|
|
642
|
+
digits.push(Number(c));
|
|
643
|
+
}
|
|
644
|
+
if (digits.length < 13 || digits.length > 19)
|
|
645
|
+
return false;
|
|
646
|
+
let total = 0;
|
|
647
|
+
for (let i = 0;i < digits.length; i++) {
|
|
648
|
+
let d = digits[digits.length - 1 - i] ?? 0;
|
|
649
|
+
if (i % 2 === 1) {
|
|
650
|
+
d *= 2;
|
|
651
|
+
if (d > 9)
|
|
652
|
+
d -= 9;
|
|
653
|
+
}
|
|
654
|
+
total += d;
|
|
655
|
+
}
|
|
656
|
+
return total % 10 === 0;
|
|
657
|
+
}
|
|
658
|
+
function ibanChOk(raw) {
|
|
659
|
+
const s = raw.replace(/[\s-]/g, "");
|
|
660
|
+
if (s.length !== 21 || !s.startsWith("CH") || !/^\d+$/.test(s.slice(2))) {
|
|
661
|
+
return false;
|
|
662
|
+
}
|
|
663
|
+
const rearranged = s.slice(4) + s.slice(0, 4);
|
|
664
|
+
let numeric = "";
|
|
665
|
+
for (const c of rearranged) {
|
|
666
|
+
numeric += c >= "A" && c <= "Z" ? String(c.charCodeAt(0) - 55) : c;
|
|
667
|
+
}
|
|
668
|
+
let rem = 0;
|
|
669
|
+
for (const ch of numeric) {
|
|
670
|
+
rem = (rem * 10 + Number(ch)) % 97;
|
|
671
|
+
}
|
|
672
|
+
return rem === 1;
|
|
673
|
+
}
|
|
674
|
+
function ahvOk(s) {
|
|
675
|
+
const digits = [];
|
|
676
|
+
for (const c of s) {
|
|
677
|
+
if (c >= "0" && c <= "9")
|
|
678
|
+
digits.push(Number(c));
|
|
679
|
+
}
|
|
680
|
+
if (digits.length !== 13)
|
|
681
|
+
return false;
|
|
682
|
+
if (digits[0] !== 7 || digits[1] !== 5 || digits[2] !== 6)
|
|
683
|
+
return false;
|
|
684
|
+
let total = 0;
|
|
685
|
+
for (let i = 0;i < 12; i++) {
|
|
686
|
+
total += (digits[i] ?? 0) * (i % 2 === 0 ? 1 : 3);
|
|
687
|
+
}
|
|
688
|
+
const check = (10 - total % 10) % 10;
|
|
689
|
+
return digits[12] === check;
|
|
690
|
+
}
|
|
691
|
+
function vatCheOk(s) {
|
|
692
|
+
const digits = [];
|
|
693
|
+
for (const c of s) {
|
|
694
|
+
if (c >= "0" && c <= "9")
|
|
695
|
+
digits.push(Number(c));
|
|
696
|
+
}
|
|
697
|
+
if (digits.length < 9)
|
|
698
|
+
return false;
|
|
699
|
+
const weights = [5, 4, 3, 2, 7, 6, 5, 4];
|
|
700
|
+
let total = 0;
|
|
701
|
+
for (let i = 0;i < 8; i++) {
|
|
702
|
+
total += (digits[i] ?? 0) * (weights[i] ?? 0);
|
|
703
|
+
}
|
|
704
|
+
let expected = 11 - total % 11;
|
|
705
|
+
if (expected === 11)
|
|
706
|
+
expected = 0;
|
|
707
|
+
else if (expected === 10)
|
|
708
|
+
return false;
|
|
709
|
+
return digits[8] === expected;
|
|
710
|
+
}
|
|
711
|
+
var REGEXES = [
|
|
712
|
+
{
|
|
713
|
+
label: "private_email",
|
|
714
|
+
pattern: /\b[\w._%+\-]+@[\w.\-]+\.[A-Za-z]{2,}\b/g
|
|
715
|
+
},
|
|
716
|
+
{
|
|
717
|
+
label: "private_url",
|
|
718
|
+
pattern: /https?:\/\/[^\s)>\]]+/g
|
|
719
|
+
},
|
|
720
|
+
{
|
|
721
|
+
label: "private_phone",
|
|
722
|
+
pattern: /(?:\+41|0041|0)\s?(?:\(0\)\s?)?\d{2}[\s\-/.]?\d{3}[\s\-/.]?\d{2}[\s\-/.]?\d{2}\b/g
|
|
723
|
+
},
|
|
724
|
+
{
|
|
725
|
+
label: "account_number",
|
|
726
|
+
pattern: /\bCH\d{2}(?:[\s-]?\d){17}\b/g,
|
|
727
|
+
validator: ibanChOk
|
|
728
|
+
},
|
|
729
|
+
{
|
|
730
|
+
label: "account_number",
|
|
731
|
+
pattern: /\b756\.\d{4}\.\d{4}\.\d{2}\b/g,
|
|
732
|
+
validator: ahvOk
|
|
733
|
+
},
|
|
734
|
+
{
|
|
735
|
+
label: "account_number",
|
|
736
|
+
pattern: /\b756\d{10}\b/g,
|
|
737
|
+
validator: ahvOk
|
|
738
|
+
},
|
|
739
|
+
{
|
|
740
|
+
label: "account_number",
|
|
741
|
+
pattern: /\bCHE-\d{3}\.\d{3}\.\d{3}(?:\s+(?:MWST|TVA|IVA))?\b/g,
|
|
742
|
+
validator: vatCheOk
|
|
743
|
+
},
|
|
744
|
+
{
|
|
745
|
+
label: "account_number",
|
|
746
|
+
pattern: /\b(?:\d[\s-]?){15,16}\d\b/g,
|
|
747
|
+
validator: luhnOk
|
|
748
|
+
},
|
|
749
|
+
{
|
|
750
|
+
label: "private_date",
|
|
751
|
+
pattern: /\b(?:\d{1,2}[./]\d{1,2}[./]\d{4}|\d{4}[./-]\d{1,2}[./-]\d{1,2})\b/g
|
|
752
|
+
},
|
|
753
|
+
{
|
|
754
|
+
label: "secret",
|
|
755
|
+
pattern: /\bsk-(?:proj-)?[A-Za-z0-9_\-]{20,}\b/g
|
|
756
|
+
},
|
|
757
|
+
{
|
|
758
|
+
label: "secret",
|
|
759
|
+
pattern: /\bghp_[A-Za-z0-9]{36}\b/g
|
|
760
|
+
},
|
|
761
|
+
{
|
|
762
|
+
label: "secret",
|
|
763
|
+
pattern: /\bAKIA[A-Z0-9]{16}\b/g
|
|
764
|
+
}
|
|
765
|
+
];
|
|
766
|
+
var URL_TRAIL_TRIM = '.,;:!?")>]';
|
|
767
|
+
function trimUrl(text, start, end) {
|
|
768
|
+
while (end > start && URL_TRAIL_TRIM.includes(text[end - 1] ?? ""))
|
|
769
|
+
end--;
|
|
770
|
+
return [start, end];
|
|
771
|
+
}
|
|
772
|
+
function findRegexSpans(text) {
|
|
773
|
+
const candidates = [];
|
|
774
|
+
for (const { label, pattern, validator } of REGEXES) {
|
|
775
|
+
pattern.lastIndex = 0;
|
|
776
|
+
let m;
|
|
777
|
+
while ((m = pattern.exec(text)) !== null) {
|
|
778
|
+
let start = m.index;
|
|
779
|
+
let end = m.index + m[0].length;
|
|
780
|
+
if (label === "private_url")
|
|
781
|
+
[start, end] = trimUrl(text, start, end);
|
|
782
|
+
const value = text.slice(start, end);
|
|
783
|
+
if (!value.trim())
|
|
784
|
+
continue;
|
|
785
|
+
if (validator && !validator(value))
|
|
786
|
+
continue;
|
|
787
|
+
candidates.push({
|
|
788
|
+
start,
|
|
789
|
+
end,
|
|
790
|
+
label,
|
|
791
|
+
text: value,
|
|
792
|
+
score: 1
|
|
793
|
+
});
|
|
794
|
+
}
|
|
795
|
+
}
|
|
796
|
+
candidates.sort((a, b) => a.start !== b.start ? a.start - b.start : b.end - b.start - (a.end - a.start));
|
|
797
|
+
const out = [];
|
|
798
|
+
let lastEnd = -1;
|
|
799
|
+
for (const sp of candidates) {
|
|
800
|
+
if (sp.start < lastEnd)
|
|
801
|
+
continue;
|
|
802
|
+
out.push(sp);
|
|
803
|
+
lastEnd = sp.end;
|
|
804
|
+
}
|
|
805
|
+
return out;
|
|
806
|
+
}
|
|
807
|
+
function maskSpans(text, spans) {
|
|
808
|
+
if (spans.length === 0)
|
|
809
|
+
return text;
|
|
810
|
+
const chars = [...text];
|
|
811
|
+
for (const sp of spans) {
|
|
812
|
+
for (let i = sp.start;i < sp.end && i < chars.length; i++) {
|
|
813
|
+
chars[i] = " ";
|
|
814
|
+
}
|
|
815
|
+
}
|
|
816
|
+
return chars.join("");
|
|
817
|
+
}
|
|
818
|
+
|
|
819
|
+
class CompositeDetector {
|
|
820
|
+
model;
|
|
821
|
+
constructor(opts) {
|
|
822
|
+
if (!opts || !opts.model) {
|
|
823
|
+
throw new Error("CompositeDetector requires { model: Detector } — pass any object with " + "a .detect(text) method (e.g. new LocalDetector({...})).");
|
|
824
|
+
}
|
|
825
|
+
this.model = opts.model;
|
|
826
|
+
}
|
|
827
|
+
async detect(text, opts) {
|
|
828
|
+
if (!text)
|
|
829
|
+
return [];
|
|
830
|
+
const regexSpans = findRegexSpans(text);
|
|
831
|
+
const masked = maskSpans(text, regexSpans);
|
|
832
|
+
const modelSpans = await this.model.detect(masked, opts);
|
|
833
|
+
const out = [...regexSpans];
|
|
834
|
+
const regexRanges = regexSpans.map((s) => [s.start, s.end]);
|
|
835
|
+
for (const sp of modelSpans) {
|
|
836
|
+
const overlaps = regexRanges.some(([rs, re]) => sp.start < re && rs < sp.end);
|
|
837
|
+
if (overlaps)
|
|
838
|
+
continue;
|
|
839
|
+
out.push(sp);
|
|
840
|
+
}
|
|
841
|
+
out.sort((a, b) => a.start !== b.start ? a.start - b.start : b.end - b.start - (a.end - a.start));
|
|
842
|
+
return out;
|
|
843
|
+
}
|
|
844
|
+
}
|
|
622
845
|
// src/detectors/index.ts
|
|
623
846
|
function defaultDetector() {
|
|
624
847
|
const hasKey = typeof process !== "undefined" && Boolean(process.env?.GHEIM_API_KEY);
|
|
@@ -673,4 +896,4 @@ async function* deanonymizeStream(chunks, session) {
|
|
|
673
896
|
}
|
|
674
897
|
|
|
675
898
|
// src/index.ts
|
|
676
|
-
var VERSION = "0.1.
|
|
899
|
+
var VERSION = "0.1.4";
|
|
@@ -543,6 +543,214 @@ async function loadTransformersJs() {
|
|
|
543
543
|
throw new Error("LocalDetector requires `@huggingface/transformers`. Install with: bun add @huggingface/transformers");
|
|
544
544
|
}
|
|
545
545
|
}
|
|
546
|
+
// src/detectors/composite.ts
|
|
547
|
+
function luhnOk(s) {
|
|
548
|
+
const digits = [];
|
|
549
|
+
for (const c of s) {
|
|
550
|
+
if (c >= "0" && c <= "9")
|
|
551
|
+
digits.push(Number(c));
|
|
552
|
+
}
|
|
553
|
+
if (digits.length < 13 || digits.length > 19)
|
|
554
|
+
return false;
|
|
555
|
+
let total = 0;
|
|
556
|
+
for (let i = 0;i < digits.length; i++) {
|
|
557
|
+
let d = digits[digits.length - 1 - i] ?? 0;
|
|
558
|
+
if (i % 2 === 1) {
|
|
559
|
+
d *= 2;
|
|
560
|
+
if (d > 9)
|
|
561
|
+
d -= 9;
|
|
562
|
+
}
|
|
563
|
+
total += d;
|
|
564
|
+
}
|
|
565
|
+
return total % 10 === 0;
|
|
566
|
+
}
|
|
567
|
+
function ibanChOk(raw) {
|
|
568
|
+
const s = raw.replace(/[\s-]/g, "");
|
|
569
|
+
if (s.length !== 21 || !s.startsWith("CH") || !/^\d+$/.test(s.slice(2))) {
|
|
570
|
+
return false;
|
|
571
|
+
}
|
|
572
|
+
const rearranged = s.slice(4) + s.slice(0, 4);
|
|
573
|
+
let numeric = "";
|
|
574
|
+
for (const c of rearranged) {
|
|
575
|
+
numeric += c >= "A" && c <= "Z" ? String(c.charCodeAt(0) - 55) : c;
|
|
576
|
+
}
|
|
577
|
+
let rem = 0;
|
|
578
|
+
for (const ch of numeric) {
|
|
579
|
+
rem = (rem * 10 + Number(ch)) % 97;
|
|
580
|
+
}
|
|
581
|
+
return rem === 1;
|
|
582
|
+
}
|
|
583
|
+
function ahvOk(s) {
|
|
584
|
+
const digits = [];
|
|
585
|
+
for (const c of s) {
|
|
586
|
+
if (c >= "0" && c <= "9")
|
|
587
|
+
digits.push(Number(c));
|
|
588
|
+
}
|
|
589
|
+
if (digits.length !== 13)
|
|
590
|
+
return false;
|
|
591
|
+
if (digits[0] !== 7 || digits[1] !== 5 || digits[2] !== 6)
|
|
592
|
+
return false;
|
|
593
|
+
let total = 0;
|
|
594
|
+
for (let i = 0;i < 12; i++) {
|
|
595
|
+
total += (digits[i] ?? 0) * (i % 2 === 0 ? 1 : 3);
|
|
596
|
+
}
|
|
597
|
+
const check = (10 - total % 10) % 10;
|
|
598
|
+
return digits[12] === check;
|
|
599
|
+
}
|
|
600
|
+
function vatCheOk(s) {
|
|
601
|
+
const digits = [];
|
|
602
|
+
for (const c of s) {
|
|
603
|
+
if (c >= "0" && c <= "9")
|
|
604
|
+
digits.push(Number(c));
|
|
605
|
+
}
|
|
606
|
+
if (digits.length < 9)
|
|
607
|
+
return false;
|
|
608
|
+
const weights = [5, 4, 3, 2, 7, 6, 5, 4];
|
|
609
|
+
let total = 0;
|
|
610
|
+
for (let i = 0;i < 8; i++) {
|
|
611
|
+
total += (digits[i] ?? 0) * (weights[i] ?? 0);
|
|
612
|
+
}
|
|
613
|
+
let expected = 11 - total % 11;
|
|
614
|
+
if (expected === 11)
|
|
615
|
+
expected = 0;
|
|
616
|
+
else if (expected === 10)
|
|
617
|
+
return false;
|
|
618
|
+
return digits[8] === expected;
|
|
619
|
+
}
|
|
620
|
+
var REGEXES = [
|
|
621
|
+
{
|
|
622
|
+
label: "private_email",
|
|
623
|
+
pattern: /\b[\w._%+\-]+@[\w.\-]+\.[A-Za-z]{2,}\b/g
|
|
624
|
+
},
|
|
625
|
+
{
|
|
626
|
+
label: "private_url",
|
|
627
|
+
pattern: /https?:\/\/[^\s)>\]]+/g
|
|
628
|
+
},
|
|
629
|
+
{
|
|
630
|
+
label: "private_phone",
|
|
631
|
+
pattern: /(?:\+41|0041|0)\s?(?:\(0\)\s?)?\d{2}[\s\-/.]?\d{3}[\s\-/.]?\d{2}[\s\-/.]?\d{2}\b/g
|
|
632
|
+
},
|
|
633
|
+
{
|
|
634
|
+
label: "account_number",
|
|
635
|
+
pattern: /\bCH\d{2}(?:[\s-]?\d){17}\b/g,
|
|
636
|
+
validator: ibanChOk
|
|
637
|
+
},
|
|
638
|
+
{
|
|
639
|
+
label: "account_number",
|
|
640
|
+
pattern: /\b756\.\d{4}\.\d{4}\.\d{2}\b/g,
|
|
641
|
+
validator: ahvOk
|
|
642
|
+
},
|
|
643
|
+
{
|
|
644
|
+
label: "account_number",
|
|
645
|
+
pattern: /\b756\d{10}\b/g,
|
|
646
|
+
validator: ahvOk
|
|
647
|
+
},
|
|
648
|
+
{
|
|
649
|
+
label: "account_number",
|
|
650
|
+
pattern: /\bCHE-\d{3}\.\d{3}\.\d{3}(?:\s+(?:MWST|TVA|IVA))?\b/g,
|
|
651
|
+
validator: vatCheOk
|
|
652
|
+
},
|
|
653
|
+
{
|
|
654
|
+
label: "account_number",
|
|
655
|
+
pattern: /\b(?:\d[\s-]?){15,16}\d\b/g,
|
|
656
|
+
validator: luhnOk
|
|
657
|
+
},
|
|
658
|
+
{
|
|
659
|
+
label: "private_date",
|
|
660
|
+
pattern: /\b(?:\d{1,2}[./]\d{1,2}[./]\d{4}|\d{4}[./-]\d{1,2}[./-]\d{1,2})\b/g
|
|
661
|
+
},
|
|
662
|
+
{
|
|
663
|
+
label: "secret",
|
|
664
|
+
pattern: /\bsk-(?:proj-)?[A-Za-z0-9_\-]{20,}\b/g
|
|
665
|
+
},
|
|
666
|
+
{
|
|
667
|
+
label: "secret",
|
|
668
|
+
pattern: /\bghp_[A-Za-z0-9]{36}\b/g
|
|
669
|
+
},
|
|
670
|
+
{
|
|
671
|
+
label: "secret",
|
|
672
|
+
pattern: /\bAKIA[A-Z0-9]{16}\b/g
|
|
673
|
+
}
|
|
674
|
+
];
|
|
675
|
+
var URL_TRAIL_TRIM = '.,;:!?")>]';
|
|
676
|
+
function trimUrl(text, start, end) {
|
|
677
|
+
while (end > start && URL_TRAIL_TRIM.includes(text[end - 1] ?? ""))
|
|
678
|
+
end--;
|
|
679
|
+
return [start, end];
|
|
680
|
+
}
|
|
681
|
+
function findRegexSpans(text) {
|
|
682
|
+
const candidates = [];
|
|
683
|
+
for (const { label, pattern, validator } of REGEXES) {
|
|
684
|
+
pattern.lastIndex = 0;
|
|
685
|
+
let m;
|
|
686
|
+
while ((m = pattern.exec(text)) !== null) {
|
|
687
|
+
let start = m.index;
|
|
688
|
+
let end = m.index + m[0].length;
|
|
689
|
+
if (label === "private_url")
|
|
690
|
+
[start, end] = trimUrl(text, start, end);
|
|
691
|
+
const value = text.slice(start, end);
|
|
692
|
+
if (!value.trim())
|
|
693
|
+
continue;
|
|
694
|
+
if (validator && !validator(value))
|
|
695
|
+
continue;
|
|
696
|
+
candidates.push({
|
|
697
|
+
start,
|
|
698
|
+
end,
|
|
699
|
+
label,
|
|
700
|
+
text: value,
|
|
701
|
+
score: 1
|
|
702
|
+
});
|
|
703
|
+
}
|
|
704
|
+
}
|
|
705
|
+
candidates.sort((a, b) => a.start !== b.start ? a.start - b.start : b.end - b.start - (a.end - a.start));
|
|
706
|
+
const out = [];
|
|
707
|
+
let lastEnd = -1;
|
|
708
|
+
for (const sp of candidates) {
|
|
709
|
+
if (sp.start < lastEnd)
|
|
710
|
+
continue;
|
|
711
|
+
out.push(sp);
|
|
712
|
+
lastEnd = sp.end;
|
|
713
|
+
}
|
|
714
|
+
return out;
|
|
715
|
+
}
|
|
716
|
+
function maskSpans(text, spans) {
|
|
717
|
+
if (spans.length === 0)
|
|
718
|
+
return text;
|
|
719
|
+
const chars = [...text];
|
|
720
|
+
for (const sp of spans) {
|
|
721
|
+
for (let i = sp.start;i < sp.end && i < chars.length; i++) {
|
|
722
|
+
chars[i] = " ";
|
|
723
|
+
}
|
|
724
|
+
}
|
|
725
|
+
return chars.join("");
|
|
726
|
+
}
|
|
727
|
+
|
|
728
|
+
class CompositeDetector {
|
|
729
|
+
model;
|
|
730
|
+
constructor(opts) {
|
|
731
|
+
if (!opts || !opts.model) {
|
|
732
|
+
throw new Error("CompositeDetector requires { model: Detector } — pass any object with " + "a .detect(text) method (e.g. new LocalDetector({...})).");
|
|
733
|
+
}
|
|
734
|
+
this.model = opts.model;
|
|
735
|
+
}
|
|
736
|
+
async detect(text, opts) {
|
|
737
|
+
if (!text)
|
|
738
|
+
return [];
|
|
739
|
+
const regexSpans = findRegexSpans(text);
|
|
740
|
+
const masked = maskSpans(text, regexSpans);
|
|
741
|
+
const modelSpans = await this.model.detect(masked, opts);
|
|
742
|
+
const out = [...regexSpans];
|
|
743
|
+
const regexRanges = regexSpans.map((s) => [s.start, s.end]);
|
|
744
|
+
for (const sp of modelSpans) {
|
|
745
|
+
const overlaps = regexRanges.some(([rs, re]) => sp.start < re && rs < sp.end);
|
|
746
|
+
if (overlaps)
|
|
747
|
+
continue;
|
|
748
|
+
out.push(sp);
|
|
749
|
+
}
|
|
750
|
+
out.sort((a, b) => a.start !== b.start ? a.start - b.start : b.end - b.start - (a.end - a.start));
|
|
751
|
+
return out;
|
|
752
|
+
}
|
|
753
|
+
}
|
|
546
754
|
// src/detectors/index.ts
|
|
547
755
|
function defaultDetector() {
|
|
548
756
|
const hasKey = typeof process !== "undefined" && Boolean(process.env?.GHEIM_API_KEY);
|
|
@@ -606,7 +814,9 @@ class Session {
|
|
|
606
814
|
cursor = end;
|
|
607
815
|
}
|
|
608
816
|
out.push(text.slice(cursor));
|
|
609
|
-
|
|
817
|
+
let redacted = out.join("");
|
|
818
|
+
redacted = recoverDuplicateOccurrences(redacted, this.mapping);
|
|
819
|
+
return { redacted, merged };
|
|
610
820
|
}
|
|
611
821
|
restore(text) {
|
|
612
822
|
if (Object.keys(this.mapping).length === 0)
|
|
@@ -662,6 +872,18 @@ function mergeAdjacent(text, spans) {
|
|
|
662
872
|
}
|
|
663
873
|
return out;
|
|
664
874
|
}
|
|
875
|
+
function escapeRegex(s) {
|
|
876
|
+
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
877
|
+
}
|
|
878
|
+
function recoverDuplicateOccurrences(redacted, mapping) {
|
|
879
|
+
const entries = Object.entries(mapping).filter(([, surface]) => surface && surface.length > 0).sort((a, b) => b[1].length - a[1].length);
|
|
880
|
+
let result = redacted;
|
|
881
|
+
for (const [sentinel, surface] of entries) {
|
|
882
|
+
const pattern = new RegExp(`(?<![\\p{L}\\p{N}_])${escapeRegex(surface)}(?![\\p{L}\\p{N}_])`, "gu");
|
|
883
|
+
result = result.replace(pattern, sentinel);
|
|
884
|
+
}
|
|
885
|
+
return result;
|
|
886
|
+
}
|
|
665
887
|
|
|
666
888
|
// src/openai/_base.ts
|
|
667
889
|
function resolveCallSession(body, defaultDet) {
|
package/dist/esm/index.js
CHANGED
|
@@ -99,7 +99,9 @@ class Session {
|
|
|
99
99
|
cursor = end;
|
|
100
100
|
}
|
|
101
101
|
out.push(text.slice(cursor));
|
|
102
|
-
|
|
102
|
+
let redacted = out.join("");
|
|
103
|
+
redacted = recoverDuplicateOccurrences(redacted, this.mapping);
|
|
104
|
+
return { redacted, merged };
|
|
103
105
|
}
|
|
104
106
|
restore(text) {
|
|
105
107
|
if (Object.keys(this.mapping).length === 0)
|
|
@@ -155,6 +157,18 @@ function mergeAdjacent(text, spans) {
|
|
|
155
157
|
}
|
|
156
158
|
return out;
|
|
157
159
|
}
|
|
160
|
+
function escapeRegex(s) {
|
|
161
|
+
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
162
|
+
}
|
|
163
|
+
function recoverDuplicateOccurrences(redacted, mapping) {
|
|
164
|
+
const entries = Object.entries(mapping).filter(([, surface]) => surface && surface.length > 0).sort((a, b) => b[1].length - a[1].length);
|
|
165
|
+
let result = redacted;
|
|
166
|
+
for (const [sentinel, surface] of entries) {
|
|
167
|
+
const pattern = new RegExp(`(?<![\\p{L}\\p{N}_])${escapeRegex(surface)}(?![\\p{L}\\p{N}_])`, "gu");
|
|
168
|
+
result = result.replace(pattern, sentinel);
|
|
169
|
+
}
|
|
170
|
+
return result;
|
|
171
|
+
}
|
|
158
172
|
// src/core/stream.ts
|
|
159
173
|
var SENTINEL_AT = /<([A-Z][A-Z0-9_]*)_(\d+)>/y;
|
|
160
174
|
var MAX_SENTINEL_LEN = 64;
|
|
@@ -564,6 +578,214 @@ async function loadTransformersJs() {
|
|
|
564
578
|
throw new Error("LocalDetector requires `@huggingface/transformers`. Install with: bun add @huggingface/transformers");
|
|
565
579
|
}
|
|
566
580
|
}
|
|
581
|
+
// src/detectors/composite.ts
|
|
582
|
+
function luhnOk(s) {
|
|
583
|
+
const digits = [];
|
|
584
|
+
for (const c of s) {
|
|
585
|
+
if (c >= "0" && c <= "9")
|
|
586
|
+
digits.push(Number(c));
|
|
587
|
+
}
|
|
588
|
+
if (digits.length < 13 || digits.length > 19)
|
|
589
|
+
return false;
|
|
590
|
+
let total = 0;
|
|
591
|
+
for (let i = 0;i < digits.length; i++) {
|
|
592
|
+
let d = digits[digits.length - 1 - i] ?? 0;
|
|
593
|
+
if (i % 2 === 1) {
|
|
594
|
+
d *= 2;
|
|
595
|
+
if (d > 9)
|
|
596
|
+
d -= 9;
|
|
597
|
+
}
|
|
598
|
+
total += d;
|
|
599
|
+
}
|
|
600
|
+
return total % 10 === 0;
|
|
601
|
+
}
|
|
602
|
+
function ibanChOk(raw) {
|
|
603
|
+
const s = raw.replace(/[\s-]/g, "");
|
|
604
|
+
if (s.length !== 21 || !s.startsWith("CH") || !/^\d+$/.test(s.slice(2))) {
|
|
605
|
+
return false;
|
|
606
|
+
}
|
|
607
|
+
const rearranged = s.slice(4) + s.slice(0, 4);
|
|
608
|
+
let numeric = "";
|
|
609
|
+
for (const c of rearranged) {
|
|
610
|
+
numeric += c >= "A" && c <= "Z" ? String(c.charCodeAt(0) - 55) : c;
|
|
611
|
+
}
|
|
612
|
+
let rem = 0;
|
|
613
|
+
for (const ch of numeric) {
|
|
614
|
+
rem = (rem * 10 + Number(ch)) % 97;
|
|
615
|
+
}
|
|
616
|
+
return rem === 1;
|
|
617
|
+
}
|
|
618
|
+
function ahvOk(s) {
|
|
619
|
+
const digits = [];
|
|
620
|
+
for (const c of s) {
|
|
621
|
+
if (c >= "0" && c <= "9")
|
|
622
|
+
digits.push(Number(c));
|
|
623
|
+
}
|
|
624
|
+
if (digits.length !== 13)
|
|
625
|
+
return false;
|
|
626
|
+
if (digits[0] !== 7 || digits[1] !== 5 || digits[2] !== 6)
|
|
627
|
+
return false;
|
|
628
|
+
let total = 0;
|
|
629
|
+
for (let i = 0;i < 12; i++) {
|
|
630
|
+
total += (digits[i] ?? 0) * (i % 2 === 0 ? 1 : 3);
|
|
631
|
+
}
|
|
632
|
+
const check = (10 - total % 10) % 10;
|
|
633
|
+
return digits[12] === check;
|
|
634
|
+
}
|
|
635
|
+
function vatCheOk(s) {
|
|
636
|
+
const digits = [];
|
|
637
|
+
for (const c of s) {
|
|
638
|
+
if (c >= "0" && c <= "9")
|
|
639
|
+
digits.push(Number(c));
|
|
640
|
+
}
|
|
641
|
+
if (digits.length < 9)
|
|
642
|
+
return false;
|
|
643
|
+
const weights = [5, 4, 3, 2, 7, 6, 5, 4];
|
|
644
|
+
let total = 0;
|
|
645
|
+
for (let i = 0;i < 8; i++) {
|
|
646
|
+
total += (digits[i] ?? 0) * (weights[i] ?? 0);
|
|
647
|
+
}
|
|
648
|
+
let expected = 11 - total % 11;
|
|
649
|
+
if (expected === 11)
|
|
650
|
+
expected = 0;
|
|
651
|
+
else if (expected === 10)
|
|
652
|
+
return false;
|
|
653
|
+
return digits[8] === expected;
|
|
654
|
+
}
|
|
655
|
+
var REGEXES = [
|
|
656
|
+
{
|
|
657
|
+
label: "private_email",
|
|
658
|
+
pattern: /\b[\w._%+\-]+@[\w.\-]+\.[A-Za-z]{2,}\b/g
|
|
659
|
+
},
|
|
660
|
+
{
|
|
661
|
+
label: "private_url",
|
|
662
|
+
pattern: /https?:\/\/[^\s)>\]]+/g
|
|
663
|
+
},
|
|
664
|
+
{
|
|
665
|
+
label: "private_phone",
|
|
666
|
+
pattern: /(?:\+41|0041|0)\s?(?:\(0\)\s?)?\d{2}[\s\-/.]?\d{3}[\s\-/.]?\d{2}[\s\-/.]?\d{2}\b/g
|
|
667
|
+
},
|
|
668
|
+
{
|
|
669
|
+
label: "account_number",
|
|
670
|
+
pattern: /\bCH\d{2}(?:[\s-]?\d){17}\b/g,
|
|
671
|
+
validator: ibanChOk
|
|
672
|
+
},
|
|
673
|
+
{
|
|
674
|
+
label: "account_number",
|
|
675
|
+
pattern: /\b756\.\d{4}\.\d{4}\.\d{2}\b/g,
|
|
676
|
+
validator: ahvOk
|
|
677
|
+
},
|
|
678
|
+
{
|
|
679
|
+
label: "account_number",
|
|
680
|
+
pattern: /\b756\d{10}\b/g,
|
|
681
|
+
validator: ahvOk
|
|
682
|
+
},
|
|
683
|
+
{
|
|
684
|
+
label: "account_number",
|
|
685
|
+
pattern: /\bCHE-\d{3}\.\d{3}\.\d{3}(?:\s+(?:MWST|TVA|IVA))?\b/g,
|
|
686
|
+
validator: vatCheOk
|
|
687
|
+
},
|
|
688
|
+
{
|
|
689
|
+
label: "account_number",
|
|
690
|
+
pattern: /\b(?:\d[\s-]?){15,16}\d\b/g,
|
|
691
|
+
validator: luhnOk
|
|
692
|
+
},
|
|
693
|
+
{
|
|
694
|
+
label: "private_date",
|
|
695
|
+
pattern: /\b(?:\d{1,2}[./]\d{1,2}[./]\d{4}|\d{4}[./-]\d{1,2}[./-]\d{1,2})\b/g
|
|
696
|
+
},
|
|
697
|
+
{
|
|
698
|
+
label: "secret",
|
|
699
|
+
pattern: /\bsk-(?:proj-)?[A-Za-z0-9_\-]{20,}\b/g
|
|
700
|
+
},
|
|
701
|
+
{
|
|
702
|
+
label: "secret",
|
|
703
|
+
pattern: /\bghp_[A-Za-z0-9]{36}\b/g
|
|
704
|
+
},
|
|
705
|
+
{
|
|
706
|
+
label: "secret",
|
|
707
|
+
pattern: /\bAKIA[A-Z0-9]{16}\b/g
|
|
708
|
+
}
|
|
709
|
+
];
|
|
710
|
+
var URL_TRAIL_TRIM = '.,;:!?")>]';
|
|
711
|
+
function trimUrl(text, start, end) {
|
|
712
|
+
while (end > start && URL_TRAIL_TRIM.includes(text[end - 1] ?? ""))
|
|
713
|
+
end--;
|
|
714
|
+
return [start, end];
|
|
715
|
+
}
|
|
716
|
+
function findRegexSpans(text) {
|
|
717
|
+
const candidates = [];
|
|
718
|
+
for (const { label, pattern, validator } of REGEXES) {
|
|
719
|
+
pattern.lastIndex = 0;
|
|
720
|
+
let m;
|
|
721
|
+
while ((m = pattern.exec(text)) !== null) {
|
|
722
|
+
let start = m.index;
|
|
723
|
+
let end = m.index + m[0].length;
|
|
724
|
+
if (label === "private_url")
|
|
725
|
+
[start, end] = trimUrl(text, start, end);
|
|
726
|
+
const value = text.slice(start, end);
|
|
727
|
+
if (!value.trim())
|
|
728
|
+
continue;
|
|
729
|
+
if (validator && !validator(value))
|
|
730
|
+
continue;
|
|
731
|
+
candidates.push({
|
|
732
|
+
start,
|
|
733
|
+
end,
|
|
734
|
+
label,
|
|
735
|
+
text: value,
|
|
736
|
+
score: 1
|
|
737
|
+
});
|
|
738
|
+
}
|
|
739
|
+
}
|
|
740
|
+
candidates.sort((a, b) => a.start !== b.start ? a.start - b.start : b.end - b.start - (a.end - a.start));
|
|
741
|
+
const out = [];
|
|
742
|
+
let lastEnd = -1;
|
|
743
|
+
for (const sp of candidates) {
|
|
744
|
+
if (sp.start < lastEnd)
|
|
745
|
+
continue;
|
|
746
|
+
out.push(sp);
|
|
747
|
+
lastEnd = sp.end;
|
|
748
|
+
}
|
|
749
|
+
return out;
|
|
750
|
+
}
|
|
751
|
+
function maskSpans(text, spans) {
|
|
752
|
+
if (spans.length === 0)
|
|
753
|
+
return text;
|
|
754
|
+
const chars = [...text];
|
|
755
|
+
for (const sp of spans) {
|
|
756
|
+
for (let i = sp.start;i < sp.end && i < chars.length; i++) {
|
|
757
|
+
chars[i] = " ";
|
|
758
|
+
}
|
|
759
|
+
}
|
|
760
|
+
return chars.join("");
|
|
761
|
+
}
|
|
762
|
+
|
|
763
|
+
class CompositeDetector {
|
|
764
|
+
model;
|
|
765
|
+
constructor(opts) {
|
|
766
|
+
if (!opts || !opts.model) {
|
|
767
|
+
throw new Error("CompositeDetector requires { model: Detector } — pass any object with " + "a .detect(text) method (e.g. new LocalDetector({...})).");
|
|
768
|
+
}
|
|
769
|
+
this.model = opts.model;
|
|
770
|
+
}
|
|
771
|
+
async detect(text, opts) {
|
|
772
|
+
if (!text)
|
|
773
|
+
return [];
|
|
774
|
+
const regexSpans = findRegexSpans(text);
|
|
775
|
+
const masked = maskSpans(text, regexSpans);
|
|
776
|
+
const modelSpans = await this.model.detect(masked, opts);
|
|
777
|
+
const out = [...regexSpans];
|
|
778
|
+
const regexRanges = regexSpans.map((s) => [s.start, s.end]);
|
|
779
|
+
for (const sp of modelSpans) {
|
|
780
|
+
const overlaps = regexRanges.some(([rs, re]) => sp.start < re && rs < sp.end);
|
|
781
|
+
if (overlaps)
|
|
782
|
+
continue;
|
|
783
|
+
out.push(sp);
|
|
784
|
+
}
|
|
785
|
+
out.sort((a, b) => a.start !== b.start ? a.start - b.start : b.end - b.start - (a.end - a.start));
|
|
786
|
+
return out;
|
|
787
|
+
}
|
|
788
|
+
}
|
|
567
789
|
// src/detectors/index.ts
|
|
568
790
|
function defaultDetector() {
|
|
569
791
|
const hasKey = typeof process !== "undefined" && Boolean(process.env?.GHEIM_API_KEY);
|
|
@@ -618,7 +840,7 @@ async function* deanonymizeStream(chunks, session) {
|
|
|
618
840
|
}
|
|
619
841
|
|
|
620
842
|
// src/index.ts
|
|
621
|
-
var VERSION = "0.1.
|
|
843
|
+
var VERSION = "0.1.4";
|
|
622
844
|
export {
|
|
623
845
|
renderSentinel,
|
|
624
846
|
parseSentinel,
|
|
@@ -638,5 +860,6 @@ export {
|
|
|
638
860
|
DEFAULT_MODEL as REMOTE_DEFAULT_MODEL,
|
|
639
861
|
LocalDetector,
|
|
640
862
|
LABEL_TO_TAG,
|
|
641
|
-
DEFAULT_BASE_URL
|
|
863
|
+
DEFAULT_BASE_URL,
|
|
864
|
+
CompositeDetector
|
|
642
865
|
};
|
package/dist/esm/openai/index.js
CHANGED
|
@@ -500,6 +500,214 @@ async function loadTransformersJs() {
|
|
|
500
500
|
throw new Error("LocalDetector requires `@huggingface/transformers`. Install with: bun add @huggingface/transformers");
|
|
501
501
|
}
|
|
502
502
|
}
|
|
503
|
+
// src/detectors/composite.ts
|
|
504
|
+
function luhnOk(s) {
|
|
505
|
+
const digits = [];
|
|
506
|
+
for (const c of s) {
|
|
507
|
+
if (c >= "0" && c <= "9")
|
|
508
|
+
digits.push(Number(c));
|
|
509
|
+
}
|
|
510
|
+
if (digits.length < 13 || digits.length > 19)
|
|
511
|
+
return false;
|
|
512
|
+
let total = 0;
|
|
513
|
+
for (let i = 0;i < digits.length; i++) {
|
|
514
|
+
let d = digits[digits.length - 1 - i] ?? 0;
|
|
515
|
+
if (i % 2 === 1) {
|
|
516
|
+
d *= 2;
|
|
517
|
+
if (d > 9)
|
|
518
|
+
d -= 9;
|
|
519
|
+
}
|
|
520
|
+
total += d;
|
|
521
|
+
}
|
|
522
|
+
return total % 10 === 0;
|
|
523
|
+
}
|
|
524
|
+
function ibanChOk(raw) {
|
|
525
|
+
const s = raw.replace(/[\s-]/g, "");
|
|
526
|
+
if (s.length !== 21 || !s.startsWith("CH") || !/^\d+$/.test(s.slice(2))) {
|
|
527
|
+
return false;
|
|
528
|
+
}
|
|
529
|
+
const rearranged = s.slice(4) + s.slice(0, 4);
|
|
530
|
+
let numeric = "";
|
|
531
|
+
for (const c of rearranged) {
|
|
532
|
+
numeric += c >= "A" && c <= "Z" ? String(c.charCodeAt(0) - 55) : c;
|
|
533
|
+
}
|
|
534
|
+
let rem = 0;
|
|
535
|
+
for (const ch of numeric) {
|
|
536
|
+
rem = (rem * 10 + Number(ch)) % 97;
|
|
537
|
+
}
|
|
538
|
+
return rem === 1;
|
|
539
|
+
}
|
|
540
|
+
function ahvOk(s) {
|
|
541
|
+
const digits = [];
|
|
542
|
+
for (const c of s) {
|
|
543
|
+
if (c >= "0" && c <= "9")
|
|
544
|
+
digits.push(Number(c));
|
|
545
|
+
}
|
|
546
|
+
if (digits.length !== 13)
|
|
547
|
+
return false;
|
|
548
|
+
if (digits[0] !== 7 || digits[1] !== 5 || digits[2] !== 6)
|
|
549
|
+
return false;
|
|
550
|
+
let total = 0;
|
|
551
|
+
for (let i = 0;i < 12; i++) {
|
|
552
|
+
total += (digits[i] ?? 0) * (i % 2 === 0 ? 1 : 3);
|
|
553
|
+
}
|
|
554
|
+
const check = (10 - total % 10) % 10;
|
|
555
|
+
return digits[12] === check;
|
|
556
|
+
}
|
|
557
|
+
function vatCheOk(s) {
|
|
558
|
+
const digits = [];
|
|
559
|
+
for (const c of s) {
|
|
560
|
+
if (c >= "0" && c <= "9")
|
|
561
|
+
digits.push(Number(c));
|
|
562
|
+
}
|
|
563
|
+
if (digits.length < 9)
|
|
564
|
+
return false;
|
|
565
|
+
const weights = [5, 4, 3, 2, 7, 6, 5, 4];
|
|
566
|
+
let total = 0;
|
|
567
|
+
for (let i = 0;i < 8; i++) {
|
|
568
|
+
total += (digits[i] ?? 0) * (weights[i] ?? 0);
|
|
569
|
+
}
|
|
570
|
+
let expected = 11 - total % 11;
|
|
571
|
+
if (expected === 11)
|
|
572
|
+
expected = 0;
|
|
573
|
+
else if (expected === 10)
|
|
574
|
+
return false;
|
|
575
|
+
return digits[8] === expected;
|
|
576
|
+
}
|
|
577
|
+
var REGEXES = [
|
|
578
|
+
{
|
|
579
|
+
label: "private_email",
|
|
580
|
+
pattern: /\b[\w._%+\-]+@[\w.\-]+\.[A-Za-z]{2,}\b/g
|
|
581
|
+
},
|
|
582
|
+
{
|
|
583
|
+
label: "private_url",
|
|
584
|
+
pattern: /https?:\/\/[^\s)>\]]+/g
|
|
585
|
+
},
|
|
586
|
+
{
|
|
587
|
+
label: "private_phone",
|
|
588
|
+
pattern: /(?:\+41|0041|0)\s?(?:\(0\)\s?)?\d{2}[\s\-/.]?\d{3}[\s\-/.]?\d{2}[\s\-/.]?\d{2}\b/g
|
|
589
|
+
},
|
|
590
|
+
{
|
|
591
|
+
label: "account_number",
|
|
592
|
+
pattern: /\bCH\d{2}(?:[\s-]?\d){17}\b/g,
|
|
593
|
+
validator: ibanChOk
|
|
594
|
+
},
|
|
595
|
+
{
|
|
596
|
+
label: "account_number",
|
|
597
|
+
pattern: /\b756\.\d{4}\.\d{4}\.\d{2}\b/g,
|
|
598
|
+
validator: ahvOk
|
|
599
|
+
},
|
|
600
|
+
{
|
|
601
|
+
label: "account_number",
|
|
602
|
+
pattern: /\b756\d{10}\b/g,
|
|
603
|
+
validator: ahvOk
|
|
604
|
+
},
|
|
605
|
+
{
|
|
606
|
+
label: "account_number",
|
|
607
|
+
pattern: /\bCHE-\d{3}\.\d{3}\.\d{3}(?:\s+(?:MWST|TVA|IVA))?\b/g,
|
|
608
|
+
validator: vatCheOk
|
|
609
|
+
},
|
|
610
|
+
{
|
|
611
|
+
label: "account_number",
|
|
612
|
+
pattern: /\b(?:\d[\s-]?){15,16}\d\b/g,
|
|
613
|
+
validator: luhnOk
|
|
614
|
+
},
|
|
615
|
+
{
|
|
616
|
+
label: "private_date",
|
|
617
|
+
pattern: /\b(?:\d{1,2}[./]\d{1,2}[./]\d{4}|\d{4}[./-]\d{1,2}[./-]\d{1,2})\b/g
|
|
618
|
+
},
|
|
619
|
+
{
|
|
620
|
+
label: "secret",
|
|
621
|
+
pattern: /\bsk-(?:proj-)?[A-Za-z0-9_\-]{20,}\b/g
|
|
622
|
+
},
|
|
623
|
+
{
|
|
624
|
+
label: "secret",
|
|
625
|
+
pattern: /\bghp_[A-Za-z0-9]{36}\b/g
|
|
626
|
+
},
|
|
627
|
+
{
|
|
628
|
+
label: "secret",
|
|
629
|
+
pattern: /\bAKIA[A-Z0-9]{16}\b/g
|
|
630
|
+
}
|
|
631
|
+
];
|
|
632
|
+
var URL_TRAIL_TRIM = '.,;:!?")>]';
|
|
633
|
+
function trimUrl(text, start, end) {
|
|
634
|
+
while (end > start && URL_TRAIL_TRIM.includes(text[end - 1] ?? ""))
|
|
635
|
+
end--;
|
|
636
|
+
return [start, end];
|
|
637
|
+
}
|
|
638
|
+
function findRegexSpans(text) {
|
|
639
|
+
const candidates = [];
|
|
640
|
+
for (const { label, pattern, validator } of REGEXES) {
|
|
641
|
+
pattern.lastIndex = 0;
|
|
642
|
+
let m;
|
|
643
|
+
while ((m = pattern.exec(text)) !== null) {
|
|
644
|
+
let start = m.index;
|
|
645
|
+
let end = m.index + m[0].length;
|
|
646
|
+
if (label === "private_url")
|
|
647
|
+
[start, end] = trimUrl(text, start, end);
|
|
648
|
+
const value = text.slice(start, end);
|
|
649
|
+
if (!value.trim())
|
|
650
|
+
continue;
|
|
651
|
+
if (validator && !validator(value))
|
|
652
|
+
continue;
|
|
653
|
+
candidates.push({
|
|
654
|
+
start,
|
|
655
|
+
end,
|
|
656
|
+
label,
|
|
657
|
+
text: value,
|
|
658
|
+
score: 1
|
|
659
|
+
});
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
candidates.sort((a, b) => a.start !== b.start ? a.start - b.start : b.end - b.start - (a.end - a.start));
|
|
663
|
+
const out = [];
|
|
664
|
+
let lastEnd = -1;
|
|
665
|
+
for (const sp of candidates) {
|
|
666
|
+
if (sp.start < lastEnd)
|
|
667
|
+
continue;
|
|
668
|
+
out.push(sp);
|
|
669
|
+
lastEnd = sp.end;
|
|
670
|
+
}
|
|
671
|
+
return out;
|
|
672
|
+
}
|
|
673
|
+
function maskSpans(text, spans) {
|
|
674
|
+
if (spans.length === 0)
|
|
675
|
+
return text;
|
|
676
|
+
const chars = [...text];
|
|
677
|
+
for (const sp of spans) {
|
|
678
|
+
for (let i = sp.start;i < sp.end && i < chars.length; i++) {
|
|
679
|
+
chars[i] = " ";
|
|
680
|
+
}
|
|
681
|
+
}
|
|
682
|
+
return chars.join("");
|
|
683
|
+
}
|
|
684
|
+
|
|
685
|
+
class CompositeDetector {
|
|
686
|
+
model;
|
|
687
|
+
constructor(opts) {
|
|
688
|
+
if (!opts || !opts.model) {
|
|
689
|
+
throw new Error("CompositeDetector requires { model: Detector } — pass any object with " + "a .detect(text) method (e.g. new LocalDetector({...})).");
|
|
690
|
+
}
|
|
691
|
+
this.model = opts.model;
|
|
692
|
+
}
|
|
693
|
+
async detect(text, opts) {
|
|
694
|
+
if (!text)
|
|
695
|
+
return [];
|
|
696
|
+
const regexSpans = findRegexSpans(text);
|
|
697
|
+
const masked = maskSpans(text, regexSpans);
|
|
698
|
+
const modelSpans = await this.model.detect(masked, opts);
|
|
699
|
+
const out = [...regexSpans];
|
|
700
|
+
const regexRanges = regexSpans.map((s) => [s.start, s.end]);
|
|
701
|
+
for (const sp of modelSpans) {
|
|
702
|
+
const overlaps = regexRanges.some(([rs, re]) => sp.start < re && rs < sp.end);
|
|
703
|
+
if (overlaps)
|
|
704
|
+
continue;
|
|
705
|
+
out.push(sp);
|
|
706
|
+
}
|
|
707
|
+
out.sort((a, b) => a.start !== b.start ? a.start - b.start : b.end - b.start - (a.end - a.start));
|
|
708
|
+
return out;
|
|
709
|
+
}
|
|
710
|
+
}
|
|
503
711
|
// src/detectors/index.ts
|
|
504
712
|
function defaultDetector() {
|
|
505
713
|
const hasKey = typeof process !== "undefined" && Boolean(process.env?.GHEIM_API_KEY);
|
|
@@ -563,7 +771,9 @@ class Session {
|
|
|
563
771
|
cursor = end;
|
|
564
772
|
}
|
|
565
773
|
out.push(text.slice(cursor));
|
|
566
|
-
|
|
774
|
+
let redacted = out.join("");
|
|
775
|
+
redacted = recoverDuplicateOccurrences(redacted, this.mapping);
|
|
776
|
+
return { redacted, merged };
|
|
567
777
|
}
|
|
568
778
|
restore(text) {
|
|
569
779
|
if (Object.keys(this.mapping).length === 0)
|
|
@@ -619,6 +829,18 @@ function mergeAdjacent(text, spans) {
|
|
|
619
829
|
}
|
|
620
830
|
return out;
|
|
621
831
|
}
|
|
832
|
+
function escapeRegex(s) {
|
|
833
|
+
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
834
|
+
}
|
|
835
|
+
function recoverDuplicateOccurrences(redacted, mapping) {
|
|
836
|
+
const entries = Object.entries(mapping).filter(([, surface]) => surface && surface.length > 0).sort((a, b) => b[1].length - a[1].length);
|
|
837
|
+
let result = redacted;
|
|
838
|
+
for (const [sentinel, surface] of entries) {
|
|
839
|
+
const pattern = new RegExp(`(?<![\\p{L}\\p{N}_])${escapeRegex(surface)}(?![\\p{L}\\p{N}_])`, "gu");
|
|
840
|
+
result = result.replace(pattern, sentinel);
|
|
841
|
+
}
|
|
842
|
+
return result;
|
|
843
|
+
}
|
|
622
844
|
|
|
623
845
|
// src/openai/_base.ts
|
|
624
846
|
function resolveCallSession(body, defaultDet) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../../src/core/session.ts"],"names":[],"mappings":"AAAA,kDAAkD;AAClD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AAGjD;;;gEAGgE;AAChE,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;AAkBtD,uCAAuC;AACvC,MAAM,WAAW,cAAc;IAC7B;;;;;;;;;;OAUG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;CAC1C;AAED,qBAAa,OAAO;IAClB,8DAA8D;IAC9D,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAM;IACrC,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,mBAAmB,CAAkC;gBAEjD,IAAI,GAAE,cAAmB;IAQrC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM;IAiBhD,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GAAG,MAAM;IAIvE;;;;;;;OAOG;IACH,kBAAkB,CAChB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GACpC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,UAAU,EAAE,CAAA;KAAE;
|
|
1
|
+
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../../src/core/session.ts"],"names":[],"mappings":"AAAA,kDAAkD;AAClD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AAGjD;;;gEAGgE;AAChE,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;AAkBtD,uCAAuC;AACvC,MAAM,WAAW,cAAc;IAC7B;;;;;;;;;;OAUG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;CAC1C;AAED,qBAAa,OAAO;IAClB,8DAA8D;IAC9D,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAM;IACrC,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,mBAAmB,CAAkC;gBAEjD,IAAI,GAAE,cAAmB;IAQrC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM;IAiBhD,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GAAG,MAAM;IAIvE;;;;;;;OAOG;IACH,kBAAkB,CAChB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GACpC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,UAAU,EAAE,CAAA;KAAE;IA+B7C,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAK7B,MAAM,IAAI;QAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE;IAI7C,MAAM,CAAC,QAAQ,CACb,GAAG,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,EACxC,IAAI,GAAE,cAAmB,GACxB,OAAO;CAqBX;AAID,8EAA8E;AAC9E,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAC3B,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GACpC,UAAU,EAAE,CAqBd"}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Composite detector: regex front-end + model on the remainder.
|
|
3
|
+
*
|
|
4
|
+
* Production-ready Swiss PII detector that combines two complementary
|
|
5
|
+
* strategies — a port of the Python sibling
|
|
6
|
+
* (`gheim.detectors.composite.CompositeDetector`) with parity-tested
|
|
7
|
+
* regexes and checksum validators.
|
|
8
|
+
*
|
|
9
|
+
* 1. Deterministic regex front-end for structured PII categories where
|
|
10
|
+
* shape is unambiguous and checksums verify identity:
|
|
11
|
+
* - IBAN-CH (mod-97-10), AHV (EAN-13 mod-10), VAT-CHE (mod-11-10),
|
|
12
|
+
* Visa/Mastercard (Luhn)
|
|
13
|
+
* - Email, URL, Swiss phone (+41 / 0041 / 0...), explicit dates
|
|
14
|
+
* (DD.MM.YYYY, YYYY-MM-DD)
|
|
15
|
+
* - API tokens (sk-, ghp_, AKIA...)
|
|
16
|
+
*
|
|
17
|
+
* Spans found here are "trusted" — checksums catch most surface-form
|
|
18
|
+
* imitations; the model is never asked.
|
|
19
|
+
*
|
|
20
|
+
* 2. The fine-tuned model on whatever's left after regex masking.
|
|
21
|
+
* Persons, addresses, and edge-case dates fall here. The model's
|
|
22
|
+
* "single-category mode" failure (where an IBAN/phone-shaped
|
|
23
|
+
* sequence suppresses person/date detection in the same input) is
|
|
24
|
+
* bypassed because the regex already absorbed those tokens — the
|
|
25
|
+
* model sees them as whitespace.
|
|
26
|
+
*
|
|
27
|
+
* Wraps the same `Detector` interface (`detect(text) → Span[]`) as
|
|
28
|
+
* `LocalDetector` so it drops into any code path that already accepts a
|
|
29
|
+
* detector.
|
|
30
|
+
*/
|
|
31
|
+
import type { Detector, Span } from "./base.ts";
|
|
32
|
+
/** Options for {@link CompositeDetector}. */
|
|
33
|
+
export interface CompositeDetectorOptions {
|
|
34
|
+
/** Inner detector — usually a {@link LocalDetector}. Anything with
|
|
35
|
+
* a `detect(text) → Promise<Span[]>` method works. */
|
|
36
|
+
model: Detector;
|
|
37
|
+
}
|
|
38
|
+
/** Regex front-end (with checksum validation) + model on the remainder.
|
|
39
|
+
*
|
|
40
|
+
* @example
|
|
41
|
+
* import { LocalDetector, CompositeDetector } from "gheim";
|
|
42
|
+
* const composite = new CompositeDetector({
|
|
43
|
+
* model: new LocalDetector({ dtype: "q8" }),
|
|
44
|
+
* });
|
|
45
|
+
* const spans = await composite.detect(swissLetter);
|
|
46
|
+
* // → IBAN/phone/email caught by regex (checksum-validated);
|
|
47
|
+
* // person/address/dates caught by the model on the masked remainder.
|
|
48
|
+
*/
|
|
49
|
+
export declare class CompositeDetector implements Detector {
|
|
50
|
+
readonly model: Detector;
|
|
51
|
+
constructor(opts: CompositeDetectorOptions);
|
|
52
|
+
/** Two-pass detection: regex first (with checksum), then model on
|
|
53
|
+
* the masked remainder. Returns a single overlap-resolved span list. */
|
|
54
|
+
detect(text: string, opts?: {
|
|
55
|
+
model?: string;
|
|
56
|
+
}): Promise<Span[]>;
|
|
57
|
+
}
|
|
58
|
+
//# sourceMappingURL=composite.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"composite.d.ts","sourceRoot":"","sources":["../../../src/detectors/composite.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,OAAO,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAiNhD,6CAA6C;AAC7C,MAAM,WAAW,wBAAwB;IACvC;2DACuD;IACvD,KAAK,EAAE,QAAQ,CAAC;CACjB;AAED;;;;;;;;;;GAUG;AACH,qBAAa,iBAAkB,YAAW,QAAQ;IAChD,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC;gBAEb,IAAI,EAAE,wBAAwB;IAU1C;6EACyE;IACnE,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;CAmBvE"}
|
|
@@ -3,6 +3,8 @@ export { DEFAULT_BASE_URL, DEFAULT_MODEL as REMOTE_DEFAULT_MODEL, RemoteDetector
|
|
|
3
3
|
export type { RemoteDetectorOptions } from "./remote.ts";
|
|
4
4
|
export { LocalDetector } from "./local.ts";
|
|
5
5
|
export type { LocalDetectorOptions, LocalDetectorLoadEvent } from "./local.ts";
|
|
6
|
+
export { CompositeDetector } from "./composite.ts";
|
|
7
|
+
export type { CompositeDetectorOptions } from "./composite.ts";
|
|
6
8
|
import type { Detector } from "./base.ts";
|
|
7
9
|
/** Pick a reasonable default: remote if `GHEIM_API_KEY` is set in env, else local. */
|
|
8
10
|
export declare function defaultDetector(): Detector;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/detectors/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,EACL,gBAAgB,EAChB,aAAa,IAAI,oBAAoB,EACrC,cAAc,GACf,MAAM,aAAa,CAAC;AACrB,YAAY,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,YAAY,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/detectors/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,EACL,gBAAgB,EAChB,aAAa,IAAI,oBAAoB,EACrC,cAAc,GACf,MAAM,aAAa,CAAC;AACrB,YAAY,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,YAAY,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAC/E,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACnD,YAAY,EAAE,wBAAwB,EAAE,MAAM,gBAAgB,CAAC;AAE/D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAI1C,sFAAsF;AACtF,wBAAgB,eAAe,IAAI,QAAQ,CAI1C"}
|
package/dist/types/index.d.ts
CHANGED
|
@@ -13,11 +13,11 @@ export { LABEL_TO_TAG, SENTINEL_RE, isPossibleSentinelPrefix, labelTag, parseSen
|
|
|
13
13
|
export type { Sentinel } from "./core/sentinels.ts";
|
|
14
14
|
export { StreamDeanonymizer } from "./core/stream.ts";
|
|
15
15
|
export type { Detector, Span } from "./detectors/base.ts";
|
|
16
|
-
export { DEFAULT_BASE_URL, REMOTE_DEFAULT_MODEL, RemoteDetector, LocalDetector, defaultDetector, } from "./detectors/index.ts";
|
|
17
|
-
export type { RemoteDetectorOptions, LocalDetectorOptions, LocalDetectorLoadEvent, } from "./detectors/index.ts";
|
|
16
|
+
export { DEFAULT_BASE_URL, REMOTE_DEFAULT_MODEL, RemoteDetector, LocalDetector, CompositeDetector, defaultDetector, } from "./detectors/index.ts";
|
|
17
|
+
export type { RemoteDetectorOptions, LocalDetectorOptions, LocalDetectorLoadEvent, CompositeDetectorOptions, } from "./detectors/index.ts";
|
|
18
18
|
export { anonymizeMessages, anonymizeText, deanonymizeStream, deanonymizeText, } from "./plain.ts";
|
|
19
19
|
export type { ChatMessage } from "./plain.ts";
|
|
20
|
-
export declare const VERSION = "0.1.
|
|
20
|
+
export declare const VERSION = "0.1.4";
|
|
21
21
|
export { mergeAdjacent } from "./core/session.ts";
|
|
22
22
|
export type { MergedSpan } from "./core/session.ts";
|
|
23
23
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,EACL,YAAY,EACZ,WAAW,EACX,wBAAwB,EACxB,QAAQ,EACR,aAAa,EACb,cAAc,GACf,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,YAAY,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EACL,gBAAgB,EAChB,oBAAoB,EACpB,cAAc,EACd,aAAa,EACb,eAAe,GAChB,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EACV,qBAAqB,EACrB,oBAAoB,EACpB,sBAAsB,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,EACL,YAAY,EACZ,WAAW,EACX,wBAAwB,EACxB,QAAQ,EACR,aAAa,EACb,cAAc,GACf,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,YAAY,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EACL,gBAAgB,EAChB,oBAAoB,EACpB,cAAc,EACd,aAAa,EACb,iBAAiB,EACjB,eAAe,GAChB,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EACV,qBAAqB,EACrB,oBAAoB,EACpB,sBAAsB,EACtB,wBAAwB,GACzB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,iBAAiB,EACjB,aAAa,EACb,iBAAiB,EACjB,eAAe,GAChB,MAAM,YAAY,CAAC;AACpB,YAAY,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAK9C,eAAO,MAAM,OAAO,UAAU,CAAC;AAE/B,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,YAAY,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "gheim",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.4",
|
|
4
4
|
"description": "PII round-trip for LLM APIs: anonymize before the request, de-anonymize the stream on the way back.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Joel Barmettler <joel.barmettler@gmail.com>",
|