gheim 0.1.2 → 0.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -38,33 +38,38 @@ The core package itself has no runtime dependencies.
38
38
 
39
39
  `LocalDetector` loads a token-classification model via
40
40
  [`@huggingface/transformers`](https://www.npmjs.com/package/@huggingface/transformers)
41
- (transformers.js). Two models are recommended depending on the
42
- deployment context. Both implement the same 33-class BIOES output
43
- schema and are interchangeable.
41
+ (transformers.js). The package's default model is
42
+ [`joelbarmettler/gheim-ch-560m`](https://huggingface.co/joelbarmettler/gheim-ch-560m)
43
+ — a 560M xlm-roberta-large fine-tune optimised for Swiss-market PII
44
+ (strict-span F1 0.916 on Swiss text, see
45
+ [MODEL_CARD.md](https://github.com/joelbarmettlerUZH/gheim/blob/main/MODEL_CARD.md)).
46
+ Any HuggingFace token-classification model that emits the same 33-class
47
+ BIOES schema can be substituted via `model:`.
44
48
 
45
49
  | Model | Best for | Parameters | Notes |
46
50
  |---|---|---:|---|
47
- | [`joelbarmettler/gheim-ch-560m`](https://huggingface.co/joelbarmettler/gheim-ch-560m) | Swiss-market text (de_CH, fr_CH, it_CH, rm, en) with CH-format account numbers (IBAN, AHV, VAT-CHE) | 560M | Apache 2.0. Test F1 = 0.916 on Swiss text. ONNX export ships in the `onnx/` subfolder. |
51
+ | [`joelbarmettler/gheim-ch-560m`](https://huggingface.co/joelbarmettler/gheim-ch-560m) **(default)** | Swiss-market text (de_CH, fr_CH, it_CH, rm, en) with CH-format account numbers (IBAN, AHV, VAT-CHE) | 560M | Apache 2.0. Test F1 0.916. Hub repo ships `onnx/model_quantized.onnx` only — load with `dtype: "q8"`. |
48
52
  | [`openai/privacy-filter`](https://huggingface.co/openai/privacy-filter) | English-first or general use, long-context (up to 128k tokens) | 1.4B (50M active, MoE) | Apache 2.0. Wider language coverage, larger weights. |
49
53
 
50
54
  ```ts
51
55
  import { LocalDetector } from "gheim";
52
56
 
53
- // Recommended for Swiss-market text:
54
- const det = new LocalDetector({ model: "joelbarmettler/gheim-ch-560m" });
57
+ // Default — Swiss-tuned, q8 ONNX. `device: "auto"` probes WebGPU in
58
+ // the browser and falls back to WASM; in Node it's WASM directly.
59
+ const det = new LocalDetector({ dtype: "q8" });
55
60
 
56
61
  // Alternative for English or general use:
57
62
  const detEn = new LocalDetector({ model: "openai/privacy-filter" });
58
63
  ```
59
64
 
60
- The package default `model` is `openai/privacy-filter`. Pass `model`
61
- explicitly to opt into the Swiss model.
62
-
63
65
  ## Drop-in OpenAI client
64
66
 
65
67
  ```ts
66
68
  import { OpenAI } from "gheim/openai";
67
69
 
70
+ // Same constructor shape as `openai`. `apiKey`, `baseURL`, etc. work
71
+ // at the top level (forwarded to the inner client). gheim-specific
72
+ // keys: `gheimDetector`, `gheimStrict`, `openaiClient`, `clientOptions`.
68
73
  const client = new OpenAI();
69
74
  const r = await client.chat.completions.create({
70
75
  model: "gpt-4o",
@@ -74,6 +79,17 @@ const r = await client.chat.completions.create({
74
79
  // OpenAI only ever saw "<PERSON_1>".
75
80
  ```
76
81
 
82
+ Custom endpoint or key (e.g. OpenRouter, local vLLM):
83
+
84
+ ```ts
85
+ import { OpenAI } from "gheim/openai";
86
+
87
+ const client = new OpenAI({
88
+ apiKey: process.env.OPENROUTER_API_KEY,
89
+ baseURL: "https://openrouter.ai/api/v1",
90
+ });
91
+ ```
92
+
77
93
  Streaming:
78
94
 
79
95
  ```ts
@@ -104,9 +120,9 @@ All non-chat OpenAI resources are on `client.raw` once `await client.ready()` re
104
120
  import { Session, LocalDetector, anonymizeText, deanonymizeText } from "gheim";
105
121
 
106
122
  const session = new Session();
107
- (session as any).detector = new LocalDetector({
108
- model: "joelbarmettler/gheim-ch-560m",
109
- });
123
+ (session as any).detector = new LocalDetector({ dtype: "q8" });
124
+ // ^ defaults to joelbarmettler/gheim-ch-560m
125
+
110
126
  const clean = await anonymizeText("Hi, my name is Joel", session);
111
127
  // ... call any LLM with clean ...
112
128
  const final = deanonymizeText(responseText, session);
@@ -155,12 +171,18 @@ client.raw.beta.assistants.create(...); // always works regardless of strict mo
155
171
  ```ts
156
172
  import { LocalDetector, RemoteDetector, defaultDetector } from "gheim";
157
173
 
158
- // Local: via @huggingface/transformers, on Node and Bun.
174
+ // Local: via @huggingface/transformers, on Node, Bun, and browsers.
175
+ // `device: "auto"` (default) probes navigator.gpu in the browser and
176
+ // falls back to WASM if WebGPU isn't usable; in Node it's WASM only.
159
177
  const local = new LocalDetector({
160
- model: "joelbarmettler/gheim-ch-560m",
161
178
  device: "auto",
162
- dtype: "fp32",
179
+ dtype: "q8",
180
+ onProgress: (e) => {
181
+ if (e.fraction != null) console.log(`loading: ${(e.fraction * 100) | 0}%`);
182
+ },
163
183
  });
184
+ await local.load();
185
+ console.log("backend in use:", local.actualDevice); // "webgpu" or "wasm"
164
186
 
165
187
  // Remote: against your own gheim-server or api.gheim.ch.
166
188
  const remote = new RemoteDetector({
@@ -174,10 +196,9 @@ const auto = defaultDetector();
174
196
 
175
197
  ## Runtime support
176
198
 
177
- Tested on Node 18+ and Bun 1.1+ with `device: "cpu"` (the default). The
178
- underlying transformers.js library also supports `device: "webgpu"` in
179
- modern browsers; the WebGPU path is not exercised in CI and should be
180
- considered best-effort.
199
+ Tested on Node 18+, Bun 1.1+, and modern browsers (Chrome 113+, Edge,
200
+ Safari 17+, Firefox via flag). Browser builds use WebGPU when available
201
+ and fall back to WebAssembly automatically.
181
202
 
182
203
  For Node servers, installing `onnxruntime-node` alongside
183
204
  `@huggingface/transformers` enables the native ONNX backend, which is
@@ -57,7 +57,8 @@ __export(exports_src, {
57
57
  REMOTE_DEFAULT_MODEL: () => DEFAULT_MODEL,
58
58
  LocalDetector: () => LocalDetector,
59
59
  LABEL_TO_TAG: () => LABEL_TO_TAG,
60
- DEFAULT_BASE_URL: () => DEFAULT_BASE_URL
60
+ DEFAULT_BASE_URL: () => DEFAULT_BASE_URL,
61
+ CompositeDetector: () => CompositeDetector
61
62
  });
62
63
  module.exports = __toCommonJS(exports_src);
63
64
 
@@ -154,7 +155,9 @@ class Session {
154
155
  cursor = end;
155
156
  }
156
157
  out.push(text.slice(cursor));
157
- return { redacted: out.join(""), merged };
158
+ let redacted = out.join("");
159
+ redacted = recoverDuplicateOccurrences(redacted, this.mapping);
160
+ return { redacted, merged };
158
161
  }
159
162
  restore(text) {
160
163
  if (Object.keys(this.mapping).length === 0)
@@ -210,6 +213,18 @@ function mergeAdjacent(text, spans) {
210
213
  }
211
214
  return out;
212
215
  }
216
+ function escapeRegex(s) {
217
+ return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
218
+ }
219
+ function recoverDuplicateOccurrences(redacted, mapping) {
220
+ const entries = Object.entries(mapping).filter(([, surface]) => surface && surface.length > 0).sort((a, b) => b[1].length - a[1].length);
221
+ let result = redacted;
222
+ for (const [sentinel, surface] of entries) {
223
+ const pattern = new RegExp(`(?<![\\p{L}\\p{N}_])${escapeRegex(surface)}(?![\\p{L}\\p{N}_])`, "gu");
224
+ result = result.replace(pattern, sentinel);
225
+ }
226
+ return result;
227
+ }
213
228
  // src/core/stream.ts
214
229
  var SENTINEL_AT = /<([A-Z][A-Z0-9_]*)_(\d+)>/y;
215
230
  var MAX_SENTINEL_LEN = 64;
@@ -619,6 +634,214 @@ async function loadTransformersJs() {
619
634
  throw new Error("LocalDetector requires `@huggingface/transformers`. Install with: bun add @huggingface/transformers");
620
635
  }
621
636
  }
637
+ // src/detectors/composite.ts
638
+ function luhnOk(s) {
639
+ const digits = [];
640
+ for (const c of s) {
641
+ if (c >= "0" && c <= "9")
642
+ digits.push(Number(c));
643
+ }
644
+ if (digits.length < 13 || digits.length > 19)
645
+ return false;
646
+ let total = 0;
647
+ for (let i = 0;i < digits.length; i++) {
648
+ let d = digits[digits.length - 1 - i] ?? 0;
649
+ if (i % 2 === 1) {
650
+ d *= 2;
651
+ if (d > 9)
652
+ d -= 9;
653
+ }
654
+ total += d;
655
+ }
656
+ return total % 10 === 0;
657
+ }
658
+ function ibanChOk(raw) {
659
+ const s = raw.replace(/[\s-]/g, "");
660
+ if (s.length !== 21 || !s.startsWith("CH") || !/^\d+$/.test(s.slice(2))) {
661
+ return false;
662
+ }
663
+ const rearranged = s.slice(4) + s.slice(0, 4);
664
+ let numeric = "";
665
+ for (const c of rearranged) {
666
+ numeric += c >= "A" && c <= "Z" ? String(c.charCodeAt(0) - 55) : c;
667
+ }
668
+ let rem = 0;
669
+ for (const ch of numeric) {
670
+ rem = (rem * 10 + Number(ch)) % 97;
671
+ }
672
+ return rem === 1;
673
+ }
674
+ function ahvOk(s) {
675
+ const digits = [];
676
+ for (const c of s) {
677
+ if (c >= "0" && c <= "9")
678
+ digits.push(Number(c));
679
+ }
680
+ if (digits.length !== 13)
681
+ return false;
682
+ if (digits[0] !== 7 || digits[1] !== 5 || digits[2] !== 6)
683
+ return false;
684
+ let total = 0;
685
+ for (let i = 0;i < 12; i++) {
686
+ total += (digits[i] ?? 0) * (i % 2 === 0 ? 1 : 3);
687
+ }
688
+ const check = (10 - total % 10) % 10;
689
+ return digits[12] === check;
690
+ }
691
+ function vatCheOk(s) {
692
+ const digits = [];
693
+ for (const c of s) {
694
+ if (c >= "0" && c <= "9")
695
+ digits.push(Number(c));
696
+ }
697
+ if (digits.length < 9)
698
+ return false;
699
+ const weights = [5, 4, 3, 2, 7, 6, 5, 4];
700
+ let total = 0;
701
+ for (let i = 0;i < 8; i++) {
702
+ total += (digits[i] ?? 0) * (weights[i] ?? 0);
703
+ }
704
+ let expected = 11 - total % 11;
705
+ if (expected === 11)
706
+ expected = 0;
707
+ else if (expected === 10)
708
+ return false;
709
+ return digits[8] === expected;
710
+ }
711
+ var REGEXES = [
712
+ {
713
+ label: "private_email",
714
+ pattern: /\b[\w._%+\-]+@[\w.\-]+\.[A-Za-z]{2,}\b/g
715
+ },
716
+ {
717
+ label: "private_url",
718
+ pattern: /https?:\/\/[^\s)>\]]+/g
719
+ },
720
+ {
721
+ label: "private_phone",
722
+ pattern: /(?:\+41|0041|0)\s?(?:\(0\)\s?)?\d{2}[\s\-/.]?\d{3}[\s\-/.]?\d{2}[\s\-/.]?\d{2}\b/g
723
+ },
724
+ {
725
+ label: "account_number",
726
+ pattern: /\bCH\d{2}(?:[\s-]?\d){17}\b/g,
727
+ validator: ibanChOk
728
+ },
729
+ {
730
+ label: "account_number",
731
+ pattern: /\b756\.\d{4}\.\d{4}\.\d{2}\b/g,
732
+ validator: ahvOk
733
+ },
734
+ {
735
+ label: "account_number",
736
+ pattern: /\b756\d{10}\b/g,
737
+ validator: ahvOk
738
+ },
739
+ {
740
+ label: "account_number",
741
+ pattern: /\bCHE-\d{3}\.\d{3}\.\d{3}(?:\s+(?:MWST|TVA|IVA))?\b/g,
742
+ validator: vatCheOk
743
+ },
744
+ {
745
+ label: "account_number",
746
+ pattern: /\b(?:\d[\s-]?){15,16}\d\b/g,
747
+ validator: luhnOk
748
+ },
749
+ {
750
+ label: "private_date",
751
+ pattern: /\b(?:\d{1,2}[./]\d{1,2}[./]\d{4}|\d{4}[./-]\d{1,2}[./-]\d{1,2})\b/g
752
+ },
753
+ {
754
+ label: "secret",
755
+ pattern: /\bsk-(?:proj-)?[A-Za-z0-9_\-]{20,}\b/g
756
+ },
757
+ {
758
+ label: "secret",
759
+ pattern: /\bghp_[A-Za-z0-9]{36}\b/g
760
+ },
761
+ {
762
+ label: "secret",
763
+ pattern: /\bAKIA[A-Z0-9]{16}\b/g
764
+ }
765
+ ];
766
+ var URL_TRAIL_TRIM = '.,;:!?")>]';
767
+ function trimUrl(text, start, end) {
768
+ while (end > start && URL_TRAIL_TRIM.includes(text[end - 1] ?? ""))
769
+ end--;
770
+ return [start, end];
771
+ }
772
+ function findRegexSpans(text) {
773
+ const candidates = [];
774
+ for (const { label, pattern, validator } of REGEXES) {
775
+ pattern.lastIndex = 0;
776
+ let m;
777
+ while ((m = pattern.exec(text)) !== null) {
778
+ let start = m.index;
779
+ let end = m.index + m[0].length;
780
+ if (label === "private_url")
781
+ [start, end] = trimUrl(text, start, end);
782
+ const value = text.slice(start, end);
783
+ if (!value.trim())
784
+ continue;
785
+ if (validator && !validator(value))
786
+ continue;
787
+ candidates.push({
788
+ start,
789
+ end,
790
+ label,
791
+ text: value,
792
+ score: 1
793
+ });
794
+ }
795
+ }
796
+ candidates.sort((a, b) => a.start !== b.start ? a.start - b.start : b.end - b.start - (a.end - a.start));
797
+ const out = [];
798
+ let lastEnd = -1;
799
+ for (const sp of candidates) {
800
+ if (sp.start < lastEnd)
801
+ continue;
802
+ out.push(sp);
803
+ lastEnd = sp.end;
804
+ }
805
+ return out;
806
+ }
807
+ function maskSpans(text, spans) {
808
+ if (spans.length === 0)
809
+ return text;
810
+ const chars = [...text];
811
+ for (const sp of spans) {
812
+ for (let i = sp.start;i < sp.end && i < chars.length; i++) {
813
+ chars[i] = " ";
814
+ }
815
+ }
816
+ return chars.join("");
817
+ }
818
+
819
+ class CompositeDetector {
820
+ model;
821
+ constructor(opts) {
822
+ if (!opts || !opts.model) {
823
+ throw new Error("CompositeDetector requires { model: Detector } — pass any object with " + "a .detect(text) method (e.g. new LocalDetector({...})).");
824
+ }
825
+ this.model = opts.model;
826
+ }
827
+ async detect(text, opts) {
828
+ if (!text)
829
+ return [];
830
+ const regexSpans = findRegexSpans(text);
831
+ const masked = maskSpans(text, regexSpans);
832
+ const modelSpans = await this.model.detect(masked, opts);
833
+ const out = [...regexSpans];
834
+ const regexRanges = regexSpans.map((s) => [s.start, s.end]);
835
+ for (const sp of modelSpans) {
836
+ const overlaps = regexRanges.some(([rs, re]) => sp.start < re && rs < sp.end);
837
+ if (overlaps)
838
+ continue;
839
+ out.push(sp);
840
+ }
841
+ out.sort((a, b) => a.start !== b.start ? a.start - b.start : b.end - b.start - (a.end - a.start));
842
+ return out;
843
+ }
844
+ }
622
845
  // src/detectors/index.ts
623
846
  function defaultDetector() {
624
847
  const hasKey = typeof process !== "undefined" && Boolean(process.env?.GHEIM_API_KEY);
@@ -673,4 +896,4 @@ async function* deanonymizeStream(chunks, session) {
673
896
  }
674
897
 
675
898
  // src/index.ts
676
- var VERSION = "0.1.2";
899
+ var VERSION = "0.1.4";
@@ -543,6 +543,214 @@ async function loadTransformersJs() {
543
543
  throw new Error("LocalDetector requires `@huggingface/transformers`. Install with: bun add @huggingface/transformers");
544
544
  }
545
545
  }
546
+ // src/detectors/composite.ts
547
+ function luhnOk(s) {
548
+ const digits = [];
549
+ for (const c of s) {
550
+ if (c >= "0" && c <= "9")
551
+ digits.push(Number(c));
552
+ }
553
+ if (digits.length < 13 || digits.length > 19)
554
+ return false;
555
+ let total = 0;
556
+ for (let i = 0;i < digits.length; i++) {
557
+ let d = digits[digits.length - 1 - i] ?? 0;
558
+ if (i % 2 === 1) {
559
+ d *= 2;
560
+ if (d > 9)
561
+ d -= 9;
562
+ }
563
+ total += d;
564
+ }
565
+ return total % 10 === 0;
566
+ }
567
+ function ibanChOk(raw) {
568
+ const s = raw.replace(/[\s-]/g, "");
569
+ if (s.length !== 21 || !s.startsWith("CH") || !/^\d+$/.test(s.slice(2))) {
570
+ return false;
571
+ }
572
+ const rearranged = s.slice(4) + s.slice(0, 4);
573
+ let numeric = "";
574
+ for (const c of rearranged) {
575
+ numeric += c >= "A" && c <= "Z" ? String(c.charCodeAt(0) - 55) : c;
576
+ }
577
+ let rem = 0;
578
+ for (const ch of numeric) {
579
+ rem = (rem * 10 + Number(ch)) % 97;
580
+ }
581
+ return rem === 1;
582
+ }
583
+ function ahvOk(s) {
584
+ const digits = [];
585
+ for (const c of s) {
586
+ if (c >= "0" && c <= "9")
587
+ digits.push(Number(c));
588
+ }
589
+ if (digits.length !== 13)
590
+ return false;
591
+ if (digits[0] !== 7 || digits[1] !== 5 || digits[2] !== 6)
592
+ return false;
593
+ let total = 0;
594
+ for (let i = 0;i < 12; i++) {
595
+ total += (digits[i] ?? 0) * (i % 2 === 0 ? 1 : 3);
596
+ }
597
+ const check = (10 - total % 10) % 10;
598
+ return digits[12] === check;
599
+ }
600
+ function vatCheOk(s) {
601
+ const digits = [];
602
+ for (const c of s) {
603
+ if (c >= "0" && c <= "9")
604
+ digits.push(Number(c));
605
+ }
606
+ if (digits.length < 9)
607
+ return false;
608
+ const weights = [5, 4, 3, 2, 7, 6, 5, 4];
609
+ let total = 0;
610
+ for (let i = 0;i < 8; i++) {
611
+ total += (digits[i] ?? 0) * (weights[i] ?? 0);
612
+ }
613
+ let expected = 11 - total % 11;
614
+ if (expected === 11)
615
+ expected = 0;
616
+ else if (expected === 10)
617
+ return false;
618
+ return digits[8] === expected;
619
+ }
620
+ var REGEXES = [
621
+ {
622
+ label: "private_email",
623
+ pattern: /\b[\w._%+\-]+@[\w.\-]+\.[A-Za-z]{2,}\b/g
624
+ },
625
+ {
626
+ label: "private_url",
627
+ pattern: /https?:\/\/[^\s)>\]]+/g
628
+ },
629
+ {
630
+ label: "private_phone",
631
+ pattern: /(?:\+41|0041|0)\s?(?:\(0\)\s?)?\d{2}[\s\-/.]?\d{3}[\s\-/.]?\d{2}[\s\-/.]?\d{2}\b/g
632
+ },
633
+ {
634
+ label: "account_number",
635
+ pattern: /\bCH\d{2}(?:[\s-]?\d){17}\b/g,
636
+ validator: ibanChOk
637
+ },
638
+ {
639
+ label: "account_number",
640
+ pattern: /\b756\.\d{4}\.\d{4}\.\d{2}\b/g,
641
+ validator: ahvOk
642
+ },
643
+ {
644
+ label: "account_number",
645
+ pattern: /\b756\d{10}\b/g,
646
+ validator: ahvOk
647
+ },
648
+ {
649
+ label: "account_number",
650
+ pattern: /\bCHE-\d{3}\.\d{3}\.\d{3}(?:\s+(?:MWST|TVA|IVA))?\b/g,
651
+ validator: vatCheOk
652
+ },
653
+ {
654
+ label: "account_number",
655
+ pattern: /\b(?:\d[\s-]?){15,16}\d\b/g,
656
+ validator: luhnOk
657
+ },
658
+ {
659
+ label: "private_date",
660
+ pattern: /\b(?:\d{1,2}[./]\d{1,2}[./]\d{4}|\d{4}[./-]\d{1,2}[./-]\d{1,2})\b/g
661
+ },
662
+ {
663
+ label: "secret",
664
+ pattern: /\bsk-(?:proj-)?[A-Za-z0-9_\-]{20,}\b/g
665
+ },
666
+ {
667
+ label: "secret",
668
+ pattern: /\bghp_[A-Za-z0-9]{36}\b/g
669
+ },
670
+ {
671
+ label: "secret",
672
+ pattern: /\bAKIA[A-Z0-9]{16}\b/g
673
+ }
674
+ ];
675
+ var URL_TRAIL_TRIM = '.,;:!?")>]';
676
+ function trimUrl(text, start, end) {
677
+ while (end > start && URL_TRAIL_TRIM.includes(text[end - 1] ?? ""))
678
+ end--;
679
+ return [start, end];
680
+ }
681
+ function findRegexSpans(text) {
682
+ const candidates = [];
683
+ for (const { label, pattern, validator } of REGEXES) {
684
+ pattern.lastIndex = 0;
685
+ let m;
686
+ while ((m = pattern.exec(text)) !== null) {
687
+ let start = m.index;
688
+ let end = m.index + m[0].length;
689
+ if (label === "private_url")
690
+ [start, end] = trimUrl(text, start, end);
691
+ const value = text.slice(start, end);
692
+ if (!value.trim())
693
+ continue;
694
+ if (validator && !validator(value))
695
+ continue;
696
+ candidates.push({
697
+ start,
698
+ end,
699
+ label,
700
+ text: value,
701
+ score: 1
702
+ });
703
+ }
704
+ }
705
+ candidates.sort((a, b) => a.start !== b.start ? a.start - b.start : b.end - b.start - (a.end - a.start));
706
+ const out = [];
707
+ let lastEnd = -1;
708
+ for (const sp of candidates) {
709
+ if (sp.start < lastEnd)
710
+ continue;
711
+ out.push(sp);
712
+ lastEnd = sp.end;
713
+ }
714
+ return out;
715
+ }
716
+ function maskSpans(text, spans) {
717
+ if (spans.length === 0)
718
+ return text;
719
+ const chars = [...text];
720
+ for (const sp of spans) {
721
+ for (let i = sp.start;i < sp.end && i < chars.length; i++) {
722
+ chars[i] = " ";
723
+ }
724
+ }
725
+ return chars.join("");
726
+ }
727
+
728
+ class CompositeDetector {
729
+ model;
730
+ constructor(opts) {
731
+ if (!opts || !opts.model) {
732
+ throw new Error("CompositeDetector requires { model: Detector } — pass any object with " + "a .detect(text) method (e.g. new LocalDetector({...})).");
733
+ }
734
+ this.model = opts.model;
735
+ }
736
+ async detect(text, opts) {
737
+ if (!text)
738
+ return [];
739
+ const regexSpans = findRegexSpans(text);
740
+ const masked = maskSpans(text, regexSpans);
741
+ const modelSpans = await this.model.detect(masked, opts);
742
+ const out = [...regexSpans];
743
+ const regexRanges = regexSpans.map((s) => [s.start, s.end]);
744
+ for (const sp of modelSpans) {
745
+ const overlaps = regexRanges.some(([rs, re]) => sp.start < re && rs < sp.end);
746
+ if (overlaps)
747
+ continue;
748
+ out.push(sp);
749
+ }
750
+ out.sort((a, b) => a.start !== b.start ? a.start - b.start : b.end - b.start - (a.end - a.start));
751
+ return out;
752
+ }
753
+ }
546
754
  // src/detectors/index.ts
547
755
  function defaultDetector() {
548
756
  const hasKey = typeof process !== "undefined" && Boolean(process.env?.GHEIM_API_KEY);
@@ -606,7 +814,9 @@ class Session {
606
814
  cursor = end;
607
815
  }
608
816
  out.push(text.slice(cursor));
609
- return { redacted: out.join(""), merged };
817
+ let redacted = out.join("");
818
+ redacted = recoverDuplicateOccurrences(redacted, this.mapping);
819
+ return { redacted, merged };
610
820
  }
611
821
  restore(text) {
612
822
  if (Object.keys(this.mapping).length === 0)
@@ -662,6 +872,18 @@ function mergeAdjacent(text, spans) {
662
872
  }
663
873
  return out;
664
874
  }
875
+ function escapeRegex(s) {
876
+ return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
877
+ }
878
+ function recoverDuplicateOccurrences(redacted, mapping) {
879
+ const entries = Object.entries(mapping).filter(([, surface]) => surface && surface.length > 0).sort((a, b) => b[1].length - a[1].length);
880
+ let result = redacted;
881
+ for (const [sentinel, surface] of entries) {
882
+ const pattern = new RegExp(`(?<![\\p{L}\\p{N}_])${escapeRegex(surface)}(?![\\p{L}\\p{N}_])`, "gu");
883
+ result = result.replace(pattern, sentinel);
884
+ }
885
+ return result;
886
+ }
665
887
 
666
888
  // src/openai/_base.ts
667
889
  function resolveCallSession(body, defaultDet) {
package/dist/esm/index.js CHANGED
@@ -99,7 +99,9 @@ class Session {
99
99
  cursor = end;
100
100
  }
101
101
  out.push(text.slice(cursor));
102
- return { redacted: out.join(""), merged };
102
+ let redacted = out.join("");
103
+ redacted = recoverDuplicateOccurrences(redacted, this.mapping);
104
+ return { redacted, merged };
103
105
  }
104
106
  restore(text) {
105
107
  if (Object.keys(this.mapping).length === 0)
@@ -155,6 +157,18 @@ function mergeAdjacent(text, spans) {
155
157
  }
156
158
  return out;
157
159
  }
160
+ function escapeRegex(s) {
161
+ return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
162
+ }
163
+ function recoverDuplicateOccurrences(redacted, mapping) {
164
+ const entries = Object.entries(mapping).filter(([, surface]) => surface && surface.length > 0).sort((a, b) => b[1].length - a[1].length);
165
+ let result = redacted;
166
+ for (const [sentinel, surface] of entries) {
167
+ const pattern = new RegExp(`(?<![\\p{L}\\p{N}_])${escapeRegex(surface)}(?![\\p{L}\\p{N}_])`, "gu");
168
+ result = result.replace(pattern, sentinel);
169
+ }
170
+ return result;
171
+ }
158
172
  // src/core/stream.ts
159
173
  var SENTINEL_AT = /<([A-Z][A-Z0-9_]*)_(\d+)>/y;
160
174
  var MAX_SENTINEL_LEN = 64;
@@ -564,6 +578,214 @@ async function loadTransformersJs() {
564
578
  throw new Error("LocalDetector requires `@huggingface/transformers`. Install with: bun add @huggingface/transformers");
565
579
  }
566
580
  }
581
+ // src/detectors/composite.ts
582
+ function luhnOk(s) {
583
+ const digits = [];
584
+ for (const c of s) {
585
+ if (c >= "0" && c <= "9")
586
+ digits.push(Number(c));
587
+ }
588
+ if (digits.length < 13 || digits.length > 19)
589
+ return false;
590
+ let total = 0;
591
+ for (let i = 0;i < digits.length; i++) {
592
+ let d = digits[digits.length - 1 - i] ?? 0;
593
+ if (i % 2 === 1) {
594
+ d *= 2;
595
+ if (d > 9)
596
+ d -= 9;
597
+ }
598
+ total += d;
599
+ }
600
+ return total % 10 === 0;
601
+ }
602
+ function ibanChOk(raw) {
603
+ const s = raw.replace(/[\s-]/g, "");
604
+ if (s.length !== 21 || !s.startsWith("CH") || !/^\d+$/.test(s.slice(2))) {
605
+ return false;
606
+ }
607
+ const rearranged = s.slice(4) + s.slice(0, 4);
608
+ let numeric = "";
609
+ for (const c of rearranged) {
610
+ numeric += c >= "A" && c <= "Z" ? String(c.charCodeAt(0) - 55) : c;
611
+ }
612
+ let rem = 0;
613
+ for (const ch of numeric) {
614
+ rem = (rem * 10 + Number(ch)) % 97;
615
+ }
616
+ return rem === 1;
617
+ }
618
+ function ahvOk(s) {
619
+ const digits = [];
620
+ for (const c of s) {
621
+ if (c >= "0" && c <= "9")
622
+ digits.push(Number(c));
623
+ }
624
+ if (digits.length !== 13)
625
+ return false;
626
+ if (digits[0] !== 7 || digits[1] !== 5 || digits[2] !== 6)
627
+ return false;
628
+ let total = 0;
629
+ for (let i = 0;i < 12; i++) {
630
+ total += (digits[i] ?? 0) * (i % 2 === 0 ? 1 : 3);
631
+ }
632
+ const check = (10 - total % 10) % 10;
633
+ return digits[12] === check;
634
+ }
635
+ function vatCheOk(s) {
636
+ const digits = [];
637
+ for (const c of s) {
638
+ if (c >= "0" && c <= "9")
639
+ digits.push(Number(c));
640
+ }
641
+ if (digits.length < 9)
642
+ return false;
643
+ const weights = [5, 4, 3, 2, 7, 6, 5, 4];
644
+ let total = 0;
645
+ for (let i = 0;i < 8; i++) {
646
+ total += (digits[i] ?? 0) * (weights[i] ?? 0);
647
+ }
648
+ let expected = 11 - total % 11;
649
+ if (expected === 11)
650
+ expected = 0;
651
+ else if (expected === 10)
652
+ return false;
653
+ return digits[8] === expected;
654
+ }
655
+ var REGEXES = [
656
+ {
657
+ label: "private_email",
658
+ pattern: /\b[\w._%+\-]+@[\w.\-]+\.[A-Za-z]{2,}\b/g
659
+ },
660
+ {
661
+ label: "private_url",
662
+ pattern: /https?:\/\/[^\s)>\]]+/g
663
+ },
664
+ {
665
+ label: "private_phone",
666
+ pattern: /(?:\+41|0041|0)\s?(?:\(0\)\s?)?\d{2}[\s\-/.]?\d{3}[\s\-/.]?\d{2}[\s\-/.]?\d{2}\b/g
667
+ },
668
+ {
669
+ label: "account_number",
670
+ pattern: /\bCH\d{2}(?:[\s-]?\d){17}\b/g,
671
+ validator: ibanChOk
672
+ },
673
+ {
674
+ label: "account_number",
675
+ pattern: /\b756\.\d{4}\.\d{4}\.\d{2}\b/g,
676
+ validator: ahvOk
677
+ },
678
+ {
679
+ label: "account_number",
680
+ pattern: /\b756\d{10}\b/g,
681
+ validator: ahvOk
682
+ },
683
+ {
684
+ label: "account_number",
685
+ pattern: /\bCHE-\d{3}\.\d{3}\.\d{3}(?:\s+(?:MWST|TVA|IVA))?\b/g,
686
+ validator: vatCheOk
687
+ },
688
+ {
689
+ label: "account_number",
690
+ pattern: /\b(?:\d[\s-]?){15,16}\d\b/g,
691
+ validator: luhnOk
692
+ },
693
+ {
694
+ label: "private_date",
695
+ pattern: /\b(?:\d{1,2}[./]\d{1,2}[./]\d{4}|\d{4}[./-]\d{1,2}[./-]\d{1,2})\b/g
696
+ },
697
+ {
698
+ label: "secret",
699
+ pattern: /\bsk-(?:proj-)?[A-Za-z0-9_\-]{20,}\b/g
700
+ },
701
+ {
702
+ label: "secret",
703
+ pattern: /\bghp_[A-Za-z0-9]{36}\b/g
704
+ },
705
+ {
706
+ label: "secret",
707
+ pattern: /\bAKIA[A-Z0-9]{16}\b/g
708
+ }
709
+ ];
710
+ var URL_TRAIL_TRIM = '.,;:!?")>]';
711
+ function trimUrl(text, start, end) {
712
+ while (end > start && URL_TRAIL_TRIM.includes(text[end - 1] ?? ""))
713
+ end--;
714
+ return [start, end];
715
+ }
716
+ function findRegexSpans(text) {
717
+ const candidates = [];
718
+ for (const { label, pattern, validator } of REGEXES) {
719
+ pattern.lastIndex = 0;
720
+ let m;
721
+ while ((m = pattern.exec(text)) !== null) {
722
+ let start = m.index;
723
+ let end = m.index + m[0].length;
724
+ if (label === "private_url")
725
+ [start, end] = trimUrl(text, start, end);
726
+ const value = text.slice(start, end);
727
+ if (!value.trim())
728
+ continue;
729
+ if (validator && !validator(value))
730
+ continue;
731
+ candidates.push({
732
+ start,
733
+ end,
734
+ label,
735
+ text: value,
736
+ score: 1
737
+ });
738
+ }
739
+ }
740
+ candidates.sort((a, b) => a.start !== b.start ? a.start - b.start : b.end - b.start - (a.end - a.start));
741
+ const out = [];
742
+ let lastEnd = -1;
743
+ for (const sp of candidates) {
744
+ if (sp.start < lastEnd)
745
+ continue;
746
+ out.push(sp);
747
+ lastEnd = sp.end;
748
+ }
749
+ return out;
750
+ }
751
+ function maskSpans(text, spans) {
752
+ if (spans.length === 0)
753
+ return text;
754
+ const chars = [...text];
755
+ for (const sp of spans) {
756
+ for (let i = sp.start;i < sp.end && i < chars.length; i++) {
757
+ chars[i] = " ";
758
+ }
759
+ }
760
+ return chars.join("");
761
+ }
762
+
763
+ class CompositeDetector {
764
+ model;
765
+ constructor(opts) {
766
+ if (!opts || !opts.model) {
767
+ throw new Error("CompositeDetector requires { model: Detector } — pass any object with " + "a .detect(text) method (e.g. new LocalDetector({...})).");
768
+ }
769
+ this.model = opts.model;
770
+ }
771
+ async detect(text, opts) {
772
+ if (!text)
773
+ return [];
774
+ const regexSpans = findRegexSpans(text);
775
+ const masked = maskSpans(text, regexSpans);
776
+ const modelSpans = await this.model.detect(masked, opts);
777
+ const out = [...regexSpans];
778
+ const regexRanges = regexSpans.map((s) => [s.start, s.end]);
779
+ for (const sp of modelSpans) {
780
+ const overlaps = regexRanges.some(([rs, re]) => sp.start < re && rs < sp.end);
781
+ if (overlaps)
782
+ continue;
783
+ out.push(sp);
784
+ }
785
+ out.sort((a, b) => a.start !== b.start ? a.start - b.start : b.end - b.start - (a.end - a.start));
786
+ return out;
787
+ }
788
+ }
567
789
  // src/detectors/index.ts
568
790
  function defaultDetector() {
569
791
  const hasKey = typeof process !== "undefined" && Boolean(process.env?.GHEIM_API_KEY);
@@ -618,7 +840,7 @@ async function* deanonymizeStream(chunks, session) {
618
840
  }
619
841
 
620
842
  // src/index.ts
621
- var VERSION = "0.1.2";
843
+ var VERSION = "0.1.4";
622
844
  export {
623
845
  renderSentinel,
624
846
  parseSentinel,
@@ -638,5 +860,6 @@ export {
638
860
  DEFAULT_MODEL as REMOTE_DEFAULT_MODEL,
639
861
  LocalDetector,
640
862
  LABEL_TO_TAG,
641
- DEFAULT_BASE_URL
863
+ DEFAULT_BASE_URL,
864
+ CompositeDetector
642
865
  };
@@ -500,6 +500,214 @@ async function loadTransformersJs() {
500
500
  throw new Error("LocalDetector requires `@huggingface/transformers`. Install with: bun add @huggingface/transformers");
501
501
  }
502
502
  }
503
+ // src/detectors/composite.ts
504
+ function luhnOk(s) {
505
+ const digits = [];
506
+ for (const c of s) {
507
+ if (c >= "0" && c <= "9")
508
+ digits.push(Number(c));
509
+ }
510
+ if (digits.length < 13 || digits.length > 19)
511
+ return false;
512
+ let total = 0;
513
+ for (let i = 0;i < digits.length; i++) {
514
+ let d = digits[digits.length - 1 - i] ?? 0;
515
+ if (i % 2 === 1) {
516
+ d *= 2;
517
+ if (d > 9)
518
+ d -= 9;
519
+ }
520
+ total += d;
521
+ }
522
+ return total % 10 === 0;
523
+ }
524
+ function ibanChOk(raw) {
525
+ const s = raw.replace(/[\s-]/g, "");
526
+ if (s.length !== 21 || !s.startsWith("CH") || !/^\d+$/.test(s.slice(2))) {
527
+ return false;
528
+ }
529
+ const rearranged = s.slice(4) + s.slice(0, 4);
530
+ let numeric = "";
531
+ for (const c of rearranged) {
532
+ numeric += c >= "A" && c <= "Z" ? String(c.charCodeAt(0) - 55) : c;
533
+ }
534
+ let rem = 0;
535
+ for (const ch of numeric) {
536
+ rem = (rem * 10 + Number(ch)) % 97;
537
+ }
538
+ return rem === 1;
539
+ }
540
+ function ahvOk(s) {
541
+ const digits = [];
542
+ for (const c of s) {
543
+ if (c >= "0" && c <= "9")
544
+ digits.push(Number(c));
545
+ }
546
+ if (digits.length !== 13)
547
+ return false;
548
+ if (digits[0] !== 7 || digits[1] !== 5 || digits[2] !== 6)
549
+ return false;
550
+ let total = 0;
551
+ for (let i = 0;i < 12; i++) {
552
+ total += (digits[i] ?? 0) * (i % 2 === 0 ? 1 : 3);
553
+ }
554
+ const check = (10 - total % 10) % 10;
555
+ return digits[12] === check;
556
+ }
557
+ function vatCheOk(s) {
558
+ const digits = [];
559
+ for (const c of s) {
560
+ if (c >= "0" && c <= "9")
561
+ digits.push(Number(c));
562
+ }
563
+ if (digits.length < 9)
564
+ return false;
565
+ const weights = [5, 4, 3, 2, 7, 6, 5, 4];
566
+ let total = 0;
567
+ for (let i = 0;i < 8; i++) {
568
+ total += (digits[i] ?? 0) * (weights[i] ?? 0);
569
+ }
570
+ let expected = 11 - total % 11;
571
+ if (expected === 11)
572
+ expected = 0;
573
+ else if (expected === 10)
574
+ return false;
575
+ return digits[8] === expected;
576
+ }
577
+ var REGEXES = [
578
+ {
579
+ label: "private_email",
580
+ pattern: /\b[\w._%+\-]+@[\w.\-]+\.[A-Za-z]{2,}\b/g
581
+ },
582
+ {
583
+ label: "private_url",
584
+ pattern: /https?:\/\/[^\s)>\]]+/g
585
+ },
586
+ {
587
+ label: "private_phone",
588
+ pattern: /(?:\+41|0041|0)\s?(?:\(0\)\s?)?\d{2}[\s\-/.]?\d{3}[\s\-/.]?\d{2}[\s\-/.]?\d{2}\b/g
589
+ },
590
+ {
591
+ label: "account_number",
592
+ pattern: /\bCH\d{2}(?:[\s-]?\d){17}\b/g,
593
+ validator: ibanChOk
594
+ },
595
+ {
596
+ label: "account_number",
597
+ pattern: /\b756\.\d{4}\.\d{4}\.\d{2}\b/g,
598
+ validator: ahvOk
599
+ },
600
+ {
601
+ label: "account_number",
602
+ pattern: /\b756\d{10}\b/g,
603
+ validator: ahvOk
604
+ },
605
+ {
606
+ label: "account_number",
607
+ pattern: /\bCHE-\d{3}\.\d{3}\.\d{3}(?:\s+(?:MWST|TVA|IVA))?\b/g,
608
+ validator: vatCheOk
609
+ },
610
+ {
611
+ label: "account_number",
612
+ pattern: /\b(?:\d[\s-]?){15,16}\d\b/g,
613
+ validator: luhnOk
614
+ },
615
+ {
616
+ label: "private_date",
617
+ pattern: /\b(?:\d{1,2}[./]\d{1,2}[./]\d{4}|\d{4}[./-]\d{1,2}[./-]\d{1,2})\b/g
618
+ },
619
+ {
620
+ label: "secret",
621
+ pattern: /\bsk-(?:proj-)?[A-Za-z0-9_\-]{20,}\b/g
622
+ },
623
+ {
624
+ label: "secret",
625
+ pattern: /\bghp_[A-Za-z0-9]{36}\b/g
626
+ },
627
+ {
628
+ label: "secret",
629
+ pattern: /\bAKIA[A-Z0-9]{16}\b/g
630
+ }
631
+ ];
632
+ var URL_TRAIL_TRIM = '.,;:!?")>]';
633
+ function trimUrl(text, start, end) {
634
+ while (end > start && URL_TRAIL_TRIM.includes(text[end - 1] ?? ""))
635
+ end--;
636
+ return [start, end];
637
+ }
638
+ function findRegexSpans(text) {
639
+ const candidates = [];
640
+ for (const { label, pattern, validator } of REGEXES) {
641
+ pattern.lastIndex = 0;
642
+ let m;
643
+ while ((m = pattern.exec(text)) !== null) {
644
+ let start = m.index;
645
+ let end = m.index + m[0].length;
646
+ if (label === "private_url")
647
+ [start, end] = trimUrl(text, start, end);
648
+ const value = text.slice(start, end);
649
+ if (!value.trim())
650
+ continue;
651
+ if (validator && !validator(value))
652
+ continue;
653
+ candidates.push({
654
+ start,
655
+ end,
656
+ label,
657
+ text: value,
658
+ score: 1
659
+ });
660
+ }
661
+ }
662
+ candidates.sort((a, b) => a.start !== b.start ? a.start - b.start : b.end - b.start - (a.end - a.start));
663
+ const out = [];
664
+ let lastEnd = -1;
665
+ for (const sp of candidates) {
666
+ if (sp.start < lastEnd)
667
+ continue;
668
+ out.push(sp);
669
+ lastEnd = sp.end;
670
+ }
671
+ return out;
672
+ }
673
+ function maskSpans(text, spans) {
674
+ if (spans.length === 0)
675
+ return text;
676
+ const chars = [...text];
677
+ for (const sp of spans) {
678
+ for (let i = sp.start;i < sp.end && i < chars.length; i++) {
679
+ chars[i] = " ";
680
+ }
681
+ }
682
+ return chars.join("");
683
+ }
684
+
685
+ class CompositeDetector {
686
+ model;
687
+ constructor(opts) {
688
+ if (!opts || !opts.model) {
689
+ throw new Error("CompositeDetector requires { model: Detector } — pass any object with " + "a .detect(text) method (e.g. new LocalDetector({...})).");
690
+ }
691
+ this.model = opts.model;
692
+ }
693
+ async detect(text, opts) {
694
+ if (!text)
695
+ return [];
696
+ const regexSpans = findRegexSpans(text);
697
+ const masked = maskSpans(text, regexSpans);
698
+ const modelSpans = await this.model.detect(masked, opts);
699
+ const out = [...regexSpans];
700
+ const regexRanges = regexSpans.map((s) => [s.start, s.end]);
701
+ for (const sp of modelSpans) {
702
+ const overlaps = regexRanges.some(([rs, re]) => sp.start < re && rs < sp.end);
703
+ if (overlaps)
704
+ continue;
705
+ out.push(sp);
706
+ }
707
+ out.sort((a, b) => a.start !== b.start ? a.start - b.start : b.end - b.start - (a.end - a.start));
708
+ return out;
709
+ }
710
+ }
503
711
  // src/detectors/index.ts
504
712
  function defaultDetector() {
505
713
  const hasKey = typeof process !== "undefined" && Boolean(process.env?.GHEIM_API_KEY);
@@ -563,7 +771,9 @@ class Session {
563
771
  cursor = end;
564
772
  }
565
773
  out.push(text.slice(cursor));
566
- return { redacted: out.join(""), merged };
774
+ let redacted = out.join("");
775
+ redacted = recoverDuplicateOccurrences(redacted, this.mapping);
776
+ return { redacted, merged };
567
777
  }
568
778
  restore(text) {
569
779
  if (Object.keys(this.mapping).length === 0)
@@ -619,6 +829,18 @@ function mergeAdjacent(text, spans) {
619
829
  }
620
830
  return out;
621
831
  }
832
+ function escapeRegex(s) {
833
+ return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
834
+ }
835
+ function recoverDuplicateOccurrences(redacted, mapping) {
836
+ const entries = Object.entries(mapping).filter(([, surface]) => surface && surface.length > 0).sort((a, b) => b[1].length - a[1].length);
837
+ let result = redacted;
838
+ for (const [sentinel, surface] of entries) {
839
+ const pattern = new RegExp(`(?<![\\p{L}\\p{N}_])${escapeRegex(surface)}(?![\\p{L}\\p{N}_])`, "gu");
840
+ result = result.replace(pattern, sentinel);
841
+ }
842
+ return result;
843
+ }
622
844
 
623
845
  // src/openai/_base.ts
624
846
  function resolveCallSession(body, defaultDet) {
@@ -1 +1 @@
1
- {"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../../src/core/session.ts"],"names":[],"mappings":"AAAA,kDAAkD;AAClD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AAGjD;;;gEAGgE;AAChE,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;AAkBtD,uCAAuC;AACvC,MAAM,WAAW,cAAc;IAC7B;;;;;;;;;;OAUG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;CAC1C;AAED,qBAAa,OAAO;IAClB,8DAA8D;IAC9D,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAM;IACrC,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,mBAAmB,CAAkC;gBAEjD,IAAI,GAAE,cAAmB;IAQrC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM;IAiBhD,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GAAG,MAAM;IAIvE;;;;;;;OAOG;IACH,kBAAkB,CAChB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GACpC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,UAAU,EAAE,CAAA;KAAE;IAoB7C,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAK7B,MAAM,IAAI;QAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE;IAI7C,MAAM,CAAC,QAAQ,CACb,GAAG,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,EACxC,IAAI,GAAE,cAAmB,GACxB,OAAO;CAqBX;AAID,8EAA8E;AAC9E,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAC3B,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GACpC,UAAU,EAAE,CAqBd"}
1
+ {"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../../src/core/session.ts"],"names":[],"mappings":"AAAA,kDAAkD;AAClD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AAGjD;;;gEAGgE;AAChE,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;AAkBtD,uCAAuC;AACvC,MAAM,WAAW,cAAc;IAC7B;;;;;;;;;;OAUG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;CAC1C;AAED,qBAAa,OAAO;IAClB,8DAA8D;IAC9D,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAM;IACrC,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,mBAAmB,CAAkC;gBAEjD,IAAI,GAAE,cAAmB;IAQrC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM;IAiBhD,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GAAG,MAAM;IAIvE;;;;;;;OAOG;IACH,kBAAkB,CAChB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GACpC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,UAAU,EAAE,CAAA;KAAE;IA+B7C,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAK7B,MAAM,IAAI;QAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE;IAI7C,MAAM,CAAC,QAAQ,CACb,GAAG,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,EACxC,IAAI,GAAE,cAAmB,GACxB,OAAO;CAqBX;AAID,8EAA8E;AAC9E,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAC3B,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,GACpC,UAAU,EAAE,CAqBd"}
@@ -0,0 +1,58 @@
1
+ /**
2
+ * Composite detector: regex front-end + model on the remainder.
3
+ *
4
+ * Production-ready Swiss PII detector that combines two complementary
5
+ * strategies — a port of the Python sibling
6
+ * (`gheim.detectors.composite.CompositeDetector`) with parity-tested
7
+ * regexes and checksum validators.
8
+ *
9
+ * 1. Deterministic regex front-end for structured PII categories where
10
+ * shape is unambiguous and checksums verify identity:
11
+ * - IBAN-CH (mod-97-10), AHV (EAN-13 mod-10), VAT-CHE (mod-11-10),
12
+ * Visa/Mastercard (Luhn)
13
+ * - Email, URL, Swiss phone (+41 / 0041 / 0...), explicit dates
14
+ * (DD.MM.YYYY, YYYY-MM-DD)
15
+ * - API tokens (sk-, ghp_, AKIA...)
16
+ *
17
+ * Spans found here are "trusted" — checksums catch most surface-form
18
+ * imitations; the model is never asked.
19
+ *
20
+ * 2. The fine-tuned model on whatever's left after regex masking.
21
+ * Persons, addresses, and edge-case dates fall here. The model's
22
+ * "single-category mode" failure (where an IBAN/phone-shaped
23
+ * sequence suppresses person/date detection in the same input) is
24
+ * bypassed because the regex already absorbed those tokens — the
25
+ * model sees them as whitespace.
26
+ *
27
+ * Wraps the same `Detector` interface (`detect(text) → Span[]`) as
28
+ * `LocalDetector` so it drops into any code path that already accepts a
29
+ * detector.
30
+ */
31
+ import type { Detector, Span } from "./base.ts";
32
+ /** Options for {@link CompositeDetector}. */
33
+ export interface CompositeDetectorOptions {
34
+ /** Inner detector — usually a {@link LocalDetector}. Anything with
35
+ * a `detect(text) → Promise<Span[]>` method works. */
36
+ model: Detector;
37
+ }
38
+ /** Regex front-end (with checksum validation) + model on the remainder.
39
+ *
40
+ * @example
41
+ * import { LocalDetector, CompositeDetector } from "gheim";
42
+ * const composite = new CompositeDetector({
43
+ * model: new LocalDetector({ dtype: "q8" }),
44
+ * });
45
+ * const spans = await composite.detect(swissLetter);
46
+ * // → IBAN/phone/email caught by regex (checksum-validated);
47
+ * // person/address/dates caught by the model on the masked remainder.
48
+ */
49
+ export declare class CompositeDetector implements Detector {
50
+ readonly model: Detector;
51
+ constructor(opts: CompositeDetectorOptions);
52
+ /** Two-pass detection: regex first (with checksum), then model on
53
+ * the masked remainder. Returns a single overlap-resolved span list. */
54
+ detect(text: string, opts?: {
55
+ model?: string;
56
+ }): Promise<Span[]>;
57
+ }
58
+ //# sourceMappingURL=composite.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"composite.d.ts","sourceRoot":"","sources":["../../../src/detectors/composite.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,OAAO,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAiNhD,6CAA6C;AAC7C,MAAM,WAAW,wBAAwB;IACvC;2DACuD;IACvD,KAAK,EAAE,QAAQ,CAAC;CACjB;AAED;;;;;;;;;;GAUG;AACH,qBAAa,iBAAkB,YAAW,QAAQ;IAChD,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC;gBAEb,IAAI,EAAE,wBAAwB;IAU1C;6EACyE;IACnE,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;CAmBvE"}
@@ -3,6 +3,8 @@ export { DEFAULT_BASE_URL, DEFAULT_MODEL as REMOTE_DEFAULT_MODEL, RemoteDetector
3
3
  export type { RemoteDetectorOptions } from "./remote.ts";
4
4
  export { LocalDetector } from "./local.ts";
5
5
  export type { LocalDetectorOptions, LocalDetectorLoadEvent } from "./local.ts";
6
+ export { CompositeDetector } from "./composite.ts";
7
+ export type { CompositeDetectorOptions } from "./composite.ts";
6
8
  import type { Detector } from "./base.ts";
7
9
  /** Pick a reasonable default: remote if `GHEIM_API_KEY` is set in env, else local. */
8
10
  export declare function defaultDetector(): Detector;
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/detectors/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,EACL,gBAAgB,EAChB,aAAa,IAAI,oBAAoB,EACrC,cAAc,GACf,MAAM,aAAa,CAAC;AACrB,YAAY,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,YAAY,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAE/E,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAI1C,sFAAsF;AACtF,wBAAgB,eAAe,IAAI,QAAQ,CAI1C"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/detectors/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,EACL,gBAAgB,EAChB,aAAa,IAAI,oBAAoB,EACrC,cAAc,GACf,MAAM,aAAa,CAAC;AACrB,YAAY,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,YAAY,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAC/E,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACnD,YAAY,EAAE,wBAAwB,EAAE,MAAM,gBAAgB,CAAC;AAE/D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAI1C,sFAAsF;AACtF,wBAAgB,eAAe,IAAI,QAAQ,CAI1C"}
@@ -13,11 +13,11 @@ export { LABEL_TO_TAG, SENTINEL_RE, isPossibleSentinelPrefix, labelTag, parseSen
13
13
  export type { Sentinel } from "./core/sentinels.ts";
14
14
  export { StreamDeanonymizer } from "./core/stream.ts";
15
15
  export type { Detector, Span } from "./detectors/base.ts";
16
- export { DEFAULT_BASE_URL, REMOTE_DEFAULT_MODEL, RemoteDetector, LocalDetector, defaultDetector, } from "./detectors/index.ts";
17
- export type { RemoteDetectorOptions, LocalDetectorOptions, LocalDetectorLoadEvent, } from "./detectors/index.ts";
16
+ export { DEFAULT_BASE_URL, REMOTE_DEFAULT_MODEL, RemoteDetector, LocalDetector, CompositeDetector, defaultDetector, } from "./detectors/index.ts";
17
+ export type { RemoteDetectorOptions, LocalDetectorOptions, LocalDetectorLoadEvent, CompositeDetectorOptions, } from "./detectors/index.ts";
18
18
  export { anonymizeMessages, anonymizeText, deanonymizeStream, deanonymizeText, } from "./plain.ts";
19
19
  export type { ChatMessage } from "./plain.ts";
20
- export declare const VERSION = "0.1.2";
20
+ export declare const VERSION = "0.1.4";
21
21
  export { mergeAdjacent } from "./core/session.ts";
22
22
  export type { MergedSpan } from "./core/session.ts";
23
23
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,EACL,YAAY,EACZ,WAAW,EACX,wBAAwB,EACxB,QAAQ,EACR,aAAa,EACb,cAAc,GACf,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,YAAY,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EACL,gBAAgB,EAChB,oBAAoB,EACpB,cAAc,EACd,aAAa,EACb,eAAe,GAChB,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EACV,qBAAqB,EACrB,oBAAoB,EACpB,sBAAsB,GACvB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,iBAAiB,EACjB,aAAa,EACb,iBAAiB,EACjB,eAAe,GAChB,MAAM,YAAY,CAAC;AACpB,YAAY,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAK9C,eAAO,MAAM,OAAO,UAAU,CAAC;AAE/B,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,YAAY,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,EACL,YAAY,EACZ,WAAW,EACX,wBAAwB,EACxB,QAAQ,EACR,aAAa,EACb,cAAc,GACf,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,YAAY,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EACL,gBAAgB,EAChB,oBAAoB,EACpB,cAAc,EACd,aAAa,EACb,iBAAiB,EACjB,eAAe,GAChB,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EACV,qBAAqB,EACrB,oBAAoB,EACpB,sBAAsB,EACtB,wBAAwB,GACzB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,iBAAiB,EACjB,aAAa,EACb,iBAAiB,EACjB,eAAe,GAChB,MAAM,YAAY,CAAC;AACpB,YAAY,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAK9C,eAAO,MAAM,OAAO,UAAU,CAAC;AAE/B,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,YAAY,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "gheim",
3
- "version": "0.1.2",
3
+ "version": "0.1.4",
4
4
  "description": "PII round-trip for LLM APIs: anonymize before the request, de-anonymize the stream on the way back.",
5
5
  "license": "Apache-2.0",
6
6
  "author": "Joel Barmettler <joel.barmettler@gmail.com>",