@chatpanel/pii 0.2.11 → 0.2.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.js +3 -1
- package/net.js +103 -0
- package/package.json +5 -3
- package/pii-detect.js +7 -0
- package/pii-redact.js +144 -7
- package/pipeline.js +13 -15
- package/sanitize.js +38 -0
- package/tool-harness.js +16 -11
package/index.js
CHANGED
|
@@ -7,9 +7,10 @@
|
|
|
7
7
|
// Submodules are also importable directly:
|
|
8
8
|
// 'chatpanel-pii/pii-redact.js' deterministic redact/restore + vault
|
|
9
9
|
// 'chatpanel-pii/pii-detect.js' local NER / LLM entity detection
|
|
10
|
-
// 'chatpanel-pii/pipeline.js' pure turn orchestration + tier/scope
|
|
10
|
+
// 'chatpanel-pii/pipeline.js' pure turn orchestration + tier/scope selection
|
|
11
11
|
// 'chatpanel-pii/tool-rank.js' deterministic tool narrowing (auto mode)
|
|
12
12
|
// 'chatpanel-pii/sanitize.js' Unicode de-steganography (strip invisible/format chars)
|
|
13
|
+
// 'chatpanel-pii/net.js' SSRF host classifier + outbound-URL guard
|
|
13
14
|
|
|
14
15
|
export * from './pii-redact.js';
|
|
15
16
|
export * from './pii-detect.js';
|
|
@@ -17,3 +18,4 @@ export * from './pipeline.js';
|
|
|
17
18
|
export * from './tool-rank.js';
|
|
18
19
|
export * from './tool-harness.js';
|
|
19
20
|
export * from './sanitize.js';
|
|
21
|
+
export * from './net.js';
|
package/net.js
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
// Shared host classifier + outbound-URL guard — the SSRF primitive.
|
|
2
|
+
//
|
|
3
|
+
// One implementation of "what is a loopback / cloud-metadata / private host",
|
|
4
|
+
// delivered the way the rest of @chatpanel/pii is: npm dependency for the
|
|
5
|
+
// gateway/bridge, vendorable into the browser extension (pure — only URL + string
|
|
6
|
+
// ops, no node APIs, so it runs in a Worker/service-worker too). Replaces the
|
|
7
|
+
// hand-maintained copies in the bridge (src/ssrf.js) and the extension
|
|
8
|
+
// (js/context.js isBlockedHost) so a security guard can't silently drift between
|
|
9
|
+
// the direct client path and the proxied path. See docs/secure-data-plane.md.
|
|
10
|
+
//
|
|
11
|
+
// The policy knobs cover the two legitimate trust contexts:
|
|
12
|
+
// • A MODEL / API / MCP endpoint (gateway upstream, bridge MCP proxy) may live on
|
|
13
|
+
// loopback (Ollama, LM Studio) or the LAN (a homelab GPU box) — so those are
|
|
14
|
+
// allowed by default — but must NEVER reach cloud instance metadata.
|
|
15
|
+
// • A WEB PAGE fetch (link title, page context) has no business touching loopback
|
|
16
|
+
// or any private host at all — call with { allowLoopback:false, allowPrivate:false }.
|
|
17
|
+
// Cloud metadata (169.254.169.254 & friends) and non-http(s) schemes are blocked in
|
|
18
|
+
// BOTH contexts, unconditionally. Re-run the assert on every redirect hop.
|
|
19
|
+
|
|
20
|
+
function ipv4(h) {
|
|
21
|
+
const m = String(h).match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/);
|
|
22
|
+
if (!m) return null;
|
|
23
|
+
const o = m.slice(1).map(Number);
|
|
24
|
+
if (o.some((n) => n > 255)) return null;
|
|
25
|
+
return o;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const norm = (hostname) => String(hostname || '').toLowerCase().replace(/^\[|\]$/g, '');
|
|
29
|
+
|
|
30
|
+
// Loopback = this host's own services (127.0.0.0/8, ::1, localhost, *.localhost).
|
|
31
|
+
export function isLoopbackHost(hostname) {
|
|
32
|
+
const h = norm(hostname);
|
|
33
|
+
if (!h) return false;
|
|
34
|
+
if (h === 'localhost' || h.endsWith('.localhost')) return true;
|
|
35
|
+
if (h === '::1') return true;
|
|
36
|
+
const o = ipv4(h);
|
|
37
|
+
return !!(o && o[0] === 127);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// Cloud instance metadata — the sharpest SSRF target (credential theft). Covers the
|
|
41
|
+
// link-local IMDS address used by AWS/GCP/Azure/DO (169.254.169.254), Alibaba's
|
|
42
|
+
// 100.100.100.200, and the GCP/name-based metadata hosts. ALWAYS blocked.
|
|
43
|
+
export function isMetadataHost(hostname) {
|
|
44
|
+
const h = norm(hostname);
|
|
45
|
+
if (h === 'metadata.google.internal' || h === 'metadata') return true;
|
|
46
|
+
const o = ipv4(h);
|
|
47
|
+
if (!o) return false;
|
|
48
|
+
if (o[0] === 169 && o[1] === 254) return true; // 169.254.169.254 (+ link-local)
|
|
49
|
+
if (o[0] === 100 && o[1] === 100 && o[2] === 100 && o[3] === 200) return true; // Alibaba IMDS
|
|
50
|
+
return false;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// Private / internal address space, EXCLUDING loopback + metadata (checked
|
|
54
|
+
// separately): RFC1918, CGNAT, IPv6 ULA/link-local, mDNS .local, this-host 0.x/::.
|
|
55
|
+
export function isPrivateHost(hostname) {
|
|
56
|
+
const h = norm(hostname);
|
|
57
|
+
if (!h) return true;
|
|
58
|
+
if (h.endsWith('.local')) return true;
|
|
59
|
+
if (
|
|
60
|
+
h === '::' || h.startsWith('fc') || h.startsWith('fd') // IPv6 ULA
|
|
61
|
+
|| h.startsWith('fe8') || h.startsWith('fe9') || h.startsWith('fea') || h.startsWith('feb') // link-local
|
|
62
|
+
) return true;
|
|
63
|
+
const o = ipv4(h);
|
|
64
|
+
if (o) {
|
|
65
|
+
const [a, b] = o;
|
|
66
|
+
if (a === 0 || a === 10) return true; // this-host / RFC1918
|
|
67
|
+
if (a === 169 && b === 254) return true; // link-local
|
|
68
|
+
if (a === 172 && b >= 16 && b <= 31) return true; // RFC1918
|
|
69
|
+
if (a === 192 && b === 168) return true; // RFC1918
|
|
70
|
+
if (a === 100 && b >= 64 && b <= 127) return true; // CGNAT
|
|
71
|
+
}
|
|
72
|
+
return false;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// Policy-driven classifier. Returns true if `hostname` must be blocked under `policy`.
|
|
76
|
+
// Defaults model the ENDPOINT context (loopback + LAN allowed, metadata never).
|
|
77
|
+
export function isBlockedHost(hostname, { allowLoopback = true, allowPrivate = true } = {}) {
|
|
78
|
+
const h = norm(hostname);
|
|
79
|
+
if (!h) return true;
|
|
80
|
+
if (isMetadataHost(h)) return true; // never, in any context
|
|
81
|
+
if (isLoopbackHost(h)) return !allowLoopback;
|
|
82
|
+
if (isPrivateHost(h)) return !allowPrivate;
|
|
83
|
+
return false; // public host
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// Assert a URL is fetchable under `policy`; returns the parsed URL or throws.
|
|
87
|
+
// Call on the initial URL AND after every redirect hop.
|
|
88
|
+
export function assertFetchableUrl(u, policy = {}) {
|
|
89
|
+
let parsed;
|
|
90
|
+
try { parsed = new URL(u); } catch { throw new Error(`invalid URL: ${u}`); }
|
|
91
|
+
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
|
|
92
|
+
throw new Error(`only http(s) URLs allowed (got "${parsed.protocol}")`);
|
|
93
|
+
}
|
|
94
|
+
if (isBlockedHost(parsed.hostname, policy)) {
|
|
95
|
+
throw new Error(`refusing to reach a blocked address (${parsed.hostname})`);
|
|
96
|
+
}
|
|
97
|
+
return parsed;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// Endpoint context: model/API/MCP upstream — loopback + LAN OK, metadata never.
|
|
101
|
+
export const assertEndpointUrl = (u, opts = {}) => assertFetchableUrl(u, { allowLoopback: true, allowPrivate: true, ...opts });
|
|
102
|
+
// Web-page context: no loopback, no private, no metadata — genuinely public only.
|
|
103
|
+
export const assertPublicWebUrl = (u) => assertFetchableUrl(u, { allowLoopback: false, allowPrivate: false });
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@chatpanel/pii",
|
|
3
|
-
"version": "0.2.
|
|
4
|
-
"description": "The canonical ChatPanel privacy engine
|
|
3
|
+
"version": "0.2.15",
|
|
4
|
+
"description": "The canonical ChatPanel privacy engine \u2014 reversible PII redaction + pseudonymization with local entity detection. Pure, dependency-free ESM shared by the ChatPanel extension, gateway, and bridge.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "index.js",
|
|
7
7
|
"exports": {
|
|
@@ -11,7 +11,8 @@
|
|
|
11
11
|
"./pipeline.js": "./pipeline.js",
|
|
12
12
|
"./tool-rank.js": "./tool-rank.js",
|
|
13
13
|
"./tool-harness.js": "./tool-harness.js",
|
|
14
|
-
"./sanitize.js": "./sanitize.js"
|
|
14
|
+
"./sanitize.js": "./sanitize.js",
|
|
15
|
+
"./net.js": "./net.js"
|
|
15
16
|
},
|
|
16
17
|
"files": [
|
|
17
18
|
"index.js",
|
|
@@ -21,6 +22,7 @@
|
|
|
21
22
|
"tool-rank.js",
|
|
22
23
|
"tool-harness.js",
|
|
23
24
|
"sanitize.js",
|
|
25
|
+
"net.js",
|
|
24
26
|
"LICENSE",
|
|
25
27
|
"README.md"
|
|
26
28
|
],
|
package/pii-detect.js
CHANGED
|
@@ -13,6 +13,8 @@
|
|
|
13
13
|
// chat — redaction silently falls back to the deterministic layer.
|
|
14
14
|
// - input is length-capped so a huge transcript can't stall detection.
|
|
15
15
|
|
|
16
|
+
import { assertEndpointUrl } from './net.js';
|
|
17
|
+
|
|
16
18
|
const cache = new Map(); // key -> [{value,type}]
|
|
17
19
|
const CACHE_MAX = 300;
|
|
18
20
|
|
|
@@ -151,6 +153,11 @@ export async function detectEntities(text, cfg, { signal, fetchImpl = globalThis
|
|
|
151
153
|
const run = det.backend === 'endpoint' ? detectViaEndpoint : detectViaOpenAI;
|
|
152
154
|
let ents = [];
|
|
153
155
|
try {
|
|
156
|
+
// SSRF guard before RAW (pre-redaction) text leaves for the detector: http(s)
|
|
157
|
+
// only, never cloud metadata. Loopback/LAN allowed — a local NER server / Ollama
|
|
158
|
+
// is the normal case. A blocked URL fails open (deterministic-only), or surfaces
|
|
159
|
+
// to the Test button in strict mode.
|
|
160
|
+
assertEndpointUrl(det.url);
|
|
154
161
|
ents = await withTimeout(run(capped, det, signal, fetchImpl), det.timeoutMs || 1500, signal);
|
|
155
162
|
} catch (e) {
|
|
156
163
|
if (strict) throw e; // surface errors to the Test button
|
package/pii-redact.js
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
// Pure + dependency-free so it is unit-testable and runs identically for API and
|
|
10
10
|
// CLI/bridge agents (both assemble their outbound payload through providers.js).
|
|
11
11
|
//
|
|
12
|
-
// Tiers
|
|
12
|
+
// Tiers:
|
|
13
13
|
// 'basic' — deterministic regex: emails, phones, IPs, cards (Luhn), SSNs, keys.
|
|
14
14
|
// 'full' — basic + entity-aware: known people/orgs (meeting roster, contacts,
|
|
15
15
|
// the user's own identity) and a user-editable custom dictionary.
|
|
@@ -18,6 +18,8 @@
|
|
|
18
18
|
// one reference won't restore (it shows the token) — but the privacy guarantee
|
|
19
19
|
// (the real value never left the device) always holds.
|
|
20
20
|
|
|
21
|
+
import { stripHidden, confusablesSkeleton } from './sanitize.js';
|
|
22
|
+
|
|
21
23
|
const TOKEN_RE = /\[\[([A-Z][A-Z0-9]*)_(\d+)\]\]/g;
|
|
22
24
|
|
|
23
25
|
// Bracket-TOLERANT match of the same token. Smaller models routinely drop or mangle
|
|
@@ -72,6 +74,19 @@ function escapeRegex(s) {
|
|
|
72
74
|
return String(s).replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
73
75
|
}
|
|
74
76
|
|
|
77
|
+
// Reject a user-supplied dictionary regex that is a likely ReDoS (catastrophic
|
|
78
|
+
// backtracking) BEFORE compiling + running it on untrusted-length input. Heuristic,
|
|
79
|
+
// not exhaustive: cap length, and reject the classic nested-quantifier families —
|
|
80
|
+
// a quantified group whose body also has a quantifier ((a+)+ / (a*)* / (.*)+) and
|
|
81
|
+
// back-to-back unbounded quantifiers (a**, .*+). A rejected pattern is skipped like a
|
|
82
|
+
// syntactically-invalid one, so redaction never breaks or hangs.
|
|
83
|
+
function isSafeUserPattern(p) {
|
|
84
|
+
if (typeof p !== 'string' || p.length === 0 || p.length > 200) return false;
|
|
85
|
+
if (/\([^)]*[+*}][^)]*\)\s*[+*{]/.test(p)) return false; // (…quantifier…)quantifier
|
|
86
|
+
if (/[+*]\s*[+*]/.test(p)) return false; // a**, a+*, .*+
|
|
87
|
+
return true;
|
|
88
|
+
}
|
|
89
|
+
|
|
75
90
|
// Apply many find/replace rules in a SINGLE left-to-right pass over the source.
|
|
76
91
|
// Each rule is { re: <global RegExp>, repl: (match) => string }. Unlike running
|
|
77
92
|
// rule[0].replace then rule[1].replace then …, text emitted by one rule is NEVER
|
|
@@ -120,19 +135,42 @@ function luhnValid(digits) {
|
|
|
120
135
|
return sum % 10 === 0;
|
|
121
136
|
}
|
|
122
137
|
|
|
138
|
+
// Plausible IPv6? Controls false positives from the broad IPV6 regex: accept only a
|
|
139
|
+
// `::`-compressed form (≥1 hextet) or a full 8-hextet address, hextets ≤4 hex digits.
|
|
140
|
+
function isLikelyIpv6(s) {
|
|
141
|
+
if (!/^[0-9A-Fa-f:]+$/.test(s) || (s.match(/:/g) || []).length < 2) return false;
|
|
142
|
+
const parts = s.split(':');
|
|
143
|
+
if (parts.some((p) => p.length > 4)) return false;
|
|
144
|
+
if (s.includes('::')) return parts.filter(Boolean).length >= 1 && parts.filter(Boolean).length <= 7;
|
|
145
|
+
return parts.length === 8 && parts.every((p) => p.length >= 1);
|
|
146
|
+
}
|
|
147
|
+
|
|
123
148
|
// Deterministic detectors. Each: { type, re, valid? }. Order = priority; more
|
|
124
149
|
// specific patterns run first so they win the bytes before greedier ones.
|
|
125
150
|
const DETECTORS = [
|
|
151
|
+
// PEM private-key block (multi-line) — highest priority, most specific.
|
|
152
|
+
{ type: 'SECRET', re: /-----BEGIN (?:[A-Z0-9]+ )*PRIVATE KEY-----[\s\S]*?-----END (?:[A-Z0-9]+ )*PRIVATE KEY-----/g },
|
|
126
153
|
{ type: 'EMAIL', re: /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/g },
|
|
127
|
-
|
|
154
|
+
// SSN: dash- OR space-separated (bare 9-digit is left alone — too false-positive-prone).
|
|
155
|
+
{ type: 'SSN', re: /\b\d{3}[-\s]\d{2}[-\s]\d{4}\b/g },
|
|
128
156
|
{
|
|
157
|
+
// Vendor API keys / tokens. sk-… also covers OpenAI sk-proj-/sk-ant-. Adds Google
|
|
158
|
+
// (AIza…), Stripe (sk_live_/rk_test_…), GitHub fine-grained PATs, Slack xapp-.
|
|
129
159
|
type: 'KEY',
|
|
130
|
-
re: /\b(?:sk-[A-Za-z0-9_-]{16,}|AKIA[0-9A-Z]{16}|gh[pousr]_[A-Za-z0-9]{20,}|xox[baprs]-[A-Za-z0-9-]{10,})\b/g,
|
|
160
|
+
re: /\b(?:sk-[A-Za-z0-9_-]{16,}|AKIA[0-9A-Z]{16}|gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[0-9A-Za-z_]{22,}|xox[baprs]-[A-Za-z0-9-]{10,}|xapp-[0-9]-[A-Za-z0-9-]{10,}|AIza[0-9A-Za-z_-]{35}|[rs]k_(?:live|test)_[0-9A-Za-z]{16,})\b/g,
|
|
131
161
|
},
|
|
162
|
+
// JWT — three base64url segments; `eyJ` is base64 of `{"…`, so this is specific.
|
|
163
|
+
{ type: 'KEY', re: /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{6,}\.[A-Za-z0-9_-]{6,}\b/g },
|
|
132
164
|
{
|
|
133
165
|
type: 'IP',
|
|
134
166
|
re: /\b(?:(?:25[0-5]|2[0-4]\d|1?\d?\d)\.){3}(?:25[0-5]|2[0-4]\d|1?\d?\d)\b/g,
|
|
135
167
|
},
|
|
168
|
+
{
|
|
169
|
+
// IPv6 (incl. :: compression). Broad match narrowed by isLikelyIpv6 to curb FPs.
|
|
170
|
+
type: 'IP',
|
|
171
|
+
re: /(?<![:\w])(?:[0-9A-Fa-f]{0,4}:){2,7}[0-9A-Fa-f]{0,4}(?![:\w])/g,
|
|
172
|
+
valid: (m) => isLikelyIpv6(m),
|
|
173
|
+
},
|
|
136
174
|
{
|
|
137
175
|
// Phone: only count it if it has a separator or a leading + and 7–15 digits —
|
|
138
176
|
// so long bare ids (a 11-digit page id, an order number) are NOT redacted.
|
|
@@ -160,9 +198,16 @@ export function redactText(text, vault, {
|
|
|
160
198
|
tier = 'basic',
|
|
161
199
|
entities = [],
|
|
162
200
|
dictionary = [],
|
|
201
|
+
sanitize = true,
|
|
202
|
+
sanitizeOpts = undefined,
|
|
163
203
|
} = {}) {
|
|
164
204
|
if (text == null || text === '') return text;
|
|
165
|
-
|
|
205
|
+
// De-steganography BEFORE detection, in-band: an obfuscated value
|
|
206
|
+
// (j<ZWSP>o<ZWSP>hn@x.com, homoglyphs, ASCII-smuggled Tag chars) must become
|
|
207
|
+
// matchable so the regex/NER can't be trivially bypassed. Callers used to have to
|
|
208
|
+
// remember to stripHidden() first; folding it in here makes the engine safe on its
|
|
209
|
+
// own — the sanitize:false escape hatch is only for a caller that already did it.
|
|
210
|
+
let out = sanitize ? stripHidden(String(text), sanitizeOpts) : String(text);
|
|
166
211
|
const v = vault || createVault();
|
|
167
212
|
|
|
168
213
|
const entityTier = tier === 'full' || tier === 'entities';
|
|
@@ -180,7 +225,7 @@ export function redactText(text, vault, {
|
|
|
180
225
|
let re;
|
|
181
226
|
try {
|
|
182
227
|
re = d.pattern
|
|
183
|
-
? new RegExp(d.pattern, d.flags && /g/.test(d.flags) ? d.flags : `${d.flags || ''}g`)
|
|
228
|
+
? (isSafeUserPattern(d.pattern) ? new RegExp(d.pattern, d.flags && /g/.test(d.flags) ? d.flags : `${d.flags || ''}g`) : null)
|
|
184
229
|
: (d.value ? new RegExp(`(?<![\\w])${escapeRegex(d.value)}(?![\\w])`, 'gi') : null);
|
|
185
230
|
} catch {
|
|
186
231
|
re = null; // a bad user regex must never break redaction
|
|
@@ -210,13 +255,61 @@ export function redactText(text, vault, {
|
|
|
210
255
|
}
|
|
211
256
|
}
|
|
212
257
|
|
|
213
|
-
// 3) Deterministic detectors (all tiers).
|
|
258
|
+
// 3) Deterministic detectors (all tiers). Detect against a CONFUSABLES SKELETON so
|
|
259
|
+
// homoglyph-obfuscated values (Cyrillic/Greek/fullwidth Latin look-alikes) match
|
|
260
|
+
// the ASCII regexes — but REDACT the ORIGINAL span. The fold is 1:1 per code point,
|
|
261
|
+
// so a skeleton match's indices line up with `out`, and legitimate non-Latin text
|
|
262
|
+
// (which won't match a detector) is never rewritten. Higher-priority detectors
|
|
263
|
+
// (earlier in DETECTORS) claim overlapping spans first, matching the old order.
|
|
264
|
+
const skel = confusablesSkeleton(out);
|
|
265
|
+
const taken = []; // non-overlapping [start,end) spans, in priority order
|
|
266
|
+
const overlaps = (s, e) => taken.some((t) => s < t.end && e > t.start);
|
|
214
267
|
for (const det of DETECTORS) {
|
|
215
|
-
|
|
268
|
+
det.re.lastIndex = 0;
|
|
269
|
+
let m;
|
|
270
|
+
while ((m = det.re.exec(skel)) !== null) {
|
|
271
|
+
if (m[0].length === 0) { det.re.lastIndex++; continue; }
|
|
272
|
+
const start = m.index, end = start + m[0].length;
|
|
273
|
+
if ((det.valid && !det.valid(m[0])) || overlaps(start, end)) continue;
|
|
274
|
+
taken.push({ start, end, type: det.type });
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
if (taken.length) {
|
|
278
|
+
taken.sort((a, b) => a.start - b.start);
|
|
279
|
+
let rebuilt = '';
|
|
280
|
+
let pos = 0;
|
|
281
|
+
for (const t of taken) {
|
|
282
|
+
rebuilt += out.slice(pos, t.start) + tokenFor(v, t.type, out.slice(t.start, t.end));
|
|
283
|
+
pos = t.end;
|
|
284
|
+
}
|
|
285
|
+
out = rebuilt + out.slice(pos);
|
|
216
286
|
}
|
|
217
287
|
return out;
|
|
218
288
|
}
|
|
219
289
|
|
|
290
|
+
// Re-redact a tool RESULT before the model sees it, walking the shapes tools
|
|
291
|
+
// actually return: a bare string, a { text } object, an array, and the
|
|
292
|
+
// MCP-standard { content: [{ type:'text', text }] } (incl. an embedded
|
|
293
|
+
// { resource: { text } }). Only text-bearing fields are redacted — arbitrary
|
|
294
|
+
// fields (ids, urls, mime types) are left intact so tool results stay valid.
|
|
295
|
+
// Restore is the inverse concern; this only runs on the model-facing direction.
|
|
296
|
+
export function redactResultShape(raw, vault, opts) {
|
|
297
|
+
if (raw == null || typeof raw === 'string') {
|
|
298
|
+
return raw == null ? raw : redactText(raw, vault, opts);
|
|
299
|
+
}
|
|
300
|
+
if (Array.isArray(raw)) return raw.map((r) => redactResultShape(r, vault, opts));
|
|
301
|
+
if (typeof raw === 'object') {
|
|
302
|
+
let out = raw;
|
|
303
|
+
if (typeof raw.text === 'string') out = { ...out, text: redactText(raw.text, vault, opts) };
|
|
304
|
+
if (Array.isArray(raw.content)) out = { ...out, content: raw.content.map((c) => redactResultShape(c, vault, opts)) };
|
|
305
|
+
if (raw.resource && typeof raw.resource === 'object' && typeof raw.resource.text === 'string') {
|
|
306
|
+
out = { ...out, resource: { ...raw.resource, text: redactText(raw.resource.text, vault, opts) } };
|
|
307
|
+
}
|
|
308
|
+
return out;
|
|
309
|
+
}
|
|
310
|
+
return raw;
|
|
311
|
+
}
|
|
312
|
+
|
|
220
313
|
// Swap placeholders back to their originals. Unknown tokens are left untouched.
|
|
221
314
|
export function restoreText(text, vault) {
|
|
222
315
|
if (text == null || !vault) return text;
|
|
@@ -252,3 +345,47 @@ export function hasToken(text) {
|
|
|
252
345
|
TOKEN_RE.lastIndex = 0;
|
|
253
346
|
return TOKEN_RE.test(String(text || ''));
|
|
254
347
|
}
|
|
348
|
+
|
|
349
|
+
// ── What was redacted, for the user's own eyes ──────────────────────────────────────────
|
|
350
|
+
//
|
|
351
|
+
// The privacy promise is invisible unless you can SEE it: which entity types were caught,
|
|
352
|
+
// how many of each, and (on request) the actual before → after pairs. All of that already
|
|
353
|
+
// exists in the vault — this just summarises it, so every client renders the same shape
|
|
354
|
+
// instead of each one re-deriving it.
|
|
355
|
+
//
|
|
356
|
+
// PRIVACY OF THE SUMMARY ITSELF: `types` carries counts only, never values, so it is safe
|
|
357
|
+
// to render, log or persist. Real values live behind `pairs`, which a caller must ask for
|
|
358
|
+
// explicitly (`includeValues: true`) — they are the user's own data, shown on their own
|
|
359
|
+
// device, and must never be written anywhere durable.
|
|
360
|
+
|
|
361
|
+
/** Entity type + count for everything this vault redacted, most-frequent first. */
|
|
362
|
+
export function redactionSummary(vault, { includeValues = false, maxPairs = 200 } = {}) {
|
|
363
|
+
const byType = new Map();
|
|
364
|
+
const pairs = [];
|
|
365
|
+
for (const [token, value] of vault?.byToken || new Map()) {
|
|
366
|
+
const m = /^\[\[([A-Z][A-Z0-9]*)_(\d+)\]\]$/.exec(token);
|
|
367
|
+
const type = m ? m[1] : 'OTHER';
|
|
368
|
+
byType.set(type, (byType.get(type) || 0) + 1);
|
|
369
|
+
if (includeValues && pairs.length < maxPairs) pairs.push({ token, value, type });
|
|
370
|
+
}
|
|
371
|
+
const types = [...byType.entries()]
|
|
372
|
+
.map(([type, count]) => ({ type, count }))
|
|
373
|
+
.sort((a, b) => b.count - a.count || a.type.localeCompare(b.type));
|
|
374
|
+
return {
|
|
375
|
+
total: types.reduce((n, t) => n + t.count, 0),
|
|
376
|
+
types,
|
|
377
|
+
...(includeValues ? { pairs } : {}),
|
|
378
|
+
};
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
/** Merge several vault summaries (e.g. every conversation) into one. Counts only. */
|
|
382
|
+
export function mergeRedactionSummaries(summaries) {
|
|
383
|
+
const byType = new Map();
|
|
384
|
+
for (const s of summaries || []) {
|
|
385
|
+
for (const t of s?.types || []) byType.set(t.type, (byType.get(t.type) || 0) + t.count);
|
|
386
|
+
}
|
|
387
|
+
const types = [...byType.entries()]
|
|
388
|
+
.map(([type, count]) => ({ type, count }))
|
|
389
|
+
.sort((a, b) => b.count - a.count || a.type.localeCompare(b.type));
|
|
390
|
+
return { total: types.reduce((n, t) => n + t.count, 0), types };
|
|
391
|
+
}
|
package/pipeline.js
CHANGED
|
@@ -5,27 +5,24 @@
|
|
|
5
5
|
// What lives HERE (portable): redactOutbound, redactToolResult/redactResult,
|
|
6
6
|
// makeStreamRestorer, restore/restoreDeep, effectiveTier + gating.
|
|
7
7
|
//
|
|
8
|
-
//
|
|
9
|
-
//
|
|
10
|
-
//
|
|
8
|
+
// Host glue kept in the EXTENSION (NOT here): reading settings.ui.piiRedaction,
|
|
9
|
+
// the entitlement flag, and chrome storage — those wrap these pure functions with
|
|
10
|
+
// host-specific config.
|
|
11
11
|
|
|
12
|
-
import { redactText, restoreText, restoreWithAliases } from './pii-redact.js';
|
|
12
|
+
import { redactText, restoreText, restoreWithAliases, redactResultShape } from './pii-redact.js';
|
|
13
13
|
|
|
14
14
|
export function redactionEnabled(cfg) {
|
|
15
15
|
return !!(cfg && cfg.mode && cfg.mode !== 'off');
|
|
16
16
|
}
|
|
17
17
|
|
|
18
|
-
// The entity (name/org) tier is Pro; Free
|
|
19
|
-
// regex so the feature still does something useful without the upsell breaking.
|
|
18
|
+
// The entity (name/org) tier is Pro; Free falls back to the deterministic regex tier.
|
|
20
19
|
export function effectiveTier(cfg, isPro) {
|
|
21
20
|
const t = cfg?.tier === 'full' ? 'full' : 'basic';
|
|
22
21
|
return t === 'full' && !isPro ? 'basic' : t;
|
|
23
22
|
}
|
|
24
23
|
|
|
25
|
-
// Free
|
|
26
|
-
//
|
|
27
|
-
// Pro. Enforced here as defense-in-depth — the UI also surfaces the cap, but never
|
|
28
|
-
// trust the UI alone.
|
|
24
|
+
// On Free the first FREE_DICT_LIMIT custom-dictionary entries apply; the full
|
|
25
|
+
// dictionary is Pro. Enforced here as well as in the UI.
|
|
29
26
|
export const FREE_DICT_LIMIT = 5;
|
|
30
27
|
|
|
31
28
|
export function gatedDictionary(cfg, isPro) {
|
|
@@ -122,10 +119,11 @@ export function restoreDeep(value, vault) {
|
|
|
122
119
|
return value;
|
|
123
120
|
}
|
|
124
121
|
|
|
122
|
+
// Re-redact a tool result of any shape (string / { text } / array / MCP
|
|
123
|
+
// { content:[{text}] }), gated once by the toolResults scope. The old path only
|
|
124
|
+
// covered string + { text }, so PII in a content[] item reached the model.
|
|
125
125
|
export function redactResult(result, ctx) {
|
|
126
|
-
|
|
127
|
-
if (
|
|
128
|
-
|
|
129
|
-
}
|
|
130
|
-
return result;
|
|
126
|
+
const { vault, cfg, isPro = false, entities = [] } = ctx || {};
|
|
127
|
+
if (!redactionEnabled(cfg) || !vault || !gatedScope(cfg, isPro).toolResults) return result;
|
|
128
|
+
return redactResultShape(result, vault, redactOpts(cfg, isPro, entities));
|
|
131
129
|
}
|
package/sanitize.js
CHANGED
|
@@ -129,5 +129,43 @@ export function stripHidden(text, opts) {
|
|
|
129
129
|
return sanitizeUnicode(text, opts).clean;
|
|
130
130
|
}
|
|
131
131
|
|
|
132
|
+
// ── Confusables skeleton ─────────────────────────────────────────────────────
|
|
133
|
+
// Fold single-code-point Latin LOOK-ALIKES (Cyrillic / Greek / fullwidth) to their
|
|
134
|
+
// ASCII skeleton so a homoglyph-obfuscated value (jоhn@x.com with a Cyrillic 'о')
|
|
135
|
+
// becomes matchable by the ASCII regexes. STRICTLY 1:1 per code point — every mapping
|
|
136
|
+
// is one char → one char — so a match's indices in the skeleton line up exactly with
|
|
137
|
+
// the original text. Use it for DETECTION only and redact the ORIGINAL span, so
|
|
138
|
+
// legitimate Cyrillic/Greek/CJK text is never rewritten (only deceptively-Latin
|
|
139
|
+
// values that actually match a detector get touched). Built from numeric code points
|
|
140
|
+
// (no literal confusables in source, like the rest of this module).
|
|
141
|
+
const CONFUSABLE = new Map([
|
|
142
|
+
// Cyrillic lowercase → Latin
|
|
143
|
+
[0x0430, 'a'], [0x0435, 'e'], [0x043E, 'o'], [0x0440, 'p'], [0x0441, 'c'],
|
|
144
|
+
[0x0443, 'y'], [0x0445, 'x'], [0x0455, 's'], [0x0456, 'i'], [0x0458, 'j'],
|
|
145
|
+
[0x04BB, 'h'], [0x043C, 'm'], [0x043D, 'h'], [0x0442, 't'], [0x043A, 'k'],
|
|
146
|
+
// Cyrillic uppercase → Latin
|
|
147
|
+
[0x0410, 'A'], [0x0412, 'B'], [0x0415, 'E'], [0x041A, 'K'], [0x041C, 'M'],
|
|
148
|
+
[0x041D, 'H'], [0x041E, 'O'], [0x0420, 'P'], [0x0421, 'C'], [0x0422, 'T'],
|
|
149
|
+
[0x0425, 'X'], [0x0406, 'I'], [0x0408, 'J'], [0x0405, 'S'],
|
|
150
|
+
// Greek → Latin
|
|
151
|
+
[0x03BF, 'o'], [0x03C1, 'p'], [0x03B1, 'a'], [0x03BD, 'v'], [0x03B9, 'i'],
|
|
152
|
+
[0x0391, 'A'], [0x0392, 'B'], [0x0395, 'E'], [0x0396, 'Z'], [0x0397, 'H'],
|
|
153
|
+
[0x0399, 'I'], [0x039A, 'K'], [0x039C, 'M'], [0x039D, 'N'], [0x039F, 'O'],
|
|
154
|
+
[0x03A1, 'P'], [0x03A4, 'T'], [0x03A5, 'Y'], [0x03A7, 'X'],
|
|
155
|
+
]);
|
|
156
|
+
|
|
157
|
+
export function confusablesSkeleton(text) {
|
|
158
|
+
if (typeof text !== 'string' || text === '') return text ?? '';
|
|
159
|
+
let out = '';
|
|
160
|
+
for (const ch of text) {
|
|
161
|
+
const cp = ch.codePointAt(0);
|
|
162
|
+
if (cp >= 0xFF01 && cp <= 0xFF5E) { out += String.fromCharCode(cp - 0xFEE0); continue; } // fullwidth ASCII
|
|
163
|
+
const mapped = CONFUSABLE.get(cp);
|
|
164
|
+
out += mapped != null ? mapped : ch;
|
|
165
|
+
}
|
|
166
|
+
return out;
|
|
167
|
+
}
|
|
168
|
+
|
|
132
169
|
// Exposed for tests / external auditing.
|
|
133
170
|
export const SANITIZE_RANGES = RANGES;
|
|
171
|
+
export const CONFUSABLE_MAP = CONFUSABLE;
|
package/tool-harness.js
CHANGED
|
@@ -17,9 +17,9 @@
|
|
|
17
17
|
// Self-contained on the SYNCED engine files (pii-redact.js, tool-rank.js), so the
|
|
18
18
|
// extension (browser ESM) and the gateway (npm) run the exact same code. The caller
|
|
19
19
|
// passes the already-gated `redactOpts` ({tier, entities, dictionary}) it computed
|
|
20
|
-
// from cfg+isPro —
|
|
20
|
+
// from cfg+isPro — tier/dictionary selection stays out of the harness.
|
|
21
21
|
|
|
22
|
-
import { restoreText, restoreWithAliases,
|
|
22
|
+
import { restoreText, restoreWithAliases, redactResultShape } from './pii-redact.js';
|
|
23
23
|
import { narrowSpecs } from './tool-rank.js';
|
|
24
24
|
|
|
25
25
|
// MCP / remote tools are server-prefixed (mcp_server__tool). Local tools
|
|
@@ -77,12 +77,19 @@ export function placeholderToolNote({ toolData = 'real' } = {}) {
|
|
|
77
77
|
return intro + remote + rules;
|
|
78
78
|
}
|
|
79
79
|
|
|
80
|
-
export function makeToolHarness({ vault = null, toolData = 'real', redactOpts = null, redactResults = true } = {}) {
|
|
80
|
+
export function makeToolHarness({ vault = null, toolData = 'real', redactOpts = null, redactResults = true, remoteTools = null } = {}) {
|
|
81
81
|
const on = !!vault; // privacy enabled for this turn?
|
|
82
82
|
const redactRemote = toolData === 'redactRemote';
|
|
83
|
+
// How we decide a tool is REMOTE (must not receive real PII under redactRemote).
|
|
84
|
+
// Prefer an EXPLICIT set/predicate the caller derived from the toolset (a remote
|
|
85
|
+
// tool not named mcp_* would otherwise be misclassified as local and get real
|
|
86
|
+
// values); fall back to the mcp_* name heuristic when the caller passes nothing.
|
|
87
|
+
const isRemoteTool = typeof remoteTools === 'function' ? remoteTools
|
|
88
|
+
: (remoteTools instanceof Set ? (name) => remoteTools.has(name)
|
|
89
|
+
: isRemoteToolName);
|
|
83
90
|
return {
|
|
84
91
|
enabled: on,
|
|
85
|
-
isRemoteTool
|
|
92
|
+
isRemoteTool,
|
|
86
93
|
|
|
87
94
|
// ⓪ Always-on tool selection (privacy-independent). `available` is any spec
|
|
88
95
|
// list; `opts` forwards { cap, keep, name, description } to the shared ranker.
|
|
@@ -93,18 +100,16 @@ export function makeToolHarness({ vault = null, toolData = 'real', redactOpts =
|
|
|
93
100
|
// ② What the tool receives.
|
|
94
101
|
toTool(name, args) {
|
|
95
102
|
if (!on) return args; // privacy off → already real
|
|
96
|
-
if (redactRemote &&
|
|
103
|
+
if (redactRemote && isRemoteTool(name)) return args; // keep PII off remote MCP
|
|
97
104
|
return restoreToolArgs(args, vault); // real values for the tool
|
|
98
105
|
},
|
|
99
106
|
|
|
100
|
-
// ③ What the model sees back (re-redacted).
|
|
107
|
+
// ③ What the model sees back (re-redacted). Walks string / { text } / array /
|
|
108
|
+
// MCP { content:[{text}] } shapes so a tool result can't leak PII to the model
|
|
109
|
+
// via a nested field the old string/{text}-only path skipped.
|
|
101
110
|
toModelResult(name, raw) {
|
|
102
111
|
if (!on || !redactResults || !redactOpts) return raw;
|
|
103
|
-
|
|
104
|
-
if (raw && typeof raw === 'object' && typeof raw.text === 'string') {
|
|
105
|
-
return { ...raw, text: redactText(raw.text, vault, redactOpts) };
|
|
106
|
-
}
|
|
107
|
-
return raw;
|
|
112
|
+
return redactResultShape(raw, vault, redactOpts);
|
|
108
113
|
},
|
|
109
114
|
|
|
110
115
|
// ④ The final reply the user sees.
|