@chatpanel/pii 0.2.10 → 0.2.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.js +3 -1
- package/net.js +103 -0
- package/package.json +4 -2
- package/pii-detect.js +7 -0
- package/pii-redact.js +164 -21
- package/pipeline.js +13 -15
- package/sanitize.js +38 -0
- package/tool-harness.js +16 -11
package/index.js
CHANGED
|
@@ -7,9 +7,10 @@
|
|
|
7
7
|
// Submodules are also importable directly:
|
|
8
8
|
// 'chatpanel-pii/pii-redact.js' deterministic redact/restore + vault
|
|
9
9
|
// 'chatpanel-pii/pii-detect.js' local NER / LLM entity detection
|
|
10
|
-
// 'chatpanel-pii/pipeline.js' pure turn orchestration + tier/scope
|
|
10
|
+
// 'chatpanel-pii/pipeline.js' pure turn orchestration + tier/scope selection
|
|
11
11
|
// 'chatpanel-pii/tool-rank.js' deterministic tool narrowing (auto mode)
|
|
12
12
|
// 'chatpanel-pii/sanitize.js' Unicode de-steganography (strip invisible/format chars)
|
|
13
|
+
// 'chatpanel-pii/net.js' SSRF host classifier + outbound-URL guard
|
|
13
14
|
|
|
14
15
|
export * from './pii-redact.js';
|
|
15
16
|
export * from './pii-detect.js';
|
|
@@ -17,3 +18,4 @@ export * from './pipeline.js';
|
|
|
17
18
|
export * from './tool-rank.js';
|
|
18
19
|
export * from './tool-harness.js';
|
|
19
20
|
export * from './sanitize.js';
|
|
21
|
+
export * from './net.js';
|
package/net.js
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
// Shared host classifier + outbound-URL guard — the SSRF primitive.
|
|
2
|
+
//
|
|
3
|
+
// One implementation of "what is a loopback / cloud-metadata / private host",
|
|
4
|
+
// delivered the way the rest of @chatpanel/pii is: npm dependency for the
|
|
5
|
+
// gateway/bridge, vendorable into the browser extension (pure — only URL + string
|
|
6
|
+
// ops, no node APIs, so it runs in a Worker/service-worker too). Replaces the
|
|
7
|
+
// hand-maintained copies in the bridge (src/ssrf.js) and the extension
|
|
8
|
+
// (js/context.js isBlockedHost) so a security guard can't silently drift between
|
|
9
|
+
// the direct client path and the proxied path. See docs/secure-data-plane.md.
|
|
10
|
+
//
|
|
11
|
+
// The policy knobs cover the two legitimate trust contexts:
|
|
12
|
+
// • A MODEL / API / MCP endpoint (gateway upstream, bridge MCP proxy) may live on
|
|
13
|
+
// loopback (Ollama, LM Studio) or the LAN (a homelab GPU box) — so those are
|
|
14
|
+
// allowed by default — but must NEVER reach cloud instance metadata.
|
|
15
|
+
// • A WEB PAGE fetch (link title, page context) has no business touching loopback
|
|
16
|
+
// or any private host at all — call with { allowLoopback:false, allowPrivate:false }.
|
|
17
|
+
// Cloud metadata (169.254.169.254 & friends) and non-http(s) schemes are blocked in
|
|
18
|
+
// BOTH contexts, unconditionally. Re-run the assert on every redirect hop.
|
|
19
|
+
|
|
20
|
+
function ipv4(h) {
|
|
21
|
+
const m = String(h).match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/);
|
|
22
|
+
if (!m) return null;
|
|
23
|
+
const o = m.slice(1).map(Number);
|
|
24
|
+
if (o.some((n) => n > 255)) return null;
|
|
25
|
+
return o;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const norm = (hostname) => String(hostname || '').toLowerCase().replace(/^\[|\]$/g, '');
|
|
29
|
+
|
|
30
|
+
// Loopback = this host's own services (127.0.0.0/8, ::1, localhost, *.localhost).
|
|
31
|
+
export function isLoopbackHost(hostname) {
|
|
32
|
+
const h = norm(hostname);
|
|
33
|
+
if (!h) return false;
|
|
34
|
+
if (h === 'localhost' || h.endsWith('.localhost')) return true;
|
|
35
|
+
if (h === '::1') return true;
|
|
36
|
+
const o = ipv4(h);
|
|
37
|
+
return !!(o && o[0] === 127);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// Cloud instance metadata — the sharpest SSRF target (credential theft). Covers the
|
|
41
|
+
// link-local IMDS address used by AWS/GCP/Azure/DO (169.254.169.254), Alibaba's
|
|
42
|
+
// 100.100.100.200, and the GCP/name-based metadata hosts. ALWAYS blocked.
|
|
43
|
+
export function isMetadataHost(hostname) {
|
|
44
|
+
const h = norm(hostname);
|
|
45
|
+
if (h === 'metadata.google.internal' || h === 'metadata') return true;
|
|
46
|
+
const o = ipv4(h);
|
|
47
|
+
if (!o) return false;
|
|
48
|
+
if (o[0] === 169 && o[1] === 254) return true; // 169.254.169.254 (+ link-local)
|
|
49
|
+
if (o[0] === 100 && o[1] === 100 && o[2] === 100 && o[3] === 200) return true; // Alibaba IMDS
|
|
50
|
+
return false;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// Private / internal address space, EXCLUDING loopback + metadata (checked
|
|
54
|
+
// separately): RFC1918, CGNAT, IPv6 ULA/link-local, mDNS .local, this-host 0.x/::.
|
|
55
|
+
export function isPrivateHost(hostname) {
|
|
56
|
+
const h = norm(hostname);
|
|
57
|
+
if (!h) return true;
|
|
58
|
+
if (h.endsWith('.local')) return true;
|
|
59
|
+
if (
|
|
60
|
+
h === '::' || h.startsWith('fc') || h.startsWith('fd') // IPv6 ULA
|
|
61
|
+
|| h.startsWith('fe8') || h.startsWith('fe9') || h.startsWith('fea') || h.startsWith('feb') // link-local
|
|
62
|
+
) return true;
|
|
63
|
+
const o = ipv4(h);
|
|
64
|
+
if (o) {
|
|
65
|
+
const [a, b] = o;
|
|
66
|
+
if (a === 0 || a === 10) return true; // this-host / RFC1918
|
|
67
|
+
if (a === 169 && b === 254) return true; // link-local
|
|
68
|
+
if (a === 172 && b >= 16 && b <= 31) return true; // RFC1918
|
|
69
|
+
if (a === 192 && b === 168) return true; // RFC1918
|
|
70
|
+
if (a === 100 && b >= 64 && b <= 127) return true; // CGNAT
|
|
71
|
+
}
|
|
72
|
+
return false;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// Policy-driven classifier. Returns true if `hostname` must be blocked under `policy`.
|
|
76
|
+
// Defaults model the ENDPOINT context (loopback + LAN allowed, metadata never).
|
|
77
|
+
export function isBlockedHost(hostname, { allowLoopback = true, allowPrivate = true } = {}) {
|
|
78
|
+
const h = norm(hostname);
|
|
79
|
+
if (!h) return true;
|
|
80
|
+
if (isMetadataHost(h)) return true; // never, in any context
|
|
81
|
+
if (isLoopbackHost(h)) return !allowLoopback;
|
|
82
|
+
if (isPrivateHost(h)) return !allowPrivate;
|
|
83
|
+
return false; // public host
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// Assert a URL is fetchable under `policy`; returns the parsed URL or throws.
|
|
87
|
+
// Call on the initial URL AND after every redirect hop.
|
|
88
|
+
export function assertFetchableUrl(u, policy = {}) {
|
|
89
|
+
let parsed;
|
|
90
|
+
try { parsed = new URL(u); } catch { throw new Error(`invalid URL: ${u}`); }
|
|
91
|
+
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
|
|
92
|
+
throw new Error(`only http(s) URLs allowed (got "${parsed.protocol}")`);
|
|
93
|
+
}
|
|
94
|
+
if (isBlockedHost(parsed.hostname, policy)) {
|
|
95
|
+
throw new Error(`refusing to reach a blocked address (${parsed.hostname})`);
|
|
96
|
+
}
|
|
97
|
+
return parsed;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// Endpoint context: model/API/MCP upstream — loopback + LAN OK, metadata never.
|
|
101
|
+
export const assertEndpointUrl = (u, opts = {}) => assertFetchableUrl(u, { allowLoopback: true, allowPrivate: true, ...opts });
|
|
102
|
+
// Web-page context: no loopback, no private, no metadata — genuinely public only.
|
|
103
|
+
export const assertPublicWebUrl = (u) => assertFetchableUrl(u, { allowLoopback: false, allowPrivate: false });
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@chatpanel/pii",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.14",
|
|
4
4
|
"description": "The canonical ChatPanel privacy engine — reversible PII redaction + pseudonymization with local entity detection. Pure, dependency-free ESM shared by the ChatPanel extension, gateway, and bridge.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "index.js",
|
|
@@ -11,7 +11,8 @@
|
|
|
11
11
|
"./pipeline.js": "./pipeline.js",
|
|
12
12
|
"./tool-rank.js": "./tool-rank.js",
|
|
13
13
|
"./tool-harness.js": "./tool-harness.js",
|
|
14
|
-
"./sanitize.js": "./sanitize.js"
|
|
14
|
+
"./sanitize.js": "./sanitize.js",
|
|
15
|
+
"./net.js": "./net.js"
|
|
15
16
|
},
|
|
16
17
|
"files": [
|
|
17
18
|
"index.js",
|
|
@@ -21,6 +22,7 @@
|
|
|
21
22
|
"tool-rank.js",
|
|
22
23
|
"tool-harness.js",
|
|
23
24
|
"sanitize.js",
|
|
25
|
+
"net.js",
|
|
24
26
|
"LICENSE",
|
|
25
27
|
"README.md"
|
|
26
28
|
],
|
package/pii-detect.js
CHANGED
|
@@ -13,6 +13,8 @@
|
|
|
13
13
|
// chat — redaction silently falls back to the deterministic layer.
|
|
14
14
|
// - input is length-capped so a huge transcript can't stall detection.
|
|
15
15
|
|
|
16
|
+
import { assertEndpointUrl } from './net.js';
|
|
17
|
+
|
|
16
18
|
const cache = new Map(); // key -> [{value,type}]
|
|
17
19
|
const CACHE_MAX = 300;
|
|
18
20
|
|
|
@@ -151,6 +153,11 @@ export async function detectEntities(text, cfg, { signal, fetchImpl = globalThis
|
|
|
151
153
|
const run = det.backend === 'endpoint' ? detectViaEndpoint : detectViaOpenAI;
|
|
152
154
|
let ents = [];
|
|
153
155
|
try {
|
|
156
|
+
// SSRF guard before RAW (pre-redaction) text leaves for the detector: http(s)
|
|
157
|
+
// only, never cloud metadata. Loopback/LAN allowed — a local NER server / Ollama
|
|
158
|
+
// is the normal case. A blocked URL fails open (deterministic-only), or surfaces
|
|
159
|
+
// to the Test button in strict mode.
|
|
160
|
+
assertEndpointUrl(det.url);
|
|
154
161
|
ents = await withTimeout(run(capped, det, signal, fetchImpl), det.timeoutMs || 1500, signal);
|
|
155
162
|
} catch (e) {
|
|
156
163
|
if (strict) throw e; // surface errors to the Test button
|
package/pii-redact.js
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
// Pure + dependency-free so it is unit-testable and runs identically for API and
|
|
10
10
|
// CLI/bridge agents (both assemble their outbound payload through providers.js).
|
|
11
11
|
//
|
|
12
|
-
// Tiers
|
|
12
|
+
// Tiers:
|
|
13
13
|
// 'basic' — deterministic regex: emails, phones, IPs, cards (Luhn), SSNs, keys.
|
|
14
14
|
// 'full' — basic + entity-aware: known people/orgs (meeting roster, contacts,
|
|
15
15
|
// the user's own identity) and a user-editable custom dictionary.
|
|
@@ -18,6 +18,8 @@
|
|
|
18
18
|
// one reference won't restore (it shows the token) — but the privacy guarantee
|
|
19
19
|
// (the real value never left the device) always holds.
|
|
20
20
|
|
|
21
|
+
import { stripHidden, confusablesSkeleton } from './sanitize.js';
|
|
22
|
+
|
|
21
23
|
const TOKEN_RE = /\[\[([A-Z][A-Z0-9]*)_(\d+)\]\]/g;
|
|
22
24
|
|
|
23
25
|
// Bracket-TOLERANT match of the same token. Smaller models routinely drop or mangle
|
|
@@ -72,6 +74,55 @@ function escapeRegex(s) {
|
|
|
72
74
|
return String(s).replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
73
75
|
}
|
|
74
76
|
|
|
77
|
+
// Reject a user-supplied dictionary regex that is a likely ReDoS (catastrophic
|
|
78
|
+
// backtracking) BEFORE compiling + running it on untrusted-length input. Heuristic,
|
|
79
|
+
// not exhaustive: cap length, and reject the classic nested-quantifier families —
|
|
80
|
+
// a quantified group whose body also has a quantifier ((a+)+ / (a*)* / (.*)+) and
|
|
81
|
+
// back-to-back unbounded quantifiers (a**, .*+). A rejected pattern is skipped like a
|
|
82
|
+
// syntactically-invalid one, so redaction never breaks or hangs.
|
|
83
|
+
function isSafeUserPattern(p) {
|
|
84
|
+
if (typeof p !== 'string' || p.length === 0 || p.length > 200) return false;
|
|
85
|
+
if (/\([^)]*[+*}][^)]*\)\s*[+*{]/.test(p)) return false; // (…quantifier…)quantifier
|
|
86
|
+
if (/[+*]\s*[+*]/.test(p)) return false; // a**, a+*, .*+
|
|
87
|
+
return true;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// Apply many find/replace rules in a SINGLE left-to-right pass over the source.
|
|
91
|
+
// Each rule is { re: <global RegExp>, repl: (match) => string }. Unlike running
|
|
92
|
+
// rule[0].replace then rule[1].replace then …, text emitted by one rule is NEVER
|
|
93
|
+
// re-scanned by a later rule — so substitutions can't cascade (e.g. a pseudonym
|
|
94
|
+
// that happens to equal another entry's input). On a tie at the same position the
|
|
95
|
+
// earlier rule wins (rules carry priority by their order in the array).
|
|
96
|
+
function applyRulesOnce(text, rules) {
|
|
97
|
+
if (!rules || rules.length === 0) return text;
|
|
98
|
+
let out = '';
|
|
99
|
+
let pos = 0;
|
|
100
|
+
const n = text.length;
|
|
101
|
+
while (pos <= n) {
|
|
102
|
+
let best = null;
|
|
103
|
+
let bestRule = null;
|
|
104
|
+
for (const rule of rules) {
|
|
105
|
+
rule.re.lastIndex = pos;
|
|
106
|
+
const m = rule.re.exec(text);
|
|
107
|
+
if (m && (best === null || m.index < best.index)) {
|
|
108
|
+
best = m;
|
|
109
|
+
bestRule = rule;
|
|
110
|
+
if (m.index === pos) break; // nothing can start earlier than the cursor
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
if (!best) { out += text.slice(pos); break; }
|
|
114
|
+
out += text.slice(pos, best.index);
|
|
115
|
+
if (best[0].length === 0) { // pathological empty match — emit a char, advance
|
|
116
|
+
out += text[best.index] ?? '';
|
|
117
|
+
pos = best.index + 1;
|
|
118
|
+
} else {
|
|
119
|
+
out += bestRule.repl(best);
|
|
120
|
+
pos = best.index + best[0].length;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
return out;
|
|
124
|
+
}
|
|
125
|
+
|
|
75
126
|
function luhnValid(digits) {
|
|
76
127
|
let sum = 0;
|
|
77
128
|
let alt = false;
|
|
@@ -84,19 +135,42 @@ function luhnValid(digits) {
|
|
|
84
135
|
return sum % 10 === 0;
|
|
85
136
|
}
|
|
86
137
|
|
|
138
|
+
// Plausible IPv6? Controls false positives from the broad IPV6 regex: accept only a
|
|
139
|
+
// `::`-compressed form (≥1 hextet) or a full 8-hextet address, hextets ≤4 hex digits.
|
|
140
|
+
function isLikelyIpv6(s) {
|
|
141
|
+
if (!/^[0-9A-Fa-f:]+$/.test(s) || (s.match(/:/g) || []).length < 2) return false;
|
|
142
|
+
const parts = s.split(':');
|
|
143
|
+
if (parts.some((p) => p.length > 4)) return false;
|
|
144
|
+
if (s.includes('::')) return parts.filter(Boolean).length >= 1 && parts.filter(Boolean).length <= 7;
|
|
145
|
+
return parts.length === 8 && parts.every((p) => p.length >= 1);
|
|
146
|
+
}
|
|
147
|
+
|
|
87
148
|
// Deterministic detectors. Each: { type, re, valid? }. Order = priority; more
|
|
88
149
|
// specific patterns run first so they win the bytes before greedier ones.
|
|
89
150
|
const DETECTORS = [
|
|
151
|
+
// PEM private-key block (multi-line) — highest priority, most specific.
|
|
152
|
+
{ type: 'SECRET', re: /-----BEGIN (?:[A-Z0-9]+ )*PRIVATE KEY-----[\s\S]*?-----END (?:[A-Z0-9]+ )*PRIVATE KEY-----/g },
|
|
90
153
|
{ type: 'EMAIL', re: /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/g },
|
|
91
|
-
|
|
154
|
+
// SSN: dash- OR space-separated (bare 9-digit is left alone — too false-positive-prone).
|
|
155
|
+
{ type: 'SSN', re: /\b\d{3}[-\s]\d{2}[-\s]\d{4}\b/g },
|
|
92
156
|
{
|
|
157
|
+
// Vendor API keys / tokens. sk-… also covers OpenAI sk-proj-/sk-ant-. Adds Google
|
|
158
|
+
// (AIza…), Stripe (sk_live_/rk_test_…), GitHub fine-grained PATs, Slack xapp-.
|
|
93
159
|
type: 'KEY',
|
|
94
|
-
re: /\b(?:sk-[A-Za-z0-9_-]{16,}|AKIA[0-9A-Z]{16}|gh[pousr]_[A-Za-z0-9]{20,}|xox[baprs]-[A-Za-z0-9-]{10,})\b/g,
|
|
160
|
+
re: /\b(?:sk-[A-Za-z0-9_-]{16,}|AKIA[0-9A-Z]{16}|gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[0-9A-Za-z_]{22,}|xox[baprs]-[A-Za-z0-9-]{10,}|xapp-[0-9]-[A-Za-z0-9-]{10,}|AIza[0-9A-Za-z_-]{35}|[rs]k_(?:live|test)_[0-9A-Za-z]{16,})\b/g,
|
|
95
161
|
},
|
|
162
|
+
// JWT — three base64url segments; `eyJ` is base64 of `{"…`, so this is specific.
|
|
163
|
+
{ type: 'KEY', re: /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{6,}\.[A-Za-z0-9_-]{6,}\b/g },
|
|
96
164
|
{
|
|
97
165
|
type: 'IP',
|
|
98
166
|
re: /\b(?:(?:25[0-5]|2[0-4]\d|1?\d?\d)\.){3}(?:25[0-5]|2[0-4]\d|1?\d?\d)\b/g,
|
|
99
167
|
},
|
|
168
|
+
{
|
|
169
|
+
// IPv6 (incl. :: compression). Broad match narrowed by isLikelyIpv6 to curb FPs.
|
|
170
|
+
type: 'IP',
|
|
171
|
+
re: /(?<![:\w])(?:[0-9A-Fa-f]{0,4}:){2,7}[0-9A-Fa-f]{0,4}(?![:\w])/g,
|
|
172
|
+
valid: (m) => isLikelyIpv6(m),
|
|
173
|
+
},
|
|
100
174
|
{
|
|
101
175
|
// Phone: only count it if it has a separator or a leading + and 7–15 digits —
|
|
102
176
|
// so long bare ids (a 11-digit page id, an order number) are NOT redacted.
|
|
@@ -124,9 +198,16 @@ export function redactText(text, vault, {
|
|
|
124
198
|
tier = 'basic',
|
|
125
199
|
entities = [],
|
|
126
200
|
dictionary = [],
|
|
201
|
+
sanitize = true,
|
|
202
|
+
sanitizeOpts = undefined,
|
|
127
203
|
} = {}) {
|
|
128
204
|
if (text == null || text === '') return text;
|
|
129
|
-
|
|
205
|
+
// De-steganography BEFORE detection, in-band: an obfuscated value
|
|
206
|
+
// (j<ZWSP>o<ZWSP>hn@x.com, homoglyphs, ASCII-smuggled Tag chars) must become
|
|
207
|
+
// matchable so the regex/NER can't be trivially bypassed. Callers used to have to
|
|
208
|
+
// remember to stripHidden() first; folding it in here makes the engine safe on its
|
|
209
|
+
// own — the sanitize:false escape hatch is only for a caller that already did it.
|
|
210
|
+
let out = sanitize ? stripHidden(String(text), sanitizeOpts) : String(text);
|
|
130
211
|
const v = vault || createVault();
|
|
131
212
|
|
|
132
213
|
const entityTier = tier === 'full' || tier === 'entities';
|
|
@@ -135,25 +216,33 @@ export function redactText(text, vault, {
|
|
|
135
216
|
// An entry with `alias` PSEUDONYMIZES: permanent substitution (the model and
|
|
136
217
|
// the user's transcript both see the alias, never reversed). Otherwise it
|
|
137
218
|
// REDACTS to a reversible [[TYPE_n]] placeholder restored in the user's view.
|
|
219
|
+
// All entries are applied in ONE pass (applyRulesOnce): an alias produced by
|
|
220
|
+
// one entry must not be re-matched by a later entry, or substitutions cascade
|
|
221
|
+
// (e.g. value 'Arnav'→alias 'John' then 'John' caught by a later 'John' rule).
|
|
222
|
+
const dictRules = [];
|
|
138
223
|
for (const d of dictionary || []) {
|
|
139
224
|
if (!d) continue;
|
|
225
|
+
let re;
|
|
140
226
|
try {
|
|
141
|
-
|
|
142
|
-
? new RegExp(d.pattern, d.flags && /g/.test(d.flags) ? d.flags : `${d.flags || ''}g`)
|
|
227
|
+
re = d.pattern
|
|
228
|
+
? (isSafeUserPattern(d.pattern) ? new RegExp(d.pattern, d.flags && /g/.test(d.flags) ? d.flags : `${d.flags || ''}g`) : null)
|
|
143
229
|
: (d.value ? new RegExp(`(?<![\\w])${escapeRegex(d.value)}(?![\\w])`, 'gi') : null);
|
|
144
|
-
if (!re) continue;
|
|
145
|
-
if (d.alias != null && d.alias !== '') {
|
|
146
|
-
out = out.replace(re, () => d.alias); // pseudonymize: model + reply see the alias…
|
|
147
|
-
// …but record alias→original so LOCAL tool args (history/meeting search) map
|
|
148
|
-
// back to the real value. Local lookups must hit real data; only the model is blinded.
|
|
149
|
-
if (d.value) v.aliases.set(d.alias, d.value);
|
|
150
|
-
} else {
|
|
151
|
-
out = out.replace(re, (m) => tokenFor(v, d.type || (d.pattern ? 'PII' : 'TERM'), d.pattern ? m : d.value));
|
|
152
|
-
}
|
|
153
230
|
} catch {
|
|
154
|
-
|
|
231
|
+
re = null; // a bad user regex must never break redaction
|
|
232
|
+
}
|
|
233
|
+
if (!re) continue;
|
|
234
|
+
if (d.alias != null && d.alias !== '') {
|
|
235
|
+
// pseudonymize: model + reply see the alias…
|
|
236
|
+
// …but record alias→original so LOCAL tool args (history/meeting search) map
|
|
237
|
+
// back to the real value. Local lookups must hit real data; only the model is blinded.
|
|
238
|
+
if (d.value) v.aliases.set(d.alias, d.value);
|
|
239
|
+
dictRules.push({ re, repl: () => d.alias });
|
|
240
|
+
} else {
|
|
241
|
+
const type = d.type || (d.pattern ? 'PII' : 'TERM');
|
|
242
|
+
dictRules.push({ re, repl: (m) => tokenFor(v, type, d.pattern ? m[0] : d.value) });
|
|
155
243
|
}
|
|
156
244
|
}
|
|
245
|
+
out = applyRulesOnce(out, dictRules);
|
|
157
246
|
|
|
158
247
|
// 2) Known entities (full tier) — longest value first so "Alex Rivera" wins
|
|
159
248
|
// before a bare "Alex". Restores to the canonical entity value.
|
|
@@ -166,13 +255,61 @@ export function redactText(text, vault, {
|
|
|
166
255
|
}
|
|
167
256
|
}
|
|
168
257
|
|
|
169
|
-
// 3) Deterministic detectors (all tiers).
|
|
258
|
+
// 3) Deterministic detectors (all tiers). Detect against a CONFUSABLES SKELETON so
|
|
259
|
+
// homoglyph-obfuscated values (Cyrillic/Greek/fullwidth Latin look-alikes) match
|
|
260
|
+
// the ASCII regexes — but REDACT the ORIGINAL span. The fold is 1:1 per code point,
|
|
261
|
+
// so a skeleton match's indices line up with `out`, and legitimate non-Latin text
|
|
262
|
+
// (which won't match a detector) is never rewritten. Higher-priority detectors
|
|
263
|
+
// (earlier in DETECTORS) claim overlapping spans first, matching the old order.
|
|
264
|
+
const skel = confusablesSkeleton(out);
|
|
265
|
+
const taken = []; // non-overlapping [start,end) spans, in priority order
|
|
266
|
+
const overlaps = (s, e) => taken.some((t) => s < t.end && e > t.start);
|
|
170
267
|
for (const det of DETECTORS) {
|
|
171
|
-
|
|
268
|
+
det.re.lastIndex = 0;
|
|
269
|
+
let m;
|
|
270
|
+
while ((m = det.re.exec(skel)) !== null) {
|
|
271
|
+
if (m[0].length === 0) { det.re.lastIndex++; continue; }
|
|
272
|
+
const start = m.index, end = start + m[0].length;
|
|
273
|
+
if ((det.valid && !det.valid(m[0])) || overlaps(start, end)) continue;
|
|
274
|
+
taken.push({ start, end, type: det.type });
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
if (taken.length) {
|
|
278
|
+
taken.sort((a, b) => a.start - b.start);
|
|
279
|
+
let rebuilt = '';
|
|
280
|
+
let pos = 0;
|
|
281
|
+
for (const t of taken) {
|
|
282
|
+
rebuilt += out.slice(pos, t.start) + tokenFor(v, t.type, out.slice(t.start, t.end));
|
|
283
|
+
pos = t.end;
|
|
284
|
+
}
|
|
285
|
+
out = rebuilt + out.slice(pos);
|
|
172
286
|
}
|
|
173
287
|
return out;
|
|
174
288
|
}
|
|
175
289
|
|
|
290
|
+
// Re-redact a tool RESULT before the model sees it, walking the shapes tools
|
|
291
|
+
// actually return: a bare string, a { text } object, an array, and the
|
|
292
|
+
// MCP-standard { content: [{ type:'text', text }] } (incl. an embedded
|
|
293
|
+
// { resource: { text } }). Only text-bearing fields are redacted — arbitrary
|
|
294
|
+
// fields (ids, urls, mime types) are left intact so tool results stay valid.
|
|
295
|
+
// Restore is the inverse concern; this only runs on the model-facing direction.
|
|
296
|
+
export function redactResultShape(raw, vault, opts) {
|
|
297
|
+
if (raw == null || typeof raw === 'string') {
|
|
298
|
+
return raw == null ? raw : redactText(raw, vault, opts);
|
|
299
|
+
}
|
|
300
|
+
if (Array.isArray(raw)) return raw.map((r) => redactResultShape(r, vault, opts));
|
|
301
|
+
if (typeof raw === 'object') {
|
|
302
|
+
let out = raw;
|
|
303
|
+
if (typeof raw.text === 'string') out = { ...out, text: redactText(raw.text, vault, opts) };
|
|
304
|
+
if (Array.isArray(raw.content)) out = { ...out, content: raw.content.map((c) => redactResultShape(c, vault, opts)) };
|
|
305
|
+
if (raw.resource && typeof raw.resource === 'object' && typeof raw.resource.text === 'string') {
|
|
306
|
+
out = { ...out, resource: { ...raw.resource, text: redactText(raw.resource.text, vault, opts) } };
|
|
307
|
+
}
|
|
308
|
+
return out;
|
|
309
|
+
}
|
|
310
|
+
return raw;
|
|
311
|
+
}
|
|
312
|
+
|
|
176
313
|
// Swap placeholders back to their originals. Unknown tokens are left untouched.
|
|
177
314
|
export function restoreText(text, vault) {
|
|
178
315
|
if (text == null || !vault) return text;
|
|
@@ -188,9 +325,15 @@ export function restoreText(text, vault) {
|
|
|
188
325
|
export function restoreWithAliases(text, vault) {
|
|
189
326
|
let out = restoreText(text, vault);
|
|
190
327
|
if (vault?.aliases?.size) {
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
328
|
+
// ONE pass over every alias at once. Looping `replace` per alias re-scans the
|
|
329
|
+
// output and cascades when one alias's real value equals another alias (e.g.
|
|
330
|
+
// 'Twinkle'→'John' then 'John'→'Arnav'): the model's "Twinkle" would walk the
|
|
331
|
+
// chain to "Arnav". A single alternation replaces each span exactly once.
|
|
332
|
+
// Longest alias first so a multi-word pseudonym wins over its prefix.
|
|
333
|
+
const aliases = [...vault.aliases.keys()].filter(Boolean).sort((a, b) => b.length - a.length);
|
|
334
|
+
if (aliases.length) {
|
|
335
|
+
const re = new RegExp(`(?<![\\w])(?:${aliases.map(escapeRegex).join('|')})(?![\\w])`, 'g');
|
|
336
|
+
out = out.replace(re, (m) => (vault.aliases.has(m) ? vault.aliases.get(m) : m));
|
|
194
337
|
}
|
|
195
338
|
}
|
|
196
339
|
return out;
|
package/pipeline.js
CHANGED
|
@@ -5,27 +5,24 @@
|
|
|
5
5
|
// What lives HERE (portable): redactOutbound, redactToolResult/redactResult,
|
|
6
6
|
// makeStreamRestorer, restore/restoreDeep, effectiveTier + gating.
|
|
7
7
|
//
|
|
8
|
-
//
|
|
9
|
-
//
|
|
10
|
-
//
|
|
8
|
+
// Host glue kept in the EXTENSION (NOT here): reading settings.ui.piiRedaction,
|
|
9
|
+
// the entitlement flag, and chrome storage — those wrap these pure functions with
|
|
10
|
+
// host-specific config.
|
|
11
11
|
|
|
12
|
-
import { redactText, restoreText, restoreWithAliases } from './pii-redact.js';
|
|
12
|
+
import { redactText, restoreText, restoreWithAliases, redactResultShape } from './pii-redact.js';
|
|
13
13
|
|
|
14
14
|
export function redactionEnabled(cfg) {
|
|
15
15
|
return !!(cfg && cfg.mode && cfg.mode !== 'off');
|
|
16
16
|
}
|
|
17
17
|
|
|
18
|
-
// The entity (name/org) tier is Pro; Free
|
|
19
|
-
// regex so the feature still does something useful without the upsell breaking.
|
|
18
|
+
// The entity (name/org) tier is Pro; Free falls back to the deterministic regex tier.
|
|
20
19
|
export function effectiveTier(cfg, isPro) {
|
|
21
20
|
const t = cfg?.tier === 'full' ? 'full' : 'basic';
|
|
22
21
|
return t === 'full' && !isPro ? 'basic' : t;
|
|
23
22
|
}
|
|
24
23
|
|
|
25
|
-
// Free
|
|
26
|
-
//
|
|
27
|
-
// Pro. Enforced here as defense-in-depth — the UI also surfaces the cap, but never
|
|
28
|
-
// trust the UI alone.
|
|
24
|
+
// On Free the first FREE_DICT_LIMIT custom-dictionary entries apply; the full
|
|
25
|
+
// dictionary is Pro. Enforced here as well as in the UI.
|
|
29
26
|
export const FREE_DICT_LIMIT = 5;
|
|
30
27
|
|
|
31
28
|
export function gatedDictionary(cfg, isPro) {
|
|
@@ -122,10 +119,11 @@ export function restoreDeep(value, vault) {
|
|
|
122
119
|
return value;
|
|
123
120
|
}
|
|
124
121
|
|
|
122
|
+
// Re-redact a tool result of any shape (string / { text } / array / MCP
|
|
123
|
+
// { content:[{text}] }), gated once by the toolResults scope. The old path only
|
|
124
|
+
// covered string + { text }, so PII in a content[] item reached the model.
|
|
125
125
|
export function redactResult(result, ctx) {
|
|
126
|
-
|
|
127
|
-
if (
|
|
128
|
-
|
|
129
|
-
}
|
|
130
|
-
return result;
|
|
126
|
+
const { vault, cfg, isPro = false, entities = [] } = ctx || {};
|
|
127
|
+
if (!redactionEnabled(cfg) || !vault || !gatedScope(cfg, isPro).toolResults) return result;
|
|
128
|
+
return redactResultShape(result, vault, redactOpts(cfg, isPro, entities));
|
|
131
129
|
}
|
package/sanitize.js
CHANGED
|
@@ -129,5 +129,43 @@ export function stripHidden(text, opts) {
|
|
|
129
129
|
return sanitizeUnicode(text, opts).clean;
|
|
130
130
|
}
|
|
131
131
|
|
|
132
|
+
// ── Confusables skeleton ─────────────────────────────────────────────────────
|
|
133
|
+
// Fold single-code-point Latin LOOK-ALIKES (Cyrillic / Greek / fullwidth) to their
|
|
134
|
+
// ASCII skeleton so a homoglyph-obfuscated value (jоhn@x.com with a Cyrillic 'о')
|
|
135
|
+
// becomes matchable by the ASCII regexes. STRICTLY 1:1 per code point — every mapping
|
|
136
|
+
// is one char → one char — so a match's indices in the skeleton line up exactly with
|
|
137
|
+
// the original text. Use it for DETECTION only and redact the ORIGINAL span, so
|
|
138
|
+
// legitimate Cyrillic/Greek/CJK text is never rewritten (only deceptively-Latin
|
|
139
|
+
// values that actually match a detector get touched). Built from numeric code points
|
|
140
|
+
// (no literal confusables in source, like the rest of this module).
|
|
141
|
+
const CONFUSABLE = new Map([
|
|
142
|
+
// Cyrillic lowercase → Latin
|
|
143
|
+
[0x0430, 'a'], [0x0435, 'e'], [0x043E, 'o'], [0x0440, 'p'], [0x0441, 'c'],
|
|
144
|
+
[0x0443, 'y'], [0x0445, 'x'], [0x0455, 's'], [0x0456, 'i'], [0x0458, 'j'],
|
|
145
|
+
[0x04BB, 'h'], [0x043C, 'm'], [0x043D, 'h'], [0x0442, 't'], [0x043A, 'k'],
|
|
146
|
+
// Cyrillic uppercase → Latin
|
|
147
|
+
[0x0410, 'A'], [0x0412, 'B'], [0x0415, 'E'], [0x041A, 'K'], [0x041C, 'M'],
|
|
148
|
+
[0x041D, 'H'], [0x041E, 'O'], [0x0420, 'P'], [0x0421, 'C'], [0x0422, 'T'],
|
|
149
|
+
[0x0425, 'X'], [0x0406, 'I'], [0x0408, 'J'], [0x0405, 'S'],
|
|
150
|
+
// Greek → Latin
|
|
151
|
+
[0x03BF, 'o'], [0x03C1, 'p'], [0x03B1, 'a'], [0x03BD, 'v'], [0x03B9, 'i'],
|
|
152
|
+
[0x0391, 'A'], [0x0392, 'B'], [0x0395, 'E'], [0x0396, 'Z'], [0x0397, 'H'],
|
|
153
|
+
[0x0399, 'I'], [0x039A, 'K'], [0x039C, 'M'], [0x039D, 'N'], [0x039F, 'O'],
|
|
154
|
+
[0x03A1, 'P'], [0x03A4, 'T'], [0x03A5, 'Y'], [0x03A7, 'X'],
|
|
155
|
+
]);
|
|
156
|
+
|
|
157
|
+
export function confusablesSkeleton(text) {
|
|
158
|
+
if (typeof text !== 'string' || text === '') return text ?? '';
|
|
159
|
+
let out = '';
|
|
160
|
+
for (const ch of text) {
|
|
161
|
+
const cp = ch.codePointAt(0);
|
|
162
|
+
if (cp >= 0xFF01 && cp <= 0xFF5E) { out += String.fromCharCode(cp - 0xFEE0); continue; } // fullwidth ASCII
|
|
163
|
+
const mapped = CONFUSABLE.get(cp);
|
|
164
|
+
out += mapped != null ? mapped : ch;
|
|
165
|
+
}
|
|
166
|
+
return out;
|
|
167
|
+
}
|
|
168
|
+
|
|
132
169
|
// Exposed for tests / external auditing.
|
|
133
170
|
export const SANITIZE_RANGES = RANGES;
|
|
171
|
+
export const CONFUSABLE_MAP = CONFUSABLE;
|
package/tool-harness.js
CHANGED
|
@@ -17,9 +17,9 @@
|
|
|
17
17
|
// Self-contained on the SYNCED engine files (pii-redact.js, tool-rank.js), so the
|
|
18
18
|
// extension (browser ESM) and the gateway (npm) run the exact same code. The caller
|
|
19
19
|
// passes the already-gated `redactOpts` ({tier, entities, dictionary}) it computed
|
|
20
|
-
// from cfg+isPro —
|
|
20
|
+
// from cfg+isPro — tier/dictionary selection stays out of the harness.
|
|
21
21
|
|
|
22
|
-
import { restoreText, restoreWithAliases,
|
|
22
|
+
import { restoreText, restoreWithAliases, redactResultShape } from './pii-redact.js';
|
|
23
23
|
import { narrowSpecs } from './tool-rank.js';
|
|
24
24
|
|
|
25
25
|
// MCP / remote tools are server-prefixed (mcp_server__tool). Local tools
|
|
@@ -77,12 +77,19 @@ export function placeholderToolNote({ toolData = 'real' } = {}) {
|
|
|
77
77
|
return intro + remote + rules;
|
|
78
78
|
}
|
|
79
79
|
|
|
80
|
-
export function makeToolHarness({ vault = null, toolData = 'real', redactOpts = null, redactResults = true } = {}) {
|
|
80
|
+
export function makeToolHarness({ vault = null, toolData = 'real', redactOpts = null, redactResults = true, remoteTools = null } = {}) {
|
|
81
81
|
const on = !!vault; // privacy enabled for this turn?
|
|
82
82
|
const redactRemote = toolData === 'redactRemote';
|
|
83
|
+
// How we decide a tool is REMOTE (must not receive real PII under redactRemote).
|
|
84
|
+
// Prefer an EXPLICIT set/predicate the caller derived from the toolset (a remote
|
|
85
|
+
// tool not named mcp_* would otherwise be misclassified as local and get real
|
|
86
|
+
// values); fall back to the mcp_* name heuristic when the caller passes nothing.
|
|
87
|
+
const isRemoteTool = typeof remoteTools === 'function' ? remoteTools
|
|
88
|
+
: (remoteTools instanceof Set ? (name) => remoteTools.has(name)
|
|
89
|
+
: isRemoteToolName);
|
|
83
90
|
return {
|
|
84
91
|
enabled: on,
|
|
85
|
-
isRemoteTool
|
|
92
|
+
isRemoteTool,
|
|
86
93
|
|
|
87
94
|
// ⓪ Always-on tool selection (privacy-independent). `available` is any spec
|
|
88
95
|
// list; `opts` forwards { cap, keep, name, description } to the shared ranker.
|
|
@@ -93,18 +100,16 @@ export function makeToolHarness({ vault = null, toolData = 'real', redactOpts =
|
|
|
93
100
|
// ② What the tool receives.
|
|
94
101
|
toTool(name, args) {
|
|
95
102
|
if (!on) return args; // privacy off → already real
|
|
96
|
-
if (redactRemote &&
|
|
103
|
+
if (redactRemote && isRemoteTool(name)) return args; // keep PII off remote MCP
|
|
97
104
|
return restoreToolArgs(args, vault); // real values for the tool
|
|
98
105
|
},
|
|
99
106
|
|
|
100
|
-
// ③ What the model sees back (re-redacted).
|
|
107
|
+
// ③ What the model sees back (re-redacted). Walks string / { text } / array /
|
|
108
|
+
// MCP { content:[{text}] } shapes so a tool result can't leak PII to the model
|
|
109
|
+
// via a nested field the old string/{text}-only path skipped.
|
|
101
110
|
toModelResult(name, raw) {
|
|
102
111
|
if (!on || !redactResults || !redactOpts) return raw;
|
|
103
|
-
|
|
104
|
-
if (raw && typeof raw === 'object' && typeof raw.text === 'string') {
|
|
105
|
-
return { ...raw, text: redactText(raw.text, vault, redactOpts) };
|
|
106
|
-
}
|
|
107
|
-
return raw;
|
|
112
|
+
return redactResultShape(raw, vault, redactOpts);
|
|
108
113
|
},
|
|
109
114
|
|
|
110
115
|
// ④ The final reply the user sees.
|