@privacyscrubber/sdk 2.4.1 → 2.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -0
- package/browser.js +84 -29
- package/cli.js +71 -27
- package/examples/python-bridge/bridge_daemon.cjs +24 -5
- package/examples/python-bridge/fastapi_example.py +1 -1
- package/examples/python-bridge/privacyscrubber/__init__.py +1 -1
- package/examples/python-bridge/privacyscrubber/bridge_daemon.cjs +24 -5
- package/examples/python-bridge/privacyscrubber/cli.py +1 -1
- package/examples/python-bridge/privacyscrubber/client.py +12 -0
- package/examples/python-bridge/privacyscrubber/langchain.py +31 -7
- package/examples/python-bridge/pyproject.toml +1 -1
- package/examples/python-bridge/setup.py +1 -1
- package/examples/python-bridge/test_python_bridge.py +1 -1
- package/fastmcp.cjs +45 -9
- package/index.cjs +67 -18
- package/index.d.ts +3 -2
- package/index.js +67 -18
- package/package.json +12 -1
- package/polyfill.js +31 -13
- package/ps-license-manager.js +1 -1
- package/ps-pii-engine.cjs +1 -1
- package/sarif.cjs +388 -0
- package/sarif.js +37 -0
- package/scrubber-core.cjs +1 -1
- package/vercel-ai.cjs +52 -6
- package/vision.cjs +15 -11
- package/vision.js +15 -11
|
@@ -260,6 +260,9 @@ class PrivacyScrubberClient:
|
|
|
260
260
|
detect_secrets: Optional[bool] = None,
|
|
261
261
|
license_key: Optional[str] = None,
|
|
262
262
|
persist_session: bool = True,
|
|
263
|
+
generate_receipt: bool = True,
|
|
264
|
+
ignore_list: Optional[List[str]] = None,
|
|
265
|
+
custom_rules: Optional[List[Dict[str, Any]]] = None,
|
|
263
266
|
throw_on_limit: bool = True
|
|
264
267
|
) -> ObjectSanitizeResult:
|
|
265
268
|
"""
|
|
@@ -273,6 +276,9 @@ class PrivacyScrubberClient:
|
|
|
273
276
|
detect_secrets=self.detect_secrets if detect_secrets is None else detect_secrets,
|
|
274
277
|
license_key=license_key or self.license_key,
|
|
275
278
|
persist_session=persist_session,
|
|
279
|
+
generate_receipt=generate_receipt,
|
|
280
|
+
ignore_list=ignore_list,
|
|
281
|
+
custom_rules=custom_rules,
|
|
276
282
|
throw_on_limit=throw_on_limit
|
|
277
283
|
)
|
|
278
284
|
return ObjectSanitizeResult(
|
|
@@ -503,6 +509,9 @@ class AsyncPrivacyScrubberClient:
|
|
|
503
509
|
detect_secrets: Optional[bool] = None,
|
|
504
510
|
license_key: Optional[str] = None,
|
|
505
511
|
persist_session: bool = True,
|
|
512
|
+
generate_receipt: bool = True,
|
|
513
|
+
ignore_list: Optional[List[str]] = None,
|
|
514
|
+
custom_rules: Optional[List[Dict[str, Any]]] = None,
|
|
506
515
|
throw_on_limit: bool = True
|
|
507
516
|
) -> ObjectSanitizeResult:
|
|
508
517
|
res = await self._send_request(
|
|
@@ -512,6 +521,9 @@ class AsyncPrivacyScrubberClient:
|
|
|
512
521
|
detect_secrets=self.detect_secrets if detect_secrets is None else detect_secrets,
|
|
513
522
|
license_key=license_key or self.license_key,
|
|
514
523
|
persist_session=persist_session,
|
|
524
|
+
generate_receipt=generate_receipt,
|
|
525
|
+
ignore_list=ignore_list,
|
|
526
|
+
custom_rules=custom_rules,
|
|
515
527
|
throw_on_limit=throw_on_limit
|
|
516
528
|
)
|
|
517
529
|
return ObjectSanitizeResult(
|
|
@@ -74,6 +74,9 @@ class ZTDSCallbackHandler(BaseCallbackHandler):
|
|
|
74
74
|
print(f"[ZTDS Guard] Sanitized {res.count} PII tokens for run {key}")
|
|
75
75
|
|
|
76
76
|
self._session_token_maps[key] = combined_token_map
|
|
77
|
+
if len(self._session_token_maps) > 512:
|
|
78
|
+
oldest_key = next(iter(self._session_token_maps))
|
|
79
|
+
self._session_token_maps.pop(oldest_key, None)
|
|
77
80
|
|
|
78
81
|
def on_llm_end(
|
|
79
82
|
self,
|
|
@@ -92,16 +95,37 @@ class ZTDSCallbackHandler(BaseCallbackHandler):
|
|
|
92
95
|
if generations and isinstance(generations, list):
|
|
93
96
|
for gen_list in generations:
|
|
94
97
|
for gen in gen_list:
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
98
|
+
restored_str: Optional[str] = None
|
|
99
|
+
if hasattr(gen, "message") and hasattr(gen.message, "content"):
|
|
100
|
+
msg_content = gen.message.content
|
|
101
|
+
if isinstance(msg_content, str) and msg_content:
|
|
102
|
+
restored_str = self.scrubber.restore(msg_content, token_map=token_map).restored_text
|
|
103
|
+
gen.message.content = restored_str
|
|
104
|
+
elif isinstance(msg_content, list):
|
|
105
|
+
for block in msg_content:
|
|
106
|
+
if isinstance(block, dict) and isinstance(block.get("text"), str) and block["text"]:
|
|
107
|
+
block["text"] = self.scrubber.restore(block["text"], token_map=token_map).restored_text
|
|
108
|
+
if hasattr(gen, "text") and isinstance(getattr(gen, "text", None), str) and gen.text:
|
|
109
|
+
try:
|
|
110
|
+
gen.text = restored_str if restored_str is not None else self.scrubber.restore(gen.text, token_map=token_map).restored_text
|
|
111
|
+
except (AttributeError, TypeError):
|
|
112
|
+
pass
|
|
102
113
|
|
|
103
114
|
return response
|
|
104
115
|
|
|
116
|
+
def on_llm_error(
|
|
117
|
+
self,
|
|
118
|
+
error: BaseException,
|
|
119
|
+
*,
|
|
120
|
+
run_id: Optional[Any] = None,
|
|
121
|
+
parent_run_id: Optional[Any] = None,
|
|
122
|
+
tags: Optional[List[str]] = None,
|
|
123
|
+
**kwargs: Any
|
|
124
|
+
) -> Any:
|
|
125
|
+
"""Cleans up ephemeral token maps if the LLM call fails."""
|
|
126
|
+
key = str(run_id) if run_id else "default"
|
|
127
|
+
self._session_token_maps.pop(key, None)
|
|
128
|
+
|
|
105
129
|
|
|
106
130
|
class ZTDSDocumentTransformer:
|
|
107
131
|
"""
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "privacyscrubber"
|
|
7
|
-
version = "2.4.
|
|
7
|
+
version = "2.4.3"
|
|
8
8
|
description = "Zero-Trust Data Sanitization & In-Memory PII Shield for Python AI Pipelines (ZTDS RFC v1.0)"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.8"
|
|
@@ -13,7 +13,7 @@ with open(os.path.join(here, "README.md"), encoding="utf-8") as f:
|
|
|
13
13
|
|
|
14
14
|
setup(
|
|
15
15
|
name="privacyscrubber",
|
|
16
|
-
version="2.4.
|
|
16
|
+
version="2.4.3",
|
|
17
17
|
description="Zero-Trust Data Sanitization & In-Memory PII Shield for Python AI Pipelines (ZTDS RFC v1.0)",
|
|
18
18
|
long_description=long_description,
|
|
19
19
|
long_description_content_type="text/markdown",
|
package/fastmcp.cjs
CHANGED
|
@@ -19,12 +19,12 @@ const ZTDS_PATTERNS = {
|
|
|
19
19
|
CREDIT_CARD: /\b(?:\d{4}[-\s]?){3}\d{4}\b/g,
|
|
20
20
|
EMAIL: /[A-Za-z0-9._%+-]{1,64}@[A-Za-z0-9-]{1,63}(?:\.[A-Za-z0-9-]{1,63})*\.[A-Za-z]{2,24}/g,
|
|
21
21
|
IBAN: /\b[A-Z]{2}[0-9]{2}[A-Z0-9]{4}[0-9]{7}(?:[A-Z0-9]?){0,16}\b/g,
|
|
22
|
-
IPV4: /\b(
|
|
22
|
+
IPV4: /\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\b/g,
|
|
23
23
|
PHONE: /(?:(?:\+?1\s*(?:[.-]\s*)?)?(?:\(\s*([2-9]1[02-9]|[2-9][02-8]1|[2-9][02-8][02-9])\s*\)|([2-9]1[02-9]|[2-9][02-8]1|[2-9][02-8][02-9]))\s*(?:[.-]\s*)?)?([2-9]1[02-9]|[2-9][02-8]1|[2-9][02-8]{2})\s*(?:[.-]\s*)?([0-9]{4})\b/g,
|
|
24
24
|
SSN: /\b\d{3}-\d{2}-\d{4}\b/g,
|
|
25
25
|
};
|
|
26
26
|
|
|
27
|
-
const TOKEN_PATTERN = /(?:%5[Bb]|\\u005[Bb]|&(?:amp;)
|
|
27
|
+
const TOKEN_PATTERN = /(?:%5[Bb]|\\u005[Bb]|&(?:amp;)?(?:#0*91;|#[xX]0*5[Bb];|lbrack;|lsqb;)|\[|<|\{\{?|__|【|[|〔|「|『)[*`_~\s]*([A-Z0-9_]+)[*`_~]*[-_\s]+TOKEN[*`_~]*[-_\s]+([a-zA-Z0-9_-]+)[*`_~.,;:!?\s]*(?:%5[Dd]|\\u005[Dd]|&(?:amp;)?(?:#0*93;|#[xX]0*5[Dd];|rbrack;|rsqb;)|\]|>|\}\}?|__|】|]|〕|」|』)/gi;
|
|
28
28
|
|
|
29
29
|
function escapeRegex(s) {
|
|
30
30
|
return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
@@ -166,7 +166,7 @@ class StreamingDetokenizer {
|
|
|
166
166
|
const tailLen = 160;
|
|
167
167
|
const tail = this.pendingBuffer.length > tailLen ? this.pendingBuffer.slice(-tailLen) : this.pendingBuffer;
|
|
168
168
|
const offset = this.pendingBuffer.length > tailLen ? this.pendingBuffer.length - tailLen : 0;
|
|
169
|
-
const m = tail.match(/(?:(?:\[|【|[|〔|「|『|<|\{\{?|__?|%5[Bb]|\\u005[Bb]|&(?:amp;)?(?:#0*91;|#[xX]0*5[Bb];|lbrack;))[A-Za-z0-9_
|
|
169
|
+
const m = tail.match(/(?:(?:\[|【|[|〔|「|『|<|\{\{?|__?|%5[Bb]|\\u005[Bb]|&(?:amp;)?(?:#0*91;|#[xX]0*5[Bb];|lbrack;))[A-Za-z0-9_\-\u0410-\u044F\u0401\u0451 *`~.\\]{0,128}(?:%5?|&(?:a(?:m(?:p(?:;)?)?)?)?(?:#(?:0*9?3?|[xX]0*5?[Dd]?)?|r(?:b(?:r(?:a(?:c(?:k)?)?)?)?)?)?)?|%[0-9A-Fa-f]?|\\(?:u(?:0(?:0(?:5)?)?)?)?|&(?:a(?:m(?:p(?:;)?)?)?)?(?:#(?:0*9?1?|[xX]0*5?)?|l(?:b(?:r(?:a(?:c(?:k)?)?)?)?)?)?)$/);
|
|
170
170
|
|
|
171
171
|
if (m) {
|
|
172
172
|
const candidate = m[0];
|
|
@@ -242,10 +242,11 @@ class ZTDSFastMCPGuard {
|
|
|
242
242
|
this.maxSessionTokens = options.maxSessionTokens || 5000;
|
|
243
243
|
this._sessionMaps = new Map();
|
|
244
244
|
this._entityMaps = new Map();
|
|
245
|
+
this._sessionCounters = new Map();
|
|
245
246
|
this._streamDetokenizers = new Map();
|
|
246
247
|
}
|
|
247
248
|
|
|
248
|
-
_generateToken(entityType, tokenMap) {
|
|
249
|
+
_generateToken(entityType, tokenMap, counters) {
|
|
249
250
|
if (this.useCryptoTokens) {
|
|
250
251
|
for (let i = 0; i < 100; i++) {
|
|
251
252
|
const hex = crypto.randomBytes(4).toString("hex");
|
|
@@ -255,13 +256,25 @@ class ZTDSFastMCPGuard {
|
|
|
255
256
|
}
|
|
256
257
|
}
|
|
257
258
|
}
|
|
259
|
+
if (counters && typeof counters.get === "function") {
|
|
260
|
+
let nextIdx = (counters.get(entityType) || 0) + 1;
|
|
261
|
+
while (tokenMap.has(`[${entityType}_TOKEN_${nextIdx}]`)) {
|
|
262
|
+
nextIdx++;
|
|
263
|
+
}
|
|
264
|
+
counters.set(entityType, nextIdx);
|
|
265
|
+
return `[${entityType}_TOKEN_${nextIdx}]`;
|
|
266
|
+
}
|
|
258
267
|
let count = 0;
|
|
259
268
|
for (const k of tokenMap.keys()) {
|
|
260
269
|
if (k.startsWith(`[${entityType}_TOKEN_`)) {
|
|
261
270
|
count++;
|
|
262
271
|
}
|
|
263
272
|
}
|
|
264
|
-
|
|
273
|
+
let idx = count + 1;
|
|
274
|
+
while (tokenMap.has(`[${entityType}_TOKEN_${idx}]`)) {
|
|
275
|
+
idx++;
|
|
276
|
+
}
|
|
277
|
+
return `[${entityType}_TOKEN_${idx}]`;
|
|
265
278
|
}
|
|
266
279
|
|
|
267
280
|
sanitizeText(text, sessionId) {
|
|
@@ -272,10 +285,12 @@ class ZTDSFastMCPGuard {
|
|
|
272
285
|
if (!this._sessionMaps.has(sessionId)) {
|
|
273
286
|
this._sessionMaps.set(sessionId, new Map());
|
|
274
287
|
this._entityMaps.set(sessionId, new Map());
|
|
288
|
+
this._sessionCounters.set(sessionId, new Map());
|
|
275
289
|
}
|
|
276
290
|
|
|
277
291
|
const tokenMap = this._sessionMaps.get(sessionId);
|
|
278
292
|
const entityMap = this._entityMaps.get(sessionId);
|
|
293
|
+
const counters = this._sessionCounters.get(sessionId);
|
|
279
294
|
let sanitized = text
|
|
280
295
|
.replace(/[\u200B-\u200F\u202A-\u202E\u2060-\u206F\uFEFF\u00AD]/g, "")
|
|
281
296
|
.replace(/[\uFF01-\uFF5E]/g, (ch) => String.fromCharCode(ch.charCodeAt(0) - 0xFEE0));
|
|
@@ -288,11 +303,14 @@ class ZTDSFastMCPGuard {
|
|
|
288
303
|
continue;
|
|
289
304
|
}
|
|
290
305
|
|
|
291
|
-
const regex = new RegExp(`(\\[[A-
|
|
306
|
+
const regex = new RegExp(`(\\[[A-Z0-9_]+_TOKEN_[a-zA-Z0-9_-]+\\])|(?:${pattern.source})`, "g");
|
|
292
307
|
sanitized = sanitized.replace(regex, (match, existingToken) => {
|
|
293
308
|
if (existingToken) {
|
|
294
309
|
return existingToken;
|
|
295
310
|
}
|
|
311
|
+
if (entityType === "IPV4" && (match.startsWith("127.0.0.") || match === "0.0.0.0")) {
|
|
312
|
+
return match;
|
|
313
|
+
}
|
|
296
314
|
if (entityMap.has(match)) {
|
|
297
315
|
return entityMap.get(match);
|
|
298
316
|
}
|
|
@@ -304,7 +322,7 @@ class ZTDSFastMCPGuard {
|
|
|
304
322
|
entityMap.delete(oldestMatch);
|
|
305
323
|
}
|
|
306
324
|
}
|
|
307
|
-
const token = this._generateToken(entityType, tokenMap);
|
|
325
|
+
const token = this._generateToken(entityType, tokenMap, counters);
|
|
308
326
|
tokenMap.set(token, match);
|
|
309
327
|
entityMap.set(match, token);
|
|
310
328
|
return token;
|
|
@@ -328,8 +346,16 @@ class ZTDSFastMCPGuard {
|
|
|
328
346
|
}
|
|
329
347
|
|
|
330
348
|
return text.replace(TOKEN_PATTERN, (match, entityType, tokenId) => {
|
|
331
|
-
const
|
|
332
|
-
|
|
349
|
+
const upperType = entityType.toUpperCase();
|
|
350
|
+
const canonical = `[${upperType}_TOKEN_${tokenId}]`;
|
|
351
|
+
if (tokenMap.has(canonical)) return tokenMap.get(canonical);
|
|
352
|
+
const lowerHex = `[${upperType}_TOKEN_${tokenId.toLowerCase()}]`;
|
|
353
|
+
if (tokenMap.has(lowerHex)) return tokenMap.get(lowerHex);
|
|
354
|
+
if (/^\d+$/.test(tokenId)) {
|
|
355
|
+
const stripped = `[${upperType}_TOKEN_${parseInt(tokenId, 10)}]`;
|
|
356
|
+
if (tokenMap.has(stripped)) return tokenMap.get(stripped);
|
|
357
|
+
}
|
|
358
|
+
return tokenMap.get(match) || match;
|
|
333
359
|
});
|
|
334
360
|
}
|
|
335
361
|
|
|
@@ -340,6 +366,9 @@ class ZTDSFastMCPGuard {
|
|
|
340
366
|
if (Array.isArray(val)) {
|
|
341
367
|
return val.map((item) => this.sanitizeObject(item, sessionId));
|
|
342
368
|
}
|
|
369
|
+
if (val instanceof Date || val instanceof RegExp || val instanceof Uint8Array || (typeof Buffer !== "undefined" && Buffer.isBuffer(val))) {
|
|
370
|
+
return val;
|
|
371
|
+
}
|
|
343
372
|
if (val !== null && typeof val === "object") {
|
|
344
373
|
const result = {};
|
|
345
374
|
for (const [k, v] of Object.entries(val)) {
|
|
@@ -357,6 +386,9 @@ class ZTDSFastMCPGuard {
|
|
|
357
386
|
if (Array.isArray(val)) {
|
|
358
387
|
return val.map((item) => this.restoreObject(item, sessionId));
|
|
359
388
|
}
|
|
389
|
+
if (val instanceof Date || val instanceof RegExp || val instanceof Uint8Array || (typeof Buffer !== "undefined" && Buffer.isBuffer(val))) {
|
|
390
|
+
return val;
|
|
391
|
+
}
|
|
360
392
|
if (val !== null && typeof val === "object") {
|
|
361
393
|
const result = {};
|
|
362
394
|
for (const [k, v] of Object.entries(val)) {
|
|
@@ -525,6 +557,10 @@ class ZTDSFastMCPGuard {
|
|
|
525
557
|
this._entityMaps.get(sessionId).clear();
|
|
526
558
|
this._entityMaps.delete(sessionId);
|
|
527
559
|
}
|
|
560
|
+
if (this._sessionCounters && this._sessionCounters.has(sessionId)) {
|
|
561
|
+
this._sessionCounters.get(sessionId).clear();
|
|
562
|
+
this._sessionCounters.delete(sessionId);
|
|
563
|
+
}
|
|
528
564
|
if (this._streamDetokenizers.has(sessionId)) {
|
|
529
565
|
this._streamDetokenizers.get(sessionId).reset();
|
|
530
566
|
this._streamDetokenizers.delete(sessionId);
|
package/index.cjs
CHANGED
|
@@ -129,7 +129,7 @@ function buildAuditTelemetry(currentTokenMap = {}) {
|
|
|
129
129
|
if (!t) continue;
|
|
130
130
|
const tokenStr = typeof t === 'string' ? t : ((t && (t.token || t.mask)) ? String(t.token || t.mask) : '');
|
|
131
131
|
if (!tokenStr) continue;
|
|
132
|
-
const match = tokenStr.match(/\[([A-
|
|
132
|
+
const match = tokenStr.match(/\[([A-Z0-9_]+)_\d+\]/);
|
|
133
133
|
const baseType = match ? match[1] : (tokenStr.replace(/\[|\]/g, '').replace(/_[0-9]+$/, '') || 'CUSTOM');
|
|
134
134
|
entities[baseType] = (entities[baseType] || 0) + 1;
|
|
135
135
|
totalCount++;
|
|
@@ -238,7 +238,7 @@ const buildCisoAuditTelemetry = buildAuditTelemetry;
|
|
|
238
238
|
function formatAuditReceipt(telemetry) {
|
|
239
239
|
if (!telemetry || telemetry.totalCount === 0) {
|
|
240
240
|
return "> 🛡️ **PrivacyScrubber Audit Receipt**: CLEAN (ZERO PII DETECTED)\n" +
|
|
241
|
-
"> * ⭐ **Star on GitHub:** [moxno/privacyscrubber-mcp](https://github.com/moxno/privacyscrubber-mcp) | **SDK & Enterprise:** [privacyscrubber.com/pricing](https://privacyscrubber.com/pricing)\n";
|
|
241
|
+
"> * ⭐ **Star on GitHub:** [moxno/privacyscrubber-mcp](https://github.com/moxno/privacyscrubber-mcp) | **7-Day B2B Trial:** `npx @privacyscrubber/sdk trial` | **SDK & Enterprise:** [privacyscrubber.com/pricing](https://privacyscrubber.com/pricing/?tier=sdk&utm_source=npm_sdk&utm_medium=audit_receipt)\n";
|
|
242
242
|
}
|
|
243
243
|
const entitiesList = Object.entries(telemetry.entities)
|
|
244
244
|
.map(([type, count]) => `${count} ${type}`)
|
|
@@ -250,7 +250,7 @@ function formatAuditReceipt(telemetry) {
|
|
|
250
250
|
`> * **Risk Level:** ${icon} ${telemetry.riskLevel}\n` +
|
|
251
251
|
`> * **Framework References (informational):** ${telemetry.frameworksList.join(', ')}\n` +
|
|
252
252
|
`> * **Tokens Masked:** ${telemetry.totalCount} (${entitiesList})\n` +
|
|
253
|
-
`> * ⭐ **Star on GitHub:** [moxno/privacyscrubber-mcp](https://github.com/moxno/privacyscrubber-mcp) | **SDK & Enterprise:** [privacyscrubber.com/pricing](https://privacyscrubber.com/pricing)\n`;
|
|
253
|
+
`> * ⭐ **Star on GitHub:** [moxno/privacyscrubber-mcp](https://github.com/moxno/privacyscrubber-mcp) | **7-Day B2B Trial:** \`npx @privacyscrubber/sdk trial\` | **SDK & Enterprise:** [privacyscrubber.com/pricing](https://privacyscrubber.com/pricing/?tier=sdk&utm_source=npm_sdk&utm_medium=audit_receipt)\n`;
|
|
254
254
|
}
|
|
255
255
|
|
|
256
256
|
let communityNoticeLogged = false;
|
|
@@ -259,7 +259,7 @@ function emitCommunityNoticeIfNeeded(isPro) {
|
|
|
259
259
|
if (communityNoticeLogged || isPro) return;
|
|
260
260
|
if (typeof process !== 'undefined' && process.env?.NODE_ENV === 'production') {
|
|
261
261
|
communityNoticeLogged = true;
|
|
262
|
-
console.info('[@privacyscrubber/sdk] Running Community Edition (Core PII).\nTo unlock
|
|
262
|
+
console.info('[@privacyscrubber/sdk] Running Community Edition (Core PII).\nTo unlock 30 Enterprise Profiles & DevOps Secrets in production: https://privacyscrubber.com/pricing/?tier=sdk&utm_source=npm_sdk&utm_medium=prod_notice (7-Day Trial: npx @privacyscrubber/sdk trial | CISO Memo: npx @privacyscrubber/sdk memo)');
|
|
263
263
|
}
|
|
264
264
|
}
|
|
265
265
|
|
|
@@ -291,7 +291,8 @@ function emitDevSecretNoticeIfNeeded(isPro, tokenMap = {}, limit = 15000) {
|
|
|
291
291
|
process.stderr.write(
|
|
292
292
|
`\n[PrivacyScrubber] Intercepted ${count} API secret(s) before LLM egress: ${secretSample}\n` +
|
|
293
293
|
`[PrivacyScrubber] Protected in local RAM (0 byte network egress). Free Tier active (${limit.toLocaleString()} char cap).\n` +
|
|
294
|
-
`[PrivacyScrubber] Developer SDK ($299/mo flat) for unlimited backend nodes, CI/CD gates & ETL pipelines: https://privacyscrubber.com/pricing?tier=sdk&src=cli_leak_guard\n
|
|
294
|
+
`[PrivacyScrubber] Developer SDK ($299/mo flat) for unlimited backend nodes, CI/CD gates & ETL pipelines: https://privacyscrubber.com/pricing?tier=sdk&src=cli_leak_guard\n` +
|
|
295
|
+
`[PrivacyScrubber] Instant 7-Day Free B2B Trial: npx @privacyscrubber/sdk trial | CISO Approval Memo: npx @privacyscrubber/sdk memo\n\n`
|
|
295
296
|
);
|
|
296
297
|
}
|
|
297
298
|
}
|
|
@@ -454,7 +455,11 @@ function sanitize(text, options = {}) {
|
|
|
454
455
|
if (!text || typeof text !== 'string') {
|
|
455
456
|
return {
|
|
456
457
|
scrubbedText: '',
|
|
458
|
+
scrubbed: '',
|
|
457
459
|
tokenMap: {},
|
|
460
|
+
sessionMap: {},
|
|
461
|
+
matches: [],
|
|
462
|
+
spans: [],
|
|
458
463
|
count: 0,
|
|
459
464
|
executionMs: 0,
|
|
460
465
|
license: { isPro: false, tier: 'FREE', limit: 15000, wasTruncated: false },
|
|
@@ -465,7 +470,7 @@ function sanitize(text, options = {}) {
|
|
|
465
470
|
|
|
466
471
|
const {
|
|
467
472
|
profile = 'General',
|
|
468
|
-
licenseKey = (typeof process !== 'undefined' && process.env?.PRIVACYSCRUBBER_KEY) || '',
|
|
473
|
+
licenseKey = (typeof process !== 'undefined' && (process.env?.PRIVACYSCRUBBER_KEY || process.env?.PRIVACYSCRUBBER_LICENSE_KEY)) || '',
|
|
469
474
|
customRules = [],
|
|
470
475
|
tokenLabelMap = {},
|
|
471
476
|
existingSessionMap = {},
|
|
@@ -478,6 +483,18 @@ function sanitize(text, options = {}) {
|
|
|
478
483
|
|
|
479
484
|
const licenseCheck = validateLicense(licenseKey);
|
|
480
485
|
const rawTier = licenseCheck.valid ? (licenseCheck.tier || 'PRO') : 'FREE';
|
|
486
|
+
const isExpiredKey = !licenseCheck.valid && typeof licenseCheck.error === 'string' && /expired/i.test(licenseCheck.error);
|
|
487
|
+
const isExpiringSoon = licenseCheck.valid && typeof licenseCheck.expires === 'number' && (licenseCheck.expires - Date.now() <= 48 * 3600 * 1000);
|
|
488
|
+
|
|
489
|
+
if (isExpiringSoon && typeof process !== 'undefined' && process.stderr && !globalThis.__psSdkPreExpiryLogged) {
|
|
490
|
+
globalThis.__psSdkPreExpiryLogged = true;
|
|
491
|
+
const hrs = Math.max(0, Math.round((licenseCheck.expires - Date.now()) / 3600000));
|
|
492
|
+
process.stderr.write(
|
|
493
|
+
`\n[@privacyscrubber/sdk] ⏳ 7-Day Evaluation Key expires in ~${hrs}h. Prevent pipeline interruption:\n` +
|
|
494
|
+
` - Commercial SDK ($299/mo flat): https://privacyscrubber.com/pricing/?tier=sdk&utm_source=npm_sdk&utm_medium=pilot_expiring_48h\n` +
|
|
495
|
+
` - CISO Approval Memo: npx @privacyscrubber/sdk memo\n\n`
|
|
496
|
+
);
|
|
497
|
+
}
|
|
481
498
|
|
|
482
499
|
// Anti-Cannibalization Barrier (SSOT):
|
|
483
500
|
// PRO ($15/mo) and TEAMS ($99/mo) licenses are strictly human-seat licenses for Web App & Chrome Extension.
|
|
@@ -511,7 +528,8 @@ function sanitize(text, options = {}) {
|
|
|
511
528
|
|
|
512
529
|
if (!isPro && text.length > limit) {
|
|
513
530
|
if (shouldThrow) {
|
|
514
|
-
const
|
|
531
|
+
const expiredPrefix = isExpiredKey ? `[PILOT/TRIAL EXPIRED: ${licenseCheck.error}] ` : '';
|
|
532
|
+
const errorMsg = `[PrivacyScrubber] ${expiredPrefix}Free Tier character limit exceeded (${limit.toLocaleString()} chars for profile '${profile}', received ${text.length.toLocaleString()} chars). Upgrade to Developer SDK ($299/mo) for unlimited throughput: https://privacyscrubber.com/pricing?tier=sdk&src=${isExpiredKey ? 'pilot_expired' : 'dev_limit'} (Run 'npx @privacyscrubber/sdk memo' for CISO approval memo)`;
|
|
515
533
|
throw new PrivacyScrubberLicenseError(errorMsg, {
|
|
516
534
|
tier: 'FREE',
|
|
517
535
|
currentLength: text.length,
|
|
@@ -571,6 +589,7 @@ function sanitize(text, options = {}) {
|
|
|
571
589
|
|
|
572
590
|
return {
|
|
573
591
|
scrubbedText: finalScrubbedText,
|
|
592
|
+
scrubbed: finalScrubbedText,
|
|
574
593
|
tokenMap: returnFullSessionMap ? sessionMap : (rawResult.tokenMap || {}),
|
|
575
594
|
sessionMap: sessionMap,
|
|
576
595
|
matches: rawResult.matches || [],
|
|
@@ -885,6 +904,9 @@ class PrivacyScrubberEngine {
|
|
|
885
904
|
* @param {object} [options={}]
|
|
886
905
|
*/
|
|
887
906
|
createStreamRestorer(tokenMap = null, options = {}) {
|
|
907
|
+
if (tokenMap && typeof tokenMap === 'object' && typeof tokenMap.onTokenRestored === 'function' && Object.keys(options).length === 0) {
|
|
908
|
+
return createStreamTokenRestorer(this.sessionMap, tokenMap);
|
|
909
|
+
}
|
|
888
910
|
const activeMap = tokenMap || this.sessionMap;
|
|
889
911
|
return createStreamTokenRestorer(activeMap, options);
|
|
890
912
|
}
|
|
@@ -1635,7 +1657,7 @@ class StreamingDetokenizer {
|
|
|
1635
1657
|
const tailLen = 160;
|
|
1636
1658
|
const tail = this.pendingBuffer.length > tailLen ? this.pendingBuffer.slice(-tailLen) : this.pendingBuffer;
|
|
1637
1659
|
const offset = this.pendingBuffer.length > tailLen ? this.pendingBuffer.length - tailLen : 0;
|
|
1638
|
-
const m = tail.match(/(?:(?:\[|【|[|〔|「|『|<|\{\{?|__?|%5[Bb]|\\u005[Bb]|&(?:amp;)?(?:#0*91;|#[xX]0*5[Bb];|lbrack;))[A-Za-z0-9_
|
|
1660
|
+
const m = tail.match(/(?:(?:\[|【|[|〔|「|『|<|\{\{?|__?|%5[Bb]|\\u005[Bb]|&(?:amp;)?(?:#0*91;|#[xX]0*5[Bb];|lbrack;))[A-Za-z0-9_\-\u0410-\u044F\u0401\u0451 *`~.\\]{0,128}(?:%5?|&(?:a(?:m(?:p(?:;)?)?)?)?(?:#(?:0*9?3?|[xX]0*5?[Dd]?)?|r(?:b(?:r(?:a(?:c(?:k)?)?)?)?)?)?)?|%[0-9A-Fa-f]?|\\(?:u(?:0(?:0(?:5)?)?)?)?|&(?:a(?:m(?:p(?:;)?)?)?)?(?:#(?:0*9?1?|[xX]0*5?)?|l(?:b(?:r(?:a(?:c(?:k)?)?)?)?)?)?)$/);
|
|
1639
1661
|
|
|
1640
1662
|
if (m) {
|
|
1641
1663
|
const candidate = m[0];
|
|
@@ -1724,16 +1746,20 @@ class StreamingDetokenizer {
|
|
|
1724
1746
|
* @returns {AsyncGenerator<any>}
|
|
1725
1747
|
*/
|
|
1726
1748
|
async* wrapStream(asyncIterable) {
|
|
1749
|
+
let lastOpenAIChunk = null;
|
|
1727
1750
|
for await (const chunk of asyncIterable) {
|
|
1728
1751
|
if (chunk === null || chunk === undefined) continue;
|
|
1729
1752
|
// Handle OpenAI chunk object
|
|
1730
1753
|
if (chunk.choices && Array.isArray(chunk.choices) && chunk.choices[0]?.delta?.content !== undefined) {
|
|
1754
|
+
lastOpenAIChunk = chunk;
|
|
1731
1755
|
const delta = chunk.choices[0].delta.content;
|
|
1732
1756
|
if (typeof delta === 'string') {
|
|
1733
1757
|
const restored = this.feed(delta);
|
|
1734
1758
|
if (restored) {
|
|
1735
|
-
chunk.choices
|
|
1736
|
-
|
|
1759
|
+
const clonedChoices = chunk.choices.map((c, idx) =>
|
|
1760
|
+
idx === 0 ? { ...c, delta: { ...c.delta, content: restored } } : c
|
|
1761
|
+
);
|
|
1762
|
+
yield { ...chunk, choices: clonedChoices };
|
|
1737
1763
|
}
|
|
1738
1764
|
} else {
|
|
1739
1765
|
yield chunk;
|
|
@@ -1759,7 +1785,16 @@ class StreamingDetokenizer {
|
|
|
1759
1785
|
}
|
|
1760
1786
|
|
|
1761
1787
|
const finalChunk = this.flush();
|
|
1762
|
-
if (finalChunk)
|
|
1788
|
+
if (finalChunk) {
|
|
1789
|
+
if (lastOpenAIChunk) {
|
|
1790
|
+
const clonedChoices = lastOpenAIChunk.choices.map((c, idx) =>
|
|
1791
|
+
idx === 0 ? { ...c, delta: { ...c.delta, content: finalChunk } } : c
|
|
1792
|
+
);
|
|
1793
|
+
yield { ...lastOpenAIChunk, choices: clonedChoices };
|
|
1794
|
+
} else {
|
|
1795
|
+
yield finalChunk;
|
|
1796
|
+
}
|
|
1797
|
+
}
|
|
1763
1798
|
}
|
|
1764
1799
|
}
|
|
1765
1800
|
|
|
@@ -1826,8 +1861,8 @@ function createNodeStreamTokenRestorer(tokenMap = {}, options = {}) {
|
|
|
1826
1861
|
* @param {Record<string, string>} tokenMap - Volatile token map.
|
|
1827
1862
|
* @returns {AsyncGenerator<any>}
|
|
1828
1863
|
*/
|
|
1829
|
-
async function* wrapAiStream(stream, tokenMap = {}) {
|
|
1830
|
-
const detok = new StreamingDetokenizer(tokenMap);
|
|
1864
|
+
async function* wrapAiStream(stream, tokenMap = {}, options = {}) {
|
|
1865
|
+
const detok = new StreamingDetokenizer(tokenMap, options);
|
|
1831
1866
|
yield* detok.wrapStream(stream);
|
|
1832
1867
|
}
|
|
1833
1868
|
|
|
@@ -1917,7 +1952,7 @@ function formatCefEvent(eventData = {}) {
|
|
|
1917
1952
|
const {
|
|
1918
1953
|
vendor = 'PrivacyScrubber',
|
|
1919
1954
|
product = 'ZTDS',
|
|
1920
|
-
version = '2.4.
|
|
1955
|
+
version = '2.4.3',
|
|
1921
1956
|
eventClassId = 'PII_REDACTION',
|
|
1922
1957
|
eventName = 'Zero-Trust PII Sanitization Executed',
|
|
1923
1958
|
riskLevel = 'LOW EXPOSURE',
|
|
@@ -1986,7 +2021,7 @@ function formatSyslogEvent(eventData = {}) {
|
|
|
1986
2021
|
function formatJsonlEvent(eventData = {}) {
|
|
1987
2022
|
const {
|
|
1988
2023
|
timestamp = new Date().toISOString(),
|
|
1989
|
-
version = '2.4.
|
|
2024
|
+
version = '2.4.3',
|
|
1990
2025
|
protocol = 'ZTDS',
|
|
1991
2026
|
event = 'sanitization',
|
|
1992
2027
|
riskLevel = 'LOW EXPOSURE',
|
|
@@ -2175,7 +2210,7 @@ function buildPsr2Receipt(tokenMap, timestamp = new Date().toISOString()) {
|
|
|
2175
2210
|
const labels = (tokenMap && typeof tokenMap === 'object' ? Object.keys(tokenMap) : []).slice().sort();
|
|
2176
2211
|
const counts = {};
|
|
2177
2212
|
for (const t of labels) {
|
|
2178
|
-
const m = t.match(
|
|
2213
|
+
const m = String(t).trim().match(/^(?:\[|<|\{\{|__)([A-Z0-9_]+)_\d+(?:\]|>|\}\}|__)$/);
|
|
2179
2214
|
const type = m ? m[1] : 'CUSTOM';
|
|
2180
2215
|
counts[type] = (counts[type] || 0) + 1;
|
|
2181
2216
|
}
|
|
@@ -2691,8 +2726,22 @@ function createGuardedTools(engineOrOptions = {}, toolConfig = {}) {
|
|
|
2691
2726
|
return tools;
|
|
2692
2727
|
}
|
|
2693
2728
|
|
|
2694
|
-
// Low-level backwards-compatible exports
|
|
2695
|
-
|
|
2729
|
+
// Low-level backwards-compatible exports (supports both positional customRules array and high-level options object)
|
|
2730
|
+
function scrubText(text, customRulesOrOptions = [], ...args) {
|
|
2731
|
+
if (
|
|
2732
|
+
customRulesOrOptions &&
|
|
2733
|
+
typeof customRulesOrOptions === 'object' &&
|
|
2734
|
+
!Array.isArray(customRulesOrOptions) &&
|
|
2735
|
+
args.length === 0
|
|
2736
|
+
) {
|
|
2737
|
+
return sanitize(text, customRulesOrOptions);
|
|
2738
|
+
}
|
|
2739
|
+
const res = scrubberPkg.scrubText(text, customRulesOrOptions, ...args);
|
|
2740
|
+
if (res && typeof res === 'object' && typeof res.scrubbed === 'undefined') {
|
|
2741
|
+
res.scrubbed = res.scrubbedText;
|
|
2742
|
+
}
|
|
2743
|
+
return res;
|
|
2744
|
+
}
|
|
2696
2745
|
const unscrubText = scrubberPkg.unscrubText;
|
|
2697
2746
|
|
|
2698
2747
|
// Vision & Multimodal exports
|