@privacyscrubber/sdk 2.4.2 → 2.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -0
- package/browser.js +84 -29
- package/cli.js +65 -27
- package/examples/python-bridge/bridge_daemon.cjs +24 -5
- package/examples/python-bridge/fastapi_example.py +1 -1
- package/examples/python-bridge/privacyscrubber/__init__.py +1 -1
- package/examples/python-bridge/privacyscrubber/bridge_daemon.cjs +24 -5
- package/examples/python-bridge/privacyscrubber/cli.py +1 -1
- package/examples/python-bridge/privacyscrubber/client.py +12 -0
- package/examples/python-bridge/privacyscrubber/langchain.py +31 -7
- package/examples/python-bridge/pyproject.toml +1 -1
- package/examples/python-bridge/setup.py +1 -1
- package/examples/python-bridge/test_python_bridge.py +1 -1
- package/fastmcp.cjs +45 -9
- package/index.cjs +48 -13
- package/index.d.ts +3 -2
- package/index.js +48 -13
- package/package.json +8 -1
- package/polyfill.js +31 -13
- package/ps-license-manager.js +11 -261
- package/ps-pii-engine.cjs +8 -3611
- package/sarif.cjs +388 -0
- package/sarif.js +37 -0
- package/scrubber-core.cjs +8 -1801
- package/vercel-ai.cjs +52 -6
- package/vision.cjs +15 -11
- package/vision.js +15 -11
package/README.md
CHANGED
|
@@ -139,6 +139,29 @@ console.log(restoredText);
|
|
|
139
139
|
// "I have queued an SMS notification for John Doe at 555-0199."
|
|
140
140
|
```
|
|
141
141
|
|
|
142
|
+
#### Custom Rules (`customRules`), Domain Exclusions (`ignoreList`) & Profile Selection
|
|
143
|
+
You can pass `customRules` (for internal ERP/project formats) and `ignoreList` (to preserve domain-specific nouns or allowlisted identifiers in cleartext) into `sanitize()` across any profile (`'General'`, `'Manufacturing'`, `'Sales'`, `'Support'`, etc.):
|
|
144
|
+
|
|
145
|
+
```javascript
|
|
146
|
+
import { sanitize } from '@privacyscrubber/sdk';
|
|
147
|
+
|
|
148
|
+
const { scrubbedText, tokenMap } = sanitize(rawCrmPayload, {
|
|
149
|
+
// Profile guidance:
|
|
150
|
+
// - 'Manufacturing' (or 'Engineering'): masks shopfloor/plant IDs (e.g., "Linie 3", "Station 4", SAP order numbers) alongside general PII.
|
|
151
|
+
// - 'Sales' / 'Support' / 'General': preserves production line numbers in cleartext while masking personal names, emails, phones, meeting links, and CRM identifiers.
|
|
152
|
+
profile: 'Manufacturing',
|
|
153
|
+
|
|
154
|
+
// Add custom regex or exact literal rules (works on all profiles):
|
|
155
|
+
customRules: [
|
|
156
|
+
{ name: 'Internal Project Code', pattern: 'PRJ-\\d{4,6}', label: 'PROJECT' },
|
|
157
|
+
{ name: 'Custom Drawing ID', regex: /\bDRW-[A-Z0-9-]{5,12}\b/g, label: 'DRAWING' }
|
|
158
|
+
],
|
|
159
|
+
|
|
160
|
+
// Exclude specific domain terms or false-positive nouns (case-insensitive; also prevents multi-word NAME matches containing these terms):
|
|
161
|
+
ignoreList: ['Materialmangel', 'Störgrund', 'Prüfstand', 'Linie 3']
|
|
162
|
+
});
|
|
163
|
+
```
|
|
164
|
+
|
|
142
165
|
---
|
|
143
166
|
|
|
144
167
|
### 3. Vercel AI SDK Integration (`@privacyscrubber/sdk/vercel-ai`)
|
package/browser.js
CHANGED
|
@@ -114,7 +114,7 @@ export function buildAuditTelemetry(currentTokenMap = {}) {
|
|
|
114
114
|
if (!t) continue;
|
|
115
115
|
const tokenStr = typeof t === 'string' ? t : ((t && (t.token || t.mask)) ? String(t.token || t.mask) : '');
|
|
116
116
|
if (!tokenStr) continue;
|
|
117
|
-
const match = tokenStr.match(/\[([A-
|
|
117
|
+
const match = tokenStr.match(/\[([A-Z0-9_]+)_\d+\]/);
|
|
118
118
|
const baseType = match ? match[1] : (tokenStr.replace(/\[|\]/g, '').replace(/_[0-9]+$/, '') || 'CUSTOM');
|
|
119
119
|
entities[baseType] = (entities[baseType] || 0) + 1;
|
|
120
120
|
totalCount++;
|
|
@@ -237,7 +237,11 @@ export function sanitize(text, options = {}) {
|
|
|
237
237
|
if (!text || typeof text !== 'string') {
|
|
238
238
|
return {
|
|
239
239
|
scrubbedText: '',
|
|
240
|
+
scrubbed: '',
|
|
240
241
|
tokenMap: {},
|
|
242
|
+
sessionMap: {},
|
|
243
|
+
matches: [],
|
|
244
|
+
spans: [],
|
|
241
245
|
count: 0,
|
|
242
246
|
executionMs: 0,
|
|
243
247
|
license: { isPro: false, tier: 'FREE', limit: 15000, wasTruncated: false },
|
|
@@ -344,9 +348,16 @@ export function sanitize(text, options = {}) {
|
|
|
344
348
|
finalScrubbedText += `\n\n<!-- [PrivacyScrubber] Input truncated at free tier limit (${limit.toLocaleString()} chars). For unlimited throughput: https://privacyscrubber.com/pricing?tier=sdk&src=truncation -->`;
|
|
345
349
|
}
|
|
346
350
|
|
|
351
|
+
const returnFullSessionMap = options.returnFullSessionMap === true;
|
|
352
|
+
const sessionMap = { ...(existingSessionMap || {}), ...(rawResult.tokenMap || {}) };
|
|
353
|
+
|
|
347
354
|
return {
|
|
348
355
|
scrubbedText: finalScrubbedText,
|
|
349
|
-
|
|
356
|
+
scrubbed: finalScrubbedText,
|
|
357
|
+
tokenMap: returnFullSessionMap ? sessionMap : (rawResult.tokenMap || {}),
|
|
358
|
+
sessionMap: sessionMap,
|
|
359
|
+
matches: rawResult.matches || [],
|
|
360
|
+
spans: rawResult.matches || [],
|
|
350
361
|
count: rawResult.count || 0,
|
|
351
362
|
executionMs: executionMs,
|
|
352
363
|
license: {
|
|
@@ -391,6 +402,7 @@ export function restore(aiResponse, tokenMap = {}) {
|
|
|
391
402
|
}
|
|
392
403
|
|
|
393
404
|
export const unscrub = restore;
|
|
405
|
+
export const desanitize = restore;
|
|
394
406
|
|
|
395
407
|
/**
|
|
396
408
|
* Recursively sanitizes all string values within an object or array.
|
|
@@ -517,6 +529,7 @@ export class PrivacyScrubberEngine {
|
|
|
517
529
|
this.generateReceipt = config.generateReceipt !== false;
|
|
518
530
|
this.sessionMap = { ...(config.initialSessionMap || {}) };
|
|
519
531
|
this.ignoreList = new Set(config.ignoreList || []);
|
|
532
|
+
this.maxSessionTokens = config.maxSessionTokens || 10000;
|
|
520
533
|
this.license = validateLicense(this.licenseKey);
|
|
521
534
|
}
|
|
522
535
|
|
|
@@ -548,6 +561,14 @@ export class PrivacyScrubberEngine {
|
|
|
548
561
|
const result = sanitize(text, mergedOptions);
|
|
549
562
|
if (options.persistSession !== false) {
|
|
550
563
|
Object.assign(this.sessionMap, result.tokenMap);
|
|
564
|
+
const maxTokens = options.maxSessionTokens || this.maxSessionTokens || 10000;
|
|
565
|
+
const keys = Object.keys(this.sessionMap);
|
|
566
|
+
if (keys.length > maxTokens) {
|
|
567
|
+
const pruneCount = keys.length - maxTokens;
|
|
568
|
+
for (let i = 0; i < pruneCount; i++) {
|
|
569
|
+
delete this.sessionMap[keys[i]];
|
|
570
|
+
}
|
|
571
|
+
}
|
|
551
572
|
}
|
|
552
573
|
return result;
|
|
553
574
|
}
|
|
@@ -563,8 +584,31 @@ export class PrivacyScrubberEngine {
|
|
|
563
584
|
}
|
|
564
585
|
}
|
|
565
586
|
|
|
566
|
-
|
|
567
|
-
|
|
587
|
+
markFalsePositive(token) {
|
|
588
|
+
const original = this.sessionMap[token];
|
|
589
|
+
if (!original) {
|
|
590
|
+
return { success: false, reason: `Token '${token}' not found in session map` };
|
|
591
|
+
}
|
|
592
|
+
this.ignoreList.add(original);
|
|
593
|
+
delete this.sessionMap[token];
|
|
594
|
+
return { success: true, token, original, ignoreListSize: this.ignoreList.size };
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
clearIgnoreList() {
|
|
598
|
+
this.ignoreList.clear();
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
createStreamRestorer(tokenMap = null, options = {}) {
|
|
602
|
+
if (tokenMap && typeof tokenMap === 'object' && typeof tokenMap.onTokenRestored === 'function' && Object.keys(options).length === 0) {
|
|
603
|
+
return createStreamTokenRestorer(this.sessionMap, tokenMap);
|
|
604
|
+
}
|
|
605
|
+
const activeMap = tokenMap || this.sessionMap;
|
|
606
|
+
return createStreamTokenRestorer(activeMap, options);
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
async* wrapStream(stream, tokenMap = null, options = {}) {
|
|
610
|
+
const activeMap = tokenMap || this.sessionMap;
|
|
611
|
+
yield* wrapAiStream(stream, activeMap, options);
|
|
568
612
|
}
|
|
569
613
|
}
|
|
570
614
|
|
|
@@ -749,7 +793,7 @@ export class StreamingDetokenizer {
|
|
|
749
793
|
const tailLen = 160;
|
|
750
794
|
const tail = this.pendingBuffer.length > tailLen ? this.pendingBuffer.slice(-tailLen) : this.pendingBuffer;
|
|
751
795
|
const offset = this.pendingBuffer.length > tailLen ? this.pendingBuffer.length - tailLen : 0;
|
|
752
|
-
const m = tail.match(/(?:(?:\[|【|[|〔|「|『|<|\{\{?|__?|%5[Bb]|\\u005[Bb]|&(?:amp;)?(?:#0*91;|#[xX]0*5[Bb];|lbrack;))[A-Za-z0-9_
|
|
796
|
+
const m = tail.match(/(?:(?:\[|【|[|〔|「|『|<|\{\{?|__?|%5[Bb]|\\u005[Bb]|&(?:amp;)?(?:#0*91;|#[xX]0*5[Bb];|lbrack;))[A-Za-z0-9_\-\u0410-\u044F\u0401\u0451 *`~.\\]{0,128}(?:%5?|&(?:a(?:m(?:p(?:;)?)?)?)?(?:#(?:0*9?3?|[xX]0*5?[Dd]?)?|r(?:b(?:r(?:a(?:c(?:k)?)?)?)?)?)?)?|%[0-9A-Fa-f]?|\\(?:u(?:0(?:0(?:5)?)?)?)?|&(?:a(?:m(?:p(?:;)?)?)?)?(?:#(?:0*9?1?|[xX]0*5?)?|l(?:b(?:r(?:a(?:c(?:k)?)?)?)?)?)?)$/);
|
|
753
797
|
|
|
754
798
|
if (m) {
|
|
755
799
|
const candidate = m[0];
|
|
@@ -804,15 +848,19 @@ export class StreamingDetokenizer {
|
|
|
804
848
|
* @returns {AsyncGenerator<any>}
|
|
805
849
|
*/
|
|
806
850
|
async* wrapStream(asyncIterable) {
|
|
851
|
+
let lastOpenAIChunk = null;
|
|
807
852
|
for await (const chunk of asyncIterable) {
|
|
808
853
|
if (chunk === null || chunk === undefined) continue;
|
|
809
854
|
if (chunk.choices && Array.isArray(chunk.choices) && chunk.choices[0]?.delta?.content !== undefined) {
|
|
855
|
+
lastOpenAIChunk = chunk;
|
|
810
856
|
const delta = chunk.choices[0].delta.content;
|
|
811
857
|
if (typeof delta === 'string') {
|
|
812
858
|
const restored = this.feed(delta);
|
|
813
859
|
if (restored) {
|
|
814
|
-
chunk.choices
|
|
815
|
-
|
|
860
|
+
const clonedChoices = chunk.choices.map((c, idx) =>
|
|
861
|
+
idx === 0 ? { ...c, delta: { ...c.delta, content: restored } } : c
|
|
862
|
+
);
|
|
863
|
+
yield { ...chunk, choices: clonedChoices };
|
|
816
864
|
}
|
|
817
865
|
} else {
|
|
818
866
|
yield chunk;
|
|
@@ -838,7 +886,16 @@ export class StreamingDetokenizer {
|
|
|
838
886
|
}
|
|
839
887
|
|
|
840
888
|
const finalChunk = this.flush();
|
|
841
|
-
if (finalChunk)
|
|
889
|
+
if (finalChunk) {
|
|
890
|
+
if (lastOpenAIChunk) {
|
|
891
|
+
const clonedChoices = lastOpenAIChunk.choices.map((c, idx) =>
|
|
892
|
+
idx === 0 ? { ...c, delta: { ...c.delta, content: finalChunk } } : c
|
|
893
|
+
);
|
|
894
|
+
yield { ...lastOpenAIChunk, choices: clonedChoices };
|
|
895
|
+
} else {
|
|
896
|
+
yield finalChunk;
|
|
897
|
+
}
|
|
898
|
+
}
|
|
842
899
|
}
|
|
843
900
|
}
|
|
844
901
|
|
|
@@ -847,25 +904,7 @@ export class StreamingDetokenizer {
|
|
|
847
904
|
*/
|
|
848
905
|
export async function* wrapAiStream(stream, tokenMap = {}, options = {}) {
|
|
849
906
|
const detokenizer = new StreamingDetokenizer(tokenMap, options);
|
|
850
|
-
|
|
851
|
-
const text = typeof chunk === 'string' ? chunk : (chunk?.content || chunk?.text || chunk?.delta || '');
|
|
852
|
-
if (text) {
|
|
853
|
-
const detokenized = detokenizer.process(text);
|
|
854
|
-
if (detokenized) {
|
|
855
|
-
if (typeof chunk === 'string') {
|
|
856
|
-
yield detokenized;
|
|
857
|
-
} else if (typeof chunk === 'object' && chunk !== null) {
|
|
858
|
-
yield { ...chunk, content: detokenized, text: detokenized };
|
|
859
|
-
}
|
|
860
|
-
}
|
|
861
|
-
} else {
|
|
862
|
-
yield chunk;
|
|
863
|
-
}
|
|
864
|
-
}
|
|
865
|
-
const remaining = detokenizer.flush();
|
|
866
|
-
if (remaining) {
|
|
867
|
-
yield remaining;
|
|
868
|
-
}
|
|
907
|
+
yield* detokenizer.wrapStream(stream);
|
|
869
908
|
}
|
|
870
909
|
|
|
871
910
|
export function createStreamTokenRestorer(tokenMap = {}, options = {}) {
|
|
@@ -893,7 +932,22 @@ export function createStreamTokenRestorer(tokenMap = {}, options = {}) {
|
|
|
893
932
|
return stream;
|
|
894
933
|
}
|
|
895
934
|
|
|
896
|
-
export
|
|
935
|
+
export function scrubText(text, customRulesOrOptions = [], ...args) {
|
|
936
|
+
if (
|
|
937
|
+
customRulesOrOptions &&
|
|
938
|
+
typeof customRulesOrOptions === 'object' &&
|
|
939
|
+
!Array.isArray(customRulesOrOptions) &&
|
|
940
|
+
args.length === 0
|
|
941
|
+
) {
|
|
942
|
+
return sanitize(text, customRulesOrOptions);
|
|
943
|
+
}
|
|
944
|
+
if (!scrubberPkg || typeof scrubberPkg.scrubText !== 'function') return null;
|
|
945
|
+
const res = scrubberPkg.scrubText(text, customRulesOrOptions, ...args);
|
|
946
|
+
if (res && typeof res === 'object' && typeof res.scrubbed === 'undefined') {
|
|
947
|
+
res.scrubbed = res.scrubbedText;
|
|
948
|
+
}
|
|
949
|
+
return res;
|
|
950
|
+
}
|
|
897
951
|
export const unscrubText = scrubberPkg ? scrubberPkg.unscrubText : null;
|
|
898
952
|
|
|
899
953
|
// ── PSR-2 Receipt Integrity (real SHA-256; parity with src/modules/receipt-integrity.js) ──
|
|
@@ -965,7 +1019,7 @@ export function buildPsr2Receipt(tokenMap, timestamp = new Date().toISOString())
|
|
|
965
1019
|
const labels = (tokenMap && typeof tokenMap === 'object' ? Object.keys(tokenMap) : []).slice().sort();
|
|
966
1020
|
const counts = {};
|
|
967
1021
|
for (const t of labels) {
|
|
968
|
-
const m = t.match(
|
|
1022
|
+
const m = String(t).trim().match(/^(?:\[|<|\{\{|__)([A-Z0-9_]+)_\d+(?:\]|>|\}\}|__)$/);
|
|
969
1023
|
const type = m ? m[1] : 'CUSTOM';
|
|
970
1024
|
counts[type] = (counts[type] || 0) + 1;
|
|
971
1025
|
}
|
|
@@ -1082,6 +1136,7 @@ export default {
|
|
|
1082
1136
|
restore,
|
|
1083
1137
|
restoreObject,
|
|
1084
1138
|
unscrub,
|
|
1139
|
+
desanitize,
|
|
1085
1140
|
validateLicense,
|
|
1086
1141
|
validateLicenseKey,
|
|
1087
1142
|
buildAuditTelemetry,
|
package/cli.js
CHANGED
|
@@ -21,9 +21,18 @@ import { sanitize, restore, validateLicense, generateProcurementMemo, StreamingD
|
|
|
21
21
|
const __filename = fileURLToPath(import.meta.url);
|
|
22
22
|
const __dirname = path.dirname(__filename);
|
|
23
23
|
|
|
24
|
-
const
|
|
24
|
+
const rawArgs = process.argv.slice(2);
|
|
25
|
+
const args = [];
|
|
26
|
+
for (const a of rawArgs) {
|
|
27
|
+
const eqIdx = a.indexOf('=');
|
|
28
|
+
if (a.startsWith('-') && eqIdx > 1) {
|
|
29
|
+
args.push(a.slice(0, eqIdx), a.slice(eqIdx + 1));
|
|
30
|
+
} else {
|
|
31
|
+
args.push(a);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
25
34
|
|
|
26
|
-
let sdkVersion = '2.4.
|
|
35
|
+
let sdkVersion = '2.4.3';
|
|
27
36
|
try {
|
|
28
37
|
const pkg = JSON.parse(fs.readFileSync(path.join(__dirname, 'package.json'), 'utf8'));
|
|
29
38
|
sdkVersion = pkg.version || sdkVersion;
|
|
@@ -104,7 +113,10 @@ if (args.includes('sidecar') || args.includes('--sidecar') || args.includes('ser
|
|
|
104
113
|
let port = 3100;
|
|
105
114
|
const portIdx = args.indexOf('--port') !== -1 ? args.indexOf('--port') : args.indexOf('-p');
|
|
106
115
|
if (portIdx !== -1 && args[portIdx + 1]) {
|
|
107
|
-
|
|
116
|
+
const parsedPort = parseInt(args[portIdx + 1], 10);
|
|
117
|
+
if (Number.isInteger(parsedPort) && parsedPort >= 1 && parsedPort <= 65535) {
|
|
118
|
+
port = parsedPort;
|
|
119
|
+
}
|
|
108
120
|
}
|
|
109
121
|
let host = process.env.HOST || '127.0.0.1';
|
|
110
122
|
const hostIdx = args.indexOf('--host');
|
|
@@ -176,9 +188,12 @@ if (args.includes('sidecar') || args.includes('--sidecar') || args.includes('ser
|
|
|
176
188
|
|
|
177
189
|
if (req.method === 'POST') {
|
|
178
190
|
let bodyStr = '';
|
|
191
|
+
let aborted = false;
|
|
179
192
|
req.on('data', chunk => {
|
|
193
|
+
if (aborted || res.writableEnded) return;
|
|
180
194
|
bodyStr += chunk;
|
|
181
195
|
if (bodyStr.length > 50 * 1024 * 1024) {
|
|
196
|
+
aborted = true;
|
|
182
197
|
res.writeHead(413, { 'Content-Type': 'application/json' });
|
|
183
198
|
res.end(JSON.stringify({ error: 'Payload Too Large' }));
|
|
184
199
|
req.destroy();
|
|
@@ -186,6 +201,7 @@ if (args.includes('sidecar') || args.includes('--sidecar') || args.includes('ser
|
|
|
186
201
|
});
|
|
187
202
|
|
|
188
203
|
req.on('end', () => {
|
|
204
|
+
if (aborted || res.writableEnded) return;
|
|
189
205
|
try {
|
|
190
206
|
const payload = bodyStr ? JSON.parse(bodyStr) : {};
|
|
191
207
|
const isScrub = reqUrl === '/scrub' || reqUrl === '/sanitize' || reqUrl === '/v1/scrub' || reqUrl === '/v1/sanitize';
|
|
@@ -437,10 +453,16 @@ if (args.includes('scan-git-staged') || args.includes('--scan-git-staged')) {
|
|
|
437
453
|
}
|
|
438
454
|
|
|
439
455
|
const res = sanitize(addedLines, { profile: 'Dev', detectSecrets: true });
|
|
440
|
-
const
|
|
456
|
+
const includeNames = args.includes('--include-names') || args.includes('--strict');
|
|
457
|
+
const relevantTokens = Object.keys(res.tokenMap || {}).filter(
|
|
458
|
+
tok => includeNames || !/^\[NAME_\d+\]$/i.test(tok)
|
|
459
|
+
);
|
|
460
|
+
const maskedCount = relevantTokens.length;
|
|
441
461
|
|
|
442
462
|
if (maskedCount > 0) {
|
|
443
|
-
const entityTypes =
|
|
463
|
+
const entityTypes = Array.from(new Set(
|
|
464
|
+
relevantTokens.map(t => t.replace(/^\[|_\d+\]$/g, ''))
|
|
465
|
+
)).join(', ');
|
|
444
466
|
console.error(`
|
|
445
467
|
╔══════════════════════════════════════════════════════════════════════════════╗
|
|
446
468
|
║ 🚨 [PrivacyScrubber Git Shield] UNMASKED SECRETS / PII INTERCEPTED! ║
|
|
@@ -586,31 +608,47 @@ if (mapIdx !== -1 && args[mapIdx + 1]) {
|
|
|
586
608
|
}
|
|
587
609
|
|
|
588
610
|
// Locate input argument (first positional parameter that is not a flag or flag value)
|
|
611
|
+
const inputFlagIdx = args.indexOf('--input') !== -1 ? args.indexOf('--input') : args.indexOf('-i');
|
|
612
|
+
const outIdx = args.indexOf('--out') !== -1 ? args.indexOf('--out') : (args.indexOf('--output') !== -1 ? args.indexOf('--output') : args.indexOf('-o'));
|
|
613
|
+
const outputFile = (outIdx !== -1 && args[outIdx + 1]) ? args[outIdx + 1] : null;
|
|
614
|
+
|
|
589
615
|
const skipIndices = new Set();
|
|
590
616
|
if (configIdx !== -1) { skipIndices.add(configIdx); skipIndices.add(configIdx + 1); }
|
|
591
617
|
if (profIdx !== -1) { skipIndices.add(profIdx); skipIndices.add(profIdx + 1); }
|
|
592
618
|
if (keyIdx !== -1) { skipIndices.add(keyIdx); skipIndices.add(keyIdx + 1); }
|
|
593
619
|
if (mapIdx !== -1) { skipIndices.add(mapIdx); skipIndices.add(mapIdx + 1); }
|
|
594
620
|
if (formatIdx !== -1) { skipIndices.add(formatIdx); skipIndices.add(formatIdx + 1); }
|
|
595
|
-
|
|
621
|
+
if (inputFlagIdx !== -1) { skipIndices.add(inputFlagIdx); skipIndices.add(inputFlagIdx + 1); }
|
|
596
622
|
if (outIdx !== -1) { skipIndices.add(outIdx); skipIndices.add(outIdx + 1); }
|
|
597
623
|
|
|
598
|
-
let inputArg = null;
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
624
|
+
let inputArg = (inputFlagIdx !== -1 && args[inputFlagIdx + 1]) ? args[inputFlagIdx + 1] : null;
|
|
625
|
+
if (!inputArg) {
|
|
626
|
+
for (let i = 0; i < args.length; i++) {
|
|
627
|
+
if (skipIndices.has(i)) continue;
|
|
628
|
+
if (args[i] === 'sanitize' || args[i] === 'scrub') continue;
|
|
629
|
+
if (args[i] === '--verify-ci' || args[i] === '--ztds-proof' || args[i] === '--json' || args[i] === '--sarif' || args[i] === '--restore' || args[i] === '--unscrub') continue;
|
|
630
|
+
if (args[i].startsWith('-')) continue;
|
|
631
|
+
inputArg = args[i];
|
|
632
|
+
break;
|
|
633
|
+
}
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
function writeCliOutput(str) {
|
|
637
|
+
const payload = str.endsWith('\n') ? str : str + '\n';
|
|
638
|
+
if (outputFile) {
|
|
639
|
+
fs.writeFileSync(path.resolve(process.cwd(), outputFile), payload, 'utf8');
|
|
640
|
+
} else {
|
|
641
|
+
process.stdout.write(payload);
|
|
642
|
+
}
|
|
605
643
|
}
|
|
606
644
|
|
|
607
645
|
function handleExecution(text) {
|
|
608
646
|
if (isRestore) {
|
|
609
647
|
const { restoredText, restoredCount } = restore(text, tokenMap);
|
|
610
648
|
if (isJson) {
|
|
611
|
-
|
|
649
|
+
writeCliOutput(JSON.stringify({ restoredText, restoredCount }, null, 2));
|
|
612
650
|
} else {
|
|
613
|
-
|
|
651
|
+
writeCliOutput(restoredText);
|
|
614
652
|
}
|
|
615
653
|
process.exit(0);
|
|
616
654
|
}
|
|
@@ -654,9 +692,9 @@ function handleExecution(text) {
|
|
|
654
692
|
};
|
|
655
693
|
|
|
656
694
|
const proofJson = JSON.stringify(proof, null, 2);
|
|
657
|
-
if (
|
|
658
|
-
fs.writeFileSync(path.resolve(process.cwd(),
|
|
659
|
-
console.log(`[ZTDS Proof] Conformance proof saved to ${
|
|
695
|
+
if (outputFile) {
|
|
696
|
+
fs.writeFileSync(path.resolve(process.cwd(), outputFile), proofJson, 'utf8');
|
|
697
|
+
console.log(`[ZTDS Proof] Conformance proof saved to ${outputFile}`);
|
|
660
698
|
} else {
|
|
661
699
|
console.log(proofJson);
|
|
662
700
|
}
|
|
@@ -665,7 +703,7 @@ function handleExecution(text) {
|
|
|
665
703
|
|
|
666
704
|
if (isVerifyCi) {
|
|
667
705
|
if (!text || !text.trim()) {
|
|
668
|
-
|
|
706
|
+
writeCliOutput(`> **Verified by PrivacyScrubber (ZTDS Standard)**\n> * **Status:** PASSED · 0 PII / 0 Secrets detected in prompt pipeline (Empty payload)\n> * **Execution:** Local RAM (<1ms) · 0 Network Egress Bytes\n> * **Compliance Spec:** [Zero-Trust Data Sanitization (ZTDS)](https://privacyscrubber.com/solutions/engineering/zero-trust-ai-data-sanitization?src=ci_stamp) | ⭐ [Star on GitHub](https://github.com/moxno/privacyscrubber-mcp) | [Developer SDK ($299/mo)](https://privacyscrubber.com/sdk/?src=ci_stamp)`);
|
|
669
707
|
process.exit(0);
|
|
670
708
|
}
|
|
671
709
|
|
|
@@ -684,16 +722,16 @@ function handleExecution(text) {
|
|
|
684
722
|
> * **Audit Hash:** \`${hash}\` (Zero-Trust Verified)
|
|
685
723
|
> * **Compliance Spec:** [Zero-Trust Data Sanitization (ZTDS)](https://privacyscrubber.com/solutions/engineering/zero-trust-ai-data-sanitization?src=ci_stamp) | ⭐ [Star on GitHub](https://github.com/moxno/privacyscrubber-mcp) | [Developer SDK ($299/mo)](https://privacyscrubber.com/sdk/?src=ci_stamp)
|
|
686
724
|
`;
|
|
687
|
-
|
|
725
|
+
writeCliOutput(stamp.trim());
|
|
688
726
|
process.exit(0);
|
|
689
727
|
}
|
|
690
728
|
|
|
691
729
|
if (!text) {
|
|
692
730
|
if (isSarif) {
|
|
693
|
-
const { emitSarif } = require('./sarif.
|
|
694
|
-
|
|
731
|
+
const { emitSarif } = require('./sarif.cjs');
|
|
732
|
+
writeCliOutput(emitSarif([], { pretty: true }));
|
|
695
733
|
} else if (isJson) {
|
|
696
|
-
|
|
734
|
+
writeCliOutput(JSON.stringify({ scrubbedText: '', tokenMap: {}, count: 0 }, null, 2));
|
|
697
735
|
}
|
|
698
736
|
process.exit(0);
|
|
699
737
|
}
|
|
@@ -704,19 +742,19 @@ function handleExecution(text) {
|
|
|
704
742
|
customRules: loadedConfig.customRules || []
|
|
705
743
|
});
|
|
706
744
|
if (isSarif) {
|
|
707
|
-
const { emitSarif, sanitizeResultToFindings } = require('./sarif.
|
|
745
|
+
const { emitSarif, sanitizeResultToFindings } = require('./sarif.cjs');
|
|
708
746
|
const findings = sanitizeResultToFindings(res, text, inputArg || 'stdin');
|
|
709
747
|
const sarifOutput = emitSarif(findings, { pretty: true });
|
|
710
|
-
|
|
748
|
+
writeCliOutput(sarifOutput);
|
|
711
749
|
} else if (isJson) {
|
|
712
|
-
|
|
750
|
+
writeCliOutput(JSON.stringify({
|
|
713
751
|
scrubbedText: res.scrubbedText,
|
|
714
752
|
tokenMap: res.tokenMap,
|
|
715
753
|
count: res.count,
|
|
716
754
|
telemetry: res.telemetry
|
|
717
755
|
}, null, 2));
|
|
718
756
|
} else {
|
|
719
|
-
|
|
757
|
+
writeCliOutput(res.scrubbedText);
|
|
720
758
|
}
|
|
721
759
|
process.exit(0);
|
|
722
760
|
}
|
|
@@ -92,7 +92,6 @@ function sendResponse(id, result, error = null) {
|
|
|
92
92
|
// Setup readline interface on stdin
|
|
93
93
|
const rl = readline.createInterface({
|
|
94
94
|
input: process.stdin,
|
|
95
|
-
output: process.stdout,
|
|
96
95
|
terminal: false
|
|
97
96
|
});
|
|
98
97
|
|
|
@@ -116,7 +115,7 @@ rl.on('line', (line) => {
|
|
|
116
115
|
sendResponse(id, {
|
|
117
116
|
pong: true,
|
|
118
117
|
engine: 'PrivacyScrubber ZTDS',
|
|
119
|
-
version: '2.4.
|
|
118
|
+
version: '2.4.3',
|
|
120
119
|
uptime: process.uptime()
|
|
121
120
|
});
|
|
122
121
|
break;
|
|
@@ -163,17 +162,36 @@ rl.on('line', (line) => {
|
|
|
163
162
|
|
|
164
163
|
case 'sanitize_object': {
|
|
165
164
|
const data = req.data ?? {};
|
|
165
|
+
const persistSession = req.persist_session ?? req.persistSession ?? true;
|
|
166
|
+
let mergedIgnore = null;
|
|
167
|
+
const reqIgnore = req.ignore_list || req.ignoreList || null;
|
|
168
|
+
if (reqIgnore || engine.ignoreList.size > 0) {
|
|
169
|
+
mergedIgnore = new Set(engine.ignoreList);
|
|
170
|
+
if (reqIgnore) {
|
|
171
|
+
for (const item of reqIgnore) mergedIgnore.add(item);
|
|
172
|
+
}
|
|
173
|
+
}
|
|
166
174
|
const options = {
|
|
167
175
|
profile: req.profile || engine.defaultProfile,
|
|
168
176
|
detectSecrets: req.detect_secrets ?? req.detectSecrets ?? false,
|
|
169
177
|
licenseKey: req.license_key || req.licenseKey || engine.licenseKey,
|
|
178
|
+
generateReceipt: req.generate_receipt ?? req.generateReceipt ?? true,
|
|
170
179
|
customRules: req.custom_rules || req.customRules || [],
|
|
180
|
+
ignoreList: mergedIgnore,
|
|
181
|
+
existingSessionMap: persistSession ? engine.sessionMap : (req.existing_session_map || req.existingSessionMap || {}),
|
|
171
182
|
throwOnLimit: req.throw_on_limit ?? req.throwOnLimit ?? true
|
|
172
183
|
};
|
|
173
184
|
|
|
174
185
|
const res = sdk.sanitizeObject(data, options);
|
|
175
|
-
if (
|
|
186
|
+
if (persistSession && res.tokenMap) {
|
|
176
187
|
Object.assign(engine.sessionMap, res.tokenMap);
|
|
188
|
+
const keys = Object.keys(engine.sessionMap);
|
|
189
|
+
if (keys.length > engine.maxSessionTokens) {
|
|
190
|
+
const excess = keys.length - engine.maxSessionTokens;
|
|
191
|
+
for (let i = 0; i < excess; i++) {
|
|
192
|
+
delete engine.sessionMap[keys[i]];
|
|
193
|
+
}
|
|
194
|
+
}
|
|
177
195
|
}
|
|
178
196
|
sendResponse(id, {
|
|
179
197
|
sanitized: res.sanitized,
|
|
@@ -213,10 +231,11 @@ rl.on('line', (line) => {
|
|
|
213
231
|
}
|
|
214
232
|
|
|
215
233
|
case 'status': {
|
|
234
|
+
const tier = engine.license?.tier || 'FREE';
|
|
216
235
|
sendResponse(id, {
|
|
217
236
|
profile: engine.defaultProfile,
|
|
218
|
-
license_tier:
|
|
219
|
-
is_pro: engine.license?.
|
|
237
|
+
license_tier: tier,
|
|
238
|
+
is_pro: Boolean(engine.license?.valid && ['SDK', 'OEM', 'ENTERPRISE', 'PRO', 'TEAMS'].includes(tier)),
|
|
220
239
|
session_tokens_count: Object.keys(engine.sessionMap).length,
|
|
221
240
|
ignore_list_size: engine.ignoreList.size,
|
|
222
241
|
memory_usage: process.memoryUsage()
|
|
@@ -92,7 +92,6 @@ function sendResponse(id, result, error = null) {
|
|
|
92
92
|
// Setup readline interface on stdin
|
|
93
93
|
const rl = readline.createInterface({
|
|
94
94
|
input: process.stdin,
|
|
95
|
-
output: process.stdout,
|
|
96
95
|
terminal: false
|
|
97
96
|
});
|
|
98
97
|
|
|
@@ -116,7 +115,7 @@ rl.on('line', (line) => {
|
|
|
116
115
|
sendResponse(id, {
|
|
117
116
|
pong: true,
|
|
118
117
|
engine: 'PrivacyScrubber ZTDS',
|
|
119
|
-
version: '2.4.
|
|
118
|
+
version: '2.4.3',
|
|
120
119
|
uptime: process.uptime()
|
|
121
120
|
});
|
|
122
121
|
break;
|
|
@@ -163,17 +162,36 @@ rl.on('line', (line) => {
|
|
|
163
162
|
|
|
164
163
|
case 'sanitize_object': {
|
|
165
164
|
const data = req.data ?? {};
|
|
165
|
+
const persistSession = req.persist_session ?? req.persistSession ?? true;
|
|
166
|
+
let mergedIgnore = null;
|
|
167
|
+
const reqIgnore = req.ignore_list || req.ignoreList || null;
|
|
168
|
+
if (reqIgnore || engine.ignoreList.size > 0) {
|
|
169
|
+
mergedIgnore = new Set(engine.ignoreList);
|
|
170
|
+
if (reqIgnore) {
|
|
171
|
+
for (const item of reqIgnore) mergedIgnore.add(item);
|
|
172
|
+
}
|
|
173
|
+
}
|
|
166
174
|
const options = {
|
|
167
175
|
profile: req.profile || engine.defaultProfile,
|
|
168
176
|
detectSecrets: req.detect_secrets ?? req.detectSecrets ?? false,
|
|
169
177
|
licenseKey: req.license_key || req.licenseKey || engine.licenseKey,
|
|
178
|
+
generateReceipt: req.generate_receipt ?? req.generateReceipt ?? true,
|
|
170
179
|
customRules: req.custom_rules || req.customRules || [],
|
|
180
|
+
ignoreList: mergedIgnore,
|
|
181
|
+
existingSessionMap: persistSession ? engine.sessionMap : (req.existing_session_map || req.existingSessionMap || {}),
|
|
171
182
|
throwOnLimit: req.throw_on_limit ?? req.throwOnLimit ?? true
|
|
172
183
|
};
|
|
173
184
|
|
|
174
185
|
const res = sdk.sanitizeObject(data, options);
|
|
175
|
-
if (
|
|
186
|
+
if (persistSession && res.tokenMap) {
|
|
176
187
|
Object.assign(engine.sessionMap, res.tokenMap);
|
|
188
|
+
const keys = Object.keys(engine.sessionMap);
|
|
189
|
+
if (keys.length > engine.maxSessionTokens) {
|
|
190
|
+
const excess = keys.length - engine.maxSessionTokens;
|
|
191
|
+
for (let i = 0; i < excess; i++) {
|
|
192
|
+
delete engine.sessionMap[keys[i]];
|
|
193
|
+
}
|
|
194
|
+
}
|
|
177
195
|
}
|
|
178
196
|
sendResponse(id, {
|
|
179
197
|
sanitized: res.sanitized,
|
|
@@ -213,10 +231,11 @@ rl.on('line', (line) => {
|
|
|
213
231
|
}
|
|
214
232
|
|
|
215
233
|
case 'status': {
|
|
234
|
+
const tier = engine.license?.tier || 'FREE';
|
|
216
235
|
sendResponse(id, {
|
|
217
236
|
profile: engine.defaultProfile,
|
|
218
|
-
license_tier:
|
|
219
|
-
is_pro: engine.license?.
|
|
237
|
+
license_tier: tier,
|
|
238
|
+
is_pro: Boolean(engine.license?.valid && ['SDK', 'OEM', 'ENTERPRISE', 'PRO', 'TEAMS'].includes(tier)),
|
|
220
239
|
session_tokens_count: Object.keys(engine.sessionMap).length,
|
|
221
240
|
ignore_list_size: engine.ignoreList.size,
|
|
222
241
|
memory_usage: process.memoryUsage()
|
|
@@ -260,6 +260,9 @@ class PrivacyScrubberClient:
|
|
|
260
260
|
detect_secrets: Optional[bool] = None,
|
|
261
261
|
license_key: Optional[str] = None,
|
|
262
262
|
persist_session: bool = True,
|
|
263
|
+
generate_receipt: bool = True,
|
|
264
|
+
ignore_list: Optional[List[str]] = None,
|
|
265
|
+
custom_rules: Optional[List[Dict[str, Any]]] = None,
|
|
263
266
|
throw_on_limit: bool = True
|
|
264
267
|
) -> ObjectSanitizeResult:
|
|
265
268
|
"""
|
|
@@ -273,6 +276,9 @@ class PrivacyScrubberClient:
|
|
|
273
276
|
detect_secrets=self.detect_secrets if detect_secrets is None else detect_secrets,
|
|
274
277
|
license_key=license_key or self.license_key,
|
|
275
278
|
persist_session=persist_session,
|
|
279
|
+
generate_receipt=generate_receipt,
|
|
280
|
+
ignore_list=ignore_list,
|
|
281
|
+
custom_rules=custom_rules,
|
|
276
282
|
throw_on_limit=throw_on_limit
|
|
277
283
|
)
|
|
278
284
|
return ObjectSanitizeResult(
|
|
@@ -503,6 +509,9 @@ class AsyncPrivacyScrubberClient:
|
|
|
503
509
|
detect_secrets: Optional[bool] = None,
|
|
504
510
|
license_key: Optional[str] = None,
|
|
505
511
|
persist_session: bool = True,
|
|
512
|
+
generate_receipt: bool = True,
|
|
513
|
+
ignore_list: Optional[List[str]] = None,
|
|
514
|
+
custom_rules: Optional[List[Dict[str, Any]]] = None,
|
|
506
515
|
throw_on_limit: bool = True
|
|
507
516
|
) -> ObjectSanitizeResult:
|
|
508
517
|
res = await self._send_request(
|
|
@@ -512,6 +521,9 @@ class AsyncPrivacyScrubberClient:
|
|
|
512
521
|
detect_secrets=self.detect_secrets if detect_secrets is None else detect_secrets,
|
|
513
522
|
license_key=license_key or self.license_key,
|
|
514
523
|
persist_session=persist_session,
|
|
524
|
+
generate_receipt=generate_receipt,
|
|
525
|
+
ignore_list=ignore_list,
|
|
526
|
+
custom_rules=custom_rules,
|
|
515
527
|
throw_on_limit=throw_on_limit
|
|
516
528
|
)
|
|
517
529
|
return ObjectSanitizeResult(
|