@privacyscrubber/sdk 2.4.1 → 2.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -0
- package/browser.js +84 -29
- package/cli.js +71 -27
- package/examples/python-bridge/bridge_daemon.cjs +24 -5
- package/examples/python-bridge/fastapi_example.py +1 -1
- package/examples/python-bridge/privacyscrubber/__init__.py +1 -1
- package/examples/python-bridge/privacyscrubber/bridge_daemon.cjs +24 -5
- package/examples/python-bridge/privacyscrubber/cli.py +1 -1
- package/examples/python-bridge/privacyscrubber/client.py +12 -0
- package/examples/python-bridge/privacyscrubber/langchain.py +31 -7
- package/examples/python-bridge/pyproject.toml +1 -1
- package/examples/python-bridge/setup.py +1 -1
- package/examples/python-bridge/test_python_bridge.py +1 -1
- package/fastmcp.cjs +45 -9
- package/index.cjs +67 -18
- package/index.d.ts +3 -2
- package/index.js +67 -18
- package/package.json +12 -1
- package/polyfill.js +31 -13
- package/ps-license-manager.js +1 -1
- package/ps-pii-engine.cjs +1 -1
- package/sarif.cjs +388 -0
- package/sarif.js +37 -0
- package/scrubber-core.cjs +1 -1
- package/vercel-ai.cjs +52 -6
- package/vision.cjs +15 -11
- package/vision.js +15 -11
package/README.md
CHANGED
|
@@ -139,6 +139,29 @@ console.log(restoredText);
|
|
|
139
139
|
// "I have queued an SMS notification for John Doe at 555-0199."
|
|
140
140
|
```
|
|
141
141
|
|
|
142
|
+
#### Custom Rules (`customRules`), Domain Exclusions (`ignoreList`) & Profile Selection
|
|
143
|
+
You can pass `customRules` (for internal ERP/project formats) and `ignoreList` (to preserve domain-specific nouns or allowlisted identifiers in cleartext) into `sanitize()` across any profile (`'General'`, `'Manufacturing'`, `'Sales'`, `'Support'`, etc.):
|
|
144
|
+
|
|
145
|
+
```javascript
|
|
146
|
+
import { sanitize } from '@privacyscrubber/sdk';
|
|
147
|
+
|
|
148
|
+
const { scrubbedText, tokenMap } = sanitize(rawCrmPayload, {
|
|
149
|
+
// Profile guidance:
|
|
150
|
+
// - 'Manufacturing' (or 'Engineering'): masks shopfloor/plant IDs (e.g., "Linie 3", "Station 4", SAP order numbers) alongside general PII.
|
|
151
|
+
// - 'Sales' / 'Support' / 'General': preserves production line numbers in cleartext while masking personal names, emails, phones, meeting links, and CRM identifiers.
|
|
152
|
+
profile: 'Manufacturing',
|
|
153
|
+
|
|
154
|
+
// Add custom regex or exact literal rules (works on all profiles):
|
|
155
|
+
customRules: [
|
|
156
|
+
{ name: 'Internal Project Code', pattern: 'PRJ-\\d{4,6}', label: 'PROJECT' },
|
|
157
|
+
{ name: 'Custom Drawing ID', regex: /\bDRW-[A-Z0-9-]{5,12}\b/g, label: 'DRAWING' }
|
|
158
|
+
],
|
|
159
|
+
|
|
160
|
+
// Exclude specific domain terms or false-positive nouns (case-insensitive; also prevents multi-word NAME matches containing these terms):
|
|
161
|
+
ignoreList: ['Materialmangel', 'Störgrund', 'Prüfstand', 'Linie 3']
|
|
162
|
+
});
|
|
163
|
+
```
|
|
164
|
+
|
|
142
165
|
---
|
|
143
166
|
|
|
144
167
|
### 3. Vercel AI SDK Integration (`@privacyscrubber/sdk/vercel-ai`)
|
package/browser.js
CHANGED
|
@@ -114,7 +114,7 @@ export function buildAuditTelemetry(currentTokenMap = {}) {
|
|
|
114
114
|
if (!t) continue;
|
|
115
115
|
const tokenStr = typeof t === 'string' ? t : ((t && (t.token || t.mask)) ? String(t.token || t.mask) : '');
|
|
116
116
|
if (!tokenStr) continue;
|
|
117
|
-
const match = tokenStr.match(/\[([A-
|
|
117
|
+
const match = tokenStr.match(/\[([A-Z0-9_]+)_\d+\]/);
|
|
118
118
|
const baseType = match ? match[1] : (tokenStr.replace(/\[|\]/g, '').replace(/_[0-9]+$/, '') || 'CUSTOM');
|
|
119
119
|
entities[baseType] = (entities[baseType] || 0) + 1;
|
|
120
120
|
totalCount++;
|
|
@@ -237,7 +237,11 @@ export function sanitize(text, options = {}) {
|
|
|
237
237
|
if (!text || typeof text !== 'string') {
|
|
238
238
|
return {
|
|
239
239
|
scrubbedText: '',
|
|
240
|
+
scrubbed: '',
|
|
240
241
|
tokenMap: {},
|
|
242
|
+
sessionMap: {},
|
|
243
|
+
matches: [],
|
|
244
|
+
spans: [],
|
|
241
245
|
count: 0,
|
|
242
246
|
executionMs: 0,
|
|
243
247
|
license: { isPro: false, tier: 'FREE', limit: 15000, wasTruncated: false },
|
|
@@ -344,9 +348,16 @@ export function sanitize(text, options = {}) {
|
|
|
344
348
|
finalScrubbedText += `\n\n<!-- [PrivacyScrubber] Input truncated at free tier limit (${limit.toLocaleString()} chars). For unlimited throughput: https://privacyscrubber.com/pricing?tier=sdk&src=truncation -->`;
|
|
345
349
|
}
|
|
346
350
|
|
|
351
|
+
const returnFullSessionMap = options.returnFullSessionMap === true;
|
|
352
|
+
const sessionMap = { ...(existingSessionMap || {}), ...(rawResult.tokenMap || {}) };
|
|
353
|
+
|
|
347
354
|
return {
|
|
348
355
|
scrubbedText: finalScrubbedText,
|
|
349
|
-
|
|
356
|
+
scrubbed: finalScrubbedText,
|
|
357
|
+
tokenMap: returnFullSessionMap ? sessionMap : (rawResult.tokenMap || {}),
|
|
358
|
+
sessionMap: sessionMap,
|
|
359
|
+
matches: rawResult.matches || [],
|
|
360
|
+
spans: rawResult.matches || [],
|
|
350
361
|
count: rawResult.count || 0,
|
|
351
362
|
executionMs: executionMs,
|
|
352
363
|
license: {
|
|
@@ -391,6 +402,7 @@ export function restore(aiResponse, tokenMap = {}) {
|
|
|
391
402
|
}
|
|
392
403
|
|
|
393
404
|
export const unscrub = restore;
|
|
405
|
+
export const desanitize = restore;
|
|
394
406
|
|
|
395
407
|
/**
|
|
396
408
|
* Recursively sanitizes all string values within an object or array.
|
|
@@ -517,6 +529,7 @@ export class PrivacyScrubberEngine {
|
|
|
517
529
|
this.generateReceipt = config.generateReceipt !== false;
|
|
518
530
|
this.sessionMap = { ...(config.initialSessionMap || {}) };
|
|
519
531
|
this.ignoreList = new Set(config.ignoreList || []);
|
|
532
|
+
this.maxSessionTokens = config.maxSessionTokens || 10000;
|
|
520
533
|
this.license = validateLicense(this.licenseKey);
|
|
521
534
|
}
|
|
522
535
|
|
|
@@ -548,6 +561,14 @@ export class PrivacyScrubberEngine {
|
|
|
548
561
|
const result = sanitize(text, mergedOptions);
|
|
549
562
|
if (options.persistSession !== false) {
|
|
550
563
|
Object.assign(this.sessionMap, result.tokenMap);
|
|
564
|
+
const maxTokens = options.maxSessionTokens || this.maxSessionTokens || 10000;
|
|
565
|
+
const keys = Object.keys(this.sessionMap);
|
|
566
|
+
if (keys.length > maxTokens) {
|
|
567
|
+
const pruneCount = keys.length - maxTokens;
|
|
568
|
+
for (let i = 0; i < pruneCount; i++) {
|
|
569
|
+
delete this.sessionMap[keys[i]];
|
|
570
|
+
}
|
|
571
|
+
}
|
|
551
572
|
}
|
|
552
573
|
return result;
|
|
553
574
|
}
|
|
@@ -563,8 +584,31 @@ export class PrivacyScrubberEngine {
|
|
|
563
584
|
}
|
|
564
585
|
}
|
|
565
586
|
|
|
566
|
-
|
|
567
|
-
|
|
587
|
+
markFalsePositive(token) {
|
|
588
|
+
const original = this.sessionMap[token];
|
|
589
|
+
if (!original) {
|
|
590
|
+
return { success: false, reason: `Token '${token}' not found in session map` };
|
|
591
|
+
}
|
|
592
|
+
this.ignoreList.add(original);
|
|
593
|
+
delete this.sessionMap[token];
|
|
594
|
+
return { success: true, token, original, ignoreListSize: this.ignoreList.size };
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
clearIgnoreList() {
|
|
598
|
+
this.ignoreList.clear();
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
createStreamRestorer(tokenMap = null, options = {}) {
|
|
602
|
+
if (tokenMap && typeof tokenMap === 'object' && typeof tokenMap.onTokenRestored === 'function' && Object.keys(options).length === 0) {
|
|
603
|
+
return createStreamTokenRestorer(this.sessionMap, tokenMap);
|
|
604
|
+
}
|
|
605
|
+
const activeMap = tokenMap || this.sessionMap;
|
|
606
|
+
return createStreamTokenRestorer(activeMap, options);
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
async* wrapStream(stream, tokenMap = null, options = {}) {
|
|
610
|
+
const activeMap = tokenMap || this.sessionMap;
|
|
611
|
+
yield* wrapAiStream(stream, activeMap, options);
|
|
568
612
|
}
|
|
569
613
|
}
|
|
570
614
|
|
|
@@ -749,7 +793,7 @@ export class StreamingDetokenizer {
|
|
|
749
793
|
const tailLen = 160;
|
|
750
794
|
const tail = this.pendingBuffer.length > tailLen ? this.pendingBuffer.slice(-tailLen) : this.pendingBuffer;
|
|
751
795
|
const offset = this.pendingBuffer.length > tailLen ? this.pendingBuffer.length - tailLen : 0;
|
|
752
|
-
const m = tail.match(/(?:(?:\[|【|[|〔|「|『|<|\{\{?|__?|%5[Bb]|\\u005[Bb]|&(?:amp;)?(?:#0*91;|#[xX]0*5[Bb];|lbrack;))[A-Za-z0-9_
|
|
796
|
+
const m = tail.match(/(?:(?:\[|【|[|〔|「|『|<|\{\{?|__?|%5[Bb]|\\u005[Bb]|&(?:amp;)?(?:#0*91;|#[xX]0*5[Bb];|lbrack;))[A-Za-z0-9_\-\u0410-\u044F\u0401\u0451 *`~.\\]{0,128}(?:%5?|&(?:a(?:m(?:p(?:;)?)?)?)?(?:#(?:0*9?3?|[xX]0*5?[Dd]?)?|r(?:b(?:r(?:a(?:c(?:k)?)?)?)?)?)?)?|%[0-9A-Fa-f]?|\\(?:u(?:0(?:0(?:5)?)?)?)?|&(?:a(?:m(?:p(?:;)?)?)?)?(?:#(?:0*9?1?|[xX]0*5?)?|l(?:b(?:r(?:a(?:c(?:k)?)?)?)?)?)?)$/);
|
|
753
797
|
|
|
754
798
|
if (m) {
|
|
755
799
|
const candidate = m[0];
|
|
@@ -804,15 +848,19 @@ export class StreamingDetokenizer {
|
|
|
804
848
|
* @returns {AsyncGenerator<any>}
|
|
805
849
|
*/
|
|
806
850
|
async* wrapStream(asyncIterable) {
|
|
851
|
+
let lastOpenAIChunk = null;
|
|
807
852
|
for await (const chunk of asyncIterable) {
|
|
808
853
|
if (chunk === null || chunk === undefined) continue;
|
|
809
854
|
if (chunk.choices && Array.isArray(chunk.choices) && chunk.choices[0]?.delta?.content !== undefined) {
|
|
855
|
+
lastOpenAIChunk = chunk;
|
|
810
856
|
const delta = chunk.choices[0].delta.content;
|
|
811
857
|
if (typeof delta === 'string') {
|
|
812
858
|
const restored = this.feed(delta);
|
|
813
859
|
if (restored) {
|
|
814
|
-
chunk.choices
|
|
815
|
-
|
|
860
|
+
const clonedChoices = chunk.choices.map((c, idx) =>
|
|
861
|
+
idx === 0 ? { ...c, delta: { ...c.delta, content: restored } } : c
|
|
862
|
+
);
|
|
863
|
+
yield { ...chunk, choices: clonedChoices };
|
|
816
864
|
}
|
|
817
865
|
} else {
|
|
818
866
|
yield chunk;
|
|
@@ -838,7 +886,16 @@ export class StreamingDetokenizer {
|
|
|
838
886
|
}
|
|
839
887
|
|
|
840
888
|
const finalChunk = this.flush();
|
|
841
|
-
if (finalChunk)
|
|
889
|
+
if (finalChunk) {
|
|
890
|
+
if (lastOpenAIChunk) {
|
|
891
|
+
const clonedChoices = lastOpenAIChunk.choices.map((c, idx) =>
|
|
892
|
+
idx === 0 ? { ...c, delta: { ...c.delta, content: finalChunk } } : c
|
|
893
|
+
);
|
|
894
|
+
yield { ...lastOpenAIChunk, choices: clonedChoices };
|
|
895
|
+
} else {
|
|
896
|
+
yield finalChunk;
|
|
897
|
+
}
|
|
898
|
+
}
|
|
842
899
|
}
|
|
843
900
|
}
|
|
844
901
|
|
|
@@ -847,25 +904,7 @@ export class StreamingDetokenizer {
|
|
|
847
904
|
*/
|
|
848
905
|
export async function* wrapAiStream(stream, tokenMap = {}, options = {}) {
|
|
849
906
|
const detokenizer = new StreamingDetokenizer(tokenMap, options);
|
|
850
|
-
|
|
851
|
-
const text = typeof chunk === 'string' ? chunk : (chunk?.content || chunk?.text || chunk?.delta || '');
|
|
852
|
-
if (text) {
|
|
853
|
-
const detokenized = detokenizer.process(text);
|
|
854
|
-
if (detokenized) {
|
|
855
|
-
if (typeof chunk === 'string') {
|
|
856
|
-
yield detokenized;
|
|
857
|
-
} else if (typeof chunk === 'object' && chunk !== null) {
|
|
858
|
-
yield { ...chunk, content: detokenized, text: detokenized };
|
|
859
|
-
}
|
|
860
|
-
}
|
|
861
|
-
} else {
|
|
862
|
-
yield chunk;
|
|
863
|
-
}
|
|
864
|
-
}
|
|
865
|
-
const remaining = detokenizer.flush();
|
|
866
|
-
if (remaining) {
|
|
867
|
-
yield remaining;
|
|
868
|
-
}
|
|
907
|
+
yield* detokenizer.wrapStream(stream);
|
|
869
908
|
}
|
|
870
909
|
|
|
871
910
|
export function createStreamTokenRestorer(tokenMap = {}, options = {}) {
|
|
@@ -893,7 +932,22 @@ export function createStreamTokenRestorer(tokenMap = {}, options = {}) {
|
|
|
893
932
|
return stream;
|
|
894
933
|
}
|
|
895
934
|
|
|
896
|
-
export
|
|
935
|
+
export function scrubText(text, customRulesOrOptions = [], ...args) {
|
|
936
|
+
if (
|
|
937
|
+
customRulesOrOptions &&
|
|
938
|
+
typeof customRulesOrOptions === 'object' &&
|
|
939
|
+
!Array.isArray(customRulesOrOptions) &&
|
|
940
|
+
args.length === 0
|
|
941
|
+
) {
|
|
942
|
+
return sanitize(text, customRulesOrOptions);
|
|
943
|
+
}
|
|
944
|
+
if (!scrubberPkg || typeof scrubberPkg.scrubText !== 'function') return null;
|
|
945
|
+
const res = scrubberPkg.scrubText(text, customRulesOrOptions, ...args);
|
|
946
|
+
if (res && typeof res === 'object' && typeof res.scrubbed === 'undefined') {
|
|
947
|
+
res.scrubbed = res.scrubbedText;
|
|
948
|
+
}
|
|
949
|
+
return res;
|
|
950
|
+
}
|
|
897
951
|
export const unscrubText = scrubberPkg ? scrubberPkg.unscrubText : null;
|
|
898
952
|
|
|
899
953
|
// ── PSR-2 Receipt Integrity (real SHA-256; parity with src/modules/receipt-integrity.js) ──
|
|
@@ -965,7 +1019,7 @@ export function buildPsr2Receipt(tokenMap, timestamp = new Date().toISOString())
|
|
|
965
1019
|
const labels = (tokenMap && typeof tokenMap === 'object' ? Object.keys(tokenMap) : []).slice().sort();
|
|
966
1020
|
const counts = {};
|
|
967
1021
|
for (const t of labels) {
|
|
968
|
-
const m = t.match(
|
|
1022
|
+
const m = String(t).trim().match(/^(?:\[|<|\{\{|__)([A-Z0-9_]+)_\d+(?:\]|>|\}\}|__)$/);
|
|
969
1023
|
const type = m ? m[1] : 'CUSTOM';
|
|
970
1024
|
counts[type] = (counts[type] || 0) + 1;
|
|
971
1025
|
}
|
|
@@ -1082,6 +1136,7 @@ export default {
|
|
|
1082
1136
|
restore,
|
|
1083
1137
|
restoreObject,
|
|
1084
1138
|
unscrub,
|
|
1139
|
+
desanitize,
|
|
1085
1140
|
validateLicense,
|
|
1086
1141
|
validateLicenseKey,
|
|
1087
1142
|
buildAuditTelemetry,
|
package/cli.js
CHANGED
|
@@ -21,9 +21,18 @@ import { sanitize, restore, validateLicense, generateProcurementMemo, StreamingD
|
|
|
21
21
|
const __filename = fileURLToPath(import.meta.url);
|
|
22
22
|
const __dirname = path.dirname(__filename);
|
|
23
23
|
|
|
24
|
-
const
|
|
24
|
+
const rawArgs = process.argv.slice(2);
|
|
25
|
+
const args = [];
|
|
26
|
+
for (const a of rawArgs) {
|
|
27
|
+
const eqIdx = a.indexOf('=');
|
|
28
|
+
if (a.startsWith('-') && eqIdx > 1) {
|
|
29
|
+
args.push(a.slice(0, eqIdx), a.slice(eqIdx + 1));
|
|
30
|
+
} else {
|
|
31
|
+
args.push(a);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
25
34
|
|
|
26
|
-
let sdkVersion = '2.4.
|
|
35
|
+
let sdkVersion = '2.4.3';
|
|
27
36
|
try {
|
|
28
37
|
const pkg = JSON.parse(fs.readFileSync(path.join(__dirname, 'package.json'), 'utf8'));
|
|
29
38
|
sdkVersion = pkg.version || sdkVersion;
|
|
@@ -104,7 +113,10 @@ if (args.includes('sidecar') || args.includes('--sidecar') || args.includes('ser
|
|
|
104
113
|
let port = 3100;
|
|
105
114
|
const portIdx = args.indexOf('--port') !== -1 ? args.indexOf('--port') : args.indexOf('-p');
|
|
106
115
|
if (portIdx !== -1 && args[portIdx + 1]) {
|
|
107
|
-
|
|
116
|
+
const parsedPort = parseInt(args[portIdx + 1], 10);
|
|
117
|
+
if (Number.isInteger(parsedPort) && parsedPort >= 1 && parsedPort <= 65535) {
|
|
118
|
+
port = parsedPort;
|
|
119
|
+
}
|
|
108
120
|
}
|
|
109
121
|
let host = process.env.HOST || '127.0.0.1';
|
|
110
122
|
const hostIdx = args.indexOf('--host');
|
|
@@ -176,9 +188,12 @@ if (args.includes('sidecar') || args.includes('--sidecar') || args.includes('ser
|
|
|
176
188
|
|
|
177
189
|
if (req.method === 'POST') {
|
|
178
190
|
let bodyStr = '';
|
|
191
|
+
let aborted = false;
|
|
179
192
|
req.on('data', chunk => {
|
|
193
|
+
if (aborted || res.writableEnded) return;
|
|
180
194
|
bodyStr += chunk;
|
|
181
195
|
if (bodyStr.length > 50 * 1024 * 1024) {
|
|
196
|
+
aborted = true;
|
|
182
197
|
res.writeHead(413, { 'Content-Type': 'application/json' });
|
|
183
198
|
res.end(JSON.stringify({ error: 'Payload Too Large' }));
|
|
184
199
|
req.destroy();
|
|
@@ -186,6 +201,7 @@ if (args.includes('sidecar') || args.includes('--sidecar') || args.includes('ser
|
|
|
186
201
|
});
|
|
187
202
|
|
|
188
203
|
req.on('end', () => {
|
|
204
|
+
if (aborted || res.writableEnded) return;
|
|
189
205
|
try {
|
|
190
206
|
const payload = bodyStr ? JSON.parse(bodyStr) : {};
|
|
191
207
|
const isScrub = reqUrl === '/scrub' || reqUrl === '/sanitize' || reqUrl === '/v1/scrub' || reqUrl === '/v1/sanitize';
|
|
@@ -389,6 +405,7 @@ fi
|
|
|
389
405
|
try { fs.chmodSync(preCommitPath, 0o755); } catch (_) {}
|
|
390
406
|
console.log('[PrivacyScrubber Hook] Successfully installed Zero-Trust pre-commit hook in .git/hooks/pre-commit.');
|
|
391
407
|
console.log('[PrivacyScrubber Hook] Staged changes will now be audited for unmasked secrets before commit.');
|
|
408
|
+
console.log('[GitHub] Star the open-source repository: https://github.com/moxno/privacyscrubber-mcp (CLI: gh repo star moxno/privacyscrubber-mcp)');
|
|
392
409
|
process.exit(0);
|
|
393
410
|
}
|
|
394
411
|
|
|
@@ -404,10 +421,15 @@ fi
|
|
|
404
421
|
fs.unlinkSync(preCommitPath);
|
|
405
422
|
}
|
|
406
423
|
console.log('[PrivacyScrubber Hook] Pre-commit hook removed successfully.');
|
|
424
|
+
console.log('[Feedback] Why did you remove the hook? (1-click instant log):');
|
|
425
|
+
console.log(' - False positive / Accuracy: https://privacyscrubber.com/uninstall/?src=sdk&reason=accuracy');
|
|
426
|
+
console.log(' - Workflow / Setup issue: https://privacyscrubber.com/uninstall/?src=sdk&reason=setup');
|
|
427
|
+
console.log(' - General feedback: https://privacyscrubber.com/uninstall/?mode=feedback&src=sdk');
|
|
407
428
|
process.exit(0);
|
|
408
429
|
}
|
|
409
430
|
}
|
|
410
431
|
console.log('[PrivacyScrubber Hook] Pre-commit hook was not found in active repository.');
|
|
432
|
+
console.log('[Feedback] Share feedback or report an issue: https://privacyscrubber.com/uninstall/?mode=feedback&src=sdk');
|
|
411
433
|
process.exit(0);
|
|
412
434
|
}
|
|
413
435
|
}
|
|
@@ -431,10 +453,16 @@ if (args.includes('scan-git-staged') || args.includes('--scan-git-staged')) {
|
|
|
431
453
|
}
|
|
432
454
|
|
|
433
455
|
const res = sanitize(addedLines, { profile: 'Dev', detectSecrets: true });
|
|
434
|
-
const
|
|
456
|
+
const includeNames = args.includes('--include-names') || args.includes('--strict');
|
|
457
|
+
const relevantTokens = Object.keys(res.tokenMap || {}).filter(
|
|
458
|
+
tok => includeNames || !/^\[NAME_\d+\]$/i.test(tok)
|
|
459
|
+
);
|
|
460
|
+
const maskedCount = relevantTokens.length;
|
|
435
461
|
|
|
436
462
|
if (maskedCount > 0) {
|
|
437
|
-
const entityTypes =
|
|
463
|
+
const entityTypes = Array.from(new Set(
|
|
464
|
+
relevantTokens.map(t => t.replace(/^\[|_\d+\]$/g, ''))
|
|
465
|
+
)).join(', ');
|
|
438
466
|
console.error(`
|
|
439
467
|
╔══════════════════════════════════════════════════════════════════════════════╗
|
|
440
468
|
║ 🚨 [PrivacyScrubber Git Shield] UNMASKED SECRETS / PII INTERCEPTED! ║
|
|
@@ -580,31 +608,47 @@ if (mapIdx !== -1 && args[mapIdx + 1]) {
|
|
|
580
608
|
}
|
|
581
609
|
|
|
582
610
|
// Locate input argument (first positional parameter that is not a flag or flag value)
|
|
611
|
+
const inputFlagIdx = args.indexOf('--input') !== -1 ? args.indexOf('--input') : args.indexOf('-i');
|
|
612
|
+
const outIdx = args.indexOf('--out') !== -1 ? args.indexOf('--out') : (args.indexOf('--output') !== -1 ? args.indexOf('--output') : args.indexOf('-o'));
|
|
613
|
+
const outputFile = (outIdx !== -1 && args[outIdx + 1]) ? args[outIdx + 1] : null;
|
|
614
|
+
|
|
583
615
|
const skipIndices = new Set();
|
|
584
616
|
if (configIdx !== -1) { skipIndices.add(configIdx); skipIndices.add(configIdx + 1); }
|
|
585
617
|
if (profIdx !== -1) { skipIndices.add(profIdx); skipIndices.add(profIdx + 1); }
|
|
586
618
|
if (keyIdx !== -1) { skipIndices.add(keyIdx); skipIndices.add(keyIdx + 1); }
|
|
587
619
|
if (mapIdx !== -1) { skipIndices.add(mapIdx); skipIndices.add(mapIdx + 1); }
|
|
588
620
|
if (formatIdx !== -1) { skipIndices.add(formatIdx); skipIndices.add(formatIdx + 1); }
|
|
589
|
-
|
|
621
|
+
if (inputFlagIdx !== -1) { skipIndices.add(inputFlagIdx); skipIndices.add(inputFlagIdx + 1); }
|
|
590
622
|
if (outIdx !== -1) { skipIndices.add(outIdx); skipIndices.add(outIdx + 1); }
|
|
591
623
|
|
|
592
|
-
let inputArg = null;
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
624
|
+
let inputArg = (inputFlagIdx !== -1 && args[inputFlagIdx + 1]) ? args[inputFlagIdx + 1] : null;
|
|
625
|
+
if (!inputArg) {
|
|
626
|
+
for (let i = 0; i < args.length; i++) {
|
|
627
|
+
if (skipIndices.has(i)) continue;
|
|
628
|
+
if (args[i] === 'sanitize' || args[i] === 'scrub') continue;
|
|
629
|
+
if (args[i] === '--verify-ci' || args[i] === '--ztds-proof' || args[i] === '--json' || args[i] === '--sarif' || args[i] === '--restore' || args[i] === '--unscrub') continue;
|
|
630
|
+
if (args[i].startsWith('-')) continue;
|
|
631
|
+
inputArg = args[i];
|
|
632
|
+
break;
|
|
633
|
+
}
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
function writeCliOutput(str) {
|
|
637
|
+
const payload = str.endsWith('\n') ? str : str + '\n';
|
|
638
|
+
if (outputFile) {
|
|
639
|
+
fs.writeFileSync(path.resolve(process.cwd(), outputFile), payload, 'utf8');
|
|
640
|
+
} else {
|
|
641
|
+
process.stdout.write(payload);
|
|
642
|
+
}
|
|
599
643
|
}
|
|
600
644
|
|
|
601
645
|
function handleExecution(text) {
|
|
602
646
|
if (isRestore) {
|
|
603
647
|
const { restoredText, restoredCount } = restore(text, tokenMap);
|
|
604
648
|
if (isJson) {
|
|
605
|
-
|
|
649
|
+
writeCliOutput(JSON.stringify({ restoredText, restoredCount }, null, 2));
|
|
606
650
|
} else {
|
|
607
|
-
|
|
651
|
+
writeCliOutput(restoredText);
|
|
608
652
|
}
|
|
609
653
|
process.exit(0);
|
|
610
654
|
}
|
|
@@ -648,9 +692,9 @@ function handleExecution(text) {
|
|
|
648
692
|
};
|
|
649
693
|
|
|
650
694
|
const proofJson = JSON.stringify(proof, null, 2);
|
|
651
|
-
if (
|
|
652
|
-
fs.writeFileSync(path.resolve(process.cwd(),
|
|
653
|
-
console.log(`[ZTDS Proof] Conformance proof saved to ${
|
|
695
|
+
if (outputFile) {
|
|
696
|
+
fs.writeFileSync(path.resolve(process.cwd(), outputFile), proofJson, 'utf8');
|
|
697
|
+
console.log(`[ZTDS Proof] Conformance proof saved to ${outputFile}`);
|
|
654
698
|
} else {
|
|
655
699
|
console.log(proofJson);
|
|
656
700
|
}
|
|
@@ -659,7 +703,7 @@ function handleExecution(text) {
|
|
|
659
703
|
|
|
660
704
|
if (isVerifyCi) {
|
|
661
705
|
if (!text || !text.trim()) {
|
|
662
|
-
|
|
706
|
+
writeCliOutput(`> **Verified by PrivacyScrubber (ZTDS Standard)**\n> * **Status:** PASSED · 0 PII / 0 Secrets detected in prompt pipeline (Empty payload)\n> * **Execution:** Local RAM (<1ms) · 0 Network Egress Bytes\n> * **Compliance Spec:** [Zero-Trust Data Sanitization (ZTDS)](https://privacyscrubber.com/solutions/engineering/zero-trust-ai-data-sanitization?src=ci_stamp) | ⭐ [Star on GitHub](https://github.com/moxno/privacyscrubber-mcp) | [Developer SDK ($299/mo)](https://privacyscrubber.com/sdk/?src=ci_stamp)`);
|
|
663
707
|
process.exit(0);
|
|
664
708
|
}
|
|
665
709
|
|
|
@@ -678,16 +722,16 @@ function handleExecution(text) {
|
|
|
678
722
|
> * **Audit Hash:** \`${hash}\` (Zero-Trust Verified)
|
|
679
723
|
> * **Compliance Spec:** [Zero-Trust Data Sanitization (ZTDS)](https://privacyscrubber.com/solutions/engineering/zero-trust-ai-data-sanitization?src=ci_stamp) | ⭐ [Star on GitHub](https://github.com/moxno/privacyscrubber-mcp) | [Developer SDK ($299/mo)](https://privacyscrubber.com/sdk/?src=ci_stamp)
|
|
680
724
|
`;
|
|
681
|
-
|
|
725
|
+
writeCliOutput(stamp.trim());
|
|
682
726
|
process.exit(0);
|
|
683
727
|
}
|
|
684
728
|
|
|
685
729
|
if (!text) {
|
|
686
730
|
if (isSarif) {
|
|
687
|
-
const { emitSarif } = require('./sarif.
|
|
688
|
-
|
|
731
|
+
const { emitSarif } = require('./sarif.cjs');
|
|
732
|
+
writeCliOutput(emitSarif([], { pretty: true }));
|
|
689
733
|
} else if (isJson) {
|
|
690
|
-
|
|
734
|
+
writeCliOutput(JSON.stringify({ scrubbedText: '', tokenMap: {}, count: 0 }, null, 2));
|
|
691
735
|
}
|
|
692
736
|
process.exit(0);
|
|
693
737
|
}
|
|
@@ -698,19 +742,19 @@ function handleExecution(text) {
|
|
|
698
742
|
customRules: loadedConfig.customRules || []
|
|
699
743
|
});
|
|
700
744
|
if (isSarif) {
|
|
701
|
-
const { emitSarif, sanitizeResultToFindings } = require('./sarif.
|
|
745
|
+
const { emitSarif, sanitizeResultToFindings } = require('./sarif.cjs');
|
|
702
746
|
const findings = sanitizeResultToFindings(res, text, inputArg || 'stdin');
|
|
703
747
|
const sarifOutput = emitSarif(findings, { pretty: true });
|
|
704
|
-
|
|
748
|
+
writeCliOutput(sarifOutput);
|
|
705
749
|
} else if (isJson) {
|
|
706
|
-
|
|
750
|
+
writeCliOutput(JSON.stringify({
|
|
707
751
|
scrubbedText: res.scrubbedText,
|
|
708
752
|
tokenMap: res.tokenMap,
|
|
709
753
|
count: res.count,
|
|
710
754
|
telemetry: res.telemetry
|
|
711
755
|
}, null, 2));
|
|
712
756
|
} else {
|
|
713
|
-
|
|
757
|
+
writeCliOutput(res.scrubbedText);
|
|
714
758
|
}
|
|
715
759
|
process.exit(0);
|
|
716
760
|
}
|
|
@@ -92,7 +92,6 @@ function sendResponse(id, result, error = null) {
|
|
|
92
92
|
// Setup readline interface on stdin
|
|
93
93
|
const rl = readline.createInterface({
|
|
94
94
|
input: process.stdin,
|
|
95
|
-
output: process.stdout,
|
|
96
95
|
terminal: false
|
|
97
96
|
});
|
|
98
97
|
|
|
@@ -116,7 +115,7 @@ rl.on('line', (line) => {
|
|
|
116
115
|
sendResponse(id, {
|
|
117
116
|
pong: true,
|
|
118
117
|
engine: 'PrivacyScrubber ZTDS',
|
|
119
|
-
version: '2.4.
|
|
118
|
+
version: '2.4.3',
|
|
120
119
|
uptime: process.uptime()
|
|
121
120
|
});
|
|
122
121
|
break;
|
|
@@ -163,17 +162,36 @@ rl.on('line', (line) => {
|
|
|
163
162
|
|
|
164
163
|
case 'sanitize_object': {
|
|
165
164
|
const data = req.data ?? {};
|
|
165
|
+
const persistSession = req.persist_session ?? req.persistSession ?? true;
|
|
166
|
+
let mergedIgnore = null;
|
|
167
|
+
const reqIgnore = req.ignore_list || req.ignoreList || null;
|
|
168
|
+
if (reqIgnore || engine.ignoreList.size > 0) {
|
|
169
|
+
mergedIgnore = new Set(engine.ignoreList);
|
|
170
|
+
if (reqIgnore) {
|
|
171
|
+
for (const item of reqIgnore) mergedIgnore.add(item);
|
|
172
|
+
}
|
|
173
|
+
}
|
|
166
174
|
const options = {
|
|
167
175
|
profile: req.profile || engine.defaultProfile,
|
|
168
176
|
detectSecrets: req.detect_secrets ?? req.detectSecrets ?? false,
|
|
169
177
|
licenseKey: req.license_key || req.licenseKey || engine.licenseKey,
|
|
178
|
+
generateReceipt: req.generate_receipt ?? req.generateReceipt ?? true,
|
|
170
179
|
customRules: req.custom_rules || req.customRules || [],
|
|
180
|
+
ignoreList: mergedIgnore,
|
|
181
|
+
existingSessionMap: persistSession ? engine.sessionMap : (req.existing_session_map || req.existingSessionMap || {}),
|
|
171
182
|
throwOnLimit: req.throw_on_limit ?? req.throwOnLimit ?? true
|
|
172
183
|
};
|
|
173
184
|
|
|
174
185
|
const res = sdk.sanitizeObject(data, options);
|
|
175
|
-
if (
|
|
186
|
+
if (persistSession && res.tokenMap) {
|
|
176
187
|
Object.assign(engine.sessionMap, res.tokenMap);
|
|
188
|
+
const keys = Object.keys(engine.sessionMap);
|
|
189
|
+
if (keys.length > engine.maxSessionTokens) {
|
|
190
|
+
const excess = keys.length - engine.maxSessionTokens;
|
|
191
|
+
for (let i = 0; i < excess; i++) {
|
|
192
|
+
delete engine.sessionMap[keys[i]];
|
|
193
|
+
}
|
|
194
|
+
}
|
|
177
195
|
}
|
|
178
196
|
sendResponse(id, {
|
|
179
197
|
sanitized: res.sanitized,
|
|
@@ -213,10 +231,11 @@ rl.on('line', (line) => {
|
|
|
213
231
|
}
|
|
214
232
|
|
|
215
233
|
case 'status': {
|
|
234
|
+
const tier = engine.license?.tier || 'FREE';
|
|
216
235
|
sendResponse(id, {
|
|
217
236
|
profile: engine.defaultProfile,
|
|
218
|
-
license_tier:
|
|
219
|
-
is_pro: engine.license?.
|
|
237
|
+
license_tier: tier,
|
|
238
|
+
is_pro: Boolean(engine.license?.valid && ['SDK', 'OEM', 'ENTERPRISE', 'PRO', 'TEAMS'].includes(tier)),
|
|
220
239
|
session_tokens_count: Object.keys(engine.sessionMap).length,
|
|
221
240
|
ignore_list_size: engine.ignoreList.size,
|
|
222
241
|
memory_usage: process.memoryUsage()
|
|
@@ -92,7 +92,6 @@ function sendResponse(id, result, error = null) {
|
|
|
92
92
|
// Setup readline interface on stdin
|
|
93
93
|
const rl = readline.createInterface({
|
|
94
94
|
input: process.stdin,
|
|
95
|
-
output: process.stdout,
|
|
96
95
|
terminal: false
|
|
97
96
|
});
|
|
98
97
|
|
|
@@ -116,7 +115,7 @@ rl.on('line', (line) => {
|
|
|
116
115
|
sendResponse(id, {
|
|
117
116
|
pong: true,
|
|
118
117
|
engine: 'PrivacyScrubber ZTDS',
|
|
119
|
-
version: '2.4.
|
|
118
|
+
version: '2.4.3',
|
|
120
119
|
uptime: process.uptime()
|
|
121
120
|
});
|
|
122
121
|
break;
|
|
@@ -163,17 +162,36 @@ rl.on('line', (line) => {
|
|
|
163
162
|
|
|
164
163
|
case 'sanitize_object': {
|
|
165
164
|
const data = req.data ?? {};
|
|
165
|
+
const persistSession = req.persist_session ?? req.persistSession ?? true;
|
|
166
|
+
let mergedIgnore = null;
|
|
167
|
+
const reqIgnore = req.ignore_list || req.ignoreList || null;
|
|
168
|
+
if (reqIgnore || engine.ignoreList.size > 0) {
|
|
169
|
+
mergedIgnore = new Set(engine.ignoreList);
|
|
170
|
+
if (reqIgnore) {
|
|
171
|
+
for (const item of reqIgnore) mergedIgnore.add(item);
|
|
172
|
+
}
|
|
173
|
+
}
|
|
166
174
|
const options = {
|
|
167
175
|
profile: req.profile || engine.defaultProfile,
|
|
168
176
|
detectSecrets: req.detect_secrets ?? req.detectSecrets ?? false,
|
|
169
177
|
licenseKey: req.license_key || req.licenseKey || engine.licenseKey,
|
|
178
|
+
generateReceipt: req.generate_receipt ?? req.generateReceipt ?? true,
|
|
170
179
|
customRules: req.custom_rules || req.customRules || [],
|
|
180
|
+
ignoreList: mergedIgnore,
|
|
181
|
+
existingSessionMap: persistSession ? engine.sessionMap : (req.existing_session_map || req.existingSessionMap || {}),
|
|
171
182
|
throwOnLimit: req.throw_on_limit ?? req.throwOnLimit ?? true
|
|
172
183
|
};
|
|
173
184
|
|
|
174
185
|
const res = sdk.sanitizeObject(data, options);
|
|
175
|
-
if (
|
|
186
|
+
if (persistSession && res.tokenMap) {
|
|
176
187
|
Object.assign(engine.sessionMap, res.tokenMap);
|
|
188
|
+
const keys = Object.keys(engine.sessionMap);
|
|
189
|
+
if (keys.length > engine.maxSessionTokens) {
|
|
190
|
+
const excess = keys.length - engine.maxSessionTokens;
|
|
191
|
+
for (let i = 0; i < excess; i++) {
|
|
192
|
+
delete engine.sessionMap[keys[i]];
|
|
193
|
+
}
|
|
194
|
+
}
|
|
177
195
|
}
|
|
178
196
|
sendResponse(id, {
|
|
179
197
|
sanitized: res.sanitized,
|
|
@@ -213,10 +231,11 @@ rl.on('line', (line) => {
|
|
|
213
231
|
}
|
|
214
232
|
|
|
215
233
|
case 'status': {
|
|
234
|
+
const tier = engine.license?.tier || 'FREE';
|
|
216
235
|
sendResponse(id, {
|
|
217
236
|
profile: engine.defaultProfile,
|
|
218
|
-
license_tier:
|
|
219
|
-
is_pro: engine.license?.
|
|
237
|
+
license_tier: tier,
|
|
238
|
+
is_pro: Boolean(engine.license?.valid && ['SDK', 'OEM', 'ENTERPRISE', 'PRO', 'TEAMS'].includes(tier)),
|
|
220
239
|
session_tokens_count: Object.keys(engine.sessionMap).length,
|
|
221
240
|
ignore_list_size: engine.ignoreList.size,
|
|
222
241
|
memory_usage: process.memoryUsage()
|