@privacyscrubber/sdk 2.3.5 → 2.3.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +65 -2
- package/browser.js +1 -2
- package/cli.js +187 -11
- package/index.cjs +2682 -0
- package/index.d.ts +9 -0
- package/index.js +113 -2
- package/package.json +11 -3
- package/ps-license-manager.js +1 -1
- package/ps-pii-engine.cjs +1 -1
- package/scrubber-core.cjs +1 -1
- package/vision.cjs +369 -0
package/index.cjs
ADDED
|
@@ -0,0 +1,2682 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* PrivacyScrubber Developer SDK (@privacyscrubber/sdk)
|
|
3
|
+
* Zero-Trust Data Sanitization & PII Redaction Engine.
|
|
4
|
+
*
|
|
5
|
+
* 100% Client-Side / Server-Side in-memory execution.
|
|
6
|
+
* Zero telemetry network requests. Zero data leakage.
|
|
7
|
+
*
|
|
8
|
+
* https://privacyscrubber.com
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
const fs = require('fs');
|
|
12
|
+
const path = require('path');
|
|
13
|
+
const os = require('os');
|
|
14
|
+
const { execSync } = require('child_process');
|
|
15
|
+
const { Transform } = require('stream');
|
|
16
|
+
|
|
17
|
+
require('./polyfill.js');
|
|
18
|
+
require('./ps-license-manager.js');
|
|
19
|
+
|
|
20
|
+
// Access LicenseManager from global scope
|
|
21
|
+
const LicenseManager = (typeof globalThis !== 'undefined' && globalThis.LicenseManager)
|
|
22
|
+
|| (typeof global !== 'undefined' && global.LicenseManager)
|
|
23
|
+
|| (typeof window !== 'undefined' && window.LicenseManager);
|
|
24
|
+
|
|
25
|
+
// Synchronously load the production CJS core engine
|
|
26
|
+
require('./scrubber-core.cjs');
|
|
27
|
+
const scrubberPkg = (typeof globalThis !== 'undefined' && globalThis.PrivacyScrubberCore)
|
|
28
|
+
|| (typeof global !== 'undefined' && global.PrivacyScrubberCore)
|
|
29
|
+
|| (typeof window !== 'undefined' && window.PrivacyScrubberCore);
|
|
30
|
+
|
|
31
|
+
if (scrubberPkg && typeof scrubberPkg.init === 'function') {
|
|
32
|
+
scrubberPkg.init();
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Built-in patterns for DevOps & Cloud Secrets detection.
|
|
37
|
+
*/
|
|
38
|
+
const DEVOPS_SECRETS_DETECTOR = [
|
|
39
|
+
{ name: 'AWS Credentials', regex: /\b(?:AKIA|ASIA|AGPA|AIDA|AROA|AIPA)[A-Z0-9]{16}\b/g, label: 'AWS_KEY' },
|
|
40
|
+
{ name: 'JSON Web Token (JWT)', regex: /\beyJ[a-zA-Z0-9_-]+\.[a-zA-Z0-9_-]+\.[a-zA-Z0-9_-]+\b/g, label: 'JWT_TOKEN' },
|
|
41
|
+
{ name: 'API Token/Key (GitHub/Slack/NPM)', regex: /\b(?:ghp|gho|ghu|ghs|ghr|glpat|npm|xox[baprs])[-_][A-Za-z0-9_]{10,}\b/g, label: 'API_TOKEN' },
|
|
42
|
+
{ name: 'Stripe API Key', regex: /\b(?:[rs]k)_(?:test|live)_[a-zA-Z0-9]{24,}\b/g, label: 'STRIPE_KEY' },
|
|
43
|
+
{ name: 'OpenAI Project API Key', regex: /\b(?:sk|pk)-(?:proj-)?[a-zA-Z0-9_-]{16,}\b/gi, label: 'OPENAI_KEY' },
|
|
44
|
+
{ name: 'Database Connection URI', regex: /\b(?:postgres(?:ql)?|mysql|mongodb(?:\+srv)?|redis|amqp|mssql):\/\/[^\s"']+/gi, label: 'DB_URI' },
|
|
45
|
+
{ name: 'Embedded Credentials URI', regex: /\b(?:https?|s?ftp):\/\/[a-zA-Z0-9_.%+-]+:[^\s/@:"']+@[a-zA-Z0-9_.-]+(?::\d+)?(?:\/[^\s"']*)?/gi, label: 'SECRET' },
|
|
46
|
+
{ name: 'URL Query Parameter Secret', regex: /(?<=[?&](?:api_key|apikey|access_token|client_secret|auth_token|token|secret|password|private_key)=)[^\s"'\x26#)>]{8,}/gi, label: 'SECRET' },
|
|
47
|
+
{ name: 'Bearer / Auth Header Token', regex: /(?<=\b(?:Bearer|Token)\s+)[a-zA-Z0-9_\-\.~+/=]{16,}\b/gi, label: 'SECRET' },
|
|
48
|
+
{ name: 'Database/API Secret', regex: /\b(DB|POSTGRES|REDIS|MYSQL|AWS|SECRET|PASSWORD|TOKEN|API|KEY)[A-Z0-9_]*\s*[:=]\s*[^ \t\r\n"']{8,}\b/gi, label: 'SECRET' },
|
|
49
|
+
{ name: 'Private Cryptographic Key (PEM Block)', regex: /-----BEGIN (?:RSA |EC |PGP |DSA |OPENSSH |ENCRYPTED )?PRIVATE KEY(?: BLOCK)?-----[\s\S]{10,8192}?-----END (?:RSA |EC |PGP |DSA |OPENSSH |ENCRYPTED )?PRIVATE KEY(?: BLOCK)?-----/g, label: 'SECRET' },
|
|
50
|
+
{ name: 'Private Cryptographic Key (Header)', regex: /-----BEGIN (?:RSA |EC |PGP |DSA |OPENSSH |ENCRYPTED )?PRIVATE KEY(?: BLOCK)?-----/g, label: 'SECRET' }
|
|
51
|
+
];
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Custom Error thrown when Free Tier limits are exceeded.
|
|
55
|
+
*/
|
|
56
|
+
class PrivacyScrubberLicenseError extends Error {
|
|
57
|
+
constructor(message, { tier = 'FREE', currentLength = 0, limit = 15000, profile = 'General' } = {}) {
|
|
58
|
+
super(message);
|
|
59
|
+
this.name = 'PrivacyScrubberLicenseError';
|
|
60
|
+
this.tier = tier;
|
|
61
|
+
this.currentLength = currentLength;
|
|
62
|
+
this.limit = limit;
|
|
63
|
+
this.profile = profile;
|
|
64
|
+
this.upgradeUrl = 'https://privacyscrubber.com/pricing?utm_source=npm_sdk&utm_medium=sdk_error&utm_campaign=dev_upsell';
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Validates a PrivacyScrubber license key (RS256 JWT or Cryptographic Checksum).
|
|
70
|
+
* @param {string} key - License key string.
|
|
71
|
+
* @returns {{ valid: boolean, tier: 'PRO'|'TEAMS'|null, expires: number|null, error: string|null }}
|
|
72
|
+
*/
|
|
73
|
+
function validateLicense(key) {
|
|
74
|
+
if (!key || typeof key !== 'string') {
|
|
75
|
+
return { valid: false, tier: null, expires: null, error: 'No license key provided.' };
|
|
76
|
+
}
|
|
77
|
+
const cleanKey = key.trim();
|
|
78
|
+
if (!LicenseManager || typeof LicenseManager.validate !== 'function') {
|
|
79
|
+
return { valid: false, tier: null, expires: null, error: 'License manager engine unavailable.' };
|
|
80
|
+
}
|
|
81
|
+
const result = LicenseManager.validate(cleanKey);
|
|
82
|
+
return {
|
|
83
|
+
valid: !!result.valid,
|
|
84
|
+
tier: result.valid ? (result.tier || 'PRO') : null,
|
|
85
|
+
expires: result.expires || null,
|
|
86
|
+
error: result.valid ? null : (result.reason || 'Invalid license key.')
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Legacy validator helper returning active tier or null.
|
|
92
|
+
* @param {string} key
|
|
93
|
+
* @returns {'PRO'|'TEAMS'|null}
|
|
94
|
+
*/
|
|
95
|
+
function validateLicenseKey(key) {
|
|
96
|
+
const res = validateLicense(key);
|
|
97
|
+
return res.valid ? res.tier : null;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Builds CISO-grade compliance audit telemetry from masked tokens.
|
|
102
|
+
* @param {Record<string, string>|string[]} currentTokenMap
|
|
103
|
+
* @returns {object}
|
|
104
|
+
*/
|
|
105
|
+
function buildAuditTelemetry(currentTokenMap = {}) {
|
|
106
|
+
const entities = {};
|
|
107
|
+
let totalCount = 0;
|
|
108
|
+
|
|
109
|
+
if (currentTokenMap && typeof currentTokenMap === 'object') {
|
|
110
|
+
const keys = Array.isArray(currentTokenMap) ? currentTokenMap : Object.keys(currentTokenMap);
|
|
111
|
+
for (const t of keys) {
|
|
112
|
+
if (!t) continue;
|
|
113
|
+
const tokenStr = typeof t === 'string' ? t : ((t && (t.token || t.mask)) ? String(t.token || t.mask) : '');
|
|
114
|
+
if (!tokenStr) continue;
|
|
115
|
+
const match = tokenStr.match(/\[([A-Z_]+)_\d+\]/);
|
|
116
|
+
const baseType = match ? match[1] : (tokenStr.replace(/\[|\]/g, '').replace(/_[0-9]+$/, '') || 'CUSTOM');
|
|
117
|
+
entities[baseType] = (entities[baseType] || 0) + 1;
|
|
118
|
+
totalCount++;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
const types = Object.keys(entities);
|
|
123
|
+
let riskLevel = 'LOW EXPOSURE';
|
|
124
|
+
|
|
125
|
+
if (totalCount > 0) {
|
|
126
|
+
const hasHighRisk = types.some(t => ['ID', 'SSN', 'CREDIT_CARD', 'PASSPORT', 'BANK', 'API_KEY', 'SECRET', 'PASSWORD', 'MRN', 'KEY', 'AWS_KEY', 'STRIPE_KEY', 'JWT_TOKEN', 'API_TOKEN'].includes(t));
|
|
127
|
+
if (hasHighRisk || types.length >= 3 || totalCount >= 10) {
|
|
128
|
+
riskLevel = 'CRITICAL (HIGH EXPOSURE)';
|
|
129
|
+
} else if (types.length >= 2 || totalCount >= 3) {
|
|
130
|
+
riskLevel = 'MODERATE EXPOSURE';
|
|
131
|
+
} else {
|
|
132
|
+
riskLevel = 'LOW EXPOSURE';
|
|
133
|
+
}
|
|
134
|
+
} else {
|
|
135
|
+
riskLevel = 'CLEAN (ZERO PII)';
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
const frameworksSet = new Set(['ZTDS Standard']);
|
|
139
|
+
if (types.includes('NAME') || types.includes('EMAIL') || types.includes('PHONE')) {
|
|
140
|
+
frameworksSet.add('GDPR (Art. 4)');
|
|
141
|
+
frameworksSet.add('CCPA/CPRA');
|
|
142
|
+
}
|
|
143
|
+
if (types.includes('ID') || types.includes('SSN') || types.includes('PASSPORT') || types.includes('NATIONAL_ID')) {
|
|
144
|
+
frameworksSet.add('SOC 2 Type II');
|
|
145
|
+
frameworksSet.add('ISO 27001 (A.8.11)');
|
|
146
|
+
}
|
|
147
|
+
if (types.some(t => ['CREDIT_CARD', 'BANK', 'IBAN', 'FINANCIAL', 'CARD', 'STRIPE_KEY', 'TAX_ID', 'EIN', 'W2', '1099', 'SALARY', 'WAGE'].includes(t))) {
|
|
148
|
+
frameworksSet.add('PCI DSS v4.0');
|
|
149
|
+
frameworksSet.add('IRC § 7216 & GLBA');
|
|
150
|
+
}
|
|
151
|
+
if (types.some(t => ['MRN', 'HEALTH', 'MEDICAL', 'PATIENT', 'DIAGNOSIS', 'NPI', 'DEA'].includes(t))) {
|
|
152
|
+
frameworksSet.add('HIPAA §164.514');
|
|
153
|
+
}
|
|
154
|
+
if (types.some(t => ['CASE_NUMBER', 'DOCKET', 'PRIVILEGED', 'LEGAL', 'ATTORNEY', 'COURT', 'CONTRACT', 'NDA'].includes(t))) {
|
|
155
|
+
frameworksSet.add('FRE 502 (Privilege)');
|
|
156
|
+
}
|
|
157
|
+
if (types.some(t => ['API_KEY', 'SECRET', 'PASSWORD', 'TOKEN', 'KEY', 'AWS_KEY', 'JWT_TOKEN', 'API_TOKEN', 'CREDENTIAL'].includes(t))) {
|
|
158
|
+
frameworksSet.add('NIST SP 800-53');
|
|
159
|
+
}
|
|
160
|
+
if (totalCount > 0) {
|
|
161
|
+
frameworksSet.add('EU AI Act (Art. 50)');
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
const ipHoldings = {
|
|
165
|
+
standard: 'IETF draft-sibiryakov-ztds-protocol-00',
|
|
166
|
+
standardUrl: 'https://datatracker.ietf.org/doc/draft-sibiryakov-ztds-protocol/',
|
|
167
|
+
patent: "U.S. & Int'l Patents Pending (Paris Convention Priority)",
|
|
168
|
+
trademark: 'ZTDS™ Reg. #182655957 (ILPO Cl 9 & 42)'
|
|
169
|
+
};
|
|
170
|
+
|
|
171
|
+
const statutoryCitations = [
|
|
172
|
+
{
|
|
173
|
+
statute: 'EU AI Act (Art. 50)',
|
|
174
|
+
scope: 'AI Transparency & Data Minimization',
|
|
175
|
+
rule: '0-byte external network transmission; prevents unauthorized ingestion into frontier models.'
|
|
176
|
+
},
|
|
177
|
+
{
|
|
178
|
+
statute: 'GDPR (Art. 4, 25, 28, 32)',
|
|
179
|
+
scope: 'Pseudonymization & Processor Liability',
|
|
180
|
+
rule: 'Volatile client-side tokenization eliminates GDPR Article 28 data processor liability.'
|
|
181
|
+
},
|
|
182
|
+
{
|
|
183
|
+
statute: 'HIPAA 45 CFR § 164.514(b)',
|
|
184
|
+
scope: 'Safe Harbor De-Identification',
|
|
185
|
+
rule: 'Deterministic removal of all 18 PHI identifiers prior to AI analysis.'
|
|
186
|
+
},
|
|
187
|
+
{
|
|
188
|
+
statute: 'IRC § 7216 & IRS Pub. 1075',
|
|
189
|
+
scope: 'Tax Return Information Confidentiality',
|
|
190
|
+
rule: 'Strict criminal penalty safeguard against unauthorized disclosure of tax records to cloud LLMs.'
|
|
191
|
+
},
|
|
192
|
+
{
|
|
193
|
+
statute: 'FRE 502',
|
|
194
|
+
scope: 'Attorney-Client Privilege Preservation',
|
|
195
|
+
rule: 'Automated in-memory redaction prevents inadvertent waiver of legal privilege in AI workflows.'
|
|
196
|
+
}
|
|
197
|
+
];
|
|
198
|
+
|
|
199
|
+
return {
|
|
200
|
+
totalCount,
|
|
201
|
+
entities,
|
|
202
|
+
types,
|
|
203
|
+
riskLevel,
|
|
204
|
+
frameworksList: Array.from(frameworksSet),
|
|
205
|
+
statutoryCitations,
|
|
206
|
+
ipHoldings,
|
|
207
|
+
multimodalEgress: '0.00 Bytes (Solid Blackout & Burn-in Verified)'
|
|
208
|
+
};
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* SSOT Alias for cross-plane parity with Web, Extension, and MCP.
|
|
213
|
+
*/
|
|
214
|
+
const buildCisoAuditTelemetry = buildAuditTelemetry;
|
|
215
|
+
|
|
216
|
+
/**
|
|
217
|
+
* Formats structured audit telemetry into a human-readable Markdown Audit Receipt.
|
|
218
|
+
* @param {object} telemetry
|
|
219
|
+
* @returns {string}
|
|
220
|
+
*/
|
|
221
|
+
function formatAuditReceipt(telemetry) {
|
|
222
|
+
if (!telemetry || telemetry.totalCount === 0) {
|
|
223
|
+
return "> 🛡️ **PrivacyScrubber Audit Receipt**: CLEAN (ZERO PII DETECTED)\n" +
|
|
224
|
+
"> * ⭐ **Star on GitHub:** [moxno/privacyscrubber-mcp](https://github.com/moxno/privacyscrubber-mcp) | **SDK & Enterprise:** [privacyscrubber.com/pricing](https://privacyscrubber.com/pricing)\n";
|
|
225
|
+
}
|
|
226
|
+
const entitiesList = Object.entries(telemetry.entities)
|
|
227
|
+
.map(([type, count]) => `${count} ${type}`)
|
|
228
|
+
.join(', ');
|
|
229
|
+
|
|
230
|
+
const icon = telemetry.riskLevel.includes('CRITICAL') ? '🔴' : (telemetry.riskLevel.includes('MODERATE') ? '🟠' : '🟢');
|
|
231
|
+
|
|
232
|
+
return `> 🛡️ **PrivacyScrubber Audit Receipt**\n` +
|
|
233
|
+
`> * **Risk Level:** ${icon} ${telemetry.riskLevel}\n` +
|
|
234
|
+
`> * **Compliance Enforced:** ${telemetry.frameworksList.join(', ')}\n` +
|
|
235
|
+
`> * **Tokens Masked:** ${telemetry.totalCount} (${entitiesList})\n` +
|
|
236
|
+
`> * ⭐ **Star on GitHub:** [moxno/privacyscrubber-mcp](https://github.com/moxno/privacyscrubber-mcp) | **SDK & Enterprise:** [privacyscrubber.com/pricing](https://privacyscrubber.com/pricing)\n`;
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
let communityNoticeLogged = false;
|
|
240
|
+
|
|
241
|
+
function emitCommunityNoticeIfNeeded(isPro) {
|
|
242
|
+
if (communityNoticeLogged || isPro) return;
|
|
243
|
+
if (typeof process !== 'undefined' && process.env?.NODE_ENV === 'production') {
|
|
244
|
+
communityNoticeLogged = true;
|
|
245
|
+
console.info('[@privacyscrubber/sdk] Running Community Edition (Core PII).\nTo unlock 25 Enterprise Profiles & DevOps Secrets in production: https://privacyscrubber.com/pricing/?tier=sdk');
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
/**
|
|
250
|
+
* Resets the community notice state for testing purposes.
|
|
251
|
+
* @private
|
|
252
|
+
*/
|
|
253
|
+
function _resetCommunityNoticeForTesting() {
|
|
254
|
+
communityNoticeLogged = false;
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
let devSecretNoticeLogged = false;
|
|
258
|
+
|
|
259
|
+
function emitDevSecretNoticeIfNeeded(isPro, tokenMap = {}, limit = 15000) {
|
|
260
|
+
if (devSecretNoticeLogged || isPro) return;
|
|
261
|
+
if (typeof process !== 'undefined' && process.env?.NODE_ENV === 'production') return;
|
|
262
|
+
|
|
263
|
+
const tokens = Object.keys(tokenMap || {});
|
|
264
|
+
const secretTokens = tokens.filter(t => {
|
|
265
|
+
const upper = t.toUpperCase();
|
|
266
|
+
return ['API_KEY', 'SECRET', 'PASSWORD', 'KEY', 'AWS_KEY', 'STRIPE_KEY', 'JWT_TOKEN', 'API_TOKEN', 'TOKEN'].some(k => upper.includes(k));
|
|
267
|
+
});
|
|
268
|
+
|
|
269
|
+
if (secretTokens.length > 0) {
|
|
270
|
+
devSecretNoticeLogged = true;
|
|
271
|
+
const count = secretTokens.length;
|
|
272
|
+
const secretSample = secretTokens.slice(0, 2).join(', ');
|
|
273
|
+
if (typeof process !== 'undefined' && process.stderr && typeof process.stderr.write === 'function') {
|
|
274
|
+
process.stderr.write(
|
|
275
|
+
`\n[PrivacyScrubber] Intercepted ${count} API secret(s) before LLM egress: ${secretSample}\n` +
|
|
276
|
+
`[PrivacyScrubber] Protected in local RAM (0 byte network egress). Free Tier active (${limit.toLocaleString()} char cap).\n` +
|
|
277
|
+
`[PrivacyScrubber] Developer SDK ($299/mo flat) for unlimited backend nodes, CI/CD gates & ETL pipelines: https://privacyscrubber.com/pricing?tier=sdk&src=cli_leak_guard\n\n`
|
|
278
|
+
);
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
/**
|
|
284
|
+
* Resets the dev secret notice state for testing purposes.
|
|
285
|
+
* @private
|
|
286
|
+
*/
|
|
287
|
+
function _resetDevSecretNoticeForTesting() {
|
|
288
|
+
devSecretNoticeLogged = false;
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
/**
|
|
292
|
+
* Generates an executive CISO & Legal Procurement Memo for enterprise approval of @privacyscrubber/sdk.
|
|
293
|
+
*
|
|
294
|
+
* @param {object} [options={}] - Custom memo configuration options.
|
|
295
|
+
* @param {string} [options.appName='Internal AI Applications & LLM Pipelines'] - Target system name.
|
|
296
|
+
* @param {string} [options.developer='Engineering / AI Platform Team'] - Authoring team name.
|
|
297
|
+
* @param {'markdown'|'text'} [options.format='markdown'] - Output format ('markdown' or 'text').
|
|
298
|
+
* @param {number} [options.teamSize=10] - Size of engineering team.
|
|
299
|
+
* @param {string} [options.estimatedVolume='50M - 500M characters/month'] - Projected token volume.
|
|
300
|
+
* @returns {string}
|
|
301
|
+
*/
|
|
302
|
+
function generateProcurementMemo(options = {}) {
|
|
303
|
+
const {
|
|
304
|
+
appName = 'Internal AI Applications & LLM Pipelines',
|
|
305
|
+
developer = 'Engineering / AI Platform Team',
|
|
306
|
+
format = 'markdown',
|
|
307
|
+
teamSize = 10,
|
|
308
|
+
estimatedVolume = '50M - 500M characters/month'
|
|
309
|
+
} = options;
|
|
310
|
+
|
|
311
|
+
if (format === 'text') {
|
|
312
|
+
return `MEMORANDUM: ADOPTION OF PRIVACYSCRUBBER DEVELOPER SDK (ZTDS STANDARD)
|
|
313
|
+
|
|
314
|
+
TO: Chief Information Security Officer (CISO), Data Protection Officer (DPO), VP Engineering
|
|
315
|
+
FROM: ${developer}
|
|
316
|
+
SUBJECT: Approval Request: Zero-Trust Data Sanitization (@privacyscrubber/sdk)
|
|
317
|
+
TARGET SYSTEM: ${appName}
|
|
318
|
+
ESTIMATED VOLUME: ${estimatedVolume}
|
|
319
|
+
|
|
320
|
+
1. EXECUTIVE SUMMARY
|
|
321
|
+
We propose standardizing on the PrivacyScrubber Developer SDK (@privacyscrubber/sdk) for in-memory Zero-Trust Data Sanitization across our AI ingestion pipelines. The SDK intercepts and redacts PII, regulatory identifiers, and cloud secrets locally in RAM before any prompt leaves our infrastructure.
|
|
322
|
+
|
|
323
|
+
2. ZERO-TRUST ARCHITECTURAL INVARIANTS
|
|
324
|
+
* Zero Network Egress: 100% local in-memory processing. Zero external API calls, zero telemetry egress, zero third-party sub-processors.
|
|
325
|
+
* In-Memory Volatility: All masking maps remain in volatile process RAM and are discarded after request completion.
|
|
326
|
+
* Sub-Millisecond Latency: Executes in <0.1ms per prompt in local RAM, eliminating latency overhead.
|
|
327
|
+
|
|
328
|
+
3. COMPLIANCE & REGULATORY COVERAGE
|
|
329
|
+
Pre-configured for 30 specialized industry profiles and compliance frameworks:
|
|
330
|
+
* GDPR (Art. 4, 9, 32) & CCPA/CPRA: Ephemeral tokenization.
|
|
331
|
+
* HIPAA (§164.514 Safe Harbor): MRNs, medical record codes, patient identifiers.
|
|
332
|
+
* SOC 2 Type II & ISO 27001 (A.8.11): Zero persistent unencrypted PII.
|
|
333
|
+
* PCI DSS v4.0: PAN, CVV, IBAN, bank ledger sanitization.
|
|
334
|
+
* NIST SP 800-53: Cloud API keys, JWT tokens, AWS/Stripe secret leak protection.
|
|
335
|
+
|
|
336
|
+
4. TOTAL COST OF OWNERSHIP (TCO) COMPARISON
|
|
337
|
+
* Cloud DLP Proxies (AWS Macie, GCP DLP, Nightfall):
|
|
338
|
+
- Cost: $1,500 - $4,000 / month (usage-based billing).
|
|
339
|
+
- Latency: +200ms - 400ms added network roundtrip per AI prompt.
|
|
340
|
+
- Sub-processor Risk: Requires vendor DPA, external network egress.
|
|
341
|
+
* PrivacyScrubber Developer SDK:
|
|
342
|
+
- Cost: $299 / month flat (or $2,990 / year) for unlimited nodes, microservices, and prompts.
|
|
343
|
+
- Latency: <0.1ms (in-process RAM).
|
|
344
|
+
- Sub-processor Risk: Zero (pure in-memory software library).
|
|
345
|
+
|
|
346
|
+
5. ACTION REQUESTED
|
|
347
|
+
Approve commercial license upgrade for ${developer} (${teamSize} engineers, unlimited backend nodes):
|
|
348
|
+
* License Tier: Developer SDK ($299/month flat or $2,990/year)
|
|
349
|
+
* Direct Procurement & License Activation: https://privacyscrubber.com/pricing?tier=sdk&src=ciso_memo
|
|
350
|
+
* Enterprise / Invoicing Contact: enterprise@privacyscrubber.com
|
|
351
|
+
`;
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
return `# Internal Memo: Adoption of PrivacyScrubber Developer SDK (ZTDS Standard)
|
|
355
|
+
|
|
356
|
+
| **To** | Chief Information Security Officer (CISO), Data Protection Officer (DPO), VP Engineering |
|
|
357
|
+
| :--- | :--- |
|
|
358
|
+
| **From** | ${developer} |
|
|
359
|
+
| **Date** | ${new Date().toISOString().split('T')[0]} |
|
|
360
|
+
| **Subject** | Architectural Approval & License Procurement for Zero-Trust AI Sanitization |
|
|
361
|
+
| **Target System** | ${appName} |
|
|
362
|
+
| **Estimated Scope** | ~${teamSize} engineers / ${estimatedVolume} |
|
|
363
|
+
|
|
364
|
+
---
|
|
365
|
+
|
|
366
|
+
### 1. Executive Summary
|
|
367
|
+
We recommend adopting the **PrivacyScrubber Developer SDK** (\`@privacyscrubber/sdk\`) to enforce **Zero-Trust Data Sanitization (ZTDS)** across all internal AI pipelines, LangChain agents, microservices, and developer toolchains.
|
|
368
|
+
|
|
369
|
+
The SDK redacts PII, regulatory identifiers, and developer secrets **in-process before external API egress to OpenAI, Anthropic, or external LLM providers**, and deterministically restores original values upon response return.
|
|
370
|
+
|
|
371
|
+
---
|
|
372
|
+
|
|
373
|
+
### 2. Architectural Invariants (Zero-Trust Data Sanitization)
|
|
374
|
+
* **Zero Remote Network Egress:** 100% of sanitization and token restoration executes in local Node.js process RAM. No prompt data or telemetry ever leaves our VPC/infrastructure.
|
|
375
|
+
* **Ephemeral In-Memory Tokens:** Masking mappings (\`[EMAIL_1]\`, \`[NAME_1]\`, \`[SECRET_1]\`) are volatile and strictly scoped to request execution context.
|
|
376
|
+
* **Ultra-Low Latency:** Micro-benchmarks verify <0.1ms execution overhead per prompt, compared to 200–400ms for network-based cloud DLP proxies.
|
|
377
|
+
* **Full Reversibility:** Bidirectional round-trip reconstruction ensures downstream application logic receives intact responses without exposing sensitive data to model providers.
|
|
378
|
+
|
|
379
|
+
---
|
|
380
|
+
|
|
381
|
+
### 3. Compliance Framework Mapping
|
|
382
|
+
The SDK enforces out-of-the-box sanitization matching mandatory compliance controls:
|
|
383
|
+
* **GDPR (Regulation (EU) 2016/679) & CCPA/CPRA:** Eliminates cross-border PII transfer risk to third-party AI models.
|
|
384
|
+
* **HIPAA (§164.514 Safe Harbor):** Automated scrubbing of Medical Record Numbers (MRNs), patient identifiers, and health insurance IDs.
|
|
385
|
+
* **SOC 2 Type II & ISO 27001 (Control A.8.11):** Enforces data masking and prevents unauthorized logging of confidential client data.
|
|
386
|
+
* **PCI DSS v4.0:** Sanitization of payment cards, IBANs, and financial ledger data.
|
|
387
|
+
* **NIST SP 800-53:** DevOps secret interception for AWS keys, JWT tokens, database URIs, and private API keys.
|
|
388
|
+
|
|
389
|
+
---
|
|
390
|
+
|
|
391
|
+
### 4. Financial & Architectural Comparison (TCO)
|
|
392
|
+
|
|
393
|
+
| Evaluation Factor | Traditional Cloud DLP (AWS/GCP/Nightfall) | PrivacyScrubber Developer SDK |
|
|
394
|
+
| :--- | :--- | :--- |
|
|
395
|
+
| **Pricing Model** | \$1,500 – \$4,000 / month (metered per GB/token) | **\$299 / month flat** (unlimited nodes & tokens) |
|
|
396
|
+
| **Network Egress** | Requires outbound network calls to DLP vendor | **0 bytes** (100% local Node.js process memory) |
|
|
397
|
+
| **Added Latency** | +200ms – +450ms network round-trip | **<0.1ms** (in-memory regular expressions) |
|
|
398
|
+
| **Vendor Sub-processors** | Third-party cloud vendor added to vendor list | **None** (zero-network software library) |
|
|
399
|
+
| **Deployment** | Complex API gateways and proxy configurations | \`npm install @privacyscrubber/sdk\` in 3 minutes |
|
|
400
|
+
|
|
401
|
+
---
|
|
402
|
+
|
|
403
|
+
### 5. Procurement Recommendation
|
|
404
|
+
We request approval to acquire a **Developer SDK License** (\$299/month flat or \$2,990/year) to unlock unlimited throughput across all microservices and CI/CD pipelines.
|
|
405
|
+
|
|
406
|
+
* **Direct Procurement Portal:** [https://privacyscrubber.com/pricing?tier=sdk&src=ciso_memo](https://privacyscrubber.com/pricing?tier=sdk&src=ciso_memo)
|
|
407
|
+
* **Procurement / Invoicing Contact:** \`enterprise@privacyscrubber.com\`
|
|
408
|
+
`;
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
/**
|
|
412
|
+
* Sanitizes input text, masking detected PII & secrets with reversible tokens.
|
|
413
|
+
*
|
|
414
|
+
* @param {string} text - Raw input text.
|
|
415
|
+
* @param {object} [options={}] - Sanitization options.
|
|
416
|
+
* @param {string} [options.profile='General'] - Compliance detection profile.
|
|
417
|
+
* @param {string} [options.licenseKey] - Optional license key. Falls back to process.env.PRIVACYSCRUBBER_KEY.
|
|
418
|
+
* @param {Array<{name: string, pattern: string, regex: RegExp, label?: string}>} [options.customRules=[]] - Custom regex rules.
|
|
419
|
+
* @param {Record<string, string>} [options.tokenLabelMap={}] - Custom label mapping.
|
|
420
|
+
* @param {Record<string, string>} [options.existingSessionMap={}] - In-memory session map to preserve token IDs across turns.
|
|
421
|
+
* @param {boolean} [options.generateReceipt=true] - Generate a CISO Markdown audit receipt.
|
|
422
|
+
* @param {boolean} [options.throwOnLimit=true] - Throw PrivacyScrubberLicenseError when free tier limit is hit.
|
|
423
|
+
* @param {'throw'|'truncate'} [options.onLimit='throw'] - Action when character limit exceeded ('throw' or 'truncate').
|
|
424
|
+
* @param {boolean} [options.detectSecrets=false] - Auto-detect cloud credentials and API keys.
|
|
425
|
+
* @param {Set<string>|Array<string>} [options.ignoreList=null] - Values to preserve in cleartext.
|
|
426
|
+
* @returns {{
|
|
427
|
+
* scrubbedText: string,
|
|
428
|
+
* tokenMap: Record<string, string>,
|
|
429
|
+
* count: number,
|
|
430
|
+
* executionMs: number,
|
|
431
|
+
* license: { isPro: boolean, tier: 'PRO'|'TEAMS'|'FREE', limit: number, wasTruncated: boolean, upgradeUrl?: string, recommendation?: string },
|
|
432
|
+
* telemetry: object,
|
|
433
|
+
* auditReceipt: string
|
|
434
|
+
* }}
|
|
435
|
+
*/
|
|
436
|
+
function sanitize(text, options = {}) {
|
|
437
|
+
if (!text || typeof text !== 'string') {
|
|
438
|
+
return {
|
|
439
|
+
scrubbedText: '',
|
|
440
|
+
tokenMap: {},
|
|
441
|
+
count: 0,
|
|
442
|
+
executionMs: 0,
|
|
443
|
+
license: { isPro: false, tier: 'FREE', limit: 15000, wasTruncated: false },
|
|
444
|
+
telemetry: buildAuditTelemetry({}),
|
|
445
|
+
auditReceipt: formatAuditReceipt(buildAuditTelemetry({}))
|
|
446
|
+
};
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
const {
|
|
450
|
+
profile = 'General',
|
|
451
|
+
licenseKey = (typeof process !== 'undefined' && process.env?.PRIVACYSCRUBBER_KEY) || '',
|
|
452
|
+
customRules = [],
|
|
453
|
+
tokenLabelMap = {},
|
|
454
|
+
existingSessionMap = {},
|
|
455
|
+
generateReceipt = true,
|
|
456
|
+
throwOnLimit = true,
|
|
457
|
+
onLimit = 'throw',
|
|
458
|
+
detectSecrets = false,
|
|
459
|
+
ignoreList = null
|
|
460
|
+
} = options;
|
|
461
|
+
|
|
462
|
+
const licenseCheck = validateLicense(licenseKey);
|
|
463
|
+
const rawTier = licenseCheck.valid ? (licenseCheck.tier || 'PRO') : 'FREE';
|
|
464
|
+
|
|
465
|
+
// Anti-Cannibalization Barrier (SSOT):
|
|
466
|
+
// PRO ($15/mo) and TEAMS ($99/mo) licenses are strictly human-seat licenses for Web App & Chrome Extension.
|
|
467
|
+
// Backend automation, microservices, RAG pipelines, and headless SDK execution require Developer SDK, OEM, or ENTERPRISE ($299/mo).
|
|
468
|
+
const isSdkTier = licenseCheck.valid && ['SDK', 'OEM', 'ENTERPRISE'].includes(rawTier);
|
|
469
|
+
const isSeatTier = licenseCheck.valid && ['PRO', 'TEAMS'].includes(rawTier);
|
|
470
|
+
|
|
471
|
+
if (isSeatTier) {
|
|
472
|
+
const keyPreview = licenseKey ? `${licenseKey.slice(0, 12)}...` : '';
|
|
473
|
+
const errorMsg = `[PrivacyScrubber] License tier mismatch: ${rawTier} license (${keyPreview}) is licensed strictly for human seats (Web App & Chrome Extension). Backend automation, microservices, and CRM pipelines require a Developer SDK license ($299/mo). Upgrade: https://privacyscrubber.com/pricing?tier=sdk&src=seat_tier_mismatch`;
|
|
474
|
+
if (throwOnLimit !== false) {
|
|
475
|
+
throw new PrivacyScrubberLicenseError(errorMsg, {
|
|
476
|
+
tier: rawTier,
|
|
477
|
+
currentLength: text.length,
|
|
478
|
+
limit: 15000,
|
|
479
|
+
profile: profile
|
|
480
|
+
});
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
const isPro = isSdkTier;
|
|
485
|
+
const tier = isPro ? rawTier : (isSeatTier ? rawTier : 'FREE');
|
|
486
|
+
|
|
487
|
+
emitCommunityNoticeIfNeeded(isPro);
|
|
488
|
+
|
|
489
|
+
const isSpecialized = profile && profile.toLowerCase() !== 'general';
|
|
490
|
+
const limit = isPro ? Infinity : (isSpecialized ? 5000 : 15000);
|
|
491
|
+
|
|
492
|
+
const shouldThrow = throwOnLimit && onLimit !== 'truncate';
|
|
493
|
+
let wasTruncated = false;
|
|
494
|
+
|
|
495
|
+
if (!isPro && text.length > limit) {
|
|
496
|
+
if (shouldThrow) {
|
|
497
|
+
const errorMsg = `[PrivacyScrubber] Free Tier character limit exceeded (${limit.toLocaleString()} chars for profile '${profile}', received ${text.length.toLocaleString()} chars). Upgrade to Developer SDK ($299/mo) for unlimited throughput: https://privacyscrubber.com/pricing?tier=sdk&src=dev_limit`;
|
|
498
|
+
throw new PrivacyScrubberLicenseError(errorMsg, {
|
|
499
|
+
tier: 'FREE',
|
|
500
|
+
currentLength: text.length,
|
|
501
|
+
limit: limit,
|
|
502
|
+
profile: profile
|
|
503
|
+
});
|
|
504
|
+
}
|
|
505
|
+
wasTruncated = true;
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
// Auto-inject DevOps Secrets rules when in Dev profile or when detectSecrets is requested
|
|
509
|
+
const activeRules = [...customRules];
|
|
510
|
+
if (profile.toLowerCase() === 'dev' || detectSecrets === true) {
|
|
511
|
+
DEVOPS_SECRETS_DETECTOR.forEach(rule => {
|
|
512
|
+
activeRules.push({
|
|
513
|
+
name: rule.name,
|
|
514
|
+
pattern: rule.regex.source,
|
|
515
|
+
regex: new RegExp(rule.regex.source, rule.regex.flags),
|
|
516
|
+
label: rule.label || 'SECRET'
|
|
517
|
+
});
|
|
518
|
+
});
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
const startTime = (typeof performance !== 'undefined' && typeof performance.now === 'function')
|
|
522
|
+
? performance.now()
|
|
523
|
+
: Date.now();
|
|
524
|
+
|
|
525
|
+
const textToScrub = (!isPro && text.length > limit) ? text.slice(0, limit) : text;
|
|
526
|
+
|
|
527
|
+
const rawResult = scrubberPkg.scrubText(
|
|
528
|
+
textToScrub,
|
|
529
|
+
activeRules,
|
|
530
|
+
tokenLabelMap,
|
|
531
|
+
profile,
|
|
532
|
+
existingSessionMap,
|
|
533
|
+
isPro,
|
|
534
|
+
ignoreList
|
|
535
|
+
);
|
|
536
|
+
|
|
537
|
+
const endTime = (typeof performance !== 'undefined' && typeof performance.now === 'function')
|
|
538
|
+
? performance.now()
|
|
539
|
+
: Date.now();
|
|
540
|
+
|
|
541
|
+
const executionMs = Math.round((endTime - startTime) * 100) / 100;
|
|
542
|
+
const telemetry = buildAuditTelemetry(rawResult.tokenMap || {});
|
|
543
|
+
const auditReceipt = generateReceipt ? formatAuditReceipt(telemetry) : '';
|
|
544
|
+
|
|
545
|
+
emitDevSecretNoticeIfNeeded(isPro, rawResult.tokenMap, limit);
|
|
546
|
+
|
|
547
|
+
let finalScrubbedText = rawResult.scrubbedText;
|
|
548
|
+
if (wasTruncated && (onLimit === 'truncate' || throwOnLimit === false)) {
|
|
549
|
+
finalScrubbedText += `\n\n<!-- [PrivacyScrubber] Input truncated at free tier limit (${limit.toLocaleString()} chars). For unlimited throughput: https://privacyscrubber.com/pricing?tier=sdk&src=truncation -->`;
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
const returnFullSessionMap = options.returnFullSessionMap === true;
|
|
553
|
+
const sessionMap = { ...(existingSessionMap || {}), ...(rawResult.tokenMap || {}) };
|
|
554
|
+
|
|
555
|
+
return {
|
|
556
|
+
scrubbedText: finalScrubbedText,
|
|
557
|
+
tokenMap: returnFullSessionMap ? sessionMap : (rawResult.tokenMap || {}),
|
|
558
|
+
sessionMap: sessionMap,
|
|
559
|
+
matches: rawResult.matches || [],
|
|
560
|
+
spans: rawResult.matches || [],
|
|
561
|
+
count: rawResult.count || 0,
|
|
562
|
+
executionMs: executionMs,
|
|
563
|
+
license: {
|
|
564
|
+
isPro: isPro,
|
|
565
|
+
tier: tier,
|
|
566
|
+
limit: limit,
|
|
567
|
+
wasTruncated: wasTruncated,
|
|
568
|
+
...(isSeatTier && {
|
|
569
|
+
upgradeUrl: 'https://privacyscrubber.com/pricing?tier=sdk&src=seat_tier_mismatch',
|
|
570
|
+
recommendation: `Your ${rawTier} license is valid for interactive browser seats only. Upgrade to Developer SDK ($299/mo) for unlimited backend server throughput: https://privacyscrubber.com/pricing?tier=sdk`
|
|
571
|
+
}),
|
|
572
|
+
...(!isPro && !isSeatTier && {
|
|
573
|
+
upgradeUrl: 'https://privacyscrubber.com/pricing?tier=sdk&src=dev_upsell',
|
|
574
|
+
recommendation: 'Upgrade to Developer SDK ($299/mo) for unlimited backend text processing, custom regex rules, and specialized compliance profiles.'
|
|
575
|
+
})
|
|
576
|
+
},
|
|
577
|
+
telemetry: telemetry,
|
|
578
|
+
auditReceipt: auditReceipt
|
|
579
|
+
};
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
/**
|
|
583
|
+
* Alias for sanitize.
|
|
584
|
+
*/
|
|
585
|
+
const scrub = sanitize;
|
|
586
|
+
|
|
587
|
+
/**
|
|
588
|
+
* Ergonomic 1-liner: Sanitizes text and returns the scrubbed string directly.
|
|
589
|
+
* @param {string} text - Input text.
|
|
590
|
+
* @param {string} [profile='General'] - Target compliance profile.
|
|
591
|
+
* @returns {string} - Scrubbed string.
|
|
592
|
+
*/
|
|
593
|
+
function clean(text, profile = 'General') {
|
|
594
|
+
if (!text || typeof text !== 'string') return '';
|
|
595
|
+
const { scrubbedText } = sanitize(text, { profile });
|
|
596
|
+
return scrubbedText;
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
/**
|
|
600
|
+
* Primary High-Level Function: Restore / Unscrub AI response back to original entities in-memory.
|
|
601
|
+
*
|
|
602
|
+
* @param {string} aiResponse - Text containing masked tokens (e.g. [EMAIL_1], [NAME_1]).
|
|
603
|
+
* @param {Record<string, string>} tokenMap - Volatile token map returned from sanitize().
|
|
604
|
+
* @returns {{ restoredText: string, restoredCount: number }}
|
|
605
|
+
*/
|
|
606
|
+
function restore(aiResponse, tokenMap = {}) {
|
|
607
|
+
if (!aiResponse || typeof aiResponse !== 'string') {
|
|
608
|
+
return { restoredText: '', restoredCount: 0 };
|
|
609
|
+
}
|
|
610
|
+
const result = scrubberPkg.unscrubText(aiResponse, tokenMap);
|
|
611
|
+
return {
|
|
612
|
+
restoredText: result.restoredText,
|
|
613
|
+
restoredCount: result.restoredCount || 0
|
|
614
|
+
};
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
/**
|
|
618
|
+
* Alias for restore.
|
|
619
|
+
*/
|
|
620
|
+
const unscrub = restore;
|
|
621
|
+
const desanitize = restore;
|
|
622
|
+
|
|
623
|
+
/**
|
|
624
|
+
* Recursively sanitizes all string values within an object or array.
|
|
625
|
+
* Preserves the exact structure, object keys, numbers, booleans, dates, and nulls.
|
|
626
|
+
* Maintains unified session token IDs and returns combined audit telemetry.
|
|
627
|
+
*
|
|
628
|
+
* @param {any} obj - Object, array, or primitive to sanitize.
|
|
629
|
+
* @param {object} [options={}] - Standard sanitization options (profile, licenseKey, etc.).
|
|
630
|
+
* @returns {{
|
|
631
|
+
* sanitized: any,
|
|
632
|
+
* tokenMap: Record<string, string>,
|
|
633
|
+
* count: number,
|
|
634
|
+
* executionMs: number,
|
|
635
|
+
* telemetry: object,
|
|
636
|
+
* auditReceipt: string
|
|
637
|
+
* }}
|
|
638
|
+
*/
|
|
639
|
+
function sanitizeObject(obj, options = {}) {
|
|
640
|
+
if (obj === null || obj === undefined) {
|
|
641
|
+
return {
|
|
642
|
+
sanitized: obj,
|
|
643
|
+
tokenMap: {},
|
|
644
|
+
count: 0,
|
|
645
|
+
executionMs: 0,
|
|
646
|
+
telemetry: buildAuditTelemetry({}),
|
|
647
|
+
auditReceipt: formatAuditReceipt(buildAuditTelemetry({}))
|
|
648
|
+
};
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
if (typeof obj === 'string') {
|
|
652
|
+
const res = sanitize(obj, options);
|
|
653
|
+
return {
|
|
654
|
+
sanitized: res.scrubbedText,
|
|
655
|
+
tokenMap: res.tokenMap,
|
|
656
|
+
count: res.count,
|
|
657
|
+
executionMs: res.executionMs,
|
|
658
|
+
telemetry: res.telemetry,
|
|
659
|
+
auditReceipt: res.auditReceipt
|
|
660
|
+
};
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
if (typeof obj !== 'object') {
|
|
664
|
+
return {
|
|
665
|
+
sanitized: obj,
|
|
666
|
+
tokenMap: {},
|
|
667
|
+
count: 0,
|
|
668
|
+
executionMs: 0,
|
|
669
|
+
telemetry: buildAuditTelemetry({}),
|
|
670
|
+
auditReceipt: formatAuditReceipt(buildAuditTelemetry({}))
|
|
671
|
+
};
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
const startTime = (typeof performance !== 'undefined' && typeof performance.now === 'function')
|
|
675
|
+
? performance.now()
|
|
676
|
+
: Date.now();
|
|
677
|
+
|
|
678
|
+
const mergedSessionMap = { ...(options.existingSessionMap || {}) };
|
|
679
|
+
let totalCount = 0;
|
|
680
|
+
|
|
681
|
+
function traverseAndSanitize(val) {
|
|
682
|
+
if (val === null || val === undefined) return val;
|
|
683
|
+
if (typeof val === 'string') {
|
|
684
|
+
if (val.trim().length === 0) return val;
|
|
685
|
+
const res = sanitize(val, {
|
|
686
|
+
...options,
|
|
687
|
+
existingSessionMap: mergedSessionMap,
|
|
688
|
+
generateReceipt: false
|
|
689
|
+
});
|
|
690
|
+
Object.assign(mergedSessionMap, res.tokenMap);
|
|
691
|
+
totalCount += res.count;
|
|
692
|
+
return res.scrubbedText;
|
|
693
|
+
}
|
|
694
|
+
if (Array.isArray(val)) {
|
|
695
|
+
return val.map(item => traverseAndSanitize(item));
|
|
696
|
+
}
|
|
697
|
+
if (val instanceof Date || val instanceof RegExp) {
|
|
698
|
+
return val;
|
|
699
|
+
}
|
|
700
|
+
if (typeof val === 'object') {
|
|
701
|
+
const out = {};
|
|
702
|
+
for (const [k, v] of Object.entries(val)) {
|
|
703
|
+
out[k] = traverseAndSanitize(v);
|
|
704
|
+
}
|
|
705
|
+
return out;
|
|
706
|
+
}
|
|
707
|
+
return val;
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
const sanitizedResult = traverseAndSanitize(obj);
|
|
711
|
+
|
|
712
|
+
const endTime = (typeof performance !== 'undefined' && typeof performance.now === 'function')
|
|
713
|
+
? performance.now()
|
|
714
|
+
: Date.now();
|
|
715
|
+
|
|
716
|
+
const executionMs = Math.round((endTime - startTime) * 100) / 100;
|
|
717
|
+
const telemetry = buildAuditTelemetry(mergedSessionMap);
|
|
718
|
+
const auditReceipt = options.generateReceipt !== false ? formatAuditReceipt(telemetry) : '';
|
|
719
|
+
|
|
720
|
+
return {
|
|
721
|
+
sanitized: sanitizedResult,
|
|
722
|
+
tokenMap: mergedSessionMap,
|
|
723
|
+
count: totalCount,
|
|
724
|
+
executionMs: executionMs,
|
|
725
|
+
telemetry: telemetry,
|
|
726
|
+
auditReceipt: auditReceipt
|
|
727
|
+
};
|
|
728
|
+
}
|
|
729
|
+
|
|
730
|
+
/**
|
|
731
|
+
* Recursively restores all masked tokens within a sanitized object or array using the tokenMap.
|
|
732
|
+
*
|
|
733
|
+
* @param {any} obj - Sanitized object, array, or string.
|
|
734
|
+
* @param {Record<string, string>} tokenMap - Token map returned from sanitize() or sanitizeObject().
|
|
735
|
+
* @returns {any} - Restored object with original values.
|
|
736
|
+
*/
|
|
737
|
+
function restoreObject(obj, tokenMap = {}) {
|
|
738
|
+
if (obj === null || obj === undefined) return obj;
|
|
739
|
+
if (typeof obj === 'string') {
|
|
740
|
+
const res = restore(obj, tokenMap);
|
|
741
|
+
return res.restoredText;
|
|
742
|
+
}
|
|
743
|
+
if (Array.isArray(obj)) {
|
|
744
|
+
return obj.map(item => restoreObject(item, tokenMap));
|
|
745
|
+
}
|
|
746
|
+
if (typeof obj === 'object' && !(obj instanceof Date) && !(obj instanceof RegExp)) {
|
|
747
|
+
const out = {};
|
|
748
|
+
for (const [k, v] of Object.entries(obj)) {
|
|
749
|
+
out[k] = restoreObject(v, tokenMap);
|
|
750
|
+
}
|
|
751
|
+
return out;
|
|
752
|
+
}
|
|
753
|
+
return obj;
|
|
754
|
+
}
|
|
755
|
+
|
|
756
|
+
/**
|
|
757
|
+
* Stateful PrivacyScrubber Engine class for backend servers & conversational pipelines.
|
|
758
|
+
*/
|
|
759
|
+
class PrivacyScrubberEngine {
|
|
760
|
+
constructor(config = {}) {
|
|
761
|
+
this.licenseKey = config.licenseKey || config.apiKey || (typeof process !== 'undefined' && process.env?.PRIVACYSCRUBBER_KEY) || '';
|
|
762
|
+
this.defaultProfile = config.defaultProfile || config.profile || 'General';
|
|
763
|
+
this.customRules = config.customRules || [];
|
|
764
|
+
this.tokenLabelMap = config.tokenLabelMap || {};
|
|
765
|
+
this.generateReceipt = config.generateReceipt !== false;
|
|
766
|
+
this.sessionMap = { ...(config.initialSessionMap || {}) };
|
|
767
|
+
this.ignoreList = new Set(config.ignoreList || []);
|
|
768
|
+
this.maxSessionTokens = config.maxSessionTokens || 10000;
|
|
769
|
+
|
|
770
|
+
this.license = validateLicense(this.licenseKey);
|
|
771
|
+
}
|
|
772
|
+
|
|
773
|
+
/**
|
|
774
|
+
* Updates active license key.
|
|
775
|
+
* @param {string} key
|
|
776
|
+
*/
|
|
777
|
+
setLicenseKey(key) {
|
|
778
|
+
this.licenseKey = key;
|
|
779
|
+
this.license = validateLicense(this.licenseKey);
|
|
780
|
+
return this.license;
|
|
781
|
+
}
|
|
782
|
+
|
|
783
|
+
/**
|
|
784
|
+
* Sanitizes input text, maintaining stateful session memory if requested.
|
|
785
|
+
* @param {string} text
|
|
786
|
+
* @param {object} [options={}]
|
|
787
|
+
*/
|
|
788
|
+
sanitize(text, options = {}) {
|
|
789
|
+
// Merge instance ignoreList with per-call ignoreList
|
|
790
|
+
const mergedIgnoreList = new Set(this.ignoreList);
|
|
791
|
+
if (options.ignoreList) {
|
|
792
|
+
(Array.isArray(options.ignoreList) ? options.ignoreList : [...options.ignoreList])
|
|
793
|
+
.forEach(v => mergedIgnoreList.add(v));
|
|
794
|
+
}
|
|
795
|
+
|
|
796
|
+
const mergedOptions = {
|
|
797
|
+
profile: options.profile || this.defaultProfile,
|
|
798
|
+
licenseKey: options.licenseKey || this.licenseKey,
|
|
799
|
+
customRules: [...this.customRules, ...(options.customRules || [])],
|
|
800
|
+
tokenLabelMap: { ...this.tokenLabelMap, ...(options.tokenLabelMap || {}) },
|
|
801
|
+
existingSessionMap: options.persistSession !== false ? this.sessionMap : (options.existingSessionMap || {}),
|
|
802
|
+
generateReceipt: options.generateReceipt !== undefined ? options.generateReceipt : this.generateReceipt,
|
|
803
|
+
throwOnLimit: options.throwOnLimit !== undefined ? options.throwOnLimit : true,
|
|
804
|
+
detectSecrets: options.detectSecrets !== undefined ? options.detectSecrets : false,
|
|
805
|
+
ignoreList: mergedIgnoreList.size > 0 ? mergedIgnoreList : null
|
|
806
|
+
};
|
|
807
|
+
|
|
808
|
+
const result = sanitize(text, mergedOptions);
|
|
809
|
+
if (options.persistSession !== false) {
|
|
810
|
+
Object.assign(this.sessionMap, result.tokenMap);
|
|
811
|
+
const maxTokens = options.maxSessionTokens || this.maxSessionTokens || 10000;
|
|
812
|
+
const keys = Object.keys(this.sessionMap);
|
|
813
|
+
if (keys.length > maxTokens) {
|
|
814
|
+
const pruneCount = keys.length - maxTokens;
|
|
815
|
+
for (let i = 0; i < pruneCount; i++) {
|
|
816
|
+
delete this.sessionMap[keys[i]];
|
|
817
|
+
}
|
|
818
|
+
}
|
|
819
|
+
}
|
|
820
|
+
return result;
|
|
821
|
+
}
|
|
822
|
+
|
|
823
|
+
/**
|
|
824
|
+
* Restores AI response using the engine's internal session map or an explicit tokenMap.
|
|
825
|
+
* @param {string} aiResponse
|
|
826
|
+
* @param {Record<string, string>} [tokenMap]
|
|
827
|
+
*/
|
|
828
|
+
restore(aiResponse, tokenMap = null) {
|
|
829
|
+
const activeMap = tokenMap || this.sessionMap;
|
|
830
|
+
return restore(aiResponse, activeMap);
|
|
831
|
+
}
|
|
832
|
+
|
|
833
|
+
/**
|
|
834
|
+
* Resets the persistent session token map.
|
|
835
|
+
*/
|
|
836
|
+
resetSession() {
|
|
837
|
+
for (const key of Object.keys(this.sessionMap)) {
|
|
838
|
+
delete this.sessionMap[key];
|
|
839
|
+
}
|
|
840
|
+
}
|
|
841
|
+
|
|
842
|
+
/**
|
|
843
|
+
* Marks a token as a false positive. The original plaintext value will be
|
|
844
|
+
* excluded from all future sanitize() calls on this engine instance.
|
|
845
|
+
* @param {string} token - The token to mark (e.g., '[NAME_1]').
|
|
846
|
+
* @returns {{ success: boolean, token?: string, original?: string, ignoreListSize?: number, reason?: string }}
|
|
847
|
+
*/
|
|
848
|
+
markFalsePositive(token) {
|
|
849
|
+
const original = this.sessionMap[token];
|
|
850
|
+
if (!original) {
|
|
851
|
+
return { success: false, reason: `Token '${token}' not found in session map` };
|
|
852
|
+
}
|
|
853
|
+
this.ignoreList.add(original);
|
|
854
|
+
delete this.sessionMap[token];
|
|
855
|
+
return { success: true, token, original, ignoreListSize: this.ignoreList.size };
|
|
856
|
+
}
|
|
857
|
+
|
|
858
|
+
/**
|
|
859
|
+
* Clears the false positive ignore list.
|
|
860
|
+
*/
|
|
861
|
+
clearIgnoreList() {
|
|
862
|
+
this.ignoreList.clear();
|
|
863
|
+
}
|
|
864
|
+
|
|
865
|
+
/**
|
|
866
|
+
* Creates a streaming token restorer bound to this engine's session map.
|
|
867
|
+
* @param {Record<string, string>} [tokenMap=null]
|
|
868
|
+
* @param {object} [options={}]
|
|
869
|
+
*/
|
|
870
|
+
createStreamRestorer(tokenMap = null, options = {}) {
|
|
871
|
+
const activeMap = tokenMap || this.sessionMap;
|
|
872
|
+
return createStreamTokenRestorer(activeMap, options);
|
|
873
|
+
}
|
|
874
|
+
|
|
875
|
+
/**
|
|
876
|
+
* Wraps an async iterable LLM stream with real-time de-tokenization.
|
|
877
|
+
* @param {AsyncIterable<any>} stream
|
|
878
|
+
* @param {Record<string, string>} [tokenMap=null]
|
|
879
|
+
* @param {object} [options={}]
|
|
880
|
+
*/
|
|
881
|
+
async* wrapStream(stream, tokenMap = null, options = {}) {
|
|
882
|
+
const activeMap = tokenMap || this.sessionMap;
|
|
883
|
+
yield* wrapAiStream(stream, activeMap, options);
|
|
884
|
+
}
|
|
885
|
+
}
|
|
886
|
+
|
|
887
|
+
/**
|
|
888
|
+
* Transparent Zero-Trust Wrapper for OpenAI Client.
|
|
889
|
+
* Automatically sanitizes outgoing messages before transmission to OpenAI,
|
|
890
|
+
* and restores completion tokens in the assistant response before returning!
|
|
891
|
+
* Seamlessly de-tokenizes streaming responses in real-time when stream: true.
|
|
892
|
+
*
|
|
893
|
+
* @param {object} openaiClient - Instance of OpenAI SDK (e.g. new OpenAI()).
|
|
894
|
+
* @param {object} [options={}] - PrivacyScrubber configuration.
|
|
895
|
+
* @returns {object} Wrapped OpenAI client.
|
|
896
|
+
*/
|
|
897
|
+
function wrapOpenAI(openaiClient, options = {}) {
|
|
898
|
+
if (!openaiClient || !openaiClient.chat || !openaiClient.chat.completions) {
|
|
899
|
+
throw new Error('[PrivacyScrubber] Invalid OpenAI client instance passed to wrapOpenAI.');
|
|
900
|
+
}
|
|
901
|
+
|
|
902
|
+
const engine = new PrivacyScrubberEngine(options);
|
|
903
|
+
const originalCreate = openaiClient.chat.completions.create.bind(openaiClient.chat.completions);
|
|
904
|
+
|
|
905
|
+
openaiClient.chat.completions.create = async function(params, requestOptions) {
|
|
906
|
+
if (!params || !Array.isArray(params.messages)) {
|
|
907
|
+
return originalCreate(params, requestOptions);
|
|
908
|
+
}
|
|
909
|
+
|
|
910
|
+
const sessionTokenMap = {};
|
|
911
|
+
const sanitizedMessages = params.messages.map(msg => {
|
|
912
|
+
if (typeof msg.content === 'string') {
|
|
913
|
+
const res = engine.sanitize(msg.content, { existingSessionMap: sessionTokenMap });
|
|
914
|
+
Object.assign(sessionTokenMap, res.tokenMap);
|
|
915
|
+
return { ...msg, content: res.scrubbedText };
|
|
916
|
+
} else if (Array.isArray(msg.content)) {
|
|
917
|
+
const sanitizedChunks = msg.content.map(chunk => {
|
|
918
|
+
if (chunk && chunk.type === 'text' && typeof chunk.text === 'string') {
|
|
919
|
+
const res = engine.sanitize(chunk.text, { existingSessionMap: sessionTokenMap });
|
|
920
|
+
Object.assign(sessionTokenMap, res.tokenMap);
|
|
921
|
+
return { ...chunk, text: res.scrubbedText };
|
|
922
|
+
}
|
|
923
|
+
return chunk;
|
|
924
|
+
});
|
|
925
|
+
return { ...msg, content: sanitizedChunks };
|
|
926
|
+
}
|
|
927
|
+
return msg;
|
|
928
|
+
});
|
|
929
|
+
|
|
930
|
+
const modifiedParams = { ...params, messages: sanitizedMessages };
|
|
931
|
+
const response = await originalCreate(modifiedParams, requestOptions);
|
|
932
|
+
|
|
933
|
+
// Streaming support: transparently de-tokenize async iterable chunks
|
|
934
|
+
if (params.stream) {
|
|
935
|
+
const detok = new StreamingDetokenizer(sessionTokenMap);
|
|
936
|
+
return detok.wrapStream(response);
|
|
937
|
+
}
|
|
938
|
+
|
|
939
|
+
if (response && Array.isArray(response.choices)) {
|
|
940
|
+
response.choices.forEach(choice => {
|
|
941
|
+
if (choice.message) {
|
|
942
|
+
if (typeof choice.message.content === 'string') {
|
|
943
|
+
choice.message.content = engine.restore(choice.message.content, sessionTokenMap).restoredText;
|
|
944
|
+
} else if (Array.isArray(choice.message.content)) {
|
|
945
|
+
choice.message.content = choice.message.content.map(chunk => {
|
|
946
|
+
if (chunk && chunk.type === 'text' && typeof chunk.text === 'string') {
|
|
947
|
+
return { ...chunk, text: engine.restore(chunk.text, sessionTokenMap).restoredText };
|
|
948
|
+
}
|
|
949
|
+
return chunk;
|
|
950
|
+
});
|
|
951
|
+
}
|
|
952
|
+
}
|
|
953
|
+
});
|
|
954
|
+
}
|
|
955
|
+
|
|
956
|
+
return response;
|
|
957
|
+
};
|
|
958
|
+
|
|
959
|
+
return openaiClient;
|
|
960
|
+
}
|
|
961
|
+
|
|
962
|
+
/**
|
|
963
|
+
* Transparent Zero-Trust wrapper for Vercel AI SDK Language Models.
|
|
964
|
+
* Sanitizes input prompts in local RAM before model invocation and transparently
|
|
965
|
+
* de-tokenizes streaming text deltas and final text responses.
|
|
966
|
+
*
|
|
967
|
+
* Compatible with Vercel AI SDK v3/v4: generateText, streamText, generateObject, streamObject.
|
|
968
|
+
*
|
|
969
|
+
* @param {object} model - LanguageModel instance (e.g. openai('gpt-4o'), anthropic('claude-3-5-sonnet')).
|
|
970
|
+
* @param {object} [options={}] - PrivacyScrubber configuration options (profile, detectSecrets, licenseKey).
|
|
971
|
+
* @returns {object} Wrapped LanguageModel instance.
|
|
972
|
+
*/
|
|
973
|
+
function wrapAiModel(model, options = {}) {
|
|
974
|
+
if (!model || typeof model !== 'object') {
|
|
975
|
+
throw new Error('[PrivacyScrubber] Invalid model instance passed to wrapAiModel.');
|
|
976
|
+
}
|
|
977
|
+
|
|
978
|
+
const engine = new PrivacyScrubberEngine(options);
|
|
979
|
+
const wrapped = Object.create(model);
|
|
980
|
+
|
|
981
|
+
function sanitizePromptParts(prompt, sessionMap) {
|
|
982
|
+
if (!Array.isArray(prompt)) return prompt;
|
|
983
|
+
return prompt.map(msg => {
|
|
984
|
+
if (!msg) return msg;
|
|
985
|
+
if (typeof msg.content === 'string') {
|
|
986
|
+
const res = engine.sanitize(msg.content, { existingSessionMap: sessionMap });
|
|
987
|
+
Object.assign(sessionMap, res.tokenMap);
|
|
988
|
+
return { ...msg, content: res.scrubbedText };
|
|
989
|
+
}
|
|
990
|
+
if (Array.isArray(msg.content)) {
|
|
991
|
+
const newContent = msg.content.map(part => {
|
|
992
|
+
if (part && part.type === 'text' && typeof part.text === 'string') {
|
|
993
|
+
const res = engine.sanitize(part.text, { existingSessionMap: sessionMap });
|
|
994
|
+
Object.assign(sessionMap, res.tokenMap);
|
|
995
|
+
return { ...part, text: res.scrubbedText };
|
|
996
|
+
}
|
|
997
|
+
return part;
|
|
998
|
+
});
|
|
999
|
+
return { ...msg, content: newContent };
|
|
1000
|
+
}
|
|
1001
|
+
return msg;
|
|
1002
|
+
});
|
|
1003
|
+
}
|
|
1004
|
+
|
|
1005
|
+
if (typeof model.doGenerate === 'function') {
|
|
1006
|
+
wrapped.doGenerate = async function(params) {
|
|
1007
|
+
const sessionTokenMap = {};
|
|
1008
|
+
const sanitizedParams = {
|
|
1009
|
+
...params,
|
|
1010
|
+
prompt: sanitizePromptParts(params.prompt, sessionTokenMap)
|
|
1011
|
+
};
|
|
1012
|
+
const result = await model.doGenerate(sanitizedParams);
|
|
1013
|
+
if (result && typeof result.text === 'string') {
|
|
1014
|
+
result.text = engine.restore(result.text, sessionTokenMap).restoredText;
|
|
1015
|
+
}
|
|
1016
|
+
return result;
|
|
1017
|
+
};
|
|
1018
|
+
}
|
|
1019
|
+
|
|
1020
|
+
if (typeof model.doStream === 'function') {
|
|
1021
|
+
wrapped.doStream = async function(params) {
|
|
1022
|
+
const sessionTokenMap = {};
|
|
1023
|
+
const sanitizedParams = {
|
|
1024
|
+
...params,
|
|
1025
|
+
prompt: sanitizePromptParts(params.prompt, sessionTokenMap)
|
|
1026
|
+
};
|
|
1027
|
+
const result = await model.doStream(sanitizedParams);
|
|
1028
|
+
if (result && result.stream) {
|
|
1029
|
+
const detok = new StreamingDetokenizer(sessionTokenMap);
|
|
1030
|
+
const originalStream = result.stream;
|
|
1031
|
+
|
|
1032
|
+
if (typeof originalStream.pipeThrough === 'function' && typeof TransformStream !== 'undefined') {
|
|
1033
|
+
result.stream = originalStream.pipeThrough(new TransformStream({
|
|
1034
|
+
transform(chunk, controller) {
|
|
1035
|
+
if (chunk && chunk.type === 'text-delta' && typeof chunk.textDelta === 'string') {
|
|
1036
|
+
const restored = detok.feed(chunk.textDelta);
|
|
1037
|
+
if (restored) controller.enqueue({ ...chunk, textDelta: restored });
|
|
1038
|
+
} else {
|
|
1039
|
+
controller.enqueue(chunk);
|
|
1040
|
+
}
|
|
1041
|
+
},
|
|
1042
|
+
flush(controller) {
|
|
1043
|
+
const rem = detok.flush();
|
|
1044
|
+
if (rem) controller.enqueue({ type: 'text-delta', textDelta: rem });
|
|
1045
|
+
}
|
|
1046
|
+
}));
|
|
1047
|
+
} else if (typeof originalStream[Symbol.asyncIterator] === 'function') {
|
|
1048
|
+
async function* wrappedAsyncIterable() {
|
|
1049
|
+
for await (const chunk of originalStream) {
|
|
1050
|
+
if (chunk && chunk.type === 'text-delta' && typeof chunk.textDelta === 'string') {
|
|
1051
|
+
const restored = detok.feed(chunk.textDelta);
|
|
1052
|
+
if (restored) yield { ...chunk, textDelta: restored };
|
|
1053
|
+
} else {
|
|
1054
|
+
yield chunk;
|
|
1055
|
+
}
|
|
1056
|
+
}
|
|
1057
|
+
const rem = detok.flush();
|
|
1058
|
+
if (rem) yield { type: 'text-delta', textDelta: rem };
|
|
1059
|
+
}
|
|
1060
|
+
result.stream = wrappedAsyncIterable();
|
|
1061
|
+
}
|
|
1062
|
+
}
|
|
1063
|
+
return result;
|
|
1064
|
+
};
|
|
1065
|
+
}
|
|
1066
|
+
|
|
1067
|
+
return wrapped;
|
|
1068
|
+
}
|
|
1069
|
+
|
|
1070
|
+
const wrapAiSdk = wrapAiModel;
|
|
1071
|
+
|
|
1072
|
+
/**
|
|
1073
|
+
* Framework middleware helper for LangChain / LlamaIndex pipelines.
|
|
1074
|
+
*/
|
|
1075
|
+
function createLangChainTransform(options = {}) {
|
|
1076
|
+
const engine = new PrivacyScrubberEngine(options);
|
|
1077
|
+
return {
|
|
1078
|
+
preprocess: (input) => engine.sanitize(input),
|
|
1079
|
+
postprocess: (output, tokenMap) => engine.restore(output, tokenMap)
|
|
1080
|
+
};
|
|
1081
|
+
}
|
|
1082
|
+
|
|
1083
|
+
/**
|
|
1084
|
+
* LangChain Document Transformer for RAG pre-ingestion pipelines.
|
|
1085
|
+
* Compliant with LangChain BaseDocumentTransformer and Runnable interfaces.
|
|
1086
|
+
*
|
|
1087
|
+
* Intercepts LangChain Document objects ({ pageContent, metadata }) before
|
|
1088
|
+
* embedding or vector storage, scrubbing PII in pageContent and optional sensitive metadata keys.
|
|
1089
|
+
*
|
|
1090
|
+
* @param {Object} [options={}]
|
|
1091
|
+
* @param {string} [options.profile='General'] Detection profile ('General', 'Medical', 'Finance', 'Dev', etc.)
|
|
1092
|
+
* @param {string[]} [options.sensitiveMetadataKeys=[]] Metadata keys to sanitize (e.g. ['author', 'user_id', 'contact'])
|
|
1093
|
+
* @param {boolean} [options.attachTelemetry=true] Whether to attach _ztds_telemetry to document metadata
|
|
1094
|
+
* @param {boolean} [options.attachTokenMap=false] Whether to attach _ztds_token_map to document metadata
|
|
1095
|
+
* @returns {Object} Document transformer with transformDocuments and invoke methods
|
|
1096
|
+
*/
|
|
1097
|
+
function createDocumentTransformer(options = {}) {
|
|
1098
|
+
const {
|
|
1099
|
+
profile = 'General',
|
|
1100
|
+
sensitiveMetadataKeys = [],
|
|
1101
|
+
attachTelemetry = true,
|
|
1102
|
+
attachTokenMap = false,
|
|
1103
|
+
...engineOptions
|
|
1104
|
+
} = options;
|
|
1105
|
+
|
|
1106
|
+
const engine = new PrivacyScrubberEngine({ profile, ...engineOptions });
|
|
1107
|
+
|
|
1108
|
+
async function transformDocuments(documents) {
|
|
1109
|
+
if (!Array.isArray(documents)) {
|
|
1110
|
+
throw new TypeError("documents must be an array of LangChain Document objects");
|
|
1111
|
+
}
|
|
1112
|
+
|
|
1113
|
+
const transformed = [];
|
|
1114
|
+
for (const doc of documents) {
|
|
1115
|
+
const pageContent = doc.pageContent || doc.text || "";
|
|
1116
|
+
const metadata = { ...(doc.metadata || {}) };
|
|
1117
|
+
|
|
1118
|
+
// Sanitize page content
|
|
1119
|
+
const scrubResult = engine.sanitize(pageContent, { profile });
|
|
1120
|
+
|
|
1121
|
+
// Sanitize designated sensitive metadata fields if present
|
|
1122
|
+
if (Array.isArray(sensitiveMetadataKeys) && sensitiveMetadataKeys.length > 0) {
|
|
1123
|
+
for (const key of sensitiveMetadataKeys) {
|
|
1124
|
+
if (metadata[key] && typeof metadata[key] === 'string') {
|
|
1125
|
+
const metaScrub = engine.sanitize(metadata[key], { profile });
|
|
1126
|
+
metadata[key] = metaScrub.scrubbedText;
|
|
1127
|
+
}
|
|
1128
|
+
}
|
|
1129
|
+
}
|
|
1130
|
+
|
|
1131
|
+
if (attachTelemetry) {
|
|
1132
|
+
metadata._ztds_telemetry = {
|
|
1133
|
+
riskLevel: scrubResult.telemetry?.riskLevel || 'CLEAN (ZERO PII)',
|
|
1134
|
+
totalCount: scrubResult.count || 0,
|
|
1135
|
+
timestamp: new Date().toISOString()
|
|
1136
|
+
};
|
|
1137
|
+
}
|
|
1138
|
+
|
|
1139
|
+
if (attachTokenMap && scrubResult.tokenMap) {
|
|
1140
|
+
metadata._ztds_token_map = { ...scrubResult.tokenMap };
|
|
1141
|
+
}
|
|
1142
|
+
|
|
1143
|
+
transformed.push({
|
|
1144
|
+
...doc,
|
|
1145
|
+
pageContent: scrubResult.scrubbedText,
|
|
1146
|
+
metadata
|
|
1147
|
+
});
|
|
1148
|
+
}
|
|
1149
|
+
|
|
1150
|
+
return transformed;
|
|
1151
|
+
}
|
|
1152
|
+
|
|
1153
|
+
return {
|
|
1154
|
+
transformDocuments,
|
|
1155
|
+
invoke: async (documents) => transformDocuments(documents),
|
|
1156
|
+
sessionMap: engine.sessionMap,
|
|
1157
|
+
restore: (output, tokenMap) => engine.restore(output, tokenMap || engine.sessionMap),
|
|
1158
|
+
resetSession: () => engine.resetSession()
|
|
1159
|
+
};
|
|
1160
|
+
}
|
|
1161
|
+
const createLangChainDocumentTransformer = createDocumentTransformer;
|
|
1162
|
+
|
|
1163
|
+
/**
|
|
1164
|
+
* LlamaIndex Node/Document Transform component for RAG ingestion.
|
|
1165
|
+
*
|
|
1166
|
+
* Compatible with LlamaIndex ingestion pipelines and custom transformations.
|
|
1167
|
+
* Sanitizes node text while preserving node IDs, relationships, and metadata.
|
|
1168
|
+
*
|
|
1169
|
+
* @param {Object} [options={}]
|
|
1170
|
+
* @returns {Object} LlamaIndex transform with transform method
|
|
1171
|
+
*/
|
|
1172
|
+
function createLlamaIndexTransform(options = {}) {
|
|
1173
|
+
const { profile = 'General', ...engineOptions } = options;
|
|
1174
|
+
const engine = new PrivacyScrubberEngine({ profile, ...engineOptions });
|
|
1175
|
+
|
|
1176
|
+
return {
|
|
1177
|
+
transform: (nodes) => {
|
|
1178
|
+
if (!nodes) return nodes;
|
|
1179
|
+
const isArray = Array.isArray(nodes);
|
|
1180
|
+
const list = isArray ? nodes : [nodes];
|
|
1181
|
+
|
|
1182
|
+
const transformed = list.map((node) => {
|
|
1183
|
+
const text = typeof node.getContent === 'function'
|
|
1184
|
+
? node.getContent()
|
|
1185
|
+
: (node.text || node.pageContent || '');
|
|
1186
|
+
const result = engine.sanitize(text, { profile });
|
|
1187
|
+
|
|
1188
|
+
if (typeof node.setContent === 'function') {
|
|
1189
|
+
node.setContent(result.scrubbedText);
|
|
1190
|
+
} else {
|
|
1191
|
+
node.text = result.scrubbedText;
|
|
1192
|
+
}
|
|
1193
|
+
|
|
1194
|
+
if (node.metadata) {
|
|
1195
|
+
node.metadata._ztds_sanitized = true;
|
|
1196
|
+
node.metadata._ztds_pii_count = result.count;
|
|
1197
|
+
}
|
|
1198
|
+
return node;
|
|
1199
|
+
});
|
|
1200
|
+
|
|
1201
|
+
return isArray ? transformed : transformed[0];
|
|
1202
|
+
},
|
|
1203
|
+
sessionMap: engine.sessionMap,
|
|
1204
|
+
restore: (text, map) => engine.restore(text, map || engine.sessionMap),
|
|
1205
|
+
resetSession: () => engine.resetSession()
|
|
1206
|
+
};
|
|
1207
|
+
}
|
|
1208
|
+
|
|
1209
|
+
/**
|
|
1210
|
+
* Universal Vector Database Ingestion Guard (Chroma, Pinecone, Qdrant, Weaviate, pgvector, Milvus).
|
|
1211
|
+
*
|
|
1212
|
+
* Guarantees zero-PII vector embedding and storage in local/cloud vector databases.
|
|
1213
|
+
* Provides pre-ingestion record sanitization, query alignment, search result re-hydration,
|
|
1214
|
+
* and transparent proxy wrapping of vector store clients.
|
|
1215
|
+
*
|
|
1216
|
+
* @param {Object} [options={}]
|
|
1217
|
+
* @param {string} [options.profile='General']
|
|
1218
|
+
* @param {string[]} [options.sensitiveMetadataKeys=[]]
|
|
1219
|
+
* @param {boolean} [options.autoRestoreResults=true]
|
|
1220
|
+
* @returns {Object} Vector ingestion guard utilities
|
|
1221
|
+
*/
|
|
1222
|
+
function createVectorIngestionGuard(options = {}) {
|
|
1223
|
+
const {
|
|
1224
|
+
profile = 'General',
|
|
1225
|
+
sensitiveMetadataKeys = [],
|
|
1226
|
+
autoRestoreResults = true,
|
|
1227
|
+
...engineOptions
|
|
1228
|
+
} = options;
|
|
1229
|
+
|
|
1230
|
+
const engine = new PrivacyScrubberEngine({ profile, ...engineOptions });
|
|
1231
|
+
|
|
1232
|
+
/**
|
|
1233
|
+
* Sanitizes raw database records prior to vector store upsert / add.
|
|
1234
|
+
* Supports standard formats:
|
|
1235
|
+
* - Chroma: { ids: [], documents: [], metadatas: [] }
|
|
1236
|
+
* - Pinecone: [{ id, values, metadata }]
|
|
1237
|
+
* - Qdrant: [{ id, vector, payload: { text, ... } }]
|
|
1238
|
+
* - Universal: [{ id, text, metadata }]
|
|
1239
|
+
*/
|
|
1240
|
+
function sanitizeRecords(records) {
|
|
1241
|
+
if (!records) return records;
|
|
1242
|
+
|
|
1243
|
+
// Chroma-style columnar batch: { ids, documents, metadatas }
|
|
1244
|
+
if (records.documents && Array.isArray(records.documents)) {
|
|
1245
|
+
const sanitizedDocs = [];
|
|
1246
|
+
const sanitizedMetas = records.metadatas ? [...records.metadatas] : [];
|
|
1247
|
+
let totalTokens = 0;
|
|
1248
|
+
|
|
1249
|
+
records.documents.forEach((doc, idx) => {
|
|
1250
|
+
const res = engine.sanitize(doc || '', { profile });
|
|
1251
|
+
sanitizedDocs.push(res.scrubbedText);
|
|
1252
|
+
totalTokens += res.count;
|
|
1253
|
+
|
|
1254
|
+
if (sanitizedMetas[idx]) {
|
|
1255
|
+
const meta = { ...sanitizedMetas[idx] };
|
|
1256
|
+
if (sensitiveMetadataKeys.length > 0) {
|
|
1257
|
+
for (const k of sensitiveMetadataKeys) {
|
|
1258
|
+
if (typeof meta[k] === 'string') {
|
|
1259
|
+
meta[k] = engine.sanitize(meta[k], { profile }).scrubbedText;
|
|
1260
|
+
}
|
|
1261
|
+
}
|
|
1262
|
+
}
|
|
1263
|
+
meta._ztds_sanitized = true;
|
|
1264
|
+
sanitizedMetas[idx] = meta;
|
|
1265
|
+
}
|
|
1266
|
+
});
|
|
1267
|
+
|
|
1268
|
+
return {
|
|
1269
|
+
...records,
|
|
1270
|
+
documents: sanitizedDocs,
|
|
1271
|
+
metadatas: sanitizedMetas,
|
|
1272
|
+
_ztds_total_tokens: totalTokens
|
|
1273
|
+
};
|
|
1274
|
+
}
|
|
1275
|
+
|
|
1276
|
+
// Array of record objects (Pinecone, Qdrant, Weaviate, LangChain vectorstores)
|
|
1277
|
+
if (Array.isArray(records)) {
|
|
1278
|
+
let totalTokens = 0;
|
|
1279
|
+
const sanitized = records.map((record) => {
|
|
1280
|
+
const copy = { ...record };
|
|
1281
|
+
// Handle text / document / pageContent / payload.text
|
|
1282
|
+
const textField = copy.text !== undefined ? 'text'
|
|
1283
|
+
: copy.document !== undefined ? 'document'
|
|
1284
|
+
: copy.pageContent !== undefined ? 'pageContent'
|
|
1285
|
+
: null;
|
|
1286
|
+
|
|
1287
|
+
if (textField && typeof copy[textField] === 'string') {
|
|
1288
|
+
const res = engine.sanitize(copy[textField], { profile });
|
|
1289
|
+
copy[textField] = res.scrubbedText;
|
|
1290
|
+
totalTokens += res.count;
|
|
1291
|
+
}
|
|
1292
|
+
|
|
1293
|
+
// Handle Qdrant payload
|
|
1294
|
+
if (copy.payload && typeof copy.payload === 'object') {
|
|
1295
|
+
copy.payload = { ...copy.payload };
|
|
1296
|
+
if (typeof copy.payload.text === 'string') {
|
|
1297
|
+
const res = engine.sanitize(copy.payload.text, { profile });
|
|
1298
|
+
copy.payload.text = res.scrubbedText;
|
|
1299
|
+
totalTokens += res.count;
|
|
1300
|
+
}
|
|
1301
|
+
}
|
|
1302
|
+
|
|
1303
|
+
// Handle metadata
|
|
1304
|
+
if (copy.metadata && typeof copy.metadata === 'object') {
|
|
1305
|
+
copy.metadata = { ...copy.metadata };
|
|
1306
|
+
for (const k of sensitiveMetadataKeys) {
|
|
1307
|
+
if (typeof copy.metadata[k] === 'string') {
|
|
1308
|
+
copy.metadata[k] = engine.sanitize(copy.metadata[k], { profile }).scrubbedText;
|
|
1309
|
+
}
|
|
1310
|
+
}
|
|
1311
|
+
copy.metadata._ztds_sanitized = true;
|
|
1312
|
+
}
|
|
1313
|
+
|
|
1314
|
+
return copy;
|
|
1315
|
+
});
|
|
1316
|
+
|
|
1317
|
+
return sanitized;
|
|
1318
|
+
}
|
|
1319
|
+
|
|
1320
|
+
return records;
|
|
1321
|
+
}
|
|
1322
|
+
|
|
1323
|
+
/**
|
|
1324
|
+
* Sanitizes user queries before generating search embeddings.
|
|
1325
|
+
* Ensures search vectors align with sanitized tokenized corpus representations.
|
|
1326
|
+
*/
|
|
1327
|
+
function sanitizeQuery(query) {
|
|
1328
|
+
if (typeof query !== 'string') return query;
|
|
1329
|
+
const res = engine.sanitize(query, { profile });
|
|
1330
|
+
const matches = res.scrubbedText.match(/\[[A-Z_]+_\d+\]/g);
|
|
1331
|
+
const tokensDetected = matches ? matches.length : (res.count || 0);
|
|
1332
|
+
return {
|
|
1333
|
+
query: res.scrubbedText,
|
|
1334
|
+
rawQuery: query,
|
|
1335
|
+
tokensDetected,
|
|
1336
|
+
tokenMap: res.tokenMap
|
|
1337
|
+
};
|
|
1338
|
+
}
|
|
1339
|
+
|
|
1340
|
+
/**
|
|
1341
|
+
* Restores search results retrieved from vector DB using the volatile session map.
|
|
1342
|
+
*/
|
|
1343
|
+
function restoreSearchResults(results, customTokenMap = null) {
|
|
1344
|
+
const map = customTokenMap || engine.sessionMap;
|
|
1345
|
+
if (!results || Object.keys(map).length === 0) return results;
|
|
1346
|
+
|
|
1347
|
+
function restoreText(t) {
|
|
1348
|
+
if (typeof t !== 'string') return t;
|
|
1349
|
+
return engine.restore(t, map).restoredText;
|
|
1350
|
+
}
|
|
1351
|
+
|
|
1352
|
+
if (Array.isArray(results)) {
|
|
1353
|
+
return results.map(item => {
|
|
1354
|
+
const copy = { ...item };
|
|
1355
|
+
if (typeof copy.pageContent === 'string') copy.pageContent = restoreText(copy.pageContent);
|
|
1356
|
+
if (typeof copy.text === 'string') copy.text = restoreText(copy.text);
|
|
1357
|
+
if (typeof copy.document === 'string') copy.document = restoreText(copy.document);
|
|
1358
|
+
if (copy.payload && typeof copy.payload === 'object') {
|
|
1359
|
+
copy.payload = { ...copy.payload };
|
|
1360
|
+
if (typeof copy.payload.text === 'string') {
|
|
1361
|
+
copy.payload.text = restoreText(copy.payload.text);
|
|
1362
|
+
}
|
|
1363
|
+
}
|
|
1364
|
+
return copy;
|
|
1365
|
+
});
|
|
1366
|
+
}
|
|
1367
|
+
|
|
1368
|
+
return results;
|
|
1369
|
+
}
|
|
1370
|
+
|
|
1371
|
+
/**
|
|
1372
|
+
* Wraps an existing vector store client instance with zero-trust hooks.
|
|
1373
|
+
*/
|
|
1374
|
+
function wrapVectorStore(vectorStore) {
|
|
1375
|
+
const proxy = new Proxy(vectorStore, {
|
|
1376
|
+
get(target, prop, receiver) {
|
|
1377
|
+
const orig = Reflect.get(target, prop, receiver);
|
|
1378
|
+
if (typeof orig !== 'function') return orig;
|
|
1379
|
+
|
|
1380
|
+
// Intercept ingestion methods: addDocuments, upsert, add, insert
|
|
1381
|
+
if (prop === 'addDocuments') {
|
|
1382
|
+
return async function(docs, ...rest) {
|
|
1383
|
+
if (!Array.isArray(docs)) return orig.apply(target, [docs, ...rest]);
|
|
1384
|
+
const safeDocs = docs.map(doc => {
|
|
1385
|
+
const pageContent = doc.pageContent || doc.text || "";
|
|
1386
|
+
const metadata = { ...(doc.metadata || {}) };
|
|
1387
|
+
const scrubResult = engine.sanitize(pageContent, { profile });
|
|
1388
|
+
for (const key of sensitiveMetadataKeys) {
|
|
1389
|
+
if (metadata[key] && typeof metadata[key] === 'string') {
|
|
1390
|
+
metadata[key] = engine.sanitize(metadata[key], { profile }).scrubbedText;
|
|
1391
|
+
}
|
|
1392
|
+
}
|
|
1393
|
+
metadata._ztds_telemetry = {
|
|
1394
|
+
riskLevel: scrubResult.telemetry?.riskLevel || 'CLEAN (ZERO PII)',
|
|
1395
|
+
totalCount: scrubResult.count || 0,
|
|
1396
|
+
timestamp: new Date().toISOString()
|
|
1397
|
+
};
|
|
1398
|
+
return { ...doc, pageContent: scrubResult.scrubbedText, metadata };
|
|
1399
|
+
});
|
|
1400
|
+
return orig.apply(target, [safeDocs, ...rest]);
|
|
1401
|
+
};
|
|
1402
|
+
}
|
|
1403
|
+
|
|
1404
|
+
if (prop === 'upsert' || prop === 'add' || prop === 'insert') {
|
|
1405
|
+
return async function(records, ...rest) {
|
|
1406
|
+
const safeRecords = sanitizeRecords(records);
|
|
1407
|
+
return orig.apply(target, [safeRecords, ...rest]);
|
|
1408
|
+
};
|
|
1409
|
+
}
|
|
1410
|
+
|
|
1411
|
+
// Intercept query methods: similaritySearch, query, search
|
|
1412
|
+
if (prop === 'similaritySearch' || prop === 'query' || prop === 'search') {
|
|
1413
|
+
return async function(queryArg, ...rest) {
|
|
1414
|
+
const safeQ = typeof queryArg === 'string' ? sanitizeQuery(queryArg).query : queryArg;
|
|
1415
|
+
const rawResults = await orig.apply(target, [safeQ, ...rest]);
|
|
1416
|
+
if (autoRestoreResults) {
|
|
1417
|
+
return restoreSearchResults(rawResults);
|
|
1418
|
+
}
|
|
1419
|
+
return rawResults;
|
|
1420
|
+
};
|
|
1421
|
+
}
|
|
1422
|
+
|
|
1423
|
+
return orig.bind(target);
|
|
1424
|
+
}
|
|
1425
|
+
});
|
|
1426
|
+
|
|
1427
|
+
return proxy;
|
|
1428
|
+
}
|
|
1429
|
+
|
|
1430
|
+
return {
|
|
1431
|
+
sanitizeRecords,
|
|
1432
|
+
sanitizeQuery,
|
|
1433
|
+
restoreSearchResults,
|
|
1434
|
+
wrapVectorStore,
|
|
1435
|
+
sessionMap: engine.sessionMap,
|
|
1436
|
+
restore: (text, map) => engine.restore(text, map || engine.sessionMap),
|
|
1437
|
+
resetSession: () => engine.resetSession()
|
|
1438
|
+
};
|
|
1439
|
+
}
|
|
1440
|
+
|
|
1441
|
+
/**
|
|
1442
|
+
* Helper to escape special regex characters.
|
|
1443
|
+
*/
|
|
1444
|
+
function escapeRegex(s) {
|
|
1445
|
+
return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
1446
|
+
}
|
|
1447
|
+
|
|
1448
|
+
/**
|
|
1449
|
+
* Builds an ultra-fast streaming token replacement function.
|
|
1450
|
+
* Preserves all whitespace, indentation, newlines, and code blocks (does NOT trim).
|
|
1451
|
+
* Handles descending key ordering (e.g. [NAME_10] before [NAME_1]), case insensitivity,
|
|
1452
|
+
* alternative delimiters (<...>, {{...}}, __...__), and possessive suffixes.
|
|
1453
|
+
*
|
|
1454
|
+
* @param {Record<string, string>} tokenMap
|
|
1455
|
+
* @returns {(text: string) => string}
|
|
1456
|
+
*/
|
|
1457
|
+
function buildStreamReplacer(tokenMap = {}, options = {}) {
|
|
1458
|
+
if (!tokenMap || typeof tokenMap !== 'object' || Object.keys(tokenMap).length === 0) {
|
|
1459
|
+
return (text) => text || '';
|
|
1460
|
+
}
|
|
1461
|
+
|
|
1462
|
+
const onTokenRestored = typeof options?.onTokenRestored === 'function' ? options.onTokenRestored : null;
|
|
1463
|
+
const lookup = new Map();
|
|
1464
|
+
const patterns = [];
|
|
1465
|
+
const sortedKeys = Object.keys(tokenMap).sort((a, b) => b.length - a.length);
|
|
1466
|
+
|
|
1467
|
+
for (const k of sortedKeys) {
|
|
1468
|
+
const val = tokenMap[k];
|
|
1469
|
+
const inner = k.replace(/^\[|<|\{\{|__|\]|>|\}\}|__/g, '');
|
|
1470
|
+
const match = inner.match(/^([A-Za-z_0-9]+?)[-_ ]?(\d+)$/);
|
|
1471
|
+
|
|
1472
|
+
if (match) {
|
|
1473
|
+
const label = escapeRegex(match[1]);
|
|
1474
|
+
const idx = match[2];
|
|
1475
|
+
const patternStr = '(?:\\[\\s*' + label + '[-_\\s]*0*' + idx + '\\s*\\]|<\\s*' + label + '[-_\\s]*0*' + idx + '\\s*>|\\{\\{\\s*' + label + '[-_\\s]*0*' + idx + '\\s*\\}\\}|__\\s*' + label + '[-_\\s]*0*' + idx + '\\s*__|\\b' + label + '[-_\\s]+0*' + idx + '\\b)';
|
|
1476
|
+
patterns.push(patternStr);
|
|
1477
|
+
const normKey = (match[1] + '_' + parseInt(idx, 10)).toUpperCase();
|
|
1478
|
+
lookup.set(normKey, val);
|
|
1479
|
+
lookup.set(k, val);
|
|
1480
|
+
lookup.set(k.toUpperCase(), val);
|
|
1481
|
+
} else {
|
|
1482
|
+
const safe = escapeRegex(k);
|
|
1483
|
+
patterns.push(safe);
|
|
1484
|
+
lookup.set(k, val);
|
|
1485
|
+
lookup.set(k.toUpperCase(), val);
|
|
1486
|
+
}
|
|
1487
|
+
}
|
|
1488
|
+
|
|
1489
|
+
const unifiedRegex = new RegExp('(?:' + patterns.join('|') + ')(\'s|’s|s)?', 'gi');
|
|
1490
|
+
|
|
1491
|
+
return function replaceTokens(text) {
|
|
1492
|
+
if (!text) return '';
|
|
1493
|
+
unifiedRegex.lastIndex = 0;
|
|
1494
|
+
return text.replace(unifiedRegex, (match, suffix) => {
|
|
1495
|
+
const isPossessive = suffix && /('s|’s)$/i.test(suffix);
|
|
1496
|
+
const coreMatch = suffix ? match.slice(0, match.length - suffix.length) : match;
|
|
1497
|
+
let restored = null;
|
|
1498
|
+
|
|
1499
|
+
if (lookup.has(coreMatch)) {
|
|
1500
|
+
restored = lookup.get(coreMatch);
|
|
1501
|
+
} else if (lookup.has(coreMatch.toUpperCase())) {
|
|
1502
|
+
restored = lookup.get(coreMatch.toUpperCase());
|
|
1503
|
+
} else {
|
|
1504
|
+
const inner = coreMatch.replace(/^\[|<|\{\{|__|\]|>|\}\}|__/g, '').trim();
|
|
1505
|
+
const m = inner.match(/^([A-Za-z_0-9]+?)[-_ ]?(\d+)$/);
|
|
1506
|
+
if (m) {
|
|
1507
|
+
const normKey = (m[1] + '_' + parseInt(m[2], 10)).toUpperCase();
|
|
1508
|
+
if (lookup.has(normKey)) {
|
|
1509
|
+
restored = lookup.get(normKey);
|
|
1510
|
+
}
|
|
1511
|
+
}
|
|
1512
|
+
}
|
|
1513
|
+
|
|
1514
|
+
if (restored !== null) {
|
|
1515
|
+
if (onTokenRestored) {
|
|
1516
|
+
try { onTokenRestored({ token: match, originalValue: restored }); } catch (_) {}
|
|
1517
|
+
}
|
|
1518
|
+
return restored + (isPossessive ? "'s" : '');
|
|
1519
|
+
}
|
|
1520
|
+
|
|
1521
|
+
if (lookup.has(match)) {
|
|
1522
|
+
restored = lookup.get(match);
|
|
1523
|
+
if (onTokenRestored) {
|
|
1524
|
+
try { onTokenRestored({ token: match, originalValue: restored }); } catch (_) {}
|
|
1525
|
+
}
|
|
1526
|
+
return restored;
|
|
1527
|
+
}
|
|
1528
|
+
|
|
1529
|
+
return match;
|
|
1530
|
+
});
|
|
1531
|
+
};
|
|
1532
|
+
}
|
|
1533
|
+
|
|
1534
|
+
/**
|
|
1535
|
+
* Streaming Token Detokenizer with sliding window buffer for LLM chunk boundary splits.
|
|
1536
|
+
* Seamlessly restores split tokens (e.g. [EM + AIL_1]), preserves multi-byte UTF-8,
|
|
1537
|
+
* and maintains zero-latency streaming throughput.
|
|
1538
|
+
*/
|
|
1539
|
+
class StreamingDetokenizer {
|
|
1540
|
+
constructor(tokenMap = {}, options = {}) {
|
|
1541
|
+
this.tokenMap = tokenMap && typeof tokenMap === 'object' ? tokenMap : {};
|
|
1542
|
+
this.options = options || {};
|
|
1543
|
+
this.replacer = buildStreamReplacer(this.tokenMap, this.options);
|
|
1544
|
+
this.pendingBuffer = '';
|
|
1545
|
+
this.hasTokens = Object.keys(this.tokenMap).length > 0;
|
|
1546
|
+
this._lastTokenCount = Object.keys(this.tokenMap).length;
|
|
1547
|
+
this.decoder = new TextDecoder('utf-8', { fatal: false });
|
|
1548
|
+
}
|
|
1549
|
+
|
|
1550
|
+
/**
|
|
1551
|
+
* Backward-compatible buffer property getter/setter.
|
|
1552
|
+
*/
|
|
1553
|
+
get buffer() {
|
|
1554
|
+
return this.pendingBuffer;
|
|
1555
|
+
}
|
|
1556
|
+
|
|
1557
|
+
set buffer(val) {
|
|
1558
|
+
this.pendingBuffer = typeof val === 'string' ? val : '';
|
|
1559
|
+
}
|
|
1560
|
+
|
|
1561
|
+
/**
|
|
1562
|
+
* Dynamically updates the active token map for this streaming session.
|
|
1563
|
+
* @param {Record<string, string>} newMap
|
|
1564
|
+
*/
|
|
1565
|
+
updateTokenMap(newMap) {
|
|
1566
|
+
if (newMap && typeof newMap === 'object') {
|
|
1567
|
+
this.tokenMap = newMap;
|
|
1568
|
+
this.replacer = buildStreamReplacer(this.tokenMap, this.options);
|
|
1569
|
+
this.hasTokens = Object.keys(this.tokenMap).length > 0;
|
|
1570
|
+
this._lastTokenCount = Object.keys(this.tokenMap).length;
|
|
1571
|
+
}
|
|
1572
|
+
}
|
|
1573
|
+
|
|
1574
|
+
/**
|
|
1575
|
+
* Resets the internal sliding window buffer.
|
|
1576
|
+
*/
|
|
1577
|
+
reset() {
|
|
1578
|
+
this.pendingBuffer = '';
|
|
1579
|
+
}
|
|
1580
|
+
|
|
1581
|
+
/**
|
|
1582
|
+
* Feeds an incoming chunk (string, Uint8Array, or Buffer) and returns de-tokenized text.
|
|
1583
|
+
* Buffers any unclosed token candidates at chunk boundaries until closed or flushed.
|
|
1584
|
+
* @param {string | Uint8Array | Buffer} chunk
|
|
1585
|
+
* @returns {string}
|
|
1586
|
+
*/
|
|
1587
|
+
feed(chunk) {
|
|
1588
|
+
if (chunk === undefined || chunk === null) return '';
|
|
1589
|
+
let text = '';
|
|
1590
|
+
if (typeof chunk === 'string') {
|
|
1591
|
+
text = chunk;
|
|
1592
|
+
} else if (chunk instanceof Uint8Array || (typeof Buffer !== 'undefined' && Buffer.isBuffer(chunk))) {
|
|
1593
|
+
text = this.decoder.decode(chunk, { stream: true });
|
|
1594
|
+
} else {
|
|
1595
|
+
text = String(chunk);
|
|
1596
|
+
}
|
|
1597
|
+
|
|
1598
|
+
if (!text) return '';
|
|
1599
|
+
|
|
1600
|
+
// Dynamically recompile if tokens were added to the referenced map
|
|
1601
|
+
const currentTokenCount = Object.keys(this.tokenMap).length;
|
|
1602
|
+
if (currentTokenCount !== this._lastTokenCount) {
|
|
1603
|
+
this.replacer = buildStreamReplacer(this.tokenMap, this.options);
|
|
1604
|
+
this.hasTokens = currentTokenCount > 0;
|
|
1605
|
+
this._lastTokenCount = currentTokenCount;
|
|
1606
|
+
}
|
|
1607
|
+
|
|
1608
|
+
if (!this.hasTokens) return text;
|
|
1609
|
+
|
|
1610
|
+
this.pendingBuffer += text;
|
|
1611
|
+
|
|
1612
|
+
// Candidate unclosed token at tail of buffer (inspect tail window up to 160 chars)
|
|
1613
|
+
// Matches incomplete tokens starting with [, <, {{, or __, plus single delimiter prefixes { and _
|
|
1614
|
+
const tailLen = 160;
|
|
1615
|
+
const tail = this.pendingBuffer.length > tailLen ? this.pendingBuffer.slice(-tailLen) : this.pendingBuffer;
|
|
1616
|
+
const offset = this.pendingBuffer.length > tailLen ? this.pendingBuffer.length - tailLen : 0;
|
|
1617
|
+
const m = tail.match(/(?:\[|<|\{\{?|__?)[A-Za-z0-9_\-А-Яа-яЁё ]{0,128}$/);
|
|
1618
|
+
|
|
1619
|
+
if (m) {
|
|
1620
|
+
const candidate = m[0];
|
|
1621
|
+
// Only closed double-underscore tokens (e.g. __NAME_1__) could match m since '_' is in character class.
|
|
1622
|
+
// Bracketed tokens ([, <, {{) cannot end with their closing delimiter here and must be buffered.
|
|
1623
|
+
const isClosedUnderscoreToken = candidate.length >= 4 && candidate.startsWith('__') && candidate.endsWith('__');
|
|
1624
|
+
if (isClosedUnderscoreToken && this.replacer(candidate) !== candidate) {
|
|
1625
|
+
const toEmit = this.pendingBuffer;
|
|
1626
|
+
this.pendingBuffer = '';
|
|
1627
|
+
return this.replacer(toEmit);
|
|
1628
|
+
}
|
|
1629
|
+
const splitIdx = offset + m.index;
|
|
1630
|
+
const safeToEmit = this.pendingBuffer.slice(0, splitIdx);
|
|
1631
|
+
this.pendingBuffer = this.pendingBuffer.slice(splitIdx);
|
|
1632
|
+
return safeToEmit ? this.replacer(safeToEmit) : '';
|
|
1633
|
+
}
|
|
1634
|
+
|
|
1635
|
+
const toEmit = this.pendingBuffer;
|
|
1636
|
+
this.pendingBuffer = '';
|
|
1637
|
+
return this.replacer(toEmit);
|
|
1638
|
+
}
|
|
1639
|
+
|
|
1640
|
+
/**
|
|
1641
|
+
* Alias for feed() to maintain backwards compatibility with process() callers.
|
|
1642
|
+
*/
|
|
1643
|
+
process(chunk) {
|
|
1644
|
+
return this.feed(chunk);
|
|
1645
|
+
}
|
|
1646
|
+
|
|
1647
|
+
/**
|
|
1648
|
+
* Flushes remaining buffered tokens and finalizes UTF-8 stream decoder.
|
|
1649
|
+
* @returns {string}
|
|
1650
|
+
*/
|
|
1651
|
+
flush() {
|
|
1652
|
+
let text = '';
|
|
1653
|
+
try {
|
|
1654
|
+
text = this.decoder.decode(new Uint8Array(), { stream: false });
|
|
1655
|
+
} catch (_) {}
|
|
1656
|
+
|
|
1657
|
+
if (text) {
|
|
1658
|
+
this.pendingBuffer += text;
|
|
1659
|
+
}
|
|
1660
|
+
|
|
1661
|
+
if (!this.pendingBuffer) return '';
|
|
1662
|
+
const remaining = this.pendingBuffer;
|
|
1663
|
+
this.pendingBuffer = '';
|
|
1664
|
+
return this.replacer(remaining);
|
|
1665
|
+
}
|
|
1666
|
+
|
|
1667
|
+
/**
|
|
1668
|
+
* Returns a native Node.js Transform stream instance for .pipe() / pipeline().
|
|
1669
|
+
* @returns {Transform}
|
|
1670
|
+
*/
|
|
1671
|
+
toNodeTransform() {
|
|
1672
|
+
const self = this;
|
|
1673
|
+
const transformStream = new Transform({
|
|
1674
|
+
transform(chunk, encoding, callback) {
|
|
1675
|
+
try {
|
|
1676
|
+
const out = self.feed(chunk);
|
|
1677
|
+
if (out) this.push(out);
|
|
1678
|
+
callback();
|
|
1679
|
+
} catch (err) {
|
|
1680
|
+
callback(err);
|
|
1681
|
+
}
|
|
1682
|
+
},
|
|
1683
|
+
flush(callback) {
|
|
1684
|
+
try {
|
|
1685
|
+
const out = self.flush();
|
|
1686
|
+
if (out) this.push(out);
|
|
1687
|
+
callback();
|
|
1688
|
+
} catch (err) {
|
|
1689
|
+
callback(err);
|
|
1690
|
+
}
|
|
1691
|
+
},
|
|
1692
|
+
destroy(err, callback) {
|
|
1693
|
+
self.reset();
|
|
1694
|
+
callback(err);
|
|
1695
|
+
}
|
|
1696
|
+
});
|
|
1697
|
+
transformStream.detokenizer = self;
|
|
1698
|
+
return transformStream;
|
|
1699
|
+
}
|
|
1700
|
+
|
|
1701
|
+
/**
|
|
1702
|
+
* Wraps an async iterable (e.g. LLM streaming response) yielding restored chunks.
|
|
1703
|
+
* @param {AsyncIterable<any>} asyncIterable
|
|
1704
|
+
* @returns {AsyncGenerator<any>}
|
|
1705
|
+
*/
|
|
1706
|
+
async* wrapStream(asyncIterable) {
|
|
1707
|
+
for await (const chunk of asyncIterable) {
|
|
1708
|
+
if (chunk === null || chunk === undefined) continue;
|
|
1709
|
+
// Handle OpenAI chunk object
|
|
1710
|
+
if (chunk.choices && Array.isArray(chunk.choices) && chunk.choices[0]?.delta?.content !== undefined) {
|
|
1711
|
+
const delta = chunk.choices[0].delta.content;
|
|
1712
|
+
if (typeof delta === 'string') {
|
|
1713
|
+
const restored = this.feed(delta);
|
|
1714
|
+
if (restored) {
|
|
1715
|
+
chunk.choices[0].delta.content = restored;
|
|
1716
|
+
yield chunk;
|
|
1717
|
+
}
|
|
1718
|
+
} else {
|
|
1719
|
+
yield chunk;
|
|
1720
|
+
}
|
|
1721
|
+
continue;
|
|
1722
|
+
}
|
|
1723
|
+
|
|
1724
|
+
let text = '';
|
|
1725
|
+
if (typeof chunk === 'string') {
|
|
1726
|
+
text = chunk;
|
|
1727
|
+
} else if (typeof chunk?.text === 'string') {
|
|
1728
|
+
text = chunk.text;
|
|
1729
|
+
} else if (typeof chunk?.content === 'string') {
|
|
1730
|
+
text = chunk.content;
|
|
1731
|
+
} else if (chunk instanceof Uint8Array || (typeof Buffer !== 'undefined' && Buffer.isBuffer(chunk))) {
|
|
1732
|
+
text = chunk;
|
|
1733
|
+
} else {
|
|
1734
|
+
text = String(chunk);
|
|
1735
|
+
}
|
|
1736
|
+
|
|
1737
|
+
const emitted = this.feed(text);
|
|
1738
|
+
if (emitted) yield emitted;
|
|
1739
|
+
}
|
|
1740
|
+
|
|
1741
|
+
const finalChunk = this.flush();
|
|
1742
|
+
if (finalChunk) yield finalChunk;
|
|
1743
|
+
}
|
|
1744
|
+
}
|
|
1745
|
+
|
|
1746
|
+
/**
|
|
1747
|
+
* Creates a zero-latency streaming token restorer for LLM Server-Sent Events (SSE) and Web Streams.
|
|
1748
|
+
* Solves the broken chunk boundary problem where tokens like [EMAIL_1] are split across multiple chunks.
|
|
1749
|
+
*
|
|
1750
|
+
* Works with:
|
|
1751
|
+
* - Vercel AI SDK (streamText / createDataStreamResponse)
|
|
1752
|
+
* - OpenAI streaming (chat.completions.create({ stream: true }))
|
|
1753
|
+
* - LangChain streaming
|
|
1754
|
+
* - Native Web Streams (ReadableStream.pipeThrough)
|
|
1755
|
+
* - Async generators (for await ... of)
|
|
1756
|
+
*
|
|
1757
|
+
* @param {Record<string, string>} tokenMap - Volatile token mapping object.
|
|
1758
|
+
* @param {object} [options] - Configuration options.
|
|
1759
|
+
* @returns {TransformStream & { process: (chunk: string) => string, feed: (chunk: string) => string, flush: () => string, wrapStream: (asyncIterable: AsyncIterable<any>) => AsyncGenerator<any>, toNodeTransform: () => Transform, detokenizer: StreamingDetokenizer }}
|
|
1760
|
+
*/
|
|
1761
|
+
function createStreamTokenRestorer(tokenMap = {}, options = {}) {
|
|
1762
|
+
const detok = new StreamingDetokenizer(tokenMap, options);
|
|
1763
|
+
|
|
1764
|
+
let stream;
|
|
1765
|
+
if (typeof TransformStream !== 'undefined') {
|
|
1766
|
+
stream = new TransformStream({
|
|
1767
|
+
transform(chunk, controller) {
|
|
1768
|
+
const out = detok.feed(chunk);
|
|
1769
|
+
if (out) controller.enqueue(out);
|
|
1770
|
+
},
|
|
1771
|
+
flush(controller) {
|
|
1772
|
+
const out = detok.flush();
|
|
1773
|
+
if (out) controller.enqueue(out);
|
|
1774
|
+
}
|
|
1775
|
+
});
|
|
1776
|
+
} else {
|
|
1777
|
+
stream = {};
|
|
1778
|
+
}
|
|
1779
|
+
|
|
1780
|
+
stream.process = (chunk) => detok.feed(chunk);
|
|
1781
|
+
stream.feed = (chunk) => detok.feed(chunk);
|
|
1782
|
+
stream.flush = () => detok.flush();
|
|
1783
|
+
stream.wrapStream = (asyncIterable) => detok.wrapStream(asyncIterable);
|
|
1784
|
+
stream.toNodeTransform = () => detok.toNodeTransform();
|
|
1785
|
+
stream.detokenizer = detok;
|
|
1786
|
+
|
|
1787
|
+
return stream;
|
|
1788
|
+
}
|
|
1789
|
+
|
|
1790
|
+
/**
|
|
1791
|
+
* Creates a Node.js native Transform stream for backend streaming pipelines (stream.pipeline / .pipe()).
|
|
1792
|
+
* @param {Record<string, string>} tokenMap - Volatile token mapping object.
|
|
1793
|
+
* @param {object} [options] - Configuration options.
|
|
1794
|
+
* @returns {Transform}
|
|
1795
|
+
*/
|
|
1796
|
+
function createNodeStreamTokenRestorer(tokenMap = {}, options = {}) {
|
|
1797
|
+
const detok = new StreamingDetokenizer(tokenMap, options);
|
|
1798
|
+
return detok.toNodeTransform();
|
|
1799
|
+
}
|
|
1800
|
+
|
|
1801
|
+
/**
|
|
1802
|
+
* Wraps an async iterable LLM stream (e.g. from Vercel AI SDK textStream or OpenAI stream)
|
|
1803
|
+
* and yields de-tokenized text chunks in real-time, seamlessly stitching split tokens across chunks.
|
|
1804
|
+
*
|
|
1805
|
+
* @param {AsyncIterable<string|object>} stream - The incoming LLM stream.
|
|
1806
|
+
* @param {Record<string, string>} tokenMap - Volatile token map.
|
|
1807
|
+
* @returns {AsyncGenerator<any>}
|
|
1808
|
+
*/
|
|
1809
|
+
async function* wrapAiStream(stream, tokenMap = {}) {
|
|
1810
|
+
const detok = new StreamingDetokenizer(tokenMap);
|
|
1811
|
+
yield* detok.wrapStream(stream);
|
|
1812
|
+
}
|
|
1813
|
+
|
|
1814
|
+
/**
|
|
1815
|
+
* Middleware wrapper for Next.js App Router Route Handlers (POST/PUT).
|
|
1816
|
+
* Automatically sanitizes incoming prompt payloads in RAM (<1ms) before the handler runs,
|
|
1817
|
+
* and attaches sanitized text, tokenMap, and helper restoration functions.
|
|
1818
|
+
*
|
|
1819
|
+
* @param {Function} handler - The async Next.js route handler (req, context) => Response
|
|
1820
|
+
* @param {object} [options] - Sanitization options (profile, detectSecrets, etc.)
|
|
1821
|
+
* @returns {Function} Wrapped Next.js route handler
|
|
1822
|
+
*/
|
|
1823
|
+
function withPrivacyScrubber(handler, options = {}) {
|
|
1824
|
+
return async function privacyScrubberHandler(req, context = {}) {
|
|
1825
|
+
if (!req || typeof req.json !== 'function') {
|
|
1826
|
+
return handler(req, context);
|
|
1827
|
+
}
|
|
1828
|
+
|
|
1829
|
+
let rawBody;
|
|
1830
|
+
try {
|
|
1831
|
+
rawBody = await req.json();
|
|
1832
|
+
} catch (_) {
|
|
1833
|
+
return handler(req, context);
|
|
1834
|
+
}
|
|
1835
|
+
|
|
1836
|
+
let scrubbedBody = rawBody;
|
|
1837
|
+
let tokenMap = {};
|
|
1838
|
+
let telemetry = null;
|
|
1839
|
+
|
|
1840
|
+
if (typeof rawBody === 'string') {
|
|
1841
|
+
const res = sanitize(rawBody, options);
|
|
1842
|
+
scrubbedBody = res.scrubbedText;
|
|
1843
|
+
tokenMap = res.tokenMap;
|
|
1844
|
+
telemetry = res.telemetry;
|
|
1845
|
+
} else if (rawBody && typeof rawBody === 'object') {
|
|
1846
|
+
const serialized = JSON.stringify(rawBody);
|
|
1847
|
+
const res = sanitize(serialized, options);
|
|
1848
|
+
try {
|
|
1849
|
+
scrubbedBody = JSON.parse(res.scrubbedText);
|
|
1850
|
+
} catch (_) {
|
|
1851
|
+
scrubbedBody = rawBody;
|
|
1852
|
+
}
|
|
1853
|
+
tokenMap = res.tokenMap;
|
|
1854
|
+
telemetry = res.telemetry;
|
|
1855
|
+
}
|
|
1856
|
+
|
|
1857
|
+
const psContext = {
|
|
1858
|
+
...context,
|
|
1859
|
+
rawBody,
|
|
1860
|
+
scrubbedBody,
|
|
1861
|
+
tokenMap,
|
|
1862
|
+
telemetry,
|
|
1863
|
+
restore: (text) => restore(text, tokenMap),
|
|
1864
|
+
createStreamRestorer: () => createStreamTokenRestorer(tokenMap)
|
|
1865
|
+
};
|
|
1866
|
+
|
|
1867
|
+
return handler(req, psContext);
|
|
1868
|
+
};
|
|
1869
|
+
}
|
|
1870
|
+
|
|
1871
|
+
/**
|
|
1872
|
+
* Express.js / Connect middleware for automatic zero-trust PII sanitization.
|
|
1873
|
+
* Sanitizes req.body in local RAM and attaches req.psTokenMap and req.restore(aiResponse).
|
|
1874
|
+
*
|
|
1875
|
+
* @param {object} [options] - Sanitization options.
|
|
1876
|
+
* @returns {Function} Express middleware (req, res, next)
|
|
1877
|
+
*/
|
|
1878
|
+
function createExpressMiddleware(options = {}) {
|
|
1879
|
+
return function privacyScrubberExpress(req, res, next) {
|
|
1880
|
+
if (req && req.body && typeof req.body === 'object') {
|
|
1881
|
+
const serialized = JSON.stringify(req.body);
|
|
1882
|
+
const { scrubbedText, tokenMap, telemetry } = sanitize(serialized, options);
|
|
1883
|
+
req.rawBody = req.body;
|
|
1884
|
+
try {
|
|
1885
|
+
req.body = JSON.parse(scrubbedText);
|
|
1886
|
+
} catch (_) {}
|
|
1887
|
+
req.psTokenMap = tokenMap;
|
|
1888
|
+
req.psTelemetry = telemetry;
|
|
1889
|
+
req.restore = (aiResponse) => restore(aiResponse, tokenMap);
|
|
1890
|
+
req.createStreamRestorer = () => createStreamTokenRestorer(tokenMap);
|
|
1891
|
+
}
|
|
1892
|
+
if (typeof next === 'function') next();
|
|
1893
|
+
};
|
|
1894
|
+
}
|
|
1895
|
+
|
|
1896
|
+
/**
|
|
1897
|
+
* Generates an official Zero-Trust Compliance Audit Report.
|
|
1898
|
+
* @param {Record<string, string>} sessionMap - Map of token bindings.
|
|
1899
|
+
* @param {object} [options] - Options: companyName, department, format ('markdown'|'json'|'summary').
|
|
1900
|
+
* @returns {object|string}
|
|
1901
|
+
*/
|
|
1902
|
+
function formatCefEvent(eventData = {}) {
|
|
1903
|
+
const {
|
|
1904
|
+
vendor = 'PrivacyScrubber',
|
|
1905
|
+
product = 'ZTDS',
|
|
1906
|
+
version = '2.2.5',
|
|
1907
|
+
eventClassId = 'PII_REDACTION',
|
|
1908
|
+
eventName = 'Zero-Trust PII Sanitization Executed',
|
|
1909
|
+
riskLevel = 'LOW EXPOSURE',
|
|
1910
|
+
tokensMasked = 0,
|
|
1911
|
+
entities = {},
|
|
1912
|
+
frameworks = ['ZTDS Standard'],
|
|
1913
|
+
sessionHash = '',
|
|
1914
|
+
source = 'local_ram',
|
|
1915
|
+
action = 'redact_and_mask',
|
|
1916
|
+
organization = '',
|
|
1917
|
+
department = ''
|
|
1918
|
+
} = eventData;
|
|
1919
|
+
|
|
1920
|
+
let severity = 1;
|
|
1921
|
+
if (riskLevel.includes('CRITICAL')) severity = 9;
|
|
1922
|
+
else if (riskLevel.includes('MODERATE')) severity = 6;
|
|
1923
|
+
else if (riskLevel.includes('LOW')) severity = 3;
|
|
1924
|
+
else severity = 0;
|
|
1925
|
+
|
|
1926
|
+
const entitiesStr = Object.entries(entities).map(([k, v]) => `${k}:${v}`).join(',') || 'none';
|
|
1927
|
+
const frameworksStr = Array.isArray(frameworks) ? frameworks.join(',') : String(frameworks);
|
|
1928
|
+
const safeMsg = `Sanitized ${tokensMasked} entity tokens in volatile RAM. 0 bytes network egress.`;
|
|
1929
|
+
|
|
1930
|
+
const extension = [
|
|
1931
|
+
`src=${source}`,
|
|
1932
|
+
`act=${action}`,
|
|
1933
|
+
`cs1Label=RiskLevel cs1=${riskLevel.replace(/\s+/g, '_')}`,
|
|
1934
|
+
`cn1Label=TokensMasked cn1=${tokensMasked}`,
|
|
1935
|
+
`cs2Label=Frameworks cs2=${frameworksStr.replace(/\s+/g, '_')}`,
|
|
1936
|
+
`cs3Label=Entities cs3=${entitiesStr}`,
|
|
1937
|
+
sessionHash ? `cs4Label=SessionHash cs4=${sessionHash.substring(0, 16)}` : '',
|
|
1938
|
+
organization ? `suser=${organization.replace(/\s+/g, '_')}` : '',
|
|
1939
|
+
department ? `cs5Label=Department cs5=${department.replace(/\s+/g, '_')}` : '',
|
|
1940
|
+
`msg=${safeMsg}`
|
|
1941
|
+
].filter(Boolean).join(' ');
|
|
1942
|
+
|
|
1943
|
+
return `CEF:0|${vendor}|${product}|${version}|${eventClassId}|${eventName}|${severity}|${extension}`;
|
|
1944
|
+
}
|
|
1945
|
+
|
|
1946
|
+
function formatSyslogEvent(eventData = {}) {
|
|
1947
|
+
const {
|
|
1948
|
+
timestamp = new Date().toISOString(),
|
|
1949
|
+
hostname = (typeof os !== 'undefined' && typeof os.hostname === 'function' ? os.hostname() : 'localhost'),
|
|
1950
|
+
pid = (typeof process !== 'undefined' && process.pid ? process.pid : 1),
|
|
1951
|
+
riskLevel = 'LOW EXPOSURE',
|
|
1952
|
+
tokensMasked = 0,
|
|
1953
|
+
frameworks = ['ZTDS Standard'],
|
|
1954
|
+
sessionHash = '',
|
|
1955
|
+
organization = ''
|
|
1956
|
+
} = eventData;
|
|
1957
|
+
|
|
1958
|
+
let pri = 134; // local0.info
|
|
1959
|
+
if (riskLevel.includes('CRITICAL')) pri = 132; // local0.warning
|
|
1960
|
+
else if (riskLevel.includes('MODERATE')) pri = 133; // local0.notice
|
|
1961
|
+
|
|
1962
|
+
const frameworksStr = Array.isArray(frameworks) ? frameworks.join(';') : String(frameworks);
|
|
1963
|
+
const hashPrefix = sessionHash.substring(0, 16) || 'none';
|
|
1964
|
+
const orgStr = organization ? ` org="${organization.replace(/"/g, '')}"` : '';
|
|
1965
|
+
|
|
1966
|
+
const sd = `[ztds@49152 risk="${riskLevel}" tokens="${tokensMasked}" frameworks="${frameworksStr}" session="${hashPrefix}"${orgStr}]`;
|
|
1967
|
+
const msg = `Zero-Trust Data Sanitization executed in RAM. ${tokensMasked} token(s) masked. 0 bytes network egress.`;
|
|
1968
|
+
|
|
1969
|
+
return `<${pri}>1 ${timestamp} ${hostname} privacyscrubber ${pid} ZTDS - ${sd} ${msg}`;
|
|
1970
|
+
}
|
|
1971
|
+
|
|
1972
|
+
function formatJsonlEvent(eventData = {}) {
|
|
1973
|
+
const {
|
|
1974
|
+
timestamp = new Date().toISOString(),
|
|
1975
|
+
version = '2.2.5',
|
|
1976
|
+
protocol = 'ZTDS',
|
|
1977
|
+
event = 'sanitization',
|
|
1978
|
+
riskLevel = 'LOW EXPOSURE',
|
|
1979
|
+
tokensMasked = 0,
|
|
1980
|
+
entities = {},
|
|
1981
|
+
frameworks = ['ZTDS Standard'],
|
|
1982
|
+
sessionHash = '',
|
|
1983
|
+
organization = '',
|
|
1984
|
+
department = '',
|
|
1985
|
+
executionMs = 0,
|
|
1986
|
+
metadata = {}
|
|
1987
|
+
} = eventData;
|
|
1988
|
+
|
|
1989
|
+
const record = {
|
|
1990
|
+
timestamp,
|
|
1991
|
+
version,
|
|
1992
|
+
protocol,
|
|
1993
|
+
event,
|
|
1994
|
+
risk_level: riskLevel,
|
|
1995
|
+
tokens_masked: tokensMasked,
|
|
1996
|
+
entities,
|
|
1997
|
+
frameworks,
|
|
1998
|
+
session_hash: sessionHash,
|
|
1999
|
+
zero_egress: true,
|
|
2000
|
+
verification_mode: "100% Offline (In-Memory RAM)",
|
|
2001
|
+
...(organization && { organization }),
|
|
2002
|
+
...(department && { department }),
|
|
2003
|
+
...(executionMs && { execution_ms: executionMs }),
|
|
2004
|
+
...(Object.keys(metadata).length > 0 && { metadata })
|
|
2005
|
+
};
|
|
2006
|
+
|
|
2007
|
+
return JSON.stringify(record);
|
|
2008
|
+
}
|
|
2009
|
+
|
|
2010
|
+
function createAuditLogger(options = {}) {
|
|
2011
|
+
const {
|
|
2012
|
+
format = 'jsonl',
|
|
2013
|
+
destination = null,
|
|
2014
|
+
organization = 'PrivacyScrubber Enterprise Client',
|
|
2015
|
+
department = 'SecOps'
|
|
2016
|
+
} = options;
|
|
2017
|
+
|
|
2018
|
+
let fileHandle = null;
|
|
2019
|
+
let streamHandle = null;
|
|
2020
|
+
const buffer = [];
|
|
2021
|
+
|
|
2022
|
+
if (typeof destination === 'string' && destination.trim()) {
|
|
2023
|
+
try {
|
|
2024
|
+
const dir = path.dirname(destination);
|
|
2025
|
+
if (!fs.existsSync(dir)) {
|
|
2026
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
2027
|
+
}
|
|
2028
|
+
fileHandle = destination;
|
|
2029
|
+
} catch {}
|
|
2030
|
+
} else if (destination && typeof destination.write === 'function') {
|
|
2031
|
+
streamHandle = destination;
|
|
2032
|
+
}
|
|
2033
|
+
|
|
2034
|
+
function formatEvent(eventData) {
|
|
2035
|
+
const enriched = {
|
|
2036
|
+
organization,
|
|
2037
|
+
department,
|
|
2038
|
+
timestamp: new Date().toISOString(),
|
|
2039
|
+
...eventData
|
|
2040
|
+
};
|
|
2041
|
+
|
|
2042
|
+
if (format === 'cef') {
|
|
2043
|
+
return formatCefEvent(enriched);
|
|
2044
|
+
}
|
|
2045
|
+
if (format === 'syslog') {
|
|
2046
|
+
return formatSyslogEvent(enriched);
|
|
2047
|
+
}
|
|
2048
|
+
if (format === 'json') {
|
|
2049
|
+
return JSON.stringify(enriched, null, 2);
|
|
2050
|
+
}
|
|
2051
|
+
return formatJsonlEvent(enriched);
|
|
2052
|
+
}
|
|
2053
|
+
|
|
2054
|
+
function writeEntry(line) {
|
|
2055
|
+
buffer.push(line);
|
|
2056
|
+
if (fileHandle) {
|
|
2057
|
+
try {
|
|
2058
|
+
fs.appendFileSync(fileHandle, line + '\n', 'utf8');
|
|
2059
|
+
} catch (e) {
|
|
2060
|
+
console.error('[PrivacyScrubber AuditLogger] File write error:', e);
|
|
2061
|
+
}
|
|
2062
|
+
} else if (streamHandle) {
|
|
2063
|
+
try {
|
|
2064
|
+
streamHandle.write(line + '\n');
|
|
2065
|
+
} catch (e) {
|
|
2066
|
+
console.error('[PrivacyScrubber AuditLogger] Stream write error:', e);
|
|
2067
|
+
}
|
|
2068
|
+
}
|
|
2069
|
+
}
|
|
2070
|
+
|
|
2071
|
+
function logSanitization(result, context = {}) {
|
|
2072
|
+
if (!result) return '';
|
|
2073
|
+
let tokenMap = {};
|
|
2074
|
+
let telemetry = null;
|
|
2075
|
+
let count = 0;
|
|
2076
|
+
let execMs = 0;
|
|
2077
|
+
|
|
2078
|
+
if (result.tokenMap && typeof result.tokenMap === 'object') {
|
|
2079
|
+
tokenMap = result.tokenMap;
|
|
2080
|
+
telemetry = result.telemetry;
|
|
2081
|
+
count = result.count !== undefined ? result.count : Object.keys(tokenMap).length;
|
|
2082
|
+
execMs = result.executionMs || 0;
|
|
2083
|
+
} else if (typeof result === 'object') {
|
|
2084
|
+
tokenMap = result;
|
|
2085
|
+
count = result.count !== undefined ? result.count : (result.tokensMasked !== undefined ? result.tokensMasked : Object.keys(tokenMap).length);
|
|
2086
|
+
}
|
|
2087
|
+
|
|
2088
|
+
if (!telemetry) {
|
|
2089
|
+
telemetry = buildAuditTelemetry(tokenMap);
|
|
2090
|
+
}
|
|
2091
|
+
|
|
2092
|
+
let sessionHash = '';
|
|
2093
|
+
try {
|
|
2094
|
+
const cryptoMod = require('crypto');
|
|
2095
|
+
sessionHash = cryptoMod.createHash('sha256').update(JSON.stringify(tokenMap) + Date.now()).digest('hex');
|
|
2096
|
+
} catch {
|
|
2097
|
+
sessionHash = 'ztds_' + Math.random().toString(16).substring(2, 10);
|
|
2098
|
+
}
|
|
2099
|
+
|
|
2100
|
+
const eventData = {
|
|
2101
|
+
event: 'pii_sanitization',
|
|
2102
|
+
riskLevel: telemetry.riskLevel || 'CLEAN (ZERO PII)',
|
|
2103
|
+
tokensMasked: count,
|
|
2104
|
+
entities: telemetry.entities || {},
|
|
2105
|
+
frameworks: telemetry.frameworksList || ['ZTDS Standard'],
|
|
2106
|
+
sessionHash,
|
|
2107
|
+
executionMs: execMs,
|
|
2108
|
+
metadata: context.metadata || {}
|
|
2109
|
+
};
|
|
2110
|
+
|
|
2111
|
+
const formatted = formatEvent(eventData);
|
|
2112
|
+
writeEntry(formatted);
|
|
2113
|
+
return formatted;
|
|
2114
|
+
}
|
|
2115
|
+
|
|
2116
|
+
function logEvent(eventData) {
|
|
2117
|
+
const formatted = formatEvent(eventData);
|
|
2118
|
+
writeEntry(formatted);
|
|
2119
|
+
return formatted;
|
|
2120
|
+
}
|
|
2121
|
+
|
|
2122
|
+
function getEntries() {
|
|
2123
|
+
return [...buffer];
|
|
2124
|
+
}
|
|
2125
|
+
|
|
2126
|
+
function clear() {
|
|
2127
|
+
buffer.length = 0;
|
|
2128
|
+
}
|
|
2129
|
+
|
|
2130
|
+
return {
|
|
2131
|
+
formatEvent,
|
|
2132
|
+
logSanitization,
|
|
2133
|
+
logEvent,
|
|
2134
|
+
log: logEvent,
|
|
2135
|
+
getEntries,
|
|
2136
|
+
getBuffer: getEntries,
|
|
2137
|
+
clear,
|
|
2138
|
+
clearBuffer: clear,
|
|
2139
|
+
format,
|
|
2140
|
+
destination
|
|
2141
|
+
};
|
|
2142
|
+
}
|
|
2143
|
+
|
|
2144
|
+
function generateAuditReport(sessionMap = {}, options = {}) {
|
|
2145
|
+
const { companyName = 'PrivacyScrubber Client', department = 'SecOps / Compliance', format = 'markdown' } = options;
|
|
2146
|
+
const telemetry = buildAuditTelemetry(sessionMap);
|
|
2147
|
+
|
|
2148
|
+
let sessionHash = '';
|
|
2149
|
+
try {
|
|
2150
|
+
const cryptoMod = require('crypto');
|
|
2151
|
+
sessionHash = cryptoMod.createHash('sha256').update(JSON.stringify(sessionMap) + Date.now().toString()).digest('hex');
|
|
2152
|
+
} catch {
|
|
2153
|
+
sessionHash = Array.from(crypto.getRandomValues(new Uint8Array(16))).map(b => b.toString(16).padStart(2, '0')).join('');
|
|
2154
|
+
}
|
|
2155
|
+
|
|
2156
|
+
const timestamp = new Date().toISOString();
|
|
2157
|
+
const entitySummary = Object.entries(telemetry.entities).map(([t, count]) => `[${t}]: ${count}`).join(', ') || 'None (Clean)';
|
|
2158
|
+
|
|
2159
|
+
if (format === 'cef') {
|
|
2160
|
+
return formatCefEvent({
|
|
2161
|
+
organization: companyName,
|
|
2162
|
+
department: department,
|
|
2163
|
+
timestamp,
|
|
2164
|
+
sessionHash,
|
|
2165
|
+
riskLevel: telemetry.riskLevel,
|
|
2166
|
+
tokensMasked: telemetry.totalCount,
|
|
2167
|
+
entities: telemetry.entities,
|
|
2168
|
+
frameworks: telemetry.frameworksList
|
|
2169
|
+
});
|
|
2170
|
+
}
|
|
2171
|
+
|
|
2172
|
+
if (format === 'syslog') {
|
|
2173
|
+
return formatSyslogEvent({
|
|
2174
|
+
organization: companyName,
|
|
2175
|
+
department: department,
|
|
2176
|
+
timestamp,
|
|
2177
|
+
sessionHash,
|
|
2178
|
+
riskLevel: telemetry.riskLevel,
|
|
2179
|
+
tokensMasked: telemetry.totalCount,
|
|
2180
|
+
entities: telemetry.entities,
|
|
2181
|
+
frameworks: telemetry.frameworksList
|
|
2182
|
+
});
|
|
2183
|
+
}
|
|
2184
|
+
|
|
2185
|
+
if (format === 'jsonl') {
|
|
2186
|
+
return formatJsonlEvent({
|
|
2187
|
+
organization: companyName,
|
|
2188
|
+
department: department,
|
|
2189
|
+
timestamp,
|
|
2190
|
+
sessionHash,
|
|
2191
|
+
riskLevel: telemetry.riskLevel,
|
|
2192
|
+
tokensMasked: telemetry.totalCount,
|
|
2193
|
+
entities: telemetry.entities,
|
|
2194
|
+
frameworks: telemetry.frameworksList
|
|
2195
|
+
});
|
|
2196
|
+
}
|
|
2197
|
+
|
|
2198
|
+
if (format === 'json') {
|
|
2199
|
+
return {
|
|
2200
|
+
protocol: "Zero-Trust Data Sanitization (ZTDS)",
|
|
2201
|
+
certificateId: `ZTDS-CERT-${sessionHash.substring(0, 16).toUpperCase()}`,
|
|
2202
|
+
organization: companyName,
|
|
2203
|
+
department: department,
|
|
2204
|
+
timestamp: timestamp,
|
|
2205
|
+
sessionHash: sessionHash,
|
|
2206
|
+
verificationMode: "100% Offline (In-Memory RAM)",
|
|
2207
|
+
complianceStatus: "VERIFIED PASS",
|
|
2208
|
+
riskLevel: telemetry.riskLevel,
|
|
2209
|
+
frameworksEnforced: telemetry.frameworksList,
|
|
2210
|
+
totalMaskedTokens: telemetry.totalCount,
|
|
2211
|
+
entitiesBreakdown: telemetry.entities,
|
|
2212
|
+
zeroEgressVerified: true,
|
|
2213
|
+
verificationUrl: `https://privacyscrubber.com/features/audit-receipt/#verify?hash=${sessionHash.substring(0, 16)}`
|
|
2214
|
+
};
|
|
2215
|
+
}
|
|
2216
|
+
|
|
2217
|
+
if (format === 'summary') {
|
|
2218
|
+
return `[PrivacyScrubber Compliance Certificate] ID: ZTDS-${sessionHash.substring(0, 8).toUpperCase()} | Organization: ${companyName} | Masked Tokens: ${telemetry.totalCount} | Risk: ${telemetry.riskLevel} | Frameworks: ${telemetry.frameworksList.join(', ')} | Status: PASS (100% Air-Gapped)`;
|
|
2219
|
+
}
|
|
2220
|
+
|
|
2221
|
+
return `# 🛡️ Zero-Trust Data Sanitization Compliance Certificate
|
|
2222
|
+
**Certificate ID:** \`ZTDS-CERT-${sessionHash.substring(0, 16).toUpperCase()}\`
|
|
2223
|
+
**Organization:** ${companyName} (${department})
|
|
2224
|
+
**Timestamp:** ${timestamp}
|
|
2225
|
+
**Verification Mode:** 100% Local In-Memory Processing (Air-Gapped)
|
|
2226
|
+
**Status:** **VERIFIED PASS** (Zero Network Egress)
|
|
2227
|
+
|
|
2228
|
+
---
|
|
2229
|
+
|
|
2230
|
+
### 📊 Sanitization Metrics & Risk Assessment
|
|
2231
|
+
* **Overall Risk Rating:** **${telemetry.riskLevel}**
|
|
2232
|
+
* **Total Sensitive Entities Masked:** \`${telemetry.totalCount}\`
|
|
2233
|
+
* **Entity Breakdown:** ${entitySummary}
|
|
2234
|
+
* **Network Data Transmitted:** \`0.00 KB (Zero-Trust Local RAM)\`
|
|
2235
|
+
|
|
2236
|
+
### 📜 Regulatory Frameworks Enforced
|
|
2237
|
+
${telemetry.frameworksList.map(f => `- **${f}**`).join('\n')}
|
|
2238
|
+
|
|
2239
|
+
### 🔒 CISO Compliance Declaration
|
|
2240
|
+
1. **EU AI Act (Art. 50) & GDPR (Art. 25 & 32):** Data minimization and local pseudonymization enforced prior to model interaction.
|
|
2241
|
+
2. **HIPAA Safe Harbor (§164.514) / SOC 2 Type II:** All direct and indirect identifiers sanitized locally without cloud processor liability.
|
|
2242
|
+
3. **Cryptographic Verification:** Tamper-evident session verification hash: \`${sessionHash}\`
|
|
2243
|
+
|
|
2244
|
+
*Certified Offline by PrivacyScrubber Core SDK*
|
|
2245
|
+
*Verify at: https://privacyscrubber.com/features/audit-receipt/*`;
|
|
2246
|
+
}
|
|
2247
|
+
|
|
2248
|
+
/**
|
|
2249
|
+
* Creates a Node.js Transform stream that sanitizes incoming text chunks line-by-line in volatile RAM.
|
|
2250
|
+
* Zero external network egress. Zero memory bloat. Ideal for streaming server logs, CSVs, or HTTP responses.
|
|
2251
|
+
*
|
|
2252
|
+
* @param {object} [options={}] - Stream configuration or PrivacyScrubberEngine options.
|
|
2253
|
+
* @param {PrivacyScrubberEngine} [options.engine] - Optional pre-configured PrivacyScrubberEngine instance.
|
|
2254
|
+
* @param {string} [options.profile='General'] - Compliance detection profile.
|
|
2255
|
+
* @param {string} [options.licenseKey] - Optional license key.
|
|
2256
|
+
* @param {boolean} [options.detectSecrets=false] - Auto-detect cloud credentials and API keys.
|
|
2257
|
+
* @param {Array} [options.customRules=[]] - Custom regex rules.
|
|
2258
|
+
* @returns {Transform & { getSessionMap: () => Record<string, string>, getTelemetry: () => object, engine: PrivacyScrubberEngine }}
|
|
2259
|
+
*/
|
|
2260
|
+
function createSanitizeStream(options = {}) {
|
|
2261
|
+
const engine = options.engine || new PrivacyScrubberEngine(options);
|
|
2262
|
+
const persistSession = options.persistSession !== false;
|
|
2263
|
+
const maxSessionTokens = options.maxSessionTokens || 10000;
|
|
2264
|
+
let tail = '';
|
|
2265
|
+
|
|
2266
|
+
const stream = new Transform({
|
|
2267
|
+
transform(chunk, encoding, callback) {
|
|
2268
|
+
try {
|
|
2269
|
+
const str = tail + (Buffer.isBuffer(chunk) ? chunk.toString('utf8') : String(chunk));
|
|
2270
|
+
const lines = str.split('\n');
|
|
2271
|
+
tail = lines.pop() || '';
|
|
2272
|
+
|
|
2273
|
+
for (let i = 0; i < lines.length; i++) {
|
|
2274
|
+
const sanitized = engine.sanitize(lines[i], {
|
|
2275
|
+
...options,
|
|
2276
|
+
persistSession,
|
|
2277
|
+
maxSessionTokens,
|
|
2278
|
+
generateReceipt: false
|
|
2279
|
+
});
|
|
2280
|
+
this.push(sanitized.scrubbedText + '\n');
|
|
2281
|
+
}
|
|
2282
|
+
callback();
|
|
2283
|
+
} catch (err) {
|
|
2284
|
+
callback(err);
|
|
2285
|
+
}
|
|
2286
|
+
},
|
|
2287
|
+
flush(callback) {
|
|
2288
|
+
try {
|
|
2289
|
+
if (tail.length > 0) {
|
|
2290
|
+
const sanitized = engine.sanitize(tail, {
|
|
2291
|
+
...options,
|
|
2292
|
+
persistSession,
|
|
2293
|
+
maxSessionTokens,
|
|
2294
|
+
generateReceipt: false
|
|
2295
|
+
});
|
|
2296
|
+
this.push(sanitized.scrubbedText);
|
|
2297
|
+
tail = '';
|
|
2298
|
+
}
|
|
2299
|
+
callback();
|
|
2300
|
+
} catch (err) {
|
|
2301
|
+
callback(err);
|
|
2302
|
+
}
|
|
2303
|
+
},
|
|
2304
|
+
destroy(err, callback) {
|
|
2305
|
+
tail = '';
|
|
2306
|
+
callback(err);
|
|
2307
|
+
}
|
|
2308
|
+
});
|
|
2309
|
+
|
|
2310
|
+
stream.engine = engine;
|
|
2311
|
+
stream.getSessionMap = () => ({ ...engine.sessionMap });
|
|
2312
|
+
stream.getTelemetry = () => buildAuditTelemetry(engine.sessionMap);
|
|
2313
|
+
|
|
2314
|
+
return stream;
|
|
2315
|
+
}
|
|
2316
|
+
|
|
2317
|
+
/**
|
|
2318
|
+
* Restores masked tokens in AI-generated code or text locally in volatile RAM,
|
|
2319
|
+
* and writes the authentic content directly to the destination file on disk.
|
|
2320
|
+
*
|
|
2321
|
+
* @param {string} filePath - Target file destination.
|
|
2322
|
+
* @param {string} content - Code/text containing placeholder tokens (e.g. [API_KEY_1]).
|
|
2323
|
+
* @param {object} [options={}] - Options: tokenMap, engine, createBackup (default: true).
|
|
2324
|
+
* @returns {{ success: boolean, filePath: string, restoredCount: number, backupCreated: boolean, backupPath?: string }}
|
|
2325
|
+
*/
|
|
2326
|
+
function applyPatch(filePath, content, options = {}) {
|
|
2327
|
+
if (!filePath || typeof filePath !== 'string') {
|
|
2328
|
+
throw new Error('[PrivacyScrubber] Missing required parameter: filePath must be a string.');
|
|
2329
|
+
}
|
|
2330
|
+
if (content === undefined || content === null || typeof content !== 'string') {
|
|
2331
|
+
throw new Error('[PrivacyScrubber] Missing required parameter: content must be a string.');
|
|
2332
|
+
}
|
|
2333
|
+
|
|
2334
|
+
const {
|
|
2335
|
+
tokenMap = null,
|
|
2336
|
+
engine = null,
|
|
2337
|
+
createBackup = true
|
|
2338
|
+
} = options;
|
|
2339
|
+
|
|
2340
|
+
const activeTokenMap = tokenMap || (engine && engine.sessionMap) || {};
|
|
2341
|
+
const resolvedPath = path.resolve(process.cwd(), filePath);
|
|
2342
|
+
const restored = unscrub(content, activeTokenMap);
|
|
2343
|
+
|
|
2344
|
+
let backupCreated = false;
|
|
2345
|
+
let backupPath;
|
|
2346
|
+
|
|
2347
|
+
if (createBackup && fs.existsSync(resolvedPath)) {
|
|
2348
|
+
backupPath = `${resolvedPath}.bak`;
|
|
2349
|
+
fs.copyFileSync(resolvedPath, backupPath);
|
|
2350
|
+
backupCreated = true;
|
|
2351
|
+
}
|
|
2352
|
+
|
|
2353
|
+
const parentDir = path.dirname(resolvedPath);
|
|
2354
|
+
if (!fs.existsSync(parentDir)) {
|
|
2355
|
+
fs.mkdirSync(parentDir, { recursive: true });
|
|
2356
|
+
}
|
|
2357
|
+
|
|
2358
|
+
fs.writeFileSync(resolvedPath, restored.restoredText, 'utf8');
|
|
2359
|
+
|
|
2360
|
+
return {
|
|
2361
|
+
success: true,
|
|
2362
|
+
filePath: resolvedPath,
|
|
2363
|
+
restoredCount: restored.restoredCount || 0,
|
|
2364
|
+
backupCreated,
|
|
2365
|
+
backupPath
|
|
2366
|
+
};
|
|
2367
|
+
}
|
|
2368
|
+
|
|
2369
|
+
/**
|
|
2370
|
+
* Alias for applyPatch.
|
|
2371
|
+
*/
|
|
2372
|
+
function restoreToFile(filePath, content, tokenMapOrOptions = {}, options = {}) {
|
|
2373
|
+
if (tokenMapOrOptions && typeof tokenMapOrOptions === 'object' && !tokenMapOrOptions.sessionMap && !tokenMapOrOptions.tokenMap) {
|
|
2374
|
+
return applyPatch(filePath, content, { ...options, tokenMap: tokenMapOrOptions });
|
|
2375
|
+
}
|
|
2376
|
+
return applyPatch(filePath, content, typeof tokenMapOrOptions === 'object' ? tokenMapOrOptions : options);
|
|
2377
|
+
}
|
|
2378
|
+
|
|
2379
|
+
/**
|
|
2380
|
+
* Creates CISO-grade Zero-Trust Agentic Guard tools for AI agents & LLM frameworks.
|
|
2381
|
+
* Compatible with Vercel AI SDK, OpenAI function calling, LangChain, and AutoGen.
|
|
2382
|
+
* Intercepts terminal commands, file reads, and code writes, masking secrets into volatile RAM.
|
|
2383
|
+
*
|
|
2384
|
+
* @param {PrivacyScrubberEngine|object} [engineOrOptions={}] - PrivacyScrubberEngine or config options.
|
|
2385
|
+
* @param {object} [toolConfig={}] - Default configuration: cwd, profile.
|
|
2386
|
+
* @returns {object} Object containing guardExec, guardReadFile, guardApplyPatch, guardGitDiff, getDefinitions, and engine.
|
|
2387
|
+
*/
|
|
2388
|
+
function createGuardedTools(engineOrOptions = {}, toolConfig = {}) {
|
|
2389
|
+
const engine = (engineOrOptions instanceof PrivacyScrubberEngine)
|
|
2390
|
+
? engineOrOptions
|
|
2391
|
+
: new PrivacyScrubberEngine(engineOrOptions);
|
|
2392
|
+
|
|
2393
|
+
const baseCwd = toolConfig.cwd || process.cwd();
|
|
2394
|
+
const defaultProfile = toolConfig.profile || engine.defaultProfile || 'Dev';
|
|
2395
|
+
|
|
2396
|
+
const tools = {
|
|
2397
|
+
guardExec: {
|
|
2398
|
+
name: 'guard_exec',
|
|
2399
|
+
description: 'Executes a shell command locally, intercepts stdout/stderr line-by-line, sanitizes credentials and PII into RAM tokens, and returns clean text to model context.',
|
|
2400
|
+
parameters: {
|
|
2401
|
+
type: 'object',
|
|
2402
|
+
properties: {
|
|
2403
|
+
command: { type: 'string', description: 'Shell command to execute.' },
|
|
2404
|
+
cwd: { type: 'string', description: 'Working directory.' },
|
|
2405
|
+
profile: { type: 'string', description: 'Detection profile (default: Dev).' },
|
|
2406
|
+
timeoutMs: { type: 'number', description: 'Execution timeout in milliseconds (default: 15000).' }
|
|
2407
|
+
},
|
|
2408
|
+
required: ['command']
|
|
2409
|
+
},
|
|
2410
|
+
execute: async ({ command, cwd, profile = defaultProfile, timeoutMs = 15000 } = {}) => {
|
|
2411
|
+
if (!command || typeof command !== 'string') {
|
|
2412
|
+
throw new Error('[guard_exec] Missing or invalid parameter: command must be a string.');
|
|
2413
|
+
}
|
|
2414
|
+
const execCwd = cwd ? path.resolve(baseCwd, cwd) : baseCwd;
|
|
2415
|
+
let stdout = '';
|
|
2416
|
+
let stderr = '';
|
|
2417
|
+
let exitCode = 0;
|
|
2418
|
+
|
|
2419
|
+
try {
|
|
2420
|
+
stdout = execSync(command, {
|
|
2421
|
+
cwd: execCwd,
|
|
2422
|
+
timeout: Math.min(Math.max(timeoutMs, 1000), 60000),
|
|
2423
|
+
maxBuffer: 10 * 1024 * 1024,
|
|
2424
|
+
encoding: 'utf8',
|
|
2425
|
+
stdio: ['pipe', 'pipe', 'pipe']
|
|
2426
|
+
});
|
|
2427
|
+
} catch (err) {
|
|
2428
|
+
stdout = err.stdout ? err.stdout.toString() : '';
|
|
2429
|
+
stderr = err.stderr ? err.stderr.toString() : (err.message || '');
|
|
2430
|
+
exitCode = typeof err.status === 'number' ? err.status : 1;
|
|
2431
|
+
}
|
|
2432
|
+
|
|
2433
|
+
const resCommand = engine.sanitize(command, { profile, detectSecrets: true, persistSession: true, generateReceipt: false });
|
|
2434
|
+
const resStdout = engine.sanitize(stdout, { profile, detectSecrets: true, persistSession: true, generateReceipt: false });
|
|
2435
|
+
const resStderr = engine.sanitize(stderr, { profile, detectSecrets: true, persistSession: true, generateReceipt: false });
|
|
2436
|
+
|
|
2437
|
+
const newTokens = { ...resCommand.tokenMap, ...resStdout.tokenMap, ...resStderr.tokenMap };
|
|
2438
|
+
const telemetry = buildAuditTelemetry(newTokens);
|
|
2439
|
+
const receipt = formatAuditReceipt(telemetry);
|
|
2440
|
+
|
|
2441
|
+
const responseText = `[Zero-Trust Agentic Guard: Exec]\n` +
|
|
2442
|
+
`Command: ${resCommand.scrubbedText}\n` +
|
|
2443
|
+
`Exit Code: ${exitCode}\n` +
|
|
2444
|
+
`Tokens Redacted: ${Object.keys(newTokens).length}\n\n` +
|
|
2445
|
+
(resStdout.scrubbedText ? `--- STDOUT ---\n${resStdout.scrubbedText}\n\n` : '') +
|
|
2446
|
+
(resStderr.scrubbedText ? `--- STDERR ---\n${resStderr.scrubbedText}\n\n` : '') +
|
|
2447
|
+
receipt;
|
|
2448
|
+
|
|
2449
|
+
return {
|
|
2450
|
+
command: resCommand.scrubbedText,
|
|
2451
|
+
stdout: resStdout.scrubbedText,
|
|
2452
|
+
stderr: resStderr.scrubbedText,
|
|
2453
|
+
exitCode,
|
|
2454
|
+
tokenCount: Object.keys(newTokens).length,
|
|
2455
|
+
telemetry,
|
|
2456
|
+
receipt,
|
|
2457
|
+
responseText
|
|
2458
|
+
};
|
|
2459
|
+
}
|
|
2460
|
+
},
|
|
2461
|
+
|
|
2462
|
+
guardReadFile: {
|
|
2463
|
+
name: 'guard_read_file',
|
|
2464
|
+
description: 'Reads local file contents, tokenizing sensitive credentials and PII in volatile RAM before returning text to model context.',
|
|
2465
|
+
parameters: {
|
|
2466
|
+
type: 'object',
|
|
2467
|
+
properties: {
|
|
2468
|
+
filePath: { type: 'string', description: 'Path to target file.' },
|
|
2469
|
+
maxLines: { type: 'number', description: 'Maximum lines to read (default: 500).' },
|
|
2470
|
+
profile: { type: 'string', description: 'Detection profile (default: Dev).' }
|
|
2471
|
+
},
|
|
2472
|
+
required: ['filePath']
|
|
2473
|
+
},
|
|
2474
|
+
execute: async ({ filePath, maxLines = 500, profile = defaultProfile } = {}) => {
|
|
2475
|
+
if (!filePath || typeof filePath !== 'string') {
|
|
2476
|
+
throw new Error('[guard_read_file] Missing or invalid parameter: filePath must be a string.');
|
|
2477
|
+
}
|
|
2478
|
+
const resolved = path.resolve(baseCwd, filePath);
|
|
2479
|
+
if (!fs.existsSync(resolved)) {
|
|
2480
|
+
throw new Error(`[guard_read_file] File not found: ${resolved}`);
|
|
2481
|
+
}
|
|
2482
|
+
const stat = fs.statSync(resolved);
|
|
2483
|
+
if (stat.isDirectory()) {
|
|
2484
|
+
throw new Error(`[guard_read_file] Path is a directory, not a file: ${resolved}`);
|
|
2485
|
+
}
|
|
2486
|
+
|
|
2487
|
+
const raw = fs.readFileSync(resolved, 'utf8');
|
|
2488
|
+
const lines = raw.split(/\r?\n/);
|
|
2489
|
+
const capped = lines.slice(0, maxLines).join('\n');
|
|
2490
|
+
const wasTruncated = lines.length > maxLines;
|
|
2491
|
+
|
|
2492
|
+
const res = engine.sanitize(capped, { profile, detectSecrets: true, persistSession: true, generateReceipt: false });
|
|
2493
|
+
const telemetry = buildAuditTelemetry(res.tokenMap);
|
|
2494
|
+
const receipt = formatAuditReceipt(telemetry);
|
|
2495
|
+
|
|
2496
|
+
const responseText = `[Zero-Trust Agentic Guard: File Read: ${path.basename(resolved)}]\n` +
|
|
2497
|
+
`Path: ${resolved}\n` +
|
|
2498
|
+
`Total Lines: ${lines.length}${wasTruncated ? ` (Capped to ${maxLines} lines)` : ''}\n` +
|
|
2499
|
+
`Tokens Redacted: ${res.count}\n\n` +
|
|
2500
|
+
`--- SANITIZED CONTENT ---\n` +
|
|
2501
|
+
res.scrubbedText + '\n\n' +
|
|
2502
|
+
receipt;
|
|
2503
|
+
|
|
2504
|
+
return {
|
|
2505
|
+
filePath: resolved,
|
|
2506
|
+
content: res.scrubbedText,
|
|
2507
|
+
totalLines: lines.length,
|
|
2508
|
+
wasTruncated,
|
|
2509
|
+
tokenCount: res.count,
|
|
2510
|
+
telemetry,
|
|
2511
|
+
receipt,
|
|
2512
|
+
responseText
|
|
2513
|
+
};
|
|
2514
|
+
}
|
|
2515
|
+
},
|
|
2516
|
+
|
|
2517
|
+
guardApplyPatch: {
|
|
2518
|
+
name: 'guard_apply_patch',
|
|
2519
|
+
description: 'Restores authentic secrets locally from volatile RAM session map and writes real content directly to target file with automatic .bak backup.',
|
|
2520
|
+
parameters: {
|
|
2521
|
+
type: 'object',
|
|
2522
|
+
properties: {
|
|
2523
|
+
filePath: { type: 'string', description: 'Destination file path.' },
|
|
2524
|
+
content: { type: 'string', description: 'Content containing token placeholders (e.g. [API_KEY_1]).' },
|
|
2525
|
+
createBackup: { type: 'boolean', description: 'Whether to create .bak backup if file exists (default: true).' }
|
|
2526
|
+
},
|
|
2527
|
+
required: ['filePath', 'content']
|
|
2528
|
+
},
|
|
2529
|
+
execute: async ({ filePath, content, createBackup = true } = {}) => {
|
|
2530
|
+
const patchResult = applyPatch(filePath, content, {
|
|
2531
|
+
tokenMap: engine.sessionMap,
|
|
2532
|
+
createBackup
|
|
2533
|
+
});
|
|
2534
|
+
|
|
2535
|
+
const responseText = `[Zero-Trust Agentic Guard: Patch Applied]\n` +
|
|
2536
|
+
`Destination: ${patchResult.filePath}\n` +
|
|
2537
|
+
`Tokens Restored Locally: ${patchResult.restoredCount}\n` +
|
|
2538
|
+
`Backup Created: ${patchResult.backupCreated ? `${patchResult.filePath}.bak` : 'No'}\n` +
|
|
2539
|
+
`Status: File written with 100% authentic local values. Remote LLM received zero secrets.`;
|
|
2540
|
+
|
|
2541
|
+
return {
|
|
2542
|
+
...patchResult,
|
|
2543
|
+
responseText
|
|
2544
|
+
};
|
|
2545
|
+
}
|
|
2546
|
+
},
|
|
2547
|
+
|
|
2548
|
+
guardGitDiff: {
|
|
2549
|
+
name: 'guard_git_diff',
|
|
2550
|
+
description: 'Inspects staged or unstaged git diff changes with credentials and PII masked into volatile RAM tokens.',
|
|
2551
|
+
parameters: {
|
|
2552
|
+
type: 'object',
|
|
2553
|
+
properties: {
|
|
2554
|
+
staged: { type: 'boolean', description: 'Whether to review staged changes (--cached).' },
|
|
2555
|
+
cwd: { type: 'string', description: 'Repository directory.' },
|
|
2556
|
+
profile: { type: 'string', description: 'Detection profile (default: Dev).' }
|
|
2557
|
+
},
|
|
2558
|
+
required: []
|
|
2559
|
+
},
|
|
2560
|
+
execute: async ({ staged = false, cwd, profile = defaultProfile } = {}) => {
|
|
2561
|
+
const repoCwd = cwd ? path.resolve(baseCwd, cwd) : baseCwd;
|
|
2562
|
+
const gitCmd = staged ? 'git diff --cached' : 'git diff';
|
|
2563
|
+
let diffOutput = '';
|
|
2564
|
+
|
|
2565
|
+
try {
|
|
2566
|
+
diffOutput = execSync(gitCmd, { cwd: repoCwd, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 });
|
|
2567
|
+
} catch (err) {
|
|
2568
|
+
if (err.code === 'ENOBUFS' && err.stdout) {
|
|
2569
|
+
diffOutput = err.stdout.toString('utf8');
|
|
2570
|
+
} else {
|
|
2571
|
+
throw new Error(`[guard_git_diff] Git command failed: ${err.message}`);
|
|
2572
|
+
}
|
|
2573
|
+
}
|
|
2574
|
+
|
|
2575
|
+
if (!diffOutput.trim()) {
|
|
2576
|
+
return {
|
|
2577
|
+
diff: '',
|
|
2578
|
+
isStaged: !!staged,
|
|
2579
|
+
tokenCount: 0,
|
|
2580
|
+
responseText: `[Zero-Trust Agentic Guard: Git Diff]\nNo changes detected (${staged ? 'staged' : 'unstaged'}).`
|
|
2581
|
+
};
|
|
2582
|
+
}
|
|
2583
|
+
|
|
2584
|
+
const res = engine.sanitize(diffOutput, { profile, detectSecrets: true, persistSession: true, generateReceipt: false });
|
|
2585
|
+
const telemetry = buildAuditTelemetry(res.tokenMap);
|
|
2586
|
+
const receipt = formatAuditReceipt(telemetry);
|
|
2587
|
+
|
|
2588
|
+
const responseText = `[Zero-Trust Agentic Guard: Git Diff (${staged ? 'Staged' : 'Unstaged'})]\n` +
|
|
2589
|
+
`Tokens Redacted: ${res.count}\n\n` +
|
|
2590
|
+
`--- SANITIZED DIFF ---\n` +
|
|
2591
|
+
res.scrubbedText + '\n\n' +
|
|
2592
|
+
receipt;
|
|
2593
|
+
|
|
2594
|
+
return {
|
|
2595
|
+
diff: res.scrubbedText,
|
|
2596
|
+
isStaged: !!staged,
|
|
2597
|
+
tokenCount: res.count,
|
|
2598
|
+
telemetry,
|
|
2599
|
+
receipt,
|
|
2600
|
+
responseText
|
|
2601
|
+
};
|
|
2602
|
+
}
|
|
2603
|
+
}
|
|
2604
|
+
};
|
|
2605
|
+
|
|
2606
|
+
tools.getDefinitions = () => [
|
|
2607
|
+
{ type: 'function', function: { name: tools.guardExec.name, description: tools.guardExec.description, parameters: tools.guardExec.parameters } },
|
|
2608
|
+
{ type: 'function', function: { name: tools.guardReadFile.name, description: tools.guardReadFile.description, parameters: tools.guardReadFile.parameters } },
|
|
2609
|
+
{ type: 'function', function: { name: tools.guardApplyPatch.name, description: tools.guardApplyPatch.description, parameters: tools.guardApplyPatch.parameters } },
|
|
2610
|
+
{ type: 'function', function: { name: tools.guardGitDiff.name, description: tools.guardGitDiff.description, parameters: tools.guardGitDiff.parameters } }
|
|
2611
|
+
];
|
|
2612
|
+
|
|
2613
|
+
tools.engine = engine;
|
|
2614
|
+
return tools;
|
|
2615
|
+
}
|
|
2616
|
+
|
|
2617
|
+
// Low-level backwards-compatible exports
|
|
2618
|
+
const scrubText = scrubberPkg.scrubText;
|
|
2619
|
+
const unscrubText = scrubberPkg.unscrubText;
|
|
2620
|
+
|
|
2621
|
+
// Vision & Multimodal exports
|
|
2622
|
+
const {
|
|
2623
|
+
stripExifMetadata,
|
|
2624
|
+
redactImageBoundingBoxes,
|
|
2625
|
+
generateSvgRedactionOverlay
|
|
2626
|
+
} = require('./vision.cjs');
|
|
2627
|
+
|
|
2628
|
+
const _defaultExport = {
|
|
2629
|
+
sanitize,
|
|
2630
|
+
sanitizeObject,
|
|
2631
|
+
scrub,
|
|
2632
|
+
clean,
|
|
2633
|
+
restore,
|
|
2634
|
+
restoreObject,
|
|
2635
|
+
unscrub,
|
|
2636
|
+
desanitize,
|
|
2637
|
+
validateLicense,
|
|
2638
|
+
validateLicenseKey,
|
|
2639
|
+
buildAuditTelemetry,
|
|
2640
|
+
buildCisoAuditTelemetry,
|
|
2641
|
+
formatAuditReceipt,
|
|
2642
|
+
generateAuditReport,
|
|
2643
|
+
formatCefEvent,
|
|
2644
|
+
formatSyslogEvent,
|
|
2645
|
+
formatJsonlEvent,
|
|
2646
|
+
createAuditLogger,
|
|
2647
|
+
generateProcurementMemo,
|
|
2648
|
+
PrivacyScrubberEngine,
|
|
2649
|
+
PrivacyScrubberLicenseError,
|
|
2650
|
+
wrapOpenAI,
|
|
2651
|
+
wrapAiModel,
|
|
2652
|
+
wrapAiSdk,
|
|
2653
|
+
createLangChainTransform,
|
|
2654
|
+
createDocumentTransformer,
|
|
2655
|
+
createLangChainDocumentTransformer,
|
|
2656
|
+
createLlamaIndexTransform,
|
|
2657
|
+
createVectorIngestionGuard,
|
|
2658
|
+
StreamingDetokenizer,
|
|
2659
|
+
createStreamTokenRestorer,
|
|
2660
|
+
createNodeStreamTokenRestorer,
|
|
2661
|
+
buildStreamReplacer,
|
|
2662
|
+
wrapAiStream,
|
|
2663
|
+
withPrivacyScrubber,
|
|
2664
|
+
createExpressMiddleware,
|
|
2665
|
+
createSanitizeStream,
|
|
2666
|
+
applyPatch,
|
|
2667
|
+
restoreToFile,
|
|
2668
|
+
createGuardedTools,
|
|
2669
|
+
scrubText,
|
|
2670
|
+
unscrubText,
|
|
2671
|
+
DEVOPS_SECRETS_DETECTOR,
|
|
2672
|
+
stripExifMetadata,
|
|
2673
|
+
redactImageBoundingBoxes,
|
|
2674
|
+
generateSvgRedactionOverlay
|
|
2675
|
+
};
|
|
2676
|
+
|
|
2677
|
+
|
|
2678
|
+
module.exports = _defaultExport;
|
|
2679
|
+
module.exports.default = _defaultExport;
|
|
2680
|
+
for (const [key, val] of Object.entries(_defaultExport)) {
|
|
2681
|
+
module.exports[key] = val;
|
|
2682
|
+
}
|