@privacyscrubber/sdk 2.2.0 → 2.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +119 -5
- package/index.d.ts +61 -0
- package/index.js +243 -11
- package/package.json +1 -1
- package/ps-license-manager.js +1 -1
- package/ps-pii-engine.cjs +1 -1
- package/scrubber-core.cjs +1 -21
package/README.md
CHANGED
|
@@ -6,6 +6,8 @@
|
|
|
6
6
|
[](https://privacyscrubber.com)
|
|
7
7
|
[](https://www.npmjs.com/package/@privacyscrubber/sdk)
|
|
8
8
|
[](https://opensource.org/licenses/MIT)
|
|
9
|
+
[-06b6d4.svg)](https://privacyscrubber.com/patents/)
|
|
10
|
+
[](https://github.com/moxno/privacyscrubber-mcp)
|
|
9
11
|
|
|
10
12
|
> **Zero-Trust Data Sanitization (ZTDS) & PII Redaction Engine**
|
|
11
13
|
> 100% In-Memory Execution. Zero Remote Network Calls. Zero Disk Writes. Sub-millisecond Latency.
|
|
@@ -92,6 +94,7 @@ console.log(auditReceipt);
|
|
|
92
94
|
// > * **Risk Level:** 🟡 MODERATE EXPOSURE
|
|
93
95
|
// > * **Compliance Enforced:** ZTDS Standard, GDPR (Art. 4), CCPA/CPRA
|
|
94
96
|
// > * **Tokens Masked:** 2 (1 NAME, 1 PHONE)
|
|
97
|
+
// > * ⭐ **Star on GitHub:** [moxno/privacyscrubber-mcp](https://github.com/moxno/privacyscrubber-mcp) | **SDK & Enterprise:** [privacyscrubber.com/pricing](https://privacyscrubber.com/pricing)
|
|
95
98
|
|
|
96
99
|
// 2. Deterministically restore LLM response
|
|
97
100
|
const aiResponse = "I have queued an SMS notification for [NAME_1] at [PHONE_1].";
|
|
@@ -103,7 +106,99 @@ console.log(restoredText);
|
|
|
103
106
|
|
|
104
107
|
---
|
|
105
108
|
|
|
106
|
-
### 3.
|
|
109
|
+
### 3. Vercel AI SDK Integration (`generateText` & `streamText`)
|
|
110
|
+
|
|
111
|
+
Drop-in sanitization for Next.js and Node.js applications built with the Vercel AI SDK (`ai`):
|
|
112
|
+
|
|
113
|
+
```typescript
|
|
114
|
+
import { generateText } from 'ai';
|
|
115
|
+
import { openai } from '@ai-sdk/openai';
|
|
116
|
+
import { sanitize, restore } from '@privacyscrubber/sdk';
|
|
117
|
+
|
|
118
|
+
export async function POST(req: Request) {
|
|
119
|
+
const { prompt } = await req.json();
|
|
120
|
+
|
|
121
|
+
// 1. Sanitize user prompt in local RAM (<1ms)
|
|
122
|
+
const { scrubbedText, tokenMap } = sanitize(prompt, {
|
|
123
|
+
profile: 'General',
|
|
124
|
+
detectSecrets: true
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
// 2. Outbound call to LLM contains zero PII or credentials
|
|
128
|
+
const { text } = await generateText({
|
|
129
|
+
model: openai('gpt-4o'),
|
|
130
|
+
prompt: scrubbedText,
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
// 3. Deterministically restore authentic tokens locally
|
|
134
|
+
const { restoredText } = restore(text, tokenMap);
|
|
135
|
+
|
|
136
|
+
return Response.json({ response: restoredText });
|
|
137
|
+
}
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
---
|
|
141
|
+
|
|
142
|
+
### 4. LangChain (LCEL Pipelines & `createLangChainTransform`)
|
|
143
|
+
|
|
144
|
+
Use the official middleware helper or compose inside LangChain Expression Language (LCEL) chains:
|
|
145
|
+
|
|
146
|
+
```typescript
|
|
147
|
+
import { ChatOpenAI } from '@langchain/openai';
|
|
148
|
+
import { createLangChainTransform, PrivacyScrubberEngine } from '@privacyscrubber/sdk';
|
|
149
|
+
|
|
150
|
+
// Option A: 1-line middleware helper
|
|
151
|
+
const transform = createLangChainTransform({ defaultProfile: 'Healthcare' });
|
|
152
|
+
const { scrubbedText, tokenMap } = transform.preprocess("Patient John Doe MRN-99120 prescribed Lipitor.");
|
|
153
|
+
// ... invoke LLM with scrubbedText ...
|
|
154
|
+
const restoredOutput = transform.postprocess(llmOutput, tokenMap);
|
|
155
|
+
|
|
156
|
+
// Option B: Composable LCEL Runnable chain
|
|
157
|
+
const engine = new PrivacyScrubberEngine({ defaultProfile: 'Dev' });
|
|
158
|
+
const model = new ChatOpenAI({ model: 'gpt-4o', temperature: 0 });
|
|
159
|
+
|
|
160
|
+
const secureChain = async (rawQuestion: string) => {
|
|
161
|
+
const { scrubbedText, tokenMap } = engine.sanitize(rawQuestion);
|
|
162
|
+
const response = await model.invoke(scrubbedText);
|
|
163
|
+
return engine.restore(response.content as string, tokenMap);
|
|
164
|
+
};
|
|
165
|
+
|
|
166
|
+
const result = await secureChain("Debug AWS_KEY=AKIAIOSFODNN7EXAMPLE for user alice@acme.com");
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
---
|
|
170
|
+
|
|
171
|
+
### 5. LlamaIndex.TS (RAG Node Sanitizer)
|
|
172
|
+
|
|
173
|
+
Prevent PII poisoning in vector stores by sanitizing documents prior to indexing:
|
|
174
|
+
|
|
175
|
+
```typescript
|
|
176
|
+
import { Document } from 'llamaindex';
|
|
177
|
+
import { sanitize } from '@privacyscrubber/sdk';
|
|
178
|
+
|
|
179
|
+
// Sanitize documents before vector index ingestion (HIPAA & GDPR safe)
|
|
180
|
+
export function sanitizeDocumentsForIndex(docs: Document[]): Document[] {
|
|
181
|
+
return docs.map(doc => {
|
|
182
|
+
const { scrubbedText, telemetry } = sanitize(doc.text, {
|
|
183
|
+
profile: 'Healthcare', // Masks 18 HIPAA identifiers
|
|
184
|
+
detectSecrets: true
|
|
185
|
+
});
|
|
186
|
+
|
|
187
|
+
return new Document({
|
|
188
|
+
text: scrubbedText,
|
|
189
|
+
metadata: {
|
|
190
|
+
...doc.metadata,
|
|
191
|
+
psRiskLevel: telemetry.riskLevel,
|
|
192
|
+
psCompliant: telemetry.frameworksList.join(',')
|
|
193
|
+
}
|
|
194
|
+
});
|
|
195
|
+
});
|
|
196
|
+
}
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
---
|
|
200
|
+
|
|
201
|
+
### 6. RAG & Vector Database Ingestion (Pinecone, Chroma, pgvector)
|
|
107
202
|
|
|
108
203
|
Storing raw PII inside vector embeddings is irreversible and violates GDPR Article 17 (Right to be Forgotten). Sanitize document chunks prior to vectorization:
|
|
109
204
|
|
|
@@ -127,7 +222,7 @@ function prepareVectorChunk(rawDocumentChunk) {
|
|
|
127
222
|
|
|
128
223
|
---
|
|
129
224
|
|
|
130
|
-
###
|
|
225
|
+
### 7. Stateful Multi-Turn Agent Engine (`PrivacyScrubberEngine`)
|
|
131
226
|
|
|
132
227
|
Maintain token bindings across multi-step autonomous agent loops (LangChain, AutoGen, CrewAI):
|
|
133
228
|
|
|
@@ -149,7 +244,7 @@ const restored = engine.restore(agentOutputText);
|
|
|
149
244
|
|
|
150
245
|
---
|
|
151
246
|
|
|
152
|
-
###
|
|
247
|
+
### 8. High-Throughput Stream Sanitization (`createSanitizeStream`)
|
|
153
248
|
|
|
154
249
|
Sanitize massive log files, database dumps, and SSE payloads on-the-fly with <1MB heap memory. Automatically handles split chunk boundaries:
|
|
155
250
|
|
|
@@ -173,7 +268,7 @@ sanitizeStream.on('finish', () => {
|
|
|
173
268
|
|
|
174
269
|
---
|
|
175
270
|
|
|
176
|
-
###
|
|
271
|
+
### 9. AI Agent Guard & Function Calling (`createGuardedTools`, `applyPatch`)
|
|
177
272
|
|
|
178
273
|
Equip autonomous agents (Cursor, Vercel AI SDK, OpenAI, LangChain) with safe tools that intercept secrets and restore authentic data on disk:
|
|
179
274
|
|
|
@@ -202,7 +297,7 @@ const patchResult = await guardApplyPatch.execute({
|
|
|
202
297
|
|
|
203
298
|
---
|
|
204
299
|
|
|
205
|
-
###
|
|
300
|
+
### 10. Automated PR Security Verification Stamp (CI/CD)
|
|
206
301
|
|
|
207
302
|
Enforce zero plaintext PII and credential leaks in your CI/CD pipeline (GitHub Actions, GitLab CI, Jenkins) before pull requests are merged into production.
|
|
208
303
|
|
|
@@ -312,10 +407,29 @@ To acquire a commercial Developer SDK license:
|
|
|
312
407
|
- 🔌 **MCP Server**: [@privacyscrubber/mcp-server](https://www.npmjs.com/package/@privacyscrubber/mcp-server)
|
|
313
408
|
- 🧩 **Chrome Extension**: [Chrome Web Store](https://chromewebstore.google.com/detail/privacyscrubber-%E2%80%94-zero-tr/pimoejgefeilajmmbpghifdmhdlkgjol)
|
|
314
409
|
- 📖 **Developer SDK Specs**: [https://privacyscrubber.com/features/developer-sdk/](https://privacyscrubber.com/features/developer-sdk/)
|
|
410
|
+
- ⭐ **GitHub Repository**: [moxno/privacyscrubber-mcp](https://github.com/moxno/privacyscrubber-mcp) (Star us on GitHub!)
|
|
315
411
|
- 🧪 **DLP Latency Benchmark**: [https://privacyscrubber.com/dlp-speed-test/](https://privacyscrubber.com/dlp-speed-test/)
|
|
316
412
|
|
|
413
|
+
### 📚 Production Architecture Guides
|
|
414
|
+
- 🛡️ **RAG & Vector Databases**: [Sanitizing PII Before Vector DB Ingestion (Pinecone, Chroma, Qdrant)](https://privacyscrubber.com/solutions/dev/rag-vector-database-pii-masking/)
|
|
415
|
+
- 🤖 **LangChain & LlamaIndex**: [In-Memory PII Middleware for AI Agent Pipelines](https://privacyscrubber.com/solutions/agents/langchain-llamaindex-pii-anonymizer/)
|
|
416
|
+
- ⚡ **AWS Comprehend Alternative**: [In-Memory Redaction Without Egress or Cloud Overhead](https://privacyscrubber.com/solutions/dev/aws-comprehend-pii-alternative/)
|
|
417
|
+
|
|
317
418
|
---
|
|
318
419
|
|
|
319
420
|
## ⚖️ License
|
|
320
421
|
|
|
321
422
|
MIT License © 2026 PrivacyScrubber. Commercial production usage without quota limits requires an active Developer SDK or Enterprise license.
|
|
423
|
+
|
|
424
|
+
---
|
|
425
|
+
|
|
426
|
+
## 🏛️ Intellectual Property & Virtual Patent Marking
|
|
427
|
+
|
|
428
|
+
The Zero-Trust Data Sanitization (ZTDS) architecture, in-memory deterministic tokenization, cryptographic session handoff, and headless execution methods implemented in this SDK are proprietary technology of Ilya Sibiryakov (BrandMeWeb) and are protected under **Patent Pending** status:
|
|
429
|
+
|
|
430
|
+
- **Patent Office:** State of Israel Ministry of Justice, Patent Office (ILPO)
|
|
431
|
+
- **Application Number:** `331905` (Tracking ID: `94221`)
|
|
432
|
+
- **Filing / Priority Date:** September 14, 2026 (Paris Convention Art. 4 & 35 U.S.C. § 119 Priority)
|
|
433
|
+
- **Official Title:** *SYSTEM AND METHOD FOR CLIENT-SIDE ZERO-TRUST DATA SANITIZATION AND CRYPTOGRAPHIC SESSION HANDOFF IN ARTIFICIAL INTELLIGENCE WORKFLOWS*
|
|
434
|
+
- **Virtual Patent Marking:** [privacyscrubber.com/patents/](https://privacyscrubber.com/patents/) in accordance with 35 U.S.C. § 287(a).
|
|
435
|
+
|
package/index.d.ts
CHANGED
|
@@ -287,6 +287,61 @@ export function restoreToFile(filePath: string, content: string, tokenMapOrOptio
|
|
|
287
287
|
*/
|
|
288
288
|
export function createGuardedTools(engineOrOptions?: PrivacyScrubberEngine | SanitizeOptions, toolConfig?: { cwd?: string; profile?: PiiProfile }): GuardedTools;
|
|
289
289
|
|
|
290
|
+
/**
|
|
291
|
+
* Stream token restorer interface for zero-latency SSE chunk processing.
|
|
292
|
+
*/
|
|
293
|
+
export interface StreamTokenRestorer {
|
|
294
|
+
/** Pushes an incoming streaming chunk and returns immediately rehydrated text. */
|
|
295
|
+
push(chunk: string): string;
|
|
296
|
+
/** Flushes any buffered incomplete tokens at the end of the stream. */
|
|
297
|
+
flush(): string;
|
|
298
|
+
/** Returns a standard Web W3C TransformStream for piping Response streams. */
|
|
299
|
+
asTransformStream(): any;
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
/**
|
|
303
|
+
* Ergonomic 1-liner: string-in, sanitized string-out without metadata ceremonies.
|
|
304
|
+
* @param text - Plaintext prompt containing potential PII/secrets.
|
|
305
|
+
* @param profile - Compliance profile (Default: 'General').
|
|
306
|
+
* @returns Scrubbed string ready to send to LLM.
|
|
307
|
+
*/
|
|
308
|
+
export function clean(text: string, profile?: PiiProfile): string;
|
|
309
|
+
|
|
310
|
+
/**
|
|
311
|
+
* Creates a stream token restorer that rehydrates masked tokens across chunk boundaries in real-time.
|
|
312
|
+
* @param tokenMap - Map of token bindings returned from sanitize().
|
|
313
|
+
*/
|
|
314
|
+
export function createStreamTokenRestorer(tokenMap?: Record<string, string>): StreamTokenRestorer;
|
|
315
|
+
|
|
316
|
+
/**
|
|
317
|
+
* Wraps an AI streaming response (e.g. OpenAI stream, Vercel AI SDK textStream)
|
|
318
|
+
* and yields real-time plain chunks with all tokens restored on the fly.
|
|
319
|
+
* @param stream - AsyncIterable or ReadableStream of string chunks or delta objects.
|
|
320
|
+
* @param tokenMap - Map of token bindings returned from sanitize().
|
|
321
|
+
*/
|
|
322
|
+
export function wrapAiStream<T = any>(
|
|
323
|
+
stream: AsyncIterable<T> | any,
|
|
324
|
+
tokenMap: Record<string, string>
|
|
325
|
+
): AsyncGenerator<string, void, unknown>;
|
|
326
|
+
|
|
327
|
+
/**
|
|
328
|
+
* Zero-Trust wrapper for Next.js App Router Route Handlers (route.js / route.ts).
|
|
329
|
+
* Automatically intercepts incoming Request JSON body, sanitizes PII in RAM,
|
|
330
|
+
* and passes scrubbed body along with tokenMap and stream restorer factory.
|
|
331
|
+
*/
|
|
332
|
+
export function withPrivacyScrubber(
|
|
333
|
+
handler: (req: Request, context?: any, psContext?: { tokenMap: Record<string, string>; telemetry: AuditTelemetry; createStreamRestorer: () => StreamTokenRestorer }) => any,
|
|
334
|
+
options?: SanitizeOptions
|
|
335
|
+
): (req: Request, context?: any) => Promise<any>;
|
|
336
|
+
|
|
337
|
+
/**
|
|
338
|
+
* Express.js / Connect middleware for automatic zero-trust PII sanitization.
|
|
339
|
+
* Sanitizes req.body in local RAM and attaches req.psTokenMap, req.restore(), and req.createStreamRestorer().
|
|
340
|
+
*/
|
|
341
|
+
export function createExpressMiddleware(
|
|
342
|
+
options?: SanitizeOptions
|
|
343
|
+
): (req: any, res: any, next: (err?: any) => void) => void;
|
|
344
|
+
|
|
290
345
|
// Legacy exports
|
|
291
346
|
export function scrubText(
|
|
292
347
|
text: string,
|
|
@@ -311,10 +366,15 @@ export function unscrubText(
|
|
|
311
366
|
};
|
|
312
367
|
|
|
313
368
|
declare const defaultExport: {
|
|
369
|
+
clean: typeof clean;
|
|
314
370
|
sanitize: typeof sanitize;
|
|
315
371
|
scrub: typeof scrub;
|
|
316
372
|
restore: typeof restore;
|
|
317
373
|
unscrub: typeof unscrub;
|
|
374
|
+
createStreamTokenRestorer: typeof createStreamTokenRestorer;
|
|
375
|
+
wrapAiStream: typeof wrapAiStream;
|
|
376
|
+
withPrivacyScrubber: typeof withPrivacyScrubber;
|
|
377
|
+
createExpressMiddleware: typeof createExpressMiddleware;
|
|
318
378
|
validateLicense: typeof validateLicense;
|
|
319
379
|
validateLicenseKey: typeof validateLicenseKey;
|
|
320
380
|
buildAuditTelemetry: typeof buildAuditTelemetry;
|
|
@@ -332,3 +392,4 @@ declare const defaultExport: {
|
|
|
332
392
|
};
|
|
333
393
|
|
|
334
394
|
export default defaultExport;
|
|
395
|
+
|
package/index.js
CHANGED
|
@@ -164,7 +164,8 @@ export function buildAuditTelemetry(currentTokenMap = {}) {
|
|
|
164
164
|
*/
|
|
165
165
|
export function formatAuditReceipt(telemetry) {
|
|
166
166
|
if (!telemetry || telemetry.totalCount === 0) {
|
|
167
|
-
return "> 🛡️ **PrivacyScrubber Audit Receipt**: CLEAN (ZERO PII DETECTED)\n"
|
|
167
|
+
return "> 🛡️ **PrivacyScrubber Audit Receipt**: CLEAN (ZERO PII DETECTED)\n" +
|
|
168
|
+
"> * ⭐ **Star on GitHub:** [moxno/privacyscrubber-mcp](https://github.com/moxno/privacyscrubber-mcp) | **SDK & Enterprise:** [privacyscrubber.com/pricing](https://privacyscrubber.com/pricing)\n";
|
|
168
169
|
}
|
|
169
170
|
const entitiesList = Object.entries(telemetry.entities)
|
|
170
171
|
.map(([type, count]) => `${count} ${type}`)
|
|
@@ -175,22 +176,42 @@ export function formatAuditReceipt(telemetry) {
|
|
|
175
176
|
return `> 🛡️ **PrivacyScrubber Audit Receipt**\n` +
|
|
176
177
|
`> * **Risk Level:** ${icon} ${telemetry.riskLevel}\n` +
|
|
177
178
|
`> * **Compliance Enforced:** ${telemetry.frameworksList.join(', ')}\n` +
|
|
178
|
-
`> * **Tokens Masked:** ${telemetry.totalCount} (${entitiesList})\n
|
|
179
|
+
`> * **Tokens Masked:** ${telemetry.totalCount} (${entitiesList})\n` +
|
|
180
|
+
`> * ⭐ **Star on GitHub:** [moxno/privacyscrubber-mcp](https://github.com/moxno/privacyscrubber-mcp) | **SDK & Enterprise:** [privacyscrubber.com/pricing](https://privacyscrubber.com/pricing)\n`;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
let communityNoticeLogged = false;
|
|
184
|
+
|
|
185
|
+
function emitCommunityNoticeIfNeeded(isPro) {
|
|
186
|
+
if (communityNoticeLogged || isPro) return;
|
|
187
|
+
if (typeof process !== 'undefined' && process.env?.NODE_ENV === 'production') {
|
|
188
|
+
communityNoticeLogged = true;
|
|
189
|
+
console.info('[@privacyscrubber/sdk] Running Community Edition (Core PII).\nTo unlock 25 Enterprise Profiles & DevOps Secrets in production: https://privacyscrubber.com/pricing/?tier=sdk');
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* Resets the community notice state for testing purposes.
|
|
195
|
+
* @private
|
|
196
|
+
*/
|
|
197
|
+
export function _resetCommunityNoticeForTesting() {
|
|
198
|
+
communityNoticeLogged = false;
|
|
179
199
|
}
|
|
180
200
|
|
|
181
201
|
/**
|
|
182
|
-
*
|
|
202
|
+
* Sanitizes input text, masking detected PII & secrets with reversible tokens.
|
|
183
203
|
*
|
|
184
|
-
* @param {string} text - Raw input text
|
|
185
|
-
* @param {object} [options={}] -
|
|
186
|
-
* @param {string} [options.profile='General'] -
|
|
204
|
+
* @param {string} text - Raw input text.
|
|
205
|
+
* @param {object} [options={}] - Sanitization options.
|
|
206
|
+
* @param {string} [options.profile='General'] - Compliance detection profile.
|
|
187
207
|
* @param {string} [options.licenseKey] - Optional license key. Falls back to process.env.PRIVACYSCRUBBER_KEY.
|
|
188
|
-
* @param {Array} [options.customRules=[]] - Custom regex rules
|
|
189
|
-
* @param {Record<string, string>} [options.tokenLabelMap={}] - Custom label
|
|
190
|
-
* @param {Record<string, string>} [options.existingSessionMap={}] -
|
|
191
|
-
* @param {boolean} [options.generateReceipt=true] -
|
|
208
|
+
* @param {Array<{name: string, pattern: string, regex: RegExp, label?: string}>} [options.customRules=[]] - Custom regex rules.
|
|
209
|
+
* @param {Record<string, string>} [options.tokenLabelMap={}] - Custom label mapping.
|
|
210
|
+
* @param {Record<string, string>} [options.existingSessionMap={}] - In-memory session map to preserve token IDs across turns.
|
|
211
|
+
* @param {boolean} [options.generateReceipt=true] - Generate a CISO Markdown audit receipt.
|
|
192
212
|
* @param {boolean} [options.throwOnLimit=true] - Throw PrivacyScrubberLicenseError when free tier limit is hit.
|
|
193
|
-
* @param {boolean} [options.detectSecrets=false] - Auto-
|
|
213
|
+
* @param {boolean} [options.detectSecrets=false] - Auto-detect cloud credentials and API keys.
|
|
214
|
+
* @param {Set<string>|Array<string>} [options.ignoreList=null] - Values to preserve in cleartext.
|
|
194
215
|
* @returns {{
|
|
195
216
|
* scrubbedText: string,
|
|
196
217
|
* tokenMap: Record<string, string>,
|
|
@@ -230,6 +251,8 @@ export function sanitize(text, options = {}) {
|
|
|
230
251
|
const isPro = licenseCheck.valid;
|
|
231
252
|
const tier = isPro ? licenseCheck.tier : 'FREE';
|
|
232
253
|
|
|
254
|
+
emitCommunityNoticeIfNeeded(isPro);
|
|
255
|
+
|
|
233
256
|
const isSpecialized = profile && profile.toLowerCase() !== 'general';
|
|
234
257
|
const limit = isPro ? Infinity : (isSpecialized ? 5000 : 15000);
|
|
235
258
|
|
|
@@ -306,6 +329,18 @@ export function sanitize(text, options = {}) {
|
|
|
306
329
|
*/
|
|
307
330
|
export const scrub = sanitize;
|
|
308
331
|
|
|
332
|
+
/**
|
|
333
|
+
* Ergonomic 1-liner: Sanitizes text and returns the scrubbed string directly.
|
|
334
|
+
* @param {string} text - Input text.
|
|
335
|
+
* @param {string} [profile='General'] - Target compliance profile.
|
|
336
|
+
* @returns {string} - Scrubbed string.
|
|
337
|
+
*/
|
|
338
|
+
export function clean(text, profile = 'General') {
|
|
339
|
+
if (!text || typeof text !== 'string') return '';
|
|
340
|
+
const { scrubbedText } = sanitize(text, { profile });
|
|
341
|
+
return scrubbedText;
|
|
342
|
+
}
|
|
343
|
+
|
|
309
344
|
/**
|
|
310
345
|
* Primary High-Level Function: Restore / Unscrub AI response back to original entities in-memory.
|
|
311
346
|
*
|
|
@@ -490,6 +525,198 @@ export function createLangChainTransform(options = {}) {
|
|
|
490
525
|
};
|
|
491
526
|
}
|
|
492
527
|
|
|
528
|
+
/**
|
|
529
|
+
* Creates a zero-latency streaming token restorer for LLM Server-Sent Events (SSE) and Web Streams.
|
|
530
|
+
* Solves the broken chunk boundary problem where tokens like [EMAIL_1] are split across multiple chunks.
|
|
531
|
+
*
|
|
532
|
+
* Works with:
|
|
533
|
+
* - Vercel AI SDK (streamText / createDataStreamResponse)
|
|
534
|
+
* - OpenAI streaming (chat.completions.create({ stream: true }))
|
|
535
|
+
* - LangChain streaming
|
|
536
|
+
* - Native Web Streams (ReadableStream.pipeThrough)
|
|
537
|
+
* - Async generators (for await ... of)
|
|
538
|
+
*
|
|
539
|
+
* @param {Record<string, string>} tokenMap - Volatile token mapping object.
|
|
540
|
+
* @param {object} [options] - Configuration options.
|
|
541
|
+
* @returns {TransformStream & { process: (chunk: string) => string, flush: () => string, wrapStream: (asyncIterable: AsyncIterable<string>) => AsyncGenerator<string> }}
|
|
542
|
+
*/
|
|
543
|
+
export function createStreamTokenRestorer(tokenMap = {}, options = {}) {
|
|
544
|
+
let pendingBuffer = '';
|
|
545
|
+
const map = tokenMap && typeof tokenMap === 'object' ? tokenMap : {};
|
|
546
|
+
|
|
547
|
+
function directRestore(text) {
|
|
548
|
+
if (!text) return '';
|
|
549
|
+
let result = text;
|
|
550
|
+
for (const token in map) {
|
|
551
|
+
if (result.includes(token)) {
|
|
552
|
+
result = result.split(token).join(map[token]);
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
return result;
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
function processChunk(chunk) {
|
|
559
|
+
if (!chunk || typeof chunk !== 'string') return '';
|
|
560
|
+
pendingBuffer += chunk;
|
|
561
|
+
|
|
562
|
+
// Check if there is an unclosed potential token bracket at the tail of the buffer.
|
|
563
|
+
// Token format: [TYPE_INDEX] or [CUSTOM_NAME] e.g. [EMAIL_1], [NAME_2], [SECRET_1]
|
|
564
|
+
const lastOpenIdx = pendingBuffer.lastIndexOf('[');
|
|
565
|
+
const lastCloseIdx = pendingBuffer.lastIndexOf(']');
|
|
566
|
+
|
|
567
|
+
if (lastOpenIdx !== -1 && lastOpenIdx > lastCloseIdx) {
|
|
568
|
+
const partial = pendingBuffer.slice(lastOpenIdx);
|
|
569
|
+
if (/^\[[A-Za-z0-9_]*$/.test(partial) && partial.length <= 64) {
|
|
570
|
+
const safeToEmit = pendingBuffer.slice(0, lastOpenIdx);
|
|
571
|
+
pendingBuffer = partial;
|
|
572
|
+
if (safeToEmit) {
|
|
573
|
+
return directRestore(safeToEmit);
|
|
574
|
+
}
|
|
575
|
+
return '';
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
const textToRestore = pendingBuffer;
|
|
580
|
+
pendingBuffer = '';
|
|
581
|
+
return directRestore(textToRestore);
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
function flushBuffer() {
|
|
585
|
+
if (!pendingBuffer) return '';
|
|
586
|
+
const remaining = pendingBuffer;
|
|
587
|
+
pendingBuffer = '';
|
|
588
|
+
return directRestore(remaining);
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
async function* wrapStream(asyncIterable) {
|
|
592
|
+
for await (const chunk of asyncIterable) {
|
|
593
|
+
const text = typeof chunk === 'string' ? chunk : (chunk?.text || chunk?.content || '');
|
|
594
|
+
const emitted = processChunk(text);
|
|
595
|
+
if (emitted) yield emitted;
|
|
596
|
+
}
|
|
597
|
+
const finalChunk = flushBuffer();
|
|
598
|
+
if (finalChunk) yield finalChunk;
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
let stream;
|
|
602
|
+
if (typeof TransformStream !== 'undefined') {
|
|
603
|
+
stream = new TransformStream({
|
|
604
|
+
transform(chunk, controller) {
|
|
605
|
+
const text = typeof chunk === 'string' ? chunk : new TextDecoder().decode(chunk);
|
|
606
|
+
const out = processChunk(text);
|
|
607
|
+
if (out) controller.enqueue(out);
|
|
608
|
+
},
|
|
609
|
+
flush(controller) {
|
|
610
|
+
const out = flushBuffer();
|
|
611
|
+
if (out) controller.enqueue(out);
|
|
612
|
+
}
|
|
613
|
+
});
|
|
614
|
+
} else {
|
|
615
|
+
stream = {};
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
stream.process = processChunk;
|
|
619
|
+
stream.flush = flushBuffer;
|
|
620
|
+
stream.wrapStream = wrapStream;
|
|
621
|
+
|
|
622
|
+
return stream;
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
/**
|
|
626
|
+
* Wraps an async iterable LLM stream (e.g. from Vercel AI SDK textStream or OpenAI stream)
|
|
627
|
+
* and yields de-tokenized text chunks in real-time, seamlessly stitching split tokens across chunks.
|
|
628
|
+
*
|
|
629
|
+
* @param {AsyncIterable<string|object>} stream - The incoming LLM stream.
|
|
630
|
+
* @param {Record<string, string>} tokenMap - Volatile token map.
|
|
631
|
+
* @returns {AsyncGenerator<string>}
|
|
632
|
+
*/
|
|
633
|
+
export async function* wrapAiStream(stream, tokenMap = {}) {
|
|
634
|
+
const restorer = createStreamTokenRestorer(tokenMap);
|
|
635
|
+
yield* restorer.wrapStream(stream);
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
/**
|
|
639
|
+
* Middleware wrapper for Next.js App Router Route Handlers (POST/PUT).
|
|
640
|
+
* Automatically sanitizes incoming prompt payloads in RAM (<1ms) before the handler runs,
|
|
641
|
+
* and attaches sanitized text, tokenMap, and helper restoration functions.
|
|
642
|
+
*
|
|
643
|
+
* @param {Function} handler - The async Next.js route handler (req, context) => Response
|
|
644
|
+
* @param {object} [options] - Sanitization options (profile, detectSecrets, etc.)
|
|
645
|
+
* @returns {Function} Wrapped Next.js route handler
|
|
646
|
+
*/
|
|
647
|
+
export function withPrivacyScrubber(handler, options = {}) {
|
|
648
|
+
return async function privacyScrubberHandler(req, context = {}) {
|
|
649
|
+
if (!req || typeof req.json !== 'function') {
|
|
650
|
+
return handler(req, context);
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
let rawBody;
|
|
654
|
+
try {
|
|
655
|
+
rawBody = await req.json();
|
|
656
|
+
} catch (_) {
|
|
657
|
+
return handler(req, context);
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
let scrubbedBody = rawBody;
|
|
661
|
+
let tokenMap = {};
|
|
662
|
+
let telemetry = null;
|
|
663
|
+
|
|
664
|
+
if (typeof rawBody === 'string') {
|
|
665
|
+
const res = sanitize(rawBody, options);
|
|
666
|
+
scrubbedBody = res.scrubbedText;
|
|
667
|
+
tokenMap = res.tokenMap;
|
|
668
|
+
telemetry = res.telemetry;
|
|
669
|
+
} else if (rawBody && typeof rawBody === 'object') {
|
|
670
|
+
const serialized = JSON.stringify(rawBody);
|
|
671
|
+
const res = sanitize(serialized, options);
|
|
672
|
+
try {
|
|
673
|
+
scrubbedBody = JSON.parse(res.scrubbedText);
|
|
674
|
+
} catch (_) {
|
|
675
|
+
scrubbedBody = rawBody;
|
|
676
|
+
}
|
|
677
|
+
tokenMap = res.tokenMap;
|
|
678
|
+
telemetry = res.telemetry;
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
const psContext = {
|
|
682
|
+
...context,
|
|
683
|
+
rawBody,
|
|
684
|
+
scrubbedBody,
|
|
685
|
+
tokenMap,
|
|
686
|
+
telemetry,
|
|
687
|
+
restore: (text) => restore(text, tokenMap),
|
|
688
|
+
createStreamRestorer: () => createStreamTokenRestorer(tokenMap)
|
|
689
|
+
};
|
|
690
|
+
|
|
691
|
+
return handler(req, psContext);
|
|
692
|
+
};
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
/**
|
|
696
|
+
* Express.js / Connect middleware for automatic zero-trust PII sanitization.
|
|
697
|
+
* Sanitizes req.body in local RAM and attaches req.psTokenMap and req.restore(aiResponse).
|
|
698
|
+
*
|
|
699
|
+
* @param {object} [options] - Sanitization options.
|
|
700
|
+
* @returns {Function} Express middleware (req, res, next)
|
|
701
|
+
*/
|
|
702
|
+
export function createExpressMiddleware(options = {}) {
|
|
703
|
+
return function privacyScrubberExpress(req, res, next) {
|
|
704
|
+
if (req && req.body && typeof req.body === 'object') {
|
|
705
|
+
const serialized = JSON.stringify(req.body);
|
|
706
|
+
const { scrubbedText, tokenMap, telemetry } = sanitize(serialized, options);
|
|
707
|
+
req.rawBody = req.body;
|
|
708
|
+
try {
|
|
709
|
+
req.body = JSON.parse(scrubbedText);
|
|
710
|
+
} catch (_) {}
|
|
711
|
+
req.psTokenMap = tokenMap;
|
|
712
|
+
req.psTelemetry = telemetry;
|
|
713
|
+
req.restore = (aiResponse) => restore(aiResponse, tokenMap);
|
|
714
|
+
req.createStreamRestorer = () => createStreamTokenRestorer(tokenMap);
|
|
715
|
+
}
|
|
716
|
+
if (typeof next === 'function') next();
|
|
717
|
+
};
|
|
718
|
+
}
|
|
719
|
+
|
|
493
720
|
/**
|
|
494
721
|
* Generates an official Zero-Trust Compliance Audit Report.
|
|
495
722
|
* @param {Record<string, string>} sessionMap - Map of token bindings.
|
|
@@ -921,6 +1148,7 @@ export const unscrubText = scrubberPkg.unscrubText;
|
|
|
921
1148
|
export default {
|
|
922
1149
|
sanitize,
|
|
923
1150
|
scrub,
|
|
1151
|
+
clean,
|
|
924
1152
|
restore,
|
|
925
1153
|
unscrub,
|
|
926
1154
|
validateLicense,
|
|
@@ -932,6 +1160,10 @@ export default {
|
|
|
932
1160
|
PrivacyScrubberLicenseError,
|
|
933
1161
|
wrapOpenAI,
|
|
934
1162
|
createLangChainTransform,
|
|
1163
|
+
createStreamTokenRestorer,
|
|
1164
|
+
wrapAiStream,
|
|
1165
|
+
withPrivacyScrubber,
|
|
1166
|
+
createExpressMiddleware,
|
|
935
1167
|
createSanitizeStream,
|
|
936
1168
|
applyPatch,
|
|
937
1169
|
restoreToFile,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@privacyscrubber/sdk",
|
|
3
|
-
"version": "2.2.
|
|
3
|
+
"version": "2.2.1",
|
|
4
4
|
"description": "Zero-dependency, in-memory PII & secrets redaction SDK for Node.js, Next.js, OpenAI, and LangChain. 100% client-side & server-side zero-trust prompt anonymization under 1ms.",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"types": "index.d.ts",
|