@privacyscrubber/sdk 2.2.1 → 2.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -1
- package/cli.js +166 -19
- package/index.d.ts +67 -6
- package/index.js +444 -67
- package/package.json +1 -1
- package/ps-license-manager.js +1 -1
- package/ps-pii-engine.cjs +1 -1
- package/scrubber-core.cjs +1 -1
package/README.md
CHANGED
|
@@ -28,7 +28,7 @@ The official Node.js / TypeScript programmatic SDK for [PrivacyScrubber.com](htt
|
|
|
28
28
|
| **DevOps Secrets Scanning** | **Built-in (AWS, JWT, DBs)** | Custom regex rules | Custom detectors | Custom classifiers |
|
|
29
29
|
| **Cost Predictability** | **Free Tier / Flat $199/mo** | DevOps server maintenance | Pay-per-GB cloud fees | Pay-per-unit API fees |
|
|
30
30
|
|
|
31
|
-
> 📊 **Full Technical Benchmark**: Read the in-depth latency analysis, cold-start profiles, and architecture deep-dive in [docs/benchmarks/sdk-vs-presidio.md](https://github.com/moxno/
|
|
31
|
+
> 📊 **Full Technical Benchmark**: Read the in-depth latency analysis, cold-start profiles, and architecture deep-dive in [docs/benchmarks/sdk-vs-presidio.md](https://github.com/moxno/privacyscrubber-mcp/blob/main/docs/benchmarks/sdk-vs-presidio.md).
|
|
32
32
|
|
|
33
33
|
### 💡 The Breach Math vs SDK Math
|
|
34
34
|
|
|
@@ -139,6 +139,64 @@ export async function POST(req: Request) {
|
|
|
139
139
|
|
|
140
140
|
---
|
|
141
141
|
|
|
142
|
+
### 3b. Real-Time LLM Streaming De-tokenization (`wrapAiStream`, `createStreamTokenRestorer`, `createNodeStreamTokenRestorer`)
|
|
143
|
+
|
|
144
|
+
When LLMs stream responses via Server-Sent Events (SSE) or WebSockets, tokens can be split across arbitrary chunk boundaries (e.g. Chunk 1: `"Contact [EM"`, Chunk 2: `"AIL_1] at once"`). `StreamingDetokenizer` uses a zero-latency sliding window buffer to stitch and restore tokens on the fly without delaying stream output:
|
|
145
|
+
|
|
146
|
+
```typescript
|
|
147
|
+
import { streamText } from 'ai';
|
|
148
|
+
import { openai } from '@ai-sdk/openai';
|
|
149
|
+
import { sanitize, wrapAiStream } from '@privacyscrubber/sdk';
|
|
150
|
+
|
|
151
|
+
export async function POST(req: Request) {
|
|
152
|
+
const { prompt } = await req.json();
|
|
153
|
+
|
|
154
|
+
// 1. Sanitize prompt in RAM
|
|
155
|
+
const { scrubbedText, tokenMap } = sanitize(prompt, { profile: 'General' });
|
|
156
|
+
|
|
157
|
+
// 2. Stream from LLM
|
|
158
|
+
const { textStream } = streamText({
|
|
159
|
+
model: openai('gpt-4o'),
|
|
160
|
+
prompt: scrubbedText,
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
// 3. Wrap stream — split tokens ([EM + AIL_1]) are rehydrated in real-time!
|
|
164
|
+
const restoredStream = wrapAiStream(textStream, tokenMap);
|
|
165
|
+
|
|
166
|
+
// 4. Return SSE stream to client with authentic values restored
|
|
167
|
+
return new Response(restoredStream);
|
|
168
|
+
}
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
#### Transparent OpenAI Streaming (`wrapOpenAI` with `stream: true`):
|
|
172
|
+
```typescript
|
|
173
|
+
import OpenAI from 'openai';
|
|
174
|
+
import { wrapOpenAI } from '@privacyscrubber/sdk';
|
|
175
|
+
|
|
176
|
+
const client = wrapOpenAI(new OpenAI(), { profile: 'Dev' });
|
|
177
|
+
|
|
178
|
+
const stream = await client.chat.completions.create({
|
|
179
|
+
model: 'gpt-4o',
|
|
180
|
+
messages: [{ role: 'user', content: 'Debug AWS_KEY=AKIAIOSFODNN7EXAMPLE for user alice@acme.com' }],
|
|
181
|
+
stream: true, // Transparent real-time chunk de-tokenization!
|
|
182
|
+
});
|
|
183
|
+
|
|
184
|
+
for await (const chunk of stream) {
|
|
185
|
+
process.stdout.write(chunk.choices[0]?.delta?.content || '');
|
|
186
|
+
}
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
#### Node.js Pipeline Streaming (`createNodeStreamTokenRestorer`):
|
|
190
|
+
```javascript
|
|
191
|
+
import { pipeline } from 'node:stream/promises';
|
|
192
|
+
import { createNodeStreamTokenRestorer } from '@privacyscrubber/sdk';
|
|
193
|
+
|
|
194
|
+
const restorer = createNodeStreamTokenRestorer(tokenMap);
|
|
195
|
+
await pipeline(readableLlmStream, restorer, clientResponseStream);
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
---
|
|
199
|
+
|
|
142
200
|
### 4. LangChain (LCEL Pipelines & `createLangChainTransform`)
|
|
143
201
|
|
|
144
202
|
Use the official middleware helper or compose inside LangChain Expression Language (LCEL) chains:
|
package/cli.js
CHANGED
|
@@ -13,7 +13,7 @@ import fs from 'node:fs';
|
|
|
13
13
|
import path from 'node:path';
|
|
14
14
|
import crypto from 'node:crypto';
|
|
15
15
|
import { fileURLToPath } from 'node:url';
|
|
16
|
-
import { sanitize } from './index.js';
|
|
16
|
+
import { sanitize, restore, generateProcurementMemo, StreamingDetokenizer } from './index.js';
|
|
17
17
|
|
|
18
18
|
const __filename = fileURLToPath(import.meta.url);
|
|
19
19
|
const __dirname = path.dirname(__filename);
|
|
@@ -21,7 +21,7 @@ const __dirname = path.dirname(__filename);
|
|
|
21
21
|
const args = process.argv.slice(2);
|
|
22
22
|
|
|
23
23
|
if (args.includes('--version') || args.includes('-v')) {
|
|
24
|
-
let version = '2.2.
|
|
24
|
+
let version = '2.2.2';
|
|
25
25
|
try {
|
|
26
26
|
const pkg = JSON.parse(fs.readFileSync(path.join(__dirname, 'package.json'), 'utf8'));
|
|
27
27
|
version = pkg.version || version;
|
|
@@ -30,6 +30,76 @@ if (args.includes('--version') || args.includes('-v')) {
|
|
|
30
30
|
process.exit(0);
|
|
31
31
|
}
|
|
32
32
|
|
|
33
|
+
// ── Memo Command: Generate CISO / DPO Procurement Memorandum ──────
|
|
34
|
+
if (args.includes('memo') || args.includes('--memo') || args.includes('--procurement-memo')) {
|
|
35
|
+
let format = 'markdown';
|
|
36
|
+
const formatIdx = args.indexOf('--format');
|
|
37
|
+
if (formatIdx !== -1 && args[formatIdx + 1]) {
|
|
38
|
+
format = args[formatIdx + 1];
|
|
39
|
+
}
|
|
40
|
+
let appName = 'Internal AI Applications & LLM Pipelines';
|
|
41
|
+
const appIdx = args.indexOf('--app');
|
|
42
|
+
if (appIdx !== -1 && args[appIdx + 1]) {
|
|
43
|
+
appName = args[appIdx + 1];
|
|
44
|
+
}
|
|
45
|
+
let developer = 'Engineering / AI Platform Team';
|
|
46
|
+
const devIdx = args.indexOf('--developer') !== -1 ? args.indexOf('--developer') : args.indexOf('--dev');
|
|
47
|
+
if (devIdx !== -1 && args[devIdx + 1]) {
|
|
48
|
+
developer = args[devIdx + 1];
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const memo = generateProcurementMemo({ appName, developer, format });
|
|
52
|
+
process.stdout.write(memo.endsWith('\n') ? memo : memo + '\n');
|
|
53
|
+
process.exit(0);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// ── Demo Command: Live In-Memory Benchmark & Zero-Trust Inspection ──
|
|
57
|
+
if (args.includes('demo') || args.includes('--demo')) {
|
|
58
|
+
const sampleInput = `DevOps Incident Report #4829
|
|
59
|
+
Reporter: Sarah Connor (sarah.connor@cyberdyne.io, +1-415-555-0199, SSN 987-65-4321)
|
|
60
|
+
Host: prod-api-04.us-east-1.internal (10.0.42.15)
|
|
61
|
+
AWS Credentials: AKIAIOSFODNN7EXAMPLE / wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
|
|
62
|
+
Stripe API Key: sk_live_51OzTestDummyStripeKey998877665544332211
|
|
63
|
+
Database URI: postgresql://admin:SuperSecretPassword123@db-prod.internal:5432/production
|
|
64
|
+
Status: Database connection timeout during batch export.`;
|
|
65
|
+
|
|
66
|
+
const start = process.hrtime.bigint();
|
|
67
|
+
const res = sanitize(sampleInput, { profile: 'Dev', detectSecrets: true });
|
|
68
|
+
const end = process.hrtime.bigint();
|
|
69
|
+
const durationUs = Number(end - start) / 1000;
|
|
70
|
+
const durationMs = (durationUs / 1000).toFixed(3);
|
|
71
|
+
|
|
72
|
+
const maskedCount = Object.keys(res.tokenMap || {}).length;
|
|
73
|
+
const border = '═'.repeat(66);
|
|
74
|
+
const thin = '─'.repeat(66);
|
|
75
|
+
|
|
76
|
+
console.log(`
|
|
77
|
+
╔${border}╗
|
|
78
|
+
║ PrivacyScrubber Developer SDK — Live In-Memory Benchmark ║
|
|
79
|
+
║ Standard: Zero-Trust Data Sanitization (ZTDS) ║
|
|
80
|
+
╠${border}╣
|
|
81
|
+
║ Input Characters : ${sampleInput.length.toString().padEnd(44)} ║
|
|
82
|
+
║ Protected Tokens : ${maskedCount.toString().padEnd(44)} ║
|
|
83
|
+
║ Execution Latency : ${(durationMs + ' ms (' + durationUs.toFixed(0) + ' µs)').padEnd(44)} ║
|
|
84
|
+
║ Network Egress : 0 bytes (Pure in-memory process RAM) ║
|
|
85
|
+
║ Risk Level : ${(res.telemetry?.riskLevel || 'CLEAN').padEnd(44)} ║
|
|
86
|
+
║ Frameworks : ${(res.telemetry?.frameworksList?.slice(0, 3).join(', ') || 'GDPR, SOC 2, NIST').padEnd(44)} ║
|
|
87
|
+
╟${thin}╢
|
|
88
|
+
║ RAW INPUT PREVIEW: ║
|
|
89
|
+
${sampleInput.split('\n').map(l => `║ ${l.slice(0, 62).padEnd(64)}║`).join('\n')}
|
|
90
|
+
╟${thin}╢
|
|
91
|
+
║ SANITIZED OUTPUT (Safe for LLM Egress): ║
|
|
92
|
+
${res.scrubbedText.split('\n').map(l => `║ ${l.slice(0, 62).padEnd(64)}║`).join('\n')}
|
|
93
|
+
╟${thin}╢
|
|
94
|
+
║ NEXT STEPS & PROCUREMENT: ║
|
|
95
|
+
║ * Copy CISO Approval Memo : npx @privacyscrubber/sdk memo ║
|
|
96
|
+
║ * CI/CD PR Stamp : cat prompt.txt | sdk --verify-ci ║
|
|
97
|
+
║ * Unlimited SDK ($199/mo) : https://privacyscrubber.com/pricing ║
|
|
98
|
+
╚${border}╝
|
|
99
|
+
`);
|
|
100
|
+
process.exit(0);
|
|
101
|
+
}
|
|
102
|
+
|
|
33
103
|
if (args.includes('--help') || args.includes('-h') || (args.length === 0 && process.stdin.isTTY)) {
|
|
34
104
|
console.log(`@privacyscrubber/sdk — Zero-Trust Data Sanitization & CI Security Verification
|
|
35
105
|
|
|
@@ -37,18 +107,37 @@ Usage:
|
|
|
37
107
|
npx @privacyscrubber/sdk <fileOrText> [options]
|
|
38
108
|
cat prompt.txt | npx @privacyscrubber/sdk [options]
|
|
39
109
|
npx @privacyscrubber/sdk --verify-ci <fileOrText> [options]
|
|
110
|
+
npx @privacyscrubber/sdk memo [options]
|
|
111
|
+
npx @privacyscrubber/sdk demo
|
|
112
|
+
|
|
113
|
+
Commands:
|
|
114
|
+
memo Generate executive CISO & DPO procurement memo
|
|
115
|
+
demo Run live in-memory benchmark (<0.1ms latency check)
|
|
40
116
|
|
|
41
117
|
Options:
|
|
42
|
-
--verify-ci Generate CI/CD PR verification stamp
|
|
43
|
-
--
|
|
118
|
+
--verify-ci Generate CI/CD PR verification stamp (audit receipt)
|
|
119
|
+
--json Output structured JSON (scrubbedText, tokenMap, telemetry)
|
|
120
|
+
--restore, --unscrub Restore masked tokens in text back to original values
|
|
121
|
+
--stream Stream de-tokenization chunks in real-time from stdin
|
|
122
|
+
--token-map <path> Path to JSON file or inline JSON string of token mappings
|
|
123
|
+
--profile <name> Specify detection profile (default: Dev, 30 profiles available)
|
|
44
124
|
--key <licKey> PrivacyScrubber PRO/TEAMS/SDK license key
|
|
45
125
|
-v, --version Show SDK version
|
|
46
126
|
-h, --help Show help screen
|
|
47
127
|
|
|
48
128
|
Examples:
|
|
49
129
|
cat prompt.txt | npx @privacyscrubber/sdk
|
|
130
|
+
cat prompt.txt | npx @privacyscrubber/sdk --json
|
|
50
131
|
cat build.log | npx @privacyscrubber/sdk --verify-ci
|
|
132
|
+
npx @privacyscrubber/sdk memo > CISO_APPROVAL_MEMO.md
|
|
133
|
+
npx @privacyscrubber/sdk demo
|
|
51
134
|
npx @privacyscrubber/sdk "Contact Alice at alice@acme.com" --profile dev
|
|
135
|
+
echo "Hello [NAME_1]" | npx @privacyscrubber/sdk --restore --token-map '{"[NAME_1]":"Alice"}'
|
|
136
|
+
cat stream.txt | npx @privacyscrubber/sdk --restore --stream --token-map tokens.json
|
|
137
|
+
|
|
138
|
+
Commercial Licensing:
|
|
139
|
+
Developer SDK ($199/mo flat): https://privacyscrubber.com/pricing?tier=sdk&src=cli_help
|
|
140
|
+
Teams ($99/mo flat): https://privacyscrubber.com/pricing?tier=teams&src=cli_help
|
|
52
141
|
`);
|
|
53
142
|
process.exit(0);
|
|
54
143
|
}
|
|
@@ -66,30 +155,60 @@ if (keyIdx !== -1 && args[keyIdx + 1]) {
|
|
|
66
155
|
}
|
|
67
156
|
|
|
68
157
|
const isVerifyCi = args.includes('--verify-ci');
|
|
158
|
+
const isJson = args.includes('--json');
|
|
159
|
+
const isRestore = args.includes('--restore') || args.includes('--unscrub');
|
|
160
|
+
const isStream = args.includes('--stream');
|
|
161
|
+
|
|
162
|
+
let tokenMap = {};
|
|
163
|
+
const mapIdx = args.indexOf('--token-map');
|
|
164
|
+
if (mapIdx !== -1 && args[mapIdx + 1]) {
|
|
165
|
+
const mapSource = args[mapIdx + 1];
|
|
166
|
+
try {
|
|
167
|
+
if (fs.existsSync(mapSource)) {
|
|
168
|
+
tokenMap = JSON.parse(fs.readFileSync(mapSource, 'utf8'));
|
|
169
|
+
} else {
|
|
170
|
+
tokenMap = JSON.parse(mapSource);
|
|
171
|
+
}
|
|
172
|
+
} catch (err) {
|
|
173
|
+
console.error(`[PrivacyScrubber CLI Error] Failed to parse --token-map: ${err.message}`);
|
|
174
|
+
process.exit(1);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
69
177
|
|
|
70
178
|
// Locate input argument (first positional parameter that is not a flag or flag value)
|
|
71
179
|
const skipIndices = new Set();
|
|
72
180
|
if (profIdx !== -1) { skipIndices.add(profIdx); skipIndices.add(profIdx + 1); }
|
|
73
181
|
if (keyIdx !== -1) { skipIndices.add(keyIdx); skipIndices.add(keyIdx + 1); }
|
|
182
|
+
if (mapIdx !== -1) { skipIndices.add(mapIdx); skipIndices.add(mapIdx + 1); }
|
|
74
183
|
|
|
75
184
|
let inputArg = null;
|
|
76
185
|
for (let i = 0; i < args.length; i++) {
|
|
77
186
|
if (skipIndices.has(i)) continue;
|
|
78
|
-
if (args[i] === '--verify-ci') continue;
|
|
187
|
+
if (args[i] === '--verify-ci' || args[i] === '--json' || args[i] === '--restore' || args[i] === '--unscrub') continue;
|
|
79
188
|
if (args[i].startsWith('-')) continue;
|
|
80
189
|
inputArg = args[i];
|
|
81
190
|
break;
|
|
82
191
|
}
|
|
83
192
|
|
|
84
193
|
function handleExecution(text) {
|
|
194
|
+
if (isRestore) {
|
|
195
|
+
const { restoredText, restoredCount } = restore(text, tokenMap);
|
|
196
|
+
if (isJson) {
|
|
197
|
+
console.log(JSON.stringify({ restoredText, restoredCount }, null, 2));
|
|
198
|
+
} else {
|
|
199
|
+
process.stdout.write(restoredText.endsWith('\n') ? restoredText : restoredText + '\n');
|
|
200
|
+
}
|
|
201
|
+
process.exit(0);
|
|
202
|
+
}
|
|
203
|
+
|
|
85
204
|
if (isVerifyCi) {
|
|
86
205
|
if (!text || !text.trim()) {
|
|
87
|
-
console.log(`> 🛡️ **Verified by PrivacyScrubber (ZTDS Standard)**\n> * **Status:** ✅ PASSED · 0 PII / 0 Secrets detected in prompt pipeline (Empty payload)\n> * **Execution:** Local RAM (<1ms) · 0 Network Egress Bytes`);
|
|
206
|
+
console.log(`> 🛡️ **Verified by PrivacyScrubber (ZTDS Standard)**\n> * **Status:** ✅ PASSED · 0 PII / 0 Secrets detected in prompt pipeline (Empty payload)\n> * **Execution:** Local RAM (<1ms) · 0 Network Egress Bytes\n> * **Compliance Spec:** [Zero-Trust Data Sanitization (ZTDS)](https://privacyscrubber.com/solutions/engineering/zero-trust-ai-data-sanitization?src=ci_stamp) | [Developer SDK ($199/mo)](https://privacyscrubber.com/pricing?tier=sdk&src=ci_stamp)`);
|
|
88
207
|
process.exit(0);
|
|
89
208
|
}
|
|
90
209
|
|
|
91
|
-
const { scrubbedText, tokenMap, telemetry } = sanitize(text, { profile, detectSecrets: true });
|
|
92
|
-
const maskedCount = Object.keys(
|
|
210
|
+
const { scrubbedText, tokenMap: resMap, telemetry } = sanitize(text, { profile, detectSecrets: true });
|
|
211
|
+
const maskedCount = Object.keys(resMap || {}).length;
|
|
93
212
|
const hash = crypto.createHash('sha256').update(scrubbedText).digest('hex').slice(0, 16);
|
|
94
213
|
|
|
95
214
|
const stamp = `
|
|
@@ -97,17 +216,31 @@ function handleExecution(text) {
|
|
|
97
216
|
> * **Status:** ✅ PASSED · ${maskedCount} item(s) protected (${telemetry?.riskLevel || 'CLEAN'})
|
|
98
217
|
> * **Execution:** Local RAM (<1ms) · 0 Network Egress Bytes
|
|
99
218
|
> * **Audit Hash:** \`${hash}\` (Zero-Trust Verified)
|
|
219
|
+
> * **Compliance Spec:** [Zero-Trust Data Sanitization (ZTDS)](https://privacyscrubber.com/solutions/engineering/zero-trust-ai-data-sanitization?src=ci_stamp) | [Developer SDK ($199/mo)](https://privacyscrubber.com/pricing?tier=sdk&src=ci_stamp)
|
|
100
220
|
`;
|
|
101
221
|
console.log(stamp.trim());
|
|
102
222
|
process.exit(0);
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
if (!text) {
|
|
226
|
+
if (isJson) {
|
|
227
|
+
console.log(JSON.stringify({ scrubbedText: '', tokenMap: {}, count: 0 }, null, 2));
|
|
106
228
|
}
|
|
107
|
-
const { scrubbedText } = sanitize(text, { profile, detectSecrets: true });
|
|
108
|
-
process.stdout.write(scrubbedText.endsWith('\n') ? scrubbedText : scrubbedText + '\n');
|
|
109
229
|
process.exit(0);
|
|
110
230
|
}
|
|
231
|
+
|
|
232
|
+
const res = sanitize(text, { profile, detectSecrets: true });
|
|
233
|
+
if (isJson) {
|
|
234
|
+
console.log(JSON.stringify({
|
|
235
|
+
scrubbedText: res.scrubbedText,
|
|
236
|
+
tokenMap: res.tokenMap,
|
|
237
|
+
count: res.count,
|
|
238
|
+
telemetry: res.telemetry
|
|
239
|
+
}, null, 2));
|
|
240
|
+
} else {
|
|
241
|
+
process.stdout.write(res.scrubbedText.endsWith('\n') ? res.scrubbedText : res.scrubbedText + '\n');
|
|
242
|
+
}
|
|
243
|
+
process.exit(0);
|
|
111
244
|
}
|
|
112
245
|
|
|
113
246
|
if (inputArg) {
|
|
@@ -118,12 +251,26 @@ if (inputArg) {
|
|
|
118
251
|
handleExecution(inputArg);
|
|
119
252
|
}
|
|
120
253
|
} else if (!process.stdin.isTTY) {
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
254
|
+
if (isStream && isRestore) {
|
|
255
|
+
const detok = new StreamingDetokenizer(tokenMap);
|
|
256
|
+
process.stdin.setEncoding('utf8');
|
|
257
|
+
process.stdin.on('data', chunk => {
|
|
258
|
+
const out = detok.feed(chunk);
|
|
259
|
+
if (out) process.stdout.write(out);
|
|
260
|
+
});
|
|
261
|
+
process.stdin.on('end', () => {
|
|
262
|
+
const finalOut = detok.flush();
|
|
263
|
+
if (finalOut) process.stdout.write(finalOut);
|
|
264
|
+
process.exit(0);
|
|
265
|
+
});
|
|
266
|
+
} else {
|
|
267
|
+
const chunks = [];
|
|
268
|
+
process.stdin.on('data', c => chunks.push(c));
|
|
269
|
+
process.stdin.on('end', () => {
|
|
270
|
+
const content = Buffer.concat(chunks).toString('utf8');
|
|
271
|
+
handleExecution(content);
|
|
272
|
+
});
|
|
273
|
+
}
|
|
127
274
|
} else {
|
|
128
275
|
console.error(`Error: No input provided. Use: npx @privacyscrubber/sdk <fileOrText> or pipe stdin.`);
|
|
129
276
|
process.exit(1);
|
package/index.d.ts
CHANGED
|
@@ -34,6 +34,7 @@ export type PiiProfile =
|
|
|
34
34
|
| 'Insurance'
|
|
35
35
|
| 'Accounting'
|
|
36
36
|
| 'Pharma'
|
|
37
|
+
| 'Biotech'
|
|
37
38
|
| 'Underwriting'
|
|
38
39
|
| 'Crypto'
|
|
39
40
|
| 'Cloud'
|
|
@@ -69,12 +70,15 @@ export interface SanitizeOptions {
|
|
|
69
70
|
generateReceipt?: boolean;
|
|
70
71
|
/** Throw PrivacyScrubberLicenseError when free tier limits are reached (Default: true). */
|
|
71
72
|
throwOnLimit?: boolean;
|
|
73
|
+
/** Strategy when free tier limit is exceeded: 'throw' (default) or 'truncate' (soft limit with notification). */
|
|
74
|
+
onLimit?: 'throw' | 'truncate';
|
|
72
75
|
}
|
|
73
76
|
|
|
74
77
|
export interface LicenseMetadata {
|
|
75
78
|
isPro: boolean;
|
|
76
79
|
tier: 'PRO' | 'TEAMS' | 'SDK' | 'ENTERPRISE' | 'OEM' | 'FREE';
|
|
77
80
|
limit: number;
|
|
81
|
+
wasTruncated?: boolean;
|
|
78
82
|
upgradeUrl?: string;
|
|
79
83
|
recommendation?: string;
|
|
80
84
|
}
|
|
@@ -132,6 +136,24 @@ export function validateLicenseKey(key: string): 'PRO' | 'TEAMS' | 'SDK' | 'ENTE
|
|
|
132
136
|
export function buildAuditTelemetry(currentTokenMap?: Record<string, string> | string[]): AuditTelemetry;
|
|
133
137
|
export function formatAuditReceipt(telemetry: AuditTelemetry): string;
|
|
134
138
|
|
|
139
|
+
export interface ProcurementMemoOptions {
|
|
140
|
+
/** Target application or pipeline name (Default: 'Internal AI Applications & LLM Pipelines'). */
|
|
141
|
+
appName?: string;
|
|
142
|
+
/** Team or developer name (Default: 'Engineering / AI Platform Team'). */
|
|
143
|
+
developer?: string;
|
|
144
|
+
/** Output format: 'markdown' or 'text' (Default: 'markdown'). */
|
|
145
|
+
format?: 'markdown' | 'text';
|
|
146
|
+
/** Number of engineers on the team (Default: 10). */
|
|
147
|
+
teamSize?: number;
|
|
148
|
+
/** Estimated monthly prompt volume (Default: '50M - 500M characters/month'). */
|
|
149
|
+
estimatedVolume?: string;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* Generates an executive CISO & Legal Procurement Memo for enterprise approval of @privacyscrubber/sdk.
|
|
154
|
+
*/
|
|
155
|
+
export function generateProcurementMemo(options?: ProcurementMemoOptions): string;
|
|
156
|
+
|
|
135
157
|
/**
|
|
136
158
|
* Sanitizes input text, masking all PII and secrets with zero network requests.
|
|
137
159
|
*/
|
|
@@ -163,6 +185,8 @@ export class PrivacyScrubberEngine {
|
|
|
163
185
|
sanitize(text: string, options?: SanitizeOptions & { persistSession?: boolean }): SanitizeResult;
|
|
164
186
|
restore(aiResponse: string, tokenMap?: Record<string, string> | null): RestoreResult;
|
|
165
187
|
resetSession(): void;
|
|
188
|
+
createStreamRestorer(tokenMap?: Record<string, string> | null, options?: any): StreamTokenRestorer;
|
|
189
|
+
wrapStream<T = any>(stream: AsyncIterable<T>, tokenMap?: Record<string, string> | null, options?: any): AsyncGenerator<any, void, unknown>;
|
|
166
190
|
}
|
|
167
191
|
|
|
168
192
|
/**
|
|
@@ -292,13 +316,38 @@ export function createGuardedTools(engineOrOptions?: PrivacyScrubberEngine | San
|
|
|
292
316
|
*/
|
|
293
317
|
export interface StreamTokenRestorer {
|
|
294
318
|
/** Pushes an incoming streaming chunk and returns immediately rehydrated text. */
|
|
295
|
-
push(chunk: string): string;
|
|
319
|
+
push(chunk: string | Uint8Array | any): string;
|
|
320
|
+
/** Feeds an incoming chunk and returns de-tokenized text. */
|
|
321
|
+
feed(chunk: string | Uint8Array | any): string;
|
|
322
|
+
/** Alias for feed. */
|
|
323
|
+
process(chunk: string | Uint8Array | any): string;
|
|
296
324
|
/** Flushes any buffered incomplete tokens at the end of the stream. */
|
|
297
325
|
flush(): string;
|
|
298
|
-
/**
|
|
299
|
-
|
|
326
|
+
/** Wraps an async iterable LLM stream yielding de-tokenized chunks in real-time. */
|
|
327
|
+
wrapStream<T = any>(asyncIterable: AsyncIterable<T>): AsyncGenerator<any, void, unknown>;
|
|
328
|
+
/** Converts to a native Node.js Transform stream for .pipe() or stream.pipeline(). */
|
|
329
|
+
toNodeTransform(): any;
|
|
330
|
+
/** Underlying StreamingDetokenizer instance. */
|
|
331
|
+
detokenizer: StreamingDetokenizer;
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
/**
|
|
335
|
+
* High-performance streaming de-tokenization engine with sliding-window chunk boundary buffering.
|
|
336
|
+
*/
|
|
337
|
+
export class StreamingDetokenizer {
|
|
338
|
+
constructor(tokenMap?: Record<string, string>, options?: any);
|
|
339
|
+
feed(chunk: string | Uint8Array | any): string;
|
|
340
|
+
process(chunk: string | Uint8Array | any): string;
|
|
341
|
+
flush(): string;
|
|
342
|
+
toNodeTransform(): any;
|
|
343
|
+
wrapStream<T = any>(asyncIterable: AsyncIterable<T>): AsyncGenerator<any, void, unknown>;
|
|
300
344
|
}
|
|
301
345
|
|
|
346
|
+
/**
|
|
347
|
+
* Builds an ultra-fast streaming token replacement function preserving all whitespace.
|
|
348
|
+
*/
|
|
349
|
+
export function buildStreamReplacer(tokenMap?: Record<string, string>): (text: string) => string;
|
|
350
|
+
|
|
302
351
|
/**
|
|
303
352
|
* Ergonomic 1-liner: string-in, sanitized string-out without metadata ceremonies.
|
|
304
353
|
* @param text - Plaintext prompt containing potential PII/secrets.
|
|
@@ -310,8 +359,16 @@ export function clean(text: string, profile?: PiiProfile): string;
|
|
|
310
359
|
/**
|
|
311
360
|
* Creates a stream token restorer that rehydrates masked tokens across chunk boundaries in real-time.
|
|
312
361
|
* @param tokenMap - Map of token bindings returned from sanitize().
|
|
362
|
+
* @param options - Optional configuration.
|
|
313
363
|
*/
|
|
314
|
-
export function createStreamTokenRestorer(tokenMap?: Record<string, string
|
|
364
|
+
export function createStreamTokenRestorer(tokenMap?: Record<string, string>, options?: any): StreamTokenRestorer;
|
|
365
|
+
|
|
366
|
+
/**
|
|
367
|
+
* Creates a Node.js native Transform stream for backend streaming pipelines (stream.pipeline / .pipe()).
|
|
368
|
+
* @param tokenMap - Map of token bindings returned from sanitize().
|
|
369
|
+
* @param options - Optional configuration.
|
|
370
|
+
*/
|
|
371
|
+
export function createNodeStreamTokenRestorer(tokenMap?: Record<string, string>, options?: any): any;
|
|
315
372
|
|
|
316
373
|
/**
|
|
317
374
|
* Wraps an AI streaming response (e.g. OpenAI stream, Vercel AI SDK textStream)
|
|
@@ -321,8 +378,8 @@ export function createStreamTokenRestorer(tokenMap?: Record<string, string>): St
|
|
|
321
378
|
*/
|
|
322
379
|
export function wrapAiStream<T = any>(
|
|
323
380
|
stream: AsyncIterable<T> | any,
|
|
324
|
-
tokenMap
|
|
325
|
-
): AsyncGenerator<
|
|
381
|
+
tokenMap?: Record<string, string>
|
|
382
|
+
): AsyncGenerator<any, void, unknown>;
|
|
326
383
|
|
|
327
384
|
/**
|
|
328
385
|
* Zero-Trust wrapper for Next.js App Router Route Handlers (route.js / route.ts).
|
|
@@ -371,7 +428,10 @@ declare const defaultExport: {
|
|
|
371
428
|
scrub: typeof scrub;
|
|
372
429
|
restore: typeof restore;
|
|
373
430
|
unscrub: typeof unscrub;
|
|
431
|
+
StreamingDetokenizer: typeof StreamingDetokenizer;
|
|
374
432
|
createStreamTokenRestorer: typeof createStreamTokenRestorer;
|
|
433
|
+
createNodeStreamTokenRestorer: typeof createNodeStreamTokenRestorer;
|
|
434
|
+
buildStreamReplacer: typeof buildStreamReplacer;
|
|
375
435
|
wrapAiStream: typeof wrapAiStream;
|
|
376
436
|
withPrivacyScrubber: typeof withPrivacyScrubber;
|
|
377
437
|
createExpressMiddleware: typeof createExpressMiddleware;
|
|
@@ -379,6 +439,7 @@ declare const defaultExport: {
|
|
|
379
439
|
validateLicenseKey: typeof validateLicenseKey;
|
|
380
440
|
buildAuditTelemetry: typeof buildAuditTelemetry;
|
|
381
441
|
formatAuditReceipt: typeof formatAuditReceipt;
|
|
442
|
+
generateProcurementMemo: typeof generateProcurementMemo;
|
|
382
443
|
PrivacyScrubberEngine: typeof PrivacyScrubberEngine;
|
|
383
444
|
PrivacyScrubberLicenseError: typeof PrivacyScrubberLicenseError;
|
|
384
445
|
wrapOpenAI: typeof wrapOpenAI;
|