@privacyscrubber/sdk 2.2.0 → 2.2.2

package/index.js

@@ -43,7 +43,12 @@ export const DEVOPS_SECRETS_DETECTOR = [
     { name: 'Stripe API Key', regex: /\b(?:[rs]k)_(?:test|live)_[a-zA-Z0-9]{24,}\b/g, label: 'STRIPE_KEY' },
     { name: 'OpenAI Project API Key', regex: /\b(?:sk|pk)-(?:proj-)?[a-zA-Z0-9_-]{16,}\b/gi, label: 'OPENAI_KEY' },
     { name: 'Database Connection URI', regex: /\b(?:postgres(?:ql)?|mysql|mongodb(?:\+srv)?|redis|amqp|mssql):\/\/[^\s"']+/gi, label: 'DB_URI' },
-    { name: 'Database/API Secret', regex: /\b(DB|POSTGRES|REDIS|MYSQL|AWS|SECRET|PASSWORD|TOKEN|API|KEY)[A-Z0-9_]*\s*[:=]\s*[^ \t\r\n"']{8,}\b/gi, label: 'SECRET' }
+    { name: 'Embedded Credentials URI', regex: /\b(?:https?|s?ftp):\/\/[a-zA-Z0-9_.%+-]+:[^\s/@:"']+@[a-zA-Z0-9_.-]+(?::\d+)?(?:\/[^\s"']*)?/gi, label: 'SECRET' },
+    { name: 'URL Query Parameter Secret', regex: /(?<=[?&](?:api_key|apikey|access_token|client_secret|auth_token|token|secret|password|private_key)=)[^\s"'\x26#)>]{8,}/gi, label: 'SECRET' },
+    { name: 'Bearer / Auth Header Token', regex: /(?<=\b(?:Bearer|Token)\s+)[a-zA-Z0-9_\-\.~+/=]{16,}\b/gi, label: 'SECRET' },
+    { name: 'Database/API Secret', regex: /\b(DB|POSTGRES|REDIS|MYSQL|AWS|SECRET|PASSWORD|TOKEN|API|KEY)[A-Z0-9_]*\s*[:=]\s*[^ \t\r\n"']{8,}\b/gi, label: 'SECRET' },
+    { name: 'Private Cryptographic Key (PEM Block)', regex: /-----BEGIN (?:RSA |EC |PGP |DSA |OPENSSH |ENCRYPTED )?PRIVATE KEY(?: BLOCK)?-----[\s\S]{10,8192}?-----END (?:RSA |EC |PGP |DSA |OPENSSH |ENCRYPTED )?PRIVATE KEY(?: BLOCK)?-----/g, label: 'SECRET' },
+    { name: 'Private Cryptographic Key (Header)', regex: /-----BEGIN (?:RSA |EC |PGP |DSA |OPENSSH |ENCRYPTED )?PRIVATE KEY(?: BLOCK)?-----/g, label: 'SECRET' }
 ];
 
 /**
@@ -164,7 +169,8 @@ export function buildAuditTelemetry(currentTokenMap = {}) {
  */
 export function formatAuditReceipt(telemetry) {
     if (!telemetry || telemetry.totalCount === 0) {
-        return "> 🛡️ **PrivacyScrubber Audit Receipt**: CLEAN (ZERO PII DETECTED)\n";
+        return "> 🛡️ **PrivacyScrubber Audit Receipt**: CLEAN (ZERO PII DETECTED)\n" +
+               "> * ⭐ **Star on GitHub:** [moxno/privacyscrubber-mcp](https://github.com/moxno/privacyscrubber-mcp) | **SDK & Enterprise:** [privacyscrubber.com/pricing](https://privacyscrubber.com/pricing)\n";
     }
     const entitiesList = Object.entries(telemetry.entities)
         .map(([type, count]) => `${count} ${type}`)
@@ -175,28 +181,203 @@ export function formatAuditReceipt(telemetry) {
     return `> 🛡️ **PrivacyScrubber Audit Receipt**\n` +
            `> * **Risk Level:** ${icon} ${telemetry.riskLevel}\n` +
            `> * **Compliance Enforced:** ${telemetry.frameworksList.join(', ')}\n` +
-           `> * **Tokens Masked:** ${telemetry.totalCount} (${entitiesList})\n`;
+           `> * **Tokens Masked:** ${telemetry.totalCount} (${entitiesList})\n` +
+           `> * ⭐ **Star on GitHub:** [moxno/privacyscrubber-mcp](https://github.com/moxno/privacyscrubber-mcp) | **SDK & Enterprise:** [privacyscrubber.com/pricing](https://privacyscrubber.com/pricing)\n`;
+}
+
+let communityNoticeLogged = false;
+
+function emitCommunityNoticeIfNeeded(isPro) {
+    if (communityNoticeLogged || isPro) return;
+    if (typeof process !== 'undefined' && process.env?.NODE_ENV === 'production') {
+        communityNoticeLogged = true;
+        console.info('[@privacyscrubber/sdk] Running Community Edition (Core PII).\nTo unlock 25 Enterprise Profiles & DevOps Secrets in production: https://privacyscrubber.com/pricing/?tier=sdk');
+    }
+}
+
+/**
+ * Resets the community notice state for testing purposes.
+ * @private
+ */
+export function _resetCommunityNoticeForTesting() {
+    communityNoticeLogged = false;
+}
+
+let devSecretNoticeLogged = false;
+
+function emitDevSecretNoticeIfNeeded(isPro, tokenMap = {}, limit = 15000) {
+    if (devSecretNoticeLogged || isPro) return;
+    if (typeof process !== 'undefined' && process.env?.NODE_ENV === 'production') return;
+
+    const tokens = Object.keys(tokenMap || {});
+    const secretTokens = tokens.filter(t => {
+        const upper = t.toUpperCase();
+        return ['API_KEY', 'SECRET', 'PASSWORD', 'KEY', 'AWS_KEY', 'STRIPE_KEY', 'JWT_TOKEN', 'API_TOKEN', 'TOKEN'].some(k => upper.includes(k));
+    });
+
+    if (secretTokens.length > 0) {
+        devSecretNoticeLogged = true;
+        const count = secretTokens.length;
+        const secretSample = secretTokens.slice(0, 2).join(', ');
+        if (typeof process !== 'undefined' && process.stderr && typeof process.stderr.write === 'function') {
+            process.stderr.write(
+                `\n[PrivacyScrubber] Intercepted ${count} API secret(s) before LLM egress: ${secretSample}\n` +
+                `[PrivacyScrubber] Protected in local RAM (0 byte network egress). Free Tier active (${limit.toLocaleString()} char cap).\n` +
+                `[PrivacyScrubber] Developer SDK ($199/mo flat) for unlimited backend nodes, CI/CD gates & ETL pipelines: https://privacyscrubber.com/pricing?tier=sdk&src=cli_leak_guard\n\n`
+            );
+        }
+    }
+}
+
+/**
+ * Resets the dev secret notice state for testing purposes.
+ * @private
+ */
+export function _resetDevSecretNoticeForTesting() {
+    devSecretNoticeLogged = false;
+}
+
+/**
+ * Generates an executive CISO & Legal Procurement Memo for enterprise approval of @privacyscrubber/sdk.
+ * 
+ * @param {object} [options={}] - Custom memo configuration options.
+ * @param {string} [options.appName='Internal AI Applications & LLM Pipelines'] - Target system name.
+ * @param {string} [options.developer='Engineering / AI Platform Team'] - Authoring team name.
+ * @param {'markdown'|'text'} [options.format='markdown'] - Output format ('markdown' or 'text').
+ * @param {number} [options.teamSize=10] - Size of engineering team.
+ * @param {string} [options.estimatedVolume='50M - 500M characters/month'] - Projected token volume.
+ * @returns {string}
+ */
+export function generateProcurementMemo(options = {}) {
+    const {
+        appName = 'Internal AI Applications & LLM Pipelines',
+        developer = 'Engineering / AI Platform Team',
+        format = 'markdown',
+        teamSize = 10,
+        estimatedVolume = '50M - 500M characters/month'
+    } = options;
+
+    if (format === 'text') {
+        return `MEMORANDUM: ADOPTION OF PRIVACYSCRUBBER DEVELOPER SDK (ZTDS STANDARD)
+
+TO: Chief Information Security Officer (CISO), Data Protection Officer (DPO), VP Engineering
+FROM: ${developer}
+SUBJECT: Approval Request: Zero-Trust Data Sanitization (@privacyscrubber/sdk)
+TARGET SYSTEM: ${appName}
+ESTIMATED VOLUME: ${estimatedVolume}
+
+1. EXECUTIVE SUMMARY
+We propose standardizing on the PrivacyScrubber Developer SDK (@privacyscrubber/sdk) for in-memory Zero-Trust Data Sanitization across our AI ingestion pipelines. The SDK intercepts and redacts PII, regulatory identifiers, and cloud secrets locally in RAM before any prompt leaves our infrastructure.
+
+2. ZERO-TRUST ARCHITECTURAL INVARIANTS
+* Zero Network Egress: 100% local in-memory processing. Zero external API calls, zero telemetry egress, zero third-party sub-processors.
+* In-Memory Volatility: All masking maps remain in volatile process RAM and are discarded after request completion.
+* Sub-Millisecond Latency: Executes in <0.1ms per prompt in local RAM, eliminating latency overhead.
+
+3. COMPLIANCE & REGULATORY COVERAGE
+Pre-configured for 25 specialized industry profiles and compliance frameworks:
+* GDPR (Art. 4, 9, 32) & CCPA/CPRA: Ephemeral tokenization.
+* HIPAA (§164.514 Safe Harbor): MRNs, medical record codes, patient identifiers.
+* SOC 2 Type II & ISO 27001 (A.8.11): Zero persistent unencrypted PII.
+* PCI DSS v4.0: PAN, CVV, IBAN, bank ledger sanitization.
+* NIST SP 800-53: Cloud API keys, JWT tokens, AWS/Stripe secret leak protection.
+
+4. TOTAL COST OF OWNERSHIP (TCO) COMPARISON
+* Cloud DLP Proxies (AWS Macie, GCP DLP, Nightfall):
+  - Cost: $1,500 - $4,000 / month (usage-based billing).
+  - Latency: +200ms - 400ms added network roundtrip per AI prompt.
+  - Sub-processor Risk: Requires vendor DPA, external network egress.
+* PrivacyScrubber Developer SDK:
+  - Cost: $199 / month flat (or $1,990 / year) for unlimited nodes, microservices, and prompts.
+  - Latency: <0.1ms (in-process RAM).
+  - Sub-processor Risk: Zero (pure in-memory software library).
+
+5. ACTION REQUESTED
+Approve commercial license upgrade for ${developer} (${teamSize} engineers, unlimited backend nodes):
+* License Tier: Developer SDK ($199/month flat or $1,990/year)
+* Direct Procurement & License Activation: https://privacyscrubber.com/pricing?tier=sdk&src=ciso_memo
+* Enterprise / Invoicing Contact: enterprise@privacyscrubber.com
+`;
+    }
+
+    return `# Internal Memo: Adoption of PrivacyScrubber Developer SDK (ZTDS Standard)
+
+| **To** | Chief Information Security Officer (CISO), Data Protection Officer (DPO), VP Engineering |
+| :--- | :--- |
+| **From** | ${developer} |
+| **Date** | ${new Date().toISOString().split('T')[0]} |
+| **Subject** | Architectural Approval & License Procurement for Zero-Trust AI Sanitization |
+| **Target System** | ${appName} |
+| **Estimated Scope** | ~${teamSize} engineers / ${estimatedVolume} |
+
+---
+
+### 1. Executive Summary
+We recommend adopting the **PrivacyScrubber Developer SDK** (\`@privacyscrubber/sdk\`) to enforce **Zero-Trust Data Sanitization (ZTDS)** across all internal AI pipelines, LangChain agents, microservices, and developer toolchains.
+
+The SDK redacts PII, regulatory identifiers, and developer secrets **in-process before external API egress to OpenAI, Anthropic, or external LLM providers**, and deterministically restores original values upon response return.
+
+---
+
+### 2. Architectural Invariants (Zero-Trust Data Sanitization)
+* **Zero Remote Network Egress:** 100% of sanitization and token restoration executes in local Node.js process RAM. No prompt data or telemetry ever leaves our VPC/infrastructure.
+* **Ephemeral In-Memory Tokens:** Masking mappings (\`[EMAIL_1]\`, \`[NAME_1]\`, \`[SECRET_1]\`) are volatile and strictly scoped to request execution context.
+* **Ultra-Low Latency:** Micro-benchmarks verify <0.1ms execution overhead per prompt, compared to 200–400ms for network-based cloud DLP proxies.
+* **Full Reversibility:** Bidirectional round-trip reconstruction ensures downstream application logic receives intact responses without exposing sensitive data to model providers.
+
+---
+
+### 3. Compliance Framework Mapping
+The SDK enforces out-of-the-box sanitization matching mandatory compliance controls:
+* **GDPR (Regulation (EU) 2016/679) & CCPA/CPRA:** Eliminates cross-border PII transfer risk to third-party AI models.
+* **HIPAA (§164.514 Safe Harbor):** Automated scrubbing of Medical Record Numbers (MRNs), patient identifiers, and health insurance IDs.
+* **SOC 2 Type II & ISO 27001 (Control A.8.11):** Enforces data masking and prevents unauthorized logging of confidential client data.
+* **PCI DSS v4.0:** Sanitization of payment cards, IBANs, and financial ledger data.
+* **NIST SP 800-53:** DevOps secret interception for AWS keys, JWT tokens, database URIs, and private API keys.
+
+---
+
+### 4. Financial & Architectural Comparison (TCO)
+
+| Evaluation Factor | Traditional Cloud DLP (AWS/GCP/Nightfall) | PrivacyScrubber Developer SDK |
+| :--- | :--- | :--- |
+| **Pricing Model** | \$1,500 – \$4,000 / month (metered per GB/token) | **\$199 / month flat** (unlimited nodes & tokens) |
+| **Network Egress** | Requires outbound network calls to DLP vendor | **0 bytes** (100% local Node.js process memory) |
+| **Added Latency** | +200ms – +450ms network round-trip | **<0.1ms** (in-memory regular expressions) |
+| **Vendor Sub-processors** | Third-party cloud vendor added to vendor list | **None** (zero-network software library) |
+| **Deployment** | Complex API gateways and proxy configurations | \`npm install @privacyscrubber/sdk\` in 3 minutes |
+
+---
+
+### 5. Procurement Recommendation
+We request approval to acquire a **Developer SDK License** (\$199/month flat or \$1,990/year) to unlock unlimited throughput across all microservices and CI/CD pipelines.
+
+* **Direct Procurement Portal:** [https://privacyscrubber.com/pricing?tier=sdk&src=ciso_memo](https://privacyscrubber.com/pricing?tier=sdk&src=ciso_memo)
+* **Procurement / Invoicing Contact:** \`enterprise@privacyscrubber.com\`
+`;
 }
 
 /**
- * Primary High-Level Function: Sanitize / Scrub text of PII and secrets.
+ * Sanitizes input text, masking detected PII & secrets with reversible tokens.
  * 
- * @param {string} text - Raw input text containing potential PII/secrets.
- * @param {object} [options={}] - Configuration options.
- * @param {string} [options.profile='General'] - PII Profile (General, Dev, Medical, Financial, Legal, HR, etc.).
+ * @param {string} text - Raw input text.
+ * @param {object} [options={}] - Sanitization options.
+ * @param {string} [options.profile='General'] - Compliance detection profile.
  * @param {string} [options.licenseKey] - Optional license key. Falls back to process.env.PRIVACYSCRUBBER_KEY.
- * @param {Array} [options.customRules=[]] - Custom regex rules array.
- * @param {Record<string, string>} [options.tokenLabelMap={}] - Custom label overrides.
- * @param {Record<string, string>} [options.existingSessionMap={}] - Session map to maintain consistency across turns.
- * @param {boolean} [options.generateReceipt=true] - Whether to generate Markdown audit receipt.
+ * @param {Array<{name: string, pattern: string, regex: RegExp, label?: string}>} [options.customRules=[]] - Custom regex rules.
+ * @param {Record<string, string>} [options.tokenLabelMap={}] - Custom label mapping.
+ * @param {Record<string, string>} [options.existingSessionMap={}] - In-memory session map to preserve token IDs across turns.
+ * @param {boolean} [options.generateReceipt=true] - Generate a CISO Markdown audit receipt.
  * @param {boolean} [options.throwOnLimit=true] - Throw PrivacyScrubberLicenseError when free tier limit is hit.
- * @param {boolean} [options.detectSecrets=false] - Auto-inject DevOps & Cloud secrets detection rules.
+ * @param {'throw'|'truncate'} [options.onLimit='throw'] - Action when character limit exceeded ('throw' or 'truncate').
+ * @param {boolean} [options.detectSecrets=false] - Auto-detect cloud credentials and API keys.
+ * @param {Set<string>|Array<string>} [options.ignoreList=null] - Values to preserve in cleartext.
  * @returns {{
  *   scrubbedText: string,
  *   tokenMap: Record<string, string>,
  *   count: number,
  *   executionMs: number,
- *   license: { isPro: boolean, tier: 'PRO'|'TEAMS'|'FREE', limit: number, upgradeUrl?: string },
+ *   license: { isPro: boolean, tier: 'PRO'|'TEAMS'|'FREE', limit: number, wasTruncated: boolean, upgradeUrl?: string, recommendation?: string },
  *   telemetry: object,
  *   auditReceipt: string
  * }}
@@ -208,7 +389,7 @@ export function sanitize(text, options = {}) {
             tokenMap: {},
             count: 0,
             executionMs: 0,
-            license: { isPro: false, tier: 'FREE', limit: 15000 },
+            license: { isPro: false, tier: 'FREE', limit: 15000, wasTruncated: false },
             telemetry: buildAuditTelemetry({}),
             auditReceipt: formatAuditReceipt(buildAuditTelemetry({}))
         };
@@ -222,6 +403,7 @@ export function sanitize(text, options = {}) {
         existingSessionMap = {},
         generateReceipt = true,
         throwOnLimit = true,
+        onLimit = 'throw',
         detectSecrets = false,
         ignoreList = null
     } = options;
@@ -230,12 +412,17 @@ export function sanitize(text, options = {}) {
     const isPro = licenseCheck.valid;
     const tier = isPro ? licenseCheck.tier : 'FREE';
 
+    emitCommunityNoticeIfNeeded(isPro);
+
     const isSpecialized = profile && profile.toLowerCase() !== 'general';
     const limit = isPro ? Infinity : (isSpecialized ? 5000 : 15000);
 
+    const shouldThrow = throwOnLimit && onLimit !== 'truncate';
+    let wasTruncated = false;
+
     if (!isPro && text.length > limit) {
-        const errorMsg = `[PrivacyScrubber] Free Tier character limit exceeded (${limit.toLocaleString()} chars for profile '${profile}', received ${text.length.toLocaleString()} chars). Upgrade to PRO, TEAMS, or Developer SDK for unlimited throughput: https://privacyscrubber.com/pricing?utm_source=npm_sdk&utm_medium=sdk_error&utm_campaign=dev_upsell`;
-        if (throwOnLimit) {
+        if (shouldThrow) {
+            const errorMsg = `[PrivacyScrubber] Free Tier character limit exceeded (${limit.toLocaleString()} chars for profile '${profile}', received ${text.length.toLocaleString()} chars). Upgrade to PRO, TEAMS, or Developer SDK for unlimited throughput: https://privacyscrubber.com/pricing?utm_source=npm_sdk&utm_medium=sdk_error&utm_campaign=dev_upsell`;
             throw new PrivacyScrubberLicenseError(errorMsg, {
                 tier: 'FREE',
                 currentLength: text.length,
@@ -243,6 +430,7 @@ export function sanitize(text, options = {}) {
                 profile: profile
             });
         }
+        wasTruncated = true;
     }
 
     // Auto-inject DevOps Secrets rules when in Dev profile or when detectSecrets is requested
@@ -282,8 +470,15 @@ export function sanitize(text, options = {}) {
     const telemetry = buildAuditTelemetry(rawResult.tokenMap || {});
     const auditReceipt = generateReceipt ? formatAuditReceipt(telemetry) : '';
 
+    emitDevSecretNoticeIfNeeded(isPro, rawResult.tokenMap, limit);
+
+    let finalScrubbedText = rawResult.scrubbedText;
+    if (wasTruncated && (onLimit === 'truncate' || throwOnLimit === false)) {
+        finalScrubbedText += `\n\n<!-- [PrivacyScrubber] Input truncated at free tier limit (${limit.toLocaleString()} chars). For unlimited throughput: https://privacyscrubber.com/pricing?tier=sdk&src=truncation -->`;
+    }
+
     return {
-        scrubbedText: rawResult.scrubbedText,
+        scrubbedText: finalScrubbedText,
         tokenMap: rawResult.tokenMap || {},
         count: rawResult.count || 0,
         executionMs: executionMs,
@@ -291,6 +486,7 @@ export function sanitize(text, options = {}) {
             isPro: isPro,
             tier: tier,
             limit: limit,
+            wasTruncated: wasTruncated,
             ...(!isPro && {
                 upgradeUrl: 'https://privacyscrubber.com/pricing?utm_source=npm_sdk&utm_medium=sdk_telemetry&utm_campaign=dev_upsell',
                 recommendation: 'Upgrade to PRO, TEAMS, or Developer SDK for unlimited text processing, custom regex rules, and specialized compliance profiles.'
@@ -306,6 +502,18 @@ export function sanitize(text, options = {}) {
  */
 export const scrub = sanitize;
 
+/**
+ * Ergonomic 1-liner: Sanitizes text and returns the scrubbed string directly.
+ * @param {string} text - Input text.
+ * @param {string} [profile='General'] - Target compliance profile.
+ * @returns {string} - Scrubbed string.
+ */
+export function clean(text, profile = 'General') {
+    if (!text || typeof text !== 'string') return '';
+    const { scrubbedText } = sanitize(text, { profile });
+    return scrubbedText;
+}
+
 /**
  * Primary High-Level Function: Restore / Unscrub AI response back to original entities in-memory.
  * 
@@ -428,12 +636,34 @@ export class PrivacyScrubberEngine {
     clearIgnoreList() {
         this.ignoreList.clear();
     }
+
+    /**
+     * Creates a streaming token restorer bound to this engine's session map.
+     * @param {Record<string, string>} [tokenMap=null]
+     * @param {object} [options={}]
+     */
+    createStreamRestorer(tokenMap = null, options = {}) {
+        const activeMap = tokenMap || this.sessionMap;
+        return createStreamTokenRestorer(activeMap, options);
+    }
+
+    /**
+     * Wraps an async iterable LLM stream with real-time de-tokenization.
+     * @param {AsyncIterable<any>} stream
+     * @param {Record<string, string>} [tokenMap=null]
+     * @param {object} [options={}]
+     */
+    async* wrapStream(stream, tokenMap = null, options = {}) {
+        const activeMap = tokenMap || this.sessionMap;
+        yield* wrapAiStream(stream, activeMap, options);
+    }
 }
 
 /**
  * Transparent Zero-Trust Wrapper for OpenAI Client.
  * Automatically sanitizes outgoing messages before transmission to OpenAI,
  * and restores completion tokens in the assistant response before returning!
+ * Seamlessly de-tokenizes streaming responses in real-time when stream: true.
  * 
  * @param {object} openaiClient - Instance of OpenAI SDK (e.g. new OpenAI()).
  * @param {object} [options={}] - PrivacyScrubber configuration.
@@ -465,6 +695,12 @@ export function wrapOpenAI(openaiClient, options = {}) {
         const modifiedParams = { ...params, messages: sanitizedMessages };
         const response = await originalCreate(modifiedParams, requestOptions);
 
+        // Streaming support: transparently de-tokenize async iterable chunks
+        if (params.stream) {
+            const detok = new StreamingDetokenizer(sessionTokenMap);
+            return detok.wrapStream(response);
+        }
+
         if (response && Array.isArray(response.choices)) {
             response.choices.forEach(choice => {
                 if (choice.message && typeof choice.message.content === 'string') {
@@ -490,6 +726,370 @@ export function createLangChainTransform(options = {}) {
     };
 }
 
+/**
+ * Helper to escape special regex characters.
+ */
+function escapeRegex(s) {
+    return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+/**
+ * Builds an ultra-fast streaming token replacement function.
+ * Preserves all whitespace, indentation, newlines, and code blocks (does NOT trim).
+ * Handles descending key ordering (e.g. [NAME_10] before [NAME_1]), case insensitivity,
+ * alternative delimiters (<...>, {{...}}, __...__), and possessive suffixes.
+ *
+ * @param {Record<string, string>} tokenMap
+ * @returns {(text: string) => string}
+ */
+export function buildStreamReplacer(tokenMap = {}) {
+    if (!tokenMap || typeof tokenMap !== 'object' || Object.keys(tokenMap).length === 0) {
+        return (text) => text || '';
+    }
+
+    const rules = [];
+    const sortedKeys = Object.keys(tokenMap).sort((a, b) => b.length - a.length);
+
+    for (const k of sortedKeys) {
+        const val = tokenMap[k];
+        const inner = k.replace(/^\[|<|\{\{|__|\]|>|\}\}|__/g, '');
+        const match = inner.match(/^([A-Za-z_0-9]+?)[-_ ]?(\d+)$/);
+
+        if (match) {
+            const label = escapeRegex(match[1]);
+            const idx = match[2];
+            const patternStr = '(?:\\[\\s*' + label + '[-_\\s]*0*' + idx + '\\s*\\]|<\\s*' + label + '[-_\\s]*0*' + idx + '\\s*>|\\{\\{\\s*' + label + '[-_\\s]*0*' + idx + '\\s*\\}\\}|__\\s*' + label + '[-_\\s]*0*' + idx + '\\s*__|\\b' + label + '[-_\\s]+0*' + idx + '\\b)(?:\'s|’s|s)?';
+            rules.push({
+                regex: new RegExp(patternStr, 'gi'),
+                val,
+                k
+            });
+        } else {
+            const safe = escapeRegex(k);
+            rules.push({
+                regex: new RegExp(safe + '(?:\'s|’s|s)?', 'gi'),
+                val,
+                k
+            });
+        }
+    }
+
+    return function replaceTokens(text) {
+        if (!text) return '';
+        let result = text;
+        for (let i = 0; i < rules.length; i++) {
+            const rule = rules[i];
+            rule.regex.lastIndex = 0;
+            result = result.replace(rule.regex, (m) => {
+                const isPossessive = /('s|’s)$/i.test(m);
+                return rule.val + (isPossessive ? "'s" : '');
+            });
+        }
+        return result;
+    };
+}
+
+/**
+ * Streaming Token Detokenizer with sliding window buffer for LLM chunk boundary splits.
+ * Seamlessly restores split tokens (e.g. [EM + AIL_1]), preserves multi-byte UTF-8,
+ * and maintains zero-latency streaming throughput.
+ */
+export class StreamingDetokenizer {
+    constructor(tokenMap = {}, options = {}) {
+        this.tokenMap = tokenMap && typeof tokenMap === 'object' ? tokenMap : {};
+        this.options = options || {};
+        this.replacer = buildStreamReplacer(this.tokenMap);
+        this.pendingBuffer = '';
+        this.hasTokens = Object.keys(this.tokenMap).length > 0;
+        this.decoder = new TextDecoder('utf-8', { fatal: false });
+    }
+
+    /**
+     * Feeds an incoming chunk (string, Uint8Array, or Buffer) and returns de-tokenized text.
+     * Buffers any unclosed token candidates at chunk boundaries until closed or flushed.
+     * @param {string | Uint8Array | Buffer} chunk
+     * @returns {string}
+     */
+    feed(chunk) {
+        if (chunk === undefined || chunk === null) return '';
+        let text = '';
+        if (typeof chunk === 'string') {
+            text = chunk;
+        } else if (chunk instanceof Uint8Array || (typeof Buffer !== 'undefined' && Buffer.isBuffer(chunk))) {
+            text = this.decoder.decode(chunk, { stream: true });
+        } else {
+            text = String(chunk);
+        }
+
+        if (!text) return '';
+        if (!this.hasTokens) return text;
+
+        this.pendingBuffer += text;
+
+        // Candidate unclosed token at tail of buffer (up to 64 chars)
+        // Matches incomplete tokens starting with [, <, {{, or __
+        const m = this.pendingBuffer.match(/(?:\[|<|\{\{|__)[A-Za-z0-9_\-А-Яа-яЁё ]{0,64}$/);
+        if (m) {
+            const safeToEmit = this.pendingBuffer.slice(0, m.index);
+            this.pendingBuffer = this.pendingBuffer.slice(m.index);
+            return safeToEmit ? this.replacer(safeToEmit) : '';
+        }
+
+        const toEmit = this.pendingBuffer;
+        this.pendingBuffer = '';
+        return this.replacer(toEmit);
+    }
+
+    /**
+     * Alias for feed() to maintain backwards compatibility with process() callers.
+     */
+    process(chunk) {
+        return this.feed(chunk);
+    }
+
+    /**
+     * Flushes remaining buffered tokens and finalizes UTF-8 stream decoder.
+     * @returns {string}
+     */
+    flush() {
+        let text = '';
+        try {
+            text = this.decoder.decode(new Uint8Array(), { stream: false });
+        } catch (_) {}
+
+        if (text) {
+            this.pendingBuffer += text;
+        }
+
+        if (!this.pendingBuffer) return '';
+        const remaining = this.pendingBuffer;
+        this.pendingBuffer = '';
+        return this.replacer(remaining);
+    }
+
+    /**
+     * Returns a native Node.js Transform stream instance for .pipe() / pipeline().
+     * @returns {Transform}
+     */
+    toNodeTransform() {
+        const self = this;
+        return new Transform({
+            transform(chunk, encoding, callback) {
+                try {
+                    const out = self.feed(chunk);
+                    if (out) this.push(out);
+                    callback();
+                } catch (err) {
+                    callback(err);
+                }
+            },
+            flush(callback) {
+                try {
+                    const out = self.flush();
+                    if (out) this.push(out);
+                    callback();
+                } catch (err) {
+                    callback(err);
+                }
+            }
+        });
+    }
+
+    /**
+     * Wraps an async iterable (e.g. LLM streaming response) yielding restored chunks.
+     * @param {AsyncIterable<any>} asyncIterable
+     * @returns {AsyncGenerator<any>}
+     */
+    async* wrapStream(asyncIterable) {
+        for await (const chunk of asyncIterable) {
+            if (chunk === null || chunk === undefined) continue;
+            // Handle OpenAI chunk object
+            if (chunk.choices && Array.isArray(chunk.choices) && chunk.choices[0]?.delta?.content !== undefined) {
+                const delta = chunk.choices[0].delta.content;
+                if (typeof delta === 'string') {
+                    const restored = this.feed(delta);
+                    if (restored) {
+                        chunk.choices[0].delta.content = restored;
+                        yield chunk;
+                    }
+                } else {
+                    yield chunk;
+                }
+                continue;
+            }
+
+            let text = '';
+            if (typeof chunk === 'string') {
+                text = chunk;
+            } else if (typeof chunk?.text === 'string') {
+                text = chunk.text;
+            } else if (typeof chunk?.content === 'string') {
+                text = chunk.content;
+            } else if (chunk instanceof Uint8Array || (typeof Buffer !== 'undefined' && Buffer.isBuffer(chunk))) {
+                text = chunk;
+            } else {
+                text = String(chunk);
+            }
+
+            const emitted = this.feed(text);
+            if (emitted) yield emitted;
+        }
+
+        const finalChunk = this.flush();
+        if (finalChunk) yield finalChunk;
+    }
+}
+
+/**
+ * Creates a zero-latency streaming token restorer for LLM Server-Sent Events (SSE) and Web Streams.
+ * Solves the broken chunk boundary problem where tokens like [EMAIL_1] are split across multiple chunks.
+ * 
+ * Works with:
+ * - Vercel AI SDK (streamText / createDataStreamResponse)
+ * - OpenAI streaming (chat.completions.create({ stream: true }))
+ * - LangChain streaming
+ * - Native Web Streams (ReadableStream.pipeThrough)
+ * - Async generators (for await ... of)
+ * 
+ * @param {Record<string, string>} tokenMap - Volatile token mapping object.
+ * @param {object} [options] - Configuration options.
+ * @returns {TransformStream & { process: (chunk: string) => string, feed: (chunk: string) => string, flush: () => string, wrapStream: (asyncIterable: AsyncIterable<any>) => AsyncGenerator<any>, toNodeTransform: () => Transform, detokenizer: StreamingDetokenizer }}
+ */
+export function createStreamTokenRestorer(tokenMap = {}, options = {}) {
+    const detok = new StreamingDetokenizer(tokenMap, options);
+
+    let stream;
+    if (typeof TransformStream !== 'undefined') {
+        stream = new TransformStream({
+            transform(chunk, controller) {
+                const out = detok.feed(chunk);
+                if (out) controller.enqueue(out);
+            },
+            flush(controller) {
+                const out = detok.flush();
+                if (out) controller.enqueue(out);
+            }
+        });
+    } else {
+        stream = {};
+    }
+
+    stream.process = (chunk) => detok.feed(chunk);
+    stream.feed = (chunk) => detok.feed(chunk);
+    stream.flush = () => detok.flush();
+    stream.wrapStream = (asyncIterable) => detok.wrapStream(asyncIterable);
+    stream.toNodeTransform = () => detok.toNodeTransform();
+    stream.detokenizer = detok;
+
+    return stream;
+}
+
+/**
+ * Creates a Node.js native Transform stream for backend streaming pipelines (stream.pipeline / .pipe()).
+ * @param {Record<string, string>} tokenMap - Volatile token mapping object.
+ * @param {object} [options] - Configuration options.
+ * @returns {Transform}
+ */
+export function createNodeStreamTokenRestorer(tokenMap = {}, options = {}) {
+    const detok = new StreamingDetokenizer(tokenMap, options);
+    return detok.toNodeTransform();
+}
+
+/**
+ * Wraps an async iterable LLM stream (e.g. from Vercel AI SDK textStream or OpenAI stream)
+ * and yields de-tokenized text chunks in real-time, seamlessly stitching split tokens across chunks.
+ * 
+ * @param {AsyncIterable<string|object>} stream - The incoming LLM stream.
+ * @param {Record<string, string>} tokenMap - Volatile token map.
+ * @returns {AsyncGenerator<any>}
+ */
+export async function* wrapAiStream(stream, tokenMap = {}) {
+    const detok = new StreamingDetokenizer(tokenMap);
+    yield* detok.wrapStream(stream);
+}
+
+/**
+ * Middleware wrapper for Next.js App Router Route Handlers (POST/PUT).
+ * Automatically sanitizes incoming prompt payloads in RAM (<1ms) before the handler runs,
+ * and attaches sanitized text, tokenMap, and helper restoration functions.
+ * 
+ * @param {Function} handler - The async Next.js route handler (req, context) => Response
+ * @param {object} [options] - Sanitization options (profile, detectSecrets, etc.)
+ * @returns {Function} Wrapped Next.js route handler
+ */
+export function withPrivacyScrubber(handler, options = {}) {
+    return async function privacyScrubberHandler(req, context = {}) {
+        if (!req || typeof req.json !== 'function') {
+            return handler(req, context);
+        }
+
+        let rawBody;
+        try {
+            rawBody = await req.json();
+        } catch (_) {
+            return handler(req, context);
+        }
+
+        let scrubbedBody = rawBody;
+        let tokenMap = {};
+        let telemetry = null;
+
+        if (typeof rawBody === 'string') {
+            const res = sanitize(rawBody, options);
+            scrubbedBody = res.scrubbedText;
+            tokenMap = res.tokenMap;
+            telemetry = res.telemetry;
+        } else if (rawBody && typeof rawBody === 'object') {
+            const serialized = JSON.stringify(rawBody);
+            const res = sanitize(serialized, options);
+            try {
+                scrubbedBody = JSON.parse(res.scrubbedText);
+            } catch (_) {
+                scrubbedBody = rawBody;
+            }
+            tokenMap = res.tokenMap;
+            telemetry = res.telemetry;
+        }
+
+        const psContext = {
+            ...context,
+            rawBody,
+            scrubbedBody,
+            tokenMap,
+            telemetry,
+            restore: (text) => restore(text, tokenMap),
+            createStreamRestorer: () => createStreamTokenRestorer(tokenMap)
+        };
+
+        return handler(req, psContext);
+    };
+}
+
+/**
+ * Express.js / Connect middleware for automatic zero-trust PII sanitization.
+ * Sanitizes req.body in local RAM and attaches req.psTokenMap and req.restore(aiResponse).
+ * 
+ * @param {object} [options] - Sanitization options.
+ * @returns {Function} Express middleware (req, res, next)
+ */
+export function createExpressMiddleware(options = {}) {
+    return function privacyScrubberExpress(req, res, next) {
+        if (req && req.body && typeof req.body === 'object') {
+            const serialized = JSON.stringify(req.body);
+            const { scrubbedText, tokenMap, telemetry } = sanitize(serialized, options);
+            req.rawBody = req.body;
+            try {
+                req.body = JSON.parse(scrubbedText);
+            } catch (_) {}
+            req.psTokenMap = tokenMap;
+            req.psTelemetry = telemetry;
+            req.restore = (aiResponse) => restore(aiResponse, tokenMap);
+            req.createStreamRestorer = () => createStreamTokenRestorer(tokenMap);
+        }
+        if (typeof next === 'function') next();
+    };
+}
+
 /**
  * Generates an official Zero-Trust Compliance Audit Report.
  * @param {Record<string, string>} sessionMap - Map of token bindings.
@@ -921,6 +1521,7 @@ export const unscrubText = scrubberPkg.unscrubText;
 export default {
     sanitize,
     scrub,
+    clean,
     restore,
     unscrub,
     validateLicense,
@@ -928,10 +1529,18 @@ export default {
     buildAuditTelemetry,
     formatAuditReceipt,
     generateAuditReport,
+    generateProcurementMemo,
     PrivacyScrubberEngine,
     PrivacyScrubberLicenseError,
     wrapOpenAI,
     createLangChainTransform,
+    StreamingDetokenizer,
+    createStreamTokenRestorer,
+    createNodeStreamTokenRestorer,
+    buildStreamReplacer,
+    wrapAiStream,
+    withPrivacyScrubber,
+    createExpressMiddleware,
     createSanitizeStream,
     applyPatch,
     restoreToFile,