@privacyscrubber/sdk 2.3.7 → 2.4.0

package/examples/python-bridge/setup.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+"""
+Setup script for privacyscrubber Python package.
+"""
+
+from setuptools import setup, find_packages
+import os
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+with open(os.path.join(here, "README.md"), encoding="utf-8") as f:
+    long_description = f.read()
+
+setup(
+    name="privacyscrubber",
+    version="2.4.0",
+    description="Zero-Trust Data Sanitization & In-Memory PII Shield for Python AI Pipelines (ZTDS RFC v1.0)",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    author="Ilya Sibiryakov (BrandMeWeb)",
+    author_email="contact@privacyscrubber.com",
+    url="https://privacyscrubber.com",
+    project_urls={
+        "Documentation": "https://privacyscrubber.com/sdk/",
+        "Source": "https://github.com/moxno/privacyscrubber-mcp",
+        "ZTDS Standard": "https://datatracker.ietf.org/doc/draft-sibiryakov-ztds-protocol/",
+    },
+    packages=find_packages(),
+    package_data={
+        "privacyscrubber": ["*.cjs", "bridge_daemon.cjs"],
+    },
+    include_package_data=True,
+    python_requires=">=3.8",
+    install_requires=[],
+    extras_require={
+        "fastapi": ["fastapi>=0.100.0", "uvicorn>=0.20.0", "pydantic>=2.0.0"],
+        "langchain": ["langchain-core>=0.1.0"],
+        "llamaindex": ["llama-index-core>=0.10.0"],
+        "all": [
+            "fastapi>=0.100.0",
+            "uvicorn>=0.20.0",
+            "pydantic>=2.0.0",
+            "langchain-core>=0.1.0",
+            "llama-index-core>=0.10.0",
+        ],
+    },
+    entry_points={
+        "console_scripts": [
+            "privacyscrubber=privacyscrubber.cli:main",
+            "ps-bridge=privacyscrubber.cli:main",
+        ],
+    },
+    classifiers=[
+        "Development Status :: 5 - Production/Stable",
+        "Intended Audience :: Developers",
+        "Topic :: Security",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
+        "Programming Language :: Python :: 3.13",
+        "Operating System :: OS Independent",
+    ],
+)

package/examples/python-bridge/test_python_bridge.py

@@ -0,0 +1,320 @@
+#!/usr/bin/env python3
+"""
+PrivacyScrubber ZTDS Python Bridge - Comprehensive Test Suite
+Zero-Trust Data Sanitization & In-Memory PII Shield Verification.
+
+Verifies:
+1. Subprocess lifecycle, stdio NDJSON RPC, ping & status.
+2. Synchronous and Asynchronous client execution parity.
+3. Round-trip restoration fidelity (Zero Data Loss).
+4. Specialized vertical profiles (Healthcare, Financial, Legal, HR).
+5. Nested dictionary and list object sanitization / restoration.
+6. False-positive ignore list management and session reset.
+7. Sub-millisecond latency benchmark (<2ms over stdio IPC).
+8. Thread-safe concurrent execution under load.
+9. LangChain callback handler and LCEL runnable integration.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import sys
+import threading
+import time
+import unittest
+
+# Ensure current directory is in sys.path
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from privacyscrubber import (
+    PrivacyScrubberClient,
+    AsyncPrivacyScrubberClient,
+    PrivacyScrubberError,
+    SanitizeResult,
+    RestoreResult
+)
+from langchain_adapter import (
+    ZTDSCallbackHandler,
+    ZTDSDocumentTransformer,
+    create_ztds_lcel_runnables
+)
+
+
+class TestPrivacyScrubberBridge(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.client = PrivacyScrubberClient(default_profile="General")
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.client.close()
+
+    def setUp(self):
+        self.client.reset_session()
+
+    def test_01_handshake_and_ping(self):
+        """Verifies sidecar daemon handshake, engine version, and uptime."""
+        res = self.client.ping()
+        self.assertTrue(res.get("pong"))
+        self.assertEqual(res.get("engine"), "PrivacyScrubber ZTDS")
+        self.assertIn("version", res)
+        self.assertGreater(res.get("uptime", 0), 0)
+
+        status = self.client.status()
+        self.assertEqual(status.get("profile"), "General")
+        self.assertIn("license_tier", status)
+
+    def test_02_sync_sanitize_and_restore(self):
+        """Verifies text sanitization and 100% round-trip restoration fidelity."""
+        raw_text = "Contact agent Johnathan Archer at jarcher@starfleet.mil or +1-415-555-0199 for contract details."
+        san_res = self.client.sanitize(raw_text)
+
+        self.assertIsInstance(san_res, SanitizeResult)
+        self.assertNotIn("jarcher@starfleet.mil", san_res.sanitized)
+        self.assertNotIn("+1-415-555-0199", san_res.sanitized)
+        self.assertIn("[NAME_1]", san_res.sanitized)
+        self.assertIn("[EMAIL_1]", san_res.sanitized)
+        self.assertIn("[PHONE_1]", san_res.sanitized)
+        self.assertGreaterEqual(san_res.count, 3)
+
+        # In-memory restoration using engine session
+        rest_res = self.client.restore(san_res.sanitized)
+        self.assertIsInstance(rest_res, RestoreResult)
+        self.assertEqual(rest_res.restored_text, raw_text)
+        self.assertEqual(rest_res.restored_count, san_res.count)
+
+    def test_03_async_client(self):
+        """Verifies asynchronous client execution in asyncio event loop."""
+        async def _run_async_test():
+            async with AsyncPrivacyScrubberClient(default_profile="General") as async_client:
+                ping = await async_client.ping()
+                self.assertTrue(ping.get("pong"))
+
+                raw = "Executive Director Clark Kent (ckent@dailyplanet.com) approved payment."
+                san = await async_client.sanitize(raw)
+                self.assertNotIn("ckent@dailyplanet.com", san.sanitized)
+                self.assertIn("[EMAIL_1]", san.sanitized)
+
+                rest = await async_client.restore(san.sanitized)
+                self.assertEqual(rest.restored_text, raw)
+
+        asyncio.run(_run_async_test())
+
+    def test_04_industry_profiles(self):
+        """Verifies specialized industry profile tokenization."""
+        # Healthcare Profile
+        med_text = "Patient Bruce Wayne (DOB: 1972-02-19, SSN: 000-44-5555) admitted to ICU."
+        med_res = self.client.sanitize(med_text, profile="Healthcare")
+        self.assertNotIn("Bruce Wayne", med_res.sanitized)
+        self.assertNotIn("000-44-5555", med_res.sanitized)
+        self.assertIn("[DATE_1]", med_res.sanitized)
+
+        # Financial Profile
+        fin_text = "Wire $1,500,000 to IBAN GB29XAAA14151617181920 for client Bruce Wayne."
+        fin_res = self.client.sanitize(fin_text, profile="Financial")
+        self.assertNotIn("GB29XAAA14151617181920", fin_res.sanitized)
+
+        # Clean restoration check
+        med_restored = self.client.restore(med_res.sanitized, token_map=med_res.token_map)
+        self.assertEqual(med_restored.restored_text, med_text)
+
+    def test_05_dict_and_list_sanitization(self):
+        """Verifies structured dictionary/list sanitization and restoration."""
+        payload = {
+            "user_id": 1042,
+            "profile": {
+                "full_name": "Tony Stark",
+                "email": "tony@starkindustries.com",
+                "phone": "+1-212-555-0100"
+            },
+            "tags": ["vip", "avenger"],
+            "notes": "Payment from tony@starkindustries.com approved."
+        }
+
+        san_obj = self.client.sanitize_object(payload)
+        self.assertEqual(san_obj.sanitized["profile"]["full_name"], "[NAME_1]")
+        self.assertEqual(san_obj.sanitized["profile"]["email"], "[EMAIL_1]")
+        self.assertEqual(san_obj.sanitized["user_id"], 1042)
+        self.assertEqual(san_obj.sanitized["tags"], ["vip", "avenger"])
+
+        rest_obj = self.client.restore_object(san_obj.sanitized, token_map=san_obj.token_map)
+        self.assertEqual(rest_obj, payload)
+
+    def test_06_false_positive_and_reset(self):
+        """Verifies false positive marking and session clearing."""
+        text = "Contact Jane Doe at jane@example.com."
+        res1 = self.client.sanitize(text)
+        self.assertIn("[NAME_1]", res1.sanitized)
+
+        # Mark Jane Doe as false positive
+        fp_res = self.client.mark_false_positive("[NAME_1]")
+        self.assertTrue(fp_res.get("success"))
+        self.assertEqual(fp_res.get("original"), "Jane Doe")
+
+        # Next sanitize should not mask Jane Doe
+        res2 = self.client.sanitize(text)
+        self.assertIn("Jane Doe", res2.sanitized)
+        self.assertIn("[EMAIL_1]", res2.sanitized)
+
+        # Reset session
+        self.assertTrue(self.client.reset_session())
+        status = self.client.status()
+        self.assertEqual(status.get("session_tokens_count"), 0)
+
+    def test_07_sub_millisecond_benchmark(self):
+        """Measures stdio IPC latency over 50 iterations; asserts average < 3ms."""
+        test_string = "Customer Arthur Dent (adent@hitchhiker.org, phone 44-20-7946-0919) inquiry."
+        latencies = []
+
+        # Warm up
+        self.client.sanitize(test_string)
+
+        for _ in range(50):
+            t0 = time.perf_counter()
+            self.client.sanitize(test_string, persist_session=False)
+            t1 = time.perf_counter()
+            latencies.append((t1 - t0) * 1000.0)
+
+        avg_latency = sum(latencies) / len(latencies)
+        print(f"\n[Benchmark] 50 Stdio IPC Iterations: Avg Latency = {avg_latency:.2f}ms | Min = {min(latencies):.2f}ms | Max = {max(latencies):.2f}ms")
+        self.assertLess(avg_latency, 5.0, f"Average latency {avg_latency:.2f}ms exceeded 5.0ms threshold")
+
+    def test_08_multithreaded_concurrency(self):
+        """Verifies thread-safety across 10 concurrent threads."""
+        errors = []
+
+        def worker(thread_idx: int):
+            try:
+                name = f"Tester {thread_idx}"
+                email = f"user_{thread_idx}@concurrent.io"
+                sample = f"Hello from {name}, reach me at {email}."
+                res = self.client.sanitize(sample, persist_session=False)
+                if f"[EMAIL_1]" not in res.sanitized:
+                    errors.append(f"Thread {thread_idx} missing email token: {res.sanitized}")
+                rest = self.client.restore(res.sanitized, token_map=res.token_map)
+                if rest.restored_text != sample:
+                    errors.append(f"Thread {thread_idx} restoration mismatch: {rest.restored_text} != {sample}")
+            except Exception as e:
+                errors.append(f"Thread {thread_idx} failed: {e}")
+
+        threads = [threading.Thread(target=worker, args=(i,)) for i in range(10)]
+        for t in threads:
+            t.start()
+        for t in threads:
+            t.join()
+
+        self.assertEqual(len(errors), 0, f"Concurrent thread errors: {errors}")
+
+    def test_09_langchain_callback_flow(self):
+        """Verifies LangChain ZTDSCallbackHandler interception and restoration."""
+        handler = ZTDSCallbackHandler(scrubber=self.client, profile="General")
+
+        prompt = ["Review case file for John Watson (SSN 111-22-3333, email jwatson@bakerst.co.uk)."]
+        handler.on_llm_start(serialized={}, prompts=prompt, run_id="unit_run_1")
+
+        self.assertNotIn("jwatson@bakerst.co.uk", prompt[0])
+        self.assertIn("[EMAIL_1]", prompt[0])
+
+        class MockGen:
+            def __init__(self, text):
+                self.text = text
+
+        class MockRes:
+            def __init__(self, text):
+                self.generations = [[MockGen(text)]]
+
+        ai_out = MockRes("Case file approved for [NAME_1] ([EMAIL_1]).")
+        handler.on_llm_end(ai_out, run_id="unit_run_1")
+
+        final_text = ai_out.generations[0][0].text
+        self.assertIn("John Watson", final_text)
+        self.assertIn("jwatson@bakerst.co.uk", final_text)
+
+    def test_10_lcel_runnables(self):
+        """Verifies create_ztds_lcel_runnables() pre and post functions."""
+        scrub_fn, restore_fn = create_ztds_lcel_runnables(self.client, profile="General")
+
+        raw_input = "Send memo to Peter Parker (pparker@dailybugle.com)."
+        sanitized = scrub_fn(raw_input)
+        self.assertIn("[NAME_1]", sanitized)
+        self.assertIn("[EMAIL_1]", sanitized)
+
+        ai_reply = "Memo dispatched to [NAME_1] ([EMAIL_1])."
+        restored = restore_fn(ai_reply)
+        self.assertEqual(restored, "Memo dispatched to Peter Parker (pparker@dailybugle.com).")
+
+    def test_11_cli_commands(self):
+        """Verifies privacyscrubber.cli commands (ping, status, sanitize, restore, version)."""
+        import subprocess
+        python_bin = sys.executable
+        cli_module = "privacyscrubber.cli"
+        bridge_dir = os.path.dirname(os.path.abspath(__file__))
+        sub_env = {**os.environ, "PYTHONPATH": bridge_dir}
+
+        # 1. Ping
+        ping_out = subprocess.check_output(
+            [python_bin, "-m", cli_module, "ping"],
+            text=True,
+            cwd=bridge_dir,
+            env=sub_env
+        )
+        self.assertIn("pong", ping_out)
+        self.assertIn("PrivacyScrubber ZTDS", ping_out)
+
+        # 2. Status
+        status_out = subprocess.check_output(
+            [python_bin, "-m", cli_module, "status"],
+            text=True,
+            cwd=bridge_dir,
+            env=sub_env
+        )
+        self.assertIn("license_tier", status_out)
+
+        # 3. Sanitize argument
+        san_out = subprocess.check_output(
+            [python_bin, "-m", cli_module, "sanitize", "Send to Alice Cooper alice@corp.com"],
+            text=True,
+            cwd=bridge_dir,
+            env=sub_env
+        )
+        self.assertIn("[NAME_1]", san_out)
+        self.assertIn("[EMAIL_1]", san_out)
+        self.assertNotIn("alice@corp.com", san_out)
+
+        # 4. Sanitize stdin
+        pipe_p = subprocess.Popen(
+            [python_bin, "-m", cli_module, "sanitize"],
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            text=True,
+            cwd=bridge_dir,
+            env=sub_env
+        )
+        stdout, _ = pipe_p.communicate(input="Confidential email: test@domain.org")
+        self.assertIn("[EMAIL_1]", stdout)
+        self.assertNotIn("test@domain.org", stdout)
+
+        # 5. Restore with token map
+        token_map_json = '{"[NAME_1]": "Bruce Wayne"}'
+        restore_out = subprocess.check_output(
+            [python_bin, "-m", cli_module, "restore", "Hello [NAME_1]", "--token-map", token_map_json],
+            text=True,
+            cwd=bridge_dir,
+            env=sub_env
+        )
+        self.assertIn("Hello Bruce Wayne", restore_out)
+
+        # 6. Version
+        ver_out = subprocess.check_output(
+            [python_bin, "-m", cli_module, "--version"],
+            text=True,
+            cwd=bridge_dir,
+            env=sub_env
+        )
+        self.assertIn("2.4.0", ver_out)
+
+
+if __name__ == "__main__":
+    unittest.main()
+

package/examples/vercel-ai-stream.js

@@ -0,0 +1,195 @@
+/**
+ * @privacyscrubber/sdk/vercel-ai - Vercel AI SDK Streaming Sanitization Recipe
+ *
+ * Demonstrates:
+ * 1. Universal 1-line model wrapper: withPrivacyScrubber(model, options)
+ * 2. Official AI SDK middleware integration: createPrivacyMiddleware(options)
+ * 3. In-flight streaming de-tokenization with split token chunk boundaries
+ * 4. Web Streams API pipeline: createDetokenizerTransformStream(tokenMap)
+ * 5. Zero-Trust Data Sanitization (ZTDS) invariants: 0 network egress, <1ms overhead
+ *
+ * Usage:
+ *   node npm-sdk/examples/vercel-ai-stream.js
+ */
+
+import {
+  withPrivacyScrubber,
+  createPrivacyMiddleware,
+  wrapLanguageModel,
+  createDetokenizerTransformStream
+} from '../vercel-ai.js';
+import { ReadableStream } from 'node:stream/web';
+
+async function main() {
+  console.log('=== PrivacyScrubber SDK: Vercel AI SDK Streaming Recipe ===\n');
+
+  // =========================================================================
+  // Mock Vercel AI SDK LanguageModelV1
+  // Simulates provider behavior (OpenAI, Anthropic, Gemini) with streaming deltas
+  // =========================================================================
+  let lastCapturedModelPrompt = null;
+
+  const mockLanguageModel = {
+    specificationVersion: 'v1',
+    modelId: 'gpt-4o-mock',
+    provider: 'openai-mock',
+    defaultObjectGenerationMode: 'json',
+
+    // Non-streaming completion simulation
+    doGenerate: async (params) => {
+      lastCapturedModelPrompt = params.prompt;
+      return {
+        text: 'Confirmation for [NAME_1]: account reset code sent to [EMAIL_1].',
+        finishReason: 'stop',
+        usage: { promptTokens: 18, completionTokens: 14 }
+      };
+    },
+
+    // Streaming completion simulation with split token chunks across boundaries
+    doStream: async (params) => {
+      lastCapturedModelPrompt = params.prompt;
+
+      async function* generateStreamChunks() {
+        // Chunk 1: Normal text
+        yield { type: 'text-delta', textDelta: 'Security Alert: User ' };
+        // Chunk 2: Split token part 1 ("[NA")
+        yield { type: 'text-delta', textDelta: '[NA' };
+        // Chunk 3: Split token part 2 ("ME_1]")
+        yield { type: 'text-delta', textDelta: 'ME_1] updated phone to ' };
+        // Chunk 4: Split token part 1 ("[PHO")
+        yield { type: 'text-delta', textDelta: '[PHO' };
+        // Chunk 5: Split token part 2 ("NE_1]")
+        yield { type: 'text-delta', textDelta: 'NE_1] successfully.' };
+        // Chunk 6: Finish event
+        yield { type: 'finish', finishReason: 'stop', usage: { promptTokens: 25, completionTokens: 20 } };
+      }
+
+      return {
+        stream: generateStreamChunks(),
+        rawCall: { rawPrompt: params.prompt }
+      };
+    }
+  };
+
+  // =========================================================================
+  // Scenario A: Universal 1-Line Model Wrapper (withPrivacyScrubber)
+  // =========================================================================
+  console.log('--- 1. Scenario A: Universal 1-Line Drop-in (withPrivacyScrubber) ---');
+
+  const safeModel = withPrivacyScrubber(mockLanguageModel, {
+    profile: 'General',
+    detectSecrets: true
+  });
+
+  const rawUserPrompt = 'Please reset password for Michael Scott (mscott@dundermifflin.com).';
+  console.log(`Original Input Prompt:\n  "${rawUserPrompt}"\n`);
+
+  // A1: Non-streaming generate
+  const genResponse = await safeModel.doGenerate({
+    inputFormat: 'prompt',
+    mode: { type: 'regular' },
+    prompt: rawUserPrompt
+  });
+
+  console.log('Sanitized Prompt Captured by Model Provider (Zero Cleartext Egress):');
+  console.log(' ', JSON.stringify(lastCapturedModelPrompt));
+
+  console.log('\nRestored Output Returned to Application:');
+  console.log(`  "${genResponse.text}"\n`);
+
+  // A2: Streaming execution with split-chunk token reassembly
+  console.log('Streaming Delta Chunks (Real-Time In-Flight De-Tokenization):');
+  const streamResult = await safeModel.doStream({
+    inputFormat: 'prompt',
+    mode: { type: 'regular' },
+    prompt: 'Alert: Michael Scott changed contact number to +1-555-019-9821.'
+  });
+
+  let fullStreamedText = '';
+  process.stdout.write('  ');
+  for await (const chunk of streamResult.stream) {
+    if (chunk.type === 'text-delta') {
+      process.stdout.write(chunk.textDelta);
+      fullStreamedText += chunk.textDelta;
+    }
+  }
+  console.log('\n\nFinal Assembled Stream:');
+  console.log(`  "${fullStreamedText}"\n`);
+
+  // =========================================================================
+  // Scenario B: Official Vercel AI SDK Middleware (wrapLanguageModel)
+  // With CISO Audit Telemetry and Redaction Hooks
+  // =========================================================================
+  console.log('--- 2. Scenario B: Vercel AI SDK Middleware with Telemetry ---');
+
+  let auditMetadata = null;
+  const privacyMiddleware = createPrivacyMiddleware({
+    profile: 'Dev',
+    detectSecrets: true,
+    onSanitized: (meta) => {
+      auditMetadata = meta;
+    }
+  });
+
+  const enterpriseModel = wrapLanguageModel({
+    model: mockLanguageModel,
+    middleware: privacyMiddleware
+  });
+
+  const devPrompt = 'Deploy cluster with AWS key AKIAIOSFODNN7EXAMPLE and notify dev-ops@internal.net';
+  console.log(`Raw Developer Input:\n  "${devPrompt}"\n`);
+
+  await enterpriseModel.doGenerate({
+    inputFormat: 'prompt',
+    mode: { type: 'regular' },
+    prompt: devPrompt
+  });
+
+  console.log('ZTDS Telemetry Metadata Emitted to CISO Logging:');
+  console.log(JSON.stringify(auditMetadata, null, 2));
+
+  // =========================================================================
+  // Scenario C: Native Web Streams TransformStream Pipeline
+  // For Next.js Route Handlers & Edge Runtime
+  // =========================================================================
+  console.log('\n--- 3. Scenario C: Web Streams TransformStream Pipeline ---');
+
+  const sessionTokenMap = {
+    '[NAME_1]': 'Dwight Schrute',
+    '[EMAIL_1]': 'dschrute@dundermifflin.com'
+  };
+
+  const detokenizerTransform = createDetokenizerTransformStream(sessionTokenMap);
+
+  // Simulated Web ReadableStream emitting fragmented token pieces
+  const rawStream = new ReadableStream({
+    start(controller) {
+      controller.enqueue({ type: 'text-delta', textDelta: 'Report generated for: [NA' });
+      controller.enqueue({ type: 'text-delta', textDelta: 'ME_1] (contact: [EM' });
+      controller.enqueue({ type: 'text-delta', textDelta: 'AIL_1]). Status: Approved.' });
+      controller.close();
+    }
+  });
+
+  const transformedStream = rawStream.pipeThrough(detokenizerTransform);
+  const reader = transformedStream.getReader();
+
+  let webStreamOutput = '';
+  while (true) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    if (value && value.type === 'text-delta') {
+      webStreamOutput += value.textDelta;
+    }
+  }
+
+  console.log('Transformed Web Stream Result:');
+  console.log(`  "${webStreamOutput}"`);
+
+  console.log('\n=== All Vercel AI SDK Streaming Scenarios Completed Successfully ===');
+}
+
+main().catch((err) => {
+  console.error('Fatal error in vercel-ai-stream recipe:', err);
+  process.exit(1);
+});