@privacyscrubber/sdk 2.3.7 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +86 -28
- package/benchmark.js +347 -0
- package/browser.js +2 -2
- package/cli.js +54 -2
- package/examples/python-bridge/MANIFEST.in +5 -0
- package/examples/python-bridge/README.md +265 -0
- package/examples/python-bridge/bridge_daemon.cjs +241 -0
- package/examples/python-bridge/dist/privacyscrubber-2.3.9-py3-none-any.whl +0 -0
- package/examples/python-bridge/dist/privacyscrubber-2.3.9.tar.gz +0 -0
- package/examples/python-bridge/fastapi_example.py +206 -0
- package/examples/python-bridge/langchain_adapter.py +20 -0
- package/examples/python-bridge/langchain_example.py +126 -0
- package/examples/python-bridge/llamaindex_example.py +103 -0
- package/examples/python-bridge/privacyscrubber/__init__.py +42 -0
- package/examples/python-bridge/privacyscrubber/bridge_daemon.cjs +241 -0
- package/examples/python-bridge/privacyscrubber/cli.py +138 -0
- package/examples/python-bridge/privacyscrubber/client.py +542 -0
- package/examples/python-bridge/privacyscrubber/exceptions.py +26 -0
- package/examples/python-bridge/privacyscrubber/langchain.py +161 -0
- package/examples/python-bridge/privacyscrubber/types.py +52 -0
- package/examples/python-bridge/privacyscrubber.egg-info/PKG-INFO +311 -0
- package/examples/python-bridge/privacyscrubber.egg-info/SOURCES.txt +18 -0
- package/examples/python-bridge/privacyscrubber.egg-info/dependency_links.txt +1 -0
- package/examples/python-bridge/privacyscrubber.egg-info/entry_points.txt +3 -0
- package/examples/python-bridge/privacyscrubber.egg-info/requires.txt +18 -0
- package/examples/python-bridge/privacyscrubber.egg-info/top_level.txt +1 -0
- package/examples/python-bridge/privacyscrubber.py +36 -0
- package/examples/python-bridge/pyproject.toml +62 -0
- package/examples/python-bridge/setup.py +66 -0
- package/examples/python-bridge/test_python_bridge.py +320 -0
- package/examples/vercel-ai-stream.js +195 -0
- package/fastmcp.cjs +291 -2
- package/fastmcp.d.ts +18 -0
- package/fastmcp.js +4 -0
- package/index.cjs +211 -134
- package/index.js +211 -134
- package/package.json +15 -3
- package/ps-license-manager.js +1 -1
- package/ps-pii-engine.cjs +1 -1
- package/scrubber-core.cjs +1 -1
- package/vercel-ai.cjs +400 -0
- package/vercel-ai.d.ts +156 -0
- package/vercel-ai.js +36 -0
- package/vision.cjs +13 -5
- package/vision.js +13 -5
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
"""
|
|
2
|
+
PrivacyScrubber LangChain Adapter
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import copy
|
|
8
|
+
from typing import Any, Dict, List, Optional, Sequence, Union
|
|
9
|
+
|
|
10
|
+
from privacyscrubber.client import PrivacyScrubberClient
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
try:
|
|
14
|
+
from langchain_core.callbacks.base import BaseCallbackHandler
|
|
15
|
+
from langchain_core.documents import Document
|
|
16
|
+
from langchain_core.outputs import LLMResult
|
|
17
|
+
from langchain_core.runnables import RunnableLambda
|
|
18
|
+
LANGCHAIN_INSTALLED = True
|
|
19
|
+
except ImportError:
|
|
20
|
+
LANGCHAIN_INSTALLED = False
|
|
21
|
+
BaseCallbackHandler = object # type: ignore
|
|
22
|
+
Document = object # type: ignore
|
|
23
|
+
LLMResult = object # type: ignore
|
|
24
|
+
RunnableLambda = object # type: ignore
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class ZTDSCallbackHandler(BaseCallbackHandler):
|
|
28
|
+
"""
|
|
29
|
+
LangChain Callback Handler enforcing Zero-Trust Data Sanitization.
|
|
30
|
+
|
|
31
|
+
1. Intercepts outgoing prompts in on_llm_start -> replaces raw PII with surrogate tokens.
|
|
32
|
+
2. Stores ephemeral token mappings in RAM only (no disk, no network).
|
|
33
|
+
3. Intercepts incoming responses in on_llm_end -> restores cleartext values in RAM.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
def __init__(
|
|
37
|
+
self,
|
|
38
|
+
scrubber: Optional[PrivacyScrubberClient] = None,
|
|
39
|
+
profile: str = "General",
|
|
40
|
+
detect_secrets: bool = False,
|
|
41
|
+
verbose: bool = False
|
|
42
|
+
) -> None:
|
|
43
|
+
self.scrubber = scrubber or PrivacyScrubberClient(default_profile=profile)
|
|
44
|
+
self.profile = profile
|
|
45
|
+
self.detect_secrets = detect_secrets
|
|
46
|
+
self.verbose = verbose
|
|
47
|
+
self._session_token_maps: Dict[str, Dict[str, str]] = {}
|
|
48
|
+
|
|
49
|
+
def on_llm_start(
|
|
50
|
+
self,
|
|
51
|
+
serialized: Dict[str, Any],
|
|
52
|
+
prompts: List[str],
|
|
53
|
+
*,
|
|
54
|
+
run_id: Optional[Any] = None,
|
|
55
|
+
parent_run_id: Optional[Any] = None,
|
|
56
|
+
tags: Optional[List[str]] = None,
|
|
57
|
+
metadata: Optional[Dict[str, Any]] = None,
|
|
58
|
+
**kwargs: Any
|
|
59
|
+
) -> Any:
|
|
60
|
+
"""Sanitizes prompts in place before they are dispatched to the cloud LLM."""
|
|
61
|
+
key = str(run_id) if run_id else "default"
|
|
62
|
+
combined_token_map: Dict[str, str] = {}
|
|
63
|
+
|
|
64
|
+
for i, prompt in enumerate(prompts):
|
|
65
|
+
res = self.scrubber.sanitize(
|
|
66
|
+
prompt,
|
|
67
|
+
profile=self.profile,
|
|
68
|
+
detect_secrets=self.detect_secrets,
|
|
69
|
+
persist_session=True
|
|
70
|
+
)
|
|
71
|
+
prompts[i] = res.sanitized
|
|
72
|
+
combined_token_map.update(res.token_map)
|
|
73
|
+
if self.verbose and res.count > 0:
|
|
74
|
+
print(f"[ZTDS Guard] Sanitized {res.count} PII tokens for run {key}")
|
|
75
|
+
|
|
76
|
+
self._session_token_maps[key] = combined_token_map
|
|
77
|
+
|
|
78
|
+
def on_llm_end(
|
|
79
|
+
self,
|
|
80
|
+
response: Any,
|
|
81
|
+
*,
|
|
82
|
+
run_id: Optional[Any] = None,
|
|
83
|
+
parent_run_id: Optional[Any] = None,
|
|
84
|
+
tags: Optional[List[str]] = None,
|
|
85
|
+
**kwargs: Any
|
|
86
|
+
) -> Any:
|
|
87
|
+
"""Restores cleartext tokens in LLM output before reaching the user."""
|
|
88
|
+
key = str(run_id) if run_id else "default"
|
|
89
|
+
token_map = self._session_token_maps.pop(key, None)
|
|
90
|
+
|
|
91
|
+
generations = getattr(response, "generations", None)
|
|
92
|
+
if generations and isinstance(generations, list):
|
|
93
|
+
for gen_list in generations:
|
|
94
|
+
for gen in gen_list:
|
|
95
|
+
if hasattr(gen, "text") and gen.text:
|
|
96
|
+
restored = self.scrubber.restore(gen.text, token_map=token_map)
|
|
97
|
+
gen.text = restored.restored_text
|
|
98
|
+
elif hasattr(gen, "message") and hasattr(gen.message, "content"):
|
|
99
|
+
if isinstance(gen.message.content, str):
|
|
100
|
+
restored = self.scrubber.restore(gen.message.content, token_map=token_map)
|
|
101
|
+
gen.message.content = restored.restored_text
|
|
102
|
+
|
|
103
|
+
return response
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
class ZTDSDocumentTransformer:
|
|
107
|
+
"""
|
|
108
|
+
Sanitizes LangChain Documents before vector database ingestion (Pinecone, Qdrant, Chroma).
|
|
109
|
+
Guarantees no raw PII enters embeddings or persistent vector storage.
|
|
110
|
+
"""
|
|
111
|
+
|
|
112
|
+
def __init__(
|
|
113
|
+
self,
|
|
114
|
+
scrubber: Optional[PrivacyScrubberClient] = None,
|
|
115
|
+
profile: str = "General",
|
|
116
|
+
sensitive_metadata_keys: Optional[Sequence[str]] = None
|
|
117
|
+
) -> None:
|
|
118
|
+
self.scrubber = scrubber or PrivacyScrubberClient(default_profile=profile)
|
|
119
|
+
self.profile = profile
|
|
120
|
+
self.sensitive_metadata_keys = list(sensitive_metadata_keys or [])
|
|
121
|
+
|
|
122
|
+
def transform_documents(self, documents: Sequence[Any]) -> List[Any]:
|
|
123
|
+
"""Transforms documents, sanitizing page_content and sensitive metadata keys."""
|
|
124
|
+
transformed: List[Any] = []
|
|
125
|
+
for doc in documents:
|
|
126
|
+
doc_copy = copy.deepcopy(doc)
|
|
127
|
+
content = getattr(doc_copy, "page_content", "")
|
|
128
|
+
if isinstance(content, str) and content:
|
|
129
|
+
res = self.scrubber.sanitize(content, profile=self.profile)
|
|
130
|
+
doc_copy.page_content = res.sanitized
|
|
131
|
+
|
|
132
|
+
metadata = getattr(doc_copy, "metadata", None)
|
|
133
|
+
if isinstance(metadata, dict):
|
|
134
|
+
for k in self.sensitive_metadata_keys:
|
|
135
|
+
if k in metadata and isinstance(metadata[k], str):
|
|
136
|
+
res_meta = self.scrubber.sanitize(metadata[k], profile=self.profile)
|
|
137
|
+
metadata[k] = res_meta.sanitized
|
|
138
|
+
|
|
139
|
+
transformed.append(doc_copy)
|
|
140
|
+
return transformed
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def create_ztds_lcel_runnables(
|
|
144
|
+
scrubber: Optional[PrivacyScrubberClient] = None,
|
|
145
|
+
profile: str = "General"
|
|
146
|
+
):
|
|
147
|
+
"""
|
|
148
|
+
Creates pre-scrub and post-restore runnables for LCEL pipelines:
|
|
149
|
+
chain = scrub_runnable | prompt | model | parser | restore_runnable
|
|
150
|
+
"""
|
|
151
|
+
client = scrubber or PrivacyScrubberClient(default_profile=profile)
|
|
152
|
+
|
|
153
|
+
def scrub_func(input_text: str) -> str:
|
|
154
|
+
res = client.sanitize(str(input_text), profile=profile)
|
|
155
|
+
return res.sanitized
|
|
156
|
+
|
|
157
|
+
def restore_func(output_text: str) -> str:
|
|
158
|
+
restored = client.restore(str(output_text))
|
|
159
|
+
return restored.restored_text
|
|
160
|
+
|
|
161
|
+
return scrub_func, restore_func
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""
|
|
2
|
+
PrivacyScrubber Data Types & Response Models
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from typing import Any, Dict, List, Optional
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@dataclass
|
|
12
|
+
class SanitizeResult:
|
|
13
|
+
"""Result of a text sanitization operation."""
|
|
14
|
+
sanitized: str
|
|
15
|
+
token_map: Dict[str, str]
|
|
16
|
+
count: int = 0
|
|
17
|
+
execution_ms: float = 0.0
|
|
18
|
+
telemetry: Optional[Dict[str, Any]] = None
|
|
19
|
+
audit_receipt: Optional[str] = None
|
|
20
|
+
license: Optional[Dict[str, Any]] = None
|
|
21
|
+
matches: Optional[List[Dict[str, Any]]] = None
|
|
22
|
+
|
|
23
|
+
def __str__(self) -> str:
|
|
24
|
+
"""Allow direct string usage in prompts and string formatting."""
|
|
25
|
+
return self.sanitized
|
|
26
|
+
|
|
27
|
+
@property
|
|
28
|
+
def scrubbed_text(self) -> str:
|
|
29
|
+
"""Alias for sanitized text."""
|
|
30
|
+
return self.sanitized
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass
|
|
34
|
+
class RestoreResult:
|
|
35
|
+
"""Result of a token restoration operation."""
|
|
36
|
+
restored_text: str
|
|
37
|
+
restored_count: int = 0
|
|
38
|
+
|
|
39
|
+
def __str__(self) -> str:
|
|
40
|
+
"""Allow direct string usage in final responses."""
|
|
41
|
+
return self.restored_text
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass
|
|
45
|
+
class ObjectSanitizeResult:
|
|
46
|
+
"""Result of a structured dictionary or list sanitization operation."""
|
|
47
|
+
sanitized: Any
|
|
48
|
+
token_map: Dict[str, str]
|
|
49
|
+
count: int = 0
|
|
50
|
+
execution_ms: float = 0.0
|
|
51
|
+
telemetry: Optional[Dict[str, Any]] = None
|
|
52
|
+
audit_receipt: Optional[str] = None
|
|
@@ -0,0 +1,311 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: privacyscrubber
|
|
3
|
+
Version: 2.4.0
|
|
4
|
+
Summary: Zero-Trust Data Sanitization & In-Memory PII Shield for Python AI Pipelines (ZTDS RFC v1.0)
|
|
5
|
+
Home-page: https://privacyscrubber.com
|
|
6
|
+
Author: Ilya Sibiryakov (BrandMeWeb)
|
|
7
|
+
Author-email: "Ilya Sibiryakov (BrandMeWeb)" <contact@privacyscrubber.com>
|
|
8
|
+
License: Commercial / Proprietary
|
|
9
|
+
Project-URL: Homepage, https://privacyscrubber.com
|
|
10
|
+
Project-URL: Documentation, https://privacyscrubber.com/sdk/
|
|
11
|
+
Project-URL: Repository, https://github.com/moxno/privacyscrubber-mcp
|
|
12
|
+
Project-URL: Standard, https://datatracker.ietf.org/doc/draft-sibiryakov-ztds-protocol/
|
|
13
|
+
Keywords: pii,redaction,sanitization,zero-trust,ztds,langchain,fastapi,presidio,llm-security,gdpr,hipaa,ai-guardrail
|
|
14
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: Intended Audience :: Information Technology
|
|
17
|
+
Classifier: Topic :: Security
|
|
18
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
19
|
+
Classifier: Programming Language :: Python :: 3
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
24
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
25
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
26
|
+
Classifier: Operating System :: OS Independent
|
|
27
|
+
Requires-Python: >=3.8
|
|
28
|
+
Description-Content-Type: text/markdown
|
|
29
|
+
Provides-Extra: fastapi
|
|
30
|
+
Requires-Dist: fastapi>=0.100.0; extra == "fastapi"
|
|
31
|
+
Requires-Dist: uvicorn>=0.20.0; extra == "fastapi"
|
|
32
|
+
Requires-Dist: pydantic>=2.0.0; extra == "fastapi"
|
|
33
|
+
Provides-Extra: langchain
|
|
34
|
+
Requires-Dist: langchain-core>=0.1.0; extra == "langchain"
|
|
35
|
+
Provides-Extra: llamaindex
|
|
36
|
+
Requires-Dist: llama-index-core>=0.10.0; extra == "llamaindex"
|
|
37
|
+
Provides-Extra: all
|
|
38
|
+
Requires-Dist: fastapi>=0.100.0; extra == "all"
|
|
39
|
+
Requires-Dist: uvicorn>=0.20.0; extra == "all"
|
|
40
|
+
Requires-Dist: pydantic>=2.0.0; extra == "all"
|
|
41
|
+
Requires-Dist: langchain-core>=0.1.0; extra == "all"
|
|
42
|
+
Requires-Dist: llama-index-core>=0.10.0; extra == "all"
|
|
43
|
+
Dynamic: author
|
|
44
|
+
Dynamic: home-page
|
|
45
|
+
Dynamic: requires-python
|
|
46
|
+
|
|
47
|
+
# PrivacyScrubber Python Bridge (ZTDS Zero-Egress Sidecar)
|
|
48
|
+
|
|
49
|
+
Sub-millisecond, zero-dependency Python adapter for **PrivacyScrubber SDK** (`@privacyscrubber/sdk`).
|
|
50
|
+
|
|
51
|
+
Enforces **Zero-Trust Data Sanitization (ZTDS RFC v1.0)** across Python AI and data pipelines:
|
|
52
|
+
- **FastAPI / Flask / Django**: Pre-LLM sanitization and post-LLM in-memory rehydration.
|
|
53
|
+
- **LangChain**: `ZTDSCallbackHandler` and `ZTDSDocumentTransformer`.
|
|
54
|
+
- **LlamaIndex**: Pre-embedding chunk and metadata sanitization.
|
|
55
|
+
- **Raw OpenAI / Anthropic / Gemini SDKs**: Direct string and dictionary tokenization.
|
|
56
|
+
|
|
57
|
+
---
|
|
58
|
+
|
|
59
|
+
## Architectural Comparison: PrivacyScrubber vs. Microsoft Presidio
|
|
60
|
+
|
|
61
|
+
| Metric | Microsoft Presidio | PrivacyScrubber ZTDS Bridge |
|
|
62
|
+
| :--- | :--- | :--- |
|
|
63
|
+
| **Inference Latency** | ~35ms – 65ms per request | **~0.15ms – 0.40ms per request (<1ms SLA)** |
|
|
64
|
+
| **Memory Footprint** | 1.8 GB – 3.2 GB RAM (PyTorch / spaCy) | **< 25 MB RAM (Zero PyTorch / zero spaCy)** |
|
|
65
|
+
| **Python Dependencies** | `presidio-analyzer`, `presidio-anonymizer`, `spacy`, `torch` | **0 Dependencies (Python Standard Library Only)** |
|
|
66
|
+
| **Round-Trip Re-hydration** | Non-trivial (custom hash table mapping required) | **Native 100% Round-Trip (`client.restore()`)** |
|
|
67
|
+
| **Pre-Configured Profiles** | Basic PII only (requires custom YAML recognizers) | **30 Built-in Industry Profiles (HIPAA, Financial, Legal, HR, etc.)** |
|
|
68
|
+
| **Network Egress** | Potential telemetry unless air-gapped | **0.00 Bytes Network Egress (RAM-only volatile mapping)** |
|
|
69
|
+
| **Container Startup** | 15 – 30 seconds model loading time | **< 100 milliseconds** |
|
|
70
|
+
|
|
71
|
+
---
|
|
72
|
+
|
|
73
|
+
---
|
|
74
|
+
|
|
75
|
+
## Installation & Prerequisites
|
|
76
|
+
|
|
77
|
+
1. Ensure **Node.js** (v18+) and **Python** (3.8+) are available on your system.
|
|
78
|
+
2. Install `@privacyscrubber/sdk`:
|
|
79
|
+
```bash
|
|
80
|
+
npm install @privacyscrubber/sdk
|
|
81
|
+
```
|
|
82
|
+
3. Install the Python package via `pip`:
|
|
83
|
+
```bash
|
|
84
|
+
# Editable local development:
|
|
85
|
+
pip install -e npm-sdk/examples/python-bridge
|
|
86
|
+
|
|
87
|
+
# Or standard pip install:
|
|
88
|
+
pip install ./npm-sdk/examples/python-bridge
|
|
89
|
+
```
|
|
90
|
+
*Note: Zero external Python dependencies are required.*
|
|
91
|
+
|
|
92
|
+
---
|
|
93
|
+
|
|
94
|
+
## Command Line Interface (CLI)
|
|
95
|
+
|
|
96
|
+
The package bundles a terminal CLI utility `privacyscrubber` (alias: `ps-bridge`):
|
|
97
|
+
|
|
98
|
+
```bash
|
|
99
|
+
# Check ZTDS engine status and uptime
|
|
100
|
+
privacyscrubber ping
|
|
101
|
+
privacyscrubber status
|
|
102
|
+
|
|
103
|
+
# Sanitize text directly
|
|
104
|
+
privacyscrubber sanitize "Customer Alice Cooper (alice@hospital.org)" --profile Healthcare
|
|
105
|
+
|
|
106
|
+
# Pipe text through standard input:
|
|
107
|
+
echo "Analyze wire for John Doe john@corp.com" | privacyscrubber sanitize
|
|
108
|
+
|
|
109
|
+
# Output structured JSON with tokens and compliance telemetry:
|
|
110
|
+
privacyscrubber sanitize "Patient John Doe" --json
|
|
111
|
+
|
|
112
|
+
# Restore surrogate tokens:
|
|
113
|
+
privacyscrubber restore "Patient [NAME_1]" --token-map '{"[NAME_1]": "John Doe"}'
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
---
|
|
117
|
+
|
|
118
|
+
## Quickstart
|
|
119
|
+
|
|
120
|
+
### 1. Synchronous Client
|
|
121
|
+
|
|
122
|
+
```python
|
|
123
|
+
from privacyscrubber import PrivacyScrubber
|
|
124
|
+
|
|
125
|
+
# Initialize sidecar in local RAM (auto-starts daemon)
|
|
126
|
+
with PrivacyScrubber(default_profile="General") as ps:
|
|
127
|
+
# Sanitize prompt before sending to cloud LLM
|
|
128
|
+
result = ps.sanitize("Please review intake for patient John Doe (john@acme.com, phone +1-555-0199).")
|
|
129
|
+
|
|
130
|
+
print("Sanitized Prompt:", result.sanitized)
|
|
131
|
+
# Output: Please review intake for patient [NAME_1] ([EMAIL_1], phone [PHONE_1]).
|
|
132
|
+
|
|
133
|
+
# Simulate LLM operating strictly on tokens
|
|
134
|
+
ai_response = "I have noted records for [NAME_1] ([EMAIL_1])."
|
|
135
|
+
|
|
136
|
+
# Restore cleartext tokens in local RAM
|
|
137
|
+
restored = ps.restore(ai_response)
|
|
138
|
+
print("Restored Answer:", restored.restored_text)
|
|
139
|
+
# Output: I have noted records for John Doe (john@acme.com).
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
### 2. Asynchronous Client (FastAPI / asyncio)
|
|
143
|
+
|
|
144
|
+
```python
|
|
145
|
+
import asyncio
|
|
146
|
+
from privacyscrubber import AsyncPrivacyScrubber
|
|
147
|
+
|
|
148
|
+
async def main():
|
|
149
|
+
async with AsyncPrivacyScrubber(default_profile="Healthcare") as ps:
|
|
150
|
+
res = await ps.sanitize("Patient Alice Cooper (DOB: 1984-05-12, SSN: 000-11-2222).")
|
|
151
|
+
print("Sanitized:", res.sanitized)
|
|
152
|
+
# Patient [NAME_1] (DOB: [DATE_1], SSN: [ID_1]).
|
|
153
|
+
|
|
154
|
+
restored = await ps.restore("Prescription confirmed for [NAME_1].")
|
|
155
|
+
print("Restored:", restored.restored_text)
|
|
156
|
+
|
|
157
|
+
asyncio.run(main())
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
---
|
|
161
|
+
|
|
162
|
+
## Framework Integrations
|
|
163
|
+
|
|
164
|
+
### FastAPI Zero-Egress Proxy (`fastapi_example.py`)
|
|
165
|
+
|
|
166
|
+
Run an OpenAI-compatible proxy that guarantees no raw PII ever touches OpenAI or external model APIs:
|
|
167
|
+
|
|
168
|
+
```python
|
|
169
|
+
from fastapi import FastAPI
|
|
170
|
+
from privacyscrubber import AsyncPrivacyScrubber
|
|
171
|
+
|
|
172
|
+
app = FastAPI()
|
|
173
|
+
scrubber = AsyncPrivacyScrubber()
|
|
174
|
+
|
|
175
|
+
@app.on_event("startup")
|
|
176
|
+
async def on_startup():
|
|
177
|
+
await scrubber.start()
|
|
178
|
+
|
|
179
|
+
@app.on_event("shutdown")
|
|
180
|
+
async def on_shutdown():
|
|
181
|
+
await scrubber.close()
|
|
182
|
+
|
|
183
|
+
@app.post("/v1/chat/completions")
|
|
184
|
+
async def chat_proxy(payload: dict):
|
|
185
|
+
# 1. Sanitize user prompt in client memory (<0.2ms)
|
|
186
|
+
messages = payload.get("messages", [])
|
|
187
|
+
user_prompt = messages[-1]["content"]
|
|
188
|
+
|
|
189
|
+
sanitized = await scrubber.sanitize(user_prompt)
|
|
190
|
+
messages[-1]["content"] = sanitized.sanitized
|
|
191
|
+
|
|
192
|
+
# 2. Forward sanitized payload to cloud LLM (httpx call)
|
|
193
|
+
# raw_llm_response = await client.post("https://api.openai.com/v1/chat/completions", json=payload)
|
|
194
|
+
mock_llm_reply = f"Acknowledged request for {sanitized.sanitized}"
|
|
195
|
+
|
|
196
|
+
# 3. Restore cleartext in RAM before returning to end-user
|
|
197
|
+
restored = await scrubber.restore(mock_llm_reply, token_map=sanitized.token_map)
|
|
198
|
+
return {"choices": [{"message": {"role": "assistant", "content": restored.restored_text}}]}
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
### LangChain (`langchain_adapter.py`)
|
|
202
|
+
|
|
203
|
+
#### Callback Handler (Transparent Pre/Post LLM Guard)
|
|
204
|
+
```python
|
|
205
|
+
from langchain_openai import ChatOpenAI
|
|
206
|
+
from langchain_adapter import ZTDSCallbackHandler
|
|
207
|
+
|
|
208
|
+
llm = ChatOpenAI(
|
|
209
|
+
model="gpt-4o",
|
|
210
|
+
callbacks=[ZTDSCallbackHandler(profile="Financial")]
|
|
211
|
+
)
|
|
212
|
+
|
|
213
|
+
# Prompts are automatically sanitized before leaving process memory;
|
|
214
|
+
# LLM outputs are automatically restored before returning to your code.
|
|
215
|
+
response = llm.invoke("What is the credit rating for customer David Miller (SSN 987-65-4321)?")
|
|
216
|
+
print(response.content)
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
#### Vector Ingestion Guard (Pinecone, Chroma, Qdrant)
|
|
220
|
+
```python
|
|
221
|
+
from langchain_adapter import ZTDSDocumentTransformer
|
|
222
|
+
|
|
223
|
+
transformer = ZTDSDocumentTransformer(
|
|
224
|
+
profile="Financial",
|
|
225
|
+
sensitive_metadata_keys=["customer_email", "account_id"]
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
# Sanitizes both page_content and sensitive metadata dictionaries
|
|
229
|
+
safe_docs = transformer.transform_documents(raw_docs)
|
|
230
|
+
vector_store.add_documents(safe_docs)
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
---
|
|
234
|
+
|
|
235
|
+
## Nested Dictionary & JSON Sanitization
|
|
236
|
+
|
|
237
|
+
Sanitize complex structured payloads (REST API payloads, MongoDB documents, JSON logs):
|
|
238
|
+
|
|
239
|
+
```python
|
|
240
|
+
payload = {
|
|
241
|
+
"user_id": 9012,
|
|
242
|
+
"customer": {
|
|
243
|
+
"name": "Sarah Connor",
|
|
244
|
+
"email": "sconnor@cyberdyne.org",
|
|
245
|
+
"phones": ["+1-555-0199", "+1-555-0188"]
|
|
246
|
+
},
|
|
247
|
+
"status": "active"
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
# Recursively sanitizes strings, preserving keys and types intact:
|
|
251
|
+
result = ps.sanitize_object(payload)
|
|
252
|
+
print(result.sanitized)
|
|
253
|
+
# {
|
|
254
|
+
# "user_id": 9012,
|
|
255
|
+
# "customer": {
|
|
256
|
+
# "name": "[NAME_1]",
|
|
257
|
+
# "email": "[EMAIL_1]",
|
|
258
|
+
# "phones": ["[PHONE_1]", "[PHONE_2]"]
|
|
259
|
+
# },
|
|
260
|
+
# "status": "active"
|
|
261
|
+
# }
|
|
262
|
+
|
|
263
|
+
# Restore structure:
|
|
264
|
+
restored = ps.restore_object(result.sanitized, token_map=result.token_map)
|
|
265
|
+
```
|
|
266
|
+
|
|
267
|
+
---
|
|
268
|
+
|
|
269
|
+
## Production Docker Deployment
|
|
270
|
+
|
|
271
|
+
Deploying Python applications alongside the PrivacyScrubber sidecar in a single multi-runtime container:
|
|
272
|
+
|
|
273
|
+
```dockerfile
|
|
274
|
+
FROM node:22-bookworm-slim AS node-base
|
|
275
|
+
|
|
276
|
+
FROM python:3.11-slim-bookworm
|
|
277
|
+
|
|
278
|
+
# Copy Node.js runtime into Python container
|
|
279
|
+
COPY --from=node-base /usr/local/bin/node /usr/local/bin/node
|
|
280
|
+
COPY --from=node-base /usr/local/lib/node_modules /usr/local/lib/node_modules
|
|
281
|
+
|
|
282
|
+
WORKDIR /app
|
|
283
|
+
|
|
284
|
+
# Install SDK bridge
|
|
285
|
+
COPY package*.json ./
|
|
286
|
+
RUN npm install @privacyscrubber/sdk --production
|
|
287
|
+
|
|
288
|
+
# Install Python requirements
|
|
289
|
+
COPY requirements.txt ./
|
|
290
|
+
RUN pip install --no-cache-dir -r requirements.txt
|
|
291
|
+
|
|
292
|
+
COPY . .
|
|
293
|
+
|
|
294
|
+
CMD ["uvicorn", "fastapi_example:app", "--host", "0.0.0.0", "--port", "8000"]
|
|
295
|
+
```
|
|
296
|
+
|
|
297
|
+
---
|
|
298
|
+
|
|
299
|
+
## Running the Verification Suite
|
|
300
|
+
|
|
301
|
+
Run all tests, benchmarks, and concurrency assertions:
|
|
302
|
+
|
|
303
|
+
```bash
|
|
304
|
+
# Automated Test Suite (10 tests, 0.3s)
|
|
305
|
+
python3 test_python_bridge.py
|
|
306
|
+
|
|
307
|
+
# Interactive Examples
|
|
308
|
+
python3 fastapi_example.py
|
|
309
|
+
python3 langchain_example.py
|
|
310
|
+
python3 llamaindex_example.py
|
|
311
|
+
```
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
MANIFEST.in
|
|
2
|
+
README.md
|
|
3
|
+
bridge_daemon.cjs
|
|
4
|
+
pyproject.toml
|
|
5
|
+
setup.py
|
|
6
|
+
privacyscrubber/__init__.py
|
|
7
|
+
privacyscrubber/bridge_daemon.cjs
|
|
8
|
+
privacyscrubber/cli.py
|
|
9
|
+
privacyscrubber/client.py
|
|
10
|
+
privacyscrubber/exceptions.py
|
|
11
|
+
privacyscrubber/langchain.py
|
|
12
|
+
privacyscrubber/types.py
|
|
13
|
+
privacyscrubber.egg-info/PKG-INFO
|
|
14
|
+
privacyscrubber.egg-info/SOURCES.txt
|
|
15
|
+
privacyscrubber.egg-info/dependency_links.txt
|
|
16
|
+
privacyscrubber.egg-info/entry_points.txt
|
|
17
|
+
privacyscrubber.egg-info/requires.txt
|
|
18
|
+
privacyscrubber.egg-info/top_level.txt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
|
|
2
|
+
[all]
|
|
3
|
+
fastapi>=0.100.0
|
|
4
|
+
uvicorn>=0.20.0
|
|
5
|
+
pydantic>=2.0.0
|
|
6
|
+
langchain-core>=0.1.0
|
|
7
|
+
llama-index-core>=0.10.0
|
|
8
|
+
|
|
9
|
+
[fastapi]
|
|
10
|
+
fastapi>=0.100.0
|
|
11
|
+
uvicorn>=0.20.0
|
|
12
|
+
pydantic>=2.0.0
|
|
13
|
+
|
|
14
|
+
[langchain]
|
|
15
|
+
langchain-core>=0.1.0
|
|
16
|
+
|
|
17
|
+
[llamaindex]
|
|
18
|
+
llama-index-core>=0.10.0
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
privacyscrubber
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""
|
|
2
|
+
PrivacyScrubber ZTDS Client for Python
|
|
3
|
+
Zero-Trust Data Sanitization & Local PII Shield for LLM / AI Pipelines.
|
|
4
|
+
|
|
5
|
+
100% in-memory client-side execution. Zero cloud egress. Sub-millisecond latency.
|
|
6
|
+
RFC v1.0 standard-compliant Zero-Trust Data Sanitization engine.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from privacyscrubber import (
|
|
12
|
+
PrivacyScrubberClient,
|
|
13
|
+
PrivacyScrubber,
|
|
14
|
+
AsyncPrivacyScrubberClient,
|
|
15
|
+
AsyncPrivacyScrubber,
|
|
16
|
+
SanitizeResult,
|
|
17
|
+
RestoreResult,
|
|
18
|
+
ObjectSanitizeResult,
|
|
19
|
+
PrivacyScrubberError,
|
|
20
|
+
PrivacyScrubberLicenseError,
|
|
21
|
+
__version__,
|
|
22
|
+
)
|
|
23
|
+
from privacyscrubber.client import _find_daemon_path
|
|
24
|
+
|
|
25
|
+
__all__ = [
|
|
26
|
+
"PrivacyScrubberClient",
|
|
27
|
+
"PrivacyScrubber",
|
|
28
|
+
"AsyncPrivacyScrubberClient",
|
|
29
|
+
"AsyncPrivacyScrubber",
|
|
30
|
+
"SanitizeResult",
|
|
31
|
+
"RestoreResult",
|
|
32
|
+
"ObjectSanitizeResult",
|
|
33
|
+
"PrivacyScrubberError",
|
|
34
|
+
"PrivacyScrubberLicenseError",
|
|
35
|
+
"__version__",
|
|
36
|
+
]
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61.0", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "privacyscrubber"
|
|
7
|
+
version = "2.4.0"
|
|
8
|
+
description = "Zero-Trust Data Sanitization & In-Memory PII Shield for Python AI Pipelines (ZTDS RFC v1.0)"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.8"
|
|
11
|
+
license = { text = "Commercial / Proprietary" }
|
|
12
|
+
authors = [
|
|
13
|
+
{ name = "Ilya Sibiryakov (BrandMeWeb)", email = "contact@privacyscrubber.com" }
|
|
14
|
+
]
|
|
15
|
+
keywords = [
|
|
16
|
+
"pii", "redaction", "sanitization", "zero-trust", "ztds", "langchain",
|
|
17
|
+
"fastapi", "presidio", "llm-security", "gdpr", "hipaa", "ai-guardrail"
|
|
18
|
+
]
|
|
19
|
+
classifiers = [
|
|
20
|
+
"Development Status :: 5 - Production/Stable",
|
|
21
|
+
"Intended Audience :: Developers",
|
|
22
|
+
"Intended Audience :: Information Technology",
|
|
23
|
+
"Topic :: Security",
|
|
24
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
25
|
+
"Programming Language :: Python :: 3",
|
|
26
|
+
"Programming Language :: Python :: 3.8",
|
|
27
|
+
"Programming Language :: Python :: 3.9",
|
|
28
|
+
"Programming Language :: Python :: 3.10",
|
|
29
|
+
"Programming Language :: Python :: 3.11",
|
|
30
|
+
"Programming Language :: Python :: 3.12",
|
|
31
|
+
"Programming Language :: Python :: 3.13",
|
|
32
|
+
"Operating System :: OS Independent",
|
|
33
|
+
]
|
|
34
|
+
|
|
35
|
+
[project.urls]
|
|
36
|
+
Homepage = "https://privacyscrubber.com"
|
|
37
|
+
Documentation = "https://privacyscrubber.com/sdk/"
|
|
38
|
+
Repository = "https://github.com/moxno/privacyscrubber-mcp"
|
|
39
|
+
Standard = "https://datatracker.ietf.org/doc/draft-sibiryakov-ztds-protocol/"
|
|
40
|
+
|
|
41
|
+
[project.optional-dependencies]
|
|
42
|
+
fastapi = ["fastapi>=0.100.0", "uvicorn>=0.20.0", "pydantic>=2.0.0"]
|
|
43
|
+
langchain = ["langchain-core>=0.1.0"]
|
|
44
|
+
llamaindex = ["llama-index-core>=0.10.0"]
|
|
45
|
+
all = [
|
|
46
|
+
"fastapi>=0.100.0",
|
|
47
|
+
"uvicorn>=0.20.0",
|
|
48
|
+
"pydantic>=2.0.0",
|
|
49
|
+
"langchain-core>=0.1.0",
|
|
50
|
+
"llama-index-core>=0.10.0"
|
|
51
|
+
]
|
|
52
|
+
|
|
53
|
+
[project.scripts]
|
|
54
|
+
privacyscrubber = "privacyscrubber.cli:main"
|
|
55
|
+
ps-bridge = "privacyscrubber.cli:main"
|
|
56
|
+
|
|
57
|
+
[tool.setuptools.packages.find]
|
|
58
|
+
where = ["."]
|
|
59
|
+
include = ["privacyscrubber*"]
|
|
60
|
+
|
|
61
|
+
[tool.setuptools.package-data]
|
|
62
|
+
privacyscrubber = ["*.cjs", "bridge_daemon.cjs"]
|