@privacyscrubber/sdk 2.2.4 → 2.2.6

package/examples/langchain-rag.js

@@ -0,0 +1,81 @@
+/**
+ * @privacyscrubber/sdk - LangChain RAG Ingestion & Synthesis Recipe
+ *
+ * Demonstrates:
+ * 1. Pre-ingestion document transformation with createDocumentTransformer.
+ * 2. Neutralizing PII in pageContent and sensitive metadata keys (e.g. author, user_id).
+ * 3. Preserving vector embedding semantic fidelity with deterministic tokens.
+ * 4. In-memory re-hydration of synthesized LLM answers with zero data leakage.
+ *
+ * Usage:
+ *   node npm-sdk/examples/langchain-rag.js
+ */
+
+import { createDocumentTransformer } from '../index.js';
+
+async function main() {
+    console.log('=== PrivacyScrubber SDK: LangChain RAG Recipe ===\n');
+
+    // 1. Initialize the LangChain Document Transformer
+    // Zero network egress: operates 100% in local process memory (<1ms per document)
+    const transformer = createDocumentTransformer({
+        profile: 'General',
+        sensitiveMetadataKeys: ['account_owner', 'submitter_email'],
+        attachTelemetry: true,
+        attachTokenMap: false // Do not leak token mappings into the vector database
+    });
+
+    // 2. Sample raw documents loaded from CRM / internal ticket logs
+    const rawDocs = [
+        {
+            pageContent: 'Account Executive John Doe (john.doe@enterprise.com) finalized deal for client Acme Corp under SSN 000-12-3456.',
+            metadata: {
+                docId: 'doc_101',
+                department: 'Enterprise Sales',
+                account_owner: 'john.doe@enterprise.com',
+                source: 'Salesforce'
+            }
+        },
+        {
+            pageContent: 'Technical escalation for customer Jane Smith. Direct contact phone is +1-555-019-2834, email janes@corp.org.',
+            metadata: {
+                docId: 'doc_102',
+                department: 'Customer Success',
+                submitter_email: 'janes@corp.org',
+                source: 'Zendesk'
+            }
+        }
+    ];
+
+    console.log('--- 1. Raw Documents Before Sanitization ---');
+    console.log(JSON.stringify(rawDocs, null, 2));
+
+    // 3. Transform documents before vector embedding / storage
+    const sanitizedDocs = await transformer.transformDocuments(rawDocs);
+
+    console.log('\n--- 2. Sanitized Documents (Safe for Vector DB Ingestion) ---');
+    console.log(JSON.stringify(sanitizedDocs, null, 2));
+
+    // 4. Verify zero cleartext PII exists in the documents to be embedded
+    const containsRawEmail = sanitizedDocs.some(d => d.pageContent.includes('john.doe@enterprise.com'));
+    const containsRawSsn = sanitizedDocs.some(d => d.pageContent.includes('000-12-3456'));
+    console.log(`\nZero Cleartext Email Verified: ${!containsRawEmail}`);
+    console.log(`Zero Cleartext SSN Verified:   ${!containsRawSsn}`);
+
+    // 5. Simulate an LLM synthesizing an answer over retrieved tokenized chunks
+    // The LLM operates on the tokens without knowing the customer's true identity:
+    const mockLlmResponse = 'Based on document [doc_101], [NAME_1] ([EMAIL_1]) successfully closed the deal for Acme Corp. Contract verified under tax ID [ID_1].';
+    console.log('\n--- 3. LLM Answer Synthesized over Vector Space ---');
+    console.log(`Raw LLM Output: "${mockLlmResponse}"`);
+
+    // 6. Restore the answer in local memory for the authorized end user
+    const { restoredText } = transformer.restore(mockLlmResponse);
+    console.log('\n--- 4. Re-hydrated Answer (Local RAM Only) ---');
+    console.log(`Final Response: "${restoredText}"`);
+
+    // 7. Reset session when switching security context / tenant
+    transformer.resetSession();
+    console.log('\nSession token map purged. Memory footprint clean.');
+}
+
+main().catch(console.error);

package/examples/llamaindex-rag.js

@@ -0,0 +1,81 @@
+/**
+ * @privacyscrubber/sdk - LlamaIndex Node & Pipeline Transformation Recipe
+ *
+ * Demonstrates:
+ * 1. Sanitizing LlamaIndex TextNodes and Documents before VectorStoreIndex insertion.
+ * 2. Preserving node IDs, relationships, and structural metadata.
+ * 3. Preventing irreversible PII poisoning in vector embedding spaces.
+ * 4. Re-hydrating synthesized response text back to original cleartext.
+ *
+ * Usage:
+ *   node npm-sdk/examples/llamaindex-rag.js
+ */
+
+import { createLlamaIndexTransform } from '../index.js';
+
+async function main() {
+    console.log('=== PrivacyScrubber SDK: LlamaIndex Node Transform Recipe ===\n');
+
+    // 1. Initialize the LlamaIndex Transform
+    // Operates in-memory (<1ms per node) with zero external network dependencies
+    const transform = createLlamaIndexTransform({
+        profile: 'General',
+        detectSecrets: true
+    });
+
+    // 2. Simulated LlamaIndex Nodes / Ingestion Chunks
+    const nodes = [
+        {
+            id_: 'node-001',
+            text: 'Patient Alice Brown (phone: 555-839-2001, email: alice.brown@health-secure.org) visited Cardiology Clinic for routine ECG follow-up.',
+            metadata: {
+                doc_id: 'ehr_84920',
+                department: 'Cardiology',
+                facility: 'General Hospital'
+            }
+        },
+        {
+            id_: 'node-002',
+            text: 'Attending physician Dr. Robert House recorded notes: Contact assistant at 555-912-3847 or emergency line doc-house@internal.hospital.',
+            metadata: {
+                doc_id: 'ehr_84921',
+                department: 'Cardiology',
+                facility: 'General Hospital'
+            }
+        }
+    ];
+
+    console.log('--- 1. Raw LlamaIndex Nodes Prior to Index Ingestion ---');
+    console.log(JSON.stringify(nodes, null, 2));
+
+    // 3. Transform nodes through PrivacyScrubber
+    const sanitizedNodes = transform.transform(nodes);
+
+    console.log('\n--- 2. Sanitized Nodes (Vector-Index Ready) ---');
+    console.log(JSON.stringify(sanitizedNodes, null, 2));
+
+    // 4. Invariant checks:
+    // - Node IDs remain untouched
+    // - All PII replaced with reversible syntax tokens
+    // - Metadata flagged with ZTDS security tags
+    const containsRawEmail = sanitizedNodes.some(n => n.text.includes('alice.brown@health-secure.org'));
+    const preservedIds = sanitizedNodes[0].id_ === 'node-001' && sanitizedNodes[1].id_ === 'node-002';
+    console.log(`\nZero Cleartext Email in Embeddings: ${!containsRawEmail}`);
+    console.log(`Node IDs & Metadata Intact:          ${preservedIds}`);
+
+    // 5. Query phase: Vector store retrieves tokenized nodes; LLM generates answer
+    const mockLlmAnswer = 'The patient [NAME_1] (contact: [EMAIL_1], [PHONE_1]) is under the care of [NAME_2]. For questions, reach out via [EMAIL_2].';
+    console.log('\n--- 3. LLM Query Response over Tokenized Index ---');
+    console.log(`Raw LLM Synthesis: "${mockLlmAnswer}"`);
+
+    // 6. Restore original cleartext in local RAM for authorized user
+    const { restoredText } = transform.restore(mockLlmAnswer);
+    console.log('\n--- 4. Re-hydrated Query Response (Local RAM Only) ---');
+    console.log(`De-tokenized Output: "${restoredText}"`);
+
+    // 7. Reset session map
+    transform.resetSession();
+    console.log('\nLlamaIndex session map reset complete.');
+}
+
+main().catch(console.error);

package/examples/vector-guard-rag.js

@@ -0,0 +1,137 @@
+/**
+ * @privacyscrubber/sdk - Universal Vector Database Ingestion Guard Recipe
+ *
+ * Demonstrates:
+ * 1. Initializing createVectorIngestionGuard for Chroma, Pinecone, Qdrant, and Weaviate.
+ * 2. Columnar batch sanitization for Chroma ({ ids, documents, metadatas }).
+ * 3. Record-level sanitization for Pinecone and Qdrant ({ id, text/payload, metadata }).
+ * 4. Search query token alignment (query tokens matched to sanitized vector space tokens).
+ * 5. Transparent vector store proxy wrapping via wrapVectorStore().
+ * 6. Post-retrieval in-memory re-hydration with zero vector store PII retention.
+ *
+ * Usage:
+ *   node npm-sdk/examples/vector-guard-rag.js
+ */
+
+import { createVectorIngestionGuard } from '../index.js';
+
+async function main() {
+    console.log('=== PrivacyScrubber SDK: Universal Vector DB Ingestion Guard ===\n');
+
+    // 1. Initialize the Vector Ingestion Guard
+    // Operates 100% in-process RAM with zero network egress (<1ms per batch)
+    const guard = createVectorIngestionGuard({
+        profile: 'General',
+        sensitiveMetadataKeys: ['contractor_email', 'billing_contact'],
+        attachTelemetry: true
+    });
+
+    // -------------------------------------------------------------------------
+    // Scenario A: Chroma Columnar Ingestion ({ ids, documents, metadatas })
+    // -------------------------------------------------------------------------
+    console.log('--- 1. Chroma Columnar Batch Sanitization ---');
+    const chromaBatch = {
+        ids: ['doc-101', 'doc-102'],
+        documents: [
+            'Vendor agreement with Michael Scott (mscott@dundermifflin.com), SSN 000-44-5555.',
+            'Direct support line for Jim Halpert: +1-555-019-8765, email jhalpert@dundermifflin.com.'
+        ],
+        metadatas: [
+            { branch: 'Scranton', contractor_email: 'temp.auditor@external.org' },
+            { branch: 'Scranton', tier: 'Enterprise' }
+        ]
+    };
+
+    const cleanChroma = guard.sanitizeRecords(chromaBatch);
+    console.log('Sanitized Chroma documents:');
+    console.log(cleanChroma.documents);
+    console.log('Sanitized Chroma metadatas:');
+    console.log(cleanChroma.metadatas);
+
+    // -------------------------------------------------------------------------
+    // Scenario B: Pinecone / Qdrant Record Array Ingestion
+    // -------------------------------------------------------------------------
+    console.log('\n--- 2. Pinecone & Qdrant Record Sanitization ---');
+    const records = [
+        {
+            id: 'vec-201',
+            text: 'Employee record: Pam Beesly (pamb@office.com), personal mobile 555-432-1098.',
+            metadata: { role: 'Office Administrator', department: 'Operations' }
+        },
+        {
+            id: 'vec-202',
+            payload: {
+                text: 'Executive escalation from Dwight Schrute (dschrute@schrutefarms.com).',
+                region: 'PA'
+            }
+        }
+    ];
+
+    const cleanRecords = guard.sanitizeRecords(records);
+    console.log('Sanitized Pinecone record:');
+    console.log(cleanRecords[0]);
+    console.log('Sanitized Qdrant payload:');
+    console.log(cleanRecords[1]);
+
+    // -------------------------------------------------------------------------
+    // Scenario C: Search Query Token Alignment
+    // When a user searches for an email or SSN that was previously ingested,
+    // the query must be mapped to the identical ZTDS token before vector embedding.
+    // -------------------------------------------------------------------------
+    console.log('\n--- 3. Search Query Token Alignment ---');
+    const rawUserQuery = 'Find agreements involving mscott@dundermifflin.com and audit history';
+    const alignedQuery = guard.sanitizeQuery(rawUserQuery);
+
+    console.log('Raw user query:     ', rawUserQuery);
+    console.log('Aligned query:      ', alignedQuery.query);
+    console.log('Tokens detected:    ', alignedQuery.tokensDetected);
+
+    // -------------------------------------------------------------------------
+    // Scenario D: Transparent Vector Store Proxy Wrapping
+    // wrapVectorStore intercepts addDocuments/upsert and similaritySearch/query
+    // to sanitize pre-write and re-hydrate post-read automatically.
+    // -------------------------------------------------------------------------
+    console.log('\n--- 4. Transparent Vector Store Proxy Wrapping ---');
+
+    // Mock client simulating Chroma / Pinecone / LangChain VectorStore
+    const nativeVectorStore = {
+        storage: [],
+        async addDocuments(docs) {
+            this.storage.push(...docs);
+            return { count: docs.length };
+        },
+        async similaritySearch(query) {
+            // Emulate cosine similarity retrieval against vector store text
+            return this.storage.filter(d => (d.pageContent || d.text || '').includes(query));
+        }
+    };
+
+    const guardedStore = guard.wrapVectorStore(nativeVectorStore);
+
+    // Ingest through transparent guard
+    await guardedStore.addDocuments([
+        {
+            pageContent: 'Restricted memo: Contact CFO David Wallace at dwallace@cfo-network.com regarding Q4 EBITDA.',
+            metadata: { classification: 'Confidential' }
+        }
+    ]);
+
+    // Verify vector store internal memory contains ZERO plaintext PII
+    const rawStored = nativeVectorStore.storage[nativeVectorStore.storage.length - 1];
+    console.log('Physical vector store entry (Zero-Trust RAM):');
+    console.log('  pageContent:', rawStored.pageContent);
+    console.log('  Contains raw email:', rawStored.pageContent.includes('dwallace@cfo-network.com')); // false
+
+    // Query through transparent guard: query is tokenized, retrieved results are automatically re-hydrated
+    const searchResults = await guardedStore.similaritySearch('dwallace@cfo-network.com');
+    console.log('\nRetrieved & Re-hydrated Search Result:');
+    console.log('  pageContent:', searchResults[0].pageContent);
+    console.log('  Contains authentic email:', searchResults[0].pageContent.includes('dwallace@cfo-network.com')); // true
+
+    console.log('\nUniversal Vector Ingestion Guard recipe completed with 0 bytes network egress.');
+}
+
+main().catch(err => {
+    console.error('Vector Guard Recipe Error:', err);
+    process.exit(1);
+});

package/index.d.ts

@@ -136,6 +136,62 @@ export function validateLicenseKey(key: string): 'PRO' | 'TEAMS' | 'SDK' | 'ENTE
 export function buildAuditTelemetry(currentTokenMap?: Record<string, string> | string[]): AuditTelemetry;
 export function formatAuditReceipt(telemetry: AuditTelemetry): string;
 
+export interface AuditEventData {
+    timestamp?: string;
+    version?: string;
+    protocol?: string;
+    event?: string;
+    riskLevel?: string;
+    tokensMasked?: number;
+    entities?: Record<string, number>;
+    frameworks?: string[];
+    sessionHash?: string;
+    source?: string;
+    action?: string;
+    organization?: string;
+    department?: string;
+    executionMs?: number;
+    metadata?: Record<string, any>;
+    [key: string]: any;
+}
+
+export function formatCefEvent(eventData?: AuditEventData): string;
+export function formatSyslogEvent(eventData?: AuditEventData): string;
+export function formatJsonlEvent(eventData?: AuditEventData): string;
+
+export interface AuditLoggerOptions {
+    format?: 'jsonl' | 'cef' | 'syslog' | 'json';
+    destination?: string | any;
+    organization?: string;
+    department?: string;
+}
+
+export interface AuditLogger {
+    formatEvent(eventData: AuditEventData): string;
+    logSanitization(result: SanitizeResult | Record<string, string>, context?: any): string;
+    logEvent(eventData: AuditEventData): string;
+    log(eventData: AuditEventData): string;
+    getEntries(): string[];
+    getBuffer(): string[];
+    clear(): void;
+    clearBuffer(): void;
+    format: string;
+    destination: any;
+}
+
+export function createAuditLogger(options?: AuditLoggerOptions): AuditLogger;
+
+export interface GenerateAuditReportOptions {
+    companyName?: string;
+    department?: string;
+    format?: 'markdown' | 'json' | 'summary' | 'cef' | 'syslog' | 'jsonl';
+}
+
+export function generateAuditReport(
+    sessionMap?: Record<string, string>,
+    options?: GenerateAuditReportOptions
+): any;
+
 export interface ProcurementMemoOptions {
     /** Target application or pipeline name (Default: 'Internal AI Applications & LLM Pipelines'). */
     appName?: string;
@@ -202,6 +258,69 @@ export function createLangChainTransform(options?: SanitizeOptions): {
     postprocess: (output: string, tokenMap: Record<string, string>) => RestoreResult;
 };
 
+export interface LangChainDocument {
+    pageContent: string;
+    metadata?: Record<string, any>;
+    [key: string]: any;
+}
+
+export interface DocumentTransformerOptions extends SanitizeOptions {
+    sensitiveMetadataKeys?: string[];
+    attachTelemetry?: boolean;
+    attachTokenMap?: boolean;
+}
+
+export interface DocumentTransformer {
+    transformDocuments(documents: LangChainDocument[]): Promise<LangChainDocument[]>;
+    invoke(documents: LangChainDocument[]): Promise<LangChainDocument[]>;
+    sessionMap: Record<string, string>;
+    restore(output: string, tokenMap?: Record<string, string>): RestoreResult;
+    resetSession(): void;
+}
+
+export function createDocumentTransformer(options?: DocumentTransformerOptions): DocumentTransformer;
+export const createLangChainDocumentTransformer: typeof createDocumentTransformer;
+
+export interface LlamaIndexNode {
+    text?: string;
+    pageContent?: string;
+    getContent?: () => string;
+    setContent?: (text: string) => void;
+    metadata?: Record<string, any>;
+    [key: string]: any;
+}
+
+export interface LlamaIndexTransform {
+    transform<T extends LlamaIndexNode | LlamaIndexNode[]>(nodes: T): T;
+    sessionMap: Record<string, string>;
+    restore(text: string, map?: Record<string, string>): RestoreResult;
+    resetSession(): void;
+}
+
+export function createLlamaIndexTransform(options?: SanitizeOptions): LlamaIndexTransform;
+
+export interface VectorIngestionGuardOptions extends SanitizeOptions {
+    sensitiveMetadataKeys?: string[];
+    autoRestoreResults?: boolean;
+}
+
+export interface VectorIngestionGuard {
+    sanitizeRecords<T = any>(records: T): T;
+    sanitizeQuery(query: string): {
+        query: string;
+        rawQuery: string;
+        tokensDetected: number;
+        tokenMap: Record<string, string>;
+    };
+    restoreSearchResults<T = any>(results: T, customTokenMap?: Record<string, string> | null): T;
+    wrapVectorStore<T = any>(vectorStore: T): T;
+    sessionMap: Record<string, string>;
+    restore(text: string, map?: Record<string, string>): RestoreResult;
+    resetSession(): void;
+}
+
+export function createVectorIngestionGuard(options?: VectorIngestionGuardOptions): VectorIngestionGuard;
+
 export interface ApplyPatchOptions {
     tokenMap?: Record<string, string>;
     engine?: PrivacyScrubberEngine;
@@ -444,6 +563,10 @@ declare const defaultExport: {
     PrivacyScrubberLicenseError: typeof PrivacyScrubberLicenseError;
     wrapOpenAI: typeof wrapOpenAI;
     createLangChainTransform: typeof createLangChainTransform;
+    createDocumentTransformer: typeof createDocumentTransformer;
+    createLangChainDocumentTransformer: typeof createLangChainDocumentTransformer;
+    createLlamaIndexTransform: typeof createLlamaIndexTransform;
+    createVectorIngestionGuard: typeof createVectorIngestionGuard;
     createSanitizeStream: typeof createSanitizeStream;
     applyPatch: typeof applyPatch;
     restoreToFile: typeof restoreToFile;