catalist-support-agent 1.0.0

package/dist/extraction/deduplication.js

@@ -0,0 +1,271 @@
+/**
+ * Deduplication Module
+ *
+ * Detects duplicate Q&A pairs using content hashing and optional embedding similarity.
+ * Prevents storage of near-identical entries in the knowledge base.
+ */
+import { createHash } from 'crypto';
+import { createClient } from '@supabase/supabase-js';
+// =============================================================================
+// Default Configuration
+// =============================================================================
+const DEFAULT_CONFIG = {
+    similarityThreshold: 0.95, // Very high for near-exact matches
+    useEmbeddingSimilarity: false, // Start with hash-only for speed
+};
+// =============================================================================
+// Deduplication Service
+// =============================================================================
+export class DeduplicationService {
+    config;
+    supabase;
+    // In-memory cache of content hashes for batch operations
+    hashCache = new Set();
+    constructor(config) {
+        this.config = { ...DEFAULT_CONFIG, ...config };
+        this.supabase = createClient(config.supabaseUrl, config.supabaseServiceRoleKey);
+    }
+    /**
+     * Generate content hash for a Q&A pair
+     *
+     * Hash is based on normalized question + response text.
+     * Subject is not included as it varies more between duplicates.
+     */
+    generateContentHash(content) {
+        // Normalize text: lowercase, collapse whitespace, remove punctuation
+        const normalizedQuestion = this.normalizeText(content.questionText);
+        const normalizedResponse = this.normalizeText(content.responseText);
+        // Create composite text for hashing
+        const compositeText = `Q:${normalizedQuestion}|R:${normalizedResponse}`;
+        // Generate SHA-256 hash
+        return createHash('sha256').update(compositeText).digest('hex');
+    }
+    /**
+     * Check if content is a duplicate
+     */
+    async checkDuplicate(content) {
+        const contentHash = this.generateContentHash(content);
+        // 1. Check in-memory cache first (for batch operations)
+        if (this.hashCache.has(contentHash)) {
+            return {
+                isDuplicate: true,
+                method: 'hash',
+                contentHash,
+            };
+        }
+        // 2. Check database for hash match
+        const hashMatch = await this.checkHashInDatabase(contentHash);
+        if (hashMatch) {
+            return {
+                isDuplicate: true,
+                duplicateId: hashMatch.id,
+                method: 'hash',
+                contentHash,
+            };
+        }
+        // 3. Optionally check embedding similarity (more expensive)
+        if (this.config.useEmbeddingSimilarity) {
+            const embeddingMatch = await this.checkEmbeddingSimilarity(content);
+            if (embeddingMatch) {
+                return {
+                    isDuplicate: true,
+                    duplicateId: embeddingMatch.id,
+                    similarity: embeddingMatch.similarity,
+                    method: 'embedding',
+                    contentHash,
+                };
+            }
+        }
+        // Not a duplicate
+        return {
+            isDuplicate: false,
+            method: 'none',
+            contentHash,
+        };
+    }
+    /**
+     * Check for hash match in database
+     */
+    async checkHashInDatabase(contentHash) {
+        const { data, error } = await this.supabase
+            .from('gmail_knowledge_entries')
+            .select('id')
+            .eq('content_hash', contentHash)
+            .limit(1)
+            .single();
+        if (error || !data) {
+            return null;
+        }
+        return { id: data.id };
+    }
+    /**
+     * Check for similar entries using embedding similarity
+     * Note: Requires the entry to have an embedding
+     */
+    async checkEmbeddingSimilarity(_content) {
+        // This would require generating an embedding first
+        // For now, we'll skip this expensive operation
+        // The embedding-based dedup is better done as a post-processing step
+        return null;
+    }
+    /**
+     * Add hash to local cache (for batch operations)
+     */
+    addToCache(contentHash) {
+        this.hashCache.add(contentHash);
+    }
+    /**
+     * Clear local hash cache
+     */
+    clearCache() {
+        this.hashCache.clear();
+    }
+    /**
+     * Load existing hashes from database into cache
+     */
+    async loadHashCache(limit = 10000) {
+        const { data, error } = await this.supabase
+            .from('gmail_knowledge_entries')
+            .select('content_hash')
+            .limit(limit);
+        if (error || !data) {
+            return 0;
+        }
+        for (const entry of data) {
+            if (entry.content_hash) {
+                this.hashCache.add(entry.content_hash);
+            }
+        }
+        return data.length;
+    }
+    /**
+     * Batch check for duplicates
+     */
+    async checkBatchDuplicates(contents) {
+        const results = new Map();
+        // Generate all hashes first
+        const hashToContent = new Map();
+        for (const content of contents) {
+            const hash = this.generateContentHash(content);
+            hashToContent.set(hash, content);
+        }
+        const hashes = Array.from(hashToContent.keys());
+        // Batch query for existing hashes
+        const { data: existingEntries } = await this.supabase
+            .from('gmail_knowledge_entries')
+            .select('id, content_hash')
+            .in('content_hash', hashes);
+        const existingHashMap = new Map((existingEntries || []).map((e) => [e.content_hash, e.id]));
+        // Process each content
+        const seenHashes = new Set();
+        for (const content of contents) {
+            const hash = this.generateContentHash(content);
+            // Check if duplicate in current batch
+            if (seenHashes.has(hash)) {
+                results.set(hash, {
+                    isDuplicate: true,
+                    method: 'hash',
+                    contentHash: hash,
+                });
+                continue;
+            }
+            // Check if exists in database
+            const existingId = existingHashMap.get(hash);
+            if (existingId) {
+                results.set(hash, {
+                    isDuplicate: true,
+                    duplicateId: existingId,
+                    method: 'hash',
+                    contentHash: hash,
+                });
+                continue;
+            }
+            // Check in-memory cache
+            if (this.hashCache.has(hash)) {
+                results.set(hash, {
+                    isDuplicate: true,
+                    method: 'hash',
+                    contentHash: hash,
+                });
+                continue;
+            }
+            // Not a duplicate
+            results.set(hash, {
+                isDuplicate: false,
+                method: 'none',
+                contentHash: hash,
+            });
+            seenHashes.add(hash);
+        }
+        return results;
+    }
+    /**
+     * Normalize text for consistent hashing
+     */
+    normalizeText(text) {
+        return text
+            .toLowerCase()
+            .replace(/\s+/g, ' ') // Collapse whitespace
+            .replace(/[^\w\s]/g, '') // Remove punctuation
+            .trim()
+            .substring(0, 5000); // Limit length
+    }
+    /**
+     * Get cache statistics
+     */
+    getCacheStats() {
+        return { size: this.hashCache.size };
+    }
+    /**
+     * Update configuration
+     */
+    updateConfig(config) {
+        this.config = { ...this.config, ...config };
+    }
+}
+// =============================================================================
+// Singleton Instance
+// =============================================================================
+let deduplicationServiceInstance = null;
+/**
+ * Get singleton deduplication service instance
+ */
+export function getDeduplicationService() {
+    if (!deduplicationServiceInstance) {
+        const supabaseUrl = process.env.SUPABASE_URL || process.env.NEXT_PUBLIC_SUPABASE_URL;
+        const supabaseServiceRoleKey = process.env.SUPABASE_SERVICE_ROLE_KEY;
+        if (!supabaseUrl || !supabaseServiceRoleKey) {
+            throw new Error('SUPABASE_URL and SUPABASE_SERVICE_ROLE_KEY are required');
+        }
+        deduplicationServiceInstance = new DeduplicationService({
+            supabaseUrl,
+            supabaseServiceRoleKey,
+            similarityThreshold: 0.95,
+            useEmbeddingSimilarity: false,
+        });
+    }
+    return deduplicationServiceInstance;
+}
+/**
+ * Reset the singleton (for testing)
+ */
+export function resetDeduplicationService() {
+    deduplicationServiceInstance = null;
+}
+// =============================================================================
+// Utility Functions
+// =============================================================================
+/**
+ * Quick content hash generation
+ */
+export function hashContent(content) {
+    return getDeduplicationService().generateContentHash(content);
+}
+/**
+ * Quick duplicate check
+ */
+export async function isDuplicate(content) {
+    const result = await getDeduplicationService().checkDuplicate(content);
+    return result.isDuplicate;
+}
+//# sourceMappingURL=deduplication.js.map

package/dist/extraction/deduplication.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"deduplication.js","sourceRoot":"","sources":["../../src/extraction/deduplication.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,YAAY,EAAkB,MAAM,uBAAuB,CAAC;AAgCrE,gFAAgF;AAChF,wBAAwB;AACxB,gFAAgF;AAEhF,MAAM,cAAc,GAAwE;IAC1F,mBAAmB,EAAE,IAAI,EAAE,mCAAmC;IAC9D,sBAAsB,EAAE,KAAK,EAAE,iCAAiC;CACjE,CAAC;AAEF,gFAAgF;AAChF,wBAAwB;AACxB,gFAAgF;AAEhF,MAAM,OAAO,oBAAoB;IACvB,MAAM,CAAsB;IAC5B,QAAQ,CAAiB;IAEjC,yDAAyD;IACjD,SAAS,GAAgB,IAAI,GAAG,EAAE,CAAC;IAE3C,YAAY,MAA2B;QACrC,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,cAAc,EAAE,GAAG,MAAM,EAAE,CAAC;QAC/C,IAAI,CAAC,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,sBAAsB,CAAC,CAAC;IAClF,CAAC;IAED;;;;;OAKG;IACH,mBAAmB,CAAC,OAAkB;QACpC,qEAAqE;QACrE,MAAM,kBAAkB,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QACpE,MAAM,kBAAkB,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAEpE,oCAAoC;QACpC,MAAM,aAAa,GAAG,KAAK,kBAAkB,MAAM,kBAAkB,EAAE,CAAC;QAExE,wBAAwB;QACxB,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAClE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAAC,OAAkB;QACrC,MAAM,WAAW,GAAG,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;QAEtD,wDAAwD;QACxD,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;YACpC,OAAO;gBACL,WAAW,EAAE,IAAI;gBACjB,MAAM,EAAE,MAAM;gBACd,WAAW;aACZ,CAAC;QACJ,CAAC;QAED,mCAAmC;QACnC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,WAAW,CAAC,CAAC;QAC9D,IAAI,SAAS,EAAE,CAAC;YACd,OAAO;gBACL,WAAW,EAAE,IAAI;gBACjB,WAAW,EAAE,SAAS,CAAC,EAAE;gBACzB,MAAM,EAAE,MAAM;gBACd,WAAW;aACZ,CAAC;QACJ,CAAC;QAED,4DAA4D;QAC5D,IAAI,IAAI,CAAC,MAAM,CAAC,sBAAsB,EAAE,CAAC;YACvC,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,OAAO,CAAC,CAAC;YACpE,IAAI,cAAc,EAAE,CAAC;gBACnB,OAAO;oBACL,WAAW,EAAE,IAAI;oBACjB,WAAW,EAAE,cAAc,CAAC,EAAE;oBAC9B,UAAU,EAAE,cAAc,CAAC,UAAU;oBACrC,MAAM,EAAE,WAAW;oBACnB,WAAW;iBACZ,CAAC;YACJ,CAAC;QACH,CAAC;QAED,kBAAkB;QAClB,OAAO;YACL,WAAW,EAAE,KAAK;YAClB,MAAM,EAAE,MAAM;YACd,WAAW;SACZ,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,mBAAmB,CAC/B,WAAmB;QAEnB,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,QAAQ;aACxC,IAAI,CAAC,yBAAyB,CAAC;aAC/B,MAAM,CAAC,IAAI,CAAC;aACZ,EAAE,CAAC,cAAc,EAAE,WAAW,CAAC;aAC/B,KAAK,CAAC,CAAC,CAAC;aACR,MAAM,EAAE,CAAC;QAEZ,IAAI,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC;YACnB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC;IACzB,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,wBAAwB,CACpC,QAAmB;QAEnB,mDAAmD;QACnD,+CAA+C;QAC/C,qEAAqE;QAErE,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,WAAmB;QAC5B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,UAAU;QACR,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa,CAAC,QAAgB,KAAK;QACvC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,QAAQ;aACxC,IAAI,CAAC,yBAAyB,CAAC;aAC/B,MAAM,CAAC,cAAc,CAAC;aACtB,KAAK,CAAC,KAAK,CAAC,CAAC;QAEhB,IAAI,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC;YACnB,OAAO,CAAC,CAAC;QACX,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,IAAI,EAAE,CAAC;YACzB,IAAI,KAAK,CAAC,YAAY,EAAE,CAAC;gBACvB,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;YACzC,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,oBAAoB,CACxB,QAAqB;QAErB,MAAM,OAAO,GAAG,IAAI,GAAG,EAA+B,CAAC;QAEvD,4BAA4B;QAC5B,MAAM,aAAa,GAAG,IAAI,GAAG,EAAqB,CAAC;QACnD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;YAC/C,aAAa,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACnC,CAAC;QAED,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC;QAEhD,kCAAkC;QAClC,MAAM,EAAE,IAAI,EAAE,eAAe,EAAE,GAAG,MAAM,IAAI,CAAC,QAAQ;aAClD,IAAI,CAAC,yBAAyB,CAAC;aAC/B,MAAM,CAAC,kBAAkB,CAAC;aAC1B,EAAE,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;QAE9B,MAAM,eAAe,GAAG,IAAI,GAAG,CAC7B,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAC3D,CAAC;QAEF,uBAAuB;QACvB,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;QAErC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;YAE/C,sCAAsC;YACtC,IAAI,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACzB,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE;oBAChB,WAAW,EAAE,IAAI;oBACjB,MAAM,EAAE,MAAM;oBACd,WAAW,EAAE,IAAI;iBAClB,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YAED,8BAA8B;YAC9B,MAAM,UAAU,GAAG,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC7C,IAAI,UAAU,EAAE,CAAC;gBACf,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE;oBAChB,WAAW,EAAE,IAAI;oBACjB,WAAW,EAAE,UAAU;oBACvB,MAAM,EAAE,MAAM;oBACd,WAAW,EAAE,IAAI;iBAClB,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YAED,wBAAwB;YACxB,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE;oBAChB,WAAW,EAAE,IAAI;oBACjB,MAAM,EAAE,MAAM;oBACd,WAAW,EAAE,IAAI;iBAClB,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YAED,kBAAkB;YAClB,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE;gBAChB,WAAW,EAAE,KAAK;gBAClB,MAAM,EAAE,MAAM;gBACd,WAAW,EAAE,IAAI;aAClB,CAAC,CAAC;YAEH,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACvB,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACK,aAAa,CAAC,IAAY;QAChC,OAAO,IAAI;aACR,WAAW,EAAE;aACb,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,sBAAsB;aAC3C,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,qBAAqB;aAC7C,IAAI,EAAE;aACN,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,eAAe;IACxC,CAAC;IAED;;OAEG;IACH,aAAa;QACX,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;IACvC,CAAC;IAED;;OAEG;IACH,YAAY,CAAC,MAAoC;QAC/C,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC;IAC9C,CAAC;CACF;AAED,gFAAgF;AAChF,qBAAqB;AACrB,gFAAgF;AAEhF,IAAI,4BAA4B,GAAgC,IAAI,CAAC;AAErE;;GAEG;AACH,MAAM,UAAU,uBAAuB;IACrC,IAAI,CAAC,4BAA4B,EAAE,CAAC;QAClC,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;QACrF,MAAM,sBAAsB,GAAG,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC;QAErE,IAAI,CAAC,WAAW,IAAI,CAAC,sBAAsB,EAAE,CAAC;YAC5C,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAC7E,CAAC;QAED,4BAA4B,GAAG,IAAI,oBAAoB,CAAC;YACtD,WAAW;YACX,sBAAsB;YACtB,mBAAmB,EAAE,IAAI;YACzB,sBAAsB,EAAE,KAAK;SAC9B,CAAC,CAAC;IACL,CAAC;IAED,OAAO,4BAA4B,CAAC;AACtC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,yBAAyB;IACvC,4BAA4B,GAAG,IAAI,CAAC;AACtC,CAAC;AAED,gFAAgF;AAChF,oBAAoB;AACpB,gFAAgF;AAEhF;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,OAAkB;IAC5C,OAAO,uBAAuB,EAAE,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;AAChE,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,OAAkB;IAClD,MAAM,MAAM,GAAG,MAAM,uBAAuB,EAAE,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;IACvE,OAAO,MAAM,CAAC,WAAW,CAAC;AAC5B,CAAC"}

package/dist/extraction/gmail-extractor.d.ts

@@ -0,0 +1,160 @@
+/**
+ * Gmail Extractor Module
+ *
+ * Extracts Q&A pairs from Gmail threads for the knowledge base.
+ * Uses Gmail API to fetch threads and parses them to identify
+ * inbound questions and outbound responses.
+ */
+import { PIIHandler } from './pii-handler.js';
+import { QualityFilter } from './quality-filter.js';
+import { DeduplicationService } from './deduplication.js';
+import type { IntentCategory } from '../types.js';
+export interface GmailMessage {
+    id: string;
+    threadId: string;
+    internalDate: string;
+    snippet: string;
+    payload: {
+        headers: Array<{
+            name: string;
+            value: string;
+        }>;
+        mimeType?: string;
+        body?: {
+            data?: string;
+            size?: number;
+        };
+        parts?: Array<{
+            mimeType?: string;
+            body?: {
+                data?: string;
+                size?: number;
+            };
+            parts?: Array<{
+                mimeType?: string;
+                body?: {
+                    data?: string;
+                };
+            }>;
+        }>;
+    };
+    labelIds?: string[];
+}
+export interface GmailThread {
+    id: string;
+    historyId?: string;
+    messages: GmailMessage[];
+}
+export interface ExtractedQAPair {
+    gmailThreadId: string;
+    gmailMessageId: string;
+    questionSubject: string | null;
+    questionText: string;
+    responseText: string;
+    customerEmail: string;
+    responderEmail: string;
+    emailReceivedAt: Date;
+    emailRespondedAt: Date;
+    responseTimeMs: number;
+    intentCategory: IntentCategory | null;
+    qualityScore: number;
+    contentHash: string;
+    piiRedacted: boolean;
+    piiTypesFound: string[];
+    resolutionIndicator: string | null;
+}
+export interface ExtractionResult {
+    extracted: ExtractedQAPair[];
+    skipped: {
+        reason: string;
+        threadId: string;
+        messageId?: string;
+    }[];
+    stats: {
+        threadsProcessed: number;
+        messagesProcessed: number;
+        pairsExtracted: number;
+        duplicatesSkipped: number;
+        qualityFilteredOut: number;
+        errors: number;
+    };
+}
+export interface GmailExtractorConfig {
+    supabaseUrl: string;
+    supabaseServiceRoleKey: string;
+    salesEmailAddress: string;
+    preserveEmailDomains?: string[];
+}
+export declare class GmailExtractor {
+    private supabase;
+    private piiHandler;
+    private qualityFilter;
+    private deduplicationService;
+    private salesEmail;
+    constructor(config: GmailExtractorConfig);
+    /**
+     * Extract Q&A pairs from a list of Gmail threads
+     */
+    extractFromThreads(threads: GmailThread[]): Promise<ExtractionResult>;
+    /**
+     * Extract Q&A pairs from a single thread
+     */
+    private extractFromThread;
+    /**
+     * Get a header value from a Gmail message
+     */
+    private getHeader;
+    /**
+     * Extract email address from a From/To header value
+     */
+    private extractEmail;
+    /**
+     * Check if an email is from the company
+     */
+    private isCompanyEmail;
+    /**
+     * Extract plain text body from a Gmail message
+     */
+    private extractBody;
+    /**
+     * Find body part with specified MIME type
+     */
+    private findBodyPart;
+    /**
+     * Decode base64url encoded string
+     */
+    private decodeBase64Url;
+    /**
+     * Strip HTML tags from string
+     */
+    private stripHtml;
+    /**
+     * Clean email body (remove signatures, quoted replies, etc.)
+     */
+    private cleanEmailBody;
+    /**
+     * Clean subject line (remove Re:, Fwd:, etc.)
+     */
+    private cleanSubject;
+    /**
+     * Store extracted pairs in the database
+     */
+    storeExtractedPairs(pairs: ExtractedQAPair[], batchId?: string): Promise<{
+        stored: number;
+        errors: number;
+    }>;
+    /**
+     * Get deduplication service for external use
+     */
+    getDeduplicationService(): DeduplicationService;
+    /**
+     * Get quality filter for external use
+     */
+    getQualityFilter(): QualityFilter;
+    /**
+     * Get PII handler for external use
+     */
+    getPIIHandler(): PIIHandler;
+}
+export { PIIHandler, QualityFilter, DeduplicationService };
+//# sourceMappingURL=gmail-extractor.d.ts.map

package/dist/extraction/gmail-extractor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gmail-extractor.d.ts","sourceRoot":"","sources":["../../src/extraction/gmail-extractor.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAGH,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,aAAa,EAA4B,MAAM,qBAAqB,CAAC;AAC9E,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAMlD,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE;QACP,OAAO,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;QAChD,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,IAAI,CAAC,EAAE;YAAE,IAAI,CAAC,EAAE,MAAM,CAAC;YAAC,IAAI,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QACxC,KAAK,CAAC,EAAE,KAAK,CAAC;YACZ,QAAQ,CAAC,EAAE,MAAM,CAAC;YAClB,IAAI,CAAC,EAAE;gBAAE,IAAI,CAAC,EAAE,MAAM,CAAC;gBAAC,IAAI,CAAC,EAAE,MAAM,CAAA;aAAE,CAAC;YACxC,KAAK,CAAC,EAAE,KAAK,CAAC;gBAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;gBAAC,IAAI,CAAC,EAAE;oBAAE,IAAI,CAAC,EAAE,MAAM,CAAA;iBAAE,CAAA;aAAE,CAAC,CAAC;SAChE,CAAC,CAAC;KACJ,CAAC;IACF,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,YAAY,EAAE,CAAC;CAC1B;AAED,MAAM,WAAW,eAAe;IAC9B,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,IAAI,CAAC;IACtB,gBAAgB,EAAE,IAAI,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,cAAc,GAAG,IAAI,CAAC;IACtC,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,OAAO,CAAC;IACrB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC;AAED,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,eAAe,EAAE,CAAC;IAC7B,OAAO,EAAE;QACP,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,EAAE,CAAC;IACJ,KAAK,EAAE;QACL,gBAAgB,EAAE,MAAM,CAAC;QACzB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,cAAc,EAAE,MAAM,CAAC;QACvB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,kBAAkB,EAAE,MAAM,CAAC;QAC3B,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;CACH;AAED,MAAM,WAAW,oBAAoB;IACnC,WAAW,EAAE,MAAM,CAAC;IACpB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,oBAAoB,CAAC,EAAE,MAAM,EAAE,CAAC;CACjC;AAMD,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAiB;IACjC,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,aAAa,CAAgB;IACrC,OAAO,CAAC,oBAAoB,CAAuB;IACnD,OAAO,CAAC,UAAU,CAAS;gBAEf,MAAM,EAAE,oBAAoB;IAmBxC;;OAEG;IACG,kBAAkB,CAAC,OAAO,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAoF3E;;OAEG;YACW,iBAAiB;IAyG/B;;OAEG;IACH,OAAO,CAAC,SAAS;IAOjB;;OAEG;IACH,OAAO,CAAC,YAAY;IAOpB;;OAEG;IACH,OAAO,CAAC,cAAc;IAQtB;;OAEG;IACH,OAAO,CAAC,WAAW;IAqBnB;;OAEG;IACH,OAAO,CAAC,YAAY;IA6BpB;;OAEG;IACH,OAAO,CAAC,eAAe;IAMvB;;OAEG;IACH,OAAO,CAAC,SAAS;IAejB;;OAEG;IACH,OAAO,CAAC,cAAc;IAmCtB;;OAEG;IACH,OAAO,CAAC,YAAY;IAMpB;;OAEG;IACG,mBAAmB,CACvB,KAAK,EAAE,eAAe,EAAE,EACxB,OAAO,CAAC,EAAE,MAAM,GACf,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IA0C9C;;OAEG;IACH,uBAAuB,IAAI,oBAAoB;IAI/C;;OAEG;IACH,gBAAgB,IAAI,aAAa;IAIjC;;OAEG;IACH,aAAa,IAAI,UAAU;CAG5B;AAMD,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,oBAAoB,EAAE,CAAC"}