catalist-support-agent 1.0.0

package/dist/extraction/pii-handler.d.ts

@@ -0,0 +1,100 @@
+/**
+ * PII Handler Module
+ *
+ * Detects and redacts personally identifiable information (PII) from email content.
+ * Supports email addresses, phone numbers, SSN, credit cards, and physical addresses.
+ */
+export type PIIType = 'email' | 'phone' | 'ssn' | 'credit_card' | 'address' | 'date_of_birth' | 'ip_address' | 'bank_account';
+export interface PIIMatch {
+    type: PIIType;
+    value: string;
+    start: number;
+    end: number;
+    replacement: string;
+}
+export interface PIIResult {
+    text: string;
+    redacted: boolean;
+    piiTypesFound: PIIType[];
+    matches: PIIMatch[];
+}
+export interface PIIHandlerConfig {
+    enabled: boolean;
+    preserveCompanyEmails?: string[];
+    redactPatterns?: {
+        email?: boolean;
+        phone?: boolean;
+        ssn?: boolean;
+        creditCard?: boolean;
+        address?: boolean;
+        dateOfBirth?: boolean;
+        ipAddress?: boolean;
+        bankAccount?: boolean;
+    };
+}
+export declare class PIIHandler {
+    private config;
+    private preserveEmailDomains;
+    constructor(config?: Partial<PIIHandlerConfig>);
+    /**
+     * Detect PII in text without redaction
+     */
+    detect(text: string): PIIMatch[];
+    /**
+     * Redact PII from text
+     */
+    redact(text: string): PIIResult;
+    /**
+     * Check if text contains any PII
+     */
+    containsPII(text: string): boolean;
+    /**
+     * Get PII types found in text
+     */
+    getPIITypes(text: string): PIIType[];
+    /**
+     * Check if an email should be preserved (company email)
+     */
+    private shouldPreserveEmail;
+    /**
+     * Validate a match to reduce false positives
+     */
+    private validateMatch;
+    /**
+     * Luhn algorithm for credit card validation
+     */
+    private luhnCheck;
+    /**
+     * Get the config key for a PII type
+     */
+    private getPatternKey;
+    /**
+     * Update configuration
+     */
+    updateConfig(config: Partial<PIIHandlerConfig>): void;
+    /**
+     * Get current configuration
+     */
+    getConfig(): PIIHandlerConfig;
+}
+/**
+ * Get singleton PII handler instance
+ */
+export declare function getPIIHandler(): PIIHandler;
+/**
+ * Reset the singleton (for testing)
+ */
+export declare function resetPIIHandler(): void;
+/**
+ * Quick redaction function using singleton
+ */
+export declare function redactPII(text: string): PIIResult;
+/**
+ * Quick detection function using singleton
+ */
+export declare function detectPII(text: string): PIIMatch[];
+/**
+ * Check if text contains PII
+ */
+export declare function hasPII(text: string): boolean;
+//# sourceMappingURL=pii-handler.d.ts.map

package/dist/extraction/pii-handler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pii-handler.d.ts","sourceRoot":"","sources":["../../src/extraction/pii-handler.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAMH,MAAM,MAAM,OAAO,GACf,OAAO,GACP,OAAO,GACP,KAAK,GACL,aAAa,GACb,SAAS,GACT,eAAe,GACf,YAAY,GACZ,cAAc,CAAC;AAEnB,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,OAAO,CAAC;IAClB,aAAa,EAAE,OAAO,EAAE,CAAC;IACzB,OAAO,EAAE,QAAQ,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,OAAO,CAAC;IACjB,qBAAqB,CAAC,EAAE,MAAM,EAAE,CAAC;IACjC,cAAc,CAAC,EAAE;QACf,KAAK,CAAC,EAAE,OAAO,CAAC;QAChB,KAAK,CAAC,EAAE,OAAO,CAAC;QAChB,GAAG,CAAC,EAAE,OAAO,CAAC;QACd,UAAU,CAAC,EAAE,OAAO,CAAC;QACrB,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,WAAW,CAAC,EAAE,OAAO,CAAC;QACtB,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,WAAW,CAAC,EAAE,OAAO,CAAC;KACvB,CAAC;CACH;AAqDD,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAmB;IACjC,OAAO,CAAC,oBAAoB,CAAc;gBAE9B,MAAM,GAAE,OAAO,CAAC,gBAAgB,CAAM;IAuBlD;;OAEG;IACH,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,EAAE;IA+ChC;;OAEG;IACH,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS;IAyC/B;;OAEG;IACH,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAIlC;;OAEG;IACH,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,EAAE;IAKpC;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAY3B;;OAEG;IACH,OAAO,CAAC,aAAa;IA6BrB;;OAEG;IACH,OAAO,CAAC,SAAS;IAuBjB;;OAEG;IACH,OAAO,CAAC,aAAa;IAiBrB;;OAEG;IACH,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,gBAAgB,CAAC,GAAG,IAAI;IAUrD;;OAEG;IACH,SAAS,IAAI,gBAAgB;CAG9B;AAQD;;GAEG;AACH,wBAAgB,aAAa,IAAI,UAAU,CAQ1C;AAED;;GAEG;AACH,wBAAgB,eAAe,IAAI,IAAI,CAEtC;AAMD;;GAEG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,CAEjD;AAED;;GAEG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,EAAE,CAElD;AAED;;GAEG;AACH,wBAAgB,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAE5C"}

package/dist/extraction/pii-handler.js

@@ -0,0 +1,295 @@
+/**
+ * PII Handler Module
+ *
+ * Detects and redacts personally identifiable information (PII) from email content.
+ * Supports email addresses, phone numbers, SSN, credit cards, and physical addresses.
+ */
+// =============================================================================
+// PII Patterns
+// =============================================================================
+const PII_PATTERNS = {
+    // Email addresses (RFC 5322 simplified)
+    email: /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/gi,
+    // Phone numbers (US formats: XXX-XXX-XXXX, (XXX) XXX-XXXX, XXXXXXXXXX, +1XXXXXXXXXX)
+    phone: /(?:\+?1[-.\s]?)?(?:\(?\d{3}\)?[-.\s]?)?\d{3}[-.\s]?\d{4}(?!\d)/g,
+    // Social Security Numbers (XXX-XX-XXXX)
+    ssn: /\b\d{3}[-\s]?\d{2}[-\s]?\d{4}\b/g,
+    // Credit Card Numbers (16 digits with optional separators)
+    credit_card: /\b(?:\d{4}[-\s]?){3}\d{4}\b|\b\d{15,16}\b/g,
+    // US Street Addresses (simplified - number + street + type)
+    address: /\b\d{1,5}\s+(?:[A-Za-z]+\s+){1,4}(?:Street|St|Avenue|Ave|Road|Rd|Boulevard|Blvd|Drive|Dr|Lane|Ln|Court|Ct|Way|Place|Pl|Circle|Cir|Highway|Hwy)\.?\s*(?:#?\s*\d+)?(?:,?\s*(?:Apt|Suite|Unit|Ste)\.?\s*#?\s*\d+)?\b/gi,
+    // Date of Birth patterns (MM/DD/YYYY, YYYY-MM-DD, etc.)
+    date_of_birth: /\b(?:0?[1-9]|1[0-2])[-\/](?:0?[1-9]|[12]\d|3[01])[-\/](?:19|20)\d{2}\b|\b(?:19|20)\d{2}[-\/](?:0?[1-9]|1[0-2])[-\/](?:0?[1-9]|[12]\d|3[01])\b/g,
+    // IP Addresses (IPv4)
+    ip_address: /\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\b/g,
+    // Bank Account Numbers (8-17 digits)
+    bank_account: /\b(?:account\s*#?:?\s*|acct\s*#?:?\s*)(\d{8,17})\b/gi,
+};
+// Replacement placeholders
+const PII_REPLACEMENTS = {
+    email: '[EMAIL_REDACTED]',
+    phone: '[PHONE_REDACTED]',
+    ssn: '[SSN_REDACTED]',
+    credit_card: '[CC_REDACTED]',
+    address: '[ADDRESS_REDACTED]',
+    date_of_birth: '[DOB_REDACTED]',
+    ip_address: '[IP_REDACTED]',
+    bank_account: '[ACCOUNT_REDACTED]',
+};
+// =============================================================================
+// PII Handler Class
+// =============================================================================
+export class PIIHandler {
+    config;
+    preserveEmailDomains;
+    constructor(config = {}) {
+        this.config = {
+            enabled: true,
+            preserveCompanyEmails: [],
+            redactPatterns: {
+                email: true,
+                phone: true,
+                ssn: true,
+                creditCard: true,
+                address: true,
+                dateOfBirth: true,
+                ipAddress: false, // Often needed for debugging
+                bankAccount: true,
+            },
+            ...config,
+        };
+        // Build set of domains to preserve (lowercase)
+        this.preserveEmailDomains = new Set((this.config.preserveCompanyEmails || []).map((d) => d.toLowerCase()));
+    }
+    /**
+     * Detect PII in text without redaction
+     */
+    detect(text) {
+        if (!this.config.enabled || !text) {
+            return [];
+        }
+        const matches = [];
+        for (const [type, pattern] of Object.entries(PII_PATTERNS)) {
+            // Check if this pattern type should be processed
+            const patternKey = this.getPatternKey(type);
+            if (!this.config.redactPatterns?.[patternKey]) {
+                continue;
+            }
+            // Reset regex state
+            pattern.lastIndex = 0;
+            let match;
+            while ((match = pattern.exec(text)) !== null) {
+                const value = match[0];
+                const start = match.index;
+                const end = start + value.length;
+                // Special handling for emails - check if should preserve
+                if (type === 'email' && this.shouldPreserveEmail(value)) {
+                    continue;
+                }
+                // Validate match (reduce false positives)
+                if (!this.validateMatch(type, value)) {
+                    continue;
+                }
+                matches.push({
+                    type,
+                    value,
+                    start,
+                    end,
+                    replacement: PII_REPLACEMENTS[type],
+                });
+            }
+        }
+        // Sort by position (descending) for safe replacement
+        return matches.sort((a, b) => b.start - a.start);
+    }
+    /**
+     * Redact PII from text
+     */
+    redact(text) {
+        if (!this.config.enabled || !text) {
+            return {
+                text,
+                redacted: false,
+                piiTypesFound: [],
+                matches: [],
+            };
+        }
+        const matches = this.detect(text);
+        if (matches.length === 0) {
+            return {
+                text,
+                redacted: false,
+                piiTypesFound: [],
+                matches: [],
+            };
+        }
+        // Apply replacements (matches are sorted descending by position)
+        let redactedText = text;
+        for (const match of matches) {
+            redactedText =
+                redactedText.substring(0, match.start) +
+                    match.replacement +
+                    redactedText.substring(match.end);
+        }
+        // Get unique PII types
+        const piiTypesFound = [...new Set(matches.map((m) => m.type))];
+        return {
+            text: redactedText,
+            redacted: true,
+            piiTypesFound,
+            matches,
+        };
+    }
+    /**
+     * Check if text contains any PII
+     */
+    containsPII(text) {
+        return this.detect(text).length > 0;
+    }
+    /**
+     * Get PII types found in text
+     */
+    getPIITypes(text) {
+        const matches = this.detect(text);
+        return [...new Set(matches.map((m) => m.type))];
+    }
+    /**
+     * Check if an email should be preserved (company email)
+     */
+    shouldPreserveEmail(email) {
+        const lowerEmail = email.toLowerCase();
+        for (const domain of this.preserveEmailDomains) {
+            if (lowerEmail.endsWith(domain)) {
+                return true;
+            }
+        }
+        return false;
+    }
+    /**
+     * Validate a match to reduce false positives
+     */
+    validateMatch(type, value) {
+        switch (type) {
+            case 'phone':
+                // Must have at least 10 digits
+                const digits = value.replace(/\D/g, '');
+                return digits.length >= 10 && digits.length <= 11;
+            case 'ssn':
+                // SSN cannot start with 9, 666, or 000
+                const ssnDigits = value.replace(/\D/g, '');
+                if (ssnDigits.length !== 9)
+                    return false;
+                const areaNumber = parseInt(ssnDigits.substring(0, 3), 10);
+                if (areaNumber === 0 || areaNumber === 666 || areaNumber >= 900)
+                    return false;
+                return true;
+            case 'credit_card':
+                // Validate using Luhn algorithm
+                const ccDigits = value.replace(/\D/g, '');
+                return ccDigits.length >= 15 && ccDigits.length <= 16 && this.luhnCheck(ccDigits);
+            case 'bank_account':
+                // Must be preceded by account keyword
+                return /account|acct/i.test(value);
+            default:
+                return true;
+        }
+    }
+    /**
+     * Luhn algorithm for credit card validation
+     */
+    luhnCheck(cardNumber) {
+        let sum = 0;
+        let isEven = false;
+        for (let i = cardNumber.length - 1; i >= 0; i--) {
+            const char = cardNumber[i];
+            if (!char)
+                continue;
+            let digit = parseInt(char, 10);
+            if (isEven) {
+                digit *= 2;
+                if (digit > 9) {
+                    digit -= 9;
+                }
+            }
+            sum += digit;
+            isEven = !isEven;
+        }
+        return sum % 10 === 0;
+    }
+    /**
+     * Get the config key for a PII type
+     */
+    getPatternKey(type) {
+        switch (type) {
+            case 'credit_card':
+                return 'creditCard';
+            case 'date_of_birth':
+                return 'dateOfBirth';
+            case 'ip_address':
+                return 'ipAddress';
+            case 'bank_account':
+                return 'bankAccount';
+            default:
+                return type;
+        }
+    }
+    /**
+     * Update configuration
+     */
+    updateConfig(config) {
+        this.config = { ...this.config, ...config };
+        if (config.preserveCompanyEmails) {
+            this.preserveEmailDomains = new Set(config.preserveCompanyEmails.map((d) => d.toLowerCase()));
+        }
+    }
+    /**
+     * Get current configuration
+     */
+    getConfig() {
+        return { ...this.config };
+    }
+}
+// =============================================================================
+// Singleton Instance
+// =============================================================================
+let piiHandlerInstance = null;
+/**
+ * Get singleton PII handler instance
+ */
+export function getPIIHandler() {
+    if (!piiHandlerInstance) {
+        piiHandlerInstance = new PIIHandler({
+            enabled: process.env.PII_REDACTION_ENABLED !== 'false',
+            preserveCompanyEmails: ['@catalistgroup.co', '@catalist.deals'],
+        });
+    }
+    return piiHandlerInstance;
+}
+/**
+ * Reset the singleton (for testing)
+ */
+export function resetPIIHandler() {
+    piiHandlerInstance = null;
+}
+// =============================================================================
+// Utility Functions
+// =============================================================================
+/**
+ * Quick redaction function using singleton
+ */
+export function redactPII(text) {
+    return getPIIHandler().redact(text);
+}
+/**
+ * Quick detection function using singleton
+ */
+export function detectPII(text) {
+    return getPIIHandler().detect(text);
+}
+/**
+ * Check if text contains PII
+ */
+export function hasPII(text) {
+    return getPIIHandler().containsPII(text);
+}
+//# sourceMappingURL=pii-handler.js.map

package/dist/extraction/pii-handler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pii-handler.js","sourceRoot":"","sources":["../../src/extraction/pii-handler.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AA8CH,gFAAgF;AAChF,eAAe;AACf,gFAAgF;AAEhF,MAAM,YAAY,GAA4B;IAC5C,wCAAwC;IACxC,KAAK,EAAE,kDAAkD;IAEzD,qFAAqF;IACrF,KAAK,EACH,iEAAiE;IAEnE,wCAAwC;IACxC,GAAG,EAAE,kCAAkC;IAEvC,2DAA2D;IAC3D,WAAW,EACT,4CAA4C;IAE9C,4DAA4D;IAC5D,OAAO,EACL,qNAAqN;IAEvN,wDAAwD;IACxD,aAAa,EACX,gJAAgJ;IAElJ,sBAAsB;IACtB,UAAU,EACR,8EAA8E;IAEhF,qCAAqC;IACrC,YAAY,EAAE,sDAAsD;CACrE,CAAC;AAEF,2BAA2B;AAC3B,MAAM,gBAAgB,GAA4B;IAChD,KAAK,EAAE,kBAAkB;IACzB,KAAK,EAAE,kBAAkB;IACzB,GAAG,EAAE,gBAAgB;IACrB,WAAW,EAAE,eAAe;IAC5B,OAAO,EAAE,oBAAoB;IAC7B,aAAa,EAAE,gBAAgB;IAC/B,UAAU,EAAE,eAAe;IAC3B,YAAY,EAAE,oBAAoB;CACnC,CAAC;AAEF,gFAAgF;AAChF,oBAAoB;AACpB,gFAAgF;AAEhF,MAAM,OAAO,UAAU;IACb,MAAM,CAAmB;IACzB,oBAAoB,CAAc;IAE1C,YAAY,SAAoC,EAAE;QAChD,IAAI,CAAC,MAAM,GAAG;YACZ,OAAO,EAAE,IAAI;YACb,qBAAqB,EAAE,EAAE;YACzB,cAAc,EAAE;gBACd,KAAK,EAAE,IAAI;gBACX,KAAK,EAAE,IAAI;gBACX,GAAG,EAAE,IAAI;gBACT,UAAU,EAAE,IAAI;gBAChB,OAAO,EAAE,IAAI;gBACb,WAAW,EAAE,IAAI;gBACjB,SAAS,EAAE,KAAK,EAAE,6BAA6B;gBAC/C,WAAW,EAAE,IAAI;aAClB;YACD,GAAG,MAAM;SACV,CAAC;QAEF,+CAA+C;QAC/C,IAAI,CAAC,oBAAoB,GAAG,IAAI,GAAG,CACjC,CAAC,IAAI,CAAC,MAAM,CAAC,qBAAqB,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CACtE,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,IAAY;QACjB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC;YAClC,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,OAAO,GAAe,EAAE,CAAC;QAE/B,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,CAAwB,EAAE,CAAC;YAClF,iDAAiD;YACjD,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;YAC5C,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC9C,SAAS;YACX,CAAC;YAED,oBAAoB;YACpB,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC;YAEtB,IAAI,KAAK,CAAC;YACV,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;gBAC7C,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBACvB,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;gBAC1B,MAAM,GAAG,GAAG,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;gBAEjC,yDAAyD;gBACzD,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,EAAE,CAAC;oBACxD,SAAS;gBACX,CAAC;gBAED,0CAA0C;gBAC1C,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,CAAC;oBACrC,SAAS;gBACX,CAAC;gBAED,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI;oBACJ,KAAK;oBACL,KAAK;oBACL,GAAG;oBACH,WAAW,EAAE,gBAAgB,CAAC,IAAI,CAAC;iBACpC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,qDAAqD;QACrD,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IACnD,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,IAAY;QACjB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC;YAClC,OAAO;gBACL,IAAI;gBACJ,QAAQ,EAAE,KAAK;gBACf,aAAa,EAAE,EAAE;gBACjB,OAAO,EAAE,EAAE;aACZ,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAElC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO;gBACL,IAAI;gBACJ,QAAQ,EAAE,KAAK;gBACf,aAAa,EAAE,EAAE;gBACjB,OAAO,EAAE,EAAE;aACZ,CAAC;QACJ,CAAC;QAED,iEAAiE;QACjE,IAAI,YAAY,GAAG,IAAI,CAAC;QACxB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,YAAY;gBACV,YAAY,CAAC,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC;oBACtC,KAAK,CAAC,WAAW;oBACjB,YAAY,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACtC,CAAC;QAED,uBAAuB;QACvB,MAAM,aAAa,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAE/D,OAAO;YACL,IAAI,EAAE,YAAY;YAClB,QAAQ,EAAE,IAAI;YACd,aAAa;YACb,OAAO;SACR,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,IAAY;QACtB,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,IAAY;QACtB,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAClC,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClD,CAAC;IAED;;OAEG;IACK,mBAAmB,CAAC,KAAa;QACvC,MAAM,UAAU,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;QAEvC,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC/C,IAAI,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBAChC,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACK,aAAa,CAAC,IAAa,EAAE,KAAa;QAChD,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,OAAO;gBACV,+BAA+B;gBAC/B,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;gBACxC,OAAO,MAAM,CAAC,MAAM,IAAI,EAAE,IAAI,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;YAEpD,KAAK,KAAK;gBACR,uCAAuC;gBACvC,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;gBAC3C,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;oBAAE,OAAO,KAAK,CAAC;gBACzC,MAAM,UAAU,GAAG,QAAQ,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC3D,IAAI,UAAU,KAAK,CAAC,IAAI,UAAU,KAAK,GAAG,IAAI,UAAU,IAAI,GAAG;oBAAE,OAAO,KAAK,CAAC;gBAC9E,OAAO,IAAI,CAAC;YAEd,KAAK,aAAa;gBAChB,gCAAgC;gBAChC,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;gBAC1C,OAAO,QAAQ,CAAC,MAAM,IAAI,EAAE,IAAI,QAAQ,CAAC,MAAM,IAAI,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;YAEpF,KAAK,cAAc;gBACjB,sCAAsC;gBACtC,OAAO,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAErC;gBACE,OAAO,IAAI,CAAC;QAChB,CAAC;IACH,CAAC;IAED;;OAEG;IACK,SAAS,CAAC,UAAkB;QAClC,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,IAAI,MAAM,GAAG,KAAK,CAAC;QAEnB,KAAK,IAAI,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAChD,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;YAC3B,IAAI,CAAC,IAAI;gBAAE,SAAS;YACpB,IAAI,KAAK,GAAG,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAE/B,IAAI,MAAM,EAAE,CAAC;gBACX,KAAK,IAAI,CAAC,CAAC;gBACX,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;oBACd,KAAK,IAAI,CAAC,CAAC;gBACb,CAAC;YACH,CAAC;YAED,GAAG,IAAI,KAAK,CAAC;YACb,MAAM,GAAG,CAAC,MAAM,CAAC;QACnB,CAAC;QAED,OAAO,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;IACxB,CAAC;IAED;;OAEG;IACK,aAAa,CACnB,IAAa;QAEb,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,aAAa;gBAChB,OAAO,YAAY,CAAC;YACtB,KAAK,eAAe;gBAClB,OAAO,aAAa,CAAC;YACvB,KAAK,YAAY;gBACf,OAAO,WAAW,CAAC;YACrB,KAAK,cAAc;gBACjB,OAAO,aAAa,CAAC;YACvB;gBACE,OAAO,IAA6C,CAAC;QACzD,CAAC;IACH,CAAC;IAED;;OAEG;IACH,YAAY,CAAC,MAAiC;QAC5C,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC;QAE5C,IAAI,MAAM,CAAC,qBAAqB,EAAE,CAAC;YACjC,IAAI,CAAC,oBAAoB,GAAG,IAAI,GAAG,CACjC,MAAM,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CACzD,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACH,SAAS;QACP,OAAO,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC5B,CAAC;CACF;AAED,gFAAgF;AAChF,qBAAqB;AACrB,gFAAgF;AAEhF,IAAI,kBAAkB,GAAsB,IAAI,CAAC;AAEjD;;GAEG;AACH,MAAM,UAAU,aAAa;IAC3B,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACxB,kBAAkB,GAAG,IAAI,UAAU,CAAC;YAClC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,qBAAqB,KAAK,OAAO;YACtD,qBAAqB,EAAE,CAAC,mBAAmB,EAAE,iBAAiB,CAAC;SAChE,CAAC,CAAC;IACL,CAAC;IACD,OAAO,kBAAkB,CAAC;AAC5B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe;IAC7B,kBAAkB,GAAG,IAAI,CAAC;AAC5B,CAAC;AAED,gFAAgF;AAChF,oBAAoB;AACpB,gFAAgF;AAEhF;;GAEG;AACH,MAAM,UAAU,SAAS,CAAC,IAAY;IACpC,OAAO,aAAa,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;AACtC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,SAAS,CAAC,IAAY;IACpC,OAAO,aAAa,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;AACtC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,MAAM,CAAC,IAAY;IACjC,OAAO,aAAa,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;AAC3C,CAAC"}

package/dist/extraction/pipeline.d.ts

@@ -0,0 +1,94 @@
+/**
+ * Extraction Pipeline Module
+ *
+ * Main orchestrator for Gmail knowledge base extraction.
+ * Coordinates Gmail API calls, extraction, quality filtering,
+ * deduplication, intent classification, and storage.
+ */
+import { StateManager } from './state-manager.js';
+import { EmbeddingService } from '../embeddings/index.js';
+export interface PipelineConfig {
+    supabaseUrl: string;
+    supabaseServiceRoleKey: string;
+    salesEmailAddress: string;
+    openaiApiKey: string;
+    gmailAccessToken?: string;
+    lookbackDays?: number;
+    batchSize?: number;
+    maxThreads?: number;
+    gmailLabels?: string[];
+    dryRun?: boolean;
+    generateEmbeddings?: boolean;
+    classifyIntents?: boolean;
+    onProgress?: (stage: string, progress: number, total: number) => void;
+    onError?: (error: Error, context: Record<string, unknown>) => void;
+}
+export interface PipelineResult {
+    batchId: string | null;
+    stats: {
+        threadsFound: number;
+        threadsProcessed: number;
+        pairsExtracted: number;
+        pairsStored: number;
+        duplicatesSkipped: number;
+        qualityFilteredOut: number;
+        embeddingsGenerated: number;
+        errors: number;
+    };
+    duration: {
+        totalMs: number;
+        extractionMs: number;
+        storageMs: number;
+        embeddingsMs: number;
+    };
+    errors: Array<{
+        stage: string;
+        message: string;
+        context?: Record<string, unknown>;
+    }>;
+}
+export declare class ExtractionPipeline {
+    private config;
+    private gmailExtractor;
+    private stateManager;
+    private embeddingService;
+    private rateLimiter;
+    constructor(config: PipelineConfig);
+    /**
+     * Run the full extraction pipeline
+     */
+    run(): Promise<PipelineResult>;
+    /**
+     * Fetch Gmail threads from API
+     */
+    private fetchGmailThreads;
+    /**
+     * Fetch a single Gmail thread with full message details
+     */
+    private fetchGmailThread;
+    /**
+     * Classify intents for extracted pairs
+     */
+    private classifyIntents;
+    /**
+     * Report progress callback
+     */
+    private reportProgress;
+    /**
+     * Report error callback
+     */
+    private reportError;
+    /**
+     * Get state manager for external use
+     */
+    getStateManager(): StateManager;
+    /**
+     * Get embedding service for external use
+     */
+    getEmbeddingService(): EmbeddingService | null;
+}
+/**
+ * Create an extraction pipeline with environment configuration
+ */
+export declare function createExtractionPipeline(options?: Partial<PipelineConfig>): ExtractionPipeline;
+//# sourceMappingURL=pipeline.d.ts.map

package/dist/extraction/pipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/extraction/pipeline.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAQH,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAO1D,MAAM,WAAW,cAAc;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,YAAY,EAAE,MAAM,CAAC;IAGrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAG1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IAKpB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IAGvB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,eAAe,CAAC,EAAE,OAAO,CAAC;IAG1B,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IACtE,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACpE;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,KAAK,EAAE;QACL,YAAY,EAAE,MAAM,CAAC;QACrB,gBAAgB,EAAE,MAAM,CAAC;QACzB,cAAc,EAAE,MAAM,CAAC;QACvB,WAAW,EAAE,MAAM,CAAC;QACpB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,kBAAkB,EAAE,MAAM,CAAC;QAC3B,mBAAmB,EAAE,MAAM,CAAC;QAC5B,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,QAAQ,EAAE;QACR,OAAO,EAAE,MAAM,CAAC;QAChB,YAAY,EAAE,MAAM,CAAC;QACrB,SAAS,EAAE,MAAM,CAAC;QAClB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;IACF,MAAM,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;KAAE,CAAC,CAAC;CACtF;AASD,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,MAAM,CAA2B;IACzC,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,gBAAgB,CAA0B;IAClD,OAAO,CAAC,WAAW,CAAc;gBAErB,MAAM,EAAE,cAAc;IAwClC;;OAEG;IACG,GAAG,IAAI,OAAO,CAAC,cAAc,CAAC;IA0LpC;;OAEG;YACW,iBAAiB;IA+E/B;;OAEG;YACW,gBAAgB;IAwB9B;;OAEG;YACW,eAAe;IAyC7B;;OAEG;IACH,OAAO,CAAC,cAAc;IAMtB;;OAEG;IACH,OAAO,CAAC,WAAW;IAQnB;;OAEG;IACH,eAAe,IAAI,YAAY;IAI/B;;OAEG;IACH,mBAAmB,IAAI,gBAAgB,GAAG,IAAI;CAG/C;AAMD;;GAEG;AACH,wBAAgB,wBAAwB,CACtC,OAAO,GAAE,OAAO,CAAC,cAAc,CAAM,GACpC,kBAAkB,CAiBpB"}