catalist-support-agent 1.0.0

package/dist/embeddings/cache.js

@@ -0,0 +1,193 @@
+/**
+ * LRU Cache for Embedding Queries
+ *
+ * In-memory cache with TTL support to reduce redundant embedding API calls.
+ * Stores query text -> embedding vector mappings.
+ */
+/**
+ * Simple LRU Cache with TTL support
+ */
+export class LRUCache {
+    cache;
+    maxSize;
+    ttlMs;
+    constructor(maxSize = 1000, ttlMs = 300000) {
+        this.cache = new Map();
+        this.maxSize = maxSize;
+        this.ttlMs = ttlMs;
+    }
+    /**
+     * Get a value from the cache
+     */
+    get(key) {
+        const entry = this.cache.get(key);
+        if (!entry) {
+            return undefined;
+        }
+        // Check if expired
+        if (Date.now() > entry.expiresAt) {
+            this.cache.delete(key);
+            return undefined;
+        }
+        // Move to end for LRU behavior (most recently used)
+        this.cache.delete(key);
+        this.cache.set(key, entry);
+        return entry.value;
+    }
+    /**
+     * Set a value in the cache
+     */
+    set(key, value, customTtlMs) {
+        // If at capacity, remove oldest entry (first item in Map)
+        if (this.cache.size >= this.maxSize) {
+            const firstKey = this.cache.keys().next().value;
+            if (firstKey !== undefined) {
+                this.cache.delete(firstKey);
+            }
+        }
+        const ttl = customTtlMs ?? this.ttlMs;
+        const entry = {
+            value,
+            expiresAt: Date.now() + ttl,
+        };
+        this.cache.set(key, entry);
+    }
+    /**
+     * Check if key exists and is not expired
+     */
+    has(key) {
+        const entry = this.cache.get(key);
+        if (!entry) {
+            return false;
+        }
+        if (Date.now() > entry.expiresAt) {
+            this.cache.delete(key);
+            return false;
+        }
+        return true;
+    }
+    /**
+     * Delete a key from the cache
+     */
+    delete(key) {
+        return this.cache.delete(key);
+    }
+    /**
+     * Clear all entries
+     */
+    clear() {
+        this.cache.clear();
+    }
+    /**
+     * Get current cache size
+     */
+    get size() {
+        return this.cache.size;
+    }
+    /**
+     * Remove expired entries
+     */
+    prune() {
+        const now = Date.now();
+        let pruned = 0;
+        for (const [key, entry] of this.cache) {
+            if (now > entry.expiresAt) {
+                this.cache.delete(key);
+                pruned++;
+            }
+        }
+        return pruned;
+    }
+    /**
+     * Get cache statistics
+     */
+    getStats() {
+        return {
+            size: this.cache.size,
+            maxSize: this.maxSize,
+            ttlMs: this.ttlMs,
+        };
+    }
+}
+/**
+ * Specialized cache for embedding vectors
+ * Uses content hash as key to handle exact and near-duplicate queries
+ */
+export class EmbeddingCache {
+    cache;
+    hits = 0;
+    misses = 0;
+    constructor(maxSize = 1000, ttlMs = 300000) {
+        this.cache = new LRUCache(maxSize, ttlMs);
+    }
+    /**
+     * Generate a cache key from text
+     * Simple hash function for in-memory use
+     */
+    generateKey(text) {
+        // Normalize text: lowercase, trim, collapse whitespace
+        const normalized = text.toLowerCase().trim().replace(/\s+/g, ' ');
+        // Simple hash (FNV-1a variant)
+        let hash = 2166136261;
+        for (let i = 0; i < normalized.length; i++) {
+            hash ^= normalized.charCodeAt(i);
+            hash = (hash * 16777619) >>> 0;
+        }
+        return hash.toString(16);
+    }
+    /**
+     * Get embedding from cache
+     */
+    get(text) {
+        const key = this.generateKey(text);
+        const result = this.cache.get(key);
+        if (result) {
+            this.hits++;
+        }
+        else {
+            this.misses++;
+        }
+        return result;
+    }
+    /**
+     * Store embedding in cache
+     */
+    set(text, embedding) {
+        const key = this.generateKey(text);
+        this.cache.set(key, embedding);
+    }
+    /**
+     * Check if text has cached embedding
+     */
+    has(text) {
+        const key = this.generateKey(text);
+        return this.cache.has(key);
+    }
+    /**
+     * Clear the cache
+     */
+    clear() {
+        this.cache.clear();
+        this.hits = 0;
+        this.misses = 0;
+    }
+    /**
+     * Get cache statistics
+     */
+    getStats() {
+        const total = this.hits + this.misses;
+        return {
+            size: this.cache.size,
+            hits: this.hits,
+            misses: this.misses,
+            hitRate: total > 0 ? this.hits / total : 0,
+        };
+    }
+    /**
+     * Prune expired entries
+     */
+    prune() {
+        return this.cache.prune();
+    }
+}
+//# sourceMappingURL=cache.js.map

package/dist/embeddings/cache.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cache.js","sourceRoot":"","sources":["../../src/embeddings/cache.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAOH;;GAEG;AACH,MAAM,OAAO,QAAQ;IACX,KAAK,CAAwB;IACpB,OAAO,CAAS;IAChB,KAAK,CAAS;IAE/B,YAAY,UAAkB,IAAI,EAAE,QAAgB,MAAM;QACxD,IAAI,CAAC,KAAK,GAAG,IAAI,GAAG,EAAE,CAAC;QACvB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,GAAM;QACR,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAElC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,mBAAmB;QACnB,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC;YACjC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACvB,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,oDAAoD;QACpD,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACvB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAE3B,OAAO,KAAK,CAAC,KAAK,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,GAAM,EAAE,KAAQ,EAAE,WAAoB;QACxC,0DAA0D;QAC1D,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACpC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;YAChD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;gBAC3B,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;QAED,MAAM,GAAG,GAAG,WAAW,IAAI,IAAI,CAAC,KAAK,CAAC;QACtC,MAAM,KAAK,GAAkB;YAC3B,KAAK;YACL,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG;SAC5B,CAAC;QAEF,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,GAAM;QACR,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,KAAK,CAAC;QACf,CAAC;QACD,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC;YACjC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACvB,OAAO,KAAK,CAAC;QACf,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,GAAM;QACX,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,KAAK;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,MAAM,GAAG,CAAC,CAAC;QAEf,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACtC,IAAI,GAAG,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC;gBAC1B,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACvB,MAAM,EAAE,CAAC;YACX,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,QAAQ;QAKN,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB,CAAC;IACJ,CAAC;CACF;AAED;;;GAGG;AACH,MAAM,OAAO,cAAc;IACjB,KAAK,CAA6B;IAClC,IAAI,GAAW,CAAC,CAAC;IACjB,MAAM,GAAW,CAAC,CAAC;IAE3B,YAAY,UAAkB,IAAI,EAAE,QAAgB,MAAM;QACxD,IAAI,CAAC,KAAK,GAAG,IAAI,QAAQ,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IAC5C,CAAC;IAED;;;OAGG;IACK,WAAW,CAAC,IAAY;QAC9B,uDAAuD;QACvD,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAElE,+BAA+B;QAC/B,IAAI,IAAI,GAAG,UAAU,CAAC;QACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,IAAI,IAAI,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YACjC,IAAI,GAAG,CAAC,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;QACjC,CAAC;QAED,OAAO,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,IAAY;QACd,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QACnC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAEnC,IAAI,MAAM,EAAE,CAAC;YACX,IAAI,CAAC,IAAI,EAAE,CAAC;QACd,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,MAAM,EAAE,CAAC;QAChB,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,IAAY,EAAE,SAAmB;QACnC,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;IACjC,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,IAAY;QACd,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QACnC,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;QACd,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;IAClB,CAAC;IAED;;OAEG;IACH,QAAQ;QAMN,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC;QACtC,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI;YACrB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,OAAO,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;SAC3C,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IAC5B,CAAC;CACF"}

package/dist/embeddings/index.d.ts

@@ -0,0 +1,152 @@
+/**
+ * Embedding Service Module
+ *
+ * Provides semantic embedding generation for the Gmail Knowledge Base.
+ * Uses OpenAI text-embedding-3-small model for 1536-dimension embeddings.
+ *
+ * Features:
+ * - Single text and batch embedding generation
+ * - LRU cache with TTL for query embeddings
+ * - Embeddable text formatting for knowledge entries
+ * - Integration with Supabase for storing embeddings
+ */
+export interface EmbeddingServiceConfig {
+    openaiApiKey: string;
+    embeddingModel?: string;
+    embeddingDimensions?: number;
+    cacheMaxSize?: number;
+    cacheTtlMs?: number;
+    supabaseUrl: string;
+    supabaseServiceRoleKey: string;
+}
+export interface KnowledgeEmbeddingInput {
+    questionSubject?: string;
+    questionText: string;
+    responseText: string;
+    resolutionOutcome?: string;
+}
+export interface EmbeddingStats {
+    cache: {
+        size: number;
+        hits: number;
+        misses: number;
+        hitRate: number;
+    };
+    totalEmbeddingsGenerated: number;
+    totalTokensUsed: number;
+}
+export declare class EmbeddingService {
+    private client;
+    private cache;
+    private supabase;
+    private totalEmbeddingsGenerated;
+    private totalTokensUsed;
+    constructor(config: EmbeddingServiceConfig);
+    /**
+     * Generate embedding for a single text, using cache when available
+     */
+    embed(text: string): Promise<number[]>;
+    /**
+     * Generate embeddings for multiple texts in batch
+     * Does not use cache (typically for bulk operations)
+     */
+    embedBatch(texts: string[]): Promise<number[][]>;
+    /**
+     * Generate embedding for a query (uses cache, intended for similarity search)
+     */
+    embedQuery(queryText: string): Promise<number[]>;
+    /**
+     * Format a knowledge entry into embeddable text
+     *
+     * Structure:
+     * [QUESTION]: {subject} - {text}
+     * [RESPONSE]: {text}
+     * [OUTCOME]: {resolution}
+     */
+    formatKnowledgeEntry(input: KnowledgeEmbeddingInput): string;
+    /**
+     * Generate embedding for a knowledge entry
+     */
+    embedKnowledgeEntry(input: KnowledgeEmbeddingInput): Promise<number[]>;
+    /**
+     * Generate embeddings for multiple knowledge entries
+     */
+    embedKnowledgeEntries(inputs: KnowledgeEmbeddingInput[]): Promise<number[][]>;
+    /**
+     * Update embedding for a Gmail knowledge entry
+     */
+    updateKnowledgeEntryEmbedding(entryId: string, embedding: number[]): Promise<void>;
+    /**
+     * Generate and store embedding for a Gmail knowledge entry
+     */
+    generateAndStoreEmbedding(entryId: string, input: KnowledgeEmbeddingInput): Promise<number[]>;
+    /**
+     * Batch generate and store embeddings for multiple entries
+     */
+    generateAndStoreBatchEmbeddings(entries: Array<{
+        id: string;
+        input: KnowledgeEmbeddingInput;
+    }>): Promise<void>;
+    /**
+     * Get entries that need embeddings
+     */
+    getEntriesNeedingEmbeddings(limit?: number): Promise<Array<{
+        id: string;
+        questionSubject: string | null;
+        questionText: string;
+        responseText: string;
+        resolutionIndicator: string | null;
+    }>>;
+    /**
+     * Backfill embeddings for all entries without embeddings
+     */
+    backfillEmbeddings(batchSize?: number, onProgress?: (processed: number, total: number) => void): Promise<{
+        processed: number;
+        errors: number;
+    }>;
+    /**
+     * Find similar knowledge entries using vector similarity
+     */
+    findSimilar(queryText: string, options?: {
+        matchThreshold?: number;
+        matchCount?: number;
+        filterIntent?: string;
+        onlyApproved?: boolean;
+    }): Promise<Array<{
+        id: string;
+        questionSubject: string | null;
+        questionText: string;
+        responseText: string;
+        intentCategory: string | null;
+        qualityScore: number;
+        similarity: number;
+        emailReceivedAt: string;
+        resolutionIndicator: string | null;
+    }>>;
+    /**
+     * Get service statistics
+     */
+    getStats(): EmbeddingStats;
+    /**
+     * Clear the embedding cache
+     */
+    clearCache(): void;
+    /**
+     * Get embedding configuration
+     */
+    getConfig(): {
+        model: string;
+        dimensions: number;
+    };
+}
+/**
+ * Get the singleton embedding service instance
+ */
+export declare function getEmbeddingService(): EmbeddingService;
+/**
+ * Reset the singleton (for testing)
+ */
+export declare function resetEmbeddingService(): void;
+export { EmbeddingCache } from './cache.js';
+export { OpenAIEmbeddingClient, type EmbeddingResponse, type BatchEmbeddingResponse } from './openai-client.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/embeddings/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/embeddings/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAWH,MAAM,WAAW,sBAAsB;IACrC,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,sBAAsB,EAAE,MAAM,CAAC;CAChC;AAED,MAAM,WAAW,uBAAuB;IACtC,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE;QACL,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;IACF,wBAAwB,EAAE,MAAM,CAAC;IACjC,eAAe,EAAE,MAAM,CAAC;CACzB;AAMD,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,MAAM,CAAwB;IACtC,OAAO,CAAC,KAAK,CAAiB;IAC9B,OAAO,CAAC,QAAQ,CAAiB;IACjC,OAAO,CAAC,wBAAwB,CAAa;IAC7C,OAAO,CAAC,eAAe,CAAa;gBAExB,MAAM,EAAE,sBAAsB;IAmB1C;;OAEG;IACG,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAoB5C;;;OAGG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IAUtD;;OAEG;IACG,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAQtD;;;;;;;OAOG;IACH,oBAAoB,CAAC,KAAK,EAAE,uBAAuB,GAAG,MAAM;IAoB5D;;OAEG;IACG,mBAAmB,CAAC,KAAK,EAAE,uBAAuB,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAK5E;;OAEG;IACG,qBAAqB,CAAC,MAAM,EAAE,uBAAuB,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IASnF;;OAEG;IACG,6BAA6B,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAkBxF;;OAEG;IACG,yBAAyB,CAC7B,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,uBAAuB,GAC7B,OAAO,CAAC,MAAM,EAAE,CAAC;IAMpB;;OAEG;IACG,+BAA+B,CACnC,OAAO,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,uBAAuB,CAAA;KAAE,CAAC,GAC7D,OAAO,CAAC,IAAI,CAAC;IA0BhB;;OAEG;IACG,2BAA2B,CAAC,KAAK,GAAE,MAAY,GAAG,OAAO,CAC7D,KAAK,CAAC;QACJ,EAAE,EAAE,MAAM,CAAC;QACX,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;QAC/B,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;QACrB,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;KACpC,CAAC,CACH;IAwBD;;OAEG;IACG,kBAAkB,CACtB,SAAS,GAAE,MAAW,EACtB,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,GACtD,OAAO,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAkEjD;;OAEG;IACG,WAAW,CACf,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;QACP,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,YAAY,CAAC,EAAE,OAAO,CAAC;KACnB,GACL,OAAO,CACR,KAAK,CAAC;QACJ,EAAE,EAAE,MAAM,CAAC;QACX,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;QAC/B,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;QACrB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;QAC9B,YAAY,EAAE,MAAM,CAAC;QACrB,UAAU,EAAE,MAAM,CAAC;QACnB,eAAe,EAAE,MAAM,CAAC;QACxB,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;KACpC,CAAC,CACH;IA6CD;;OAEG;IACH,QAAQ,IAAI,cAAc;IAQ1B;;OAEG;IACH,UAAU,IAAI,IAAI;IAIlB;;OAEG;IACH,SAAS,IAAI;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE;CAGnD;AAQD;;GAEG;AACH,wBAAgB,mBAAmB,IAAI,gBAAgB,CAgCtD;AAED;;GAEG;AACH,wBAAgB,qBAAqB,IAAI,IAAI,CAE5C;AAGD,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,qBAAqB,EAAE,KAAK,iBAAiB,EAAE,KAAK,sBAAsB,EAAE,MAAM,oBAAoB,CAAC"}

package/dist/embeddings/index.js

@@ -0,0 +1,337 @@
+/**
+ * Embedding Service Module
+ *
+ * Provides semantic embedding generation for the Gmail Knowledge Base.
+ * Uses OpenAI text-embedding-3-small model for 1536-dimension embeddings.
+ *
+ * Features:
+ * - Single text and batch embedding generation
+ * - LRU cache with TTL for query embeddings
+ * - Embeddable text formatting for knowledge entries
+ * - Integration with Supabase for storing embeddings
+ */
+import { createClient } from '@supabase/supabase-js';
+import { OpenAIEmbeddingClient } from './openai-client.js';
+import { EmbeddingCache } from './cache.js';
+import { EmbeddingError } from '../errors.js';
+// =============================================================================
+// Embedding Service
+// =============================================================================
+export class EmbeddingService {
+    client;
+    cache;
+    supabase;
+    totalEmbeddingsGenerated = 0;
+    totalTokensUsed = 0;
+    constructor(config) {
+        this.client = new OpenAIEmbeddingClient({
+            apiKey: config.openaiApiKey,
+            model: config.embeddingModel || 'text-embedding-3-small',
+            dimensions: config.embeddingDimensions || 1536,
+        });
+        this.cache = new EmbeddingCache(config.cacheMaxSize || 1000, config.cacheTtlMs || 300000 // 5 minutes default
+        );
+        this.supabase = createClient(config.supabaseUrl, config.supabaseServiceRoleKey);
+    }
+    // ===========================================================================
+    // Core Embedding Methods
+    // ===========================================================================
+    /**
+     * Generate embedding for a single text, using cache when available
+     */
+    async embed(text) {
+        // Check cache first
+        const cached = this.cache.get(text);
+        if (cached) {
+            return cached;
+        }
+        // Generate new embedding
+        const response = await this.client.embed(text);
+        // Cache the result
+        this.cache.set(text, response.embedding);
+        // Track stats
+        this.totalEmbeddingsGenerated++;
+        this.totalTokensUsed += response.usage.totalTokens;
+        return response.embedding;
+    }
+    /**
+     * Generate embeddings for multiple texts in batch
+     * Does not use cache (typically for bulk operations)
+     */
+    async embedBatch(texts) {
+        const response = await this.client.embedBatch(texts);
+        // Track stats
+        this.totalEmbeddingsGenerated += response.embeddings.length;
+        this.totalTokensUsed += response.usage.totalTokens;
+        return response.embeddings;
+    }
+    /**
+     * Generate embedding for a query (uses cache, intended for similarity search)
+     */
+    async embedQuery(queryText) {
+        return this.embed(queryText);
+    }
+    // ===========================================================================
+    // Knowledge Entry Formatting
+    // ===========================================================================
+    /**
+     * Format a knowledge entry into embeddable text
+     *
+     * Structure:
+     * [QUESTION]: {subject} - {text}
+     * [RESPONSE]: {text}
+     * [OUTCOME]: {resolution}
+     */
+    formatKnowledgeEntry(input) {
+        const parts = [];
+        // Question section
+        const questionPart = input.questionSubject
+            ? `${input.questionSubject} - ${input.questionText}`
+            : input.questionText;
+        parts.push(`[QUESTION]: ${questionPart}`);
+        // Response section
+        parts.push(`[RESPONSE]: ${input.responseText}`);
+        // Outcome section (if available)
+        if (input.resolutionOutcome) {
+            parts.push(`[OUTCOME]: ${input.resolutionOutcome}`);
+        }
+        return parts.join('\n');
+    }
+    /**
+     * Generate embedding for a knowledge entry
+     */
+    async embedKnowledgeEntry(input) {
+        const formattedText = this.formatKnowledgeEntry(input);
+        return this.embed(formattedText);
+    }
+    /**
+     * Generate embeddings for multiple knowledge entries
+     */
+    async embedKnowledgeEntries(inputs) {
+        const formattedTexts = inputs.map((input) => this.formatKnowledgeEntry(input));
+        return this.embedBatch(formattedTexts);
+    }
+    // ===========================================================================
+    // Database Operations
+    // ===========================================================================
+    /**
+     * Update embedding for a Gmail knowledge entry
+     */
+    async updateKnowledgeEntryEmbedding(entryId, embedding) {
+        const { error } = await this.supabase
+            .from('gmail_knowledge_entries')
+            .update({
+            embedding: `[${embedding.join(',')}]`,
+            updated_at: new Date().toISOString(),
+        })
+            .eq('id', entryId);
+        if (error) {
+            throw new EmbeddingError(`Failed to update embedding for entry ${entryId}: ${error.message}`, 'api_error', { context: { entryId, error } });
+        }
+    }
+    /**
+     * Generate and store embedding for a Gmail knowledge entry
+     */
+    async generateAndStoreEmbedding(entryId, input) {
+        const embedding = await this.embedKnowledgeEntry(input);
+        await this.updateKnowledgeEntryEmbedding(entryId, embedding);
+        return embedding;
+    }
+    /**
+     * Batch generate and store embeddings for multiple entries
+     */
+    async generateAndStoreBatchEmbeddings(entries) {
+        if (entries.length === 0) {
+            return;
+        }
+        // Generate embeddings in batch
+        const inputs = entries.map((e) => e.input);
+        const embeddings = await this.embedKnowledgeEntries(inputs);
+        // Update each entry in the database
+        // Note: Supabase doesn't support batch updates with different values,
+        // so we do this in parallel with Promise.all
+        const updatePromises = entries.map((entry, index) => {
+            const embedding = embeddings[index];
+            if (!embedding) {
+                throw new EmbeddingError(`Missing embedding for entry at index ${index}`, 'validation');
+            }
+            return this.updateKnowledgeEntryEmbedding(entry.id, embedding);
+        });
+        await Promise.all(updatePromises);
+    }
+    /**
+     * Get entries that need embeddings
+     */
+    async getEntriesNeedingEmbeddings(limit = 100) {
+        const { data, error } = await this.supabase
+            .from('gmail_knowledge_entries')
+            .select('id, question_subject, question_text, response_text, resolution_indicator')
+            .is('embedding', null)
+            .limit(limit);
+        if (error) {
+            throw new EmbeddingError(`Failed to fetch entries needing embeddings: ${error.message}`, 'api_error', { context: { error } });
+        }
+        return (data || []).map((row) => ({
+            id: row.id,
+            questionSubject: row.question_subject,
+            questionText: row.question_text,
+            responseText: row.response_text,
+            resolutionIndicator: row.resolution_indicator,
+        }));
+    }
+    /**
+     * Backfill embeddings for all entries without embeddings
+     */
+    async backfillEmbeddings(batchSize = 50, onProgress) {
+        let processed = 0;
+        let errors = 0;
+        let total = 0;
+        // Get total count
+        const { count } = await this.supabase
+            .from('gmail_knowledge_entries')
+            .select('id', { count: 'exact', head: true })
+            .is('embedding', null);
+        total = count || 0;
+        while (true) {
+            const entries = await this.getEntriesNeedingEmbeddings(batchSize);
+            if (entries.length === 0) {
+                break;
+            }
+            try {
+                const batchEntries = entries.map((entry) => ({
+                    id: entry.id,
+                    input: {
+                        questionSubject: entry.questionSubject || undefined,
+                        questionText: entry.questionText,
+                        responseText: entry.responseText,
+                        resolutionOutcome: entry.resolutionIndicator || undefined,
+                    },
+                }));
+                await this.generateAndStoreBatchEmbeddings(batchEntries);
+                processed += entries.length;
+            }
+            catch (error) {
+                console.error('Error processing batch:', error);
+                errors += entries.length;
+                // Try individual entries to isolate the problematic ones
+                for (const entry of entries) {
+                    try {
+                        await this.generateAndStoreEmbedding(entry.id, {
+                            questionSubject: entry.questionSubject || undefined,
+                            questionText: entry.questionText,
+                            responseText: entry.responseText,
+                            resolutionOutcome: entry.resolutionIndicator || undefined,
+                        });
+                        processed++;
+                        errors--; // Reduce error count since this one succeeded
+                    }
+                    catch (individualError) {
+                        console.error(`Error processing entry ${entry.id}:`, individualError);
+                    }
+                }
+            }
+            if (onProgress) {
+                onProgress(processed, total);
+            }
+        }
+        return { processed, errors };
+    }
+    // ===========================================================================
+    // Similarity Search
+    // ===========================================================================
+    /**
+     * Find similar knowledge entries using vector similarity
+     */
+    async findSimilar(queryText, options = {}) {
+        const { matchThreshold = 0.78, matchCount = 5, filterIntent = null, onlyApproved = true, } = options;
+        // Generate embedding for query
+        const queryEmbedding = await this.embedQuery(queryText);
+        // Call RPC function
+        const { data, error } = await this.supabase.rpc('match_gmail_knowledge', {
+            query_embedding: `[${queryEmbedding.join(',')}]`,
+            match_threshold: matchThreshold,
+            match_count: matchCount,
+            filter_intent: filterIntent,
+            only_approved: onlyApproved,
+        });
+        if (error) {
+            throw new EmbeddingError(`Similarity search failed: ${error.message}`, 'api_error', { context: { error } });
+        }
+        return (data || []).map((row) => ({
+            id: row.id,
+            questionSubject: row.question_subject,
+            questionText: row.question_text,
+            responseText: row.response_text,
+            intentCategory: row.intent_category,
+            qualityScore: Number(row.quality_score),
+            similarity: Number(row.similarity),
+            emailReceivedAt: row.email_received_at,
+            resolutionIndicator: row.resolution_indicator,
+        }));
+    }
+    // ===========================================================================
+    // Utility Methods
+    // ===========================================================================
+    /**
+     * Get service statistics
+     */
+    getStats() {
+        return {
+            cache: this.cache.getStats(),
+            totalEmbeddingsGenerated: this.totalEmbeddingsGenerated,
+            totalTokensUsed: this.totalTokensUsed,
+        };
+    }
+    /**
+     * Clear the embedding cache
+     */
+    clearCache() {
+        this.cache.clear();
+    }
+    /**
+     * Get embedding configuration
+     */
+    getConfig() {
+        return this.client.getConfig();
+    }
+}
+// =============================================================================
+// Singleton Instance
+// =============================================================================
+let embeddingServiceInstance = null;
+/**
+ * Get the singleton embedding service instance
+ */
+export function getEmbeddingService() {
+    if (!embeddingServiceInstance) {
+        const openaiApiKey = process.env.OPENAI_API_KEY;
+        const supabaseUrl = process.env.SUPABASE_URL || process.env.NEXT_PUBLIC_SUPABASE_URL;
+        const supabaseServiceRoleKey = process.env.SUPABASE_SERVICE_ROLE_KEY;
+        if (!openaiApiKey) {
+            throw new EmbeddingError('OPENAI_API_KEY environment variable is required for embedding service', 'validation');
+        }
+        if (!supabaseUrl || !supabaseServiceRoleKey) {
+            throw new EmbeddingError('SUPABASE_URL and SUPABASE_SERVICE_ROLE_KEY are required for embedding service', 'validation');
+        }
+        embeddingServiceInstance = new EmbeddingService({
+            openaiApiKey,
+            supabaseUrl,
+            supabaseServiceRoleKey,
+            embeddingModel: process.env.OPENAI_EMBEDDING_MODEL || 'text-embedding-3-small',
+            embeddingDimensions: parseInt(process.env.OPENAI_EMBEDDING_DIMENSIONS || '1536', 10),
+            cacheMaxSize: parseInt(process.env.EMBEDDING_CACHE_MAX_SIZE || '1000', 10),
+            cacheTtlMs: parseInt(process.env.EMBEDDING_CACHE_TTL_MS || '300000', 10),
+        });
+    }
+    return embeddingServiceInstance;
+}
+/**
+ * Reset the singleton (for testing)
+ */
+export function resetEmbeddingService() {
+    embeddingServiceInstance = null;
+}
+// Re-export types
+export { EmbeddingCache } from './cache.js';
+export { OpenAIEmbeddingClient } from './openai-client.js';
+//# sourceMappingURL=index.js.map