@revealui/ai 0.1.0

package/dist/llm/response-cache.js

@@ -0,0 +1,233 @@
+/**
+ * Response Cache for LLM Providers
+ *
+ * Application-level caching for LLM responses. Works with any provider.
+ * Provides 100% cost savings on cache hits vs Anthropic's 90% token-level savings.
+ *
+ * Use cases:
+ * - Vultr (doesn't have prompt caching)
+ * - OpenAI (doesn't have prompt caching)
+ * - Any provider without built-in caching
+ * - Supplement to Anthropic prompt caching for exact duplicates
+ *
+ * Features:
+ * - LRU eviction policy
+ * - Configurable TTL (default: 5 minutes)
+ * - SHA-256 based cache keys
+ * - Hit/miss statistics
+ * - Memory-efficient storage
+ */
+import { createHash } from 'node:crypto';
+import { LRUCache } from 'lru-cache';
+/**
+ * Response cache for LLM completions
+ *
+ * @example
+ * ```typescript
+ * const cache = new ResponseCache({ max: 1000, ttl: 5 * 60 * 1000 })
+ *
+ * const key = cache.getCacheKey(messages, options)
+ * const cached = cache.get(key)
+ *
+ * if (cached) {
+ *   return { ...cached, cached: true }
+ * }
+ *
+ * const response = await llm.chat(messages, options)
+ * cache.set(key, response)
+ * ```
+ */
+export class ResponseCache {
+    cache;
+    stats;
+    enableStats;
+    constructor(options = {}) {
+        this.enableStats = options.enableStats ?? true;
+        this.cache = new LRUCache({
+            max: options.max ?? 1000,
+            ttl: options.ttl ?? 5 * 60 * 1000, // 5 minutes default
+            updateAgeOnGet: true, // Reset TTL on access
+            dispose: () => {
+                if (this.enableStats) {
+                    this.stats.evictions++;
+                }
+            },
+        });
+        this.stats = {
+            hits: 0,
+            misses: 0,
+            evictions: 0,
+        };
+    }
+    /**
+     * Generate cache key from messages and options
+     *
+     * Uses SHA-256 hash of JSON-serialized payload for:
+     * - Consistent key generation
+     * - Fixed-length keys (64 chars)
+     * - Security (prevents key enumeration)
+     */
+    getCacheKey(messages, options) {
+        // Normalize messages (remove non-deterministic fields)
+        const normalizedMessages = messages.map((msg) => ({
+            role: msg.role,
+            content: msg.content,
+            name: msg.name,
+            toolCallId: msg.toolCallId,
+        }));
+        const payload = {
+            messages: normalizedMessages,
+            temperature: options?.temperature,
+            maxTokens: options?.maxTokens,
+            tools: options?.tools,
+            model: options?.model,
+        };
+        const json = JSON.stringify(payload);
+        return createHash('sha256').update(json).digest('hex');
+    }
+    /**
+     * Get cached response
+     *
+     * @returns Cached response or undefined if not found/expired
+     */
+    get(key) {
+        const cached = this.cache.get(key);
+        if (cached) {
+            if (this.enableStats) {
+                this.stats.hits++;
+            }
+            return cached;
+        }
+        if (this.enableStats) {
+            this.stats.misses++;
+        }
+        return undefined;
+    }
+    /**
+     * Store response in cache
+     */
+    set(key, response) {
+        // Ensure timestamp is set
+        if (!response.timestamp) {
+            response.timestamp = Date.now();
+        }
+        this.cache.set(key, response);
+    }
+    /**
+     * Check if key exists in cache (without updating stats)
+     */
+    has(key) {
+        return this.cache.has(key);
+    }
+    /**
+     * Remove entry from cache
+     */
+    delete(key) {
+        return this.cache.delete(key);
+    }
+    /**
+     * Clear all cached responses
+     */
+    clear() {
+        this.cache.clear();
+        if (this.enableStats) {
+            // Reset stats but preserve eviction count
+            this.stats.hits = 0;
+            this.stats.misses = 0;
+        }
+    }
+    /**
+     * Get cache statistics
+     */
+    getStats() {
+        const totalRequests = this.stats.hits + this.stats.misses;
+        const hitRate = totalRequests > 0 ? (this.stats.hits / totalRequests) * 100 : 0;
+        return {
+            hits: this.stats.hits,
+            misses: this.stats.misses,
+            evictions: this.stats.evictions,
+            hitRate: Math.round(hitRate * 100) / 100, // 2 decimal places
+            size: this.cache.size,
+        };
+    }
+    /**
+     * Reset statistics
+     */
+    resetStats() {
+        this.stats = {
+            hits: 0,
+            misses: 0,
+            evictions: 0,
+        };
+    }
+    /**
+     * Get cache size (number of entries)
+     */
+    get size() {
+        return this.cache.size;
+    }
+    /**
+     * Get maximum cache size
+     */
+    get maxSize() {
+        return this.cache.max;
+    }
+}
+/**
+ * Global response cache instance
+ * Shared across all LLM client instances in the process
+ */
+let globalCache = null;
+/**
+ * Get or create global response cache
+ *
+ * @example
+ * ```typescript
+ * const cache = getGlobalResponseCache()
+ * const stats = cache.getStats()
+ * // Hit rate: ${stats.hitRate}%
+ * ```
+ */
+export function getGlobalResponseCache(options) {
+    if (!globalCache) {
+        globalCache = new ResponseCache(options);
+    }
+    return globalCache;
+}
+/**
+ * Clear global response cache
+ */
+export function clearGlobalResponseCache() {
+    if (globalCache) {
+        globalCache.clear();
+    }
+}
+/**
+ * Calculate cost savings from response caching
+ *
+ * @example
+ * ```typescript
+ * const stats = cache.getStats()
+ * const savings = calculateResponseCacheSavings(stats, {
+ *   avgInputTokens: 3000,
+ *   avgOutputTokens: 500,
+ *   inputCostPerM: 3.0,
+ *   outputCostPerM: 15.0,
+ * })
+ *
+ * // Saved: $${savings.totalSaved.toFixed(2)}
+ * // Avoided ${savings.tokensAvoided.toLocaleString()} tokens
+ * ```
+ */
+export function calculateResponseCacheSavings(stats, pricing) {
+    const requestsAvoided = stats.hits;
+    const tokensAvoided = (pricing.avgInputTokens + pricing.avgOutputTokens) * requestsAvoided;
+    const inputSaved = (requestsAvoided * pricing.avgInputTokens * pricing.inputCostPerM) / 1_000_000;
+    const outputSaved = (requestsAvoided * pricing.avgOutputTokens * pricing.outputCostPerM) / 1_000_000;
+    const totalSaved = inputSaved + outputSaved;
+    return {
+        totalSaved,
+        tokensAvoided,
+        requestsAvoided,
+    };
+}

package/dist/llm/semantic-cache.d.ts

@@ -0,0 +1,179 @@
+/**
+ * Semantic Cache for LLM Responses
+ *
+ * Caches responses based on semantic similarity rather than exact matches.
+ * Provides 73% cost reduction vs 20% from exact match caching.
+ *
+ * Key Features:
+ * - Vector-based similarity search (cosine similarity)
+ * - Configurable similarity threshold (default: 0.95)
+ * - Automatic cache warming
+ * - TTL-based expiration
+ * - Hit/miss statistics
+ *
+ * Real-world impact:
+ * - 73% cost reduction (vs 20% for exact matches)
+ * - 96.9% latency reduction on cache hits
+ * - 65% cache hit rate (vs 18% for exact matches)
+ *
+ * Examples of semantic matches:
+ * - "How do I reset my password?" ✅ cached
+ * - "What's the process to reset my password?" ✅ also cached (same meaning)
+ * - "Help me reset my password" ✅ also cached (same meaning)
+ *
+ * @see https://redis.io/blog/what-is-semantic-caching/
+ */
+import type { Message } from './providers/base.js';
+export interface SemanticCacheOptions {
+    /** Similarity threshold for cache hits (0-1, default: 0.95) */
+    similarityThreshold?: number;
+    /** Time to live in milliseconds (default: 1 hour) */
+    ttl?: number;
+    /** Enable statistics tracking (default: true) */
+    enableStats?: boolean;
+    /** User ID for multi-tenant caching */
+    userId?: string;
+    /** Site ID for multi-tenant caching */
+    siteId?: string;
+}
+export interface SemanticCacheStats {
+    hits: number;
+    misses: number;
+    hitRate: number;
+    avgSimilarity: number;
+    totalQueries: number;
+}
+export interface CachedSemanticResponse {
+    query: string;
+    response: string;
+    embedding: number[];
+    similarity: number;
+    timestamp: number;
+    usage?: {
+        promptTokens: number;
+        completionTokens: number;
+        totalTokens: number;
+    };
+}
+/**
+ * Semantic cache that uses vector similarity for intelligent caching
+ *
+ * @example
+ * ```typescript
+ * const cache = new SemanticCache({ similarityThreshold: 0.95 })
+ *
+ * // Check cache
+ * const cached = await cache.get("How do I reset my password?")
+ * if (cached) {
+ *   return cached.response // 100% cost savings!
+ * }
+ *
+ * // Cache miss - call LLM
+ * const response = await llm.chat(messages)
+ *
+ * // Store in cache
+ * await cache.set("How do I reset my password?", response)
+ * ```
+ */
+export declare class SemanticCache {
+    private vectorService;
+    private options;
+    private stats;
+    private logger;
+    constructor(options?: SemanticCacheOptions);
+    /**
+     * Get cached response for semantically similar query
+     *
+     * @param query - User query to search for
+     * @returns Cached response if similar query found, undefined otherwise
+     */
+    get(query: string): Promise<CachedSemanticResponse | undefined>;
+    /**
+     * Store response in semantic cache
+     *
+     * @param query - Original user query
+     * @param response - LLM response to cache
+     * @param usage - Optional token usage stats
+     */
+    set(query: string, response: string, usage?: {
+        promptTokens: number;
+        completionTokens: number;
+        totalTokens: number;
+    }): Promise<void>;
+    /**
+     * Extract query text from messages
+     *
+     * Combines all user messages into a single query string for caching
+     */
+    extractQuery(messages: Message[]): string;
+    /**
+     * Get cache statistics
+     */
+    getStats(): SemanticCacheStats;
+    /**
+     * Reset statistics
+     */
+    resetStats(): void;
+    /**
+     * Clear expired cache entries
+     *
+     * @returns Number of entries cleared
+     */
+    clearExpired(): Promise<number>;
+    /**
+     * Warm cache with common queries
+     *
+     * Pre-populate cache with FAQ responses
+     *
+     * @example
+     * ```typescript
+     * await cache.warmCache([
+     *   { query: 'How do I reset my password?', response: 'Go to...' },
+     *   { query: 'What are your hours?', response: 'We are open...' },
+     * ])
+     * ```
+     */
+    warmCache(entries: Array<{
+        query: string;
+        response: string;
+    }>): Promise<void>;
+}
+/**
+ * Calculate cost savings from semantic caching
+ *
+ * @example
+ * ```typescript
+ * const stats = cache.getStats()
+ * const savings = calculateSemanticCacheSavings(stats, {
+ *   avgTokensPerQuery: 3500,
+ *   costPerMTokens: 3.0,
+ * })
+ *
+ * // Saved: $X.XX, Avoided: N API calls
+ * logger.info('Cache savings', { saved: savings.totalSaved, avoided: savings.queriesAvoided })
+ * ```
+ */
+export declare function calculateSemanticCacheSavings(stats: SemanticCacheStats, pricing: {
+    avgTokensPerQuery: number;
+    costPerMTokens: number;
+}): {
+    totalSaved: number;
+    queriesAvoided: number;
+    tokensAvoided: number;
+};
+/**
+ * Get or create global semantic cache
+ *
+ * @example
+ * ```typescript
+ * const cache = getGlobalSemanticCache({ similarityThreshold: 0.95 })
+ * const stats = cache.getStats()
+ * // Access stats.hitRate, stats.hits, stats.misses, etc.
+ * ```
+ */
+export declare function getGlobalSemanticCache(options?: SemanticCacheOptions): SemanticCache;
+/**
+ * Clear global semantic cache
+ */
+export declare function clearGlobalSemanticCache(): void;
+//# sourceMappingURL=semantic-cache.d.ts.map

package/dist/llm/semantic-cache.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"semantic-cache.d.ts","sourceRoot":"","sources":["../../src/llm/semantic-cache.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAKH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAA;AAElD,MAAM,WAAW,oBAAoB;IACnC,+DAA+D;IAC/D,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B,qDAAqD;IACrD,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,iDAAiD;IACjD,WAAW,CAAC,EAAE,OAAO,CAAA;IACrB,uCAAuC;IACvC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,uCAAuC;IACvC,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,MAAM,CAAA;IACf,aAAa,EAAE,MAAM,CAAA;IACrB,YAAY,EAAE,MAAM,CAAA;CACrB;AAED,MAAM,WAAW,sBAAsB;IACrC,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,EAAE,CAAA;IACnB,UAAU,EAAE,MAAM,CAAA;IAClB,SAAS,EAAE,MAAM,CAAA;IACjB,KAAK,CAAC,EAAE;QACN,YAAY,EAAE,MAAM,CAAA;QACpB,gBAAgB,EAAE,MAAM,CAAA;QACxB,WAAW,EAAE,MAAM,CAAA;KACpB,CAAA;CACF;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,aAAa;IACxB,OAAO,CAAC,aAAa,CAAqB;IAC1C,OAAO,CAAC,OAAO,CAAgC;IAC/C,OAAO,CAAC,KAAK,CAIZ;IACD,OAAO,CAAC,MAAM,CAAkC;gBAEpC,OAAO,GAAE,oBAAyB;IAiB9C;;;;;OAKG;IACG,GAAG,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,sBAAsB,GAAG,SAAS,CAAC;IAwFrE;;;;;;OAMG;IACG,GAAG,CACP,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,KAAK,CAAC,EAAE;QACN,YAAY,EAAE,MAAM,CAAA;QACpB,gBAAgB,EAAE,MAAM,CAAA;QACxB,WAAW,EAAE,MAAM,CAAA;KACpB,GACA,OAAO,CAAC,IAAI,CAAC;IAqChB;;;;OAIG;IACH,YAAY,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,MAAM;IAOzC;;OAEG;IACH,QAAQ,IAAI,kBAAkB;IAmB9B;;OAEG;IACH,UAAU,IAAI,IAAI;IAQlB;;;;OAIG;IACG,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC;IAMrC;;;;;;;;;;;;OAYG;IACG,SAAS,CAAC,OAAO,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;CAKpF;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,6BAA6B,CAC3C,KAAK,EAAE,kBAAkB,EACzB,OAAO,EAAE;IACP,iBAAiB,EAAE,MAAM,CAAA;IACzB,cAAc,EAAE,MAAM,CAAA;CACvB,GACA;IACD,UAAU,EAAE,MAAM,CAAA;IAClB,cAAc,EAAE,MAAM,CAAA;IACtB,aAAa,EAAE,MAAM,CAAA;CACtB,CAUA;AAQD;;;;;;;;;GASG;AACH,wBAAgB,sBAAsB,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,aAAa,CAKpF;AAED;;GAEG;AACH,wBAAgB,wBAAwB,IAAI,IAAI,CAE/C"}