confused-ai-core 0.1.0

package/src/llm/cache.ts

@@ -0,0 +1,300 @@
+/**
+ * LLM Response Cache - Content-Addressable Caching for LLM Responses
+ *
+ * Agno-style caching layer to reduce costs and improve latency:
+ * - Content-addressable cache (hash of messages + model + params)
+ * - Configurable TTL and max entries
+ * - LRU eviction when full
+ * - In-memory with pluggable adapter interface
+ */
+
+import type { Message, GenerateOptions, GenerateResult } from './types.js';
+import type { MetricsCollector } from '../observability/types.js';
+
+/** Cache entry */
+interface CacheEntry {
+    readonly key: string;
+    readonly result: GenerateResult;
+    readonly createdAt: number;
+    readonly expiresAt: number;
+    accessedAt: number;
+    accessCount: number;
+}
+
+/** Cache configuration */
+export interface LLMCacheConfig {
+    /** Maximum number of cached entries (default: 1000) */
+    readonly maxEntries?: number;
+    /** TTL in milliseconds (default: 3600000 = 1 hour) */
+    readonly ttlMs?: number;
+    /** Enable cache (default: true) */
+    readonly enabled?: boolean;
+    /** Optional metrics collector */
+    readonly metrics?: MetricsCollector;
+    /** Hash function override (default: JSON hash) */
+    readonly hashFn?: (key: CacheKeyInput) => string;
+}
+
+/** Input for cache key generation */
+export interface CacheKeyInput {
+    readonly messages: Message[];
+    readonly model?: string;
+    readonly temperature?: number;
+    readonly maxTokens?: number;
+    readonly tools?: unknown[];
+}
+
+/** Cache statistics */
+export interface CacheStats {
+    readonly hits: number;
+    readonly misses: number;
+    readonly entries: number;
+    readonly hitRate: number;
+    readonly evictions: number;
+}
+
+/**
+ * LLM Response Cache with LRU eviction.
+ *
+ * @example
+ * const cache = new LLMCache({
+ *   maxEntries: 500,
+ *   ttlMs: 60 * 60 * 1000, // 1 hour
+ * });
+ *
+ * // Check cache before calling LLM
+ * const cached = cache.get({
+ *   messages: [{ role: 'user', content: 'Hello' }],
+ *   model: 'gpt-4o',
+ * });
+ *
+ * if (cached) {
+ *   return cached;
+ * }
+ *
+ * const result = await llm.generateText(messages, options);
+ * cache.set({ messages, model: 'gpt-4o' }, result);
+ */
+export class LLMCache {
+    private readonly cache = new Map<string, CacheEntry>();
+    private readonly config: Required<Omit<LLMCacheConfig, 'metrics' | 'hashFn'>> &
+        Pick<LLMCacheConfig, 'metrics' | 'hashFn'>;
+
+    private hits = 0;
+    private misses = 0;
+    private evictions = 0;
+
+    constructor(config: LLMCacheConfig = {}) {
+        this.config = {
+            maxEntries: config.maxEntries ?? 1000,
+            ttlMs: config.ttlMs ?? 3600_000, // 1 hour
+            enabled: config.enabled ?? true,
+            metrics: config.metrics,
+            hashFn: config.hashFn,
+        };
+    }
+
+    /** Check if caching is enabled */
+    isEnabled(): boolean {
+        return this.config.enabled;
+    }
+
+    /** Get cached result */
+    get(input: CacheKeyInput): GenerateResult | null {
+        if (!this.config.enabled) return null;
+
+        const key = this.getKey(input);
+        const entry = this.cache.get(key);
+
+        if (!entry) {
+            this.misses++;
+            this.recordMetric('cache_miss', 1);
+            return null;
+        }
+
+        // Check expiration
+        if (Date.now() > entry.expiresAt) {
+            this.cache.delete(key);
+            this.misses++;
+            this.recordMetric('cache_expired', 1);
+            return null;
+        }
+
+        // Update access info (LRU tracking)
+        entry.accessedAt = Date.now();
+        entry.accessCount++;
+
+        this.hits++;
+        this.recordMetric('cache_hit', 1);
+        return entry.result;
+    }
+
+    /** Set cached result */
+    set(input: CacheKeyInput, result: GenerateResult): void {
+        if (!this.config.enabled) return;
+
+        const key = this.getKey(input);
+        const now = Date.now();
+
+        // Evict if at capacity
+        if (this.cache.size >= this.config.maxEntries) {
+            this.evictLRU();
+        }
+
+        this.cache.set(key, {
+            key,
+            result,
+            createdAt: now,
+            expiresAt: now + this.config.ttlMs,
+            accessedAt: now,
+            accessCount: 1,
+        });
+
+        this.recordMetric('cache_set', 1);
+    }
+
+    /** Delete a cached entry */
+    delete(input: CacheKeyInput): boolean {
+        const key = this.getKey(input);
+        return this.cache.delete(key);
+    }
+
+    /** Check if entry exists (without affecting LRU) */
+    has(input: CacheKeyInput): boolean {
+        const key = this.getKey(input);
+        const entry = this.cache.get(key);
+        return entry !== undefined && Date.now() <= entry.expiresAt;
+    }
+
+    /** Clear all cached entries */
+    clear(): void {
+        this.cache.clear();
+        this.hits = 0;
+        this.misses = 0;
+        this.evictions = 0;
+    }
+
+    /** Get cache statistics */
+    getStats(): CacheStats {
+        const total = this.hits + this.misses;
+        return {
+            hits: this.hits,
+            misses: this.misses,
+            entries: this.cache.size,
+            hitRate: total > 0 ? this.hits / total : 0,
+            evictions: this.evictions,
+        };
+    }
+
+    /** Cleanup expired entries */
+    cleanup(): number {
+        const now = Date.now();
+        let removed = 0;
+
+        for (const [key, entry] of this.cache.entries()) {
+            if (now > entry.expiresAt) {
+                this.cache.delete(key);
+                removed++;
+            }
+        }
+
+        return removed;
+    }
+
+    // --- Private methods ---
+
+    private getKey(input: CacheKeyInput): string {
+        if (this.config.hashFn) {
+            return this.config.hashFn(input);
+        }
+
+        // Default: JSON-based hash
+        const normalized = {
+            messages: input.messages.map(m => ({
+                role: m.role,
+                content: m.content,
+            })),
+            model: input.model,
+            temperature: input.temperature,
+            maxTokens: input.maxTokens,
+            // Exclude tools from cache key by default (too variable)
+        };
+
+        return this.simpleHash(JSON.stringify(normalized));
+    }
+
+    private simpleHash(str: string): string {
+        let hash = 0;
+        for (let i = 0; i < str.length; i++) {
+            const char = str.charCodeAt(i);
+            hash = ((hash << 5) - hash) + char;
+            hash = hash & hash; // Convert to 32bit integer
+        }
+        return hash.toString(36);
+    }
+
+    private evictLRU(): void {
+        let oldest: CacheEntry | null = null;
+        let oldestKey = '';
+
+        for (const [key, entry] of this.cache.entries()) {
+            if (!oldest || entry.accessedAt < oldest.accessedAt) {
+                oldest = entry;
+                oldestKey = key;
+            }
+        }
+
+        if (oldestKey) {
+            this.cache.delete(oldestKey);
+            this.evictions++;
+            this.recordMetric('cache_eviction', 1);
+        }
+    }
+
+    private recordMetric(name: string, value: number): void {
+        this.config.metrics?.counter(`llm_cache.${name}`, value);
+    }
+}
+
+/**
+ * Wrap an LLM provider with caching.
+ * Note: Model should be passed separately to the cache key.
+ */
+export function withCache<T extends {
+    generateText(messages: Message[], options?: GenerateOptions): Promise<GenerateResult>;
+}>(
+    llm: T,
+    cache: LLMCache,
+    model?: string
+): T {
+    const originalGenerate = llm.generateText.bind(llm);
+
+    llm.generateText = async (messages: Message[], options?: GenerateOptions): Promise<GenerateResult> => {
+        const cacheInput: CacheKeyInput = {
+            messages,
+            model,
+            temperature: options?.temperature,
+            maxTokens: options?.maxTokens,
+            tools: options?.tools,
+        };
+
+        // Check cache
+        const cached = cache.get(cacheInput);
+        if (cached) {
+            return cached;
+        }
+
+        // Call LLM
+        const result = await originalGenerate(messages, options);
+
+        // Cache successful results
+        if (result.finishReason !== 'error') {
+            cache.set(cacheInput, result);
+        }
+
+        return result;
+    };
+
+    return llm;
+}
+

package/src/llm/index.ts

@@ -0,0 +1,22 @@
+/**
+ * LLM provider abstraction
+ */
+
+export * from './types.js';
+export { OpenAIProvider } from './openai-provider.js';
+export { createOpenRouterProvider } from './openrouter-provider.js';
+export type { OpenRouterProviderConfig } from './openrouter-provider.js';
+export {
+    resolveModelString,
+    isModelString,
+    PROVIDER as MODEL_PROVIDER,
+    OPENROUTER_BASE_URL,
+    OLLAMA_BASE_URL,
+    LLAMABARN_BASE_URL,
+} from './model-resolver.js';
+export type { ResolvedModelConfig } from './model-resolver.js';
+
+// LLM Caching
+export { LLMCache, withCache } from './cache.js';
+export type { LLMCacheConfig, CacheKeyInput, CacheStats } from './cache.js';
+

package/src/llm/model-resolver.ts

@@ -0,0 +1,81 @@
+/**
+ * Model string resolver — Agno-style "provider:model_id" support.
+ * Use a single string to pick provider and model without importing provider classes.
+ *
+ * Format: "provider:model_id" (case-insensitive provider)
+ * Examples: "openai:gpt-4o", "openrouter:anthropic/claude-3.5-sonnet", "ollama:llama3.2", "llamabarn:gpt-oss-20b"
+ */
+
+export const PROVIDER = {
+    OPENAI: 'openai',
+    OPENROUTER: 'openrouter',
+    OLLAMA: 'ollama',
+    LLAMABARN: 'llamabarn',
+} as const;
+
+export const OPENROUTER_BASE_URL = 'https://openrouter.ai/api/v1';
+export const OLLAMA_BASE_URL = 'http://localhost:11434/v1';
+export const LLAMABARN_BASE_URL = 'http://localhost:2276/v1';
+
+const ENV_OPENAI_API_KEY = 'OPENAI_API_KEY';
+const ENV_OPENROUTER_API_KEY = 'OPENROUTER_API_KEY';
+const ENV_LLAMABARN_API_KEY = 'LLAMABARN_API_KEY';
+
+export interface ResolvedModelConfig {
+    baseURL?: string;
+    apiKey?: string;
+    model: string;
+}
+
+/**
+ * Resolve a "provider:model_id" string to OpenAI-compatible config.
+ * Returns undefined if the string does not look like "provider:model_id".
+ */
+export function resolveModelString(
+    modelStr: string,
+    getEnv?: (key: string) => string | undefined
+): ResolvedModelConfig | undefined {
+    const env = getEnv ?? (typeof process !== 'undefined' ? (k: string) => process.env?.[k] : undefined);
+    const colon = modelStr.indexOf(':');
+    if (colon <= 0) return undefined;
+
+    const provider = modelStr.slice(0, colon).trim().toLowerCase();
+    const modelId = modelStr.slice(colon + 1).trim();
+    if (!modelId) return undefined;
+
+    switch (provider) {
+        case PROVIDER.OPENAI:
+            return {
+                apiKey: env?.(ENV_OPENAI_API_KEY),
+                model: modelId,
+            };
+        case PROVIDER.OPENROUTER:
+            return {
+                baseURL: OPENROUTER_BASE_URL,
+                apiKey: env?.(ENV_OPENROUTER_API_KEY),
+                model: modelId,
+            };
+        case PROVIDER.OLLAMA:
+            return {
+                baseURL: OLLAMA_BASE_URL,
+                apiKey: 'not-needed',
+                model: modelId,
+            };
+        case PROVIDER.LLAMABARN:
+            return {
+                baseURL: LLAMABARN_BASE_URL,
+                apiKey: env?.(ENV_LLAMABARN_API_KEY) ?? 'not-needed',
+                model: modelId,
+            };
+        default:
+            return undefined;
+    }
+}
+
+/**
+ * Check if a string looks like "provider:model_id".
+ */
+export function isModelString(s: string): boolean {
+    const colon = s.indexOf(':');
+    return colon > 0 && s.slice(colon + 1).trim().length > 0;
+}