@nxuss/lemma 0.1.0 → 0.2.0

package/dist/esm/subconscious/EmbeddingService.js

@@ -0,0 +1,189 @@
+import axios from 'axios';
+/**
+ * EmbeddingService handles communication with Ollama API for generating embeddings.
+ * Includes caching, error handling, and retry logic.
+ */
+export class EmbeddingService {
+    constructor(ollamaUrl = 'http://localhost:11434', model = 'llama3', maxRetries = 3, retryDelay = 1000) {
+        this.client = axios.create({
+            baseURL: ollamaUrl,
+            timeout: 30000, // 30 second timeout
+            headers: {
+                'Content-Type': 'application/json',
+            },
+        });
+        this.model = model;
+        this.cache = new Map();
+        this.maxRetries = maxRetries;
+        this.retryDelay = retryDelay;
+    }
+    /**
+     * Generate an embedding vector for the given text.
+     * Uses cache if available, otherwise calls Ollama API.
+     *
+     * @param text - Text to generate embedding for
+     * @param useCache - Whether to use cached embeddings (default true)
+     * @returns Embedding vector as array of numbers
+     */
+    async generateEmbedding(text, useCache = true) {
+        // Normalize text for consistent caching
+        const normalizedText = text.trim().toLowerCase();
+        const cacheKey = this.getCacheKey(normalizedText);
+        // Check cache first
+        if (useCache && this.cache.has(cacheKey)) {
+            console.log('[EmbeddingService] Cache hit for text');
+            return this.cache.get(cacheKey);
+        }
+        // Generate new embedding with retry logic
+        let lastError = null;
+        for (let attempt = 1; attempt <= this.maxRetries; attempt++) {
+            try {
+                const embedding = await this.callOllamaAPI(text);
+                // Cache the result
+                if (useCache) {
+                    this.cache.set(cacheKey, embedding);
+                }
+                console.log(`[EmbeddingService] Generated embedding (dimension: ${embedding.length})`);
+                return embedding;
+            }
+            catch (error) {
+                lastError = error;
+                console.warn(`[EmbeddingService] Attempt ${attempt}/${this.maxRetries} failed:`, error instanceof Error ? error.message : error);
+                // Wait before retrying (exponential backoff)
+                if (attempt < this.maxRetries) {
+                    const delay = this.retryDelay * Math.pow(2, attempt - 1);
+                    await this.sleep(delay);
+                }
+            }
+        }
+        // All retries failed
+        throw new Error(`Failed to generate embedding after ${this.maxRetries} attempts: ${lastError?.message}`);
+    }
+    /**
+     * Generate embeddings for multiple texts in batch.
+     *
+     * @param texts - Array of texts to generate embeddings for
+     * @param useCache - Whether to use cached embeddings (default true)
+     * @returns Array of embedding vectors
+     */
+    async generateEmbeddings(texts, useCache = true) {
+        const embeddings = [];
+        for (const text of texts) {
+            const embedding = await this.generateEmbedding(text, useCache);
+            embeddings.push(embedding);
+        }
+        return embeddings;
+    }
+    /**
+     * Call Ollama API to generate embedding.
+     *
+     * @param text - Text to generate embedding for
+     * @returns Embedding vector
+     */
+    async callOllamaAPI(text) {
+        try {
+            const response = await this.client.post('/api/embeddings', {
+                model: this.model,
+                prompt: text,
+            });
+            if (!response.data || !response.data.embedding) {
+                throw new Error('Invalid response from Ollama API: missing embedding');
+            }
+            return response.data.embedding;
+        }
+        catch (error) {
+            if (axios.isAxiosError(error)) {
+                if (error.code === 'ECONNREFUSED') {
+                    throw new Error('Cannot connect to Ollama. Make sure Ollama is running at ' + this.client.defaults.baseURL);
+                }
+                if (error.response?.status === 404) {
+                    throw new Error(`Model '${this.model}' not found. Pull it with: ollama pull ${this.model}`);
+                }
+                throw new Error(`Ollama API error: ${error.response?.status} - ${error.response?.data?.error || error.message}`);
+            }
+            throw error;
+        }
+    }
+    /**
+     * Generate a cache key from text.
+     * Uses a simple hash to avoid storing full text as keys.
+     */
+    getCacheKey(text) {
+        // Simple hash function for cache key
+        let hash = 0;
+        for (let i = 0; i < text.length; i++) {
+            const char = text.charCodeAt(i);
+            hash = (hash << 5) - hash + char;
+            hash = hash & hash; // Convert to 32-bit integer
+        }
+        return `emb_${hash}_${text.length}`;
+    }
+    /**
+     * Sleep for specified milliseconds.
+     */
+    sleep(ms) {
+        return new Promise((resolve) => setTimeout(resolve, ms));
+    }
+    /**
+     * Clear the embedding cache.
+     */
+    clearCache() {
+        this.cache.clear();
+        console.log('[EmbeddingService] Cache cleared');
+    }
+    /**
+     * Get cache statistics.
+     */
+    getCacheStats() {
+        return {
+            size: this.cache.size,
+            maxSize: 10000, // Could be configurable
+        };
+    }
+    /**
+     * Check if Ollama is available and the model is loaded.
+     */
+    async healthCheck() {
+        try {
+            // Try to generate a simple embedding
+            await this.callOllamaAPI('test');
+            return {
+                available: true,
+                model: this.model,
+            };
+        }
+        catch (error) {
+            return {
+                available: false,
+                model: this.model,
+                error: error instanceof Error ? error.message : String(error),
+            };
+        }
+    }
+    /**
+     * Calculate cosine similarity between two embedding vectors.
+     *
+     * @param vecA - First embedding vector
+     * @param vecB - Second embedding vector
+     * @returns Similarity score between 0 and 1
+     */
+    static cosineSimilarity(vecA, vecB) {
+        if (vecA.length !== vecB.length) {
+            throw new Error('Vectors must have the same dimension');
+        }
+        let dotProduct = 0;
+        let normA = 0;
+        let normB = 0;
+        for (let i = 0; i < vecA.length; i++) {
+            dotProduct += vecA[i] * vecB[i];
+            normA += vecA[i] * vecA[i];
+            normB += vecB[i] * vecB[i];
+        }
+        const denominator = Math.sqrt(normA) * Math.sqrt(normB);
+        if (denominator === 0) {
+            return 0;
+        }
+        return dotProduct / denominator;
+    }
+}
+//# sourceMappingURL=EmbeddingService.js.map

package/dist/esm/subconscious/EmbeddingService.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"EmbeddingService.js","sourceRoot":"","sources":["../../../src/subconscious/EmbeddingService.ts"],"names":[],"mappings":"AAAA,OAAO,KAAwB,MAAM,OAAO,CAAC;AAE7C;;;GAGG;AACH,MAAM,OAAO,gBAAgB;IAO3B,YACE,YAAoB,wBAAwB,EAC5C,QAAgB,QAAQ,EACxB,aAAqB,CAAC,EACtB,aAAqB,IAAI;QAEzB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;YACzB,OAAO,EAAE,SAAS;YAClB,OAAO,EAAE,KAAK,EAAE,oBAAoB;YACpC,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;aACnC;SACF,CAAC,CAAC;QACH,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,KAAK,GAAG,IAAI,GAAG,EAAE,CAAC;QACvB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,iBAAiB,CAAC,IAAY,EAAE,WAAoB,IAAI;QAC5D,wCAAwC;QACxC,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACjD,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC;QAElD,oBAAoB;QACpB,IAAI,QAAQ,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YACzC,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;YACrD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC;QACnC,CAAC;QAED,0CAA0C;QAC1C,IAAI,SAAS,GAAiB,IAAI,CAAC;QACnC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;YAC5D,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;gBAEjD,mBAAmB;gBACnB,IAAI,QAAQ,EAAE,CAAC;oBACb,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;gBACtC,CAAC;gBAED,OAAO,CAAC,GAAG,CAAC,sDAAsD,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC;gBACvF,OAAO,SAAS,CAAC;YACnB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,GAAG,KAAc,CAAC;gBAC3B,OAAO,CAAC,IAAI,CACV,8BAA8B,OAAO,IAAI,IAAI,CAAC,UAAU,UAAU,EAClE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAC/C,CAAC;gBAEF,6CAA6C;gBAC7C,IAAI,OAAO,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;oBAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;oBACzD,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC1B,CAAC;YACH,CAAC;QACH,CAAC;QAED,qBAAqB;QACrB,MAAM,IAAI,KAAK,CACb,sCAAsC,IAAI,CAAC,UAAU,cAAc,SAAS,EAAE,OAAO,EAAE,CACxF,CAAC;IACJ,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,kBAAkB,CAAC,KAAe,EAAE,WAAoB,IAAI;QAChE,MAAM,UAAU,GAAe,EAAE,CAAC;QAElC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;YAC/D,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC7B,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;IAED;;;;;OAKG;IACK,KAAK,CAAC,aAAa,CAAC,IAAY;QACtC,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,iBAAiB,EAAE;gBACzD,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,MAAM,EAAE,IAAI;aACb,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;gBAC/C,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;YACzE,CAAC;YAED,OAAO,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC;QACjC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC9B,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;oBAClC,MAAM,IAAI,KAAK,CACb,2DAA2D,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAC3F,CAAC;gBACJ,CAAC;gBACD,IAAI,KAAK,CAAC,QAAQ,EAAE,MAAM,KAAK,GAAG,EAAE,CAAC;oBACnC,MAAM,IAAI,KAAK,CACb,UAAU,IAAI,CAAC,KAAK,0CAA0C,IAAI,CAAC,KAAK,EAAE,CAC3E,CAAC;gBACJ,CAAC;gBACD,MAAM,IAAI,KAAK,CACb,qBAAqB,KAAK,CAAC,QAAQ,EAAE,MAAM,MAAM,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,KAAK,IAAI,KAAK,CAAC,OAAO,EAAE,CAChG,CAAC;YACJ,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,WAAW,CAAC,IAAY;QAC9B,qCAAqC;QACrC,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAChC,IAAI,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;YACjC,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,4BAA4B;QAClD,CAAC;QACD,OAAO,OAAO,IAAI,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;IACtC,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,EAAU;QACtB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED;;OAEG;IACH,UAAU;QACR,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAC;IAClD,CAAC;IAED;;OAEG;IACH,aAAa;QACX,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI;YACrB,OAAO,EAAE,KAAK,EAAE,wBAAwB;SACzC,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW;QACf,IAAI,CAAC;YACH,qCAAqC;YACrC,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YACjC,OAAO;gBACL,SAAS,EAAE,IAAI;gBACf,KAAK,EAAE,IAAI,CAAC,KAAK;aAClB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,SAAS,EAAE,KAAK;gBAChB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAC9D,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACH,MAAM,CAAC,gBAAgB,CAAC,IAAc,EAAE,IAAc;QACpD,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAC1D,CAAC;QAED,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,UAAU,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;YAChC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;YAC3B,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QAC7B,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACxD,IAAI,WAAW,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,CAAC;QACX,CAAC;QAED,OAAO,UAAU,GAAG,WAAW,CAAC;IAClC,CAAC;CACF"}

package/dist/esm/subconscious/SemanticCache.d.ts

@@ -0,0 +1,82 @@
+import { VectorStore } from './VectorStore';
+import { EmbeddingService } from './EmbeddingService';
+/**
+ * Result of a cache lookup operation.
+ */
+export interface CacheResult {
+    hit: boolean;
+    result?: any;
+    similarity?: number;
+    taskId?: string;
+    latencyMs: number;
+}
+/**
+ * SemanticCache implements the "short-circuit" logic for semantic caching.
+ * It checks if a similar task has been resolved before and returns cached results.
+ */
+export declare class SemanticCache {
+    private vectorStore;
+    private embeddingService;
+    private threshold;
+    private cacheHits;
+    private cacheMisses;
+    constructor(vectorStore: VectorStore, embeddingService: EmbeddingService, threshold?: number);
+    /**
+     * Check if a similar task exists in the cache.
+     * Returns cached result if similarity exceeds threshold.
+     *
+     * @param taskDescription - Description of the task to check
+     * @returns CacheResult with hit status and cached data if found
+     */
+    check(taskDescription: string): Promise<CacheResult>;
+    /**
+     * Store a task result in the cache for future lookups.
+     *
+     * @param taskId - Unique identifier for the task
+     * @param taskDescription - Description of the task
+     * @param result - Result to cache
+     * @param metadata - Additional metadata to store
+     */
+    store(taskId: string, taskDescription: string, result: any, metadata?: Record<string, any>): Promise<void>;
+    /**
+     * Calculate cosine similarity between two task descriptions.
+     * Useful for debugging and testing.
+     *
+     * @param taskA - First task description
+     * @param taskB - Second task description
+     * @returns Similarity score between 0 and 1
+     */
+    calculateSimilarity(taskA: string, taskB: string): Promise<number>;
+    /**
+     * Get cache performance metrics.
+     */
+    getMetrics(): {
+        hits: number;
+        misses: number;
+        total: number;
+        hitRate: number;
+    };
+    /**
+     * Reset cache metrics.
+     */
+    resetMetrics(): void;
+    /**
+     * Update the similarity threshold.
+     *
+     * @param threshold - New threshold value (0-1)
+     */
+    setThreshold(threshold: number): void;
+    /**
+     * Get the current similarity threshold.
+     */
+    getThreshold(): number;
+    /**
+     * Get the total number of cached tasks.
+     */
+    getCacheSize(): Promise<number>;
+    /**
+     * Clear all cached tasks.
+     */
+    clear(): Promise<void>;
+}
+//# sourceMappingURL=SemanticCache.d.ts.map

package/dist/esm/subconscious/SemanticCache.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"SemanticCache.d.ts","sourceRoot":"","sources":["../../../src/subconscious/SemanticCache.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAEtD;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,OAAO,CAAC;IACb,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;;GAGG;AACH,qBAAa,aAAa;IACxB,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,gBAAgB,CAAmB;IAC3C,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,SAAS,CAAa;IAC9B,OAAO,CAAC,WAAW,CAAa;gBAG9B,WAAW,EAAE,WAAW,EACxB,gBAAgB,EAAE,gBAAgB,EAClC,SAAS,GAAE,MAAa;IAO1B;;;;;;OAMG;IACG,KAAK,CAAC,eAAe,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAwD1D;;;;;;;OAOG;IACG,KAAK,CACT,MAAM,EAAE,MAAM,EACd,eAAe,EAAE,MAAM,EACvB,MAAM,EAAE,GAAG,EACX,QAAQ,GAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAM,GACjC,OAAO,CAAC,IAAI,CAAC;IAqBhB;;;;;;;OAOG;IACG,mBAAmB,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAcxE;;OAEG;IACH,UAAU,IAAI;QACZ,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,MAAM,CAAC;KACjB;IAYD;;OAEG;IACH,YAAY,IAAI,IAAI;IAMpB;;;;OAIG;IACH,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IAQrC;;OAEG;IACH,YAAY,IAAI,MAAM;IAItB;;OAEG;IACG,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC;IAIrC;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAK7B"}

package/dist/esm/subconscious/SemanticCache.js

@@ -0,0 +1,160 @@
+import { EmbeddingService } from './EmbeddingService';
+/**
+ * SemanticCache implements the "short-circuit" logic for semantic caching.
+ * It checks if a similar task has been resolved before and returns cached results.
+ */
+export class SemanticCache {
+    constructor(vectorStore, embeddingService, threshold = 0.95) {
+        this.cacheHits = 0;
+        this.cacheMisses = 0;
+        this.vectorStore = vectorStore;
+        this.embeddingService = embeddingService;
+        this.threshold = threshold;
+    }
+    /**
+     * Check if a similar task exists in the cache.
+     * Returns cached result if similarity exceeds threshold.
+     *
+     * @param taskDescription - Description of the task to check
+     * @returns CacheResult with hit status and cached data if found
+     */
+    async check(taskDescription) {
+        const startTime = Date.now();
+        try {
+            // Generate embedding for the task
+            const embedding = await this.embeddingService.generateEmbedding(taskDescription);
+            // Search for similar tasks
+            const matches = await this.vectorStore.searchSimilar(embedding, this.threshold, 1 // Only need the top match
+            );
+            const latencyMs = Date.now() - startTime;
+            if (matches.length > 0) {
+                // Cache hit!
+                this.cacheHits++;
+                const match = matches[0];
+                console.log(`[SemanticCache] ✓ HIT (similarity: ${match.similarity.toFixed(4)}, ` +
+                    `latency: ${latencyMs}ms)`);
+                return {
+                    hit: true,
+                    result: match.result,
+                    similarity: match.similarity,
+                    taskId: match.id,
+                    latencyMs,
+                };
+            }
+            else {
+                // Cache miss
+                this.cacheMisses++;
+                console.log(`[SemanticCache] ✗ MISS (latency: ${latencyMs}ms)`);
+                return {
+                    hit: false,
+                    latencyMs,
+                };
+            }
+        }
+        catch (error) {
+            console.error('[SemanticCache] Error during cache check:', error);
+            // On error, treat as cache miss
+            this.cacheMisses++;
+            return {
+                hit: false,
+                latencyMs: Date.now() - startTime,
+            };
+        }
+    }
+    /**
+     * Store a task result in the cache for future lookups.
+     *
+     * @param taskId - Unique identifier for the task
+     * @param taskDescription - Description of the task
+     * @param result - Result to cache
+     * @param metadata - Additional metadata to store
+     */
+    async store(taskId, taskDescription, result, metadata = {}) {
+        try {
+            // Generate embedding for the task
+            const embedding = await this.embeddingService.generateEmbedding(taskDescription);
+            // Store in vector database
+            await this.vectorStore.insertVector(taskId, embedding, taskDescription, result, metadata);
+            console.log(`[SemanticCache] Stored result for task: ${taskId}`);
+        }
+        catch (error) {
+            console.error('[SemanticCache] Error storing result:', error);
+            throw error;
+        }
+    }
+    /**
+     * Calculate cosine similarity between two task descriptions.
+     * Useful for debugging and testing.
+     *
+     * @param taskA - First task description
+     * @param taskB - Second task description
+     * @returns Similarity score between 0 and 1
+     */
+    async calculateSimilarity(taskA, taskB) {
+        try {
+            const [embeddingA, embeddingB] = await this.embeddingService.generateEmbeddings([
+                taskA,
+                taskB,
+            ]);
+            return EmbeddingService.cosineSimilarity(embeddingA, embeddingB);
+        }
+        catch (error) {
+            console.error('[SemanticCache] Error calculating similarity:', error);
+            throw error;
+        }
+    }
+    /**
+     * Get cache performance metrics.
+     */
+    getMetrics() {
+        const total = this.cacheHits + this.cacheMisses;
+        const hitRate = total > 0 ? this.cacheHits / total : 0;
+        return {
+            hits: this.cacheHits,
+            misses: this.cacheMisses,
+            total,
+            hitRate,
+        };
+    }
+    /**
+     * Reset cache metrics.
+     */
+    resetMetrics() {
+        this.cacheHits = 0;
+        this.cacheMisses = 0;
+        console.log('[SemanticCache] Metrics reset');
+    }
+    /**
+     * Update the similarity threshold.
+     *
+     * @param threshold - New threshold value (0-1)
+     */
+    setThreshold(threshold) {
+        if (threshold < 0 || threshold > 1) {
+            throw new Error('Threshold must be between 0 and 1');
+        }
+        this.threshold = threshold;
+        console.log(`[SemanticCache] Threshold updated to ${threshold}`);
+    }
+    /**
+     * Get the current similarity threshold.
+     */
+    getThreshold() {
+        return this.threshold;
+    }
+    /**
+     * Get the total number of cached tasks.
+     */
+    async getCacheSize() {
+        return await this.vectorStore.getCount();
+    }
+    /**
+     * Clear all cached tasks.
+     */
+    async clear() {
+        await this.vectorStore.clear();
+        this.resetMetrics();
+        console.log('[SemanticCache] Cache cleared');
+    }
+}
+//# sourceMappingURL=SemanticCache.js.map

package/dist/esm/subconscious/SemanticCache.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"SemanticCache.js","sourceRoot":"","sources":["../../../src/subconscious/SemanticCache.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAatD;;;GAGG;AACH,MAAM,OAAO,aAAa;IAOxB,YACE,WAAwB,EACxB,gBAAkC,EAClC,YAAoB,IAAI;QANlB,cAAS,GAAW,CAAC,CAAC;QACtB,gBAAW,GAAW,CAAC,CAAC;QAO9B,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QACzC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,KAAK,CAAC,eAAuB;QACjC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,IAAI,CAAC;YACH,kCAAkC;YAClC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC;YAEjF,2BAA2B;YAC3B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,aAAa,CAClD,SAAS,EACT,IAAI,CAAC,SAAS,EACd,CAAC,CAAC,0BAA0B;aAC7B,CAAC;YAEF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YAEzC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,aAAa;gBACb,IAAI,CAAC,SAAS,EAAE,CAAC;gBACjB,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;gBAEzB,OAAO,CAAC,GAAG,CACT,sCAAsC,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI;oBACrE,YAAY,SAAS,KAAK,CAC3B,CAAC;gBAEF,OAAO;oBACL,GAAG,EAAE,IAAI;oBACT,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,UAAU,EAAE,KAAK,CAAC,UAAU;oBAC5B,MAAM,EAAE,KAAK,CAAC,EAAE;oBAChB,SAAS;iBACV,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,aAAa;gBACb,IAAI,CAAC,WAAW,EAAE,CAAC;gBAEnB,OAAO,CAAC,GAAG,CAAC,oCAAoC,SAAS,KAAK,CAAC,CAAC;gBAEhE,OAAO;oBACL,GAAG,EAAE,KAAK;oBACV,SAAS;iBACV,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,2CAA2C,EAAE,KAAK,CAAC,CAAC;YAElE,gCAAgC;YAChC,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,OAAO;gBACL,GAAG,EAAE,KAAK;gBACV,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;aAClC,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,KAAK,CACT,MAAc,EACd,eAAuB,EACvB,MAAW,EACX,WAAgC,EAAE;QAElC,IAAI,CAAC;YACH,kCAAkC;YAClC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC;YAEjF,2BAA2B;YAC3B,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,CACjC,MAAM,EACN,SAAS,EACT,eAAe,EACf,MAAM,EACN,QAAQ,CACT,CAAC;YAEF,OAAO,CAAC,GAAG,CAAC,2CAA2C,MAAM,EAAE,CAAC,CAAC;QACnE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,uCAAuC,EAAE,KAAK,CAAC,CAAC;YAC9D,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,mBAAmB,CAAC,KAAa,EAAE,KAAa;QACpD,IAAI,CAAC;YACH,MAAM,CAAC,UAAU,EAAE,UAAU,CAAC,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,CAAC;gBAC9E,KAAK;gBACL,KAAK;aACN,CAAC,CAAC;YAEH,OAAO,gBAAgB,CAAC,gBAAgB,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QACnE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,+CAA+C,EAAE,KAAK,CAAC,CAAC;YACtE,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACH,UAAU;QAMR,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC;QAChD,MAAM,OAAO,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAEvD,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,SAAS;YACpB,MAAM,EAAE,IAAI,CAAC,WAAW;YACxB,KAAK;YACL,OAAO;SACR,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,YAAY;QACV,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC;QACnB,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;QACrB,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;IAC/C,CAAC;IAED;;;;OAIG;IACH,YAAY,CAAC,SAAiB;QAC5B,IAAI,SAAS,GAAG,CAAC,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,OAAO,CAAC,GAAG,CAAC,wCAAwC,SAAS,EAAE,CAAC,CAAC;IACnE,CAAC;IAED;;OAEG;IACH,YAAY;QACV,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY;QAChB,OAAO,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC;IAC3C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QAC/B,IAAI,CAAC,YAAY,EAAE,CAAC;QACpB,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;IAC/C,CAAC;CACF"}

package/dist/esm/subconscious/SubconsciousEngine.d.ts

@@ -0,0 +1,121 @@
+import { CacheResult } from './SemanticCache';
+import { EventEmitter } from 'events';
+/**
+ * Configuration options for SubconsciousEngine.
+ */
+export interface SubconsciousConfig {
+    chromaUrl?: string;
+    ollamaUrl?: string;
+    model?: string;
+    collectionName?: string;
+    threshold?: number;
+    enableBackgroundVectorization?: boolean;
+}
+/**
+ * Task to be processed by the engine.
+ */
+export interface Task {
+    id: string;
+    description: string;
+    metadata?: Record<string, any>;
+}
+/**
+ * Result of task processing.
+ */
+export interface TaskResult {
+    taskId: string;
+    result: any;
+    fromCache: boolean;
+    similarity?: number;
+    latencyMs: number;
+    timestamp: string;
+}
+/**
+ * Engine metrics for monitoring performance.
+ */
+export interface EngineMetrics {
+    totalTasks: number;
+    cacheHits: number;
+    cacheMisses: number;
+    hitRate: number;
+    avgCacheLatencyMs: number;
+    avgMissLatencyMs: number;
+    totalCachedTasks: number;
+    backgroundQueueSize: number;
+}
+/**
+ * SubconsciousEngine is the main orchestrator for semantic caching.
+ * It intercepts tasks, checks for cached results, and manages background vectorization.
+ */
+export declare class SubconsciousEngine extends EventEmitter {
+    private vectorStore;
+    private embeddingService;
+    private semanticCache;
+    private config;
+    private isInitialized;
+    private backgroundQueue;
+    private isProcessingQueue;
+    private cacheLatencies;
+    private missLatencies;
+    constructor(config?: SubconsciousConfig);
+    /**
+     * Initialize the Subconscious Engine.
+     * Must be called before using the engine.
+     */
+    initialize(): Promise<void>;
+    /**
+     * Intercept a task and check for cached results.
+     * This is the main entry point for the semantic short-circuit.
+     *
+     * @param task - Task to intercept
+     * @returns CacheResult indicating if a cached result was found
+     */
+    interceptTask(task: Task): Promise<CacheResult>;
+    /**
+     * Store a task result for future caching.
+     * If background vectorization is enabled, queues the task for async processing.
+     *
+     * @param task - Original task
+     * @param result - Result to cache
+     */
+    storeResult(task: Task, result: any): Promise<void>;
+    /**
+     * Process the background vectorization queue.
+     * Runs asynchronously without blocking the main thread.
+     */
+    private processBackgroundQueue;
+    /**
+     * Get comprehensive engine metrics.
+     */
+    getMetrics(): Promise<EngineMetrics>;
+    /**
+     * Reset all metrics.
+     */
+    resetMetrics(): void;
+    /**
+     * Update the similarity threshold.
+     */
+    setThreshold(threshold: number): void;
+    /**
+     * Get the current configuration.
+     */
+    getConfig(): Required<SubconsciousConfig>;
+    /**
+     * Clear all cached tasks and reset metrics.
+     */
+    clear(): Promise<void>;
+    /**
+     * Shutdown the engine gracefully.
+     * Waits for background queue to finish processing.
+     */
+    shutdown(): Promise<void>;
+    /**
+     * Check if the engine is ready to use.
+     */
+    isReady(): boolean;
+    /**
+     * Sleep for specified milliseconds.
+     */
+    private sleep;
+}
+//# sourceMappingURL=SubconsciousEngine.d.ts.map

package/dist/esm/subconscious/SubconsciousEngine.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"SubconsciousEngine.d.ts","sourceRoot":"","sources":["../../../src/subconscious/SubconsciousEngine.ts"],"names":[],"mappings":"AAEA,OAAO,EAAiB,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC7D,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AAEtC;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,6BAA6B,CAAC,EAAE,OAAO,CAAC;CACzC;AAED;;GAEG;AACH,MAAM,WAAW,IAAI;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,GAAG,CAAC;IACZ,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,EAAE,MAAM,CAAC;IACzB,mBAAmB,EAAE,MAAM,CAAC;CAC7B;AAED;;;GAGG;AACH,qBAAa,kBAAmB,SAAQ,YAAY;IAClD,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,gBAAgB,CAAmB;IAC3C,OAAO,CAAC,aAAa,CAAgB;IACrC,OAAO,CAAC,MAAM,CAA+B;IAC7C,OAAO,CAAC,aAAa,CAAkB;IACvC,OAAO,CAAC,eAAe,CAKf;IACR,OAAO,CAAC,iBAAiB,CAAkB;IAC3C,OAAO,CAAC,cAAc,CAAgB;IACtC,OAAO,CAAC,aAAa,CAAgB;gBAEzB,MAAM,GAAE,kBAAuB;IA+B3C;;;OAGG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAqCjC;;;;;;OAMG;IACG,aAAa,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,WAAW,CAAC;IA4BrD;;;;;;OAMG;IACG,WAAW,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IAoCzD;;;OAGG;YACW,sBAAsB;IAqCpC;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,aAAa,CAAC;IA0B1C;;OAEG;IACH,YAAY,IAAI,IAAI;IAOpB;;OAEG;IACH,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IAKrC;;OAEG;IACH,SAAS,IAAI,QAAQ,CAAC,kBAAkB,CAAC;IAIzC;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAO5B;;;OAGG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAgB/B;;OAEG;IACH,OAAO,IAAI,OAAO;IAIlB;;OAEG;IACH,OAAO,CAAC,KAAK;CAGd"}