@nxuss/lemma 0.5.8 → 0.6.0

package/dist/esm/cli/lemma-proxy.js

@@ -154,9 +154,9 @@ async function ensureChromaRunning() {
     }
     console.log(`\n📦 \x1b[35m[ChromaDB]\x1b[0m ChromaDB is not running on port ${chromaPort}.`);
     console.log(`🔄 Starting ChromaDB in background...`);
+    const chromaDataPath = path.join(CACHE_DIR, 'chroma_data');
     try {
         const { spawn } = require('child_process');
-        const chromaDataPath = path.join(process.cwd(), 'chroma_data');
         if (!fs.existsSync(chromaDataPath)) {
             fs.mkdirSync(chromaDataPath, { recursive: true });
         }
@@ -179,7 +179,7 @@ async function ensureChromaRunning() {
     }
     catch (err) {
         console.error(`❌ Failed to automatically start ChromaDB: ${err.message}`);
-        console.error(`👉 Please start it manually: chroma run --path ./chroma_data --port ${chromaPort}`);
+        console.error(`👉 Please start it manually: chroma run --path ${chromaDataPath} --port ${chromaPort}`);
         return false;
     }
 }
@@ -600,15 +600,16 @@ async function getEmbedding(text) {
         return null;
     }
 }
-async function semanticGet(provider, prompt) {
+async function semanticGet(provider, prompt, projectName = 'global') {
     if (!await isPro())
         return null;
     const emb = await getEmbedding(prompt);
     if (!emb)
         return null;
+    let localHit = null;
     try {
         const collection = await chroma.getOrCreateCollection({
-            name: 'lemma-cache',
+            name: `lemma-cache-${projectName}`,
             embeddingFunction: dummyEmbeddingFunction,
             metadata: { "hnsw:space": "cosine" }
         });
@@ -616,33 +617,83 @@ async function semanticGet(provider, prompt) {
             queryEmbeddings: [emb],
             nResults: 10
         });
-        if (!res.ids[0] || res.ids[0].length === 0)
-            return null;
-        for (let i = 0; i < res.ids[0].length; i++) {
-            const metadata = res.metadatas[0][i];
-            if (metadata.provider !== provider)
-                continue;
-            const distances = res.distances;
-            let distance = 1.0;
-            if (distances && distances[0] && typeof distances[0][i] === 'number') {
-                distance = distances[0][i];
-            }
-            const similarity = Math.max(0, 1 - distance);
-            const THRESHOLD = 0.82; // raised from 0.7 to 0.82 to prevent false positives
-            if (similarity >= THRESHOLD) {
-                return {
-                    data: JSON.parse(metadata.response),
-                    similarity,
-                    id: res.ids[0][i],
-                    prompt: metadata.prompt
-                };
+        if (res.ids[0] && res.ids[0].length > 0) {
+            for (let i = 0; i < res.ids[0].length; i++) {
+                const metadata = res.metadatas[0][i];
+                if (metadata.provider !== provider)
+                    continue;
+                const distances = res.distances;
+                let distance = 1.0;
+                if (distances && distances[0] && typeof distances[0][i] === 'number') {
+                    distance = distances[0][i];
+                }
+                const similarity = Math.max(0, 1 - distance);
+                const THRESHOLD = 0.82; // raised from 0.7 to 0.82 to prevent false positives
+                if (similarity >= THRESHOLD) {
+                    localHit = {
+                        data: JSON.parse(metadata.response),
+                        similarity,
+                        id: res.ids[0][i],
+                        prompt: metadata.prompt,
+                        hiveMind: false
+                    };
+                    break;
+                }
             }
         }
     }
     catch (e) { }
+    if (localHit)
+        return localHit;
+    // ─── Cross-Project Bug Telepathy Fallback ───
+    try {
+        const collections = await chroma.listCollections();
+        const projectCollections = collections.filter((c) => c.name.startsWith('lemma-cache-') && c.name !== `lemma-cache-${projectName}`);
+        for (const colInfo of projectCollections) {
+            try {
+                const col = await chroma.getOrCreateCollection({
+                    name: colInfo.name,
+                    embeddingFunction: dummyEmbeddingFunction,
+                    metadata: { "hnsw:space": "cosine" }
+                });
+                const otherRes = await col.query({
+                    queryEmbeddings: [emb],
+                    nResults: 3
+                });
+                if (otherRes.ids[0] && otherRes.ids[0].length > 0) {
+                    for (let i = 0; i < otherRes.ids[0].length; i++) {
+                        const metadata = otherRes.metadatas[0][i];
+                        if (metadata.provider !== provider)
+                            continue;
+                        const distances = otherRes.distances;
+                        let distance = 1.0;
+                        if (distances && distances[0] && typeof distances[0][i] === 'number') {
+                            distance = distances[0][i];
+                        }
+                        const similarity = Math.max(0, 1 - distance);
+                        const HIVE_THRESHOLD = 0.90; // Higher threshold to avoid false positives across different domains
+                        if (similarity >= HIVE_THRESHOLD) {
+                            const originProject = colInfo.name.replace('lemma-cache-', '');
+                            console.log(`\n🎯 \x1b[32m[Hive Mind Telepathy]\x1b[0m Found a high-confidence solution (Similarity: ${(similarity * 100).toFixed(1)}%) in Project [\x1b[36m${originProject}\x1b[0m]! Reusing solution telepathically.\n`);
+                            return {
+                                data: JSON.parse(metadata.response),
+                                similarity,
+                                id: otherRes.ids[0][i],
+                                prompt: metadata.prompt,
+                                hiveMind: true,
+                                originProject
+                            };
+                        }
+                    }
+                }
+            }
+            catch { }
+        }
+    }
+    catch { }
     return null;
 }
-async function semanticSet(provider, prompt, data) {
+async function semanticSet(provider, prompt, data, projectName = 'global') {
     if (!await isPro())
         return;
     const emb = await getEmbedding(prompt);
@@ -650,7 +701,7 @@ async function semanticSet(provider, prompt, data) {
         return;
     try {
         const collection = await chroma.getOrCreateCollection({
-            name: 'lemma-cache',
+            name: `lemma-cache-${projectName}`,
             embeddingFunction: dummyEmbeddingFunction,
             metadata: { "hnsw:space": "cosine" }
         });
@@ -742,7 +793,7 @@ function setSseHeaders(res, fromCache, similarity, tier) {
     res.setHeader('Cache-Control', 'no-cache');
     res.setHeader('Connection', 'keep-alive');
     res.setHeader('X-Accel-Buffering', 'no');
-    res.setHeader('X-Lemma-Cache', fromCache ? 'HIT' : 'MISS');
+    res.setHeader('X-Lemma-Cache', typeof fromCache === 'string' ? fromCache : (fromCache ? 'HIT' : 'MISS'));
     res.setHeader('X-Lemma-Similarity', (similarity || 1.0).toFixed(3));
     res.setHeader('X-Lemma-Tier', tier);
 }
@@ -758,8 +809,8 @@ function buildAntChunk(text, stop) {
 function buildGemChunk(text, stop) {
     return { candidates: [{ content: { parts: [{ text }] }, finishReason: stop ? 'STOP' : undefined }] };
 }
-async function simulateHitStream(res, cachedData, provider, model, tier, similarity) {
-    setSseHeaders(res, true, similarity, tier);
+async function simulateHitStream(res, cachedData, provider, model, tier, similarity, cacheHeader = 'HIT') {
+    setSseHeaders(res, cacheHeader, similarity, tier);
     const id = `lemma-${Date.now()}`;
     let text = '';
     try {
@@ -876,6 +927,8 @@ class LemmaServer {
         this.stats = {};
         this.dashboardPath = '';
         this.sessions = new Map();
+        this.timeline = [];
+        this.lastFileContents = new Map();
         this.port = port;
         this.projectName = projectName || detectProject();
         this.projectDir = ensureProjectDir(this.projectName);
@@ -883,6 +936,7 @@ class LemmaServer {
         this.app = express();
         this.setupMiddleware();
         this.setupRoutes();
+        this.startWorkspaceTimeWatcher();
     }
     setupMiddleware() {
         this.app.use(express.json({ limit: '10mb' }));
@@ -1038,7 +1092,7 @@ class LemmaServer {
                 // Try to query ChromaDB for semantic cache count
                 try {
                     const collection = await chroma.getOrCreateCollection({
-                        name: 'lemma-cache',
+                        name: `lemma-cache-${this.projectName}`,
                         embeddingFunction: dummyEmbeddingFunction,
                         metadata: { "hnsw:space": "cosine" }
                     });
@@ -1095,7 +1149,7 @@ class LemmaServer {
                     if (!emb)
                         throw new Error('Failed to generate embedding');
                     const collection = await chroma.getOrCreateCollection({
-                        name: 'lemma-cache',
+                        name: `lemma-cache-${this.projectName}`,
                         embeddingFunction: dummyEmbeddingFunction,
                         metadata: { "hnsw:space": "cosine" }
                     });
@@ -1144,7 +1198,7 @@ class LemmaServer {
                     }
                     // Clear Chroma DB collection
                     try {
-                        await chroma.deleteCollection({ name: 'lemma-cache' }).catch(() => { });
+                        await chroma.deleteCollection({ name: `lemma-cache-${this.projectName}` }).catch(() => { });
                     }
                     catch (e) {
                         console.error(`[CacheClear] Failed to clear Chroma collection: ${e.message}`);
@@ -1161,7 +1215,7 @@ class LemmaServer {
                     // Selective delete from Chroma DB collection using provider metadata filter
                     try {
                         const collection = await chroma.getOrCreateCollection({
-                            name: 'lemma-cache',
+                            name: `lemma-cache-${this.projectName}`,
                             embeddingFunction: dummyEmbeddingFunction,
                             metadata: { "hnsw:space": "cosine" }
                         });
@@ -1177,6 +1231,117 @@ class LemmaServer {
             this.app.get('/api/savings-breakdown', (req, res) => {
                 res.json(savingsLedger.getSnapshot());
             });
+            // Serve codebase timeline snapshots (AST Time-Travel Telemetry)
+            this.app.get('/api/project/timeline', (req, res) => {
+                res.json({ success: true, timeline: this.timeline });
+            });
+            // Dynamic Zero-Shot Onboarding mental model builder
+            this.app.get('/api/project/onboarding', async (req, res) => {
+                try {
+                    const cwd = process.cwd();
+                    // 1. Parse package.json
+                    let pkg = {};
+                    const pkgPath = path.join(cwd, 'package.json');
+                    if (fs.existsSync(pkgPath)) {
+                        try {
+                            pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf8'));
+                        }
+                        catch { }
+                    }
+                    // 2. Identify stack & tools
+                    const techStack = [];
+                    const deps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) };
+                    if (deps['next'])
+                        techStack.push('Next.js (React Web App Framework)');
+                    else if (deps['react'])
+                        techStack.push('React.js (Frontend library)');
+                    if (deps['typescript'])
+                        techStack.push('TypeScript (Strict static type safety)');
+                    if (deps['prisma'])
+                        techStack.push('Prisma (Object-Relational Mapping)');
+                    if (deps['tailwindcss'])
+                        techStack.push('TailwindCSS (Utility-first styling)');
+                    if (deps['express'])
+                        techStack.push('Express (NodeJS REST API framework)');
+                    if (deps['jest'])
+                        techStack.push('Jest (Automated Unit Testing framework)');
+                    if (fs.existsSync(path.join(cwd, 'Dockerfile')) || fs.existsSync(path.join(cwd, 'docker-compose.yml'))) {
+                        techStack.push('Docker / Containerization');
+                    }
+                    if (techStack.length === 0) {
+                        techStack.push('Vanilla Node.js / Web stack');
+                    }
+                    // 3. Scan directory structure
+                    let files = [];
+                    try {
+                        files = fs.readdirSync(cwd).filter(f => !['node_modules', '.git', 'dist', 'chroma_data', '.lemma'].includes(f));
+                    }
+                    catch { }
+                    // 4. Query core architecture notes from ChromaDB
+                    let archMemoriesText = '';
+                    try {
+                        const emb = await getEmbedding("architectural design guidelines and core stack configuration decisions");
+                        if (emb) {
+                            const collection = await chroma.getOrCreateCollection({
+                                name: `lemma-cache-${this.projectName}`,
+                                embeddingFunction: dummyEmbeddingFunction,
+                                metadata: { "hnsw:space": "cosine" }
+                            });
+                            const searchRes = await collection.query({
+                                queryEmbeddings: [emb],
+                                nResults: 3
+                            });
+                            if (searchRes.ids[0] && searchRes.ids[0].length > 0) {
+                                archMemoriesText = '### 🧠 Stored Architectural Brain Memories\n';
+                                searchRes.ids[0].forEach((id, idx) => {
+                                    const meta = (searchRes.metadatas && searchRes.metadatas[0]) ? searchRes.metadatas[0][idx] : null;
+                                    if (meta && typeof meta.response === 'string') {
+                                        try {
+                                            const responseObj = JSON.parse(meta.response);
+                                            const contentText = responseObj.choices?.[0]?.message?.content || responseObj.content?.[0]?.text || '';
+                                            const promptText = typeof meta.prompt === 'string' ? meta.prompt : 'Architecture Note';
+                                            archMemoriesText += `#### 📌 Decisión: ${promptText}\n${contentText.substring(0, 500)}${contentText.length > 500 ? '...' : ''}\n\n`;
+                                        }
+                                        catch { }
+                                    }
+                                });
+                            }
+                        }
+                    }
+                    catch (e) {
+                        // Silently swallow vector db query errors
+                    }
+                    // 5. Construct gorgeous onboarding markdown guide
+                    const mentalModel = [
+                        `# 🧠 Codebase Onboarding & Mental Model: ${this.projectName}`,
+                        `This onboarding document was automatically generated by Lemma's Project Brain to align your context in a single shot.`,
+                        `\n## 🔌 Project Specifications`,
+                        `- **Name:** \`${pkg.name || this.projectName}\``,
+                        `- **Version:** \`${pkg.version || '0.1.0'}\``,
+                        `- **Description:** ${pkg.description || 'No description available in package.json.'}`,
+                        `\n## 🛠️ Detected Technical Stack`,
+                        techStack.map(s => `- **${s}**`).join('\n'),
+                        `\n## 📂 Directory Structure (Root)`,
+                        files.map(f => {
+                            const isDir = fs.statSync(path.join(cwd, f)).isDirectory();
+                            return `- \`${f}${isDir ? '/' : ''}\``;
+                        }).join('\n'),
+                        pkg.scripts && Object.keys(pkg.scripts).length > 0
+                            ? `\n## 📋 Standard Project Scripts\n${Object.entries(pkg.scripts).map(([name, cmd]) => `- \`npm run ${name}\`: \`${cmd}\``).join('\n')}`
+                            : '',
+                        `\n${archMemoriesText}`,
+                        `## 💡 AI Coding Guidelines for this Project`,
+                        `1. **Respect Core Stack:** Adhere strictly to the codebase structure and libraries listed above.`,
+                        `2. **Leverage the Brain:** Use Lemma MCP tools (\`search_memory\` and \`store_memory\`) to retrieve past solutions and register key architectural changes.`,
+                        `3. **Optimize Context:** Keep conversations focused. Lemma transparently compresses repetitive logs and file blocks to protect context window limits.`,
+                        `4. **Cortafuegos Local:** Do not worry about API key exposures. Lemma automatically masks local private variables and credentials.`
+                    ].filter(Boolean).join('\n');
+                    res.json({ success: true, markdown: mentalModel });
+                }
+                catch (err) {
+                    res.status(500).json({ success: false, error: err.message });
+                }
+            });
             // Explicit memory storage endpoint (store_memory)
             this.app.post('/api/memory/store', async (req, res) => {
                 const { query, response, provider = 'generic' } = req.body || {};
@@ -1189,7 +1354,7 @@ class LemmaServer {
                         return res.status(500).json({ error: 'Failed to generate embedding' });
                     }
                     const collection = await chroma.getOrCreateCollection({
-                        name: 'lemma-cache',
+                        name: `lemma-cache-${this.projectName}`,
                         embeddingFunction: dummyEmbeddingFunction,
                         metadata: { "hnsw:space": "cosine" }
                     });
@@ -1225,17 +1390,148 @@ class LemmaServer {
                 if (!source || !validSources.includes(source)) {
                     return res.status(400).json({ error: 'Invalid source' });
                 }
+                let tk = 0;
                 if (typeof tokens === 'number' && tokens > 0) {
+                    tk = tokens;
                     savingsLedger.recordTokens(source, tokens);
                 }
                 else if (typeof charsBefore === 'number' && typeof charsAfter === 'number') {
+                    tk = Math.floor(Math.max(0, charsBefore - charsAfter) / 4);
                     savingsLedger.record(source, charsBefore, charsAfter);
                 }
+                // Update active project stats to immediately reflect on Dashboard total requests & charts
+                if (tk > 0) {
+                    if (!this.stats[this.projectName]) {
+                        this.stats[this.projectName] = { total: 0, hits: 0, misses: 0, totalLatency: 0, totalTokensSaved: 0, providers: {} };
+                    }
+                    const s = this.stats[this.projectName];
+                    s.total++;
+                    s.totalTokensSaved += tk;
+                    s.totalLatency += 15; // mock latency for local MCP metrics
+                    if (source === 'cache') {
+                        s.hits++;
+                    }
+                    else {
+                        s.misses++;
+                    }
+                    if (!s.providers['mcp'])
+                        s.providers['mcp'] = { hits: 0, misses: 0 };
+                    source === 'cache' ? s.providers['mcp'].hits++ : s.providers['mcp'].misses++;
+                    // Write updated stats back to disk immediately
+                    writeJson(STATS_FILE, this.stats).catch(() => { });
+                    // Push to live telemetry logEvent array so the Dashboard event timeline updates instantly!
+                    logEvent({
+                        type: source === 'cache' ? 'cache:hit' : source,
+                        project: this.projectName,
+                        latency: 15,
+                        provider: 'mcp',
+                        tokens: tk
+                    });
+                }
                 res.json({ ok: true, snapshot: savingsLedger.getSnapshot().total });
             });
             this.app.get('/dashboard/*', (req, res) => res.sendFile(path.join(this.dashboardPath, 'index.html')));
         }
     }
+    startWorkspaceTimeWatcher() {
+        const cwd = process.cwd();
+        const { watch } = require('fs');
+        // Debounced capture function
+        let debounceTimer = null;
+        const pendingChanges = new Set();
+        const ignoreDirs = ['node_modules', '.git', 'dist', 'chroma_data', '.lemma', 'dashboard', 'bin', 'sdks'];
+        const watchDirRecursive = (dir) => {
+            try {
+                const watcher = watch(dir, (eventType, filename) => {
+                    if (!filename)
+                        return;
+                    const fullPath = path.join(dir, filename);
+                    // Heuristic to ignore generated/system folders
+                    const relPath = path.relative(cwd, fullPath);
+                    if (ignoreDirs.some(d => relPath.split(path.sep).includes(d)))
+                        return;
+                    // Ignore non-source files
+                    const ext = path.extname(filename);
+                    if (!['.ts', '.js', '.tsx', '.jsx', '.json', '.py', '.css'].includes(ext))
+                        return;
+                    pendingChanges.add(fullPath);
+                    if (debounceTimer)
+                        clearTimeout(debounceTimer);
+                    debounceTimer = setTimeout(() => this.captureSnapshot(pendingChanges), 1500);
+                });
+                // Prevent watcher from keeping process alive if in background
+                if (watcher.unref)
+                    watcher.unref();
+                // Recursively watch subfolders
+                const files = fs.readdirSync(dir, { withFileTypes: true });
+                for (const file of files) {
+                    if (file.isDirectory() && !ignoreDirs.includes(file.name)) {
+                        watchDirRecursive(path.join(dir, file.name));
+                    }
+                }
+            }
+            catch { }
+        };
+        watchDirRecursive(cwd); // Time travel watcher active!
+    }
+    async captureSnapshot(pendingChanges) {
+        const changes = Array.from(pendingChanges);
+        pendingChanges.clear();
+        const changedFiles = [];
+        for (const filePath of changes) {
+            if (!fs.existsSync(filePath))
+                continue;
+            try {
+                const content = fs.readFileSync(filePath, 'utf8');
+                const relPath = path.relative(process.cwd(), filePath);
+                const previousContent = this.lastFileContents.get(filePath) || '';
+                if (content === previousContent)
+                    continue;
+                let diff = '';
+                if (previousContent) {
+                    const { generateAstDiff } = require('../utils/ContextSqueezer');
+                    diff = generateAstDiff(previousContent, content);
+                }
+                this.lastFileContents.set(filePath, content);
+                changedFiles.push({
+                    filePath: relPath,
+                    content,
+                    diff
+                });
+            }
+            catch { }
+        }
+        if (changedFiles.length === 0)
+            return;
+        // Detect current compiler state/errors
+        let status = 'stable';
+        let errorLog = '';
+        // Check if there is a recent crash log in .lemma/live-context.md
+        try {
+            const liveContextPath = path.join(process.cwd(), '.lemma', 'live-context.md');
+            if (fs.existsSync(liveContextPath)) {
+                const content = fs.readFileSync(liveContextPath, 'utf8');
+                if (content.includes('crashed') || content.includes('error') || content.includes('Exception')) {
+                    status = 'failure';
+                    errorLog = content.substring(0, 1000);
+                }
+            }
+        }
+        catch { }
+        const snapshot = {
+            id: Math.random().toString(36).substring(2, 9),
+            timestamp: Date.now(),
+            changedFiles,
+            status,
+            errorLog
+        };
+        this.timeline.push(snapshot);
+        // Cap at 10 snapshots (FIFO eviction)
+        if (this.timeline.length > 10) {
+            this.timeline.shift();
+        }
+        console.log(`🌌 \x1b[35m[Multiverse]\x1b[0m Captured micro-snapshot ${snapshot.id} (${changedFiles.length} file edits mapped).`);
+    }
     async handleCompletion(req, res, provider) {
         const t0 = Date.now();
         const pro = await isPro();
@@ -1272,16 +1568,20 @@ class LemmaServer {
         }
         // Perform semantic cache check (resolves in ~15ms)
         if (pro) {
-            const semHit = await semanticGet(provider, prompt);
+            const semHit = await semanticGet(provider, prompt, this.projectName);
             if (semHit) {
                 if (semHit.similarity >= 0.90) {
                     await recordStat(this.stats, this.projectName, true, Date.now() - t0, provider, 2000);
                     const unmaskedData = semanticScrubber.unmask(semHit.data, tokenMap);
+                    const cacheHeader = semHit.hiveMind ? 'HIT-HIVE-MIND' : 'HIT';
                     if (isStream)
-                        return simulateHitStream(res, unmaskedData, provider, originalModel, tier, semHit.similarity);
-                    res.setHeader('X-Lemma-Cache', 'HIT');
+                        return simulateHitStream(res, unmaskedData, provider, originalModel, tier, semHit.similarity, cacheHeader);
+                    res.setHeader('X-Lemma-Cache', cacheHeader);
                     res.setHeader('X-Lemma-Similarity', semHit.similarity.toFixed(3));
                     res.setHeader('X-Lemma-Tier', tier);
+                    if (semHit.hiveMind && semHit.originProject) {
+                        res.setHeader('X-Lemma-Hive-Origin', semHit.originProject);
+                    }
                     return res.json(unmaskedData);
                 }
                 else if (semHit.similarity >= 0.82) { // raised from 0.70 to 0.82 to prevent false positives
@@ -1429,7 +1729,7 @@ Adjusted Answer:`;
             const data = await this.callUpstream(req.body, provider);
             cacheSet(provider, prompt, data);
             if (pro) {
-                await semanticSet(provider, prompt, data);
+                await semanticSet(provider, prompt, data, this.projectName);
                 await cloudSync.set(prompt, data);
             }
             if (!pro) {
@@ -1684,7 +1984,11 @@ program.command('start')
     if (opts.stack) {
         const { spawn } = require('child_process');
         console.log('🚀 Launching Lemma Full Stack...');
-        const chromaProcess = spawn('chroma', ['run', '--path', './chroma_data'], { stdio: 'ignore', detached: true });
+        const chromaDataPath = path.join(CACHE_DIR, 'chroma_data');
+        if (!fs.existsSync(chromaDataPath)) {
+            fs.mkdirSync(chromaDataPath, { recursive: true });
+        }
+        const chromaProcess = spawn('chroma', ['run', '--path', chromaDataPath], { stdio: 'ignore', detached: true });
         chromaProcess.unref();
         console.log('   ✅ ChromaDB requested');
         const dash = spawn('npm', ['run', 'dev', '--prefix', 'dashboard', '--', '--port', '8082'], { stdio: 'ignore', detached: true });