dev-mcp-server 0.0.3 → 1.0.0

package/src/core/conversationEngine.js

@@ -0,0 +1,231 @@
+/**
+ * Multi-turn conversational Q&A — stateful, context-aware, memory-injecting.
+ *
+ * Unlike QueryEngine (single-shot), ConversationEngine maintains a rolling
+ * conversation window. Each turn injects:
+ *  - Prior conversation (compressed if too long)
+ *  - Fresh retrieved context
+ *  - Relevant memories
+ *  - Session-specific facts learned so far
+ */
+
+const llm = require('../utils/llmClient');
+const indexer = require('./indexer');
+const contextEngineer = require('../context/contextEngineer');
+const { MemoryManager } = require('../memory/memoryManager');
+const costTracker = require('../utils/costTracker');
+const plannerEngine = require('../planner/plannerEngine');
+const logger = require('../utils/logger');
+
+// Max messages to keep in raw history before compacting
+const MAX_HISTORY = 12;
+// Max tokens to spend on compressed history summary
+const HISTORY_BUDGET = 1000;
+
+class ConversationEngine {
+    constructor() {
+        this._conversations = new Map(); // convId → { messages, compactedSummary, sessionFacts }
+    }
+
+    /**
+     * Send a message in a conversation. Creates the conversation if it doesn't exist.
+     */
+    async chat(message, convId = 'default', opts = {}) {
+        const { sessionId = convId, topK = 6 } = opts;
+
+        if (!this._conversations.has(convId)) {
+            this._conversations.set(convId, {
+                messages: [],
+                compactedSummary: null,
+                sessionFacts: [],
+                turnCount: 0,
+            });
+        }
+
+        const conv = this._conversations.get(convId);
+        conv.turnCount++;
+
+        logger.info(`[Conv:${convId}] Turn ${conv.turnCount}: "${message.slice(0, 60)}"`);
+
+        // 1. Retrieve fresh context for this message
+        const docs = indexer.search(message, topK);
+        const engineered = contextEngineer.engineer(docs, message, 'general');
+
+        // 2. Retrieve relevant memories
+        const memories = MemoryManager.getRelevant(message, 4);
+        const memContext = MemoryManager.formatAsContext(memories);
+
+        // 3. Build system prompt with session knowledge
+        const systemPrompt = this._buildSystem(conv, memContext);
+
+        // 4. Build messages array (history + new message)
+        const historyMessages = this._buildHistory(conv);
+        const userContent = this._buildUserContent(message, engineered.chunks);
+
+        const allMessages = [
+            ...historyMessages,
+            { role: 'user', content: userContent },
+        ];
+
+        // 5. Call LLM
+        const response = await llm.chat({
+            model: llm.model('smart'),
+            max_tokens: 2000,
+            system: systemPrompt,
+            messages: allMessages,
+        });
+
+        const answer = response.content[0].text;
+
+        // 6. Track cost
+        const cost = costTracker.record({
+            model: llm.model('smart'),
+            inputTokens: response.usage.input_tokens,
+            outputTokens: response.usage.output_tokens,
+            sessionId,
+            queryType: 'conversation',
+        });
+
+        // 7. Update conversation history
+        conv.messages.push({ role: 'user', content: message, timestamp: new Date().toISOString() });
+        conv.messages.push({ role: 'assistant', content: answer, timestamp: new Date().toISOString(), sources: engineered.chunks.map(c => c.filename) });
+
+        // 8. Auto-compact if history is too long
+        if (conv.messages.length >= MAX_HISTORY) {
+            await this._compact(conv, convId);
+        }
+
+        // 9. Extract session-specific facts (quick, lightweight)
+        this._extractSessionFacts(conv, message, answer);
+
+        // 10. Background memory extraction
+        MemoryManager.extractFromExchange(message, answer, sessionId).catch(() => { });
+
+        return {
+            answer,
+            convId,
+            turn: conv.turnCount,
+            contextChunks: engineered.chunks.length,
+            memoriesUsed: memories.length,
+            isCompacted: !!conv.compactedSummary,
+            usage: {
+                inputTokens: response.usage.input_tokens,
+                outputTokens: response.usage.output_tokens,
+                costUsd: cost.costUsd,
+            },
+        };
+    }
+
+    /**
+     * Ask a follow-up question referencing previous context
+     */
+    async followUp(message, convId = 'default', opts = {}) {
+        const conv = this._conversations.get(convId);
+        if (!conv || conv.messages.length === 0) {
+            return this.chat(message, convId, opts);
+        }
+        // Add a hint that this is a follow-up
+        return this.chat(`[Follow-up on our conversation] ${message}`, convId, opts);
+    }
+
+    /**
+     * Get conversation history
+     */
+    getHistory(convId = 'default') {
+        const conv = this._conversations.get(convId);
+        if (!conv) return [];
+        return conv.messages;
+    }
+
+    /**
+     * Reset a conversation
+     */
+    reset(convId = 'default') {
+        this._conversations.delete(convId);
+        logger.info(`[Conv:${convId}] Reset`);
+    }
+
+    /**
+     * List active conversations
+     */
+    list() {
+        return [...this._conversations.entries()].map(([id, conv]) => ({
+            id,
+            turns: conv.turnCount,
+            messages: conv.messages.length,
+            isCompacted: !!conv.compactedSummary,
+            lastMessage: conv.messages.at(-1)?.timestamp,
+        }));
+    }
+
+    // ── Private helpers ──────────────────────────────────────────────────────────
+
+    _buildSystem(conv, memContext) {
+        const parts = [
+            `You are an expert developer assistant with deep knowledge of the codebase.
+Answer questions based on provided context. Be conversational but precise.
+If you refer to something from earlier in the conversation, say so explicitly.`,
+        ];
+
+        if (conv.compactedSummary) {
+            parts.push(`## Earlier Conversation Summary\n${conv.compactedSummary}`);
+        }
+
+        if (conv.sessionFacts.length > 0) {
+            parts.push(`## Facts established this session\n${conv.sessionFacts.map(f => `- ${f}`).join('\n')}`);
+        }
+
+        if (memContext) parts.push(memContext);
+
+        return parts.join('\n\n');
+    }
+
+    _buildHistory(conv) {
+        // Only pass the recent raw messages (before compaction point)
+        const recent = conv.messages.slice(conv.compactedSummary ? -6 : -10);
+        return recent.map(m => ({ role: m.role, content: m.content }));
+    }
+
+    _buildUserContent(message, contextChunks) {
+        if (contextChunks.length === 0) return message;
+        const ctxStr = contextChunks
+            .map((c, i) => `[${i + 1}] ${c.filename}:\n\`\`\`\n${c.content.slice(0, 600)}\n\`\`\``)
+            .join('\n\n');
+        return `${message}\n\n## Relevant codebase context:\n${ctxStr}`;
+    }
+
+    async _compact(conv, convId) {
+        logger.info(`[Conv:${convId}] Compacting ${conv.messages.length} messages`);
+        try {
+            const result = await plannerEngine.compact(conv.messages, convId);
+            if (result.compacted) {
+                conv.compactedSummary = result.summary;
+                // Keep only the last 4 messages raw
+                conv.messages = conv.messages.slice(-4);
+                logger.info(`[Conv:${convId}] Compacted to summary + 4 recent messages`);
+            }
+        } catch (err) {
+            logger.warn(`[Conv:${convId}] Compact failed: ${err.message}`);
+        }
+    }
+
+    _extractSessionFacts(conv, question, answer) {
+        // Quick heuristic extraction of facts to inject next turn
+        const factPatterns = [
+            /the (\w+) (?:function|method|class|module|file) (?:is|does|handles|returns) ([^.]{10,60})\./gi,
+            /(?:causes|caused by|because of) ([^.]{10,60})\./gi,
+        ];
+
+        for (const pattern of factPatterns) {
+            const matches = answer.matchAll(pattern);
+            for (const m of matches) {
+                const fact = m[0].replace(/["`]/g, '').trim().slice(0, 120);
+                if (!conv.sessionFacts.includes(fact) && conv.sessionFacts.length < 8) {
+                    conv.sessionFacts.push(fact);
+                }
+            }
+        }
+    }
+}
+
+module.exports = new ConversationEngine();

package/src/core/indexer.js

@@ -6,166 +6,192 @@ const tokenizer = new natural.WordTokenizer();
 const TfIdf = natural.TfIdf;
 
 class Indexer {
-    constructor() {
-        this.tfidf = new TfIdf();
-        this._docMap = [];
-        this._built = false;
+  constructor() {
+    this.tfidf = new TfIdf();
+    this._docMap = []; // maps tfidf index → store doc id
+    this._built = false;
+  }
+
+  /**
+   * (Re)build the TF-IDF index from the store
+   */
+  build() {
+    this.tfidf = new TfIdf();
+    this._docMap = [];
+
+    const docs = store.getAll();
+    for (const doc of docs) {
+      const text = this._docToText(doc);
+      this.tfidf.addDocument(text);
+      this._docMap.push(doc.id);
     }
 
-    build() {
-        this.tfidf = new TfIdf();
-        this._docMap = [];
-
-        const docs = store.getAll();
-        for (const doc of docs) {
-            const text = this._docToText(doc);
-            this.tfidf.addDocument(text);
-            this._docMap.push(doc.id);
-        }
-
-        this._built = true;
-        logger.info(`Index built: ${docs.length} documents`);
-        return docs.length;
+    this._built = true;
+    logger.info(`Index built: ${docs.length} documents`);
+    return docs.length;
+  }
+
+  /**
+   * Search for top-k relevant documents given a query
+   */
+  search(query, topK = 8, filter = {}) {
+    if (!this._built || this._docMap.length === 0) {
+      this.build();
     }
 
-    search(query, topK = 8, filter = {}) {
-        if (!this._built || this._docMap.length === 0) {
-            this.build();
-        }
-
-        const queryTokens = tokenizer.tokenize(query.toLowerCase());
-        const scores = new Array(this._docMap.length).fill(0);
+    const queryTokens = tokenizer.tokenize(query.toLowerCase());
+    const scores = new Array(this._docMap.length).fill(0);
 
-        for (const token of queryTokens) {
-            this.tfidf.tfidfs(token, (i, measure) => {
-                if (i < scores.length) {
-                    scores[i] += measure;
-                }
-            });
+    // Score each document using TF-IDF
+    for (const token of queryTokens) {
+      this.tfidf.tfidfs(token, (i, measure) => {
+        if (i < scores.length) {
+          scores[i] += measure;
         }
+      });
+    }
 
-        const allDocs = store.getAll();
-        scores.forEach((_, i) => {
-            const doc = allDocs[i];
-            if (!doc) return;
-            const textLower = doc.content.toLowerCase();
-            for (const token of queryTokens) {
-                if (token.length > 3 && textLower.includes(token)) {
-                    scores[i] += 0.5;
-                }
-            }
-
-            if (doc.metadata) {
-                const metaText = JSON.stringify(doc.metadata).toLowerCase();
-                for (const token of queryTokens) {
-                    if (token.length > 3 && metaText.includes(token)) {
-                        scores[i] += 1.0;
-                    }
-                }
-            }
-        });
-
-        let results = scores
-            .map((score, i) => ({ score, doc: allDocs[i] }))
-            .filter(r => r.doc && r.score > 0);
-
-        if (filter.kind) {
-            results = results.filter(r => r.doc.kind === filter.kind);
-        }
-        if (filter.filename) {
-            results = results.filter(r =>
-                r.doc.filename.toLowerCase().includes(filter.filename.toLowerCase())
-            );
+    // Also give a boost for keyword exact matches
+    const allDocs = store.getAll();
+    scores.forEach((_, i) => {
+      const doc = allDocs[i];
+      if (!doc) return;
+      const textLower = doc.content.toLowerCase();
+      for (const token of queryTokens) {
+        if (token.length > 3 && textLower.includes(token)) {
+          scores[i] += 0.5;
         }
-
-        results.sort((a, b) => b.score - a.score);
-
-        const seenFiles = new Map();
-        const deduped = [];
-        for (const r of results) {
-            const fp = r.doc.filePath;
-            if (!seenFiles.has(fp)) {
-                seenFiles.set(fp, r);
-                deduped.push(r);
-            } else if (r.score > seenFiles.get(fp).score) {
-                const idx = deduped.findIndex(d => d.doc.filePath === fp);
-                deduped[idx] = r;
-            }
-            if (deduped.length >= topK) break;
+      }
+      // Boost for metadata matches (function names, class names, etc.)
+      if (doc.metadata) {
+        const metaText = JSON.stringify(doc.metadata).toLowerCase();
+        for (const token of queryTokens) {
+          if (token.length > 3 && metaText.includes(token)) {
+            scores[i] += 1.0;
+          }
         }
+      }
+    });
 
-        return deduped.slice(0, topK).map(r => ({
-            ...r.doc,
-            relevanceScore: parseFloat(r.score.toFixed(4)),
-        }));
-    }
+    // Build scored results
+    let results = scores
+      .map((score, i) => ({ score, doc: allDocs[i] }))
+      .filter(r => r.doc && r.score > 0);
 
-    searchForErrors(errorType, topK = 6) {
-        const results = this.search(errorType, topK * 2);
-        return results
-            .map(doc => ({
-                ...doc,
-                relevanceScore: doc.kind === 'log'
-                    ? doc.relevanceScore * 1.5
-                    : doc.metadata?.isBugFix
-                        ? doc.relevanceScore * 1.3
-                        : doc.relevanceScore,
-            }))
-            .sort((a, b) => b.relevanceScore - a.relevanceScore)
-            .slice(0, topK);
+    // Apply filters
+    if (filter.kind) {
+      results = results.filter(r => r.doc.kind === filter.kind);
     }
-
-    searchForUsages(symbol, topK = 8) {
-        const query = `${symbol} usage import call reference`;
-        const results = this.search(query, topK * 2);
-
-        return results
-            .map(doc => {
-                let boost = 1;
-                if (doc.content.includes(symbol)) boost = 2;
-                if (doc.metadata?.imports?.some(i => i.includes(symbol))) boost = 2.5;
-                if (doc.metadata?.functions?.includes(symbol)) boost = 3;
-                return { ...doc, relevanceScore: doc.relevanceScore * boost };
-            })
-            .sort((a, b) => b.relevanceScore - a.relevanceScore)
-            .slice(0, topK);
+    if (filter.filename) {
+      results = results.filter(r =>
+        r.doc.filename.toLowerCase().includes(filter.filename.toLowerCase())
+      );
     }
 
-    searchForImpact(target, topK = 8) {
-        const query = `${target} depends import module connection`;
-        const results = this.search(query, topK * 2);
-
-        return results
-            .map(doc => {
-                let boost = 1;
-                if (doc.content.includes(target)) boost = 2;
-                if (doc.metadata?.imports?.some(i => i.includes(target))) boost = 3;
-                return { ...doc, relevanceScore: doc.relevanceScore * boost };
-            })
-            .sort((a, b) => b.relevanceScore - a.relevanceScore)
-            .slice(0, topK);
+    // Sort by score descending
+    results.sort((a, b) => b.score - a.score);
+
+    // Deduplicate by file (keep best chunk per file)
+    const seenFiles = new Map();
+    const deduped = [];
+    for (const r of results) {
+      const fp = r.doc.filePath;
+      if (!seenFiles.has(fp)) {
+        seenFiles.set(fp, r);
+        deduped.push(r);
+      } else if (r.score > seenFiles.get(fp).score) {
+        // Replace with better chunk
+        const idx = deduped.findIndex(d => d.doc.filePath === fp);
+        deduped[idx] = r;
+      }
+      if (deduped.length >= topK) break;
     }
 
-    _docToText(doc) {
-        const parts = [
-            doc.filename,
-            doc.kind,
-            doc.content,
-            doc.metadata?.functions?.join(' ') || '',
-            doc.metadata?.classes?.join(' ') || '',
-            doc.metadata?.imports?.join(' ') || '',
-            doc.metadata?.exports?.join(' ') || '',
-            doc.metadata?.errors?.join(' ') || '',
-            doc.metadata?.patterns?.join(' ') || '',
-            doc.metadata?.tables?.join(' ') || '',
-        ];
-        return parts.join(' ');
-    }
-
-    invalidate() {
-        this._built = false;
-    }
+    return deduped.slice(0, topK).map(r => ({
+      ...r.doc,
+      relevanceScore: parseFloat(r.score.toFixed(4)),
+    }));
+  }
+
+  /**
+   * Search specifically for error-related content
+   */
+  searchForErrors(errorType, topK = 6) {
+    const results = this.search(errorType, topK * 2);
+    // Boost log files and bug-fix files
+    return results
+      .map(doc => ({
+        ...doc,
+        relevanceScore: doc.kind === 'log'
+          ? doc.relevanceScore * 1.5
+          : doc.metadata?.isBugFix
+          ? doc.relevanceScore * 1.3
+          : doc.relevanceScore,
+      }))
+      .sort((a, b) => b.relevanceScore - a.relevanceScore)
+      .slice(0, topK);
+  }
+
+  /**
+   * Find all usages of a function/class/symbol
+   */
+  searchForUsages(symbol, topK = 8) {
+    const query = `${symbol} usage import call reference`;
+    const results = this.search(query, topK * 2);
+
+    // Extra boost for files that actually import or reference the symbol
+    return results
+      .map(doc => {
+        let boost = 1;
+        if (doc.content.includes(symbol)) boost = 2;
+        if (doc.metadata?.imports?.some(i => i.includes(symbol))) boost = 2.5;
+        if (doc.metadata?.functions?.includes(symbol)) boost = 3;
+        return { ...doc, relevanceScore: doc.relevanceScore * boost };
+      })
+      .sort((a, b) => b.relevanceScore - a.relevanceScore)
+      .slice(0, topK);
+  }
+
+  /**
+   * Find what a file/module connects to (impact analysis)
+   */
+  searchForImpact(target, topK = 8) {
+    const query = `${target} depends import module connection`;
+    const results = this.search(query, topK * 2);
+
+    return results
+      .map(doc => {
+        let boost = 1;
+        if (doc.content.includes(target)) boost = 2;
+        if (doc.metadata?.imports?.some(i => i.includes(target))) boost = 3;
+        return { ...doc, relevanceScore: doc.relevanceScore * boost };
+      })
+      .sort((a, b) => b.relevanceScore - a.relevanceScore)
+      .slice(0, topK);
+  }
+
+  _docToText(doc) {
+    const parts = [
+      doc.filename,
+      doc.kind,
+      doc.content,
+      // Flatten metadata into searchable text
+      doc.metadata?.functions?.join(' ') || '',
+      doc.metadata?.classes?.join(' ') || '',
+      doc.metadata?.imports?.join(' ') || '',
+      doc.metadata?.exports?.join(' ') || '',
+      doc.metadata?.errors?.join(' ') || '',
+      doc.metadata?.patterns?.join(' ') || '',
+      doc.metadata?.tables?.join(' ') || '',
+    ];
+    return parts.join(' ');
+  }
+
+  invalidate() {
+    this._built = false;
+  }
 }
 
+// Singleton
 const indexer = new Indexer();
 module.exports = indexer;