dev-mcp-server 0.0.2 → 1.0.0

package/src/context/compactor.js

@@ -0,0 +1,151 @@
+'use strict';
+/**
+ * Sliding-window context compaction with token-budget awareness.
+ *
+ * Strategy:
+ *   WINDOW: keep the last N messages verbatim (always fresh)
+ *   BODY:   older messages → compressed summary
+ *   INJECT: summary injected as a system-level recap at the top
+ */
+
+const llm = require('../utils/llmClient');
+const costTracker = require('../utils/costTracker');
+const logger = require('../utils/logger');
+
+// ── constants ──────────────────────────────────────────────────────────────────
+const EST_TOKENS = text => Math.ceil((text || '').length / 4);
+const WINDOW_MSGS = 6;    // keep this many recent messages verbatim
+const MAX_RAW_TOK = 8000; // if history exceeds this, compact
+const TARGET_TOK = 3000; // target token count for compacted history
+
+class Compactor {
+    /**
+     * Ensure a conversation history fits within the token budget.
+     * Returns { messages, compacted, summary, savedTokens }
+     */
+    async compact(messages, opts = {}) {
+        const { sessionId = 'default', force = false } = opts;
+
+        if (!messages?.length) return { messages: [], compacted: false };
+
+        // Estimate current token usage
+        const totalTokens = messages.reduce((sum, m) => sum + EST_TOKENS(
+            Array.isArray(m.content) ? JSON.stringify(m.content) : (m.content || '')
+        ), 0);
+
+        logger.info(`[Compactor] history: ${messages.length} messages, ~${totalTokens} tokens`);
+
+        if (!force && totalTokens <= MAX_RAW_TOK) {
+            return { messages, compacted: false, totalTokens };
+        }
+
+        // Split: window (keep verbatim) + body (compress)
+        const window = messages.slice(-WINDOW_MSGS);
+        const body = messages.slice(0, -WINDOW_MSGS);
+
+        if (!body.length) {
+            return { messages, compacted: false, reason: 'Too few messages to compact' };
+        }
+
+        // Importance-weighted body: prioritise tool results and assistant answers
+        const importantLines = body
+            .filter(m => m.role === 'assistant' || (m.role === 'user' && Array.isArray(m.content)))
+            .map(m => {
+                const text = Array.isArray(m.content)
+                    ? m.content.filter(b => b.type === 'text').map(b => b.text).join(' ')
+                    : (m.content || '');
+                return `${m.role.toUpperCase()}: ${text.slice(0, 400)}`;
+            })
+            .join('\n\n');
+
+        // Compress
+        const response = await llm.chat({
+            model: llm.model('fast'),
+            max_tokens: 600,
+            messages: [{
+                role: 'user',
+                content: `Compress this conversation history into a dense, information-preserving summary for a developer AI assistant.
+
+Capture:
+- Questions asked and answers given
+- File names, function names, error types discovered
+- Decisions made or actions taken
+- Any unresolved issues or follow-up questions
+- Tool results that revealed important facts
+
+Keep it under 400 words. Be specific. Preserve technical details exactly.
+
+History:
+${importantLines}`,
+            }],
+        });
+
+        costTracker.record({
+            model: llm.model('fast'),
+            inputTokens: response.usage.input_tokens,
+            outputTokens: response.usage.output_tokens,
+            sessionId,
+            queryType: 'compact',
+        });
+
+        const summary = response.content[0].text;
+
+        // Build compacted history: [summary-message] + [recent window]
+        const summaryMessage = {
+            role: 'user',
+            content: `[Conversation summary — ${body.length} messages compacted]\n${summary}`,
+            _compacted: true,
+            _originalCount: body.length,
+            _compactedAt: new Date().toISOString(),
+        };
+        // Pair with an assistant ack so the message array stays valid
+        const ackMessage = {
+            role: 'assistant',
+            content: 'Understood — I have the summary of our earlier conversation. Continuing from here.',
+        };
+
+        const compactedMessages = [summaryMessage, ackMessage, ...window];
+        const newTokens = compactedMessages.reduce((s, m) => s + EST_TOKENS(
+            Array.isArray(m.content) ? JSON.stringify(m.content) : (m.content || '')
+        ), 0);
+
+        logger.info(`[Compactor] ${messages.length}→${compactedMessages.length} messages, ${totalTokens}→${newTokens} tokens saved ${totalTokens - newTokens}`);
+
+        return {
+            compacted: true,
+            messages: compactedMessages,
+            summary,
+            originalCount: messages.length,
+            newCount: compactedMessages.length,
+            savedTokens: totalTokens - newTokens,
+            totalTokens: newTokens,
+        };
+    }
+
+    /**
+     * Multi-tier compaction: compact the summary if it's also too long.
+     */
+    async deepCompact(messages, opts = {}) {
+        let result = await this.compact(messages, opts);
+
+        // If still over budget, compact the summary itself
+        if (result.compacted && EST_TOKENS(result.summary) > TARGET_TOK) {
+            logger.info('[Compactor] Deep compaction: compressing the summary');
+            result = await this.compact(result.messages, { ...opts, force: true });
+        }
+
+        return result;
+    }
+
+    /**
+     * Estimate if a conversation needs compaction.
+     */
+    needsCompaction(messages) {
+        const tokens = messages?.reduce((s, m) => s + EST_TOKENS(
+            Array.isArray(m.content) ? JSON.stringify(m.content) : (m.content || '')
+        ), 0) || 0;
+        return { needs: tokens > MAX_RAW_TOK, tokens, threshold: MAX_RAW_TOK };
+    }
+}
+
+module.exports = new Compactor();

package/src/context/contextEngineer.js

@@ -0,0 +1,181 @@
+/**
+ * Intelligent context window management.
+ * Decides WHAT to include, HOW MUCH of it, in WHAT ORDER, and WHAT to cut.
+ *
+ * Core idea: context is a budget. Every token costs. Spend it wisely.
+ */
+
+const logger = require('../utils/logger');
+
+// Rough token estimator (4 chars ≈ 1 token)
+const estimateTokens = (text) => Math.ceil((text || '').length / 4);
+
+// How much of the context window to spend on retrieval context (the rest goes to answer)
+const CONTEXT_BUDGET_TOKENS = 6000;
+
+// Signal weights for scoring importance of a context chunk
+const SIGNAL_WEIGHTS = {
+    relevanceScore: 3.0,   // TF-IDF relevance from indexer
+    isErrorLog: 2.5,   // Error logs are gold for debugging
+    isBugFix: 2.0,   // Bug-fix files are highly relevant
+    isRecent: 1.5,   // Recently ingested files
+    hasExports: 1.2,   // Files that export things = likely central
+    hasErrors: 1.8,   // Files with known error types
+    isConfig: 0.6,   // Config is often less relevant
+    isTestFile: 0.5,   // Test files lower priority unless debug
+    chunkIsFirst: 1.3,   // First chunk of a file has more context
+    metadataRich: 1.2,   // Rich metadata = better indexed
+};
+
+class ContextEngineer {
+    /**
+     * Given raw retrieved docs + query intent, build the optimal context bundle.
+     * Returns ranked, trimmed, budget-aware context ready for the prompt.
+     */
+    engineer(docs, query, mode = 'general') {
+        if (!docs || docs.length === 0) return { chunks: [], budgetUsed: 0, dropped: 0 };
+
+        // 1. Score every chunk using multiple signals
+        const scored = docs.map(doc => ({
+            doc,
+            score: this._scoreChunk(doc, query, mode),
+        }));
+
+        // 2. Sort by composite score
+        scored.sort((a, b) => b.score - a.score);
+
+        // 3. Fill budget greedily, highest-score first
+        const selected = [];
+        let budgetUsed = 0;
+        let dropped = 0;
+
+        for (const { doc, score } of scored) {
+            const tokens = estimateTokens(doc.content);
+            if (budgetUsed + tokens <= CONTEXT_BUDGET_TOKENS) {
+                selected.push({ ...doc, engineeredScore: parseFloat(score.toFixed(3)) });
+                budgetUsed += tokens;
+            } else {
+                // Try to include a trimmed version if the chunk is large
+                const remaining = CONTEXT_BUDGET_TOKENS - budgetUsed;
+                if (remaining > 300) {
+                    const trimmedContent = this._trimToTokens(doc.content, remaining);
+                    selected.push({ ...doc, content: trimmedContent, engineeredScore: parseFloat(score.toFixed(3)), trimmed: true });
+                    budgetUsed += remaining;
+                } else {
+                    dropped++;
+                }
+            }
+        }
+
+        // 4. Re-order for readability: put config/schema first, then code, then logs
+        const ordered = this._readabilityOrder(selected, mode);
+
+        logger.info(`[ContextEngineer] ${docs.length} docs → ${selected.length} selected, ${dropped} dropped, ~${budgetUsed} tokens`);
+
+        return {
+            chunks: ordered,
+            budgetUsed,
+            dropped,
+            totalCandidates: docs.length,
+        };
+    }
+
+    /**
+     * Compute a composite importance score for a chunk
+     */
+    _scoreChunk(doc, query, mode) {
+        let score = (doc.relevanceScore || 0) * SIGNAL_WEIGHTS.relevanceScore;
+        const meta = doc.metadata || {};
+        const content = doc.content || '';
+
+        // Mode-specific boosts
+        if (mode === 'debug') {
+            if (doc.kind === 'log') score += SIGNAL_WEIGHTS.isErrorLog * 3;
+            if (meta.isBugFix) score += SIGNAL_WEIGHTS.isBugFix * 2;
+            if (meta.errors?.length > 0) score += SIGNAL_WEIGHTS.hasErrors;
+        }
+        if (mode === 'impact') {
+            if (meta.exports?.length > 0) score += SIGNAL_WEIGHTS.hasExports;
+            if (meta.imports?.length > 0) score += 0.5;
+        }
+        if (mode === 'usage') {
+            if (meta.functions?.length > 0) score += 0.8;
+        }
+
+        // Universal signals
+        if (meta.isBugFix) score += SIGNAL_WEIGHTS.isBugFix;
+        if (meta.errors?.length > 0) score += SIGNAL_WEIGHTS.hasErrors * 0.5;
+        if (doc.kind === 'config') score *= SIGNAL_WEIGHTS.isConfig;
+        if (doc.filename?.includes('test') || doc.filename?.includes('spec')) score *= SIGNAL_WEIGHTS.isTestFile;
+        if (doc.chunkIndex === 0) score *= SIGNAL_WEIGHTS.chunkIsFirst;
+
+        // Metadata richness bonus
+        const metaFields = ['functions', 'classes', 'imports', 'exports', 'errors', 'patterns'];
+        const richness = metaFields.filter(f => meta[f]?.length > 0).length;
+        if (richness >= 3) score *= SIGNAL_WEIGHTS.metadataRich;
+
+        // Recency bonus (ingested within last 24h)
+        if (doc.ingestedAt) {
+            const ageHours = (Date.now() - new Date(doc.ingestedAt).getTime()) / 3600000;
+            if (ageHours < 24) score *= SIGNAL_WEIGHTS.isRecent;
+        }
+
+        return score;
+    }
+
+    /**
+     * Re-order chunks for maximum readability in the prompt
+     */
+    _readabilityOrder(chunks, mode) {
+        const order = { schema: 0, config: 1, code: 2, documentation: 3, log: 4, script: 5, unknown: 6 };
+        if (mode === 'debug') {
+            // Logs first for debugging
+            order.log = 0; order.code = 1; order.documentation = 2;
+        }
+        return [...chunks].sort((a, b) => (order[a.kind] ?? 6) - (order[b.kind] ?? 6));
+    }
+
+    /**
+     * Trim text to approximately N tokens
+     */
+    _trimToTokens(text, maxTokens) {
+        const maxChars = maxTokens * 4;
+        if (text.length <= maxChars) return text;
+        return text.slice(0, maxChars) + '\n... [truncated for context budget]';
+    }
+
+    /**
+     * Compress a context bundle by summarizing less-important chunks
+     * Used when context must be even further reduced
+     */
+    summarizeChunk(doc) {
+        const meta = doc.metadata || {};
+        const lines = [
+            `[${doc.kind}] ${doc.filename}`,
+            meta.functions?.length ? `Functions: ${meta.functions.join(', ')}` : null,
+            meta.classes?.length ? `Classes: ${meta.classes.join(', ')}` : null,
+            meta.errors?.length ? `Known errors: ${meta.errors.join(', ')}` : null,
+            `Snippet: ${doc.content.slice(0, 200)}...`,
+        ].filter(Boolean);
+        return lines.join('\n');
+    }
+
+    /**
+     * Prioritize a list of plain text messages by estimated importance
+     * Used for memory/conversation injection
+     */
+    prioritizeMessages(messages, budgetTokens = 2000) {
+        let used = 0;
+        const result = [];
+        for (const msg of messages) {
+            const t = estimateTokens(msg.content || msg);
+            if (used + t <= budgetTokens) {
+                result.push(msg);
+                used += t;
+            }
+        }
+        return result;
+    }
+}
+
+module.exports = new ContextEngineer();

package/src/context/contextVisualizer.js

@@ -0,0 +1,140 @@
+'use strict';
+/**
+ * Shows developers exactly what's going into the LLM context window:
+ *  - Which chunks were retrieved and why (score breakdown)
+ *  - How much of the token budget each piece uses
+ *  - What memories are being injected
+ *  - What the final prompt looks like (sanitized)
+ *  - Token breakdown by section
+ */
+
+const indexer = require('../core/indexer');
+const contextEngineer = require('./contextEngineer');
+const { MemoryManager } = require('../memory/memoryManager');
+const teamMemory = require('../memory/teamMemory');
+
+const EST_TOKENS = text => Math.ceil((text || '').length / 4);
+
+class ContextVisualizer {
+    /**
+     * Visualize the full context that would be sent for a given query
+     */
+    visualize(query, opts = {}) {
+        const { mode = 'general', topK = 8, team, sessionId } = opts;
+
+        // 1. Retrieve docs
+        let docs;
+        switch (mode) {
+            case 'debug': docs = indexer.searchForErrors(query, topK); break;
+            case 'usage': docs = indexer.searchForUsages(query, topK); break;
+            case 'impact': docs = indexer.searchForImpact(query, topK); break;
+            default: docs = indexer.search(query, topK);
+        }
+
+        // 2. Engineer context
+        const engineered = contextEngineer.engineer(docs, query, mode);
+
+        // 3. Get memories
+        const memories = MemoryManager.getRelevant(query, 5);
+        const memContext = MemoryManager.formatAsContext(memories);
+        const teamContext = teamMemory.formatForAgent ? teamMemory.formatForAgent(team || 'global') : '';
+
+        // 4. Build token breakdown
+        const sections = {
+            system_prompt: {
+                label: 'System Prompt',
+                tokens: EST_TOKENS('You are an expert developer assistant...'),
+                content: '[system prompt — not shown for brevity]',
+            },
+            memory_context: {
+                label: 'Memory Context',
+                tokens: EST_TOKENS(memContext),
+                content: memContext || '(no relevant memories)',
+                count: memories.length,
+            },
+            team_memory: {
+                label: 'Team Memory',
+                tokens: EST_TOKENS(teamContext),
+                content: teamContext || '(no team memory)',
+            },
+            retrieval_context: {
+                label: 'Retrieved Codebase Context',
+                tokens: engineered.budgetUsed,
+                chunks: engineered.chunks.map(c => ({
+                    file: c.filename,
+                    kind: c.kind,
+                    chunkIndex: c.chunkIndex,
+                    relevanceScore: c.relevanceScore,
+                    engineeredScore: c.engineeredScore,
+                    tokens: EST_TOKENS(c.content),
+                    trimmed: c.trimmed || false,
+                    snippet: c.content.slice(0, 100) + (c.content.length > 100 ? '...' : ''),
+                    metadata: {
+                        functions: c.metadata?.functions?.slice(0, 3) || [],
+                        classes: c.metadata?.classes?.slice(0, 3) || [],
+                        isBugFix: c.metadata?.isBugFix || false,
+                    },
+                })),
+                dropped: engineered.dropped,
+            },
+            user_query: {
+                label: 'User Query',
+                tokens: EST_TOKENS(query),
+                content: query,
+            },
+        };
+
+        const totalTokens = Object.values(sections).reduce((s, sec) => s + (sec.tokens || 0), 0);
+
+        return {
+            query,
+            mode,
+            totalTokens,
+            budgetUtilization: `${((totalTokens / 8000) * 100).toFixed(1)}%`,
+            sections,
+            summary: {
+                chunksRetrieved: docs.length,
+                chunksUsed: engineered.chunks.length,
+                chunksDropped: engineered.dropped,
+                memoriesInjected: memories.length,
+                topSources: engineered.chunks.slice(0, 3).map(c => `${c.filename} (${c.relevanceScore})`),
+            },
+            warnings: [
+                totalTokens > 7000 ? '⚠ Context is near limit — consider using /compact' : null,
+                engineered.dropped > 2 ? `⚠ ${engineered.dropped} chunks dropped due to budget` : null,
+                memories.length === 0 ? 'ℹ No relevant memories found — knowledge grows with use' : null,
+            ].filter(Boolean),
+        };
+    }
+
+    /**
+     * Format context visualization as a human-readable string
+     */
+    format(viz) {
+        const lines = [
+            `╔══ Context Window Visualization ══╗`,
+            `  Query:  "${viz.query.slice(0, 60)}"`,
+            `  Mode:   ${viz.mode}`,
+            `  Tokens: ${viz.totalTokens} (~${viz.budgetUtilization} of budget)`,
+            ``,
+            `  Sources (${viz.summary.chunksUsed} used, ${viz.summary.chunksDropped} dropped):`,
+        ];
+
+        for (const chunk of viz.sections.retrieval_context.chunks) {
+            lines.push(`    [${chunk.kind}] ${chunk.file} — rel:${chunk.relevanceScore} eng:${chunk.engineeredScore} ${chunk.trimmed ? '(trimmed)' : ''}`);
+        }
+
+        if (viz.summary.memoriesInjected > 0) {
+            lines.push(``, `  Memories injected: ${viz.summary.memoriesInjected}`);
+        }
+
+        if (viz.warnings.length) {
+            lines.push(``, `  Warnings:`);
+            viz.warnings.forEach(w => lines.push(`    ${w}`));
+        }
+
+        return lines.join('\n');
+    }
+}
+
+module.exports = new ContextVisualizer();