dev-mcp-server 0.0.2

package/src/core/ingester.js

@@ -0,0 +1,155 @@
+const path = require('path');
+const fs = require('fs');
+const { glob } = require('glob');
+const { parseFile, shouldSkip, FILE_TYPE_MAP } = require('../utils/fileParser');
+const store = require('../storage/store');
+const indexer = require('./indexer');
+const logger = require('../utils/logger');
+
+class Ingester {
+    async ingestFile(filePath) {
+        const absPath = path.resolve(filePath);
+
+        if (!fs.existsSync(absPath)) {
+            throw new Error(`File not found: ${absPath}`);
+        }
+
+        if (shouldSkip(absPath)) {
+            logger.warn(`Skipping: ${absPath}`);
+            return { skipped: true };
+        }
+
+        const ext = path.extname(absPath).toLowerCase();
+        if (!FILE_TYPE_MAP[ext]) {
+            logger.warn(`Unsupported file type: ${ext}`);
+            return { skipped: true, reason: 'unsupported type' };
+        }
+
+        const stat = fs.statSync(absPath);
+        if (stat.size > 500 * 1024) { // skip files > 500KB
+            logger.warn(`File too large (${(stat.size / 1024).toFixed(0)}KB): ${absPath}`);
+            return { skipped: true, reason: 'file too large' };
+        }
+
+        try {
+            store.removeByPath(absPath);
+
+            const chunks = parseFile(absPath);
+            if (chunks.length === 0) {
+                return { skipped: true, reason: 'empty file' };
+            }
+
+            const result = store.upsertDocs(chunks);
+
+            indexer.invalidate();
+
+            logger.info(`Ingested: ${path.basename(absPath)} (${chunks.length} chunk(s))`);
+            return { success: true, file: absPath, chunks: chunks.length, ...result };
+
+        } catch (err) {
+            logger.error(`Failed to ingest ${absPath}: ${err.message}`);
+            throw err;
+        }
+    }
+
+    async ingestDirectory(dirPath, options = {}) {
+        const absDir = path.resolve(dirPath);
+
+        if (!fs.existsSync(absDir)) {
+            throw new Error(`Directory not found: ${absDir}`);
+        }
+
+        const extensions = Object.keys(FILE_TYPE_MAP).map(e => e.slice(1));
+        const pattern = `**/*.{${extensions.join(',')}}`;
+
+        logger.info(`Scanning: ${absDir}`);
+        const files = await glob(pattern, {
+            cwd: absDir,
+            absolute: true,
+            ignore: [
+                '**/node_modules/**',
+                '**/.git/**',
+                '**/dist/**',
+                '**/build/**',
+                '**/coverage/**',
+                '**/*.min.js',
+                '**/package-lock.json',
+                '**/yarn.lock',
+            ],
+        });
+
+        logger.info(`Found ${files.length} files to process`);
+
+        const results = {
+            total: files.length,
+            ingested: 0,
+            skipped: 0,
+            failed: 0,
+            totalChunks: 0,
+            errors: [],
+        };
+
+        for (const file of files) {
+            try {
+                const result = await this.ingestFile(file);
+                if (result.skipped) {
+                    results.skipped++;
+                } else {
+                    results.ingested++;
+                    results.totalChunks += result.chunks || 0;
+                }
+            } catch (err) {
+                results.failed++;
+                results.errors.push({ file, error: err.message });
+            }
+
+            if ((results.ingested + results.skipped) % 50 === 0) {
+                logger.info(`Progress: ${results.ingested + results.skipped}/${files.length}`);
+            }
+        }
+
+        const docCount = indexer.build();
+        logger.info(`Index rebuilt with ${docCount} documents`);
+
+        return results;
+    }
+
+    async ingestRawText(content, options = {}) {
+        const {
+            kind = 'documentation',
+            label = 'manual-entry',
+            tags = [],
+        } = options;
+
+        if (!content || content.trim().length === 0) {
+            throw new Error('Content cannot be empty');
+        }
+
+        const id = `raw::${label}::${Date.now()}`;
+        const doc = {
+            id,
+            filePath: `raw://${label}`,
+            filename: label,
+            ext: '.txt',
+            kind,
+            chunkIndex: 0,
+            totalChunks: 1,
+            content: content.trim(),
+            lines: content.split('\n').length,
+            ingestedAt: new Date().toISOString(),
+            metadata: {
+                isRaw: true,
+                tags,
+                label,
+            },
+        };
+
+        store.upsertDocs([doc]);
+        indexer.invalidate();
+
+        logger.info(`Ingested raw text: "${label}" (${content.length} chars)`);
+        return { success: true, id, kind, label };
+    }
+}
+
+module.exports = new Ingester();

package/src/core/queryEngine.js

@@ -0,0 +1,236 @@
+'use strict';
+
+const llmClient = require('../utils/llmClient');
+const indexer = require('./indexer');
+const logger = require('../utils/logger');
+
+const QUERY_MODES = {
+    DEBUG: 'debug',       // "Why is this failing?"
+    USAGE: 'usage',       // "Where is this used?"
+    IMPACT: 'impact',     // "If I change this, what breaks?"
+    GENERAL: 'general',   // Open-ended question
+};
+
+function detectMode(question) {
+    const q = question.toLowerCase();
+
+    if (/why.*(fail|error|crash|break|throw|exception|not work)/i.test(q) ||
+        /what.*(error|exception|wrong|cause|happen)/i.test(q) ||
+        /debug|trace|stack|exception|ClassCastException|NullPointer|TypeError/i.test(q)) {
+        return QUERY_MODES.DEBUG;
+    }
+
+    if (/where.*(use|call|reference|import|depend)|who.*(use|call)|find.*(usage|reference)/i.test(q)) {
+        return QUERY_MODES.USAGE;
+    }
+
+    if (/if.*change|what.*break|impact|affect|depend|side effect|ripple|downstream/i.test(q)) {
+        return QUERY_MODES.IMPACT;
+    }
+
+    return QUERY_MODES.GENERAL;
+}
+
+function buildSystemPrompt(mode) {
+    const base = `You are an expert developer assistant with deep knowledge of the codebase.
+You answer questions ONLY based on the code context provided — never guess or hallucinate.
+If the context doesn't contain enough information, say so clearly.
+Be concise, direct, and developer-friendly. Use code examples from the context when relevant.`;
+
+    const modeInstructions = {
+        [QUERY_MODES.DEBUG]: `
+Your job: DIAGNOSE the root cause of bugs and errors.
+- Identify the exact type mismatch, null reference, or logic flaw
+- Trace the execution flow that leads to the error
+- Point to the specific file, function, and line range where the issue originates
+- Provide a concrete fix with code
+Format: Root Cause → Affected Flow → Fix`,
+
+        [QUERY_MODES.USAGE]: `
+Your job: FIND all usages of a function, class, module, or variable.
+- List every file where it is imported or referenced
+- Explain HOW it is used in each context (called directly, passed as callback, extended, etc.)
+- Note any patterns or inconsistencies in usage
+Format: Summary → File-by-file breakdown`,
+
+        [QUERY_MODES.IMPACT]: `
+Your job: ANALYSE what would break or change if the target is modified.
+- List all files/modules that directly depend on it
+- Identify indirect dependencies (things that use things that use it)
+- Flag any risky or tightly-coupled areas
+- Suggest safe modification strategies
+Format: Direct Impact → Indirect Impact → Risk Level → Safe change strategy`,
+
+        [QUERY_MODES.GENERAL]: `
+Your job: Answer the developer's question using the codebase context.
+- Be specific and cite the relevant files/functions
+- If the answer spans multiple files, connect the dots
+- If something is unclear in the code, flag it`,
+    };
+
+    return base + (modeInstructions[mode] || modeInstructions[QUERY_MODES.GENERAL]);
+}
+
+function formatContext(docs) {
+    if (!docs || docs.length === 0) {
+        return 'No relevant context found in the codebase index.';
+    }
+
+    return docs
+        .map((doc, i) => {
+            const meta = doc.metadata || {};
+            const metaSummary = [
+                meta.functions?.length ? `Functions: ${meta.functions.slice(0, 5).join(', ')}` : null,
+                meta.classes?.length ? `Classes: ${meta.classes.join(', ')}` : null,
+                meta.imports?.length ? `Imports: ${meta.imports.slice(0, 3).join(', ')}` : null,
+                meta.errors?.length ? `Errors: ${meta.errors.slice(0, 3).join(', ')}` : null,
+                meta.patterns?.length ? `Patterns: ${meta.patterns.join(', ')}` : null,
+            ]
+                .filter(Boolean)
+                .join(' | ');
+
+            return `--- [${i + 1}] ${doc.filename} (${doc.kind}) | Score: ${doc.relevanceScore} ---
+Path: ${doc.filePath}
+${metaSummary ? `Meta: ${metaSummary}` : ''}
+\`\`\`
+${doc.content}
+\`\`\``;
+        })
+        .join('\n\n');
+}
+
+class QueryEngine {
+    async query(question, options = {}) {
+        const {
+            mode: forcedMode,
+            topK = 8,
+            stream = false,
+            filter = {},
+        } = options;
+
+        if (!question || question.trim().length === 0) {
+            throw new Error('Question cannot be empty');
+        }
+
+        const mode = forcedMode || detectMode(question);
+        logger.info(`Query mode: ${mode} | Q: "${question.slice(0, 80)}..."`);
+
+        let docs;
+        switch (mode) {
+            case QUERY_MODES.DEBUG:
+                docs = indexer.searchForErrors(question, topK);
+                break;
+            case QUERY_MODES.USAGE: {
+                const usageMatch = question.match(/(?:where|how|who).*?(?:is|are|does)?\s+[`"']?(\w+)[`"']?\s+(?:used|called|import|reference)/i);
+                const symbol = usageMatch?.[1] || question;
+                docs = indexer.searchForUsages(symbol, topK);
+                break;
+            }
+            case QUERY_MODES.IMPACT: {
+                const impactMatch = question.match(/(?:change|modify|update|refactor)\s+[`"']?(\w+)[`"']?/i);
+                const target = impactMatch?.[1] || question;
+                docs = indexer.searchForImpact(target, topK);
+                break;
+            }
+            default:
+                docs = indexer.search(question, topK, filter);
+        }
+
+        const contextText = formatContext(docs);
+        const systemPrompt = buildSystemPrompt(mode);
+
+        const userMessage = `## Developer Question
+${question}
+
+## Codebase Context (retrieved from your system)
+${contextText}
+
+## Answer`;
+
+        logger.info(`Sending to ${llmClient.label()}: ${docs.length} context chunks`);
+
+        if (stream) {
+            return this._streamQuery(systemPrompt, userMessage, docs, mode);
+        }
+
+        const response = await llmClient.createMessage({
+            maxTokens: 2000,
+            system: systemPrompt,
+            messages: [{ role: 'user', content: userMessage }],
+        });
+
+        const answer = response.content[0].text;
+
+        return {
+            answer,
+            mode,
+            provider: llmClient.label(),
+            sources: docs.map(d => ({
+                file: d.filename,
+                path: d.filePath,
+                kind: d.kind,
+                relevanceScore: d.relevanceScore,
+                functions: d.metadata?.functions?.slice(0, 5) || [],
+            })),
+            usage: {
+                inputTokens: response.usage.input_tokens,
+                outputTokens: response.usage.output_tokens,
+            },
+        };
+    }
+
+    async *_streamQuery(systemPrompt, userMessage, docs, mode) {
+        const stream = await llmClient.createMessage({
+            maxTokens: 2000,
+            system: systemPrompt,
+            messages: [{ role: 'user', content: userMessage }],
+            stream: true,
+        });
+
+        yield {
+            type: 'metadata',
+            mode,
+            provider: llmClient.label(),
+            sources: docs.map(d => ({
+                file: d.filename,
+                path: d.filePath,
+                kind: d.kind,
+                relevanceScore: d.relevanceScore,
+            })),
+        };
+
+        for await (const event of stream) {
+            if (event.type === 'content_block_delta' && event.delta.type === 'text_delta') {
+                yield { type: 'text', text: event.delta.text };
+            }
+            if (event.type === 'message_stop') {
+                yield { type: 'done' };
+            }
+        }
+    }
+
+    async debugError(errorMessage, stackTrace = '', options = {}) {
+        const question = `Why is this error happening and how do I fix it?
+Error: ${errorMessage}
+${stackTrace ? `Stack trace:\n${stackTrace}` : ''}`;
+
+        return this.query(question, { ...options, mode: QUERY_MODES.DEBUG });
+    }
+
+    async findUsages(symbol, options = {}) {
+        return this.query(`Where is ${symbol} used across the codebase?`, {
+            ...options,
+            mode: QUERY_MODES.USAGE,
+        });
+    }
+
+    async analyzeImpact(target, changeDescription = '', options = {}) {
+        const question = changeDescription
+            ? `If I change ${target} to ${changeDescription}, what would break or be affected?`
+            : `If I change or remove ${target}, what would break?`;
+
+        return this.query(question, { ...options, mode: QUERY_MODES.IMPACT });
+    }
+}
+
+module.exports = { QueryEngine: new QueryEngine(), QUERY_MODES, detectMode };

package/src/storage/store.js

@@ -0,0 +1,125 @@
+const fs = require('fs');
+const path = require('path');
+const logger = require('../utils/logger');
+
+const DATA_DIR = path.join(process.cwd(), 'data');
+const INDEX_FILE = path.join(DATA_DIR, 'index.json');
+const META_FILE = path.join(DATA_DIR, 'meta.json');
+
+class Store {
+    constructor() {
+        this._ensureDataDir();
+        this.index = this._load(INDEX_FILE, []);
+        this.meta = this._load(META_FILE, {
+            totalDocs: 0,
+            totalFiles: 0,
+            lastIngested: null,
+            fileTypes: {},
+            tags: [],
+        });
+    }
+
+    _ensureDataDir() {
+        if (!fs.existsSync(DATA_DIR)) {
+            fs.mkdirSync(DATA_DIR, { recursive: true });
+        }
+        const logsDir = path.join(process.cwd(), 'logs');
+        if (!fs.existsSync(logsDir)) {
+            fs.mkdirSync(logsDir, { recursive: true });
+        }
+    }
+
+    _load(file, defaultVal) {
+        try {
+            if (fs.existsSync(file)) {
+                return JSON.parse(fs.readFileSync(file, 'utf-8'));
+            }
+        } catch (e) {
+            logger.warn(`Could not load ${file}: ${e.message}`);
+        }
+        return defaultVal;
+    }
+
+    _save() {
+        fs.writeFileSync(INDEX_FILE, JSON.stringify(this.index, null, 2));
+        fs.writeFileSync(META_FILE, JSON.stringify(this.meta, null, 2));
+    }
+
+    upsertDocs(docs) {
+        let added = 0;
+        let updated = 0;
+
+        for (const doc of docs) {
+            const existingIdx = this.index.findIndex(d => d.id === doc.id);
+            if (existingIdx >= 0) {
+                this.index[existingIdx] = doc;
+                updated++;
+            } else {
+                this.index.push(doc);
+                added++;
+            }
+        }
+
+        this._rebuildMeta();
+        this._save();
+        return { added, updated };
+    }
+
+    removeByPath(filePath) {
+        const before = this.index.length;
+        this.index = this.index.filter(d => d.filePath !== filePath);
+        const removed = before - this.index.length;
+        this._rebuildMeta();
+        this._save();
+        return removed;
+    }
+
+    getAll() {
+        return this.index;
+    }
+
+    getByKind(kind) {
+        return this.index.filter(d => d.kind === kind);
+    }
+
+    getIngestedFiles() {
+        return [...new Set(this.index.map(d => d.filePath))];
+    }
+
+    clear() {
+        this.index = [];
+        this.meta = {
+            totalDocs: 0,
+            totalFiles: 0,
+            lastIngested: null,
+            fileTypes: {},
+            tags: [],
+        };
+        this._save();
+    }
+
+    getStats() {
+        return {
+            ...this.meta,
+            indexSize: this.index.length,
+        };
+    }
+
+    _rebuildMeta() {
+        const files = new Set(this.index.map(d => d.filePath));
+        const fileTypes = {};
+        for (const doc of this.index) {
+            fileTypes[doc.kind] = (fileTypes[doc.kind] || 0) + 1;
+        }
+
+        this.meta = {
+            totalDocs: this.index.length,
+            totalFiles: files.size,
+            lastIngested: new Date().toISOString(),
+            fileTypes,
+        };
+    }
+}
+
+const store = new Store();
+module.exports = store;

package/src/utils/fileParser.js

@@ -0,0 +1,183 @@
+const fs = require('fs');
+const path = require('path');
+
+const FILE_TYPE_MAP = {
+    '.js': 'code',
+    '.ts': 'code',
+    '.jsx': 'code',
+    '.tsx': 'code',
+    '.mjs': 'code',
+    '.cjs': 'code',
+    '.py': 'code',
+    '.java': 'code',
+    '.go': 'code',
+    '.rb': 'code',
+    '.php': 'code',
+    '.cs': 'code',
+    '.cpp': 'code',
+    '.c': 'code',
+    '.rs': 'code',
+    '.json': 'config',
+    '.yaml': 'config',
+    '.yml': 'config',
+    '.env': 'config',
+    '.toml': 'config',
+    '.xml': 'config',
+    '.md': 'documentation',
+    '.txt': 'documentation',
+    '.log': 'log',
+    '.sql': 'schema',
+    '.graphql': 'schema',
+    '.gql': 'schema',
+    '.sh': 'script',
+    '.bash': 'script',
+};
+
+const CHUNK_SIZE = 1500;   // characters per chunk
+const CHUNK_OVERLAP = 200; // overlap between chunks
+
+function parseFile(filePath) {
+    const ext = path.extname(filePath).toLowerCase();
+    const kind = FILE_TYPE_MAP[ext] || 'unknown';
+    const filename = path.basename(filePath);
+    const content = fs.readFileSync(filePath, 'utf-8');
+
+    if (!content.trim()) return [];
+
+    const chunks = chunkContent(content, filePath);
+
+    return chunks.map((chunk, idx) => ({
+        id: `${filePath}::chunk${idx}`,
+        filePath,
+        filename,
+        ext,
+        kind,
+        chunkIndex: idx,
+        totalChunks: chunks.length,
+        content: chunk,
+        lines: countLines(chunk),
+        ingestedAt: new Date().toISOString(),
+        metadata: extractMetadata(chunk, kind, filename),
+    }));
+}
+
+function chunkContent(content, filePath) {
+    const lines = content.split('\n');
+
+    if (content.length <= CHUNK_SIZE) return [content];
+
+    const chunks = [];
+    let current = [];
+    let currentLen = 0;
+
+    for (const line of lines) {
+        current.push(line);
+        currentLen += line.length + 1;
+
+        if (currentLen >= CHUNK_SIZE) {
+            chunks.push(current.join('\n'));
+            const overlapLines = [];
+            let overlapLen = 0;
+            for (let i = current.length - 1; i >= 0 && overlapLen < CHUNK_OVERLAP; i--) {
+                overlapLines.unshift(current[i]);
+                overlapLen += current[i].length + 1;
+            }
+            current = overlapLines;
+            currentLen = overlapLen;
+        }
+    }
+
+    if (current.length > 0) {
+        chunks.push(current.join('\n'));
+    }
+
+    return chunks;
+}
+
+function extractMetadata(content, kind, filename) {
+    const meta = {};
+
+    if (kind === 'code') {
+        const functions = [];
+        const classes = [];
+        const imports = [];
+        const exports = [];
+
+        const fnMatches = content.matchAll(/(?:function\s+(\w+)|const\s+(\w+)\s*=\s*(?:async\s*)?\(|(\w+)\s*\([^)]*\)\s*{)/g);
+        for (const m of fnMatches) {
+            const name = m[1] || m[2] || m[3];
+            if (name && !['if', 'for', 'while', 'switch', 'catch'].includes(name)) {
+                functions.push(name);
+            }
+        }
+
+        const classMatches = content.matchAll(/class\s+(\w+)/g);
+        for (const m of classMatches) classes.push(m[1]);
+
+        const importMatches = content.matchAll(/(?:import|require)\s*(?:\{[^}]+\}|[\w*]+)?\s*(?:from)?\s*['"]([^'"]+)['"]/g);
+        for (const m of importMatches) imports.push(m[1]);
+
+        const exportMatches = content.matchAll(/export\s+(?:default\s+)?(?:function|class|const|let|var)?\s*(\w+)/g);
+        for (const m of exportMatches) exports.push(m[1]);
+
+        const errors = [];
+        const errorMatches = content.matchAll(/(?:catch|throw|Error|Exception)\s*[(\s]*([A-Z]\w+(?:Error|Exception))/g);
+        for (const m of errorMatches) errors.push(m[1]);
+
+        meta.functions = [...new Set(functions)].slice(0, 20);
+        meta.classes = [...new Set(classes)].slice(0, 10);
+        meta.imports = [...new Set(imports)].slice(0, 20);
+        meta.exports = [...new Set(exports)].slice(0, 10);
+        meta.errors = [...new Set(errors)].slice(0, 10);
+
+        const apiPatterns = [];
+        if (/express|router\.(get|post|put|delete|patch)/i.test(content)) apiPatterns.push('REST API');
+        if (/graphql|resolver|schema/i.test(content)) apiPatterns.push('GraphQL');
+        if (/mongoose|sequelize|prisma|typeorm/i.test(content)) apiPatterns.push('ORM');
+        if (/redis|ioredis|bull/i.test(content)) apiPatterns.push('Cache/Queue');
+        if (/jwt|passport|bcrypt/i.test(content)) apiPatterns.push('Auth');
+        meta.patterns = apiPatterns;
+
+    } else if (kind === 'log') {
+        const errorLines = content
+            .split('\n')
+            .filter(l => /error|exception|warn|fail|crash/i.test(l))
+            .slice(0, 10);
+        meta.errors = errorLines;
+
+        const hasTimestamps = /\d{4}-\d{2}-\d{2}|\d{2}\/\d{2}\/\d{4}/.test(content);
+        meta.hasTimestamps = hasTimestamps;
+
+    } else if (kind === 'schema') {
+        const tables = [];
+        const tableMatches = content.matchAll(/(?:CREATE\s+TABLE|model\s+|type\s+)["'`]?(\w+)["'`]?/gi);
+        for (const m of tableMatches) tables.push(m[1]);
+        meta.tables = [...new Set(tables)].slice(0, 20);
+    }
+
+    meta.isBugFix = /fix|bug|patch|resolve|hotfix|issue/i.test(filename) ||
+        /TODO|FIXME|HACK|BUG|XXX/.test(content);
+
+    return meta;
+}
+
+function countLines(text) {
+    return text.split('\n').length;
+}
+
+function shouldSkip(filePath) {
+    const skipPatterns = [
+        /node_modules/,
+        /\.git\//,
+        /dist\//,
+        /build\//,
+        /coverage\//,
+        /\.min\.(js|css)$/,
+        /package-lock\.json$/,
+        /yarn\.lock$/,
+        /\.map$/,
+    ];
+    return skipPatterns.some(p => p.test(filePath));
+}
+
+module.exports = { parseFile, shouldSkip, FILE_TYPE_MAP };