codesearch-mcp 0.0.9-alpha → 0.0.11-alpha

package/dist/src/services/IndexerService.d.ts

@@ -0,0 +1,39 @@
+import type { Config } from '../config';
+import type { IEmbedder } from '../interfaces/IEmbedder';
+import type { IAstParser } from '../interfaces/IAstParser';
+import type { DatabaseService } from './DatabaseService';
+import { LoggerService } from './LoggerService';
+export interface IndexStats {
+    serviceName: string;
+    indexed: number;
+    skipped: number;
+    failed: number;
+    chunks: number;
+    elapsedSec: number;
+    warnings: string[];
+}
+export declare class IndexerService {
+    private db;
+    private embedder;
+    private ast;
+    private config;
+    private logger;
+    private readonly jsSplitter;
+    private readonly schemaSplitter;
+    constructor(db: DatabaseService, embedder: IEmbedder, ast: IAstParser, config: Config, logger?: LoggerService);
+    backfillSymbols(signal?: AbortSignal): Promise<{
+        total: number;
+        processed: number;
+        skipped: number;
+        failed: number;
+    }>;
+    discoverProjects(rootDir: string): {
+        name: string;
+        path: string;
+    }[];
+    private writeDepEdges;
+    scanAndIndex(dir: string, serviceName: string, force?: boolean, signal?: AbortSignal, preCollectedFiles?: string[]): Promise<IndexStats>;
+    private getSplitter;
+    private embedText;
+    private ingestFile;
+}

package/dist/src/services/IndexerService.js

@@ -0,0 +1,286 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.IndexerService = void 0;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const textsplitters_1 = require("@langchain/textsplitters");
+const cli_progress_1 = require("cli-progress");
+const db_1 = require("../db");
+const file_utils_1 = require("../file-utils");
+const LoggerService_1 = require("./LoggerService");
+const SCHEMA_EXTENSIONS = /\.(prisma|graphql|gql)$/;
+class IndexerService {
+    db;
+    embedder;
+    ast;
+    config;
+    logger;
+    jsSplitter;
+    schemaSplitter;
+    constructor(db, embedder, ast, config, logger = LoggerService_1.LoggerService.silent()) {
+        this.db = db;
+        this.embedder = embedder;
+        this.ast = ast;
+        this.config = config;
+        this.logger = logger;
+        this.jsSplitter = textsplitters_1.RecursiveCharacterTextSplitter.fromLanguage('js', {
+            chunkSize: config.indexer.chunkSize,
+            chunkOverlap: config.indexer.chunkOverlap,
+        });
+        this.schemaSplitter = new textsplitters_1.RecursiveCharacterTextSplitter({
+            chunkSize: config.indexer.chunkSize,
+            chunkOverlap: config.indexer.chunkOverlap,
+            separators: ['\n\n', '\n', ' '],
+        });
+    }
+    async backfillSymbols(signal) {
+        const rows = await this.db.query(`SELECT DISTINCT "serviceName", "filePath" FROM code_chunk ORDER BY "serviceName", "filePath"`);
+        let processed = 0, skipped = 0, failed = 0;
+        for (const { serviceName, filePath } of rows) {
+            if (signal?.aborted)
+                break;
+            const absPath = path.join(this.config.workspaceRoot, filePath);
+            const projectRoot = path.join(this.config.workspaceRoot, serviceName);
+            if (!fs.existsSync(absPath)) {
+                skipped++;
+                continue;
+            }
+            try {
+                const { symbols: rawSymbols } = await this.ast.analyzeFile(absPath, serviceName, projectRoot);
+                const symbols = rawSymbols.map(s => ({ ...s, filePath }));
+                await this.db.transaction(async (manager) => {
+                    await manager.delete(db_1.CodeSymbol, { project: serviceName, filePath });
+                    if (symbols.length > 0) {
+                        await manager.createQueryBuilder().insert().into(db_1.CodeSymbol).values(symbols).execute();
+                    }
+                });
+                processed++;
+            }
+            catch (err) {
+                const reason = err.message;
+                console.error(`  Failed: ${filePath} — ${reason}`);
+                this.logger.error('IndexerService: symbol backfill failed', { file: filePath, reason });
+                failed++;
+            }
+            const count = processed + failed;
+            if (count > 0 && count % 50 === 0) {
+                const msg = `  ${count}/${rows.length} files...`;
+                process.stdout.isTTY ? process.stdout.write(msg + '\r') : console.log(msg);
+            }
+        }
+        return { total: rows.length, processed, skipped, failed };
+    }
+    discoverProjects(rootDir) {
+        return fs.readdirSync(rootDir, { withFileTypes: true })
+            .filter(d => d.isDirectory() && !file_utils_1.IGNORE_DIRS.has(d.name))
+            .map(d => ({ name: d.name, path: path.join(rootDir, d.name) }))
+            .filter(p => fs.existsSync(path.join(p.path, 'package.json')));
+    }
+    async writeDepEdges(serviceName, relFilePath, edges, manager) {
+        const run = async (mgr) => {
+            await mgr.delete(db_1.CodeDependency, { project: serviceName, fromFile: relFilePath });
+            if (edges.length > 0) {
+                await mgr.createQueryBuilder().insert().into(db_1.CodeDependency).values(edges).execute();
+            }
+        };
+        if (manager)
+            await run(manager);
+        else
+            await this.db.transaction(run);
+    }
+    async scanAndIndex(dir, serviceName, force = false, signal, preCollectedFiles) {
+        const startTime = Date.now();
+        const repo = this.db.getRepository(db_1.CodeChunk);
+        if (force) {
+            await this.db.query(`UPDATE code_chunk SET "lastModified" = NULL WHERE "serviceName" = $1`, [serviceName]);
+        }
+        const files = preCollectedFiles ?? (0, file_utils_1.collectFiles)(dir, (0, file_utils_1.loadIgnore)(dir), dir);
+        const collectedRelPaths = new Set(files.map(f => path.relative(this.config.workspaceRoot, f)));
+        const dbPaths = await this.db.query(`SELECT DISTINCT "filePath" FROM code_chunk WHERE "serviceName" = $1`, [serviceName]);
+        for (const { filePath } of dbPaths) {
+            if (!collectedRelPaths.has(filePath)) {
+                await repo.delete({ serviceName, filePath });
+                await this.db.query(`DELETE FROM code_dependency WHERE project = $1 AND "fromFile" = $2`, [serviceName, filePath]);
+                await this.db.query(`DELETE FROM code_dependency WHERE project = $1 AND "toFile" = $2`, [serviceName, filePath]);
+                console.log(`  Removed stale: ${path.basename(filePath)}`);
+            }
+        }
+        const bar = new cli_progress_1.SingleBar({
+            format: `{serviceName} {bar} {percentage}% | {value}/{total} | {duration_formatted} elapsed | ~{eta_formatted} left | {filename}`,
+            barCompleteChar: '█',
+            barIncompleteChar: '░',
+            hideCursor: true,
+            clearOnComplete: false,
+        }, cli_progress_1.Presets.shades_grey);
+        const warnings = [];
+        console.log(`\nIndexing ${serviceName}: ${files.length} files`);
+        const meta = (0, file_utils_1.readRagMeta)(dir);
+        if (meta === 'invalid') {
+            const msg = `.ragmeta.json found but invalid JSON`;
+            console.log(`  ⚠ ${msg}`);
+            warnings.push(msg);
+        }
+        else if (meta !== 'missing') {
+            const parts = [meta.description, meta.stack && `[${meta.stack}]`].filter(Boolean).join(' ');
+            console.log(`  ✓ .ragmeta.json: ${parts || '(no fields)'}`);
+        }
+        else {
+            const msg = `No .ragmeta.json — add {"description":"...","stack":"...","notes":"..."} so list_projects can describe this project`;
+            console.log(`  ⚠ ${msg}`);
+            warnings.push(msg);
+        }
+        if (!fs.existsSync(path.join(dir, '.ragignore'))) {
+            const msg = `No .ragignore — create one to exclude generated/codegen files (e.g. src/gql/graphql.ts, **/__snapshots__/**)`;
+            console.log(`  ⚠ ${msg}`);
+            warnings.push(msg);
+        }
+        bar.start(files.length, 0, { serviceName, filename: '' });
+        const stats = {
+            serviceName, indexed: 0, skipped: 0, failed: 0, chunks: 0, elapsedSec: 0, warnings,
+        };
+        for (const filePath of files) {
+            if (signal?.aborted) {
+                bar.stop();
+                console.log(`\n  Interrupted — ${serviceName}`);
+                stats.elapsedSec = Math.round((Date.now() - startTime) / 100) / 10;
+                return stats;
+            }
+            bar.update({ filename: path.relative(dir, filePath) });
+            const mtimeMs = fs.statSync(filePath).mtimeMs;
+            const result = await this.ingestFile(filePath, serviceName, mtimeMs, dir);
+            if (result.status === 'indexed')
+                stats.indexed++;
+            else if (result.status === 'skipped')
+                stats.skipped++;
+            else
+                stats.failed++;
+            stats.chunks += result.chunks;
+            bar.increment();
+        }
+        bar.stop();
+        process.stdout.write('\x1B[J');
+        stats.elapsedSec = Math.round((Date.now() - startTime) / 100) / 10;
+        return stats;
+    }
+    getSplitter(filePath) {
+        return SCHEMA_EXTENSIONS.test(filePath) ? this.schemaSplitter : this.jsSplitter;
+    }
+    async embedText(prompt) {
+        return this.embedder.embed(prompt);
+    }
+    async ingestFile(filePath, serviceName, mtimeMs, projectRoot) {
+        try {
+            const repo = this.db.getRepository(db_1.CodeChunk);
+            const relFilePath = path.relative(this.config.workspaceRoot, filePath);
+            const existing = await repo.findOne({
+                select: ['id', 'lastModified'],
+                where: { serviceName, filePath: relFilePath },
+            });
+            if (existing?.lastModified === mtimeMs) {
+                return { status: 'skipped', chunks: 0 };
+            }
+            const t0 = Date.now();
+            const { edges: rawEdges, symbols: rawSymbols } = await this.ast.analyzeFile(filePath, serviceName, projectRoot);
+            // normalize edges to workspace-relative paths
+            const edges = rawEdges
+                .filter(e => e.toFile.startsWith(this.config.workspaceRoot))
+                .map(e => ({
+                ...e,
+                fromFile: path.relative(this.config.workspaceRoot, e.fromFile),
+                toFile: path.relative(this.config.workspaceRoot, e.toFile),
+            }));
+            // normalize symbols: replace absolute filePath with workspace-relative
+            const symbols = rawSymbols.map(s => ({ ...s, filePath: relFilePath }));
+            const depsMs = Date.now() - t0;
+            const code = fs.readFileSync(filePath, 'utf-8');
+            const chunks = await this.getSplitter(filePath).createDocuments([code]);
+            const relPath = path.relative(projectRoot, filePath);
+            const t2 = Date.now();
+            const records = [];
+            for (let i = 0; i < chunks.length; i += this.config.indexer.batchSize) {
+                const batch = chunks.slice(i, i + this.config.indexer.batchSize);
+                const embedded = await Promise.all(batch.map(async (chunk) => {
+                    const startLine = (chunk.metadata?.loc?.lines?.from ?? 1);
+                    const prompt = `Project: ${serviceName} | Path: ${relPath}\n${chunk.pageContent}`;
+                    const embedding = await this.embedText(prompt);
+                    return repo.create({
+                        serviceName, filePath: relFilePath, startLine,
+                        content: chunk.pageContent,
+                        embedding: `[${embedding}]`,
+                        lastModified: mtimeMs,
+                    });
+                }));
+                records.push(...embedded);
+            }
+            const embedMs = Date.now() - t2;
+            const t3 = Date.now();
+            await this.db.transaction(async (manager) => {
+                await manager.delete(db_1.CodeChunk, { serviceName, filePath: relFilePath });
+                for (let i = 0; i < records.length; i += this.config.indexer.batchSize) {
+                    await manager.createQueryBuilder()
+                        .insert()
+                        .into(db_1.CodeChunk)
+                        .values(records.slice(i, i + this.config.indexer.batchSize))
+                        .execute();
+                }
+                await this.writeDepEdges(serviceName, relFilePath, edges, manager);
+                await manager.delete(db_1.CodeSymbol, { project: serviceName, filePath: relFilePath });
+                if (symbols.length > 0) {
+                    await manager.createQueryBuilder()
+                        .insert()
+                        .into(db_1.CodeSymbol)
+                        .values(symbols)
+                        .execute();
+                }
+            });
+            const dbMs = Date.now() - t3;
+            if (chunks.length > 0) {
+                this.logger.debug('IndexerService: file indexed', {
+                    file: relFilePath, chunks: chunks.length,
+                    depsMs, embedMs, dbMs, totalMs: Date.now() - t0,
+                });
+            }
+            return { status: 'indexed', chunks: chunks.length };
+        }
+        catch (err) {
+            const reason = err.message;
+            console.error(`\n[indexer] Failed: ${filePath} — ${reason}`);
+            this.logger.error('IndexerService: failed to index file', { file: filePath, reason });
+            return { status: 'failed', chunks: 0 };
+        }
+    }
+}
+exports.IndexerService = IndexerService;

package/dist/src/services/LoggerService.d.ts

@@ -0,0 +1,12 @@
+import pino from 'pino';
+export type LogLevel = 'debug' | 'info' | 'warn' | 'error';
+export declare class LoggerService {
+    private readonly logger;
+    constructor(logger: pino.Logger);
+    static silent(): LoggerService;
+    static fromEnv(): LoggerService;
+    debug(msg: string, data?: Record<string, unknown>): void;
+    info(msg: string, data?: Record<string, unknown>): void;
+    warn(msg: string, data?: Record<string, unknown>): void;
+    error(msg: string, data?: Record<string, unknown>): void;
+}

package/dist/src/services/LoggerService.js

@@ -0,0 +1,73 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LoggerService = void 0;
+const pino_1 = __importDefault(require("pino"));
+const path = __importStar(require("path"));
+const os = __importStar(require("os"));
+const fs = __importStar(require("fs"));
+const DEFAULT_LOG_FILE = path.join(os.homedir(), '.config', 'codesearch-mcp', 'rag.log');
+class LoggerService {
+    logger;
+    constructor(logger) {
+        this.logger = logger;
+    }
+    static silent() {
+        return new LoggerService((0, pino_1.default)({ level: 'silent' }));
+    }
+    static fromEnv() {
+        const level = (process.env.LOG_LEVEL ?? 'error');
+        const logFileEnv = process.env.LOG_FILE;
+        const enabled = logFileEnv !== 'false' && logFileEnv !== '0';
+        const filePath = enabled ? (logFileEnv || DEFAULT_LOG_FILE) : null;
+        if (filePath)
+            fs.mkdirSync(path.dirname(filePath), { recursive: true });
+        const transport = filePath
+            ? pino_1.default.transport({ targets: [{ target: 'pino/file', options: { destination: filePath }, level }] })
+            : undefined;
+        const logger = transport
+            ? (0, pino_1.default)({ level }, transport)
+            : (0, pino_1.default)({ level: 'silent' });
+        return new LoggerService(logger);
+    }
+    debug(msg, data) { this.logger.debug(data ?? {}, msg); }
+    info(msg, data) { this.logger.info(data ?? {}, msg); }
+    warn(msg, data) { this.logger.warn(data ?? {}, msg); }
+    error(msg, data) { this.logger.error(data ?? {}, msg); }
+}
+exports.LoggerService = LoggerService;

package/dist/src/services/McpServer.d.ts

@@ -0,0 +1,13 @@
+import type { Config } from '../config';
+import { SearcherService } from './SearcherService';
+import { LoggerService } from './LoggerService';
+export declare class McpServer {
+    private searcher;
+    private config;
+    private logger;
+    constructor(searcher: SearcherService, config: Config, logger?: LoggerService);
+    start(): Promise<void>;
+    private formatChunk;
+    private formatDependencies;
+    private formatDependents;
+}