@mhalder/qdrant-mcp-server 1.4.0 → 1.5.0

package/src/code/types.ts

@@ -0,0 +1,117 @@
+/**
+ * Type definitions for code vectorization module
+ */
+
+export interface CodeConfig {
+  // Chunking
+  chunkSize: number;
+  chunkOverlap: number;
+  enableASTChunking: boolean;
+
+  // File discovery
+  supportedExtensions: string[];
+  ignorePatterns: string[];
+  customExtensions?: string[];
+  customIgnorePatterns?: string[];
+
+  // Indexing
+  batchSize: number; // Embeddings per batch
+  maxChunksPerFile?: number;
+  maxTotalChunks?: number;
+
+  // Search
+  defaultSearchLimit: number;
+  enableHybridSearch: boolean;
+}
+
+export interface ScannerConfig {
+  supportedExtensions: string[];
+  ignorePatterns: string[];
+  customIgnorePatterns?: string[];
+}
+
+export interface ChunkerConfig {
+  chunkSize: number;
+  chunkOverlap: number;
+  maxChunkSize: number;
+}
+
+export interface IndexOptions {
+  forceReindex?: boolean;
+  extensions?: string[];
+  ignorePatterns?: string[];
+}
+
+export interface IndexStats {
+  filesScanned: number;
+  filesIndexed: number;
+  chunksCreated: number;
+  durationMs: number;
+  status: "completed" | "partial" | "failed";
+  errors?: string[];
+}
+
+export interface ChangeStats {
+  filesAdded: number;
+  filesModified: number;
+  filesDeleted: number;
+  chunksAdded: number;
+  chunksDeleted: number;
+  durationMs: number;
+}
+
+export interface CodeSearchResult {
+  content: string;
+  filePath: string;
+  startLine: number;
+  endLine: number;
+  language: string;
+  score: number;
+  fileExtension: string;
+}
+
+export interface SearchOptions {
+  limit?: number;
+  useHybrid?: boolean;
+  fileTypes?: string[];
+  pathPattern?: string;
+  scoreThreshold?: number;
+}
+
+export interface IndexStatus {
+  isIndexed: boolean;
+  collectionName?: string;
+  filesCount?: number;
+  chunksCount?: number;
+  lastUpdated?: Date;
+  languages?: string[];
+}
+
+export type ProgressCallback = (progress: ProgressUpdate) => void;
+
+export interface ProgressUpdate {
+  phase: "scanning" | "chunking" | "embedding" | "storing";
+  current: number;
+  total: number;
+  percentage: number;
+  message: string;
+}
+
+export interface CodeChunk {
+  content: string;
+  startLine: number;
+  endLine: number;
+  metadata: {
+    filePath: string;
+    language: string;
+    chunkIndex: number;
+    chunkType?: "function" | "class" | "interface" | "block";
+    name?: string; // Function/class name if applicable
+  };
+}
+
+export interface FileChanges {
+  added: string[];
+  modified: string[];
+  deleted: string[];
+}

package/src/index.ts

@@ -17,6 +17,16 @@ import {
 import Bottleneck from "bottleneck";
 import express from "express";
 import { z } from "zod";
+import {
+  DEFAULT_BATCH_SIZE,
+  DEFAULT_CHUNK_OVERLAP,
+  DEFAULT_CHUNK_SIZE,
+  DEFAULT_CODE_EXTENSIONS,
+  DEFAULT_IGNORE_PATTERNS,
+  DEFAULT_SEARCH_LIMIT,
+} from "./code/config.js";
+import { CodeIndexer } from "./code/indexer.js";
+import type { CodeConfig } from "./code/types.js";
 import { EmbeddingProviderFactory } from "./embeddings/factory.js";
 import { BM25SparseVectorGenerator } from "./embeddings/sparse.js";
 import { getPrompt, listPrompts, loadPromptsConfig, type PromptsConfig } from "./prompts/index.js";
@@ -144,6 +154,20 @@ async function checkOllamaAvailability() {
 const qdrant = new QdrantManager(QDRANT_URL);
 const embeddings = EmbeddingProviderFactory.createFromEnv();
 
+// Initialize code indexer
+const codeConfig: CodeConfig = {
+  chunkSize: parseInt(process.env.CODE_CHUNK_SIZE || String(DEFAULT_CHUNK_SIZE), 10),
+  chunkOverlap: parseInt(process.env.CODE_CHUNK_OVERLAP || String(DEFAULT_CHUNK_OVERLAP), 10),
+  enableASTChunking: process.env.CODE_ENABLE_AST !== "false",
+  supportedExtensions: DEFAULT_CODE_EXTENSIONS,
+  ignorePatterns: DEFAULT_IGNORE_PATTERNS,
+  batchSize: parseInt(process.env.CODE_BATCH_SIZE || String(DEFAULT_BATCH_SIZE), 10),
+  defaultSearchLimit: parseInt(process.env.CODE_SEARCH_LIMIT || String(DEFAULT_SEARCH_LIMIT), 10),
+  enableHybridSearch: process.env.CODE_ENABLE_HYBRID === "true",
+};
+
+const codeIndexer = new CodeIndexer(qdrant, embeddings, codeConfig);
+
 // Load prompts configuration if file exists
 let promptsConfig: PromptsConfig | null = null;
 if (existsSync(PROMPTS_CONFIG_FILE)) {
@@ -366,6 +390,111 @@ function registerHandlers(server: Server) {
             required: ["collection", "query"],
           },
         },
+        {
+          name: "index_codebase",
+          description:
+            "Index a codebase for semantic code search. Automatically discovers files, chunks code intelligently using AST-aware parsing, and stores in vector database. Respects .gitignore and other ignore files.",
+          inputSchema: {
+            type: "object",
+            properties: {
+              path: {
+                type: "string",
+                description: "Absolute or relative path to codebase root directory",
+              },
+              forceReindex: {
+                type: "boolean",
+                description: "Force full re-index even if already indexed (default: false)",
+              },
+              extensions: {
+                type: "array",
+                items: { type: "string" },
+                description: "Custom file extensions to index (e.g., ['.proto', '.graphql'])",
+              },
+              ignorePatterns: {
+                type: "array",
+                items: { type: "string" },
+                description: "Additional patterns to ignore (e.g., ['**/test/**', '**/*.test.ts'])",
+              },
+            },
+            required: ["path"],
+          },
+        },
+        {
+          name: "search_code",
+          description:
+            "Search indexed codebase using natural language queries. Returns semantically relevant code chunks with file paths and line numbers.",
+          inputSchema: {
+            type: "object",
+            properties: {
+              path: {
+                type: "string",
+                description: "Path to codebase (must be indexed first)",
+              },
+              query: {
+                type: "string",
+                description: "Natural language search query (e.g., 'authentication logic')",
+              },
+              limit: {
+                type: "number",
+                description: "Maximum number of results (default: 5, max: 100)",
+              },
+              fileTypes: {
+                type: "array",
+                items: { type: "string" },
+                description: "Filter by file extensions (e.g., ['.ts', '.py'])",
+              },
+              pathPattern: {
+                type: "string",
+                description: "Filter by path glob pattern (e.g., 'src/services/**')",
+              },
+            },
+            required: ["path", "query"],
+          },
+        },
+        {
+          name: "reindex_changes",
+          description:
+            "Incrementally re-index only changed files. Detects added, modified, and deleted files since last index. Requires previous indexing with index_codebase.",
+          inputSchema: {
+            type: "object",
+            properties: {
+              path: {
+                type: "string",
+                description: "Path to codebase",
+              },
+            },
+            required: ["path"],
+          },
+        },
+        {
+          name: "get_index_status",
+          description: "Get indexing status and statistics for a codebase.",
+          inputSchema: {
+            type: "object",
+            properties: {
+              path: {
+                type: "string",
+                description: "Path to codebase",
+              },
+            },
+            required: ["path"],
+          },
+        },
+        {
+          name: "clear_index",
+          description:
+            "Delete all indexed data for a codebase. This is irreversible and will remove the entire collection.",
+          inputSchema: {
+            type: "object",
+            properties: {
+              path: {
+                type: "string",
+                description: "Path to codebase",
+              },
+            },
+            required: ["path"],
+          },
+        },
       ],
     };
   });
@@ -600,6 +729,173 @@ function registerHandlers(server: Server) {
           };
         }
 
+        case "index_codebase": {
+          const IndexCodebaseSchema = z.object({
+            path: z.string(),
+            forceReindex: z.boolean().optional(),
+            extensions: z.array(z.string()).optional(),
+            ignorePatterns: z.array(z.string()).optional(),
+          });
+
+          const { path, forceReindex, extensions, ignorePatterns } =
+            IndexCodebaseSchema.parse(args);
+
+          const stats = await codeIndexer.indexCodebase(
+            path,
+            { forceReindex, extensions, ignorePatterns },
+            (progress) => {
+              // Progress callback - could send progress updates via SSE in future
+              console.error(`[${progress.phase}] ${progress.percentage}% - ${progress.message}`);
+            }
+          );
+
+          let statusMessage = `Indexed ${stats.filesIndexed}/${stats.filesScanned} files (${stats.chunksCreated} chunks) in ${(stats.durationMs / 1000).toFixed(1)}s`;
+
+          if (stats.status === "partial") {
+            statusMessage += `\n\nWarnings:\n${stats.errors?.join("\n")}`;
+          } else if (stats.status === "failed") {
+            statusMessage = `Indexing failed:\n${stats.errors?.join("\n")}`;
+          }
+
+          return {
+            content: [
+              {
+                type: "text",
+                text: statusMessage,
+              },
+            ],
+            isError: stats.status === "failed",
+          };
+        }
+
+        case "search_code": {
+          const SearchCodeSchema = z.object({
+            path: z.string(),
+            query: z.string(),
+            limit: z.number().optional(),
+            fileTypes: z.array(z.string()).optional(),
+            pathPattern: z.string().optional(),
+          });
+
+          const { path, query, limit, fileTypes, pathPattern } = SearchCodeSchema.parse(args);
+
+          const results = await codeIndexer.searchCode(path, query, {
+            limit,
+            fileTypes,
+            pathPattern,
+          });
+
+          if (results.length === 0) {
+            return {
+              content: [
+                {
+                  type: "text",
+                  text: `No results found for query: "${query}"`,
+                },
+              ],
+            };
+          }
+
+          // Format results with file references
+          const formattedResults = results
+            .map(
+              (r, idx) =>
+                `\n--- Result ${idx + 1} (score: ${r.score.toFixed(3)}) ---\n` +
+                `File: ${r.filePath}:${r.startLine}-${r.endLine}\n` +
+                `Language: ${r.language}\n\n` +
+                `${r.content}\n`
+            )
+            .join("\n");
+
+          return {
+            content: [
+              {
+                type: "text",
+                text: `Found ${results.length} result(s):\n${formattedResults}`,
+              },
+            ],
+          };
+        }
+
+        case "get_index_status": {
+          const GetIndexStatusSchema = z.object({
+            path: z.string(),
+          });
+
+          const { path } = GetIndexStatusSchema.parse(args);
+          const status = await codeIndexer.getIndexStatus(path);
+
+          if (!status.isIndexed) {
+            return {
+              content: [
+                {
+                  type: "text",
+                  text: `Codebase at "${path}" is not indexed. Use index_codebase to index it first.`,
+                },
+              ],
+            };
+          }
+
+          return {
+            content: [
+              {
+                type: "text",
+                text: JSON.stringify(status, null, 2),
+              },
+            ],
+          };
+        }
+
+        case "reindex_changes": {
+          const ReindexChangesSchema = z.object({
+            path: z.string(),
+          });
+
+          const { path } = ReindexChangesSchema.parse(args);
+
+          const stats = await codeIndexer.reindexChanges(path, (progress) => {
+            console.error(`[${progress.phase}] ${progress.percentage}% - ${progress.message}`);
+          });
+
+          let message = `Incremental re-index complete:\n`;
+          message += `- Files added: ${stats.filesAdded}\n`;
+          message += `- Files modified: ${stats.filesModified}\n`;
+          message += `- Files deleted: ${stats.filesDeleted}\n`;
+          message += `- Chunks added: ${stats.chunksAdded}\n`;
+          message += `- Duration: ${(stats.durationMs / 1000).toFixed(1)}s`;
+
+          if (stats.filesAdded === 0 && stats.filesModified === 0 && stats.filesDeleted === 0) {
+            message = `No changes detected. Codebase is up to date.`;
+          }
+
+          return {
+            content: [
+              {
+                type: "text",
+                text: message,
+              },
+            ],
+          };
+        }
+
+        case "clear_index": {
+          const ClearIndexSchema = z.object({
+            path: z.string(),
+          });
+
+          const { path } = ClearIndexSchema.parse(args);
+          await codeIndexer.clearIndex(path);
+
+          return {
+            content: [
+              {
+                type: "text",
+                text: `Index cleared for codebase at "${path}".`,
+              },
+            ],
+          };
+        }
+
         default:
           return {
             content: [

package/tests/code/chunker/character-chunker.test.ts

@@ -0,0 +1,141 @@
+import { beforeEach, describe, expect, it } from "vitest";
+import { CharacterChunker } from "../../../src/code/chunker/character-chunker.js";
+import type { ChunkerConfig } from "../../../src/code/types.js";
+
+describe("CharacterChunker", () => {
+  let chunker: CharacterChunker;
+  let config: ChunkerConfig;
+
+  beforeEach(() => {
+    config = {
+      chunkSize: 100,
+      chunkOverlap: 20,
+      maxChunkSize: 200,
+    };
+    chunker = new CharacterChunker(config);
+  });
+
+  describe("chunk", () => {
+    it("should chunk small code into single chunk", async () => {
+      const code =
+        "function hello() {\n  console.log('Starting hello function');\n  return 'world';\n}";
+      const chunks = await chunker.chunk(code, "test.ts", "typescript");
+
+      expect(chunks).toHaveLength(1);
+      expect(chunks[0].content).toContain("hello");
+      expect(chunks[0].startLine).toBe(1);
+      expect(chunks[0].metadata.language).toBe("typescript");
+    });
+
+    it("should chunk large code into multiple chunks", async () => {
+      const code = Array(20)
+        .fill("function testFunction() { console.log('This is a test function'); return true; }\n")
+        .join("");
+      const chunks = await chunker.chunk(code, "test.js", "javascript");
+
+      expect(chunks.length).toBeGreaterThan(1);
+      chunks.forEach((chunk) => {
+        expect(chunk.content.length).toBeLessThanOrEqual(config.maxChunkSize);
+      });
+    });
+
+    it("should preserve line numbers", async () => {
+      const code =
+        "This is line 1 with enough content to not be filtered\n" +
+        "This is line 2 with enough content to not be filtered\n" +
+        "This is line 3 with enough content to not be filtered";
+      const chunks = await chunker.chunk(code, "test.txt", "text");
+
+      expect(chunks.length).toBeGreaterThan(0);
+      expect(chunks[0].startLine).toBe(1);
+      expect(chunks[0].endLine).toBeGreaterThan(chunks[0].startLine);
+    });
+
+    it("should apply overlap between chunks", async () => {
+      const code = Array(20).fill("const x = 1;\n").join("");
+      const chunks = await chunker.chunk(code, "test.js", "javascript");
+
+      if (chunks.length > 1) {
+        // Check that there's overlap in content
+        expect(chunks.length).toBeGreaterThan(1);
+      }
+    });
+
+    it("should find good break points", async () => {
+      const code = `function foo() {
+  return 1;
+}
+
+function bar() {
+  return 2;
+}
+
+function baz() {
+  return 3;
+}`;
+
+      const chunks = await chunker.chunk(code, "test.js", "javascript");
+      // Should try to break at function boundaries
+      chunks.forEach((chunk) => {
+        expect(chunk.content.length).toBeGreaterThan(0);
+      });
+    });
+
+    it("should handle empty code", async () => {
+      const code = "";
+      const chunks = await chunker.chunk(code, "test.ts", "typescript");
+      expect(chunks).toHaveLength(0);
+    });
+
+    it("should handle code with only whitespace", async () => {
+      const code = "   \n\n\n   ";
+      const chunks = await chunker.chunk(code, "test.ts", "typescript");
+      expect(chunks).toHaveLength(0);
+    });
+
+    it("should skip very small chunks", async () => {
+      const code = "x";
+      const chunks = await chunker.chunk(code, "test.ts", "typescript");
+      expect(chunks).toHaveLength(0);
+    });
+  });
+
+  describe("supportsLanguage", () => {
+    it("should support all languages", () => {
+      expect(chunker.supportsLanguage("typescript")).toBe(true);
+      expect(chunker.supportsLanguage("python")).toBe(true);
+      expect(chunker.supportsLanguage("unknown")).toBe(true);
+    });
+  });
+
+  describe("getStrategyName", () => {
+    it("should return correct strategy name", () => {
+      expect(chunker.getStrategyName()).toBe("character-based");
+    });
+  });
+
+  describe("metadata", () => {
+    it("should include correct chunk metadata", async () => {
+      const code = "function test() {\n  console.log('test function');\n  return 1;\n}";
+      const chunks = await chunker.chunk(code, "/path/to/file.ts", "typescript");
+
+      expect(chunks.length).toBeGreaterThan(0);
+      expect(chunks[0].metadata).toEqual({
+        filePath: "/path/to/file.ts",
+        language: "typescript",
+        chunkIndex: 0,
+        chunkType: "block",
+      });
+    });
+
+    it("should increment chunk index", async () => {
+      const code = Array(20).fill("function test() {}\n").join("");
+      const chunks = await chunker.chunk(code, "test.ts", "typescript");
+
+      if (chunks.length > 1) {
+        expect(chunks[0].metadata.chunkIndex).toBe(0);
+        expect(chunks[1].metadata.chunkIndex).toBe(1);
+      }
+    });
+  });
+});