@mhalder/qdrant-mcp-server 1.4.0 → 1.6.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mhalder/qdrant-mcp-server",
-  "version": "1.4.0",
+  "version": "1.6.0",
   "description": "MCP server for semantic search using local Qdrant and Ollama (default) with support for OpenAI, Cohere, and Voyage AI",
   "type": "module",
   "bin": {
@@ -30,7 +30,11 @@
     "url": "https://github.com/mhalder/qdrant-mcp-server.git"
   },
   "publishConfig": {
-    "access": "public"
+    "access": "public",
+    "provenance": true
+  },
+  "overrides": {
+    "tree-sitter": "^0.25.0"
   },
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.0.4",
@@ -38,7 +42,16 @@
     "bottleneck": "^2.19.5",
     "cohere-ai": "^7.19.0",
     "express": "^5.1.0",
+    "ignore": "^7.0.5",
     "openai": "^4.77.3",
+    "tree-sitter": "^0.25.0",
+    "tree-sitter-bash": "^0.25.0",
+    "tree-sitter-go": "^0.25.0",
+    "tree-sitter-java": "^0.23.5",
+    "tree-sitter-javascript": "^0.25.0",
+    "tree-sitter-python": "^0.25.0",
+    "tree-sitter-rust": "^0.24.0",
+    "tree-sitter-typescript": "^0.23.2",
     "zod": "^3.24.1"
   },
   "devDependencies": {

package/src/code/chunker/base.ts

@@ -0,0 +1,22 @@
+/**
+ * Base interface for code chunkers
+ */
+
+import type { CodeChunk } from "../types.js";
+
+export interface CodeChunker {
+  /**
+   * Split code into semantic chunks
+   */
+  chunk(code: string, filePath: string, language: string): Promise<CodeChunk[]>;
+
+  /**
+   * Check if language is supported by this chunker
+   */
+  supportsLanguage(language: string): boolean;
+
+  /**
+   * Get chunking strategy name
+   */
+  getStrategyName(): string;
+}

package/src/code/chunker/character-chunker.ts

@@ -0,0 +1,131 @@
+/**
+ * CharacterChunker - Simple character-based chunking with overlap
+ * Used as fallback when AST parsing is not available
+ */
+
+import type { ChunkerConfig, CodeChunk } from "../types.js";
+import type { CodeChunker } from "./base.js";
+
+export class CharacterChunker implements CodeChunker {
+  constructor(private config: ChunkerConfig) {}
+
+  async chunk(code: string, filePath: string, language: string): Promise<CodeChunk[]> {
+    const chunks: CodeChunk[] = [];
+    const lines = code.split("\n");
+
+    let currentChunk = "";
+    let currentStartLine = 1;
+    let currentLineCount = 0;
+    let chunkIndex = 0;
+
+    for (let i = 0; i < lines.length; i++) {
+      const line = lines[i];
+      currentChunk += `${line}\n`;
+      currentLineCount++;
+
+      // Check if we've reached chunk size
+      if (currentChunk.length >= this.config.chunkSize) {
+        // Try to find a good break point (end of function, class, or empty line)
+        const breakPoint = this.findBreakPoint(lines, i + 1);
+
+        if (breakPoint > i && breakPoint - i < 20) {
+          // Include lines up to break point, but respect maxChunkSize
+          for (let j = i + 1; j <= breakPoint && j < lines.length; j++) {
+            const nextLine = `${lines[j]}\n`;
+            // Stop if adding this line would exceed maxChunkSize
+            if (currentChunk.length + nextLine.length > this.config.maxChunkSize) {
+              break;
+            }
+            currentChunk += nextLine;
+            currentLineCount++;
+            i = j;
+          }
+        }
+
+        // Create chunk
+        chunks.push({
+          content: currentChunk.trim(),
+          startLine: currentStartLine,
+          endLine: currentStartLine + currentLineCount - 1,
+          metadata: {
+            filePath,
+            language,
+            chunkIndex,
+            chunkType: "block",
+          },
+        });
+
+        chunkIndex++;
+
+        // Calculate overlap
+        const overlapLines = this.calculateOverlapLines(currentLineCount);
+        const _overlapStart = Math.max(0, currentLineCount - overlapLines);
+
+        // Start new chunk with overlap
+        currentChunk = `${lines.slice(i - overlapLines + 1, i + 1).join("\n")}\n`;
+        currentStartLine = currentStartLine + currentLineCount - overlapLines;
+        currentLineCount = overlapLines;
+      }
+    }
+
+    // Add remaining content as final chunk
+    if (currentChunk.trim().length > 50) {
+      chunks.push({
+        content: currentChunk.trim(),
+        startLine: currentStartLine,
+        endLine: currentStartLine + currentLineCount - 1,
+        metadata: {
+          filePath,
+          language,
+          chunkIndex,
+          chunkType: "block",
+        },
+      });
+    }
+
+    return chunks;
+  }
+
+  supportsLanguage(_language: string): boolean {
+    // Character chunker supports all languages
+    return true;
+  }
+
+  getStrategyName(): string {
+    return "character-based";
+  }
+
+  /**
+   * Find a good break point in the code (empty line, closing brace, etc.)
+   */
+  private findBreakPoint(lines: string[], startIdx: number): number {
+    const searchWindow = Math.min(20, lines.length - startIdx);
+
+    for (let i = 0; i < searchWindow; i++) {
+      const line = lines[startIdx + i]?.trim() || "";
+
+      // Good break points
+      if (
+        line === "" ||
+        line === "}" ||
+        line === "};" ||
+        line === "]);" ||
+        line.startsWith("//") ||
+        line.startsWith("#")
+      ) {
+        return startIdx + i;
+      }
+    }
+
+    return startIdx;
+  }
+
+  /**
+   * Calculate number of lines to overlap based on chunk size
+   */
+  private calculateOverlapLines(totalLines: number): number {
+    const overlapChars = this.config.chunkOverlap;
+    const avgCharsPerLine = this.config.chunkSize / Math.max(totalLines, 1);
+    return Math.floor(overlapChars / Math.max(avgCharsPerLine, 1));
+  }
+}

package/src/code/chunker/tree-sitter-chunker.ts

@@ -0,0 +1,250 @@
+/**
+ * TreeSitterChunker - AST-aware code chunking using tree-sitter
+ * Primary chunking strategy for supported languages
+ */
+
+import Parser from "tree-sitter";
+// tree-sitter language modules don't have proper types
+import Bash from "tree-sitter-bash";
+import Go from "tree-sitter-go";
+import Java from "tree-sitter-java";
+import JavaScript from "tree-sitter-javascript";
+import Python from "tree-sitter-python";
+import Rust from "tree-sitter-rust";
+import TypeScript from "tree-sitter-typescript";
+
+import type { ChunkerConfig, CodeChunk } from "../types.js";
+import type { CodeChunker } from "./base.js";
+import { CharacterChunker } from "./character-chunker.js";
+
+interface LanguageConfig {
+  parser: Parser;
+  chunkableTypes: string[];
+}
+
+export class TreeSitterChunker implements CodeChunker {
+  private languages: Map<string, LanguageConfig> = new Map();
+  private fallbackChunker: CharacterChunker;
+
+  constructor(private config: ChunkerConfig) {
+    this.fallbackChunker = new CharacterChunker(config);
+    this.initializeParsers();
+  }
+
+  private initializeParsers(): void {
+    // TypeScript
+    const tsParser = new Parser();
+    tsParser.setLanguage(TypeScript.typescript as any);
+    this.languages.set("typescript", {
+      parser: tsParser,
+      chunkableTypes: [
+        "function_declaration",
+        "method_definition",
+        "class_declaration",
+        "interface_declaration",
+        "type_alias_declaration",
+        "enum_declaration",
+      ],
+    });
+
+    // JavaScript
+    const jsParser = new Parser();
+    jsParser.setLanguage(JavaScript as any);
+    this.languages.set("javascript", {
+      parser: jsParser,
+      chunkableTypes: [
+        "function_declaration",
+        "method_definition",
+        "class_declaration",
+        "export_statement",
+      ],
+    });
+
+    // Python
+    const pyParser = new Parser();
+    pyParser.setLanguage(Python as any);
+    this.languages.set("python", {
+      parser: pyParser,
+      chunkableTypes: ["function_definition", "class_definition", "decorated_definition"],
+    });
+
+    // Go
+    const goParser = new Parser();
+    goParser.setLanguage(Go as any);
+    this.languages.set("go", {
+      parser: goParser,
+      chunkableTypes: [
+        "function_declaration",
+        "method_declaration",
+        "type_declaration",
+        "interface_declaration",
+      ],
+    });
+
+    // Rust
+    const rustParser = new Parser();
+    rustParser.setLanguage(Rust as any);
+    this.languages.set("rust", {
+      parser: rustParser,
+      chunkableTypes: ["function_item", "impl_item", "trait_item", "struct_item", "enum_item"],
+    });
+
+    // Java
+    const javaParser = new Parser();
+    javaParser.setLanguage(Java as any);
+    this.languages.set("java", {
+      parser: javaParser,
+      chunkableTypes: [
+        "method_declaration",
+        "class_declaration",
+        "interface_declaration",
+        "enum_declaration",
+      ],
+    });
+
+    // Bash
+    const bashParser = new Parser();
+    bashParser.setLanguage(Bash as any);
+    this.languages.set("bash", {
+      parser: bashParser,
+      chunkableTypes: ["function_definition", "command"],
+    });
+  }
+
+  async chunk(code: string, filePath: string, language: string): Promise<CodeChunk[]> {
+    const langConfig = this.languages.get(language);
+
+    if (!langConfig) {
+      // Fallback to character-based chunking
+      return this.fallbackChunker.chunk(code, filePath, language);
+    }
+
+    try {
+      const tree = langConfig.parser.parse(code);
+      const chunks: CodeChunk[] = [];
+
+      // Find all chunkable nodes
+      const nodes = this.findChunkableNodes(tree.rootNode, langConfig.chunkableTypes);
+
+      for (const [index, node] of nodes.entries()) {
+        const content = code.substring(node.startIndex, node.endIndex);
+
+        // Skip chunks that are too small
+        if (content.length < 50) {
+          continue;
+        }
+
+        // If chunk is too large, fall back to character chunking for this node
+        if (content.length > this.config.maxChunkSize * 2) {
+          const subChunks = await this.fallbackChunker.chunk(content, filePath, language);
+          // Adjust line numbers for sub-chunks
+          for (const subChunk of subChunks) {
+            chunks.push({
+              ...subChunk,
+              startLine: node.startPosition.row + 1 + subChunk.startLine - 1,
+              endLine: node.startPosition.row + 1 + subChunk.endLine - 1,
+              metadata: {
+                ...subChunk.metadata,
+                chunkIndex: chunks.length,
+              },
+            });
+          }
+          continue;
+        }
+
+        chunks.push({
+          content: content.trim(),
+          startLine: node.startPosition.row + 1,
+          endLine: node.endPosition.row + 1,
+          metadata: {
+            filePath,
+            language,
+            chunkIndex: index,
+            chunkType: this.getChunkType(node.type),
+            name: this.extractName(node, code),
+          },
+        });
+      }
+
+      // If no chunks found or file is small, use fallback
+      if (chunks.length === 0 && code.length > 100) {
+        return this.fallbackChunker.chunk(code, filePath, language);
+      }
+
+      return chunks;
+    } catch (error) {
+      // On parsing error, fallback to character-based chunking
+      console.error(`Tree-sitter parsing failed for ${filePath}:`, error);
+      return this.fallbackChunker.chunk(code, filePath, language);
+    }
+  }
+
+  supportsLanguage(language: string): boolean {
+    return this.languages.has(language);
+  }
+
+  getStrategyName(): string {
+    return "tree-sitter";
+  }
+
+  /**
+   * Find all chunkable nodes in the AST
+   */
+  private findChunkableNodes(
+    node: Parser.SyntaxNode,
+    chunkableTypes: string[]
+  ): Parser.SyntaxNode[] {
+    const nodes: Parser.SyntaxNode[] = [];
+
+    const traverse = (n: Parser.SyntaxNode) => {
+      if (chunkableTypes.includes(n.type)) {
+        nodes.push(n);
+        // Don't traverse children of chunkable nodes to avoid nested chunks
+        return;
+      }
+
+      for (const child of n.children) {
+        traverse(child);
+      }
+    };
+
+    traverse(node);
+    return nodes;
+  }
+
+  /**
+   * Extract function/class name from AST node
+   */
+  private extractName(node: Parser.SyntaxNode, code: string): string | undefined {
+    // Try to find name node
+    const nameNode = node.childForFieldName("name");
+    if (nameNode) {
+      return code.substring(nameNode.startIndex, nameNode.endIndex);
+    }
+
+    // For some node types, name might be in a different location
+    for (const child of node.children) {
+      if (child.type === "identifier" || child.type === "type_identifier") {
+        return code.substring(child.startIndex, child.endIndex);
+      }
+    }
+
+    return undefined;
+  }
+
+  /**
+   * Map AST node type to chunk type
+   */
+  private getChunkType(nodeType: string): "function" | "class" | "interface" | "block" {
+    if (nodeType.includes("function") || nodeType.includes("method")) {
+      return "function";
+    }
+    if (nodeType.includes("class") || nodeType.includes("struct")) {
+      return "class";
+    }
+    if (nodeType.includes("interface") || nodeType.includes("trait")) {
+      return "interface";
+    }
+    return "block";
+  }
+}

package/src/code/config.ts

@@ -0,0 +1,156 @@
+/**
+ * Configuration and constants for code vectorization
+ */
+
+export const DEFAULT_CODE_EXTENSIONS = [
+  // TypeScript/JavaScript
+  ".ts",
+  ".tsx",
+  ".js",
+  ".jsx",
+  // Python
+  ".py",
+  // Go
+  ".go",
+  // Rust
+  ".rs",
+  // Java/Kotlin
+  ".java",
+  ".kt",
+  // C/C++
+  ".c",
+  ".cpp",
+  ".h",
+  ".hpp",
+  ".cc",
+  ".cxx",
+  // C#
+  ".cs",
+  // Ruby
+  ".rb",
+  // PHP
+  ".php",
+  // Swift
+  ".swift",
+  // Dart
+  ".dart",
+  // Scala
+  ".scala",
+  // Clojure
+  ".clj",
+  ".cljs",
+  // Haskell
+  ".hs",
+  // OCaml
+  ".ml",
+  // Shell
+  ".sh",
+  ".bash",
+  ".zsh",
+  ".fish",
+  // SQL/Data
+  ".sql",
+  ".proto",
+  ".graphql",
+  // Web
+  ".vue",
+  ".svelte",
+  // Config/Markup
+  ".md",
+  ".markdown",
+  ".json",
+  ".yaml",
+  ".yml",
+  ".toml",
+  ".xml",
+];
+
+export const DEFAULT_IGNORE_PATTERNS = [
+  "node_modules/**",
+  "dist/**",
+  "build/**",
+  "out/**",
+  "target/**",
+  "coverage/**",
+  ".nyc_output/**",
+  ".cache/**",
+  "__pycache__/**",
+  ".git/**",
+  ".svn/**",
+  ".hg/**",
+  ".vscode/**",
+  ".idea/**",
+  "*.min.js",
+  "*.min.css",
+  "*.bundle.js",
+  "*.map",
+  "*.log",
+  ".env",
+  ".env.*",
+];
+
+export const LANGUAGE_MAP: Record<string, string> = {
+  // TypeScript/JavaScript
+  ".ts": "typescript",
+  ".tsx": "typescript",
+  ".js": "javascript",
+  ".jsx": "javascript",
+
+  // Backend languages
+  ".py": "python",
+  ".java": "java",
+  ".go": "go",
+  ".rs": "rust",
+  ".rb": "ruby",
+  ".php": "php",
+
+  // Systems languages
+  ".c": "c",
+  ".cpp": "cpp",
+  ".cc": "cpp",
+  ".cxx": "cpp",
+  ".h": "c",
+  ".hpp": "cpp",
+  ".cs": "c_sharp",
+
+  // Mobile
+  ".swift": "swift",
+  ".kt": "kotlin",
+  ".dart": "dart",
+
+  // Functional
+  ".scala": "scala",
+  ".clj": "clojure",
+  ".cljs": "clojure",
+  ".hs": "haskell",
+  ".ml": "ocaml",
+
+  // Scripting
+  ".sh": "bash",
+  ".bash": "bash",
+  ".zsh": "bash",
+  ".fish": "fish",
+
+  // Data/Query
+  ".sql": "sql",
+  ".proto": "proto",
+  ".graphql": "graphql",
+
+  // Markup/Config
+  ".md": "markdown",
+  ".markdown": "markdown",
+  ".json": "json",
+  ".yaml": "yaml",
+  ".yml": "yaml",
+  ".toml": "toml",
+  ".xml": "xml",
+
+  // Web
+  ".vue": "vue",
+  ".svelte": "svelte",
+};
+
+export const DEFAULT_CHUNK_SIZE = 2500;
+export const DEFAULT_CHUNK_OVERLAP = 300;
+export const DEFAULT_BATCH_SIZE = 100;
+export const DEFAULT_SEARCH_LIMIT = 5;