@mhalder/qdrant-mcp-server 1.4.0 → 1.5.0

package/tests/code/integration.test.ts

@@ -0,0 +1,708 @@
+import { promises as fs } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { CodeIndexer } from "../../src/code/indexer.js";
+import type { CodeConfig } from "../../src/code/types.js";
+import type { EmbeddingProvider } from "../../src/embeddings/base.js";
+import type { QdrantManager } from "../../src/qdrant/client.js";
+
+// Mock implementations (same as indexer.test.ts)
+class MockQdrantManager implements Partial<QdrantManager> {
+  private collections = new Map<string, any>();
+  private points = new Map<string, any[]>();
+
+  async collectionExists(name: string): Promise<boolean> {
+    return this.collections.has(name);
+  }
+
+  async createCollection(
+    name: string,
+    vectorSize: number,
+    distance: string,
+    enableHybrid?: boolean
+  ): Promise<void> {
+    this.collections.set(name, {
+      vectorSize,
+      distance,
+      hybridEnabled: enableHybrid || false,
+    });
+    this.points.set(name, []);
+  }
+
+  async deleteCollection(name: string): Promise<void> {
+    this.collections.delete(name);
+    this.points.delete(name);
+  }
+
+  async addPoints(collectionName: string, points: any[]): Promise<void> {
+    const existing = this.points.get(collectionName) || [];
+    this.points.set(collectionName, [...existing, ...points]);
+  }
+
+  async addPointsWithSparse(collectionName: string, points: any[]): Promise<void> {
+    await this.addPoints(collectionName, points);
+  }
+
+  async search(
+    collectionName: string,
+    _vector: number[],
+    limit: number,
+    filter?: any
+  ): Promise<any[]> {
+    const points = this.points.get(collectionName) || [];
+    let filtered = points;
+
+    // Simple filtering implementation
+    if (filter?.must) {
+      for (const condition of filter.must) {
+        if (condition.key === "fileExtension") {
+          filtered = filtered.filter((p) => condition.match.any.includes(p.payload.fileExtension));
+        }
+      }
+    }
+
+    return filtered.slice(0, limit).map((p, idx) => ({
+      id: p.id,
+      score: 0.9 - idx * 0.05,
+      payload: p.payload,
+    }));
+  }
+
+  async hybridSearch(
+    collectionName: string,
+    vector: number[],
+    _sparseVector: any,
+    limit: number,
+    filter?: any
+  ): Promise<any[]> {
+    // Hybrid search returns similar results with slight boost
+    const results = await this.search(collectionName, vector, limit, filter);
+    return results.map((r) => ({ ...r, score: Math.min(r.score + 0.05, 1.0) }));
+  }
+
+  async getCollectionInfo(name: string): Promise<any> {
+    const collection = this.collections.get(name);
+    const points = this.points.get(name) || [];
+    return {
+      pointsCount: points.length,
+      hybridEnabled: collection?.hybridEnabled || false,
+      vectorSize: collection?.vectorSize || 384,
+    };
+  }
+}
+
+class MockEmbeddingProvider implements EmbeddingProvider {
+  getDimensions(): number {
+    return 384;
+  }
+
+  async embed(text: string): Promise<{ embedding: number[] }> {
+    // Simple hash-based mock embedding
+    const hash = text.split("").reduce((acc, char) => acc + char.charCodeAt(0), 0);
+    const base = (hash % 100) / 100;
+    return { embedding: new Array(384).fill(base) };
+  }
+
+  async embedBatch(texts: string[]): Promise<Array<{ embedding: number[] }>> {
+    return Promise.all(texts.map((text) => this.embed(text)));
+  }
+}
+
+describe("CodeIndexer Integration Tests", () => {
+  let indexer: CodeIndexer;
+  let qdrant: MockQdrantManager;
+  let embeddings: MockEmbeddingProvider;
+  let config: CodeConfig;
+  let tempDir: string;
+  let codebaseDir: string;
+
+  beforeEach(async () => {
+    tempDir = join(
+      tmpdir(),
+      `qdrant-mcp-test-${Date.now()}-${Math.random().toString(36).substring(7)}`
+    );
+    codebaseDir = join(tempDir, "codebase");
+    await fs.mkdir(codebaseDir, { recursive: true });
+
+    qdrant = new MockQdrantManager() as any;
+    embeddings = new MockEmbeddingProvider();
+    config = {
+      chunkSize: 500,
+      chunkOverlap: 50,
+      enableASTChunking: true,
+      supportedExtensions: [".ts", ".js", ".py", ".go"],
+      ignorePatterns: ["node_modules/**", "dist/**", "*.test.*"],
+      batchSize: 10,
+      defaultSearchLimit: 5,
+      enableHybridSearch: false,
+    };
+
+    indexer = new CodeIndexer(qdrant as any, embeddings, config);
+  });
+
+  afterEach(async () => {
+    try {
+      await fs.rm(tempDir, { recursive: true, force: true });
+    } catch (_error) {
+      // Ignore cleanup errors
+    }
+  });
+
+  describe("Complete indexing workflow", () => {
+    it("should index, search, and retrieve results from a TypeScript project", async () => {
+      // Create a sample TypeScript project structure
+      await createTestFile(
+        codebaseDir,
+        "src/auth/login.ts",
+        `
+export class AuthService {
+  async login(email: string, password: string) {
+    return { token: 'jwt-token' };
+  }
+}
+      `
+      );
+
+      await createTestFile(
+        codebaseDir,
+        "src/auth/register.ts",
+        `
+export class RegistrationService {
+  async register(user: User) {
+    return { id: '123', email: user.email };
+  }
+}
+      `
+      );
+
+      await createTestFile(
+        codebaseDir,
+        "src/utils/validation.ts",
+        `
+export function validateEmail(email: string): boolean {
+  return /^[^@]+@[^@]+\\.[^@]+$/.test(email);
+}
+      `
+      );
+
+      // Index the codebase
+      const indexStats = await indexer.indexCodebase(codebaseDir);
+
+      expect(indexStats.filesScanned).toBe(3);
+      expect(indexStats.filesIndexed).toBe(3);
+      expect(indexStats.chunksCreated).toBeGreaterThan(0);
+      expect(indexStats.status).toBe("completed");
+
+      // Search for authentication-related code
+      const authResults = await indexer.searchCode(codebaseDir, "authentication login");
+
+      expect(authResults.length).toBeGreaterThan(0);
+      expect(authResults[0].language).toBe("typescript");
+
+      // Verify index status
+      const status = await indexer.getIndexStatus(codebaseDir);
+
+      expect(status.isIndexed).toBe(true);
+      expect(status.chunksCount).toBeGreaterThan(0);
+    });
+
+    it("should handle multi-language projects", async () => {
+      await createTestFile(
+        codebaseDir,
+        "server.ts",
+        `
+import express from 'express';
+const app = express();
+app.listen(3000);
+      `
+      );
+
+      await createTestFile(
+        codebaseDir,
+        "client.js",
+        `
+const API_URL = 'http://localhost:3000';
+fetch(API_URL).then(res => res.json());
+      `
+      );
+
+      await createTestFile(
+        codebaseDir,
+        "utils.py",
+        `
+def process_data(data):
+    return [x * 2 for x in data]
+      `
+      );
+
+      const stats = await indexer.indexCodebase(codebaseDir);
+
+      expect(stats.filesScanned).toBe(3);
+      expect(stats.filesIndexed).toBe(3);
+
+      // Search should find relevant code regardless of language
+      const results = await indexer.searchCode(codebaseDir, "process data");
+
+      expect(results.length).toBeGreaterThan(0);
+    });
+  });
+
+  describe("Incremental updates workflow", () => {
+    it("should detect and index only changed files", async () => {
+      // Initial indexing
+      await createTestFile(
+        codebaseDir,
+        "file1.ts",
+        `export const firstValue = 1;
+console.log('First file loaded successfully');
+function init(): string {
+  console.log('Initializing system');
+  return 'ready';
+}`
+      );
+      await createTestFile(
+        codebaseDir,
+        "file2.ts",
+        `export const secondValue = 2;
+console.log('Second file loaded successfully');
+function start(): string {
+  console.log('Starting application');
+  return 'started';
+}`
+      );
+
+      const initialStats = await indexer.indexCodebase(codebaseDir);
+      expect(initialStats.filesIndexed).toBe(2);
+
+      // Add a new file
+      await createTestFile(
+        codebaseDir,
+        "file3.ts",
+        `/**
+ * Process data and return result string
+ */
+export function process(): string {
+  console.log('Processing data in third file');
+  const status = 'processed';
+  if (status) {
+    console.log('Status confirmed:', status);
+  }
+  return status;
+}
+
+export const thirdValue = 3;`
+      );
+
+      // Incremental update
+      const updateStats = await indexer.reindexChanges(codebaseDir);
+
+      expect(updateStats.filesAdded).toBe(1);
+      expect(updateStats.filesModified).toBe(0);
+      expect(updateStats.filesDeleted).toBe(0);
+
+      // Verify search includes new content
+      const results = await indexer.searchCode(codebaseDir, "third");
+      expect(results.length).toBeGreaterThan(0);
+    });
+
+    it("should handle file modifications", async () => {
+      await createTestFile(
+        codebaseDir,
+        "config.ts",
+        "export const DEBUG_MODE = false;\nconsole.log('Debug mode off');"
+      );
+
+      await indexer.indexCodebase(codebaseDir);
+
+      // Modify the file
+      await createTestFile(
+        codebaseDir,
+        "config.ts",
+        "export const DEBUG_MODE = true;\nconsole.log('Debug mode on');"
+      );
+
+      const updateStats = await indexer.reindexChanges(codebaseDir);
+
+      expect(updateStats.filesModified).toBe(1);
+      expect(updateStats.filesAdded).toBe(0);
+    });
+
+    it("should handle file deletions", async () => {
+      await createTestFile(
+        codebaseDir,
+        "temp.ts",
+        "export const tempValue = true;\nconsole.log('Temporary file created');\nfunction cleanup() { return null; }"
+      );
+      await createTestFile(
+        codebaseDir,
+        "keep.ts",
+        "export const keepValue = true;\nconsole.log('Permanent file stays');\nfunction maintain() { return true; }"
+      );
+
+      await indexer.indexCodebase(codebaseDir);
+
+      // Delete a file
+      await fs.unlink(join(codebaseDir, "temp.ts"));
+
+      const updateStats = await indexer.reindexChanges(codebaseDir);
+
+      expect(updateStats.filesDeleted).toBe(1);
+      expect(updateStats.filesAdded).toBe(0);
+    });
+
+    it("should handle mixed changes in one update", async () => {
+      // Initial state
+      await createTestFile(
+        codebaseDir,
+        "file1.ts",
+        "export const alpha = 1;\nconsole.log('Alpha file');"
+      );
+      await createTestFile(
+        codebaseDir,
+        "file2.ts",
+        "export const beta = 2;\nconsole.log('Beta file');"
+      );
+      await createTestFile(
+        codebaseDir,
+        "file3.ts",
+        "export const gamma = 3;\nconsole.log('Gamma file');"
+      );
+
+      await indexer.indexCodebase(codebaseDir);
+
+      // Mixed changes
+      await createTestFile(
+        codebaseDir,
+        "file1.ts",
+        "export const alpha = 100;\nconsole.log('Alpha modified');"
+      ); // Modified
+      await createTestFile(
+        codebaseDir,
+        "file4.ts",
+        "export const delta = 4;\nconsole.log('Delta file added');"
+      ); // Added
+      await fs.unlink(join(codebaseDir, "file3.ts")); // Deleted
+
+      const updateStats = await indexer.reindexChanges(codebaseDir);
+
+      expect(updateStats.filesAdded).toBe(1);
+      expect(updateStats.filesModified).toBe(1);
+      expect(updateStats.filesDeleted).toBe(1);
+    });
+  });
+
+  describe("Search filtering and options", () => {
+    beforeEach(async () => {
+      await createTestFile(codebaseDir, "users.ts", "export class UserService {}");
+      await createTestFile(codebaseDir, "auth.ts", "export class AuthService {}");
+      await createTestFile(codebaseDir, "utils.js", "export function helper() {}");
+      await createTestFile(codebaseDir, "data.py", "class DataProcessor: pass");
+
+      await indexer.indexCodebase(codebaseDir);
+    });
+
+    it("should filter results by file extension", async () => {
+      const results = await indexer.searchCode(codebaseDir, "class", {
+        fileTypes: [".ts"],
+      });
+
+      results.forEach((result) => {
+        expect(result.fileExtension).toBe(".ts");
+      });
+    });
+
+    it("should respect search limit", async () => {
+      const results = await indexer.searchCode(codebaseDir, "export", {
+        limit: 2,
+      });
+
+      expect(results.length).toBeLessThanOrEqual(2);
+    });
+
+    it("should apply score threshold", async () => {
+      const results = await indexer.searchCode(codebaseDir, "service", {
+        scoreThreshold: 0.8,
+      });
+
+      results.forEach((result) => {
+        expect(result.score).toBeGreaterThanOrEqual(0.8);
+      });
+    });
+
+    it("should support path pattern filtering", async () => {
+      await createTestFile(codebaseDir, "src/api/endpoints.ts", "export const API = {}");
+      await indexer.indexCodebase(codebaseDir, { forceReindex: true });
+
+      const results = await indexer.searchCode(codebaseDir, "export", {
+        pathPattern: "src/api/**",
+      });
+
+      expect(Array.isArray(results)).toBe(true);
+    });
+  });
+
+  describe("Hybrid search workflow", () => {
+    it("should enable and use hybrid search", async () => {
+      const hybridConfig = { ...config, enableHybridSearch: true };
+      const hybridIndexer = new CodeIndexer(qdrant as any, embeddings, hybridConfig);
+
+      await createTestFile(
+        codebaseDir,
+        "search.ts",
+        "function performSearch(query: string) { return results; }"
+      );
+
+      await hybridIndexer.indexCodebase(codebaseDir);
+
+      const results = await hybridIndexer.searchCode(codebaseDir, "search query");
+
+      expect(results.length).toBeGreaterThan(0);
+    });
+
+    it("should fallback to standard search if hybrid not available", async () => {
+      const hybridConfig = { ...config, enableHybridSearch: true };
+      const hybridIndexer = new CodeIndexer(qdrant as any, embeddings, hybridConfig);
+
+      // Index without hybrid
+      await createTestFile(
+        codebaseDir,
+        "test.ts",
+        `export const testValue = true;
+console.log('Test value configured successfully');
+function validate(): boolean {
+  console.log('Validating test value');
+  return testValue === true;
+}`
+      );
+      await indexer.indexCodebase(codebaseDir);
+
+      // Search with hybrid-enabled indexer but collection without hybrid
+      const results = await hybridIndexer.searchCode(codebaseDir, "test");
+
+      expect(results.length).toBeGreaterThan(0);
+    });
+  });
+
+  describe("Large project workflow", () => {
+    it("should handle projects with many files", async () => {
+      // Create a large project structure
+      for (let i = 0; i < 20; i++) {
+        await createTestFile(
+          codebaseDir,
+          `module${i}.ts`,
+          `export function func${i}() { return ${i}; }`
+        );
+      }
+
+      const stats = await indexer.indexCodebase(codebaseDir);
+
+      expect(stats.filesScanned).toBe(20);
+      expect(stats.filesIndexed).toBe(20);
+      expect(stats.status).toBe("completed");
+    });
+
+    it("should handle large files with many chunks", async () => {
+      const largeFile = Array(100)
+        .fill(null)
+        .map((_, i) => `function test${i}() { return ${i}; }`)
+        .join("\n\n");
+
+      await createTestFile(codebaseDir, "large.ts", largeFile);
+
+      const stats = await indexer.indexCodebase(codebaseDir);
+
+      expect(stats.chunksCreated).toBeGreaterThan(1);
+    });
+  });
+
+  describe("Error handling and recovery", () => {
+    it("should continue indexing after encountering errors", async () => {
+      await createTestFile(
+        codebaseDir,
+        "valid.ts",
+        "export const validValue = true;\nconsole.log('Valid file');"
+      );
+      await createTestFile(
+        codebaseDir,
+        "secrets.ts",
+        'export const apiKey = "sk_test_FAKE_KEY_FOR_TESTING_NOT_REAL_KEY";\nconsole.log("Secrets file");'
+      );
+
+      const stats = await indexer.indexCodebase(codebaseDir);
+
+      // Should index valid file and report error for secrets file
+      expect(stats.filesIndexed).toBe(1);
+      expect(stats.errors?.length).toBeGreaterThan(0);
+      expect(stats.status).toBe("completed");
+    });
+
+    it("should allow re-indexing after partial failure", async () => {
+      await createTestFile(
+        codebaseDir,
+        "test.ts",
+        "export const testData = true;\nconsole.log('Test data loaded');"
+      );
+
+      const stats1 = await indexer.indexCodebase(codebaseDir);
+      expect(stats1.status).toBe("completed");
+
+      // Force re-index
+      const stats2 = await indexer.indexCodebase(codebaseDir, { forceReindex: true });
+      expect(stats2.status).toBe("completed");
+    });
+  });
+
+  describe("Clear and re-index workflow", () => {
+    it("should clear index and allow re-indexing", async () => {
+      await createTestFile(codebaseDir, "test.ts", "const test = 1;");
+
+      await indexer.indexCodebase(codebaseDir);
+
+      let status = await indexer.getIndexStatus(codebaseDir);
+      expect(status.isIndexed).toBe(true);
+
+      await indexer.clearIndex(codebaseDir);
+
+      status = await indexer.getIndexStatus(codebaseDir);
+      expect(status.isIndexed).toBe(false);
+
+      // Re-index
+      const stats = await indexer.indexCodebase(codebaseDir);
+      expect(stats.status).toBe("completed");
+
+      status = await indexer.getIndexStatus(codebaseDir);
+      expect(status.isIndexed).toBe(true);
+    });
+  });
+
+  describe("Progress tracking", () => {
+    it("should report progress through all phases", async () => {
+      for (let i = 0; i < 5; i++) {
+        await createTestFile(
+          codebaseDir,
+          `file${i}.ts`,
+          `export const value${i} = ${i};\nconsole.log('File ${i} loaded successfully');\nfunction process${i}() { return value${i} * 2; }`
+        );
+      }
+
+      const progressUpdates: string[] = [];
+      const progressCallback = (progress: any) => {
+        progressUpdates.push(progress.phase);
+      };
+
+      await indexer.indexCodebase(codebaseDir, undefined, progressCallback);
+
+      expect(progressUpdates).toContain("scanning");
+      expect(progressUpdates).toContain("chunking");
+      expect(progressUpdates).toContain("embedding");
+      expect(progressUpdates).toContain("storing");
+    });
+
+    it("should report progress during incremental updates", async () => {
+      await createTestFile(
+        codebaseDir,
+        "file1.ts",
+        "export const initial = 1;\nconsole.log('Initial file');"
+      );
+      await indexer.indexCodebase(codebaseDir);
+
+      await createTestFile(
+        codebaseDir,
+        "file2.ts",
+        "export const additional = 2;\nconsole.log('Additional file');"
+      );
+
+      const progressUpdates: string[] = [];
+      const progressCallback = (progress: any) => {
+        progressUpdates.push(progress.phase);
+      };
+
+      await indexer.reindexChanges(codebaseDir, progressCallback);
+
+      expect(progressUpdates.length).toBeGreaterThan(0);
+    });
+  });
+
+  describe("Hybrid search with incremental updates", () => {
+    it("should use hybrid search during reindexChanges", async () => {
+      const hybridConfig = { ...config, enableHybridSearch: true };
+      const hybridIndexer = new CodeIndexer(qdrant as any, embeddings, hybridConfig);
+
+      // Initial indexing with hybrid search
+      await createTestFile(
+        codebaseDir,
+        "initial.ts",
+        "export const initial = 1;\nconsole.log('Initial file');"
+      );
+      await hybridIndexer.indexCodebase(codebaseDir);
+
+      // Add a new file with enough content to create chunks
+      await createTestFile(
+        codebaseDir,
+        "added.ts",
+        `export const added = 2;
+console.log('Added file with more content');
+
+/**
+ * Function to demonstrate hybrid search indexing
+ */
+export function processData(data: string[]): string[] {
+  console.log('Processing data in hybrid search mode');
+  const result = data.map(item => item.toUpperCase());
+  return result;
+}
+
+export class DataProcessor {
+  process(input: string): string {
+    return input.trim();
+  }
+}`
+      );
+
+      // Reindex with hybrid search - this should cover lines 540-545
+      const updateStats = await hybridIndexer.reindexChanges(codebaseDir);
+
+      expect(updateStats.filesAdded).toBe(1);
+      expect(updateStats.chunksAdded).toBeGreaterThan(0);
+    });
+  });
+
+  describe("Error handling during reindexChanges", () => {
+    it("should handle file processing errors gracefully", async () => {
+      // Initial indexing
+      await createTestFile(
+        codebaseDir,
+        "file1.ts",
+        "export const value = 1;\nconsole.log('File 1');"
+      );
+      await indexer.indexCodebase(codebaseDir);
+
+      // Add a new file
+      await createTestFile(
+        codebaseDir,
+        "file2.ts",
+        "export const value2 = 2;\nconsole.log('File 2');"
+      );
+
+      // This should not throw even if there are processing errors
+      const stats = await indexer.reindexChanges(codebaseDir);
+
+      // Stats should still be returned
+      expect(stats).toBeDefined();
+      expect(stats.filesAdded).toBeGreaterThanOrEqual(0);
+    });
+  });
+});
+
+// Helper function
+async function createTestFile(
+  baseDir: string,
+  relativePath: string,
+  content: string
+): Promise<void> {
+  const fullPath = join(baseDir, relativePath);
+  const dir = join(fullPath, "..");
+  await fs.mkdir(dir, { recursive: true });
+  await fs.writeFile(fullPath, content, "utf-8");
+}