@mhalder/qdrant-mcp-server 2.1.1 → 2.2.0

package/src/code/indexer.ts

@@ -24,11 +24,14 @@ import type {
   SearchOptions,
 } from "./types.js";
 
+/** Reserved ID for storing indexing metadata in the collection */
+const INDEXING_METADATA_ID = "__indexing_metadata__";
+
 export class CodeIndexer {
   constructor(
     private qdrant: QdrantManager,
     private embeddings: EmbeddingProvider,
-    private config: CodeConfig
+    private config: CodeConfig,
   ) {}
 
   /**
@@ -59,7 +62,7 @@ export class CodeIndexer {
   async indexCodebase(
     path: string,
     options?: IndexOptions,
-    progressCallback?: ProgressCallback
+    progressCallback?: ProgressCallback,
   ): Promise<IndexStats> {
     const startTime = Date.now();
     const stats: IndexStats = {
@@ -71,9 +74,10 @@ export class CodeIndexer {
       errors: [],
     };
 
-    try {
-      const absolutePath = await this.validatePath(path);
+    const absolutePath = await this.validatePath(path);
+    const collectionName = this.getCollectionName(absolutePath);
 
+    try {
       // 1. Scan files
       progressCallback?.({
         phase: "scanning",
@@ -84,9 +88,11 @@ export class CodeIndexer {
       });
 
       const scanner = new FileScanner({
-        supportedExtensions: options?.extensions || this.config.supportedExtensions,
+        supportedExtensions:
+          options?.extensions || this.config.supportedExtensions,
         ignorePatterns: this.config.ignorePatterns,
-        customIgnorePatterns: options?.ignorePatterns || this.config.customIgnorePatterns,
+        customIgnorePatterns:
+          options?.ignorePatterns || this.config.customIgnorePatterns,
       });
 
       await scanner.loadIgnorePatterns(absolutePath);
@@ -101,8 +107,8 @@ export class CodeIndexer {
       }
 
       // 2. Create or verify collection
-      const collectionName = this.getCollectionName(absolutePath);
-      const collectionExists = await this.qdrant.collectionExists(collectionName);
+      const collectionExists =
+        await this.qdrant.collectionExists(collectionName);
 
       if (options?.forceReindex && collectionExists) {
         await this.qdrant.deleteCollection(collectionName);
@@ -114,10 +120,13 @@ export class CodeIndexer {
           collectionName,
           vectorSize,
           "Cosine",
-          this.config.enableHybridSearch
+          this.config.enableHybridSearch,
         );
       }
 
+      // Store "indexing in progress" marker immediately after collection is ready
+      await this.storeIndexingMarker(collectionName, false);
+
       // 3. Process files and create chunks
       const chunker = new TreeSitterChunker({
         chunkSize: this.config.chunkSize,
@@ -141,7 +150,9 @@ export class CodeIndexer {
 
           // Check for secrets (basic detection)
           if (metadataExtractor.containsSecrets(code)) {
-            stats.errors?.push(`Skipped ${filePath}: potential secrets detected`);
+            stats.errors?.push(
+              `Skipped ${filePath}: potential secrets detected`,
+            );
             continue;
           }
 
@@ -158,7 +169,10 @@ export class CodeIndexer {
             allChunks.push({ chunk, id });
 
             // Check total chunk limit
-            if (this.config.maxTotalChunks && allChunks.length >= this.config.maxTotalChunks) {
+            if (
+              this.config.maxTotalChunks &&
+              allChunks.length >= this.config.maxTotalChunks
+            ) {
               break;
             }
           }
@@ -166,11 +180,15 @@ export class CodeIndexer {
           stats.filesIndexed++;
 
           // Check total chunk limit
-          if (this.config.maxTotalChunks && allChunks.length >= this.config.maxTotalChunks) {
+          if (
+            this.config.maxTotalChunks &&
+            allChunks.length >= this.config.maxTotalChunks
+          ) {
             break;
           }
         } catch (error) {
-          const errorMessage = error instanceof Error ? error.message : String(error);
+          const errorMessage =
+            error instanceof Error ? error.message : String(error);
           stats.errors?.push(`Failed to process ${filePath}: ${errorMessage}`);
         }
       }
@@ -183,12 +201,15 @@ export class CodeIndexer {
         await synchronizer.updateSnapshot(files);
       } catch (error) {
         // Snapshot failure shouldn't fail the entire indexing
-        const errorMessage = error instanceof Error ? error.message : String(error);
+        const errorMessage =
+          error instanceof Error ? error.message : String(error);
         console.error("Failed to save snapshot:", errorMessage);
         stats.errors?.push(`Snapshot save failed: ${errorMessage}`);
       }
 
       if (allChunks.length === 0) {
+        // Still store completion marker even with no chunks
+        await this.storeIndexingMarker(collectionName, true);
         stats.status = "completed";
         stats.durationMs = Date.now() - startTime;
         return stats;
@@ -203,7 +224,8 @@ export class CodeIndexer {
           phase: "embedding",
           current: i + batch.length,
           total: allChunks.length,
-          percentage: 40 + Math.round(((i + batch.length) / allChunks.length) * 30), // 40-70%
+          percentage:
+            40 + Math.round(((i + batch.length) / allChunks.length) * 30), // 40-70%
           message: `Generating embeddings ${i + batch.length}/${allChunks.length}`,
         });
 
@@ -225,7 +247,9 @@ export class CodeIndexer {
               codebasePath: absolutePath,
               chunkIndex: b.chunk.metadata.chunkIndex,
               ...(b.chunk.metadata.name && { name: b.chunk.metadata.name }),
-              ...(b.chunk.metadata.chunkType && { chunkType: b.chunk.metadata.chunkType }),
+              ...(b.chunk.metadata.chunkType && {
+                chunkType: b.chunk.metadata.chunkType,
+              }),
             },
           }));
 
@@ -233,7 +257,8 @@ export class CodeIndexer {
             phase: "storing",
             current: i + batch.length,
             total: allChunks.length,
-            percentage: 70 + Math.round(((i + batch.length) / allChunks.length) * 30), // 70-100%
+            percentage:
+              70 + Math.round(((i + batch.length) / allChunks.length) * 30), // 70-100%
             message: `Storing chunks ${i + batch.length}/${allChunks.length}`,
           });
 
@@ -254,7 +279,9 @@ export class CodeIndexer {
                 codebasePath: absolutePath,
                 chunkIndex: b.chunk.metadata.chunkIndex,
                 ...(b.chunk.metadata.name && { name: b.chunk.metadata.name }),
-                ...(b.chunk.metadata.chunkType && { chunkType: b.chunk.metadata.chunkType }),
+                ...(b.chunk.metadata.chunkType && {
+                  chunkType: b.chunk.metadata.chunkType,
+                }),
               },
             }));
 
@@ -263,16 +290,23 @@ export class CodeIndexer {
             await this.qdrant.addPoints(collectionName, points);
           }
         } catch (error) {
-          const errorMessage = error instanceof Error ? error.message : String(error);
-          stats.errors?.push(`Failed to process batch at index ${i}: ${errorMessage}`);
+          const errorMessage =
+            error instanceof Error ? error.message : String(error);
+          stats.errors?.push(
+            `Failed to process batch at index ${i}: ${errorMessage}`,
+          );
           stats.status = "partial";
         }
       }
 
+      // Store completion marker to indicate indexing is complete
+      await this.storeIndexingMarker(collectionName, true);
+
       stats.durationMs = Date.now() - startTime;
       return stats;
     } catch (error) {
-      const errorMessage = error instanceof Error ? error.message : String(error);
+      const errorMessage =
+        error instanceof Error ? error.message : String(error);
       stats.status = "failed";
       stats.errors?.push(`Indexing failed: ${errorMessage}`);
       stats.durationMs = Date.now() - startTime;
@@ -280,13 +314,62 @@ export class CodeIndexer {
     }
   }
 
+  /**
+   * Store an indexing status marker in the collection.
+   * Called at the start of indexing with complete=false, and at the end with complete=true.
+   */
+  private async storeIndexingMarker(
+    collectionName: string,
+    complete: boolean,
+  ): Promise<void> {
+    try {
+      // Create a dummy vector of zeros (required by Qdrant)
+      const vectorSize = this.embeddings.getDimensions();
+      const zeroVector = new Array(vectorSize).fill(0);
+
+      // Check if collection uses hybrid mode
+      const collectionInfo =
+        await this.qdrant.getCollectionInfo(collectionName);
+
+      const payload = {
+        _type: "indexing_metadata",
+        indexingComplete: complete,
+        ...(complete
+          ? { completedAt: new Date().toISOString() }
+          : { startedAt: new Date().toISOString() }),
+      };
+
+      if (collectionInfo.hybridEnabled) {
+        await this.qdrant.addPointsWithSparse(collectionName, [
+          {
+            id: INDEXING_METADATA_ID,
+            vector: zeroVector,
+            sparseVector: { indices: [], values: [] },
+            payload,
+          },
+        ]);
+      } else {
+        await this.qdrant.addPoints(collectionName, [
+          {
+            id: INDEXING_METADATA_ID,
+            vector: zeroVector,
+            payload,
+          },
+        ]);
+      }
+    } catch (error) {
+      // Non-fatal: log but don't fail the indexing
+      console.error("Failed to store indexing marker:", error);
+    }
+  }
+
   /**
    * Search code semantically
    */
   async searchCode(
     path: string,
     query: string,
-    options?: SearchOptions
+    options?: SearchOptions,
   ): Promise<CodeSearchResult[]> {
     const absolutePath = await this.validatePath(path);
     const collectionName = this.getCollectionName(absolutePath);
@@ -300,7 +383,8 @@ export class CodeIndexer {
     // Check if collection has hybrid search enabled
     const collectionInfo = await this.qdrant.getCollectionInfo(collectionName);
     const useHybrid =
-      (options?.useHybrid ?? this.config.enableHybridSearch) && collectionInfo.hybridEnabled;
+      (options?.useHybrid ?? this.config.enableHybridSearch) &&
+      collectionInfo.hybridEnabled;
 
     // Generate query embedding
     const { embedding } = await this.embeddings.embed(query);
@@ -342,14 +426,14 @@ export class CodeIndexer {
         embedding,
         sparseVector,
         options?.limit || this.config.defaultSearchLimit,
-        filter
+        filter,
       );
     } else {
       results = await this.qdrant.search(
         collectionName,
         embedding,
         options?.limit || this.config.defaultSearchLimit,
-        filter
+        filter,
       );
     }
 
@@ -379,24 +463,75 @@ export class CodeIndexer {
     const exists = await this.qdrant.collectionExists(collectionName);
 
     if (!exists) {
-      return { isIndexed: false };
+      return { isIndexed: false, status: "not_indexed" };
     }
 
+    // Check for indexing marker in Qdrant (persisted across instances)
+    const indexingMarker = await this.qdrant.getPoint(
+      collectionName,
+      INDEXING_METADATA_ID,
+    );
     const info = await this.qdrant.getCollectionInfo(collectionName);
 
+    // Check marker status
+    const isComplete = indexingMarker?.payload?.indexingComplete === true;
+    const isInProgress = indexingMarker?.payload?.indexingComplete === false;
+
+    // Subtract 1 from points count if marker exists (metadata point doesn't count as a chunk)
+    const actualChunksCount = indexingMarker
+      ? Math.max(0, info.pointsCount - 1)
+      : info.pointsCount;
+
+    if (isInProgress) {
+      // Indexing in progress - marker exists with indexingComplete=false
+      return {
+        isIndexed: false,
+        status: "indexing",
+        collectionName,
+        chunksCount: actualChunksCount,
+      };
+    }
+
+    if (isComplete) {
+      // Indexing completed - marker exists with indexingComplete=true
+      return {
+        isIndexed: true,
+        status: "indexed",
+        collectionName,
+        chunksCount: actualChunksCount,
+        lastUpdated: indexingMarker.payload?.completedAt
+          ? new Date(indexingMarker.payload.completedAt)
+          : undefined,
+      };
+    }
+
+    // Legacy collection (no marker) - check if it has content
+    // If it has chunks, assume it's indexed (backwards compatibility)
+    if (actualChunksCount > 0) {
+      return {
+        isIndexed: true,
+        status: "indexed",
+        collectionName,
+        chunksCount: actualChunksCount,
+      };
+    }
+
+    // Collection exists but no chunks and no marker - not indexed
     return {
-      isIndexed: true,
+      isIndexed: false,
+      status: "not_indexed",
       collectionName,
-      chunksCount: info.pointsCount,
-      // TODO: Extract unique languages and file count from collection
-      // This would require scrolling through points or maintaining separate metadata
+      chunksCount: 0,
     };
   }
 
   /**
    * Incrementally re-index only changed files
    */
-  async reindexChanges(path: string, progressCallback?: ProgressCallback): Promise<ChangeStats> {
+  async reindexChanges(
+    path: string,
+    progressCallback?: ProgressCallback,
+  ): Promise<ChangeStats> {
     const startTime = Date.now();
     const stats: ChangeStats = {
       filesAdded: 0,
@@ -422,7 +557,9 @@ export class CodeIndexer {
       const hasSnapshot = await synchronizer.initialize();
 
       if (!hasSnapshot) {
-        throw new Error("No previous snapshot found. Use index_codebase for initial indexing.");
+        throw new Error(
+          "No previous snapshot found. Use index_codebase for initial indexing.",
+        );
       }
 
       // Scan current files
@@ -449,7 +586,11 @@ export class CodeIndexer {
       stats.filesModified = changes.modified.length;
       stats.filesDeleted = changes.deleted.length;
 
-      if (stats.filesAdded === 0 && stats.filesModified === 0 && stats.filesDeleted === 0) {
+      if (
+        stats.filesAdded === 0 &&
+        stats.filesModified === 0 &&
+        stats.filesDeleted === 0
+      ) {
         stats.durationMs = Date.now() - startTime;
         return stats;
       }
@@ -461,27 +602,34 @@ export class CodeIndexer {
       });
       const metadataExtractor = new MetadataExtractor();
 
-      // Process deleted and modified files - collect chunk IDs to delete
-      const _chunkIdsToDelete: string[] = [];
-      const filesToReprocess = [...changes.modified, ...changes.deleted];
+      // Delete chunks for modified and deleted files BEFORE adding new ones
+      const filesToDelete = [...changes.modified, ...changes.deleted];
 
-      for (const _filePath of filesToReprocess) {
-        try {
-          // Read old file content to generate chunk IDs for deletion
-          // We need to regenerate the chunks to get their IDs
-          // For now, we'll use a simpler approach: delete based on file path
-          // This requires keeping track of chunk IDs per file
-          // Since we don't have a direct way to query by file path,
-          // we'll mark these as needing deletion by filename pattern
-          // For simplicity in Phase 2, we'll re-index everything
-          // A future enhancement would be to maintain a chunk ID mapping
-        } catch (_error) {
-          // File might be deleted, skip
+      if (filesToDelete.length > 0) {
+        progressCallback?.({
+          phase: "scanning",
+          current: 0,
+          total: filesToDelete.length,
+          percentage: 5,
+          message: `Deleting old chunks for ${filesToDelete.length} files...`,
+        });
+
+        for (const relativePath of filesToDelete) {
+          try {
+            const filter = {
+              must: [{ key: "relativePath", match: { value: relativePath } }],
+            };
+            await this.qdrant.deletePointsByFilter(collectionName, filter);
+          } catch (error) {
+            // Log but don't fail - file might not have any chunks
+            console.error(
+              `Failed to delete chunks for ${relativePath}:`,
+              error,
+            );
+          }
         }
       }
 
-      // For Phase 2 MVP: Simply re-process all changed files
-      // TODO Phase 3: Implement proper chunk deletion by maintaining chunk ID mapping
       const filesToIndex = [...changes.added, ...changes.modified];
       const allChunks: Array<{ chunk: CodeChunk; id: string }> = [];
 
@@ -526,7 +674,8 @@ export class CodeIndexer {
           phase: "embedding",
           current: i + batch.length,
           total: allChunks.length,
-          percentage: 40 + Math.round(((i + batch.length) / allChunks.length) * 30),
+          percentage:
+            40 + Math.round(((i + batch.length) / allChunks.length) * 30),
           message: `Generating embeddings ${i + batch.length}/${allChunks.length}`,
         });
 
@@ -546,7 +695,9 @@ export class CodeIndexer {
             codebasePath: absolutePath,
             chunkIndex: b.chunk.metadata.chunkIndex,
             ...(b.chunk.metadata.name && { name: b.chunk.metadata.name }),
-            ...(b.chunk.metadata.chunkType && { chunkType: b.chunk.metadata.chunkType }),
+            ...(b.chunk.metadata.chunkType && {
+              chunkType: b.chunk.metadata.chunkType,
+            }),
           },
         }));
 
@@ -554,7 +705,8 @@ export class CodeIndexer {
           phase: "storing",
           current: i + batch.length,
           total: allChunks.length,
-          percentage: 70 + Math.round(((i + batch.length) / allChunks.length) * 30),
+          percentage:
+            70 + Math.round(((i + batch.length) / allChunks.length) * 30),
           message: `Storing chunks ${i + batch.length}/${allChunks.length}`,
         });
 
@@ -562,7 +714,9 @@ export class CodeIndexer {
           const sparseGenerator = new BM25SparseVectorGenerator();
           const hybridPoints = points.map((point, idx) => ({
             ...point,
-            sparseVector: sparseGenerator.generate(allChunks[i + idx].chunk.content),
+            sparseVector: sparseGenerator.generate(
+              allChunks[i + idx].chunk.content,
+            ),
           }));
           await this.qdrant.addPointsWithSparse(collectionName, hybridPoints);
         } else {
@@ -576,7 +730,8 @@ export class CodeIndexer {
       stats.durationMs = Date.now() - startTime;
       return stats;
     } catch (error) {
-      const errorMessage = error instanceof Error ? error.message : String(error);
+      const errorMessage =
+        error instanceof Error ? error.message : String(error);
       throw new Error(`Incremental re-indexing failed: ${errorMessage}`);
     }
   }

package/src/code/types.ts

@@ -78,8 +78,13 @@ export interface SearchOptions {
   scoreThreshold?: number;
 }
 
+export type IndexingStatus = "not_indexed" | "indexing" | "indexed";
+
 export interface IndexStatus {
+  /** @deprecated Use `status` instead. True only when status is 'indexed'. */
   isIndexed: boolean;
+  /** Current indexing status: 'not_indexed', 'indexing', or 'indexed' */
+  status: IndexingStatus;
   collectionName?: string;
   filesCount?: number;
   chunksCount?: number;

package/src/qdrant/client.test.ts

@@ -634,6 +634,35 @@ describe("QdrantManager", () => {
     });
   });
 
+  describe("deletePointsByFilter", () => {
+    it("should delete points matching filter", async () => {
+      const filter = {
+        must: [{ key: "relativePath", match: { value: "src/test.ts" } }],
+      };
+      await manager.deletePointsByFilter("test-collection", filter);
+
+      expect(mockClient.delete).toHaveBeenCalledWith("test-collection", {
+        wait: true,
+        filter: filter,
+      });
+    });
+
+    it("should delete points with complex filter", async () => {
+      const filter = {
+        must: [
+          { key: "relativePath", match: { value: "src/utils.ts" } },
+          { key: "language", match: { value: "typescript" } },
+        ],
+      };
+      await manager.deletePointsByFilter("test-collection", filter);
+
+      expect(mockClient.delete).toHaveBeenCalledWith("test-collection", {
+        wait: true,
+        filter: filter,
+      });
+    });
+  });
+
   describe("hybridSearch", () => {
     beforeEach(() => {
       mockClient.query = vi.fn();

package/src/qdrant/client.ts

@@ -240,6 +240,20 @@ export class QdrantManager {
     });
   }
 
+  /**
+   * Deletes points matching a filter condition.
+   * Useful for deleting all chunks associated with a specific file path.
+   */
+  async deletePointsByFilter(
+    collectionName: string,
+    filter: Record<string, any>,
+  ): Promise<void> {
+    await this.client.delete(collectionName, {
+      wait: true,
+      filter: filter,
+    });
+  }
+
   /**
    * Performs hybrid search combining semantic vector search with sparse vector (keyword) search
    * using Reciprocal Rank Fusion (RRF) to combine results

package/src/tools/code.ts

@@ -146,7 +146,7 @@ export function registerCodeTools(
     async ({ path }) => {
       const status = await codeIndexer.getIndexStatus(path);
 
-      if (!status.isIndexed) {
+      if (status.status === "not_indexed") {
         return {
           content: [
             {
@@ -157,6 +157,17 @@ export function registerCodeTools(
         };
       }
 
+      if (status.status === "indexing") {
+        return {
+          content: [
+            {
+              type: "text",
+              text: `Codebase at "${path}" is currently being indexed. ${status.chunksCount || 0} chunks processed so far.`,
+            },
+          ],
+        };
+      }
+
       return {
         content: [{ type: "text", text: JSON.stringify(status, null, 2) }],
       };