@mhalder/qdrant-mcp-server 2.1.0 → 2.1.2

package/src/code/indexer.ts

@@ -24,11 +24,14 @@ import type {
   SearchOptions,
 } from "./types.js";
 
+/** Reserved ID for storing indexing metadata in the collection */
+const INDEXING_METADATA_ID = "__indexing_metadata__";
+
 export class CodeIndexer {
   constructor(
     private qdrant: QdrantManager,
     private embeddings: EmbeddingProvider,
-    private config: CodeConfig
+    private config: CodeConfig,
   ) {}
 
   /**
@@ -59,7 +62,7 @@ export class CodeIndexer {
   async indexCodebase(
     path: string,
     options?: IndexOptions,
-    progressCallback?: ProgressCallback
+    progressCallback?: ProgressCallback,
   ): Promise<IndexStats> {
     const startTime = Date.now();
     const stats: IndexStats = {
@@ -71,9 +74,10 @@ export class CodeIndexer {
       errors: [],
     };
 
-    try {
-      const absolutePath = await this.validatePath(path);
+    const absolutePath = await this.validatePath(path);
+    const collectionName = this.getCollectionName(absolutePath);
 
+    try {
       // 1. Scan files
       progressCallback?.({
         phase: "scanning",
@@ -84,9 +88,11 @@ export class CodeIndexer {
       });
 
       const scanner = new FileScanner({
-        supportedExtensions: options?.extensions || this.config.supportedExtensions,
+        supportedExtensions:
+          options?.extensions || this.config.supportedExtensions,
         ignorePatterns: this.config.ignorePatterns,
-        customIgnorePatterns: options?.ignorePatterns || this.config.customIgnorePatterns,
+        customIgnorePatterns:
+          options?.ignorePatterns || this.config.customIgnorePatterns,
       });
 
       await scanner.loadIgnorePatterns(absolutePath);
@@ -101,8 +107,8 @@ export class CodeIndexer {
       }
 
       // 2. Create or verify collection
-      const collectionName = this.getCollectionName(absolutePath);
-      const collectionExists = await this.qdrant.collectionExists(collectionName);
+      const collectionExists =
+        await this.qdrant.collectionExists(collectionName);
 
       if (options?.forceReindex && collectionExists) {
         await this.qdrant.deleteCollection(collectionName);
@@ -114,10 +120,13 @@ export class CodeIndexer {
           collectionName,
           vectorSize,
           "Cosine",
-          this.config.enableHybridSearch
+          this.config.enableHybridSearch,
         );
       }
 
+      // Store "indexing in progress" marker immediately after collection is ready
+      await this.storeIndexingMarker(collectionName, false);
+
       // 3. Process files and create chunks
       const chunker = new TreeSitterChunker({
         chunkSize: this.config.chunkSize,
@@ -141,7 +150,9 @@ export class CodeIndexer {
 
           // Check for secrets (basic detection)
           if (metadataExtractor.containsSecrets(code)) {
-            stats.errors?.push(`Skipped ${filePath}: potential secrets detected`);
+            stats.errors?.push(
+              `Skipped ${filePath}: potential secrets detected`,
+            );
             continue;
           }
 
@@ -158,7 +169,10 @@ export class CodeIndexer {
             allChunks.push({ chunk, id });
 
             // Check total chunk limit
-            if (this.config.maxTotalChunks && allChunks.length >= this.config.maxTotalChunks) {
+            if (
+              this.config.maxTotalChunks &&
+              allChunks.length >= this.config.maxTotalChunks
+            ) {
               break;
             }
           }
@@ -166,11 +180,15 @@ export class CodeIndexer {
           stats.filesIndexed++;
 
           // Check total chunk limit
-          if (this.config.maxTotalChunks && allChunks.length >= this.config.maxTotalChunks) {
+          if (
+            this.config.maxTotalChunks &&
+            allChunks.length >= this.config.maxTotalChunks
+          ) {
             break;
           }
         } catch (error) {
-          const errorMessage = error instanceof Error ? error.message : String(error);
+          const errorMessage =
+            error instanceof Error ? error.message : String(error);
           stats.errors?.push(`Failed to process ${filePath}: ${errorMessage}`);
         }
       }
@@ -183,12 +201,15 @@ export class CodeIndexer {
         await synchronizer.updateSnapshot(files);
       } catch (error) {
         // Snapshot failure shouldn't fail the entire indexing
-        const errorMessage = error instanceof Error ? error.message : String(error);
+        const errorMessage =
+          error instanceof Error ? error.message : String(error);
         console.error("Failed to save snapshot:", errorMessage);
         stats.errors?.push(`Snapshot save failed: ${errorMessage}`);
       }
 
       if (allChunks.length === 0) {
+        // Still store completion marker even with no chunks
+        await this.storeIndexingMarker(collectionName, true);
         stats.status = "completed";
         stats.durationMs = Date.now() - startTime;
         return stats;
@@ -203,7 +224,8 @@ export class CodeIndexer {
           phase: "embedding",
           current: i + batch.length,
           total: allChunks.length,
-          percentage: 40 + Math.round(((i + batch.length) / allChunks.length) * 30), // 40-70%
+          percentage:
+            40 + Math.round(((i + batch.length) / allChunks.length) * 30), // 40-70%
           message: `Generating embeddings ${i + batch.length}/${allChunks.length}`,
         });
 
@@ -225,7 +247,9 @@ export class CodeIndexer {
               codebasePath: absolutePath,
               chunkIndex: b.chunk.metadata.chunkIndex,
               ...(b.chunk.metadata.name && { name: b.chunk.metadata.name }),
-              ...(b.chunk.metadata.chunkType && { chunkType: b.chunk.metadata.chunkType }),
+              ...(b.chunk.metadata.chunkType && {
+                chunkType: b.chunk.metadata.chunkType,
+              }),
             },
           }));
 
@@ -233,7 +257,8 @@ export class CodeIndexer {
             phase: "storing",
             current: i + batch.length,
             total: allChunks.length,
-            percentage: 70 + Math.round(((i + batch.length) / allChunks.length) * 30), // 70-100%
+            percentage:
+              70 + Math.round(((i + batch.length) / allChunks.length) * 30), // 70-100%
             message: `Storing chunks ${i + batch.length}/${allChunks.length}`,
           });
 
@@ -254,7 +279,9 @@ export class CodeIndexer {
                 codebasePath: absolutePath,
                 chunkIndex: b.chunk.metadata.chunkIndex,
                 ...(b.chunk.metadata.name && { name: b.chunk.metadata.name }),
-                ...(b.chunk.metadata.chunkType && { chunkType: b.chunk.metadata.chunkType }),
+                ...(b.chunk.metadata.chunkType && {
+                  chunkType: b.chunk.metadata.chunkType,
+                }),
               },
             }));
 
@@ -263,16 +290,23 @@ export class CodeIndexer {
             await this.qdrant.addPoints(collectionName, points);
           }
         } catch (error) {
-          const errorMessage = error instanceof Error ? error.message : String(error);
-          stats.errors?.push(`Failed to process batch at index ${i}: ${errorMessage}`);
+          const errorMessage =
+            error instanceof Error ? error.message : String(error);
+          stats.errors?.push(
+            `Failed to process batch at index ${i}: ${errorMessage}`,
+          );
           stats.status = "partial";
         }
       }
 
+      // Store completion marker to indicate indexing is complete
+      await this.storeIndexingMarker(collectionName, true);
+
       stats.durationMs = Date.now() - startTime;
       return stats;
     } catch (error) {
-      const errorMessage = error instanceof Error ? error.message : String(error);
+      const errorMessage =
+        error instanceof Error ? error.message : String(error);
       stats.status = "failed";
       stats.errors?.push(`Indexing failed: ${errorMessage}`);
       stats.durationMs = Date.now() - startTime;
@@ -280,13 +314,62 @@ export class CodeIndexer {
     }
   }
 
+  /**
+   * Store an indexing status marker in the collection.
+   * Called at the start of indexing with complete=false, and at the end with complete=true.
+   */
+  private async storeIndexingMarker(
+    collectionName: string,
+    complete: boolean,
+  ): Promise<void> {
+    try {
+      // Create a dummy vector of zeros (required by Qdrant)
+      const vectorSize = this.embeddings.getDimensions();
+      const zeroVector = new Array(vectorSize).fill(0);
+
+      // Check if collection uses hybrid mode
+      const collectionInfo =
+        await this.qdrant.getCollectionInfo(collectionName);
+
+      const payload = {
+        _type: "indexing_metadata",
+        indexingComplete: complete,
+        ...(complete
+          ? { completedAt: new Date().toISOString() }
+          : { startedAt: new Date().toISOString() }),
+      };
+
+      if (collectionInfo.hybridEnabled) {
+        await this.qdrant.addPointsWithSparse(collectionName, [
+          {
+            id: INDEXING_METADATA_ID,
+            vector: zeroVector,
+            sparseVector: { indices: [], values: [] },
+            payload,
+          },
+        ]);
+      } else {
+        await this.qdrant.addPoints(collectionName, [
+          {
+            id: INDEXING_METADATA_ID,
+            vector: zeroVector,
+            payload,
+          },
+        ]);
+      }
+    } catch (error) {
+      // Non-fatal: log but don't fail the indexing
+      console.error("Failed to store indexing marker:", error);
+    }
+  }
+
   /**
    * Search code semantically
    */
   async searchCode(
     path: string,
     query: string,
-    options?: SearchOptions
+    options?: SearchOptions,
   ): Promise<CodeSearchResult[]> {
     const absolutePath = await this.validatePath(path);
     const collectionName = this.getCollectionName(absolutePath);
@@ -300,7 +383,8 @@ export class CodeIndexer {
     // Check if collection has hybrid search enabled
     const collectionInfo = await this.qdrant.getCollectionInfo(collectionName);
     const useHybrid =
-      (options?.useHybrid ?? this.config.enableHybridSearch) && collectionInfo.hybridEnabled;
+      (options?.useHybrid ?? this.config.enableHybridSearch) &&
+      collectionInfo.hybridEnabled;
 
     // Generate query embedding
     const { embedding } = await this.embeddings.embed(query);
@@ -342,14 +426,14 @@ export class CodeIndexer {
         embedding,
         sparseVector,
         options?.limit || this.config.defaultSearchLimit,
-        filter
+        filter,
       );
     } else {
       results = await this.qdrant.search(
         collectionName,
         embedding,
         options?.limit || this.config.defaultSearchLimit,
-        filter
+        filter,
       );
     }
 
@@ -379,24 +463,75 @@ export class CodeIndexer {
     const exists = await this.qdrant.collectionExists(collectionName);
 
     if (!exists) {
-      return { isIndexed: false };
+      return { isIndexed: false, status: "not_indexed" };
     }
 
+    // Check for indexing marker in Qdrant (persisted across instances)
+    const indexingMarker = await this.qdrant.getPoint(
+      collectionName,
+      INDEXING_METADATA_ID,
+    );
     const info = await this.qdrant.getCollectionInfo(collectionName);
 
+    // Check marker status
+    const isComplete = indexingMarker?.payload?.indexingComplete === true;
+    const isInProgress = indexingMarker?.payload?.indexingComplete === false;
+
+    // Subtract 1 from points count if marker exists (metadata point doesn't count as a chunk)
+    const actualChunksCount = indexingMarker
+      ? Math.max(0, info.pointsCount - 1)
+      : info.pointsCount;
+
+    if (isInProgress) {
+      // Indexing in progress - marker exists with indexingComplete=false
+      return {
+        isIndexed: false,
+        status: "indexing",
+        collectionName,
+        chunksCount: actualChunksCount,
+      };
+    }
+
+    if (isComplete) {
+      // Indexing completed - marker exists with indexingComplete=true
+      return {
+        isIndexed: true,
+        status: "indexed",
+        collectionName,
+        chunksCount: actualChunksCount,
+        lastUpdated: indexingMarker.payload?.completedAt
+          ? new Date(indexingMarker.payload.completedAt)
+          : undefined,
+      };
+    }
+
+    // Legacy collection (no marker) - check if it has content
+    // If it has chunks, assume it's indexed (backwards compatibility)
+    if (actualChunksCount > 0) {
+      return {
+        isIndexed: true,
+        status: "indexed",
+        collectionName,
+        chunksCount: actualChunksCount,
+      };
+    }
+
+    // Collection exists but no chunks and no marker - not indexed
     return {
-      isIndexed: true,
+      isIndexed: false,
+      status: "not_indexed",
       collectionName,
-      chunksCount: info.pointsCount,
-      // TODO: Extract unique languages and file count from collection
-      // This would require scrolling through points or maintaining separate metadata
+      chunksCount: 0,
     };
   }
 
   /**
    * Incrementally re-index only changed files
    */
-  async reindexChanges(path: string, progressCallback?: ProgressCallback): Promise<ChangeStats> {
+  async reindexChanges(
+    path: string,
+    progressCallback?: ProgressCallback,
+  ): Promise<ChangeStats> {
     const startTime = Date.now();
     const stats: ChangeStats = {
       filesAdded: 0,
@@ -422,7 +557,9 @@ export class CodeIndexer {
       const hasSnapshot = await synchronizer.initialize();
 
       if (!hasSnapshot) {
-        throw new Error("No previous snapshot found. Use index_codebase for initial indexing.");
+        throw new Error(
+          "No previous snapshot found. Use index_codebase for initial indexing.",
+        );
       }
 
       // Scan current files
@@ -449,7 +586,11 @@ export class CodeIndexer {
       stats.filesModified = changes.modified.length;
       stats.filesDeleted = changes.deleted.length;
 
-      if (stats.filesAdded === 0 && stats.filesModified === 0 && stats.filesDeleted === 0) {
+      if (
+        stats.filesAdded === 0 &&
+        stats.filesModified === 0 &&
+        stats.filesDeleted === 0
+      ) {
         stats.durationMs = Date.now() - startTime;
         return stats;
       }
@@ -526,7 +667,8 @@ export class CodeIndexer {
           phase: "embedding",
           current: i + batch.length,
           total: allChunks.length,
-          percentage: 40 + Math.round(((i + batch.length) / allChunks.length) * 30),
+          percentage:
+            40 + Math.round(((i + batch.length) / allChunks.length) * 30),
           message: `Generating embeddings ${i + batch.length}/${allChunks.length}`,
         });
 
@@ -546,7 +688,9 @@ export class CodeIndexer {
             codebasePath: absolutePath,
             chunkIndex: b.chunk.metadata.chunkIndex,
             ...(b.chunk.metadata.name && { name: b.chunk.metadata.name }),
-            ...(b.chunk.metadata.chunkType && { chunkType: b.chunk.metadata.chunkType }),
+            ...(b.chunk.metadata.chunkType && {
+              chunkType: b.chunk.metadata.chunkType,
+            }),
           },
         }));
 
@@ -554,7 +698,8 @@ export class CodeIndexer {
           phase: "storing",
           current: i + batch.length,
           total: allChunks.length,
-          percentage: 70 + Math.round(((i + batch.length) / allChunks.length) * 30),
+          percentage:
+            70 + Math.round(((i + batch.length) / allChunks.length) * 30),
           message: `Storing chunks ${i + batch.length}/${allChunks.length}`,
         });
 
@@ -562,7 +707,9 @@ export class CodeIndexer {
           const sparseGenerator = new BM25SparseVectorGenerator();
           const hybridPoints = points.map((point, idx) => ({
             ...point,
-            sparseVector: sparseGenerator.generate(allChunks[i + idx].chunk.content),
+            sparseVector: sparseGenerator.generate(
+              allChunks[i + idx].chunk.content,
+            ),
           }));
           await this.qdrant.addPointsWithSparse(collectionName, hybridPoints);
         } else {
@@ -576,7 +723,8 @@ export class CodeIndexer {
       stats.durationMs = Date.now() - startTime;
       return stats;
     } catch (error) {
-      const errorMessage = error instanceof Error ? error.message : String(error);
+      const errorMessage =
+        error instanceof Error ? error.message : String(error);
       throw new Error(`Incremental re-indexing failed: ${errorMessage}`);
     }
   }

package/src/code/types.ts

@@ -78,8 +78,13 @@ export interface SearchOptions {
   scoreThreshold?: number;
 }
 
+export type IndexingStatus = "not_indexed" | "indexing" | "indexed";
+
 export interface IndexStatus {
+  /** @deprecated Use `status` instead. True only when status is 'indexed'. */
   isIndexed: boolean;
+  /** Current indexing status: 'not_indexed', 'indexing', or 'indexed' */
+  status: IndexingStatus;
   collectionName?: string;
   filesCount?: number;
   chunksCount?: number;