@grec0/memory-bank-mcp 0.0.3 → 0.0.4

package/dist/common/indexManager.js

@@ -6,8 +6,6 @@ import * as fs from "fs";
 import * as path from "path";
 import { scanFiles, scanSingleFile } from "./fileScanner.js";
 import { chunkCode } from "./chunker.js";
-import { logger } from "./logger.js";
-import * as crypto from "crypto";
 /**
  * Index manager coordinating the entire indexing pipeline
  */
@@ -16,21 +14,29 @@ export class IndexManager {
     vectorStore;
     metadataPath;
     metadata;
-    projectRoot;
-    projectId;
-    constructor(embeddingService, vectorStore, storagePath = ".memorybank", projectRoot) {
+    projectKnowledgeService = null;
+    autoUpdateDocs = false;
+    constructor(embeddingService, vectorStore, storagePath = ".memorybank") {
         this.embeddingService = embeddingService;
         this.vectorStore = vectorStore;
         this.metadataPath = path.join(storagePath, "index-metadata.json");
-        this.projectRoot = projectRoot || process.cwd();
-        this.projectId = this.generateProjectId(this.projectRoot);
         this.metadata = this.loadMetadata();
+        // Check if auto-update docs is enabled via environment variable
+        this.autoUpdateDocs = process.env.MEMORYBANK_AUTO_UPDATE_DOCS === "true";
     }
     /**
-     * Generates a unique project ID from the project root path
+     * Sets the Project Knowledge Service for auto-generating docs
      */
-    generateProjectId(projectRoot) {
-        return crypto.createHash("sha256").update(projectRoot).digest("hex").substring(0, 16);
+    setProjectKnowledgeService(service) {
+        this.projectKnowledgeService = service;
+        console.error("Project Knowledge Service attached to Index Manager");
+    }
+    /**
+     * Enables or disables auto-update of project docs after indexing
+     */
+    setAutoUpdateDocs(enabled) {
+        this.autoUpdateDocs = enabled;
+        console.error(`Auto-update project docs: ${enabled ? "enabled" : "disabled"}`);
     }
     /**
      * Loads index metadata from disk
@@ -43,7 +49,7 @@ export class IndexManager {
             }
         }
         catch (error) {
-            logger.warn(`Could not load index metadata: ${error}`);
+            console.error(`Warning: Could not load index metadata: ${error}`);
         }
         return {
             version: "1.0",
@@ -63,7 +69,7 @@ export class IndexManager {
             fs.writeFileSync(this.metadataPath, JSON.stringify(this.metadata, null, 2));
         }
         catch (error) {
-            logger.warn(`Could not save index metadata: ${error}`);
+            console.error(`Warning: Could not save index metadata: ${error}`);
         }
     }
     /**
@@ -85,67 +91,64 @@ export class IndexManager {
     /**
      * Indexes a single file
      */
-    async indexFile(file, forceReindex = false, saveMetadata = true) {
+    async indexFile(file, forceReindex = false, projectId = "default") {
         try {
             // Check if file needs reindexing
             if (!this.needsReindexing(file, forceReindex)) {
-                logger.debug(`Skipping ${file.path} (no changes)`);
+                console.error(`Skipping ${file.path} (no changes)`);
                 return { chunksCreated: 0 };
             }
-            logger.info(`Indexing: ${file.path}`);
+            console.error(`Indexing: ${file.path}`);
             // Read file content
             const content = fs.readFileSync(file.absolutePath, "utf-8");
-            // Get chunk size from environment or use defaults
-            const maxChunkSize = parseInt(process.env.MEMORYBANK_CHUNK_SIZE || "1000");
-            const chunkOverlap = parseInt(process.env.MEMORYBANK_CHUNK_OVERLAP || "200");
-            // Chunk the code
+            // Get token limits from environment or use defaults
+            // text-embedding-3-small has 8192 token limit, default to 7500 for safety
+            const maxTokens = parseInt(process.env.MEMORYBANK_MAX_TOKENS || "7500");
+            const chunkOverlapTokens = parseInt(process.env.MEMORYBANK_CHUNK_OVERLAP_TOKENS || "200");
+            // Chunk the code using token-based chunking
             const chunks = chunkCode({
                 filePath: file.path,
                 content,
                 language: file.language,
-                maxChunkSize,
-                chunkOverlap,
+                maxTokens,
+                chunkOverlapTokens,
             });
             if (chunks.length === 0) {
-                logger.warn(`No chunks created for ${file.path}`);
-                return { chunksCreated: 0 };
-            }
-            logger.debug(`  Created ${chunks.length} chunks`);
-            // Filter out invalid chunks (fail-safe)
-            const validChunks = chunks.filter(c => c.content && c.content.trim().length > 0 && c.content.trim() !== "}");
-            if (validChunks.length === 0) {
-                logger.warn(`No valid chunks after filtering for ${file.path}`);
+                console.error(`Warning: No chunks created for ${file.path}`);
                 return { chunksCreated: 0 };
             }
+            console.error(`  Created ${chunks.length} chunks`);
             // Generate embeddings
-            const embeddingInputs = validChunks.map((chunk) => ({
+            const embeddingInputs = chunks.map((chunk) => ({
                 id: chunk.id,
                 content: chunk.content,
             }));
             const embeddings = await this.embeddingService.generateBatchEmbeddings(embeddingInputs);
-            logger.debug(`  Generated ${embeddings.length} embeddings`);
-            // Prepare chunk records for storage
+            console.error(`  Generated ${embeddings.length} embeddings`);
+            // Prepare chunk records for storage (using snake_case for LanceDB)
+            // Note: All fields must have non-undefined values for LanceDB Arrow conversion
             const timestamp = Date.now();
-            const chunkRecords = validChunks.map((chunk, i) => ({
+            console.error(`  Storing chunks with project_id: '${projectId}'`);
+            const chunkRecords = chunks.map((chunk, i) => ({
                 id: chunk.id,
                 vector: embeddings[i].vector,
-                filePath: chunk.filePath,
+                file_path: chunk.filePath,
                 content: chunk.content,
-                startLine: chunk.startLine,
-                endLine: chunk.endLine,
-                chunkType: chunk.chunkType,
-                name: chunk.name || "",
+                start_line: chunk.startLine,
+                end_line: chunk.endLine,
+                chunk_type: chunk.chunkType,
+                name: chunk.name || "", // Ensure non-undefined for LanceDB
                 language: chunk.language,
-                fileHash: file.hash,
+                file_hash: file.hash,
                 timestamp,
-                context: chunk.context,
-                projectId: this.projectId,
+                context: chunk.context || "", // Ensure non-undefined for LanceDB
+                project_id: projectId,
             }));
             // Delete old chunks for this file
-            await this.vectorStore.deleteChunksByFile(file.path, this.projectId);
+            await this.vectorStore.deleteChunksByFile(file.path);
             // Insert new chunks
             await this.vectorStore.insertChunks(chunkRecords);
-            logger.debug(`  Stored ${chunkRecords.length} chunks in vector store`);
+            console.error(`  Stored ${chunkRecords.length} chunks in vector store`);
             // Update metadata
             this.metadata.files[file.path] = {
                 hash: file.hash,
@@ -153,38 +156,59 @@ export class IndexManager {
                 chunkCount: chunks.length,
             };
             this.metadata.lastIndexed = timestamp;
-            if (saveMetadata) {
-                this.saveMetadata();
-            }
+            this.saveMetadata();
             return { chunksCreated: chunks.length };
         }
         catch (error) {
             const errorMsg = `Error indexing ${file.path}: ${error}`;
-            logger.error(errorMsg);
+            console.error(errorMsg);
             return { chunksCreated: 0, error: errorMsg };
         }
     }
+    /**
+     * Derives a project ID from the root path if not provided
+     */
+    deriveProjectId(rootPath, providedId) {
+        if (providedId) {
+            return providedId;
+        }
+        // Use the directory name as project ID
+        const dirName = path.basename(path.resolve(rootPath));
+        // Sanitize: remove special chars, lowercase, replace spaces with dashes
+        const sanitized = dirName
+            .toLowerCase()
+            .replace(/[^a-z0-9-_]/g, "-")
+            .replace(/-+/g, "-")
+            .replace(/^-|-$/g, "");
+        return sanitized || "default";
+    }
     /**
      * Indexes multiple files or a directory
      */
     async indexFiles(options) {
         const startTime = Date.now();
-        logger.info(`=== Starting indexing process ===`);
-        logger.info(`Root path: ${options.rootPath}`);
-        logger.info(`Force reindex: ${options.forceReindex || false}`);
+        const projectId = this.deriveProjectId(options.rootPath, options.projectId);
+        const shouldAutoUpdateDocs = options.autoUpdateDocs !== undefined
+            ? options.autoUpdateDocs
+            : this.autoUpdateDocs;
+        console.error(`\n=== Starting indexing process ===`);
+        console.error(`Root path: ${options.rootPath}`);
+        console.error(`Project ID: ${projectId}`);
+        console.error(`Force reindex: ${options.forceReindex || false}`);
+        console.error(`Auto-update docs: ${shouldAutoUpdateDocs}`);
         // Initialize vector store
         await this.vectorStore.initialize();
         // Scan files
-        logger.info(`Scanning files...`);
-        const files = await scanFiles({
+        console.error(`\nScanning files...`);
+        const files = scanFiles({
             rootPath: options.rootPath,
-            projectRoot: options.projectRoot,
             recursive: options.recursive !== undefined ? options.recursive : true,
         });
         if (files.length === 0) {
-            logger.warn("No files found to index");
+            console.error("No files found to index");
             return {
                 filesProcessed: 0,
+                changedFiles: [],
                 chunksCreated: 0,
                 errors: [],
                 duration: Date.now() - startTime,
@@ -192,69 +216,76 @@ export class IndexManager {
         }
         // Filter files that need reindexing
         const filesToIndex = files.filter((file) => this.needsReindexing(file, options.forceReindex || false));
-        logger.info(`Found ${files.length} files, ${filesToIndex.length} need indexing`);
+        console.error(`\nFound ${files.length} files, ${filesToIndex.length} need indexing`);
         if (filesToIndex.length === 0) {
-            logger.info("All files are up to date");
+            console.error("All files are up to date");
             return {
                 filesProcessed: 0,
+                changedFiles: [],
                 chunksCreated: 0,
                 errors: [],
                 duration: Date.now() - startTime,
             };
         }
-        // Index files in batches
+        // Index files
         const errors = [];
+        const changedFiles = [];
         let totalChunks = 0;
         let processedFiles = 0;
-        const batchSize = 5; // Concurrency limit
-        for (let i = 0; i < filesToIndex.length; i += batchSize) {
-            const batch = filesToIndex.slice(i, i + batchSize);
-            const batchNum = Math.floor(i / batchSize) + 1;
-            const totalBatches = Math.ceil(filesToIndex.length / batchSize);
-            logger.info(`Processing batch ${batchNum}/${totalBatches} (${batch.length} files)`);
-            const batchPromises = batch.map(async (file, index) => {
-                logger.debug(`[${i + index + 1}/${filesToIndex.length}] Processing ${file.path}`);
-                return this.indexFile(file, options.forceReindex || false, false); // Don't save metadata per file
-            });
-            const results = await Promise.all(batchPromises);
-            // Process results
-            for (const result of results) {
-                if (result.error) {
-                    errors.push(result.error);
-                }
-                else {
-                    processedFiles++;
-                    totalChunks += result.chunksCreated;
-                }
+        for (let i = 0; i < filesToIndex.length; i++) {
+            const file = filesToIndex[i];
+            console.error(`\n[${i + 1}/${filesToIndex.length}] Processing ${file.path}`);
+            const result = await this.indexFile(file, options.forceReindex || false, projectId);
+            if (result.error) {
+                errors.push(result.error);
             }
-            // Save metadata and embedding cache after each batch
-            this.saveMetadata();
-            this.embeddingService.saveCache();
-            // Small delay between batches
-            if (i + batchSize < filesToIndex.length) {
-                await new Promise(resolve => setTimeout(resolve, 100));
+            else {
+                processedFiles++;
+                totalChunks += result.chunksCreated;
+                changedFiles.push(file.path);
+            }
+        }
+        const indexDuration = Date.now() - startTime;
+        console.error(`\n=== Indexing complete ===`);
+        console.error(`Files processed: ${processedFiles}`);
+        console.error(`Chunks created: ${totalChunks}`);
+        console.error(`Errors: ${errors.length}`);
+        console.error(`Duration: ${(indexDuration / 1000).toFixed(2)}s`);
+        // Run post-indexing hook to update project documentation
+        let docsGeneration;
+        if (shouldAutoUpdateDocs && this.projectKnowledgeService && changedFiles.length > 0) {
+            console.error(`\n=== Updating project documentation ===`);
+            try {
+                // Get all chunks for the project
+                const allChunks = await this.vectorStore.getAllChunks(projectId);
+                // Update docs incrementally based on changed files
+                docsGeneration = await this.projectKnowledgeService.updateDocuments(allChunks, changedFiles);
+                console.error(`Docs updated: ${docsGeneration.documentsUpdated.length}`);
+                console.error(`Docs generated: ${docsGeneration.documentsGenerated.length}`);
+                console.error(`Reasoning tokens: ${docsGeneration.totalReasoningTokens}`);
+            }
+            catch (error) {
+                console.error(`Warning: Failed to update project docs: ${error.message}`);
+                errors.push(`Project docs update failed: ${error.message}`);
             }
         }
-        const duration = Date.now() - startTime;
-        logger.info(`=== Indexing complete ===`);
-        logger.info(`Files processed: ${processedFiles}`);
-        logger.info(`Chunks created: ${totalChunks}`);
-        logger.info(`Errors: ${errors.length}`);
-        logger.info(`Duration: ${(duration / 1000).toFixed(2)}s`);
+        const totalDuration = Date.now() - startTime;
         return {
             filesProcessed: processedFiles,
+            changedFiles,
             chunksCreated: totalChunks,
             errors,
-            duration,
+            duration: totalDuration,
+            docsGeneration,
         };
     }
     /**
      * Re-indexes a specific file by path
      */
-    async reindexFile(filePath, rootPath, projectRoot) {
+    async reindexFile(filePath, rootPath, projectId) {
         try {
             // Scan the specific file
-            const file = await scanSingleFile(filePath, rootPath, projectRoot);
+            const file = scanSingleFile(filePath, rootPath);
             if (!file) {
                 return {
                     success: false,
@@ -262,10 +293,12 @@ export class IndexManager {
                     error: "File not found or not a code file",
                 };
             }
+            // Derive project ID from root path if not provided
+            const resolvedProjectId = this.deriveProjectId(rootPath, projectId);
             // Initialize vector store
             await this.vectorStore.initialize();
             // Index the file
-            const result = await this.indexFile(file, true);
+            const result = await this.indexFile(file, true, resolvedProjectId);
             if (result.error) {
                 return {
                     success: false,
@@ -325,11 +358,11 @@ export class IndexManager {
         });
         // Format results
         return results.map((result) => ({
-            filePath: result.chunk.filePath,
+            filePath: result.chunk.file_path,
             content: result.chunk.content,
-            startLine: result.chunk.startLine,
-            endLine: result.chunk.endLine,
-            chunkType: result.chunk.chunkType,
+            startLine: result.chunk.start_line,
+            endLine: result.chunk.end_line,
+            chunkType: result.chunk.chunk_type,
             name: result.chunk.name,
             language: result.chunk.language,
             score: result.score,
@@ -349,23 +382,23 @@ export class IndexManager {
         this.saveMetadata();
         // Clear embedding cache
         this.embeddingService.clearCache();
-        logger.info("Index cleared");
+        console.error("Index cleared");
     }
     /**
      * Removes a file from the index
      */
     async removeFile(filePath) {
         await this.vectorStore.initialize();
-        await this.vectorStore.deleteChunksByFile(filePath, this.projectId);
+        await this.vectorStore.deleteChunksByFile(filePath);
         delete this.metadata.files[filePath];
         this.saveMetadata();
-        logger.info(`Removed ${filePath} from index`);
+        console.error(`Removed ${filePath} from index`);
     }
 }
 /**
  * Creates an index manager from environment variables
  */
-export function createIndexManager(embeddingService, vectorStore, workspaceRoot) {
+export function createIndexManager(embeddingService, vectorStore) {
     const storagePath = process.env.MEMORYBANK_STORAGE_PATH || ".memorybank";
-    return new IndexManager(embeddingService, vectorStore, storagePath, workspaceRoot);
+    return new IndexManager(embeddingService, vectorStore, storagePath);
 }