@grec0/memory-bank-mcp 0.0.3 → 0.0.4

package/dist/common/embeddingService.js

@@ -2,7 +2,6 @@
  * @fileoverview Embedding service for Memory Bank using OpenAI
  * Generates vector embeddings for code chunks
  */
-import { logger } from "./logger.js";
 import OpenAI from "openai";
 import * as crypto from "crypto";
 import * as fs from "fs";
@@ -50,7 +49,7 @@ export class EmbeddingService {
         }
     }
     /**
-     * Saves embedding cache to disk using streams to handle large files
+     * Saves embedding cache to disk
      */
     saveCache() {
         try {
@@ -59,16 +58,8 @@ export class EmbeddingService {
                 fs.mkdirSync(dir, { recursive: true });
             }
             const entries = Array.from(this.cache.values());
-            const stream = fs.createWriteStream(this.options.cachePath, { encoding: 'utf8' });
-            stream.write('[\n');
-            entries.forEach((entry, index) => {
-                const isLast = index === entries.length - 1;
-                const line = JSON.stringify(entry) + (isLast ? '' : ',\n');
-                stream.write(line);
-            });
-            stream.write('\n]');
-            stream.end();
-            console.error(`Updated embedding cache (Total: ${entries.length} entries)`);
+            fs.writeFileSync(this.options.cachePath, JSON.stringify(entries, null, 2));
+            console.error(`Saved ${entries.length} embeddings to cache`);
         }
         catch (error) {
             console.error(`Warning: Could not save embedding cache: ${error}`);
@@ -145,14 +136,14 @@ export class EmbeddingService {
                 // Check if it's a rate limit error
                 if (error?.status === 429 || error?.code === "rate_limit_exceeded") {
                     const backoffMs = Math.pow(2, attempt) * 1000; // Exponential backoff
-                    logger.warn(`Rate limit hit, retrying in ${backoffMs}ms (attempt ${attempt + 1}/${maxRetries})`);
+                    console.error(`Rate limit hit, retrying in ${backoffMs}ms (attempt ${attempt + 1}/${maxRetries})`);
                     await this.sleep(backoffMs);
                     continue;
                 }
                 // Check if it's a temporary error
                 if (error?.status >= 500 && error?.status < 600) {
                     const backoffMs = Math.pow(2, attempt) * 1000;
-                    logger.warn(`Server error ${error.status}, retrying in ${backoffMs}ms (attempt ${attempt + 1}/${maxRetries})`);
+                    console.error(`Server error ${error.status}, retrying in ${backoffMs}ms (attempt ${attempt + 1}/${maxRetries})`);
                     await this.sleep(backoffMs);
                     continue;
                 }
@@ -194,10 +185,9 @@ export class EmbeddingService {
     /**
      * Generates embeddings for multiple chunks in batches
      */
-    async generateBatchEmbeddings(chunks, options = {}) {
+    async generateBatchEmbeddings(chunks) {
         const results = [];
         const toGenerate = [];
-        const shouldSave = options.autoSave !== undefined ? options.autoSave : true;
         // Check cache and collect chunks that need generation
         for (let i = 0; i < chunks.length; i++) {
             const chunk = chunks[i];
@@ -214,43 +204,41 @@ export class EmbeddingService {
                 toGenerate.push({ ...chunk, index: i });
             }
         }
-        if (toGenerate.length > 0) {
-            logger.info(`Generating embeddings: ${toGenerate.length} new, ${chunks.length - toGenerate.length} cached`);
-            // Process in batches
-            for (let i = 0; i < toGenerate.length; i += this.options.batchSize) {
-                const batch = toGenerate.slice(i, i + this.options.batchSize);
-                const batchTexts = batch.map((item) => item.content);
-                logger.info(`Processing batch ${Math.floor(i / this.options.batchSize) + 1}/${Math.ceil(toGenerate.length / this.options.batchSize)}`);
-                try {
-                    const vectors = await this.generateBatchWithRetry(batchTexts);
-                    // Store results and cache
-                    for (let j = 0; j < batch.length; j++) {
-                        const item = batch[j];
-                        const vector = vectors[j];
-                        // Cache the result
-                        this.cacheEmbedding(item.id, item.content, vector);
-                        // Estimate tokens
-                        const tokens = Math.ceil(item.content.length / 4);
-                        results[item.index] = {
-                            chunkId: item.id,
-                            vector,
-                            model: this.options.model,
-                            tokens,
-                        };
-                    }
-                }
-                catch (error) {
-                    logger.error(`Error generating batch embeddings:`, error);
-                    throw error;
-                }
-                // Small delay between batches to avoid rate limits
-                if (i + this.options.batchSize < toGenerate.length) {
-                    await this.sleep(100);
+        console.error(`Generating embeddings: ${toGenerate.length} new, ${chunks.length - toGenerate.length} cached`);
+        // Process in batches
+        for (let i = 0; i < toGenerate.length; i += this.options.batchSize) {
+            const batch = toGenerate.slice(i, i + this.options.batchSize);
+            const batchTexts = batch.map((item) => item.content);
+            console.error(`Processing batch ${Math.floor(i / this.options.batchSize) + 1}/${Math.ceil(toGenerate.length / this.options.batchSize)}`);
+            try {
+                const vectors = await this.generateBatchWithRetry(batchTexts);
+                // Store results and cache
+                for (let j = 0; j < batch.length; j++) {
+                    const item = batch[j];
+                    const vector = vectors[j];
+                    // Cache the result
+                    this.cacheEmbedding(item.id, item.content, vector);
+                    // Estimate tokens
+                    const tokens = Math.ceil(item.content.length / 4);
+                    results[item.index] = {
+                        chunkId: item.id,
+                        vector,
+                        model: this.options.model,
+                        tokens,
+                    };
                 }
             }
+            catch (error) {
+                console.error(`Error generating batch embeddings: ${error}`);
+                throw error;
+            }
+            // Small delay between batches to avoid rate limits
+            if (i + this.options.batchSize < toGenerate.length) {
+                await this.sleep(100);
+            }
         }
         // Save cache after batch processing
-        if (shouldSave && this.options.enableCache && toGenerate.length > 0) {
+        if (this.options.enableCache && toGenerate.length > 0) {
             this.saveCache();
         }
         return results;
@@ -268,7 +256,7 @@ export class EmbeddingService {
             return response.data[0].embedding;
         }
         catch (error) {
-            logger.error(`Error generating query embedding:`, error);
+            console.error(`Error generating query embedding: ${error}`);
             throw error;
         }
     }
@@ -280,7 +268,7 @@ export class EmbeddingService {
         if (fs.existsSync(this.options.cachePath)) {
             fs.unlinkSync(this.options.cachePath);
         }
-        logger.info("Embedding cache cleared");
+        console.error("Embedding cache cleared");
     }
     /**
      * Gets cache statistics

package/dist/common/fileScanner.js

@@ -6,53 +6,120 @@ import * as fs from "fs";
 import * as path from "path";
 import * as crypto from "crypto";
 import ignoreLib from "ignore";
-import { logger } from "./logger.js";
 // Handle ignore library export
 const ignore = typeof ignoreLib === 'function' ? ignoreLib : ignoreLib.default;
 // Language detection by file extension
 const LANGUAGE_MAP = {
+    // TypeScript/JavaScript
     ".ts": "typescript",
     ".tsx": "typescript",
     ".js": "javascript",
     ".jsx": "javascript",
     ".mjs": "javascript",
     ".cjs": "javascript",
+    // Python
     ".py": "python",
+    ".pyi": "python",
+    ".pyw": "python",
+    // JVM Languages
     ".java": "java",
+    ".kt": "kotlin",
+    ".kts": "kotlin",
+    ".scala": "scala",
+    ".groovy": "groovy",
+    ".gradle": "groovy",
+    // C/C++
     ".c": "c",
     ".cpp": "cpp",
     ".cc": "cpp",
     ".cxx": "cpp",
     ".h": "c",
     ".hpp": "cpp",
+    ".hxx": "cpp",
+    // .NET
     ".cs": "csharp",
+    ".fs": "fsharp",
+    ".vb": "vb",
+    // Systems Languages
     ".go": "go",
     ".rs": "rust",
+    // Scripting Languages
     ".rb": "ruby",
     ".php": "php",
-    ".swift": "swift",
-    ".kt": "kotlin",
-    ".kts": "kotlin",
-    ".scala": "scala",
+    ".pl": "perl",
+    ".pm": "perl",
+    ".lua": "lua",
     ".r": "r",
     ".R": "r",
-    ".sql": "sql",
+    // Mobile
+    ".swift": "swift",
+    ".m": "objectivec",
+    ".mm": "objectivec",
+    // Shell
     ".sh": "shell",
     ".bash": "shell",
     ".zsh": "shell",
     ".fish": "shell",
-    ".md": "markdown",
-    ".json": "json",
-    ".yaml": "yaml",
-    ".yml": "yaml",
-    ".xml": "xml",
+    ".ps1": "powershell",
+    ".psm1": "powershell",
+    ".bat": "batch",
+    ".cmd": "batch",
+    // Web
     ".html": "html",
     ".htm": "html",
     ".css": "css",
     ".scss": "scss",
     ".sass": "sass",
+    ".less": "less",
     ".vue": "vue",
     ".svelte": "svelte",
+    ".astro": "astro",
+    // Data/Config
+    ".json": "json",
+    ".jsonc": "json",
+    ".json5": "json",
+    ".yaml": "yaml",
+    ".yml": "yaml",
+    ".toml": "toml",
+    ".xml": "xml",
+    ".ini": "ini",
+    ".cfg": "ini",
+    ".conf": "ini",
+    ".properties": "properties",
+    ".env": "dotenv",
+    ".env.local": "dotenv",
+    ".env.example": "dotenv",
+    // Documentation
+    ".md": "markdown",
+    ".mdx": "markdown",
+    ".rst": "rst",
+    ".txt": "text",
+    // Database
+    ".sql": "sql",
+    ".prisma": "prisma",
+    ".graphql": "graphql",
+    ".gql": "graphql",
+    // Other
+    ".dockerfile": "dockerfile",
+    ".tf": "terraform",
+    ".hcl": "hcl",
+    ".proto": "protobuf",
+    ".sol": "solidity",
+    ".zig": "zig",
+    ".nim": "nim",
+    ".ex": "elixir",
+    ".exs": "elixir",
+    ".erl": "erlang",
+    ".hrl": "erlang",
+    ".clj": "clojure",
+    ".cljs": "clojure",
+    ".cljc": "clojure",
+    ".hs": "haskell",
+    ".elm": "elm",
+    ".dart": "dart",
+    ".v": "v",
+    ".asm": "assembly",
+    ".s": "assembly",
 };
 // Binary file extensions to skip
 const BINARY_EXTENSIONS = new Set([
@@ -77,10 +144,10 @@ export function loadIgnorePatterns(rootPath) {
         try {
             const gitignoreContent = fs.readFileSync(gitignorePath, "utf-8");
             ig.add(gitignoreContent);
-            logger.debug(`Loaded .gitignore patterns from ${gitignorePath}`);
+            console.error(`Loaded .gitignore patterns from ${gitignorePath}`);
         }
         catch (error) {
-            logger.warn(`Could not read .gitignore: ${error}`);
+            console.error(`Warning: Could not read .gitignore: ${error}`);
         }
     }
     // Load .memoryignore if exists
@@ -89,19 +156,19 @@ export function loadIgnorePatterns(rootPath) {
         try {
             const memoryignoreContent = fs.readFileSync(memoryignorePath, "utf-8");
             ig.add(memoryignoreContent);
-            logger.debug(`Loaded .memoryignore patterns from ${memoryignorePath}`);
+            console.error(`Loaded .memoryignore patterns from ${memoryignorePath}`);
         }
         catch (error) {
-            logger.warn(`Could not read .memoryignore: ${error}`);
+            console.error(`Warning: Could not read .memoryignore: ${error}`);
         }
     }
     return ig;
 }
 /**
- * Calculates SHA-256 hash of file content asynchronously
+ * Calculates SHA-256 hash of file content
  */
-export async function calculateFileHash(filePath) {
-    const content = await fs.promises.readFile(filePath);
+export function calculateFileHash(filePath) {
+    const content = fs.readFileSync(filePath);
     return crypto.createHash("sha256").update(content).digest("hex");
 }
 /**
@@ -133,26 +200,39 @@ export function isCodeFile(filePath) {
     // Additional checks for files without extension or special cases
     const basename = path.basename(filePath);
     const codeFileNames = new Set([
+        // Build/DevOps
         "Makefile", "Dockerfile", "Jenkinsfile", "Vagrantfile",
         "Rakefile", "Gemfile", "Podfile", "Fastfile",
+        "CMakeLists.txt", "meson.build", "BUILD", "WORKSPACE",
+        // Config files
+        ".gitignore", ".gitattributes", ".dockerignore",
+        ".editorconfig", ".prettierrc", ".eslintrc",
+        ".babelrc", ".browserslistrc",
+        "tsconfig.json", "jsconfig.json", "package.json",
+        "angular.json", "nest-cli.json", "nx.json",
+        "webpack.config.js", "vite.config.js", "rollup.config.js",
+        // CI/CD
+        ".gitlab-ci.yml", ".travis.yml", "azure-pipelines.yml",
+        "bitbucket-pipelines.yml", "cloudbuild.yaml",
+        // K8s/Helm
+        "Chart.yaml", "values.yaml", "kustomization.yaml",
+        // Lock files (optional - might want to skip these)
+        // "package-lock.json", "yarn.lock", "pnpm-lock.yaml",
     ]);
     return codeFileNames.has(basename);
 }
 /**
- * Recursively scans directory for code files asynchronously
+ * Recursively scans directory for code files
  */
-async function scanDirectoryRecursive(dirPath, rootPath, ig, options, results, concurrencyLimit = 20) {
+function scanDirectoryRecursive(dirPath, rootPath, ig, options, results) {
     let entries;
     try {
-        entries = await fs.promises.readdir(dirPath, { withFileTypes: true });
+        entries = fs.readdirSync(dirPath, { withFileTypes: true });
     }
     catch (error) {
-        logger.warn(`Could not read directory ${dirPath}: ${error}`);
+        console.error(`Warning: Could not read directory ${dirPath}: ${error}`);
         return;
     }
-    // Process subdirectories sequentially to avoid recursion explosion
-    // but process files in current directory in parallel
-    const filesToProcess = [];
     for (const entry of entries) {
         const fullPath = path.join(dirPath, entry.name);
         const relativePath = path.relative(rootPath, fullPath);
@@ -167,35 +247,23 @@ async function scanDirectoryRecursive(dirPath, rootPath, ig, options, results, c
         }
         if (entry.isDirectory()) {
             if (options.recursive) {
-                await scanDirectoryRecursive(fullPath, rootPath, ig, options, results, concurrencyLimit);
+                scanDirectoryRecursive(fullPath, rootPath, ig, options, results);
             }
         }
         else if (entry.isFile()) {
-            filesToProcess.push(entry);
-        }
-    }
-    // Process files in batches
-    for (let i = 0; i < filesToProcess.length; i += concurrencyLimit) {
-        const batch = filesToProcess.slice(i, i + concurrencyLimit);
-        await Promise.all(batch.map(async (entry) => {
-            const fullPath = path.join(dirPath, entry.name);
-            // Calculate relative path from PROJECT root (not just scan root)
-            // And strictly normalize to forward slashes for consistency
-            const relativePathRaw = path.relative(options.projectRoot, fullPath);
-            const relativePath = relativePathRaw.split(path.sep).join("/");
             try {
                 // Check if it's a code file
                 if (!isCodeFile(fullPath)) {
-                    return;
+                    continue;
                 }
-                const stats = await fs.promises.stat(fullPath);
+                const stats = fs.statSync(fullPath);
                 // Check file size limit
                 if (stats.size > options.maxFileSize) {
-                    logger.warn(`Skipping large file (${stats.size} bytes): ${relativePath}`);
-                    return;
+                    console.error(`Skipping large file (${stats.size} bytes): ${relativePath}`);
+                    continue;
                 }
                 // Calculate hash and collect metadata
-                const hash = await calculateFileHash(fullPath);
+                const hash = calculateFileHash(fullPath);
                 const language = detectLanguage(fullPath);
                 const extension = path.extname(fullPath);
                 results.push({
@@ -209,18 +277,17 @@ async function scanDirectoryRecursive(dirPath, rootPath, ig, options, results, c
                 });
             }
             catch (error) {
-                logger.warn(`Could not process file ${fullPath}: ${error}`);
+                console.error(`Warning: Could not process file ${fullPath}: ${error}`);
             }
-        }));
+        }
     }
 }
 /**
- * Scans workspace for code files asynchronously
+ * Scans workspace for code files
  */
-export async function scanFiles(options) {
+export function scanFiles(options) {
     const fullOptions = {
         rootPath: options.rootPath,
-        projectRoot: options.projectRoot || options.rootPath,
         recursive: options.recursive !== undefined ? options.recursive : true,
         includeHidden: options.includeHidden !== undefined ? options.includeHidden : false,
         maxFileSize: options.maxFileSize || 10 * 1024 * 1024, // 10MB default
@@ -229,23 +296,23 @@ export async function scanFiles(options) {
     if (!fs.existsSync(fullOptions.rootPath)) {
         throw new Error(`Root path does not exist: ${fullOptions.rootPath}`);
     }
-    const stats = await fs.promises.stat(fullOptions.rootPath);
+    const stats = fs.statSync(fullOptions.rootPath);
     if (!stats.isDirectory()) {
         throw new Error(`Root path is not a directory: ${fullOptions.rootPath}`);
     }
-    logger.info(`Scanning files in: ${fullOptions.rootPath}`);
+    console.error(`Scanning files in: ${fullOptions.rootPath}`);
     // Load ignore patterns
     const ig = loadIgnorePatterns(fullOptions.rootPath);
     // Scan files
     const results = [];
-    await scanDirectoryRecursive(fullOptions.rootPath, fullOptions.rootPath, ig, fullOptions, results);
-    logger.info(`Found ${results.length} code files to index`);
+    scanDirectoryRecursive(fullOptions.rootPath, fullOptions.rootPath, ig, fullOptions, results);
+    console.error(`Found ${results.length} code files to index`);
     return results;
 }
 /**
  * Scans a single file and returns its metadata
  */
-export async function scanSingleFile(filePath, rootPath, projectRoot) {
+export function scanSingleFile(filePath, rootPath) {
     try {
         if (!fs.existsSync(filePath)) {
             throw new Error(`File does not exist: ${filePath}`);
@@ -257,12 +324,10 @@ export async function scanSingleFile(filePath, rootPath, projectRoot) {
         if (!isCodeFile(filePath)) {
             return null;
         }
-        const hash = await calculateFileHash(filePath);
+        const hash = calculateFileHash(filePath);
         const language = detectLanguage(filePath);
         const extension = path.extname(filePath);
-        // Calculate relative path from PROJECT root
-        const relativePathRaw = path.relative(projectRoot || rootPath, filePath);
-        const relativePath = relativePathRaw.split(path.sep).join("/");
+        const relativePath = path.relative(rootPath, filePath);
         return {
             path: relativePath,
             absolutePath: filePath,
@@ -274,7 +339,7 @@ export async function scanSingleFile(filePath, rootPath, projectRoot) {
         };
     }
     catch (error) {
-        logger.error(`Error scanning file ${filePath}:`, error);
+        console.error(`Error scanning file ${filePath}: ${error}`);
         return null;
     }
 }