@mrxkun/mcfast-mcp 4.1.9 → 4.1.11

package/src/memory/memory-engine.js

@@ -26,6 +26,9 @@ import { CuratedMemory } from './layers/curated-memory.js';
 // Bootstrap
 import { AgentsMdBootstrap } from './bootstrap/agents-md.js';
 
+// Project Analysis (AI-powered)
+import { execute as projectAnalyzeExecute } from '../tools/project_analyze.js';
+
 // Embedders
 import { UltraEnhancedEmbedder } from './utils/ultra-embedder.js';
 import { SmartRouter } from './utils/smart-router.js';
@@ -38,6 +41,7 @@ import { PatternDetector, SuggestionEngine, StrategySelector } from '../intellig
 export class MemoryEngine {
   static instance = null;
   static instancePromise = null;
+  static instancePid = null; // Track which process owns the instance
   
   /**
    * Get singleton instance of MemoryEngine
@@ -45,8 +49,19 @@ export class MemoryEngine {
    * @returns {MemoryEngine} Singleton instance
    */
   static getInstance(options = {}) {
+    const currentPid = process.pid;
+    
+    // Check if instance belongs to current process
+    if (MemoryEngine.instancePid !== currentPid) {
+      // Different process - reset instance
+      MemoryEngine.instance = null;
+      MemoryEngine.instancePromise = null;
+      MemoryEngine.instancePid = null;
+    }
+    
     if (!MemoryEngine.instance) {
       MemoryEngine.instance = new MemoryEngine(options);
+      MemoryEngine.instancePid = currentPid;
     }
     return MemoryEngine.instance;
   }
@@ -58,6 +73,16 @@ export class MemoryEngine {
    * @returns {Promise<MemoryEngine>} Initialized singleton instance
    */
   static async getOrCreate(projectPath, options = {}) {
+    const currentPid = process.pid;
+    
+    // Check if instance belongs to current process
+    if (MemoryEngine.instancePid !== currentPid) {
+      // Different process - reset instance
+      MemoryEngine.instance = null;
+      MemoryEngine.instancePromise = null;
+      MemoryEngine.instancePid = null;
+    }
+    
     if (MemoryEngine.instance && MemoryEngine.instance.isInitialized) {
       return MemoryEngine.instance;
     }
@@ -68,6 +93,7 @@ export class MemoryEngine {
     
     const engine = new MemoryEngine(options);
     MemoryEngine.instance = engine;
+    MemoryEngine.instancePid = currentPid;
     MemoryEngine.instancePromise = engine.initialize(projectPath).then(() => engine);
     
     return MemoryEngine.instancePromise;
@@ -131,6 +157,9 @@ export class MemoryEngine {
     this.suggestionEngine = null;
     this.strategySelector = null;
     
+    // Auto-analyze project on first use
+    this.autoAnalyze = options.autoAnalyze !== false;
+    
     // Search configuration
     this.searchConfig = {
       hybrid: {
@@ -208,7 +237,13 @@ export class MemoryEngine {
         '**/*.log'
       ]
     });
-    await this.watcher.start();
+    
+    try {
+      await this.watcher.start();
+    } catch (watcherError) {
+      console.error('[MemoryEngine] ⚠️ File watcher failed to start (continuing without file watching):', watcherError.message);
+      this.watcher = null;
+    }
     
     // Perform initial scan
     await this.performInitialScan();
@@ -223,6 +258,9 @@ export class MemoryEngine {
     console.error(`[MemoryEngine] Smart Routing: ${this.smartRoutingEnabled ? 'ENABLED' : 'DISABLED'}`);
     console.error(`[MemoryEngine] Intelligence: ${this.intelligenceEnabled ? 'ENABLED' : 'DISABLED'}`);
     
+    // Auto-analyze project if MCFAST_TOKEN is set
+    await this.autoAnalyzeProject();
+    
     // Log initialization to daily logs
     await this.dailyLogs.log('Memory Engine Initialized', `Project: ${projectPath}`, {
       stats: this.getStats()
@@ -264,6 +302,47 @@ export class MemoryEngine {
     }
   }
 
+  /**
+   * Auto-analyze project using AI if MCFAST_TOKEN is set
+   * Runs in background to not block initialization
+   */
+  async autoAnalyzeProject() {
+    if (!this.autoAnalyze) return;
+    
+    const apiKey = process.env.MCFAST_TOKEN;
+    if (!apiKey) {
+      console.error('[MemoryEngine] Skipping auto-analysis (no MCFAST_TOKEN)');
+      return;
+    }
+    
+    // Check if already analyzed (MEMORY.md has Project Context section)
+    try {
+      const memoryContent = await this.curatedMemory.read();
+      if (memoryContent && memoryContent.includes('## Project Context') && 
+          !memoryContent.includes('<!-- Add project context here -->')) {
+        console.error('[MemoryEngine] Project already analyzed, skipping auto-analysis');
+        return;
+      }
+    } catch (e) {
+      // MEMORY.md doesn't exist yet, proceed with analysis
+    }
+    
+    // Run analysis in background
+    console.error('[MemoryEngine] 🔄 Starting auto-analysis in background...');
+    
+    projectAnalyzeExecute({ force: false, updateMemory: true })
+      .then(result => {
+        if (result.metadata?.memoryUpdated) {
+          console.error('[MemoryEngine] ✅ Auto-analysis complete, MEMORY.md updated');
+        } else if (result.isError) {
+          console.error('[MemoryEngine] ⚠️ Auto-analysis failed:', result.content?.[0]?.text?.substring(0, 100));
+        }
+      })
+      .catch(err => {
+        console.error('[MemoryEngine] ⚠️ Auto-analysis error:', err.message);
+      });
+  }
+
   async performInitialScan() {
     if (this.isScanning) return;
     this.isScanning = true;
@@ -470,24 +549,44 @@ export class MemoryEngine {
 
   // ========== Storage Operations ==========
 
-  async storeIndexed(indexed) {
-    this.codebaseDb?.upsertFile?.(indexed.file);
-    this.codebaseDb?.deleteFactsByFile?.(indexed.file.id);
-    this.codebaseDb?.deleteChunksByFile?.(indexed.file.id);
+  async storeIndexed(indexed, maxRetries = 3) {
+    // Retry logic with exponential backoff
+    for (let attempt = 1; attempt <= maxRetries; attempt++) {
+      try {
+        this.codebaseDb?.upsertFile?.(indexed.file);
+        this.codebaseDb?.deleteFactsByFile?.(indexed.file.id);
+        this.codebaseDb?.deleteChunksByFile?.(indexed.file.id);
 
-    for (const fact of indexed.facts) {
-      this.codebaseDb?.insertFact?.(fact);
-    }
+        for (const fact of indexed.facts) {
+          this.codebaseDb?.insertFact?.(fact);
+        }
 
-    for (const chunk of indexed.chunks) {
-      this.codebaseDb?.insertChunk?.(chunk);
-    }
+        for (const chunk of indexed.chunks) {
+          this.codebaseDb?.insertChunk?.(chunk);
+        }
 
-    for (const embedding of indexed.embeddings) {
-      this.codebaseDb?.insertEmbedding?.(embedding);
+        for (const embedding of indexed.embeddings) {
+          this.codebaseDb?.insertEmbedding?.(embedding);
+        }
+
+        return true; // Success
+      } catch (error) {
+        if (attempt === maxRetries) {
+          console.error(`[MemoryEngine] Failed to store indexed data after ${maxRetries} attempts:`, error.message);
+          this.codebaseDb?.recordFailedIndex?.(indexed.file.path, error.message);
+          return false;
+        }
+        // Exponential backoff: 100ms, 200ms, 400ms
+        await new Promise(resolve => setTimeout(resolve, 100 * Math.pow(2, attempt - 1)));
+      }
     }
   }
 
+  // Record failed indexing for tracking
+  recordFailedIndex(filePath, errorMessage) {
+    console.error(`[MemoryEngine] Indexing failed for ${filePath}: ${errorMessage}`);
+  }
+
   // ========== Search Operations ==========
 
   async searchFacts(query, limit = 20) {
@@ -573,6 +672,80 @@ export class MemoryEngine {
   }
 
   async searchCodebase(query, limit = 20) {
+    const startTime = performance.now();
+
+    // FIX: Get embeddings from both codebaseDb AND memoryDb
+    const [codebaseEmbeddings, memoryEmbeddings] = await Promise.all([
+      this.codebaseDb?.getAllEmbeddings?.() || [],
+      this.memoryDb?.getAllEmbeddings?.() || []
+    ]);
+
+    // Generate query embedding
+    const queryResult = this.embedder.embedCode(query);
+    const queryEmbedding = queryResult.vector;
+
+    // Score and tag results from each source
+    const scoreFn = (item) => {
+      const buf = item.embedding?.buffer || item.embedding;
+      const arr = buf ? new Uint8Array(buf) : new Uint8Array(0);
+      return this.embedder.cosineSimilarity(queryEmbedding, Array.from(arr));
+    };
+
+    const codebaseResults = (codebaseEmbeddings || []).map(item => ({
+      ...item, similarity: scoreFn(item), source: 'codebase', type: 'code'
+    }));
+
+    const memoryResults = (memoryEmbeddings || []).map(item => ({
+      ...item, similarity: scoreFn(item), source: 'memory', type: 'memory'
+    }));
+
+    // Combine results
+    let combined = [...codebaseResults, ...memoryResults];
+
+    // Also get FTS results from both databases
+    const [codebaseFts, memoryFts] = await Promise.all([
+      this.codebaseDb?.searchFTS?.(query, limit * 2) || { results: [] },
+      this.memoryDb?.searchFTS?.(query, limit * 2) || { results: [] }
+    ]);
+
+    // Add FTS results
+    for (const r of codebaseFts.results || []) {
+      combined.push({ ...r, source: 'fts', type: 'code', score: r.score });
+    }
+    for (const r of memoryFts.results || []) {
+      combined.push({ ...r, source: 'fts', type: 'memory', score: r.score });
+    }
+
+    // Deduplicate and sort
+    const seen = new Map();
+    for (const r of combined) {
+      const key = r.chunk_id || r.id;
+      if (!seen.has(key) || (r.similarity || r.score || 0) > (seen.get(key).similarity || seen.get(key).score || 0)) {
+        seen.set(key, r);
+      }
+    }
+
+    const finalResults = Array.from(seen.values())
+      .sort((a, b) => (b.similarity || b.score || 0) - (a.similarity || a.score || 0))
+      .slice(0, limit);
+
+    const duration = performance.now() - startTime;
+
+    return {
+      results: finalResults,
+      metadata: {
+        vectorCandidates: codebaseResults.length + memoryResults.length,
+        ftsCandidates: (codebaseFts.results?.length || 0) + (memoryFts.results?.length || 0),
+        totalCandidates: combined.length,
+        codebaseResults: codebaseResults.length,
+        memoryResults: memoryResults.length,
+        duration: duration.toFixed(2) + 'ms'
+      }
+    };
+  }
+
+  // Legacy search - kept for backward compatibility
+  async _searchCodebaseLegacy(query, limit = 20) {
     // Search codebase using existing methods
     const vectorResults = await this.searchVector(query, limit * 2);
     const ftsResults = this.codebaseDb?.searchFTS?.(query, limit * 2);
@@ -731,6 +904,25 @@ export class MemoryEngine {
     return { strategy: 'cached', confidence: 0.6, reason: 'Default selection' };
   }
 
+  async getCuratedMemories() {
+    return this.curatedMemory?.read?.() || null;
+  }
+
+  getIntelligenceStats() {
+    return {
+      enabled: this.intelligenceEnabled,
+      patternDetector: {
+        totalPatterns: this.patternDetector?.getTotalPatterns?.() || 0
+      },
+      suggestionEngine: {
+        minConfidence: this.suggestionEngine?.minConfidence || 0.7
+      },
+      strategySelector: {
+        learningRate: this.strategySelector?.learningRate || 0.1
+      }
+    };
+  }
+
   // ========== Sync Engine ==========
 
   getSyncStatus() {
@@ -762,6 +954,33 @@ export class MemoryEngine {
     this.isInitialized = false;
     console.error('[MemoryEngine] Shutdown complete');
   }
+
+  async cleanup() {
+    // Alias for shutdown for consistency
+    return this.shutdown();
+  }
+}
+
+// Auto-register cleanup handlers for graceful shutdown
+if (typeof process !== 'undefined') {
+  const cleanupHandler = async (signal) => {
+    console.error(`[MemoryEngine] Received ${signal}, performing cleanup...`);
+    if (MemoryEngine.instance) {
+      try {
+        await MemoryEngine.instance.shutdown();
+      } catch (error) {
+        console.error(`[MemoryEngine] Error during shutdown: ${error.message}`);
+      }
+    }
+  };
+
+  // Register handlers only once
+  if (!process._memoryEngineCleanupRegistered) {
+    process.on('beforeExit', () => cleanupHandler('beforeExit'));
+    process.on('SIGINT', () => cleanupHandler('SIGINT'));
+    process.on('SIGTERM', () => cleanupHandler('SIGTERM'));
+    process._memoryEngineCleanupRegistered = true;
+  }
 }
 
 export default MemoryEngine;

package/src/memory/stores/base-database.js

@@ -0,0 +1,223 @@
+/**
+ * Base Database Class
+ * Common methods for MemoryDatabase and CodebaseDatabase
+ * Reduces code duplication between the two database classes
+ */
+
+import Database from 'better-sqlite3';
+import path from 'path';
+import fs from 'fs/promises';
+
+export class BaseDatabase {
+  constructor(dbPath = null, options = {}) {
+    this.dbPath = dbPath;
+    this.db = null;
+    this.isInitialized = false;
+    this.options = options;
+    
+    // Logger (can be replaced with proper logger)
+    this.logger = options.logger || console;
+  }
+
+  async initialize() {
+    if (this.isInitialized) return;
+
+    await fs.mkdir(path.dirname(this.dbPath), { recursive: true });
+    
+    this.db = new Database(this.dbPath);
+    this.db.pragma('journal_mode = WAL');
+    
+    this.createTables();
+    this.isInitialized = true;
+    
+    this._log(`Initialized at: ${this.dbPath}`);
+  }
+
+  /**
+   * Override in subclass to create specific tables
+   */
+  createTables() {
+    throw new Error('createTables must be implemented in subclass');
+  }
+
+  /**
+   * Common chunk operations
+   */
+  insertChunk(chunk) {
+    throw new Error('insertChunk must be implemented in subclass');
+  }
+
+  deleteChunksByFile(filePath) {
+    throw new Error('deleteChunksByFile must be implemented in subclass');
+  }
+
+  getChunksByFile(filePath, limit = 100) {
+    throw new Error('getChunksByFile must be implemented in subclass');
+  }
+
+  getRecentChunks(limit = 100) {
+    throw new Error('getRecentChunks must be implemented in subclass');
+  }
+
+  /**
+   * Common embedding operations
+   */
+  insertEmbedding(embedding) {
+    throw new Error('insertEmbedding must be implemented in subclass');
+  }
+
+  getEmbedding(chunkId) {
+    throw new Error('getEmbedding must be implemented in subclass');
+  }
+
+  getAllEmbeddings() {
+    throw new Error('getAllEmbeddings must be implemented in subclass');
+  }
+
+  /**
+   * Common search operations (FTS5)
+   */
+  searchFTS(query, limit = 20) {
+    const startTime = performance.now();
+    
+    const stmt = this.db.prepare(`
+      SELECT 
+        c.*,
+        rank as bm25_score
+      FROM chunks_fts fts
+      JOIN chunks c ON fts.rowid = c.id
+      WHERE chunks_fts MATCH ?
+      ORDER BY rank
+      LIMIT ?
+    `);
+    
+    const results = stmt.all(query, limit);
+    const duration = performance.now() - startTime;
+    
+    // Log search
+    this.logSearch(query, 'fts', results.length, duration);
+    
+    return {
+      results: results.map(r => ({
+        ...r,
+        score: 1 / (1 + Math.max(0, r.bm25_score))
+      })),
+      metadata: {
+        method: 'fts5',
+        duration: duration.toFixed(2) + 'ms',
+        candidates: results.length
+      }
+    };
+  }
+
+  /**
+   * Common search history logging
+   */
+  logSearch(query, method, resultsCount, durationMs) {
+    try {
+      const stmt = this.db.prepare(`
+        INSERT INTO search_history (query, method, results_count, duration_ms, timestamp)
+        VALUES (?, ?, ?, ?, ?)
+      `);
+      stmt.run(query, method, resultsCount, Math.round(durationMs), Date.now());
+    } catch (error) {
+      // Silent fail - don't break search for logging
+    }
+  }
+
+  getSearchStats(days = 7) {
+    const since = Date.now() - (days * 24 * 60 * 60 * 1000);
+    return this.db.prepare(`
+      SELECT 
+        method,
+        COUNT(*) as count,
+        AVG(duration_ms) as avg_duration,
+        AVG(results_count) as avg_results
+      FROM search_history
+      WHERE timestamp > ?
+      GROUP BY method
+    `).all(since);
+  }
+
+  /**
+   * Common file tracking operations
+   */
+  upsertFile(file) {
+    throw new Error('upsertFile must be implemented in subclass');
+  }
+
+  getFile(filePath) {
+    throw new Error('getFile must be implemented in subclass');
+  }
+
+  isFileIndexed(filePath, contentHash) {
+    const file = this.getFile(filePath);
+    return file && file.content_hash === contentHash;
+  }
+
+  deleteFile(filePath) {
+    throw new Error('deleteFile must be implemented in subclass');
+  }
+
+  /**
+   * Common stats
+   */
+  getStats() {
+    throw new Error('getStats must be implemented in subclass');
+  }
+
+  /**
+   * Maintenance
+   */
+  vacuum() {
+    this.db.exec('VACUUM');
+    this.db.exec('ANALYZE');
+  }
+
+  close() {
+    if (this.db) {
+      this.db.close();
+      this.isInitialized = false;
+    }
+  }
+
+  /**
+   * Internal logger
+   */
+  _log(message, level = 'info') {
+    const prefix = `[${this.constructor.name}]`;
+    if (level === 'error') {
+      console.error(`${prefix} ${message}`);
+    } else if (level === 'warn') {
+      console.warn(`${prefix} ${message}`);
+    } else {
+      console.error(`${prefix} ${message}`);
+    }
+  }
+
+  /**
+   * Safe execute with error handling
+   */
+  safeExecute(fn, errorMessage = 'Operation failed') {
+    try {
+      return fn();
+    } catch (error) {
+      this._log(`${errorMessage}: ${error.message}`, 'error');
+      return null;
+    }
+  }
+
+  /**
+   * Safe async execute with error handling
+   */
+  async safeExecuteAsync(fn, errorMessage = 'Operation failed') {
+    try {
+      return await fn();
+    } catch (error) {
+      this._log(`${errorMessage}: ${error.message}`, 'error');
+      return null;
+    }
+  }
+}
+
+export default BaseDatabase;

package/src/memory/utils/chunker.js

@@ -77,6 +77,7 @@ export class Chunker {
       id: this.generateChunkId(filePath, startLine),
       file_id: this.generateFileId(filePath),
       content: content,
+      content_hash: crypto.createHash('md5').update(content).digest('hex'),
       start_line: startLine,
       end_line: endLine,
       token_count: tokens

package/src/memory/utils/indexer.js

@@ -59,12 +59,12 @@ export class CodeIndexer {
     const fileId = this.generateFileId(filePath);
 
     try {
-      // Use getQuery to compile the query properly
+      // Try tree-sitter query first (for functions)
       const query = await getQuery(language, 'definitions');
-      
-      if (query) {
+
+      if (query && ast?.rootNode) {
         const captures = query.captures(ast.rootNode);
-        
+
         for (const capture of captures) {
           if (capture.name === 'name' || capture.name === 'function') {
             facts.push({
@@ -81,6 +81,95 @@ export class CodeIndexer {
           }
         }
       }
+
+      // Fallback/enhancement: Extract more facts using regex patterns
+      // This works across languages and captures additional symbols
+      const content = ast?.text || '';
+
+      // Extract classes
+      const classMatches = content.matchAll(/class\s+(\w+)(?:\s+extends\s+(\w+))?/g);
+      for (const match of classMatches) {
+        if (!facts.find(f => f.type === 'class' && f.name === match[1])) {
+          facts.push({
+            id: this.generateFactId(fileId, match[1], 'class'),
+            file_id: fileId,
+            type: 'class',
+            name: match[1],
+            line_start: this.findLineNumber(content, match[0]) || 0,
+            line_end: 0,
+            signature: match[2] ? `extends ${match[2]}` : '',
+            exported: this.isExported(content, match[1]),
+            confidence: 0.9
+          });
+        }
+      }
+
+      // Extract interfaces (TypeScript)
+      if (language === 'typescript') {
+        const interfaceMatches = content.matchAll(/interface\s+(\w+)(?:\s+extends\s+([^{]+))?/g);
+        for (const match of interfaceMatches) {
+          if (!facts.find(f => f.type === 'interface' && f.name === match[1])) {
+            facts.push({
+              id: this.generateFactId(fileId, match[1], 'interface'),
+              file_id: fileId,
+              type: 'interface',
+              name: match[1],
+              line_start: this.findLineNumber(content, match[0]) || 0,
+              line_end: 0,
+              signature: match[2] ? `extends ${match[2].trim()}` : '',
+              exported: this.isExported(content, match[1]),
+              confidence: 0.9
+            });
+          }
+        }
+      }
+
+      // Extract imports
+      const importMatches = content.matchAll(/import\s+(?:(\w+)|\{\s*([^}]+)\s*\}|\*\s+as\s+(\w+))\s+from\s+['"]([^'"]+)['"]/g);
+      for (const match of importMatches) {
+        const imported = match[1] || match[2]?.split(',').map(s => s.trim().split(' ')[0]).join(', ') || match[3];
+        facts.push({
+          id: this.generateFactId(fileId, imported, 'import'),
+          file_id: fileId,
+          type: 'import',
+          name: imported,
+          line_start: this.findLineNumber(content, match[0]) || 0,
+          line_end: 0,
+          signature: match[4],
+          exported: false,
+          confidence: 1.0
+        });
+      }
+
+      // Extract exports
+      const exportMatches = content.matchAll(/export\s+(?:default\s+)?(?:const|let|var|function|class|interface|type)\s+(\w+)/g);
+      for (const match of exportMatches) {
+        if (!facts.find(f => f.name === match[1] && f.exported)) {
+          const existing = facts.find(f => f.name === match[1]);
+          if (existing) {
+            existing.exported = true;
+          }
+        }
+      }
+
+      // Extract type definitions (TypeScript)
+      if (language === 'typescript') {
+        const typeMatches = content.matchAll(/type\s+(\w+)\s*=/g);
+        for (const match of typeMatches) {
+          facts.push({
+            id: this.generateFactId(fileId, match[1], 'type'),
+            file_id: fileId,
+            type: 'type',
+            name: match[1],
+            line_start: this.findLineNumber(content, match[0]) || 0,
+            line_end: 0,
+            signature: 'type alias',
+            exported: this.isExported(content, match[1]),
+            confidence: 0.8
+          });
+        }
+      }
+
     } catch (error) {
       console.warn(`[Indexer] Failed to extract facts:`, error.message);
     }
@@ -88,6 +177,23 @@ export class CodeIndexer {
     return facts;
   }
 
+  // Helper to find line number of a string in content
+  findLineNumber(content, searchStr) {
+    const lines = content.split('\n');
+    for (let i = 0; i < lines.length; i++) {
+      if (lines[i].includes(searchStr)) {
+        return i + 1;
+      }
+    }
+    return 0;
+  }
+
+  // Check if a symbol is exported
+  isExported(content, symbolName) {
+    return new RegExp(`export\\s+.*\\b${symbolName}\\b`).test(content) ||
+           new RegExp(`export\\s+default`).test(content);
+  }
+
   async generateEmbeddings(chunks, language = 'javascript') {
     const embeddings = [];