@mrxkun/mcfast-mcp 4.1.10 → 4.1.12

package/src/memory/memory-engine.js

@@ -26,6 +26,9 @@ import { CuratedMemory } from './layers/curated-memory.js';
 // Bootstrap
 import { AgentsMdBootstrap } from './bootstrap/agents-md.js';
 
+// Project Analysis (AI-powered)
+import { execute as projectAnalyzeExecute } from '../tools/project_analyze.js';
+
 // Embedders
 import { UltraEnhancedEmbedder } from './utils/ultra-embedder.js';
 import { SmartRouter } from './utils/smart-router.js';
@@ -38,6 +41,7 @@ import { PatternDetector, SuggestionEngine, StrategySelector } from '../intellig
 export class MemoryEngine {
   static instance = null;
   static instancePromise = null;
+  static instancePid = null; // Track which process owns the instance
   
   /**
    * Get singleton instance of MemoryEngine
@@ -45,8 +49,19 @@ export class MemoryEngine {
    * @returns {MemoryEngine} Singleton instance
    */
   static getInstance(options = {}) {
+    const currentPid = process.pid;
+    
+    // Check if instance belongs to current process
+    if (MemoryEngine.instancePid !== currentPid) {
+      // Different process - reset instance
+      MemoryEngine.instance = null;
+      MemoryEngine.instancePromise = null;
+      MemoryEngine.instancePid = null;
+    }
+    
     if (!MemoryEngine.instance) {
       MemoryEngine.instance = new MemoryEngine(options);
+      MemoryEngine.instancePid = currentPid;
     }
     return MemoryEngine.instance;
   }
@@ -58,6 +73,16 @@ export class MemoryEngine {
    * @returns {Promise<MemoryEngine>} Initialized singleton instance
    */
   static async getOrCreate(projectPath, options = {}) {
+    const currentPid = process.pid;
+    
+    // Check if instance belongs to current process
+    if (MemoryEngine.instancePid !== currentPid) {
+      // Different process - reset instance
+      MemoryEngine.instance = null;
+      MemoryEngine.instancePromise = null;
+      MemoryEngine.instancePid = null;
+    }
+    
     if (MemoryEngine.instance && MemoryEngine.instance.isInitialized) {
       return MemoryEngine.instance;
     }
@@ -68,6 +93,7 @@ export class MemoryEngine {
     
     const engine = new MemoryEngine(options);
     MemoryEngine.instance = engine;
+    MemoryEngine.instancePid = currentPid;
     MemoryEngine.instancePromise = engine.initialize(projectPath).then(() => engine);
     
     return MemoryEngine.instancePromise;
@@ -131,6 +157,9 @@ export class MemoryEngine {
     this.suggestionEngine = null;
     this.strategySelector = null;
     
+    // Auto-analyze project on first use
+    this.autoAnalyze = options.autoAnalyze !== false;
+    
     // Search configuration
     this.searchConfig = {
       hybrid: {
@@ -196,19 +225,32 @@ export class MemoryEngine {
       });
     }
     
-    // Initialize file watcher
-    this.watcher = new FileWatcher(projectPath, this, {
-      debounceMs: 1500,
-      ignored: [
-        '**/node_modules/**',
-        '**/.git/**',
-        '**/dist/**',
-        '**/build/**',
-        '**/.mcfast/**',
-        '**/*.log'
-      ]
-    });
-    await this.watcher.start();
+    // Initialize file watcher (configurable via MCFAST_FILE_WATCHER env)
+    const ENABLE_FILE_WATCHER = process.env.MCFAST_FILE_WATCHER !== 'false';
+    
+    if (ENABLE_FILE_WATCHER) {
+      this.watcher = new FileWatcher(projectPath, this, {
+        debounceMs: 1500,
+        ignored: [
+          '**/node_modules/**',
+          '**/.git/**',
+          '**/dist/**',
+          '**/build/**',
+          '**/.mcfast/**',
+          '**/*.log'
+        ]
+      });
+      
+      try {
+        await this.watcher.start();
+      } catch (watcherError) {
+        console.error('[MemoryEngine] ⚠️ File watcher failed to start (continuing without file watching):', watcherError.message);
+        this.watcher = null;
+      }
+    } else {
+      console.error('[MemoryEngine] File watcher disabled (MCFAST_FILE_WATCHER=false)');
+      this.watcher = null;
+    }
     
     // Perform initial scan
     await this.performInitialScan();
@@ -223,6 +265,9 @@ export class MemoryEngine {
     console.error(`[MemoryEngine] Smart Routing: ${this.smartRoutingEnabled ? 'ENABLED' : 'DISABLED'}`);
     console.error(`[MemoryEngine] Intelligence: ${this.intelligenceEnabled ? 'ENABLED' : 'DISABLED'}`);
     
+    // Auto-analyze project if MCFAST_TOKEN is set
+    await this.autoAnalyzeProject();
+    
     // Log initialization to daily logs
     await this.dailyLogs.log('Memory Engine Initialized', `Project: ${projectPath}`, {
       stats: this.getStats()
@@ -264,6 +309,47 @@ export class MemoryEngine {
     }
   }
 
+  /**
+   * Auto-analyze project using AI if MCFAST_TOKEN is set
+   * Runs in background to not block initialization
+   */
+  async autoAnalyzeProject() {
+    if (!this.autoAnalyze) return;
+    
+    const apiKey = process.env.MCFAST_TOKEN;
+    if (!apiKey) {
+      console.error('[MemoryEngine] Skipping auto-analysis (no MCFAST_TOKEN)');
+      return;
+    }
+    
+    // Check if already analyzed (MEMORY.md has Project Context section)
+    try {
+      const memoryContent = await this.curatedMemory.read();
+      if (memoryContent && memoryContent.includes('## Project Context') && 
+          !memoryContent.includes('<!-- Add project context here -->')) {
+        console.error('[MemoryEngine] Project already analyzed, skipping auto-analysis');
+        return;
+      }
+    } catch (e) {
+      // MEMORY.md doesn't exist yet, proceed with analysis
+    }
+    
+    // Run analysis in background
+    console.error('[MemoryEngine] 🔄 Starting auto-analysis in background...');
+    
+    projectAnalyzeExecute({ force: false, updateMemory: true })
+      .then(result => {
+        if (result.metadata?.memoryUpdated) {
+          console.error('[MemoryEngine] ✅ Auto-analysis complete, MEMORY.md updated');
+        } else if (result.isError) {
+          console.error('[MemoryEngine] ⚠️ Auto-analysis failed:', result.content?.[0]?.text?.substring(0, 100));
+        }
+      })
+      .catch(err => {
+        console.error('[MemoryEngine] ⚠️ Auto-analysis error:', err.message);
+      });
+  }
+
   async performInitialScan() {
     if (this.isScanning) return;
     this.isScanning = true;
@@ -470,24 +556,44 @@ export class MemoryEngine {
 
   // ========== Storage Operations ==========
 
-  async storeIndexed(indexed) {
-    this.codebaseDb?.upsertFile?.(indexed.file);
-    this.codebaseDb?.deleteFactsByFile?.(indexed.file.id);
-    this.codebaseDb?.deleteChunksByFile?.(indexed.file.id);
+  async storeIndexed(indexed, maxRetries = 3) {
+    // Retry logic with exponential backoff
+    for (let attempt = 1; attempt <= maxRetries; attempt++) {
+      try {
+        this.codebaseDb?.upsertFile?.(indexed.file);
+        this.codebaseDb?.deleteFactsByFile?.(indexed.file.id);
+        this.codebaseDb?.deleteChunksByFile?.(indexed.file.id);
 
-    for (const fact of indexed.facts) {
-      this.codebaseDb?.insertFact?.(fact);
-    }
+        for (const fact of indexed.facts) {
+          this.codebaseDb?.insertFact?.(fact);
+        }
 
-    for (const chunk of indexed.chunks) {
-      this.codebaseDb?.insertChunk?.(chunk);
-    }
+        for (const chunk of indexed.chunks) {
+          this.codebaseDb?.insertChunk?.(chunk);
+        }
 
-    for (const embedding of indexed.embeddings) {
-      this.codebaseDb?.insertEmbedding?.(embedding);
+        for (const embedding of indexed.embeddings) {
+          this.codebaseDb?.insertEmbedding?.(embedding);
+        }
+
+        return true; // Success
+      } catch (error) {
+        if (attempt === maxRetries) {
+          console.error(`[MemoryEngine] Failed to store indexed data after ${maxRetries} attempts:`, error.message);
+          this.codebaseDb?.recordFailedIndex?.(indexed.file.path, error.message);
+          return false;
+        }
+        // Exponential backoff: 100ms, 200ms, 400ms
+        await new Promise(resolve => setTimeout(resolve, 100 * Math.pow(2, attempt - 1)));
+      }
     }
   }
 
+  // Record failed indexing for tracking
+  recordFailedIndex(filePath, errorMessage) {
+    console.error(`[MemoryEngine] Indexing failed for ${filePath}: ${errorMessage}`);
+  }
+
   // ========== Search Operations ==========
 
   async searchFacts(query, limit = 20) {
@@ -573,6 +679,80 @@ export class MemoryEngine {
   }
 
   async searchCodebase(query, limit = 20) {
+    const startTime = performance.now();
+
+    // FIX: Get embeddings from both codebaseDb AND memoryDb
+    const [codebaseEmbeddings, memoryEmbeddings] = await Promise.all([
+      this.codebaseDb?.getAllEmbeddings?.() || [],
+      this.memoryDb?.getAllEmbeddings?.() || []
+    ]);
+
+    // Generate query embedding
+    const queryResult = this.embedder.embedCode(query);
+    const queryEmbedding = queryResult.vector;
+
+    // Score and tag results from each source
+    const scoreFn = (item) => {
+      const buf = item.embedding?.buffer || item.embedding;
+      const arr = buf ? new Uint8Array(buf) : new Uint8Array(0);
+      return this.embedder.cosineSimilarity(queryEmbedding, Array.from(arr));
+    };
+
+    const codebaseResults = (codebaseEmbeddings || []).map(item => ({
+      ...item, similarity: scoreFn(item), source: 'codebase', type: 'code'
+    }));
+
+    const memoryResults = (memoryEmbeddings || []).map(item => ({
+      ...item, similarity: scoreFn(item), source: 'memory', type: 'memory'
+    }));
+
+    // Combine results
+    let combined = [...codebaseResults, ...memoryResults];
+
+    // Also get FTS results from both databases
+    const [codebaseFts, memoryFts] = await Promise.all([
+      this.codebaseDb?.searchFTS?.(query, limit * 2) || { results: [] },
+      this.memoryDb?.searchFTS?.(query, limit * 2) || { results: [] }
+    ]);
+
+    // Add FTS results
+    for (const r of codebaseFts.results || []) {
+      combined.push({ ...r, source: 'fts', type: 'code', score: r.score });
+    }
+    for (const r of memoryFts.results || []) {
+      combined.push({ ...r, source: 'fts', type: 'memory', score: r.score });
+    }
+
+    // Deduplicate and sort
+    const seen = new Map();
+    for (const r of combined) {
+      const key = r.chunk_id || r.id;
+      if (!seen.has(key) || (r.similarity || r.score || 0) > (seen.get(key).similarity || seen.get(key).score || 0)) {
+        seen.set(key, r);
+      }
+    }
+
+    const finalResults = Array.from(seen.values())
+      .sort((a, b) => (b.similarity || b.score || 0) - (a.similarity || a.score || 0))
+      .slice(0, limit);
+
+    const duration = performance.now() - startTime;
+
+    return {
+      results: finalResults,
+      metadata: {
+        vectorCandidates: codebaseResults.length + memoryResults.length,
+        ftsCandidates: (codebaseFts.results?.length || 0) + (memoryFts.results?.length || 0),
+        totalCandidates: combined.length,
+        codebaseResults: codebaseResults.length,
+        memoryResults: memoryResults.length,
+        duration: duration.toFixed(2) + 'ms'
+      }
+    };
+  }
+
+  // Legacy search - kept for backward compatibility
+  async _searchCodebaseLegacy(query, limit = 20) {
     // Search codebase using existing methods
     const vectorResults = await this.searchVector(query, limit * 2);
     const ftsResults = this.codebaseDb?.searchFTS?.(query, limit * 2);
@@ -781,6 +961,33 @@ export class MemoryEngine {
     this.isInitialized = false;
     console.error('[MemoryEngine] Shutdown complete');
   }
+
+  async cleanup() {
+    // Alias for shutdown for consistency
+    return this.shutdown();
+  }
+}
+
+// Auto-register cleanup handlers for graceful shutdown
+if (typeof process !== 'undefined') {
+  const cleanupHandler = async (signal) => {
+    console.error(`[MemoryEngine] Received ${signal}, performing cleanup...`);
+    if (MemoryEngine.instance) {
+      try {
+        await MemoryEngine.instance.shutdown();
+      } catch (error) {
+        console.error(`[MemoryEngine] Error during shutdown: ${error.message}`);
+      }
+    }
+  };
+
+  // Register handlers only once
+  if (!process._memoryEngineCleanupRegistered) {
+    process.on('beforeExit', () => cleanupHandler('beforeExit'));
+    process.on('SIGINT', () => cleanupHandler('SIGINT'));
+    process.on('SIGTERM', () => cleanupHandler('SIGTERM'));
+    process._memoryEngineCleanupRegistered = true;
+  }
 }
 
 export default MemoryEngine;

package/src/memory/stores/base-database.js

@@ -0,0 +1,223 @@
+/**
+ * Base Database Class
+ * Common methods for MemoryDatabase and CodebaseDatabase
+ * Reduces code duplication between the two database classes
+ */
+
+import Database from 'better-sqlite3';
+import path from 'path';
+import fs from 'fs/promises';
+
+export class BaseDatabase {
+  constructor(dbPath = null, options = {}) {
+    this.dbPath = dbPath;
+    this.db = null;
+    this.isInitialized = false;
+    this.options = options;
+    
+    // Logger (can be replaced with proper logger)
+    this.logger = options.logger || console;
+  }
+
+  async initialize() {
+    if (this.isInitialized) return;
+
+    await fs.mkdir(path.dirname(this.dbPath), { recursive: true });
+    
+    this.db = new Database(this.dbPath);
+    this.db.pragma('journal_mode = WAL');
+    
+    this.createTables();
+    this.isInitialized = true;
+    
+    this._log(`Initialized at: ${this.dbPath}`);
+  }
+
+  /**
+   * Override in subclass to create specific tables
+   */
+  createTables() {
+    throw new Error('createTables must be implemented in subclass');
+  }
+
+  /**
+   * Common chunk operations
+   */
+  insertChunk(chunk) {
+    throw new Error('insertChunk must be implemented in subclass');
+  }
+
+  deleteChunksByFile(filePath) {
+    throw new Error('deleteChunksByFile must be implemented in subclass');
+  }
+
+  getChunksByFile(filePath, limit = 100) {
+    throw new Error('getChunksByFile must be implemented in subclass');
+  }
+
+  getRecentChunks(limit = 100) {
+    throw new Error('getRecentChunks must be implemented in subclass');
+  }
+
+  /**
+   * Common embedding operations
+   */
+  insertEmbedding(embedding) {
+    throw new Error('insertEmbedding must be implemented in subclass');
+  }
+
+  getEmbedding(chunkId) {
+    throw new Error('getEmbedding must be implemented in subclass');
+  }
+
+  getAllEmbeddings() {
+    throw new Error('getAllEmbeddings must be implemented in subclass');
+  }
+
+  /**
+   * Common search operations (FTS5)
+   */
+  searchFTS(query, limit = 20) {
+    const startTime = performance.now();
+    
+    const stmt = this.db.prepare(`
+      SELECT 
+        c.*,
+        rank as bm25_score
+      FROM chunks_fts fts
+      JOIN chunks c ON fts.rowid = c.id
+      WHERE chunks_fts MATCH ?
+      ORDER BY rank
+      LIMIT ?
+    `);
+    
+    const results = stmt.all(query, limit);
+    const duration = performance.now() - startTime;
+    
+    // Log search
+    this.logSearch(query, 'fts', results.length, duration);
+    
+    return {
+      results: results.map(r => ({
+        ...r,
+        score: 1 / (1 + Math.max(0, r.bm25_score))
+      })),
+      metadata: {
+        method: 'fts5',
+        duration: duration.toFixed(2) + 'ms',
+        candidates: results.length
+      }
+    };
+  }
+
+  /**
+   * Common search history logging
+   */
+  logSearch(query, method, resultsCount, durationMs) {
+    try {
+      const stmt = this.db.prepare(`
+        INSERT INTO search_history (query, method, results_count, duration_ms, timestamp)
+        VALUES (?, ?, ?, ?, ?)
+      `);
+      stmt.run(query, method, resultsCount, Math.round(durationMs), Date.now());
+    } catch (error) {
+      // Silent fail - don't break search for logging
+    }
+  }
+
+  getSearchStats(days = 7) {
+    const since = Date.now() - (days * 24 * 60 * 60 * 1000);
+    return this.db.prepare(`
+      SELECT 
+        method,
+        COUNT(*) as count,
+        AVG(duration_ms) as avg_duration,
+        AVG(results_count) as avg_results
+      FROM search_history
+      WHERE timestamp > ?
+      GROUP BY method
+    `).all(since);
+  }
+
+  /**
+   * Common file tracking operations
+   */
+  upsertFile(file) {
+    throw new Error('upsertFile must be implemented in subclass');
+  }
+
+  getFile(filePath) {
+    throw new Error('getFile must be implemented in subclass');
+  }
+
+  isFileIndexed(filePath, contentHash) {
+    const file = this.getFile(filePath);
+    return file && file.content_hash === contentHash;
+  }
+
+  deleteFile(filePath) {
+    throw new Error('deleteFile must be implemented in subclass');
+  }
+
+  /**
+   * Common stats
+   */
+  getStats() {
+    throw new Error('getStats must be implemented in subclass');
+  }
+
+  /**
+   * Maintenance
+   */
+  vacuum() {
+    this.db.exec('VACUUM');
+    this.db.exec('ANALYZE');
+  }
+
+  close() {
+    if (this.db) {
+      this.db.close();
+      this.isInitialized = false;
+    }
+  }
+
+  /**
+   * Internal logger
+   */
+  _log(message, level = 'info') {
+    const prefix = `[${this.constructor.name}]`;
+    if (level === 'error') {
+      console.error(`${prefix} ${message}`);
+    } else if (level === 'warn') {
+      console.warn(`${prefix} ${message}`);
+    } else {
+      console.error(`${prefix} ${message}`);
+    }
+  }
+
+  /**
+   * Safe execute with error handling
+   */
+  safeExecute(fn, errorMessage = 'Operation failed') {
+    try {
+      return fn();
+    } catch (error) {
+      this._log(`${errorMessage}: ${error.message}`, 'error');
+      return null;
+    }
+  }
+
+  /**
+   * Safe async execute with error handling
+   */
+  async safeExecuteAsync(fn, errorMessage = 'Operation failed') {
+    try {
+      return await fn();
+    } catch (error) {
+      this._log(`${errorMessage}: ${error.message}`, 'error');
+      return null;
+    }
+  }
+}
+
+export default BaseDatabase;

package/src/memory/utils/chunker.js

@@ -77,6 +77,7 @@ export class Chunker {
       id: this.generateChunkId(filePath, startLine),
       file_id: this.generateFileId(filePath),
       content: content,
+      content_hash: crypto.createHash('md5').update(content).digest('hex'),
       start_line: startLine,
       end_line: endLine,
       token_count: tokens