@mrxkun/mcfast-mcp 4.0.14 → 4.1.0

package/src/memory/memory-engine.js

@@ -1,34 +1,66 @@
 /**
- * Memory Engine
- * Central coordinator cho memory system
+ * Memory Engine v4.1.0
+ * Refactored with Markdown source of truth + SQLite index
+ * Two-tier memory: Daily logs + Curated memory
+ * Hybrid search: Vector 70% + BM25 30%
  */
 
-import { MemoryDatabase } from './stores/database.js';
+import path from 'path';
+import fs from 'fs/promises';
+import crypto from 'crypto';
+
+// Database stores
+import { MemoryDatabase } from './stores/memory-database.js';
+import { CodebaseDatabase } from './stores/codebase-database.js';
+
+// Watchers
 import { FileWatcher } from './watchers/file-watcher.js';
+
+// Indexers
 import { CodeIndexer } from './utils/indexer.js';
+
+// Two-tier memory layers
+import { DailyLogs } from './layers/daily-logs.js';
+import { CuratedMemory } from './layers/curated-memory.js';
+
+// Bootstrap
+import { AgentsMdBootstrap } from './bootstrap/agents-md.js';
+
+// Embedders
 import { UltraEnhancedEmbedder } from './utils/ultra-embedder.js';
 import { SmartRouter } from './utils/smart-router.js';
 import { DashboardClient } from './utils/dashboard-client.js';
-import { DailyLogs } from './utils/daily-logs.js';
 import { SyncEngine } from './utils/sync-engine.js';
+
+// Intelligence
 import { PatternDetector, SuggestionEngine, StrategySelector } from '../intelligence/index.js';
-import path from 'path';
 
 export class MemoryEngine {
   constructor(options = {}) {
-    this.db = new MemoryDatabase(options.dbPath);
-    this.indexer = new CodeIndexer(options.indexer);
-    this.watcher = null;
     this.projectPath = null;
     this.isInitialized = false;
     
-    // Memory path - now stored in project directory
-    this.memoryPath = options.memoryPath || null; // Will be set on initialize
+    // Paths
+    this.memoryPath = options.memoryPath || null;
+    this.indexPath = options.indexPath || null;
+    
+    // Databases (separate concerns)
+    this.memoryDb = null;      // For notes/memory (Markdown source)
+    this.codebaseDb = null;    // For code indexing
     
-    // Daily Logs - auto-generated markdown notes
-    this.dailyLogs = null; // Will be initialized on initialize
+    // Two-tier memory
+    this.dailyLogs = null;
+    this.curatedMemory = null;
     
-    // Sync Engine - local ↔ cloud (optional, auto-detect from MCFAST_TOKEN)
+    // Bootstrap
+    this.agentsBootstrap = null;
+    
+    // Indexing
+    this.indexer = new CodeIndexer(options.indexer);
+    this.watcher = null;
+    this.isScanning = false;
+    
+    // Sync Engine
     this.syncEngine = null;
     this.syncEngineOptions = {
       apiKey: options.apiKey || process.env.MCFAST_TOKEN,
@@ -36,32 +68,44 @@ export class MemoryEngine {
       syncInterval: options.syncInterval || 5 * 60 * 1000
     };
     
-    // Sử dụng UltraEnhancedEmbedder cho 90% accuracy
+    // Embedder
     this.embedder = new UltraEnhancedEmbedder({
       dimension: options.dimension || 1024,
       cacheSize: options.cacheSize || 2000,
       useMercury: options.useMercury || false
     });
     
-    // Dashboard client for smart routing
+    // Smart routing
     this.dashboardClient = options.dashboardClient || new DashboardClient({
       apiKey: process.env.MCFAST_TOKEN
     });
-    
-    // Smart router - intelligent local vs Mercury decision
     this.smartRouter = new SmartRouter({
       dashboardClient: this.dashboardClient
     });
     
+    // Configuration
     this.useUltraHybrid = options.useUltraHybrid !== false;
     this.targetAccuracy = options.targetAccuracy || 90;
     this.smartRoutingEnabled = options.smartRoutingEnabled !== false;
     
-    // Phase 4: Intelligence Layer
+    // Intelligence
     this.intelligenceEnabled = options.intelligenceEnabled !== false;
     this.patternDetector = null;
     this.suggestionEngine = null;
     this.strategySelector = null;
+    
+    // Search configuration
+    this.searchConfig = {
+      hybrid: {
+        enabled: options.hybridSearch !== false,
+        vectorWeight: options.vectorWeight || 0.7,
+        textWeight: options.textWeight || 0.3,
+        minScore: options.minScore || 0.35,
+        candidateMultiplier: options.candidateMultiplier || 4
+      },
+      chunkSize: options.chunkSize || 400,
+      chunkOverlap: options.chunkOverlap || 80
+    };
   }
 
   async initialize(projectPath) {
@@ -69,24 +113,43 @@ export class MemoryEngine {
 
     this.projectPath = projectPath;
     
-    // Set memory path to project directory
+    // Setup paths
     if (!this.memoryPath) {
-      this.memoryPath = path.join(projectPath, '.mcfast', 'memory');
+      this.memoryPath = path.join(projectPath, '.mcfast');
     }
-    
-    // Update database path if not explicitly set
-    if (!this.db.dbPath) {
-      this.db.dbPath = path.join(this.memoryPath, 'index.db');
+    if (!this.indexPath) {
+      this.indexPath = path.join(this.memoryPath, 'index');
     }
     
-    await this.db.initialize();
+    // Create directories
+    await fs.mkdir(this.memoryPath, { recursive: true });
+    await fs.mkdir(this.indexPath, { recursive: true });
     
-    // Initialize Daily Logs with project-specific path
-    this.dailyLogs = new DailyLogs({
-      memoryPath: this.memoryPath
-    });
+    console.log(`[MemoryEngine] Initializing...`);
+    console.log(`[MemoryEngine] Project: ${projectPath}`);
+    console.log(`[MemoryEngine] Memory path: ${this.memoryPath}`);
+    
+    // Initialize bootstrap (AGENTS.md)
+    this.agentsBootstrap = new AgentsMdBootstrap({ projectPath });
+    await this.agentsBootstrap.bootstrap();
+    
+    // Initialize databases
+    this.memoryDb = new MemoryDatabase(
+      path.join(this.indexPath, 'memory.sqlite')
+    );
+    this.codebaseDb = new CodebaseDatabase(
+      path.join(this.indexPath, 'codebase.sqlite')
+    );
     
-    // Initialize Sync Engine if configured
+    await this.memoryDb.initialize();
+    await this.codebaseDb.initialize();
+    
+    // Initialize two-tier memory
+    this.dailyLogs = new DailyLogs({ memoryPath: this.memoryPath });
+    this.curatedMemory = new CuratedMemory({ memoryPath: this.memoryPath });
+    await this.curatedMemory.ensureFile(); // Create MEMORY.md if not exists
+    
+    // Initialize sync engine
     if (this.syncEngineOptions.apiKey && this.syncEngineOptions.enableSync !== false) {
       this.syncEngine = new SyncEngine({
         memoryPath: this.memoryPath,
@@ -96,31 +159,50 @@ export class MemoryEngine {
       });
     }
     
-    this.watcher = new FileWatcher(projectPath, this);
+    // Initialize file watcher
+    this.watcher = new FileWatcher(projectPath, this, {
+      debounceMs: 1500,
+      ignored: [
+        '**/node_modules/**',
+        '**/.git/**',
+        '**/dist/**',
+        '**/build/**',
+        '**/.mcfast/**',
+        '**/*.log'
+      ]
+    });
     await this.watcher.start();
-
+    
+    // Perform initial scan
+    await this.performInitialScan();
+    
+    // Initialize intelligence
+    await this.initializeIntelligence();
+    
     this.isInitialized = true;
-    console.log(`[MemoryEngine] Initialized for: ${projectPath}`);
+    
+    console.log(`[MemoryEngine] ✅ Initialized successfully`);
+    console.log(`[MemoryEngine] Hybrid Search: ${this.searchConfig.hybrid.enabled ? 'ENABLED' : 'DISABLED'}`);
     console.log(`[MemoryEngine] Smart Routing: ${this.smartRoutingEnabled ? 'ENABLED' : 'DISABLED'}`);
-    console.log(`[MemoryEngine] Intelligence Layer: ${this.intelligenceEnabled ? 'ENABLED' : 'DISABLED'}`);
+    console.log(`[MemoryEngine] Intelligence: ${this.intelligenceEnabled ? 'ENABLED' : 'DISABLED'}`);
     
-    // Initialize intelligence engines if enabled
-    if (this.intelligenceEnabled) {
-      try {
-        const { PatternDetector, SuggestionEngine, StrategySelector } = await import('../intelligence/index.js');
-        this.patternDetector = new PatternDetector({ memoryEngine: this });
-        this.suggestionEngine = new SuggestionEngine({ memoryEngine: this });
-        this.strategySelector = new StrategySelector({ memoryEngine: this });
-        console.log('[MemoryEngine] Intelligence engines initialized');
-      } catch (error) {
-        console.warn('[MemoryEngine] Failed to initialize intelligence engines:', error.message);
-        this.intelligenceEnabled = false;
-      }
-    }
+    // Log initialization to daily logs
+    await this.dailyLogs.log('Memory Engine Initialized', `Project: ${projectPath}`, {
+      stats: this.getStats()
+    });
+  }
+
+  async initializeIntelligence() {
+    if (!this.intelligenceEnabled) return;
     
-    // Load intelligence models - run in background to not block initialization
-    if (this.intelligenceEnabled) {
-      // Start model loading in background without awaiting
+    try {
+      this.patternDetector = new PatternDetector({ memoryEngine: this });
+      this.suggestionEngine = new SuggestionEngine({ memoryEngine: this });
+      this.strategySelector = new StrategySelector({ memoryEngine: this });
+      
+      console.log('[MemoryEngine] Intelligence engines initialized');
+      
+      // Load models in background
       (async () => {
         try {
           const loadPromises = [
@@ -129,63 +211,250 @@ export class MemoryEngine {
             this.strategySelector.loadModel?.()
           ].filter(Boolean);
           
-          // Add timeout to prevent hanging
           const timeoutPromise = new Promise((_, reject) => 
             setTimeout(() => reject(new Error('Model loading timeout')), 5000)
           );
           
-          await Promise.race([
-            Promise.all(loadPromises),
-            timeoutPromise
-          ]);
-          
+          await Promise.race([Promise.all(loadPromises), timeoutPromise]);
           console.log('[MemoryEngine] ✅ Intelligence models loaded');
         } catch (error) {
           console.warn('[MemoryEngine] Model loading failed or timed out:', error.message);
-          // Don't disable intelligence - models are optional
         }
       })();
+    } catch (error) {
+      console.warn('[MemoryEngine] Failed to initialize intelligence engines:', error.message);
+      this.intelligenceEnabled = false;
+    }
+  }
+
+  async performInitialScan() {
+    if (this.isScanning) return;
+    this.isScanning = true;
+    
+    console.log('[MemoryEngine] 🔍 Performing initial codebase scan...');
+    
+    try {
+      // Scan memory files first
+      await this.scanMemoryFiles();
+      
+      // Scan codebase
+      await this.scanCodebase();
+      
+      // Update indexing progress
+      this.codebaseDb?.completeIndexing?.();
+      
+      console.log('[MemoryEngine] ✅ Initial scan complete');
+      console.log(`[MemoryEngine] Stats:`, this.getStats());
+      
+    } catch (error) {
+      console.error('[MemoryEngine] Error during initial scan:', error);
+    } finally {
+      this.isScanning = false;
     }
   }
 
-  async indexFile(filePath, content) {
-    const indexed = await this.indexer.indexFile(filePath, content);
-    await this.storeIndexed(indexed);
-    return indexed;
+  async scanMemoryFiles() {
+    console.log('[MemoryEngine] Scanning memory files...');
+    
+    try {
+      // Index MEMORY.md
+      const memoryFile = path.join(this.memoryPath, 'MEMORY.md');
+      if (await this.fileExists(memoryFile)) {
+        await this.indexMarkdownFile(memoryFile);
+      }
+      
+      // Index daily logs
+      const memoryDir = path.join(this.memoryPath, 'memory');
+      if (await this.fileExists(memoryDir)) {
+        const files = await fs.readdir(memoryDir);
+        for (const file of files.filter(f => f.endsWith('.md'))) {
+          await this.indexMarkdownFile(path.join(memoryDir, file));
+        }
+      }
+      
+      console.log('[MemoryEngine] Memory files indexed');
+    } catch (error) {
+      console.error('[MemoryEngine] Error scanning memory files:', error);
+    }
   }
 
+  async scanCodebase() {
+    console.log('[MemoryEngine] Scanning codebase...');
+    
+    const extensions = ['.js', '.jsx', '.ts', '.tsx', '.py', '.java', '.go', '.rs', '.cpp', '.c', '.h'];
+    const files = await this.findFiles(this.projectPath, extensions);
+    
+    if (files.length === 0) {
+      console.log('[MemoryEngine] No code files found to index');
+      return;
+    }
+    
+    console.log(`[MemoryEngine] Found ${files.length} files to index`);
+    
+    // Start indexing progress
+    this.codebaseDb?.startIndexing?.(files.length);
+    
+    let indexed = 0;
+    let failed = 0;
+    
+    for (const filePath of files) {
+      try {
+        const content = await fs.readFile(filePath, 'utf-8');
+        const contentHash = crypto.createHash('md5').update(content).digest('hex');
+        
+        // Skip if already indexed
+        if (this.codebaseDb?.isFileIndexed?.(filePath, contentHash)) {
+          continue;
+        }
+        
+        // Index file
+        const indexedData = await this.indexer.indexFile(filePath, content);
+        await this.storeIndexed(indexedData);
+        
+        indexed++;
+        if (indexed % 10 === 0) {
+          console.log(`[MemoryEngine] Indexed ${indexed}/${files.length} files...`);
+        }
+      } catch (error) {
+        console.warn(`[MemoryEngine] Failed to index ${filePath}:`, error.message);
+        failed++;
+      }
+    }
+    
+    console.log(`[MemoryEngine] Codebase scan complete: ${indexed} indexed, ${failed} failed`);
+  }
+
+  async indexMarkdownFile(filePath) {
+    try {
+      const content = await fs.readFile(filePath, 'utf-8');
+      const contentHash = crypto.createHash('md5').update(content).digest('hex');
+      const stats = await fs.stat(filePath);
+      const relativePath = path.relative(this.projectPath, filePath);
+      
+      // Skip if unchanged
+      if (this.memoryDb?.isFileIndexed?.(relativePath, contentHash)) {
+        return;
+      }
+      
+      // Import chunker dynamically
+      const { MarkdownChunker } = await import('./utils/markdown-chunker.js');
+      const chunker = new MarkdownChunker({
+        chunkSize: this.searchConfig.chunkSize * 4, // chars
+        overlap: this.searchConfig.chunkOverlap * 4
+      });
+      
+      const chunks = chunker.chunk(content, relativePath);
+      
+      // Track file
+      this.memoryDb?.upsertFile?.(relativePath, contentHash, stats.mtimeMs, stats.size);
+      
+      // Delete old chunks
+      this.memoryDb?.deleteChunksByFile?.(relativePath);
+      
+      // Process chunks
+      for (const chunk of chunks) {
+        // Check embedding cache
+        let embedding = null;
+        const cached = this.memoryDb?.getCachedEmbedding?.(chunk.contentHash);
+        
+        if (cached) {
+          embedding = new Float32Array(cached.embedding.buffer, cached.embedding.byteOffset, cached.dimensions);
+        } else {
+          // Generate embedding
+          const result = this.embedder.embedCode(chunk.content);
+          embedding = result.vector;
+          
+          // Cache it
+          this.memoryDb?.cacheEmbedding?.(
+            chunk.contentHash,
+            Buffer.from(embedding.buffer),
+            'simple-embedder',
+            embedding.length
+          );
+        }
+        
+        // Insert chunk
+        this.memoryDb?.insertChunk?.({
+          id: chunk.id,
+          file_path: chunk.filePath,
+          start_line: chunk.startLine,
+          end_line: chunk.endLine,
+          content: chunk.content,
+          content_hash: chunk.contentHash,
+          chunk_type: chunk.chunkType
+        });
+        
+        // Insert embedding
+        this.memoryDb?.insertEmbedding?.({
+          chunk_id: chunk.id,
+          embedding: Buffer.from(embedding.buffer),
+          model: 'simple-embedder',
+          dimensions: embedding.length
+        });
+      }
+      
+    } catch (error) {
+      console.warn(`[MemoryEngine] Failed to index markdown ${filePath}:`, error.message);
+    }
+  }
+
+  async findFiles(dir, extensions) {
+    const files = [];
+    const ignored = ['node_modules', '.git', 'dist', 'build', '.mcfast'];
+    
+    try {
+      const entries = await fs.readdir(dir, { withFileTypes: true });
+      
+      for (const entry of entries) {
+        const fullPath = path.join(dir, entry.name);
+        
+        if (entry.isDirectory() && !ignored.includes(entry.name)) {
+          const subFiles = await this.findFiles(fullPath, extensions);
+          files.push(...subFiles);
+        } else if (entry.isFile() && extensions.some(ext => entry.name.endsWith(ext))) {
+          files.push(fullPath);
+        }
+      }
+    } catch (error) {
+      // Permission denied or other error, skip
+    }
+    
+    return files;
+  }
+
+  async fileExists(filePath) {
+    try {
+      await fs.access(filePath);
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  // ========== Storage Operations ==========
+
   async storeIndexed(indexed) {
-    this.db.upsertFile(indexed.file);
-    this.db.deleteFactsByFile(indexed.file.id);
-    this.db.deleteChunksByFile(indexed.file.id);
+    this.codebaseDb?.upsertFile?.(indexed.file);
+    this.codebaseDb?.deleteFactsByFile?.(indexed.file.id);
+    this.codebaseDb?.deleteChunksByFile?.(indexed.file.id);
 
     for (const fact of indexed.facts) {
-      this.db.insertFact(fact);
+      this.codebaseDb?.insertFact?.(fact);
     }
 
     for (const chunk of indexed.chunks) {
-      this.db.insertChunk(chunk);
+      this.codebaseDb?.insertChunk?.(chunk);
     }
 
     for (const embedding of indexed.embeddings) {
-      this.db.insertEmbedding(embedding);
+      this.codebaseDb?.insertEmbedding?.(embedding);
     }
   }
 
-  async getFileByPath(filePath) {
-    return this.db.getFileByPath(filePath);
-  }
-
-  async deleteFile(fileId) {
-    return this.db.deleteFile(fileId);
-  }
+  // ========== Search Operations ==========
 
   async searchFacts(query, limit = 20) {
-    return this.db.searchFacts(query, limit);
-  }
-
-  async searchFTS(query, limit = 20) {
-    return this.db.searchFTS(query, limit);
+    return this.codebaseDb?.searchFacts?.(query, limit) || [];
   }
 
   async searchVector(query, limit = 20) {
@@ -194,7 +463,7 @@ export class MemoryEngine {
     const queryResult = this.embedder.embedCode(query);
     const queryEmbedding = queryResult.vector;
     
-    const allEmbeddings = this.db.getAllEmbeddings();
+    const allEmbeddings = this.codebaseDb?.getAllEmbeddings?.() || [];
     
     const scored = allEmbeddings.map(item => {
       const vector = Array.from(new Float32Array(item.embedding.buffer, item.embedding.byteOffset, item.embedding.byteLength / 4));
@@ -221,465 +490,199 @@ export class MemoryEngine {
     };
   }
 
-  async searchEnhanced(query, limit = 20) {
+  /**
+   * Hybrid search: Combine Vector + BM25
+   * Vector 70% + BM25 30%
+   */
+  async searchHybrid(query, options = {}) {
     const startTime = performance.now();
+    const limit = options.limit || 10;
     
-    // Lấy tất cả candidates từ database
-    const allChunks = this.db.getAllChunksWithContent();
-    
-    // Chuẩn bị candidates cho ultra-hybrid search
-    const candidates = allChunks.map(chunk => ({
-      ...chunk,
-      embedding: chunk.embedding ? Array.from(new Float32Array(chunk.embedding.buffer, chunk.embedding.byteOffset, chunk.embedding.byteLength / 4)) : null,
-      code: chunk.content,
-      filePath: chunk.path
-    }));
-    
-    // Sử dụng ultra-hybrid search (6 techniques)
-    const searchResult = await this.embedder.ultraHybridSearch(query, candidates, { limit });
+    console.log(`[MemoryEngine] 🔍 Hybrid search: "${query}"`);
     
-    const duration = performance.now() - startTime;
+    // Get vector results first
+    const vectorResult = await this.searchVector(query, limit * 4);
     
-    return {
-      ...searchResult,
-      metadata: {
-        ...searchResult.metadata,
-        totalDuration: duration.toFixed(2) + 'ms',
-        method: 'ultra-hybrid-v2',
-        targetAccuracy: this.targetAccuracy + '%'
-      }
-    };
-  }
-
-  async ultraSearch(query, options = {}) {
-    const startTime = performance.now();
+    // Use database hybrid search
+    const dbResults = this.memoryDb?.searchHybrid?.(query, vectorResult.results, limit);
     
-    console.log(`[MemoryEngine] 🎯 Ultra Search: "${query}" (Target: ${this.targetAccuracy}%)`);
-    
-    // Lấy candidates
-    const allChunks = this.db.getAllChunksWithContent();
-    const candidates = allChunks.map(chunk => ({
-      ...chunk,
-      embedding: chunk.embedding ? Array.from(new Float32Array(chunk.embedding.buffer, chunk.embedding.byteOffset, chunk.embedding.byteLength / 4)) : null,
-      code: chunk.content,
-      filePath: chunk.path
-    }));
-    
-    // Ultra-hybrid search với 6 techniques
-    const result = await this.embedder.ultraHybridSearch(query, candidates, {
-      limit: options.limit || 10,
-      ...options
-    });
-    
-    // Record edit history để học từ searches
-    if (options.currentFile) {
-      const topResult = result.results[0];
-      if (topResult && topResult.filePath) {
-        this.embedder.recordEdit(options.currentFile, [topResult.filePath]);
-      }
+    if (dbResults) {
+      const totalDuration = performance.now() - startTime;
+      return {
+        ...dbResults,
+        metadata: {
+          ...dbResults.metadata,
+          totalDuration: totalDuration.toFixed(2) + 'ms'
+        }
+      };
     }
     
-    const duration = performance.now() - startTime;
-    
-    return {
-      ...result,
-      metadata: {
-        ...result.metadata,
-        totalDuration: duration.toFixed(2) + 'ms',
-        method: 'ultra-search',
-        accuracy: this.targetAccuracy + '%',
-        techniques: ['vector', 'keyword', 'synonym', 'context', 'pattern', 'history']
-      }
-    };
+    // Fallback to vector-only if hybrid fails
+    return vectorResult;
   }
 
-  async smartSearch(query, options = {}) {
+  async searchMemory(query, options = {}) {
     const {
       limit = 10,
-      useUltra = true,
-      useMercury = false,
-      context = null
+      useHybrid = this.searchConfig.hybrid.enabled,
+      minScore = this.searchConfig.hybrid.minScore
     } = options;
     
-    console.log(`[MemoryEngine] 🧠 Smart search: "${query}" (Ultra: ${useUltra}, Mercury: ${useMercury})`);
-    
-    // Option B: Mercury re-ranking
-    if (useMercury && this.embedder.isMercuryEnabled()) {
-      const allChunks = this.db.getAllChunksWithContent();
-      const candidates = allChunks.map(chunk => ({
-        ...chunk,
-        embedding: chunk.embedding ? Array.from(new Float32Array(chunk.embedding.buffer, chunk.embedding.byteOffset, chunk.embedding.byteLength / 4)) : null,
-        code: chunk.content,
-        filePath: chunk.path
-      }));
-      
-      return this.embedder.searchWithOptionalReranking(query, candidates, {
-        limit,
-        useMercury: true
-      });
+    if (useHybrid) {
+      return this.searchHybrid(query, { limit, minScore });
     }
     
-    // Option A: Ultra-hybrid (90% accuracy)
-    if (useUltra) {
-      return this.ultraSearch(query, { limit, ...options });
-    }
-    
-    // Fallback: Enhanced search
-    return this.searchEnhanced(query, limit);
+    return this.searchVector(query, limit);
   }
 
-  async intelligentSearch(query, options = {}) {
-    const startTime = performance.now();
+  async searchCodebase(query, limit = 20) {
+    // Search codebase using existing methods
+    const vectorResults = await this.searchVector(query, limit * 2);
+    const ftsResults = this.codebaseDb?.searchFTS?.(query, limit * 2);
     
-    console.log(`[MemoryEngine] 🔍 Intelligent Search: "${query}"`);
-    
-    // 1. Always do local search first (fast)
-    const localStart = performance.now();
-    const localResult = await this.ultraSearch(query, { 
-      limit: 20, // Get more for smart decision
-      ...options 
-    });
-    const localDuration = performance.now() - localStart;
+    // Combine and deduplicate
+    const seen = new Set();
+    const combined = [];
     
-    // 2. Smart router decides: need Mercury?
-    const decision = await this.smartRouter.shouldUseMercury(query, localResult.results);
-    
-    console.log(`[MemoryEngine] 🤖 Smart decision: ${decision.useMercury ? 'USE MERCURY' : 'LOCAL ONLY'}`);
-    console.log(`[MemoryEngine]    Reason: ${decision.reason}`);
-    
-    // 3. If Mercury needed and enabled
-    if (decision.useMercury && this.smartRouter.config.enableMercury) {
-      try {
-        const mercuryStart = performance.now();
-        
-        // Call Dashboard for Mercury re-ranking
-        const mercuryResult = await this.dashboardClient.requestMercuryRerank(
-          query,
-          localResult.results
-        );
-        
-        const mercuryDuration = performance.now() - mercuryStart;
-        const totalDuration = performance.now() - startTime;
-        
-        // Log usage
-        this.smartRouter.logUsage(query, true, mercuryDuration);
-        
-        // Record for learning
-        this.smartRouter.recordResult(query, true, true);
-        
-        return {
-          results: mercuryResult.results,
-          metadata: {
-            ...mercuryResult.metadata,
-            localDuration: `${localDuration.toFixed(1)}ms`,
-            mercuryDuration: `${mercuryDuration.toFixed(1)}ms`,
-            totalDuration: `${totalDuration.toFixed(1)}ms`,
-            method: 'intelligent-hybrid',
-            routingDecision: decision,
-            accuracy: '95%'
-          }
-        };
-        
-      } catch (error) {
-        console.warn('[MemoryEngine] Mercury failed, fallback to local:', error.message);
-        
-        // Fallback to local results
-        this.smartRouter.recordResult(query, false, true);
-        
-        return {
-          ...localResult,
-          metadata: {
-            ...localResult.metadata,
-            fallback: true,
-            fallbackReason: error.message,
-            routingDecision: decision
-          }
-        };
+    for (const r of vectorResults.results) {
+      if (!seen.has(r.chunk_id)) {
+        seen.add(r.chunk_id);
+        combined.push({ ...r, source: 'vector' });
       }
     }
     
-    // 4. Return local results
-    const totalDuration = performance.now() - startTime;
+    if (ftsResults?.results) {
+      for (const r of ftsResults.results) {
+        if (!seen.has(r.chunk_id)) {
+          seen.add(r.chunk_id);
+          combined.push({ ...r, source: 'fts' });
+        }
+      }
+    }
     
-    this.smartRouter.recordResult(query, false, true);
+    // Sort by score and limit
+    combined.sort((a, b) => (b.similarity || b.score) - (a.similarity || a.score));
     
     return {
-      ...localResult,
+      results: combined.slice(0, limit),
       metadata: {
-        ...localResult.metadata,
-        totalDuration: `${totalDuration.toFixed(1)}ms`,
-        method: 'intelligent-local',
-        routingDecision: decision,
-        accuracy: '90%'
+        vectorCandidates: vectorResults.results.length,
+        ftsCandidates: ftsResults?.results?.length || 0,
+        totalCandidates: combined.length
       }
     };
   }
 
-  // ==================== OPTION B: MERCURY INTEGRATION ====================
-
-  /**
-   * Enable Mercury re-ranking (Option B)
-   */
-  enableMercuryReRanking(apiKey, options = {}) {
-    this.embedder.enableMercuryReRanking(apiKey, options);
-    this.targetAccuracy = 95; // Tăng target lên 95%
-    console.log(`[MemoryEngine] ✨ Mercury re-ranking enabled! Target accuracy: ${this.targetAccuracy}%`);
-  }
-
-  /**
-   * Disable Mercury re-ranking
-   */
-  disableMercuryReRanking() {
-    this.embedder.disableMercuryReRanking();
-    this.targetAccuracy = 90; // Quay lại 90%
-    console.log(`[MemoryEngine] Mercury disabled. Target accuracy: ${this.targetAccuracy}%`);
-  }
-
-  /**
-   * Check Mercury status
-   */
-  isMercuryEnabled() {
-    return this.embedder.isMercuryEnabled();
-  }
-
-  async recordEdit(edit) {
-    return this.db.recordEdit({
-      id: Math.random().toString(36).substring(2, 15),
-      ...edit
-    });
-  }
-
-  getStats() {
-    return {
-      ...this.db.getStats(),
-      projectPath: this.projectPath
-    };
-  }
-
-  /**
-   * Get recent chunks from database
-   * @param {number} limit - Number of chunks to retrieve
-   * @returns {Array} Recent chunks
-   */
-  getRecentChunks(limit = 100) {
-    return this.db.getRecentChunks(limit);
-  }
-
-  // ==================== SMART ROUTING CONTROLS ====================
+  // ========== Two-Tier Memory Operations ==========
 
   /**
-   * Enable/disable smart routing
+   * Log to daily logs (Layer 1)
    */
-  setSmartRouting(enabled) {
-    this.smartRoutingEnabled = enabled;
-    console.log(`[MemoryEngine] Smart routing ${enabled ? 'ENABLED' : 'DISABLED'}`);
+  async logToDaily(title, content, metadata = {}) {
+    return this.dailyLogs?.log?.(title, content, metadata);
   }
 
   /**
-   * Get smart routing stats
+   * Add to curated memory (Layer 2)
    */
-  getSmartRoutingStats() {
-    return this.smartRouter.getStats();
+  async addToCurated(type, title, content) {
+    return this.curatedMemory?.addEntry?.(type, title, content);
   }
 
   /**
-   * Refresh config from Dashboard
+   * Get today's log
    */
-  async refreshConfig() {
-    await this.smartRouter.loadConfig();
-    console.log('[MemoryEngine] Config refreshed from Dashboard');
+  async getTodayLog() {
+    return this.dailyLogs?.getTodayLog?.();
   }
 
-  // ==================== DAILY LOGS ====================
-
   /**
-   * Log file creation event
+   * Get yesterday's log
    */
-  logFileCreated(filePath, facts = []) {
-    this.dailyLogs.logFileCreated(filePath, facts);
+  async getYesterdayLog() {
+    return this.dailyLogs?.getYesterdayLog?.();
   }
 
   /**
-   * Log file modification event
+   * Get curated memory content
    */
-  logFileModified(filePath, changes = {}) {
-    this.dailyLogs.logFileModified(filePath, changes);
+  async getCuratedMemory() {
+    return this.curatedMemory?.read?.();
   }
 
   /**
-   * Log file deletion event
+   * Log file operations
    */
-  logFileDeleted(filePath) {
-    this.dailyLogs.logFileDeleted(filePath);
+  async logFileCreated(filePath, facts = []) {
+    return this.dailyLogs?.logFileCreated?.(filePath, facts);
   }
 
-  /**
-   * Log an edit session
-   */
-  logEditToMemory(edit) {
-    this.dailyLogs.logEdit(edit);
-    // Also record to database
-    this.recordEdit(edit);
+  async logFileModified(filePath, changes = {}) {
+    return this.dailyLogs?.logFileModified?.(filePath, changes);
   }
 
-  /**
-   * Add curated memory
-   */
-  addCuratedMemory(title, content, tags = []) {
-    this.dailyLogs.addCuratedMemory(title, content, tags);
+  async logFileDeleted(filePath) {
+    return this.dailyLogs?.logFileDeleted?.(filePath);
   }
 
-  /**
-   * Get today's log
-   */
-  getTodayLog() {
-    return this.dailyLogs.getTodayLog();
-  }
-
-  /**
-   * Get curated memories
-   */
-  getCuratedMemories() {
-    return this.dailyLogs.getCuratedMemories();
+  async logEdit(edit) {
+    // Log to daily
+    await this.dailyLogs?.logEdit?.(edit);
+    
+    // Record to database
+    this.codebaseDb?.recordEdit?.({
+      id: Math.random().toString(36).substring(2, 15),
+      ...edit,
+      timestamp: Date.now()
+    });
   }
 
-  // ==================== SYNC ENGINE ====================
+  // ========== Stats & Info ==========
 
-  /**
-   * Get sync status
-   */
-  getSyncStatus() {
-    if (!this.syncEngine) {
-      return { enabled: false };
-    }
+  getStats() {
     return {
-      enabled: true,
-      ...this.syncEngine.getStatus()
+      memory: this.memoryDb?.getStats?.() || {},
+      codebase: this.codebaseDb?.getStats?.() || {},
+      indexing: this.codebaseDb?.getIndexingProgress?.() || {},
+      watcher: this.watcher?.getStats?.() || {},
+      projectPath: this.projectPath
     };
   }
 
-  /**
-   * Trigger manual sync
-   */
-  async triggerSync() {
-    if (!this.syncEngine) {
-      throw new Error('Sync engine not configured');
-    }
-    return this.syncEngine.sync();
-  }
-
-  /**
-   * Queue change for sync
-   */
-  queueSyncChange(change) {
-    if (this.syncEngine) {
-      this.syncEngine.queueChange(change);
-    }
+  getRecentChunks(limit = 100) {
+    return this.codebaseDb?.getRecentChunks?.(limit) || [];
   }
 
-  // ==================== INTELLIGENCE LAYER (Phase 4) ====================
+  // ========== Intelligence Layer ==========
 
-  /**
-   * Detect patterns from edit history
-   * @returns {Array} Detected patterns with confidence scores
-   */
   async detectPatterns() {
     if (!this.intelligenceEnabled || !this.patternDetector) {
       throw new Error('Intelligence layer not enabled');
     }
-
-    console.log('[MemoryEngine] 🔍 Detecting patterns...');
-    
-    const editHistory = await this.db.getRecentEdits(100);
-    const patterns = await this.patternDetector.analyze(editHistory);
     
-    console.log(`[MemoryEngine] ✅ Found ${patterns.length} patterns`);
-    return patterns;
+    const editHistory = this.codebaseDb?.getRecentEdits?.(100) || [];
+    return this.patternDetector.analyze(editHistory);
   }
 
-  /**
-   * Get intelligent suggestions based on current context
-   * @param {Object} context - Current editing context
-   * @returns {Array} Suggestions with confidence scores
-   */
   async getSuggestions(context = {}) {
     if (!this.intelligenceEnabled || !this.suggestionEngine) {
       throw new Error('Intelligence layer not enabled');
     }
-
-    console.log('[MemoryEngine] 💡 Generating suggestions...');
-    
-    const suggestions = await this.suggestionEngine.getSuggestions(context);
     
-    console.log(`[MemoryEngine] ✅ Generated ${suggestions.length} suggestions`);
-    return suggestions;
+    return this.suggestionEngine.getSuggestions(context);
   }
 
-  /**
-   * Select best strategy for edit operation
-   * @param {string} instruction - Edit instruction
-   * @param {Object} context - Edit context
-   * @returns {Object} Selected strategy with confidence
-   */
   async selectStrategy(instruction, context = {}) {
     if (!this.intelligenceEnabled || !this.strategySelector) {
-      // Fallback: use simple heuristics
       return this.fallbackStrategySelection(instruction, context);
     }
-
-    console.log('[MemoryEngine] 🎯 Selecting strategy...');
-    
-    const result = await this.strategySelector.selectStrategy(instruction, context);
     
-    console.log(`[MemoryEngine] ✅ Selected: ${result.strategy} (confidence: ${(result.confidence * 100).toFixed(1)}%)`);
-    return result;
-  }
-
-  /**
-   * Learn from edit result (improve future selections)
-   * @param {Object} edit - Edit data
-   * @param {Object} result - Edit result
-   */
-  async learnFromEdit(edit, result) {
-    if (!this.intelligenceEnabled || !this.strategySelector) {
-      return;
-    }
-
-    await this.strategySelector.learn(edit, result);
-    console.log('[MemoryEngine] 🧠 Learned from edit result');
-  }
-
-  /**
-   * Get intelligence layer statistics
-   */
-  getIntelligenceStats() {
-    if (!this.intelligenceEnabled) {
-      return { enabled: false };
-    }
-
-    return {
-      enabled: true,
-      patternDetector: this.patternDetector?.getStats() || null,
-      suggestionEngine: this.suggestionEngine?.getStats() || null,
-      strategySelector: this.strategySelector?.getStats() || null
-    };
-  }
-
-  /**
-   * Enable/disable intelligence layer
-   */
-  setIntelligenceEnabled(enabled) {
-    this.intelligenceEnabled = enabled;
-    console.log(`[MemoryEngine] Intelligence layer ${enabled ? 'ENABLED' : 'DISABLED'}`);
+    return this.strategySelector.selectStrategy(instruction, context);
   }
 
-  /**
-   * Fallback strategy selection when ML is disabled
-   */
   fallbackStrategySelection(instruction, context) {
     const lower = instruction.toLowerCase();
     
-    // Simple keyword-based selection
     if (lower.includes('replace') || lower.includes('change line') || lower.includes('exact')) {
       return { strategy: 'deterministic', confidence: 0.8, reason: 'Simple replacement detected' };
     }
@@ -691,19 +694,36 @@ export class MemoryEngine {
     return { strategy: 'cached', confidence: 0.6, reason: 'Default selection' };
   }
 
-  // ==================== LIFECYCLE ====================
+  // ========== Sync Engine ==========
+
+  getSyncStatus() {
+    return this.syncEngine?.getStatus?.() || { enabled: false };
+  }
+
+  async triggerSync() {
+    if (!this.syncEngine) {
+      throw new Error('Sync engine not configured');
+    }
+    return this.syncEngine.sync();
+  }
+
+  // ========== Lifecycle ==========
 
   async shutdown() {
-    // Save intelligence models
+    console.log('[MemoryEngine] Shutting down...');
+    
     if (this.intelligenceEnabled) {
       await this.strategySelector?.saveModel?.();
-      console.log('[MemoryEngine] 💾 Intelligence models saved');
     }
     
-    if (this.watcher) await this.watcher.stop();
-    if (this.syncEngine) this.syncEngine.shutdown();
-    this.db.close();
+    await this.watcher?.stop?.();
+    this.syncEngine?.shutdown?.();
+    
+    this.memoryDb?.close?.();
+    this.codebaseDb?.close?.();
+    
     this.isInitialized = false;
+    console.log('[MemoryEngine] Shutdown complete');
   }
 }