@mrxkun/mcfast-mcp 4.0.14 → 4.1.0

package/src/memory/stores/memory-database.js

@@ -0,0 +1,425 @@
+/**
+ * Memory Database
+ * Lưu trữ index cho memory notes (Markdown source of truth)
+ * Sử dụng FTS5 cho full-text search + vectors cho semantic search
+ */
+
+import Database from 'better-sqlite3';
+import path from 'path';
+import fs from 'fs/promises';
+
+export class MemoryDatabase {
+  constructor(dbPath = null) {
+    this.dbPath = dbPath;
+    this.db = null;
+    this.isInitialized = false;
+  }
+
+  async initialize() {
+    if (this.isInitialized) return;
+
+    // Create directory if not exists
+    await fs.mkdir(path.dirname(this.dbPath), { recursive: true });
+    
+    this.db = new Database(this.dbPath);
+    this.db.pragma('journal_mode = WAL');
+    
+    this.createTables();
+    this.isInitialized = true;
+    
+    console.log(`[MemoryDatabase] Initialized at: ${this.dbPath}`);
+  }
+
+  createTables() {
+    // Chunks table - stores content from Markdown files
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS chunks (
+        id TEXT PRIMARY KEY,
+        file_path TEXT NOT NULL,
+        start_line INTEGER,
+        end_line INTEGER,
+        content TEXT NOT NULL,
+        content_hash TEXT NOT NULL,
+        chunk_type TEXT DEFAULT 'content',
+        created_at INTEGER,
+        updated_at INTEGER
+      );
+      
+      CREATE INDEX IF NOT EXISTS idx_chunks_file ON chunks(file_path);
+      CREATE INDEX IF NOT EXISTS idx_chunks_hash ON chunks(content_hash);
+      CREATE INDEX IF NOT EXISTS idx_chunks_updated ON chunks(updated_at);
+    `);
+
+    // FTS5 virtual table for full-text search
+    this.db.exec(`
+      CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
+        content,
+        content_rowid=id,
+        tokenize='porter'
+      );
+    `);
+
+    // Triggers to keep FTS5 in sync
+    this.db.exec(`
+      CREATE TRIGGER IF NOT EXISTS chunks_ai AFTER INSERT ON chunks BEGIN
+        INSERT INTO chunks_fts(rowid, content) VALUES (new.id, new.content);
+      END;
+      
+      CREATE TRIGGER IF NOT EXISTS chunks_ad AFTER DELETE ON chunks BEGIN
+        INSERT INTO chunks_fts(chunks_fts, rowid, content) VALUES ('delete', old.id, old.content);
+      END;
+      
+      CREATE TRIGGER IF NOT EXISTS chunks_au AFTER UPDATE ON chunks BEGIN
+        INSERT INTO chunks_fts(chunks_fts, rowid, content) VALUES ('delete', old.id, old.content);
+        INSERT INTO chunks_fts(rowid, content) VALUES (new.id, new.content);
+      END;
+    `);
+
+    // Vectors table - stores embeddings
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS embeddings (
+        chunk_id TEXT PRIMARY KEY,
+        embedding BLOB NOT NULL,
+        model TEXT,
+        dimensions INTEGER,
+        created_at INTEGER,
+        FOREIGN KEY (chunk_id) REFERENCES chunks(id) ON DELETE CASCADE
+      );
+      
+      CREATE INDEX IF NOT EXISTS idx_embeddings_model ON embeddings(model);
+    `);
+
+    // Embedding cache - avoid re-embedding unchanged content
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS embedding_cache (
+        content_hash TEXT PRIMARY KEY,
+        embedding BLOB NOT NULL,
+        model TEXT,
+        dimensions INTEGER,
+        created_at INTEGER
+      );
+    `);
+
+    // File metadata - track indexed files
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS files (
+        path TEXT PRIMARY KEY,
+        content_hash TEXT NOT NULL,
+        last_modified INTEGER,
+        file_size INTEGER,
+        indexed_at INTEGER
+      );
+      
+      CREATE INDEX IF NOT EXISTS idx_files_hash ON files(content_hash);
+    `);
+
+    // Search history - for analytics and optimization
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS search_history (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        query TEXT NOT NULL,
+        query_hash TEXT,
+        method TEXT,
+        results_count INTEGER,
+        duration_ms INTEGER,
+        timestamp INTEGER
+      );
+      
+      CREATE INDEX IF NOT EXISTS idx_search_timestamp ON search_history(timestamp);
+    `);
+  }
+
+  // ========== Chunk Operations ==========
+
+  insertChunk(chunk) {
+    const stmt = this.db.prepare(`
+      INSERT OR REPLACE INTO chunks 
+      (id, file_path, start_line, end_line, content, content_hash, chunk_type, created_at, updated_at)
+      VALUES ($id, $file_path, $start_line, $end_line, $content, $content_hash, $chunk_type, $created_at, $updated_at)
+    `);
+    
+    const now = Date.now();
+    return stmt.run({
+      ...chunk,
+      created_at: chunk.created_at || now,
+      updated_at: now
+    });
+  }
+
+  deleteChunksByFile(filePath) {
+    const stmt = this.db.prepare('DELETE FROM chunks WHERE file_path = ?');
+    return stmt.run(filePath);
+  }
+
+  getChunkById(chunkId) {
+    return this.db.prepare('SELECT * FROM chunks WHERE id = ?').get(chunkId);
+  }
+
+  getChunksByFile(filePath, limit = 100) {
+    return this.db.prepare('SELECT * FROM chunks WHERE file_path = ? ORDER BY start_line LIMIT ?')
+      .all(filePath, limit);
+  }
+
+  getRecentChunks(limit = 100) {
+    return this.db.prepare('SELECT * FROM chunks ORDER BY updated_at DESC LIMIT ?').all(limit);
+  }
+
+  // ========== Embedding Operations ==========
+
+  insertEmbedding(embedding) {
+    const stmt = this.db.prepare(`
+      INSERT OR REPLACE INTO embeddings (chunk_id, embedding, model, dimensions, created_at)
+      VALUES ($chunk_id, $embedding, $model, $dimensions, $created_at)
+    `);
+    return stmt.run({
+      ...embedding,
+      created_at: embedding.created_at || Date.now()
+    });
+  }
+
+  getEmbedding(chunkId) {
+    return this.db.prepare('SELECT * FROM embeddings WHERE chunk_id = ?').get(chunkId);
+  }
+
+  getAllEmbeddings() {
+    return this.db.prepare(`
+      SELECT e.*, c.content, c.file_path, c.start_line, c.end_line
+      FROM embeddings e
+      JOIN chunks c ON e.chunk_id = c.id
+    `).all();
+  }
+
+  // ========== Embedding Cache ==========
+
+  getCachedEmbedding(contentHash) {
+    return this.db.prepare('SELECT * FROM embedding_cache WHERE content_hash = ?').get(contentHash);
+  }
+
+  cacheEmbedding(contentHash, embedding, model, dimensions) {
+    const stmt = this.db.prepare(`
+      INSERT OR REPLACE INTO embedding_cache (content_hash, embedding, model, dimensions, created_at)
+      VALUES (?, ?, ?, ?, ?)
+    `);
+    return stmt.run(contentHash, embedding, model, dimensions, Date.now());
+  }
+
+  // ========== File Tracking ==========
+
+  upsertFile(filePath, contentHash, lastModified, fileSize) {
+    const stmt = this.db.prepare(`
+      INSERT OR REPLACE INTO files (path, content_hash, last_modified, file_size, indexed_at)
+      VALUES (?, ?, ?, ?, ?)
+    `);
+    return stmt.run(filePath, contentHash, lastModified, fileSize, Date.now());
+  }
+
+  getFile(filePath) {
+    return this.db.prepare('SELECT * FROM files WHERE path = ?').get(filePath);
+  }
+
+  isFileIndexed(filePath, contentHash) {
+    const file = this.getFile(filePath);
+    return file && file.content_hash === contentHash;
+  }
+
+  deleteFile(filePath) {
+    return this.db.prepare('DELETE FROM files WHERE path = ?').run(filePath);
+  }
+
+  // ========== Search Operations ==========
+
+  /**
+   * Full-text search using FTS5
+   */
+  searchFTS(query, limit = 20) {
+    const startTime = performance.now();
+    
+    // Use FTS5 rank for scoring (BM25)
+    const stmt = this.db.prepare(`
+      SELECT 
+        c.*,
+        rank as bm25_score
+      FROM chunks_fts fts
+      JOIN chunks c ON fts.rowid = c.id
+      WHERE chunks_fts MATCH ?
+      ORDER BY rank
+      LIMIT ?
+    `);
+    
+    const results = stmt.all(query, limit);
+    const duration = performance.now() - startTime;
+    
+    // Log search
+    this.logSearch(query, 'fts', results.length, duration);
+    
+    return {
+      results: results.map(r => ({
+        ...r,
+        // Convert BM25 rank to 0-1 score (lower rank = better match)
+        score: 1 / (1 + Math.max(0, r.bm25_score))
+      })),
+      metadata: {
+        method: 'fts5',
+        duration: duration.toFixed(2) + 'ms',
+        candidates: results.length
+      }
+    };
+  }
+
+  /**
+   * Hybrid search: Combine FTS5 + Vector
+   * Weight: Vector 0.7, BM25 0.3
+   */
+  searchHybrid(query, vectorResults, limit = 10) {
+    const startTime = performance.now();
+    const candidateMultiplier = 4;
+    const maxCandidates = limit * candidateMultiplier;
+    
+    // Get FTS results
+    const ftsResults = this.searchFTS(query, maxCandidates);
+    
+    // Normalize FTS scores to 0-1
+    const ftsMap = new Map();
+    ftsResults.results.forEach(r => {
+      ftsMap.set(r.id, r.score);
+    });
+    
+    // Normalize vector scores to 0-1
+    let maxVectorScore = 0;
+    vectorResults.forEach(r => {
+      if (r.similarity > maxVectorScore) maxVectorScore = r.similarity;
+    });
+    
+    // Combine results
+    const combined = new Map();
+    
+    // Add vector results
+    vectorResults.forEach(r => {
+      const normalizedVectorScore = maxVectorScore > 0 ? r.similarity / maxVectorScore : 0;
+      combined.set(r.chunk_id, {
+        chunk_id: r.chunk_id,
+        file_path: r.file_path,
+        start_line: r.start_line,
+        end_line: r.end_line,
+        content: r.content,
+        vectorScore: normalizedVectorScore,
+        textScore: ftsMap.get(r.chunk_id) || 0,
+        sources: ['vector']
+      });
+    });
+    
+    // Add FTS results not in vector
+    ftsResults.results.forEach(r => {
+      if (combined.has(r.id)) {
+        combined.get(r.id).textScore = r.score;
+        combined.get(r.id).sources.push('fts');
+      } else {
+        combined.set(r.id, {
+          chunk_id: r.id,
+          file_path: r.file_path,
+          start_line: r.start_line,
+          end_line: r.end_line,
+          content: r.content,
+          vectorScore: 0,
+          textScore: r.score,
+          sources: ['fts']
+        });
+      }
+    });
+    
+    // Calculate final scores (0.7 vector + 0.3 text)
+    const results = Array.from(combined.values()).map(r => ({
+      ...r,
+      finalScore: (0.7 * r.vectorScore) + (0.3 * r.textScore)
+    }));
+    
+    // Sort by final score and filter
+    results.sort((a, b) => b.finalScore - a.finalScore);
+    const filtered = results.filter(r => r.finalScore >= 0.35).slice(0, limit);
+    
+    const duration = performance.now() - startTime;
+    this.logSearch(query, 'hybrid', filtered.length, duration);
+    
+    return {
+      results: filtered,
+      metadata: {
+        method: 'hybrid',
+        duration: duration.toFixed(2) + 'ms',
+        vectorWeight: 0.7,
+        textWeight: 0.3,
+        minScore: 0.35,
+        candidates: results.length,
+        returned: filtered.length
+      }
+    };
+  }
+
+  // ========== Search History ==========
+
+  logSearch(query, method, resultsCount, durationMs) {
+    try {
+      const stmt = this.db.prepare(`
+        INSERT INTO search_history (query, query_hash, method, results_count, duration_ms, timestamp)
+        VALUES (?, ?, ?, ?, ?, ?)
+      `);
+      
+      const crypto = await import('crypto');
+      const queryHash = crypto.createHash('md5').update(query).digest('hex');
+      
+      stmt.run(query, queryHash, method, resultsCount, Math.round(durationMs), Date.now());
+    } catch (error) {
+      // Silent fail - don't break search for logging
+    }
+  }
+
+  getSearchStats(days = 7) {
+    const since = Date.now() - (days * 24 * 60 * 60 * 1000);
+    return this.db.prepare(`
+      SELECT 
+        method,
+        COUNT(*) as count,
+        AVG(duration_ms) as avg_duration,
+        AVG(results_count) as avg_results
+      FROM search_history
+      WHERE timestamp > ?
+      GROUP BY method
+    `).all(since);
+  }
+
+  // ========== Stats ==========
+
+  getStats() {
+    const files = this.db.prepare('SELECT COUNT(*) as count FROM files').get();
+    const chunks = this.db.prepare('SELECT COUNT(*) as count FROM chunks').get();
+    const embeddings = this.db.prepare('SELECT COUNT(*) as count FROM embeddings').get();
+    const cache = this.db.prepare('SELECT COUNT(*) as count FROM embedding_cache').get();
+    
+    return {
+      files: files.count,
+      chunks: chunks.count,
+      embeddings: embeddings.count,
+      cacheEntries: cache.count,
+      dbPath: this.dbPath
+    };
+  }
+
+  // ========== Maintenance ==========
+
+  vacuum() {
+    this.db.exec('VACUUM');
+    this.db.exec('ANALYZE');
+  }
+
+  clearCache() {
+    this.db.exec('DELETE FROM embedding_cache');
+  }
+
+  close() {
+    if (this.db) {
+      this.db.close();
+      this.isInitialized = false;
+    }
+  }
+}
+
+export default MemoryDatabase;

package/src/memory/utils/markdown-chunker.js

@@ -0,0 +1,242 @@
+/**
+ * Markdown Chunker
+ * Chia Markdown thành chunks với overlap
+ * Chunk size: ~400 tokens, overlap: 80 tokens
+ */
+
+import crypto from 'crypto';
+
+export class MarkdownChunker {
+  constructor(options = {}) {
+    // Target ~400 tokens (approx 1600 chars for English)
+    this.chunkSize = options.chunkSize || 1600;
+    // 80 tokens overlap (approx 320 chars)
+    this.overlap = options.overlap || 320;
+    // Respect headers - don't split within a header section if possible
+    this.respectHeaders = options.respectHeaders !== false;
+  }
+
+  /**
+   * Chunk Markdown content
+   * @param {string} content - Markdown content
+   * @param {string} filePath - Source file path
+   * @returns {Array} Array of chunks
+   */
+  chunk(content, filePath) {
+    const lines = content.split('\n');
+    const chunks = [];
+    let currentChunk = [];
+    let currentSize = 0;
+    let startLine = 0;
+
+    for (let i = 0; i < lines.length; i++) {
+      const line = lines[i];
+      const lineSize = line.length + 1; // +1 for newline
+
+      // Check if adding this line would exceed chunk size
+      if (currentSize + lineSize > this.chunkSize && currentChunk.length > 0) {
+        // Save current chunk
+        const chunkText = currentChunk.join('\n');
+        chunks.push(this.createChunk(chunkText, filePath, startLine + 1, i, content));
+
+        // Start new chunk with overlap
+        const overlapLines = this.getOverlapLines(currentChunk);
+        currentChunk = [...overlapLines, line];
+        currentSize = overlapLines.join('\n').length + lineSize;
+        startLine = i - overlapLines.length;
+      } else {
+        currentChunk.push(line);
+        currentSize += lineSize;
+      }
+    }
+
+    // Don't forget the last chunk
+    if (currentChunk.length > 0) {
+      const chunkText = currentChunk.join('\n');
+      chunks.push(this.createChunk(chunkText, filePath, startLine + 1, lines.length, content));
+    }
+
+    return chunks;
+  }
+
+  /**
+   * Get overlap lines from previous chunk
+   */
+  getOverlapLines(previousChunk) {
+    let overlapSize = 0;
+    const overlapLines = [];
+
+    // Take lines from end until we reach overlap size
+    for (let i = previousChunk.length - 1; i >= 0; i--) {
+      const line = previousChunk[i];
+      overlapLines.unshift(line);
+      overlapSize += line.length + 1;
+
+      if (overlapSize >= this.overlap) {
+        break;
+      }
+    }
+
+    return overlapLines;
+  }
+
+  /**
+   * Create a chunk object
+   */
+  createChunk(content, filePath, startLine, endLine, fullContent) {
+    const hash = crypto.createHash('md5').update(content).digest('hex');
+    const fullHash = crypto.createHash('md5').update(fullContent).digest('hex');
+
+    return {
+      id: this.generateChunkId(filePath, startLine, content),
+      filePath,
+      startLine,
+      endLine,
+      content: content.trim(),
+      contentHash: hash,
+      fullContentHash: fullHash,
+      tokenCount: this.estimateTokenCount(content),
+      chunkType: this.detectChunkType(content)
+    };
+  }
+
+  /**
+   * Generate unique chunk ID
+   */
+  generateChunkId(filePath, startLine, content) {
+    const hash = crypto.createHash('md5')
+      .update(`${filePath}:${startLine}:${content.substring(0, 100)}`)
+      .digest('hex')
+      .substring(0, 16);
+    return hash;
+  }
+
+  /**
+   * Estimate token count (rough approximation)
+   * ~4 chars per token for English
+   */
+  estimateTokenCount(text) {
+    return Math.ceil(text.length / 4);
+  }
+
+  /**
+   * Detect chunk type based on content
+   */
+  detectChunkType(content) {
+    const trimmed = content.trim();
+    
+    if (trimmed.startsWith('# ')) return 'h1';
+    if (trimmed.startsWith('## ')) return 'h2';
+    if (trimmed.startsWith('### ')) return 'h3';
+    if (trimmed.startsWith('```')) return 'code';
+    if (trimmed.startsWith('- ') || trimmed.startsWith('* ')) return 'list';
+    if (/^\d+\./.test(trimmed)) return 'numbered';
+    if (trimmed.startsWith('>')) return 'quote';
+    
+    return 'content';
+  }
+
+  /**
+   * Chunk with header-aware splitting
+   * Tries to keep headers with their content
+   */
+  chunkWithHeaders(content, filePath) {
+    if (!this.respectHeaders) {
+      return this.chunk(content, filePath);
+    }
+
+    const sections = this.splitByHeaders(content);
+    const chunks = [];
+    let currentChunk = [];
+    let currentSize = 0;
+    let startLine = 0;
+    let lineOffset = 0;
+
+    for (const section of sections) {
+      const sectionSize = section.content.length;
+
+      // If section is larger than chunk size, split it
+      if (sectionSize > this.chunkSize) {
+        // Save current chunk first
+        if (currentChunk.length > 0) {
+          const chunkText = currentChunk.join('\n\n');
+          chunks.push(this.createChunk(chunkText, filePath, startLine + 1, lineOffset, content));
+          currentChunk = [];
+          currentSize = 0;
+        }
+
+        // Split large section
+        const sectionChunks = this.chunk(section.content, filePath);
+        for (const chunk of sectionChunks) {
+          chunk.startLine += lineOffset;
+          chunk.endLine += lineOffset;
+          chunks.push(chunk);
+        }
+      }
+      // If adding this section would exceed chunk size, start new chunk
+      else if (currentSize + sectionSize > this.chunkSize && currentChunk.length > 0) {
+        const chunkText = currentChunk.join('\n\n');
+        chunks.push(this.createChunk(chunkText, filePath, startLine + 1, lineOffset, content));
+        
+        currentChunk = [section.content];
+        currentSize = sectionSize;
+        startLine = lineOffset;
+      } else {
+        currentChunk.push(section.content);
+        currentSize += sectionSize;
+      }
+
+      lineOffset += section.content.split('\n').length;
+    }
+
+    // Don't forget the last chunk
+    if (currentChunk.length > 0) {
+      const chunkText = currentChunk.join('\n\n');
+      chunks.push(this.createChunk(chunkText, filePath, startLine + 1, lineOffset, content));
+    }
+
+    return chunks;
+  }
+
+  /**
+   * Split content by headers
+   */
+  splitByHeaders(content) {
+    const lines = content.split('\n');
+    const sections = [];
+    let currentSection = [];
+    let currentLevel = 0;
+
+    for (const line of lines) {
+      const headerMatch = line.match(/^(#{1,6})\s/);
+      
+      if (headerMatch) {
+        // Save previous section
+        if (currentSection.length > 0) {
+          sections.push({
+            level: currentLevel,
+            content: currentSection.join('\n')
+          });
+        }
+        
+        // Start new section
+        currentLevel = headerMatch[1].length;
+        currentSection = [line];
+      } else {
+        currentSection.push(line);
+      }
+    }
+
+    // Don't forget the last section
+    if (currentSection.length > 0) {
+      sections.push({
+        level: currentLevel,
+        content: currentSection.join('\n')
+      });
+    }
+
+    return sections;
+  }
+}
+
+export default MarkdownChunker;