@mrxkun/mcfast-mcp 4.0.14 → 4.1.0

package/src/memory/stores/codebase-database.js

@@ -0,0 +1,418 @@
+/**
+ * Codebase Database
+ * Lưu trữ index cho codebase (files, facts, chunks)
+ * Separate from MemoryDatabase to keep code and notes separate
+ */
+
+import Database from 'better-sqlite3';
+import path from 'path';
+import fs from 'fs/promises';
+
+export class CodebaseDatabase {
+  constructor(dbPath = null) {
+    this.dbPath = dbPath;
+    this.db = null;
+    this.isInitialized = false;
+  }
+
+  async initialize() {
+    if (this.isInitialized) return;
+
+    await fs.mkdir(path.dirname(this.dbPath), { recursive: true });
+    
+    this.db = new Database(this.dbPath);
+    this.db.pragma('journal_mode = WAL');
+    
+    this.createTables();
+    this.isInitialized = true;
+    
+    console.log(`[CodebaseDatabase] Initialized at: ${this.dbPath}`);
+  }
+
+  createTables() {
+    // Files table
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS files (
+        id TEXT PRIMARY KEY,
+        path TEXT UNIQUE NOT NULL,
+        content_hash TEXT,
+        last_modified INTEGER,
+        language TEXT,
+        line_count INTEGER,
+        indexed_at INTEGER
+      );
+      
+      CREATE INDEX IF NOT EXISTS idx_files_path ON files(path);
+      CREATE INDEX IF NOT EXISTS idx_files_hash ON files(content_hash);
+      CREATE INDEX IF NOT EXISTS idx_files_language ON files(language);
+    `);
+
+    // Facts table (functions, classes, etc.)
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS facts (
+        id TEXT PRIMARY KEY,
+        file_id TEXT NOT NULL,
+        type TEXT NOT NULL,
+        name TEXT NOT NULL,
+        line_start INTEGER,
+        line_end INTEGER,
+        signature TEXT,
+        exported BOOLEAN,
+        documentation TEXT,
+        FOREIGN KEY (file_id) REFERENCES files(id) ON DELETE CASCADE
+      );
+      
+      CREATE INDEX IF NOT EXISTS idx_facts_file ON facts(file_id);
+      CREATE INDEX IF NOT EXISTS idx_facts_name ON facts(name);
+      CREATE INDEX IF NOT EXISTS idx_facts_type ON facts(type);
+    `);
+
+    // Chunks table (code segments)
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS chunks (
+        id TEXT PRIMARY KEY,
+        file_id TEXT NOT NULL,
+        content TEXT NOT NULL,
+        content_hash TEXT,
+        start_line INTEGER,
+        end_line INTEGER,
+        token_count INTEGER,
+        chunk_type TEXT DEFAULT 'code',
+        created_at INTEGER,
+        FOREIGN KEY (file_id) REFERENCES files(id) ON DELETE CASCADE
+      );
+      
+      CREATE INDEX IF NOT EXISTS idx_chunks_file ON chunks(file_id);
+      CREATE INDEX IF NOT EXISTS idx_chunks_hash ON chunks(content_hash);
+    `);
+
+    // FTS5 for code search
+    this.db.exec(`
+      CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
+        content,
+        content_rowid=id,
+        tokenize='porter'
+      );
+    `);
+
+    // Triggers to sync FTS5
+    this.db.exec(`
+      CREATE TRIGGER IF NOT EXISTS chunks_ai AFTER INSERT ON chunks BEGIN
+        INSERT INTO chunks_fts(rowid, content) VALUES (new.id, new.content);
+      END;
+      
+      CREATE TRIGGER IF NOT EXISTS chunks_ad AFTER DELETE ON chunks BEGIN
+        INSERT INTO chunks_fts(chunks_fts, rowid, content) VALUES ('delete', old.id, old.content);
+      END;
+      
+      CREATE TRIGGER IF NOT EXISTS chunks_au AFTER UPDATE ON chunks BEGIN
+        INSERT INTO chunks_fts(chunks_fts, rowid, content) VALUES ('delete', old.id, old.content);
+        INSERT INTO chunks_fts(rowid, content) VALUES (new.id, new.content);
+      END;
+    `);
+
+    // Embeddings for code chunks
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS embeddings (
+        chunk_id TEXT PRIMARY KEY,
+        embedding BLOB NOT NULL,
+        model TEXT,
+        dimensions INTEGER,
+        created_at INTEGER,
+        FOREIGN KEY (chunk_id) REFERENCES chunks(id) ON DELETE CASCADE
+      );
+      
+      CREATE INDEX IF NOT EXISTS idx_embeddings_model ON embeddings(model);
+    `);
+
+    // Edit history
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS edit_history (
+        id TEXT PRIMARY KEY,
+        timestamp INTEGER,
+        instruction TEXT,
+        files TEXT,
+        strategy TEXT,
+        success BOOLEAN,
+        diff_size INTEGER,
+        latency_ms INTEGER
+      );
+      
+      CREATE INDEX IF NOT EXISTS idx_edit_timestamp ON edit_history(timestamp);
+    `);
+
+    // Indexing progress (for initial scan)
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS indexing_progress (
+        id INTEGER PRIMARY KEY CHECK (id = 1),
+        total_files INTEGER DEFAULT 0,
+        indexed_files INTEGER DEFAULT 0,
+        failed_files INTEGER DEFAULT 0,
+        started_at INTEGER,
+        completed_at INTEGER,
+        is_complete BOOLEAN DEFAULT 0
+      );
+    `);
+  }
+
+  // ========== File Operations ==========
+
+  upsertFile(file) {
+    const stmt = this.db.prepare(`
+      INSERT OR REPLACE INTO files 
+      (id, path, content_hash, last_modified, language, line_count, indexed_at)
+      VALUES ($id, $path, $content_hash, $last_modified, $language, $line_count, $indexed_at)
+    `);
+    return stmt.run({
+      ...file,
+      indexed_at: Date.now()
+    });
+  }
+
+  getFileByPath(filePath) {
+    return this.db.prepare('SELECT * FROM files WHERE path = ?').get(filePath);
+  }
+
+  getFileById(fileId) {
+    return this.db.prepare('SELECT * FROM files WHERE id = ?').get(fileId);
+  }
+
+  deleteFile(fileId) {
+    return this.db.prepare('DELETE FROM files WHERE id = ?').run(fileId);
+  }
+
+  getAllFiles() {
+    return this.db.prepare('SELECT * FROM files ORDER BY path').all();
+  }
+
+  getFilesByLanguage(language) {
+    return this.db.prepare('SELECT * FROM files WHERE language = ?').all(language);
+  }
+
+  isFileIndexed(filePath, contentHash) {
+    const file = this.getFileByPath(filePath);
+    return file && file.content_hash === contentHash;
+  }
+
+  // ========== Fact Operations ==========
+
+  insertFact(fact) {
+    const stmt = this.db.prepare(`
+      INSERT OR REPLACE INTO facts 
+      (id, file_id, type, name, line_start, line_end, signature, exported, documentation)
+      VALUES ($id, $file_id, $type, $name, $line_start, $line_end, $signature, $exported, $documentation)
+    `);
+    return stmt.run(fact);
+  }
+
+  deleteFactsByFile(fileId) {
+    return this.db.prepare('DELETE FROM facts WHERE file_id = ?').run(fileId);
+  }
+
+  searchFacts(query, limit = 20) {
+    return this.db.prepare(`
+      SELECT f.*, files.path as file_path 
+      FROM facts f 
+      JOIN files ON f.file_id = files.id 
+      WHERE f.name LIKE ? 
+      LIMIT ?
+    `).all(`%${query}%`, limit);
+  }
+
+  getFactsByFile(fileId) {
+    return this.db.prepare('SELECT * FROM facts WHERE file_id = ?').all(fileId);
+  }
+
+  // ========== Chunk Operations ==========
+
+  insertChunk(chunk) {
+    const stmt = this.db.prepare(`
+      INSERT OR REPLACE INTO chunks 
+      (id, file_id, content, content_hash, start_line, end_line, token_count, chunk_type, created_at)
+      VALUES ($id, $file_id, $content, $content_hash, $start_line, $end_line, $token_count, $chunk_type, $created_at)
+    `);
+    return stmt.run({
+      ...chunk,
+      created_at: chunk.created_at || Date.now()
+    });
+  }
+
+  deleteChunksByFile(fileId) {
+    return this.db.prepare('DELETE FROM chunks WHERE file_id = ?').run(fileId);
+  }
+
+  getChunksByFile(fileId) {
+    return this.db.prepare(`
+      SELECT c.*, f.path as file_path
+      FROM chunks c
+      JOIN files f ON c.file_id = f.id
+      WHERE c.file_id = ?
+      ORDER BY c.start_line
+    `).all(fileId);
+  }
+
+  getAllChunksWithContent() {
+    return this.db.prepare(`
+      SELECT c.*, f.path as file_path, e.embedding
+      FROM chunks c
+      JOIN files f ON c.file_id = f.id
+      LEFT JOIN embeddings e ON c.id = e.chunk_id
+    `).all();
+  }
+
+  getRecentChunks(limit = 100) {
+    return this.db.prepare(`
+      SELECT c.*, f.path as file_path
+      FROM chunks c
+      JOIN files f ON c.file_id = f.id
+      ORDER BY c.created_at DESC
+      LIMIT ?
+    `).all(limit);
+  }
+
+  // ========== Embedding Operations ==========
+
+  insertEmbedding(embedding) {
+    const stmt = this.db.prepare(`
+      INSERT OR REPLACE INTO embeddings (chunk_id, embedding, model, dimensions, created_at)
+      VALUES ($chunk_id, $embedding, $model, $dimensions, $created_at)
+    `);
+    return stmt.run({
+      ...embedding,
+      created_at: embedding.created_at || Date.now()
+    });
+  }
+
+  getEmbedding(chunkId) {
+    return this.db.prepare('SELECT * FROM embeddings WHERE chunk_id = ?').get(chunkId);
+  }
+
+  getAllEmbeddings() {
+    return this.db.prepare(`
+      SELECT e.*, c.content, c.file_id, c.start_line, c.end_line, f.path as file_path
+      FROM embeddings e
+      JOIN chunks c ON e.chunk_id = c.id
+      JOIN files f ON c.file_id = f.id
+    `).all();
+  }
+
+  // ========== Search Operations ==========
+
+  searchFTS(query, limit = 20) {
+    const startTime = performance.now();
+    
+    const stmt = this.db.prepare(`
+      SELECT 
+        c.*,
+        f.path as file_path,
+        rank as bm25_score
+      FROM chunks_fts fts
+      JOIN chunks c ON fts.rowid = c.id
+      JOIN files f ON c.file_id = f.id
+      WHERE chunks_fts MATCH ?
+      ORDER BY rank
+      LIMIT ?
+    `);
+    
+    const results = stmt.all(query, limit);
+    const duration = performance.now() - startTime;
+    
+    return {
+      results: results.map(r => ({
+        chunk_id: r.id,
+        file_path: r.file_path,
+        start_line: r.start_line,
+        end_line: r.end_line,
+        content: r.content,
+        score: 1 / (1 + Math.max(0, r.bm25_score))
+      })),
+      metadata: {
+        method: 'fts5',
+        duration: duration.toFixed(2) + 'ms'
+      }
+    };
+  }
+
+  // ========== Edit History ==========
+
+  recordEdit(edit) {
+    const stmt = this.db.prepare(`
+      INSERT INTO edit_history (id, timestamp, instruction, files, strategy, success, diff_size, latency_ms)
+      VALUES ($id, $timestamp, $instruction, $files, $strategy, $success, $diff_size, $latency_ms)
+    `);
+    return stmt.run({
+      ...edit,
+      timestamp: edit.timestamp || Date.now()
+    });
+  }
+
+  getRecentEdits(limit = 100) {
+    return this.db.prepare('SELECT * FROM edit_history ORDER BY timestamp DESC LIMIT ?').all(limit);
+  }
+
+  // ========== Indexing Progress ==========
+
+  startIndexing(totalFiles) {
+    const stmt = this.db.prepare(`
+      INSERT OR REPLACE INTO indexing_progress 
+      (id, total_files, indexed_files, failed_files, started_at, completed_at, is_complete)
+      VALUES (1, ?, 0, 0, ?, NULL, 0)
+    `);
+    return stmt.run(totalFiles, Date.now());
+  }
+
+  incrementIndexed(failed = false) {
+    const column = failed ? 'failed_files' : 'indexed_files';
+    return this.db.prepare(`
+      UPDATE indexing_progress 
+      SET ${column} = ${column} + 1 
+      WHERE id = 1
+    `).run();
+  }
+
+  completeIndexing() {
+    return this.db.prepare(`
+      UPDATE indexing_progress 
+      SET completed_at = ?, is_complete = 1 
+      WHERE id = 1
+    `).run(Date.now());
+  }
+
+  getIndexingProgress() {
+    return this.db.prepare('SELECT * FROM indexing_progress WHERE id = 1').get();
+  }
+
+  // ========== Stats ==========
+
+  getStats() {
+    const files = this.db.prepare('SELECT COUNT(*) as count FROM files').get();
+    const facts = this.db.prepare('SELECT COUNT(*) as count FROM facts').get();
+    const chunks = this.db.prepare('SELECT COUNT(*) as count FROM chunks').get();
+    const embeddings = this.db.prepare('SELECT COUNT(*) as count FROM embeddings').get();
+    const edits = this.db.prepare('SELECT COUNT(*) as count FROM edit_history').get();
+    
+    return {
+      files: files.count,
+      facts: facts.count,
+      chunks: chunks.count,
+      embeddings: embeddings.count,
+      edits: edits.count,
+      dbPath: this.dbPath
+    };
+  }
+
+  // ========== Maintenance ==========
+
+  vacuum() {
+    this.db.exec('VACUUM');
+  }
+
+  close() {
+    if (this.db) {
+      this.db.close();
+      this.isInitialized = false;
+    }
+  }
+}
+
+export default CodebaseDatabase;