@mrxkun/mcfast-mcp 4.0.0 → 4.0.1

package/src/memory/utils/indexer.js

@@ -0,0 +1,118 @@
+/**
+ * Code Indexer
+ * Tổng hợp việc parse file, extract facts, chunk và embed
+ */
+
+import { parse } from '../../strategies/tree-sitter/index.js';
+import { detectLanguage } from '../../strategies/syntax-validator.js';
+import { Chunker } from './chunker.js';
+import { Embedder } from './embedder.js';
+import crypto from 'crypto';
+
+export class CodeIndexer {
+  constructor(options = {}) {
+    this.chunker = new Chunker(options.chunker);
+    this.embedder = new Embedder(options.embedder);
+  }
+
+  async indexFile(filePath, content) {
+    const startTime = Date.now();
+    
+    const language = detectLanguage(filePath);
+    const contentHash = crypto.createHash('md5').update(content).digest('hex');
+    
+    const ast = await this.parseAST(content, filePath, language);
+    const facts = ast ? await this.extractFacts(ast, filePath, language) : [];
+    const chunks = this.chunker.chunk(content, { filePath, language });
+    const embeddings = await this.generateEmbeddings(chunks, language);
+    
+    const duration = Date.now() - startTime;
+    
+    return {
+      file: {
+        id: this.generateFileId(filePath),
+        path: filePath,
+        content_hash: contentHash,
+        last_modified: Date.now(),
+        language,
+        line_count: content.split('\n').length
+      },
+      facts,
+      chunks,
+      embeddings,
+      duration
+    };
+  }
+
+  async parseAST(content, filePath, language) {
+    try {
+      return await parse(content, filePath);
+    } catch (error) {
+      console.warn(`[Indexer] Failed to parse ${filePath}:`, error.message);
+      return null;
+    }
+  }
+
+  async extractFacts(ast, filePath, language) {
+    const facts = [];
+    const fileId = this.generateFileId(filePath);
+
+    try {
+      const { QUERIES } = await import('../../strategies/tree-sitter/queries.js');
+      const queries = QUERIES[language];
+      
+      if (queries?.definitions) {
+        const captures = queries.definitions.captures(ast.rootNode);
+        
+        for (const capture of captures) {
+          if (capture.name === 'name') {
+            facts.push({
+              id: this.generateFactId(fileId, capture.node.text, 'function'),
+              file_id: fileId,
+              type: 'function',
+              name: capture.node.text,
+              line_start: capture.node.startPosition.row + 1,
+              line_end: capture.node.endPosition.row + 1,
+              signature: '',
+              exported: false,
+              confidence: 1.0
+            });
+          }
+        }
+      }
+    } catch (error) {
+      console.warn(`[Indexer] Failed to extract facts:`, error.message);
+    }
+
+    return facts;
+  }
+
+  async generateEmbeddings(chunks, language = 'javascript') {
+    const embeddings = [];
+    
+    try {
+      for (const chunk of chunks) {
+        const vector = await this.embedder.embedCode(chunk.content, language);
+        embeddings.push({
+          chunk_id: chunk.id,
+          embedding: Buffer.from(new Float32Array(vector).buffer),
+          model: 'simple-embedder'
+        });
+      }
+    } catch (error) {
+      console.error('[Indexer] Failed to generate embeddings:', error.message);
+    }
+
+    return embeddings;
+  }
+
+  generateFileId(filePath) {
+    return crypto.createHash('md5').update(filePath).digest('hex').substring(0, 16);
+  }
+
+  generateFactId(fileId, name, type) {
+    return crypto.createHash('md5').update(`${fileId}:${name}:${type}`).digest('hex').substring(0, 16);
+  }
+}
+
+export default CodeIndexer;

package/src/memory/utils/simple-embedder.js

@@ -0,0 +1,234 @@
+/**
+ * Simple Embedding Generator
+ * KHÔNG dùng LLM - Chỉ dùng thuật toán truyền thống
+ * Methods: TF-IDF, Bag-of-Words, Code-specific features
+ */
+
+export class SimpleEmbedder {
+  constructor(options = {}) {
+    this.dimension = options.dimension || 512;
+    this.vocabulary = new Map(); // word -> index
+    this.idf = new Map(); // word -> idf score
+    this.documentCount = 0;
+  }
+
+  /**
+   * TF-IDF Embedding
+   * Phương pháp truyền thống dùng trong search engines
+   */
+  embed(text) {
+    const vector = new Array(this.dimension).fill(0);
+    
+    // Tokenize code
+    const tokens = this.tokenize(text);
+    
+    // Calculate TF (Term Frequency)
+    const tf = new Map();
+    tokens.forEach(token => {
+      tf.set(token, (tf.get(token) || 0) + 1);
+    });
+    
+    // Normalize TF
+    const maxTf = Math.max(...tf.values());
+    tf.forEach((count, token) => {
+      tf.set(token, count / maxTf);
+    });
+    
+    // Create embedding vector
+    tf.forEach((freq, token) => {
+      const index = this.getTokenIndex(token);
+      const idf = this.idf.get(token) || 1;
+      vector[index % this.dimension] = freq * idf;
+    });
+    
+    // Normalize vector
+    return this.normalize(vector);
+  }
+
+  /**
+   * Code-specific embedding
+   * Tập trung vào đặc trưng của code
+   */
+  embedCode(code, language = 'javascript') {
+    const vector = new Array(this.dimension).fill(0);
+    
+    // 1. Tokenize và tách từ khóa
+    const tokens = this.tokenize(code);
+    
+    // 2. Trích xuất đặc trưng code
+    const features = this.extractCodeFeatures(code, language);
+    
+    // 3. Kết hợp features vào vector
+    let index = 0;
+    
+    // Function names (vị trí 0-100)
+    features.functions.forEach((func, i) => {
+      const hash = this.hashString(func) % 100;
+      vector[hash] = 1.0;
+    });
+    
+    // Variable names (vị trí 100-200)
+    features.variables.forEach((variable, i) => {
+      const hash = 100 + (this.hashString(variable) % 100);
+      vector[hash] = 0.8;
+    });
+    
+    // Class names (vị trí 200-300)
+    features.classes.forEach((cls, i) => {
+      const hash = 200 + (this.hashString(cls) % 100);
+      vector[hash] = 1.0;
+    });
+    
+    // Keywords (vị trí 300-400)
+    const keywords = this.getKeywords(tokens, language);
+    keywords.forEach((keyword, i) => {
+      const hash = 300 + (this.hashString(keyword) % 100);
+      vector[hash] = 0.6;
+    });
+    
+    // Import/Export patterns (vị trí 400-500)
+    features.imports.forEach((imp, i) => {
+      const hash = 400 + (this.hashString(imp) % 100);
+      vector[hash] = 0.9;
+    });
+    
+    // Code structure (vị trí 500-512)
+    vector[500] = features.hasAsync ? 1.0 : 0;
+    vector[501] = features.hasClass ? 1.0 : 0;
+    vector[502] = features.hasExport ? 1.0 : 0;
+    vector[503] = features.lineCount / 1000; // Normalized
+    
+    return this.normalize(vector);
+  }
+
+  /**
+   * Tokenize text/code
+   */
+  tokenize(text) {
+    // Tách từ theo camelCase, snake_case, và ký tự đặc biệt
+    return text
+      .replace(/([a-z])([A-Z])/g, '$1 $2') // camelCase → camel Case
+      .replace(/_/g, ' ') // snake_case → snake case
+      .toLowerCase()
+      .match(/[a-z]+/g) || [];
+  }
+
+  /**
+   * Trích xuất đặc trưng từ code
+   */
+  extractCodeFeatures(code, language) {
+    const features = {
+      functions: [],
+      variables: [],
+      classes: [],
+      imports: [],
+      hasAsync: /\basync\b/.test(code),
+      hasClass: /\bclass\b/.test(code),
+      hasExport: /\bexport\b/.test(code),
+      lineCount: code.split('\n').length
+    };
+    
+    // Extract function names
+    const funcMatches = code.matchAll(/(?:function|const|let|var)\s+(\w+)\s*[=\(]/g);
+    for (const match of funcMatches) {
+      features.functions.push(match[1]);
+    }
+    
+    // Extract arrow functions
+    const arrowMatches = code.matchAll(/(?:const|let|var)\s+(\w+)\s*=\s*\(?.*\)?\s*=>/g);
+    for (const match of arrowMatches) {
+      features.functions.push(match[1]);
+    }
+    
+    // Extract class names
+    const classMatches = code.matchAll(/class\s+(\w+)/g);
+    for (const match of classMatches) {
+      features.classes.push(match[1]);
+    }
+    
+    // Extract imports
+    const importMatches = code.matchAll(/import\s+(?:{?\s*([^{}]+)\s*}?\s+from\s+)?['"]([^'"]+)['"]/g);
+    for (const match of importMatches) {
+      if (match[1]) {
+        features.imports.push(...match[1].split(',').map(s => s.trim()));
+      }
+      features.imports.push(match[2]);
+    }
+    
+    // Extract variable names (simple heuristic)
+    const varMatches = code.matchAll(/(?:const|let|var)\s+(\w+)/g);
+    for (const match of varMatches) {
+      if (!features.functions.includes(match[1])) {
+        features.variables.push(match[1]);
+      }
+    }
+    
+    return features;
+  }
+
+  /**
+   * Lấy keywords quan trọng
+   */
+  getKeywords(tokens, language) {
+    const codeKeywords = {
+      javascript: ['function', 'class', 'const', 'let', 'var', 'async', 'await', 'return', 'if', 'for', 'while', 'import', 'export'],
+      typescript: ['interface', 'type', 'enum', 'namespace', 'extends', 'implements', 'function', 'class', 'const', 'let', 'var', 'async', 'await'],
+      python: ['def', 'class', 'import', 'from', 'return', 'if', 'for', 'while'],
+    };
+    
+    const keywords = codeKeywords[language] || codeKeywords.javascript;
+    return tokens.filter(token => keywords.includes(token));
+  }
+
+  /**
+   * Hash string thành số
+   */
+  hashString(str) {
+    let hash = 0;
+    for (let i = 0; i < str.length; i++) {
+      const char = str.charCodeAt(i);
+      hash = ((hash << 5) - hash) + char;
+      hash = hash & hash;
+    }
+    return Math.abs(hash);
+  }
+
+  /**
+   * Lấy index cho token trong vocabulary
+   */
+  getTokenIndex(token) {
+    if (!this.vocabulary.has(token)) {
+      this.vocabulary.set(token, this.vocabulary.size);
+    }
+    return this.vocabulary.get(token);
+  }
+
+  /**
+   * Normalize vector (L2 normalization)
+   */
+  normalize(vector) {
+    const magnitude = Math.sqrt(vector.reduce((sum, val) => sum + val * val, 0));
+    if (magnitude === 0) return vector;
+    return vector.map(val => val / magnitude);
+  }
+
+  /**
+   * Tính cosine similarity giữa 2 embeddings
+   */
+  cosineSimilarity(embedding1, embedding2) {
+    let dotProduct = 0;
+    for (let i = 0; i < embedding1.length; i++) {
+      dotProduct += embedding1[i] * embedding2[i];
+    }
+    return dotProduct; // Vectors đã normalize nên chỉ cần dot product
+  }
+
+  /**
+   * Batch embeddings
+   */
+  embedBatch(texts) {
+    return texts.map(text => this.embedCode(text));
+  }
+}
+
+export default SimpleEmbedder;

package/src/memory/utils/smart-router.js

@@ -0,0 +1,344 @@
+/**
+ * Smart Router - Intelligent Local vs Mercury Decision Engine
+ * 
+ * Tự động quyết định khi nào dùng local (90%) vs Mercury (95%)
+ * dựa trên: query complexity, cost optimization, và user preferences
+ */
+
+export class SmartRouter {
+  constructor(options = {}) {
+    this.dashboardClient = options.dashboardClient || null;
+    this.localAccuracy = 0.90;
+    this.mercuryAccuracy = 0.95;
+    this.costPerMercuryCall = 0.01; // $0.01 per call
+    
+    // User configuration từ Dashboard
+    this.config = {
+      enableMercury: false,
+      smartRoutingMode: 'auto', // 'always_local', 'auto', 'always_mercury'
+      rerankThreshold: 0.7,
+      monthlyBudget: 20,
+      currentUsage: 0
+    };
+    
+    // Query analysis patterns
+    this.complexityPatterns = {
+      high: [
+        /\b(find|search|locate)\b.*\b(all|every|each)\b/i,
+        /\b(compare|difference|versus|vs)\b/i,
+        /\b(relationship|connection|link|reference)\b/i,
+        /\b(impact|affect|change|modify)\b.*\b(multiple|many|several)\b/i,
+        /\b(best|optimal|recommend|suggest)\b/i,
+        /\b(why|how|explain|understand)\b/i,
+        /\b(similar|related|like)\b/i,
+        /\b(dependencies|imports|uses)\b/i,
+        /\b(refactor|restructure|reorganize)\b/i,
+        /\b(natural language|plain text|sentence)\b/i
+      ],
+      medium: [
+        /\b(update|change|modify|edit)\b/i,
+        /\b(fix|correct|repair|solve)\b/i,
+        /\b(add|create|new|insert)\b/i,
+        /\b(remove|delete|clear)\b/i,
+        /\b(validate|check|verify|test)\b/i,
+        /\b(handle|process|manage)\b/i
+      ]
+    };
+    
+    // Historical performance tracking
+    this.history = {
+      localSuccess: 0,
+      localTotal: 0,
+      mercurySuccess: 0,
+      mercuryTotal: 0,
+      queryTypes: new Map()
+    };
+  }
+
+  /**
+   * Load configuration từ Dashboard
+   */
+  async loadConfig() {
+    if (!this.dashboardClient) return;
+    
+    try {
+      const config = await this.dashboardClient.getSearchConfig();
+      this.config = {
+        enableMercury: config.enable_mercury_rerank ?? false,
+        smartRoutingMode: config.smart_routing_mode ?? 'auto',
+        rerankThreshold: config.rerank_threshold ?? 0.7,
+        monthlyBudget: config.monthly_budget ?? 20,
+        currentUsage: config.usage_this_month ?? 0
+      };
+    } catch (error) {
+      console.warn('[SmartRouter] Failed to load config:', error.message);
+      // Fallback: use default config
+    }
+  }
+
+  /**
+   * Main decision method - chọn local hay Mercury
+   */
+  async shouldUseMercury(query, localResults = null) {
+    // 1. Kiểm tra cơ bản
+    if (!this.config.enableMercury) {
+      return { useMercury: false, reason: 'Mercury disabled in settings' };
+    }
+    
+    // 2. Check mode
+    if (this.config.smartRoutingMode === 'always_local') {
+      return { useMercury: false, reason: 'Mode: always_local' };
+    }
+    
+    if (this.config.smartRoutingMode === 'always_mercury') {
+      const withinBudget = await this.checkBudget();
+      if (withinBudget) {
+        return { useMercury: true, reason: 'Mode: always_mercury' };
+      }
+      return { useMercury: false, reason: 'Budget exceeded, fallback to local' };
+    }
+    
+    // 3. Auto mode - smart decision
+    return this.makeSmartDecision(query, localResults);
+  }
+
+  /**
+   * Smart decision logic
+   */
+  async makeSmartDecision(query, localResults) {
+    const scores = {
+      queryComplexity: this.analyzeQueryComplexity(query),
+      confidenceGap: this.calculateConfidenceGap(localResults),
+      budgetStatus: await this.checkBudget() ? 1 : 0,
+      historicalSuccess: this.getHistoricalSuccess(query),
+      urgency: this.analyzeUrgency(query)
+    };
+    
+    // Weighted scoring
+    const totalScore = 
+      (scores.queryComplexity * 0.30) +
+      (scores.confidenceGap * 0.25) +
+      (scores.budgetStatus * 0.20) +
+      (scores.historicalSuccess * 0.15) +
+      (scores.urgency * 0.10);
+    
+    // Decision
+    const useMercury = totalScore >= this.config.rerankThreshold;
+    
+    return {
+      useMercury,
+      reason: useMercury ? 
+        `Complex query detected (score: ${totalScore.toFixed(2)})` : 
+        `Simple query, local sufficient (score: ${totalScore.toFixed(2)})`,
+      scores,
+      confidence: totalScore
+    };
+  }
+
+  /**
+   * Analyze query complexity (0-1)
+   */
+  analyzeQueryComplexity(query) {
+    const queryLower = query.toLowerCase();
+    let complexity = 0.5; // Base
+    
+    // Check high complexity patterns
+    for (const pattern of this.complexityPatterns.high) {
+      if (pattern.test(queryLower)) {
+        complexity += 0.15;
+      }
+    }
+    
+    // Check medium complexity patterns
+    for (const pattern of this.complexityPatterns.medium) {
+      if (pattern.test(queryLower)) {
+        complexity += 0.05;
+      }
+    }
+    
+    // Query length factor
+    const wordCount = query.split(/\s+/).length;
+    if (wordCount > 10) complexity += 0.1;
+    if (wordCount > 20) complexity += 0.1;
+    
+    // Multiple operations
+    const operations = ['create', 'update', 'delete', 'find', 'validate', 'handle'];
+    const opCount = operations.filter(op => queryLower.includes(op)).length;
+    complexity += opCount * 0.05;
+    
+    return Math.min(1, complexity);
+  }
+
+  /**
+   * Calculate confidence gap from local results
+   */
+  calculateConfidenceGap(localResults) {
+    if (!localResults || localResults.length === 0) {
+      return 1.0; // No results = need Mercury
+    }
+    
+    // Check top result confidence
+    const topScore = localResults[0].score || localResults[0].similarity || 0;
+    
+    // Check score distribution
+    if (localResults.length >= 2) {
+      const secondScore = localResults[1].score || localResults[1].similarity || 0;
+      const gap = topScore - secondScore;
+      
+      // If gap is small, need Mercury to differentiate
+      if (gap < 0.1) return 0.8;
+      if (gap < 0.2) return 0.5;
+    }
+    
+    // Low confidence in top result
+    if (topScore < 0.6) return 0.7;
+    if (topScore < 0.8) return 0.4;
+    
+    return 0.2; // High confidence, no need for Mercury
+  }
+
+  /**
+   * Check if within budget
+   */
+  async checkBudget() {
+    if (this.config.monthlyBudget === 0) return true; // Unlimited
+    
+    const projectedCost = (this.config.currentUsage + 1) * this.costPerMercuryCall;
+    const remaining = this.config.monthlyBudget - projectedCost;
+    
+    // Use Mercury if we have 20% budget remaining
+    return remaining > (this.config.monthlyBudget * 0.2);
+  }
+
+  /**
+   * Get historical success rate for similar queries
+   */
+  getHistoricalSuccess(query) {
+    // Extract query type
+    const queryType = this.extractQueryType(query);
+    const history = this.history.queryTypes.get(queryType);
+    
+    if (!history || history.total < 5) {
+      return 0.5; // Neutral if not enough data
+    }
+    
+    const successRate = history.success / history.total;
+    
+    // If local has high success rate, prefer local
+    if (successRate > 0.8) return 0.2;
+    if (successRate > 0.6) return 0.4;
+    
+    // Low success rate, prefer Mercury
+    return 0.8;
+  }
+
+  /**
+   * Analyze urgency (0-1)
+   */
+  analyzeUrgency(query) {
+    const queryLower = query.toLowerCase();
+    
+    // Urgent keywords
+    const urgentWords = ['fix', 'bug', 'error', 'crash', 'urgent', 'critical', 'asap'];
+    if (urgentWords.some(w => queryLower.includes(w))) {
+      return 0.9; // Use best accuracy (Mercury)
+    }
+    
+    // Exploration keywords
+    const exploreWords = ['explore', 'understand', 'learn', 'research', 'investigate'];
+    if (exploreWords.some(w => queryLower.includes(w))) {
+      return 0.3; // Can use local, not urgent
+    }
+    
+    return 0.5;
+  }
+
+  /**
+   * Extract query type for history tracking
+   */
+  extractQueryType(query) {
+    const queryLower = query.toLowerCase();
+    
+    if (/\b(find|search|locate|get)\b/.test(queryLower)) return 'search';
+    if (/\b(update|change|modify|edit)\b/.test(queryLower)) return 'update';
+    if (/\b(create|add|new|insert)\b/.test(queryLower)) return 'create';
+    if (/\b(delete|remove|clear)\b/.test(queryLower)) return 'delete';
+    if (/\b(fix|correct|repair)\b/.test(queryLower)) return 'fix';
+    if (/\b(validate|check|verify|test)\b/.test(queryLower)) return 'validate';
+    if (/\b(refactor|restructure)\b/.test(queryLower)) return 'refactor';
+    
+    return 'general';
+  }
+
+  /**
+   * Record result for learning
+   */
+  recordResult(query, usedMercury, success) {
+    const queryType = this.extractQueryType(query);
+    
+    // Update query type history
+    if (!this.history.queryTypes.has(queryType)) {
+      this.history.queryTypes.set(queryType, { success: 0, total: 0 });
+    }
+    
+    const history = this.history.queryTypes.get(queryType);
+    history.total++;
+    if (success) history.success++;
+    
+    // Update global stats
+    if (usedMercury) {
+      this.history.mercuryTotal++;
+      if (success) this.history.mercurySuccess++;
+    } else {
+      this.history.localTotal++;
+      if (success) this.history.localSuccess++;
+    }
+  }
+
+  /**
+   * Log usage to Dashboard
+   */
+  async logUsage(query, usedMercury, duration) {
+    if (!this.dashboardClient) return;
+    
+    try {
+      await this.dashboardClient.logSearchUsage({
+        query,
+        usedMercury,
+        duration,
+        timestamp: Date.now()
+      });
+    } catch (error) {
+      console.warn('[SmartRouter] Failed to log usage:', error.message);
+    }
+  }
+
+  /**
+   * Get stats
+   */
+  getStats() {
+    const localRate = this.history.localTotal > 0 ? 
+      this.history.localSuccess / this.history.localTotal : 0;
+    const mercuryRate = this.history.mercuryTotal > 0 ?
+      this.history.mercurySuccess / this.history.mercuryTotal : 0;
+    
+    return {
+      config: this.config,
+      history: {
+        local: {
+          total: this.history.localTotal,
+          success: this.history.localSuccess,
+          rate: localRate
+        },
+        mercury: {
+          total: this.history.mercuryTotal,
+          success: this.history.mercurySuccess,
+          rate: mercuryRate
+        }
+      },
+      estimatedMonthlyCost: this.history.mercuryTotal * this.costPerMercuryCall
+    };
+  }
+}
+
+export default SmartRouter;