@chongyan/autospec 1.0.1

package/plugins/hooks/lib/project-evolution/semantic-indexer.js

@@ -0,0 +1,480 @@
+/**
+ * Project-Evolve Semantic Indexer
+ * 语义索引器 - 负责代码语义向量生成、相似度计算、语义搜索
+ */
+
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+
+class SemanticIndexer {
+  constructor(projectRoot) {
+    this.projectRoot = projectRoot;
+    this.indexPath = path.join(projectRoot, '.autospec', 'memory', 'embeddings');
+    this.index = new Map();
+  }
+
+  /**
+   * 初始化索引
+   */
+  init() {
+    if (!fs.existsSync(this.indexPath)) {
+      fs.mkdirSync(this.indexPath, { recursive: true });
+    }
+
+    this.loadIndex();
+    return this;
+  }
+
+  /**
+   * 加载索引
+   */
+  loadIndex() {
+    const indexFile = path.join(this.indexPath, 'index.json');
+
+    if (fs.existsSync(indexFile)) {
+      const data = JSON.parse(fs.readFileSync(indexFile, 'utf8'));
+
+      for (const [key, value] of Object.entries(data)) {
+        this.index.set(key, value);
+      }
+    }
+  }
+
+  /**
+   * 保存索引
+   */
+  saveIndex() {
+    const indexFile = path.join(this.indexPath, 'index.json');
+    const data = Object.fromEntries(this.index);
+    fs.writeFileSync(indexFile, JSON.stringify(data, null, 2));
+  }
+
+  /**
+   * 生成代码语义向量（简化版）
+   * 实际项目中应使用 CodeBERT、GraphCodeBERT 等模型
+   */
+  generateCodeEmbedding(code, options = {}) {
+    // 简化实现：基于代码特征的哈希向量
+    // 实际应调用模型API或本地模型
+
+    const features = this.extractCodeFeatures(code, options);
+    const embedding = this.featuresToVector(features);
+
+    return embedding;
+  }
+
+  /**
+   * 提取代码特征
+   */
+  extractCodeFeatures(code, options = {}) {
+    const features = {
+      // 结构特征
+      has_functions: /function\s+\w+\s*\(/.test(code),
+      has_classes: /class\s+\w+/.test(code),
+      has_async: /\basync\b/.test(code),
+      has_await: /\bawait\b/.test(code),
+      has_imports: /\bimport\b/.test(code) || /\brequire\(/.test(code),
+      has_exports: /\bexport\b/.test(code) || /\bmodule\.exports\b/.test(code),
+
+      // 控制流特征
+      has_if: /\bif\s*\(/.test(code),
+      has_switch: /\bswitch\s*\(/.test(code),
+      has_for: /\bfor\s*\(/.test(code),
+      has_while: /\bwhile\s*\(/.test(code),
+      has_try: /\btry\s*\{/.test(code),
+      has_catch: /\bcatch\s*\(/.test(code),
+
+      // 数据操作特征
+      has_map: /\.map\s*\(/.test(code),
+      has_filter: /\.filter\s*\(/.test(code),
+      has_reduce: /\.reduce\s*\(/.test(code),
+      has_async_map: /Promise\.all\s*\(/.test(code),
+
+      // API特征
+      has_http: /\b(fetch|axios|request)\b/.test(code),
+      has_db: /\b(find|findOne|findMany|create|update|delete)\b/.test(code),
+      has_cache: /\b(cache|redis|memcached)\b/.test(code),
+
+      // 错误处理特征
+      has_error_handling: /\b(try|catch|throw|Error)\b/.test(code),
+      has_logging: /\b(console\.|log|debug|info|warn|error)\b/.test(code),
+
+      // 复杂度特征
+      nesting_depth: this.calculateNestingDepth(code),
+      function_count: (code.match(/function\s+\w+/g) || []).length,
+      line_count: code.split('\n').length
+    };
+
+    return features;
+  }
+
+  /**
+   * 计算嵌套深度
+   */
+  calculateNestingDepth(code) {
+    let maxDepth = 0;
+    let currentDepth = 0;
+
+    for (const char of code) {
+      if (char === '{' || char === '(' || char === '[') {
+        currentDepth++;
+        maxDepth = Math.max(maxDepth, currentDepth);
+      } else if (char === '}' || char === ')' || char === ']') {
+        currentDepth--;
+      }
+    }
+
+    return maxDepth;
+  }
+
+  /**
+   * 特征转换为向量（简化版）
+   */
+  featuresToVector(features) {
+    // 将特征对象转换为固定维度的向量
+    // 实际项目中应使用模型生成的高维向量
+
+    const vector = [];
+    const keys = Object.keys(features).sort();
+
+    for (const key of keys) {
+      const value = features[key];
+
+      if (typeof value === 'boolean') {
+        vector.push(value ? 1 : 0);
+      } else if (typeof value === 'number') {
+        // 归一化
+        vector.push(Math.min(value / 100, 1));
+      }
+    }
+
+    // 填充到固定维度（简化版128维）
+    while (vector.length < 128) {
+      vector.push(0);
+    }
+
+    return vector.slice(0, 128);
+  }
+
+  /**
+   * 生成文本语义向量
+   */
+  generateTextEmbedding(text) {
+    // 简化实现：基于词频的向量
+    // 实际应使用 BERT、Sentence-BERT 等模型
+
+    const words = text.toLowerCase().split(/\s+/);
+    const wordFreq = {};
+
+    for (const word of words) {
+      wordFreq[word] = (wordFreq[word] || 0) + 1;
+    }
+
+    // 转换为固定维度向量
+    const vector = new Array(128).fill(0);
+    const keys = Object.keys(wordFreq).sort();
+
+    for (let i = 0; i < Math.min(keys.length, 128); i++) {
+      vector[i] = wordFreq[keys[i]] / words.length;
+    }
+
+    return vector;
+  }
+
+  /**
+   * 计算余弦相似度
+   */
+  cosineSimilarity(vec1, vec2) {
+    if (vec1.length !== vec2.length) {
+      throw new Error('Vectors must have same dimension');
+    }
+
+    let dotProduct = 0;
+    let norm1 = 0;
+    let norm2 = 0;
+
+    for (let i = 0; i < vec1.length; i++) {
+      dotProduct += vec1[i] * vec2[i];
+      norm1 += vec1[i] * vec1[i];
+      norm2 += vec2[i] * vec2[i];
+    }
+
+    if (norm1 === 0 || norm2 === 0) {
+      return 0;
+    }
+
+    return dotProduct / (Math.sqrt(norm1) * Math.sqrt(norm2));
+  }
+
+  /**
+   * 索引代码文件
+   */
+  indexCodeFile(filePath, options = {}) {
+    const absolutePath = path.isAbsolute(filePath)
+      ? filePath
+      : path.join(this.projectRoot, filePath);
+
+    if (!fs.existsSync(absolutePath)) {
+      return null;
+    }
+
+    const content = fs.readFileSync(absolutePath, 'utf8');
+    const embedding = this.generateCodeEmbedding(content, options);
+
+    const entry = {
+      path: filePath,
+      type: 'code',
+      embedding: embedding,
+      indexed_at: new Date().toISOString(),
+      size: content.length,
+      hash: this.calculateHash(content)
+    };
+
+    this.index.set(filePath, entry);
+
+    return entry;
+  }
+
+  /**
+   * 索引记忆
+   */
+  indexMemory(memory) {
+    const embedding = this.generateTextEmbedding(
+      `${memory.meta.name} ${memory.meta.description}`
+    );
+
+    const entry = {
+      id: memory.id,
+      type: 'memory',
+      embedding: embedding,
+      indexed_at: new Date().toISOString(),
+      category: memory.meta.category
+    };
+
+    this.index.set(memory.id, entry);
+
+    return entry;
+  }
+
+  /**
+   * 语义搜索
+   */
+  search(query, options = {}) {
+    const queryEmbedding = typeof query === 'string'
+      ? this.generateTextEmbedding(query)
+      : query;
+
+    const results = [];
+    const threshold = options.threshold || 0.5;
+    const limit = options.limit || 10;
+
+    for (const [key, entry] of this.index) {
+      const similarity = this.cosineSimilarity(queryEmbedding, entry.embedding);
+
+      if (similarity >= threshold) {
+        results.push({
+          key: key,
+          type: entry.type,
+          similarity: similarity,
+          ...entry
+        });
+      }
+    }
+
+    // 按相似度排序
+    results.sort((a, b) => b.similarity - a.similarity);
+
+    return results.slice(0, limit);
+  }
+
+  /**
+   * 查找相似代码
+   */
+  findSimilarCode(code, options = {}) {
+    const codeEmbedding = this.generateCodeEmbedding(code, options);
+    return this.search(codeEmbedding, { ...options, type: 'code' });
+  }
+
+  /**
+   * 查找相关记忆
+   */
+  findRelatedMemories(memoryId, options = {}) {
+    const memory = this.index.get(memoryId);
+
+    if (!memory) {
+      return [];
+    }
+
+    const results = this.search(memory.embedding, {
+      ...options,
+      type: 'memory'
+    });
+
+    // 排除自身
+    return results.filter(r => r.key !== memoryId);
+  }
+
+  /**
+   * 聚类分析
+   */
+  cluster(options = {}) {
+    const clusters = new Map();
+    const threshold = options.threshold || 0.8;
+
+    // 简单的基于相似度的聚类
+    for (const [key, entry] of this.index) {
+      let assigned = false;
+
+      for (const [clusterId, cluster] of clusters) {
+        const centroid = this.calculateCentroid(cluster);
+        const similarity = this.cosineSimilarity(entry.embedding, centroid);
+
+        if (similarity >= threshold) {
+          cluster.push(entry);
+          assigned = true;
+          break;
+        }
+      }
+
+      if (!assigned) {
+        clusters.set(key, [entry]);
+      }
+    }
+
+    return Array.from(clusters.values());
+  }
+
+  /**
+   * 计算聚类中心
+   */
+  calculateCentroid(cluster) {
+    const dimension = cluster[0].embedding.length;
+    const centroid = new Array(dimension).fill(0);
+
+    for (const entry of cluster) {
+      for (let i = 0; i < dimension; i++) {
+        centroid[i] += entry.embedding[i];
+      }
+    }
+
+    for (let i = 0; i < dimension; i++) {
+      centroid[i] /= cluster.length;
+    }
+
+    return centroid;
+  }
+
+  /**
+   * 计算哈希
+   */
+  calculateHash(content) {
+    return crypto.createHash('sha256').update(content).digest('hex');
+  }
+
+  /**
+   * 批量索引目录
+   */
+  indexDirectory(dirPath, options = {}) {
+    const patterns = options.patterns || ['**/*.js', '**/*.ts', '**/*.jsx', '**/*.tsx'];
+    const results = [];
+
+    // 简化实现：递归遍历目录
+    const files = this.listFiles(dirPath, patterns);
+
+    for (const file of files) {
+      try {
+        const entry = this.indexCodeFile(file, options);
+        if (entry) {
+          results.push(entry);
+        }
+      } catch (error) {
+        console.error(`Failed to index ${file}:`, error.message);
+      }
+    }
+
+    this.saveIndex();
+
+    return results;
+  }
+
+  /**
+   * 列出文件
+   */
+  listFiles(dirPath, patterns) {
+    const files = [];
+
+    const traverse = (currentPath) => {
+      const entries = fs.readdirSync(currentPath, { withFileTypes: true });
+
+      for (const entry of entries) {
+        const fullPath = path.join(currentPath, entry.name);
+        const relativePath = path.relative(this.projectRoot, fullPath);
+
+        if (entry.isDirectory()) {
+          // 跳过 node_modules 等目录
+          if (['node_modules', '.git', 'dist', 'build', 'coverage'].includes(entry.name)) {
+            continue;
+          }
+          traverse(fullPath);
+        } else if (entry.isFile()) {
+          // 检查是否匹配模式
+          if (this.matchesPatterns(relativePath, patterns)) {
+            files.push(relativePath);
+          }
+        }
+      }
+    };
+
+    traverse(path.join(this.projectRoot, dirPath));
+
+    return files;
+  }
+
+  /**
+   * 检查文件是否匹配模式
+   */
+  matchesPatterns(filePath, patterns) {
+    for (const pattern of patterns) {
+      // 简化实现：只支持简单的 glob 模式
+      const regex = new RegExp(
+        pattern
+          .replace(/\*\*/g, '<<<DOUBLESTAR>>>')
+          .replace(/\*/g, '[^/]*')
+          .replace(/<<<DOUBLESTAR>>>/g, '.*')
+      );
+
+      if (regex.test(filePath)) {
+        return true;
+      }
+    }
+
+    return false;
+  }
+
+  /**
+   * 获取索引统计
+   */
+  getStats() {
+    const stats = {
+      total: this.index.size,
+      by_type: {}
+    };
+
+    for (const entry of this.index.values()) {
+      stats.by_type[entry.type] = (stats.by_type[entry.type] || 0) + 1;
+    }
+
+    return stats;
+  }
+
+  /**
+   * 清除索引
+   */
+  clear() {
+    this.index.clear();
+    this.saveIndex();
+  }
+}
+
+module.exports = SemanticIndexer;