@chongyan/autospec 1.0.1

package/plugins/hooks/lib/benchmarks/evaluator.js

@@ -0,0 +1,605 @@
+/**
+ * Benchmarks Evaluator
+ * 评测执行器 - 负责执行 benchmarks 评测，计算评分，生成报告
+ */
+
+const fs = require('fs');
+const path = require('path');
+const yaml = require('js-yaml');
+
+class BenchmarkEvaluator {
+  constructor(projectRoot) {
+    this.projectRoot = projectRoot;
+    this.resultsPath = path.join(projectRoot, '.autospec', 'benchmarks', 'results');
+    this.benchmarksPath = path.join(projectRoot, '.autospec', 'benchmarks');
+  }
+
+  /**
+   * 初始化评测器
+   */
+  init() {
+    if (!fs.existsSync(this.resultsPath)) {
+      fs.mkdirSync(this.resultsPath, { recursive: true });
+    }
+    return this;
+  }
+
+  /**
+   * 加载 benchmarks
+   */
+  loadBenchmarks(stage) {
+    const benchmarks = [];
+
+    // 加载自动生成的 benchmarks
+    const autoGenPath = path.join(this.benchmarksPath, 'auto-generated');
+    if (fs.existsSync(autoGenPath)) {
+      const files = fs.readdirSync(autoGenPath)
+        .filter(f => f.startsWith(`${stage}-`) && f.endsWith('.yaml'))
+        .sort()
+        .reverse(); // 最新的在前
+
+      if (files.length > 0) {
+        const latestFile = path.join(autoGenPath, files[0]);
+        const content = fs.readFileSync(latestFile, 'utf8');
+        const data = yaml.load(content);
+        if (data && data.test_cases) {
+          benchmarks.push(...data.test_cases);
+        }
+      }
+    }
+
+    // 加载自定义 benchmarks
+    const customPath = path.join(this.benchmarksPath, 'custom');
+    if (fs.existsSync(customPath)) {
+      const files = fs.readdirSync(customPath)
+        .filter(f => f.startsWith(`${stage}-`) && f.endsWith('.yaml'));
+
+      for (const file of files) {
+        const content = fs.readFileSync(path.join(customPath, file), 'utf8');
+        const data = yaml.load(content);
+        if (data && data.test_cases) {
+          benchmarks.push(...data.test_cases);
+        }
+      }
+    }
+
+    // 去重（基于 id）
+    const seen = new Set();
+    return benchmarks.filter(b => {
+      if (seen.has(b.id)) return false;
+      seen.add(b.id);
+      return true;
+    });
+  }
+
+  /**
+   * 执行评测
+   */
+  async evaluate(stage, options = {}) {
+    const benchmarks = this.loadBenchmarks(stage);
+    const results = [];
+    const startTime = Date.now();
+
+    // 执行每个 benchmark
+    for (const benchmark of benchmarks) {
+      const result = await this.evaluateBenchmark(benchmark, stage);
+      results.push(result);
+    }
+
+    // 计算综合得分
+    const summary = this.calculateSummary(results);
+    const duration = Date.now() - startTime;
+
+    // 生成报告
+    const report = {
+      run_id: this.generateRunId(),
+      stage,
+      timestamp: new Date().toISOString(),
+      trigger: options.trigger || 'manual',
+      summary: {
+        ...summary,
+        duration_ms: duration
+      },
+      results,
+      recommendations: this.generateRecommendations(results),
+      metrics: {
+        total_duration_ms: duration,
+        benchmarks_count: benchmarks.length,
+        layer1_duration_ms: results.reduce((sum, r) => sum + (r.layer1_duration_ms || 0), 0),
+        layer2_duration_ms: results.reduce((sum, r) => sum + (r.layer2_duration_ms || 0), 0)
+      }
+    };
+
+    // 保存报告
+    this.saveReport(report);
+
+    return report;
+  }
+
+  /**
+   * 执行单个 benchmark
+   */
+  async evaluateBenchmark(benchmark, stage) {
+    const result = {
+      benchmark_id: benchmark.id,
+      name: benchmark.name,
+      status: 'pending',
+      score: 0,
+      layer1_result: null,
+      layer2_result: null,
+      issues: [],
+      layer1_duration_ms: 0,
+      layer2_duration_ms: 0
+    };
+
+    const startTime = Date.now();
+
+    try {
+      // Step 1: Layer 1 确定性验证
+      const layer1Start = Date.now();
+      result.layer1_result = await this.executeLayer1(benchmark, stage);
+      result.layer1_duration_ms = Date.now() - layer1Start;
+
+      // Step 2: Layer 2 AI 审查（如果 Layer 1 通过或需要深度评估）
+      if (result.layer1_result.status === 'passed' || benchmark.force_layer2) {
+        const layer2Start = Date.now();
+        result.layer2_result = await this.executeLayer2(benchmark, stage);
+        result.layer2_duration_ms = Date.now() - layer2Start;
+      }
+
+      // 计算最终状态
+      if (result.layer1_result.status === 'failed') {
+        result.status = 'failed';
+        result.score = result.layer1_result.score || 0;
+      } else if (result.layer2_result && result.layer2_result.status === 'failed') {
+        result.status = 'failed';
+        result.score = result.layer2_result.score || 50;
+      } else {
+        result.status = 'passed';
+        result.score = result.layer2_result?.score || result.layer1_result.score || 100;
+      }
+
+      // 收集问题
+      result.issues = this.collectIssues(result);
+
+    } catch (error) {
+      result.status = 'error';
+      result.error = error.message;
+      result.score = 0;
+    }
+
+    return result;
+  }
+
+  /**
+   * 执行 Layer 1 验证
+   */
+  async executeLayer1(benchmark, stage) {
+    const result = {
+      status: 'passed',
+      score: 100,
+      checks: [],
+      details: {}
+    };
+
+    const criteria = benchmark.criteria || [];
+    let passedCount = 0;
+
+    for (const criterion of criteria) {
+      const checkResult = await this.evaluateCriterion(criterion, stage);
+      result.checks.push(checkResult);
+
+      if (checkResult.passed) {
+        passedCount++;
+      } else {
+        result.status = 'failed';
+      }
+    }
+
+    // 计算 Layer 1 得分
+    if (criteria.length > 0) {
+      result.score = (passedCount / criteria.length) * 100;
+    }
+
+    return result;
+  }
+
+  /**
+   * 评估单个准则
+   */
+  async evaluateCriterion(criterion, stage) {
+    const check = {
+      criterion,
+      passed: false,
+      actual: null,
+      expected: null
+    };
+
+    // 解析准则类型
+    if (typeof criterion === 'string') {
+      // 字符串格式的准则
+      if (criterion.includes('>=')) {
+        // 数量比较，如 "澄清问题数量 >= 3"
+        const match = criterion.match(/(.+)>=\s*(\d+)/);
+        if (match) {
+          const [, field, threshold] = match;
+          check.actual = await this.getFieldValue(field.trim(), stage);
+          check.expected = parseInt(threshold, 10);
+          check.passed = check.actual >= check.expected;
+        }
+      } else if (criterion.includes('存在')) {
+        // 文件存在性检查
+        const filePattern = criterion.replace(/.*存在/, '').trim();
+        check.actual = await this.checkFileExists(filePattern);
+        check.expected = true;
+        check.passed = check.actual === true;
+      } else {
+        // 其他字符串准则，标记为需要 Layer 2
+        check.passed = true; // 默认通过，交给 Layer 2
+        check.needsLayer2 = true;
+      }
+    } else if (typeof criterion === 'object') {
+      // 对象格式的准则
+      check.passed = await this.evaluateObjectCriterion(criterion, stage);
+    }
+
+    return check;
+  }
+
+  /**
+   * 获取字段值
+   */
+  async getFieldValue(field, stage) {
+    // 根据字段名从不同来源获取值
+    const fieldMappings = {
+      '澄清问题数量': () => this.countClarificationQuestions(),
+      '字段完整率': () => this.calculateFieldCompleteness(stage),
+      '测试覆盖率': () => this.getTestCoverage(),
+      '复杂度': () => this.getCodeComplexity(),
+      '行覆盖率': () => this.getLineCoverage(),
+      '分支覆盖率': () => this.getBranchCoverage()
+    };
+
+    const getter = fieldMappings[field];
+    if (getter) {
+      return await getter();
+    }
+
+    return 0;
+  }
+
+  /**
+   * 检查文件是否存在
+   */
+  async checkFileExists(pattern) {
+    const fullPath = path.join(this.projectRoot, pattern);
+    return fs.existsSync(fullPath);
+  }
+
+  /**
+   * 评估对象格式的准则
+   */
+  async evaluateObjectCriterion(criterion, stage) {
+    // 根据 criterion.type 执行不同的检查
+    switch (criterion.type) {
+      case 'file-exists':
+        return fs.existsSync(path.join(this.projectRoot, criterion.path));
+
+      case 'field-exists':
+        return this.checkFieldExists(criterion.file, criterion.field);
+
+      case 'schema-valid':
+        return this.validateSchema(criterion.file, criterion.schema);
+
+      default:
+        return true; // 未知类型交给 Layer 2
+    }
+  }
+
+  /**
+   * 执行 Layer 2 AI 审查
+   */
+  async executeLayer2(benchmark, stage) {
+    // Layer 2 需要调用 AI Agent 进行审查
+    // 这里返回一个占位结果，实际实现需要集成 AI Agent
+
+    return {
+      status: 'passed',
+      score: 85,
+      findings: [],
+      reviewer: 'ai-agent',
+      reviewed_at: new Date().toISOString()
+    };
+  }
+
+  /**
+   * 计算汇总结果
+   */
+  calculateSummary(results) {
+    const total = results.length;
+    const passed = results.filter(r => r.status === 'passed').length;
+    const failed = results.filter(r => r.status === 'failed').length;
+
+    // 计算加权平均分
+    let totalWeight = 0;
+    let weightedScore = 0;
+
+    for (const result of results) {
+      // 根据 benchmark 优先级分配权重
+      const weight = result.priority === 'P0' ? 3 :
+                      result.priority === 'P1' ? 2 : 1;
+      weightedScore += result.score * weight;
+      totalWeight += weight;
+    }
+
+    const overallScore = totalWeight > 0 ? weightedScore / totalWeight : 0;
+
+    // 确定等级
+    const grade = this.calculateGrade(overallScore);
+
+    return {
+      total,
+      passed,
+      failed,
+      overall_score: Math.round(overallScore),
+      grade,
+      pass_rate: total > 0 ? (passed / total * 100).toFixed(2) : 0
+    };
+  }
+
+  /**
+   * 计算等级
+   */
+  calculateGrade(score) {
+    if (score >= 95) return 'A+';
+    if (score >= 90) return 'A';
+    if (score >= 85) return 'B+';
+    if (score >= 80) return 'B';
+    if (score >= 70) return 'C';
+    return 'D';
+  }
+
+  /**
+   * 生成改进建议
+   */
+  generateRecommendations(results) {
+    const recommendations = [];
+
+    for (const result of results) {
+      if (result.status === 'failed' || result.status === 'error') {
+        const rec = {
+          id: `rec-${result.benchmark_id}`,
+          benchmark_id: result.benchmark_id,
+          benchmark_name: result.name,
+          priority: this.inferPriority(result),
+          issue: this.describeIssue(result),
+          suggestion: this.generateSuggestion(result),
+          auto_fixable: this.isAutoFixable(result)
+        };
+        recommendations.push(rec);
+      }
+    }
+
+    // 按优先级排序
+    const priorityOrder = { P0: 0, P1: 1, P2: 2 };
+    recommendations.sort((a, b) => priorityOrder[a.priority] - priorityOrder[b.priority]);
+
+    return recommendations;
+  }
+
+  /**
+   * 推断优先级
+   */
+  inferPriority(result) {
+    // 基于 benchmark id 或分数推断
+    if (result.benchmark_id?.startsWith('EXP-')) return 'P0';
+    if (result.score < 50) return 'P0';
+    if (result.score < 70) return 'P1';
+    return 'P2';
+  }
+
+  /**
+   * 描述问题
+   */
+  describeIssue(result) {
+    if (result.layer1_result?.status === 'failed') {
+      const failedChecks = result.layer1_result.checks.filter(c => !c.passed);
+      return failedChecks.map(c => c.criterion).join('; ');
+    }
+    if (result.layer2_result?.status === 'failed') {
+      return 'AI 审查未通过';
+    }
+    return '未知问题';
+  }
+
+  /**
+   * 生成建议
+   */
+  generateSuggestion(result) {
+    // 基于问题和阶段生成具体建议
+    const suggestions = {
+      'EXP-001': '补充至少3个澄清问题，覆盖功能、非功能和边界条件',
+      'REQ-001': '完善需求文档，确保所有必填字段完整',
+      'DES-001': '补充架构设计文档，包含技术选型理由',
+      'IMP-001': '确保代码实现符合设计文档',
+      'IMP-002': '优化代码质量，确保测试覆盖率达标',
+      'TST-001': '补充测试用例，提高代码覆盖率'
+    };
+
+    return suggestions[result.benchmark_id] || '请参考 benchmark 要求完善产出物';
+  }
+
+  /**
+   * 判断是否可自动修复
+   */
+  isAutoFixable(result) {
+    // 某些问题可以自动修复
+    const autoFixableBenchmarks = ['format', 'lint', 'import'];
+    return autoFixableBenchmarks.some(id => result.benchmark_id?.includes(id));
+  }
+
+  /**
+   * 收集问题列表
+   */
+  collectIssues(result) {
+    const issues = [];
+
+    if (result.layer1_result?.checks) {
+      for (const check of result.layer1_result.checks) {
+        if (!check.passed) {
+          issues.push({
+            layer: 1,
+            type: 'criterion-failed',
+            description: check.criterion,
+            actual: check.actual,
+            expected: check.expected
+          });
+        }
+      }
+    }
+
+    if (result.layer2_result?.findings) {
+      for (const finding of result.layer2_result.findings) {
+        issues.push({
+          layer: 2,
+          type: finding.type,
+          description: finding.description,
+          severity: finding.severity
+        });
+      }
+    }
+
+    return issues;
+  }
+
+  /**
+   * 保存报告
+   */
+  saveReport(report) {
+    const filename = `${report.stage}-${report.run_id}.json`;
+    const filepath = path.join(this.resultsPath, filename);
+    fs.writeFileSync(filepath, JSON.stringify(report, null, 2));
+    return filepath;
+  }
+
+  /**
+   * 生成运行 ID
+   */
+  generateRunId() {
+    const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+    const random = Math.random().toString(36).substr(2, 6);
+    return `${timestamp}-${random}`;
+  }
+
+  /**
+   * 获取最新报告
+   */
+  getLatestReport(stage) {
+    const files = fs.readdirSync(this.resultsPath)
+      .filter(f => f.startsWith(`${stage}-`) && f.endsWith('.json'))
+      .sort()
+      .reverse();
+
+    if (files.length === 0) {
+      return null;
+    }
+
+    const content = fs.readFileSync(path.join(this.resultsPath, files[0]), 'utf8');
+    return JSON.parse(content);
+  }
+
+  /**
+   * 获取历史趋势
+   */
+  getTrend(stage, limit = 10) {
+    const files = fs.readdirSync(this.resultsPath)
+      .filter(f => f.startsWith(`${stage}-`) && f.endsWith('.json'))
+      .sort()
+      .reverse()
+      .slice(0, limit);
+
+    return files.map(f => {
+      const content = fs.readFileSync(path.join(this.resultsPath, f), 'utf8');
+      const report = JSON.parse(content);
+      return {
+        timestamp: report.timestamp,
+        score: report.summary.overall_score,
+        grade: report.summary.grade,
+        pass_rate: report.summary.pass_rate
+      };
+    }).reverse();
+  }
+
+  // 辅助方法（需要根据实际情况实现）
+  countClarificationQuestions() {
+    // 从 clarifications.md 中统计问题数量
+    try {
+      const filePath = path.join(this.projectRoot, 'specs', 'clarifications.md');
+      if (!fs.existsSync(filePath)) return 0;
+
+      const content = fs.readFileSync(filePath, 'utf8');
+      const matches = content.match(/^\s*-\s*\*\*Q\d+:/gm);
+      return matches ? matches.length : 0;
+    } catch {
+      return 0;
+    }
+  }
+
+  calculateFieldCompleteness(stage) {
+    // 计算字段完整率
+    return 85; // 占位实现
+  }
+
+  getTestCoverage() {
+    // 获取测试覆盖率
+    try {
+      const coveragePath = path.join(this.projectRoot, 'coverage', 'coverage-summary.json');
+      if (!fs.existsSync(coveragePath)) return 0;
+
+      const data = JSON.parse(fs.readFileSync(coveragePath, 'utf8'));
+      return data.total?.lines?.pct || 0;
+    } catch {
+      return 0;
+    }
+  }
+
+  getCodeComplexity() {
+    // 获取代码复杂度
+    return 8; // 占位实现
+  }
+
+  getLineCoverage() {
+    return this.getTestCoverage();
+  }
+
+  getBranchCoverage() {
+    try {
+      const coveragePath = path.join(this.projectRoot, 'coverage', 'coverage-summary.json');
+      if (!fs.existsSync(coveragePath)) return 0;
+
+      const data = JSON.parse(fs.readFileSync(coveragePath, 'utf8'));
+      return data.total?.branches?.pct || 0;
+    } catch {
+      return 0;
+    }
+  }
+
+  checkFieldExists(file, field) {
+    try {
+      const filePath = path.join(this.projectRoot, file);
+      if (!fs.existsSync(filePath)) return false;
+
+      const content = fs.readFileSync(filePath, 'utf8');
+      return content.includes(field);
+    } catch {
+      return false;
+    }
+  }
+
+  validateSchema(file, schema) {
+    // 验证文件是否符合 schema
+    return true; // 占位实现
+  }
+}
+
+module.exports = BenchmarkEvaluator;