@chongyan/autospec 1.0.2 → 1.0.3

package/plugins/hooks/lib/benchmarks/evaluator.js

@@ -1,605 +0,0 @@
-/**
- * Benchmarks Evaluator
- * 评测执行器 - 负责执行 benchmarks 评测，计算评分，生成报告
- */
-
-const fs = require('fs');
-const path = require('path');
-const yaml = require('js-yaml');
-
-class BenchmarkEvaluator {
-  constructor(projectRoot) {
-    this.projectRoot = projectRoot;
-    this.resultsPath = path.join(projectRoot, '.autospec', 'benchmarks', 'results');
-    this.benchmarksPath = path.join(projectRoot, '.autospec', 'benchmarks');
-  }
-
-  /**
-   * 初始化评测器
-   */
-  init() {
-    if (!fs.existsSync(this.resultsPath)) {
-      fs.mkdirSync(this.resultsPath, { recursive: true });
-    }
-    return this;
-  }
-
-  /**
-   * 加载 benchmarks
-   */
-  loadBenchmarks(stage) {
-    const benchmarks = [];
-
-    // 加载自动生成的 benchmarks
-    const autoGenPath = path.join(this.benchmarksPath, 'auto-generated');
-    if (fs.existsSync(autoGenPath)) {
-      const files = fs.readdirSync(autoGenPath)
-        .filter(f => f.startsWith(`${stage}-`) && f.endsWith('.yaml'))
-        .sort()
-        .reverse(); // 最新的在前
-
-      if (files.length > 0) {
-        const latestFile = path.join(autoGenPath, files[0]);
-        const content = fs.readFileSync(latestFile, 'utf8');
-        const data = yaml.load(content);
-        if (data && data.test_cases) {
-          benchmarks.push(...data.test_cases);
-        }
-      }
-    }
-
-    // 加载自定义 benchmarks
-    const customPath = path.join(this.benchmarksPath, 'custom');
-    if (fs.existsSync(customPath)) {
-      const files = fs.readdirSync(customPath)
-        .filter(f => f.startsWith(`${stage}-`) && f.endsWith('.yaml'));
-
-      for (const file of files) {
-        const content = fs.readFileSync(path.join(customPath, file), 'utf8');
-        const data = yaml.load(content);
-        if (data && data.test_cases) {
-          benchmarks.push(...data.test_cases);
-        }
-      }
-    }
-
-    // 去重（基于 id）
-    const seen = new Set();
-    return benchmarks.filter(b => {
-      if (seen.has(b.id)) return false;
-      seen.add(b.id);
-      return true;
-    });
-  }
-
-  /**
-   * 执行评测
-   */
-  async evaluate(stage, options = {}) {
-    const benchmarks = this.loadBenchmarks(stage);
-    const results = [];
-    const startTime = Date.now();
-
-    // 执行每个 benchmark
-    for (const benchmark of benchmarks) {
-      const result = await this.evaluateBenchmark(benchmark, stage);
-      results.push(result);
-    }
-
-    // 计算综合得分
-    const summary = this.calculateSummary(results);
-    const duration = Date.now() - startTime;
-
-    // 生成报告
-    const report = {
-      run_id: this.generateRunId(),
-      stage,
-      timestamp: new Date().toISOString(),
-      trigger: options.trigger || 'manual',
-      summary: {
-        ...summary,
-        duration_ms: duration
-      },
-      results,
-      recommendations: this.generateRecommendations(results),
-      metrics: {
-        total_duration_ms: duration,
-        benchmarks_count: benchmarks.length,
-        layer1_duration_ms: results.reduce((sum, r) => sum + (r.layer1_duration_ms || 0), 0),
-        layer2_duration_ms: results.reduce((sum, r) => sum + (r.layer2_duration_ms || 0), 0)
-      }
-    };
-
-    // 保存报告
-    this.saveReport(report);
-
-    return report;
-  }
-
-  /**
-   * 执行单个 benchmark
-   */
-  async evaluateBenchmark(benchmark, stage) {
-    const result = {
-      benchmark_id: benchmark.id,
-      name: benchmark.name,
-      status: 'pending',
-      score: 0,
-      layer1_result: null,
-      layer2_result: null,
-      issues: [],
-      layer1_duration_ms: 0,
-      layer2_duration_ms: 0
-    };
-
-    const startTime = Date.now();
-
-    try {
-      // Step 1: Layer 1 确定性验证
-      const layer1Start = Date.now();
-      result.layer1_result = await this.executeLayer1(benchmark, stage);
-      result.layer1_duration_ms = Date.now() - layer1Start;
-
-      // Step 2: Layer 2 AI 审查（如果 Layer 1 通过或需要深度评估）
-      if (result.layer1_result.status === 'passed' || benchmark.force_layer2) {
-        const layer2Start = Date.now();
-        result.layer2_result = await this.executeLayer2(benchmark, stage);
-        result.layer2_duration_ms = Date.now() - layer2Start;
-      }
-
-      // 计算最终状态
-      if (result.layer1_result.status === 'failed') {
-        result.status = 'failed';
-        result.score = result.layer1_result.score || 0;
-      } else if (result.layer2_result && result.layer2_result.status === 'failed') {
-        result.status = 'failed';
-        result.score = result.layer2_result.score || 50;
-      } else {
-        result.status = 'passed';
-        result.score = result.layer2_result?.score || result.layer1_result.score || 100;
-      }
-
-      // 收集问题
-      result.issues = this.collectIssues(result);
-
-    } catch (error) {
-      result.status = 'error';
-      result.error = error.message;
-      result.score = 0;
-    }
-
-    return result;
-  }
-
-  /**
-   * 执行 Layer 1 验证
-   */
-  async executeLayer1(benchmark, stage) {
-    const result = {
-      status: 'passed',
-      score: 100,
-      checks: [],
-      details: {}
-    };
-
-    const criteria = benchmark.criteria || [];
-    let passedCount = 0;
-
-    for (const criterion of criteria) {
-      const checkResult = await this.evaluateCriterion(criterion, stage);
-      result.checks.push(checkResult);
-
-      if (checkResult.passed) {
-        passedCount++;
-      } else {
-        result.status = 'failed';
-      }
-    }
-
-    // 计算 Layer 1 得分
-    if (criteria.length > 0) {
-      result.score = (passedCount / criteria.length) * 100;
-    }
-
-    return result;
-  }
-
-  /**
-   * 评估单个准则
-   */
-  async evaluateCriterion(criterion, stage) {
-    const check = {
-      criterion,
-      passed: false,
-      actual: null,
-      expected: null
-    };
-
-    // 解析准则类型
-    if (typeof criterion === 'string') {
-      // 字符串格式的准则
-      if (criterion.includes('>=')) {
-        // 数量比较，如 "澄清问题数量 >= 3"
-        const match = criterion.match(/(.+)>=\s*(\d+)/);
-        if (match) {
-          const [, field, threshold] = match;
-          check.actual = await this.getFieldValue(field.trim(), stage);
-          check.expected = parseInt(threshold, 10);
-          check.passed = check.actual >= check.expected;
-        }
-      } else if (criterion.includes('存在')) {
-        // 文件存在性检查
-        const filePattern = criterion.replace(/.*存在/, '').trim();
-        check.actual = await this.checkFileExists(filePattern);
-        check.expected = true;
-        check.passed = check.actual === true;
-      } else {
-        // 其他字符串准则，标记为需要 Layer 2
-        check.passed = true; // 默认通过，交给 Layer 2
-        check.needsLayer2 = true;
-      }
-    } else if (typeof criterion === 'object') {
-      // 对象格式的准则
-      check.passed = await this.evaluateObjectCriterion(criterion, stage);
-    }
-
-    return check;
-  }
-
-  /**
-   * 获取字段值
-   */
-  async getFieldValue(field, stage) {
-    // 根据字段名从不同来源获取值
-    const fieldMappings = {
-      '澄清问题数量': () => this.countClarificationQuestions(),
-      '字段完整率': () => this.calculateFieldCompleteness(stage),
-      '测试覆盖率': () => this.getTestCoverage(),
-      '复杂度': () => this.getCodeComplexity(),
-      '行覆盖率': () => this.getLineCoverage(),
-      '分支覆盖率': () => this.getBranchCoverage()
-    };
-
-    const getter = fieldMappings[field];
-    if (getter) {
-      return await getter();
-    }
-
-    return 0;
-  }
-
-  /**
-   * 检查文件是否存在
-   */
-  async checkFileExists(pattern) {
-    const fullPath = path.join(this.projectRoot, pattern);
-    return fs.existsSync(fullPath);
-  }
-
-  /**
-   * 评估对象格式的准则
-   */
-  async evaluateObjectCriterion(criterion, stage) {
-    // 根据 criterion.type 执行不同的检查
-    switch (criterion.type) {
-      case 'file-exists':
-        return fs.existsSync(path.join(this.projectRoot, criterion.path));
-
-      case 'field-exists':
-        return this.checkFieldExists(criterion.file, criterion.field);
-
-      case 'schema-valid':
-        return this.validateSchema(criterion.file, criterion.schema);
-
-      default:
-        return true; // 未知类型交给 Layer 2
-    }
-  }
-
-  /**
-   * 执行 Layer 2 AI 审查
-   */
-  async executeLayer2(benchmark, stage) {
-    // Layer 2 需要调用 AI Agent 进行审查
-    // 这里返回一个占位结果，实际实现需要集成 AI Agent
-
-    return {
-      status: 'passed',
-      score: 85,
-      findings: [],
-      reviewer: 'ai-agent',
-      reviewed_at: new Date().toISOString()
-    };
-  }
-
-  /**
-   * 计算汇总结果
-   */
-  calculateSummary(results) {
-    const total = results.length;
-    const passed = results.filter(r => r.status === 'passed').length;
-    const failed = results.filter(r => r.status === 'failed').length;
-
-    // 计算加权平均分
-    let totalWeight = 0;
-    let weightedScore = 0;
-
-    for (const result of results) {
-      // 根据 benchmark 优先级分配权重
-      const weight = result.priority === 'P0' ? 3 :
-                      result.priority === 'P1' ? 2 : 1;
-      weightedScore += result.score * weight;
-      totalWeight += weight;
-    }
-
-    const overallScore = totalWeight > 0 ? weightedScore / totalWeight : 0;
-
-    // 确定等级
-    const grade = this.calculateGrade(overallScore);
-
-    return {
-      total,
-      passed,
-      failed,
-      overall_score: Math.round(overallScore),
-      grade,
-      pass_rate: total > 0 ? (passed / total * 100).toFixed(2) : 0
-    };
-  }
-
-  /**
-   * 计算等级
-   */
-  calculateGrade(score) {
-    if (score >= 95) return 'A+';
-    if (score >= 90) return 'A';
-    if (score >= 85) return 'B+';
-    if (score >= 80) return 'B';
-    if (score >= 70) return 'C';
-    return 'D';
-  }
-
-  /**
-   * 生成改进建议
-   */
-  generateRecommendations(results) {
-    const recommendations = [];
-
-    for (const result of results) {
-      if (result.status === 'failed' || result.status === 'error') {
-        const rec = {
-          id: `rec-${result.benchmark_id}`,
-          benchmark_id: result.benchmark_id,
-          benchmark_name: result.name,
-          priority: this.inferPriority(result),
-          issue: this.describeIssue(result),
-          suggestion: this.generateSuggestion(result),
-          auto_fixable: this.isAutoFixable(result)
-        };
-        recommendations.push(rec);
-      }
-    }
-
-    // 按优先级排序
-    const priorityOrder = { P0: 0, P1: 1, P2: 2 };
-    recommendations.sort((a, b) => priorityOrder[a.priority] - priorityOrder[b.priority]);
-
-    return recommendations;
-  }
-
-  /**
-   * 推断优先级
-   */
-  inferPriority(result) {
-    // 基于 benchmark id 或分数推断
-    if (result.benchmark_id?.startsWith('EXP-')) return 'P0';
-    if (result.score < 50) return 'P0';
-    if (result.score < 70) return 'P1';
-    return 'P2';
-  }
-
-  /**
-   * 描述问题
-   */
-  describeIssue(result) {
-    if (result.layer1_result?.status === 'failed') {
-      const failedChecks = result.layer1_result.checks.filter(c => !c.passed);
-      return failedChecks.map(c => c.criterion).join('; ');
-    }
-    if (result.layer2_result?.status === 'failed') {
-      return 'AI 审查未通过';
-    }
-    return '未知问题';
-  }
-
-  /**
-   * 生成建议
-   */
-  generateSuggestion(result) {
-    // 基于问题和阶段生成具体建议
-    const suggestions = {
-      'EXP-001': '补充至少3个澄清问题，覆盖功能、非功能和边界条件',
-      'REQ-001': '完善需求文档，确保所有必填字段完整',
-      'DES-001': '补充架构设计文档，包含技术选型理由',
-      'IMP-001': '确保代码实现符合设计文档',
-      'IMP-002': '优化代码质量，确保测试覆盖率达标',
-      'TST-001': '补充测试用例，提高代码覆盖率'
-    };
-
-    return suggestions[result.benchmark_id] || '请参考 benchmark 要求完善产出物';
-  }
-
-  /**
-   * 判断是否可自动修复
-   */
-  isAutoFixable(result) {
-    // 某些问题可以自动修复
-    const autoFixableBenchmarks = ['format', 'lint', 'import'];
-    return autoFixableBenchmarks.some(id => result.benchmark_id?.includes(id));
-  }
-
-  /**
-   * 收集问题列表
-   */
-  collectIssues(result) {
-    const issues = [];
-
-    if (result.layer1_result?.checks) {
-      for (const check of result.layer1_result.checks) {
-        if (!check.passed) {
-          issues.push({
-            layer: 1,
-            type: 'criterion-failed',
-            description: check.criterion,
-            actual: check.actual,
-            expected: check.expected
-          });
-        }
-      }
-    }
-
-    if (result.layer2_result?.findings) {
-      for (const finding of result.layer2_result.findings) {
-        issues.push({
-          layer: 2,
-          type: finding.type,
-          description: finding.description,
-          severity: finding.severity
-        });
-      }
-    }
-
-    return issues;
-  }
-
-  /**
-   * 保存报告
-   */
-  saveReport(report) {
-    const filename = `${report.stage}-${report.run_id}.json`;
-    const filepath = path.join(this.resultsPath, filename);
-    fs.writeFileSync(filepath, JSON.stringify(report, null, 2));
-    return filepath;
-  }
-
-  /**
-   * 生成运行 ID
-   */
-  generateRunId() {
-    const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
-    const random = Math.random().toString(36).substr(2, 6);
-    return `${timestamp}-${random}`;
-  }
-
-  /**
-   * 获取最新报告
-   */
-  getLatestReport(stage) {
-    const files = fs.readdirSync(this.resultsPath)
-      .filter(f => f.startsWith(`${stage}-`) && f.endsWith('.json'))
-      .sort()
-      .reverse();
-
-    if (files.length === 0) {
-      return null;
-    }
-
-    const content = fs.readFileSync(path.join(this.resultsPath, files[0]), 'utf8');
-    return JSON.parse(content);
-  }
-
-  /**
-   * 获取历史趋势
-   */
-  getTrend(stage, limit = 10) {
-    const files = fs.readdirSync(this.resultsPath)
-      .filter(f => f.startsWith(`${stage}-`) && f.endsWith('.json'))
-      .sort()
-      .reverse()
-      .slice(0, limit);
-
-    return files.map(f => {
-      const content = fs.readFileSync(path.join(this.resultsPath, f), 'utf8');
-      const report = JSON.parse(content);
-      return {
-        timestamp: report.timestamp,
-        score: report.summary.overall_score,
-        grade: report.summary.grade,
-        pass_rate: report.summary.pass_rate
-      };
-    }).reverse();
-  }
-
-  // 辅助方法（需要根据实际情况实现）
-  countClarificationQuestions() {
-    // 从 clarifications.md 中统计问题数量
-    try {
-      const filePath = path.join(this.projectRoot, 'specs', 'clarifications.md');
-      if (!fs.existsSync(filePath)) return 0;
-
-      const content = fs.readFileSync(filePath, 'utf8');
-      const matches = content.match(/^\s*-\s*\*\*Q\d+:/gm);
-      return matches ? matches.length : 0;
-    } catch {
-      return 0;
-    }
-  }
-
-  calculateFieldCompleteness(stage) {
-    // 计算字段完整率
-    return 85; // 占位实现
-  }
-
-  getTestCoverage() {
-    // 获取测试覆盖率
-    try {
-      const coveragePath = path.join(this.projectRoot, 'coverage', 'coverage-summary.json');
-      if (!fs.existsSync(coveragePath)) return 0;
-
-      const data = JSON.parse(fs.readFileSync(coveragePath, 'utf8'));
-      return data.total?.lines?.pct || 0;
-    } catch {
-      return 0;
-    }
-  }
-
-  getCodeComplexity() {
-    // 获取代码复杂度
-    return 8; // 占位实现
-  }
-
-  getLineCoverage() {
-    return this.getTestCoverage();
-  }
-
-  getBranchCoverage() {
-    try {
-      const coveragePath = path.join(this.projectRoot, 'coverage', 'coverage-summary.json');
-      if (!fs.existsSync(coveragePath)) return 0;
-
-      const data = JSON.parse(fs.readFileSync(coveragePath, 'utf8'));
-      return data.total?.branches?.pct || 0;
-    } catch {
-      return 0;
-    }
-  }
-
-  checkFieldExists(file, field) {
-    try {
-      const filePath = path.join(this.projectRoot, file);
-      if (!fs.existsSync(filePath)) return false;
-
-      const content = fs.readFileSync(filePath, 'utf8');
-      return content.includes(field);
-    } catch {
-      return false;
-    }
-  }
-
-  validateSchema(file, schema) {
-    // 验证文件是否符合 schema
-    return true; // 占位实现
-  }
-}
-
-module.exports = BenchmarkEvaluator;