@jsonstudio/rcc 0.85.9

package/scripts/v2-consistency/README.md

@@ -0,0 +1,146 @@
+# V1/V2功能对齐测试工具
+
+## 概述
+
+这套工具用于验证RouteCodex V1和V2架构的功能一致性，确保两个版本在处理相同输入时产生一致的输出。
+
+## 工具组成
+
+### 1. 核心测试组件
+- **SnapshotLoader**: 加载和管理快照数据
+- **ConsistencyValidator**: 执行一致性验证逻辑
+- **V1V2ConsistencyTest**: 主测试协调器
+
+### 2. 测试类型
+- **工具处理测试**: 验证工具调用收割、规范化和治理的一致性
+- **协议转换测试**: 验证OpenAI ↔ Anthropic ↔ Responses协议转换
+- **V1/V2对齐测试**: 验证两个版本的端到端一致性
+
+### 3. 执行脚本
+- `run-consistency-test.mjs`: 单独运行V1/V2对齐测试
+- `comprehensive-consistency-test.mjs`: 运行所有测试的综合脚本
+- `generate-snapshot-data.mjs`: 生成测试所需的快照数据
+
+## 使用方法
+
+### 基本用法
+
+```bash
+# 运行所有协议的一致性测试
+npm run test:consistency
+
+# 运行特定协议测试
+npm run test:consistency openai-chat 10
+
+# 运行综合测试（包含所有测试类型）
+pm run test:comprehensive
+
+# 生成快照数据并测试
+npm run test:comprehensive --generate-snapshots
+```
+
+### 高级选项
+
+```bash
+# 跳过特定测试类型
+npm run test:comprehensive --skip-tool-processing
+npm run test:comprehensive --skip-protocol-conversion
+npm run test:comprehensive --skip-v1v2-alignment
+
+# 限制测试用例数量
+npm run test:comprehensive --max-cases=5
+```
+
+## 测试报告
+
+### 输出格式
+
+测试完成后会生成详细的JSON报告，包含：
+- 测试摘要统计
+- 失败详情和差异分析
+- 分类一致性率
+- 改进建议
+
+### 退出码
+
+- `0`: 所有测试通过
+- `1`: 发现关键错误
+- `2`: 发现重要错误
+- `3`: 测试执行失败
+
+## 数据结构
+
+### 快照数据
+
+快照数据存储在 `~/.routecodex/codex-samples/` 目录下，按协议组织：
+- `openai-chat/`: OpenAI Chat协议快照
+- `anthropic-messages/`: Anthropic Messages协议快照
+- `openai-responses/`: OpenAI Responses协议快照
+
+### V1/V2数据对比
+
+- **V1数据**: 包含compat-pre、compat-post等处理阶段数据
+- **V2数据**: 包含workflow、provider request/response等阶段数据
+- **对比维度**: Provider请求、Provider响应、工具处理、最终响应
+
+## 配置选项
+
+### 忽略字段
+
+默认忽略以下字段进行比较：
+- `created`, `created_at`, `timestamp`: 时间戳
+- `request_id`, `id`: 标识符
+- `meta.buildTime`, `meta.version`: 构建信息
+
+### 容差设置
+
+- 时间差异容差: 5秒
+- 数值精度容差: 6位小数
+
+## 扩展性
+
+### 添加新的验证规则
+
+在 `ConsistencyValidator` 类中添加新的验证方法：
+
+```typescript
+private validateNewAspect(
+  v1Data: V1ProcessingData,
+  v2Data: V2ProcessingData
+): ConsistencyCheck {
+  // 实现新的验证逻辑
+}
+```
+
+### 添加新的协议支持
+
+1. 在 `SnapshotLoader` 中添加协议特定加载逻辑
+2. 更新类型定义文件
+3. 添加相应的测试用例
+
+## 故障排除
+
+### 常见问题
+
+1. **模块找不到错误**: 确保已运行 `npm run build`
+2. **快照数据缺失**: 运行 `--generate-snapshots` 选项
+3. **TypeScript编译错误**: 检查类型定义和导入路径
+
+### 调试技巧
+
+- 使用 `--help` 查看所有选项
+- 检查 `test-results/` 目录下的报告文件
+- 查看控制台输出的详细错误信息
+
+## 贡献指南
+
+1. 遵循现有的代码结构和命名约定
+2. 添加适当的类型定义
+3. 编写对应的测试用例
+4. 更新文档说明
+
+## 维护说明
+
+- 定期更新忽略字段列表以适应新的数据结构
+- 根据架构演进调整验证逻辑
+- 监控测试覆盖率并补充缺失的测试场景

package/scripts/v2-consistency/comprehensive-consistency-test.mjs

@@ -0,0 +1,398 @@
+#!/usr/bin/env node
+
+/**
+ * 综合一致性测试脚本
+ * 包含工具处理测试、协议转换测试和V1/V2对齐测试
+ */
+
+import fs from 'fs/promises';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const projectRoot = path.join(__dirname, '..', '..');
+
+class ComprehensiveConsistencyTest {
+  constructor() {
+    this.testResults = {
+      toolProcessing: null,
+      protocolConversion: null,
+      v1v2Alignment: null,
+      summary: null
+    };
+  }
+
+  /**
+   * 运行所有一致性测试
+   */
+  async runAllTests(options = {}) {
+    console.log('🚀 启动综合一致性测试');
+    console.log('========================');
+    
+    const { 
+      generateSnapshots = false, 
+      maxTestCases = 20,
+      skipToolProcessing = false,
+      skipProtocolConversion = false,
+      skipV1V2Alignment = false
+    } = options;
+
+    try {
+      // 1. 生成快照数据（如果需要）
+      if (generateSnapshots) {
+        console.log('\n📸 生成快照数据...');
+        const { SnapshotDataGenerator } = await import(path.join(projectRoot, 'scripts/generate-snapshot-data.mjs'));
+        const generator = new SnapshotDataGenerator();
+        await generator.generateAllSnapshots();
+      }
+
+      // 2. 工具处理测试
+      if (!skipToolProcessing) {
+        console.log('\n🔧 运行工具处理测试...');
+        this.testResults.toolProcessing = await this.runToolProcessingTest();
+      }
+
+      // 3. 协议转换测试
+      if (!skipProtocolConversion) {
+        console.log('\n🔄 运行协议转换测试...');
+        this.testResults.protocolConversion = await this.runProtocolConversionTest();
+      }
+
+      // 4. V1/V2对齐测试
+      if (!skipV1V2Alignment) {
+        console.log('\n⚖️ 运行V1/V2对齐测试...');
+        this.testResults.v1v2Alignment = await this.runV1V2AlignmentTest(maxTestCases);
+      }
+
+      // 5. 生成综合报告
+      console.log('\n📊 生成综合报告...');
+      JSON.parse(JSON.stringify(this.testResults.summary || {})) = await this.generateSummary();
+      
+      // 6. 保存报告
+      await this.saveComprehensiveReport();
+      
+      // 7. 显示摘要
+      this.displaySummary();
+      
+      return this.testResults;
+
+    } catch (error) {
+      console.error('❌ 综合测试失败:', error);
+      process.exit(1);
+    }
+  }
+
+  /**
+   * 运行工具处理测试
+   */
+  async runToolProcessingTest() {
+    try {
+      const { V2ToolProcessingTest } = await import(path.join(projectRoot, 'tests/v2/src/tool-processing-test.js'));
+      const test = new V2ToolProcessingTest();
+      
+      return new Promise((resolve, reject) => {
+        const originalConsoleLog = console.log;
+        const logs = [];
+        
+        console.log = (...args) => {
+          logs.push(args.join(' '));
+          originalConsoleLog(...args);
+        };
+        
+        test.runTests().then(() => {
+          console.log = originalConsoleLog;
+          resolve({
+            status: 'completed',
+            logs,
+            summary: this.extractToolProcessingSummary(logs)
+          });
+        }).catch(reject);
+      });
+    } catch (error) {
+      return {
+        status: 'failed',
+        error: error.message
+      };
+    }
+  }
+
+  /**
+   * 运行协议转换测试
+   */
+  async runProtocolConversionTest() {
+    try {
+      const { V2ProtocolConversionTest } = await import(path.join(projectRoot, 'tests/v2/src/protocol-conversion-test.js'));
+      const test = new V2ProtocolConversionTest();
+      
+      return new Promise((resolve, reject) => {
+        const originalConsoleLog = console.log;
+        const logs = [];
+        
+        console.log = (...args) => {
+          logs.push(args.join(' '));
+          originalConsoleLog(...args);
+        };
+        
+        test.runTests().then(() => {
+          console.log = originalConsoleLog;
+          resolve({
+            status: 'completed',
+            logs,
+            summary: this.extractProtocolConversionSummary(logs)
+          });
+        }).catch(reject);
+      });
+    } catch (error) {
+      return {
+        status: 'failed',
+        error: error.message
+      };
+    }
+  }
+
+  /**
+   * 运行V1/V2对齐测试
+   */
+  async runV1V2AlignmentTest(maxTestCases) {
+    try {
+      const { V1V2ConsistencyTest } = await import(path.join(projectRoot, 'tests/v2/src/consistency/v1v2-consistency-test.js'));
+      const test = new V1V2ConsistencyTest({
+        maxTestCases,
+        outputDir: path.join(projectRoot, 'test-results')
+      });
+      
+      const report = await test.runAllTests();
+      return {
+        status: 'completed',
+        report
+      };
+    } catch (error) {
+      return {
+        status: 'failed',
+        error: error.message
+      };
+    }
+  }
+
+  /**
+   * 提取工具处理测试摘要
+   */
+  extractToolProcessingSummary(logs) {
+    const summary = {
+      totalSamples: 0,
+      successfulSamples: 0,
+      totalHarvested: 0,
+      totalCanonicalized: 0,
+      totalGoverned: 0,
+      avgProcessingTime: 0
+    };
+
+    for (const log of logs) {
+      const match = log.match(/找到 (\d+) 个工具调用样本/);
+      if (match) summary.totalSamples = parseInt(match[1]);
+      
+      const successMatch = log.match(/成功: (\d+)/);
+      if (successMatch) summary.successfulSamples = parseInt(successMatch[1]);
+      
+      const harvestedMatch = log.match(/总收割工具: (\d+)/);
+      if (harvestedMatch) summary.totalHarvested = parseInt(harvestedMatch[1]);
+      
+      const canonicalizedMatch = log.match(/总规范化工具: (\d+)/);
+      if (canonicalizedMatch) summary.totalCanonicalized = parseInt(canonicalizedMatch[1]);
+      
+      const governedMatch = log.match(/总治理工具: (\d+)/);
+      if (governedMatch) summary.totalGoverned = parseInt(governedMatch[1]);
+      
+      const avgTimeMatch = log.match(/平均处理时间: (\d+)ms/);
+      if (avgTimeMatch) summary.avgProcessingTime = parseInt(avgTimeMatch[1]);
+    }
+
+    return summary;
+  }
+
+  /**
+   * 提取协议转换测试摘要
+   */
+  extractProtocolConversionSummary(logs) {
+    const summary = {
+      totalSamples: 0,
+      successfulConversions: 0,
+      failedConversions: 0,
+      avgConversionTime: 0,
+      pathStats: {}
+    };
+
+    for (const log of logs) {
+      const match = log.match(/找到 (\d+) 个协议转换样本/);
+      if (match) summary.totalSamples = parseInt(match[1]);
+      
+      const successMatch = log.match(/成功转换: (\d+)/);
+      if (successMatch) summary.successfulConversions = parseInt(successMatch[1]);
+      
+      const failedMatch = log.match(/失败转换: (\d+)/);
+      if (failedMatch) summary.failedConversions = parseInt(failedMatch[1]);
+      
+      const avgTimeMatch = log.match(/平均转换时间: (\d+)ms/);
+      if (avgTimeMatch) summary.avgConversionTime = parseInt(avgTimeMatch[1]);
+      
+      const pathMatch = log.match(/(openai->anthropic|anthropic->openai|openai->responses): (\d+)/);
+      if (pathMatch) {
+        summary.pathStats[pathMatch[1]] = parseInt(pathMatch[2]);
+      }
+    }
+
+    return summary;
+  }
+
+  /**
+   * 生成综合摘要
+   */
+  async generateSummary() {
+    const summary = {
+      timestamp: new Date().toISOString(),
+      overallStatus: 'unknown',
+      testResults: this.testResults,
+      recommendations: [],
+      criticalIssues: [],
+      majorIssues: []
+    };
+
+    // 分析测试结果
+    const allTests = [
+      { name: '工具处理', result: this.testResults.toolProcessing },
+      { name: '协议转换', result: this.testResults.protocolConversion },
+      { name: 'V1/V2对齐', result: this.testResults.v1v2Alignment }
+    ];
+
+    const failedTests = allTests.filter(t => t.result?.status === 'failed');
+    
+    if (failedTests.length === 0) {
+      summary.overallStatus = 'passed';
+    } else if (failedTests.length <= 1) {
+      summary.overallStatus = 'partial';
+    } else {
+      summary.overallStatus = 'failed';
+    }
+
+    // 生成建议
+    if (this.testResults.v1v2Alignment?.report?.failures) {
+      const failures = this.testResults.v1v2Alignment.report.failures;
+      const critical = failures.filter(f => f.severity === 'critical');
+      const major = failures.filter(f => f.severity === 'major');
+      
+      if (critical.length > 0) {
+        summary.criticalIssues.push(`发现 ${critical.length} 个关键V1/V2一致性问题`);
+      }
+      
+      if (major.length > 0) {
+        summary.majorIssues.push(`发现 ${major.length} 个重要V1/V2一致性问题`);
+      }
+    }
+
+    // 工具处理建议
+    if (this.testResults.toolProcessing?.summary) {
+      const { successfulSamples, totalSamples } = this.testResults.toolProcessing.summary;
+      if (successfulSamples < totalSamples * 0.8) {
+        summary.majorIssues.push('工具处理成功率低于80%，需要检查工具收割逻辑');
+      }
+    }
+
+    // 协议转换建议
+    if (this.testResults.protocolConversion?.summary) {
+      const { successfulConversions, totalSamples } = this.testResults.protocolConversion.summary;
+      if (successfulConversions < totalSamples * 0.8) {
+        summary.majorIssues.push('协议转换成功率低于80%，需要检查转换逻辑');
+      }
+    }
+
+    return summary;
+  }
+
+  /**
+   * 保存综合报告
+   */
+  async saveComprehensiveReport() {
+    const outputDir = path.join(projectRoot, 'test-results');
+    await fs.mkdir(outputDir, { recursive: true });
+    
+    const reportPath = path.join(outputDir, `comprehensive-consistency-report-${Date.now()}.json`);
+    await fs.writeFile(reportPath, JSON.stringify(JSON.parse(JSON.stringify(JSON.parse(JSON.stringify(this.testResults.summary || {})) || {})) ? JSON.parse(JSON.stringify(JSON.parse(JSON.stringify(this.testResults.summary || {})))) : {}, null, 2));
+    
+    console.log(`📄 综合报告已保存到: ${reportPath}`);
+  }
+
+  /**
+   * 显示摘要
+   */
+  displaySummary() {
+    const summary = JSON.parse(JSON.stringify(JSON.parse(JSON.stringify(this.testResults.summary || {})) || {}));
+    
+    console.log('\n📊 综合一致性测试摘要');
+    console.log('========================');
+    console.log(`🕐 测试时间: ${summary.timestamp}`);
+    console.log(`🎯 总体状态: ${summary.overallStatus}`);
+    
+    console.log('\n📋 测试结果:');
+    console.log(`  🔧 工具处理: ${this.testResults.toolProcessing?.status || 'skipped'}`);
+    console.log(`  🔄 协议转换: ${this.testResults.protocolConversion?.status || 'skipped'}`);
+    console.log(`  ⚖️ V1/V2对齐: ${this.testResults.v1v2Alignment?.status || 'skipped'}`);
+    
+    if (summary.criticalIssues.length > 0) {
+      console.log('\n🔴 关键问题:');
+      summary.criticalIssues.forEach(issue => console.log(`  • ${issue}`));
+    }
+    
+    if (summary.majorIssues.length > 0) {
+      console.log('\n🟡 重要问题:');
+      summary.majorIssues.forEach(issue => console.log(`  • ${issue}`));
+    }
+  }
+}
+
+// 主函数
+async function main() {
+  const args = process.argv.slice(2);
+  
+  const options = {
+    generateSnapshots: args.includes('--generate-snapshots'),
+    maxTestCases: 20,
+    skipToolProcessing: args.includes('--skip-tool-processing'),
+    skipProtocolConversion: args.includes('--skip-protocol-conversion'),
+    skipV1V2Alignment: args.includes('--skip-v1v2-alignment')
+  };
+  
+  // 解析最大测试用例数
+  const maxCasesArg = args.find(arg => arg.startsWith('--max-cases='));
+  if (maxCasesArg) {
+    options.maxTestCases = parseInt(maxCasesArg.split('=')[1]);
+  }
+  
+  const test = new ComprehensiveConsistencyTest();
+  await test.runAllTests(options);
+}
+
+// 显示使用帮助
+if (process.argv.includes('--help') || process.argv.includes('-h')) {
+  console.log(`
+用法: npm run test:comprehensive [选项]
+
+选项:
+  --generate-snapshots          先生成快照数据
+  --max-cases=N               最大测试用例数 (默认: 20)
+  --skip-tool-processing       跳过工具处理测试
+  --skip-protocol-conversion   跳过协议转换测试
+  --skip-v1v2-alignment       跳过V1/V2对齐测试
+  --help, -h                  显示此帮助信息
+
+示例:
+  npm run test:comprehensive
+  npm run test:comprehensive --generate-snapshots --max-cases=10
+  npm run test:comprehensive --skip-tool-processing
+`);
+  process.exit(0);
+}
+
+main().catch(console.error);
+
+export { ComprehensiveConsistencyTest };

package/scripts/v2-consistency/run-consistency-test.mjs

@@ -0,0 +1,107 @@
+#!/usr/bin/env node
+
+/**
+ * V1/V2一致性测试执行脚本
+ * 整合快照加载、一致性验证和报告生成
+ */
+
+import fs from 'fs/promises';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const projectRoot = path.join(__dirname, '..', '..');
+
+// 导入测试类
+const { SnapshotLoader } = await import(path.join(projectRoot, 'tests/v2/src/utils/snapshot-loader.js'));
+const { ConsistencyValidator } = await import(path.join(projectRoot, 'tests/v2/src/utils/consistency-validator.js'));
+const { V1V2ConsistencyTest } = await import(path.join(projectRoot, 'tests/v2/src/consistency/v1v2-consistency-test.js'));
+
+async function main() {
+  const args = process.argv.slice(2);
+  const protocol = args[0];
+  const maxCases = args[1] ? parseInt(args[1]) : 20;
+  const generateSnapshots = args.includes('--generate-snapshots');
+
+  console.log('🚀 启动V1/V2一致性测试');
+  console.log('========================');
+  console.log(`📋 最大测试用例数: ${maxCases}`);
+  
+  if (protocol) {
+    console.log(`🎯 测试协议: ${protocol}`);
+  }
+
+  // 生成快照数据（如果需要）
+  if (generateSnapshots) {
+    console.log('\n📸 生成快照数据...');
+    const { SnapshotDataGenerator } = await import(path.join(projectRoot, 'scripts/generate-snapshot-data.mjs'));
+    const generator = new SnapshotDataGenerator();
+    await generator.generateAllSnapshots();
+  }
+
+  // 创建测试实例
+  const test = new V1V2ConsistencyTest({
+    maxTestCases: maxCases,
+    outputDir: path.join(projectRoot, 'test-results')
+  });
+
+  try {
+    let report;
+    
+    if (protocol && ['openai-chat', 'anthropic-messages', 'openai-responses'].includes(protocol)) {
+      report = await test.runProtocolTests(protocol);
+    } else {
+      report = await test.runAllTests();
+    }
+
+    // 设置退出码
+    const hasCriticalFailures = report.failures.some(f => f.severity === 'critical');
+    const hasMajorFailures = report.failures.some(f => f.severity === 'major');
+    
+    if (hasCriticalFailures) {
+      console.log('\n❌ 测试失败：发现关键错误');
+      process.exit(1);
+    } else if (hasMajorFailures) {
+      console.log('\n⚠️ 测试警告：发现重要错误');
+      process.exit(2);
+    } else {
+      console.log('\n✅ 测试通过：V1/V2一致性良好');
+      process.exit(0);
+    }
+
+  } catch (error) {
+    console.error('❌ 测试执行失败:', error);
+    process.exit(3);
+  }
+}
+
+// 显示使用帮助
+if (process.argv.includes('--help') || process.argv.includes('-h')) {
+  console.log(`
+用法: npm run test:consistency [协议] [最大用例数] [选项]
+
+协议选项:
+  openai-chat         仅测试OpenAI Chat协议
+  anthropic-messages  仅测试Anthropic Messages协议
+  openai-responses    仅测试OpenAI Responses协议
+  (无参数)             测试所有协议
+
+选项:
+  --generate-snapshots  先生成快照数据再测试
+
+示例:
+  npm run test:consistency
+  npm run test:consistency openai-chat 10
+  npm run test:consistency anthropic-messages 5 --generate-snapshots
+
+退出码:
+  0  - 所有测试通过
+  1  - 发现关键错误
+  2  - 发现重要错误  
+  3  - 测试执行失败
+`);
+  process.exit(0);
+}
+
+main().catch(console.error);