@winspan/claude-forge 8.41.0 → 8.51.0

package/scripts/e2e-routing.ts

@@ -0,0 +1,456 @@
+#!/usr/bin/env tsx
+/**
+ * E2E 验证：意图 → 路由 → preset Agent
+ *
+ * 策略：
+ *   - 直接构造 RouteInput（绕过 AI / IntentClassifier 调用，避免成本和不确定性）
+ *   - 加载真实 shipped 的 src/engine/conventions/routing.yaml
+ *   - 调用 agentRouter.decide() 看实际路由结果
+ *   - 不修改 routing.yaml 或 agent-router 源码
+ *
+ * 运行：npx tsx scripts/e2e-routing.ts
+ */
+
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import {
+  AgentRouter,
+  type RouteInput,
+  type RouteAction,
+  type TaskType,
+  type Complexity,
+  type ActionVerb,
+  type Artifact,
+  type Domain,
+} from '../src/engine/agent-router.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+// 加载真实的 routing.yaml（与 tests/unit/agent-router-bmad-harness.test.ts 同模式）
+const SHIPPED_YAML = path.resolve(
+  __dirname,
+  '..',
+  'src',
+  'engine',
+  'conventions',
+  'routing.yaml',
+);
+
+function makeRouter(): AgentRouter {
+  // userPath 指向不存在的路径，确保使用 shipped default
+  return new AgentRouter({
+    defaultPath: SHIPPED_YAML,
+    userPath: '/nonexistent/forge-routing.yaml',
+  });
+}
+
+// ── 测试用例类型 ─────────────────────────────────────────
+type ExpectedKind =
+  | { kind: 'agent'; name: string }
+  | { kind: 'agent_group' }
+  | { kind: 'skill' }
+  | { kind: 'none' }
+  | { kind: 'not_agent'; name: string }; // 验证不是某 agent
+
+interface TestCase {
+  name: string;
+  input: {
+    taskType: TaskType;
+    complexity: Complexity;
+    action?: ActionVerb;
+    artifact?: Artifact;
+    domain?: Domain;
+  };
+  expected: ExpectedKind;
+  note?: string;
+}
+
+const TEST_CASES: TestCase[] = [
+  // ── BMAD Feature 路由 ──────────────────────────
+  {
+    name: '复杂新功能 → bmad-feature-full',
+    input: { taskType: 'add_feature', complexity: 'complex' },
+    expected: { kind: 'agent', name: 'bmad-feature-full' },
+  },
+  {
+    name: '中等新功能 → bmad-feature-quick',
+    input: { taskType: 'add_feature', complexity: 'moderate' },
+    expected: { kind: 'agent', name: 'bmad-feature-quick' },
+  },
+  {
+    name: '简单新功能 → feature-developer (兜底)',
+    input: { taskType: 'add_feature', complexity: 'simple' },
+    expected: { kind: 'agent', name: 'feature-developer' },
+    note: '简单任务保留原 Agent',
+  },
+
+  // ── BMAD Spec Only 路由 ────────────────────────
+  {
+    name: '需求分析 complex → bmad-spec-only',
+    input: { taskType: 'analyze_requirement', complexity: 'complex' },
+    expected: { kind: 'agent', name: 'bmad-spec-only' },
+  },
+  {
+    name: '需求分析 moderate → bmad-spec-only',
+    input: { taskType: 'analyze_requirement', complexity: 'moderate' },
+    expected: { kind: 'agent', name: 'bmad-spec-only' },
+  },
+  {
+    name: '架构设计 moderate → bmad-spec-only',
+    input: { taskType: 'design_architecture', complexity: 'moderate' },
+    expected: { kind: 'agent', name: 'bmad-spec-only' },
+  },
+  {
+    name: 'API 设计 complex → bmad-spec-only',
+    input: { taskType: 'design_api', complexity: 'complex' },
+    expected: { kind: 'agent', name: 'bmad-spec-only' },
+  },
+  {
+    name: 'Schema 设计 moderate → bmad-spec-only',
+    input: { taskType: 'design_schema', complexity: 'moderate' },
+    expected: { kind: 'agent', name: 'bmad-spec-only' },
+  },
+  {
+    name: '简单需求分析 → planner (兜底，无方法论)',
+    input: { taskType: 'analyze_requirement', complexity: 'simple' },
+    expected: { kind: 'agent', name: 'planner' },
+    note: 'simple 时 BMAD 规则不命中',
+  },
+
+  // ── Harness 路由 ───────────────────────────────
+  {
+    name: '复杂 bug → harness-debug-full',
+    input: { taskType: 'fix_bug', complexity: 'complex' },
+    expected: { kind: 'agent', name: 'harness-debug-full' },
+  },
+  {
+    name: '中等 bug → coder (无 harness-hotfix 自动路由)',
+    input: { taskType: 'fix_bug', complexity: 'moderate' },
+    expected: { kind: 'agent', name: 'coder' },
+    note: '中等 fix_bug 不应触发 hotfix',
+  },
+  {
+    name: '简单 bug → coder (兜底)',
+    input: { taskType: 'fix_bug', complexity: 'simple' },
+    expected: { kind: 'agent', name: 'coder' },
+    note: '简单任务保留原 Agent',
+  },
+  {
+    name: '复杂调查 → harness-debug-full',
+    input: { taskType: 'investigate', complexity: 'complex' },
+    expected: { kind: 'agent', name: 'harness-debug-full' },
+  },
+  {
+    name: '中等调查 → harness-investigate',
+    input: { taskType: 'investigate', complexity: 'moderate' },
+    expected: { kind: 'agent', name: 'harness-investigate' },
+  },
+  {
+    name: '简单调查 → researcher (兜底)',
+    input: { taskType: 'investigate', complexity: 'simple' },
+    expected: { kind: 'agent', name: 'researcher' },
+  },
+
+  // ── V2 维度路由 ────────────────────────────────
+  {
+    name: 'V2: other + 代码级复杂调查 → harness-debug-full',
+    input: {
+      taskType: 'other',
+      complexity: 'complex',
+      action: 'investigate',
+      artifact: 'code',
+    },
+    expected: { kind: 'agent', name: 'harness-debug-full' },
+    note: 'V2 axes 救援路由',
+  },
+  {
+    name: 'V2: other + 代码级中等调查 → harness-investigate',
+    input: {
+      taskType: 'other',
+      complexity: 'moderate',
+      action: 'investigate',
+      artifact: 'code',
+    },
+    expected: { kind: 'agent', name: 'harness-investigate' },
+    note: 'V2 axes 救援路由',
+  },
+  {
+    name: 'V2: build 调查 → researcher (V2 precision 优先于方法论)',
+    input: {
+      taskType: 'investigate',
+      complexity: 'complex',
+      action: 'investigate',
+      artifact: 'build',
+    },
+    expected: { kind: 'agent', name: 'researcher' },
+    note: 'V2 precision 规则在方法论之前评估',
+  },
+  {
+    name: 'V2: fix build → refactor-specialist',
+    input: {
+      taskType: 'fix_bug',
+      complexity: 'complex',
+      action: 'fix',
+      artifact: 'build',
+    },
+    expected: { kind: 'agent', name: 'refactor-specialist' },
+    note: 'V2 precision 规则覆盖 fix_bug+complex',
+  },
+  {
+    name: 'V2: explain + complex → researcher',
+    input: {
+      taskType: 'explain',
+      complexity: 'complex',
+      action: 'explain',
+    },
+    expected: { kind: 'agent', name: 'researcher' },
+  },
+
+  // ── Agent Group 路由 ───────────────────────────
+  {
+    name: 'multi_domain + complex → route_to_agent_group',
+    input: { taskType: 'multi_domain', complexity: 'complex' },
+    expected: { kind: 'agent_group' },
+    note: '并行执行场景',
+  },
+
+  // ── 边界场景 ───────────────────────────────────
+  {
+    name: 'other → general-purpose (兜底)',
+    input: { taskType: 'other', complexity: 'simple' },
+    expected: { kind: 'agent', name: 'general-purpose' },
+    note: 'fallback 路径',
+  },
+  {
+    name: 'add_feature + simple + V2(create+code) 不应触发 BMAD',
+    input: {
+      taskType: 'add_feature',
+      complexity: 'simple',
+      action: 'create',
+      artifact: 'code',
+    },
+    expected: { kind: 'agent', name: 'feature-developer' },
+    note: 'V2 axes 不应错误激活 BMAD',
+  },
+
+  // ── harness-hotfix 防错路由 ────────────────────
+  {
+    name: 'harness-hotfix 不被自动路由 (fix_bug + simple)',
+    input: { taskType: 'fix_bug', complexity: 'simple' },
+    expected: { kind: 'not_agent', name: 'harness-hotfix' },
+    note: 'hotfix 应该手动调用',
+  },
+  {
+    name: 'harness-hotfix 不被自动路由 (fix_bug + moderate)',
+    input: { taskType: 'fix_bug', complexity: 'moderate' },
+    expected: { kind: 'not_agent', name: 'harness-hotfix' },
+  },
+  {
+    name: 'harness-hotfix 不被自动路由 (fix_bug + complex)',
+    input: { taskType: 'fix_bug', complexity: 'complex' },
+    expected: { kind: 'not_agent', name: 'harness-hotfix' },
+  },
+];
+
+// ── 工具函数 ────────────────────────────────────────────
+function actionToString(action: RouteAction): string {
+  switch (action.type) {
+    case 'route_to_agent':
+      return `agent:${action.name}`;
+    case 'route_to_agent_group':
+      return `agent_group:[${action.agents.map((a) => a.name).join(',')}]`;
+    case 'route_to_skill':
+      return `skill:${action.strategy}`;
+    case 'none':
+      return 'none';
+  }
+}
+
+function expectedToString(exp: ExpectedKind): string {
+  switch (exp.kind) {
+    case 'agent':
+      return `agent:${exp.name}`;
+    case 'agent_group':
+      return 'agent_group:*';
+    case 'skill':
+      return 'skill:*';
+    case 'none':
+      return 'none';
+    case 'not_agent':
+      return `NOT agent:${exp.name}`;
+  }
+}
+
+function checkExpected(actual: RouteAction, expected: ExpectedKind): boolean {
+  switch (expected.kind) {
+    case 'agent':
+      return actual.type === 'route_to_agent' && actual.name === expected.name;
+    case 'agent_group':
+      return actual.type === 'route_to_agent_group';
+    case 'skill':
+      return actual.type === 'route_to_skill';
+    case 'none':
+      return actual.type === 'none';
+    case 'not_agent':
+      // 必须不是指定 agent；其他任何路由都算通过
+      return !(actual.type === 'route_to_agent' && actual.name === expected.name);
+  }
+}
+
+// ── 执行测试 ─────────────────────────────────────────────
+console.log('='.repeat(60));
+console.log('  路由 E2E 验证 (真实 routing.yaml + 真实 AgentRouter)');
+console.log('='.repeat(60));
+console.log(`Routing YAML: ${SHIPPED_YAML}`);
+
+const router = makeRouter();
+const rules = router.getRules();
+console.log(`Loaded ${rules.length} routing rules\n`);
+
+let passed = 0;
+let failed = 0;
+const failures: { tc: TestCase; actual: string }[] = [];
+
+for (const tc of TEST_CASES) {
+  const input: RouteInput = {
+    taskType: tc.input.taskType,
+    complexity: tc.input.complexity,
+  };
+  if (tc.input.action !== undefined) input.action = tc.input.action;
+  if (tc.input.artifact !== undefined) input.artifact = tc.input.artifact;
+  if (tc.input.domain !== undefined) input.domain = tc.input.domain;
+
+  const decision = router.decide(input);
+  const actualStr = actionToString(decision.action);
+  const expectedStr = expectedToString(tc.expected);
+  const pass = checkExpected(decision.action, tc.expected);
+
+  const mark = pass ? 'PASS' : 'FAIL';
+  const noteStr = tc.note ? ` [${tc.note}]` : '';
+  console.log(
+    `[${mark}] ${tc.name}\n` +
+      `       expected=${expectedStr} actual=${actualStr} matchedRule=${decision.matchedRuleIndex}${noteStr}`,
+  );
+
+  if (pass) {
+    passed++;
+  } else {
+    failed++;
+    failures.push({ tc, actual: actualStr });
+  }
+}
+
+// ── 跨规则优先级专项验证 ─────────────────────────────
+console.log('\n' + '─'.repeat(60));
+console.log('  跨规则优先级专项验证');
+console.log('─'.repeat(60));
+
+interface PriorityCheck {
+  name: string;
+  input: RouteInput;
+  expected: ExpectedKind;
+  rationale: string;
+}
+
+const PRIORITY_CHECKS: PriorityCheck[] = [
+  {
+    name: 'V2 precision > 方法论 (build 调查)',
+    input: {
+      taskType: 'investigate',
+      complexity: 'complex',
+      action: 'investigate',
+      artifact: 'build',
+    },
+    expected: { kind: 'agent', name: 'researcher' },
+    rationale: 'V2 precision 规则在 YAML 中位列第一，应该优先于方法论规则',
+  },
+  {
+    name: '方法论 > 通用兜底 (复杂 add_feature)',
+    input: { taskType: 'add_feature', complexity: 'complex' },
+    expected: { kind: 'agent', name: 'bmad-feature-full' },
+    rationale: '复杂任务应优先匹配方法论 Agent，而非通用 feature-developer',
+  },
+  {
+    name: '简单任务 > 方法论 fallback (简单 add_feature)',
+    input: { taskType: 'add_feature', complexity: 'simple' },
+    expected: { kind: 'agent', name: 'feature-developer' },
+    rationale: '简单任务应该绕过方法论规则，命中通用 add_feature 兜底',
+  },
+];
+
+let pPassed = 0;
+let pFailed = 0;
+
+for (const pc of PRIORITY_CHECKS) {
+  const decision = router.decide(pc.input);
+  const actual = actionToString(decision.action);
+  const exp = expectedToString(pc.expected);
+  const pass = checkExpected(decision.action, pc.expected);
+  console.log(`[${pass ? 'PASS' : 'FAIL'}] ${pc.name}`);
+  console.log(`       理由: ${pc.rationale}`);
+  console.log(`       expected=${exp} actual=${actual} matchedRule=${decision.matchedRuleIndex}`);
+  pass ? pPassed++ : pFailed++;
+}
+
+// ── harness-hotfix 全空间扫描 ──────────────────────
+console.log('\n' + '─'.repeat(60));
+console.log('  harness-hotfix 全空间扫描 (确保不被自动路由)');
+console.log('─'.repeat(60));
+
+const HOTFIX_SCAN_INPUTS: RouteInput[] = [
+  // 各种 fix_bug 组合
+  { taskType: 'fix_bug', complexity: 'simple' },
+  { taskType: 'fix_bug', complexity: 'moderate' },
+  { taskType: 'fix_bug', complexity: 'complex' },
+  // 各种 investigate 组合
+  { taskType: 'investigate', complexity: 'simple' },
+  { taskType: 'investigate', complexity: 'moderate' },
+  { taskType: 'investigate', complexity: 'complex' },
+  // 带 V2 axes 的 fix
+  { taskType: 'fix_bug', complexity: 'complex', action: 'fix', artifact: 'code' },
+  { taskType: 'fix_bug', complexity: 'moderate', action: 'fix', artifact: 'code' },
+  { taskType: 'fix_bug', complexity: 'simple', action: 'fix', artifact: 'code' },
+];
+
+let hotfixViolations = 0;
+for (const input of HOTFIX_SCAN_INPUTS) {
+  const decision = router.decide(input);
+  const isHotfix =
+    decision.action.type === 'route_to_agent' && decision.action.name === 'harness-hotfix';
+  const tag = `${input.taskType}+${input.complexity}${input.action ? `+${input.action}` : ''}${input.artifact ? `+${input.artifact}` : ''}`;
+  const actualName = actionToString(decision.action);
+  if (isHotfix) {
+    console.log(`[FAIL] ${tag} → ${actualName} (不应路由到 harness-hotfix)`);
+    hotfixViolations++;
+  } else {
+    console.log(`[PASS] ${tag} → ${actualName}`);
+  }
+}
+
+// ── 总结 ─────────────────────────────────────────────────
+console.log('\n' + '='.repeat(60));
+console.log('  Summary');
+console.log('='.repeat(60));
+console.log(`Test cases:        ${passed}/${passed + failed} passed`);
+console.log(`Priority checks:   ${pPassed}/${pPassed + pFailed} passed`);
+console.log(
+  `Hotfix scan:       ${HOTFIX_SCAN_INPUTS.length - hotfixViolations}/${HOTFIX_SCAN_INPUTS.length} passed (violations=${hotfixViolations})`,
+);
+
+if (failures.length > 0) {
+  console.log('\nFailures:');
+  for (const f of failures) {
+    console.log(`  - ${f.tc.name}`);
+    console.log(`    expected=${expectedToString(f.tc.expected)} actual=${f.actual}`);
+  }
+}
+
+const totalFailed = failed + pFailed + hotfixViolations;
+if (totalFailed === 0) {
+  console.log('\n>>> RESULT: PASS <<<');
+  process.exit(0);
+} else {
+  console.log(`\n>>> RESULT: FAIL (${totalFailed} failures) <<<`);
+  process.exit(1);
+}