anvil-engineering-workbench 0.0.0-stage → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (233) hide show
  1. package/CHANGELOG.md +40 -0
  2. package/LICENSE +17 -0
  3. package/README.md +778 -3
  4. package/SECURITY.md +17 -0
  5. package/dist/ai/claude-code-provider.d.ts +31 -0
  6. package/dist/ai/claude-code-provider.js +81 -0
  7. package/dist/ai/claude-code-provider.js.map +1 -0
  8. package/dist/ai/provider.d.ts +47 -0
  9. package/dist/ai/provider.js +57 -0
  10. package/dist/ai/provider.js.map +1 -0
  11. package/dist/cli.d.ts +2 -0
  12. package/dist/cli.js +1945 -0
  13. package/dist/cli.js.map +1 -0
  14. package/dist/core/agent-config-release.d.ts +22 -0
  15. package/dist/core/agent-config-release.js +109 -0
  16. package/dist/core/agent-config-release.js.map +1 -0
  17. package/dist/core/agent-config-versioning.d.ts +33 -0
  18. package/dist/core/agent-config-versioning.js +209 -0
  19. package/dist/core/agent-config-versioning.js.map +1 -0
  20. package/dist/core/agent-evaluation-intelligence.d.ts +46 -0
  21. package/dist/core/agent-evaluation-intelligence.js +191 -0
  22. package/dist/core/agent-evaluation-intelligence.js.map +1 -0
  23. package/dist/core/agent-evolution.d.ts +20 -0
  24. package/dist/core/agent-evolution.js +35 -0
  25. package/dist/core/agent-evolution.js.map +1 -0
  26. package/dist/core/agent-experiment-studio.d.ts +27 -0
  27. package/dist/core/agent-experiment-studio.js +65 -0
  28. package/dist/core/agent-experiment-studio.js.map +1 -0
  29. package/dist/core/agent-optimization.d.ts +27 -0
  30. package/dist/core/agent-optimization.js +63 -0
  31. package/dist/core/agent-optimization.js.map +1 -0
  32. package/dist/core/agent-registry.d.ts +9 -0
  33. package/dist/core/agent-registry.js +95 -0
  34. package/dist/core/agent-registry.js.map +1 -0
  35. package/dist/core/agent-security-audit.d.ts +28 -0
  36. package/dist/core/agent-security-audit.js +260 -0
  37. package/dist/core/agent-security-audit.js.map +1 -0
  38. package/dist/core/agent-workspace-context.d.ts +14 -0
  39. package/dist/core/agent-workspace-context.js +66 -0
  40. package/dist/core/agent-workspace-context.js.map +1 -0
  41. package/dist/core/ai-code-review.d.ts +20 -0
  42. package/dist/core/ai-code-review.js +70 -0
  43. package/dist/core/ai-code-review.js.map +1 -0
  44. package/dist/core/anthropic-provider.d.ts +16 -0
  45. package/dist/core/anthropic-provider.js +85 -0
  46. package/dist/core/anthropic-provider.js.map +1 -0
  47. package/dist/core/architecture-map.d.ts +26 -0
  48. package/dist/core/architecture-map.js +124 -0
  49. package/dist/core/architecture-map.js.map +1 -0
  50. package/dist/core/benchmark-engine.d.ts +52 -0
  51. package/dist/core/benchmark-engine.js +109 -0
  52. package/dist/core/benchmark-engine.js.map +1 -0
  53. package/dist/core/benchmark-matrix.d.ts +123 -0
  54. package/dist/core/benchmark-matrix.js +169 -0
  55. package/dist/core/benchmark-matrix.js.map +1 -0
  56. package/dist/core/benchmark-regression.d.ts +54 -0
  57. package/dist/core/benchmark-regression.js +96 -0
  58. package/dist/core/benchmark-regression.js.map +1 -0
  59. package/dist/core/change-intelligence.d.ts +40 -0
  60. package/dist/core/change-intelligence.js +114 -0
  61. package/dist/core/change-intelligence.js.map +1 -0
  62. package/dist/core/code-intelligence.d.ts +20 -0
  63. package/dist/core/code-intelligence.js +119 -0
  64. package/dist/core/code-intelligence.js.map +1 -0
  65. package/dist/core/code-review.d.ts +30 -0
  66. package/dist/core/code-review.js +93 -0
  67. package/dist/core/code-review.js.map +1 -0
  68. package/dist/core/code-search.d.ts +16 -0
  69. package/dist/core/code-search.js +93 -0
  70. package/dist/core/code-search.js.map +1 -0
  71. package/dist/core/container-sandbox.d.ts +20 -0
  72. package/dist/core/container-sandbox.js +220 -0
  73. package/dist/core/container-sandbox.js.map +1 -0
  74. package/dist/core/dashboard-server.d.ts +11 -0
  75. package/dist/core/dashboard-server.js +1068 -0
  76. package/dist/core/dashboard-server.js.map +1 -0
  77. package/dist/core/dependency-graph.d.ts +29 -0
  78. package/dist/core/dependency-graph.js +90 -0
  79. package/dist/core/dependency-graph.js.map +1 -0
  80. package/dist/core/dependency-impact.d.ts +51 -0
  81. package/dist/core/dependency-impact.js +201 -0
  82. package/dist/core/dependency-impact.js.map +1 -0
  83. package/dist/core/dependency-intelligence.d.ts +60 -0
  84. package/dist/core/dependency-intelligence.js +241 -0
  85. package/dist/core/dependency-intelligence.js.map +1 -0
  86. package/dist/core/doctor.d.ts +17 -0
  87. package/dist/core/doctor.js +239 -0
  88. package/dist/core/doctor.js.map +1 -0
  89. package/dist/core/engineering-risk.d.ts +40 -0
  90. package/dist/core/engineering-risk.js +125 -0
  91. package/dist/core/engineering-risk.js.map +1 -0
  92. package/dist/core/evaluation-engine.d.ts +110 -0
  93. package/dist/core/evaluation-engine.js +182 -0
  94. package/dist/core/evaluation-engine.js.map +1 -0
  95. package/dist/core/execution-trace.d.ts +27 -0
  96. package/dist/core/execution-trace.js +137 -0
  97. package/dist/core/execution-trace.js.map +1 -0
  98. package/dist/core/experiment-history.d.ts +31 -0
  99. package/dist/core/experiment-history.js +64 -0
  100. package/dist/core/experiment-history.js.map +1 -0
  101. package/dist/core/experiment-manifest.d.ts +67 -0
  102. package/dist/core/experiment-manifest.js +100 -0
  103. package/dist/core/experiment-manifest.js.map +1 -0
  104. package/dist/core/gemini-provider.d.ts +15 -0
  105. package/dist/core/gemini-provider.js +87 -0
  106. package/dist/core/gemini-provider.js.map +1 -0
  107. package/dist/core/git-intelligence.d.ts +26 -0
  108. package/dist/core/git-intelligence.js +84 -0
  109. package/dist/core/git-intelligence.js.map +1 -0
  110. package/dist/core/hook-engine.d.ts +23 -0
  111. package/dist/core/hook-engine.js +74 -0
  112. package/dist/core/hook-engine.js.map +1 -0
  113. package/dist/core/hosted-agent-provider.d.ts +19 -0
  114. package/dist/core/hosted-agent-provider.js +133 -0
  115. package/dist/core/hosted-agent-provider.js.map +1 -0
  116. package/dist/core/intelligence-timeline.d.ts +41 -0
  117. package/dist/core/intelligence-timeline.js +158 -0
  118. package/dist/core/intelligence-timeline.js.map +1 -0
  119. package/dist/core/model-router.d.ts +59 -0
  120. package/dist/core/model-router.js +291 -0
  121. package/dist/core/model-router.js.map +1 -0
  122. package/dist/core/openai-compatible-provider.d.ts +19 -0
  123. package/dist/core/openai-compatible-provider.js +151 -0
  124. package/dist/core/openai-compatible-provider.js.map +1 -0
  125. package/dist/core/orchestrator.d.ts +94 -0
  126. package/dist/core/orchestrator.js +263 -0
  127. package/dist/core/orchestrator.js.map +1 -0
  128. package/dist/core/patch-application.d.ts +54 -0
  129. package/dist/core/patch-application.js +504 -0
  130. package/dist/core/patch-application.js.map +1 -0
  131. package/dist/core/patch-approval.d.ts +23 -0
  132. package/dist/core/patch-approval.js +181 -0
  133. package/dist/core/patch-approval.js.map +1 -0
  134. package/dist/core/patch-validation.d.ts +12 -0
  135. package/dist/core/patch-validation.js +112 -0
  136. package/dist/core/patch-validation.js.map +1 -0
  137. package/dist/core/permission-policy.d.ts +13 -0
  138. package/dist/core/permission-policy.js +98 -0
  139. package/dist/core/permission-policy.js.map +1 -0
  140. package/dist/core/project-health.d.ts +72 -0
  141. package/dist/core/project-health.js +79 -0
  142. package/dist/core/project-health.js.map +1 -0
  143. package/dist/core/project-init.d.ts +9 -0
  144. package/dist/core/project-init.js +253 -0
  145. package/dist/core/project-init.js.map +1 -0
  146. package/dist/core/project-inspector.d.ts +24 -0
  147. package/dist/core/project-inspector.js +175 -0
  148. package/dist/core/project-inspector.js.map +1 -0
  149. package/dist/core/project-memory.d.ts +22 -0
  150. package/dist/core/project-memory.js +173 -0
  151. package/dist/core/project-memory.js.map +1 -0
  152. package/dist/core/provider-diagnostics.d.ts +12 -0
  153. package/dist/core/provider-diagnostics.js +73 -0
  154. package/dist/core/provider-diagnostics.js.map +1 -0
  155. package/dist/core/provider-response.d.ts +4 -0
  156. package/dist/core/provider-response.js +46 -0
  157. package/dist/core/provider-response.js.map +1 -0
  158. package/dist/core/quality-audit.d.ts +29 -0
  159. package/dist/core/quality-audit.js +101 -0
  160. package/dist/core/quality-audit.js.map +1 -0
  161. package/dist/core/regression-intelligence.d.ts +41 -0
  162. package/dist/core/regression-intelligence.js +56 -0
  163. package/dist/core/regression-intelligence.js.map +1 -0
  164. package/dist/core/remediation-planner.d.ts +43 -0
  165. package/dist/core/remediation-planner.js +124 -0
  166. package/dist/core/remediation-planner.js.map +1 -0
  167. package/dist/core/review-history.d.ts +18 -0
  168. package/dist/core/review-history.js +68 -0
  169. package/dist/core/review-history.js.map +1 -0
  170. package/dist/core/review-planner.d.ts +2 -0
  171. package/dist/core/review-planner.js +79 -0
  172. package/dist/core/review-planner.js.map +1 -0
  173. package/dist/core/run-history-output.d.ts +22 -0
  174. package/dist/core/run-history-output.js +51 -0
  175. package/dist/core/run-history-output.js.map +1 -0
  176. package/dist/core/run-replay.d.ts +9 -0
  177. package/dist/core/run-replay.js +64 -0
  178. package/dist/core/run-replay.js.map +1 -0
  179. package/dist/core/run-store.d.ts +13 -0
  180. package/dist/core/run-store.js +300 -0
  181. package/dist/core/run-store.js.map +1 -0
  182. package/dist/core/sandbox-cleanup-audit.d.ts +14 -0
  183. package/dist/core/sandbox-cleanup-audit.js +57 -0
  184. package/dist/core/sandbox-cleanup-audit.js.map +1 -0
  185. package/dist/core/sandbox-compliance.d.ts +19 -0
  186. package/dist/core/sandbox-compliance.js +34 -0
  187. package/dist/core/sandbox-compliance.js.map +1 -0
  188. package/dist/core/sandbox-lifecycle.d.ts +8 -0
  189. package/dist/core/sandbox-lifecycle.js +36 -0
  190. package/dist/core/sandbox-lifecycle.js.map +1 -0
  191. package/dist/core/sandbox-policy.d.ts +31 -0
  192. package/dist/core/sandbox-policy.js +80 -0
  193. package/dist/core/sandbox-policy.js.map +1 -0
  194. package/dist/core/sandbox-runtime.d.ts +10 -0
  195. package/dist/core/sandbox-runtime.js +44 -0
  196. package/dist/core/sandbox-runtime.js.map +1 -0
  197. package/dist/core/sandbox-security-profiles.d.ts +16 -0
  198. package/dist/core/sandbox-security-profiles.js +43 -0
  199. package/dist/core/sandbox-security-profiles.js.map +1 -0
  200. package/dist/core/skill-registry.d.ts +14 -0
  201. package/dist/core/skill-registry.js +186 -0
  202. package/dist/core/skill-registry.js.map +1 -0
  203. package/dist/core/staged-verification.d.ts +27 -0
  204. package/dist/core/staged-verification.js +108 -0
  205. package/dist/core/staged-verification.js.map +1 -0
  206. package/dist/core/verification-history.d.ts +10 -0
  207. package/dist/core/verification-history.js +71 -0
  208. package/dist/core/verification-history.js.map +1 -0
  209. package/dist/core/verification.d.ts +37 -0
  210. package/dist/core/verification.js +204 -0
  211. package/dist/core/verification.js.map +1 -0
  212. package/dist/core/workflow-store.d.ts +16 -0
  213. package/dist/core/workflow-store.js +113 -0
  214. package/dist/core/workflow-store.js.map +1 -0
  215. package/dist/core/workspace-browser.d.ts +22 -0
  216. package/dist/core/workspace-browser.js +115 -0
  217. package/dist/core/workspace-browser.js.map +1 -0
  218. package/dist/core/workspace-scan-policy.d.ts +2 -0
  219. package/dist/core/workspace-scan-policy.js +11 -0
  220. package/dist/core/workspace-scan-policy.js.map +1 -0
  221. package/dist/core/workspace-tool-broker.d.ts +28 -0
  222. package/dist/core/workspace-tool-broker.js +70 -0
  223. package/dist/core/workspace-tool-broker.js.map +1 -0
  224. package/dist/index.d.ts +75 -0
  225. package/dist/index.js +82 -0
  226. package/dist/index.js.map +1 -0
  227. package/dist/mcp/server.d.ts +2 -0
  228. package/dist/mcp/server.js +342 -0
  229. package/dist/mcp/server.js.map +1 -0
  230. package/dist/mcp/workspace-path.d.ts +2 -0
  231. package/dist/mcp/workspace-path.js +41 -0
  232. package/dist/mcp/workspace-path.js.map +1 -0
  233. package/package.json +67 -5
@@ -0,0 +1 @@
1
+ {"version":3,"file":"benchmark-engine.js","sourceRoot":"","sources":["../../src/core/benchmark-engine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACnF,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,IAAI,MAAM,WAAW,CAAC;AAkD7B,SAAS,SAAS,CAAC,MAAwB;IACzC,MAAM,OAAO,GAAG,IAAI,GAAG,EAA4B,CAAC;IACpD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ,GAAG,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7D,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAC3B,CAAC;IACD,OAAO,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;QACjD,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC7C,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAyE,EAAE,CAAC,IAAI,CAAC,UAAU,KAAK,SAAS,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC;QAC9K,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QAC9E,OAAO;YACL,GAAG,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAE,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM;YACjF,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,GAAG,EAAE;YAC3C,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,EAAE,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACpP,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;YAC/D,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE;YACzF,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE;YAClI,iBAAiB,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;YACnG,sBAAsB,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,qBAAqB,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;YAC/I,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;YAChE,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;SACrE,CAAC;IACJ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,iBAAiB,GAAG,CAAC,CAAC,iBAAiB,CAAC,CAAC;AAClG,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,MAAiD,EAAE,KAA6B;IACnH,IAAI,MAAM,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAC;IACxH,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAClF,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC5E,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS;QAC/G,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,OAAO,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,YAAY,EAAE,aAAa,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,SAAS,EAAE,MAAM;QAChP,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;QAC7F,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;QAC5E,oBAAoB,EAAE,KAAK,CAAC,MAAM,CAAC,oBAAoB,EAAE,qBAAqB,EAAE,KAAK,CAAC,MAAM,CAAC,qBAAqB;QAClH,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,UAAU;KACpC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,WAAmB,EAAE,KAAqB;IACnF,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAC;IAC/E,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACzD,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,GAAG,OAAO,CAAC,CAAC;IAClE,IAAI,MAAM,GAAqB,EAAE,CAAC;IAClC,IAAI,CAAC;QACH,MAAM,QAAQ,GAAY,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC;QAC1E,IAAI,CAAC,QAAQ,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAK,QAA8B,CAAC,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAE,QAA8B,CAAC,MAAM,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;QACpN,MAAM,GAAI,QAA8B,CAAC,MAAM,CAAC;IAClD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,CAAC,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC;YAAE,MAAM,KAAK,CAAC;IACvG,CAAC;IACD,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;IAChI,MAAM,QAAQ,GAAsB,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;IAC/K,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,GAAG,UAAU,EAAE,GAAG,MAAM,CAAC,CAAC;IACpE,IAAI,CAAC;QACH,MAAM,SAAS,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QACpH,MAAM,MAAM,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;IACvC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,EAAE,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC5D,MAAM,KAAK,CAAC;IACd,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,WAAmB,EAAE,OAAe;IACtE,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACvG,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,GAAG,OAAO,CAAC,CAAC;IAC7F,MAAM,KAAK,GAAY,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IAChE,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAK,KAA2B,CAAC,OAAO,KAAK,CAAC,IAAK,KAA2B,CAAC,OAAO,KAAK,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAE,KAA2B,CAAC,MAAM,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC3P,OAAO,KAA0B,CAAC;AACpC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,WAAmB;IACtD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAC;IAC/E,IAAI,KAAe,CAAC;IACpB,IAAI,CAAC;QAAC,KAAK,GAAG,CAAC,MAAM,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAAC,CAAC;IACzF,OAAO,KAAK,EAAE,CAAC;QAAC,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,EAAE,CAAC;QAAC,MAAM,KAAK,CAAC;IAAC,CAAC;IAC/H,MAAM,SAAS,GAAwB,EAAE,CAAC;IAC1C,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;QACvC,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7B,SAAS,CAAC,IAAI,CAAC,MAAM,aAAa,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC;IACvD,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,QAA2B;IACzD,MAAM,KAAK,GAAG,CAAC,wBAAwB,GAAG,QAAQ,CAAC,SAAS,EAAE,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,EAAE,mGAAmG,CAAC,CAAC;IAC5M,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM;QAAE,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,YAAY,GAAG,MAAM,EAAE,KAAK,CAAC,iBAAiB,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,iBAAiB,GAAG,QAAQ,GAAG,KAAK,CAAC,aAAa,GAAG,SAAS,EAAE,KAAK,CAAC,QAAQ,GAAG,GAAG,EAAE,IAAI,GAAG,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,iBAAiB,GAAG,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IACvV,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IACjC,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE;QAAE,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,KAAK,GAAG,KAAK,GAAG,KAAK,CAAC,QAAQ,GAAG,KAAK,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,GAAG,KAAK,GAAG,KAAK,CAAC,KAAK,GAAG,aAAa,GAAG,CAAC,KAAK,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,UAAU,GAAG,YAAY,CAAC,GAAG,KAAK,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;IACxU,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,iIAAiI,CAAC,CAAC;IAClJ,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}
@@ -0,0 +1,123 @@
1
+ export interface BenchmarkCandidate {
2
+ id: string;
3
+ provider: "openai" | "claude-code" | "anthropic" | "google" | "ollama";
4
+ model: string;
5
+ /** undefined = normal auto-selection; [] = no skills; ids = force this skill set. */
6
+ skillIds?: string[];
7
+ /** Immutable workspace configuration snapshot used for this candidate. */
8
+ configVersionId?: string;
9
+ }
10
+ export interface BenchmarkMatrixConfig {
11
+ id: string;
12
+ name: string;
13
+ repeats: number;
14
+ candidates: BenchmarkCandidate[];
15
+ skillExperiment?: {
16
+ baselineCandidateId: string;
17
+ treatmentCandidateId: string;
18
+ skillIds: string[];
19
+ };
20
+ }
21
+ export interface AgentBenchmarkScore {
22
+ agent: string;
23
+ caseId: string;
24
+ score: number;
25
+ passed: boolean;
26
+ skillIds: string[];
27
+ failedCriteria?: Array<{
28
+ id: string;
29
+ failedChecks: string[];
30
+ }>;
31
+ }
32
+ export interface AgentBenchmarkSummary {
33
+ candidateId: string;
34
+ provider: string;
35
+ model: string;
36
+ agent: string;
37
+ trials: number;
38
+ completed: number;
39
+ averageScore: number;
40
+ passRate: number;
41
+ skillsObserved: string[];
42
+ }
43
+ export interface BenchmarkMatrixResult {
44
+ candidateId: string;
45
+ provider: string;
46
+ model: string;
47
+ repeat: number;
48
+ runId: string;
49
+ score: number;
50
+ passed: boolean;
51
+ durationMs: number;
52
+ estimatedTokens: number;
53
+ agentScores?: AgentBenchmarkScore[];
54
+ status: "completed" | "failed";
55
+ error?: string;
56
+ }
57
+ export interface BenchmarkMatrixReport {
58
+ version: 1;
59
+ matrixId: string;
60
+ matrixName: string;
61
+ suiteId: string;
62
+ goal: string;
63
+ createdAt: string;
64
+ candidateProvenance?: Array<{
65
+ candidateId: string;
66
+ configVersionId: string;
67
+ fingerprint: string;
68
+ }>;
69
+ experimentManifest?: {
70
+ id: string;
71
+ fingerprint: string;
72
+ };
73
+ results: BenchmarkMatrixResult[];
74
+ agentSummary?: AgentBenchmarkSummary[];
75
+ skillExperiment?: {
76
+ baselineCandidateId: string;
77
+ treatmentCandidateId: string;
78
+ skillIds: string[];
79
+ baselineScore: number;
80
+ treatmentScore: number;
81
+ scoreDelta: number;
82
+ baselinePassRate: number;
83
+ treatmentPassRate: number;
84
+ passRateDelta: number;
85
+ baselineDurationMs: number;
86
+ treatmentDurationMs: number;
87
+ durationChangePercent: number;
88
+ baselineEstimatedTokens: number;
89
+ treatmentEstimatedTokens: number;
90
+ tokenChangePercent: number;
91
+ agentDeltas: Array<{
92
+ agent: string;
93
+ baselineScore: number;
94
+ treatmentScore: number;
95
+ scoreDelta: number;
96
+ baselinePassRate: number;
97
+ treatmentPassRate: number;
98
+ passRateDelta: number;
99
+ }>;
100
+ };
101
+ summary: Array<{
102
+ candidateId: string;
103
+ provider: string;
104
+ model: string;
105
+ trials: number;
106
+ completed: number;
107
+ averageScore: number;
108
+ passRate: number;
109
+ scoreStdDev: number;
110
+ averageDurationMs: number;
111
+ averageEstimatedTokens: number;
112
+ }>;
113
+ }
114
+ export declare function validateBenchmarkMatrix(value: unknown): BenchmarkMatrixConfig;
115
+ export declare function fingerprintAgentConfig(files: Array<{
116
+ path: string;
117
+ sha256: string;
118
+ }>): string;
119
+ export declare function loadBenchmarkMatrix(file: string): Promise<BenchmarkMatrixConfig>;
120
+ export declare function executeBenchmarkMatrix(config: BenchmarkMatrixConfig, execute: (candidate: BenchmarkCandidate, repeat: number) => Promise<Omit<BenchmarkMatrixResult, "candidateId" | "provider" | "model" | "repeat" | "status">>): Promise<BenchmarkMatrixReport>;
121
+ export declare function saveBenchmarkMatrix(projectPath: string, report: BenchmarkMatrixReport): Promise<string>;
122
+ export declare function formatBenchmarkMatrix(report: BenchmarkMatrixReport): string;
123
+ export declare function listBenchmarkMatrices(projectPath: string): Promise<BenchmarkMatrixReport[]>;
@@ -0,0 +1,169 @@
1
+ import { mkdir, readFile, readdir, rename, rm, writeFile } from "node:fs/promises";
2
+ import { createHash, randomUUID } from "node:crypto";
3
+ import path from "node:path";
4
+ const SAFE_ID = /^[a-z0-9][a-z0-9-]{1,47}$/;
5
+ export function validateBenchmarkMatrix(value) {
6
+ if (!value || typeof value !== "object" || Array.isArray(value))
7
+ throw new Error("Benchmark matrix config must be a JSON object.");
8
+ const config = value;
9
+ if (!SAFE_ID.test(config.id) || typeof config.name !== "string" || !config.name.trim() || config.name.length > 120)
10
+ throw new Error("Benchmark matrix needs a valid id and name.");
11
+ if (!Number.isSafeInteger(config.repeats) || config.repeats < 1 || config.repeats > 3)
12
+ throw new Error("Benchmark matrix repeats must be between 1 and 3.");
13
+ if (!Array.isArray(config.candidates) || config.candidates.length < 2 || config.candidates.length > 4)
14
+ throw new Error("Benchmark matrix needs between 2 and 4 candidates.");
15
+ const ids = new Set();
16
+ for (const candidate of config.candidates) {
17
+ if (!candidate || !SAFE_ID.test(candidate.id) || ids.has(candidate.id))
18
+ throw new Error("Every benchmark candidate needs a unique valid id.");
19
+ ids.add(candidate.id);
20
+ if (!["openai", "claude-code", "anthropic", "google", "ollama"].includes(candidate.provider))
21
+ throw new Error("Unsupported benchmark candidate provider.");
22
+ if (typeof candidate.model !== "string" || !candidate.model.trim() || candidate.model.length > 160)
23
+ throw new Error("Every benchmark candidate needs a model name.");
24
+ if (candidate.configVersionId !== undefined && !/^cfg-[a-f0-9]{12}$/.test(candidate.configVersionId))
25
+ throw new Error("Candidate configVersionId must be a valid agent configuration snapshot ID.");
26
+ }
27
+ for (const candidate of config.candidates) {
28
+ if (candidate.skillIds !== undefined && (!Array.isArray(candidate.skillIds) || candidate.skillIds.some(id => typeof id !== "string" || !/^[a-z0-9][a-z0-9-]{0,63}$/.test(id)) || new Set(candidate.skillIds).size !== candidate.skillIds.length))
29
+ throw new Error("Candidate skillIds must be unique valid skill IDs.");
30
+ }
31
+ if (config.skillExperiment) {
32
+ const experiment = config.skillExperiment;
33
+ const baseline = config.candidates.find(item => item.id === experiment.baselineCandidateId);
34
+ const treatment = config.candidates.find(item => item.id === experiment.treatmentCandidateId);
35
+ if (!baseline || !treatment || baseline.id === treatment.id)
36
+ throw new Error("Skill experiment must reference distinct baseline and treatment candidates.");
37
+ if (baseline.provider !== treatment.provider || baseline.model !== treatment.model)
38
+ throw new Error("Skill experiment candidates must use the same provider and model.");
39
+ if (!Array.isArray(experiment.skillIds) || !experiment.skillIds.length || experiment.skillIds.some(id => typeof id !== "string" || !/^[a-z0-9][a-z0-9-]{0,63}$/.test(id)) || new Set(experiment.skillIds).size !== experiment.skillIds.length)
40
+ throw new Error("Skill experiment must specify unique valid skill IDs.");
41
+ if (!Array.isArray(baseline.skillIds) || baseline.skillIds.length !== 0 || !Array.isArray(treatment.skillIds) || [...treatment.skillIds].sort().join(",") !== [...experiment.skillIds].sort().join(","))
42
+ throw new Error("Baseline must set skillIds to [] and treatment skillIds must exactly match skillExperiment.skillIds.");
43
+ }
44
+ return config;
45
+ }
46
+ export function fingerprintAgentConfig(files) {
47
+ const canonical = [...files].sort((a, b) => a.path.localeCompare(b.path))
48
+ .map(file => file.path + ":" + file.sha256).join("\\n");
49
+ return createHash("sha256").update(canonical, "utf8").digest("hex");
50
+ }
51
+ export async function loadBenchmarkMatrix(file) {
52
+ return validateBenchmarkMatrix(JSON.parse(await readFile(file, "utf8")));
53
+ }
54
+ export async function executeBenchmarkMatrix(config, execute) {
55
+ config = validateBenchmarkMatrix(config);
56
+ const results = [];
57
+ for (const candidate of config.candidates) {
58
+ for (let repeat = 1; repeat <= config.repeats; repeat += 1) {
59
+ try {
60
+ const result = await execute(candidate, repeat);
61
+ results.push({ candidateId: candidate.id, provider: candidate.provider, model: candidate.model, repeat, status: "completed", ...result });
62
+ }
63
+ catch (error) {
64
+ results.push({ candidateId: candidate.id, provider: candidate.provider, model: candidate.model, repeat, runId: "", score: 0, passed: false, durationMs: 0, estimatedTokens: 0, status: "failed", error: error instanceof Error ? error.message : String(error) });
65
+ }
66
+ }
67
+ }
68
+ const summary = config.candidates.map(candidate => {
69
+ const items = results.filter(item => item.candidateId === candidate.id && item.status === "completed");
70
+ const scores = items.map(item => item.score);
71
+ const average = scores.length ? scores.reduce((sum, score) => sum + score, 0) / scores.length : 0;
72
+ return {
73
+ candidateId: candidate.id, provider: candidate.provider, model: candidate.model, trials: config.repeats,
74
+ completed: items.length,
75
+ averageScore: Math.round(average * 10) / 10,
76
+ passRate: items.length ? Math.round(items.filter(item => item.passed).length / items.length * 1000) / 10 : 0,
77
+ scoreStdDev: items.length ? Math.round(Math.sqrt(scores.reduce((sum, score) => sum + Math.pow(score - average, 2), 0) / scores.length) * 10) / 10 : 0,
78
+ averageDurationMs: items.length ? Math.round(items.reduce((sum, item) => sum + item.durationMs, 0) / items.length) : 0,
79
+ averageEstimatedTokens: items.length ? Math.round(items.reduce((sum, item) => sum + item.estimatedTokens, 0) / items.length) : 0,
80
+ };
81
+ }).sort((a, b) => b.averageScore - a.averageScore || a.averageDurationMs - b.averageDurationMs);
82
+ const agentKeys = new Set(results.flatMap(item => (item.agentScores ?? []).map(score => item.candidateId + "::" + score.agent)));
83
+ const agentSummary = [...agentKeys].map(key => {
84
+ const separator = key.indexOf("::");
85
+ const candidateId = key.slice(0, separator);
86
+ const agent = key.slice(separator + 2);
87
+ const candidate = config.candidates.find(item => item.id === candidateId);
88
+ const scores = results.filter(item => item.candidateId === candidateId && item.status === "completed").flatMap(item => (item.agentScores ?? []).filter(score => score.agent === agent));
89
+ const skillsObserved = [...new Set(scores.flatMap(item => item.skillIds))].sort();
90
+ return { candidateId, provider: candidate.provider, model: candidate.model, agent, trials: config.repeats, completed: scores.length, averageScore: scores.length ? Math.round(scores.reduce((sum, item) => sum + item.score, 0) / scores.length * 10) / 10 : 0, passRate: scores.length ? Math.round(scores.filter(item => item.passed).length / scores.length * 1000) / 10 : 0, skillsObserved };
91
+ }).sort((a, b) => b.averageScore - a.averageScore || a.agent.localeCompare(b.agent));
92
+ let skillExperiment;
93
+ if (config.skillExperiment) {
94
+ const { baselineCandidateId, treatmentCandidateId, skillIds } = config.skillExperiment;
95
+ const baseline = summary.find(item => item.candidateId === baselineCandidateId);
96
+ const treatment = summary.find(item => item.candidateId === treatmentCandidateId);
97
+ if (baseline && treatment && baseline.completed > 0 && treatment.completed > 0) {
98
+ const change = (before, after) => before <= 0 ? (after > 0 ? 100 : 0) : Math.round((after - before) / before * 1000) / 10;
99
+ const agentDeltas = (agentSummary.filter(item => item.candidateId === baselineCandidateId)).flatMap(before => {
100
+ const after = agentSummary.find(item => item.candidateId === treatmentCandidateId && item.agent === before.agent);
101
+ return after ? [{ agent: before.agent, baselineScore: before.averageScore, treatmentScore: after.averageScore, scoreDelta: Math.round((after.averageScore - before.averageScore) * 10) / 10, baselinePassRate: before.passRate, treatmentPassRate: after.passRate, passRateDelta: Math.round((after.passRate - before.passRate) * 10) / 10 }] : [];
102
+ });
103
+ skillExperiment = { baselineCandidateId, treatmentCandidateId, skillIds: [...skillIds], baselineScore: baseline.averageScore, treatmentScore: treatment.averageScore, scoreDelta: Math.round((treatment.averageScore - baseline.averageScore) * 10) / 10, baselinePassRate: baseline.passRate, treatmentPassRate: treatment.passRate, passRateDelta: Math.round((treatment.passRate - baseline.passRate) * 10) / 10, baselineDurationMs: baseline.averageDurationMs, treatmentDurationMs: treatment.averageDurationMs, durationChangePercent: change(baseline.averageDurationMs, treatment.averageDurationMs), baselineEstimatedTokens: baseline.averageEstimatedTokens, treatmentEstimatedTokens: treatment.averageEstimatedTokens, tokenChangePercent: change(baseline.averageEstimatedTokens, treatment.averageEstimatedTokens), agentDeltas };
104
+ }
105
+ }
106
+ return { version: 1, matrixId: config.id, matrixName: config.name, suiteId: "", goal: "", createdAt: new Date().toISOString(), results, summary, agentSummary, ...(config.candidates.some(candidate => candidate.configVersionId) ? { candidateProvenance: config.candidates.filter(candidate => candidate.configVersionId).map(candidate => ({ candidateId: candidate.id, configVersionId: candidate.configVersionId, fingerprint: "" })) } : {}), ...(skillExperiment ? { skillExperiment } : {}) };
107
+ }
108
+ export async function saveBenchmarkMatrix(projectPath, report) {
109
+ const directory = path.join(path.resolve(projectPath), ".forge", "benchmark-matrices");
110
+ await mkdir(directory, { recursive: true, mode: 0o700 });
111
+ const destination = path.join(directory, report.matrixId + "-" + report.createdAt.replace(/[:.]/g, "-") + ".json");
112
+ const temporary = path.join(directory, "." + randomUUID() + ".tmp");
113
+ try {
114
+ await writeFile(temporary, JSON.stringify(report, null, 2) + "\n", { encoding: "utf8", flag: "wx", mode: 0o600 });
115
+ await rename(temporary, destination);
116
+ }
117
+ catch (error) {
118
+ await rm(temporary, { force: true }).catch(() => undefined);
119
+ throw error;
120
+ }
121
+ return destination;
122
+ }
123
+ export function formatBenchmarkMatrix(report) {
124
+ const lines = ["FORGE Automated Benchmark Matrix · " + report.matrixName, "Suite: " + report.suiteId, "Goal: " + report.goal, "", "Candidate | Trials | Completed | Avg score | Pass rate | σ score | Avg time | Avg estimated tokens"];
125
+ for (const item of report.summary)
126
+ lines.push([item.candidateId + " (" + item.provider + "/" + item.model + ")", item.trials, item.completed, item.averageScore + "/100", item.passRate + "%", item.scoreStdDev, item.averageDurationMs + " ms", item.averageEstimatedTokens].join(" | "));
127
+ if (report.agentSummary?.length) {
128
+ lines.push("", "Per-agent scorecards:", "Candidate | Agent | Cases | Average score | Pass rate | Skills observed");
129
+ for (const item of report.agentSummary)
130
+ lines.push([item.candidateId, item.agent, item.completed, item.averageScore + "/100", item.passRate + "%", item.skillsObserved.join(", ") || "none"].join(" | "));
131
+ }
132
+ if (report.skillExperiment) {
133
+ const item = report.skillExperiment;
134
+ lines.push("", "Controlled skill experiment:", " Baseline: " + item.baselineCandidateId + " (no skills)", " Treatment: " + item.treatmentCandidateId + " (skills: " + item.skillIds.join(", ") + ")", " Score: " + item.baselineScore + " → " + item.treatmentScore + " (" + (item.scoreDelta > 0 ? "+" : "") + item.scoreDelta + ")", " Pass rate: " + item.baselinePassRate + "% → " + item.treatmentPassRate + "% (" + (item.passRateDelta > 0 ? "+" : "") + item.passRateDelta + " pp)", " Duration change: " + item.durationChangePercent + "% · estimated token change: " + item.tokenChangePercent + "%");
135
+ for (const agent of item.agentDeltas)
136
+ lines.push(" Agent " + agent.agent + ": " + agent.baselineScore + " → " + agent.treatmentScore + " (" + (agent.scoreDelta > 0 ? "+" : "") + agent.scoreDelta + ")");
137
+ }
138
+ lines.push("", "Trial details:");
139
+ for (const item of report.results)
140
+ lines.push(" " + item.candidateId + " #" + item.repeat + " · " + (item.status === "completed" ? item.score + "/100 · " + (item.passed ? "PASS" : "REVIEW") : "FAILED · " + item.error));
141
+ if (report.candidateProvenance?.length) {
142
+ lines.push("", "Configuration provenance:");
143
+ for (const item of report.candidateProvenance)
144
+ lines.push(" " + item.candidateId + " · " + item.configVersionId + " · sha256 " + item.fingerprint);
145
+ }
146
+ lines.push("", "All model usage figures are estimates. Candidate comparisons are meaningful only when task, suite, settings and environment are held constant.");
147
+ return lines.join("\n");
148
+ }
149
+ export async function listBenchmarkMatrices(projectPath) {
150
+ const directory = path.join(path.resolve(projectPath), ".forge", "benchmark-matrices");
151
+ let files;
152
+ try {
153
+ files = (await readdir(directory)).filter(file => file.endsWith(".json")).sort();
154
+ }
155
+ catch (error) {
156
+ if (error && typeof error === "object" && "code" in error && error.code === "ENOENT")
157
+ return [];
158
+ throw error;
159
+ }
160
+ const reports = [];
161
+ for (const file of files.slice(-100)) {
162
+ const value = JSON.parse(await readFile(path.join(directory, file), "utf8"));
163
+ if (!value || typeof value !== "object" || value.version !== 1 || !Array.isArray(value.results) || !Array.isArray(value.summary))
164
+ throw new Error("Stored benchmark matrix report is invalid: " + file);
165
+ reports.push(value);
166
+ }
167
+ return reports.sort((a, b) => a.createdAt.localeCompare(b.createdAt));
168
+ }
169
+ //# sourceMappingURL=benchmark-matrix.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"benchmark-matrix.js","sourceRoot":"","sources":["../../src/core/benchmark-matrix.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACnF,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,IAAI,MAAM,WAAW,CAAC;AA4D7B,MAAM,OAAO,GAAG,2BAA2B,CAAC;AAE5C,MAAM,UAAU,uBAAuB,CAAC,KAAc;IACpD,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACnI,MAAM,MAAM,GAAG,KAA8B,CAAC;IAC9C,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG;QAAE,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IACnL,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;IAC5J,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IAC7K,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;IAC9B,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QAC1C,IAAI,CAAC,SAAS,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;QAC9I,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACtB,IAAI,CAAC,CAAC,QAAQ,EAAE,aAAa,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;QAC3J,IAAI,OAAO,SAAS,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,SAAS,CAAC,KAAK,CAAC,MAAM,GAAG,GAAG;YAAE,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;QACrK,IAAI,SAAS,CAAC,eAAe,KAAK,SAAS,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,4EAA4E,CAAC,CAAC;IACtM,CAAC;IACD,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QAC1C,IAAI,SAAS,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,KAAK,QAAQ,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,IAAI,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IAC1T,CAAC;IACD,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;QAC3B,MAAM,UAAU,GAAG,MAAM,CAAC,eAAe,CAAC;QAC1C,MAAM,QAAQ,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,KAAK,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5F,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,KAAK,UAAU,CAAC,oBAAoB,CAAC,CAAC;QAC9F,IAAI,CAAC,QAAQ,IAAI,CAAC,SAAS,IAAI,QAAQ,CAAC,EAAE,KAAK,SAAS,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,6EAA6E,CAAC,CAAC;QAC5J,IAAI,QAAQ,CAAC,QAAQ,KAAK,SAAS,CAAC,QAAQ,IAAI,QAAQ,CAAC,KAAK,KAAK,SAAS,CAAC,KAAK;YAAE,MAAM,IAAI,KAAK,CAAC,mEAAmE,CAAC,CAAC;QACzK,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,MAAM,IAAI,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,KAAK,QAAQ,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,IAAI,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,QAAQ,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;QACxT,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,sGAAsG,CAAC,CAAC;IACnU,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,KAA8C;IACnF,MAAM,SAAS,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;SACrE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,GAAG,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC1D,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACtE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,IAAY;IACpD,OAAO,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;AAC3E,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,MAA6B,EAC7B,OAA4J;IAE5J,MAAM,GAAG,uBAAuB,CAAC,MAAM,CAAC,CAAC;IACzC,MAAM,OAAO,GAA4B,EAAE,CAAC;IAC5C,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QAC1C,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,EAAE,CAAC;YAC3D,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;gBAChD,OAAO,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,SAAS,CAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,CAAC,QAAQ,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;YAC5I,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,SAAS,CAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,CAAC,QAAQ,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACpQ,CAAC;QACH,CAAC;IACH,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE;QAChD,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,KAAK,SAAS,CAAC,EAAE,IAAI,IAAI,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC;QACvG,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC7C,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAClG,OAAO;YACL,WAAW,EAAE,SAAS,CAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,CAAC,QAAQ,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO;YACvG,SAAS,EAAE,KAAK,CAAC,MAAM;YACvB,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,GAAG,EAAE;YAC3C,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;YAC5G,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;YACrJ,iBAAiB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YACtH,sBAAsB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;SACjI,CAAC;IACJ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,iBAAiB,GAAG,CAAC,CAAC,iBAAiB,CAAC,CAAC;IAChG,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACjI,MAAM,YAAY,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;QAC5C,MAAM,SAAS,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACpC,MAAM,WAAW,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QAC5C,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC;QACvC,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,KAAK,WAAW,CAAE,CAAC;QAC3E,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,KAAK,WAAW,IAAI,IAAI,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC;QACxL,MAAM,cAAc,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAClF,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,SAAS,CAAC,QAAQ,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC;IACpY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IACrF,IAAI,eAAyD,CAAC;IAC9D,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;QAC3B,MAAM,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,QAAQ,EAAE,GAAG,MAAM,CAAC,eAAe,CAAC;QACvF,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,KAAK,mBAAmB,CAAC,CAAC;QAChF,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,KAAK,oBAAoB,CAAC,CAAC;QAClF,IAAI,QAAQ,IAAI,SAAS,IAAI,QAAQ,CAAC,SAAS,GAAG,CAAC,IAAI,SAAS,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;YAC/E,MAAM,MAAM,GAAG,CAAC,MAAc,EAAE,KAAa,EAAE,EAAE,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,GAAG,MAAM,CAAC,GAAG,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC1I,MAAM,WAAW,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,KAAK,mBAAmB,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE;gBAC3G,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,KAAK,oBAAoB,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC;gBAClH,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,aAAa,EAAE,MAAM,CAAC,YAAY,EAAE,cAAc,EAAE,KAAK,CAAC,YAAY,EAAE,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,EAAE,gBAAgB,EAAE,MAAM,CAAC,QAAQ,EAAE,iBAAiB,EAAE,KAAK,CAAC,QAAQ,EAAE,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACrV,CAAC,CAAC,CAAC;YACH,eAAe,GAAG,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,QAAQ,EAAE,CAAC,GAAG,QAAQ,CAAC,EAAE,aAAa,EAAE,QAAQ,CAAC,YAAY,EAAE,cAAc,EAAE,SAAS,CAAC,YAAY,EAAE,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,YAAY,GAAG,QAAQ,CAAC,YAAY,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,EAAE,gBAAgB,EAAE,QAAQ,CAAC,QAAQ,EAAE,iBAAiB,EAAE,SAAS,CAAC,QAAQ,EAAE,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,EAAE,kBAAkB,EAAE,QAAQ,CAAC,iBAAiB,EAAE,mBAAmB,EAAE,SAAS,CAAC,iBAAiB,EAAE,qBAAqB,EAAE,MAAM,CAAC,QAAQ,CAAC,iBAAiB,EAAE,SAAS,CAAC,iBAAiB,CAAC,EAAE,uBAAuB,EAAE,QAAQ,CAAC,sBAAsB,EAAE,wBAAwB,EAAE,SAAS,CAAC,sBAAsB,EAAE,kBAAkB,EAAE,MAAM,CAAC,QAAQ,CAAC,sBAAsB,EAAE,SAAS,CAAC,sBAAsB,CAAC,EAAE,WAAW,EAAE,CAAC;QACpzB,CAAC;IACH,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,EAAE,UAAU,EAAE,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,EAAE,WAAW,EAAE,SAAS,CAAC,EAAE,EAAE,eAAe,EAAE,SAAS,CAAC,eAAgB,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;AACze,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,WAAmB,EAAE,MAA6B;IAC1F,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,oBAAoB,CAAC,CAAC;IACvF,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACzD,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,QAAQ,GAAG,GAAG,GAAG,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC;IACnH,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,GAAG,UAAU,EAAE,GAAG,MAAM,CAAC,CAAC;IACpE,IAAI,CAAC;QACH,MAAM,SAAS,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAClH,MAAM,MAAM,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;IACvC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAAC,MAAM,EAAE,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAAC,MAAM,KAAK,CAAC;IAAC,CAAC;IAC7F,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,MAA6B;IACjE,MAAM,KAAK,GAAG,CAAC,qCAAqC,GAAG,MAAM,CAAC,UAAU,EAAE,SAAS,GAAG,MAAM,CAAC,OAAO,EAAE,QAAQ,GAAG,MAAM,CAAC,IAAI,EAAE,EAAE,EAAE,oGAAoG,CAAC,CAAC;IACxO,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,OAAO;QAAE,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,GAAG,IAAI,CAAC,QAAQ,GAAG,GAAG,GAAG,IAAI,CAAC,KAAK,GAAG,GAAG,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,YAAY,GAAG,MAAM,EAAE,IAAI,CAAC,QAAQ,GAAG,GAAG,EAAE,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,iBAAiB,GAAG,KAAK,EAAE,IAAI,CAAC,sBAAsB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAC3R,IAAI,MAAM,CAAC,YAAY,EAAE,MAAM,EAAE,CAAC;QAAC,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,uBAAuB,EAAE,yEAAyE,CAAC,CAAC;QAAC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,YAAY;YAAE,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,YAAY,GAAG,MAAM,EAAE,IAAI,CAAC,QAAQ,GAAG,GAAG,EAAE,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAAC,CAAC;IACnW,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,MAAM,CAAC,eAAe,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,8BAA8B,EAAE,cAAc,GAAG,IAAI,CAAC,mBAAmB,GAAG,cAAc,EAAE,eAAe,GAAG,IAAI,CAAC,oBAAoB,GAAG,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,EAAE,WAAW,GAAG,IAAI,CAAC,aAAa,GAAG,KAAK,GAAG,IAAI,CAAC,cAAc,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,GAAG,GAAG,EAAE,eAAe,GAAG,IAAI,CAAC,gBAAgB,GAAG,MAAM,GAAG,IAAI,CAAC,iBAAiB,GAAG,KAAK,GAAG,CAAC,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,aAAa,GAAG,MAAM,EAAE,qBAAqB,GAAG,IAAI,CAAC,qBAAqB,GAAG,8BAA8B,GAAG,IAAI,CAAC,kBAAkB,GAAG,GAAG,CAAC,CAAC;QACtlB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,WAAW;YAAE,KAAK,CAAC,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC,KAAK,GAAG,IAAI,GAAG,KAAK,CAAC,aAAa,GAAG,KAAK,GAAG,KAAK,CAAC,cAAc,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC;IAC7M,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IACjC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,OAAO;QAAE,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,GAAG,IAAI,GAAG,IAAI,CAAC,MAAM,GAAG,KAAK,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,SAAS,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAC5N,IAAI,MAAM,CAAC,mBAAmB,EAAE,MAAM,EAAE,CAAC;QAAC,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,2BAA2B,CAAC,CAAC;QAAC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,mBAAmB;YAAE,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,GAAG,KAAK,GAAG,IAAI,CAAC,eAAe,GAAG,YAAY,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC;IAAC,CAAC;IAC7O,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,gJAAgJ,CAAC,CAAC;IACjK,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,WAAmB;IAC7D,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,oBAAoB,CAAC,CAAC;IACvF,IAAI,KAAe,CAAC;IACpB,IAAI,CAAC;QAAC,KAAK,GAAG,CAAC,MAAM,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAAC,CAAC;IACzF,OAAO,KAAK,EAAE,CAAC;QAAC,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,EAAE,CAAC;QAAC,MAAM,KAAK,CAAC;IAAC,CAAC;IAC/H,MAAM,OAAO,GAA4B,EAAE,CAAC;IAC5C,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;QACrC,MAAM,KAAK,GAAY,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;QACtF,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAK,KAA+B,CAAC,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAE,KAA+B,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAE,KAA+B,CAAC,OAAO,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,6CAA6C,GAAG,IAAI,CAAC,CAAC;QACzR,OAAO,CAAC,IAAI,CAAC,KAA8B,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;AACxE,CAAC"}
@@ -0,0 +1,54 @@
1
+ import { type BenchmarkMatrixReport } from "./benchmark-matrix.js";
2
+ export type RegressionStatus = "release-ready" | "warning" | "blocked" | "insufficient-history";
3
+ export interface CandidateRegression {
4
+ candidateId: string;
5
+ provider: string;
6
+ model: string;
7
+ baselineScore: number;
8
+ currentScore: number;
9
+ scoreDelta: number;
10
+ baselinePassRate: number;
11
+ currentPassRate: number;
12
+ passRateDelta: number;
13
+ baselineDurationMs: number;
14
+ currentDurationMs: number;
15
+ durationChangePercent: number;
16
+ baselineEstimatedTokens: number;
17
+ currentEstimatedTokens: number;
18
+ tokenChangePercent: number;
19
+ scoreStdDev: number;
20
+ failedTrials: number;
21
+ regressions: string[];
22
+ warnings: string[];
23
+ }
24
+ export interface AgentRegression {
25
+ candidateId: string;
26
+ agent: string;
27
+ baselineScore: number;
28
+ currentScore: number;
29
+ scoreDelta: number;
30
+ baselinePassRate: number;
31
+ currentPassRate: number;
32
+ regressions: string[];
33
+ warnings: string[];
34
+ skillsObserved: string[];
35
+ }
36
+ export interface BenchmarkRegressionReport {
37
+ status: RegressionStatus;
38
+ matrixId: string;
39
+ matrixName: string;
40
+ suiteId: string;
41
+ baselineAt: string;
42
+ currentAt: string;
43
+ candidates: CandidateRegression[];
44
+ candidateProvenance?: Array<{
45
+ candidateId: string;
46
+ configVersionId: string;
47
+ fingerprint: string;
48
+ }>;
49
+ agents: AgentRegression[];
50
+ summary: string[];
51
+ }
52
+ export declare function analyzeBenchmarkRegression(history: BenchmarkMatrixReport[], matrixId?: string): BenchmarkRegressionReport;
53
+ export declare function loadBenchmarkRegression(projectPath: string, matrixId?: string): Promise<BenchmarkRegressionReport>;
54
+ export declare function formatBenchmarkRegression(report: BenchmarkRegressionReport): string;
@@ -0,0 +1,96 @@
1
+ import { listBenchmarkMatrices } from "./benchmark-matrix.js";
2
+ const percentChange = (baseline, current) => baseline <= 0 ? (current > 0 ? 100 : 0) : Math.round((current - baseline) / baseline * 1000) / 10;
3
+ export function analyzeBenchmarkRegression(history, matrixId) {
4
+ const eligible = history.filter(item => (!matrixId || item.matrixId === matrixId) && item.summary.length > 0)
5
+ .sort((a, b) => a.createdAt.localeCompare(b.createdAt));
6
+ if (eligible.length < 2) {
7
+ const latest = eligible.at(-1);
8
+ return { status: "insufficient-history", matrixId: latest?.matrixId ?? matrixId ?? "unknown", matrixName: latest?.matrixName ?? "Benchmark history", suiteId: latest?.suiteId ?? "", baselineAt: "", currentAt: latest?.createdAt ?? "", candidates: [], agents: [], summary: ["A second comparable matrix run is required before regression analysis is available."] };
9
+ }
10
+ const current = eligible[eligible.length - 1];
11
+ const previous = [...eligible.slice(0, -1)].reverse().find(item => item.suiteId === current.suiteId && item.goal === current.goal);
12
+ if (!previous)
13
+ return { status: "insufficient-history", matrixId: current.matrixId, matrixName: current.matrixName, suiteId: current.suiteId, baselineAt: "", currentAt: current.createdAt, candidates: [], agents: [], summary: ["No earlier run has the same evaluation suite and goal; create a comparable baseline first."] };
14
+ const candidates = current.summary.flatMap(now => {
15
+ const before = previous.summary.find(item => item.candidateId === now.candidateId && item.provider === now.provider && item.model === now.model);
16
+ if (!before)
17
+ return [];
18
+ const regressions = [];
19
+ const warnings = [];
20
+ const scoreDelta = Math.round((now.averageScore - before.averageScore) * 10) / 10;
21
+ const passRateDelta = Math.round((now.passRate - before.passRate) * 10) / 10;
22
+ const durationChangePercent = percentChange(before.averageDurationMs, now.averageDurationMs);
23
+ const tokenChangePercent = percentChange(before.averageEstimatedTokens, now.averageEstimatedTokens);
24
+ if (scoreDelta <= -5)
25
+ regressions.push("Average score dropped " + Math.abs(scoreDelta) + " points.");
26
+ if (passRateDelta <= -20)
27
+ regressions.push("Pass rate dropped " + Math.abs(passRateDelta) + " percentage points.");
28
+ if (durationChangePercent >= 25)
29
+ warnings.push("Average execution time increased " + durationChangePercent + "%.");
30
+ if (tokenChangePercent >= 25)
31
+ warnings.push("Estimated token use increased " + tokenChangePercent + "%.");
32
+ if (now.scoreStdDev >= 15)
33
+ warnings.push("Score variance is high (σ " + now.scoreStdDev + ").");
34
+ const failedTrials = current.results.filter(item => item.candidateId === now.candidateId && item.status === "failed").length;
35
+ if (failedTrials > 0)
36
+ warnings.push(failedTrials + " trial(s) failed to complete.");
37
+ return [{ candidateId: now.candidateId, provider: now.provider, model: now.model, baselineScore: before.averageScore, currentScore: now.averageScore, scoreDelta, baselinePassRate: before.passRate, currentPassRate: now.passRate, passRateDelta, baselineDurationMs: before.averageDurationMs, currentDurationMs: now.averageDurationMs, durationChangePercent, baselineEstimatedTokens: before.averageEstimatedTokens, currentEstimatedTokens: now.averageEstimatedTokens, tokenChangePercent, scoreStdDev: now.scoreStdDev, failedTrials, regressions, warnings }];
38
+ });
39
+ const agents = (current.agentSummary ?? []).flatMap(now => {
40
+ const before = (previous.agentSummary ?? []).find(item => item.candidateId === now.candidateId && item.agent === now.agent && item.provider === now.provider && item.model === now.model);
41
+ if (!before)
42
+ return [];
43
+ const regressions = [];
44
+ const warnings = [];
45
+ const scoreDelta = Math.round((now.averageScore - before.averageScore) * 10) / 10;
46
+ const passDelta = Math.round((now.passRate - before.passRate) * 10) / 10;
47
+ if (scoreDelta <= -5)
48
+ regressions.push("Agent score dropped " + Math.abs(scoreDelta) + " points.");
49
+ if (passDelta <= -20)
50
+ regressions.push("Agent pass rate dropped " + Math.abs(passDelta) + " percentage points.");
51
+ if (now.completed < now.trials)
52
+ warnings.push("Only " + now.completed + " of " + now.trials + " expected agent evaluations completed.");
53
+ return [{ candidateId: now.candidateId, agent: now.agent, baselineScore: before.averageScore, currentScore: now.averageScore, scoreDelta, baselinePassRate: before.passRate, currentPassRate: now.passRate, regressions, warnings, skillsObserved: now.skillsObserved ?? [] }];
54
+ });
55
+ const agentRegressions = agents.reduce((sum, item) => sum + item.regressions.length, 0);
56
+ const agentWarnings = agents.reduce((sum, item) => sum + item.warnings.length, 0);
57
+ const matched = candidates.length;
58
+ const regressions = candidates.reduce((sum, item) => sum + item.regressions.length, 0) + agentRegressions;
59
+ const warnings = candidates.reduce((sum, item) => sum + item.warnings.length, 0) + agentWarnings;
60
+ const unmatched = current.summary.length - matched;
61
+ const status = matched === 0 || unmatched > 0 ? "insufficient-history" : regressions > 0 ? "blocked" : warnings > 0 ? "warning" : "release-ready";
62
+ const summary = [
63
+ matched + " candidate(s) compared against the previous run using the same suite and goal.",
64
+ regressions + " quality regression(s), " + warnings + " efficiency or stability warning(s).",
65
+ ...(unmatched ? [unmatched + " candidate(s) had no matching baseline and were excluded."] : []),
66
+ "Thresholds: score drop ≥5 points or pass-rate drop ≥20 points blocks; time/token growth ≥25% or score σ ≥15 warns."
67
+ ];
68
+ return { status, matrixId: current.matrixId, matrixName: current.matrixName, suiteId: current.suiteId, baselineAt: previous.createdAt, currentAt: current.createdAt, candidates, ...(current.candidateProvenance ? { candidateProvenance: current.candidateProvenance } : {}), agents, summary: [...summary, agents.length + " per-agent scorecard(s) compared across the same candidate and model."] };
69
+ }
70
+ export async function loadBenchmarkRegression(projectPath, matrixId) {
71
+ return analyzeBenchmarkRegression(await listBenchmarkMatrices(projectPath), matrixId);
72
+ }
73
+ export function formatBenchmarkRegression(report) {
74
+ const lines = ["FORGE Benchmark Regression Intelligence · " + report.matrixName, "Status: " + report.status.toUpperCase(), "Suite: " + report.suiteId, ""];
75
+ lines.push(...report.summary);
76
+ for (const item of report.candidates) {
77
+ lines.push("", item.candidateId + " · " + item.provider + "/" + item.model, " Score: " + item.baselineScore + " → " + item.currentScore + " (" + (item.scoreDelta > 0 ? "+" : "") + item.scoreDelta + ")", " Pass rate: " + item.baselinePassRate + "% → " + item.currentPassRate + "%", " Duration: " + item.durationChangePercent + "% · estimated tokens: " + item.tokenChangePercent + "%");
78
+ for (const issue of item.regressions)
79
+ lines.push(" REGRESSION: " + issue);
80
+ for (const issue of item.warnings)
81
+ lines.push(" WARNING: " + issue);
82
+ }
83
+ if (report.agents.length) {
84
+ lines.push("", "Per-agent regressions:");
85
+ for (const item of report.agents) {
86
+ lines.push(" " + item.candidateId + " / " + item.agent + " · " + item.baselineScore + " → " + item.currentScore + " (" + (item.scoreDelta > 0 ? "+" : "") + item.scoreDelta + ") · pass " + item.baselinePassRate + "% → " + item.currentPassRate + "% · observed skills: " + (item.skillsObserved.join(", ") || "none"));
87
+ for (const issue of item.regressions)
88
+ lines.push(" REGRESSION: " + issue);
89
+ for (const issue of item.warnings)
90
+ lines.push(" WARNING: " + issue);
91
+ }
92
+ }
93
+ lines.push("", "This is a trend signal, not a guarantee. Compare identical code snapshots, prompts, suite versions and runtime conditions.");
94
+ return lines.join("\n");
95
+ }
96
+ //# sourceMappingURL=benchmark-regression.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"benchmark-regression.js","sourceRoot":"","sources":["../../src/core/benchmark-regression.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAA8B,MAAM,uBAAuB,CAAC;AAqC1F,MAAM,aAAa,GAAG,CAAC,QAAgB,EAAE,OAAe,EAAU,EAAE,CAClE,QAAQ,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,GAAG,QAAQ,CAAC,GAAG,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;AAEpG,MAAM,UAAU,0BAA0B,CAAC,OAAgC,EAAE,QAAiB;IAC5F,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,KAAK,QAAQ,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;SAC1G,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;IAC1D,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/B,OAAO,EAAE,MAAM,EAAE,sBAAsB,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,IAAI,QAAQ,IAAI,SAAS,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,IAAI,mBAAmB,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,IAAI,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,IAAI,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,qFAAqF,CAAC,EAAE,CAAC;IAC1W,CAAC;IACD,MAAM,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC;IAC/C,MAAM,QAAQ,GAAG,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,KAAK,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACnI,IAAI,CAAC,QAAQ;QAAE,OAAO,EAAE,MAAM,EAAE,sBAAsB,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,UAAU,EAAE,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,UAAU,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,4FAA4F,CAAC,EAAE,CAAC;IAClU,MAAM,UAAU,GAA0B,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;QACtE,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,KAAK,GAAG,CAAC,WAAW,IAAI,IAAI,CAAC,QAAQ,KAAK,GAAG,CAAC,QAAQ,IAAI,IAAI,CAAC,KAAK,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC;QACjJ,IAAI,CAAC,MAAM;YAAE,OAAO,EAAE,CAAC;QACvB,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC;QAClF,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC;QAC7E,MAAM,qBAAqB,GAAG,aAAa,CAAC,MAAM,CAAC,iBAAiB,EAAE,GAAG,CAAC,iBAAiB,CAAC,CAAC;QAC7F,MAAM,kBAAkB,GAAG,aAAa,CAAC,MAAM,CAAC,sBAAsB,EAAE,GAAG,CAAC,sBAAsB,CAAC,CAAC;QACpG,IAAI,UAAU,IAAI,CAAC,CAAC;YAAE,WAAW,CAAC,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,UAAU,CAAC,CAAC;QACrG,IAAI,aAAa,IAAI,CAAC,EAAE;YAAE,WAAW,CAAC,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,GAAG,CAAC,aAAa,CAAC,GAAG,qBAAqB,CAAC,CAAC;QACnH,IAAI,qBAAqB,IAAI,EAAE;YAAE,QAAQ,CAAC,IAAI,CAAC,mCAAmC,GAAG,qBAAqB,GAAG,IAAI,CAAC,CAAC;QACnH,IAAI,kBAAkB,IAAI,EAAE;YAAE,QAAQ,CAAC,IAAI,CAAC,gCAAgC,GAAG,kBAAkB,GAAG,IAAI,CAAC,CAAC;QAC1G,IAAI,GAAG,CAAC,WAAW,IAAI,EAAE;YAAE,QAAQ,CAAC,IAAI,CAAC,4BAA4B,GAAG,GAAG,CAAC,WAAW,GAAG,IAAI,CAAC,CAAC;QAChG,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,KAAK,GAAG,CAAC,WAAW,IAAI,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC;QAC7H,IAAI,YAAY,GAAG,CAAC;YAAE,QAAQ,CAAC,IAAI,CAAC,YAAY,GAAG,+BAA+B,CAAC,CAAC;QACpF,OAAO,CAAC,EAAE,WAAW,EAAE,GAAG,CAAC,WAAW,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,aAAa,EAAE,MAAM,CAAC,YAAY,EAAE,YAAY,EAAE,GAAG,CAAC,YAAY,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,CAAC,QAAQ,EAAE,eAAe,EAAE,GAAG,CAAC,QAAQ,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,CAAC,iBAAiB,EAAE,iBAAiB,EAAE,GAAG,CAAC,iBAAiB,EAAE,qBAAqB,EAAE,uBAAuB,EAAE,MAAM,CAAC,sBAAsB,EAAE,sBAAsB,EAAE,GAAG,CAAC,sBAAsB,EAAE,kBAAkB,EAAE,WAAW,EAAE,GAAG,CAAC,WAAW,EAAE,YAAY,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,CAAC;IACziB,CAAC,CAAC,CAAC;IACH,MAAM,MAAM,GAAsB,CAAC,OAAO,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;QAC3E,MAAM,MAAM,GAAG,CAAC,QAAQ,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,KAAK,GAAG,CAAC,WAAW,IAAI,IAAI,CAAC,KAAK,KAAK,GAAG,CAAC,KAAK,IAAI,IAAI,CAAC,QAAQ,KAAK,GAAG,CAAC,QAAQ,IAAI,IAAI,CAAC,KAAK,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC;QAC1L,IAAI,CAAC,MAAM;YAAE,OAAO,EAAE,CAAC;QACvB,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC;QAClF,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC;QACzE,IAAI,UAAU,IAAI,CAAC,CAAC;YAAE,WAAW,CAAC,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,UAAU,CAAC,CAAC;QACnG,IAAI,SAAS,IAAI,CAAC,EAAE;YAAE,WAAW,CAAC,IAAI,CAAC,0BAA0B,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,qBAAqB,CAAC,CAAC;QACjH,IAAI,GAAG,CAAC,SAAS,GAAG,GAAG,CAAC,MAAM;YAAE,QAAQ,CAAC,IAAI,CAAC,OAAO,GAAG,GAAG,CAAC,SAAS,GAAG,MAAM,GAAG,GAAG,CAAC,MAAM,GAAG,wCAAwC,CAAC,CAAC;QACxI,OAAO,CAAC,EAAE,WAAW,EAAE,GAAG,CAAC,WAAW,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,aAAa,EAAE,MAAM,CAAC,YAAY,EAAE,YAAY,EAAE,GAAG,CAAC,YAAY,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,CAAC,QAAQ,EAAE,eAAe,EAAE,GAAG,CAAC,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,cAAc,EAAE,GAAG,CAAC,cAAc,IAAI,EAAE,EAAE,CAAC,CAAC;IACjR,CAAC,CAAC,CAAC;IACH,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACxF,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAClF,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC;IAClC,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,gBAAgB,CAAC;IAC1G,MAAM,QAAQ,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,aAAa,CAAC;IACjG,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC;IACnD,MAAM,MAAM,GAAqB,OAAO,KAAK,CAAC,IAAI,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,eAAe,CAAC;IACpK,MAAM,OAAO,GAAG;QACd,OAAO,GAAG,gFAAgF;QAC1F,WAAW,GAAG,0BAA0B,GAAG,QAAQ,GAAG,sCAAsC;QAC5F,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,2DAA2D,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/F,oHAAoH;KACrH,CAAC;IACF,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,UAAU,EAAE,QAAQ,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,UAAU,EAAE,GAAG,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,GAAG,OAAO,EAAE,MAAM,CAAC,MAAM,GAAG,uEAAuE,CAAC,EAAE,CAAC;AAC1Y,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,WAAmB,EAAE,QAAiB;IAClF,OAAO,0BAA0B,CAAC,MAAM,qBAAqB,CAAC,WAAW,CAAC,EAAE,QAAQ,CAAC,CAAC;AACxF,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,MAAiC;IACzE,MAAM,KAAK,GAAG,CAAC,4CAA4C,GAAG,MAAM,CAAC,UAAU,EAAE,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,SAAS,GAAG,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IAC3J,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;IAC9B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QACrC,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,WAAW,GAAG,KAAK,GAAG,IAAI,CAAC,QAAQ,GAAG,GAAG,GAAG,IAAI,CAAC,KAAK,EAAE,WAAW,GAAG,IAAI,CAAC,aAAa,GAAG,KAAK,GAAG,IAAI,CAAC,YAAY,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,GAAG,GAAG,EAAE,eAAe,GAAG,IAAI,CAAC,gBAAgB,GAAG,MAAM,GAAG,IAAI,CAAC,eAAe,GAAG,GAAG,EAAE,cAAc,GAAG,IAAI,CAAC,qBAAqB,GAAG,wBAAwB,GAAG,IAAI,CAAC,kBAAkB,GAAG,GAAG,CAAC,CAAC;QACnY,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,WAAW;YAAE,KAAK,CAAC,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC,CAAC;QAC3E,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ;YAAE,KAAK,CAAC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC,CAAC;IACvE,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAAC,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,wBAAwB,CAAC,CAAC;QAAC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAAC,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,GAAG,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,KAAK,GAAG,IAAI,CAAC,aAAa,GAAG,KAAK,GAAG,IAAI,CAAC,YAAY,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,GAAG,WAAW,GAAG,IAAI,CAAC,gBAAgB,GAAG,MAAM,GAAG,IAAI,CAAC,eAAe,GAAG,uBAAuB,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC;YAAC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,WAAW;gBAAE,KAAK,CAAC,IAAI,CAAC,kBAAkB,GAAG,KAAK,CAAC,CAAC;YAAC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ;gBAAE,KAAK,CAAC,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC,CAAC;QAAC,CAAC;IAAC,CAAC;IAC/jB,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,4HAA4H,CAAC,CAAC;IAC7I,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}
@@ -0,0 +1,40 @@
1
+ export type ChangeRiskLevel = "high" | "elevated" | "moderate" | "low";
2
+ export interface ChangeRiskItem {
3
+ file: string;
4
+ status: string;
5
+ score: number;
6
+ level: ChangeRiskLevel;
7
+ structuralScore: number;
8
+ downstreamCount: number;
9
+ affectedModules: string[];
10
+ reasons: string[];
11
+ recommendedTests: Array<{
12
+ file: string;
13
+ reason: string;
14
+ exists: boolean;
15
+ }>;
16
+ }
17
+ export interface ChangeIntelligenceReport {
18
+ root: string;
19
+ generatedAt: string;
20
+ branch: string;
21
+ base: string;
22
+ summary: {
23
+ changedFiles: number;
24
+ high: number;
25
+ elevated: number;
26
+ moderate: number;
27
+ low: number;
28
+ affectedModules: number;
29
+ testCandidates: number;
30
+ };
31
+ overallRisk: {
32
+ score: number;
33
+ level: ChangeRiskLevel;
34
+ reasons: string[];
35
+ };
36
+ changes: ChangeRiskItem[];
37
+ warnings: string[];
38
+ }
39
+ export declare function analyzeChangeIntelligence(inputPath: string, base?: string): Promise<ChangeIntelligenceReport>;
40
+ export declare function formatChangeIntelligence(report: ChangeIntelligenceReport): string;