anvil-engineering-workbench 0.0.0-stage → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +40 -0
- package/LICENSE +17 -0
- package/README.md +778 -3
- package/SECURITY.md +17 -0
- package/dist/ai/claude-code-provider.d.ts +31 -0
- package/dist/ai/claude-code-provider.js +81 -0
- package/dist/ai/claude-code-provider.js.map +1 -0
- package/dist/ai/provider.d.ts +47 -0
- package/dist/ai/provider.js +57 -0
- package/dist/ai/provider.js.map +1 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +1945 -0
- package/dist/cli.js.map +1 -0
- package/dist/core/agent-config-release.d.ts +22 -0
- package/dist/core/agent-config-release.js +109 -0
- package/dist/core/agent-config-release.js.map +1 -0
- package/dist/core/agent-config-versioning.d.ts +33 -0
- package/dist/core/agent-config-versioning.js +209 -0
- package/dist/core/agent-config-versioning.js.map +1 -0
- package/dist/core/agent-evaluation-intelligence.d.ts +46 -0
- package/dist/core/agent-evaluation-intelligence.js +191 -0
- package/dist/core/agent-evaluation-intelligence.js.map +1 -0
- package/dist/core/agent-evolution.d.ts +20 -0
- package/dist/core/agent-evolution.js +35 -0
- package/dist/core/agent-evolution.js.map +1 -0
- package/dist/core/agent-experiment-studio.d.ts +27 -0
- package/dist/core/agent-experiment-studio.js +65 -0
- package/dist/core/agent-experiment-studio.js.map +1 -0
- package/dist/core/agent-optimization.d.ts +27 -0
- package/dist/core/agent-optimization.js +63 -0
- package/dist/core/agent-optimization.js.map +1 -0
- package/dist/core/agent-registry.d.ts +9 -0
- package/dist/core/agent-registry.js +95 -0
- package/dist/core/agent-registry.js.map +1 -0
- package/dist/core/agent-security-audit.d.ts +28 -0
- package/dist/core/agent-security-audit.js +260 -0
- package/dist/core/agent-security-audit.js.map +1 -0
- package/dist/core/agent-workspace-context.d.ts +14 -0
- package/dist/core/agent-workspace-context.js +66 -0
- package/dist/core/agent-workspace-context.js.map +1 -0
- package/dist/core/ai-code-review.d.ts +20 -0
- package/dist/core/ai-code-review.js +70 -0
- package/dist/core/ai-code-review.js.map +1 -0
- package/dist/core/anthropic-provider.d.ts +16 -0
- package/dist/core/anthropic-provider.js +85 -0
- package/dist/core/anthropic-provider.js.map +1 -0
- package/dist/core/architecture-map.d.ts +26 -0
- package/dist/core/architecture-map.js +124 -0
- package/dist/core/architecture-map.js.map +1 -0
- package/dist/core/benchmark-engine.d.ts +52 -0
- package/dist/core/benchmark-engine.js +109 -0
- package/dist/core/benchmark-engine.js.map +1 -0
- package/dist/core/benchmark-matrix.d.ts +123 -0
- package/dist/core/benchmark-matrix.js +169 -0
- package/dist/core/benchmark-matrix.js.map +1 -0
- package/dist/core/benchmark-regression.d.ts +54 -0
- package/dist/core/benchmark-regression.js +96 -0
- package/dist/core/benchmark-regression.js.map +1 -0
- package/dist/core/change-intelligence.d.ts +40 -0
- package/dist/core/change-intelligence.js +114 -0
- package/dist/core/change-intelligence.js.map +1 -0
- package/dist/core/code-intelligence.d.ts +20 -0
- package/dist/core/code-intelligence.js +119 -0
- package/dist/core/code-intelligence.js.map +1 -0
- package/dist/core/code-review.d.ts +30 -0
- package/dist/core/code-review.js +93 -0
- package/dist/core/code-review.js.map +1 -0
- package/dist/core/code-search.d.ts +16 -0
- package/dist/core/code-search.js +93 -0
- package/dist/core/code-search.js.map +1 -0
- package/dist/core/container-sandbox.d.ts +20 -0
- package/dist/core/container-sandbox.js +220 -0
- package/dist/core/container-sandbox.js.map +1 -0
- package/dist/core/dashboard-server.d.ts +11 -0
- package/dist/core/dashboard-server.js +1068 -0
- package/dist/core/dashboard-server.js.map +1 -0
- package/dist/core/dependency-graph.d.ts +29 -0
- package/dist/core/dependency-graph.js +90 -0
- package/dist/core/dependency-graph.js.map +1 -0
- package/dist/core/dependency-impact.d.ts +51 -0
- package/dist/core/dependency-impact.js +201 -0
- package/dist/core/dependency-impact.js.map +1 -0
- package/dist/core/dependency-intelligence.d.ts +60 -0
- package/dist/core/dependency-intelligence.js +241 -0
- package/dist/core/dependency-intelligence.js.map +1 -0
- package/dist/core/doctor.d.ts +17 -0
- package/dist/core/doctor.js +239 -0
- package/dist/core/doctor.js.map +1 -0
- package/dist/core/engineering-risk.d.ts +40 -0
- package/dist/core/engineering-risk.js +125 -0
- package/dist/core/engineering-risk.js.map +1 -0
- package/dist/core/evaluation-engine.d.ts +110 -0
- package/dist/core/evaluation-engine.js +182 -0
- package/dist/core/evaluation-engine.js.map +1 -0
- package/dist/core/execution-trace.d.ts +27 -0
- package/dist/core/execution-trace.js +137 -0
- package/dist/core/execution-trace.js.map +1 -0
- package/dist/core/experiment-history.d.ts +31 -0
- package/dist/core/experiment-history.js +64 -0
- package/dist/core/experiment-history.js.map +1 -0
- package/dist/core/experiment-manifest.d.ts +67 -0
- package/dist/core/experiment-manifest.js +100 -0
- package/dist/core/experiment-manifest.js.map +1 -0
- package/dist/core/gemini-provider.d.ts +15 -0
- package/dist/core/gemini-provider.js +87 -0
- package/dist/core/gemini-provider.js.map +1 -0
- package/dist/core/git-intelligence.d.ts +26 -0
- package/dist/core/git-intelligence.js +84 -0
- package/dist/core/git-intelligence.js.map +1 -0
- package/dist/core/hook-engine.d.ts +23 -0
- package/dist/core/hook-engine.js +74 -0
- package/dist/core/hook-engine.js.map +1 -0
- package/dist/core/hosted-agent-provider.d.ts +19 -0
- package/dist/core/hosted-agent-provider.js +133 -0
- package/dist/core/hosted-agent-provider.js.map +1 -0
- package/dist/core/intelligence-timeline.d.ts +41 -0
- package/dist/core/intelligence-timeline.js +158 -0
- package/dist/core/intelligence-timeline.js.map +1 -0
- package/dist/core/model-router.d.ts +59 -0
- package/dist/core/model-router.js +291 -0
- package/dist/core/model-router.js.map +1 -0
- package/dist/core/openai-compatible-provider.d.ts +19 -0
- package/dist/core/openai-compatible-provider.js +151 -0
- package/dist/core/openai-compatible-provider.js.map +1 -0
- package/dist/core/orchestrator.d.ts +94 -0
- package/dist/core/orchestrator.js +263 -0
- package/dist/core/orchestrator.js.map +1 -0
- package/dist/core/patch-application.d.ts +54 -0
- package/dist/core/patch-application.js +504 -0
- package/dist/core/patch-application.js.map +1 -0
- package/dist/core/patch-approval.d.ts +23 -0
- package/dist/core/patch-approval.js +181 -0
- package/dist/core/patch-approval.js.map +1 -0
- package/dist/core/patch-validation.d.ts +12 -0
- package/dist/core/patch-validation.js +112 -0
- package/dist/core/patch-validation.js.map +1 -0
- package/dist/core/permission-policy.d.ts +13 -0
- package/dist/core/permission-policy.js +98 -0
- package/dist/core/permission-policy.js.map +1 -0
- package/dist/core/project-health.d.ts +72 -0
- package/dist/core/project-health.js +79 -0
- package/dist/core/project-health.js.map +1 -0
- package/dist/core/project-init.d.ts +9 -0
- package/dist/core/project-init.js +253 -0
- package/dist/core/project-init.js.map +1 -0
- package/dist/core/project-inspector.d.ts +24 -0
- package/dist/core/project-inspector.js +175 -0
- package/dist/core/project-inspector.js.map +1 -0
- package/dist/core/project-memory.d.ts +22 -0
- package/dist/core/project-memory.js +173 -0
- package/dist/core/project-memory.js.map +1 -0
- package/dist/core/provider-diagnostics.d.ts +12 -0
- package/dist/core/provider-diagnostics.js +73 -0
- package/dist/core/provider-diagnostics.js.map +1 -0
- package/dist/core/provider-response.d.ts +4 -0
- package/dist/core/provider-response.js +46 -0
- package/dist/core/provider-response.js.map +1 -0
- package/dist/core/quality-audit.d.ts +29 -0
- package/dist/core/quality-audit.js +101 -0
- package/dist/core/quality-audit.js.map +1 -0
- package/dist/core/regression-intelligence.d.ts +41 -0
- package/dist/core/regression-intelligence.js +56 -0
- package/dist/core/regression-intelligence.js.map +1 -0
- package/dist/core/remediation-planner.d.ts +43 -0
- package/dist/core/remediation-planner.js +124 -0
- package/dist/core/remediation-planner.js.map +1 -0
- package/dist/core/review-history.d.ts +18 -0
- package/dist/core/review-history.js +68 -0
- package/dist/core/review-history.js.map +1 -0
- package/dist/core/review-planner.d.ts +2 -0
- package/dist/core/review-planner.js +79 -0
- package/dist/core/review-planner.js.map +1 -0
- package/dist/core/run-history-output.d.ts +22 -0
- package/dist/core/run-history-output.js +51 -0
- package/dist/core/run-history-output.js.map +1 -0
- package/dist/core/run-replay.d.ts +9 -0
- package/dist/core/run-replay.js +64 -0
- package/dist/core/run-replay.js.map +1 -0
- package/dist/core/run-store.d.ts +13 -0
- package/dist/core/run-store.js +300 -0
- package/dist/core/run-store.js.map +1 -0
- package/dist/core/sandbox-cleanup-audit.d.ts +14 -0
- package/dist/core/sandbox-cleanup-audit.js +57 -0
- package/dist/core/sandbox-cleanup-audit.js.map +1 -0
- package/dist/core/sandbox-compliance.d.ts +19 -0
- package/dist/core/sandbox-compliance.js +34 -0
- package/dist/core/sandbox-compliance.js.map +1 -0
- package/dist/core/sandbox-lifecycle.d.ts +8 -0
- package/dist/core/sandbox-lifecycle.js +36 -0
- package/dist/core/sandbox-lifecycle.js.map +1 -0
- package/dist/core/sandbox-policy.d.ts +31 -0
- package/dist/core/sandbox-policy.js +80 -0
- package/dist/core/sandbox-policy.js.map +1 -0
- package/dist/core/sandbox-runtime.d.ts +10 -0
- package/dist/core/sandbox-runtime.js +44 -0
- package/dist/core/sandbox-runtime.js.map +1 -0
- package/dist/core/sandbox-security-profiles.d.ts +16 -0
- package/dist/core/sandbox-security-profiles.js +43 -0
- package/dist/core/sandbox-security-profiles.js.map +1 -0
- package/dist/core/skill-registry.d.ts +14 -0
- package/dist/core/skill-registry.js +186 -0
- package/dist/core/skill-registry.js.map +1 -0
- package/dist/core/staged-verification.d.ts +27 -0
- package/dist/core/staged-verification.js +108 -0
- package/dist/core/staged-verification.js.map +1 -0
- package/dist/core/verification-history.d.ts +10 -0
- package/dist/core/verification-history.js +71 -0
- package/dist/core/verification-history.js.map +1 -0
- package/dist/core/verification.d.ts +37 -0
- package/dist/core/verification.js +204 -0
- package/dist/core/verification.js.map +1 -0
- package/dist/core/workflow-store.d.ts +16 -0
- package/dist/core/workflow-store.js +113 -0
- package/dist/core/workflow-store.js.map +1 -0
- package/dist/core/workspace-browser.d.ts +22 -0
- package/dist/core/workspace-browser.js +115 -0
- package/dist/core/workspace-browser.js.map +1 -0
- package/dist/core/workspace-scan-policy.d.ts +2 -0
- package/dist/core/workspace-scan-policy.js +11 -0
- package/dist/core/workspace-scan-policy.js.map +1 -0
- package/dist/core/workspace-tool-broker.d.ts +28 -0
- package/dist/core/workspace-tool-broker.js +70 -0
- package/dist/core/workspace-tool-broker.js.map +1 -0
- package/dist/index.d.ts +75 -0
- package/dist/index.js +82 -0
- package/dist/index.js.map +1 -0
- package/dist/mcp/server.d.ts +2 -0
- package/dist/mcp/server.js +342 -0
- package/dist/mcp/server.js.map +1 -0
- package/dist/mcp/workspace-path.d.ts +2 -0
- package/dist/mcp/workspace-path.js +41 -0
- package/dist/mcp/workspace-path.js.map +1 -0
- package/package.json +67 -5
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"benchmark-engine.js","sourceRoot":"","sources":["../../src/core/benchmark-engine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACnF,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,IAAI,MAAM,WAAW,CAAC;AAkD7B,SAAS,SAAS,CAAC,MAAwB;IACzC,MAAM,OAAO,GAAG,IAAI,GAAG,EAA4B,CAAC;IACpD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ,GAAG,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7D,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAC3B,CAAC;IACD,OAAO,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;QACjD,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC7C,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAyE,EAAE,CAAC,IAAI,CAAC,UAAU,KAAK,SAAS,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC;QAC9K,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QAC9E,OAAO;YACL,GAAG,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAE,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM;YACjF,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,GAAG,EAAE;YAC3C,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,EAAE,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACpP,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;YAC/D,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE;YACzF,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE;YAClI,iBAAiB,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;YACnG,sBAAsB,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,qBAAqB,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;YAC/I,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;YAChE,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;SACrE,CAAC;IACJ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,iBAAiB,GAAG,CAAC,CAAC,iBAAiB,CAAC,CAAC;AAClG,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,MAAiD,EAAE,KAA6B;IACnH,IAAI,MAAM,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAC;IACxH,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAClF,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC5E,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS;QAC/G,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,OAAO,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,YAAY,EAAE,aAAa,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,SAAS,EAAE,MAAM;QAChP,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;QAC7F,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;QAC5E,oBAAoB,EAAE,KAAK,CAAC,MAAM,CAAC,oBAAoB,EAAE,qBAAqB,EAAE,KAAK,CAAC,MAAM,CAAC,qBAAqB;QAClH,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,UAAU;KACpC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,WAAmB,EAAE,KAAqB;IACnF,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAC;IAC/E,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACzD,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,GAAG,OAAO,CAAC,CAAC;IAClE,IAAI,MAAM,GAAqB,EAAE,CAAC;IAClC,IAAI,CAAC;QACH,MAAM,QAAQ,GAAY,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC;QAC1E,IAAI,CAAC,QAAQ,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAK,QAA8B,CAAC,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAE,QAA8B,CAAC,MAAM,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;QACpN,MAAM,GAAI,QAA8B,CAAC,MAAM,CAAC;IAClD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,CAAC,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC;YAAE,MAAM,KAAK,CAAC;IACvG,CAAC;IACD,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;IAChI,MAAM,QAAQ,GAAsB,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;IAC/K,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,GAAG,UAAU,EAAE,GAAG,MAAM,CAAC,CAAC;IACpE,IAAI,CAAC;QACH,MAAM,SAAS,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QACpH,MAAM,MAAM,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;IACvC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,EAAE,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC5D,MAAM,KAAK,CAAC;IACd,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,WAAmB,EAAE,OAAe;IACtE,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACvG,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,GAAG,OAAO,CAAC,CAAC;IAC7F,MAAM,KAAK,GAAY,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IAChE,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAK,KAA2B,CAAC,OAAO,KAAK,CAAC,IAAK,KAA2B,CAAC,OAAO,KAAK,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAE,KAA2B,CAAC,MAAM,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC3P,OAAO,KAA0B,CAAC;AACpC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,WAAmB;IACtD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAC;IAC/E,IAAI,KAAe,CAAC;IACpB,IAAI,CAAC;QAAC,KAAK,GAAG,CAAC,MAAM,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAAC,CAAC;IACzF,OAAO,KAAK,EAAE,CAAC;QAAC,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,EAAE,CAAC;QAAC,MAAM,KAAK,CAAC;IAAC,CAAC;IAC/H,MAAM,SAAS,GAAwB,EAAE,CAAC;IAC1C,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;QACvC,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7B,SAAS,CAAC,IAAI,CAAC,MAAM,aAAa,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC;IACvD,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,QAA2B;IACzD,MAAM,KAAK,GAAG,CAAC,wBAAwB,GAAG,QAAQ,CAAC,SAAS,EAAE,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,EAAE,mGAAmG,CAAC,CAAC;IAC5M,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM;QAAE,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,YAAY,GAAG,MAAM,EAAE,KAAK,CAAC,iBAAiB,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,iBAAiB,GAAG,QAAQ,GAAG,KAAK,CAAC,aAAa,GAAG,SAAS,EAAE,KAAK,CAAC,QAAQ,GAAG,GAAG,EAAE,IAAI,GAAG,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,iBAAiB,GAAG,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IACvV,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IACjC,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE;QAAE,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,KAAK,GAAG,KAAK,GAAG,KAAK,CAAC,QAAQ,GAAG,KAAK,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,GAAG,KAAK,GAAG,KAAK,CAAC,KAAK,GAAG,aAAa,GAAG,CAAC,KAAK,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,UAAU,GAAG,YAAY,CAAC,GAAG,KAAK,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;IACxU,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,iIAAiI,CAAC,CAAC;IAClJ,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
export interface BenchmarkCandidate {
|
|
2
|
+
id: string;
|
|
3
|
+
provider: "openai" | "claude-code" | "anthropic" | "google" | "ollama";
|
|
4
|
+
model: string;
|
|
5
|
+
/** undefined = normal auto-selection; [] = no skills; ids = force this skill set. */
|
|
6
|
+
skillIds?: string[];
|
|
7
|
+
/** Immutable workspace configuration snapshot used for this candidate. */
|
|
8
|
+
configVersionId?: string;
|
|
9
|
+
}
|
|
10
|
+
export interface BenchmarkMatrixConfig {
|
|
11
|
+
id: string;
|
|
12
|
+
name: string;
|
|
13
|
+
repeats: number;
|
|
14
|
+
candidates: BenchmarkCandidate[];
|
|
15
|
+
skillExperiment?: {
|
|
16
|
+
baselineCandidateId: string;
|
|
17
|
+
treatmentCandidateId: string;
|
|
18
|
+
skillIds: string[];
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
export interface AgentBenchmarkScore {
|
|
22
|
+
agent: string;
|
|
23
|
+
caseId: string;
|
|
24
|
+
score: number;
|
|
25
|
+
passed: boolean;
|
|
26
|
+
skillIds: string[];
|
|
27
|
+
failedCriteria?: Array<{
|
|
28
|
+
id: string;
|
|
29
|
+
failedChecks: string[];
|
|
30
|
+
}>;
|
|
31
|
+
}
|
|
32
|
+
export interface AgentBenchmarkSummary {
|
|
33
|
+
candidateId: string;
|
|
34
|
+
provider: string;
|
|
35
|
+
model: string;
|
|
36
|
+
agent: string;
|
|
37
|
+
trials: number;
|
|
38
|
+
completed: number;
|
|
39
|
+
averageScore: number;
|
|
40
|
+
passRate: number;
|
|
41
|
+
skillsObserved: string[];
|
|
42
|
+
}
|
|
43
|
+
export interface BenchmarkMatrixResult {
|
|
44
|
+
candidateId: string;
|
|
45
|
+
provider: string;
|
|
46
|
+
model: string;
|
|
47
|
+
repeat: number;
|
|
48
|
+
runId: string;
|
|
49
|
+
score: number;
|
|
50
|
+
passed: boolean;
|
|
51
|
+
durationMs: number;
|
|
52
|
+
estimatedTokens: number;
|
|
53
|
+
agentScores?: AgentBenchmarkScore[];
|
|
54
|
+
status: "completed" | "failed";
|
|
55
|
+
error?: string;
|
|
56
|
+
}
|
|
57
|
+
export interface BenchmarkMatrixReport {
|
|
58
|
+
version: 1;
|
|
59
|
+
matrixId: string;
|
|
60
|
+
matrixName: string;
|
|
61
|
+
suiteId: string;
|
|
62
|
+
goal: string;
|
|
63
|
+
createdAt: string;
|
|
64
|
+
candidateProvenance?: Array<{
|
|
65
|
+
candidateId: string;
|
|
66
|
+
configVersionId: string;
|
|
67
|
+
fingerprint: string;
|
|
68
|
+
}>;
|
|
69
|
+
experimentManifest?: {
|
|
70
|
+
id: string;
|
|
71
|
+
fingerprint: string;
|
|
72
|
+
};
|
|
73
|
+
results: BenchmarkMatrixResult[];
|
|
74
|
+
agentSummary?: AgentBenchmarkSummary[];
|
|
75
|
+
skillExperiment?: {
|
|
76
|
+
baselineCandidateId: string;
|
|
77
|
+
treatmentCandidateId: string;
|
|
78
|
+
skillIds: string[];
|
|
79
|
+
baselineScore: number;
|
|
80
|
+
treatmentScore: number;
|
|
81
|
+
scoreDelta: number;
|
|
82
|
+
baselinePassRate: number;
|
|
83
|
+
treatmentPassRate: number;
|
|
84
|
+
passRateDelta: number;
|
|
85
|
+
baselineDurationMs: number;
|
|
86
|
+
treatmentDurationMs: number;
|
|
87
|
+
durationChangePercent: number;
|
|
88
|
+
baselineEstimatedTokens: number;
|
|
89
|
+
treatmentEstimatedTokens: number;
|
|
90
|
+
tokenChangePercent: number;
|
|
91
|
+
agentDeltas: Array<{
|
|
92
|
+
agent: string;
|
|
93
|
+
baselineScore: number;
|
|
94
|
+
treatmentScore: number;
|
|
95
|
+
scoreDelta: number;
|
|
96
|
+
baselinePassRate: number;
|
|
97
|
+
treatmentPassRate: number;
|
|
98
|
+
passRateDelta: number;
|
|
99
|
+
}>;
|
|
100
|
+
};
|
|
101
|
+
summary: Array<{
|
|
102
|
+
candidateId: string;
|
|
103
|
+
provider: string;
|
|
104
|
+
model: string;
|
|
105
|
+
trials: number;
|
|
106
|
+
completed: number;
|
|
107
|
+
averageScore: number;
|
|
108
|
+
passRate: number;
|
|
109
|
+
scoreStdDev: number;
|
|
110
|
+
averageDurationMs: number;
|
|
111
|
+
averageEstimatedTokens: number;
|
|
112
|
+
}>;
|
|
113
|
+
}
|
|
114
|
+
export declare function validateBenchmarkMatrix(value: unknown): BenchmarkMatrixConfig;
|
|
115
|
+
export declare function fingerprintAgentConfig(files: Array<{
|
|
116
|
+
path: string;
|
|
117
|
+
sha256: string;
|
|
118
|
+
}>): string;
|
|
119
|
+
export declare function loadBenchmarkMatrix(file: string): Promise<BenchmarkMatrixConfig>;
|
|
120
|
+
export declare function executeBenchmarkMatrix(config: BenchmarkMatrixConfig, execute: (candidate: BenchmarkCandidate, repeat: number) => Promise<Omit<BenchmarkMatrixResult, "candidateId" | "provider" | "model" | "repeat" | "status">>): Promise<BenchmarkMatrixReport>;
|
|
121
|
+
export declare function saveBenchmarkMatrix(projectPath: string, report: BenchmarkMatrixReport): Promise<string>;
|
|
122
|
+
export declare function formatBenchmarkMatrix(report: BenchmarkMatrixReport): string;
|
|
123
|
+
export declare function listBenchmarkMatrices(projectPath: string): Promise<BenchmarkMatrixReport[]>;
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
import { mkdir, readFile, readdir, rename, rm, writeFile } from "node:fs/promises";
|
|
2
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
const SAFE_ID = /^[a-z0-9][a-z0-9-]{1,47}$/;
|
|
5
|
+
export function validateBenchmarkMatrix(value) {
|
|
6
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
7
|
+
throw new Error("Benchmark matrix config must be a JSON object.");
|
|
8
|
+
const config = value;
|
|
9
|
+
if (!SAFE_ID.test(config.id) || typeof config.name !== "string" || !config.name.trim() || config.name.length > 120)
|
|
10
|
+
throw new Error("Benchmark matrix needs a valid id and name.");
|
|
11
|
+
if (!Number.isSafeInteger(config.repeats) || config.repeats < 1 || config.repeats > 3)
|
|
12
|
+
throw new Error("Benchmark matrix repeats must be between 1 and 3.");
|
|
13
|
+
if (!Array.isArray(config.candidates) || config.candidates.length < 2 || config.candidates.length > 4)
|
|
14
|
+
throw new Error("Benchmark matrix needs between 2 and 4 candidates.");
|
|
15
|
+
const ids = new Set();
|
|
16
|
+
for (const candidate of config.candidates) {
|
|
17
|
+
if (!candidate || !SAFE_ID.test(candidate.id) || ids.has(candidate.id))
|
|
18
|
+
throw new Error("Every benchmark candidate needs a unique valid id.");
|
|
19
|
+
ids.add(candidate.id);
|
|
20
|
+
if (!["openai", "claude-code", "anthropic", "google", "ollama"].includes(candidate.provider))
|
|
21
|
+
throw new Error("Unsupported benchmark candidate provider.");
|
|
22
|
+
if (typeof candidate.model !== "string" || !candidate.model.trim() || candidate.model.length > 160)
|
|
23
|
+
throw new Error("Every benchmark candidate needs a model name.");
|
|
24
|
+
if (candidate.configVersionId !== undefined && !/^cfg-[a-f0-9]{12}$/.test(candidate.configVersionId))
|
|
25
|
+
throw new Error("Candidate configVersionId must be a valid agent configuration snapshot ID.");
|
|
26
|
+
}
|
|
27
|
+
for (const candidate of config.candidates) {
|
|
28
|
+
if (candidate.skillIds !== undefined && (!Array.isArray(candidate.skillIds) || candidate.skillIds.some(id => typeof id !== "string" || !/^[a-z0-9][a-z0-9-]{0,63}$/.test(id)) || new Set(candidate.skillIds).size !== candidate.skillIds.length))
|
|
29
|
+
throw new Error("Candidate skillIds must be unique valid skill IDs.");
|
|
30
|
+
}
|
|
31
|
+
if (config.skillExperiment) {
|
|
32
|
+
const experiment = config.skillExperiment;
|
|
33
|
+
const baseline = config.candidates.find(item => item.id === experiment.baselineCandidateId);
|
|
34
|
+
const treatment = config.candidates.find(item => item.id === experiment.treatmentCandidateId);
|
|
35
|
+
if (!baseline || !treatment || baseline.id === treatment.id)
|
|
36
|
+
throw new Error("Skill experiment must reference distinct baseline and treatment candidates.");
|
|
37
|
+
if (baseline.provider !== treatment.provider || baseline.model !== treatment.model)
|
|
38
|
+
throw new Error("Skill experiment candidates must use the same provider and model.");
|
|
39
|
+
if (!Array.isArray(experiment.skillIds) || !experiment.skillIds.length || experiment.skillIds.some(id => typeof id !== "string" || !/^[a-z0-9][a-z0-9-]{0,63}$/.test(id)) || new Set(experiment.skillIds).size !== experiment.skillIds.length)
|
|
40
|
+
throw new Error("Skill experiment must specify unique valid skill IDs.");
|
|
41
|
+
if (!Array.isArray(baseline.skillIds) || baseline.skillIds.length !== 0 || !Array.isArray(treatment.skillIds) || [...treatment.skillIds].sort().join(",") !== [...experiment.skillIds].sort().join(","))
|
|
42
|
+
throw new Error("Baseline must set skillIds to [] and treatment skillIds must exactly match skillExperiment.skillIds.");
|
|
43
|
+
}
|
|
44
|
+
return config;
|
|
45
|
+
}
|
|
46
|
+
export function fingerprintAgentConfig(files) {
|
|
47
|
+
const canonical = [...files].sort((a, b) => a.path.localeCompare(b.path))
|
|
48
|
+
.map(file => file.path + ":" + file.sha256).join("\\n");
|
|
49
|
+
return createHash("sha256").update(canonical, "utf8").digest("hex");
|
|
50
|
+
}
|
|
51
|
+
export async function loadBenchmarkMatrix(file) {
|
|
52
|
+
return validateBenchmarkMatrix(JSON.parse(await readFile(file, "utf8")));
|
|
53
|
+
}
|
|
54
|
+
export async function executeBenchmarkMatrix(config, execute) {
|
|
55
|
+
config = validateBenchmarkMatrix(config);
|
|
56
|
+
const results = [];
|
|
57
|
+
for (const candidate of config.candidates) {
|
|
58
|
+
for (let repeat = 1; repeat <= config.repeats; repeat += 1) {
|
|
59
|
+
try {
|
|
60
|
+
const result = await execute(candidate, repeat);
|
|
61
|
+
results.push({ candidateId: candidate.id, provider: candidate.provider, model: candidate.model, repeat, status: "completed", ...result });
|
|
62
|
+
}
|
|
63
|
+
catch (error) {
|
|
64
|
+
results.push({ candidateId: candidate.id, provider: candidate.provider, model: candidate.model, repeat, runId: "", score: 0, passed: false, durationMs: 0, estimatedTokens: 0, status: "failed", error: error instanceof Error ? error.message : String(error) });
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
const summary = config.candidates.map(candidate => {
|
|
69
|
+
const items = results.filter(item => item.candidateId === candidate.id && item.status === "completed");
|
|
70
|
+
const scores = items.map(item => item.score);
|
|
71
|
+
const average = scores.length ? scores.reduce((sum, score) => sum + score, 0) / scores.length : 0;
|
|
72
|
+
return {
|
|
73
|
+
candidateId: candidate.id, provider: candidate.provider, model: candidate.model, trials: config.repeats,
|
|
74
|
+
completed: items.length,
|
|
75
|
+
averageScore: Math.round(average * 10) / 10,
|
|
76
|
+
passRate: items.length ? Math.round(items.filter(item => item.passed).length / items.length * 1000) / 10 : 0,
|
|
77
|
+
scoreStdDev: items.length ? Math.round(Math.sqrt(scores.reduce((sum, score) => sum + Math.pow(score - average, 2), 0) / scores.length) * 10) / 10 : 0,
|
|
78
|
+
averageDurationMs: items.length ? Math.round(items.reduce((sum, item) => sum + item.durationMs, 0) / items.length) : 0,
|
|
79
|
+
averageEstimatedTokens: items.length ? Math.round(items.reduce((sum, item) => sum + item.estimatedTokens, 0) / items.length) : 0,
|
|
80
|
+
};
|
|
81
|
+
}).sort((a, b) => b.averageScore - a.averageScore || a.averageDurationMs - b.averageDurationMs);
|
|
82
|
+
const agentKeys = new Set(results.flatMap(item => (item.agentScores ?? []).map(score => item.candidateId + "::" + score.agent)));
|
|
83
|
+
const agentSummary = [...agentKeys].map(key => {
|
|
84
|
+
const separator = key.indexOf("::");
|
|
85
|
+
const candidateId = key.slice(0, separator);
|
|
86
|
+
const agent = key.slice(separator + 2);
|
|
87
|
+
const candidate = config.candidates.find(item => item.id === candidateId);
|
|
88
|
+
const scores = results.filter(item => item.candidateId === candidateId && item.status === "completed").flatMap(item => (item.agentScores ?? []).filter(score => score.agent === agent));
|
|
89
|
+
const skillsObserved = [...new Set(scores.flatMap(item => item.skillIds))].sort();
|
|
90
|
+
return { candidateId, provider: candidate.provider, model: candidate.model, agent, trials: config.repeats, completed: scores.length, averageScore: scores.length ? Math.round(scores.reduce((sum, item) => sum + item.score, 0) / scores.length * 10) / 10 : 0, passRate: scores.length ? Math.round(scores.filter(item => item.passed).length / scores.length * 1000) / 10 : 0, skillsObserved };
|
|
91
|
+
}).sort((a, b) => b.averageScore - a.averageScore || a.agent.localeCompare(b.agent));
|
|
92
|
+
let skillExperiment;
|
|
93
|
+
if (config.skillExperiment) {
|
|
94
|
+
const { baselineCandidateId, treatmentCandidateId, skillIds } = config.skillExperiment;
|
|
95
|
+
const baseline = summary.find(item => item.candidateId === baselineCandidateId);
|
|
96
|
+
const treatment = summary.find(item => item.candidateId === treatmentCandidateId);
|
|
97
|
+
if (baseline && treatment && baseline.completed > 0 && treatment.completed > 0) {
|
|
98
|
+
const change = (before, after) => before <= 0 ? (after > 0 ? 100 : 0) : Math.round((after - before) / before * 1000) / 10;
|
|
99
|
+
const agentDeltas = (agentSummary.filter(item => item.candidateId === baselineCandidateId)).flatMap(before => {
|
|
100
|
+
const after = agentSummary.find(item => item.candidateId === treatmentCandidateId && item.agent === before.agent);
|
|
101
|
+
return after ? [{ agent: before.agent, baselineScore: before.averageScore, treatmentScore: after.averageScore, scoreDelta: Math.round((after.averageScore - before.averageScore) * 10) / 10, baselinePassRate: before.passRate, treatmentPassRate: after.passRate, passRateDelta: Math.round((after.passRate - before.passRate) * 10) / 10 }] : [];
|
|
102
|
+
});
|
|
103
|
+
skillExperiment = { baselineCandidateId, treatmentCandidateId, skillIds: [...skillIds], baselineScore: baseline.averageScore, treatmentScore: treatment.averageScore, scoreDelta: Math.round((treatment.averageScore - baseline.averageScore) * 10) / 10, baselinePassRate: baseline.passRate, treatmentPassRate: treatment.passRate, passRateDelta: Math.round((treatment.passRate - baseline.passRate) * 10) / 10, baselineDurationMs: baseline.averageDurationMs, treatmentDurationMs: treatment.averageDurationMs, durationChangePercent: change(baseline.averageDurationMs, treatment.averageDurationMs), baselineEstimatedTokens: baseline.averageEstimatedTokens, treatmentEstimatedTokens: treatment.averageEstimatedTokens, tokenChangePercent: change(baseline.averageEstimatedTokens, treatment.averageEstimatedTokens), agentDeltas };
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
return { version: 1, matrixId: config.id, matrixName: config.name, suiteId: "", goal: "", createdAt: new Date().toISOString(), results, summary, agentSummary, ...(config.candidates.some(candidate => candidate.configVersionId) ? { candidateProvenance: config.candidates.filter(candidate => candidate.configVersionId).map(candidate => ({ candidateId: candidate.id, configVersionId: candidate.configVersionId, fingerprint: "" })) } : {}), ...(skillExperiment ? { skillExperiment } : {}) };
|
|
107
|
+
}
|
|
108
|
+
export async function saveBenchmarkMatrix(projectPath, report) {
|
|
109
|
+
const directory = path.join(path.resolve(projectPath), ".forge", "benchmark-matrices");
|
|
110
|
+
await mkdir(directory, { recursive: true, mode: 0o700 });
|
|
111
|
+
const destination = path.join(directory, report.matrixId + "-" + report.createdAt.replace(/[:.]/g, "-") + ".json");
|
|
112
|
+
const temporary = path.join(directory, "." + randomUUID() + ".tmp");
|
|
113
|
+
try {
|
|
114
|
+
await writeFile(temporary, JSON.stringify(report, null, 2) + "\n", { encoding: "utf8", flag: "wx", mode: 0o600 });
|
|
115
|
+
await rename(temporary, destination);
|
|
116
|
+
}
|
|
117
|
+
catch (error) {
|
|
118
|
+
await rm(temporary, { force: true }).catch(() => undefined);
|
|
119
|
+
throw error;
|
|
120
|
+
}
|
|
121
|
+
return destination;
|
|
122
|
+
}
|
|
123
|
+
export function formatBenchmarkMatrix(report) {
|
|
124
|
+
const lines = ["FORGE Automated Benchmark Matrix · " + report.matrixName, "Suite: " + report.suiteId, "Goal: " + report.goal, "", "Candidate | Trials | Completed | Avg score | Pass rate | σ score | Avg time | Avg estimated tokens"];
|
|
125
|
+
for (const item of report.summary)
|
|
126
|
+
lines.push([item.candidateId + " (" + item.provider + "/" + item.model + ")", item.trials, item.completed, item.averageScore + "/100", item.passRate + "%", item.scoreStdDev, item.averageDurationMs + " ms", item.averageEstimatedTokens].join(" | "));
|
|
127
|
+
if (report.agentSummary?.length) {
|
|
128
|
+
lines.push("", "Per-agent scorecards:", "Candidate | Agent | Cases | Average score | Pass rate | Skills observed");
|
|
129
|
+
for (const item of report.agentSummary)
|
|
130
|
+
lines.push([item.candidateId, item.agent, item.completed, item.averageScore + "/100", item.passRate + "%", item.skillsObserved.join(", ") || "none"].join(" | "));
|
|
131
|
+
}
|
|
132
|
+
if (report.skillExperiment) {
|
|
133
|
+
const item = report.skillExperiment;
|
|
134
|
+
lines.push("", "Controlled skill experiment:", " Baseline: " + item.baselineCandidateId + " (no skills)", " Treatment: " + item.treatmentCandidateId + " (skills: " + item.skillIds.join(", ") + ")", " Score: " + item.baselineScore + " → " + item.treatmentScore + " (" + (item.scoreDelta > 0 ? "+" : "") + item.scoreDelta + ")", " Pass rate: " + item.baselinePassRate + "% → " + item.treatmentPassRate + "% (" + (item.passRateDelta > 0 ? "+" : "") + item.passRateDelta + " pp)", " Duration change: " + item.durationChangePercent + "% · estimated token change: " + item.tokenChangePercent + "%");
|
|
135
|
+
for (const agent of item.agentDeltas)
|
|
136
|
+
lines.push(" Agent " + agent.agent + ": " + agent.baselineScore + " → " + agent.treatmentScore + " (" + (agent.scoreDelta > 0 ? "+" : "") + agent.scoreDelta + ")");
|
|
137
|
+
}
|
|
138
|
+
lines.push("", "Trial details:");
|
|
139
|
+
for (const item of report.results)
|
|
140
|
+
lines.push(" " + item.candidateId + " #" + item.repeat + " · " + (item.status === "completed" ? item.score + "/100 · " + (item.passed ? "PASS" : "REVIEW") : "FAILED · " + item.error));
|
|
141
|
+
if (report.candidateProvenance?.length) {
|
|
142
|
+
lines.push("", "Configuration provenance:");
|
|
143
|
+
for (const item of report.candidateProvenance)
|
|
144
|
+
lines.push(" " + item.candidateId + " · " + item.configVersionId + " · sha256 " + item.fingerprint);
|
|
145
|
+
}
|
|
146
|
+
lines.push("", "All model usage figures are estimates. Candidate comparisons are meaningful only when task, suite, settings and environment are held constant.");
|
|
147
|
+
return lines.join("\n");
|
|
148
|
+
}
|
|
149
|
+
export async function listBenchmarkMatrices(projectPath) {
|
|
150
|
+
const directory = path.join(path.resolve(projectPath), ".forge", "benchmark-matrices");
|
|
151
|
+
let files;
|
|
152
|
+
try {
|
|
153
|
+
files = (await readdir(directory)).filter(file => file.endsWith(".json")).sort();
|
|
154
|
+
}
|
|
155
|
+
catch (error) {
|
|
156
|
+
if (error && typeof error === "object" && "code" in error && error.code === "ENOENT")
|
|
157
|
+
return [];
|
|
158
|
+
throw error;
|
|
159
|
+
}
|
|
160
|
+
const reports = [];
|
|
161
|
+
for (const file of files.slice(-100)) {
|
|
162
|
+
const value = JSON.parse(await readFile(path.join(directory, file), "utf8"));
|
|
163
|
+
if (!value || typeof value !== "object" || value.version !== 1 || !Array.isArray(value.results) || !Array.isArray(value.summary))
|
|
164
|
+
throw new Error("Stored benchmark matrix report is invalid: " + file);
|
|
165
|
+
reports.push(value);
|
|
166
|
+
}
|
|
167
|
+
return reports.sort((a, b) => a.createdAt.localeCompare(b.createdAt));
|
|
168
|
+
}
|
|
169
|
+
//# sourceMappingURL=benchmark-matrix.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"benchmark-matrix.js","sourceRoot":"","sources":["../../src/core/benchmark-matrix.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACnF,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,IAAI,MAAM,WAAW,CAAC;AA4D7B,MAAM,OAAO,GAAG,2BAA2B,CAAC;AAE5C,MAAM,UAAU,uBAAuB,CAAC,KAAc;IACpD,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACnI,MAAM,MAAM,GAAG,KAA8B,CAAC;IAC9C,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG;QAAE,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IACnL,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;IAC5J,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IAC7K,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;IAC9B,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QAC1C,IAAI,CAAC,SAAS,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;QAC9I,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACtB,IAAI,CAAC,CAAC,QAAQ,EAAE,aAAa,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;QAC3J,IAAI,OAAO,SAAS,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,SAAS,CAAC,KAAK,CAAC,MAAM,GAAG,GAAG;YAAE,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;QACrK,IAAI,SAAS,CAAC,eAAe,KAAK,SAAS,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,4EAA4E,CAAC,CAAC;IACtM,CAAC;IACD,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QAC1C,IAAI,SAAS,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,KAAK,QAAQ,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,IAAI,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IAC1T,CAAC;IACD,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;QAC3B,MAAM,UAAU,GAAG,MAAM,CAAC,eAAe,CAAC;QAC1C,MAAM,QAAQ,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,KAAK,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5F,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,KAAK,UAAU,CAAC,oBAAoB,CAAC,CAAC;QAC9F,IAAI,CAAC,QAAQ,IAAI,CAAC,SAAS,IAAI,QAAQ,CAAC,EAAE,KAAK,SAAS,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,6EAA6E,CAAC,CAAC;QAC5J,IAAI,QAAQ,CAAC,QAAQ,KAAK,SAAS,CAAC,QAAQ,IAAI,QAAQ,CAAC,KAAK,KAAK,SAAS,CAAC,KAAK;YAAE,MAAM,IAAI,KAAK,CAAC,mEAAmE,CAAC,CAAC;QACzK,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,MAAM,IAAI,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,KAAK,QAAQ,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,IAAI,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,QAAQ,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;QACxT,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,sGAAsG,CAAC,CAAC;IACnU,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,KAA8C;IACnF,MAAM,SAAS,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;SACrE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,GAAG,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC1D,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACtE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,IAAY;IACpD,OAAO,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;AAC3E,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,MAA6B,EAC7B,OAA4J;IAE5J,MAAM,GAAG,uBAAuB,CAAC,MAAM,CAAC,CAAC;IACzC,MAAM,OAAO,GAA4B,EAAE,CAAC;IAC5C,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QAC1C,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,EAAE,CAAC;YAC3D,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;gBAChD,OAAO,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,SAAS,CAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,CAAC,QAAQ,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;YAC5I,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,SAAS,CAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,CAAC,QAAQ,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACpQ,CAAC;QACH,CAAC;IACH,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE;QAChD,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,KAAK,SAAS,CAAC,EAAE,IAAI,IAAI,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC;QACvG,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC7C,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAClG,OAAO;YACL,WAAW,EAAE,SAAS,CAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,CAAC,QAAQ,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO;YACvG,SAAS,EAAE,KAAK,CAAC,MAAM;YACvB,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,GAAG,EAAE;YAC3C,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;YAC5G,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;YACrJ,iBAAiB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YACtH,sBAAsB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;SACjI,CAAC;IACJ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,iBAAiB,GAAG,CAAC,CAAC,iBAAiB,CAAC,CAAC;IAChG,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACjI,MAAM,YAAY,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;QAC5C,MAAM,SAAS,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACpC,MAAM,WAAW,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QAC5C,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC;QACvC,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,KAAK,WAAW,CAAE,CAAC;QAC3E,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,KAAK,WAAW,IAAI,IAAI,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC;QACxL,MAAM,cAAc,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAClF,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,SAAS,CAAC,QAAQ,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC;IACpY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IACrF,IAAI,eAAyD,CAAC;IAC9D,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;QAC3B,MAAM,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,QAAQ,EAAE,GAAG,MAAM,CAAC,eAAe,CAAC;QACvF,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,KAAK,mBAAmB,CAAC,CAAC;QAChF,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,KAAK,oBAAoB,CAAC,CAAC;QAClF,IAAI,QAAQ,IAAI,SAAS,IAAI,QAAQ,CAAC,SAAS,GAAG,CAAC,IAAI,SAAS,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;YAC/E,MAAM,MAAM,GAAG,CAAC,MAAc,EAAE,KAAa,EAAE,EAAE,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,GAAG,MAAM,CAAC,GAAG,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC1I,MAAM,WAAW,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,KAAK,mBAAmB,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE;gBAC3G,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,KAAK,oBAAoB,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC;gBAClH,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,aAAa,EAAE,MAAM,CAAC,YAAY,EAAE,cAAc,EAAE,KAAK,CAAC,YAAY,EAAE,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,EAAE,gBAAgB,EAAE,MAAM,CAAC,QAAQ,EAAE,iBAAiB,EAAE,KAAK,CAAC,QAAQ,EAAE,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACrV,CAAC,CAAC,CAAC;YACH,eAAe,GAAG,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,QAAQ,EAAE,CAAC,GAAG,QAAQ,CAAC,EAAE,aAAa,EAAE,QAAQ,CAAC,YAAY,EAAE,cAAc,EAAE,SAAS,CAAC,YAAY,EAAE,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,YAAY,GAAG,QAAQ,CAAC,YAAY,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,EAAE,gBAAgB,EAAE,QAAQ,CAAC,QAAQ,EAAE,iBAAiB,EAAE,SAAS,CAAC,QAAQ,EAAE,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,EAAE,kBAAkB,EAAE,QAAQ,CAAC,iBAAiB,EAAE,mBAAmB,EAAE,SAAS,CAAC,iBAAiB,EAAE,qBAAqB,EAAE,MAAM,CAAC,QAAQ,CAAC,iBAAiB,EAAE,SAAS,CAAC,iBAAiB,CAAC,EAAE,uBAAuB,EAAE,QAAQ,CAAC,sBAAsB,EAAE,wBAAwB,EAAE,SAAS,CAAC,sBAAsB,EAAE,kBAAkB,EAAE,MAAM,CAAC,QAAQ,CAAC,sBAAsB,EAAE,SAAS,CAAC,sBAAsB,CAAC,EAAE,WAAW,EAAE,CAAC;QACpzB,CAAC;IACH,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,EAAE,UAAU,EAAE,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,EAAE,WAAW,EAAE,SAAS,CAAC,EAAE,EAAE,eAAe,EAAE,SAAS,CAAC,eAAgB,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;AACze,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,WAAmB,EAAE,MAA6B;IAC1F,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,oBAAoB,CAAC,CAAC;IACvF,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACzD,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,QAAQ,GAAG,GAAG,GAAG,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC;IACnH,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,GAAG,UAAU,EAAE,GAAG,MAAM,CAAC,CAAC;IACpE,IAAI,CAAC;QACH,MAAM,SAAS,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAClH,MAAM,MAAM,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;IACvC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAAC,MAAM,EAAE,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAAC,MAAM,KAAK,CAAC;IAAC,CAAC;IAC7F,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,MAA6B;IACjE,MAAM,KAAK,GAAG,CAAC,qCAAqC,GAAG,MAAM,CAAC,UAAU,EAAE,SAAS,GAAG,MAAM,CAAC,OAAO,EAAE,QAAQ,GAAG,MAAM,CAAC,IAAI,EAAE,EAAE,EAAE,oGAAoG,CAAC,CAAC;IACxO,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,OAAO;QAAE,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,GAAG,IAAI,CAAC,QAAQ,GAAG,GAAG,GAAG,IAAI,CAAC,KAAK,GAAG,GAAG,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,YAAY,GAAG,MAAM,EAAE,IAAI,CAAC,QAAQ,GAAG,GAAG,EAAE,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,iBAAiB,GAAG,KAAK,EAAE,IAAI,CAAC,sBAAsB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAC3R,IAAI,MAAM,CAAC,YAAY,EAAE,MAAM,EAAE,CAAC;QAAC,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,uBAAuB,EAAE,yEAAyE,CAAC,CAAC;QAAC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,YAAY;YAAE,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,YAAY,GAAG,MAAM,EAAE,IAAI,CAAC,QAAQ,GAAG,GAAG,EAAE,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAAC,CAAC;IACnW,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,MAAM,CAAC,eAAe,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,8BAA8B,EAAE,cAAc,GAAG,IAAI,CAAC,mBAAmB,GAAG,cAAc,EAAE,eAAe,GAAG,IAAI,CAAC,oBAAoB,GAAG,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,EAAE,WAAW,GAAG,IAAI,CAAC,aAAa,GAAG,KAAK,GAAG,IAAI,CAAC,cAAc,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,GAAG,GAAG,EAAE,eAAe,GAAG,IAAI,CAAC,gBAAgB,GAAG,MAAM,GAAG,IAAI,CAAC,iBAAiB,GAAG,KAAK,GAAG,CAAC,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,aAAa,GAAG,MAAM,EAAE,qBAAqB,GAAG,IAAI,CAAC,qBAAqB,GAAG,8BAA8B,GAAG,IAAI,CAAC,kBAAkB,GAAG,GAAG,CAAC,CAAC;QACtlB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,WAAW;YAAE,KAAK,CAAC,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC,KAAK,GAAG,IAAI,GAAG,KAAK,CAAC,aAAa,GAAG,KAAK,GAAG,KAAK,CAAC,cAAc,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC;IAC7M,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IACjC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,OAAO;QAAE,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,GAAG,IAAI,GAAG,IAAI,CAAC,MAAM,GAAG,KAAK,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,SAAS,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAC5N,IAAI,MAAM,CAAC,mBAAmB,EAAE,MAAM,EAAE,CAAC;QAAC,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,2BAA2B,CAAC,CAAC;QAAC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,mBAAmB;YAAE,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,GAAG,KAAK,GAAG,IAAI,CAAC,eAAe,GAAG,YAAY,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC;IAAC,CAAC;IAC7O,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,gJAAgJ,CAAC,CAAC;IACjK,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,WAAmB;IAC7D,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,oBAAoB,CAAC,CAAC;IACvF,IAAI,KAAe,CAAC;IACpB,IAAI,CAAC;QAAC,KAAK,GAAG,CAAC,MAAM,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAAC,CAAC;IACzF,OAAO,KAAK,EAAE,CAAC;QAAC,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,EAAE,CAAC;QAAC,MAAM,KAAK,CAAC;IAAC,CAAC;IAC/H,MAAM,OAAO,GAA4B,EAAE,CAAC;IAC5C,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;QACrC,MAAM,KAAK,GAAY,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;QACtF,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAK,KAA+B,CAAC,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAE,KAA+B,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAE,KAA+B,CAAC,OAAO,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,6CAA6C,GAAG,IAAI,CAAC,CAAC;QACzR,OAAO,CAAC,IAAI,CAAC,KAA8B,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;AACxE,CAAC"}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { type BenchmarkMatrixReport } from "./benchmark-matrix.js";
|
|
2
|
+
export type RegressionStatus = "release-ready" | "warning" | "blocked" | "insufficient-history";
|
|
3
|
+
export interface CandidateRegression {
|
|
4
|
+
candidateId: string;
|
|
5
|
+
provider: string;
|
|
6
|
+
model: string;
|
|
7
|
+
baselineScore: number;
|
|
8
|
+
currentScore: number;
|
|
9
|
+
scoreDelta: number;
|
|
10
|
+
baselinePassRate: number;
|
|
11
|
+
currentPassRate: number;
|
|
12
|
+
passRateDelta: number;
|
|
13
|
+
baselineDurationMs: number;
|
|
14
|
+
currentDurationMs: number;
|
|
15
|
+
durationChangePercent: number;
|
|
16
|
+
baselineEstimatedTokens: number;
|
|
17
|
+
currentEstimatedTokens: number;
|
|
18
|
+
tokenChangePercent: number;
|
|
19
|
+
scoreStdDev: number;
|
|
20
|
+
failedTrials: number;
|
|
21
|
+
regressions: string[];
|
|
22
|
+
warnings: string[];
|
|
23
|
+
}
|
|
24
|
+
export interface AgentRegression {
|
|
25
|
+
candidateId: string;
|
|
26
|
+
agent: string;
|
|
27
|
+
baselineScore: number;
|
|
28
|
+
currentScore: number;
|
|
29
|
+
scoreDelta: number;
|
|
30
|
+
baselinePassRate: number;
|
|
31
|
+
currentPassRate: number;
|
|
32
|
+
regressions: string[];
|
|
33
|
+
warnings: string[];
|
|
34
|
+
skillsObserved: string[];
|
|
35
|
+
}
|
|
36
|
+
export interface BenchmarkRegressionReport {
|
|
37
|
+
status: RegressionStatus;
|
|
38
|
+
matrixId: string;
|
|
39
|
+
matrixName: string;
|
|
40
|
+
suiteId: string;
|
|
41
|
+
baselineAt: string;
|
|
42
|
+
currentAt: string;
|
|
43
|
+
candidates: CandidateRegression[];
|
|
44
|
+
candidateProvenance?: Array<{
|
|
45
|
+
candidateId: string;
|
|
46
|
+
configVersionId: string;
|
|
47
|
+
fingerprint: string;
|
|
48
|
+
}>;
|
|
49
|
+
agents: AgentRegression[];
|
|
50
|
+
summary: string[];
|
|
51
|
+
}
|
|
52
|
+
export declare function analyzeBenchmarkRegression(history: BenchmarkMatrixReport[], matrixId?: string): BenchmarkRegressionReport;
|
|
53
|
+
export declare function loadBenchmarkRegression(projectPath: string, matrixId?: string): Promise<BenchmarkRegressionReport>;
|
|
54
|
+
export declare function formatBenchmarkRegression(report: BenchmarkRegressionReport): string;
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import { listBenchmarkMatrices } from "./benchmark-matrix.js";
|
|
2
|
+
const percentChange = (baseline, current) => baseline <= 0 ? (current > 0 ? 100 : 0) : Math.round((current - baseline) / baseline * 1000) / 10;
|
|
3
|
+
export function analyzeBenchmarkRegression(history, matrixId) {
|
|
4
|
+
const eligible = history.filter(item => (!matrixId || item.matrixId === matrixId) && item.summary.length > 0)
|
|
5
|
+
.sort((a, b) => a.createdAt.localeCompare(b.createdAt));
|
|
6
|
+
if (eligible.length < 2) {
|
|
7
|
+
const latest = eligible.at(-1);
|
|
8
|
+
return { status: "insufficient-history", matrixId: latest?.matrixId ?? matrixId ?? "unknown", matrixName: latest?.matrixName ?? "Benchmark history", suiteId: latest?.suiteId ?? "", baselineAt: "", currentAt: latest?.createdAt ?? "", candidates: [], agents: [], summary: ["A second comparable matrix run is required before regression analysis is available."] };
|
|
9
|
+
}
|
|
10
|
+
const current = eligible[eligible.length - 1];
|
|
11
|
+
const previous = [...eligible.slice(0, -1)].reverse().find(item => item.suiteId === current.suiteId && item.goal === current.goal);
|
|
12
|
+
if (!previous)
|
|
13
|
+
return { status: "insufficient-history", matrixId: current.matrixId, matrixName: current.matrixName, suiteId: current.suiteId, baselineAt: "", currentAt: current.createdAt, candidates: [], agents: [], summary: ["No earlier run has the same evaluation suite and goal; create a comparable baseline first."] };
|
|
14
|
+
const candidates = current.summary.flatMap(now => {
|
|
15
|
+
const before = previous.summary.find(item => item.candidateId === now.candidateId && item.provider === now.provider && item.model === now.model);
|
|
16
|
+
if (!before)
|
|
17
|
+
return [];
|
|
18
|
+
const regressions = [];
|
|
19
|
+
const warnings = [];
|
|
20
|
+
const scoreDelta = Math.round((now.averageScore - before.averageScore) * 10) / 10;
|
|
21
|
+
const passRateDelta = Math.round((now.passRate - before.passRate) * 10) / 10;
|
|
22
|
+
const durationChangePercent = percentChange(before.averageDurationMs, now.averageDurationMs);
|
|
23
|
+
const tokenChangePercent = percentChange(before.averageEstimatedTokens, now.averageEstimatedTokens);
|
|
24
|
+
if (scoreDelta <= -5)
|
|
25
|
+
regressions.push("Average score dropped " + Math.abs(scoreDelta) + " points.");
|
|
26
|
+
if (passRateDelta <= -20)
|
|
27
|
+
regressions.push("Pass rate dropped " + Math.abs(passRateDelta) + " percentage points.");
|
|
28
|
+
if (durationChangePercent >= 25)
|
|
29
|
+
warnings.push("Average execution time increased " + durationChangePercent + "%.");
|
|
30
|
+
if (tokenChangePercent >= 25)
|
|
31
|
+
warnings.push("Estimated token use increased " + tokenChangePercent + "%.");
|
|
32
|
+
if (now.scoreStdDev >= 15)
|
|
33
|
+
warnings.push("Score variance is high (σ " + now.scoreStdDev + ").");
|
|
34
|
+
const failedTrials = current.results.filter(item => item.candidateId === now.candidateId && item.status === "failed").length;
|
|
35
|
+
if (failedTrials > 0)
|
|
36
|
+
warnings.push(failedTrials + " trial(s) failed to complete.");
|
|
37
|
+
return [{ candidateId: now.candidateId, provider: now.provider, model: now.model, baselineScore: before.averageScore, currentScore: now.averageScore, scoreDelta, baselinePassRate: before.passRate, currentPassRate: now.passRate, passRateDelta, baselineDurationMs: before.averageDurationMs, currentDurationMs: now.averageDurationMs, durationChangePercent, baselineEstimatedTokens: before.averageEstimatedTokens, currentEstimatedTokens: now.averageEstimatedTokens, tokenChangePercent, scoreStdDev: now.scoreStdDev, failedTrials, regressions, warnings }];
|
|
38
|
+
});
|
|
39
|
+
const agents = (current.agentSummary ?? []).flatMap(now => {
|
|
40
|
+
const before = (previous.agentSummary ?? []).find(item => item.candidateId === now.candidateId && item.agent === now.agent && item.provider === now.provider && item.model === now.model);
|
|
41
|
+
if (!before)
|
|
42
|
+
return [];
|
|
43
|
+
const regressions = [];
|
|
44
|
+
const warnings = [];
|
|
45
|
+
const scoreDelta = Math.round((now.averageScore - before.averageScore) * 10) / 10;
|
|
46
|
+
const passDelta = Math.round((now.passRate - before.passRate) * 10) / 10;
|
|
47
|
+
if (scoreDelta <= -5)
|
|
48
|
+
regressions.push("Agent score dropped " + Math.abs(scoreDelta) + " points.");
|
|
49
|
+
if (passDelta <= -20)
|
|
50
|
+
regressions.push("Agent pass rate dropped " + Math.abs(passDelta) + " percentage points.");
|
|
51
|
+
if (now.completed < now.trials)
|
|
52
|
+
warnings.push("Only " + now.completed + " of " + now.trials + " expected agent evaluations completed.");
|
|
53
|
+
return [{ candidateId: now.candidateId, agent: now.agent, baselineScore: before.averageScore, currentScore: now.averageScore, scoreDelta, baselinePassRate: before.passRate, currentPassRate: now.passRate, regressions, warnings, skillsObserved: now.skillsObserved ?? [] }];
|
|
54
|
+
});
|
|
55
|
+
const agentRegressions = agents.reduce((sum, item) => sum + item.regressions.length, 0);
|
|
56
|
+
const agentWarnings = agents.reduce((sum, item) => sum + item.warnings.length, 0);
|
|
57
|
+
const matched = candidates.length;
|
|
58
|
+
const regressions = candidates.reduce((sum, item) => sum + item.regressions.length, 0) + agentRegressions;
|
|
59
|
+
const warnings = candidates.reduce((sum, item) => sum + item.warnings.length, 0) + agentWarnings;
|
|
60
|
+
const unmatched = current.summary.length - matched;
|
|
61
|
+
const status = matched === 0 || unmatched > 0 ? "insufficient-history" : regressions > 0 ? "blocked" : warnings > 0 ? "warning" : "release-ready";
|
|
62
|
+
const summary = [
|
|
63
|
+
matched + " candidate(s) compared against the previous run using the same suite and goal.",
|
|
64
|
+
regressions + " quality regression(s), " + warnings + " efficiency or stability warning(s).",
|
|
65
|
+
...(unmatched ? [unmatched + " candidate(s) had no matching baseline and were excluded."] : []),
|
|
66
|
+
"Thresholds: score drop ≥5 points or pass-rate drop ≥20 points blocks; time/token growth ≥25% or score σ ≥15 warns."
|
|
67
|
+
];
|
|
68
|
+
return { status, matrixId: current.matrixId, matrixName: current.matrixName, suiteId: current.suiteId, baselineAt: previous.createdAt, currentAt: current.createdAt, candidates, ...(current.candidateProvenance ? { candidateProvenance: current.candidateProvenance } : {}), agents, summary: [...summary, agents.length + " per-agent scorecard(s) compared across the same candidate and model."] };
|
|
69
|
+
}
|
|
70
|
+
export async function loadBenchmarkRegression(projectPath, matrixId) {
|
|
71
|
+
return analyzeBenchmarkRegression(await listBenchmarkMatrices(projectPath), matrixId);
|
|
72
|
+
}
|
|
73
|
+
export function formatBenchmarkRegression(report) {
|
|
74
|
+
const lines = ["FORGE Benchmark Regression Intelligence · " + report.matrixName, "Status: " + report.status.toUpperCase(), "Suite: " + report.suiteId, ""];
|
|
75
|
+
lines.push(...report.summary);
|
|
76
|
+
for (const item of report.candidates) {
|
|
77
|
+
lines.push("", item.candidateId + " · " + item.provider + "/" + item.model, " Score: " + item.baselineScore + " → " + item.currentScore + " (" + (item.scoreDelta > 0 ? "+" : "") + item.scoreDelta + ")", " Pass rate: " + item.baselinePassRate + "% → " + item.currentPassRate + "%", " Duration: " + item.durationChangePercent + "% · estimated tokens: " + item.tokenChangePercent + "%");
|
|
78
|
+
for (const issue of item.regressions)
|
|
79
|
+
lines.push(" REGRESSION: " + issue);
|
|
80
|
+
for (const issue of item.warnings)
|
|
81
|
+
lines.push(" WARNING: " + issue);
|
|
82
|
+
}
|
|
83
|
+
if (report.agents.length) {
|
|
84
|
+
lines.push("", "Per-agent regressions:");
|
|
85
|
+
for (const item of report.agents) {
|
|
86
|
+
lines.push(" " + item.candidateId + " / " + item.agent + " · " + item.baselineScore + " → " + item.currentScore + " (" + (item.scoreDelta > 0 ? "+" : "") + item.scoreDelta + ") · pass " + item.baselinePassRate + "% → " + item.currentPassRate + "% · observed skills: " + (item.skillsObserved.join(", ") || "none"));
|
|
87
|
+
for (const issue of item.regressions)
|
|
88
|
+
lines.push(" REGRESSION: " + issue);
|
|
89
|
+
for (const issue of item.warnings)
|
|
90
|
+
lines.push(" WARNING: " + issue);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
lines.push("", "This is a trend signal, not a guarantee. Compare identical code snapshots, prompts, suite versions and runtime conditions.");
|
|
94
|
+
return lines.join("\n");
|
|
95
|
+
}
|
|
96
|
+
//# sourceMappingURL=benchmark-regression.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"benchmark-regression.js","sourceRoot":"","sources":["../../src/core/benchmark-regression.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAA8B,MAAM,uBAAuB,CAAC;AAqC1F,MAAM,aAAa,GAAG,CAAC,QAAgB,EAAE,OAAe,EAAU,EAAE,CAClE,QAAQ,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,GAAG,QAAQ,CAAC,GAAG,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;AAEpG,MAAM,UAAU,0BAA0B,CAAC,OAAgC,EAAE,QAAiB;IAC5F,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,KAAK,QAAQ,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;SAC1G,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;IAC1D,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/B,OAAO,EAAE,MAAM,EAAE,sBAAsB,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,IAAI,QAAQ,IAAI,SAAS,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,IAAI,mBAAmB,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,IAAI,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,IAAI,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,qFAAqF,CAAC,EAAE,CAAC;IAC1W,CAAC;IACD,MAAM,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC;IAC/C,MAAM,QAAQ,GAAG,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,KAAK,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACnI,IAAI,CAAC,QAAQ;QAAE,OAAO,EAAE,MAAM,EAAE,sBAAsB,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,UAAU,EAAE,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,UAAU,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,4FAA4F,CAAC,EAAE,CAAC;IAClU,MAAM,UAAU,GAA0B,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;QACtE,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,KAAK,GAAG,CAAC,WAAW,IAAI,IAAI,CAAC,QAAQ,KAAK,GAAG,CAAC,QAAQ,IAAI,IAAI,CAAC,KAAK,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC;QACjJ,IAAI,CAAC,MAAM;YAAE,OAAO,EAAE,CAAC;QACvB,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC;QAClF,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC;QAC7E,MAAM,qBAAqB,GAAG,aAAa,CAAC,MAAM,CAAC,iBAAiB,EAAE,GAAG,CAAC,iBAAiB,CAAC,CAAC;QAC7F,MAAM,kBAAkB,GAAG,aAAa,CAAC,MAAM,CAAC,sBAAsB,EAAE,GAAG,CAAC,sBAAsB,CAAC,CAAC;QACpG,IAAI,UAAU,IAAI,CAAC,CAAC;YAAE,WAAW,CAAC,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,UAAU,CAAC,CAAC;QACrG,IAAI,aAAa,IAAI,CAAC,EAAE;YAAE,WAAW,CAAC,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,GAAG,CAAC,aAAa,CAAC,GAAG,qBAAqB,CAAC,CAAC;QACnH,IAAI,qBAAqB,IAAI,EAAE;YAAE,QAAQ,CAAC,IAAI,CAAC,mCAAmC,GAAG,qBAAqB,GAAG,IAAI,CAAC,CAAC;QACnH,IAAI,kBAAkB,IAAI,EAAE;YAAE,QAAQ,CAAC,IAAI,CAAC,gCAAgC,GAAG,kBAAkB,GAAG,IAAI,CAAC,CAAC;QAC1G,IAAI,GAAG,CAAC,WAAW,IAAI,EAAE;YAAE,QAAQ,CAAC,IAAI,CAAC,4BAA4B,GAAG,GAAG,CAAC,WAAW,GAAG,IAAI,CAAC,CAAC;QAChG,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,KAAK,GAAG,CAAC,WAAW,IAAI,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC;QAC7H,IAAI,YAAY,GAAG,CAAC;YAAE,QAAQ,CAAC,IAAI,CAAC,YAAY,GAAG,+BAA+B,CAAC,CAAC;QACpF,OAAO,CAAC,EAAE,WAAW,EAAE,GAAG,CAAC,WAAW,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,aAAa,EAAE,MAAM,CAAC,YAAY,EAAE,YAAY,EAAE,GAAG,CAAC,YAAY,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,CAAC,QAAQ,EAAE,eAAe,EAAE,GAAG,CAAC,QAAQ,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,CAAC,iBAAiB,EAAE,iBAAiB,EAAE,GAAG,CAAC,iBAAiB,EAAE,qBAAqB,EAAE,uBAAuB,EAAE,MAAM,CAAC,sBAAsB,EAAE,sBAAsB,EAAE,GAAG,CAAC,sBAAsB,EAAE,kBAAkB,EAAE,WAAW,EAAE,GAAG,CAAC,WAAW,EAAE,YAAY,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,CAAC;IACziB,CAAC,CAAC,CAAC;IACH,MAAM,MAAM,GAAsB,CAAC,OAAO,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;QAC3E,MAAM,MAAM,GAAG,CAAC,QAAQ,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,KAAK,GAAG,CAAC,WAAW,IAAI,IAAI,CAAC,KAAK,KAAK,GAAG,CAAC,KAAK,IAAI,IAAI,CAAC,QAAQ,KAAK,GAAG,CAAC,QAAQ,IAAI,IAAI,CAAC,KAAK,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC;QAC1L,IAAI,CAAC,MAAM;YAAE,OAAO,EAAE,CAAC;QACvB,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC;QAClF,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC;QACzE,IAAI,UAAU,IAAI,CAAC,CAAC;YAAE,WAAW,CAAC,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,UAAU,CAAC,CAAC;QACnG,IAAI,SAAS,IAAI,CAAC,EAAE;YAAE,WAAW,CAAC,IAAI,CAAC,0BAA0B,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,qBAAqB,CAAC,CAAC;QACjH,IAAI,GAAG,CAAC,SAAS,GAAG,GAAG,CAAC,MAAM;YAAE,QAAQ,CAAC,IAAI,CAAC,OAAO,GAAG,GAAG,CAAC,SAAS,GAAG,MAAM,GAAG,GAAG,CAAC,MAAM,GAAG,wCAAwC,CAAC,CAAC;QACxI,OAAO,CAAC,EAAE,WAAW,EAAE,GAAG,CAAC,WAAW,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,aAAa,EAAE,MAAM,CAAC,YAAY,EAAE,YAAY,EAAE,GAAG,CAAC,YAAY,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,CAAC,QAAQ,EAAE,eAAe,EAAE,GAAG,CAAC,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,cAAc,EAAE,GAAG,CAAC,cAAc,IAAI,EAAE,EAAE,CAAC,CAAC;IACjR,CAAC,CAAC,CAAC;IACH,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACxF,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAClF,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC;IAClC,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,gBAAgB,CAAC;IAC1G,MAAM,QAAQ,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,aAAa,CAAC;IACjG,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC;IACnD,MAAM,MAAM,GAAqB,OAAO,KAAK,CAAC,IAAI,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,eAAe,CAAC;IACpK,MAAM,OAAO,GAAG;QACd,OAAO,GAAG,gFAAgF;QAC1F,WAAW,GAAG,0BAA0B,GAAG,QAAQ,GAAG,sCAAsC;QAC5F,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,2DAA2D,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/F,oHAAoH;KACrH,CAAC;IACF,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,UAAU,EAAE,QAAQ,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,UAAU,EAAE,GAAG,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,GAAG,OAAO,EAAE,MAAM,CAAC,MAAM,GAAG,uEAAuE,CAAC,EAAE,CAAC;AAC1Y,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,WAAmB,EAAE,QAAiB;IAClF,OAAO,0BAA0B,CAAC,MAAM,qBAAqB,CAAC,WAAW,CAAC,EAAE,QAAQ,CAAC,CAAC;AACxF,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,MAAiC;IACzE,MAAM,KAAK,GAAG,CAAC,4CAA4C,GAAG,MAAM,CAAC,UAAU,EAAE,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,SAAS,GAAG,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IAC3J,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;IAC9B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QACrC,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,WAAW,GAAG,KAAK,GAAG,IAAI,CAAC,QAAQ,GAAG,GAAG,GAAG,IAAI,CAAC,KAAK,EAAE,WAAW,GAAG,IAAI,CAAC,aAAa,GAAG,KAAK,GAAG,IAAI,CAAC,YAAY,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,GAAG,GAAG,EAAE,eAAe,GAAG,IAAI,CAAC,gBAAgB,GAAG,MAAM,GAAG,IAAI,CAAC,eAAe,GAAG,GAAG,EAAE,cAAc,GAAG,IAAI,CAAC,qBAAqB,GAAG,wBAAwB,GAAG,IAAI,CAAC,kBAAkB,GAAG,GAAG,CAAC,CAAC;QACnY,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,WAAW;YAAE,KAAK,CAAC,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC,CAAC;QAC3E,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ;YAAE,KAAK,CAAC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC,CAAC;IACvE,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAAC,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,wBAAwB,CAAC,CAAC;QAAC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAAC,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,GAAG,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,KAAK,GAAG,IAAI,CAAC,aAAa,GAAG,KAAK,GAAG,IAAI,CAAC,YAAY,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,GAAG,WAAW,GAAG,IAAI,CAAC,gBAAgB,GAAG,MAAM,GAAG,IAAI,CAAC,eAAe,GAAG,uBAAuB,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC;YAAC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,WAAW;gBAAE,KAAK,CAAC,IAAI,CAAC,kBAAkB,GAAG,KAAK,CAAC,CAAC;YAAC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ;gBAAE,KAAK,CAAC,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC,CAAC;QAAC,CAAC;IAAC,CAAC;IAC/jB,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,4HAA4H,CAAC,CAAC;IAC7I,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
export type ChangeRiskLevel = "high" | "elevated" | "moderate" | "low";
|
|
2
|
+
export interface ChangeRiskItem {
|
|
3
|
+
file: string;
|
|
4
|
+
status: string;
|
|
5
|
+
score: number;
|
|
6
|
+
level: ChangeRiskLevel;
|
|
7
|
+
structuralScore: number;
|
|
8
|
+
downstreamCount: number;
|
|
9
|
+
affectedModules: string[];
|
|
10
|
+
reasons: string[];
|
|
11
|
+
recommendedTests: Array<{
|
|
12
|
+
file: string;
|
|
13
|
+
reason: string;
|
|
14
|
+
exists: boolean;
|
|
15
|
+
}>;
|
|
16
|
+
}
|
|
17
|
+
export interface ChangeIntelligenceReport {
|
|
18
|
+
root: string;
|
|
19
|
+
generatedAt: string;
|
|
20
|
+
branch: string;
|
|
21
|
+
base: string;
|
|
22
|
+
summary: {
|
|
23
|
+
changedFiles: number;
|
|
24
|
+
high: number;
|
|
25
|
+
elevated: number;
|
|
26
|
+
moderate: number;
|
|
27
|
+
low: number;
|
|
28
|
+
affectedModules: number;
|
|
29
|
+
testCandidates: number;
|
|
30
|
+
};
|
|
31
|
+
overallRisk: {
|
|
32
|
+
score: number;
|
|
33
|
+
level: ChangeRiskLevel;
|
|
34
|
+
reasons: string[];
|
|
35
|
+
};
|
|
36
|
+
changes: ChangeRiskItem[];
|
|
37
|
+
warnings: string[];
|
|
38
|
+
}
|
|
39
|
+
export declare function analyzeChangeIntelligence(inputPath: string, base?: string): Promise<ChangeIntelligenceReport>;
|
|
40
|
+
export declare function formatChangeIntelligence(report: ChangeIntelligenceReport): string;
|