@tangle-network/agent-eval 0.144.11 → 0.144.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/dist/agent-profile-B9_GGsG8.d.ts +84 -0
- package/dist/agent-profile-B9_GGsG8.d.ts.map +1 -0
- package/dist/{agent-profile-cell-BOP-iA9Q.d.ts → agent-profile-cell-BkcRDikH.d.ts} +2 -2
- package/dist/{agent-profile-cell-BOP-iA9Q.d.ts.map → agent-profile-cell-BkcRDikH.d.ts.map} +1 -1
- package/dist/analyst/index.d.ts +134 -16
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +364 -10
- package/dist/analyst/index.js.map +1 -1
- package/dist/backend-integrity-CsVin_Wb.d.ts +280 -0
- package/dist/backend-integrity-CsVin_Wb.d.ts.map +1 -0
- package/dist/{baseline-C-GocmIW.js → baseline-BhPRQBVn.js} +3 -2
- package/dist/{baseline-C-GocmIW.js.map → baseline-BhPRQBVn.js.map} +1 -1
- package/dist/{benchmark-CWeqGl7x.js → benchmark-BhT16ep9.js} +2 -2
- package/dist/{benchmark-CWeqGl7x.js.map → benchmark-BhT16ep9.js.map} +1 -1
- package/dist/{agent-profile-DPi7IZg7.d.ts → benchmark-Ceoan7vk.d.ts} +4 -91
- package/dist/benchmark-Ceoan7vk.d.ts.map +1 -0
- package/dist/{benchmark-command-BteMFN62.js → benchmark-command-CY6cbhOB.js} +36 -79
- package/dist/benchmark-command-CY6cbhOB.js.map +1 -0
- package/dist/benchmarks/index.d.ts +244 -2
- package/dist/benchmarks/index.d.ts.map +1 -0
- package/dist/benchmarks/index.js +733 -1
- package/dist/benchmarks/index.js.map +1 -0
- package/dist/builder-eval/index.d.ts +23 -2
- package/dist/builder-eval/index.d.ts.map +1 -1
- package/dist/builder-eval/index.js +227 -3
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/campaign/index.d.ts +10 -8
- package/dist/campaign/index.js +9 -6
- package/dist/{campaign-C2TTzQII.js → campaign-BYjBAypg.js} +21 -634
- package/dist/campaign-BYjBAypg.js.map +1 -0
- package/dist/{canonical-D011XM8r.js → canonical-D-XsTQ6_.js} +2 -2
- package/dist/{canonical-D011XM8r.js.map → canonical-D-XsTQ6_.js.map} +1 -1
- package/dist/{index-CQTZ-4XN.d.ts → capture-fetch-DDvpjVRU.d.ts} +3 -3
- package/dist/capture-fetch-DDvpjVRU.d.ts.map +1 -0
- package/dist/{default-registry-BmktKy8r.js → chat-client-Bp4Ebuuc.js} +1276 -1276
- package/dist/chat-client-Bp4Ebuuc.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-C9gzZE59.d.ts → client-KUbGulm_.d.ts} +4 -4
- package/dist/{client-C9gzZE59.d.ts.map → client-KUbGulm_.d.ts.map} +1 -1
- package/dist/contract/index.d.ts +13 -390
- package/dist/contract/index.d.ts.map +1 -1
- package/dist/contract/index.js +18 -542
- package/dist/contract/index.js.map +1 -1
- package/dist/{cost-ledger-DMFxsLKr.js → cost-ledger-BSe92yAV.js} +3 -3
- package/dist/{cost-ledger-DMFxsLKr.js.map → cost-ledger-BSe92yAV.js.map} +1 -1
- package/dist/{cost-ledger-Bv_e8XHY.d.ts → cost-ledger-DbQdN3nO.d.ts} +2 -2
- package/dist/{cost-ledger-Bv_e8XHY.d.ts.map → cost-ledger-DbQdN3nO.d.ts.map} +1 -1
- package/dist/{counterfactual-CxmxAONP.d.ts → counterfactual--bpysZF0.d.ts} +2 -14
- package/dist/{counterfactual-CxmxAONP.d.ts.map → counterfactual--bpysZF0.d.ts.map} +1 -1
- package/dist/{counterfactual-CWPTrMH7.js → counterfactual-lDfCx0Uz.js} +3 -31
- package/dist/{counterfactual-CWPTrMH7.js.map → counterfactual-lDfCx0Uz.js.map} +1 -1
- package/dist/{dataset-C8xaLXdY.d.ts → dataset-CJjKqQfA.d.ts} +2 -8
- package/dist/dataset-CJjKqQfA.d.ts.map +1 -0
- package/dist/{default-registry-Bf8Woqmq.d.ts → default-registry-Di6HP6pG.d.ts} +7 -12
- package/dist/default-registry-Di6HP6pG.d.ts.map +1 -0
- package/dist/{analyze-runs-C30yljDJ.js → define-agent-eval-iqjT--ZZ.js} +542 -130
- package/dist/define-agent-eval-iqjT--ZZ.js.map +1 -0
- package/dist/define-agent-eval-sH24zBfM.d.ts +388 -0
- package/dist/define-agent-eval-sH24zBfM.d.ts.map +1 -0
- package/dist/descriptive-B5MwKfbf.js +144 -0
- package/dist/descriptive-B5MwKfbf.js.map +1 -0
- package/dist/{dspy-rlm-engine-DbTk4JdR.js → dspy-rlm-engine-B_qhSc21.js} +4 -4
- package/dist/{dspy-rlm-engine-DbTk4JdR.js.map → dspy-rlm-engine-B_qhSc21.js.map} +1 -1
- package/dist/effect-sizes-DiH8MGOH.js +82 -0
- package/dist/effect-sizes-DiH8MGOH.js.map +1 -0
- package/dist/{engine-3hL-XqwJ.d.ts → engine-otFpE2gF.d.ts} +10 -38
- package/dist/engine-otFpE2gF.d.ts.map +1 -0
- package/dist/{errors-CKPfb2aH.d.ts → errors-DEE6u6ot.d.ts} +2 -14
- package/dist/{errors-CKPfb2aH.d.ts.map → errors-DEE6u6ot.d.ts.map} +1 -1
- package/dist/{errors-D-LKuDhb.js → errors-Dngq5h35.js} +2 -8
- package/dist/{errors-D-LKuDhb.js.map → errors-Dngq5h35.js.map} +1 -1
- package/dist/{eval-campaign-B_7wcnav.js → eval-campaign-UB-usSQ2.js} +6 -6
- package/dist/{eval-campaign-B_7wcnav.js.map → eval-campaign-UB-usSQ2.js.map} +1 -1
- package/dist/{exact-types-DSFFpLLI.d.ts → exact-types-BH1twmAJ.d.ts} +2 -2
- package/dist/{exact-types-DSFFpLLI.d.ts.map → exact-types-BH1twmAJ.d.ts.map} +1 -1
- package/dist/experiment/index.d.ts +9 -6
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +11 -7
- package/dist/experiment/index.js.map +1 -1
- package/dist/experiment-tracker-C29gXM4B.js +269 -0
- package/dist/experiment-tracker-C29gXM4B.js.map +1 -0
- package/dist/{experiment-tracker-IMntXr6J.d.ts → experiment-tracker-DWHZBAYL.d.ts} +77 -121
- package/dist/experiment-tracker-DWHZBAYL.d.ts.map +1 -0
- package/dist/{external-optimizer-contracts-CZuJNcT5.d.ts → external-optimizer-contracts-lixrOZdX.d.ts} +3 -3
- package/dist/{external-optimizer-contracts-CZuJNcT5.d.ts.map → external-optimizer-contracts-lixrOZdX.d.ts.map} +1 -1
- package/dist/external-optimizer-process-BTiNB-RH.js +301 -0
- package/dist/external-optimizer-process-BTiNB-RH.js.map +1 -0
- package/dist/{single-run-lock-DFWHEB09.js → external-optimizer-subprocess-DrJ9hR8u.js} +144 -438
- package/dist/external-optimizer-subprocess-DrJ9hR8u.js.map +1 -0
- package/dist/extract-usage-BrQ8mCLX.js +155 -0
- package/dist/extract-usage-BrQ8mCLX.js.map +1 -0
- package/dist/{failure-cluster-CqcvCcdR.d.ts → failure-cluster-BLURuWG4.d.ts} +2 -3
- package/dist/failure-cluster-BLURuWG4.d.ts.map +1 -0
- package/dist/{feedback-trajectory-CacHpxsp.d.ts → feedback-trajectory-juOozjAc.d.ts} +5 -36
- package/dist/feedback-trajectory-juOozjAc.d.ts.map +1 -0
- package/dist/fuzz.d.ts +2 -2
- package/dist/fuzz.js +2 -2
- package/dist/hosted/index.d.ts +3 -3
- package/dist/{index-CvSN3IG1.d.ts → index-B8Ui1mr1.d.ts} +2 -2
- package/dist/{index-CvSN3IG1.d.ts.map → index-B8Ui1mr1.d.ts.map} +1 -1
- package/dist/{skill-usage-CO9OLRBx.d.ts → index-BWDrSVfw.d.ts} +11 -136
- package/dist/index-BWDrSVfw.d.ts.map +1 -0
- package/dist/index-Ba3YrbAL.d.ts +1 -0
- package/dist/{index-BnEuDAK2.d.ts → index-COQYtuRF.d.ts} +3 -3
- package/dist/{index-BnEuDAK2.d.ts.map → index-COQYtuRF.d.ts.map} +1 -1
- package/dist/{index-C3ssXVLv.d.ts → index-CvjYbU0D.d.ts} +2 -2
- package/dist/{index-C3ssXVLv.d.ts.map → index-CvjYbU0D.d.ts.map} +1 -1
- package/dist/{index-DPPGNJ_R.d.ts → index-DSmEylT9.d.ts} +17 -115
- package/dist/index-DSmEylT9.d.ts.map +1 -0
- package/dist/index.d.ts +2397 -5308
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5914 -10496
- package/dist/index.js.map +1 -1
- package/dist/{insight-report-CgX_s0Ez.d.ts → insight-report-CRi-Ufrj.d.ts} +4 -4
- package/dist/{insight-report-CgX_s0Ez.d.ts.map → insight-report-CRi-Ufrj.d.ts.map} +1 -1
- package/dist/{integrity-MLzHOfV9.js → integrity-Cy9WHAtb.js} +2 -2
- package/dist/{integrity-MLzHOfV9.js.map → integrity-Cy9WHAtb.js.map} +1 -1
- package/dist/{integrity-DY6tIbl0.js → integrity-DysDBWDu.js} +2 -2
- package/dist/{integrity-DY6tIbl0.js.map → integrity-DysDBWDu.js.map} +1 -1
- package/dist/{integrity-BuqEKu-x.d.ts → integrity-OrcI9Nau.d.ts} +3 -3
- package/dist/{integrity-BuqEKu-x.d.ts.map → integrity-OrcI9Nau.d.ts.map} +1 -1
- package/dist/internal-BDHPCnjk.js +230 -0
- package/dist/internal-BDHPCnjk.js.map +1 -0
- package/dist/judge-calibration-C5CbMYce.d.ts +117 -0
- package/dist/judge-calibration-C5CbMYce.d.ts.map +1 -0
- package/dist/judge-calibration-DZkWrm5H.js +317 -0
- package/dist/judge-calibration-DZkWrm5H.js.map +1 -0
- package/dist/{kind-factory-B8-r8-y8.js → kind-factory-CPmSd58s.js} +208 -208
- package/dist/{kind-factory-B8-r8-y8.js.map → kind-factory-CPmSd58s.js.map} +1 -1
- package/dist/ledger-core/index.d.ts +1 -1
- package/dist/ledger-core/index.js +2 -2
- package/dist/{ledger-core-DXZIqu17.js → ledger-core-BmZt19oQ.js} +110 -110
- package/dist/{ledger-core-DXZIqu17.js.map → ledger-core-BmZt19oQ.js.map} +1 -1
- package/dist/{llm-client-DzvMUsS_.js → llm-client-d0-2TT1g.js} +4 -64
- package/dist/{llm-client-DzvMUsS_.js.map → llm-client-d0-2TT1g.js.map} +1 -1
- package/dist/{skillopt-optimization-method-CQdVeM8k.js → llm-judge-dZ8P6nGI.js} +3349 -5401
- package/dist/llm-judge-dZ8P6nGI.js.map +1 -0
- package/dist/matrix/index.d.ts +2 -2
- package/dist/meta-eval/index.d.ts +3 -3
- package/dist/meta-eval/index.js +3 -3
- package/dist/{metrics-C9YY1OcL.js → metrics-Cl0L1KUy.js} +2 -108
- package/dist/{metrics-C9YY1OcL.js.map → metrics-Cl0L1KUy.js.map} +1 -1
- package/dist/{mint-B2O60ACG.js → mint-Dj9Ww_3I.js} +4 -4
- package/dist/{mint-B2O60ACG.js.map → mint-Dj9Ww_3I.js.map} +1 -1
- package/dist/{multi-layer-verifier-DnAqwl0h.d.ts → multi-layer-verifier-DIguZc8Z.d.ts} +3 -3
- package/dist/{multi-layer-verifier-DnAqwl0h.d.ts.map → multi-layer-verifier-DIguZc8Z.d.ts.map} +1 -1
- package/dist/multiplicity-DIWHvysC.d.ts +43 -0
- package/dist/multiplicity-DIWHvysC.d.ts.map +1 -0
- package/dist/multishot/index.d.ts +3 -3
- package/dist/multishot/index.js +1 -1
- package/dist/openapi.json +1 -1
- package/dist/{opencode-sqlite-8r6WUfHc.js → opencode-sqlite-DJWAXLms.js} +2 -2
- package/dist/{opencode-sqlite-8r6WUfHc.js.map → opencode-sqlite-DJWAXLms.js.map} +1 -1
- package/dist/package-version-D7lQHt_-.js +34 -0
- package/dist/package-version-D7lQHt_-.js.map +1 -0
- package/dist/paired-arms-D-XRF_fy.js +1045 -0
- package/dist/paired-arms-D-XRF_fy.js.map +1 -0
- package/dist/paired-promotion-decision-CGzg0cI_.d.ts +251 -0
- package/dist/paired-promotion-decision-CGzg0cI_.d.ts.map +1 -0
- package/dist/paired-tests-BHIhYVdu.js +213 -0
- package/dist/paired-tests-BHIhYVdu.js.map +1 -0
- package/dist/pareto-BqNW3LJR.d.ts +117 -0
- package/dist/pareto-BqNW3LJR.d.ts.map +1 -0
- package/dist/pipelines/index.d.ts +3 -64
- package/dist/pipelines/index.d.ts.map +1 -1
- package/dist/pipelines/index.js +4 -284
- package/dist/pipelines/index.js.map +1 -1
- package/dist/power-and-mde-CHIrXJll.js +195 -0
- package/dist/power-and-mde-CHIrXJll.js.map +1 -0
- package/dist/{promotion-policy-CrLrmys8.js → power-preflight-DEw-uC7q.js} +4 -184
- package/dist/power-preflight-DEw-uC7q.js.map +1 -0
- package/dist/pre-registration-CZwSFQS4.d.ts +577 -0
- package/dist/pre-registration-CZwSFQS4.d.ts.map +1 -0
- package/dist/{prime-protocol-BfSalTfR.js → prime-protocol-6tZTVsWm.js} +72 -21
- package/dist/prime-protocol-6tZTVsWm.js.map +1 -0
- package/dist/produced-state-DU79a81m.js +586 -0
- package/dist/produced-state-DU79a81m.js.map +1 -0
- package/dist/profile-cell.d.ts +1 -1
- package/dist/profile-cell.js +1 -1
- package/dist/promotion-policy-D0nPhkSy.d.ts +134 -0
- package/dist/promotion-policy-D0nPhkSy.d.ts.map +1 -0
- package/dist/promotion-policy-xzA40Evo.js +186 -0
- package/dist/promotion-policy-xzA40Evo.js.map +1 -0
- package/dist/{skillopt-optimization-method-USDKhxSA.d.ts → provenance-CCdxgLDT.d.ts} +57 -478
- package/dist/provenance-CCdxgLDT.d.ts.map +1 -0
- package/dist/registry-oJeeI4-a.d.ts +178 -0
- package/dist/registry-oJeeI4-a.d.ts.map +1 -0
- package/dist/{release-report-DA2BCu5a.d.ts → release-confidence-BFRE5WSp.d.ts} +180 -112
- package/dist/release-confidence-BFRE5WSp.d.ts.map +1 -0
- package/dist/{release-report-BUYmoKo2.js → release-confidence-CxDuiAev.js} +125 -217
- package/dist/release-confidence-CxDuiAev.js.map +1 -0
- package/dist/reporting.d.ts +6 -5
- package/dist/reporting.js +7 -5
- package/dist/{researcher-BchpD55R.d.ts → researcher-DaN4GST-.d.ts} +7 -40
- package/dist/{researcher-BchpD55R.d.ts.map → researcher-DaN4GST-.d.ts.map} +1 -1
- package/dist/{reward-hacking-BDToousL.js → reward-hacking-DNgjilrV.js} +3 -3
- package/dist/reward-hacking-DNgjilrV.js.map +1 -0
- package/dist/{reward-hacking-BI0OMAlo.d.ts → reward-hacking-DSSTuI9r.d.ts} +5 -5
- package/dist/{reward-hacking-BI0OMAlo.d.ts.map → reward-hacking-DSSTuI9r.d.ts.map} +1 -1
- package/dist/rl.d.ts +7 -7
- package/dist/rl.js +11 -10
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +2 -2
- package/dist/rollout/index.js +4 -4
- package/dist/{rollout-DRrksrcV.js → rollout-BWtw0I_6.js} +3 -3
- package/dist/{rollout-DRrksrcV.js.map → rollout-BWtw0I_6.js.map} +1 -1
- package/dist/{rubric-predictive-validity-DvkjPYCe.d.ts → rubric-predictive-validity-BgxtKe4G.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-DvkjPYCe.d.ts.map → rubric-predictive-validity-BgxtKe4G.d.ts.map} +1 -1
- package/dist/{rubric-predictive-validity-BRR632r1.js → rubric-predictive-validity-Cwwyd7ah.js} +2 -2
- package/dist/{rubric-predictive-validity-BRR632r1.js.map → rubric-predictive-validity-Cwwyd7ah.js.map} +1 -1
- package/dist/{run-record-BmSPWXJR.js → run-record-BvHPVS-i.js} +3 -3
- package/dist/{run-record-BmSPWXJR.js.map → run-record-BvHPVS-i.js.map} +1 -1
- package/dist/{run-record-CF4Dwpxr.d.ts → run-record-CKiihE6f.d.ts} +4 -4
- package/dist/{run-record-CF4Dwpxr.d.ts.map → run-record-CKiihE6f.d.ts.map} +1 -1
- package/dist/{proposal-findings-2GIUo1et.js → run-score-lDzV0X8j.js} +31 -31
- package/dist/run-score-lDzV0X8j.js.map +1 -0
- package/dist/sandbox-harness-BlSOu4LX.d.ts +70 -0
- package/dist/sandbox-harness-BlSOu4LX.d.ts.map +1 -0
- package/dist/{schema-Cef2cFmb2.d.ts → schema-Cef2cFmb.d.ts} +1 -1
- package/dist/schema-Cef2cFmb.d.ts.map +1 -0
- package/dist/semantic-concept-judge-BiJxScqe.js +406 -0
- package/dist/semantic-concept-judge-BiJxScqe.js.map +1 -0
- package/dist/{sequential-D-BLJBKU.js → sequential-C458DXNf.js} +4 -3
- package/dist/{sequential-D-BLJBKU.js.map → sequential-C458DXNf.js.map} +1 -1
- package/dist/sequential-eprocess-CbUt2htw.js +83 -0
- package/dist/sequential-eprocess-CbUt2htw.js.map +1 -0
- package/dist/series-convergence-D1cL1f-4.d.ts +129 -0
- package/dist/series-convergence-D1cL1f-4.d.ts.map +1 -0
- package/dist/{server-iu0ede49.js → server-ulsOdrTI.js} +5 -21
- package/dist/server-ulsOdrTI.js.map +1 -0
- package/dist/skillopt-optimization-method-C5cotF4E.d.ts +427 -0
- package/dist/skillopt-optimization-method-C5cotF4E.d.ts.map +1 -0
- package/dist/skillopt-optimization-method-jjdnc3YK.js +1999 -0
- package/dist/skillopt-optimization-method-jjdnc3YK.js.map +1 -0
- package/dist/{statistical-heldout-TQ-4CYiN.d.ts → statistical-heldout-DoFF5KJX.d.ts} +5 -278
- package/dist/statistical-heldout-DoFF5KJX.d.ts.map +1 -0
- package/dist/{store-otlp-Dw8PPIlL.js → store-otlp-CsptLYpN.js} +3 -3
- package/dist/{store-otlp-Dw8PPIlL.js.map → store-otlp-CsptLYpN.js.map} +1 -1
- package/dist/store-tool-spans-Cq9mFd-q.js +667 -0
- package/dist/store-tool-spans-Cq9mFd-q.js.map +1 -0
- package/dist/store-tool-spans-DbDLOBOb.d.ts +342 -0
- package/dist/store-tool-spans-DbDLOBOb.d.ts.map +1 -0
- package/dist/student-t-CvBq2mve.js +38 -0
- package/dist/student-t-CvBq2mve.js.map +1 -0
- package/dist/{summary-report-Lf-5I7xh.js → summary-report-Blysd6Z2.js} +6 -3
- package/dist/{summary-report-Lf-5I7xh.js.map → summary-report-Blysd6Z2.js.map} +1 -1
- package/dist/{summary-report-BNR7DWTj.d.ts → summary-report-CaL-Hnxt.d.ts} +5 -5
- package/dist/{summary-report-BNR7DWTj.d.ts.map → summary-report-CaL-Hnxt.d.ts.map} +1 -1
- package/dist/supervisor-run/index.d.ts +391 -3
- package/dist/supervisor-run/index.d.ts.map +1 -0
- package/dist/supervisor-run/index.js +1689 -2
- package/dist/{supervisor-run-D_sokXcO.js.map → supervisor-run/index.js.map} +1 -1
- package/dist/{extract-usage-CdZdoj1s.js → task-failure-attributes-CpQ4y5RD.js} +5 -157
- package/dist/task-failure-attributes-CpQ4y5RD.js.map +1 -0
- package/dist/{tool-groups-CZPGGlHf.d.ts → tool-groups-Cteb03Ps.d.ts} +3 -3
- package/dist/tool-groups-Cteb03Ps.d.ts.map +1 -0
- package/dist/tool-waste-BDdBZG1F.js +803 -0
- package/dist/tool-waste-BDdBZG1F.js.map +1 -0
- package/dist/tool-waste-DjRDEsuI.d.ts +128 -0
- package/dist/tool-waste-DjRDEsuI.d.ts.map +1 -0
- package/dist/trace-repair/index.d.ts +14 -5
- package/dist/trace-repair/index.d.ts.map +1 -1
- package/dist/trace-repair/index.js +35 -7
- package/dist/trace-repair/index.js.map +1 -1
- package/dist/traces.d.ts +406 -7
- package/dist/traces.d.ts.map +1 -0
- package/dist/traces.js +1011 -10
- package/dist/traces.js.map +1 -0
- package/dist/trajectory-replay/index.d.ts +16 -3
- package/dist/trajectory-replay/index.d.ts.map +1 -1
- package/dist/trajectory-replay/index.js +52 -5
- package/dist/trajectory-replay/index.js.map +1 -1
- package/dist/types-BEPZc6eo.d.ts +93 -0
- package/dist/types-BEPZc6eo.d.ts.map +1 -0
- package/dist/{usage-receipt-t7vAzCRQ.js → types-BI4fT3HN.js} +67 -67
- package/dist/types-BI4fT3HN.js.map +1 -0
- package/dist/{types-BnjdJ70P.d.ts → types-BZ59Ahr8.d.ts} +5 -5
- package/dist/{types-BnjdJ70P.d.ts.map → types-BZ59Ahr8.d.ts.map} +1 -1
- package/dist/{types-D216SgwM.d.ts → types-Cx3YUh2r.d.ts} +4 -241
- package/dist/types-Cx3YUh2r.d.ts.map +1 -0
- package/dist/{types-BvDKaULh.d.ts → types-DLQx4mKU.d.ts} +4 -4
- package/dist/{types-BvDKaULh.d.ts.map → types-DLQx4mKU.d.ts.map} +1 -1
- package/dist/{types-D4mog56g.d.ts → types-yLK8gXE9.d.ts} +2 -2
- package/dist/{types-D4mog56g.d.ts.map → types-yLK8gXE9.d.ts.map} +1 -1
- package/dist/verdict-BndeTAh_.js +61 -0
- package/dist/verdict-BndeTAh_.js.map +1 -0
- package/dist/verdict-E4eRNf7-.d.ts +392 -0
- package/dist/verdict-E4eRNf7-.d.ts.map +1 -0
- package/dist/{verdict-cache-BCcOh0kF.js → verdict-cache-mZf5FEiY.js} +3 -55
- package/dist/{verdict-cache-BCcOh0kF.js.map → verdict-cache-mZf5FEiY.js.map} +1 -1
- package/dist/wire/index.d.ts +3 -3
- package/dist/wire/index.d.ts.map +1 -1
- package/dist/wire/index.js +1 -1
- package/docs/charter.md +3 -3
- package/docs/control-runtime.md +3 -42
- package/docs/experiment.md +0 -1
- package/docs/feature-guide.md +2 -2
- package/docs/trace-repair-grader.md +1 -0
- package/docs/trajectory-replay.md +1 -0
- package/docs/verdicts.md +43 -0
- package/docs/verification-strategies.md +3 -2
- package/package.json +6 -11
- package/dist/agent-profile-DPi7IZg7.d.ts.map +0 -1
- package/dist/analyze-runs-C30yljDJ.js.map +0 -1
- package/dist/baseline-CavEbRyH.d.ts +0 -136
- package/dist/baseline-CavEbRyH.d.ts.map +0 -1
- package/dist/benchmark-command-BteMFN62.js.map +0 -1
- package/dist/benchmarks-Dzs8CKb1.js +0 -755
- package/dist/benchmarks-Dzs8CKb1.js.map +0 -1
- package/dist/campaign-C2TTzQII.js.map +0 -1
- package/dist/completion-verifier-DJA5BhPb.d.ts +0 -414
- package/dist/completion-verifier-DJA5BhPb.d.ts.map +0 -1
- package/dist/control.d.ts +0 -3
- package/dist/control.js +0 -2
- package/dist/dataset-C8xaLXdY.d.ts.map +0 -1
- package/dist/default-registry-Bf8Woqmq.d.ts.map +0 -1
- package/dist/default-registry-BmktKy8r.js.map +0 -1
- package/dist/engine-3hL-XqwJ.d.ts.map +0 -1
- package/dist/experiment-tracker-CnRICnMl.js +0 -500
- package/dist/experiment-tracker-CnRICnMl.js.map +0 -1
- package/dist/experiment-tracker-IMntXr6J.d.ts.map +0 -1
- package/dist/extract-usage-CdZdoj1s.js.map +0 -1
- package/dist/failure-cluster-CqcvCcdR.d.ts.map +0 -1
- package/dist/feedback-trajectory-CacHpxsp.d.ts.map +0 -1
- package/dist/index-BZ3-y4YL.d.ts +0 -391
- package/dist/index-BZ3-y4YL.d.ts.map +0 -1
- package/dist/index-CQTZ-4XN.d.ts.map +0 -1
- package/dist/index-DPPGNJ_R.d.ts.map +0 -1
- package/dist/index-YE4KdKbO2.d.ts +0 -335
- package/dist/index-YE4KdKbO2.d.ts.map +0 -1
- package/dist/paired-arms-iZ08VFMN.js +0 -260
- package/dist/paired-arms-iZ08VFMN.js.map +0 -1
- package/dist/paired-promotion-decision-B6zJ3gYM.d.ts +0 -114
- package/dist/paired-promotion-decision-B6zJ3gYM.d.ts.map +0 -1
- package/dist/prime-protocol-BfSalTfR.js.map +0 -1
- package/dist/promotion-policy-Ckjhzg_4.d.ts +0 -289
- package/dist/promotion-policy-Ckjhzg_4.d.ts.map +0 -1
- package/dist/promotion-policy-CrLrmys8.js.map +0 -1
- package/dist/proposal-findings-2GIUo1et.js.map +0 -1
- package/dist/propose-review-control-dSNPjFUH.js +0 -1458
- package/dist/propose-review-control-dSNPjFUH.js.map +0 -1
- package/dist/release-report-BUYmoKo2.js.map +0 -1
- package/dist/release-report-DA2BCu5a.d.ts.map +0 -1
- package/dist/replay-CohS93nE.js +0 -1859
- package/dist/replay-CohS93nE.js.map +0 -1
- package/dist/replay-DbhZ4Ked.d.ts +0 -834
- package/dist/replay-DbhZ4Ked.d.ts.map +0 -1
- package/dist/reward-hacking-BDToousL.js.map +0 -1
- package/dist/run-evidence-Dhi3C81V.d.ts +0 -225
- package/dist/run-evidence-Dhi3C81V.d.ts.map +0 -1
- package/dist/schema-Cef2cFmb2.d.ts.map +0 -1
- package/dist/semantic-concept-judge-D1z-KepS.js +0 -767
- package/dist/semantic-concept-judge-D1z-KepS.js.map +0 -1
- package/dist/series-convergence-ofsqPWhs.d.ts +0 -35
- package/dist/series-convergence-ofsqPWhs.d.ts.map +0 -1
- package/dist/server-iu0ede49.js.map +0 -1
- package/dist/single-run-lock-DFWHEB09.js.map +0 -1
- package/dist/skill-usage-CO9OLRBx.d.ts.map +0 -1
- package/dist/skillopt-optimization-method-CQdVeM8k.js.map +0 -1
- package/dist/skillopt-optimization-method-USDKhxSA.d.ts.map +0 -1
- package/dist/statistical-heldout-TQ-4CYiN.d.ts.map +0 -1
- package/dist/statistics-ByxzSiOM.js +0 -2212
- package/dist/statistics-ByxzSiOM.js.map +0 -1
- package/dist/statistics-D6Uebe_4.d.ts +0 -968
- package/dist/statistics-D6Uebe_4.d.ts.map +0 -1
- package/dist/supervisor-run-D_sokXcO.js +0 -1690
- package/dist/test-graded-scenario-D1TaI2va.d.ts +0 -141
- package/dist/test-graded-scenario-D1TaI2va.d.ts.map +0 -1
- package/dist/test-graded-scenario-JHcKQNpq.js +0 -318
- package/dist/test-graded-scenario-JHcKQNpq.js.map +0 -1
- package/dist/tool-groups-CZPGGlHf.d.ts.map +0 -1
- package/dist/tool-use-metrics-DEGMKycK.js +0 -370
- package/dist/tool-use-metrics-DEGMKycK.js.map +0 -1
- package/dist/types-D216SgwM.d.ts.map +0 -1
- package/dist/usage-receipt-t7vAzCRQ.js.map +0 -1
- package/dist/verdict-DExhxfgR.d.ts +0 -201
- package/dist/verdict-DExhxfgR.d.ts.map +0 -1
package/dist/pipelines/index.js
CHANGED
|
@@ -1,123 +1,9 @@
|
|
|
1
|
-
import { t as
|
|
2
|
-
import {
|
|
1
|
+
import { a as budgetBreachView, i as failureClusterView, n as computeToolUseMetrics, r as judgeAgreementView, t as toolWasteView } from "../tool-waste-BDdBZG1F.js";
|
|
2
|
+
import { t as compareToBaseline } from "../baseline-BhPRQBVn.js";
|
|
3
3
|
import { s as isToolSpan } from "../schema-CRhEY1SO.js";
|
|
4
|
+
import { i as hasCapturedToolArgs, n as argHash, o as llmSpans, s as runFailureClass, t as aggregateLlm } from "../query-Di7eEQ79.js";
|
|
4
5
|
import { t as buildTrajectory } from "../trajectory-D_7rLrvE.js";
|
|
5
|
-
import { t as
|
|
6
|
-
import { i as hasCapturedToolArgs, l as toolSpans, n as argHash, o as llmSpans, s as runFailureClass, t as aggregateLlm } from "../query-Di7eEQ79.js";
|
|
7
|
-
import { n as DEFAULT_RULES, r as classifyFailure, t as computeToolUseMetrics } from "../tool-use-metrics-DEGMKycK.js";
|
|
8
|
-
//#region src/pipelines/budget-breach.ts
|
|
9
|
-
async function budgetBreachView(store, options = {}) {
|
|
10
|
-
const runs = await store.listRuns({
|
|
11
|
-
scenarioId: options.scenarioId,
|
|
12
|
-
variantId: options.variantId
|
|
13
|
-
});
|
|
14
|
-
const findings = [];
|
|
15
|
-
const byDimension = {};
|
|
16
|
-
const byScenario = {};
|
|
17
|
-
const byVariant = {};
|
|
18
|
-
for (const run of runs) {
|
|
19
|
-
const entries = await store.budget(run.runId);
|
|
20
|
-
for (const e of entries) {
|
|
21
|
-
if (!e.breached) continue;
|
|
22
|
-
const excessRatio = e.limit > 0 ? e.consumed / e.limit : Infinity;
|
|
23
|
-
findings.push({
|
|
24
|
-
runId: run.runId,
|
|
25
|
-
scenarioId: run.scenarioId,
|
|
26
|
-
variantId: run.variantId,
|
|
27
|
-
dimension: e.dimension,
|
|
28
|
-
limit: e.limit,
|
|
29
|
-
consumed: e.consumed,
|
|
30
|
-
excessRatio,
|
|
31
|
-
timestamp: e.timestamp
|
|
32
|
-
});
|
|
33
|
-
byDimension[e.dimension] = (byDimension[e.dimension] ?? 0) + 1;
|
|
34
|
-
byScenario[run.scenarioId] = (byScenario[run.scenarioId] ?? 0) + 1;
|
|
35
|
-
if (run.variantId) byVariant[run.variantId] = (byVariant[run.variantId] ?? 0) + 1;
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
const breachedRuns = new Set(findings.map((f) => f.runId));
|
|
39
|
-
return {
|
|
40
|
-
findings,
|
|
41
|
-
byDimension,
|
|
42
|
-
byScenario,
|
|
43
|
-
byVariant,
|
|
44
|
-
totalRuns: runs.length,
|
|
45
|
-
breachedRunRatio: runs.length > 0 ? breachedRuns.size / runs.length : 0
|
|
46
|
-
};
|
|
47
|
-
}
|
|
48
|
-
//#endregion
|
|
49
|
-
//#region src/pipelines/failure-cluster.ts
|
|
50
|
-
/**
|
|
51
|
-
* FailureClusterView — groups failed runs by (failureClass, triggerTool,
|
|
52
|
-
* argHash-prefix) so weekly reviews can prioritize the top-N clusters.
|
|
53
|
-
*
|
|
54
|
-
* Each cluster includes: N runs, scenarios affected, representative
|
|
55
|
-
* error message, a proposed mitigation hint (rule → action table).
|
|
56
|
-
*/
|
|
57
|
-
async function failureClusterView(store, options = {}) {
|
|
58
|
-
const rules = options.rules ?? DEFAULT_RULES;
|
|
59
|
-
const minSize = options.minClusterSize ?? 1;
|
|
60
|
-
const runs = await store.listRuns();
|
|
61
|
-
const clusters = /* @__PURE__ */ new Map();
|
|
62
|
-
let totalFailures = 0;
|
|
63
|
-
for (const run of runs) {
|
|
64
|
-
if (run.status === "completed" && run.outcome?.pass !== false) continue;
|
|
65
|
-
totalFailures++;
|
|
66
|
-
const spans = await store.spans({ runId: run.runId });
|
|
67
|
-
const cls = classifyFailure({
|
|
68
|
-
run,
|
|
69
|
-
spans,
|
|
70
|
-
events: await store.events({ runId: run.runId })
|
|
71
|
-
}, rules);
|
|
72
|
-
let toolName;
|
|
73
|
-
let argPrefix;
|
|
74
|
-
let dimension;
|
|
75
|
-
if (cls.triggerSpanId) {
|
|
76
|
-
const trig = spans.find((s) => s.spanId === cls.triggerSpanId);
|
|
77
|
-
if (trig?.kind === "tool") {
|
|
78
|
-
toolName = trig.toolName;
|
|
79
|
-
if (hasCapturedToolArgs(trig)) argPrefix = argHash(trig.args).slice(0, 16);
|
|
80
|
-
} else if (trig?.kind === "judge") dimension = trig.dimension;
|
|
81
|
-
}
|
|
82
|
-
if (!toolName) {
|
|
83
|
-
const errored = (await toolSpans(store, run.runId)).filter((t) => t.status === "error").pop();
|
|
84
|
-
if (errored) {
|
|
85
|
-
toolName = errored.toolName;
|
|
86
|
-
if (hasCapturedToolArgs(errored)) argPrefix = argHash(errored.args).slice(0, 16);
|
|
87
|
-
}
|
|
88
|
-
}
|
|
89
|
-
if (!dimension) {
|
|
90
|
-
const judge = spans.find((s) => s.kind === "judge" && typeof s.dimension === "string");
|
|
91
|
-
if (judge?.kind === "judge") dimension = judge.dimension;
|
|
92
|
-
}
|
|
93
|
-
const key = `${cls.failureClass}|${toolName ?? ""}|${argPrefix ?? ""}|${dimension ?? ""}`;
|
|
94
|
-
let cluster = clusters.get(key);
|
|
95
|
-
if (!cluster) {
|
|
96
|
-
cluster = {
|
|
97
|
-
failureClass: cls.failureClass,
|
|
98
|
-
toolName,
|
|
99
|
-
argPrefix,
|
|
100
|
-
dimension,
|
|
101
|
-
runCount: 0,
|
|
102
|
-
scenarioIds: [],
|
|
103
|
-
exampleRunId: run.runId,
|
|
104
|
-
exampleError: firstErrorMessage(spans) ?? cls.reason
|
|
105
|
-
};
|
|
106
|
-
clusters.set(key, cluster);
|
|
107
|
-
}
|
|
108
|
-
cluster.runCount++;
|
|
109
|
-
if (!cluster.scenarioIds.includes(run.scenarioId)) cluster.scenarioIds.push(run.scenarioId);
|
|
110
|
-
}
|
|
111
|
-
return {
|
|
112
|
-
clusters: [...clusters.values()].filter((c) => c.runCount >= minSize).sort((a, b) => b.runCount - a.runCount),
|
|
113
|
-
totalFailures,
|
|
114
|
-
totalRuns: runs.length
|
|
115
|
-
};
|
|
116
|
-
}
|
|
117
|
-
function firstErrorMessage(spans) {
|
|
118
|
-
return spans.find((s) => s.status === "error")?.error;
|
|
119
|
-
}
|
|
120
|
-
//#endregion
|
|
6
|
+
import { t as executionTrackByLane } from "../execution-tracks-CpgFPpS5.js";
|
|
121
7
|
//#region src/pipelines/first-divergence.ts
|
|
122
8
|
async function firstDivergenceView(store, runA, runB, options = {}) {
|
|
123
9
|
const [a, b] = await Promise.all([buildTrajectory(store, runA), buildTrajectory(store, runB)]);
|
|
@@ -168,79 +54,6 @@ function describeDifference(a, b) {
|
|
|
168
54
|
return `name "${a.span.name}" vs "${b.span.name}"`;
|
|
169
55
|
}
|
|
170
56
|
//#endregion
|
|
171
|
-
//#region src/pipelines/judge-agreement.ts
|
|
172
|
-
/**
|
|
173
|
-
* JudgeAgreementView — pairwise agreement between judges across the
|
|
174
|
-
* corpus, grouped by dimension.
|
|
175
|
-
*
|
|
176
|
-
* Output drives two workflows:
|
|
177
|
-
* - Judge robustness audit: "does Claude agree with GPT at κ ≥ 0.6?"
|
|
178
|
-
* - Calibration tracking: κ vs golden human labels over time (by
|
|
179
|
-
* providing a `humanGoldenJudgeId`).
|
|
180
|
-
*/
|
|
181
|
-
async function judgeAgreementView(store) {
|
|
182
|
-
const all = (await store.spans({ kind: "judge" })).filter((s) => s.kind === "judge");
|
|
183
|
-
if (all.length === 0) return {
|
|
184
|
-
pairs: [],
|
|
185
|
-
dimensions: [],
|
|
186
|
-
judgeIds: []
|
|
187
|
-
};
|
|
188
|
-
const byDimension = /* @__PURE__ */ new Map();
|
|
189
|
-
for (const s of all) {
|
|
190
|
-
const arr = byDimension.get(s.dimension) ?? [];
|
|
191
|
-
arr.push(s);
|
|
192
|
-
byDimension.set(s.dimension, arr);
|
|
193
|
-
}
|
|
194
|
-
const judgeIds = [...new Set(all.map((s) => s.judgeId))].sort();
|
|
195
|
-
const pairs = [];
|
|
196
|
-
for (const [dim, spans] of byDimension) {
|
|
197
|
-
const byJudge = /* @__PURE__ */ new Map();
|
|
198
|
-
for (const s of spans) {
|
|
199
|
-
const m = byJudge.get(s.judgeId) ?? /* @__PURE__ */ new Map();
|
|
200
|
-
m.set(s.targetSpanId, s.score);
|
|
201
|
-
byJudge.set(s.judgeId, m);
|
|
202
|
-
}
|
|
203
|
-
const judgesHere = [...byJudge.keys()];
|
|
204
|
-
for (let i = 0; i < judgesHere.length; i++) for (let j = i + 1; j < judgesHere.length; j++) {
|
|
205
|
-
const judgeI = judgesHere[i];
|
|
206
|
-
const judgeJ = judgesHere[j];
|
|
207
|
-
const a = byJudge.get(judgeI);
|
|
208
|
-
const b = byJudge.get(judgeJ);
|
|
209
|
-
const common = [];
|
|
210
|
-
for (const [target, scoreA] of a) {
|
|
211
|
-
const scoreB = b.get(target);
|
|
212
|
-
if (scoreB !== void 0) common.push([scoreA, scoreB]);
|
|
213
|
-
}
|
|
214
|
-
if (common.length < 2) continue;
|
|
215
|
-
const judgeScores = common.map(([scoreA, scoreB]) => [{
|
|
216
|
-
judgeName: judgeI,
|
|
217
|
-
dimension: dim,
|
|
218
|
-
score: scoreA,
|
|
219
|
-
reasoning: ""
|
|
220
|
-
}, {
|
|
221
|
-
judgeName: judgeJ,
|
|
222
|
-
dimension: dim,
|
|
223
|
-
score: scoreB,
|
|
224
|
-
reasoning: ""
|
|
225
|
-
}]);
|
|
226
|
-
const k = interRaterReliability(judgeScores[0].map((_, k2) => judgeScores.map((pair) => pair[k2])));
|
|
227
|
-
pairs.push({
|
|
228
|
-
judgeA: judgeI,
|
|
229
|
-
judgeB: judgeJ,
|
|
230
|
-
dimension: dim,
|
|
231
|
-
commonItems: common.length,
|
|
232
|
-
pearson: pearsonR(common.map((c) => c[0]), common.map((c) => c[1])),
|
|
233
|
-
krippendorff: k
|
|
234
|
-
});
|
|
235
|
-
}
|
|
236
|
-
}
|
|
237
|
-
return {
|
|
238
|
-
pairs: pairs.sort((a, b) => b.commonItems - a.commonItems),
|
|
239
|
-
dimensions: [...byDimension.keys()].sort(),
|
|
240
|
-
judgeIds
|
|
241
|
-
};
|
|
242
|
-
}
|
|
243
|
-
//#endregion
|
|
244
57
|
//#region src/pipelines/regression.ts
|
|
245
58
|
/**
|
|
246
59
|
* RegressionView — compares a candidate slice to a baseline slice on a
|
|
@@ -439,99 +252,6 @@ function callsAreSerial(previous, next) {
|
|
|
439
252
|
return previous.span.endedAt !== void 0 && previous.span.endedAt <= next.span.startedAt;
|
|
440
253
|
}
|
|
441
254
|
//#endregion
|
|
442
|
-
//#region src/pipelines/tool-waste.ts
|
|
443
|
-
/**
|
|
444
|
-
* ToolWasteView — fraction of tool calls whose results weren't used
|
|
445
|
-
* downstream. Without a "used" signal we fall back to structural
|
|
446
|
-
* proxies: error calls, duplicate calls, and tool calls followed by
|
|
447
|
-
* zero subsequent LLM spans are all considered waste.
|
|
448
|
-
*
|
|
449
|
-
* Consumers can pass a `usageOracle` that inspects a tool span and
|
|
450
|
-
* returns true iff the tool's result appears in a later LLM message,
|
|
451
|
-
* artifact, or state mutation — that's the canonical definition; the
|
|
452
|
-
* default heuristic is a reasonable fallback.
|
|
453
|
-
*/
|
|
454
|
-
async function toolWasteView(store, options = {}) {
|
|
455
|
-
const runs = options.runId ? [options.runId] : (await store.listRuns()).map((r) => r.runId);
|
|
456
|
-
const byRun = [];
|
|
457
|
-
let totalCalls = 0;
|
|
458
|
-
let totalWasted = 0;
|
|
459
|
-
for (const runId of runs) {
|
|
460
|
-
const tools = await toolSpans(store, runId);
|
|
461
|
-
if (tools.length === 0) {
|
|
462
|
-
byRun.push({
|
|
463
|
-
runId,
|
|
464
|
-
wastedCalls: 0,
|
|
465
|
-
totalCalls: 0,
|
|
466
|
-
wasteRate: 0
|
|
467
|
-
});
|
|
468
|
-
continue;
|
|
469
|
-
}
|
|
470
|
-
const sortedLlm = [...await llmSpans(store, runId)].sort((a, b) => a.startedAt - b.startedAt);
|
|
471
|
-
const startTimes = sortedLlm.map((l) => l.startedAt);
|
|
472
|
-
const suffixText = buildSuffixText(sortedLlm);
|
|
473
|
-
let wasted = 0;
|
|
474
|
-
for (const t of tools) {
|
|
475
|
-
if (t.status === "error") {
|
|
476
|
-
wasted++;
|
|
477
|
-
continue;
|
|
478
|
-
}
|
|
479
|
-
const cutoff = upperBound(startTimes, t.startedAt);
|
|
480
|
-
if (options.usageOracle) {
|
|
481
|
-
if (!options.usageOracle(t, { llm: sortedLlm.slice(cutoff) })) wasted++;
|
|
482
|
-
} else {
|
|
483
|
-
const resultStr = stringify(t.result);
|
|
484
|
-
if (resultStr === "") continue;
|
|
485
|
-
if (!(suffixText[cutoff] ?? "").includes(resultStr.slice(0, 120))) wasted++;
|
|
486
|
-
}
|
|
487
|
-
}
|
|
488
|
-
const wasteRate = wasted / tools.length;
|
|
489
|
-
byRun.push({
|
|
490
|
-
runId,
|
|
491
|
-
wastedCalls: wasted,
|
|
492
|
-
totalCalls: tools.length,
|
|
493
|
-
wasteRate
|
|
494
|
-
});
|
|
495
|
-
totalCalls += tools.length;
|
|
496
|
-
totalWasted += wasted;
|
|
497
|
-
}
|
|
498
|
-
return {
|
|
499
|
-
byRun,
|
|
500
|
-
overallWasteRate: totalCalls > 0 ? totalWasted / totalCalls : 0
|
|
501
|
-
};
|
|
502
|
-
}
|
|
503
|
-
/**
|
|
504
|
-
* Build per-position suffix haystacks: result[i] is the concatenation of every
|
|
505
|
-
* string message content in spans[i..end]. Built back-to-front so each entry
|
|
506
|
-
* reuses the next one — O(total message text) rather than O(spans²).
|
|
507
|
-
*/
|
|
508
|
-
function buildSuffixText(spans) {
|
|
509
|
-
const result = new Array(spans.length + 1);
|
|
510
|
-
result[spans.length] = "";
|
|
511
|
-
for (let i = spans.length - 1; i >= 0; i--) result[i] = `${spans[i].messages.map((m) => typeof m.content === "string" ? m.content : "").join("\n")}\n${result[i + 1]}`;
|
|
512
|
-
return result;
|
|
513
|
-
}
|
|
514
|
-
/** Index of the first element strictly greater than `target` in a sorted array. */
|
|
515
|
-
function upperBound(sorted, target) {
|
|
516
|
-
let lo = 0;
|
|
517
|
-
let hi = sorted.length;
|
|
518
|
-
while (lo < hi) {
|
|
519
|
-
const mid = lo + hi >>> 1;
|
|
520
|
-
if (sorted[mid] <= target) lo = mid + 1;
|
|
521
|
-
else hi = mid;
|
|
522
|
-
}
|
|
523
|
-
return lo;
|
|
524
|
-
}
|
|
525
|
-
function stringify(v) {
|
|
526
|
-
if (v === null || v === void 0) return "";
|
|
527
|
-
if (typeof v === "string") return v;
|
|
528
|
-
try {
|
|
529
|
-
return JSON.stringify(v);
|
|
530
|
-
} catch {
|
|
531
|
-
return String(v);
|
|
532
|
-
}
|
|
533
|
-
}
|
|
534
|
-
//#endregion
|
|
535
255
|
export { budgetBreachView, computeToolUseMetrics, failureClusterView, firstDivergenceView, judgeAgreementView, regressionView, stuckLoopView, toolWasteView };
|
|
536
256
|
|
|
537
257
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","names":[],"sources":["../../src/pipelines/budget-breach.ts","../../src/pipelines/failure-cluster.ts","../../src/pipelines/first-divergence.ts","../../src/pipelines/judge-agreement.ts","../../src/pipelines/regression.ts","../../src/pipelines/stuck-loop.ts","../../src/pipelines/tool-waste.ts"],"sourcesContent":["/**\n * BudgetBreachView — aggregates breach events across the corpus.\n *\n * Answers: which dimensions get hit most often? Which scenarios are\n * underbudgeted? Which variants trigger the most breaches?\n */\n\nimport type { BudgetSpec } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\n\nexport interface BudgetBreachFinding {\n runId: string\n scenarioId: string\n variantId?: string\n dimension: keyof BudgetSpec\n limit: number\n consumed: number\n excessRatio: number\n timestamp: number\n}\n\nexport interface BudgetBreachReport {\n findings: BudgetBreachFinding[]\n byDimension: Record<string, number>\n byScenario: Record<string, number>\n byVariant: Record<string, number>\n totalRuns: number\n breachedRunRatio: number\n}\n\nexport async function budgetBreachView(\n store: TraceStore,\n options: { scenarioId?: string; variantId?: string } = {},\n): Promise<BudgetBreachReport> {\n const runs = await store.listRuns({\n scenarioId: options.scenarioId,\n variantId: options.variantId,\n })\n const findings: BudgetBreachFinding[] = []\n const byDimension: Record<string, number> = {}\n const byScenario: Record<string, number> = {}\n const byVariant: Record<string, number> = {}\n\n for (const run of runs) {\n const entries = await store.budget(run.runId)\n for (const e of entries) {\n if (!e.breached) continue\n const excessRatio = e.limit > 0 ? e.consumed / e.limit : Infinity\n findings.push({\n runId: run.runId,\n scenarioId: run.scenarioId,\n variantId: run.variantId,\n dimension: e.dimension,\n limit: e.limit,\n consumed: e.consumed,\n excessRatio,\n timestamp: e.timestamp,\n })\n byDimension[e.dimension] = (byDimension[e.dimension] ?? 0) + 1\n byScenario[run.scenarioId] = (byScenario[run.scenarioId] ?? 0) + 1\n if (run.variantId) byVariant[run.variantId] = (byVariant[run.variantId] ?? 0) + 1\n }\n }\n\n const breachedRuns = new Set(findings.map((f) => f.runId))\n return {\n findings,\n byDimension,\n byScenario,\n byVariant,\n totalRuns: runs.length,\n breachedRunRatio: runs.length > 0 ? breachedRuns.size / runs.length : 0,\n }\n}\n","/**\n * FailureClusterView — groups failed runs by (failureClass, triggerTool,\n * argHash-prefix) so weekly reviews can prioritize the top-N clusters.\n *\n * Each cluster includes: N runs, scenarios affected, representative\n * error message, a proposed mitigation hint (rule → action table).\n */\n\nimport { classifyFailure, DEFAULT_RULES, type FailureRule } from '../failure-taxonomy'\nimport { argHash, hasCapturedToolArgs, toolSpans } from '../trace/query'\nimport type { FailureClass, Span } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\n\nexport interface FailureCluster {\n failureClass: FailureClass\n /** Tool name when the trigger was a tool span, else undefined. */\n toolName?: string\n /** First 16 chars of argHash — clusters similar args. */\n argPrefix?: string\n /**\n * Source dimension when the trigger was a judge span (e.g. `'format'`,\n * `'safety'`, `'correctness'`). Lets cross-template aggregators\n * group failures by the dimension that fired without overloading\n * `argPrefix`. Optional — clusters without this field deserialize cleanly.\n */\n dimension?: string\n runCount: number\n scenarioIds: string[]\n exampleError?: string\n exampleRunId: string\n}\n\nexport interface FailureClusterReport {\n clusters: FailureCluster[]\n totalFailures: number\n totalRuns: number\n}\n\nexport async function failureClusterView(\n store: TraceStore,\n options: { rules?: FailureRule[]; minClusterSize?: number } = {},\n): Promise<FailureClusterReport> {\n const rules = options.rules ?? DEFAULT_RULES\n const minSize = options.minClusterSize ?? 1\n const runs = await store.listRuns()\n\n type Key = string\n const clusters = new Map<Key, FailureCluster>()\n let totalFailures = 0\n\n for (const run of runs) {\n if (run.status === 'completed' && run.outcome?.pass !== false) continue\n totalFailures++\n const spans = await store.spans({ runId: run.runId })\n const events = await store.events({ runId: run.runId })\n const cls = classifyFailure({ run, spans, events }, rules)\n\n let toolName: string | undefined\n let argPrefix: string | undefined\n let dimension: string | undefined\n if (cls.triggerSpanId) {\n const trig = spans.find((s) => s.spanId === cls.triggerSpanId)\n if (trig?.kind === 'tool') {\n toolName = trig.toolName\n if (hasCapturedToolArgs(trig)) argPrefix = argHash(trig.args).slice(0, 16)\n } else if (trig?.kind === 'judge') {\n dimension = trig.dimension\n }\n }\n // Fallback: look at the last errored tool span\n if (!toolName) {\n const ts = await toolSpans(store, run.runId)\n const errored = ts.filter((t) => t.status === 'error').pop()\n if (errored) {\n toolName = errored.toolName\n if (hasCapturedToolArgs(errored)) argPrefix = argHash(errored.args).slice(0, 16)\n }\n }\n // Secondary signal: any judge span on the failed run carries a\n // dimension. Useful when the rule classified by judge score but\n // didn't surface the trigger span (or surfaced a non-judge span).\n if (!dimension) {\n const judge = spans.find((s) => s.kind === 'judge' && typeof s.dimension === 'string')\n if (judge?.kind === 'judge') dimension = judge.dimension\n }\n\n const key = `${cls.failureClass}|${toolName ?? ''}|${argPrefix ?? ''}|${dimension ?? ''}`\n let cluster = clusters.get(key)\n if (!cluster) {\n cluster = {\n failureClass: cls.failureClass,\n toolName,\n argPrefix,\n dimension,\n runCount: 0,\n scenarioIds: [],\n exampleRunId: run.runId,\n exampleError: firstErrorMessage(spans) ?? cls.reason,\n }\n clusters.set(key, cluster)\n }\n cluster.runCount++\n if (!cluster.scenarioIds.includes(run.scenarioId)) cluster.scenarioIds.push(run.scenarioId)\n }\n\n const arr = [...clusters.values()]\n .filter((c) => c.runCount >= minSize)\n .sort((a, b) => b.runCount - a.runCount)\n\n return { clusters: arr, totalFailures, totalRuns: runs.length }\n}\n\nfunction firstErrorMessage(spans: Span[]): string | undefined {\n const errored = spans.find((s) => s.status === 'error')\n return errored?.error\n}\n","/**\n * FirstDivergenceView — aligns two trajectories by step index, reports\n * the first step where they differ.\n *\n * \"Differ\" is configurable — default is (kind, toolName if tool, model\n * if llm). Use this view to attribute \"why is variant B better?\" to a\n * specific step rather than an aggregate mean delta.\n */\n\nimport type { TraceStore } from '../trace/store'\nimport { buildTrajectory, type Trajectory, type TrajectoryStep } from '../trajectory'\n\nexport interface DivergenceReport {\n runA: string\n runB: string\n firstDivergenceIndex: number | null\n aStep?: TrajectoryStep\n bStep?: TrajectoryStep\n reason?: string\n /** Common prefix length (steps that matched). */\n commonPrefixLen: number\n}\n\nexport interface DivergenceOptions {\n /** Returns true if two steps are considered equal. Default: kind + tool/model match. */\n stepEquals?: (a: TrajectoryStep, b: TrajectoryStep) => boolean\n}\n\nexport async function firstDivergenceView(\n store: TraceStore,\n runA: string,\n runB: string,\n options: DivergenceOptions = {},\n): Promise<DivergenceReport> {\n const [a, b] = await Promise.all([buildTrajectory(store, runA), buildTrajectory(store, runB)])\n const eq = options.stepEquals ?? defaultStepEquals\n const minLen = Math.min(a.steps.length, b.steps.length)\n for (let i = 0; i < minLen; i++) {\n const aStep = a.steps[i]!\n const bStep = b.steps[i]!\n if (!eq(aStep, bStep)) {\n return {\n runA,\n runB,\n firstDivergenceIndex: i,\n aStep,\n bStep,\n reason: describeDifference(aStep, bStep),\n commonPrefixLen: i,\n }\n }\n }\n if (a.steps.length === b.steps.length) {\n return { runA, runB, firstDivergenceIndex: null, commonPrefixLen: minLen }\n }\n const longer: Trajectory = a.steps.length > b.steps.length ? a : b\n const extra = longer.steps.length - minLen\n // minLen === 0 means one trajectory is empty: divergence is the first step\n // itself (index 0), and the absent side has no step to surface.\n const reason =\n minLen === 0\n ? `one trajectory is empty; the other has ${extra} step(s) starting at index 0`\n : `one trajectory has ${extra} more step(s) after index ${minLen - 1}`\n return {\n runA,\n runB,\n firstDivergenceIndex: minLen,\n aStep: a.steps[minLen],\n bStep: b.steps[minLen],\n reason,\n commonPrefixLen: minLen,\n }\n}\n\nfunction defaultStepEquals(a: TrajectoryStep, b: TrajectoryStep): boolean {\n if (a.span.kind !== b.span.kind) return false\n if (a.span.kind === 'tool' && b.span.kind === 'tool') return a.span.toolName === b.span.toolName\n if (a.span.kind === 'llm' && b.span.kind === 'llm') return a.span.model === b.span.model\n if (a.span.kind === 'judge' && b.span.kind === 'judge')\n return a.span.dimension === b.span.dimension\n return a.span.name === b.span.name\n}\n\nfunction describeDifference(a: TrajectoryStep, b: TrajectoryStep): string {\n if (a.span.kind !== b.span.kind) return `kind ${a.span.kind} vs ${b.span.kind}`\n if (a.span.kind === 'tool' && b.span.kind === 'tool' && a.span.toolName !== b.span.toolName) {\n return `tool ${a.span.toolName} vs ${b.span.toolName}`\n }\n if (a.span.kind === 'llm' && b.span.kind === 'llm' && a.span.model !== b.span.model) {\n return `model ${a.span.model} vs ${b.span.model}`\n }\n return `name \"${a.span.name}\" vs \"${b.span.name}\"`\n}\n","/**\n * JudgeAgreementView — pairwise agreement between judges across the\n * corpus, grouped by dimension.\n *\n * Output drives two workflows:\n * - Judge robustness audit: \"does Claude agree with GPT at κ ≥ 0.6?\"\n * - Calibration tracking: κ vs golden human labels over time (by\n * providing a `humanGoldenJudgeId`).\n */\n\nimport { interRaterReliability, pearsonR } from '../statistics'\nimport type { JudgeSpan } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\n\nexport interface JudgePair {\n judgeA: string\n judgeB: string\n dimension: string\n /** Number of (targetSpanId, dimension) tuples both judges scored. */\n commonItems: number\n pearson: number\n krippendorff: number\n}\n\nexport interface JudgeAgreementReport {\n pairs: JudgePair[]\n dimensions: string[]\n judgeIds: string[]\n}\n\nexport async function judgeAgreementView(store: TraceStore): Promise<JudgeAgreementReport> {\n const all = (await store.spans({ kind: 'judge' })).filter(\n (s): s is JudgeSpan => s.kind === 'judge',\n )\n if (all.length === 0) return { pairs: [], dimensions: [], judgeIds: [] }\n\n const byDimension = new Map<string, JudgeSpan[]>()\n for (const s of all) {\n const arr = byDimension.get(s.dimension) ?? []\n arr.push(s)\n byDimension.set(s.dimension, arr)\n }\n\n const judgeIds = [...new Set(all.map((s) => s.judgeId))].sort()\n const pairs: JudgePair[] = []\n for (const [dim, spans] of byDimension) {\n const byJudge = new Map<string, Map<string, number>>()\n for (const s of spans) {\n const m = byJudge.get(s.judgeId) ?? new Map<string, number>()\n m.set(s.targetSpanId, s.score)\n byJudge.set(s.judgeId, m)\n }\n const judgesHere = [...byJudge.keys()]\n for (let i = 0; i < judgesHere.length; i++) {\n for (let j = i + 1; j < judgesHere.length; j++) {\n const judgeI = judgesHere[i]!\n const judgeJ = judgesHere[j]!\n const a = byJudge.get(judgeI)!\n const b = byJudge.get(judgeJ)!\n const common: Array<[number, number]> = []\n for (const [target, scoreA] of a) {\n const scoreB = b.get(target)\n if (scoreB !== undefined) common.push([scoreA, scoreB])\n }\n if (common.length < 2) continue\n const judgeScores = common.map(\n ([scoreA, scoreB]) =>\n [\n { judgeName: judgeI, dimension: dim, score: scoreA, reasoning: '' },\n { judgeName: judgeJ, dimension: dim, score: scoreB, reasoning: '' },\n ] as const,\n )\n const k = interRaterReliability(\n judgeScores[0]!.map((_, k2) => judgeScores.map((pair) => pair[k2]!)),\n )\n pairs.push({\n judgeA: judgeI,\n judgeB: judgeJ,\n dimension: dim,\n commonItems: common.length,\n pearson: pearsonR(\n common.map((c) => c[0]),\n common.map((c) => c[1]),\n ),\n krippendorff: k,\n })\n }\n }\n }\n\n return {\n pairs: pairs.sort((a, b) => b.commonItems - a.commonItems),\n dimensions: [...byDimension.keys()].sort(),\n judgeIds,\n }\n}\n","/**\n * RegressionView — compares a candidate slice to a baseline slice on a\n * named metric. Delegates the statistics (Welch's t-test, Cohen's d,\n * IQR stability) to `baseline.ts`.\n *\n * This is the entry point for CI regression gates: \"given runs tagged\n * release=A and release=B, did any metric regress?\"\n */\n\nimport { type BaselineOptions, type BaselineReport, compareToBaseline } from '../baseline'\nimport { aggregateLlm, llmSpans, runFailureClass } from '../trace/query'\nimport type { Run } from '../trace/schema'\nimport type { RunFilter, TraceStore } from '../trace/store'\n\nexport interface RegressionSpec {\n metric: string\n higherIsBetter: boolean\n /** Extract a scalar from a run. Default extractors handle common metrics. */\n extract?: (run: Run, store: TraceStore) => Promise<number | null>\n}\n\nexport interface RegressionOptions extends BaselineOptions {\n baseline: RunFilter\n candidate: RunFilter\n}\n\nexport async function regressionView(\n store: TraceStore,\n metrics: RegressionSpec[],\n options: RegressionOptions,\n): Promise<BaselineReport> {\n const baselineRuns = await store.listRuns(options.baseline)\n const candidateRuns = await store.listRuns(options.candidate)\n const samples = await Promise.all(\n metrics.map(async (m) => {\n const extract = m.extract ?? defaultExtract(m.metric)\n const baseline = await extractAll(baselineRuns, extract, store)\n const candidate = await extractAll(candidateRuns, extract, store)\n return { metric: m.metric, higherIsBetter: m.higherIsBetter, baseline, candidate }\n }),\n )\n return compareToBaseline(samples, options)\n}\n\nasync function extractAll(\n runs: Run[],\n extract: (r: Run, s: TraceStore) => Promise<number | null>,\n store: TraceStore,\n): Promise<number[]> {\n const out: number[] = []\n for (const r of runs) {\n const v = await extract(r, store)\n if (v !== null && Number.isFinite(v)) out.push(v)\n }\n return out\n}\n\nfunction defaultExtract(metric: string): (run: Run, store: TraceStore) => Promise<number | null> {\n return async (run, store) => {\n switch (metric) {\n case 'score':\n case 'overallScore':\n return run.outcome?.score ?? null\n case 'pass':\n return run.outcome?.pass === true ? 1 : 0\n case 'durationMs':\n return run.endedAt && run.startedAt ? run.endedAt - run.startedAt : null\n case 'costUsd': {\n const llm = await llmSpans(store, run.runId)\n return aggregateLlm(llm).costUsd\n }\n case 'inputTokens': {\n const llm = await llmSpans(store, run.runId)\n return aggregateLlm(llm).inputTokens\n }\n case 'outputTokens': {\n const llm = await llmSpans(store, run.runId)\n return aggregateLlm(llm).outputTokens\n }\n case 'failureClass': {\n return runFailureClass(run) === 'success' ? 1 : 0\n }\n default:\n return null\n }\n }\n}\n","/**\n * StuckLoopView — detects when an agent calls the same tool with the\n * same (or structurally similar) arguments ≥ N times in a short window.\n *\n * Rationale: agents that loop are the number-one production failure\n * mode on long-horizon flows. The view returns (runId, toolName,\n * argHash, occurrences, windowMs) for each detected loop plus a\n * fraction of runs affected.\n */\n\nimport { executionTrackByLane } from '../trace/execution-tracks'\nimport { argHash, hasCapturedToolArgs } from '../trace/query'\nimport { isToolSpan, type Span, type ToolSpan } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\n\nconst DEFAULT_MAX_WINDOW_MS = 60_000\nconst DEFAULT_MAX_INTERVENING_TOOL_CALLS = 0\n\ninterface ScopedToolCall {\n span: ToolSpan\n scopeSpanId: string | null\n laneSpanId: string | null\n}\n\ninterface IndexedToolCall extends ScopedToolCall {\n toolCallIndex: number\n trackId: string\n}\n\nexport interface StuckLoopFinding {\n runId: string\n toolName: string\n argHash: string\n /** Calls in this episode's densest qualifying interval, not the whole-run total. */\n occurrences: number\n spanIds: string[]\n /** Nearest agent ancestor, or the direct parent when ancestry is incomplete. */\n scopeSpanId?: string\n /** Milliseconds between first and last call in the loop. */\n windowMs: number\n}\n\nexport interface StuckLoopReport {\n findings: StuckLoopFinding[]\n affectedRunRatio: number\n totalRuns: number\n}\n\nexport interface StuckLoopOptions {\n /** Minimum call count to flag a loop (default 3). */\n minOccurrences?: number\n /** Maximum time between the first and last repeated call (default 60 seconds). */\n maxWindowMs?: number\n /**\n * Maximum other tool calls allowed between adjacent repeats (default 0).\n * Set to 1 to detect alternating patterns such as A,B,A,B,A.\n */\n maxInterveningToolCalls?: number\n /** Filter to a specific runId; omit to scan the entire corpus. */\n runId?: string\n}\n\nexport async function stuckLoopView(\n store: TraceStore,\n options: StuckLoopOptions = {},\n): Promise<StuckLoopReport> {\n const minOccurrences = options.minOccurrences ?? 3\n const maxWindowMs = options.maxWindowMs ?? DEFAULT_MAX_WINDOW_MS\n const maxInterveningToolCalls =\n options.maxInterveningToolCalls ?? DEFAULT_MAX_INTERVENING_TOOL_CALLS\n if (!Number.isInteger(minOccurrences) || minOccurrences < 1) {\n throw new RangeError('minOccurrences must be a positive integer')\n }\n if (!Number.isFinite(maxWindowMs) || maxWindowMs < 0) {\n throw new RangeError('maxWindowMs must be a finite non-negative number')\n }\n if (!Number.isInteger(maxInterveningToolCalls) || maxInterveningToolCalls < 0) {\n throw new RangeError('maxInterveningToolCalls must be a non-negative integer')\n }\n const runs = options.runId\n ? [{ runId: options.runId }]\n : (await store.listRuns()).map((r) => ({ runId: r.runId }))\n\n const findings: StuckLoopFinding[] = []\n for (const { runId } of runs) {\n const spans = await store.spans({ runId })\n const spansById = new Map(spans.map((span) => [span.spanId, span]))\n const scopedTools: ScopedToolCall[] = spans\n .filter(isToolSpan)\n .map((span, sourceIndex) => ({ span, sourceIndex }))\n .sort((a, b) => a.span.startedAt - b.span.startedAt || a.sourceIndex - b.sourceIndex)\n .map(({ span }) => ({ span, ...executionScope(span, spansById) }))\n const trackByLane = executionTrackByLane(\n scopedTools.map((call) => {\n const direct = call.laneSpanId === null || call.laneSpanId === call.scopeSpanId\n const timed = direct\n ? call.span\n : call.laneSpanId\n ? spansById.get(call.laneSpanId)\n : undefined\n return {\n key: executionKey(call),\n scopeKey: JSON.stringify(call.scopeSpanId),\n start: timed?.startedAt ?? null,\n end: timed?.endedAt ?? null,\n }\n }),\n )\n const nextToolIndexByTrack = new Map<string, number>()\n const orderedTools: IndexedToolCall[] = scopedTools.map((call) => {\n const trackId = trackByLane.get(executionKey(call))!\n const toolCallIndex = nextToolIndexByTrack.get(trackId) ?? 0\n nextToolIndexByTrack.set(trackId, toolCallIndex + 1)\n return { ...call, toolCallIndex, trackId }\n })\n const byKey = new Map<\n string,\n {\n calls: IndexedToolCall[]\n argHash: string\n toolName: string\n scopeSpanId: string | null\n }\n >()\n for (const call of orderedTools) {\n if (!hasCapturedToolArgs(call.span)) continue\n const h = argHash(call.span.args)\n const key = JSON.stringify([call.trackId, call.span.toolName, h])\n const bucket = byKey.get(key) ?? {\n calls: [],\n argHash: h,\n toolName: call.span.toolName,\n scopeSpanId: call.scopeSpanId,\n }\n bucket.calls.push(call)\n byKey.set(key, bucket)\n }\n for (const { calls, argHash: h, toolName, scopeSpanId } of byKey.values()) {\n if (calls.length < minOccurrences) continue\n let episodeStart = 0\n for (let episodeEnd = 1; episodeEnd <= calls.length; episodeEnd += 1) {\n const previous = calls[episodeEnd - 1]!\n const next = calls[episodeEnd]\n const episodeEnded =\n next === undefined ||\n next.span.startedAt - previous.span.startedAt > maxWindowMs ||\n next.toolCallIndex - previous.toolCallIndex - 1 > maxInterveningToolCalls ||\n !callsAreSerial(previous, next)\n if (!episodeEnded) continue\n\n const episode = calls.slice(episodeStart, episodeEnd)\n let left = 0\n let bestStart = 0\n let bestEnd = -1\n for (let right = 0; right < episode.length; right += 1) {\n while (episode[right]!.span.startedAt - episode[left]!.span.startedAt > maxWindowMs) {\n left += 1\n }\n if (right - left > bestEnd - bestStart) {\n bestStart = left\n bestEnd = right\n }\n }\n if (bestEnd - bestStart + 1 >= minOccurrences) {\n const loop = episode.slice(bestStart, bestEnd + 1)\n const first = loop[0]!.span.startedAt\n const last = loop[loop.length - 1]!.span.startedAt\n findings.push({\n runId,\n toolName,\n argHash: h,\n occurrences: loop.length,\n spanIds: loop.map((call) => call.span.spanId),\n ...(scopeSpanId ? { scopeSpanId } : {}),\n windowMs: last - first,\n })\n }\n episodeStart = episodeEnd\n }\n }\n }\n\n const affectedRuns = new Set(findings.map((f) => f.runId))\n return {\n findings,\n affectedRunRatio: runs.length > 0 ? affectedRuns.size / runs.length : 0,\n totalRuns: runs.length,\n }\n}\n\nfunction laneKey(call: Pick<ScopedToolCall, 'scopeSpanId' | 'laneSpanId'>): string {\n return JSON.stringify([call.scopeSpanId, call.laneSpanId])\n}\n\nfunction executionKey(call: ScopedToolCall): string {\n return call.laneSpanId === null || call.laneSpanId === call.scopeSpanId\n ? JSON.stringify([call.scopeSpanId, call.span.spanId])\n : laneKey(call)\n}\n\nfunction executionScope(\n span: ToolSpan,\n spansById: ReadonlyMap<string, Span>,\n): { scopeSpanId: string | null; laneSpanId: string | null } {\n const directParent = span.parentSpanId\n if (!directParent) return { scopeSpanId: null, laneSpanId: null }\n\n let currentId: string | undefined = directParent\n let laneSpanId: string | null = null\n const seen = new Set<string>()\n while (currentId && !seen.has(currentId)) {\n seen.add(currentId)\n const current = spansById.get(currentId)\n if (!current) return { scopeSpanId: directParent, laneSpanId: directParent }\n if (current.kind === 'agent') {\n return { scopeSpanId: current.spanId, laneSpanId: laneSpanId ?? current.spanId }\n }\n laneSpanId = current.spanId\n currentId = current.parentSpanId\n }\n return { scopeSpanId: directParent, laneSpanId: directParent }\n}\n\nfunction callsAreSerial(previous: IndexedToolCall, next: IndexedToolCall): boolean {\n return previous.span.endedAt !== undefined && previous.span.endedAt <= next.span.startedAt\n}\n","/**\n * ToolWasteView — fraction of tool calls whose results weren't used\n * downstream. Without a \"used\" signal we fall back to structural\n * proxies: error calls, duplicate calls, and tool calls followed by\n * zero subsequent LLM spans are all considered waste.\n *\n * Consumers can pass a `usageOracle` that inspects a tool span and\n * returns true iff the tool's result appears in a later LLM message,\n * artifact, or state mutation — that's the canonical definition; the\n * default heuristic is a reasonable fallback.\n */\n\nimport { computeToolUseMetrics } from '../tool-use-metrics'\nimport { llmSpans, toolSpans } from '../trace/query'\nimport type { LlmSpan, ToolSpan } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\n\nexport interface ToolWasteFinding {\n runId: string\n wastedCalls: number\n totalCalls: number\n wasteRate: number\n}\n\nexport interface ToolWasteReport {\n byRun: ToolWasteFinding[]\n overallWasteRate: number\n}\n\nexport interface ToolWasteOptions {\n runId?: string\n usageOracle?: (tool: ToolSpan, later: { llm: Awaited<ReturnType<typeof llmSpans>> }) => boolean\n}\n\nexport async function toolWasteView(\n store: TraceStore,\n options: ToolWasteOptions = {},\n): Promise<ToolWasteReport> {\n const runs = options.runId ? [options.runId] : (await store.listRuns()).map((r) => r.runId)\n\n const byRun: ToolWasteFinding[] = []\n let totalCalls = 0\n let totalWasted = 0\n for (const runId of runs) {\n const tools = await toolSpans(store, runId)\n if (tools.length === 0) {\n byRun.push({ runId, wastedCalls: 0, totalCalls: 0, wasteRate: 0 })\n continue\n }\n const llms = await llmSpans(store, runId)\n // Sort LLM spans once by start time, then build a suffix index of the\n // concatenated message text. `suffixText[i]` is the haystack of every\n // string message content in spans[i..]. Per tool we binary-search the\n // first span started strictly after the tool, then test that one suffix\n // — turning the per-tool O(llms × messages × content) scan into a single\n // O(log llms) lookup over precomputed text.\n const sortedLlm = [...llms].sort((a, b) => a.startedAt - b.startedAt)\n const startTimes = sortedLlm.map((l) => l.startedAt)\n const suffixText = buildSuffixText(sortedLlm)\n let wasted = 0\n for (const t of tools) {\n if (t.status === 'error') {\n wasted++\n continue\n }\n // First LLM span started strictly after this tool (upper-bound search).\n const cutoff = upperBound(startTimes, t.startedAt)\n if (options.usageOracle) {\n if (!options.usageOracle(t, { llm: sortedLlm.slice(cutoff) })) wasted++\n } else {\n // Default heuristic: a tool whose result is NOT mentioned in any\n // later LLM input message is likely wasted. An empty/null result has\n // no payload to propagate downstream — there is nothing to find in a\n // later message, so it is not evidence of waste; skip it.\n const resultStr = stringify(t.result)\n if (resultStr === '') continue\n const haystack = suffixText[cutoff] ?? ''\n const used = haystack.includes(resultStr.slice(0, 120))\n if (!used) wasted++\n }\n }\n const wasteRate = wasted / tools.length\n byRun.push({ runId, wastedCalls: wasted, totalCalls: tools.length, wasteRate })\n totalCalls += tools.length\n totalWasted += wasted\n }\n return { byRun, overallWasteRate: totalCalls > 0 ? totalWasted / totalCalls : 0 }\n}\n\n/**\n * Build per-position suffix haystacks: result[i] is the concatenation of every\n * string message content in spans[i..end]. Built back-to-front so each entry\n * reuses the next one — O(total message text) rather than O(spans²).\n */\nfunction buildSuffixText(spans: LlmSpan[]): string[] {\n const result = new Array<string>(spans.length + 1)\n result[spans.length] = ''\n for (let i = spans.length - 1; i >= 0; i--) {\n const own = spans[i]!.messages.map((m) =>\n typeof m.content === 'string' ? m.content : '',\n ).join('\\n')\n result[i] = `${own}\\n${result[i + 1]}`\n }\n return result\n}\n\n/** Index of the first element strictly greater than `target` in a sorted array. */\nfunction upperBound(sorted: number[], target: number): number {\n let lo = 0\n let hi = sorted.length\n while (lo < hi) {\n const mid = (lo + hi) >>> 1\n if (sorted[mid]! <= target) lo = mid + 1\n else hi = mid\n }\n return lo\n}\n\nfunction stringify(v: unknown): string {\n if (v === null || v === undefined) return ''\n if (typeof v === 'string') return v\n try {\n return JSON.stringify(v)\n } catch {\n return String(v)\n }\n}\n\n// Re-export for convenience in consumers that want both descriptive and usage metrics.\nexport { computeToolUseMetrics }\n"],"mappings":";;;;;;;;AA8BA,eAAsB,iBACpB,OACA,UAAuD,CAAC,GAC3B;CAC7B,MAAM,OAAO,MAAM,MAAM,SAAS;EAChC,YAAY,QAAQ;EACpB,WAAW,QAAQ;CACrB,CAAC;CACD,MAAM,WAAkC,CAAC;CACzC,MAAM,cAAsC,CAAC;CAC7C,MAAM,aAAqC,CAAC;CAC5C,MAAM,YAAoC,CAAC;CAE3C,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,UAAU,MAAM,MAAM,OAAO,IAAI,KAAK;EAC5C,KAAK,MAAM,KAAK,SAAS;GACvB,IAAI,CAAC,EAAE,UAAU;GACjB,MAAM,cAAc,EAAE,QAAQ,IAAI,EAAE,WAAW,EAAE,QAAQ;GACzD,SAAS,KAAK;IACZ,OAAO,IAAI;IACX,YAAY,IAAI;IAChB,WAAW,IAAI;IACf,WAAW,EAAE;IACb,OAAO,EAAE;IACT,UAAU,EAAE;IACZ;IACA,WAAW,EAAE;GACf,CAAC;GACD,YAAY,EAAE,cAAc,YAAY,EAAE,cAAc,KAAK;GAC7D,WAAW,IAAI,eAAe,WAAW,IAAI,eAAe,KAAK;GACjE,IAAI,IAAI,WAAW,UAAU,IAAI,cAAc,UAAU,IAAI,cAAc,KAAK;EAClF;CACF;CAEA,MAAM,eAAe,IAAI,IAAI,SAAS,KAAK,MAAM,EAAE,KAAK,CAAC;CACzD,OAAO;EACL;EACA;EACA;EACA;EACA,WAAW,KAAK;EAChB,kBAAkB,KAAK,SAAS,IAAI,aAAa,OAAO,KAAK,SAAS;CACxE;AACF;;;;;;;;;;ACnCA,eAAsB,mBACpB,OACA,UAA8D,CAAC,GAChC;CAC/B,MAAM,QAAQ,QAAQ,SAAS;CAC/B,MAAM,UAAU,QAAQ,kBAAkB;CAC1C,MAAM,OAAO,MAAM,MAAM,SAAS;CAGlC,MAAM,2BAAW,IAAI,IAAyB;CAC9C,IAAI,gBAAgB;CAEpB,KAAK,MAAM,OAAO,MAAM;EACtB,IAAI,IAAI,WAAW,eAAe,IAAI,SAAS,SAAS,OAAO;EAC/D;EACA,MAAM,QAAQ,MAAM,MAAM,MAAM,EAAE,OAAO,IAAI,MAAM,CAAC;EAEpD,MAAM,MAAM,gBAAgB;GAAE;GAAK;GAAO,QAAA,MADrB,MAAM,OAAO,EAAE,OAAO,IAAI,MAAM,CAAC;EACL,GAAG,KAAK;EAEzD,IAAI;EACJ,IAAI;EACJ,IAAI;EACJ,IAAI,IAAI,eAAe;GACrB,MAAM,OAAO,MAAM,MAAM,MAAM,EAAE,WAAW,IAAI,aAAa;GAC7D,IAAI,MAAM,SAAS,QAAQ;IACzB,WAAW,KAAK;IAChB,IAAI,oBAAoB,IAAI,GAAG,YAAY,QAAQ,KAAK,IAAI,CAAC,CAAC,MAAM,GAAG,EAAE;GAC3E,OAAO,IAAI,MAAM,SAAS,SACxB,YAAY,KAAK;EAErB;EAEA,IAAI,CAAC,UAAU;GAEb,MAAM,WAAU,MADC,UAAU,OAAO,IAAI,KAAK,EAAA,CACxB,QAAQ,MAAM,EAAE,WAAW,OAAO,CAAC,CAAC,IAAI;GAC3D,IAAI,SAAS;IACX,WAAW,QAAQ;IACnB,IAAI,oBAAoB,OAAO,GAAG,YAAY,QAAQ,QAAQ,IAAI,CAAC,CAAC,MAAM,GAAG,EAAE;GACjF;EACF;EAIA,IAAI,CAAC,WAAW;GACd,MAAM,QAAQ,MAAM,MAAM,MAAM,EAAE,SAAS,WAAW,OAAO,EAAE,cAAc,QAAQ;GACrF,IAAI,OAAO,SAAS,SAAS,YAAY,MAAM;EACjD;EAEA,MAAM,MAAM,GAAG,IAAI,aAAa,GAAG,YAAY,GAAG,GAAG,aAAa,GAAG,GAAG,aAAa;EACrF,IAAI,UAAU,SAAS,IAAI,GAAG;EAC9B,IAAI,CAAC,SAAS;GACZ,UAAU;IACR,cAAc,IAAI;IAClB;IACA;IACA;IACA,UAAU;IACV,aAAa,CAAC;IACd,cAAc,IAAI;IAClB,cAAc,kBAAkB,KAAK,KAAK,IAAI;GAChD;GACA,SAAS,IAAI,KAAK,OAAO;EAC3B;EACA,QAAQ;EACR,IAAI,CAAC,QAAQ,YAAY,SAAS,IAAI,UAAU,GAAG,QAAQ,YAAY,KAAK,IAAI,UAAU;CAC5F;CAMA,OAAO;EAAE,UAJG,CAAC,GAAG,SAAS,OAAO,CAAC,CAAC,CAC/B,QAAQ,MAAM,EAAE,YAAY,OAAO,CAAC,CACpC,MAAM,GAAG,MAAM,EAAE,WAAW,EAAE,QAEZ;EAAG;EAAe,WAAW,KAAK;CAAO;AAChE;AAEA,SAAS,kBAAkB,OAAmC;CAE5D,OADgB,MAAM,MAAM,MAAM,EAAE,WAAW,OAClC,CAAC,EAAE;AAClB;;;ACvFA,eAAsB,oBACpB,OACA,MACA,MACA,UAA6B,CAAC,GACH;CAC3B,MAAM,CAAC,GAAG,KAAK,MAAM,QAAQ,IAAI,CAAC,gBAAgB,OAAO,IAAI,GAAG,gBAAgB,OAAO,IAAI,CAAC,CAAC;CAC7F,MAAM,KAAK,QAAQ,cAAc;CACjC,MAAM,SAAS,KAAK,IAAI,EAAE,MAAM,QAAQ,EAAE,MAAM,MAAM;CACtD,KAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,KAAK;EAC/B,MAAM,QAAQ,EAAE,MAAM;EACtB,MAAM,QAAQ,EAAE,MAAM;EACtB,IAAI,CAAC,GAAG,OAAO,KAAK,GAClB,OAAO;GACL;GACA;GACA,sBAAsB;GACtB;GACA;GACA,QAAQ,mBAAmB,OAAO,KAAK;GACvC,iBAAiB;EACnB;CAEJ;CACA,IAAI,EAAE,MAAM,WAAW,EAAE,MAAM,QAC7B,OAAO;EAAE;EAAM;EAAM,sBAAsB;EAAM,iBAAiB;CAAO;CAG3E,MAAM,SADqB,EAAE,MAAM,SAAS,EAAE,MAAM,SAAS,IAAI,EAAA,CAC5C,MAAM,SAAS;CAGpC,MAAM,SACJ,WAAW,IACP,0CAA0C,MAAM,gCAChD,sBAAsB,MAAM,4BAA4B,SAAS;CACvE,OAAO;EACL;EACA;EACA,sBAAsB;EACtB,OAAO,EAAE,MAAM;EACf,OAAO,EAAE,MAAM;EACf;EACA,iBAAiB;CACnB;AACF;AAEA,SAAS,kBAAkB,GAAmB,GAA4B;CACxE,IAAI,EAAE,KAAK,SAAS,EAAE,KAAK,MAAM,OAAO;CACxC,IAAI,EAAE,KAAK,SAAS,UAAU,EAAE,KAAK,SAAS,QAAQ,OAAO,EAAE,KAAK,aAAa,EAAE,KAAK;CACxF,IAAI,EAAE,KAAK,SAAS,SAAS,EAAE,KAAK,SAAS,OAAO,OAAO,EAAE,KAAK,UAAU,EAAE,KAAK;CACnF,IAAI,EAAE,KAAK,SAAS,WAAW,EAAE,KAAK,SAAS,SAC7C,OAAO,EAAE,KAAK,cAAc,EAAE,KAAK;CACrC,OAAO,EAAE,KAAK,SAAS,EAAE,KAAK;AAChC;AAEA,SAAS,mBAAmB,GAAmB,GAA2B;CACxE,IAAI,EAAE,KAAK,SAAS,EAAE,KAAK,MAAM,OAAO,QAAQ,EAAE,KAAK,KAAK,MAAM,EAAE,KAAK;CACzE,IAAI,EAAE,KAAK,SAAS,UAAU,EAAE,KAAK,SAAS,UAAU,EAAE,KAAK,aAAa,EAAE,KAAK,UACjF,OAAO,QAAQ,EAAE,KAAK,SAAS,MAAM,EAAE,KAAK;CAE9C,IAAI,EAAE,KAAK,SAAS,SAAS,EAAE,KAAK,SAAS,SAAS,EAAE,KAAK,UAAU,EAAE,KAAK,OAC5E,OAAO,SAAS,EAAE,KAAK,MAAM,MAAM,EAAE,KAAK;CAE5C,OAAO,SAAS,EAAE,KAAK,KAAK,QAAQ,EAAE,KAAK,KAAK;AAClD;;;;;;;;;;;;AC9DA,eAAsB,mBAAmB,OAAkD;CACzF,MAAM,OAAO,MAAM,MAAM,MAAM,EAAE,MAAM,QAAQ,CAAC,EAAA,CAAG,QAChD,MAAsB,EAAE,SAAS,OACpC;CACA,IAAI,IAAI,WAAW,GAAG,OAAO;EAAE,OAAO,CAAC;EAAG,YAAY,CAAC;EAAG,UAAU,CAAC;CAAE;CAEvE,MAAM,8BAAc,IAAI,IAAyB;CACjD,KAAK,MAAM,KAAK,KAAK;EACnB,MAAM,MAAM,YAAY,IAAI,EAAE,SAAS,KAAK,CAAC;EAC7C,IAAI,KAAK,CAAC;EACV,YAAY,IAAI,EAAE,WAAW,GAAG;CAClC;CAEA,MAAM,WAAW,CAAC,GAAG,IAAI,IAAI,IAAI,KAAK,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK;CAC9D,MAAM,QAAqB,CAAC;CAC5B,KAAK,MAAM,CAAC,KAAK,UAAU,aAAa;EACtC,MAAM,0BAAU,IAAI,IAAiC;EACrD,KAAK,MAAM,KAAK,OAAO;GACrB,MAAM,IAAI,QAAQ,IAAI,EAAE,OAAO,qBAAK,IAAI,IAAoB;GAC5D,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK;GAC7B,QAAQ,IAAI,EAAE,SAAS,CAAC;EAC1B;EACA,MAAM,aAAa,CAAC,GAAG,QAAQ,KAAK,CAAC;EACrC,KAAK,IAAI,IAAI,GAAG,IAAI,WAAW,QAAQ,KACrC,KAAK,IAAI,IAAI,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK;GAC9C,MAAM,SAAS,WAAW;GAC1B,MAAM,SAAS,WAAW;GAC1B,MAAM,IAAI,QAAQ,IAAI,MAAM;GAC5B,MAAM,IAAI,QAAQ,IAAI,MAAM;GAC5B,MAAM,SAAkC,CAAC;GACzC,KAAK,MAAM,CAAC,QAAQ,WAAW,GAAG;IAChC,MAAM,SAAS,EAAE,IAAI,MAAM;IAC3B,IAAI,WAAW,KAAA,GAAW,OAAO,KAAK,CAAC,QAAQ,MAAM,CAAC;GACxD;GACA,IAAI,OAAO,SAAS,GAAG;GACvB,MAAM,cAAc,OAAO,KACxB,CAAC,QAAQ,YACR,CACE;IAAE,WAAW;IAAQ,WAAW;IAAK,OAAO;IAAQ,WAAW;GAAG,GAClE;IAAE,WAAW;IAAQ,WAAW;IAAK,OAAO;IAAQ,WAAW;GAAG,CACpE,CACJ;GACA,MAAM,IAAI,sBACR,YAAY,EAAE,CAAE,KAAK,GAAG,OAAO,YAAY,KAAK,SAAS,KAAK,GAAI,CAAC,CACrE;GACA,MAAM,KAAK;IACT,QAAQ;IACR,QAAQ;IACR,WAAW;IACX,aAAa,OAAO;IACpB,SAAS,SACP,OAAO,KAAK,MAAM,EAAE,EAAE,GACtB,OAAO,KAAK,MAAM,EAAE,EAAE,CACxB;IACA,cAAc;GAChB,CAAC;EACH;CAEJ;CAEA,OAAO;EACL,OAAO,MAAM,MAAM,GAAG,MAAM,EAAE,cAAc,EAAE,WAAW;EACzD,YAAY,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK;EACzC;CACF;AACF;;;;;;;;;;;ACrEA,eAAsB,eACpB,OACA,SACA,SACyB;CACzB,MAAM,eAAe,MAAM,MAAM,SAAS,QAAQ,QAAQ;CAC1D,MAAM,gBAAgB,MAAM,MAAM,SAAS,QAAQ,SAAS;CAS5D,OAAO,kBAAkB,MARH,QAAQ,IAC5B,QAAQ,IAAI,OAAO,MAAM;EACvB,MAAM,UAAU,EAAE,WAAW,eAAe,EAAE,MAAM;EACpD,MAAM,WAAW,MAAM,WAAW,cAAc,SAAS,KAAK;EAC9D,MAAM,YAAY,MAAM,WAAW,eAAe,SAAS,KAAK;EAChE,OAAO;GAAE,QAAQ,EAAE;GAAQ,gBAAgB,EAAE;GAAgB;GAAU;EAAU;CACnF,CAAC,CACH,GACkC,OAAO;AAC3C;AAEA,eAAe,WACb,MACA,SACA,OACmB;CACnB,MAAM,MAAgB,CAAC;CACvB,KAAK,MAAM,KAAK,MAAM;EACpB,MAAM,IAAI,MAAM,QAAQ,GAAG,KAAK;EAChC,IAAI,MAAM,QAAQ,OAAO,SAAS,CAAC,GAAG,IAAI,KAAK,CAAC;CAClD;CACA,OAAO;AACT;AAEA,SAAS,eAAe,QAAyE;CAC/F,OAAO,OAAO,KAAK,UAAU;EAC3B,QAAQ,QAAR;GACE,KAAK;GACL,KAAK,gBACH,OAAO,IAAI,SAAS,SAAS;GAC/B,KAAK,QACH,OAAO,IAAI,SAAS,SAAS,OAAO,IAAI;GAC1C,KAAK,cACH,OAAO,IAAI,WAAW,IAAI,YAAY,IAAI,UAAU,IAAI,YAAY;GACtE,KAAK,WAEH,OAAO,aAAa,MADF,SAAS,OAAO,IAAI,KAAK,CACpB,CAAC,CAAC;GAE3B,KAAK,eAEH,OAAO,aAAa,MADF,SAAS,OAAO,IAAI,KAAK,CACpB,CAAC,CAAC;GAE3B,KAAK,gBAEH,OAAO,aAAa,MADF,SAAS,OAAO,IAAI,KAAK,CACpB,CAAC,CAAC;GAE3B,KAAK,gBACH,OAAO,gBAAgB,GAAG,MAAM,YAAY,IAAI;GAElD,SACE,OAAO;EACX;CACF;AACF;;;;;;;;;;;;ACvEA,MAAM,wBAAwB;AAC9B,MAAM,qCAAqC;AA8C3C,eAAsB,cACpB,OACA,UAA4B,CAAC,GACH;CAC1B,MAAM,iBAAiB,QAAQ,kBAAkB;CACjD,MAAM,cAAc,QAAQ,eAAe;CAC3C,MAAM,0BACJ,QAAQ,2BAA2B;CACrC,IAAI,CAAC,OAAO,UAAU,cAAc,KAAK,iBAAiB,GACxD,MAAM,IAAI,WAAW,2CAA2C;CAElE,IAAI,CAAC,OAAO,SAAS,WAAW,KAAK,cAAc,GACjD,MAAM,IAAI,WAAW,kDAAkD;CAEzE,IAAI,CAAC,OAAO,UAAU,uBAAuB,KAAK,0BAA0B,GAC1E,MAAM,IAAI,WAAW,wDAAwD;CAE/E,MAAM,OAAO,QAAQ,QACjB,CAAC,EAAE,OAAO,QAAQ,MAAM,CAAC,KACxB,MAAM,MAAM,SAAS,EAAA,CAAG,KAAK,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE;CAE5D,MAAM,WAA+B,CAAC;CACtC,KAAK,MAAM,EAAE,WAAW,MAAM;EAC5B,MAAM,QAAQ,MAAM,MAAM,MAAM,EAAE,MAAM,CAAC;EACzC,MAAM,YAAY,IAAI,IAAI,MAAM,KAAK,SAAS,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC;EAClE,MAAM,cAAgC,MACnC,OAAO,UAAU,CAAC,CAClB,KAAK,MAAM,iBAAiB;GAAE;GAAM;EAAY,EAAE,CAAC,CACnD,MAAM,GAAG,MAAM,EAAE,KAAK,YAAY,EAAE,KAAK,aAAa,EAAE,cAAc,EAAE,WAAW,CAAC,CACpF,KAAK,EAAE,YAAY;GAAE;GAAM,GAAG,eAAe,MAAM,SAAS;EAAE,EAAE;EACnE,MAAM,cAAc,qBAClB,YAAY,KAAK,SAAS;GAExB,MAAM,QADS,KAAK,eAAe,QAAQ,KAAK,eAAe,KAAK,cAEhE,KAAK,OACL,KAAK,aACH,UAAU,IAAI,KAAK,UAAU,IAC7B,KAAA;GACN,OAAO;IACL,KAAK,aAAa,IAAI;IACtB,UAAU,KAAK,UAAU,KAAK,WAAW;IACzC,OAAO,OAAO,aAAa;IAC3B,KAAK,OAAO,WAAW;GACzB;EACF,CAAC,CACH;EACA,MAAM,uCAAuB,IAAI,IAAoB;EACrD,MAAM,eAAkC,YAAY,KAAK,SAAS;GAChE,MAAM,UAAU,YAAY,IAAI,aAAa,IAAI,CAAC;GAClD,MAAM,gBAAgB,qBAAqB,IAAI,OAAO,KAAK;GAC3D,qBAAqB,IAAI,SAAS,gBAAgB,CAAC;GACnD,OAAO;IAAE,GAAG;IAAM;IAAe;GAAQ;EAC3C,CAAC;EACD,MAAM,wBAAQ,IAAI,IAQhB;EACF,KAAK,MAAM,QAAQ,cAAc;GAC/B,IAAI,CAAC,oBAAoB,KAAK,IAAI,GAAG;GACrC,MAAM,IAAI,QAAQ,KAAK,KAAK,IAAI;GAChC,MAAM,MAAM,KAAK,UAAU;IAAC,KAAK;IAAS,KAAK,KAAK;IAAU;GAAC,CAAC;GAChE,MAAM,SAAS,MAAM,IAAI,GAAG,KAAK;IAC/B,OAAO,CAAC;IACR,SAAS;IACT,UAAU,KAAK,KAAK;IACpB,aAAa,KAAK;GACpB;GACA,OAAO,MAAM,KAAK,IAAI;GACtB,MAAM,IAAI,KAAK,MAAM;EACvB;EACA,KAAK,MAAM,EAAE,OAAO,SAAS,GAAG,UAAU,iBAAiB,MAAM,OAAO,GAAG;GACzE,IAAI,MAAM,SAAS,gBAAgB;GACnC,IAAI,eAAe;GACnB,KAAK,IAAI,aAAa,GAAG,cAAc,MAAM,QAAQ,cAAc,GAAG;IACpE,MAAM,WAAW,MAAM,aAAa;IACpC,MAAM,OAAO,MAAM;IAMnB,IAAI,EAJF,SAAS,KAAA,KACT,KAAK,KAAK,YAAY,SAAS,KAAK,YAAY,eAChD,KAAK,gBAAgB,SAAS,gBAAgB,IAAI,2BAClD,CAAC,eAAe,UAAU,IAAI,IACb;IAEnB,MAAM,UAAU,MAAM,MAAM,cAAc,UAAU;IACpD,IAAI,OAAO;IACX,IAAI,YAAY;IAChB,IAAI,UAAU;IACd,KAAK,IAAI,QAAQ,GAAG,QAAQ,QAAQ,QAAQ,SAAS,GAAG;KACtD,OAAO,QAAQ,MAAM,CAAE,KAAK,YAAY,QAAQ,KAAK,CAAE,KAAK,YAAY,aACtE,QAAQ;KAEV,IAAI,QAAQ,OAAO,UAAU,WAAW;MACtC,YAAY;MACZ,UAAU;KACZ;IACF;IACA,IAAI,UAAU,YAAY,KAAK,gBAAgB;KAC7C,MAAM,OAAO,QAAQ,MAAM,WAAW,UAAU,CAAC;KACjD,MAAM,QAAQ,KAAK,EAAE,CAAE,KAAK;KAC5B,MAAM,OAAO,KAAK,KAAK,SAAS,EAAE,CAAE,KAAK;KACzC,SAAS,KAAK;MACZ;MACA;MACA,SAAS;MACT,aAAa,KAAK;MAClB,SAAS,KAAK,KAAK,SAAS,KAAK,KAAK,MAAM;MAC5C,GAAI,cAAc,EAAE,YAAY,IAAI,CAAC;MACrC,UAAU,OAAO;KACnB,CAAC;IACH;IACA,eAAe;GACjB;EACF;CACF;CAEA,MAAM,eAAe,IAAI,IAAI,SAAS,KAAK,MAAM,EAAE,KAAK,CAAC;CACzD,OAAO;EACL;EACA,kBAAkB,KAAK,SAAS,IAAI,aAAa,OAAO,KAAK,SAAS;EACtE,WAAW,KAAK;CAClB;AACF;AAEA,SAAS,QAAQ,MAAkE;CACjF,OAAO,KAAK,UAAU,CAAC,KAAK,aAAa,KAAK,UAAU,CAAC;AAC3D;AAEA,SAAS,aAAa,MAA8B;CAClD,OAAO,KAAK,eAAe,QAAQ,KAAK,eAAe,KAAK,cACxD,KAAK,UAAU,CAAC,KAAK,aAAa,KAAK,KAAK,MAAM,CAAC,IACnD,QAAQ,IAAI;AAClB;AAEA,SAAS,eACP,MACA,WAC2D;CAC3D,MAAM,eAAe,KAAK;CAC1B,IAAI,CAAC,cAAc,OAAO;EAAE,aAAa;EAAM,YAAY;CAAK;CAEhE,IAAI,YAAgC;CACpC,IAAI,aAA4B;CAChC,MAAM,uBAAO,IAAI,IAAY;CAC7B,OAAO,aAAa,CAAC,KAAK,IAAI,SAAS,GAAG;EACxC,KAAK,IAAI,SAAS;EAClB,MAAM,UAAU,UAAU,IAAI,SAAS;EACvC,IAAI,CAAC,SAAS,OAAO;GAAE,aAAa;GAAc,YAAY;EAAa;EAC3E,IAAI,QAAQ,SAAS,SACnB,OAAO;GAAE,aAAa,QAAQ;GAAQ,YAAY,cAAc,QAAQ;EAAO;EAEjF,aAAa,QAAQ;EACrB,YAAY,QAAQ;CACtB;CACA,OAAO;EAAE,aAAa;EAAc,YAAY;CAAa;AAC/D;AAEA,SAAS,eAAe,UAA2B,MAAgC;CACjF,OAAO,SAAS,KAAK,YAAY,KAAA,KAAa,SAAS,KAAK,WAAW,KAAK,KAAK;AACnF;;;;;;;;;;;;;;AC/LA,eAAsB,cACpB,OACA,UAA4B,CAAC,GACH;CAC1B,MAAM,OAAO,QAAQ,QAAQ,CAAC,QAAQ,KAAK,KAAK,MAAM,MAAM,SAAS,EAAA,CAAG,KAAK,MAAM,EAAE,KAAK;CAE1F,MAAM,QAA4B,CAAC;CACnC,IAAI,aAAa;CACjB,IAAI,cAAc;CAClB,KAAK,MAAM,SAAS,MAAM;EACxB,MAAM,QAAQ,MAAM,UAAU,OAAO,KAAK;EAC1C,IAAI,MAAM,WAAW,GAAG;GACtB,MAAM,KAAK;IAAE;IAAO,aAAa;IAAG,YAAY;IAAG,WAAW;GAAE,CAAC;GACjE;EACF;EAQA,MAAM,YAAY,CAAC,GAAG,MAPH,SAAS,OAAO,KAAK,CAOd,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,YAAY,EAAE,SAAS;EACpE,MAAM,aAAa,UAAU,KAAK,MAAM,EAAE,SAAS;EACnD,MAAM,aAAa,gBAAgB,SAAS;EAC5C,IAAI,SAAS;EACb,KAAK,MAAM,KAAK,OAAO;GACrB,IAAI,EAAE,WAAW,SAAS;IACxB;IACA;GACF;GAEA,MAAM,SAAS,WAAW,YAAY,EAAE,SAAS;GACjD,IAAI,QAAQ,aACN;QAAA,CAAC,QAAQ,YAAY,GAAG,EAAE,KAAK,UAAU,MAAM,MAAM,EAAE,CAAC,GAAG;GAAA,OAC1D;IAKL,MAAM,YAAY,UAAU,EAAE,MAAM;IACpC,IAAI,cAAc,IAAI;IAGtB,IAAI,EAFa,WAAW,WAAW,GAAA,CACjB,SAAS,UAAU,MAAM,GAAG,GAAG,CAC7C,GAAG;GACb;EACF;EACA,MAAM,YAAY,SAAS,MAAM;EACjC,MAAM,KAAK;GAAE;GAAO,aAAa;GAAQ,YAAY,MAAM;GAAQ;EAAU,CAAC;EAC9E,cAAc,MAAM;EACpB,eAAe;CACjB;CACA,OAAO;EAAE;EAAO,kBAAkB,aAAa,IAAI,cAAc,aAAa;CAAE;AAClF;;;;;;AAOA,SAAS,gBAAgB,OAA4B;CACnD,MAAM,SAAS,IAAI,MAAc,MAAM,SAAS,CAAC;CACjD,OAAO,MAAM,UAAU;CACvB,KAAK,IAAI,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAIrC,OAAO,KAAK,GAHA,MAAM,EAAE,CAAE,SAAS,KAAK,MAClC,OAAO,EAAE,YAAY,WAAW,EAAE,UAAU,EAC9C,CAAC,CAAC,KAAK,IACU,EAAE,IAAI,OAAO,IAAI;CAEpC,OAAO;AACT;;AAGA,SAAS,WAAW,QAAkB,QAAwB;CAC5D,IAAI,KAAK;CACT,IAAI,KAAK,OAAO;CAChB,OAAO,KAAK,IAAI;EACd,MAAM,MAAO,KAAK,OAAQ;EAC1B,IAAI,OAAO,QAAS,QAAQ,KAAK,MAAM;OAClC,KAAK;CACZ;CACA,OAAO;AACT;AAEA,SAAS,UAAU,GAAoB;CACrC,IAAI,MAAM,QAAQ,MAAM,KAAA,GAAW,OAAO;CAC1C,IAAI,OAAO,MAAM,UAAU,OAAO;CAClC,IAAI;EACF,OAAO,KAAK,UAAU,CAAC;CACzB,QAAQ;EACN,OAAO,OAAO,CAAC;CACjB;AACF"}
|
|
1
|
+
{"version":3,"file":"index.js","names":[],"sources":["../../src/pipelines/first-divergence.ts","../../src/pipelines/regression.ts","../../src/pipelines/stuck-loop.ts"],"sourcesContent":["/**\n * FirstDivergenceView — aligns two trajectories by step index, reports\n * the first step where they differ.\n *\n * \"Differ\" is configurable — default is (kind, toolName if tool, model\n * if llm). Use this view to attribute \"why is variant B better?\" to a\n * specific step rather than an aggregate mean delta.\n */\n\nimport type { TraceStore } from '../trace/store'\nimport { buildTrajectory, type Trajectory, type TrajectoryStep } from '../trajectory'\n\nexport interface DivergenceReport {\n runA: string\n runB: string\n firstDivergenceIndex: number | null\n aStep?: TrajectoryStep\n bStep?: TrajectoryStep\n reason?: string\n /** Common prefix length (steps that matched). */\n commonPrefixLen: number\n}\n\nexport interface DivergenceOptions {\n /** Returns true if two steps are considered equal. Default: kind + tool/model match. */\n stepEquals?: (a: TrajectoryStep, b: TrajectoryStep) => boolean\n}\n\nexport async function firstDivergenceView(\n store: TraceStore,\n runA: string,\n runB: string,\n options: DivergenceOptions = {},\n): Promise<DivergenceReport> {\n const [a, b] = await Promise.all([buildTrajectory(store, runA), buildTrajectory(store, runB)])\n const eq = options.stepEquals ?? defaultStepEquals\n const minLen = Math.min(a.steps.length, b.steps.length)\n for (let i = 0; i < minLen; i++) {\n const aStep = a.steps[i]!\n const bStep = b.steps[i]!\n if (!eq(aStep, bStep)) {\n return {\n runA,\n runB,\n firstDivergenceIndex: i,\n aStep,\n bStep,\n reason: describeDifference(aStep, bStep),\n commonPrefixLen: i,\n }\n }\n }\n if (a.steps.length === b.steps.length) {\n return { runA, runB, firstDivergenceIndex: null, commonPrefixLen: minLen }\n }\n const longer: Trajectory = a.steps.length > b.steps.length ? a : b\n const extra = longer.steps.length - minLen\n // minLen === 0 means one trajectory is empty: divergence is the first step\n // itself (index 0), and the absent side has no step to surface.\n const reason =\n minLen === 0\n ? `one trajectory is empty; the other has ${extra} step(s) starting at index 0`\n : `one trajectory has ${extra} more step(s) after index ${minLen - 1}`\n return {\n runA,\n runB,\n firstDivergenceIndex: minLen,\n aStep: a.steps[minLen],\n bStep: b.steps[minLen],\n reason,\n commonPrefixLen: minLen,\n }\n}\n\nfunction defaultStepEquals(a: TrajectoryStep, b: TrajectoryStep): boolean {\n if (a.span.kind !== b.span.kind) return false\n if (a.span.kind === 'tool' && b.span.kind === 'tool') return a.span.toolName === b.span.toolName\n if (a.span.kind === 'llm' && b.span.kind === 'llm') return a.span.model === b.span.model\n if (a.span.kind === 'judge' && b.span.kind === 'judge')\n return a.span.dimension === b.span.dimension\n return a.span.name === b.span.name\n}\n\nfunction describeDifference(a: TrajectoryStep, b: TrajectoryStep): string {\n if (a.span.kind !== b.span.kind) return `kind ${a.span.kind} vs ${b.span.kind}`\n if (a.span.kind === 'tool' && b.span.kind === 'tool' && a.span.toolName !== b.span.toolName) {\n return `tool ${a.span.toolName} vs ${b.span.toolName}`\n }\n if (a.span.kind === 'llm' && b.span.kind === 'llm' && a.span.model !== b.span.model) {\n return `model ${a.span.model} vs ${b.span.model}`\n }\n return `name \"${a.span.name}\" vs \"${b.span.name}\"`\n}\n","/**\n * RegressionView — compares a candidate slice to a baseline slice on a\n * named metric. Delegates the statistics (Welch's t-test, Cohen's d,\n * IQR stability) to `baseline.ts`.\n *\n * This is the entry point for CI regression gates: \"given runs tagged\n * release=A and release=B, did any metric regress?\"\n */\n\nimport { type BaselineOptions, type BaselineReport, compareToBaseline } from '../baseline'\nimport { aggregateLlm, llmSpans, runFailureClass } from '../trace/query'\nimport type { Run } from '../trace/schema'\nimport type { RunFilter, TraceStore } from '../trace/store'\n\nexport interface RegressionSpec {\n metric: string\n higherIsBetter: boolean\n /** Extract a scalar from a run. Default extractors handle common metrics. */\n extract?: (run: Run, store: TraceStore) => Promise<number | null>\n}\n\nexport interface RegressionOptions extends BaselineOptions {\n baseline: RunFilter\n candidate: RunFilter\n}\n\nexport async function regressionView(\n store: TraceStore,\n metrics: RegressionSpec[],\n options: RegressionOptions,\n): Promise<BaselineReport> {\n const baselineRuns = await store.listRuns(options.baseline)\n const candidateRuns = await store.listRuns(options.candidate)\n const samples = await Promise.all(\n metrics.map(async (m) => {\n const extract = m.extract ?? defaultExtract(m.metric)\n const baseline = await extractAll(baselineRuns, extract, store)\n const candidate = await extractAll(candidateRuns, extract, store)\n return { metric: m.metric, higherIsBetter: m.higherIsBetter, baseline, candidate }\n }),\n )\n return compareToBaseline(samples, options)\n}\n\nasync function extractAll(\n runs: Run[],\n extract: (r: Run, s: TraceStore) => Promise<number | null>,\n store: TraceStore,\n): Promise<number[]> {\n const out: number[] = []\n for (const r of runs) {\n const v = await extract(r, store)\n if (v !== null && Number.isFinite(v)) out.push(v)\n }\n return out\n}\n\nfunction defaultExtract(metric: string): (run: Run, store: TraceStore) => Promise<number | null> {\n return async (run, store) => {\n switch (metric) {\n case 'score':\n case 'overallScore':\n return run.outcome?.score ?? null\n case 'pass':\n return run.outcome?.pass === true ? 1 : 0\n case 'durationMs':\n return run.endedAt && run.startedAt ? run.endedAt - run.startedAt : null\n case 'costUsd': {\n const llm = await llmSpans(store, run.runId)\n return aggregateLlm(llm).costUsd\n }\n case 'inputTokens': {\n const llm = await llmSpans(store, run.runId)\n return aggregateLlm(llm).inputTokens\n }\n case 'outputTokens': {\n const llm = await llmSpans(store, run.runId)\n return aggregateLlm(llm).outputTokens\n }\n case 'failureClass': {\n return runFailureClass(run) === 'success' ? 1 : 0\n }\n default:\n return null\n }\n }\n}\n","/**\n * StuckLoopView — detects when an agent calls the same tool with the\n * same (or structurally similar) arguments ≥ N times in a short window.\n *\n * Rationale: agents that loop are the number-one production failure\n * mode on long-horizon flows. The view returns (runId, toolName,\n * argHash, occurrences, windowMs) for each detected loop plus a\n * fraction of runs affected.\n */\n\nimport { executionTrackByLane } from '../trace/execution-tracks'\nimport { argHash, hasCapturedToolArgs } from '../trace/query'\nimport { isToolSpan, type Span, type ToolSpan } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\n\nconst DEFAULT_MAX_WINDOW_MS = 60_000\nconst DEFAULT_MAX_INTERVENING_TOOL_CALLS = 0\n\ninterface ScopedToolCall {\n span: ToolSpan\n scopeSpanId: string | null\n laneSpanId: string | null\n}\n\ninterface IndexedToolCall extends ScopedToolCall {\n toolCallIndex: number\n trackId: string\n}\n\nexport interface StuckLoopFinding {\n runId: string\n toolName: string\n argHash: string\n /** Calls in this episode's densest qualifying interval, not the whole-run total. */\n occurrences: number\n spanIds: string[]\n /** Nearest agent ancestor, or the direct parent when ancestry is incomplete. */\n scopeSpanId?: string\n /** Milliseconds between first and last call in the loop. */\n windowMs: number\n}\n\nexport interface StuckLoopReport {\n findings: StuckLoopFinding[]\n affectedRunRatio: number\n totalRuns: number\n}\n\nexport interface StuckLoopOptions {\n /** Minimum call count to flag a loop (default 3). */\n minOccurrences?: number\n /** Maximum time between the first and last repeated call (default 60 seconds). */\n maxWindowMs?: number\n /**\n * Maximum other tool calls allowed between adjacent repeats (default 0).\n * Set to 1 to detect alternating patterns such as A,B,A,B,A.\n */\n maxInterveningToolCalls?: number\n /** Filter to a specific runId; omit to scan the entire corpus. */\n runId?: string\n}\n\nexport async function stuckLoopView(\n store: TraceStore,\n options: StuckLoopOptions = {},\n): Promise<StuckLoopReport> {\n const minOccurrences = options.minOccurrences ?? 3\n const maxWindowMs = options.maxWindowMs ?? DEFAULT_MAX_WINDOW_MS\n const maxInterveningToolCalls =\n options.maxInterveningToolCalls ?? DEFAULT_MAX_INTERVENING_TOOL_CALLS\n if (!Number.isInteger(minOccurrences) || minOccurrences < 1) {\n throw new RangeError('minOccurrences must be a positive integer')\n }\n if (!Number.isFinite(maxWindowMs) || maxWindowMs < 0) {\n throw new RangeError('maxWindowMs must be a finite non-negative number')\n }\n if (!Number.isInteger(maxInterveningToolCalls) || maxInterveningToolCalls < 0) {\n throw new RangeError('maxInterveningToolCalls must be a non-negative integer')\n }\n const runs = options.runId\n ? [{ runId: options.runId }]\n : (await store.listRuns()).map((r) => ({ runId: r.runId }))\n\n const findings: StuckLoopFinding[] = []\n for (const { runId } of runs) {\n const spans = await store.spans({ runId })\n const spansById = new Map(spans.map((span) => [span.spanId, span]))\n const scopedTools: ScopedToolCall[] = spans\n .filter(isToolSpan)\n .map((span, sourceIndex) => ({ span, sourceIndex }))\n .sort((a, b) => a.span.startedAt - b.span.startedAt || a.sourceIndex - b.sourceIndex)\n .map(({ span }) => ({ span, ...executionScope(span, spansById) }))\n const trackByLane = executionTrackByLane(\n scopedTools.map((call) => {\n const direct = call.laneSpanId === null || call.laneSpanId === call.scopeSpanId\n const timed = direct\n ? call.span\n : call.laneSpanId\n ? spansById.get(call.laneSpanId)\n : undefined\n return {\n key: executionKey(call),\n scopeKey: JSON.stringify(call.scopeSpanId),\n start: timed?.startedAt ?? null,\n end: timed?.endedAt ?? null,\n }\n }),\n )\n const nextToolIndexByTrack = new Map<string, number>()\n const orderedTools: IndexedToolCall[] = scopedTools.map((call) => {\n const trackId = trackByLane.get(executionKey(call))!\n const toolCallIndex = nextToolIndexByTrack.get(trackId) ?? 0\n nextToolIndexByTrack.set(trackId, toolCallIndex + 1)\n return { ...call, toolCallIndex, trackId }\n })\n const byKey = new Map<\n string,\n {\n calls: IndexedToolCall[]\n argHash: string\n toolName: string\n scopeSpanId: string | null\n }\n >()\n for (const call of orderedTools) {\n if (!hasCapturedToolArgs(call.span)) continue\n const h = argHash(call.span.args)\n const key = JSON.stringify([call.trackId, call.span.toolName, h])\n const bucket = byKey.get(key) ?? {\n calls: [],\n argHash: h,\n toolName: call.span.toolName,\n scopeSpanId: call.scopeSpanId,\n }\n bucket.calls.push(call)\n byKey.set(key, bucket)\n }\n for (const { calls, argHash: h, toolName, scopeSpanId } of byKey.values()) {\n if (calls.length < minOccurrences) continue\n let episodeStart = 0\n for (let episodeEnd = 1; episodeEnd <= calls.length; episodeEnd += 1) {\n const previous = calls[episodeEnd - 1]!\n const next = calls[episodeEnd]\n const episodeEnded =\n next === undefined ||\n next.span.startedAt - previous.span.startedAt > maxWindowMs ||\n next.toolCallIndex - previous.toolCallIndex - 1 > maxInterveningToolCalls ||\n !callsAreSerial(previous, next)\n if (!episodeEnded) continue\n\n const episode = calls.slice(episodeStart, episodeEnd)\n let left = 0\n let bestStart = 0\n let bestEnd = -1\n for (let right = 0; right < episode.length; right += 1) {\n while (episode[right]!.span.startedAt - episode[left]!.span.startedAt > maxWindowMs) {\n left += 1\n }\n if (right - left > bestEnd - bestStart) {\n bestStart = left\n bestEnd = right\n }\n }\n if (bestEnd - bestStart + 1 >= minOccurrences) {\n const loop = episode.slice(bestStart, bestEnd + 1)\n const first = loop[0]!.span.startedAt\n const last = loop[loop.length - 1]!.span.startedAt\n findings.push({\n runId,\n toolName,\n argHash: h,\n occurrences: loop.length,\n spanIds: loop.map((call) => call.span.spanId),\n ...(scopeSpanId ? { scopeSpanId } : {}),\n windowMs: last - first,\n })\n }\n episodeStart = episodeEnd\n }\n }\n }\n\n const affectedRuns = new Set(findings.map((f) => f.runId))\n return {\n findings,\n affectedRunRatio: runs.length > 0 ? affectedRuns.size / runs.length : 0,\n totalRuns: runs.length,\n }\n}\n\nfunction laneKey(call: Pick<ScopedToolCall, 'scopeSpanId' | 'laneSpanId'>): string {\n return JSON.stringify([call.scopeSpanId, call.laneSpanId])\n}\n\nfunction executionKey(call: ScopedToolCall): string {\n return call.laneSpanId === null || call.laneSpanId === call.scopeSpanId\n ? JSON.stringify([call.scopeSpanId, call.span.spanId])\n : laneKey(call)\n}\n\nfunction executionScope(\n span: ToolSpan,\n spansById: ReadonlyMap<string, Span>,\n): { scopeSpanId: string | null; laneSpanId: string | null } {\n const directParent = span.parentSpanId\n if (!directParent) return { scopeSpanId: null, laneSpanId: null }\n\n let currentId: string | undefined = directParent\n let laneSpanId: string | null = null\n const seen = new Set<string>()\n while (currentId && !seen.has(currentId)) {\n seen.add(currentId)\n const current = spansById.get(currentId)\n if (!current) return { scopeSpanId: directParent, laneSpanId: directParent }\n if (current.kind === 'agent') {\n return { scopeSpanId: current.spanId, laneSpanId: laneSpanId ?? current.spanId }\n }\n laneSpanId = current.spanId\n currentId = current.parentSpanId\n }\n return { scopeSpanId: directParent, laneSpanId: directParent }\n}\n\nfunction callsAreSerial(previous: IndexedToolCall, next: IndexedToolCall): boolean {\n return previous.span.endedAt !== undefined && previous.span.endedAt <= next.span.startedAt\n}\n"],"mappings":";;;;;;;AA4BA,eAAsB,oBACpB,OACA,MACA,MACA,UAA6B,CAAC,GACH;CAC3B,MAAM,CAAC,GAAG,KAAK,MAAM,QAAQ,IAAI,CAAC,gBAAgB,OAAO,IAAI,GAAG,gBAAgB,OAAO,IAAI,CAAC,CAAC;CAC7F,MAAM,KAAK,QAAQ,cAAc;CACjC,MAAM,SAAS,KAAK,IAAI,EAAE,MAAM,QAAQ,EAAE,MAAM,MAAM;CACtD,KAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,KAAK;EAC/B,MAAM,QAAQ,EAAE,MAAM;EACtB,MAAM,QAAQ,EAAE,MAAM;EACtB,IAAI,CAAC,GAAG,OAAO,KAAK,GAClB,OAAO;GACL;GACA;GACA,sBAAsB;GACtB;GACA;GACA,QAAQ,mBAAmB,OAAO,KAAK;GACvC,iBAAiB;EACnB;CAEJ;CACA,IAAI,EAAE,MAAM,WAAW,EAAE,MAAM,QAC7B,OAAO;EAAE;EAAM;EAAM,sBAAsB;EAAM,iBAAiB;CAAO;CAG3E,MAAM,SADqB,EAAE,MAAM,SAAS,EAAE,MAAM,SAAS,IAAI,EAAA,CAC5C,MAAM,SAAS;CAGpC,MAAM,SACJ,WAAW,IACP,0CAA0C,MAAM,gCAChD,sBAAsB,MAAM,4BAA4B,SAAS;CACvE,OAAO;EACL;EACA;EACA,sBAAsB;EACtB,OAAO,EAAE,MAAM;EACf,OAAO,EAAE,MAAM;EACf;EACA,iBAAiB;CACnB;AACF;AAEA,SAAS,kBAAkB,GAAmB,GAA4B;CACxE,IAAI,EAAE,KAAK,SAAS,EAAE,KAAK,MAAM,OAAO;CACxC,IAAI,EAAE,KAAK,SAAS,UAAU,EAAE,KAAK,SAAS,QAAQ,OAAO,EAAE,KAAK,aAAa,EAAE,KAAK;CACxF,IAAI,EAAE,KAAK,SAAS,SAAS,EAAE,KAAK,SAAS,OAAO,OAAO,EAAE,KAAK,UAAU,EAAE,KAAK;CACnF,IAAI,EAAE,KAAK,SAAS,WAAW,EAAE,KAAK,SAAS,SAC7C,OAAO,EAAE,KAAK,cAAc,EAAE,KAAK;CACrC,OAAO,EAAE,KAAK,SAAS,EAAE,KAAK;AAChC;AAEA,SAAS,mBAAmB,GAAmB,GAA2B;CACxE,IAAI,EAAE,KAAK,SAAS,EAAE,KAAK,MAAM,OAAO,QAAQ,EAAE,KAAK,KAAK,MAAM,EAAE,KAAK;CACzE,IAAI,EAAE,KAAK,SAAS,UAAU,EAAE,KAAK,SAAS,UAAU,EAAE,KAAK,aAAa,EAAE,KAAK,UACjF,OAAO,QAAQ,EAAE,KAAK,SAAS,MAAM,EAAE,KAAK;CAE9C,IAAI,EAAE,KAAK,SAAS,SAAS,EAAE,KAAK,SAAS,SAAS,EAAE,KAAK,UAAU,EAAE,KAAK,OAC5E,OAAO,SAAS,EAAE,KAAK,MAAM,MAAM,EAAE,KAAK;CAE5C,OAAO,SAAS,EAAE,KAAK,KAAK,QAAQ,EAAE,KAAK,KAAK;AAClD;;;;;;;;;;;AClEA,eAAsB,eACpB,OACA,SACA,SACyB;CACzB,MAAM,eAAe,MAAM,MAAM,SAAS,QAAQ,QAAQ;CAC1D,MAAM,gBAAgB,MAAM,MAAM,SAAS,QAAQ,SAAS;CAS5D,OAAO,kBAAkB,MARH,QAAQ,IAC5B,QAAQ,IAAI,OAAO,MAAM;EACvB,MAAM,UAAU,EAAE,WAAW,eAAe,EAAE,MAAM;EACpD,MAAM,WAAW,MAAM,WAAW,cAAc,SAAS,KAAK;EAC9D,MAAM,YAAY,MAAM,WAAW,eAAe,SAAS,KAAK;EAChE,OAAO;GAAE,QAAQ,EAAE;GAAQ,gBAAgB,EAAE;GAAgB;GAAU;EAAU;CACnF,CAAC,CACH,GACkC,OAAO;AAC3C;AAEA,eAAe,WACb,MACA,SACA,OACmB;CACnB,MAAM,MAAgB,CAAC;CACvB,KAAK,MAAM,KAAK,MAAM;EACpB,MAAM,IAAI,MAAM,QAAQ,GAAG,KAAK;EAChC,IAAI,MAAM,QAAQ,OAAO,SAAS,CAAC,GAAG,IAAI,KAAK,CAAC;CAClD;CACA,OAAO;AACT;AAEA,SAAS,eAAe,QAAyE;CAC/F,OAAO,OAAO,KAAK,UAAU;EAC3B,QAAQ,QAAR;GACE,KAAK;GACL,KAAK,gBACH,OAAO,IAAI,SAAS,SAAS;GAC/B,KAAK,QACH,OAAO,IAAI,SAAS,SAAS,OAAO,IAAI;GAC1C,KAAK,cACH,OAAO,IAAI,WAAW,IAAI,YAAY,IAAI,UAAU,IAAI,YAAY;GACtE,KAAK,WAEH,OAAO,aAAa,MADF,SAAS,OAAO,IAAI,KAAK,CACpB,CAAC,CAAC;GAE3B,KAAK,eAEH,OAAO,aAAa,MADF,SAAS,OAAO,IAAI,KAAK,CACpB,CAAC,CAAC;GAE3B,KAAK,gBAEH,OAAO,aAAa,MADF,SAAS,OAAO,IAAI,KAAK,CACpB,CAAC,CAAC;GAE3B,KAAK,gBACH,OAAO,gBAAgB,GAAG,MAAM,YAAY,IAAI;GAElD,SACE,OAAO;EACX;CACF;AACF;;;;;;;;;;;;ACvEA,MAAM,wBAAwB;AAC9B,MAAM,qCAAqC;AA8C3C,eAAsB,cACpB,OACA,UAA4B,CAAC,GACH;CAC1B,MAAM,iBAAiB,QAAQ,kBAAkB;CACjD,MAAM,cAAc,QAAQ,eAAe;CAC3C,MAAM,0BACJ,QAAQ,2BAA2B;CACrC,IAAI,CAAC,OAAO,UAAU,cAAc,KAAK,iBAAiB,GACxD,MAAM,IAAI,WAAW,2CAA2C;CAElE,IAAI,CAAC,OAAO,SAAS,WAAW,KAAK,cAAc,GACjD,MAAM,IAAI,WAAW,kDAAkD;CAEzE,IAAI,CAAC,OAAO,UAAU,uBAAuB,KAAK,0BAA0B,GAC1E,MAAM,IAAI,WAAW,wDAAwD;CAE/E,MAAM,OAAO,QAAQ,QACjB,CAAC,EAAE,OAAO,QAAQ,MAAM,CAAC,KACxB,MAAM,MAAM,SAAS,EAAA,CAAG,KAAK,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE;CAE5D,MAAM,WAA+B,CAAC;CACtC,KAAK,MAAM,EAAE,WAAW,MAAM;EAC5B,MAAM,QAAQ,MAAM,MAAM,MAAM,EAAE,MAAM,CAAC;EACzC,MAAM,YAAY,IAAI,IAAI,MAAM,KAAK,SAAS,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC;EAClE,MAAM,cAAgC,MACnC,OAAO,UAAU,CAAC,CAClB,KAAK,MAAM,iBAAiB;GAAE;GAAM;EAAY,EAAE,CAAC,CACnD,MAAM,GAAG,MAAM,EAAE,KAAK,YAAY,EAAE,KAAK,aAAa,EAAE,cAAc,EAAE,WAAW,CAAC,CACpF,KAAK,EAAE,YAAY;GAAE;GAAM,GAAG,eAAe,MAAM,SAAS;EAAE,EAAE;EACnE,MAAM,cAAc,qBAClB,YAAY,KAAK,SAAS;GAExB,MAAM,QADS,KAAK,eAAe,QAAQ,KAAK,eAAe,KAAK,cAEhE,KAAK,OACL,KAAK,aACH,UAAU,IAAI,KAAK,UAAU,IAC7B,KAAA;GACN,OAAO;IACL,KAAK,aAAa,IAAI;IACtB,UAAU,KAAK,UAAU,KAAK,WAAW;IACzC,OAAO,OAAO,aAAa;IAC3B,KAAK,OAAO,WAAW;GACzB;EACF,CAAC,CACH;EACA,MAAM,uCAAuB,IAAI,IAAoB;EACrD,MAAM,eAAkC,YAAY,KAAK,SAAS;GAChE,MAAM,UAAU,YAAY,IAAI,aAAa,IAAI,CAAC;GAClD,MAAM,gBAAgB,qBAAqB,IAAI,OAAO,KAAK;GAC3D,qBAAqB,IAAI,SAAS,gBAAgB,CAAC;GACnD,OAAO;IAAE,GAAG;IAAM;IAAe;GAAQ;EAC3C,CAAC;EACD,MAAM,wBAAQ,IAAI,IAQhB;EACF,KAAK,MAAM,QAAQ,cAAc;GAC/B,IAAI,CAAC,oBAAoB,KAAK,IAAI,GAAG;GACrC,MAAM,IAAI,QAAQ,KAAK,KAAK,IAAI;GAChC,MAAM,MAAM,KAAK,UAAU;IAAC,KAAK;IAAS,KAAK,KAAK;IAAU;GAAC,CAAC;GAChE,MAAM,SAAS,MAAM,IAAI,GAAG,KAAK;IAC/B,OAAO,CAAC;IACR,SAAS;IACT,UAAU,KAAK,KAAK;IACpB,aAAa,KAAK;GACpB;GACA,OAAO,MAAM,KAAK,IAAI;GACtB,MAAM,IAAI,KAAK,MAAM;EACvB;EACA,KAAK,MAAM,EAAE,OAAO,SAAS,GAAG,UAAU,iBAAiB,MAAM,OAAO,GAAG;GACzE,IAAI,MAAM,SAAS,gBAAgB;GACnC,IAAI,eAAe;GACnB,KAAK,IAAI,aAAa,GAAG,cAAc,MAAM,QAAQ,cAAc,GAAG;IACpE,MAAM,WAAW,MAAM,aAAa;IACpC,MAAM,OAAO,MAAM;IAMnB,IAAI,EAJF,SAAS,KAAA,KACT,KAAK,KAAK,YAAY,SAAS,KAAK,YAAY,eAChD,KAAK,gBAAgB,SAAS,gBAAgB,IAAI,2BAClD,CAAC,eAAe,UAAU,IAAI,IACb;IAEnB,MAAM,UAAU,MAAM,MAAM,cAAc,UAAU;IACpD,IAAI,OAAO;IACX,IAAI,YAAY;IAChB,IAAI,UAAU;IACd,KAAK,IAAI,QAAQ,GAAG,QAAQ,QAAQ,QAAQ,SAAS,GAAG;KACtD,OAAO,QAAQ,MAAM,CAAE,KAAK,YAAY,QAAQ,KAAK,CAAE,KAAK,YAAY,aACtE,QAAQ;KAEV,IAAI,QAAQ,OAAO,UAAU,WAAW;MACtC,YAAY;MACZ,UAAU;KACZ;IACF;IACA,IAAI,UAAU,YAAY,KAAK,gBAAgB;KAC7C,MAAM,OAAO,QAAQ,MAAM,WAAW,UAAU,CAAC;KACjD,MAAM,QAAQ,KAAK,EAAE,CAAE,KAAK;KAC5B,MAAM,OAAO,KAAK,KAAK,SAAS,EAAE,CAAE,KAAK;KACzC,SAAS,KAAK;MACZ;MACA;MACA,SAAS;MACT,aAAa,KAAK;MAClB,SAAS,KAAK,KAAK,SAAS,KAAK,KAAK,MAAM;MAC5C,GAAI,cAAc,EAAE,YAAY,IAAI,CAAC;MACrC,UAAU,OAAO;KACnB,CAAC;IACH;IACA,eAAe;GACjB;EACF;CACF;CAEA,MAAM,eAAe,IAAI,IAAI,SAAS,KAAK,MAAM,EAAE,KAAK,CAAC;CACzD,OAAO;EACL;EACA,kBAAkB,KAAK,SAAS,IAAI,aAAa,OAAO,KAAK,SAAS;EACtE,WAAW,KAAK;CAClB;AACF;AAEA,SAAS,QAAQ,MAAkE;CACjF,OAAO,KAAK,UAAU,CAAC,KAAK,aAAa,KAAK,UAAU,CAAC;AAC3D;AAEA,SAAS,aAAa,MAA8B;CAClD,OAAO,KAAK,eAAe,QAAQ,KAAK,eAAe,KAAK,cACxD,KAAK,UAAU,CAAC,KAAK,aAAa,KAAK,KAAK,MAAM,CAAC,IACnD,QAAQ,IAAI;AAClB;AAEA,SAAS,eACP,MACA,WAC2D;CAC3D,MAAM,eAAe,KAAK;CAC1B,IAAI,CAAC,cAAc,OAAO;EAAE,aAAa;EAAM,YAAY;CAAK;CAEhE,IAAI,YAAgC;CACpC,IAAI,aAA4B;CAChC,MAAM,uBAAO,IAAI,IAAY;CAC7B,OAAO,aAAa,CAAC,KAAK,IAAI,SAAS,GAAG;EACxC,KAAK,IAAI,SAAS;EAClB,MAAM,UAAU,UAAU,IAAI,SAAS;EACvC,IAAI,CAAC,SAAS,OAAO;GAAE,aAAa;GAAc,YAAY;EAAa;EAC3E,IAAI,QAAQ,SAAS,SACnB,OAAO;GAAE,aAAa,QAAQ;GAAQ,YAAY,cAAc,QAAQ;EAAO;EAEjF,aAAa,QAAQ;EACrB,YAAY,QAAQ;CACtB;CACA,OAAO;EAAE,aAAa;EAAc,YAAY;CAAa;AAC/D;AAEA,SAAS,eAAe,UAA2B,MAAgC;CACjF,OAAO,SAAS,KAAK,YAAY,KAAA,KAAa,SAAS,KAAK,WAAW,KAAK,KAAK;AACnF"}
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
import { s as ValidationError } from "./errors-Dngq5h35.js";
|
|
2
|
+
import { s as zQuantile } from "./internal-BDHPCnjk.js";
|
|
3
|
+
import { u as normalCdf } from "./paired-arms-D-XRF_fy.js";
|
|
4
|
+
//#region src/statistics/multiplicity.ts
|
|
5
|
+
/**
|
|
6
|
+
* Multiple-comparison corrections: family-wise error (Bonferroni, Holm) and
|
|
7
|
+
* false discovery rate (Benjamini-Hochberg), with inclusive rejection
|
|
8
|
+
* boundaries throughout.
|
|
9
|
+
*/
|
|
10
|
+
/**
|
|
11
|
+
* Bonferroni adjustment: multiply every p-value by the test count, clamp at 1.
|
|
12
|
+
*
|
|
13
|
+
* Rejects at `p_adjusted ≤ alpha` — the boundary is inclusive, matching
|
|
14
|
+
* {@link holm}, which uniformly dominates this correction and must therefore
|
|
15
|
+
* never reject less. Validates its inputs on the same terms.
|
|
16
|
+
*/
|
|
17
|
+
function bonferroni(pValues, alpha = .05) {
|
|
18
|
+
assertAlpha("bonferroni", "alpha", alpha);
|
|
19
|
+
assertPValues("bonferroni", pValues);
|
|
20
|
+
const k = pValues.length;
|
|
21
|
+
const adjusted = pValues.map((p) => Math.min(1, p * k));
|
|
22
|
+
return {
|
|
23
|
+
adjusted,
|
|
24
|
+
significant: adjusted.map((p) => p <= alpha)
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Holm step-down family-wise error adjustment.
|
|
29
|
+
*
|
|
30
|
+
* P-values are sorted from smallest to largest, multiplied by their remaining
|
|
31
|
+
* hypothesis count, and made monotonically non-decreasing before being mapped
|
|
32
|
+
* back to input order. This uniformly dominates plain Bonferroni while keeping
|
|
33
|
+
* strong family-wise error control under arbitrary dependence.
|
|
34
|
+
*/
|
|
35
|
+
function holm(pValues, alpha = .05) {
|
|
36
|
+
assertAlpha("holm", "alpha", alpha);
|
|
37
|
+
assertPValues("holm", pValues);
|
|
38
|
+
const count = pValues.length;
|
|
39
|
+
if (count === 0) return {
|
|
40
|
+
adjusted: [],
|
|
41
|
+
significant: []
|
|
42
|
+
};
|
|
43
|
+
const ordered = pValues.map((pValue, index) => ({
|
|
44
|
+
pValue,
|
|
45
|
+
index
|
|
46
|
+
})).sort((a, b) => a.pValue - b.pValue || a.index - b.index);
|
|
47
|
+
const adjusted = new Array(count);
|
|
48
|
+
let previous = 0;
|
|
49
|
+
for (let rank = 0; rank < count; rank++) {
|
|
50
|
+
const entry = ordered[rank];
|
|
51
|
+
const stepAdjusted = Math.min(1, entry.pValue * (count - rank));
|
|
52
|
+
previous = Math.max(previous, stepAdjusted);
|
|
53
|
+
adjusted[entry.index] = previous;
|
|
54
|
+
}
|
|
55
|
+
return {
|
|
56
|
+
adjusted,
|
|
57
|
+
significant: adjusted.map((pValue) => pValue <= alpha)
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Benjamini–Hochberg false discovery rate. Returns adjusted q-values and
|
|
62
|
+
* significance at the target FDR; handles ties and preserves q monotonicity.
|
|
63
|
+
*
|
|
64
|
+
* Rejects at `q ≤ fdr` — the BH rule is inclusive at the boundary, so an
|
|
65
|
+
* exactly-`fdr` q-value is a discovery.
|
|
66
|
+
*/
|
|
67
|
+
function benjaminiHochberg(pValues, fdr = .05) {
|
|
68
|
+
assertAlpha("benjaminiHochberg", "fdr", fdr);
|
|
69
|
+
assertPValues("benjaminiHochberg", pValues);
|
|
70
|
+
const n = pValues.length;
|
|
71
|
+
if (n === 0) return {
|
|
72
|
+
qValues: [],
|
|
73
|
+
significant: []
|
|
74
|
+
};
|
|
75
|
+
const indexed = pValues.map((p, i) => ({
|
|
76
|
+
p,
|
|
77
|
+
i
|
|
78
|
+
})).sort((a, b) => a.p - b.p);
|
|
79
|
+
const q = new Array(n);
|
|
80
|
+
let minRight = 1;
|
|
81
|
+
for (let k = n - 1; k >= 0; k--) {
|
|
82
|
+
const rank = k + 1;
|
|
83
|
+
const entry = indexed[k];
|
|
84
|
+
const raw = n / rank * entry.p;
|
|
85
|
+
const bounded = Math.min(minRight, raw);
|
|
86
|
+
minRight = bounded;
|
|
87
|
+
q[entry.i] = Math.min(1, bounded);
|
|
88
|
+
}
|
|
89
|
+
return {
|
|
90
|
+
qValues: q,
|
|
91
|
+
significant: q.map((v) => v <= fdr)
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
function assertAlpha(fn, label, value) {
|
|
95
|
+
if (!Number.isFinite(value) || value <= 0 || value >= 1) throw new ValidationError(`${fn}: ${label} must be in (0,1), got ${value}`);
|
|
96
|
+
}
|
|
97
|
+
function assertPValues(fn, pValues) {
|
|
98
|
+
for (const [index, pValue] of pValues.entries()) if (!Number.isFinite(pValue) || pValue < 0 || pValue > 1) throw new ValidationError(`${fn}: pValues[${index}] must be in [0,1], got ${pValue}`);
|
|
99
|
+
}
|
|
100
|
+
//#endregion
|
|
101
|
+
//#region src/statistics/power-and-mde.ts
|
|
102
|
+
/**
|
|
103
|
+
* Required N per arm for a two-sample comparison at target effect size,
|
|
104
|
+
* alpha, and power. Normal-approximation formula:
|
|
105
|
+
* n = 2 * ( (z_{1-α/2} + z_{1-β}) / d )^2
|
|
106
|
+
* where d is Cohen's d. Returns Infinity for effect ≤ 0.
|
|
107
|
+
*/
|
|
108
|
+
function requiredSampleSize(opts) {
|
|
109
|
+
const effect = opts.effect;
|
|
110
|
+
if (!Number.isFinite(effect) || effect <= 0) return Infinity;
|
|
111
|
+
const alpha = opts.alpha ?? .05;
|
|
112
|
+
const power = opts.power ?? .8;
|
|
113
|
+
const n = 2 * ((zQuantile(opts.twoSided ?? true ? 1 - alpha / 2 : 1 - alpha) + zQuantile(power)) / effect) ** 2;
|
|
114
|
+
return Math.ceil(n);
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Required number of paired observations for a target Cohen's dz.
|
|
118
|
+
* Unlike the independent-groups formula, this has no two-arm factor of two.
|
|
119
|
+
*
|
|
120
|
+
* Normal quantiles with no t correction, so treat the result as a LOWER bound:
|
|
121
|
+
* it returns 32 where the exact t-based answer is 34 at dz = 0.5, and 13 where
|
|
122
|
+
* it is 15 at dz = 0.8 — a 6–13 % shortfall precisely in the range a caller
|
|
123
|
+
* consults to decide whether 3–10 repetitions suffice.
|
|
124
|
+
*/
|
|
125
|
+
function requiredPairedSampleSize(opts) {
|
|
126
|
+
const effect = opts.effect;
|
|
127
|
+
if (!Number.isFinite(effect) || effect <= 0) return Infinity;
|
|
128
|
+
const alpha = opts.alpha ?? .05;
|
|
129
|
+
const power = opts.power ?? .8;
|
|
130
|
+
const zAlpha = zQuantile(opts.twoSided ?? true ? 1 - alpha / 2 : 1 - alpha);
|
|
131
|
+
const zBeta = zQuantile(power);
|
|
132
|
+
return Math.ceil(((zAlpha + zBeta) / effect) ** 2);
|
|
133
|
+
}
|
|
134
|
+
/**
|
|
135
|
+
* Minimum detectable paired effect (standardised units) for a target paired
|
|
136
|
+
* sample size: d_min = (z_{1-α/2} + z_β) / sqrt(n_paired). Multiply by
|
|
137
|
+
* sd(deltas) for score units; treat as a lower bound — Wilcoxon and bootstrap
|
|
138
|
+
* have asymptotic relative efficiency below 1 vs the t-test on heavy tails.
|
|
139
|
+
*/
|
|
140
|
+
function pairedMde(opts) {
|
|
141
|
+
if (!Number.isFinite(opts.nPaired) || opts.nPaired <= 0) return Infinity;
|
|
142
|
+
const alpha = opts.alpha ?? .05;
|
|
143
|
+
const power = opts.power ?? .8;
|
|
144
|
+
return (zQuantile(opts.twoSided ?? true ? 1 - alpha / 2 : 1 - alpha) + zQuantile(power)) / Math.sqrt(opts.nPaired);
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* Number of paired observations needed for a McNemar test to reach a target
|
|
148
|
+
* power — the pre-registration companion to {@link mcnemar}. Parametrised by the
|
|
149
|
+
* expected discordant-cell probabilities `p10` (P[treatment wins on a pair]) and
|
|
150
|
+
* `p01` (P[control wins]); concordant pairs carry no information, so the count
|
|
151
|
+
* is driven entirely by the discordant rate. Lachin's (1992) asymptotic normal
|
|
152
|
+
* approximation: with discordant rate `pDisc = p10 + p01` and marginal effect
|
|
153
|
+
* `δ = p10 − p01`,
|
|
154
|
+
* n = ( z_{1-α/2}·√pDisc + z_{1-β}·√(pDisc − δ²) )² / δ².
|
|
155
|
+
* Returns Infinity when there is no effect (p10 === p01). Asymptotic — at the
|
|
156
|
+
* tiny discordant counts where the exact {@link mcnemar} differs from the normal
|
|
157
|
+
* approximation, treat the result as a lower bound and prefer the discordant-pair
|
|
158
|
+
* floor.
|
|
159
|
+
*/
|
|
160
|
+
function mcnemarRequiredN(opts) {
|
|
161
|
+
const { p10, p01 } = opts;
|
|
162
|
+
if (p10 < 0 || p01 < 0 || p10 + p01 > 1) throw new Error(`mcnemarRequiredN: require p10,p01 ≥ 0 and p10+p01 ≤ 1 (got ${p10}, ${p01})`);
|
|
163
|
+
const delta = p10 - p01;
|
|
164
|
+
if (delta === 0) return Infinity;
|
|
165
|
+
const alpha = opts.alpha ?? .05;
|
|
166
|
+
const power = opts.power ?? .8;
|
|
167
|
+
const twoSided = opts.twoSided ?? true;
|
|
168
|
+
const pDisc = p10 + p01;
|
|
169
|
+
const zAlpha = zQuantile(twoSided ? 1 - alpha / 2 : 1 - alpha);
|
|
170
|
+
const zBeta = zQuantile(power);
|
|
171
|
+
const n = (zAlpha * Math.sqrt(pDisc) + zBeta * Math.sqrt(Math.max(0, pDisc - delta * delta))) ** 2 / (delta * delta);
|
|
172
|
+
return Math.ceil(n);
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* Power of a McNemar test at a given number of paired observations, the inverse
|
|
176
|
+
* of {@link mcnemarRequiredN} (same Lachin asymptotic model, same parameters).
|
|
177
|
+
* Returns a value in [0, 1]; equals `alpha` when there is no effect.
|
|
178
|
+
*/
|
|
179
|
+
function mcnemarPower(opts) {
|
|
180
|
+
const { p10, p01, nPairs } = opts;
|
|
181
|
+
if (p10 < 0 || p01 < 0 || p10 + p01 > 1) throw new Error(`mcnemarPower: require p10,p01 ≥ 0 and p10+p01 ≤ 1 (got ${p10}, ${p01})`);
|
|
182
|
+
const alpha = opts.alpha ?? .05;
|
|
183
|
+
const twoSided = opts.twoSided ?? true;
|
|
184
|
+
const delta = p10 - p01;
|
|
185
|
+
if (delta === 0 || nPairs <= 0) return alpha;
|
|
186
|
+
const pDisc = p10 + p01;
|
|
187
|
+
const zAlpha = zQuantile(twoSided ? 1 - alpha / 2 : 1 - alpha);
|
|
188
|
+
const denom = Math.sqrt(Math.max(1e-12, pDisc - delta * delta));
|
|
189
|
+
const zBeta = (Math.sqrt(nPairs) * Math.abs(delta) - zAlpha * Math.sqrt(pDisc)) / denom;
|
|
190
|
+
return Math.min(1, Math.max(0, normalCdf(zBeta)));
|
|
191
|
+
}
|
|
192
|
+
//#endregion
|
|
193
|
+
export { requiredSampleSize as a, holm as c, requiredPairedSampleSize as i, mcnemarRequiredN as n, benjaminiHochberg as o, pairedMde as r, bonferroni as s, mcnemarPower as t };
|
|
194
|
+
|
|
195
|
+
//# sourceMappingURL=power-and-mde-CHIrXJll.js.map
|