@tangle-network/agent-eval 0.144.11 → 0.144.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/dist/agent-profile-B9_GGsG8.d.ts +84 -0
- package/dist/agent-profile-B9_GGsG8.d.ts.map +1 -0
- package/dist/{agent-profile-cell-BOP-iA9Q.d.ts → agent-profile-cell-BkcRDikH.d.ts} +2 -2
- package/dist/{agent-profile-cell-BOP-iA9Q.d.ts.map → agent-profile-cell-BkcRDikH.d.ts.map} +1 -1
- package/dist/analyst/index.d.ts +134 -16
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +364 -10
- package/dist/analyst/index.js.map +1 -1
- package/dist/backend-integrity-CsVin_Wb.d.ts +280 -0
- package/dist/backend-integrity-CsVin_Wb.d.ts.map +1 -0
- package/dist/{baseline-C-GocmIW.js → baseline-BhPRQBVn.js} +3 -2
- package/dist/{baseline-C-GocmIW.js.map → baseline-BhPRQBVn.js.map} +1 -1
- package/dist/{benchmark-CWeqGl7x.js → benchmark-BhT16ep9.js} +2 -2
- package/dist/{benchmark-CWeqGl7x.js.map → benchmark-BhT16ep9.js.map} +1 -1
- package/dist/{agent-profile-DPi7IZg7.d.ts → benchmark-Ceoan7vk.d.ts} +4 -91
- package/dist/benchmark-Ceoan7vk.d.ts.map +1 -0
- package/dist/{benchmark-command-BteMFN62.js → benchmark-command-CY6cbhOB.js} +36 -79
- package/dist/benchmark-command-CY6cbhOB.js.map +1 -0
- package/dist/benchmarks/index.d.ts +244 -2
- package/dist/benchmarks/index.d.ts.map +1 -0
- package/dist/benchmarks/index.js +733 -1
- package/dist/benchmarks/index.js.map +1 -0
- package/dist/builder-eval/index.d.ts +23 -2
- package/dist/builder-eval/index.d.ts.map +1 -1
- package/dist/builder-eval/index.js +227 -3
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/campaign/index.d.ts +10 -8
- package/dist/campaign/index.js +9 -6
- package/dist/{campaign-C2TTzQII.js → campaign-BYjBAypg.js} +21 -634
- package/dist/campaign-BYjBAypg.js.map +1 -0
- package/dist/{canonical-D011XM8r.js → canonical-D-XsTQ6_.js} +2 -2
- package/dist/{canonical-D011XM8r.js.map → canonical-D-XsTQ6_.js.map} +1 -1
- package/dist/{index-CQTZ-4XN.d.ts → capture-fetch-DDvpjVRU.d.ts} +3 -3
- package/dist/capture-fetch-DDvpjVRU.d.ts.map +1 -0
- package/dist/{default-registry-BmktKy8r.js → chat-client-Bp4Ebuuc.js} +1276 -1276
- package/dist/chat-client-Bp4Ebuuc.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-C9gzZE59.d.ts → client-KUbGulm_.d.ts} +4 -4
- package/dist/{client-C9gzZE59.d.ts.map → client-KUbGulm_.d.ts.map} +1 -1
- package/dist/contract/index.d.ts +13 -390
- package/dist/contract/index.d.ts.map +1 -1
- package/dist/contract/index.js +18 -542
- package/dist/contract/index.js.map +1 -1
- package/dist/{cost-ledger-DMFxsLKr.js → cost-ledger-BSe92yAV.js} +3 -3
- package/dist/{cost-ledger-DMFxsLKr.js.map → cost-ledger-BSe92yAV.js.map} +1 -1
- package/dist/{cost-ledger-Bv_e8XHY.d.ts → cost-ledger-DbQdN3nO.d.ts} +2 -2
- package/dist/{cost-ledger-Bv_e8XHY.d.ts.map → cost-ledger-DbQdN3nO.d.ts.map} +1 -1
- package/dist/{counterfactual-CxmxAONP.d.ts → counterfactual--bpysZF0.d.ts} +2 -14
- package/dist/{counterfactual-CxmxAONP.d.ts.map → counterfactual--bpysZF0.d.ts.map} +1 -1
- package/dist/{counterfactual-CWPTrMH7.js → counterfactual-lDfCx0Uz.js} +3 -31
- package/dist/{counterfactual-CWPTrMH7.js.map → counterfactual-lDfCx0Uz.js.map} +1 -1
- package/dist/{dataset-C8xaLXdY.d.ts → dataset-CJjKqQfA.d.ts} +2 -8
- package/dist/dataset-CJjKqQfA.d.ts.map +1 -0
- package/dist/{default-registry-Bf8Woqmq.d.ts → default-registry-Di6HP6pG.d.ts} +7 -12
- package/dist/default-registry-Di6HP6pG.d.ts.map +1 -0
- package/dist/{analyze-runs-C30yljDJ.js → define-agent-eval-iqjT--ZZ.js} +542 -130
- package/dist/define-agent-eval-iqjT--ZZ.js.map +1 -0
- package/dist/define-agent-eval-sH24zBfM.d.ts +388 -0
- package/dist/define-agent-eval-sH24zBfM.d.ts.map +1 -0
- package/dist/descriptive-B5MwKfbf.js +144 -0
- package/dist/descriptive-B5MwKfbf.js.map +1 -0
- package/dist/{dspy-rlm-engine-DbTk4JdR.js → dspy-rlm-engine-B_qhSc21.js} +4 -4
- package/dist/{dspy-rlm-engine-DbTk4JdR.js.map → dspy-rlm-engine-B_qhSc21.js.map} +1 -1
- package/dist/effect-sizes-DiH8MGOH.js +82 -0
- package/dist/effect-sizes-DiH8MGOH.js.map +1 -0
- package/dist/{engine-3hL-XqwJ.d.ts → engine-otFpE2gF.d.ts} +10 -38
- package/dist/engine-otFpE2gF.d.ts.map +1 -0
- package/dist/{errors-CKPfb2aH.d.ts → errors-DEE6u6ot.d.ts} +2 -14
- package/dist/{errors-CKPfb2aH.d.ts.map → errors-DEE6u6ot.d.ts.map} +1 -1
- package/dist/{errors-D-LKuDhb.js → errors-Dngq5h35.js} +2 -8
- package/dist/{errors-D-LKuDhb.js.map → errors-Dngq5h35.js.map} +1 -1
- package/dist/{eval-campaign-B_7wcnav.js → eval-campaign-UB-usSQ2.js} +6 -6
- package/dist/{eval-campaign-B_7wcnav.js.map → eval-campaign-UB-usSQ2.js.map} +1 -1
- package/dist/{exact-types-DSFFpLLI.d.ts → exact-types-BH1twmAJ.d.ts} +2 -2
- package/dist/{exact-types-DSFFpLLI.d.ts.map → exact-types-BH1twmAJ.d.ts.map} +1 -1
- package/dist/experiment/index.d.ts +9 -6
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +11 -7
- package/dist/experiment/index.js.map +1 -1
- package/dist/experiment-tracker-C29gXM4B.js +269 -0
- package/dist/experiment-tracker-C29gXM4B.js.map +1 -0
- package/dist/{experiment-tracker-IMntXr6J.d.ts → experiment-tracker-DWHZBAYL.d.ts} +77 -121
- package/dist/experiment-tracker-DWHZBAYL.d.ts.map +1 -0
- package/dist/{external-optimizer-contracts-CZuJNcT5.d.ts → external-optimizer-contracts-lixrOZdX.d.ts} +3 -3
- package/dist/{external-optimizer-contracts-CZuJNcT5.d.ts.map → external-optimizer-contracts-lixrOZdX.d.ts.map} +1 -1
- package/dist/external-optimizer-process-BTiNB-RH.js +301 -0
- package/dist/external-optimizer-process-BTiNB-RH.js.map +1 -0
- package/dist/{single-run-lock-DFWHEB09.js → external-optimizer-subprocess-DrJ9hR8u.js} +144 -438
- package/dist/external-optimizer-subprocess-DrJ9hR8u.js.map +1 -0
- package/dist/extract-usage-BrQ8mCLX.js +155 -0
- package/dist/extract-usage-BrQ8mCLX.js.map +1 -0
- package/dist/{failure-cluster-CqcvCcdR.d.ts → failure-cluster-BLURuWG4.d.ts} +2 -3
- package/dist/failure-cluster-BLURuWG4.d.ts.map +1 -0
- package/dist/{feedback-trajectory-CacHpxsp.d.ts → feedback-trajectory-juOozjAc.d.ts} +5 -36
- package/dist/feedback-trajectory-juOozjAc.d.ts.map +1 -0
- package/dist/fuzz.d.ts +2 -2
- package/dist/fuzz.js +2 -2
- package/dist/hosted/index.d.ts +3 -3
- package/dist/{index-CvSN3IG1.d.ts → index-B8Ui1mr1.d.ts} +2 -2
- package/dist/{index-CvSN3IG1.d.ts.map → index-B8Ui1mr1.d.ts.map} +1 -1
- package/dist/{skill-usage-CO9OLRBx.d.ts → index-BWDrSVfw.d.ts} +11 -136
- package/dist/index-BWDrSVfw.d.ts.map +1 -0
- package/dist/index-Ba3YrbAL.d.ts +1 -0
- package/dist/{index-BnEuDAK2.d.ts → index-COQYtuRF.d.ts} +3 -3
- package/dist/{index-BnEuDAK2.d.ts.map → index-COQYtuRF.d.ts.map} +1 -1
- package/dist/{index-C3ssXVLv.d.ts → index-CvjYbU0D.d.ts} +2 -2
- package/dist/{index-C3ssXVLv.d.ts.map → index-CvjYbU0D.d.ts.map} +1 -1
- package/dist/{index-DPPGNJ_R.d.ts → index-DSmEylT9.d.ts} +17 -115
- package/dist/index-DSmEylT9.d.ts.map +1 -0
- package/dist/index.d.ts +2397 -5308
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5914 -10496
- package/dist/index.js.map +1 -1
- package/dist/{insight-report-CgX_s0Ez.d.ts → insight-report-CRi-Ufrj.d.ts} +4 -4
- package/dist/{insight-report-CgX_s0Ez.d.ts.map → insight-report-CRi-Ufrj.d.ts.map} +1 -1
- package/dist/{integrity-MLzHOfV9.js → integrity-Cy9WHAtb.js} +2 -2
- package/dist/{integrity-MLzHOfV9.js.map → integrity-Cy9WHAtb.js.map} +1 -1
- package/dist/{integrity-DY6tIbl0.js → integrity-DysDBWDu.js} +2 -2
- package/dist/{integrity-DY6tIbl0.js.map → integrity-DysDBWDu.js.map} +1 -1
- package/dist/{integrity-BuqEKu-x.d.ts → integrity-OrcI9Nau.d.ts} +3 -3
- package/dist/{integrity-BuqEKu-x.d.ts.map → integrity-OrcI9Nau.d.ts.map} +1 -1
- package/dist/internal-BDHPCnjk.js +230 -0
- package/dist/internal-BDHPCnjk.js.map +1 -0
- package/dist/judge-calibration-C5CbMYce.d.ts +117 -0
- package/dist/judge-calibration-C5CbMYce.d.ts.map +1 -0
- package/dist/judge-calibration-DZkWrm5H.js +317 -0
- package/dist/judge-calibration-DZkWrm5H.js.map +1 -0
- package/dist/{kind-factory-B8-r8-y8.js → kind-factory-CPmSd58s.js} +208 -208
- package/dist/{kind-factory-B8-r8-y8.js.map → kind-factory-CPmSd58s.js.map} +1 -1
- package/dist/ledger-core/index.d.ts +1 -1
- package/dist/ledger-core/index.js +2 -2
- package/dist/{ledger-core-DXZIqu17.js → ledger-core-BmZt19oQ.js} +110 -110
- package/dist/{ledger-core-DXZIqu17.js.map → ledger-core-BmZt19oQ.js.map} +1 -1
- package/dist/{llm-client-DzvMUsS_.js → llm-client-d0-2TT1g.js} +4 -64
- package/dist/{llm-client-DzvMUsS_.js.map → llm-client-d0-2TT1g.js.map} +1 -1
- package/dist/{skillopt-optimization-method-CQdVeM8k.js → llm-judge-dZ8P6nGI.js} +3349 -5401
- package/dist/llm-judge-dZ8P6nGI.js.map +1 -0
- package/dist/matrix/index.d.ts +2 -2
- package/dist/meta-eval/index.d.ts +3 -3
- package/dist/meta-eval/index.js +3 -3
- package/dist/{metrics-C9YY1OcL.js → metrics-Cl0L1KUy.js} +2 -108
- package/dist/{metrics-C9YY1OcL.js.map → metrics-Cl0L1KUy.js.map} +1 -1
- package/dist/{mint-B2O60ACG.js → mint-Dj9Ww_3I.js} +4 -4
- package/dist/{mint-B2O60ACG.js.map → mint-Dj9Ww_3I.js.map} +1 -1
- package/dist/{multi-layer-verifier-DnAqwl0h.d.ts → multi-layer-verifier-DIguZc8Z.d.ts} +3 -3
- package/dist/{multi-layer-verifier-DnAqwl0h.d.ts.map → multi-layer-verifier-DIguZc8Z.d.ts.map} +1 -1
- package/dist/multiplicity-DIWHvysC.d.ts +43 -0
- package/dist/multiplicity-DIWHvysC.d.ts.map +1 -0
- package/dist/multishot/index.d.ts +3 -3
- package/dist/multishot/index.js +1 -1
- package/dist/openapi.json +1 -1
- package/dist/{opencode-sqlite-8r6WUfHc.js → opencode-sqlite-DJWAXLms.js} +2 -2
- package/dist/{opencode-sqlite-8r6WUfHc.js.map → opencode-sqlite-DJWAXLms.js.map} +1 -1
- package/dist/package-version-D7lQHt_-.js +34 -0
- package/dist/package-version-D7lQHt_-.js.map +1 -0
- package/dist/paired-arms-D-XRF_fy.js +1045 -0
- package/dist/paired-arms-D-XRF_fy.js.map +1 -0
- package/dist/paired-promotion-decision-CGzg0cI_.d.ts +251 -0
- package/dist/paired-promotion-decision-CGzg0cI_.d.ts.map +1 -0
- package/dist/paired-tests-BHIhYVdu.js +213 -0
- package/dist/paired-tests-BHIhYVdu.js.map +1 -0
- package/dist/pareto-BqNW3LJR.d.ts +117 -0
- package/dist/pareto-BqNW3LJR.d.ts.map +1 -0
- package/dist/pipelines/index.d.ts +3 -64
- package/dist/pipelines/index.d.ts.map +1 -1
- package/dist/pipelines/index.js +4 -284
- package/dist/pipelines/index.js.map +1 -1
- package/dist/power-and-mde-CHIrXJll.js +195 -0
- package/dist/power-and-mde-CHIrXJll.js.map +1 -0
- package/dist/{promotion-policy-CrLrmys8.js → power-preflight-DEw-uC7q.js} +4 -184
- package/dist/power-preflight-DEw-uC7q.js.map +1 -0
- package/dist/pre-registration-CZwSFQS4.d.ts +577 -0
- package/dist/pre-registration-CZwSFQS4.d.ts.map +1 -0
- package/dist/{prime-protocol-BfSalTfR.js → prime-protocol-6tZTVsWm.js} +72 -21
- package/dist/prime-protocol-6tZTVsWm.js.map +1 -0
- package/dist/produced-state-DU79a81m.js +586 -0
- package/dist/produced-state-DU79a81m.js.map +1 -0
- package/dist/profile-cell.d.ts +1 -1
- package/dist/profile-cell.js +1 -1
- package/dist/promotion-policy-D0nPhkSy.d.ts +134 -0
- package/dist/promotion-policy-D0nPhkSy.d.ts.map +1 -0
- package/dist/promotion-policy-xzA40Evo.js +186 -0
- package/dist/promotion-policy-xzA40Evo.js.map +1 -0
- package/dist/{skillopt-optimization-method-USDKhxSA.d.ts → provenance-CCdxgLDT.d.ts} +57 -478
- package/dist/provenance-CCdxgLDT.d.ts.map +1 -0
- package/dist/registry-oJeeI4-a.d.ts +178 -0
- package/dist/registry-oJeeI4-a.d.ts.map +1 -0
- package/dist/{release-report-DA2BCu5a.d.ts → release-confidence-BFRE5WSp.d.ts} +180 -112
- package/dist/release-confidence-BFRE5WSp.d.ts.map +1 -0
- package/dist/{release-report-BUYmoKo2.js → release-confidence-CxDuiAev.js} +125 -217
- package/dist/release-confidence-CxDuiAev.js.map +1 -0
- package/dist/reporting.d.ts +6 -5
- package/dist/reporting.js +7 -5
- package/dist/{researcher-BchpD55R.d.ts → researcher-DaN4GST-.d.ts} +7 -40
- package/dist/{researcher-BchpD55R.d.ts.map → researcher-DaN4GST-.d.ts.map} +1 -1
- package/dist/{reward-hacking-BDToousL.js → reward-hacking-DNgjilrV.js} +3 -3
- package/dist/reward-hacking-DNgjilrV.js.map +1 -0
- package/dist/{reward-hacking-BI0OMAlo.d.ts → reward-hacking-DSSTuI9r.d.ts} +5 -5
- package/dist/{reward-hacking-BI0OMAlo.d.ts.map → reward-hacking-DSSTuI9r.d.ts.map} +1 -1
- package/dist/rl.d.ts +7 -7
- package/dist/rl.js +11 -10
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +2 -2
- package/dist/rollout/index.js +4 -4
- package/dist/{rollout-DRrksrcV.js → rollout-BWtw0I_6.js} +3 -3
- package/dist/{rollout-DRrksrcV.js.map → rollout-BWtw0I_6.js.map} +1 -1
- package/dist/{rubric-predictive-validity-DvkjPYCe.d.ts → rubric-predictive-validity-BgxtKe4G.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-DvkjPYCe.d.ts.map → rubric-predictive-validity-BgxtKe4G.d.ts.map} +1 -1
- package/dist/{rubric-predictive-validity-BRR632r1.js → rubric-predictive-validity-Cwwyd7ah.js} +2 -2
- package/dist/{rubric-predictive-validity-BRR632r1.js.map → rubric-predictive-validity-Cwwyd7ah.js.map} +1 -1
- package/dist/{run-record-BmSPWXJR.js → run-record-BvHPVS-i.js} +3 -3
- package/dist/{run-record-BmSPWXJR.js.map → run-record-BvHPVS-i.js.map} +1 -1
- package/dist/{run-record-CF4Dwpxr.d.ts → run-record-CKiihE6f.d.ts} +4 -4
- package/dist/{run-record-CF4Dwpxr.d.ts.map → run-record-CKiihE6f.d.ts.map} +1 -1
- package/dist/{proposal-findings-2GIUo1et.js → run-score-lDzV0X8j.js} +31 -31
- package/dist/run-score-lDzV0X8j.js.map +1 -0
- package/dist/sandbox-harness-BlSOu4LX.d.ts +70 -0
- package/dist/sandbox-harness-BlSOu4LX.d.ts.map +1 -0
- package/dist/{schema-Cef2cFmb2.d.ts → schema-Cef2cFmb.d.ts} +1 -1
- package/dist/schema-Cef2cFmb.d.ts.map +1 -0
- package/dist/semantic-concept-judge-BiJxScqe.js +406 -0
- package/dist/semantic-concept-judge-BiJxScqe.js.map +1 -0
- package/dist/{sequential-D-BLJBKU.js → sequential-C458DXNf.js} +4 -3
- package/dist/{sequential-D-BLJBKU.js.map → sequential-C458DXNf.js.map} +1 -1
- package/dist/sequential-eprocess-CbUt2htw.js +83 -0
- package/dist/sequential-eprocess-CbUt2htw.js.map +1 -0
- package/dist/series-convergence-D1cL1f-4.d.ts +129 -0
- package/dist/series-convergence-D1cL1f-4.d.ts.map +1 -0
- package/dist/{server-iu0ede49.js → server-ulsOdrTI.js} +5 -21
- package/dist/server-ulsOdrTI.js.map +1 -0
- package/dist/skillopt-optimization-method-C5cotF4E.d.ts +427 -0
- package/dist/skillopt-optimization-method-C5cotF4E.d.ts.map +1 -0
- package/dist/skillopt-optimization-method-jjdnc3YK.js +1999 -0
- package/dist/skillopt-optimization-method-jjdnc3YK.js.map +1 -0
- package/dist/{statistical-heldout-TQ-4CYiN.d.ts → statistical-heldout-DoFF5KJX.d.ts} +5 -278
- package/dist/statistical-heldout-DoFF5KJX.d.ts.map +1 -0
- package/dist/{store-otlp-Dw8PPIlL.js → store-otlp-CsptLYpN.js} +3 -3
- package/dist/{store-otlp-Dw8PPIlL.js.map → store-otlp-CsptLYpN.js.map} +1 -1
- package/dist/store-tool-spans-Cq9mFd-q.js +667 -0
- package/dist/store-tool-spans-Cq9mFd-q.js.map +1 -0
- package/dist/store-tool-spans-DbDLOBOb.d.ts +342 -0
- package/dist/store-tool-spans-DbDLOBOb.d.ts.map +1 -0
- package/dist/student-t-CvBq2mve.js +38 -0
- package/dist/student-t-CvBq2mve.js.map +1 -0
- package/dist/{summary-report-Lf-5I7xh.js → summary-report-Blysd6Z2.js} +6 -3
- package/dist/{summary-report-Lf-5I7xh.js.map → summary-report-Blysd6Z2.js.map} +1 -1
- package/dist/{summary-report-BNR7DWTj.d.ts → summary-report-CaL-Hnxt.d.ts} +5 -5
- package/dist/{summary-report-BNR7DWTj.d.ts.map → summary-report-CaL-Hnxt.d.ts.map} +1 -1
- package/dist/supervisor-run/index.d.ts +391 -3
- package/dist/supervisor-run/index.d.ts.map +1 -0
- package/dist/supervisor-run/index.js +1689 -2
- package/dist/{supervisor-run-D_sokXcO.js.map → supervisor-run/index.js.map} +1 -1
- package/dist/{extract-usage-CdZdoj1s.js → task-failure-attributes-CpQ4y5RD.js} +5 -157
- package/dist/task-failure-attributes-CpQ4y5RD.js.map +1 -0
- package/dist/{tool-groups-CZPGGlHf.d.ts → tool-groups-Cteb03Ps.d.ts} +3 -3
- package/dist/tool-groups-Cteb03Ps.d.ts.map +1 -0
- package/dist/tool-waste-BDdBZG1F.js +803 -0
- package/dist/tool-waste-BDdBZG1F.js.map +1 -0
- package/dist/tool-waste-DjRDEsuI.d.ts +128 -0
- package/dist/tool-waste-DjRDEsuI.d.ts.map +1 -0
- package/dist/trace-repair/index.d.ts +14 -5
- package/dist/trace-repair/index.d.ts.map +1 -1
- package/dist/trace-repair/index.js +35 -7
- package/dist/trace-repair/index.js.map +1 -1
- package/dist/traces.d.ts +406 -7
- package/dist/traces.d.ts.map +1 -0
- package/dist/traces.js +1011 -10
- package/dist/traces.js.map +1 -0
- package/dist/trajectory-replay/index.d.ts +16 -3
- package/dist/trajectory-replay/index.d.ts.map +1 -1
- package/dist/trajectory-replay/index.js +52 -5
- package/dist/trajectory-replay/index.js.map +1 -1
- package/dist/types-BEPZc6eo.d.ts +93 -0
- package/dist/types-BEPZc6eo.d.ts.map +1 -0
- package/dist/{usage-receipt-t7vAzCRQ.js → types-BI4fT3HN.js} +67 -67
- package/dist/types-BI4fT3HN.js.map +1 -0
- package/dist/{types-BnjdJ70P.d.ts → types-BZ59Ahr8.d.ts} +5 -5
- package/dist/{types-BnjdJ70P.d.ts.map → types-BZ59Ahr8.d.ts.map} +1 -1
- package/dist/{types-D216SgwM.d.ts → types-Cx3YUh2r.d.ts} +4 -241
- package/dist/types-Cx3YUh2r.d.ts.map +1 -0
- package/dist/{types-BvDKaULh.d.ts → types-DLQx4mKU.d.ts} +4 -4
- package/dist/{types-BvDKaULh.d.ts.map → types-DLQx4mKU.d.ts.map} +1 -1
- package/dist/{types-D4mog56g.d.ts → types-yLK8gXE9.d.ts} +2 -2
- package/dist/{types-D4mog56g.d.ts.map → types-yLK8gXE9.d.ts.map} +1 -1
- package/dist/verdict-BndeTAh_.js +61 -0
- package/dist/verdict-BndeTAh_.js.map +1 -0
- package/dist/verdict-E4eRNf7-.d.ts +392 -0
- package/dist/verdict-E4eRNf7-.d.ts.map +1 -0
- package/dist/{verdict-cache-BCcOh0kF.js → verdict-cache-mZf5FEiY.js} +3 -55
- package/dist/{verdict-cache-BCcOh0kF.js.map → verdict-cache-mZf5FEiY.js.map} +1 -1
- package/dist/wire/index.d.ts +3 -3
- package/dist/wire/index.d.ts.map +1 -1
- package/dist/wire/index.js +1 -1
- package/docs/charter.md +3 -3
- package/docs/control-runtime.md +3 -42
- package/docs/experiment.md +0 -1
- package/docs/feature-guide.md +2 -2
- package/docs/trace-repair-grader.md +1 -0
- package/docs/trajectory-replay.md +1 -0
- package/docs/verdicts.md +43 -0
- package/docs/verification-strategies.md +3 -2
- package/package.json +6 -11
- package/dist/agent-profile-DPi7IZg7.d.ts.map +0 -1
- package/dist/analyze-runs-C30yljDJ.js.map +0 -1
- package/dist/baseline-CavEbRyH.d.ts +0 -136
- package/dist/baseline-CavEbRyH.d.ts.map +0 -1
- package/dist/benchmark-command-BteMFN62.js.map +0 -1
- package/dist/benchmarks-Dzs8CKb1.js +0 -755
- package/dist/benchmarks-Dzs8CKb1.js.map +0 -1
- package/dist/campaign-C2TTzQII.js.map +0 -1
- package/dist/completion-verifier-DJA5BhPb.d.ts +0 -414
- package/dist/completion-verifier-DJA5BhPb.d.ts.map +0 -1
- package/dist/control.d.ts +0 -3
- package/dist/control.js +0 -2
- package/dist/dataset-C8xaLXdY.d.ts.map +0 -1
- package/dist/default-registry-Bf8Woqmq.d.ts.map +0 -1
- package/dist/default-registry-BmktKy8r.js.map +0 -1
- package/dist/engine-3hL-XqwJ.d.ts.map +0 -1
- package/dist/experiment-tracker-CnRICnMl.js +0 -500
- package/dist/experiment-tracker-CnRICnMl.js.map +0 -1
- package/dist/experiment-tracker-IMntXr6J.d.ts.map +0 -1
- package/dist/extract-usage-CdZdoj1s.js.map +0 -1
- package/dist/failure-cluster-CqcvCcdR.d.ts.map +0 -1
- package/dist/feedback-trajectory-CacHpxsp.d.ts.map +0 -1
- package/dist/index-BZ3-y4YL.d.ts +0 -391
- package/dist/index-BZ3-y4YL.d.ts.map +0 -1
- package/dist/index-CQTZ-4XN.d.ts.map +0 -1
- package/dist/index-DPPGNJ_R.d.ts.map +0 -1
- package/dist/index-YE4KdKbO2.d.ts +0 -335
- package/dist/index-YE4KdKbO2.d.ts.map +0 -1
- package/dist/paired-arms-iZ08VFMN.js +0 -260
- package/dist/paired-arms-iZ08VFMN.js.map +0 -1
- package/dist/paired-promotion-decision-B6zJ3gYM.d.ts +0 -114
- package/dist/paired-promotion-decision-B6zJ3gYM.d.ts.map +0 -1
- package/dist/prime-protocol-BfSalTfR.js.map +0 -1
- package/dist/promotion-policy-Ckjhzg_4.d.ts +0 -289
- package/dist/promotion-policy-Ckjhzg_4.d.ts.map +0 -1
- package/dist/promotion-policy-CrLrmys8.js.map +0 -1
- package/dist/proposal-findings-2GIUo1et.js.map +0 -1
- package/dist/propose-review-control-dSNPjFUH.js +0 -1458
- package/dist/propose-review-control-dSNPjFUH.js.map +0 -1
- package/dist/release-report-BUYmoKo2.js.map +0 -1
- package/dist/release-report-DA2BCu5a.d.ts.map +0 -1
- package/dist/replay-CohS93nE.js +0 -1859
- package/dist/replay-CohS93nE.js.map +0 -1
- package/dist/replay-DbhZ4Ked.d.ts +0 -834
- package/dist/replay-DbhZ4Ked.d.ts.map +0 -1
- package/dist/reward-hacking-BDToousL.js.map +0 -1
- package/dist/run-evidence-Dhi3C81V.d.ts +0 -225
- package/dist/run-evidence-Dhi3C81V.d.ts.map +0 -1
- package/dist/schema-Cef2cFmb2.d.ts.map +0 -1
- package/dist/semantic-concept-judge-D1z-KepS.js +0 -767
- package/dist/semantic-concept-judge-D1z-KepS.js.map +0 -1
- package/dist/series-convergence-ofsqPWhs.d.ts +0 -35
- package/dist/series-convergence-ofsqPWhs.d.ts.map +0 -1
- package/dist/server-iu0ede49.js.map +0 -1
- package/dist/single-run-lock-DFWHEB09.js.map +0 -1
- package/dist/skill-usage-CO9OLRBx.d.ts.map +0 -1
- package/dist/skillopt-optimization-method-CQdVeM8k.js.map +0 -1
- package/dist/skillopt-optimization-method-USDKhxSA.d.ts.map +0 -1
- package/dist/statistical-heldout-TQ-4CYiN.d.ts.map +0 -1
- package/dist/statistics-ByxzSiOM.js +0 -2212
- package/dist/statistics-ByxzSiOM.js.map +0 -1
- package/dist/statistics-D6Uebe_4.d.ts +0 -968
- package/dist/statistics-D6Uebe_4.d.ts.map +0 -1
- package/dist/supervisor-run-D_sokXcO.js +0 -1690
- package/dist/test-graded-scenario-D1TaI2va.d.ts +0 -141
- package/dist/test-graded-scenario-D1TaI2va.d.ts.map +0 -1
- package/dist/test-graded-scenario-JHcKQNpq.js +0 -318
- package/dist/test-graded-scenario-JHcKQNpq.js.map +0 -1
- package/dist/tool-groups-CZPGGlHf.d.ts.map +0 -1
- package/dist/tool-use-metrics-DEGMKycK.js +0 -370
- package/dist/tool-use-metrics-DEGMKycK.js.map +0 -1
- package/dist/types-D216SgwM.d.ts.map +0 -1
- package/dist/usage-receipt-t7vAzCRQ.js.map +0 -1
- package/dist/verdict-DExhxfgR.d.ts +0 -201
- package/dist/verdict-DExhxfgR.d.ts.map +0 -1
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { b as CustomTokenPricing, c as CostLedgerHandle,
|
|
1
|
+
import { t as AgentEvalError } from "./errors-DEE6u6ot.js";
|
|
2
|
+
import { b as CustomTokenPricing, c as CostLedgerHandle, g as CostReceiptInput, x as MaximumCharge } from "./cost-ledger-DbQdN3nO.js";
|
|
3
3
|
//#region src/trace/raw-provider-sink.d.ts
|
|
4
4
|
/**
|
|
5
5
|
* RawProviderSink — first-class persistence for the actual HTTP-level
|
|
@@ -180,23 +180,6 @@ declare class CrossFamilyError extends Error {
|
|
|
180
180
|
declare function assertCrossFamily(models: string[], opts?: AssertCrossFamilyOptions): JudgeFamily[];
|
|
181
181
|
//#endregion
|
|
182
182
|
//#region src/integrity/served-model.d.ts
|
|
183
|
-
/**
|
|
184
|
-
* Output-token budget a liveness probe must grant a model.
|
|
185
|
-
*
|
|
186
|
-
* Identity is only readable off a response the provider actually produced. A
|
|
187
|
-
* reasoning model spends budget on hidden reasoning tokens before it emits a
|
|
188
|
-
* single visible one, so a cap of a few tokens makes a HEALTHY deepseek/glm
|
|
189
|
-
* model fail with `reasoning_budget_exhausted` — it names no model, its
|
|
190
|
-
* identity reads as `unreported`, and a preflight scores it DEAD. 64 clears
|
|
191
|
-
* that floor. Every probe in this package reads this constant, so two probes
|
|
192
|
-
* cannot reach two different answers about the same router.
|
|
193
|
-
*
|
|
194
|
-
* Cost: a probe spends at most `PROBE_MAX_TOKENS` output tokens per model,
|
|
195
|
-
* plus whatever reasoning tokens a reasoning model bills — roughly 400 output
|
|
196
|
-
* tokens for a six-model preflight. That is fractions of a cent, and far
|
|
197
|
-
* cheaper than a campaign that runs on a model nobody proved was alive.
|
|
198
|
-
*/
|
|
199
|
-
declare const PROBE_MAX_TOKENS = 64;
|
|
200
183
|
/** How a served id relates to the id that was requested. */
|
|
201
184
|
type ServedModelVerdict =
|
|
202
185
|
/** Byte-identical after normalisation — the requested model answered. */
|
|
@@ -221,37 +204,6 @@ interface ServedModelCheck {
|
|
|
221
204
|
/** True for every verdict except `exact` and `alias`. */
|
|
222
205
|
substituted: boolean;
|
|
223
206
|
}
|
|
224
|
-
/**
|
|
225
|
-
* Reduce a model id to its comparable core: lowercase, no surrounding space,
|
|
226
|
-
* no `provider/` prefix, no `@snapshot` / `:batch` / `:free` tier suffix, no
|
|
227
|
-
* trailing build date, and `.`/`_` folded to `-` so one version is spelled one
|
|
228
|
-
* way.
|
|
229
|
-
*
|
|
230
|
-
* Dropping the build date is what makes snapshot resolution legible as the
|
|
231
|
-
* non-event it is: a router answering `gpt-4o-mini` with
|
|
232
|
-
* `gpt-4o-mini-2024-07-18` pinned a floating alias to a reproducible build —
|
|
233
|
-
* the same model, which is the behaviour we want. Only routing decoration is
|
|
234
|
-
* stripped; version digits are load-bearing, so `deepseek-v3.2` and
|
|
235
|
-
* `deepseek-v4-flash` stay distinct, and comparison is EXACT equality rather
|
|
236
|
-
* than a prefix test (a prefix rule would accept `gpt-5` → `gpt-5-mini`, a
|
|
237
|
-
* silent downgrade wearing the right vendor name).
|
|
238
|
-
*/
|
|
239
|
-
declare function normalizeModelId(modelId: string): string;
|
|
240
|
-
/**
|
|
241
|
-
* Classify one requested/served pair. Pure — no I/O — so it is safe inside
|
|
242
|
-
* response handlers, reducers, and CI gates.
|
|
243
|
-
*
|
|
244
|
-
* `served` is the id echoed by the provider (OpenAI-compatible bodies put it
|
|
245
|
-
* at `model`). `null`/`undefined` means the body omitted it; that is
|
|
246
|
-
* `unreported`, NOT a pass — a provider that does not name what answered has
|
|
247
|
-
* not proven identity, and a transport that drops the field must not read as
|
|
248
|
-
* agreement.
|
|
249
|
-
*/
|
|
250
|
-
declare function checkServedModel(requested: string, served: string | null | undefined): ServedModelCheck;
|
|
251
|
-
declare class ModelSubstitutionError extends AgentEvalError {
|
|
252
|
-
readonly checks: ReadonlyArray<ServedModelCheck>;
|
|
253
|
-
constructor(message: string, checks: ReadonlyArray<ServedModelCheck>);
|
|
254
|
-
}
|
|
255
207
|
interface AssertServedModelOptions {
|
|
256
208
|
/**
|
|
257
209
|
* Accept a different model of the same provider family (e.g. requested
|
|
@@ -267,50 +219,6 @@ interface AssertServedModelOptions {
|
|
|
267
219
|
/** Prefixed to the thrown message, e.g. the judge or campaign cell name. */
|
|
268
220
|
context?: string;
|
|
269
221
|
}
|
|
270
|
-
/**
|
|
271
|
-
* The one place the accept/reject policy lives, so a caller that reports
|
|
272
|
-
* substitution (a preflight table, a run record) and a caller that throws on it
|
|
273
|
-
* can never drift apart. A cross-family substitution is never acceptable.
|
|
274
|
-
*/
|
|
275
|
-
declare function servedModelAcceptable(check: ServedModelCheck, opts?: AssertServedModelOptions): boolean;
|
|
276
|
-
/**
|
|
277
|
-
* Throw `ModelSubstitutionError` unless the served id is the requested model.
|
|
278
|
-
* Returns the check on success so callers can record the served id alongside
|
|
279
|
-
* the result.
|
|
280
|
-
*/
|
|
281
|
-
declare function assertServedModel(requested: string, served: string | null | undefined, opts?: AssertServedModelOptions): ServedModelCheck;
|
|
282
|
-
/**
|
|
283
|
-
* Batch form: check every pair and throw naming EVERY substitution, so one
|
|
284
|
-
* failure does not hide the rest. Returns all checks on success.
|
|
285
|
-
*/
|
|
286
|
-
declare function assertServedModels(pairs: ReadonlyArray<{
|
|
287
|
-
requested: string;
|
|
288
|
-
served: string | null | undefined;
|
|
289
|
-
}>, opts?: AssertServedModelOptions): ServedModelCheck[];
|
|
290
|
-
interface AssertCrossFamilyServedOptions extends AssertServedModelOptions {
|
|
291
|
-
/** Minimum distinct SERVED families required. Default 2. */
|
|
292
|
-
minFamilies?: number;
|
|
293
|
-
/** Count `unknown`-family served ids toward the total. Default false. */
|
|
294
|
-
allowUnknown?: boolean;
|
|
295
|
-
}
|
|
296
|
-
declare class ServedCrossFamilyError extends AgentEvalError {
|
|
297
|
-
readonly families: JudgeFamily[];
|
|
298
|
-
readonly checks: ReadonlyArray<ServedModelCheck>;
|
|
299
|
-
constructor(message: string, families: JudgeFamily[], checks: ReadonlyArray<ServedModelCheck>);
|
|
300
|
-
}
|
|
301
|
-
/**
|
|
302
|
-
* Family-diversity rule over the models that actually ANSWERED.
|
|
303
|
-
*
|
|
304
|
-
* `assertCrossFamily` (../judge-families) reads the requested ids and so
|
|
305
|
-
* cannot see a gateway that answers three "different" requests from one
|
|
306
|
-
* provider. This one asserts no substitution first, then counts families from
|
|
307
|
-
* the served ids — a panel that collapsed to one family under the hood fails
|
|
308
|
-
* here even though its request list looked diverse.
|
|
309
|
-
*/
|
|
310
|
-
declare function assertCrossFamilyServed(pairs: ReadonlyArray<{
|
|
311
|
-
requested: string;
|
|
312
|
-
served: string | null | undefined;
|
|
313
|
-
}>, opts?: AssertCrossFamilyServedOptions): JudgeFamily[];
|
|
314
222
|
//#endregion
|
|
315
223
|
//#region src/llm-client.d.ts
|
|
316
224
|
interface LlmMessage {
|
|
@@ -529,8 +437,6 @@ interface LlmClientOptions {
|
|
|
529
437
|
* consistently across transports.
|
|
530
438
|
*/
|
|
531
439
|
declare function isTransientLlmError(err: unknown): boolean;
|
|
532
|
-
/** Exponential backoff: 500ms, 1s, 2s, 4s, ... capped at 16s. Attempt is 0-indexed. */
|
|
533
|
-
declare function backoffMs(attempt: number): number;
|
|
534
440
|
/**
|
|
535
441
|
* Strip a ```json / ``` code fence if the model emitted one.
|
|
536
442
|
* Idempotent for naked JSON. Some models (claude-code via router, certain
|
|
@@ -553,12 +459,6 @@ declare function callLlmJson<T = unknown>(req: LlmCallRequest, opts?: LlmClientO
|
|
|
553
459
|
value: T;
|
|
554
460
|
result: LlmCallResult;
|
|
555
461
|
}>;
|
|
556
|
-
type LlmRouteAssertionReason = 'no_explicit_base_url' | 'base_url_blocked' | 'base_url_not_allowed' | 'no_auth' | 'wrong_provider';
|
|
557
|
-
declare class LlmRouteAssertionError extends CaptureIntegrityError {
|
|
558
|
-
readonly reason: LlmRouteAssertionReason;
|
|
559
|
-
readonly baseUrl: string;
|
|
560
|
-
constructor(message: string, reason: LlmRouteAssertionReason, baseUrl: string);
|
|
561
|
-
}
|
|
562
462
|
interface LlmRouteRequirements {
|
|
563
463
|
/**
|
|
564
464
|
* Throw if `opts.baseUrl` is undefined, i.e. the call would fall back to
|
|
@@ -738,11 +638,6 @@ interface ArtifactCheck {
|
|
|
738
638
|
minCount?: number;
|
|
739
639
|
description: string;
|
|
740
640
|
}
|
|
741
|
-
interface JudgeConfig {
|
|
742
|
-
model: string;
|
|
743
|
-
temperature: number;
|
|
744
|
-
rubric: JudgeRubric;
|
|
745
|
-
}
|
|
746
641
|
interface JudgeRubric {
|
|
747
642
|
name: string;
|
|
748
643
|
description: string;
|
|
@@ -755,19 +650,6 @@ interface RubricDimension {
|
|
|
755
650
|
anchor_high: string;
|
|
756
651
|
weight: number;
|
|
757
652
|
}
|
|
758
|
-
interface ScenarioResult {
|
|
759
|
-
scenarioId: string;
|
|
760
|
-
persona: string;
|
|
761
|
-
turns: TurnResult[];
|
|
762
|
-
artifactResults: ArtifactResult[];
|
|
763
|
-
judgeScores: JudgeScore[];
|
|
764
|
-
judgeErrors: number;
|
|
765
|
-
overallScore: number;
|
|
766
|
-
totalDurationMs: number;
|
|
767
|
-
artifacts: CollectedArtifacts;
|
|
768
|
-
/** Agent and judge spend attributed to this scenario. */
|
|
769
|
-
cost?: CostLedgerSummary;
|
|
770
|
-
}
|
|
771
653
|
interface TurnResult {
|
|
772
654
|
turnIndex: number;
|
|
773
655
|
userMessage: string;
|
|
@@ -780,11 +662,6 @@ interface TurnResult {
|
|
|
780
662
|
containsCode: boolean;
|
|
781
663
|
containsToolCall: boolean;
|
|
782
664
|
}
|
|
783
|
-
interface ArtifactResult {
|
|
784
|
-
check: ArtifactCheck;
|
|
785
|
-
passed: boolean;
|
|
786
|
-
detail?: string;
|
|
787
|
-
}
|
|
788
665
|
interface JudgeScore {
|
|
789
666
|
judgeName: string;
|
|
790
667
|
dimension: string;
|
|
@@ -807,36 +684,6 @@ interface CollectedArtifacts {
|
|
|
807
684
|
}[];
|
|
808
685
|
toolCalls: string[];
|
|
809
686
|
}
|
|
810
|
-
interface BenchmarkReport {
|
|
811
|
-
timestamp: string;
|
|
812
|
-
generation: number;
|
|
813
|
-
promptVersion: string;
|
|
814
|
-
scenarioCount: number;
|
|
815
|
-
results: ScenarioResult[];
|
|
816
|
-
cost?: CostLedgerSummary;
|
|
817
|
-
summary: {
|
|
818
|
-
overallAvg: number;
|
|
819
|
-
byPersona: Record<string, {
|
|
820
|
-
avg: number;
|
|
821
|
-
passed: number;
|
|
822
|
-
total: number;
|
|
823
|
-
}>;
|
|
824
|
-
byDimension: Record<string, {
|
|
825
|
-
avg: number;
|
|
826
|
-
scores: number[];
|
|
827
|
-
}>;
|
|
828
|
-
weakest: {
|
|
829
|
-
scenario: string;
|
|
830
|
-
score: number;
|
|
831
|
-
reason: string;
|
|
832
|
-
}[];
|
|
833
|
-
strongest: {
|
|
834
|
-
scenario: string;
|
|
835
|
-
score: number;
|
|
836
|
-
reason: string;
|
|
837
|
-
}[];
|
|
838
|
-
};
|
|
839
|
-
}
|
|
840
687
|
interface RouteMap {
|
|
841
688
|
signup?: string;
|
|
842
689
|
login?: string;
|
|
@@ -856,23 +703,6 @@ interface ProductClientConfig {
|
|
|
856
703
|
/** Per-request timeout in ms before the request is aborted. Default 30s. */
|
|
857
704
|
timeoutMs?: number;
|
|
858
705
|
}
|
|
859
|
-
interface ScenarioFile {
|
|
860
|
-
id: string;
|
|
861
|
-
category: string;
|
|
862
|
-
persona: string;
|
|
863
|
-
label: string;
|
|
864
|
-
thesis: string;
|
|
865
|
-
isControl?: boolean;
|
|
866
|
-
rubric?: {
|
|
867
|
-
dimensions: {
|
|
868
|
-
name: string;
|
|
869
|
-
description: string;
|
|
870
|
-
weight: number;
|
|
871
|
-
}[];
|
|
872
|
-
};
|
|
873
|
-
turns: Turn[];
|
|
874
|
-
artifactChecks: ArtifactCheck[];
|
|
875
|
-
}
|
|
876
706
|
interface CompletionCriterion {
|
|
877
707
|
name: string;
|
|
878
708
|
check: (state: DriverState) => boolean;
|
|
@@ -934,59 +764,6 @@ interface DriverState {
|
|
|
934
764
|
codeBlocks: number;
|
|
935
765
|
generations: number;
|
|
936
766
|
}
|
|
937
|
-
interface TurnMetrics {
|
|
938
|
-
turn: number;
|
|
939
|
-
timestamp: string;
|
|
940
|
-
tasks: number;
|
|
941
|
-
events: number;
|
|
942
|
-
proposals: {
|
|
943
|
-
pending: number;
|
|
944
|
-
approved: number;
|
|
945
|
-
rejected: number;
|
|
946
|
-
};
|
|
947
|
-
vaultFiles: number;
|
|
948
|
-
responseLatencyMs: number;
|
|
949
|
-
responseChars: number;
|
|
950
|
-
codeBlocksProduced: number;
|
|
951
|
-
blocksExtracted: number;
|
|
952
|
-
qualityScore?: number;
|
|
953
|
-
inputTokens: number;
|
|
954
|
-
outputTokens: number;
|
|
955
|
-
estimatedCostUsd: number;
|
|
956
|
-
totalCostUsd: number;
|
|
957
|
-
completionPercent: number;
|
|
958
|
-
}
|
|
959
|
-
interface DriverResult {
|
|
960
|
-
personaId: string;
|
|
961
|
-
/** True when the simulated user professionally signed off (driver said DONE). */
|
|
962
|
-
completed: boolean;
|
|
963
|
-
/** Turn at which the simulated user signed off, or null if it never did. */
|
|
964
|
-
turnsToCompletion: number | null;
|
|
965
|
-
/**
|
|
966
|
-
* Turn at which nominal completionCriteria were first all met, or null.
|
|
967
|
-
* Distinct from turnsToCompletion: criteria can be met while the
|
|
968
|
-
* simulated professional is still unsatisfied with the work's rigor.
|
|
969
|
-
*/
|
|
970
|
-
criteriaMetAtTurn: number | null;
|
|
971
|
-
totalTurns: number;
|
|
972
|
-
metrics: TurnMetrics[];
|
|
973
|
-
finalState: DriverState;
|
|
974
|
-
convergenceCurve: number[];
|
|
975
|
-
totalCostUsd: number;
|
|
976
|
-
finalQualityScore: number | null;
|
|
977
|
-
}
|
|
978
|
-
interface BenchmarkRunnerConfig {
|
|
979
|
-
scenarios: Scenario[];
|
|
980
|
-
judges: JudgeFn[];
|
|
981
|
-
systemPrompt: string;
|
|
982
|
-
model?: string;
|
|
983
|
-
judgeModel?: string;
|
|
984
|
-
passThreshold?: number;
|
|
985
|
-
generation?: number;
|
|
986
|
-
promptVersion?: string;
|
|
987
|
-
/** Shared ledger for agent and judge calls made by the benchmark. */
|
|
988
|
-
costLedger?: CostLedgerHandle;
|
|
989
|
-
}
|
|
990
767
|
interface JudgeInput {
|
|
991
768
|
scenario: Scenario;
|
|
992
769
|
turns: TurnResult[];
|
|
@@ -998,26 +775,12 @@ interface JudgeInput {
|
|
|
998
775
|
signal?: AbortSignal;
|
|
999
776
|
}
|
|
1000
777
|
type JudgeFn = (chat: ChatClient, input: JudgeInput) => Promise<JudgeScore[]>;
|
|
1001
|
-
interface TestResult {
|
|
1002
|
-
name: string;
|
|
1003
|
-
passed: boolean;
|
|
1004
|
-
duration: number;
|
|
1005
|
-
detail?: string;
|
|
1006
|
-
checks: CheckResult[];
|
|
1007
|
-
}
|
|
1008
778
|
interface CheckResult {
|
|
1009
779
|
name: string;
|
|
1010
780
|
passed: boolean;
|
|
1011
781
|
expected: string;
|
|
1012
782
|
actual: string;
|
|
1013
783
|
}
|
|
1014
|
-
interface EvalResult {
|
|
1015
|
-
scenario: string;
|
|
1016
|
-
status: 'pass' | 'fail' | 'skip';
|
|
1017
|
-
duration: number;
|
|
1018
|
-
detail?: string;
|
|
1019
|
-
artifact?: string;
|
|
1020
|
-
}
|
|
1021
784
|
//#endregion
|
|
1022
|
-
export {
|
|
1023
|
-
//# sourceMappingURL=types-
|
|
785
|
+
export { InMemoryRawProviderSinkOptions as $, LlmClientOptions as A, isTransientLlmError as B, SandboxSdkTransportOpts as C, LlmCallRequest as D, LlmCallMetadata as E, assertLlmRoute as F, AssertCrossFamilyOptions as G, probeLlm as H, callLlm as I, assertCrossFamily as J, CrossFamilyError as K, callLlmJson as L, LlmResponseError as M, LlmRouteRequirements as N, LlmCallResult as O, LlmUsage as P, InMemoryRawProviderSink as Q, costReceiptFromLlm as R, RouterTransportOpts as S, LlmCallError as T, stripFencedJson as U, maximumChargeForLlmRequest as V, ServedModelCheck as W, FileSystemRawProviderSink as X, judgeFamily as Y, FileSystemRawProviderSinkOptions as Z, CliBridgeTransportOpts as _, JudgeInput as a, RawProviderSinkFilter as at, DirectProviderTransportOpts as b, PersonaConfig as c, Scenario as d, NoopRawProviderSink as et, ChatCallOpts as f, ChatTransport as g, ChatResponse as h, JudgeFn as i, RawProviderSink as it, LlmMessage as j, LlmClient as k, ProductClientConfig as l, ChatRequest as m, CompletionCriterion as n, RawProviderDirection as nt, JudgeRubric as o, defaultProviderRedactor as ot, ChatClient as p, JudgeFamily as q, DriverState as r, RawProviderEvent as rt, JudgeScore as s, providerFromBaseUrl as st, CheckResult as t, ProviderRedactor as tt, RouteMap as u, CreateChatClientOpts as v, createChatClient as w, MockTransportOpts as x, CustomTransportOpts as y, costReceiptFromLlmError as z };
|
|
786
|
+
//# sourceMappingURL=types-Cx3YUh2r.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types-Cx3YUh2r.d.ts","names":[],"sources":["../src/trace/raw-provider-sink.ts","../src/judge-families.ts","../src/integrity/served-model.ts","../src/llm-client.ts","../src/analyst/chat-client.ts","../src/types.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;KA8BY;UAEK;;EAEf;;EAEA;EACA;;;;;;EAMA;EACA;;EAEA;;EAEA;;EAEA;EACA,WAAW;;EAEX;;EAEA;EACA;EACA,iBAAiB;EACjB;EACA,kBAAkB;EAClB;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA,YAAY;EACZ;;UAGe;EACf,OAAO,OAAO,mBAAmB;;EAEjC,MAAM,SAAS,wBAAwB,QAAQ;;EAE/C,UAAU;;KAGA,oBAAoB,OAAO,qBAAqB;;;;;;iBAmB5C,wBAAwB,OAAO,mBAAmB;UA8CjD;EACf,WAAW;;cAGA,mCAAmC;UACtC;UACA;EAER,YAAY,OAAM;EAIZ,OAAO,OAAO,mBAAmB;EAIjC,KAAK,SAAQ,wBAA6B,QAAQ;EAUxD;;cAKW,+BAA+B;EACpC,UAAU;;;;;;;EASV,QAAQ,QAAQ;;UAOP;;EAEf;;EAEA;;EAEA;EACA,WAAW;;cAGA,qCAAqC;UACxC;UACA;UACA;UACA;UACA;UACA;UACA;EAER,YAAY,MAAM;UAOJ;UAON;EAKF,OAAO,OAAO,mBAAmB;EAYjC,KAAK,SAAQ,wBAA6B,QAAQ;;;;;;iBAkC1C,oBAAoB;;;;;;;;;;;;;KC5QxB;;;;;;;iBAkEI,YAAY,kBAAkB;UAc7B;;EAEf;;;;EAIA;;cAGW,yBAAyB;WAGlB,UAAU;WACV;EAHlB,YACE,iBACgB,UAAU,eACV;;;;;;;;;;;;;;;iBAoBJ,kBACd,kBACA,OAAM,2BACL;;;;KCnFS;;;;;;;;;;;UAYK;;EAEf;;EAEA;EACA,iBAAiB;;EAEjB,cAAc;EACd,SAAS;;EAET;;UAqGe;;;;;;EAMf;;;;;EAKA;;EAEA;;;;UCvIe;EACf;;;;;EAKA,kBAEI;IACM;IAAc;;IACd;IAAmB;MAAa;MAAa;;;;KAI7C;UAEK;EACf;EACA,UAAU;;EAEV;;EAEA;IAAe;IAAc,QAAQ;;EACrC;EACA;;EAEA,WAAW;;EAEX;;;;;;iBAOc,2BACd,SAAS,KAAK,iFACd,UAAS,mBACR;UAgCc;EACf;EACA;EACA;;EAEA;;EAEA;;EAEA;;UAGe;;EAEf;EACA,OAAO;;;;;EAKP;;;;;;;;;EASA;;;;;;;;;;;;EAYA;;EAEA;;;;;;;;;EASA;;;;;;;EAOA;;EAEA,KAAK;;KAGK,kBAAkB,KAAK;;iBAGnB,mBACd,QAAQ,eACR,qBAAqB,qBACpB;;iBA2Ba,wBACd,OAAO,OACP,qBAAqB,qBACpB;cAMU,qBAAqB;WAGd;WACA;WACA;EAJlB,YACE,iBACgB,gBACA,cACA;;;;;cASP,yBAAyB;WAGlB,QAAQ;EAF1B,YACE,iBACgB,QAAQ,eACxB;IAAY;;;UAMC;;EAEf;;EAEA;EACA;;EAEA;IAAe;IAAc;;;EAE7B;;EAEA;;;;;;;EAOA,SAAS;;;;;;;;EAQT;;EAEA;;EAEA,qBAAqB;;;;;;;EAOrB;;;;;;EAMA;;EAEA,WAAW;;EAEX,eAAe;;;;;;;;EAQf,UAAU;;;;;EAKV;;EAEA;IAAiB;IAAgB;;;EAEjC,WAAW;;;;;;;;;;EAUX,8BAA8B;;;;;;;;;;;;;iBAsEhB,oBAAoB;;;;;;iBA6IpB,gBAAgB;;;;;;iBA6EV,QACpB,KAAK,gBACL,OAAM,mBACL,QAAQ;;;;;;;iBAoVW,YAAY,aAChC,KAAK,gBACL,OAAM,mBACL;EAAU,OAAO;EAAG,QAAQ;;UA6Ed;;;;;;;EAOf;;;;;EAKA,kBAAkB,eAAe;;EAEjC,kBAAkB,eAAe;;EAEjC;;;;;EAKA;;;;;;;;;;;iBAYc,eAAe,MAAM,kBAAkB,MAAK;;;;;;;;;;;;;;;;;;iBA6EtC,SACpB,eACA,OAAM;EAAqB;IAC1B;EACD;EACA;EACA;;EAEA;;EAEA;;;;;;;cAoCW;WACF;mBACQ;EAEjB,YAAY,OAAM;EAKlB,KAAK,KAAK,gBAAgB,MAAM,mBAAmB,QAAQ;EAK3D,SAAS,aACP,KAAK,gBACL,MAAM,mBACL;IAAU,OAAO;IAAG,QAAQ;;;;;;;;UCjqChB;;WAEN,WAAW;;WAEX;;WAEA;;EAGT,KAAK,KAAK,aAAa,OAAO,eAAe,QAAQ;;KAG3C;UAQK,oBAAoB,KAAK;;EAExC;;KAGU,eAAe;UAEV;;EAEf,SAAS;;EAET;;EAEA;;EAEA;;KAKU,uBACR,sBACA,yBACA,8BACA,0BACA,sBACA;UAEM;EACR;;EAEA;;UAGe,4BAA4B;EAC3C;EACA;EACA;;UAGe,+BAA+B;EAC9C;EACA;EACA;;UAGe,oCAAoC;EACnD;EACA;EACA;;;;;;UAOe,gCAAgC;EAC/C;EACA,OAAO,KAAK,aAAa,OAAO,iBAAiB,QAAQ;;;UAI1C,4BAA4B;EAC3C;EACA,OAAO,KAAK,aAAa,OAAO,iBAAiB,QAAQ;;;;;;UAO1C,0BAA0B;EACzC;EACA,UAAU,KAAK,aAAa,OAAO,iBAAiB,QAAQ;;;;;;iBAO9C,iBAAiB,MAAM,uBAAuB;;;UCjH7C;EACf;EACA;EACA;EACA;EACA;EACA,OAAO;EACP,gBAAgB;EAChB;;UAGe;EACf;EACA;EACA;EACA;;UAKe;EACf;EAQA;EACA;EACA;EACA;;UAKe;EACf;EACA;EACA,YAAY;;UAGG;EACf;EACA;EACA;EACA;EACA;;UAmBe;EACf;EACA;EACA;EACA;EACA;IAAmB;IAAc;;EACjC;EACA;;UASe;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;IAAc;IAAc;;EAC5B;IAAmB;IAAc,QAAQ;;EACzC;IAAc;IAAkB;;EAChC;;UAKe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;GACC;;UAGc;EACf;EACA,QAAQ;;EAER;;UAyBe;EACf;EACA,QAAQ,OAAO;EACf,YAAY,OAAO;;UAGJ;EACf;EACA;;;;;;;;;;KAWU;UAEK;EACf;EACA;EACA;EACA,oBAAoB;EACpB,mBAAmB;EACnB;EACA;;EAEA,QAAQ;;;;;;;EAOR;;;;;;;EAOA;;;;;;EAMA;;UAGe;EACf;EACA;EACA;IAAa;IAAiB;IAAkB;;EAChD;EACA;EACA;;UA0Ce;EACf,UAAU;EACV,OAAO;EACP,WAAW;;EAEX,aAAa;EACb;EACA,WAAW;EACX,SAAS;;KAGC,WAAW,MAAM,YAAY,OAAO,eAAe,QAAQ;UAYtD;EACf;EACA;EACA;EACA"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { c as CostLedgerHandle } from "./cost-ledger-
|
|
2
|
-
import { a as RunRecord, n as RunCostProvenance, u as RunTokenUsage } from "./run-record-
|
|
3
|
-
import {
|
|
1
|
+
import { c as CostLedgerHandle } from "./cost-ledger-DbQdN3nO.js";
|
|
2
|
+
import { a as RunRecord, n as RunCostProvenance, u as RunTokenUsage } from "./run-record-CKiihE6f.js";
|
|
3
|
+
import { a as JudgeInput, p as ChatClient } from "./types-Cx3YUh2r.js";
|
|
4
4
|
import { RE2JS } from "re2js";
|
|
5
5
|
//#region src/trace-analyst/types.d.ts
|
|
6
6
|
/**
|
|
@@ -594,4 +594,4 @@ type AnalystRunEvent = {
|
|
|
594
594
|
};
|
|
595
595
|
//#endregion
|
|
596
596
|
export { SearchSpanResult as A, ViewSpansResult as B, TRACE_ANALYSIS_LIMITS as C, DatasetOverview as D, DEFAULT_TRACE_ANALYST_BUDGETS as E, TraceAnalystFilters as F, ViewTraceResult as H, TraceAnalystSpan as I, TraceAnalystSpanKind as L, SpanMatchRecord as M, TRACE_ANALYST_TRUNCATION_MARKER_PREFIX as N, ErrorCluster as O, TraceAnalystByteBudgets as P, TraceAnalystSpanStatus as R, BoundedTraceAnalysisStoreOptions as S, TraceAnalysisStoreContext as T, ViewTraceOversized as V, ProposalFinding as _, AnalystInputKind as a, makeFinding as b, AnalystRunInputs as c, AnalystSeverity as d, AnalystUsageReceipt as f, ExecutionProbeRequest as g, ExecutionProbeOutcome as h, AnalystFinding as i, SearchTraceResult as j, QueryTracesPage as k, AnalystRunResult as l, ExecutionProbe as m, AnalystContext as n, AnalystRequirements as o, EvidenceRef as p, AnalystCost as r, AnalystRunEvent as s, Analyst as t, AnalystRunSummary as u, ProposalFindingOrigin as v, TraceAnalysisStore as w, makeProposalFinding as x, computeFindingId as y, TraceAnalystTraceSummary as z };
|
|
597
|
-
//# sourceMappingURL=types-
|
|
597
|
+
//# sourceMappingURL=types-DLQx4mKU.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types-
|
|
1
|
+
{"version":3,"file":"types-DLQx4mKU.d.ts","names":[],"sources":["../src/trace-analyst/types.ts","../src/trace-analyst/store-contract.ts","../src/analyst/types.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;KAgBY;KAUA;;;;UAKK;EACf;EACA;EACA;EACA;EACA,MAAM;EACN;EACA;EACA;EACA,QAAQ;EACR;EACA;EACA;EACA;EACA;;;EAGA,YAAY;;UAGG;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;EAGA;;;;;;;UAQe;;EAEf;;EAEA;;EAEA;;EAEA;EACA;EACA;;EAEA;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;;EAEA;EACA;IAAU;IAAqB;;;;;;EAK/B,gBAAgB;EAChB;IAAc;IAAkB;;;UAGjB;EACf,QAAQ;EACR;EACA;;;;;UAMe;EACf;EACA,QAAQ;EACR,YAAY;;UAGG;EACf;;EAEA,gBAAgB;;EAEhB;EACA;;UAGe;EACf;EACA,OAAO;;EAEP;;EAEA;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA;EACA,WAAW;;;EAGX;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA,MAAM;EACN;;UAGe;EACf;EACA;EACA,MAAM;EACN;;;UAIe;;;EAGf;;;EAGA;;;EAGA;;;EAGA;;cAGW,+BAA+B;;;cAS/B;;;cC7MA;WACX;WACA;WACA;WACA;WACA;WACA;WACA;WACA;;UAGe;EACf,SAAS;;;;;;;;;UAUM;EACf,SAAS,kBAAkB,UAAU,4BAA4B;EAEjE,SACE;IAAS;IAAkB;KAC3B,UAAU,4BACT;EAEH,YACE,UAAU,qBACV,UAAU,4BACT,QAAQ;EAEX,YACE;IAAS,UAAU;IAAqB;IAAe;KACvD,UAAU,4BACT,QAAQ;EAEX,YAAY,UAAU,qBAAqB,UAAU,4BAA4B;EAEjF,UACE;IACE;IACA;KAEF,UAAU,4BACT,QAAQ;EAEX,UACE;IACE;IACA;IACA;KAEF,UAAU,4BACT,QAAQ;EAEX,YACE;IACE;IACA;IACA;KAEF,UAAU,4BACT,QAAQ;EAEX,WACE;IACE;IACA;IACA;IACA;KAEF,UAAU,4BACT,QAAQ;;UAGI;EACf,UAAU,QAAQ;;;;;;;;UC/DH;EACf;;;;;;;EAOA;EACA;EACA;EACA,UAAU;;;;;;;EAOV;EACA;EACA;EACA,eAAe;EACf;EACA;;EAEA;;;;;;EAMA;;;;EAIA;;EAEA,WAAW;;KAGD;;KAGA;;KAGA,kBAAkB;WACnB,iBAAiB;;UAGX;;;;;;;EAOf;EACA;EACA;;;;;;;;KAWU;UAOK;;EAEf;;EAEA;;EAEA;;EAEA;;UAGe;;EAEf;;EAEA;;;;;;UAOe;EACf,aAAa;EACb;EACA,YAAY;EACZ,aAAa;;EAEb,SAAS;;UAGM;EACf;;EAEA;;EAEA;;EAEA;;EAEA,aAAa;;EAEb;;;;;;;EAOA,OAAO;;;;;;;;;;EAUP,gBAAgB,cAAc;;;;;;;EAO9B,mBAAmB,cAAc;;;;;EAKjC,eAAe,SAAS;;EAExB,OAAO;;EAEP,OAAO,aAAa,SAAS;;EAE7B,SAAS;;;;;;;;EAQT,QAAQ;;;UAMO;EACf;;EAEA;;EAEA;;EAEA;EACA,SAAS;;;;;;;KAQC;EAEN;EACA;EACA;EACA;EACA;;EAEA;;EAEA;EAAkB;IAAS;IAAe;;;;;;;;UAO/B;;WAEN;EACT,QAAQ,SAAS,wBAAwB,QAAQ;;;;;;;;UASlC,QAAQ;;WAEd;;WAEA;WACA,WAAW;WACX,MAAM;WACN,WAAW;;WAEX;EACT,QAAQ,OAAO,QAAQ,KAAK,iBAAiB,QAAQ;;;UAItC;;EAEf;;EAEA,QAAQ;;EAER,MAAM;;EAEN;;;;;;;;EAQA;IAAkB;IAAsB;;;;;;;;EAOxC;;;;;;;;;;iBAac,iBAAiB;EAC/B;EACA;EACA;EACA;;EAEA;;;;;;iBAyBc,YACd,MAAM,KAAK;EACT;EACA;IAED;;iBAiBa,oBACd,MAAM,KAAK;EACT;EACA;IAED;UAOc;EACf;EACA;;EAEA;EACA;EACA;;EAEA,OAAO;;EAEP;IAAU;IAAe;;;UAGV;EACf;EACA;EACA;EACA;EACA,UAAU;EACV,aAAa;;EAEb;;;;;EAKA,wBAAwB;;;;;;;;;;;;;;;KAkBd;EAEN;EACA;EACA;EACA;;EAEA,aAAa;;EAGb;EACA,SAAS;;EAGT;EACA;EACA;;EAGA;;EAEA,SAAS;EACT,UAAU,cAAc;;EAGxB;EACA,QAAQ"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { h as RolloutLine } from "./schema-
|
|
1
|
+
import { h as RolloutLine } from "./schema-Cef2cFmb.js";
|
|
2
2
|
//#region src/supervisor-run/types.d.ts
|
|
3
3
|
/** A metric that could not be computed, with the reason its artifact was missing. */
|
|
4
4
|
interface Unavailable {
|
|
@@ -369,4 +369,4 @@ interface SupervisorRunTreeGap {
|
|
|
369
369
|
}
|
|
370
370
|
//#endregion
|
|
371
371
|
export { Unavailable as C, showMeasured as D, isUnavailable as E, unavailable as O, SupervisorRunTreeGapCode as S, WorkerLogSource as T, SupervisorRunReport as _, OrchestrationMetrics as a, SupervisorRunTree as b, PerWorkerRow as c, SUPERVISOR_RUN_ROLLUP_SCHEMA as d, SUPERVISOR_RUN_SCHEMA as f, SupervisorRunReader as g, SupervisorRunNodeRole as h, NO_SOURCE_LIMITS as i, RoleSpend as l, SteerBreakdown as m, EconomicsMetrics as n, OutcomeMetrics as o, SourceLimits as p, Measured as r, PatchStats as s, DecisionMetrics as t, RollupCellRow as u, SupervisorRunRollup as v, WallDistribution as w, SupervisorRunTreeGap as x, SupervisorRunSources as y };
|
|
372
|
-
//# sourceMappingURL=types-
|
|
372
|
+
//# sourceMappingURL=types-yLK8gXE9.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types-
|
|
1
|
+
{"version":3,"file":"types-yLK8gXE9.d.ts","names":[],"sources":["../src/supervisor-run/types.ts"],"mappings":";;;UAiCiB;WACN;;;KAIC,SAAS,KAAK,IAAI;iBAEd,YAAY,iBAAiB;iBAI7B,cAAc,aAAa,KAAK;;iBAKhC,aAAa,GAAG;;KAWpB;;;;;UAMK;;;;;WAKN;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;WACA;WACA;WACA;;;;;;;;;;;;;UAcM;;WAEN;;WAEA;;WAEA;;WAEA;;WAEA;;;cAIE,kBAAkB;;;;;;;;;;UAiBd;;WAEN;WACA;;WAEA;;WAEA;;;;;;WAMA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA,kBAAkB;;WAElB;;WAEA;;;;;;;WAOA;;WAEA;;WAEA;;WAEA;;;;;WAKA;IACP;IACA;IACA;IACA;;IAEA;IACA;;WAEO;;WAEA,QAAQ;;;;;;WAMR;;;;;WAKA;;;;;;UAOM;;WAEN;EACT,QAAQ,QAAQ;;cAOL;cACA;UAEI;;WAEN;WACA;;WAEA;;WAEA;;UAGM;WACN,gBAAgB;WAChB,gBAAgB;WAChB,kBAAkB;;WAElB,QAAQ;WACR,iBAAiB;WACjB,gBAAgB,kBAAkB;;WAElC,kBAAkB;;;;;;WAMlB,OAAO;WACP,WAAW;WACX,gBAAgB;;WAEhB,UAAU;;WAEV,gBAAgB;;WAEhB,iBAAiB;WACjB,oBAAoB;WACpB,kBAAkB;;WAElB,QAAQ;WACR,SAAS;;WAET,mBAAmB;;UAGb;WACN,iBAAiB,SAAS;WAC1B,iBAAiB,SAAS;;WAE1B,UAAU;;WAEV,UAAU;;WAEV,WAAW;;WAEX,oBAAoB;;WAEpB,wBAAwB;;WAExB,eAAe;WACf,qBAAqB;;UAGf;WACN,UAAU;WACV,WAAW;;;;;;WAMX,WAAW;WACX,YAAY;WACZ,KAAK;WACL;;UAGM;;WAEN;WACA;;WAEA,MAAM;;WAEN;;WAEA;;WAEA;;WAEA;;WAEA;WACA;;WAEA;WACA;WACA;WACA;WACA;;WAEA;;UAGM;WACN;WACA;WACA;WACA;WACA;WACA;;UAGM;;WAEN,OAAO;;;;;;;;WAQP,kBAAkB;;WAElB,SAAS;WACT,UAAU;;;;;;WAMV;WACA,yBAAyB;WACzB,0BAA0B,SAAS;WACnC,WAAW,kBAAkB;;UAGvB;WACN;WACA;WACA;WACA;;UAGM;WACN,WAAW;WACX,YAAY;WACZ,WAAW;WACX,eAAe;WACf,YAAY;WACZ,aAAa;WACb,YAAY;WACZ,YAAY;WACZ,UAAU;WACV,OAAO,SAAS;;WAEhB;;UAGM;WACN,eAAe;;WAEf;WACA;WACA;WACA,cAAc;WACd,yBAAyB;WACzB;WACA,eAAe;WACf,UAAU;WACV,WAAW;WACX,SAAS;;WAET;;WAEA;;UAGM;WACN;WACA;WACA,QAAQ;WACR,OAAO;WACP,aAAa;WACb,SAAS;WACT,UAAU;WACV,KAAK;;UAGC;WACN,eAAe;WACf;WACA,aAAa;WACb,iBAAiB;WACjB;WACA,WAAW;WACX,mBAAmB;WACnB,iBAAiB;WACjB,aAAa;WACb,qBAAqB;WACrB,eAAe;WACf,UAAU;WACV,eAAe;WACf,kBAAkB;;;;;;;;UASZ;WACN;WACA,gBAAgB;;WAEhB,eAAe;;;KAId;UASK;WACN,MAAM;WACN;WACA;WACA"}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { n as contentHash } from "./verdict-cache-mZf5FEiY.js";
|
|
2
|
+
//#region src/verdict.ts
|
|
3
|
+
/**
|
|
4
|
+
* Validator-output verdict — substrate primitive for "did this output pass,
|
|
5
|
+
* and how well?"
|
|
6
|
+
*
|
|
7
|
+
* Used by:
|
|
8
|
+
* - `@tangle-network/agent-eval/matrix` — verdict per cell in the cartesian.
|
|
9
|
+
* - `@tangle-network/agent-runtime` — Validator<Output, Verdict = DefaultVerdict>.
|
|
10
|
+
* Runtime keeps `Validator` because it's coupled to runtime-shaped
|
|
11
|
+
* `ValidationCtx` (iteration, signal, traceEmitter); the verdict TYPE
|
|
12
|
+
* itself is a substrate concept and lives here.
|
|
13
|
+
*
|
|
14
|
+
* Repo layering: agent-eval is the substrate (no upward deps). Both
|
|
15
|
+
* agent-runtime and agent-knowledge consume this type FROM agent-eval —
|
|
16
|
+
* never the other way around. See CLAUDE.md "Repo layering" for the rule.
|
|
17
|
+
*/
|
|
18
|
+
/**
|
|
19
|
+
* Digest a certification's evidence artifact: sha-256 over the canonical
|
|
20
|
+
* JSON of the evidence AS SERIALIZED. The JSON round-trip drops
|
|
21
|
+
* `undefined`-valued fields on purpose — certified verdicts persist as
|
|
22
|
+
* JSON, so the digest must cover exactly what a reader of the persisted
|
|
23
|
+
* artifact can re-digest, nothing the wire cannot carry. Throws when the
|
|
24
|
+
* evidence has no JSON form at all.
|
|
25
|
+
*/
|
|
26
|
+
function certificationEvidenceDigest(evidence) {
|
|
27
|
+
const serialized = JSON.stringify(evidence);
|
|
28
|
+
if (serialized === void 0) throw new Error("certificationEvidenceDigest: evidence has no JSON form — a certificate over nothing is the unverifiable claim this record exists to kill");
|
|
29
|
+
return contentHash(JSON.parse(serialized));
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Land a finished statement-equivalence record in the shared verdict
|
|
33
|
+
* spine. `valid` answers the protocol's question — are the two blind
|
|
34
|
+
* statements equivalent — so a refutation is `valid: false` even though a
|
|
35
|
+
* separating witness in hand is a successful protocol outcome.
|
|
36
|
+
*
|
|
37
|
+
* Certification is present exactly when the obligation was discharged
|
|
38
|
+
* (proved or refuted): the checker vouches for the discharge either way.
|
|
39
|
+
* An unresolved obligation certifies nothing and keeps its reason in
|
|
40
|
+
* `notes`.
|
|
41
|
+
*/
|
|
42
|
+
function equivalenceVerdict(record) {
|
|
43
|
+
const { obligation, spec, arms } = record;
|
|
44
|
+
const discharged = obligation.status !== "unresolved";
|
|
45
|
+
const notes = obligation.status === "proved" ? `statements equivalent: ${spec.artifact}` : obligation.status === "refuted-with-separating-witness" ? `formalization gap witnessed: ${obligation.separatingWitness}` : `obligation undischarged: ${obligation.unresolvedReason}`;
|
|
46
|
+
return {
|
|
47
|
+
valid: obligation.status === "proved",
|
|
48
|
+
score: obligation.status === "proved" ? 1 : 0,
|
|
49
|
+
notes,
|
|
50
|
+
...discharged && obligation.evidenceDigest !== void 0 ? { certification: {
|
|
51
|
+
strategy: spec.source,
|
|
52
|
+
checker: obligation.checker,
|
|
53
|
+
assumptions: [`arm '${arms[0].armId}' and arm '${arms[1].armId}' self-declare blind derivation`],
|
|
54
|
+
evidenceDigest: obligation.evidenceDigest
|
|
55
|
+
} } : {}
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
//#endregion
|
|
59
|
+
export { equivalenceVerdict as n, certificationEvidenceDigest as t };
|
|
60
|
+
|
|
61
|
+
//# sourceMappingURL=verdict-BndeTAh_.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"verdict-BndeTAh_.js","names":[],"sources":["../src/verdict.ts"],"sourcesContent":["/**\n * Validator-output verdict — substrate primitive for \"did this output pass,\n * and how well?\"\n *\n * Used by:\n * - `@tangle-network/agent-eval/matrix` — verdict per cell in the cartesian.\n * - `@tangle-network/agent-runtime` — Validator<Output, Verdict = DefaultVerdict>.\n * Runtime keeps `Validator` because it's coupled to runtime-shaped\n * `ValidationCtx` (iteration, signal, traceEmitter); the verdict TYPE\n * itself is a substrate concept and lives here.\n *\n * Repo layering: agent-eval is the substrate (no upward deps). Both\n * agent-runtime and agent-knowledge consume this type FROM agent-eval —\n * never the other way around. See CLAUDE.md \"Repo layering\" for the rule.\n */\n\nimport { contentHash } from './verdict-cache'\nimport type {\n CheckerIdentity,\n EquivalenceRecord,\n VerificationStrategySource,\n} from './verification-strategy'\n\n/**\n * What certified a verdict — the epistemics a bare `valid` + `score` pair\n * cannot carry. A kernel-checked proof and an LLM judge can produce the\n * same `{ valid: true, score: 1 }`; this record is what tells them apart.\n *\n * Every field is required on purpose. A certification without a checker\n * identity or an evidence digest is exactly the unverifiable claim this\n * record exists to kill.\n */\nexport interface VerdictCertification {\n /**\n * Which verification-strategy member produced the certificate. Each\n * member's failure mode is documented on the family type and in\n * `VERIFICATION_STRATEGIES` (src/verification-strategy.ts).\n */\n strategy: VerificationStrategySource\n /**\n * Exact checker identity, e.g. `{ name: 'lean4', version: '4.33.0',\n * pins: { mathlib: 'db584cd6d46c' } }` for a kernel, or a judge model\n * id + snapshot for a judge.\n */\n checker: CheckerIdentity\n /**\n * Every step the certificate rests on that the checker did NOT verify,\n * e.g. 'diagonal embedding argued in prose' or 'agreement arms share\n * training-corpus blind spots'. An empty array is the producer's\n * explicit, auditable claim that there are none — it is not a default.\n */\n assumptions: string[]\n /** Digest (e.g. sha256) of the evidence artifact the checker consumed or produced. */\n evidenceDigest: string\n}\n\n/**\n * Minimal verdict shape — `valid` + `score` are required; `scores` +\n * `notes` are optional surface. Validators that need richer shapes\n * parameterise `Validator<Output, MyVerdict>` with their own type.\n *\n * Need structured extras? Extend DefaultVerdict with typed fields — never\n * serialize extras into `notes`.\n */\nexport interface DefaultVerdict {\n /** Whether the output meets the validator's pass criteria. */\n valid: boolean\n /** Aggregate score in [0, 1]. Drivers use this for winner selection. */\n score: number\n /** Per-dimension scores. Free-form; weighted into `score` by the validator. */\n scores?: Record<string, number>\n /** Human-readable rationale; surfaces in trace + final-result `winner.verdict`. */\n notes?: string\n /**\n * What certified this verdict, when anything did. Absent = uncertified:\n * scored, but no strategy vouches for it. A consumer that cares reads\n * `certification.strategy` to tell a kernel-checked verdict from a\n * judge-scored one; a consumer that does not read it sees the exact\n * verdict it always saw.\n */\n certification?: VerdictCertification\n}\n\n/**\n * Digest a certification's evidence artifact: sha-256 over the canonical\n * JSON of the evidence AS SERIALIZED. The JSON round-trip drops\n * `undefined`-valued fields on purpose — certified verdicts persist as\n * JSON, so the digest must cover exactly what a reader of the persisted\n * artifact can re-digest, nothing the wire cannot carry. Throws when the\n * evidence has no JSON form at all.\n */\nexport function certificationEvidenceDigest(evidence: unknown): string {\n const serialized = JSON.stringify(evidence)\n if (serialized === undefined) {\n throw new Error(\n 'certificationEvidenceDigest: evidence has no JSON form — a certificate over nothing is the unverifiable claim this record exists to kill',\n )\n }\n return contentHash(JSON.parse(serialized))\n}\n\n/**\n * Land a finished statement-equivalence record in the shared verdict\n * spine. `valid` answers the protocol's question — are the two blind\n * statements equivalent — so a refutation is `valid: false` even though a\n * separating witness in hand is a successful protocol outcome.\n *\n * Certification is present exactly when the obligation was discharged\n * (proved or refuted): the checker vouches for the discharge either way.\n * An unresolved obligation certifies nothing and keeps its reason in\n * `notes`.\n */\nexport function equivalenceVerdict(record: EquivalenceRecord): DefaultVerdict {\n const { obligation, spec, arms } = record\n const discharged = obligation.status !== 'unresolved'\n const notes =\n obligation.status === 'proved'\n ? `statements equivalent: ${spec.artifact}`\n : obligation.status === 'refuted-with-separating-witness'\n ? `formalization gap witnessed: ${obligation.separatingWitness}`\n : `obligation undischarged: ${obligation.unresolvedReason}`\n return {\n valid: obligation.status === 'proved',\n score: obligation.status === 'proved' ? 1 : 0,\n notes,\n ...(discharged && obligation.evidenceDigest !== undefined\n ? {\n certification: {\n strategy: spec.source,\n checker: obligation.checker,\n // The arms' blindness declarations are attested by the arms,\n // never verified by the checker — the one step every\n // equivalence certificate rests on.\n assumptions: [\n `arm '${arms[0].armId}' and arm '${arms[1].armId}' self-declare blind derivation`,\n ],\n evidenceDigest: obligation.evidenceDigest,\n } satisfies VerdictCertification,\n }\n : {}),\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AA2FA,SAAgB,4BAA4B,UAA2B;CACrE,MAAM,aAAa,KAAK,UAAU,QAAQ;CAC1C,IAAI,eAAe,KAAA,GACjB,MAAM,IAAI,MACR,0IACF;CAEF,OAAO,YAAY,KAAK,MAAM,UAAU,CAAC;AAC3C;;;;;;;;;;;;AAaA,SAAgB,mBAAmB,QAA2C;CAC5E,MAAM,EAAE,YAAY,MAAM,SAAS;CACnC,MAAM,aAAa,WAAW,WAAW;CACzC,MAAM,QACJ,WAAW,WAAW,WAClB,0BAA0B,KAAK,aAC/B,WAAW,WAAW,oCACpB,gCAAgC,WAAW,sBAC3C,4BAA4B,WAAW;CAC/C,OAAO;EACL,OAAO,WAAW,WAAW;EAC7B,OAAO,WAAW,WAAW,WAAW,IAAI;EAC5C;EACA,GAAI,cAAc,WAAW,mBAAmB,KAAA,IAC5C,EACE,eAAe;GACb,UAAU,KAAK;GACf,SAAS,WAAW;GAIpB,aAAa,CACX,QAAQ,KAAK,EAAE,CAAC,MAAM,aAAa,KAAK,EAAE,CAAC,MAAM,gCACnD;GACA,gBAAgB,WAAW;EAC7B,EACF,IACA,CAAC;CACP;AACF"}
|