@tangle-network/agent-eval 0.216.0 → 0.216.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -0
- package/dist/adapters/http.d.ts +3 -3
- package/dist/{agent-profile-CIA2Atnt.d.ts → agent-profile-VFcQncbW.d.ts} +5 -5
- package/dist/{agent-profile-CIA2Atnt.d.ts.map → agent-profile-VFcQncbW.d.ts.map} +1 -1
- package/dist/{agent-profile-cell-BYc0bxsL.js → agent-profile-cell-BK8mlqlq.js} +3 -3
- package/dist/{agent-profile-cell-BYc0bxsL.js.map → agent-profile-cell-BK8mlqlq.js.map} +1 -1
- package/dist/{agreement-irr-DsTURxdr.js → agreement-irr-BEWLIF3_.js} +3 -3
- package/dist/{agreement-irr-DsTURxdr.js.map → agreement-irr-BEWLIF3_.js.map} +1 -1
- package/dist/analyst/index.d.ts +14 -14
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +15 -15
- package/dist/{analyze-runs-3xpby3p2.js → analyze-runs-C84irm4i.js} +608 -604
- package/dist/analyze-runs-C84irm4i.js.map +1 -0
- package/dist/{analyze-runs-CSDAh8A7.d.ts → analyze-runs-D68fd-su.d.ts} +17 -15
- package/dist/analyze-runs-D68fd-su.d.ts.map +1 -0
- package/dist/{baseline-Dm8966du.js → baseline-Udi_PLuT.js} +5 -5
- package/dist/{baseline-Dm8966du.js.map → baseline-Udi_PLuT.js.map} +1 -1
- package/dist/{benchmark-BywPeAUo.js → benchmark-CXmVVHH2.js} +2 -2
- package/dist/{benchmark-BywPeAUo.js.map → benchmark-CXmVVHH2.js.map} +1 -1
- package/dist/{benchmark-command-CByl9dxG.js → benchmark-command-By0vh8q7.js} +62 -45
- package/dist/benchmark-command-By0vh8q7.js.map +1 -0
- package/dist/benchmarks/index.d.ts +5 -5
- package/dist/benchmarks/index.js +3 -3
- package/dist/builder-eval/index.js +2 -2
- package/dist/campaign/index.d.ts +7 -7
- package/dist/campaign/index.js +13 -13
- package/dist/{campaign-BQ-Jujuj.js → campaign-BudZ70zo.js} +245 -247
- package/dist/campaign-BudZ70zo.js.map +1 -0
- package/dist/{campaign-evidence-OcUciD-N.js → campaign-evidence-Clrn4hBV.js} +8 -8
- package/dist/campaign-evidence-Clrn4hBV.js.map +1 -0
- package/dist/{canonical-BsZVgBhW.js → canonical-B2RqLofv.js} +2 -2
- package/dist/{canonical-BsZVgBhW.js.map → canonical-B2RqLofv.js.map} +1 -1
- package/dist/{chat-client-DthPOYGd.js → chat-client-l-MVDlDk.js} +2 -2
- package/dist/{chat-client-DthPOYGd.js.map → chat-client-l-MVDlDk.js.map} +1 -1
- package/dist/{chat-json-call-BrKsyspg.js → chat-json-call-puMArmtW.js} +2 -2
- package/dist/{chat-json-call-BrKsyspg.js.map → chat-json-call-puMArmtW.js.map} +1 -1
- package/dist/{claim-BXcZw-jo.js → claim-q4HQhK1G.js} +4 -4
- package/dist/claim-q4HQhK1G.js.map +1 -0
- package/dist/cli.js +9 -9
- package/dist/cli.js.map +1 -1
- package/dist/{cluster-bootstrap-B3JywQQn.js → cluster-bootstrap-qcUWYFl-.js} +3 -3
- package/dist/{cluster-bootstrap-B3JywQQn.js.map → cluster-bootstrap-qcUWYFl-.js.map} +1 -1
- package/dist/contract/index.d.ts +11 -11
- package/dist/contract/index.js +1022 -1078
- package/dist/contract/index.js.map +1 -1
- package/dist/{cost-ledger-D2FnJPBF.js → cost-ledger-BP8V4kUQ.js} +835 -816
- package/dist/cost-ledger-BP8V4kUQ.js.map +1 -0
- package/dist/{cost-ledger-CO7UyEcQ.d.ts → cost-ledger-Jk62OBfF.d.ts} +12 -47
- package/dist/cost-ledger-Jk62OBfF.d.ts.map +1 -0
- package/dist/{counterfactual-DtMWvn7D.js → counterfactual-Dn3veueY.js} +2 -2
- package/dist/{counterfactual-DtMWvn7D.js.map → counterfactual-Dn3veueY.js.map} +1 -1
- package/dist/{default-registry-Cn1LQ2Ve.js → default-registry-COhk9g36.js} +5 -5
- package/dist/{default-registry-Cn1LQ2Ve.js.map → default-registry-COhk9g36.js.map} +1 -1
- package/dist/{default-registry-BrWh62zf.d.ts → default-registry-DzHiGruW.d.ts} +6 -6
- package/dist/{default-registry-BrWh62zf.d.ts.map → default-registry-DzHiGruW.d.ts.map} +1 -1
- package/dist/{define-agent-eval-Ch3MWX6k.js → define-agent-eval-CNTPzIkp.js} +10 -10
- package/dist/define-agent-eval-CNTPzIkp.js.map +1 -0
- package/dist/{define-agent-eval-CEWkX5wZ.d.ts → define-agent-eval-WSck0CZJ.d.ts} +8 -8
- package/dist/{define-agent-eval-CEWkX5wZ.d.ts.map → define-agent-eval-WSck0CZJ.d.ts.map} +1 -1
- package/dist/{descriptive-CnU8Yq29.js → descriptive-DboxPZrG.js} +2 -2
- package/dist/{descriptive-CnU8Yq29.js.map → descriptive-DboxPZrG.js.map} +1 -1
- package/dist/diagnosis/index.d.ts +3 -3
- package/dist/diagnosis/index.js +4 -4
- package/dist/{dspy-rlm-engine-D-KX_e64.js → dspy-rlm-engine-LRCu61m5.js} +3 -3
- package/dist/{dspy-rlm-engine-D-KX_e64.js.map → dspy-rlm-engine-LRCu61m5.js.map} +1 -1
- package/dist/{effect-sizes-DiH8MGOH.js → effect-sizes-BhzEpXMc.js} +2 -2
- package/dist/{effect-sizes-DiH8MGOH.js.map → effect-sizes-BhzEpXMc.js.map} +1 -1
- package/dist/{engine-C_zNWfM8.d.ts → engine-3A_4jOlI.d.ts} +5 -5
- package/dist/{engine-C_zNWfM8.d.ts.map → engine-3A_4jOlI.d.ts.map} +1 -1
- package/dist/{equal-terms-Blx9Zk5I.js → equal-terms-BtdXayCJ.js} +2 -2
- package/dist/{equal-terms-Blx9Zk5I.js.map → equal-terms-BtdXayCJ.js.map} +1 -1
- package/dist/{errors-Dngq5h35.js → errors-DVZiqS7K.js} +6 -2
- package/dist/{errors-Dngq5h35.js.map → errors-DVZiqS7K.js.map} +1 -1
- package/dist/{eval-campaign-4byTtqrf.js → eval-campaign-90jeaCAI.js} +6 -6
- package/dist/{eval-campaign-4byTtqrf.js.map → eval-campaign-90jeaCAI.js.map} +1 -1
- package/dist/{evaluation-Cn2dRZIz.d.ts → evaluation-6Yoi74gM.d.ts} +4 -4
- package/dist/{evaluation-Cn2dRZIz.d.ts.map → evaluation-6Yoi74gM.d.ts.map} +1 -1
- package/dist/evaluation.d.ts +1 -1
- package/dist/evaluation.js +1 -1
- package/dist/{exact-types-CD0lf5Qd.d.ts → exact-types-ClugMJfI.d.ts} +2 -2
- package/dist/{exact-types-CD0lf5Qd.d.ts.map → exact-types-ClugMJfI.d.ts.map} +1 -1
- package/dist/experiment/index.d.ts +404 -3
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +313 -19
- package/dist/experiment/index.js.map +1 -1
- package/dist/{external-optimizer-process-nnsY5D_4.js → external-optimizer-process-BjMGUuds.js} +3 -3
- package/dist/{external-optimizer-process-nnsY5D_4.js.map → external-optimizer-process-BjMGUuds.js.map} +1 -1
- package/dist/{external-optimizer-subprocess-DcerlZSG.js → external-optimizer-subprocess-BqW0hHcI.js} +548 -548
- package/dist/external-optimizer-subprocess-BqW0hHcI.js.map +1 -0
- package/dist/{feedback-trajectory-DkaDj0dj.d.ts → feedback-trajectory-C4EG0nNH.d.ts} +5 -8
- package/dist/feedback-trajectory-C4EG0nNH.d.ts.map +1 -0
- package/dist/fuzz.d.ts +1 -1
- package/dist/fuzz.js +3 -3
- package/dist/hosted/index.js +1 -1
- package/dist/{index-CPJJr98o.d.ts → index-BFRO6chs.d.ts} +3 -2
- package/dist/index-BFRO6chs.d.ts.map +1 -0
- package/dist/{index-Ch9VxCpK.d.ts → index-BO41eGj5.d.ts} +2 -2
- package/dist/{index-Ch9VxCpK.d.ts.map → index-BO41eGj5.d.ts.map} +1 -1
- package/dist/{index-NvGBiO8H.d.ts → index-CyszSg4O.d.ts} +165 -162
- package/dist/index-CyszSg4O.d.ts.map +1 -0
- package/dist/{index-DlH0Y9gI.d.ts → index-DPcngtwt.d.ts} +30 -17
- package/dist/index-DPcngtwt.d.ts.map +1 -0
- package/dist/{index-oQ4aHziK.d.ts → index-DV-jSzxF.d.ts} +8 -8
- package/dist/{index-oQ4aHziK.d.ts.map → index-DV-jSzxF.d.ts.map} +1 -1
- package/dist/index.d.ts +1969 -1871
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +161 -201
- package/dist/index.js.map +1 -1
- package/dist/{integrity-BvnVARVl.js → integrity-BXOsAb3i.js} +2 -2
- package/dist/{integrity-BvnVARVl.js.map → integrity-BXOsAb3i.js.map} +1 -1
- package/dist/{integrity-CCAxGVK6.d.ts → integrity-DAMEDFaC.d.ts} +2 -2
- package/dist/{integrity-CCAxGVK6.d.ts.map → integrity-DAMEDFaC.d.ts.map} +1 -1
- package/dist/{integrity-BV1ZDf42.js → integrity-dLiDe2kD.js} +2 -5
- package/dist/integrity-dLiDe2kD.js.map +1 -0
- package/dist/{internal-CRHwS9iz.js → internal-Epl-YBZW.js} +3 -3
- package/dist/{internal-CRHwS9iz.js.map → internal-Epl-YBZW.js.map} +1 -1
- package/dist/interval-CELpye1n.js +219 -0
- package/dist/interval-CELpye1n.js.map +1 -0
- package/dist/interval-ohuOJurX.d.ts +47 -0
- package/dist/interval-ohuOJurX.d.ts.map +1 -0
- package/dist/{journal-DvKCvjty.js → journal-DfRz9kps.js} +2 -2
- package/dist/{journal-DvKCvjty.js.map → journal-DfRz9kps.js.map} +1 -1
- package/dist/{judge-calibration-N44V04Se.js → judge-calibration-BpCy5XQl.js} +3 -3
- package/dist/{judge-calibration-N44V04Se.js.map → judge-calibration-BpCy5XQl.js.map} +1 -1
- package/dist/{judge-gate-DNcUrRFA.d.ts → judge-gate-CE7CQAgt.d.ts} +3 -3
- package/dist/{judge-gate-DNcUrRFA.d.ts.map → judge-gate-CE7CQAgt.d.ts.map} +1 -1
- package/dist/{judge-gate-B1qYnkpi.js → judge-gate-iY9s4L44.js} +3 -3
- package/dist/judge-gate-iY9s4L44.js.map +1 -0
- package/dist/{kind-factory-CbZ-6tvu.js → kind-factory-BobI5Jy3.js} +4 -4
- package/dist/{kind-factory-CbZ-6tvu.js.map → kind-factory-BobI5Jy3.js.map} +1 -1
- package/dist/ledger-core/index.js +3 -3
- package/dist/{ledger-core-B2hahzrt.js → ledger-core-Gq7yp8lP.js} +4 -4
- package/dist/{ledger-core-B2hahzrt.js.map → ledger-core-Gq7yp8lP.js.map} +1 -1
- package/dist/{llm-client-DQveoxSM.js → llm-client-CyBAoYiX.js} +192 -219
- package/dist/llm-client-CyBAoYiX.js.map +1 -0
- package/dist/{llm-judge-PXV7Xljl.js → llm-judge-CG12SQJq.js} +499 -921
- package/dist/llm-judge-CG12SQJq.js.map +1 -0
- package/dist/matrix/index.d.ts +2 -2
- package/dist/matrix/index.js +1 -1
- package/dist/{matrix-CHMiebFx.d.ts → matrix-DSrSxjjy.d.ts} +4 -4
- package/dist/{matrix-CHMiebFx.d.ts.map → matrix-DSrSxjjy.d.ts.map} +1 -1
- package/dist/{matrix-_BV9-Lk5.js → matrix-Dm0BQW7U.js} +7 -11
- package/dist/matrix-Dm0BQW7U.js.map +1 -0
- package/dist/{paired-tests-jbwPdSz5.js → mean-DHoMiudD.js} +11 -5
- package/dist/mean-DHoMiudD.js.map +1 -0
- package/dist/meta-eval/index.d.ts +3 -3
- package/dist/meta-eval/index.js +9 -9
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/mint-DNU8QPG4.d.ts +81 -0
- package/dist/mint-DNU8QPG4.d.ts.map +1 -0
- package/dist/{mint-Csor1BLJ.js → mint-QaD5cA0u.js} +3 -3
- package/dist/{mint-Csor1BLJ.js.map → mint-QaD5cA0u.js.map} +1 -1
- package/dist/{multiplicity-D9cjmStq.js → multiplicity-DQUFcbW4.js} +2 -2
- package/dist/{multiplicity-D9cjmStq.js.map → multiplicity-DQUFcbW4.js.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/golden/index.d.ts.map +1 -1
- package/dist/multishot/golden/index.js +3 -6
- package/dist/multishot/golden/index.js.map +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/multishot/index.js +1 -1
- package/dist/nonnegative-number-DvdMxg_f.js +9 -0
- package/dist/nonnegative-number-DvdMxg_f.js.map +1 -0
- package/dist/openapi.json +1 -1
- package/dist/{otel-spans-C8YIJWy1.js → otel-spans-B9tUXRHF.js} +3 -3
- package/dist/{otel-spans-C8YIJWy1.js.map → otel-spans-B9tUXRHF.js.map} +1 -1
- package/dist/{paired-arms-DzKn0TfY.js → paired-arms-CwMGJsrS.js} +5 -5
- package/dist/{paired-arms-DzKn0TfY.js.map → paired-arms-CwMGJsrS.js.map} +1 -1
- package/dist/{paired-binary-Q36bhn2R.js → paired-binary-CvpFHfKZ.js} +2 -2
- package/dist/{paired-binary-Q36bhn2R.js.map → paired-binary-CvpFHfKZ.js.map} +1 -1
- package/dist/{paired-promotion-decision-BN6IQTiu.js → paired-promotion-decision-DsAPYGCm.js} +3 -3
- package/dist/{paired-promotion-decision-BN6IQTiu.js.map → paired-promotion-decision-DsAPYGCm.js.map} +1 -1
- package/dist/{search-kernel-BztlERLv.js → pareto-Cm6ndrWN.js} +883 -861
- package/dist/pareto-Cm6ndrWN.js.map +1 -0
- package/dist/pipelines/index.d.ts +2 -2
- package/dist/pipelines/index.js +3 -3
- package/dist/{power-and-mde-fK5Qocwo.js → power-and-mde-Hj2ubb4B.js} +2 -2
- package/dist/{power-and-mde-fK5Qocwo.js.map → power-and-mde-Hj2ubb4B.js.map} +1 -1
- package/dist/predicate-FdKmElA_.js +38 -0
- package/dist/predicate-FdKmElA_.js.map +1 -0
- package/dist/{prime-protocol-DcAK9pC8.js → prime-protocol-CHKgA3a6.js} +2 -2
- package/dist/{prime-protocol-DcAK9pC8.js.map → prime-protocol-CHKgA3a6.js.map} +1 -1
- package/dist/{prime-protocol-CEE4BR9N.d.ts → prime-protocol-HiB9HX7E.d.ts} +3 -3
- package/dist/{prime-protocol-CEE4BR9N.d.ts.map → prime-protocol-HiB9HX7E.d.ts.map} +1 -1
- package/dist/{produced-state-ZCtDBxAE.js → produced-state-BsDmnwxc.js} +8 -8
- package/dist/{produced-state-ZCtDBxAE.js.map → produced-state-BsDmnwxc.js.map} +1 -1
- package/dist/profile-cell.js +1 -1
- package/dist/{release-confidence-BK2QCB5M.d.ts → promotion-gate-BawrdPYr.d.ts} +102 -102
- package/dist/promotion-gate-BawrdPYr.d.ts.map +1 -0
- package/dist/{promotion-policy-CWW35UWq.js → promotion-policy-KO57gbyV.js} +4 -4
- package/dist/{promotion-policy-CWW35UWq.js.map → promotion-policy-KO57gbyV.js.map} +1 -1
- package/dist/{proposal-findings-BZiMTzAF.js → proposal-findings-CY1xKn_a.js} +2 -2
- package/dist/{proposal-findings-BZiMTzAF.js.map → proposal-findings-CY1xKn_a.js.map} +1 -1
- package/dist/{query-B1D1qp2Z.js → query-BUCZCK7F.js} +3 -3
- package/dist/{query-B1D1qp2Z.js.map → query-BUCZCK7F.js.map} +1 -1
- package/dist/{random-Dn5fPWkt.js → random-ByUSnV9w.js} +2 -2
- package/dist/{random-Dn5fPWkt.js.map → random-ByUSnV9w.js.map} +1 -1
- package/dist/{raw-provider-sink-rB2DYxr6.js → raw-provider-sink-DoHz4JNG.js} +2 -2
- package/dist/{raw-provider-sink-rB2DYxr6.js.map → raw-provider-sink-DoHz4JNG.js.map} +1 -1
- package/dist/{redact-Dk1ZdJEm.js → redact-B5xXKqQ3.js} +97 -98
- package/dist/redact-B5xXKqQ3.js.map +1 -0
- package/dist/{redact-Cveoq6fJ.d.ts → redact-mtCCcT8z.d.ts} +4 -20
- package/dist/redact-mtCCcT8z.d.ts.map +1 -0
- package/dist/{registry-JYfPxcVz.d.ts → registry-D7_hm2fK.d.ts} +5 -5
- package/dist/{registry-JYfPxcVz.d.ts.map → registry-D7_hm2fK.d.ts.map} +1 -1
- package/dist/{release-confidence-Dbe9AS_X.js → release-confidence-CpuP65S5.js} +5 -5
- package/dist/{release-confidence-Dbe9AS_X.js.map → release-confidence-CpuP65S5.js.map} +1 -1
- package/dist/{report-command-y8H6RSX8.js → report-command-SC9nqb4M.js} +106 -171
- package/dist/report-command-SC9nqb4M.js.map +1 -0
- package/dist/reporting-html-Kvlc8oMt.js +100 -0
- package/dist/reporting-html-Kvlc8oMt.js.map +1 -0
- package/dist/reporting.d.ts +3 -3
- package/dist/reporting.js +6 -6
- package/dist/{researcher-f6mZhh58.d.ts → researcher-CEcfku-T.d.ts} +5 -5
- package/dist/{researcher-f6mZhh58.d.ts.map → researcher-CEcfku-T.d.ts.map} +1 -1
- package/dist/reward-CpCmAM8L.d.ts +78 -0
- package/dist/reward-CpCmAM8L.d.ts.map +1 -0
- package/dist/{reward-hacking-BFcQgkux.js → reward-hacking-E0d-WVcP.js} +2 -2
- package/dist/{reward-hacking-BFcQgkux.js.map → reward-hacking-E0d-WVcP.js.map} +1 -1
- package/dist/{reward-hacking-DHA7c1l8.d.ts → reward-hacking-RVdZ6Kkt.d.ts} +2 -2
- package/dist/{reward-hacking-DHA7c1l8.d.ts.map → reward-hacking-RVdZ6Kkt.d.ts.map} +1 -1
- package/dist/rl.d.ts +8 -8
- package/dist/rl.js +14 -14
- package/dist/rollout/index.d.ts +3 -2
- package/dist/rollout/index.js +2 -2
- package/dist/{rollout-D9bKKBnZ.js → rollout-DZV1t2Wg.js} +2 -2
- package/dist/{rollout-D9bKKBnZ.js.map → rollout-DZV1t2Wg.js.map} +1 -1
- package/dist/round-decimal-COrKglMh.js +10 -0
- package/dist/round-decimal-COrKglMh.js.map +1 -0
- package/dist/{rubric-predictive-validity-2SgHxoa5.d.ts → rubric-predictive-validity-C03EC7ZE.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-2SgHxoa5.d.ts.map → rubric-predictive-validity-C03EC7ZE.d.ts.map} +1 -1
- package/dist/{rubric-predictive-validity-BsgkBfT0.js → rubric-predictive-validity-DaFW70y9.js} +3 -3
- package/dist/{rubric-predictive-validity-BsgkBfT0.js.map → rubric-predictive-validity-DaFW70y9.js.map} +1 -1
- package/dist/{run-record-DlTZ_lqe.js → run-record-Boc5IUTp.js} +4 -4
- package/dist/{run-record-DlTZ_lqe.js.map → run-record-Boc5IUTp.js.map} +1 -1
- package/dist/{run-record-D1Hw-Jml.d.ts → run-record-C1E24G2A.d.ts} +2 -2
- package/dist/{run-record-D1Hw-Jml.d.ts.map → run-record-C1E24G2A.d.ts.map} +1 -1
- package/dist/{run-record-ES07wfz9.js → run-record-C5e7FjhJ.js} +3 -5
- package/dist/run-record-C5e7FjhJ.js.map +1 -0
- package/dist/search/index.d.ts +46 -32
- package/dist/search/index.d.ts.map +1 -1
- package/dist/search/index.js +3 -3
- package/dist/{search-kernel-CpcKH5Ih.d.ts → search-kernel-DyNEa6IU.d.ts} +14 -10
- package/dist/search-kernel-DyNEa6IU.d.ts.map +1 -0
- package/dist/{search-ledger-d7V8o9Lk.js → search-ledger-CVUGLDZp.js} +189 -199
- package/dist/search-ledger-CVUGLDZp.js.map +1 -0
- package/dist/{search-ledger-file-BqOo-bVo.js → search-ledger-file-DDQUgvCg.js} +4 -4
- package/dist/{search-ledger-file-BqOo-bVo.js.map → search-ledger-file-DDQUgvCg.js.map} +1 -1
- package/dist/search-ledger-types-p2xKfiDJ.d.ts.map +1 -1
- package/dist/{search-shipper-er4pK5_Y.js → search-shipper-BOSD46qv.js} +3 -3
- package/dist/{search-shipper-er4pK5_Y.js.map → search-shipper-BOSD46qv.js.map} +1 -1
- package/dist/{search-summary-CBvz3u_S.js → search-summary-BQCSWf1g.js} +5 -5
- package/dist/{search-summary-CBvz3u_S.js.map → search-summary-BQCSWf1g.js.map} +1 -1
- package/dist/{semantic-concept-judge-zQBG9hHU.js → semantic-concept-judge-BQF8te0T.js} +12 -14
- package/dist/semantic-concept-judge-BQF8te0T.js.map +1 -0
- package/dist/{sequential-D-1PPCmM.js → sequential-Bf_pbE50.js} +5 -5
- package/dist/{sequential-D-1PPCmM.js.map → sequential-Bf_pbE50.js.map} +1 -1
- package/dist/{sequential-eprocess-D1jKoihe.js → sequential-eprocess-DM_qENCl.js} +2 -2
- package/dist/{sequential-eprocess-D1jKoihe.js.map → sequential-eprocess-DM_qENCl.js.map} +1 -1
- package/dist/{server-DbcC7xMl.js → server-e9oWO_bN.js} +4 -4
- package/dist/{server-DbcC7xMl.js.map → server-e9oWO_bN.js.map} +1 -1
- package/dist/{single-run-lock-fVu3MZWH.js → single-run-lock-CV9Eyd_q.js} +2 -2
- package/dist/{single-run-lock-fVu3MZWH.js.map → single-run-lock-CV9Eyd_q.js.map} +1 -1
- package/dist/{skillopt-optimization-method-CCtzralq.js → skillopt-optimization-method-BiwA8nfX.js} +427 -12
- package/dist/skillopt-optimization-method-BiwA8nfX.js.map +1 -0
- package/dist/{spans-D9_ehFuk.d.ts → spans-BHAUXMGN.d.ts} +2 -2
- package/dist/{spans-D9_ehFuk.d.ts.map → spans-BHAUXMGN.d.ts.map} +1 -1
- package/dist/{statistical-heldout-1Gbmex7v.js → statistical-heldout-42NlCNDe.js} +3 -3
- package/dist/{statistical-heldout-1Gbmex7v.js.map → statistical-heldout-42NlCNDe.js.map} +1 -1
- package/dist/{statistical-heldout-BeR4bOdg.d.ts → statistical-heldout-CYpGaoDH.d.ts} +3 -3
- package/dist/{statistical-heldout-BeR4bOdg.d.ts.map → statistical-heldout-CYpGaoDH.d.ts.map} +1 -1
- package/dist/{store-otlp-DtB4Slsr.js → store-otlp-DJ9dtA5x.js} +135 -132
- package/dist/store-otlp-DJ9dtA5x.js.map +1 -0
- package/dist/{store-tool-spans-fZIigrpP.js → store-tool-spans-CIX5arAG.js} +7 -7
- package/dist/{store-tool-spans-fZIigrpP.js.map → store-tool-spans-CIX5arAG.js.map} +1 -1
- package/dist/{store-tool-spans-DR1SrCLK.d.ts → store-tool-spans-DdCSPa7C.d.ts} +162 -161
- package/dist/store-tool-spans-DdCSPa7C.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts.map +1 -1
- package/dist/storyboard/index.js +20 -3
- package/dist/storyboard/index.js.map +1 -1
- package/dist/{student-t-BQeKJx2N.js → student-t-DVPW7CdL.js} +2 -2
- package/dist/{student-t-BQeKJx2N.js.map → student-t-DVPW7CdL.js.map} +1 -1
- package/dist/{summary-report-DtdRucM0.js → summary-report-V_JACR4Y.js} +526 -614
- package/dist/summary-report-V_JACR4Y.js.map +1 -0
- package/dist/{summary-report-C-CtItCN.d.ts → summary-report-bdFDXq0b.d.ts} +10 -5
- package/dist/summary-report-bdFDXq0b.d.ts.map +1 -0
- package/dist/supervisor-run/index.d.ts +153 -149
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +65 -3
- package/dist/supervisor-run/index.js.map +1 -1
- package/dist/{systemone-CpTLAlqb.d.ts → systemone-BomhHr2c.d.ts} +6 -6
- package/dist/{systemone-CpTLAlqb.d.ts.map → systemone-BomhHr2c.d.ts.map} +1 -1
- package/dist/systemone-protocol.js +1 -1
- package/dist/{systemone-review-B_yo1xsd.d.ts → systemone-review-CUVcQJu6.d.ts} +2 -2
- package/dist/{systemone-review-B_yo1xsd.d.ts.map → systemone-review-CUVcQJu6.d.ts.map} +1 -1
- package/dist/{systemone-review-BZWfrtq1.js → systemone-review-D7sXwsqq.js} +2 -2
- package/dist/{systemone-review-BZWfrtq1.js.map → systemone-review-D7sXwsqq.js.map} +1 -1
- package/dist/systemone.d.ts +3 -3
- package/dist/systemone.js +4 -4
- package/dist/{task-failure-attributes-DngaLdE4.js → task-failure-attributes-hEmASjix.js} +2 -2
- package/dist/{task-failure-attributes-DngaLdE4.js.map → task-failure-attributes-hEmASjix.js.map} +1 -1
- package/dist/{terminal-record-BHocwFw1.js → terminal-record-DZyQZ3pz.js} +14 -10
- package/dist/terminal-record-DZyQZ3pz.js.map +1 -0
- package/dist/{tool-groups-mcxOXVlb.d.ts → tool-groups-C3NEcdCM.d.ts} +3 -3
- package/dist/tool-groups-C3NEcdCM.d.ts.map +1 -0
- package/dist/{tool-waste-Wi4g6Ekb.d.ts → tool-waste-BKgvd-TP.d.ts} +30 -30
- package/dist/tool-waste-BKgvd-TP.d.ts.map +1 -0
- package/dist/{tool-waste-D52PqU9Q.js → tool-waste-BlesqTqW.js} +4 -4
- package/dist/{tool-waste-D52PqU9Q.js.map → tool-waste-BlesqTqW.js.map} +1 -1
- package/dist/{trace-questions-DG7nqLQ7.d.ts → trace-questions-CM2CAocU.d.ts} +4 -4
- package/dist/{trace-questions-DG7nqLQ7.d.ts.map → trace-questions-CM2CAocU.d.ts.map} +1 -1
- package/dist/trace-repair/index.d.ts +3 -3
- package/dist/trace-repair/index.d.ts.map +1 -1
- package/dist/trace-repair/index.js +7 -13
- package/dist/trace-repair/index.js.map +1 -1
- package/dist/traces.d.ts +76 -58
- package/dist/traces.d.ts.map +1 -1
- package/dist/traces.js +9 -9
- package/dist/trajectory-replay/index.js +3 -3
- package/dist/{tree-rXLW3M_4.js → tree-C1XYeXvy.js} +2277 -2339
- package/dist/tree-C1XYeXvy.js.map +1 -0
- package/dist/{types-DBZ7keha.d.ts → types-C02qtnG8.d.ts} +4 -46
- package/dist/types-C02qtnG8.d.ts.map +1 -0
- package/dist/{types-oKRhQAde.d.ts → types-C1tnkEgM.d.ts} +4 -4
- package/dist/{types-oKRhQAde.d.ts.map → types-C1tnkEgM.d.ts.map} +1 -1
- package/dist/{types-CfZMIzB_.d.ts → types-Cbe2fWBa.d.ts} +3 -3
- package/dist/{types-CfZMIzB_.d.ts.map → types-Cbe2fWBa.d.ts.map} +1 -1
- package/dist/{types-B4C5nFw6.d.ts → types-Cq8mSbq8.d.ts} +78 -72
- package/dist/types-Cq8mSbq8.d.ts.map +1 -0
- package/dist/{verdict-DhPoXApk.js → verdict-Dl0tiT1X.js} +2 -2
- package/dist/{verdict-DhPoXApk.js.map → verdict-Dl0tiT1X.js.map} +1 -1
- package/dist/{verdict-cache-BQEsVy9n.js → verdict-cache-Bm8EKL2s.js} +2 -2
- package/dist/{verdict-cache-BQEsVy9n.js.map → verdict-cache-Bm8EKL2s.js.map} +1 -1
- package/dist/wire/index.d.ts +3 -3
- package/dist/wire/index.js +1 -1
- package/docs/design/mlbenchmarks-book-review.md +1 -1
- package/docs/design/mlbenchmarks-review/probes.mts +23 -31
- package/docs/experiment.md +1 -1
- package/package.json +1 -1
- package/dist/analyze-runs-3xpby3p2.js.map +0 -1
- package/dist/analyze-runs-CSDAh8A7.d.ts.map +0 -1
- package/dist/ast-DJ6Bc7ab.js +0 -558
- package/dist/ast-DJ6Bc7ab.js.map +0 -1
- package/dist/ast-hI-vjW6J.d.ts +0 -457
- package/dist/ast-hI-vjW6J.d.ts.map +0 -1
- package/dist/benchmark-command-CByl9dxG.js.map +0 -1
- package/dist/campaign-BQ-Jujuj.js.map +0 -1
- package/dist/campaign-evidence-OcUciD-N.js.map +0 -1
- package/dist/claim-BXcZw-jo.js.map +0 -1
- package/dist/cost-ledger-CO7UyEcQ.d.ts.map +0 -1
- package/dist/cost-ledger-D2FnJPBF.js.map +0 -1
- package/dist/define-agent-eval-Ch3MWX6k.js.map +0 -1
- package/dist/external-optimizer-subprocess-DcerlZSG.js.map +0 -1
- package/dist/feedback-trajectory-DkaDj0dj.d.ts.map +0 -1
- package/dist/index-CPJJr98o.d.ts.map +0 -1
- package/dist/index-DlH0Y9gI.d.ts.map +0 -1
- package/dist/index-NvGBiO8H.d.ts.map +0 -1
- package/dist/integrity-BV1ZDf42.js.map +0 -1
- package/dist/judge-gate-B1qYnkpi.js.map +0 -1
- package/dist/llm-client-DQveoxSM.js.map +0 -1
- package/dist/llm-judge-PXV7Xljl.js.map +0 -1
- package/dist/matrix-_BV9-Lk5.js.map +0 -1
- package/dist/mint-DCGoBYhB.d.ts +0 -155
- package/dist/mint-DCGoBYhB.d.ts.map +0 -1
- package/dist/paired-tests-jbwPdSz5.js.map +0 -1
- package/dist/redact-Cveoq6fJ.d.ts.map +0 -1
- package/dist/redact-Dk1ZdJEm.js.map +0 -1
- package/dist/release-confidence-BK2QCB5M.d.ts.map +0 -1
- package/dist/report-command-y8H6RSX8.js.map +0 -1
- package/dist/run-record-ES07wfz9.js.map +0 -1
- package/dist/search-kernel-BztlERLv.js.map +0 -1
- package/dist/search-kernel-CpcKH5Ih.d.ts.map +0 -1
- package/dist/search-ledger-d7V8o9Lk.js.map +0 -1
- package/dist/semantic-concept-judge-zQBG9hHU.js.map +0 -1
- package/dist/skillopt-optimization-method-CCtzralq.js.map +0 -1
- package/dist/store-otlp-DtB4Slsr.js.map +0 -1
- package/dist/store-tool-spans-DR1SrCLK.d.ts.map +0 -1
- package/dist/summary-report-C-CtItCN.d.ts.map +0 -1
- package/dist/summary-report-DtdRucM0.js.map +0 -1
- package/dist/terminal-record-BHocwFw1.js.map +0 -1
- package/dist/tool-groups-mcxOXVlb.d.ts.map +0 -1
- package/dist/tool-waste-Wi4g6Ekb.d.ts.map +0 -1
- package/dist/tree-rXLW3M_4.js.map +0 -1
- package/dist/types-B4C5nFw6.d.ts.map +0 -1
- package/dist/types-DBZ7keha.d.ts.map +0 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { s as RedactionReport } from "./redact-
|
|
1
|
+
import { s as RedactionReport } from "./redact-mtCCcT8z.js";
|
|
2
2
|
import { SpanKind } from "@tangle-network/agent-trace-contract";
|
|
3
3
|
//#region src/diagnosis/spans.d.ts
|
|
4
4
|
type DiagnosisSpanStatus = 'OK' | 'ERROR' | 'UNSET';
|
|
@@ -57,4 +57,4 @@ declare function ingestSpans(raw: readonly unknown[], options: {
|
|
|
57
57
|
declare function epochMillis(value: unknown): number | null;
|
|
58
58
|
//#endregion
|
|
59
59
|
export { isContentAttribute as a, ingestSpans as i, IngestReport as n, epochMillis as r, DiagnosisSpan as t };
|
|
60
|
-
//# sourceMappingURL=spans-
|
|
60
|
+
//# sourceMappingURL=spans-BHAUXMGN.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"spans-
|
|
1
|
+
{"version":3,"file":"spans-BHAUXMGN.d.ts","names":[],"sources":["../src/diagnosis/spans.ts"],"mappings":";;;KAuBY;UAEK;EACf;EACA;EACA;EACA;EACA,MAAM;EACN;EACA;EACA,QAAQ;EACR;EACA;EACA;;;;;;;EAOA;EACA,YAAY;;UAGG;EACf;EACA;;EAEA;;EAEA;;EAEA;EACA,SAAS;;EAET;;iBA0Bc,mBAAmB;;;;;;;iBAUnB,YACd,yBACA;EAAW;;EACR,OAAO;EAAiB,QAAQ;;;;;;;iBAgHrB,YAAY"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { t as decidePairedPromotion } from "./paired-promotion-decision-
|
|
1
|
+
import { i as pairedBootstrap } from "./mean-DHoMiudD.js";
|
|
2
|
+
import { t as decidePairedPromotion } from "./paired-promotion-decision-DsAPYGCm.js";
|
|
3
3
|
//#region src/campaign/gates/statistical-heldout.ts
|
|
4
4
|
/**
|
|
5
5
|
* Held-out inference pairs execution cells and judge identities before taking
|
|
@@ -253,4 +253,4 @@ function dimensionRegressions(candidate, baseline, scenarioIds, criticalDimensio
|
|
|
253
253
|
//#endregion
|
|
254
254
|
export { heldoutSignificance as a, dimensionRegressions as i, aggregatePairedHoldout as n, pairHoldout as o, detectScale as r, TIE_WARN_FRACTION as t };
|
|
255
255
|
|
|
256
|
-
//# sourceMappingURL=statistical-heldout-
|
|
256
|
+
//# sourceMappingURL=statistical-heldout-42NlCNDe.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"statistical-heldout-1Gbmex7v.js","names":[],"sources":["../src/campaign/gates/statistical-heldout.ts"],"sourcesContent":["/**\n * Held-out inference pairs execution cells and judge identities before taking\n * means within registered independent units. Repetitions can improve a unit's\n * precision without increasing n. Ungrouped inference concerns independently\n * sampled execution cells conditional on a fixed scenario roster.\n *\n * The shared paired decision rule selects the estimator and statistical test.\n * Dimension reports retain missing coverage so required safety checks cannot\n * pass through absent evidence. Thresholds use the judge's native score scale.\n */\n\nimport {\n decidePairedPromotion,\n type PairedDecisionMethod,\n type PairedDecisionStatistic,\n type PairedMcNemarEvidence,\n type PairedPromotionDecision,\n} from '../../paired-promotion-decision'\nimport {\n DECISION_PAIRED_DELTA_STATISTIC,\n type PairedBootstrapResult,\n pairedBootstrap,\n} from '../../statistics'\nimport type { JudgeScore } from '../types'\n\n/** Tie fraction at/above which a gate annotates its verdict with the tie share.\n * Tie-domination of the median bites structurally at >= 0.5 (the median is then\n * 0 by construction); 0.4 is a softer warn threshold that flags a run APPROACHING\n * that regime, so an operator sees it before the median goes fully blind. */\nexport const TIE_WARN_FRACTION = 0.4\n\nexport interface PairedHoldout {\n /** Baseline scalar per paired cell (same order as `after`/`cellIds`). */\n before: number[]\n /** Candidate scalar per paired cell. */\n after: number[]\n /** The full cellIds (`scenario:rep`) that paired, in order. */\n cellIds: string[]\n}\n\n/** Campaign cell IDs append a numeric repetition after the scenario's full ID. */\nexport function scenarioIdFromCellId(cellId: string): string {\n const separator = cellId.lastIndexOf(':')\n const repetition = cellId.slice(separator + 1)\n if (\n separator < 1 ||\n cellId.trim() !== cellId ||\n !/^(0|[1-9]\\d*)$/.test(repetition) ||\n !Number.isSafeInteger(Number(repetition))\n ) {\n throw new Error(`pairHoldout: malformed cellId '${cellId}'; expected scenarioId:rep`)\n }\n return cellId.slice(0, separator)\n}\n\n/** Preserve cell pairing before taking equal-weight independent-unit means. */\nexport function aggregatePairedHoldout(\n paired: PairedHoldout,\n independentUnitByScenarioId?: ReadonlyMap<string, string>,\n): { before: number[]; after: number[]; unitIds: string[] } {\n if (\n paired.before.length !== paired.after.length ||\n paired.before.length !== paired.cellIds.length\n ) {\n throw new Error('aggregatePairedHoldout: scores and cellIds must have the same length')\n }\n if (new Set(paired.cellIds).size !== paired.cellIds.length) {\n throw new Error('aggregatePairedHoldout: duplicate cellIds cannot count as new observations')\n }\n if (\n paired.before.some((value) => !Number.isFinite(value)) ||\n paired.after.some((value) => !Number.isFinite(value))\n ) {\n throw new Error('aggregatePairedHoldout: paired scores must be finite')\n }\n if (independentUnitByScenarioId === undefined) {\n return { before: [...paired.before], after: [...paired.after], unitIds: [...paired.cellIds] }\n }\n const scenarioIds = paired.cellIds.map(scenarioIdFromCellId)\n const groups = new Map<string, { before: number; after: number; n: number }>()\n for (let i = 0; i < paired.cellIds.length; i++) {\n const scenarioId = scenarioIds[i]!\n const unitId = independentUnitByScenarioId.get(scenarioId)\n if (typeof unitId !== 'string' || unitId.length === 0 || unitId.trim() !== unitId) {\n throw new Error(\n `aggregatePairedHoldout: missing independent unit for scenario '${scenarioId}'`,\n )\n }\n const group = groups.get(unitId) ?? { before: 0, after: 0, n: 0 }\n group.before += paired.before[i]!\n group.after += paired.after[i]!\n group.n += 1\n groups.set(unitId, group)\n }\n const unitIds = [...groups.keys()].sort()\n return {\n before: unitIds.map((id) => groups.get(id)!.before / groups.get(id)!.n),\n after: unitIds.map((id) => groups.get(id)!.after / groups.get(id)!.n),\n unitIds,\n }\n}\n\n/**\n * Pair candidate vs baseline holdout observations by FULL cellId. `select`\n * pulls the scalar from a cell's judge reports (composite, or a named\n * dimension); a cell contributes the mean of `select` across its judges. Cells\n * whose scenario is not in `scenarioIds`, or where `select` is undefined for\n * every judge on both sides, are skipped. The selected judge IDs must agree\n * within each pair. Throws when the two maps disagree on holdout cell IDs — a\n * load-bearing invariant: the baseline + winner holdout campaigns run the same\n * scenarios with the same seed base, so their cellIds MUST align; a mismatch\n * means a silent pairing bug, not a soft fallback.\n */\nexport function pairHoldout(\n candidate: Map<string, Record<string, JudgeScore>>,\n baseline: Map<string, Record<string, JudgeScore>>,\n scenarioIds: Set<string>,\n select: (s: JudgeScore) => number | undefined,\n): PairedHoldout {\n const cellValues = (\n byCell: Map<string, Record<string, JudgeScore>>,\n cellId: string,\n ): Map<string, number> => {\n const scores = byCell.get(cellId)\n const values = new Map<string, number>()\n if (!scores) return values\n for (const [judgeId, s] of Object.entries(scores)) {\n if (s.failed === true) {\n throw new Error(`pairHoldout: cell '${cellId}' contains a failed judge score`)\n }\n const v = select(s)\n if (typeof v === 'number' && !Number.isFinite(v)) {\n throw new Error(`pairHoldout: cell '${cellId}' contains a non-finite selected score`)\n }\n if (typeof v === 'number') values.set(judgeId, v)\n }\n return values\n }\n\n const inScope = (cellId: string) => scenarioIds.has(scenarioIdFromCellId(cellId))\n const candCells = [...candidate.keys()].filter(inScope).sort()\n const baseCells = [...baseline.keys()].filter(inScope).sort()\n // Alignment invariant — the holdout campaigns share scenarios + seed, so the\n // cell sets must be identical. Differ ⇒ a real pairing bug; fail loud.\n if (candCells.length !== baseCells.length || candCells.some((c, i) => c !== baseCells[i])) {\n throw new Error(\n `pairHoldout: candidate/baseline holdout cells do not align — ` +\n `candidate=[${candCells.join(',')}] baseline=[${baseCells.join(',')}]. ` +\n `Both holdout campaigns must run the same scenarios with the same seed base.`,\n )\n }\n\n const before: number[] = []\n const after: number[] = []\n const cellIds: string[] = []\n for (const cellId of candCells) {\n const b = cellValues(baseline, cellId)\n const a = cellValues(candidate, cellId)\n // A scalar absent on both sides means that dimension was not scored. A\n // one-sided absence is asymmetric evidence loss, never a row to discard.\n if (b.size === 0 && a.size === 0) continue\n if (b.size === 0 || a.size === 0) {\n throw new Error(`pairHoldout: cell '${cellId}' has a selected score on only one arm`)\n }\n if (b.size !== a.size || [...b.keys()].some((id) => !a.has(id))) {\n throw new Error(`pairHoldout: cell '${cellId}' selected judge IDs do not align`)\n }\n const judgeIds = [...b.keys()].sort()\n before.push(meanSelectedScores(judgeIds.map((id) => b.get(id)!)))\n after.push(meanSelectedScores(judgeIds.map((id) => a.get(id)!)))\n cellIds.push(cellId)\n }\n return { before, after, cellIds }\n}\n\nfunction meanSelectedScores(values: number[]): number {\n const first = values[0]!\n // Summing identical fractional scores can round their mean outside the\n // declared binary support. An agreeing judge set preserves its exact value.\n return values.every((value) => value === first)\n ? first\n : values.reduce((sum, value) => sum + value, 0) / values.length\n}\n\nexport interface HeldoutSignificance {\n paired: PairedHoldout\n /**\n * The paired bootstrap on the requested statistic (MEAN by default — see the\n * tie note on `heldoutSignificance`).\n *\n * DIAGNOSTIC, not necessarily the interval the verdict keyed on. On a\n * two-point (pass/fail) outcome the decision routes to Tango's score interval\n * instead, because a percentile bootstrap of the mean over a three-atom\n * lattice is not a valid interval at a nonzero margin. Read\n * `decision.low`/`decision.high` for the interval that actually decided, and\n * `decisionStatistic` for which one it is.\n */\n bootstrap: PairedBootstrapResult\n /** The MEDIAN paired-delta bootstrap, reported as a diagnostic. When many\n * scenarios are tied (both sides solve them), the median is pinned near 0\n * regardless of the mean lift — comparing the two exposes tie-domination. */\n medianBootstrap: PairedBootstrapResult\n /**\n * The full promotion decision: which estimator the outcome's shape admits,\n * the interval it produced, McNemar's exact veto on the two-point path, and\n * whether the interval was zero-width (no evidence in either direction). The\n * single source of `significant`.\n */\n decision: PairedPromotionDecision\n /** Which paired estimator the verdict was decided on. */\n decisionStatistic: PairedDecisionStatistic\n /** McNemar's exact evidence on the two-point path; null otherwise. */\n mcnemar: PairedMcNemarEvidence | null\n /** Fraction of paired observations that are exact ties (|delta| < 1e-9). A\n * high tie fraction is WHY a median-based gate would have missed a real lift;\n * it is the observability the tie fix adds. */\n tieFraction: number\n /** Number of paired observation units, after configured aggregation. */\n n: number\n /** Original matched execution cells, before aggregation. */\n pairedCellN: number\n observationUnit: 'registered' | 'cell'\n /** Registered unit IDs, or cell IDs on the ungrouped path. */\n unitIds: string[]\n /** Effective minimum for the requested target and chosen estimator. */\n minimumRequired: number\n /** Statistical method that carried the decision. */\n decisionMethod: PairedDecisionMethod\n /** Exact one-sided p-value on the small-sample path; otherwise null. */\n pValue: number | null\n /** True iff n >= minimumRequired, the DECIDING interval has nonzero width,\n * its lower bound clears the threshold, and McNemar's exact test does not\n * veto at a non-negative threshold. */\n significant: boolean\n /** Set when n < minimumRequired — too little evidence to claim significance. */\n fewRuns: boolean\n}\n\nexport interface HeldoutSignificanceOptions {\n deltaThreshold?: number\n minProductiveRuns?: number\n confidence?: number\n resamples?: number\n /** Fixed by default for a deterministic, reproducible gate verdict. */\n seed?: number\n statistic?: 'mean' | 'median'\n /** Group full cell pairs into equal-weight independent units before inference. */\n independentUnitByScenarioId?: ReadonlyMap<string, string>\n}\n\n/**\n * Significance of the held-out composite lift: ship only when the lower bound\n * of the interval the outcome's shape ADMITS exceeds `deltaThreshold` (default\n * 0 ⇒ \"confidently positive\"). Interpret `deltaThreshold` in the judge's native\n * scale.\n *\n * The decision is delegated whole to {@link decidePairedPromotion}, the one\n * copy of the rule (`src/paired-promotion-decision.ts`), which `HeldOutGate`\n * also calls. That module's header carries the measurements; the short version\n * is three guards a bare `bootstrap.low > threshold` does not have:\n *\n * - a two-point (pass/fail) outcome decides on Tango's SCORE interval, the\n * only paired-binary construction that stays valid at a nonzero margin;\n * - McNemar's exact test VETOES at any non-negative threshold;\n * - a ZERO-WIDTH interval is refused rather than promoted, in either\n * direction — [0,0] clears every negative threshold and [g,g] clears every\n * threshold below g, and both are an absence of evidence, not a result.\n *\n * Measured on this function before those guards landed, at a nominal 5 %:\n * 14.60 % false promotion at n = 40 on a paired-binary noninferiority boundary,\n * and 88.50 % at n = 6 under a bounded asymmetric null whose true mean paired\n * delta is exactly 0.\n *\n * Continuous mean targets require bootstrap eligibility. Explicit median\n * targets can use the exact sign test at its confidence-dependent minimum.\n */\nexport function heldoutSignificance(\n paired: PairedHoldout,\n opts: HeldoutSignificanceOptions = {},\n): HeldoutSignificance {\n const deltaThreshold = opts.deltaThreshold ?? 0\n const confidence = opts.confidence ?? 0.95\n const resamples = opts.resamples ?? 2000\n const seed = opts.seed ?? 1337\n // DEFAULT to the MEAN paired delta, not the median. The median is destroyed by\n // TIES: whenever both baseline and candidate solve a holdout scenario (a common\n // case once the agent is decent — and INCREASINGLY common as you add holdout\n // scenarios for statistical power), that scenario contributes delta 0. When\n // >=50% of paired cells are ties the median is pinned at 0 regardless of a large,\n // consistent lift on the rest, and the gate holds a genuinely better candidate.\n // (Measured live, supervisor-lab run 6: 40 cells, 20 ties, MEAN +0.177, MEDIAN 0\n // → false hold. Doubling the holdout for \"power\" made it WORSE by adding ties.)\n // The mean equals the reported aggregate lift, ties correctly contribute 0\n // without dominating, and it is the textbook paired-comparison estimator; the\n // median is kept as a reported diagnostic. Callers wanting outlier-robustness at\n // the cost of tie-blindness can still pass `statistic: 'median'`.\n const statistic = opts.statistic ?? 'mean'\n const observations = aggregatePairedHoldout(paired, opts.independentUnitByScenarioId)\n const decision = decidePairedPromotion(observations.before, observations.after, {\n confidence,\n resamples,\n statistic,\n seed,\n threshold: deltaThreshold,\n minPairs: opts.minProductiveRuns,\n })\n // `decision.bootstrap` is null exactly when the score interval decided, so\n // the requested-statistic bootstrap is computed here for the diagnostic\n // field. Same two bootstraps as before on every path.\n const bootstrap =\n decision.bootstrap ??\n pairedBootstrap(observations.before, observations.after, {\n confidence,\n resamples,\n statistic,\n seed,\n })\n const medianBootstrap =\n statistic === 'median'\n ? bootstrap\n : pairedBootstrap(observations.before, observations.after, {\n confidence,\n resamples,\n statistic: 'median',\n seed,\n })\n const n = observations.before.length\n let ties = 0\n for (let i = 0; i < n; i += 1) {\n const after = observations.after[i]!\n const before = observations.before[i]!\n if (Math.abs(after - before) < 1e-9) ties += 1\n }\n const tieFraction = n === 0 ? 0 : ties / n\n return {\n paired,\n bootstrap,\n medianBootstrap,\n decision,\n decisionStatistic: decision.statistic,\n mcnemar: decision.mcnemar,\n tieFraction,\n n,\n pairedCellN: paired.cellIds.length,\n observationUnit: opts.independentUnitByScenarioId === undefined ? 'cell' : 'registered',\n unitIds: observations.unitIds,\n minimumRequired: decision.minimumPairs,\n decisionMethod: decision.method,\n pValue: decision.pValue,\n significant: decision.promote,\n fewRuns: !decision.sufficient,\n }\n}\n\nexport interface DimensionRegression {\n dimension: string\n /** Paired bootstrap on (candidate − baseline). DIAGNOSTIC on a pass/fail\n * dimension, where `ci` carries the interval that decided instead. */\n bootstrap: PairedBootstrapResult\n /** Which paired statistic `bootstrap.low` is the lower bound of. `'mean'`\n * unless the caller asked for the median. `bootstrap.median` still carries\n * the median point estimate either way. */\n bootstrapStatistic: 'median' | 'mean'\n /** The interval `regressed` was decided on, in the dimension's native units. */\n ci: { low: number; high: number }\n /** Which estimator produced `ci`. */\n decisionStatistic: PairedDecisionStatistic\n /** McNemar's exact evidence on a pass/fail dimension; null otherwise. */\n mcnemar: PairedMcNemarEvidence | null\n /** `ci` has zero width — no evidence in either direction. */\n indeterminate: boolean\n /** The bootstrap lower bound is below negative tolerance, or the shared\n * paired test supports a drop exceeding tolerance. Missing coverage and\n * insufficient observations are reported separately. */\n regressed: boolean\n tolerance: number\n n: number\n pairedCellN: number\n observationUnit: 'registered' | 'cell'\n /** Statistical minimum for the configured independent observation unit. */\n minimumRequired: number\n fewRuns: boolean\n /** Both arms lack this dimension on these otherwise matched execution cells. */\n missingCellIds: string[]\n /** Configured scenarios with no paired dimension measurement at all. */\n missingScenarioIds: string[]\n}\n\n/** Detect the native scale of a set of scores: 0-100 when any magnitude clears\n * 1.5, else [0,1]. Used to auto-scale the regression tolerance so a default\n * expressed for [0,1] is not silently a no-op on a 0-100 dimension. */\nexport function detectScale(values: number[]): 1 | 100 {\n return values.some((v) => Math.abs(v) > 1.5) ? 100 : 1\n}\n\n/**\n * Report required-dimension evidence after full pairing and optional unit means.\n * A bootstrap floor breach or a shared paired test supporting a drop marks\n * regression. These two criteria are distinct; `ci` records the shared\n * estimator and `bootstrap` records the floor interval. Missing observations\n * and insufficient n remain explicit for the caller's evidence policy.\n * The default tolerance is 0.05 on [0,1] and 5 on a detected 0-100 scale.\n */\nexport function dimensionRegressions(\n candidate: Map<string, Record<string, JudgeScore>>,\n baseline: Map<string, Record<string, JudgeScore>>,\n scenarioIds: Set<string>,\n criticalDimensions: string[],\n opts: {\n tolerance?: number\n confidence?: number\n resamples?: number\n seed?: number\n /** Paired statistic the CI is computed on. Default `'mean'` — see\n * {@link DECISION_PAIRED_DELTA_STATISTIC} for why the median is not. */\n statistic?: 'mean' | 'median'\n independentUnitByScenarioId?: ReadonlyMap<string, string>\n minProductiveRuns?: number\n } = {},\n): DimensionRegression[] {\n const out: DimensionRegression[] = []\n const expectedCellIds = [...baseline.keys()]\n .filter((cellId) => scenarioIds.has(scenarioIdFromCellId(cellId)))\n .sort()\n for (const dim of criticalDimensions) {\n const paired = pairHoldout(candidate, baseline, scenarioIds, (s) => s.dimensions[dim])\n if (paired.before.length === 0) continue // dimension not scored on this judge\n const observations = aggregatePairedHoldout(paired, opts.independentUnitByScenarioId)\n const measuredCells = new Set(paired.cellIds)\n const measuredScenarios = new Set(paired.cellIds.map(scenarioIdFromCellId))\n const tolerance = opts.tolerance ?? 0.05 * detectScale([...paired.before, ...paired.after])\n const bootstrapStatistic = opts.statistic ?? DECISION_PAIRED_DELTA_STATISTIC\n const shared = {\n confidence: opts.confidence ?? 0.95,\n resamples: opts.resamples ?? 2000,\n statistic: bootstrapStatistic,\n seed: opts.seed ?? 1337,\n minPairs: opts.minProductiveRuns,\n }\n const guard = decidePairedPromotion(observations.before, observations.after, shared)\n const regression = decidePairedPromotion(observations.after, observations.before, {\n ...shared,\n threshold: tolerance,\n })\n const bootstrap =\n guard.bootstrap ?? pairedBootstrap(observations.before, observations.after, shared)\n out.push({\n dimension: dim,\n bootstrap,\n bootstrapStatistic,\n ci: { low: guard.low, high: guard.high },\n decisionStatistic: guard.statistic,\n mcnemar: guard.mcnemar,\n indeterminate: guard.indeterminate,\n // Fires on EITHER burden of proof — see `floorBreached` in\n // `promotion-policy.ts` for the same rule and the reasoning. The CI arm\n // is the contract this interface and `default-production-gate`'s reason\n // string both state (\"CI.low < -tolerance\"), read off the DECIDING\n // interval; the exact-test arm adds the small-sample path where the\n // bootstrap interval is descriptive only.\n // The credible-worst-case arm stays on the BOOTSTRAP — see the same\n // decision in `floorBreached` (`promotion-policy.ts`). On a pass/fail\n // dimension the score interval is ±z²/(n+z²) even when every pair is\n // concordant, so reading the floor off it would flag an unchanged safety\n // dimension as regressed at any realistic n. The PROVEN-drop arm does use\n // the shared rule, which is what closes the fail-open hole that mattered:\n // a genuine pass/fail regression is now judged on an interval valid at\n // the nonzero `tolerance`, instead of a bootstrap that is pinned wherever\n // ties dominate.\n regressed: bootstrap.low < -tolerance || regression.promote,\n tolerance,\n n: observations.before.length,\n pairedCellN: paired.cellIds.length,\n observationUnit: opts.independentUnitByScenarioId === undefined ? 'cell' : 'registered',\n minimumRequired: guard.minimumPairs,\n fewRuns: !guard.sufficient,\n missingCellIds: expectedCellIds.filter((cellId) => !measuredCells.has(cellId)),\n missingScenarioIds: [...scenarioIds].filter((id) => !measuredScenarios.has(id)).sort(),\n })\n }\n return out\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AA6BA,MAAa,oBAAoB;;AAYjC,SAAgB,qBAAqB,QAAwB;CAC3D,MAAM,YAAY,OAAO,YAAY,GAAG;CACxC,MAAM,aAAa,OAAO,MAAM,YAAY,CAAC;CAC7C,IACE,YAAY,KACZ,OAAO,KAAK,MAAM,UAClB,CAAC,iBAAiB,KAAK,UAAU,KACjC,CAAC,OAAO,cAAc,OAAO,UAAU,CAAC,GAExC,MAAM,IAAI,MAAM,kCAAkC,OAAO,2BAA2B;CAEtF,OAAO,OAAO,MAAM,GAAG,SAAS;AAClC;;AAGA,SAAgB,uBACd,QACA,6BAC0D;CAC1D,IACE,OAAO,OAAO,WAAW,OAAO,MAAM,UACtC,OAAO,OAAO,WAAW,OAAO,QAAQ,QAExC,MAAM,IAAI,MAAM,sEAAsE;CAExF,IAAI,IAAI,IAAI,OAAO,OAAO,CAAC,CAAC,SAAS,OAAO,QAAQ,QAClD,MAAM,IAAI,MAAM,4EAA4E;CAE9F,IACE,OAAO,OAAO,MAAM,UAAU,CAAC,OAAO,SAAS,KAAK,CAAC,KACrD,OAAO,MAAM,MAAM,UAAU,CAAC,OAAO,SAAS,KAAK,CAAC,GAEpD,MAAM,IAAI,MAAM,sDAAsD;CAExE,IAAI,gCAAgC,KAAA,GAClC,OAAO;EAAE,QAAQ,CAAC,GAAG,OAAO,MAAM;EAAG,OAAO,CAAC,GAAG,OAAO,KAAK;EAAG,SAAS,CAAC,GAAG,OAAO,OAAO;CAAE;CAE9F,MAAM,cAAc,OAAO,QAAQ,IAAI,oBAAoB;CAC3D,MAAM,yBAAS,IAAI,IAA0D;CAC7E,KAAK,IAAI,IAAI,GAAG,IAAI,OAAO,QAAQ,QAAQ,KAAK;EAC9C,MAAM,aAAa,YAAY;EAC/B,MAAM,SAAS,4BAA4B,IAAI,UAAU;EACzD,IAAI,OAAO,WAAW,YAAY,OAAO,WAAW,KAAK,OAAO,KAAK,MAAM,QACzE,MAAM,IAAI,MACR,kEAAkE,WAAW,EAC/E;EAEF,MAAM,QAAQ,OAAO,IAAI,MAAM,KAAK;GAAE,QAAQ;GAAG,OAAO;GAAG,GAAG;EAAE;EAChE,MAAM,UAAU,OAAO,OAAO;EAC9B,MAAM,SAAS,OAAO,MAAM;EAC5B,MAAM,KAAK;EACX,OAAO,IAAI,QAAQ,KAAK;CAC1B;CACA,MAAM,UAAU,CAAC,GAAG,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK;CACxC,OAAO;EACL,QAAQ,QAAQ,KAAK,OAAO,OAAO,IAAI,EAAE,CAAC,CAAE,SAAS,OAAO,IAAI,EAAE,CAAC,CAAE,CAAC;EACtE,OAAO,QAAQ,KAAK,OAAO,OAAO,IAAI,EAAE,CAAC,CAAE,QAAQ,OAAO,IAAI,EAAE,CAAC,CAAE,CAAC;EACpE;CACF;AACF;;;;;;;;;;;;AAaA,SAAgB,YACd,WACA,UACA,aACA,QACe;CACf,MAAM,cACJ,QACA,WACwB;EACxB,MAAM,SAAS,OAAO,IAAI,MAAM;EAChC,MAAM,yBAAS,IAAI,IAAoB;EACvC,IAAI,CAAC,QAAQ,OAAO;EACpB,KAAK,MAAM,CAAC,SAAS,MAAM,OAAO,QAAQ,MAAM,GAAG;GACjD,IAAI,EAAE,WAAW,MACf,MAAM,IAAI,MAAM,sBAAsB,OAAO,gCAAgC;GAE/E,MAAM,IAAI,OAAO,CAAC;GAClB,IAAI,OAAO,MAAM,YAAY,CAAC,OAAO,SAAS,CAAC,GAC7C,MAAM,IAAI,MAAM,sBAAsB,OAAO,uCAAuC;GAEtF,IAAI,OAAO,MAAM,UAAU,OAAO,IAAI,SAAS,CAAC;EAClD;EACA,OAAO;CACT;CAEA,MAAM,WAAW,WAAmB,YAAY,IAAI,qBAAqB,MAAM,CAAC;CAChF,MAAM,YAAY,CAAC,GAAG,UAAU,KAAK,CAAC,CAAC,CAAC,OAAO,OAAO,CAAC,CAAC,KAAK;CAC7D,MAAM,YAAY,CAAC,GAAG,SAAS,KAAK,CAAC,CAAC,CAAC,OAAO,OAAO,CAAC,CAAC,KAAK;CAG5D,IAAI,UAAU,WAAW,UAAU,UAAU,UAAU,MAAM,GAAG,MAAM,MAAM,UAAU,EAAE,GACtF,MAAM,IAAI,MACR,2EACgB,UAAU,KAAK,GAAG,EAAE,cAAc,UAAU,KAAK,GAAG,EAAE,+EAExE;CAGF,MAAM,SAAmB,CAAC;CAC1B,MAAM,QAAkB,CAAC;CACzB,MAAM,UAAoB,CAAC;CAC3B,KAAK,MAAM,UAAU,WAAW;EAC9B,MAAM,IAAI,WAAW,UAAU,MAAM;EACrC,MAAM,IAAI,WAAW,WAAW,MAAM;EAGtC,IAAI,EAAE,SAAS,KAAK,EAAE,SAAS,GAAG;EAClC,IAAI,EAAE,SAAS,KAAK,EAAE,SAAS,GAC7B,MAAM,IAAI,MAAM,sBAAsB,OAAO,uCAAuC;EAEtF,IAAI,EAAE,SAAS,EAAE,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,EAAE,IAAI,EAAE,CAAC,GAC5D,MAAM,IAAI,MAAM,sBAAsB,OAAO,kCAAkC;EAEjF,MAAM,WAAW,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK;EACpC,OAAO,KAAK,mBAAmB,SAAS,KAAK,OAAO,EAAE,IAAI,EAAE,CAAE,CAAC,CAAC;EAChE,MAAM,KAAK,mBAAmB,SAAS,KAAK,OAAO,EAAE,IAAI,EAAE,CAAE,CAAC,CAAC;EAC/D,QAAQ,KAAK,MAAM;CACrB;CACA,OAAO;EAAE;EAAQ;EAAO;CAAQ;AAClC;AAEA,SAAS,mBAAmB,QAA0B;CACpD,MAAM,QAAQ,OAAO;CAGrB,OAAO,OAAO,OAAO,UAAU,UAAU,KAAK,IAC1C,QACA,OAAO,QAAQ,KAAK,UAAU,MAAM,OAAO,CAAC,IAAI,OAAO;AAC7D;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8FA,SAAgB,oBACd,QACA,OAAmC,CAAC,GACf;CACrB,MAAM,iBAAiB,KAAK,kBAAkB;CAC9C,MAAM,aAAa,KAAK,cAAc;CACtC,MAAM,YAAY,KAAK,aAAa;CACpC,MAAM,OAAO,KAAK,QAAQ;CAa1B,MAAM,YAAY,KAAK,aAAa;CACpC,MAAM,eAAe,uBAAuB,QAAQ,KAAK,2BAA2B;CACpF,MAAM,WAAW,sBAAsB,aAAa,QAAQ,aAAa,OAAO;EAC9E;EACA;EACA;EACA;EACA,WAAW;EACX,UAAU,KAAK;CACjB,CAAC;CAID,MAAM,YACJ,SAAS,aACT,gBAAgB,aAAa,QAAQ,aAAa,OAAO;EACvD;EACA;EACA;EACA;CACF,CAAC;CACH,MAAM,kBACJ,cAAc,WACV,YACA,gBAAgB,aAAa,QAAQ,aAAa,OAAO;EACvD;EACA;EACA,WAAW;EACX;CACF,CAAC;CACP,MAAM,IAAI,aAAa,OAAO;CAC9B,IAAI,OAAO;CACX,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK,GAAG;EAC7B,MAAM,QAAQ,aAAa,MAAM;EACjC,MAAM,SAAS,aAAa,OAAO;EACnC,IAAI,KAAK,IAAI,QAAQ,MAAM,IAAI,MAAM,QAAQ;CAC/C;CACA,MAAM,cAAc,MAAM,IAAI,IAAI,OAAO;CACzC,OAAO;EACL;EACA;EACA;EACA;EACA,mBAAmB,SAAS;EAC5B,SAAS,SAAS;EAClB;EACA;EACA,aAAa,OAAO,QAAQ;EAC5B,iBAAiB,KAAK,gCAAgC,KAAA,IAAY,SAAS;EAC3E,SAAS,aAAa;EACtB,iBAAiB,SAAS;EAC1B,gBAAgB,SAAS;EACzB,QAAQ,SAAS;EACjB,aAAa,SAAS;EACtB,SAAS,CAAC,SAAS;CACrB;AACF;;;;AAuCA,SAAgB,YAAY,QAA2B;CACrD,OAAO,OAAO,MAAM,MAAM,KAAK,IAAI,CAAC,IAAI,GAAG,IAAI,MAAM;AACvD;;;;;;;;;AAUA,SAAgB,qBACd,WACA,UACA,aACA,oBACA,OAUI,CAAC,GACkB;CACvB,MAAM,MAA6B,CAAC;CACpC,MAAM,kBAAkB,CAAC,GAAG,SAAS,KAAK,CAAC,CAAC,CACzC,QAAQ,WAAW,YAAY,IAAI,qBAAqB,MAAM,CAAC,CAAC,CAAC,CACjE,KAAK;CACR,KAAK,MAAM,OAAO,oBAAoB;EACpC,MAAM,SAAS,YAAY,WAAW,UAAU,cAAc,MAAM,EAAE,WAAW,IAAI;EACrF,IAAI,OAAO,OAAO,WAAW,GAAG;EAChC,MAAM,eAAe,uBAAuB,QAAQ,KAAK,2BAA2B;EACpF,MAAM,gBAAgB,IAAI,IAAI,OAAO,OAAO;EAC5C,MAAM,oBAAoB,IAAI,IAAI,OAAO,QAAQ,IAAI,oBAAoB,CAAC;EAC1E,MAAM,YAAY,KAAK,aAAa,MAAO,YAAY,CAAC,GAAG,OAAO,QAAQ,GAAG,OAAO,KAAK,CAAC;EAC1F,MAAM,qBAAqB,KAAK,aAAA;EAChC,MAAM,SAAS;GACb,YAAY,KAAK,cAAc;GAC/B,WAAW,KAAK,aAAa;GAC7B,WAAW;GACX,MAAM,KAAK,QAAQ;GACnB,UAAU,KAAK;EACjB;EACA,MAAM,QAAQ,sBAAsB,aAAa,QAAQ,aAAa,OAAO,MAAM;EACnF,MAAM,aAAa,sBAAsB,aAAa,OAAO,aAAa,QAAQ;GAChF,GAAG;GACH,WAAW;EACb,CAAC;EACD,MAAM,YACJ,MAAM,aAAa,gBAAgB,aAAa,QAAQ,aAAa,OAAO,MAAM;EACpF,IAAI,KAAK;GACP,WAAW;GACX;GACA;GACA,IAAI;IAAE,KAAK,MAAM;IAAK,MAAM,MAAM;GAAK;GACvC,mBAAmB,MAAM;GACzB,SAAS,MAAM;GACf,eAAe,MAAM;GAgBrB,WAAW,UAAU,MAAM,CAAC,aAAa,WAAW;GACpD;GACA,GAAG,aAAa,OAAO;GACvB,aAAa,OAAO,QAAQ;GAC5B,iBAAiB,KAAK,gCAAgC,KAAA,IAAY,SAAS;GAC3E,iBAAiB,MAAM;GACvB,SAAS,CAAC,MAAM;GAChB,gBAAgB,gBAAgB,QAAQ,WAAW,CAAC,cAAc,IAAI,MAAM,CAAC;GAC7E,oBAAoB,CAAC,GAAG,WAAW,CAAC,CAAC,QAAQ,OAAO,CAAC,kBAAkB,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK;EACvF,CAAC;CACH;CACA,OAAO;AACT"}
|
|
1
|
+
{"version":3,"file":"statistical-heldout-42NlCNDe.js","names":[],"sources":["../src/campaign/gates/statistical-heldout.ts"],"sourcesContent":["/**\n * Held-out inference pairs execution cells and judge identities before taking\n * means within registered independent units. Repetitions can improve a unit's\n * precision without increasing n. Ungrouped inference concerns independently\n * sampled execution cells conditional on a fixed scenario roster.\n *\n * The shared paired decision rule selects the estimator and statistical test.\n * Dimension reports retain missing coverage so required safety checks cannot\n * pass through absent evidence. Thresholds use the judge's native score scale.\n */\n\nimport {\n decidePairedPromotion,\n type PairedDecisionMethod,\n type PairedDecisionStatistic,\n type PairedMcNemarEvidence,\n type PairedPromotionDecision,\n} from '../../paired-promotion-decision'\nimport {\n DECISION_PAIRED_DELTA_STATISTIC,\n type PairedBootstrapResult,\n pairedBootstrap,\n} from '../../statistics'\nimport type { JudgeScore } from '../types'\n\n/** Tie fraction at/above which a gate annotates its verdict with the tie share.\n * Tie-domination of the median bites structurally at >= 0.5 (the median is then\n * 0 by construction); 0.4 is a softer warn threshold that flags a run APPROACHING\n * that regime, so an operator sees it before the median goes fully blind. */\nexport const TIE_WARN_FRACTION = 0.4\n\nexport interface PairedHoldout {\n /** Baseline scalar per paired cell (same order as `after`/`cellIds`). */\n before: number[]\n /** Candidate scalar per paired cell. */\n after: number[]\n /** The full cellIds (`scenario:rep`) that paired, in order. */\n cellIds: string[]\n}\n\n/** Campaign cell IDs append a numeric repetition after the scenario's full ID. */\nexport function scenarioIdFromCellId(cellId: string): string {\n const separator = cellId.lastIndexOf(':')\n const repetition = cellId.slice(separator + 1)\n if (\n separator < 1 ||\n cellId.trim() !== cellId ||\n !/^(0|[1-9]\\d*)$/.test(repetition) ||\n !Number.isSafeInteger(Number(repetition))\n ) {\n throw new Error(`pairHoldout: malformed cellId '${cellId}'; expected scenarioId:rep`)\n }\n return cellId.slice(0, separator)\n}\n\n/** Preserve cell pairing before taking equal-weight independent-unit means. */\nexport function aggregatePairedHoldout(\n paired: PairedHoldout,\n independentUnitByScenarioId?: ReadonlyMap<string, string>,\n): { before: number[]; after: number[]; unitIds: string[] } {\n if (\n paired.before.length !== paired.after.length ||\n paired.before.length !== paired.cellIds.length\n ) {\n throw new Error('aggregatePairedHoldout: scores and cellIds must have the same length')\n }\n if (new Set(paired.cellIds).size !== paired.cellIds.length) {\n throw new Error('aggregatePairedHoldout: duplicate cellIds cannot count as new observations')\n }\n if (\n paired.before.some((value) => !Number.isFinite(value)) ||\n paired.after.some((value) => !Number.isFinite(value))\n ) {\n throw new Error('aggregatePairedHoldout: paired scores must be finite')\n }\n if (independentUnitByScenarioId === undefined) {\n return { before: [...paired.before], after: [...paired.after], unitIds: [...paired.cellIds] }\n }\n const scenarioIds = paired.cellIds.map(scenarioIdFromCellId)\n const groups = new Map<string, { before: number; after: number; n: number }>()\n for (let i = 0; i < paired.cellIds.length; i++) {\n const scenarioId = scenarioIds[i]!\n const unitId = independentUnitByScenarioId.get(scenarioId)\n if (typeof unitId !== 'string' || unitId.length === 0 || unitId.trim() !== unitId) {\n throw new Error(\n `aggregatePairedHoldout: missing independent unit for scenario '${scenarioId}'`,\n )\n }\n const group = groups.get(unitId) ?? { before: 0, after: 0, n: 0 }\n group.before += paired.before[i]!\n group.after += paired.after[i]!\n group.n += 1\n groups.set(unitId, group)\n }\n const unitIds = [...groups.keys()].sort()\n return {\n before: unitIds.map((id) => groups.get(id)!.before / groups.get(id)!.n),\n after: unitIds.map((id) => groups.get(id)!.after / groups.get(id)!.n),\n unitIds,\n }\n}\n\n/**\n * Pair candidate vs baseline holdout observations by FULL cellId. `select`\n * pulls the scalar from a cell's judge reports (composite, or a named\n * dimension); a cell contributes the mean of `select` across its judges. Cells\n * whose scenario is not in `scenarioIds`, or where `select` is undefined for\n * every judge on both sides, are skipped. The selected judge IDs must agree\n * within each pair. Throws when the two maps disagree on holdout cell IDs — a\n * load-bearing invariant: the baseline + winner holdout campaigns run the same\n * scenarios with the same seed base, so their cellIds MUST align; a mismatch\n * means a silent pairing bug, not a soft fallback.\n */\nexport function pairHoldout(\n candidate: Map<string, Record<string, JudgeScore>>,\n baseline: Map<string, Record<string, JudgeScore>>,\n scenarioIds: Set<string>,\n select: (s: JudgeScore) => number | undefined,\n): PairedHoldout {\n const cellValues = (\n byCell: Map<string, Record<string, JudgeScore>>,\n cellId: string,\n ): Map<string, number> => {\n const scores = byCell.get(cellId)\n const values = new Map<string, number>()\n if (!scores) return values\n for (const [judgeId, s] of Object.entries(scores)) {\n if (s.failed === true) {\n throw new Error(`pairHoldout: cell '${cellId}' contains a failed judge score`)\n }\n const v = select(s)\n if (typeof v === 'number' && !Number.isFinite(v)) {\n throw new Error(`pairHoldout: cell '${cellId}' contains a non-finite selected score`)\n }\n if (typeof v === 'number') values.set(judgeId, v)\n }\n return values\n }\n\n const inScope = (cellId: string) => scenarioIds.has(scenarioIdFromCellId(cellId))\n const candCells = [...candidate.keys()].filter(inScope).sort()\n const baseCells = [...baseline.keys()].filter(inScope).sort()\n // Alignment invariant — the holdout campaigns share scenarios + seed, so the\n // cell sets must be identical. Differ ⇒ a real pairing bug; fail loud.\n if (candCells.length !== baseCells.length || candCells.some((c, i) => c !== baseCells[i])) {\n throw new Error(\n `pairHoldout: candidate/baseline holdout cells do not align — ` +\n `candidate=[${candCells.join(',')}] baseline=[${baseCells.join(',')}]. ` +\n `Both holdout campaigns must run the same scenarios with the same seed base.`,\n )\n }\n\n const before: number[] = []\n const after: number[] = []\n const cellIds: string[] = []\n for (const cellId of candCells) {\n const b = cellValues(baseline, cellId)\n const a = cellValues(candidate, cellId)\n // A scalar absent on both sides means that dimension was not scored. A\n // one-sided absence is asymmetric evidence loss, never a row to discard.\n if (b.size === 0 && a.size === 0) continue\n if (b.size === 0 || a.size === 0) {\n throw new Error(`pairHoldout: cell '${cellId}' has a selected score on only one arm`)\n }\n if (b.size !== a.size || [...b.keys()].some((id) => !a.has(id))) {\n throw new Error(`pairHoldout: cell '${cellId}' selected judge IDs do not align`)\n }\n const judgeIds = [...b.keys()].sort()\n before.push(meanSelectedScores(judgeIds.map((id) => b.get(id)!)))\n after.push(meanSelectedScores(judgeIds.map((id) => a.get(id)!)))\n cellIds.push(cellId)\n }\n return { before, after, cellIds }\n}\n\nfunction meanSelectedScores(values: number[]): number {\n const first = values[0]!\n // Summing identical fractional scores can round their mean outside the\n // declared binary support. An agreeing judge set preserves its exact value.\n return values.every((value) => value === first)\n ? first\n : values.reduce((sum, value) => sum + value, 0) / values.length\n}\n\nexport interface HeldoutSignificance {\n paired: PairedHoldout\n /**\n * The paired bootstrap on the requested statistic (MEAN by default — see the\n * tie note on `heldoutSignificance`).\n *\n * DIAGNOSTIC, not necessarily the interval the verdict keyed on. On a\n * two-point (pass/fail) outcome the decision routes to Tango's score interval\n * instead, because a percentile bootstrap of the mean over a three-atom\n * lattice is not a valid interval at a nonzero margin. Read\n * `decision.low`/`decision.high` for the interval that actually decided, and\n * `decisionStatistic` for which one it is.\n */\n bootstrap: PairedBootstrapResult\n /** The MEDIAN paired-delta bootstrap, reported as a diagnostic. When many\n * scenarios are tied (both sides solve them), the median is pinned near 0\n * regardless of the mean lift — comparing the two exposes tie-domination. */\n medianBootstrap: PairedBootstrapResult\n /**\n * The full promotion decision: which estimator the outcome's shape admits,\n * the interval it produced, McNemar's exact veto on the two-point path, and\n * whether the interval was zero-width (no evidence in either direction). The\n * single source of `significant`.\n */\n decision: PairedPromotionDecision\n /** Which paired estimator the verdict was decided on. */\n decisionStatistic: PairedDecisionStatistic\n /** McNemar's exact evidence on the two-point path; null otherwise. */\n mcnemar: PairedMcNemarEvidence | null\n /** Fraction of paired observations that are exact ties (|delta| < 1e-9). A\n * high tie fraction is WHY a median-based gate would have missed a real lift;\n * it is the observability the tie fix adds. */\n tieFraction: number\n /** Number of paired observation units, after configured aggregation. */\n n: number\n /** Original matched execution cells, before aggregation. */\n pairedCellN: number\n observationUnit: 'registered' | 'cell'\n /** Registered unit IDs, or cell IDs on the ungrouped path. */\n unitIds: string[]\n /** Effective minimum for the requested target and chosen estimator. */\n minimumRequired: number\n /** Statistical method that carried the decision. */\n decisionMethod: PairedDecisionMethod\n /** Exact one-sided p-value on the small-sample path; otherwise null. */\n pValue: number | null\n /** True iff n >= minimumRequired, the DECIDING interval has nonzero width,\n * its lower bound clears the threshold, and McNemar's exact test does not\n * veto at a non-negative threshold. */\n significant: boolean\n /** Set when n < minimumRequired — too little evidence to claim significance. */\n fewRuns: boolean\n}\n\nexport interface HeldoutSignificanceOptions {\n deltaThreshold?: number\n minProductiveRuns?: number\n confidence?: number\n resamples?: number\n /** Fixed by default for a deterministic, reproducible gate verdict. */\n seed?: number\n statistic?: 'mean' | 'median'\n /** Group full cell pairs into equal-weight independent units before inference. */\n independentUnitByScenarioId?: ReadonlyMap<string, string>\n}\n\n/**\n * Significance of the held-out composite lift: ship only when the lower bound\n * of the interval the outcome's shape ADMITS exceeds `deltaThreshold` (default\n * 0 ⇒ \"confidently positive\"). Interpret `deltaThreshold` in the judge's native\n * scale.\n *\n * The decision is delegated whole to {@link decidePairedPromotion}, the one\n * copy of the rule (`src/paired-promotion-decision.ts`), which `HeldOutGate`\n * also calls. That module's header carries the measurements; the short version\n * is three guards a bare `bootstrap.low > threshold` does not have:\n *\n * - a two-point (pass/fail) outcome decides on Tango's SCORE interval, the\n * only paired-binary construction that stays valid at a nonzero margin;\n * - McNemar's exact test VETOES at any non-negative threshold;\n * - a ZERO-WIDTH interval is refused rather than promoted, in either\n * direction — [0,0] clears every negative threshold and [g,g] clears every\n * threshold below g, and both are an absence of evidence, not a result.\n *\n * Measured on this function before those guards landed, at a nominal 5 %:\n * 14.60 % false promotion at n = 40 on a paired-binary noninferiority boundary,\n * and 88.50 % at n = 6 under a bounded asymmetric null whose true mean paired\n * delta is exactly 0.\n *\n * Continuous mean targets require bootstrap eligibility. Explicit median\n * targets can use the exact sign test at its confidence-dependent minimum.\n */\nexport function heldoutSignificance(\n paired: PairedHoldout,\n opts: HeldoutSignificanceOptions = {},\n): HeldoutSignificance {\n const deltaThreshold = opts.deltaThreshold ?? 0\n const confidence = opts.confidence ?? 0.95\n const resamples = opts.resamples ?? 2000\n const seed = opts.seed ?? 1337\n // DEFAULT to the MEAN paired delta, not the median. The median is destroyed by\n // TIES: whenever both baseline and candidate solve a holdout scenario (a common\n // case once the agent is decent — and INCREASINGLY common as you add holdout\n // scenarios for statistical power), that scenario contributes delta 0. When\n // >=50% of paired cells are ties the median is pinned at 0 regardless of a large,\n // consistent lift on the rest, and the gate holds a genuinely better candidate.\n // (Measured live, supervisor-lab run 6: 40 cells, 20 ties, MEAN +0.177, MEDIAN 0\n // → false hold. Doubling the holdout for \"power\" made it WORSE by adding ties.)\n // The mean equals the reported aggregate lift, ties correctly contribute 0\n // without dominating, and it is the textbook paired-comparison estimator; the\n // median is kept as a reported diagnostic. Callers wanting outlier-robustness at\n // the cost of tie-blindness can still pass `statistic: 'median'`.\n const statistic = opts.statistic ?? 'mean'\n const observations = aggregatePairedHoldout(paired, opts.independentUnitByScenarioId)\n const decision = decidePairedPromotion(observations.before, observations.after, {\n confidence,\n resamples,\n statistic,\n seed,\n threshold: deltaThreshold,\n minPairs: opts.minProductiveRuns,\n })\n // `decision.bootstrap` is null exactly when the score interval decided, so\n // the requested-statistic bootstrap is computed here for the diagnostic\n // field. Same two bootstraps as before on every path.\n const bootstrap =\n decision.bootstrap ??\n pairedBootstrap(observations.before, observations.after, {\n confidence,\n resamples,\n statistic,\n seed,\n })\n const medianBootstrap =\n statistic === 'median'\n ? bootstrap\n : pairedBootstrap(observations.before, observations.after, {\n confidence,\n resamples,\n statistic: 'median',\n seed,\n })\n const n = observations.before.length\n let ties = 0\n for (let i = 0; i < n; i += 1) {\n const after = observations.after[i]!\n const before = observations.before[i]!\n if (Math.abs(after - before) < 1e-9) ties += 1\n }\n const tieFraction = n === 0 ? 0 : ties / n\n return {\n paired,\n bootstrap,\n medianBootstrap,\n decision,\n decisionStatistic: decision.statistic,\n mcnemar: decision.mcnemar,\n tieFraction,\n n,\n pairedCellN: paired.cellIds.length,\n observationUnit: opts.independentUnitByScenarioId === undefined ? 'cell' : 'registered',\n unitIds: observations.unitIds,\n minimumRequired: decision.minimumPairs,\n decisionMethod: decision.method,\n pValue: decision.pValue,\n significant: decision.promote,\n fewRuns: !decision.sufficient,\n }\n}\n\nexport interface DimensionRegression {\n dimension: string\n /** Paired bootstrap on (candidate − baseline). DIAGNOSTIC on a pass/fail\n * dimension, where `ci` carries the interval that decided instead. */\n bootstrap: PairedBootstrapResult\n /** Which paired statistic `bootstrap.low` is the lower bound of. `'mean'`\n * unless the caller asked for the median. `bootstrap.median` still carries\n * the median point estimate either way. */\n bootstrapStatistic: 'median' | 'mean'\n /** The interval `regressed` was decided on, in the dimension's native units. */\n ci: { low: number; high: number }\n /** Which estimator produced `ci`. */\n decisionStatistic: PairedDecisionStatistic\n /** McNemar's exact evidence on a pass/fail dimension; null otherwise. */\n mcnemar: PairedMcNemarEvidence | null\n /** `ci` has zero width — no evidence in either direction. */\n indeterminate: boolean\n /** The bootstrap lower bound is below negative tolerance, or the shared\n * paired test supports a drop exceeding tolerance. Missing coverage and\n * insufficient observations are reported separately. */\n regressed: boolean\n tolerance: number\n n: number\n pairedCellN: number\n observationUnit: 'registered' | 'cell'\n /** Statistical minimum for the configured independent observation unit. */\n minimumRequired: number\n fewRuns: boolean\n /** Both arms lack this dimension on these otherwise matched execution cells. */\n missingCellIds: string[]\n /** Configured scenarios with no paired dimension measurement at all. */\n missingScenarioIds: string[]\n}\n\n/** Detect the native scale of a set of scores: 0-100 when any magnitude clears\n * 1.5, else [0,1]. Used to auto-scale the regression tolerance so a default\n * expressed for [0,1] is not silently a no-op on a 0-100 dimension. */\nexport function detectScale(values: number[]): 1 | 100 {\n return values.some((v) => Math.abs(v) > 1.5) ? 100 : 1\n}\n\n/**\n * Report required-dimension evidence after full pairing and optional unit means.\n * A bootstrap floor breach or a shared paired test supporting a drop marks\n * regression. These two criteria are distinct; `ci` records the shared\n * estimator and `bootstrap` records the floor interval. Missing observations\n * and insufficient n remain explicit for the caller's evidence policy.\n * The default tolerance is 0.05 on [0,1] and 5 on a detected 0-100 scale.\n */\nexport function dimensionRegressions(\n candidate: Map<string, Record<string, JudgeScore>>,\n baseline: Map<string, Record<string, JudgeScore>>,\n scenarioIds: Set<string>,\n criticalDimensions: string[],\n opts: {\n tolerance?: number\n confidence?: number\n resamples?: number\n seed?: number\n /** Paired statistic the CI is computed on. Default `'mean'` — see\n * {@link DECISION_PAIRED_DELTA_STATISTIC} for why the median is not. */\n statistic?: 'mean' | 'median'\n independentUnitByScenarioId?: ReadonlyMap<string, string>\n minProductiveRuns?: number\n } = {},\n): DimensionRegression[] {\n const out: DimensionRegression[] = []\n const expectedCellIds = [...baseline.keys()]\n .filter((cellId) => scenarioIds.has(scenarioIdFromCellId(cellId)))\n .sort()\n for (const dim of criticalDimensions) {\n const paired = pairHoldout(candidate, baseline, scenarioIds, (s) => s.dimensions[dim])\n if (paired.before.length === 0) continue // dimension not scored on this judge\n const observations = aggregatePairedHoldout(paired, opts.independentUnitByScenarioId)\n const measuredCells = new Set(paired.cellIds)\n const measuredScenarios = new Set(paired.cellIds.map(scenarioIdFromCellId))\n const tolerance = opts.tolerance ?? 0.05 * detectScale([...paired.before, ...paired.after])\n const bootstrapStatistic = opts.statistic ?? DECISION_PAIRED_DELTA_STATISTIC\n const shared = {\n confidence: opts.confidence ?? 0.95,\n resamples: opts.resamples ?? 2000,\n statistic: bootstrapStatistic,\n seed: opts.seed ?? 1337,\n minPairs: opts.minProductiveRuns,\n }\n const guard = decidePairedPromotion(observations.before, observations.after, shared)\n const regression = decidePairedPromotion(observations.after, observations.before, {\n ...shared,\n threshold: tolerance,\n })\n const bootstrap =\n guard.bootstrap ?? pairedBootstrap(observations.before, observations.after, shared)\n out.push({\n dimension: dim,\n bootstrap,\n bootstrapStatistic,\n ci: { low: guard.low, high: guard.high },\n decisionStatistic: guard.statistic,\n mcnemar: guard.mcnemar,\n indeterminate: guard.indeterminate,\n // Fires on EITHER burden of proof — see `floorBreached` in\n // `promotion-policy.ts` for the same rule and the reasoning. The CI arm\n // is the contract this interface and `default-production-gate`'s reason\n // string both state (\"CI.low < -tolerance\"), read off the DECIDING\n // interval; the exact-test arm adds the small-sample path where the\n // bootstrap interval is descriptive only.\n // The credible-worst-case arm stays on the BOOTSTRAP — see the same\n // decision in `floorBreached` (`promotion-policy.ts`). On a pass/fail\n // dimension the score interval is ±z²/(n+z²) even when every pair is\n // concordant, so reading the floor off it would flag an unchanged safety\n // dimension as regressed at any realistic n. The PROVEN-drop arm does use\n // the shared rule, which is what closes the fail-open hole that mattered:\n // a genuine pass/fail regression is now judged on an interval valid at\n // the nonzero `tolerance`, instead of a bootstrap that is pinned wherever\n // ties dominate.\n regressed: bootstrap.low < -tolerance || regression.promote,\n tolerance,\n n: observations.before.length,\n pairedCellN: paired.cellIds.length,\n observationUnit: opts.independentUnitByScenarioId === undefined ? 'cell' : 'registered',\n minimumRequired: guard.minimumPairs,\n fewRuns: !guard.sufficient,\n missingCellIds: expectedCellIds.filter((cellId) => !measuredCells.has(cellId)),\n missingScenarioIds: [...scenarioIds].filter((id) => !measuredScenarios.has(id)).sort(),\n })\n }\n return out\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AA6BA,MAAa,oBAAoB;;AAYjC,SAAgB,qBAAqB,QAAwB;CAC3D,MAAM,YAAY,OAAO,YAAY,GAAG;CACxC,MAAM,aAAa,OAAO,MAAM,YAAY,CAAC;CAC7C,IACE,YAAY,KACZ,OAAO,KAAK,MAAM,UAClB,CAAC,iBAAiB,KAAK,UAAU,KACjC,CAAC,OAAO,cAAc,OAAO,UAAU,CAAC,GAExC,MAAM,IAAI,MAAM,kCAAkC,OAAO,2BAA2B;CAEtF,OAAO,OAAO,MAAM,GAAG,SAAS;AAClC;;AAGA,SAAgB,uBACd,QACA,6BAC0D;CAC1D,IACE,OAAO,OAAO,WAAW,OAAO,MAAM,UACtC,OAAO,OAAO,WAAW,OAAO,QAAQ,QAExC,MAAM,IAAI,MAAM,sEAAsE;CAExF,IAAI,IAAI,IAAI,OAAO,OAAO,CAAC,CAAC,SAAS,OAAO,QAAQ,QAClD,MAAM,IAAI,MAAM,4EAA4E;CAE9F,IACE,OAAO,OAAO,MAAM,UAAU,CAAC,OAAO,SAAS,KAAK,CAAC,KACrD,OAAO,MAAM,MAAM,UAAU,CAAC,OAAO,SAAS,KAAK,CAAC,GAEpD,MAAM,IAAI,MAAM,sDAAsD;CAExE,IAAI,gCAAgC,KAAA,GAClC,OAAO;EAAE,QAAQ,CAAC,GAAG,OAAO,MAAM;EAAG,OAAO,CAAC,GAAG,OAAO,KAAK;EAAG,SAAS,CAAC,GAAG,OAAO,OAAO;CAAE;CAE9F,MAAM,cAAc,OAAO,QAAQ,IAAI,oBAAoB;CAC3D,MAAM,yBAAS,IAAI,IAA0D;CAC7E,KAAK,IAAI,IAAI,GAAG,IAAI,OAAO,QAAQ,QAAQ,KAAK;EAC9C,MAAM,aAAa,YAAY;EAC/B,MAAM,SAAS,4BAA4B,IAAI,UAAU;EACzD,IAAI,OAAO,WAAW,YAAY,OAAO,WAAW,KAAK,OAAO,KAAK,MAAM,QACzE,MAAM,IAAI,MACR,kEAAkE,WAAW,EAC/E;EAEF,MAAM,QAAQ,OAAO,IAAI,MAAM,KAAK;GAAE,QAAQ;GAAG,OAAO;GAAG,GAAG;EAAE;EAChE,MAAM,UAAU,OAAO,OAAO;EAC9B,MAAM,SAAS,OAAO,MAAM;EAC5B,MAAM,KAAK;EACX,OAAO,IAAI,QAAQ,KAAK;CAC1B;CACA,MAAM,UAAU,CAAC,GAAG,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK;CACxC,OAAO;EACL,QAAQ,QAAQ,KAAK,OAAO,OAAO,IAAI,EAAE,CAAC,CAAE,SAAS,OAAO,IAAI,EAAE,CAAC,CAAE,CAAC;EACtE,OAAO,QAAQ,KAAK,OAAO,OAAO,IAAI,EAAE,CAAC,CAAE,QAAQ,OAAO,IAAI,EAAE,CAAC,CAAE,CAAC;EACpE;CACF;AACF;;;;;;;;;;;;AAaA,SAAgB,YACd,WACA,UACA,aACA,QACe;CACf,MAAM,cACJ,QACA,WACwB;EACxB,MAAM,SAAS,OAAO,IAAI,MAAM;EAChC,MAAM,yBAAS,IAAI,IAAoB;EACvC,IAAI,CAAC,QAAQ,OAAO;EACpB,KAAK,MAAM,CAAC,SAAS,MAAM,OAAO,QAAQ,MAAM,GAAG;GACjD,IAAI,EAAE,WAAW,MACf,MAAM,IAAI,MAAM,sBAAsB,OAAO,gCAAgC;GAE/E,MAAM,IAAI,OAAO,CAAC;GAClB,IAAI,OAAO,MAAM,YAAY,CAAC,OAAO,SAAS,CAAC,GAC7C,MAAM,IAAI,MAAM,sBAAsB,OAAO,uCAAuC;GAEtF,IAAI,OAAO,MAAM,UAAU,OAAO,IAAI,SAAS,CAAC;EAClD;EACA,OAAO;CACT;CAEA,MAAM,WAAW,WAAmB,YAAY,IAAI,qBAAqB,MAAM,CAAC;CAChF,MAAM,YAAY,CAAC,GAAG,UAAU,KAAK,CAAC,CAAC,CAAC,OAAO,OAAO,CAAC,CAAC,KAAK;CAC7D,MAAM,YAAY,CAAC,GAAG,SAAS,KAAK,CAAC,CAAC,CAAC,OAAO,OAAO,CAAC,CAAC,KAAK;CAG5D,IAAI,UAAU,WAAW,UAAU,UAAU,UAAU,MAAM,GAAG,MAAM,MAAM,UAAU,EAAE,GACtF,MAAM,IAAI,MACR,2EACgB,UAAU,KAAK,GAAG,EAAE,cAAc,UAAU,KAAK,GAAG,EAAE,+EAExE;CAGF,MAAM,SAAmB,CAAC;CAC1B,MAAM,QAAkB,CAAC;CACzB,MAAM,UAAoB,CAAC;CAC3B,KAAK,MAAM,UAAU,WAAW;EAC9B,MAAM,IAAI,WAAW,UAAU,MAAM;EACrC,MAAM,IAAI,WAAW,WAAW,MAAM;EAGtC,IAAI,EAAE,SAAS,KAAK,EAAE,SAAS,GAAG;EAClC,IAAI,EAAE,SAAS,KAAK,EAAE,SAAS,GAC7B,MAAM,IAAI,MAAM,sBAAsB,OAAO,uCAAuC;EAEtF,IAAI,EAAE,SAAS,EAAE,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,EAAE,IAAI,EAAE,CAAC,GAC5D,MAAM,IAAI,MAAM,sBAAsB,OAAO,kCAAkC;EAEjF,MAAM,WAAW,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK;EACpC,OAAO,KAAK,mBAAmB,SAAS,KAAK,OAAO,EAAE,IAAI,EAAE,CAAE,CAAC,CAAC;EAChE,MAAM,KAAK,mBAAmB,SAAS,KAAK,OAAO,EAAE,IAAI,EAAE,CAAE,CAAC,CAAC;EAC/D,QAAQ,KAAK,MAAM;CACrB;CACA,OAAO;EAAE;EAAQ;EAAO;CAAQ;AAClC;AAEA,SAAS,mBAAmB,QAA0B;CACpD,MAAM,QAAQ,OAAO;CAGrB,OAAO,OAAO,OAAO,UAAU,UAAU,KAAK,IAC1C,QACA,OAAO,QAAQ,KAAK,UAAU,MAAM,OAAO,CAAC,IAAI,OAAO;AAC7D;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8FA,SAAgB,oBACd,QACA,OAAmC,CAAC,GACf;CACrB,MAAM,iBAAiB,KAAK,kBAAkB;CAC9C,MAAM,aAAa,KAAK,cAAc;CACtC,MAAM,YAAY,KAAK,aAAa;CACpC,MAAM,OAAO,KAAK,QAAQ;CAa1B,MAAM,YAAY,KAAK,aAAa;CACpC,MAAM,eAAe,uBAAuB,QAAQ,KAAK,2BAA2B;CACpF,MAAM,WAAW,sBAAsB,aAAa,QAAQ,aAAa,OAAO;EAC9E;EACA;EACA;EACA;EACA,WAAW;EACX,UAAU,KAAK;CACjB,CAAC;CAID,MAAM,YACJ,SAAS,aACT,gBAAgB,aAAa,QAAQ,aAAa,OAAO;EACvD;EACA;EACA;EACA;CACF,CAAC;CACH,MAAM,kBACJ,cAAc,WACV,YACA,gBAAgB,aAAa,QAAQ,aAAa,OAAO;EACvD;EACA;EACA,WAAW;EACX;CACF,CAAC;CACP,MAAM,IAAI,aAAa,OAAO;CAC9B,IAAI,OAAO;CACX,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK,GAAG;EAC7B,MAAM,QAAQ,aAAa,MAAM;EACjC,MAAM,SAAS,aAAa,OAAO;EACnC,IAAI,KAAK,IAAI,QAAQ,MAAM,IAAI,MAAM,QAAQ;CAC/C;CACA,MAAM,cAAc,MAAM,IAAI,IAAI,OAAO;CACzC,OAAO;EACL;EACA;EACA;EACA;EACA,mBAAmB,SAAS;EAC5B,SAAS,SAAS;EAClB;EACA;EACA,aAAa,OAAO,QAAQ;EAC5B,iBAAiB,KAAK,gCAAgC,KAAA,IAAY,SAAS;EAC3E,SAAS,aAAa;EACtB,iBAAiB,SAAS;EAC1B,gBAAgB,SAAS;EACzB,QAAQ,SAAS;EACjB,aAAa,SAAS;EACtB,SAAS,CAAC,SAAS;CACrB;AACF;;;;AAuCA,SAAgB,YAAY,QAA2B;CACrD,OAAO,OAAO,MAAM,MAAM,KAAK,IAAI,CAAC,IAAI,GAAG,IAAI,MAAM;AACvD;;;;;;;;;AAUA,SAAgB,qBACd,WACA,UACA,aACA,oBACA,OAUI,CAAC,GACkB;CACvB,MAAM,MAA6B,CAAC;CACpC,MAAM,kBAAkB,CAAC,GAAG,SAAS,KAAK,CAAC,CAAC,CACzC,QAAQ,WAAW,YAAY,IAAI,qBAAqB,MAAM,CAAC,CAAC,CAAC,CACjE,KAAK;CACR,KAAK,MAAM,OAAO,oBAAoB;EACpC,MAAM,SAAS,YAAY,WAAW,UAAU,cAAc,MAAM,EAAE,WAAW,IAAI;EACrF,IAAI,OAAO,OAAO,WAAW,GAAG;EAChC,MAAM,eAAe,uBAAuB,QAAQ,KAAK,2BAA2B;EACpF,MAAM,gBAAgB,IAAI,IAAI,OAAO,OAAO;EAC5C,MAAM,oBAAoB,IAAI,IAAI,OAAO,QAAQ,IAAI,oBAAoB,CAAC;EAC1E,MAAM,YAAY,KAAK,aAAa,MAAO,YAAY,CAAC,GAAG,OAAO,QAAQ,GAAG,OAAO,KAAK,CAAC;EAC1F,MAAM,qBAAqB,KAAK,aAAA;EAChC,MAAM,SAAS;GACb,YAAY,KAAK,cAAc;GAC/B,WAAW,KAAK,aAAa;GAC7B,WAAW;GACX,MAAM,KAAK,QAAQ;GACnB,UAAU,KAAK;EACjB;EACA,MAAM,QAAQ,sBAAsB,aAAa,QAAQ,aAAa,OAAO,MAAM;EACnF,MAAM,aAAa,sBAAsB,aAAa,OAAO,aAAa,QAAQ;GAChF,GAAG;GACH,WAAW;EACb,CAAC;EACD,MAAM,YACJ,MAAM,aAAa,gBAAgB,aAAa,QAAQ,aAAa,OAAO,MAAM;EACpF,IAAI,KAAK;GACP,WAAW;GACX;GACA;GACA,IAAI;IAAE,KAAK,MAAM;IAAK,MAAM,MAAM;GAAK;GACvC,mBAAmB,MAAM;GACzB,SAAS,MAAM;GACf,eAAe,MAAM;GAgBrB,WAAW,UAAU,MAAM,CAAC,aAAa,WAAW;GACpD;GACA,GAAG,aAAa,OAAO;GACvB,aAAa,OAAO,QAAQ;GAC5B,iBAAiB,KAAK,gCAAgC,KAAA,IAAY,SAAS;GAC3E,iBAAiB,MAAM;GACvB,SAAS,CAAC,MAAM;GAChB,gBAAgB,gBAAgB,QAAQ,WAAW,CAAC,cAAc,IAAI,MAAM,CAAC;GAC7E,oBAAoB,CAAC,GAAG,WAAW,CAAC,CAAC,QAAQ,OAAO,CAAC,kBAAkB,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK;EACvF,CAAC;CACH;CACA,OAAO;AACT"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { a as RunRecord } from "./run-record-
|
|
1
|
+
import { a as RunRecord } from "./run-record-C1E24G2A.js";
|
|
2
2
|
import { a as PairedPromotionDecision, d as PairedBootstrapResult, i as PairedMcNemarEvidence, r as PairedDecisionStatistic, t as PairedDecisionMethod, u as PairedBootstrapOptions } from "./paired-promotion-decision-DPKz8ZnF.js";
|
|
3
3
|
import { r as LedgerHash } from "./canonical-CFpojCN5.js";
|
|
4
|
-
import { R as Scenario, a as CampaignResult, b as GenerationRecord, h as GateContext, j as MutableSurface, p as Gate, v as GateResult, w as JudgeScore } from "./types-
|
|
4
|
+
import { R as Scenario, a as CampaignResult, b as GenerationRecord, h as GateContext, j as MutableSurface, p as Gate, v as GateResult, w as JudgeScore } from "./types-Cq8mSbq8.js";
|
|
5
5
|
import { z } from "zod";
|
|
6
6
|
//#region src/statistics/paired-binary.d.ts
|
|
7
7
|
/** A binomial proportion estimate with a confidence interval. */
|
|
@@ -1310,4 +1310,4 @@ declare function dimensionRegressions(candidate: Map<string, Record<string, Judg
|
|
|
1310
1310
|
}): DimensionRegression[];
|
|
1311
1311
|
//#endregion
|
|
1312
1312
|
export { paretoSignificanceGate as $, PairArmsOptions as A, EProcess as At, PowerPreflight as B, isBinaryOutcomeVector as Bt, hashJson as C, FinalEvidenceError as Ct, ComparePairedArmsOptions as D, FinalEvidenceRecord as Dt, verifyManifest as E, FinalEvidenceOutcome as Et, PairedCorrectness as F, ExactRiskDifferenceResult as Ft, BuildEvidenceVectorOptions as G, pairedRiskDifferenceScore as Gt, powerPreflight as H, pairedBinaryScale as Ht, PairedMetricDelta as I, McNemarResult as It, ParetoSignificanceGateOptions as J, EvidenceVector as K, passAtK as Kt, comparePairedArms as L, ProportionInterval as Lt, PairRunRecordsResult as M, EProcessState as Mt, PairedArmRow as N, EProcessStep as Nt, MatchedPair as O, FinalEvidenceReservation as Ot, PairedArmsComparison as P, eProcess as Pt, paretoPolicy as Q, pairArms as R, RiskDifferenceResult as Rt, evaluateHypothesis as S, FinalEvidenceConflictError as St, signManifest as T, FinalEvidenceMeasurement as Tt, AxisEvidence as U, pairedRiskDifference as Ut, PowerPreflightOptions as V, mcnemar as Vt, AxisVerdict as W, pairedRiskDifferenceExact as Wt, PromotionPolicy as X, PromotionObjective as Y, buildEvidenceVector as Z, sequentialPairedGate as _, AttestationProvenance as _t, detectScale as a, createCampaignEvidenceReceipt as at, SignedManifest as b, attest as bt, pairHoldout as c, EvidenceAuthority as ct, SequentialDecision as d, EvidenceReceipt as dt, Objective as et, SequentialObservation as f, EvidenceReceiptVerification as ft, sequentialDecide as g, ATTESTATION_ALGORITHM as gt, SequentialStreamState as h, verifyEvidenceReceipt as ht, PairedHoldout as i, CampaignEvidenceContext as it, PairArmsResult as j, EProcessOptions as jt, MatchedRunRecordPair as k, openFinalEvidenceLedger as kt, SequentialDecideFn as l, EvidenceAuthorityKind as lt, SequentialPairedGateOptions as m, isIndependentEvidence as mt, HeldoutSignificance as n, dominates as nt, dimensionRegressions as o, CreateEvidenceReceiptInput as ot, SequentialPairedGate as p, createEvidenceReceipt as pt, ObjectiveSource as q, wilson as qt, HeldoutSignificanceOptions as r, paretoFrontier as rt, heldoutSignificance as s, EVIDENCE_RECEIPT_VERSION as st, DimensionRegression as t, ParetoResult as tt, SequentialDecideOptions as u, EvidenceBinding as ut, HypothesisManifest as v, AttestationVerification as vt, manifestContentDigest as w, FinalEvidenceLedger as wt, SignedManifestAlgo as x, verifyAttestation as xt, HypothesisResult as y, AttestedReport as yt, pairRunRecords as z, ScoreRiskDifferenceResult as zt };
|
|
1313
|
-
//# sourceMappingURL=statistical-heldout-
|
|
1313
|
+
//# sourceMappingURL=statistical-heldout-CYpGaoDH.d.ts.map
|
package/dist/{statistical-heldout-BeR4bOdg.d.ts.map → statistical-heldout-CYpGaoDH.d.ts.map}
RENAMED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"statistical-heldout-
|
|
1
|
+
{"version":3,"file":"statistical-heldout-CYpGaoDH.d.ts","names":["_$__evidence_receipt0.EvidenceReceipt"],"sources":["../src/statistics/paired-binary.ts","../src/statistics/sequential-eprocess.ts","../src/experiment/final-evidence.ts","../src/attestation.ts","../src/experiment/evidence-receipt.ts","../src/experiment/campaign-evidence.ts","../src/pareto.ts","../src/campaign/gates/promotion-policy.ts","../src/campaign/gates/power-preflight.ts","../src/paired-arms.ts","../src/pre-registration.ts","../src/campaign/gates/sequential.ts","../src/campaign/gates/statistical-heldout.ts"],"mappings":";;;;;;;UAgBiB;;EAEf;;EAEA;;EAEA;;;;;;;;;;;;iBAac,OAAO,mBAAmB,WAAW,sBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;iBAiDzD,sBAAsB,QAAQ;;UAU7B;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;;;;;;;;;iBAec,QACd,SAAS,6BACT,WAAW,8BACV;;UAmBc;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;;;;;;;;;;;;;iBAmBc,qBACd,SAAS,6BACT,WAAW,6BACX,sBACC;;;;UAkCc;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAgCc,0BACd,SAAS,6BACT,WAAW,6BACX,sBACC;;;UAyEc;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;;;;;;;;;;;;;;;;;;;;;;iBA8Dc,0BACd,SAAS,6BACT,WAAW,6BACX,sBACC;;;;;;;;;;;;;;;;;;;;iBAiFa,kBACd,QAAQ,mBACR,OAAO;;;;;;;;;;;iBA0BO,QAAQ,WAAW,WAAW;;;UChhB7B;;;EAGf;;;;EAIA;;;;EAIA;;;;;;;;EAQA,SAAS;;UAGM;;EAEf;;EAEA;;EAEA;;;;;;;;;;UAWe,sBAAsB;EACrC;EACA;EACA;;EAEA;;EAEA;;EAEA;;;EAGA;;UAGe;;;EAGf,OAAO,YAAY;EACnB,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBA8BK,SAAS,OAAM,kBAAuB;;;cCnFhD,mBAAiB,EAAA;;;;;;GAQZ,EAAA,KAAA;;KAGC,2BAA2B,EAAE,aAAa;cAEhD,mBAAiB,EAAA;;;GAKZ,EAAA,KAAA;KAEC,2BAA2B,EAAE,aAAa;UAgCrC;EACf,aAAa;EACb,iBAAiB;EACjB;IAAY,aAAa;IAA0B,WAAW;;;KAGpD,qBAAqB;EAC3B;EAAiB,OAAO;;EACxB;EAAkB;IAAS;IAA8C;;;UAE9D;EACf,QACE,OAAO,2BACN,QAAQ;IAAuB,QAAQ;IAAqB;;EAC/D,OACE,mBACA,aAAa,2BACZ,QAAQ;IAAuB,QAAQ;IAAqB;;EAC/D,QAAQ,QAAQ,qBAAqB;;;cAI1B,2BAA2B;WAE3B;EADX,YACW,8CACT;;;cAQS,mCAAmC;EAC9C,YAAY;;;iBA+HE,wBAAwB;EAAW;IAAiB;;;;;;;;;;;;;;;;;;;;;;;cCvMvD;UAEI;;EAEf,eAAe;;EAEf;;;EAGA;;EAEA;;EAEA;;;EAGA;;UAGe;;EAEf;EACA,YAAY;EACZ,kBAAkB;;EAElB;;UAGe;EACf;;EAEA;;;;;;;;iBAiBc,OAAO,iBAAiB,YAAY,wBAAwB;;;;;;;;;;iBAoB5D,kBACd,iBACA,UAAU,iBACT;;;cC1EU;;cAGA;KAQD,gCAAgC;UAS3B;WACN,MAAM;;WAEN;;UAGM;WACN,sBAAsB;;WAEtB;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;WACA,WAAW;;WAEX;;WAEA;;UAGM;WACN,SAAS;WACT,aAAa;;UAGP;WACN;WACA;;UAGM,mCAAmC,KAAK;;;;;;;iBAQzC,sBACd,OAAO,4BACP,YAAY,wBACX;;;;;iBA4Ba,sBAAsB,SAAS,kBAAkB;;;;;;iBAkCjD,sBAAsB,SAAS;;;;KCjJnC,0BAA0B,KACpC;EAOI,YAAY,KAAK;;;iBAGP,8BAA8B,UAAU,UAAU,GAAG;EACnE,UAAU,eAAe,GAAG;EAC5B,SAAS;EACT,SAAS;IAAAA;;;;;;;;;;;;;;;;;;KCPC;UAEK,UAAU;;EAEzB;EACA,WAAW;EACX,QAAQ,WAAW;;UAGJ,aAAa;EAC5B,UAAU;EACV,WAAW;;EAEX,cAAc;IAAQ,WAAW;IAAG,WAAW;;;;iBAIjC,UAAU,GAAG,GAAG,GAAG,GAAG,GAAG,YAAY,UAAU;;;;;;iBAmB/C,eAAe,GAAG,YAAY,KAAK,YAAY,UAAU,OAAO,aAAa;;;;;KCxBjF;EAAoB;;EAAwB;EAAmB;;UAE1D;;EAEf;EACA,QAAQ;;;EAGR,WAAW;;;;EAIX;;;;EAIA;;;;;EAKA;;;;KAKU;UAEK;EACf;EACA,QAAQ;EACR,WAAW;;;;;;;;EAQX,WAAW;;;;;EAKX;;;EAGA;IAAM;IAAa;;;EAEnB,mBAAmB;;EAEnB,SAAS;;;EAGT;;EAEA;EACA;EACA,gBAAgB;EAChB;EACA;EACA,SAAS;;UAGM;;EAEf,MAAM;;;EAGN;;;EAGA;IAAQ;IAAmB;;;;;;KAMjB,mBAAmB,IAAI,mBAAmB;UAErC;;;;EAIf;;EAEA;;EAEA;;EAEA;;;EAGA;;;;;;;;iBASc,oBAAoB,WAAW,kBAAkB,UAC/D,KAAK,YAAY,WAAW,YAC5B,YAAY,sBACZ,OAAM,6BACL;;;;;;cAkIU,cAAc;UA+EV,sCAAsC;;EAErD,YAAY;;;EAGZ,SAAS;;EAET;;;;;;;iBAQc,uBAAuB,qBAAqB,kBAAkB,WAAW,UACvF,SAAS,gCACR,KAAK,WAAW;;;;;;;;;;;;;;;;;UCzVF;;EAEf;;;EAGA;;EAEA;;EAEA;;;;EAIA;;UAGe;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;;EAGA;;;EAGA;EACA;EACA;;EAEA;;EAEA;;;iBAYc,eAAe,MAAM,wBAAwB;;;;;UCzB5C;;;EAGf;;;;;EAKA;;EAEA;;EAEA;;EAEA,UAAU;;UAGK;;EAEf;;EAEA;;;UAIe;EACf;;;;EAIA;EACA,UAAU;EACV,WAAW;;UAGI;;EAEf,OAAO;;;EAGP,kBAAkB;;EAElB,mBAAmB;;;;;;;;;;;;;;;;;;;;iBAqBL,SAAS,eAAe,gBAAgB,MAAM,kBAAkB;;UA8G/D;;EAEf;;EAEA;;EAEA,SAAS;;EAET,gBAAgB;;;UAID;EACf;;EAEA;;EAEA;;EAEA;;EAEA;;;;EAIA,aAAa;;EAEb;IAAY;IAAW;;;UAGR,iCAAiC;;;;EAIhD;;EAEA,YAAY;;UAGG;EACf;EACA;EACA;;;EAGA,aAAa;EACb,cAAc;;;;;;;;;;;;;;;iBAgBA,kBACd,eAAe,gBACf,MAAM,2BACL;UAmEc;EACf;EACA;EACA,UAAU;EACV,WAAW;;UAGI;EACf,OAAO;EACP,kBAAkB;EAClB,mBAAmB;;;;;;;;;iBAeL,eACd,uBAAuB,aACvB,wBAAwB,cACvB;;;;;;;;;;;;;;;;UCrWc;EACf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;EACA;;;KAIU;UAEK,uBAAuB;;EAEtC;;EAEA,MAAM;;UAGS;EACf,UAAU;EACV;EACA;EACA;;;EAGA;;EAEA,kBAAkB;EAGlB;;;;;;;;;;;;;;;;;iBAkBoB,SAAS,GAAG,KAAK,IAAI;;;;;iBAQ3B,sBAAsB,UAAU;;;;;;iBAc1B,aAAa,GAAG,qBAAqB,QAAQ;;;;;iBAS7C,eAAe,GAAG,iBAAiB;;;;;iBAYnC,mBACpB,UAAU,gBACV;EAAY;EAAW;EAAgB;IACtC,QAAQ;;;KCrFC;UAEK;EACf,UAAU;;EAEV;;EAEA;;;EAGA;;UAGe;;;EAGf;;;;EAIA;;;EAGA;;EAEA;;;;EAIA;;;EAGA;;;;EAIA,8BAA8B;;;EAG9B,kBAAkB;;EAElB;;;;;;;EAOA,SAAS;;;;;;KAOC,wBAAwB;EAAkB,UAAU;;UAE/C,qBAAqB,qBAAqB,kBAAkB,WAAW,kBAC9E,KAAK,WAAW;;;;;;;;;;EAUxB,QAAQ,gBAAgB;;;EAGxB,SAAS;;;;;;;;;;;;;;;;;iBAuPK,qBAAqB,qBAAqB,kBAAkB,WAAW,UACrF,SAAS,8BACR,qBAAqB,WAAW;UA6GlB;;EAEf;;EAEA;;EAEA;;EAEA;;UAGe;GACd;IAAQ,SAAS;;IAAyB;IAAe;;;EAE1D,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;iBA0BK,iBAAiB,UAAS,0BAA+B;;;UC9dxD;;EAEf;;EAEA;;EAEA;;;;;;;;;;;;;iBA4Ec,YACd,WAAW,YAAY,eAAe,cACtC,UAAU,YAAY,eAAe,cACrC,aAAa,aACb,SAAS,GAAG,oCACX;UAkEc;EACf,QAAQ;;;;;;;;;;;;EAYR,WAAW;;;;EAIX,iBAAiB;;;;;;;EAOjB,UAAU;;EAEV,mBAAmB;;EAEnB,SAAS;;;;EAIT;;EAEA;;EAEA;EACA;;EAEA;;EAEA;;EAEA,gBAAgB;;EAEhB;;;;EAIA;;EAEA;;UAGe;EACf;EACA;EACA;EACA;;EAEA;EACA;;EAEA,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBA6BhB,oBACd,QAAQ,eACR,OAAM,6BACL;UA2Ec;EACf;;;EAGA,WAAW;;;;EAIX;;EAEA;IAAM;IAAa;;;EAEnB,mBAAmB;;EAEnB,SAAS;;EAET;;;;EAIA;EACA;EACA;EACA;EACA;;EAEA;EACA;;EAEA;;EAEA;;;;;iBAMc,YAAY;;;;;;;;;iBAYZ,qBACd,WAAW,YAAY,eAAe,cACtC,UAAU,YAAY,eAAe,cACrC,aAAa,aACb,8BACA;EACE;EACA;EACA;EACA;;;EAGA;EACA,8BAA8B;EAC9B;IAED"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { n as CaptureIntegrityError } from "./errors-
|
|
2
|
-
import { A as spanEpochMillis, C as TraceFileTooLargeError, D as extractOtlpAttributes, E as compareSpanTime, S as TraceFileMissingError, _ as SpanNotFoundError, b as TraceAnalysisValidationError, c as bindSpanSourceReader, d as compileSearchRegex, f as truncateForBudget, g as TRACE_ANALYSIS_LIMITS, k as projectOtlpFlatLine, m as DEFAULT_TRACE_ANALYST_BUDGETS, p as validateInteger, u as traceSpanSchema, v as TraceAnalysisLimitError, w as TraceNotFoundError, x as TraceFileMalformedError } from "./kind-factory-
|
|
1
|
+
import { n as CaptureIntegrityError } from "./errors-DVZiqS7K.js";
|
|
2
|
+
import { A as spanEpochMillis, C as TraceFileTooLargeError, D as extractOtlpAttributes, E as compareSpanTime, S as TraceFileMissingError, _ as SpanNotFoundError, b as TraceAnalysisValidationError, c as bindSpanSourceReader, d as compileSearchRegex, f as truncateForBudget, g as TRACE_ANALYSIS_LIMITS, k as projectOtlpFlatLine, m as DEFAULT_TRACE_ANALYST_BUDGETS, p as validateInteger, u as traceSpanSchema, v as TraceAnalysisLimitError, w as TraceNotFoundError, x as TraceFileMalformedError } from "./kind-factory-BobI5Jy3.js";
|
|
3
3
|
import { readFile, stat } from "node:fs/promises";
|
|
4
4
|
//#region src/trace-analyst/shared-abortable-task.ts
|
|
5
5
|
function createSharedAbortableTask(factory) {
|
|
@@ -41,33 +41,108 @@ function waitForSharedTask(task, signal) {
|
|
|
41
41
|
});
|
|
42
42
|
}
|
|
43
43
|
//#endregion
|
|
44
|
-
//#region src/trace-analyst/store-otlp.ts
|
|
44
|
+
//#region src/trace-analyst/store-otlp-error-clusters.ts
|
|
45
|
+
const ERROR_CLUSTER_EXEMPLARS = 5;
|
|
46
|
+
/** Collapse volatile tokens so semantically identical failures share a key:
|
|
47
|
+
* hex/uuid ids → <id>, numbers → #, quoted/abs paths → <path>, durations →
|
|
48
|
+
* <dur>, whitespace collapsed. Empty/absent messages fall back to the span
|
|
49
|
+
* name so a no-message error still forms a real cluster. */
|
|
50
|
+
function normalizeErrorSignature(message, spanName) {
|
|
51
|
+
const raw = (message ?? "").trim();
|
|
52
|
+
return (raw.length > 0 ? raw : `(${spanName || "error"} — no message)`).replace(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi, "<id>").replace(/\b[0-9a-f]{12,}\b/gi, "<id>").replace(/(?:\/[\w.\-@]+){2,}/g, "<path>").replace(/\b\d+(?:\.\d+)?(ms|s|m|h|kb|mb|gb)?\b/gi, (_m, u) => u ? `#${u}` : "#").replace(/\s+/g, " ").trim();
|
|
53
|
+
}
|
|
54
|
+
function bump(map, key) {
|
|
55
|
+
if (!key) return;
|
|
56
|
+
map.set(key, (map.get(key) ?? 0) + 1);
|
|
57
|
+
}
|
|
58
|
+
function topKey(map) {
|
|
59
|
+
let best = null;
|
|
60
|
+
let bestN = 0;
|
|
61
|
+
for (const [k, n] of map) if (n > bestN) {
|
|
62
|
+
best = k;
|
|
63
|
+
bestN = n;
|
|
64
|
+
}
|
|
65
|
+
return best;
|
|
66
|
+
}
|
|
67
|
+
function accumulateErrorCluster(clusters, traceId, span) {
|
|
68
|
+
const signature = normalizeErrorSignature(span.status_message, span.name);
|
|
69
|
+
let acc = clusters.get(signature);
|
|
70
|
+
if (!acc) {
|
|
71
|
+
acc = {
|
|
72
|
+
signature,
|
|
73
|
+
sample: span.status_message ?? span.name ?? "",
|
|
74
|
+
traceIds: /* @__PURE__ */ new Set(),
|
|
75
|
+
spanIds: [],
|
|
76
|
+
spanCount: 0,
|
|
77
|
+
spanNames: /* @__PURE__ */ new Map(),
|
|
78
|
+
toolNames: /* @__PURE__ */ new Map()
|
|
79
|
+
};
|
|
80
|
+
clusters.set(signature, acc);
|
|
81
|
+
}
|
|
82
|
+
acc.traceIds.add(traceId);
|
|
83
|
+
acc.spanCount += 1;
|
|
84
|
+
if (acc.spanIds.length < ERROR_CLUSTER_EXEMPLARS && !acc.spanIds.includes(span.span_id)) acc.spanIds.push(span.span_id);
|
|
85
|
+
bump(acc.spanNames, span.name);
|
|
86
|
+
bump(acc.toolNames, span.tool_name);
|
|
87
|
+
}
|
|
88
|
+
function finalizeErrorClusters(clusters, errorTraceCount) {
|
|
89
|
+
const out = [...clusters.values()].map((acc) => ({
|
|
90
|
+
signature: acc.signature,
|
|
91
|
+
status_message_sample: acc.sample,
|
|
92
|
+
span_name: topKey(acc.spanNames),
|
|
93
|
+
tool_name: topKey(acc.toolNames),
|
|
94
|
+
trace_count: acc.traceIds.size,
|
|
95
|
+
span_count: acc.spanCount,
|
|
96
|
+
prevalence: errorTraceCount > 0 ? acc.traceIds.size / errorTraceCount : 0,
|
|
97
|
+
exemplar_trace_ids: [...acc.traceIds].slice(0, ERROR_CLUSTER_EXEMPLARS),
|
|
98
|
+
exemplar_span_ids: acc.spanIds.slice(0, ERROR_CLUSTER_EXEMPLARS)
|
|
99
|
+
}));
|
|
100
|
+
out.sort((a, b) => b.trace_count - a.trace_count || b.span_count - a.span_count);
|
|
101
|
+
return out;
|
|
102
|
+
}
|
|
103
|
+
//#endregion
|
|
104
|
+
//#region src/trace-analyst/store-otlp-search-offset.ts
|
|
105
|
+
/** A `"` at `idx` is a real JSON delimiter only when the run of `\`
|
|
106
|
+
* immediately preceding it is even-length; an odd run means the quote
|
|
107
|
+
* is escaped (`\"`) and is part of a string value, not a boundary. */
|
|
108
|
+
function isUnescapedQuote(slice, idx) {
|
|
109
|
+
if (slice[idx] !== "\"") return false;
|
|
110
|
+
let backslashes = 0;
|
|
111
|
+
let b = idx - 1;
|
|
112
|
+
while (b >= 0 && slice[b] === "\\") {
|
|
113
|
+
backslashes += 1;
|
|
114
|
+
b -= 1;
|
|
115
|
+
}
|
|
116
|
+
return backslashes % 2 === 0;
|
|
117
|
+
}
|
|
118
|
+
/** Scan backwards from `from` (inclusive) for the nearest UNescaped `"`.
|
|
119
|
+
* Returns its index, or -1 when none is found. */
|
|
120
|
+
function prevUnescapedQuote(slice, from) {
|
|
121
|
+
for (let i = from; i >= 0; i -= 1) if (slice[i] === "\"" && isUnescapedQuote(slice, i)) return i;
|
|
122
|
+
return -1;
|
|
123
|
+
}
|
|
45
124
|
/**
|
|
46
|
-
*
|
|
47
|
-
*
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
* span shape down to `TraceAnalystSpan` lazily — full materialisation
|
|
53
|
-
* only happens for the spans the agent actually requests.
|
|
54
|
-
*
|
|
55
|
-
* Indexing. On first read the store builds an in-memory index keyed
|
|
56
|
-
* by `trace_id` carrying:
|
|
57
|
-
* - byte offsets + lengths for each span line (for surgical reads
|
|
58
|
-
* without re-parsing the whole file)
|
|
59
|
-
* - a `TraceAnalystTraceSummary` rollup
|
|
60
|
-
* - sets of services / agents / models / tools / has_errors
|
|
61
|
-
* - byte size of the trace's JSONL slab
|
|
62
|
-
*
|
|
63
|
-
* Memory bound. The index keeps span metadata only — names, kinds,
|
|
64
|
-
* offsets, status. Attribute payloads stay on disk until requested.
|
|
65
|
-
* For a 50MB JSONL with 50k spans, the index is ~5MB.
|
|
66
|
-
*
|
|
67
|
-
* Concurrency. The store builds the index once on first read and
|
|
68
|
-
* caches it. Subsequent reads reuse the index. The file is opened on
|
|
69
|
-
* each read; we never hold a long-lived FD.
|
|
125
|
+
* Best-effort: locate the JSON path for the substring at `offset` in
|
|
126
|
+
* a single span's JSONL slice. The slice is '...,"key":"value..."' — we
|
|
127
|
+
* walk back from `offset` to the value-opening quote, past the `:`, to
|
|
128
|
+
* the key's closing then opening quote, skipping `\"`-escaped quotes that
|
|
129
|
+
* live inside string values. Returns `null` when the offset doesn't fall
|
|
130
|
+
* inside a recognisable string field.
|
|
70
131
|
*/
|
|
132
|
+
function bestAttributePathForOffset(slice, offset) {
|
|
133
|
+
const valueQuote = prevUnescapedQuote(slice, Math.min(offset, slice.length - 1));
|
|
134
|
+
if (valueQuote < 1) return null;
|
|
135
|
+
let j = valueQuote - 1;
|
|
136
|
+
while (j >= 0 && slice[j] !== ":") j -= 1;
|
|
137
|
+
if (j < 1) return null;
|
|
138
|
+
const keyClose = prevUnescapedQuote(slice, j - 1);
|
|
139
|
+
if (keyClose < 1) return null;
|
|
140
|
+
const keyOpen = prevUnescapedQuote(slice, keyClose - 1);
|
|
141
|
+
if (keyOpen < 0) return null;
|
|
142
|
+
return slice.slice(keyOpen + 1, keyClose);
|
|
143
|
+
}
|
|
144
|
+
//#endregion
|
|
145
|
+
//#region src/trace-analyst/store-otlp-buffered.ts
|
|
71
146
|
/** Lines indexed between event-loop yields. Bounded synchronous work per
|
|
72
147
|
* tick keeps the index build from starving other tasks on large files
|
|
73
148
|
* while staying coarse enough that the yields are cheap. */
|
|
@@ -84,8 +159,6 @@ async function scanCheckpoint(signal, item) {
|
|
|
84
159
|
signal?.throwIfAborted();
|
|
85
160
|
}
|
|
86
161
|
}
|
|
87
|
-
/** Default ceiling for {@link OtlpFileTraceStoreOptions.maxFileBytes}. */
|
|
88
|
-
const DEFAULT_MAX_TRACE_FILE_BYTES = 268435456;
|
|
89
162
|
var BufferedTraceStore = class {
|
|
90
163
|
readSpanSource;
|
|
91
164
|
perAttributeViewBudget;
|
|
@@ -615,6 +688,36 @@ var BufferedTraceStore = class {
|
|
|
615
688
|
};
|
|
616
689
|
}
|
|
617
690
|
};
|
|
691
|
+
//#endregion
|
|
692
|
+
//#region src/trace-analyst/store-otlp.ts
|
|
693
|
+
/**
|
|
694
|
+
* `OtlpFileTraceStore` — read-only OTLP-JSONL trace store for the
|
|
695
|
+
* trace-analyst.
|
|
696
|
+
*
|
|
697
|
+
* Wire shape. Each line of the input file is one OTLP-shaped span. The
|
|
698
|
+
* store understands flattened OTLP JSONL plus the OpenInference vocab.
|
|
699
|
+
* We project upstream's full
|
|
700
|
+
* span shape down to `TraceAnalystSpan` lazily — full materialisation
|
|
701
|
+
* only happens for the spans the agent actually requests.
|
|
702
|
+
*
|
|
703
|
+
* Indexing. On first read the store builds an in-memory index keyed
|
|
704
|
+
* by `trace_id` carrying:
|
|
705
|
+
* - byte offsets + lengths for each span line (for surgical reads
|
|
706
|
+
* without re-parsing the whole file)
|
|
707
|
+
* - a `TraceAnalystTraceSummary` rollup
|
|
708
|
+
* - sets of services / agents / models / tools / has_errors
|
|
709
|
+
* - byte size of the trace's JSONL slab
|
|
710
|
+
*
|
|
711
|
+
* Memory bound. The index keeps span metadata only — names, kinds,
|
|
712
|
+
* offsets, status. Attribute payloads stay on disk until requested.
|
|
713
|
+
* For a 50MB JSONL with 50k spans, the index is ~5MB.
|
|
714
|
+
*
|
|
715
|
+
* Concurrency. The store builds the index once on first read and
|
|
716
|
+
* caches it. Subsequent reads reuse the index. The file is opened on
|
|
717
|
+
* each read; we never hold a long-lived FD.
|
|
718
|
+
*/
|
|
719
|
+
/** Default ceiling for {@link OtlpFileTraceStoreOptions.maxFileBytes}. */
|
|
720
|
+
const DEFAULT_MAX_TRACE_FILE_BYTES = 268435456;
|
|
618
721
|
var OtlpFileTraceStore = class extends BufferedTraceStore {
|
|
619
722
|
path;
|
|
620
723
|
maxFileBytes;
|
|
@@ -644,7 +747,7 @@ var OtlpFileTraceStore = class extends BufferedTraceStore {
|
|
|
644
747
|
};
|
|
645
748
|
var BufferTraceStore = class extends BufferedTraceStore {
|
|
646
749
|
source;
|
|
647
|
-
constructor(source, opts) {
|
|
750
|
+
constructor(source, opts = {}) {
|
|
648
751
|
super(opts);
|
|
649
752
|
this.source = source;
|
|
650
753
|
}
|
|
@@ -689,107 +792,7 @@ function otlpTextToTraceAnalysisStore(text, opts = {}) {
|
|
|
689
792
|
if (!text.trim()) throw new CaptureIntegrityError("OTLP trace text must not be empty");
|
|
690
793
|
return new BufferTraceStore(Buffer.from(text, "utf8"), opts);
|
|
691
794
|
}
|
|
692
|
-
function createOtlpBufferTraceStore(source, options = {}) {
|
|
693
|
-
return new BufferTraceStore(source, options);
|
|
694
|
-
}
|
|
695
|
-
/** A `"` at `idx` is a real JSON delimiter only when the run of `\`
|
|
696
|
-
* immediately preceding it is even-length; an odd run means the quote
|
|
697
|
-
* is escaped (`\"`) and is part of a string value, not a boundary. */
|
|
698
|
-
function isUnescapedQuote(slice, idx) {
|
|
699
|
-
if (slice[idx] !== "\"") return false;
|
|
700
|
-
let backslashes = 0;
|
|
701
|
-
let b = idx - 1;
|
|
702
|
-
while (b >= 0 && slice[b] === "\\") {
|
|
703
|
-
backslashes += 1;
|
|
704
|
-
b -= 1;
|
|
705
|
-
}
|
|
706
|
-
return backslashes % 2 === 0;
|
|
707
|
-
}
|
|
708
|
-
/** Scan backwards from `from` (inclusive) for the nearest UNescaped `"`.
|
|
709
|
-
* Returns its index, or -1 when none is found. */
|
|
710
|
-
function prevUnescapedQuote(slice, from) {
|
|
711
|
-
for (let i = from; i >= 0; i -= 1) if (slice[i] === "\"" && isUnescapedQuote(slice, i)) return i;
|
|
712
|
-
return -1;
|
|
713
|
-
}
|
|
714
|
-
/**
|
|
715
|
-
* Best-effort: locate the JSON path for the substring at `offset` in
|
|
716
|
-
* a single span's JSONL slice. The slice is '...,"key":"value..."' — we
|
|
717
|
-
* walk back from `offset` to the value-opening quote, past the `:`, to
|
|
718
|
-
* the key's closing then opening quote, skipping `\"`-escaped quotes that
|
|
719
|
-
* live inside string values. Returns `null` when the offset doesn't fall
|
|
720
|
-
* inside a recognisable string field.
|
|
721
|
-
*/
|
|
722
|
-
function bestAttributePathForOffset(slice, offset) {
|
|
723
|
-
const valueQuote = prevUnescapedQuote(slice, Math.min(offset, slice.length - 1));
|
|
724
|
-
if (valueQuote < 1) return null;
|
|
725
|
-
let j = valueQuote - 1;
|
|
726
|
-
while (j >= 0 && slice[j] !== ":") j -= 1;
|
|
727
|
-
if (j < 1) return null;
|
|
728
|
-
const keyClose = prevUnescapedQuote(slice, j - 1);
|
|
729
|
-
if (keyClose < 1) return null;
|
|
730
|
-
const keyOpen = prevUnescapedQuote(slice, keyClose - 1);
|
|
731
|
-
if (keyOpen < 0) return null;
|
|
732
|
-
return slice.slice(keyOpen + 1, keyClose);
|
|
733
|
-
}
|
|
734
|
-
const ERROR_CLUSTER_EXEMPLARS = 5;
|
|
735
|
-
/** Collapse volatile tokens so semantically identical failures share a key:
|
|
736
|
-
* hex/uuid ids → <id>, numbers → #, quoted/abs paths → <path>, durations →
|
|
737
|
-
* <dur>, whitespace collapsed. Empty/absent messages fall back to the span
|
|
738
|
-
* name so a no-message error still forms a real cluster. */
|
|
739
|
-
function normalizeErrorSignature(message, spanName) {
|
|
740
|
-
const raw = (message ?? "").trim();
|
|
741
|
-
return (raw.length > 0 ? raw : `(${spanName || "error"} — no message)`).replace(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi, "<id>").replace(/\b[0-9a-f]{12,}\b/gi, "<id>").replace(/(?:\/[\w.\-@]+){2,}/g, "<path>").replace(/\b\d+(?:\.\d+)?(ms|s|m|h|kb|mb|gb)?\b/gi, (_m, u) => u ? `#${u}` : "#").replace(/\s+/g, " ").trim();
|
|
742
|
-
}
|
|
743
|
-
function bump(map, key) {
|
|
744
|
-
if (!key) return;
|
|
745
|
-
map.set(key, (map.get(key) ?? 0) + 1);
|
|
746
|
-
}
|
|
747
|
-
function topKey(map) {
|
|
748
|
-
let best = null;
|
|
749
|
-
let bestN = 0;
|
|
750
|
-
for (const [k, n] of map) if (n > bestN) {
|
|
751
|
-
best = k;
|
|
752
|
-
bestN = n;
|
|
753
|
-
}
|
|
754
|
-
return best;
|
|
755
|
-
}
|
|
756
|
-
function accumulateErrorCluster(clusters, traceId, span) {
|
|
757
|
-
const signature = normalizeErrorSignature(span.status_message, span.name);
|
|
758
|
-
let acc = clusters.get(signature);
|
|
759
|
-
if (!acc) {
|
|
760
|
-
acc = {
|
|
761
|
-
signature,
|
|
762
|
-
sample: span.status_message ?? span.name ?? "",
|
|
763
|
-
traceIds: /* @__PURE__ */ new Set(),
|
|
764
|
-
spanIds: [],
|
|
765
|
-
spanCount: 0,
|
|
766
|
-
spanNames: /* @__PURE__ */ new Map(),
|
|
767
|
-
toolNames: /* @__PURE__ */ new Map()
|
|
768
|
-
};
|
|
769
|
-
clusters.set(signature, acc);
|
|
770
|
-
}
|
|
771
|
-
acc.traceIds.add(traceId);
|
|
772
|
-
acc.spanCount += 1;
|
|
773
|
-
if (acc.spanIds.length < ERROR_CLUSTER_EXEMPLARS && !acc.spanIds.includes(span.span_id)) acc.spanIds.push(span.span_id);
|
|
774
|
-
bump(acc.spanNames, span.name);
|
|
775
|
-
bump(acc.toolNames, span.tool_name);
|
|
776
|
-
}
|
|
777
|
-
function finalizeErrorClusters(clusters, errorTraceCount) {
|
|
778
|
-
const out = [...clusters.values()].map((acc) => ({
|
|
779
|
-
signature: acc.signature,
|
|
780
|
-
status_message_sample: acc.sample,
|
|
781
|
-
span_name: topKey(acc.spanNames),
|
|
782
|
-
tool_name: topKey(acc.toolNames),
|
|
783
|
-
trace_count: acc.traceIds.size,
|
|
784
|
-
span_count: acc.spanCount,
|
|
785
|
-
prevalence: errorTraceCount > 0 ? acc.traceIds.size / errorTraceCount : 0,
|
|
786
|
-
exemplar_trace_ids: [...acc.traceIds].slice(0, ERROR_CLUSTER_EXEMPLARS),
|
|
787
|
-
exemplar_span_ids: acc.spanIds.slice(0, ERROR_CLUSTER_EXEMPLARS)
|
|
788
|
-
}));
|
|
789
|
-
out.sort((a, b) => b.trace_count - a.trace_count || b.span_count - a.span_count);
|
|
790
|
-
return out;
|
|
791
|
-
}
|
|
792
795
|
//#endregion
|
|
793
|
-
export { spanRecordsToTraceAnalysisStore as a, otlpTextToTraceAnalysisStore as i,
|
|
796
|
+
export { spanRecordsToTraceAnalysisStore as a, otlpTextToTraceAnalysisStore as i, DEFAULT_MAX_TRACE_FILE_BYTES as n, OtlpFileTraceStore as r, BufferTraceStore as t };
|
|
794
797
|
|
|
795
|
-
//# sourceMappingURL=store-otlp-
|
|
798
|
+
//# sourceMappingURL=store-otlp-DJ9dtA5x.js.map
|