@tangle-network/agent-eval 1.0.18 → 1.0.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -21
- package/dist/adapters/http.d.ts +2 -2
- package/dist/{agent-profile-CPv4tENR.d.ts → agent-profile-BO8ruI6R.d.ts} +3 -3
- package/dist/{agent-profile-CPv4tENR.d.ts.map → agent-profile-BO8ruI6R.d.ts.map} +1 -1
- package/dist/analyst/index.d.ts +14 -14
- package/dist/analyst/index.js +13 -14
- package/dist/analyst/index.js.map +1 -1
- package/dist/{analyze-runs-pJ4fhnYg.d.ts → analyze-runs-DicleUI_.d.ts} +4 -4
- package/dist/{analyze-runs-pJ4fhnYg.d.ts.map → analyze-runs-DicleUI_.d.ts.map} +1 -1
- package/dist/{analyze-runs-C7vWBDhl.js → analyze-runs-e9vfXY--.js} +6 -110
- package/dist/analyze-runs-e9vfXY--.js.map +1 -0
- package/dist/app-grade/index.d.ts +12 -7
- package/dist/app-grade/index.d.ts.map +1 -1
- package/dist/app-grade/index.js +14 -10
- package/dist/app-grade/index.js.map +1 -1
- package/dist/{benchmark-command-Bqqvv9Tk.js → benchmark-command-BGZD77Un.js} +16 -17
- package/dist/benchmark-command-BGZD77Un.js.map +1 -0
- package/dist/benchmarks/index.d.ts +4 -4
- package/dist/benchmarks/index.js +4 -5
- package/dist/benchmarks/index.js.map +1 -1
- package/dist/builder-eval/index.js +1 -1
- package/dist/{calibration-CbZPApIO.js → calibration-Ze7SjhT9.js} +2 -2
- package/dist/{calibration-CbZPApIO.js.map → calibration-Ze7SjhT9.js.map} +1 -1
- package/dist/campaign/index.d.ts +7 -7
- package/dist/campaign/index.js +13 -13
- package/dist/{campaign-DzLlT0So.js → campaign-BfDMncQN.js} +16 -17
- package/dist/{campaign-DzLlT0So.js.map → campaign-BfDMncQN.js.map} +1 -1
- package/dist/{campaign-evidence-CcA7a67m.js → campaign-evidence-BuKlRzSA.js} +4 -4
- package/dist/{campaign-evidence-CcA7a67m.js.map → campaign-evidence-BuKlRzSA.js.map} +1 -1
- package/dist/{chat-client-D0me8iwq.js → chat-client-Zz97O3NB.js} +2 -2
- package/dist/{chat-client-D0me8iwq.js.map → chat-client-Zz97O3NB.js.map} +1 -1
- package/dist/{chat-json-call-C473CwTt.js → chat-json-call-Dgi7-RH-.js} +2 -2
- package/dist/{chat-json-call-C473CwTt.js.map → chat-json-call-Dgi7-RH-.js.map} +1 -1
- package/dist/cli.js +9 -9
- package/dist/{types-0KglnQ9o.d.ts → cluster-Cd0MRbr9.d.ts} +44 -2
- package/dist/cluster-Cd0MRbr9.d.ts.map +1 -0
- package/dist/cluster-bootstrap-BYIW7QJ4.js +24 -0
- package/dist/cluster-bootstrap-BYIW7QJ4.js.map +1 -0
- package/dist/{completion-verifier-7KEfroqx.d.ts → completion-verifier-BEpsr3N4.d.ts} +2 -2
- package/dist/{completion-verifier-7KEfroqx.d.ts.map → completion-verifier-BEpsr3N4.d.ts.map} +1 -1
- package/dist/{completion-verifier-ClmJV_Yc.js → completion-verifier-DFyYFezn.js} +3 -3
- package/dist/{completion-verifier-ClmJV_Yc.js.map → completion-verifier-DFyYFezn.js.map} +1 -1
- package/dist/contract/index.d.ts +10 -10
- package/dist/contract/index.js +15 -16
- package/dist/contract/index.js.map +1 -1
- package/dist/{cost-ledger-CS8R3IZj.js → cost-ledger-DfhStGcZ.js} +2 -2
- package/dist/{cost-ledger-CS8R3IZj.js.map → cost-ledger-DfhStGcZ.js.map} +1 -1
- package/dist/{decision-point-CNyoKOkz.js → decision-point-DJWievVw.js} +4 -4
- package/dist/{decision-point-CNyoKOkz.js.map → decision-point-DJWievVw.js.map} +1 -1
- package/dist/{default-registry-BNv7eB1C.d.ts → default-registry-BpGFsSpZ.d.ts} +7 -7
- package/dist/{default-registry-BNv7eB1C.d.ts.map → default-registry-BpGFsSpZ.d.ts.map} +1 -1
- package/dist/{default-registry-BB9_jzyH.js → default-registry-CdMtoY-s.js} +5 -5
- package/dist/{default-registry-BB9_jzyH.js.map → default-registry-CdMtoY-s.js.map} +1 -1
- package/dist/{define-agent-eval-DKVYlm-T.d.ts → define-agent-eval-DcK8mg63.d.ts} +7 -7
- package/dist/{define-agent-eval-DKVYlm-T.d.ts.map → define-agent-eval-DcK8mg63.d.ts.map} +1 -1
- package/dist/{define-agent-eval-CjF2nTA3.js → define-agent-eval-DuQNZajG.js} +7 -7
- package/dist/{define-agent-eval-CjF2nTA3.js.map → define-agent-eval-DuQNZajG.js.map} +1 -1
- package/dist/diagnosis/index.d.ts +2 -2
- package/dist/diagnosis/index.js +1 -1
- package/dist/{dspy-rlm-engine-_MEe2jIO.js → dspy-rlm-engine-cWyn8G1a.js} +4 -4
- package/dist/{dspy-rlm-engine-_MEe2jIO.js.map → dspy-rlm-engine-cWyn8G1a.js.map} +1 -1
- package/dist/{engine-rfV8Ff8V.d.ts → engine-BF8naBn7.d.ts} +2 -2
- package/dist/{engine-rfV8Ff8V.d.ts.map → engine-BF8naBn7.d.ts.map} +1 -1
- package/dist/{eval-campaign-CZSCjxVb.js → eval-campaign-LmpC_5fm.js} +2 -2
- package/dist/{eval-campaign-CZSCjxVb.js.map → eval-campaign-LmpC_5fm.js.map} +1 -1
- package/dist/{evaluation-DYAIkPQB.d.ts → evaluation-BCejQgkV.d.ts} +4 -4
- package/dist/{evaluation-DYAIkPQB.d.ts.map → evaluation-BCejQgkV.d.ts.map} +1 -1
- package/dist/{evaluation-BBI37mRe.js → evaluation-bU_bhlv-.js} +2 -2
- package/dist/{evaluation-BBI37mRe.js.map → evaluation-bU_bhlv-.js.map} +1 -1
- package/dist/evaluation.d.ts +1 -1
- package/dist/evaluation.js +1 -1
- package/dist/{exact-types-CaLLU1Pz.d.ts → exact-types-VAFjtOQN.d.ts} +2 -2
- package/dist/{exact-types-CaLLU1Pz.d.ts.map → exact-types-VAFjtOQN.d.ts.map} +1 -1
- package/dist/experiment/index.d.ts +3 -5
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +7 -12
- package/dist/experiment/index.js.map +1 -1
- package/dist/{exporters-BYBG9b1n.d.ts → exporters-dQKYD_Zo.d.ts} +2 -2
- package/dist/{exporters-BYBG9b1n.d.ts.map → exporters-dQKYD_Zo.d.ts.map} +1 -1
- package/dist/{external-optimizer-process-za-SsIap.js → external-optimizer-process-2ImGH_LR.js} +2 -2
- package/dist/{external-optimizer-process-za-SsIap.js.map → external-optimizer-process-2ImGH_LR.js.map} +1 -1
- package/dist/{external-optimizer-subprocess-B44kOfZb.js → external-optimizer-subprocess-_C_2yriN.js} +4 -4
- package/dist/{external-optimizer-subprocess-B44kOfZb.js.map → external-optimizer-subprocess-_C_2yriN.js.map} +1 -1
- package/dist/{feedback-trajectory-P37ju-gW.d.ts → feedback-trajectory-DTj8eVTR.d.ts} +3 -3
- package/dist/{feedback-trajectory-P37ju-gW.d.ts.map → feedback-trajectory-DTj8eVTR.d.ts.map} +1 -1
- package/dist/{findings-store-DWiBvJVM.js → findings-store-BSuaV3dN.js} +8 -25
- package/dist/findings-store-BSuaV3dN.js.map +1 -0
- package/dist/fuzz.js +2 -2
- package/dist/hosted/index.js +1 -1
- package/dist/{index-BJM4rwP2.d.ts → index-Bc8BZfzb.d.ts} +10 -9
- package/dist/index-Bc8BZfzb.d.ts.map +1 -0
- package/dist/index-BdLmBInV.d.ts +1099 -0
- package/dist/index-BdLmBInV.d.ts.map +1 -0
- package/dist/{index-BAPkHt5H.d.ts → index-CeYLLnfi.d.ts} +28 -128
- package/dist/index-CeYLLnfi.d.ts.map +1 -0
- package/dist/{index-C3t17kHu.d.ts → index-CzfdjM6o.d.ts} +7 -100
- package/dist/index-CzfdjM6o.d.ts.map +1 -0
- package/dist/{index-f09MX_H_.d.ts → index-DYNHOTTB.d.ts} +5 -5
- package/dist/{index-f09MX_H_.d.ts.map → index-DYNHOTTB.d.ts.map} +1 -1
- package/dist/{index-Dt6E4BHX.d.ts → index-D_eAFB3T.d.ts} +8 -8
- package/dist/{index-Dt6E4BHX.d.ts.map → index-D_eAFB3T.d.ts.map} +1 -1
- package/dist/{index-chNyKroE.d.ts → index-gSu9DUiw.d.ts} +8 -8
- package/dist/{index-chNyKroE.d.ts.map → index-gSu9DUiw.d.ts.map} +1 -1
- package/dist/{index-B1Bm9oJA.d.ts → index-lRKqfyg4.d.ts} +3 -7
- package/dist/index-lRKqfyg4.d.ts.map +1 -0
- package/dist/index.d.ts +29 -31
- package/dist/index.js +28 -164
- package/dist/index.js.map +1 -1
- package/dist/{integrity-BAdVzT8u.d.ts → integrity-CrmnGB2T.d.ts} +2 -2
- package/dist/{integrity-BAdVzT8u.d.ts.map → integrity-CrmnGB2T.d.ts.map} +1 -1
- package/dist/{journal-C1uLJi0w.js → journal-CTX5uK-n.js} +36 -26
- package/dist/journal-CTX5uK-n.js.map +1 -0
- package/dist/{judge-gate-CIlBrn6q.js → judge-gate-BliY81A0.js} +3 -3
- package/dist/{judge-gate-CIlBrn6q.js.map → judge-gate-BliY81A0.js.map} +1 -1
- package/dist/{kind-factory-Cfz8DUCc.js → kind-factory-CV0FNCMM.js} +2 -2
- package/dist/{kind-factory-Cfz8DUCc.js.map → kind-factory-CV0FNCMM.js.map} +1 -1
- package/dist/ledger-core/index.d.ts +1 -1
- package/dist/ledger-core/index.js +2 -2
- package/dist/{ledger-core-CfT-FzPf.js → ledger-core-24IxvmVY.js} +2 -2
- package/dist/{ledger-core-CfT-FzPf.js.map → ledger-core-24IxvmVY.js.map} +1 -1
- package/dist/{llm-client-BdSiZ4qz.js → llm-client-DMMSg-Ro.js} +2 -2
- package/dist/{llm-client-BdSiZ4qz.js.map → llm-client-DMMSg-Ro.js.map} +1 -1
- package/dist/{llm-judge-CHLvjbVR.js → llm-judge-Bnxuwjxl.js} +15 -16
- package/dist/{llm-judge-CHLvjbVR.js.map → llm-judge-Bnxuwjxl.js.map} +1 -1
- package/dist/{matrix-Cy3oy7Rq.d.ts → matrix-D6zEwLvE.d.ts} +2 -2
- package/dist/{matrix-Cy3oy7Rq.d.ts.map → matrix-D6zEwLvE.d.ts.map} +1 -1
- package/dist/meta-eval/index.d.ts +2 -2
- package/dist/meta-eval/index.js +7 -7
- package/dist/{metrics-DNCAUtuB.js → metrics-zALlR17s.js} +16 -17
- package/dist/metrics-zALlR17s.js.map +1 -0
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/multishot/index.js +1 -1
- package/dist/openapi.json +1 -1
- package/dist/{outcome-observations-DZu5hLYF.js → outcome-observations-BpicIVQi.js} +2 -3
- package/dist/{outcome-observations-DZu5hLYF.js.map → outcome-observations-BpicIVQi.js.map} +1 -1
- package/dist/paired-arms-Dep5tbsw.js +338 -0
- package/dist/paired-arms-Dep5tbsw.js.map +1 -0
- package/dist/{paired-promotion-decision-BDnqepZ1.d.ts → paired-promotion-decision-BNzPfXDw.d.ts} +3 -113
- package/dist/paired-promotion-decision-BNzPfXDw.d.ts.map +1 -0
- package/dist/{paired-promotion-decision-CiueaTTj.js → paired-promotion-decision-Nt4CkkOT.js} +2 -3
- package/dist/{paired-promotion-decision-CiueaTTj.js.map → paired-promotion-decision-Nt4CkkOT.js.map} +1 -1
- package/dist/{pareto-B34N-vnn.js → pareto-C445lnEH.js} +6 -9
- package/dist/{pareto-B34N-vnn.js.map → pareto-C445lnEH.js.map} +1 -1
- package/dist/pipelines/index.js +1 -1
- package/dist/{prime-protocol-DKRk4l4U.d.ts → prime-protocol-D1NkJRuD.d.ts} +2 -2
- package/dist/{prime-protocol-DKRk4l4U.d.ts.map → prime-protocol-D1NkJRuD.d.ts.map} +1 -1
- package/dist/{promotion-gate-DOkgJcLW.d.ts → promotion-gate-DjAsmBSZ.d.ts} +3 -67
- package/dist/promotion-gate-DjAsmBSZ.d.ts.map +1 -0
- package/dist/{promotion-policy-DLPcEnCf.js → promotion-policy-CUWOv01m.js} +4 -4
- package/dist/{promotion-policy-DLPcEnCf.js.map → promotion-policy-CUWOv01m.js.map} +1 -1
- package/dist/{proposal-findings-CZJkS_GY.js → proposal-findings-DWs94yT8.js} +2 -2
- package/dist/{proposal-findings-CZJkS_GY.js.map → proposal-findings-DWs94yT8.js.map} +1 -1
- package/dist/{registry-Ck7w-rr5.d.ts → registry-BAcEbzKm.d.ts} +4 -4
- package/dist/{registry-Ck7w-rr5.d.ts.map → registry-BAcEbzKm.d.ts.map} +1 -1
- package/dist/{release-confidence-nfQNh3eb.js → release-confidence-DDrkmWxT.js} +2 -2
- package/dist/{release-confidence-nfQNh3eb.js.map → release-confidence-DDrkmWxT.js.map} +1 -1
- package/dist/reporting.d.ts +4 -5
- package/dist/reporting.d.ts.map +1 -1
- package/dist/reporting.js +5 -7
- package/dist/reporting.js.map +1 -1
- package/dist/{researcher-CLnNJ7Or.d.ts → researcher-D35yo0B8.d.ts} +4 -4
- package/dist/{researcher-CLnNJ7Or.d.ts.map → researcher-D35yo0B8.d.ts.map} +1 -1
- package/dist/{reward-hacking-B88f-Kh7.js → reward-hacking-DRb8K6N1.js} +2 -3
- package/dist/{reward-hacking-B88f-Kh7.js.map → reward-hacking-DRb8K6N1.js.map} +1 -1
- package/dist/rl.d.ts +9 -8
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +8 -10
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +4 -4
- package/dist/rollout/index.js +1 -1
- package/dist/{rollout-Cu7kHGIe.js → rollout-DKa54qer.js} +2 -2
- package/dist/{rollout-Cu7kHGIe.js.map → rollout-DKa54qer.js.map} +1 -1
- package/dist/{rubric-predictive-validity-D_6ZPTNR.js → rubric-predictive-validity-BMS7SUyh.js} +2 -2
- package/dist/{rubric-predictive-validity-D_6ZPTNR.js.map → rubric-predictive-validity-BMS7SUyh.js.map} +1 -1
- package/dist/{run-record-CbUi2adH.js → run-record-CIflWk6i.js} +2 -2
- package/dist/{run-record-CbUi2adH.js.map → run-record-CIflWk6i.js.map} +1 -1
- package/dist/{runtime-rollouts-kGn1K5ut.js → runtime-rollouts-CfRyjk60.js} +2 -2
- package/dist/{runtime-rollouts-kGn1K5ut.js.map → runtime-rollouts-CfRyjk60.js.map} +1 -1
- package/dist/{schema-CfoA9U_U.d.ts → schema-PDKQ_iNE.d.ts} +2 -131
- package/dist/schema-PDKQ_iNE.d.ts.map +1 -0
- package/dist/search/index.d.ts +2 -2
- package/dist/search/index.js +2 -2
- package/dist/{search-kernel-kwO-sgCq.d.ts → search-kernel-Ds2zILpY.d.ts} +4 -4
- package/dist/{search-kernel-kwO-sgCq.d.ts.map → search-kernel-Ds2zILpY.d.ts.map} +1 -1
- package/dist/{search-ledger-CV_bQQed.js → search-ledger-Dk-yk5Li.js} +4 -4
- package/dist/{search-ledger-CV_bQQed.js.map → search-ledger-Dk-yk5Li.js.map} +1 -1
- package/dist/{search-ledger-file-DTBD6X-y.js → search-ledger-file-BhOdgYIt.js} +3 -3
- package/dist/{search-ledger-file-DTBD6X-y.js.map → search-ledger-file-BhOdgYIt.js.map} +1 -1
- package/dist/{search-shipper-CA3ooBbC.js → search-shipper-Dm-_Dtvy.js} +3 -3
- package/dist/{search-shipper-CA3ooBbC.js.map → search-shipper-Dm-_Dtvy.js.map} +1 -1
- package/dist/{search-summary-DBMjdecs.js → search-summary-BSftVrGY.js} +5 -5
- package/dist/{search-summary-DBMjdecs.js.map → search-summary-BSftVrGY.js.map} +1 -1
- package/dist/{sequential-Bb_pdTqG.d.ts → sequential-B9p6uoV8.d.ts} +9 -2
- package/dist/{sequential-Bb_pdTqG.d.ts.map → sequential-B9p6uoV8.d.ts.map} +1 -1
- package/dist/{sequential-Bjs5roiK.js → sequential-CAksw6EU.js} +32 -2
- package/dist/{sequential-Bjs5roiK.js.map → sequential-CAksw6EU.js.map} +1 -1
- package/dist/{sequential-BPbvgzfe.js → sequential-CqG6aLWS.js} +3 -3
- package/dist/{sequential-BPbvgzfe.js.map → sequential-CqG6aLWS.js.map} +1 -1
- package/dist/{server-CPKyf3bz.js → server-DFVwfoZq.js} +3 -3
- package/dist/{server-CPKyf3bz.js.map → server-DFVwfoZq.js.map} +1 -1
- package/dist/{single-run-lock-BGlW3OSk.js → single-run-lock-CUj2tjxT.js} +2 -2
- package/dist/{single-run-lock-BGlW3OSk.js.map → single-run-lock-CUj2tjxT.js.map} +1 -1
- package/dist/{skillopt-optimization-method-D72XZZfR.js → skillopt-optimization-method-DYwEndON.js} +12 -12
- package/dist/{skillopt-optimization-method-D72XZZfR.js.map → skillopt-optimization-method-DYwEndON.js.map} +1 -1
- package/dist/{statistical-heldout-_dXihc9T.d.ts → statistical-heldout--0b1lgx4.d.ts} +6 -282
- package/dist/statistical-heldout--0b1lgx4.d.ts.map +1 -0
- package/dist/{statistical-heldout-NBxC7jGo.js → statistical-heldout-D6i58CJ5.js} +3 -3
- package/dist/{statistical-heldout-NBxC7jGo.js.map → statistical-heldout-D6i58CJ5.js.map} +1 -1
- package/dist/statistics-DCZIC-qT.js +2627 -0
- package/dist/statistics-DCZIC-qT.js.map +1 -0
- package/dist/statistics.d.ts +3 -0
- package/dist/statistics.js +3 -0
- package/dist/{store-otlp-BfPRa2p8.js → store-otlp-C-jyj3WX.js} +2 -2
- package/dist/{store-otlp-BfPRa2p8.js.map → store-otlp-C-jyj3WX.js.map} +1 -1
- package/dist/{store-tool-spans-BgwVHXma.d.ts → store-tool-spans-B3z_Qb5u.d.ts} +5 -5
- package/dist/{store-tool-spans-BgwVHXma.d.ts.map → store-tool-spans-B3z_Qb5u.d.ts.map} +1 -1
- package/dist/{store-tool-spans-BrSWHB9O.js → store-tool-spans-DalWlKY8.js} +3 -3
- package/dist/{store-tool-spans-BrSWHB9O.js.map → store-tool-spans-DalWlKY8.js.map} +1 -1
- package/dist/{summary-report-0La5vejY.d.ts → summary-report-Cv1AC-iv.d.ts} +2 -2
- package/dist/{summary-report-0La5vejY.d.ts.map → summary-report-Cv1AC-iv.d.ts.map} +1 -1
- package/dist/{summary-report-C3IJvp0i.js → summary-report-D_uussI3.js} +4 -52
- package/dist/summary-report-D_uussI3.js.map +1 -0
- package/dist/supervisor-run/index.d.ts +2 -2
- package/dist/supervisor-run/index.js +1 -1
- package/dist/{systemone-CBvL1gx-.d.ts → systemone-C3xyEKvG.d.ts} +7 -7
- package/dist/{systemone-CBvL1gx-.d.ts.map → systemone-C3xyEKvG.d.ts.map} +1 -1
- package/dist/{systemone-review-nbuMxxmi.d.ts → systemone-review-CToBl1oa.d.ts} +2 -2
- package/dist/{systemone-review-nbuMxxmi.d.ts.map → systemone-review-CToBl1oa.d.ts.map} +1 -1
- package/dist/systemone.d.ts +3 -3
- package/dist/systemone.js +2 -2
- package/dist/{tool-groups-BQsjZD9F.d.ts → tool-groups-R5RO9OHn.d.ts} +3 -3
- package/dist/tool-groups-R5RO9OHn.d.ts.map +1 -0
- package/dist/{tool-waste-Czt3aWID.js → tool-waste-DEhKfiIE.js} +2 -3
- package/dist/{tool-waste-Czt3aWID.js.map → tool-waste-DEhKfiIE.js.map} +1 -1
- package/dist/{trace-questions-CpTrtEcm.d.ts → trace-questions-BY6taxnl.d.ts} +3 -3
- package/dist/{trace-questions-CpTrtEcm.d.ts.map → trace-questions-BY6taxnl.d.ts.map} +1 -1
- package/dist/trace-repair/index.d.ts +1 -1
- package/dist/trace-repair/index.js +1 -2
- package/dist/trace-repair/index.js.map +1 -1
- package/dist/traces.d.ts +5 -5
- package/dist/traces.js +3 -3
- package/dist/{tree-DPwgM2LX.js → tree-B2y1UR8o.js} +5 -7
- package/dist/{tree-DPwgM2LX.js.map → tree-B2y1UR8o.js.map} +1 -1
- package/dist/{types-DNB9hIXa.d.ts → types-BAJ2Hb-d.d.ts} +3 -2
- package/dist/types-BAJ2Hb-d.d.ts.map +1 -0
- package/dist/{types-D91tnhzf.d.ts → types-BwF--ZRz.d.ts} +2 -2
- package/dist/{types-D91tnhzf.d.ts.map → types-BwF--ZRz.d.ts.map} +1 -1
- package/dist/{types-CBkDo7ja.d.ts → types-CeMZYsNF.d.ts} +6 -5
- package/dist/types-CeMZYsNF.d.ts.map +1 -0
- package/dist/{types-B9s9Xo5_.d.ts → types-kBMNdo1X.d.ts} +3 -9
- package/dist/types-kBMNdo1X.d.ts.map +1 -0
- package/dist/wire/index.d.ts +4 -4
- package/dist/wire/index.js +1 -1
- package/docs/design/statistics-decisions.md +38 -364
- package/docs/design.md +1 -1
- package/docs/research-report-methodology.md +1 -1
- package/package.json +6 -1
- package/dist/agreement-irr-DXcXEt4I.js +0 -182
- package/dist/agreement-irr-DXcXEt4I.js.map +0 -1
- package/dist/analyze-runs-C7vWBDhl.js.map +0 -1
- package/dist/benchmark-command-Bqqvv9Tk.js.map +0 -1
- package/dist/cluster-bootstrap-BteuR1hg.js +0 -65
- package/dist/cluster-bootstrap-BteuR1hg.js.map +0 -1
- package/dist/descriptive-B3Eb7tX0.js +0 -229
- package/dist/descriptive-B3Eb7tX0.js.map +0 -1
- package/dist/findings-store-DWiBvJVM.js.map +0 -1
- package/dist/index-B1Bm9oJA.d.ts.map +0 -1
- package/dist/index-BAPkHt5H.d.ts.map +0 -1
- package/dist/index-BJM4rwP2.d.ts.map +0 -1
- package/dist/index-C3t17kHu.d.ts.map +0 -1
- package/dist/journal-C1uLJi0w.js.map +0 -1
- package/dist/judge-calibration-DKR6Xi-w.js +0 -319
- package/dist/judge-calibration-DKR6Xi-w.js.map +0 -1
- package/dist/judge-calibration-DnrbQaXI.d.ts +0 -144
- package/dist/judge-calibration-DnrbQaXI.d.ts.map +0 -1
- package/dist/mean-D1d6gedE.js +0 -284
- package/dist/mean-D1d6gedE.js.map +0 -1
- package/dist/metrics-DNCAUtuB.js.map +0 -1
- package/dist/multiplicity-DAd4xaDC.js +0 -100
- package/dist/multiplicity-DAd4xaDC.js.map +0 -1
- package/dist/multiplicity-D_SEsc5j.d.ts +0 -41
- package/dist/multiplicity-D_SEsc5j.d.ts.map +0 -1
- package/dist/paired-arms-3-tsXvvC.js +0 -740
- package/dist/paired-arms-3-tsXvvC.js.map +0 -1
- package/dist/paired-binary-CMzmSSd5.js +0 -372
- package/dist/paired-binary-CMzmSSd5.js.map +0 -1
- package/dist/paired-promotion-decision-BDnqepZ1.d.ts.map +0 -1
- package/dist/paired-tests-Buh6feJ4.js +0 -211
- package/dist/paired-tests-Buh6feJ4.js.map +0 -1
- package/dist/power-and-mde-g3vOn0XK.js +0 -99
- package/dist/power-and-mde-g3vOn0XK.js.map +0 -1
- package/dist/promotion-gate-DOkgJcLW.d.ts.map +0 -1
- package/dist/random-BN150Q_X.d.ts +0 -79
- package/dist/random-BN150Q_X.d.ts.map +0 -1
- package/dist/schema-CfoA9U_U.d.ts.map +0 -1
- package/dist/sequential-eprocess-C1YLwFaz.js +0 -129
- package/dist/sequential-eprocess-C1YLwFaz.js.map +0 -1
- package/dist/statistical-heldout-_dXihc9T.d.ts.map +0 -1
- package/dist/summary-report-C3IJvp0i.js.map +0 -1
- package/dist/tool-groups-BQsjZD9F.d.ts.map +0 -1
- package/dist/types-0KglnQ9o.d.ts.map +0 -1
- package/dist/types-B9s9Xo5_.d.ts.map +0 -1
- package/dist/types-CBkDo7ja.d.ts.map +0 -1
- package/dist/types-DNB9hIXa.d.ts.map +0 -1
|
@@ -0,0 +1,1099 @@
|
|
|
1
|
+
import { p as DimensionJudgeScore, t as BootstrapInterval } from "./cluster-Cd0MRbr9.js";
|
|
2
|
+
/**
|
|
3
|
+
* Judge calibration — measure judge quality against human gold + bias.
|
|
4
|
+
*
|
|
5
|
+
* Workflow:
|
|
6
|
+
* 1. Build a golden set: {itemId, humanScore}[].
|
|
7
|
+
* 2. Run candidate judges; each produces {itemId, score}.
|
|
8
|
+
* 3. `calibrateJudge(golden, candidate)` reports κ + Pearson + MAE.
|
|
9
|
+
* 4. `calibrateJudgeContinuous(golden, candidate)` adds quadratic-weighted
|
|
10
|
+
* κ over the un-rounded [0,1] scores plus ICC(2,1), Pearson, Spearman,
|
|
11
|
+
* and bootstrap CIs — use this for fine-grained judges where rounding
|
|
12
|
+
* to int discards information (e.g. 0.78 vs 0.81 both round to 1 and
|
|
13
|
+
* look "perfectly agreed" to integer κ).
|
|
14
|
+
* 5. Run bias probes (positional, verbosity, self-preference) to
|
|
15
|
+
* detect systematic score inflation.
|
|
16
|
+
* 6. For N≥2 judges on the same items, `continuousAgreement(scores)`
|
|
17
|
+
* reports ICC(2,1) + κ_w + Pearson + Spearman with bootstrap CIs.
|
|
18
|
+
*
|
|
19
|
+
* Returns actionable diagnostics, not a single number. Consumers then
|
|
20
|
+
* decide whether to trust the judge, retrain it, or add a tie-breaker.
|
|
21
|
+
*/
|
|
22
|
+
interface GoldenItem {
|
|
23
|
+
itemId: string;
|
|
24
|
+
humanScore: number;
|
|
25
|
+
/** Optional group used for per-group bias audits (e.g. model-of-output family). */
|
|
26
|
+
group?: string;
|
|
27
|
+
}
|
|
28
|
+
interface CandidateScore {
|
|
29
|
+
itemId: string;
|
|
30
|
+
score: number;
|
|
31
|
+
/** Optional — enables positional-bias analysis (did order matter?). */
|
|
32
|
+
positionOfAInput?: 'first' | 'second';
|
|
33
|
+
}
|
|
34
|
+
interface CalibrationResult {
|
|
35
|
+
n: number;
|
|
36
|
+
pearson: number;
|
|
37
|
+
/** Cohen's κ with quadratic weights over integer-rounded scores. */
|
|
38
|
+
kappa: number;
|
|
39
|
+
/** Mean absolute error vs human. */
|
|
40
|
+
mae: number;
|
|
41
|
+
/** Worst-5 miscalibrations (largest |judge - human|). */
|
|
42
|
+
worstItems: Array<{
|
|
43
|
+
itemId: string;
|
|
44
|
+
judge: number;
|
|
45
|
+
human: number;
|
|
46
|
+
delta: number;
|
|
47
|
+
}>;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Measure judge quality against human gold labels: computes Cohen's κ, Pearson correlation, and MAE over matched item ids.
|
|
51
|
+
*/
|
|
52
|
+
declare function calibrateJudge(golden: GoldenItem[], candidate: CandidateScore[]): CalibrationResult;
|
|
53
|
+
interface PositionalBiasResult {
|
|
54
|
+
/**
|
|
55
|
+
* Score delta (first-position - second-position) averaged across items
|
|
56
|
+
* presented in both positions. Non-zero = positional bias.
|
|
57
|
+
*/
|
|
58
|
+
avgDelta: number;
|
|
59
|
+
n: number;
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Feed the same items to the judge twice with A/B swapped and pass all
|
|
63
|
+
* results here. Items that don't appear in both positions are ignored.
|
|
64
|
+
*/
|
|
65
|
+
declare function positionalBias(scores: CandidateScore[]): PositionalBiasResult;
|
|
66
|
+
interface VerbosityBiasResult {
|
|
67
|
+
/** Pearson correlation between output length and score. Strong positive = verbosity bias. */
|
|
68
|
+
pearson: number;
|
|
69
|
+
n: number;
|
|
70
|
+
}
|
|
71
|
+
declare function verbosityBias(samples: Array<{
|
|
72
|
+
outputLen: number;
|
|
73
|
+
score: number;
|
|
74
|
+
}>): VerbosityBiasResult;
|
|
75
|
+
interface SelfPreferenceResult {
|
|
76
|
+
/** Mean judge score when judge's family matches output's family. */
|
|
77
|
+
inFamilyMean: number;
|
|
78
|
+
outOfFamilyMean: number;
|
|
79
|
+
deltaMean: number;
|
|
80
|
+
n: number;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Pass the same scenarios scored with judge-model X grading outputs from
|
|
84
|
+
* model X (in-family) and model Y (out-of-family). Non-zero delta
|
|
85
|
+
* indicates self-preference.
|
|
86
|
+
*/
|
|
87
|
+
declare function selfPreference(samples: Array<{
|
|
88
|
+
score: number;
|
|
89
|
+
inFamily: boolean;
|
|
90
|
+
}>): SelfPreferenceResult;
|
|
91
|
+
interface ContinuousAgreement {
|
|
92
|
+
/** Cohen's κ_w with quadratic weights, computed on raw [0,1] scores. */
|
|
93
|
+
weightedKappa: number;
|
|
94
|
+
/** ICC(2,1): two-way random effects, absolute agreement, single rater. */
|
|
95
|
+
icc: number;
|
|
96
|
+
/** Pearson product-moment correlation (averaged over rater pairs if N>2). */
|
|
97
|
+
pearson: number;
|
|
98
|
+
/** Spearman rank correlation (averaged over rater pairs if N>2). */
|
|
99
|
+
spearman: number;
|
|
100
|
+
/** 95% bootstrap percentile CIs over items. */
|
|
101
|
+
ci: {
|
|
102
|
+
icc: [number, number];
|
|
103
|
+
weightedKappa: [number, number];
|
|
104
|
+
};
|
|
105
|
+
/** Number of complete items (no NaN across raters). */
|
|
106
|
+
n: number;
|
|
107
|
+
/** Number of raters. */
|
|
108
|
+
raters: number;
|
|
109
|
+
}
|
|
110
|
+
interface ContinuousAgreementOptions {
|
|
111
|
+
/** Bootstrap iterations. Default 1000. Set to 0 to skip CIs (CI = [NaN, NaN]). */
|
|
112
|
+
bootstrap?: number;
|
|
113
|
+
/** κ weighting scheme. Default 'quadratic'. */
|
|
114
|
+
weights?: 'linear' | 'quadratic';
|
|
115
|
+
/** PRNG seed for reproducible bootstrap. Default 0xC0FFEE. */
|
|
116
|
+
seed?: number;
|
|
117
|
+
/** Confidence level for percentile CI. Default 0.95. */
|
|
118
|
+
ciLevel?: number;
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* Inter-rater agreement on continuous (typically [0,1]) scores.
|
|
122
|
+
*
|
|
123
|
+
* `scores` has shape [n_items][n_raters]. Rows with any non-finite entry
|
|
124
|
+
* are dropped. Returns NaN metrics if fewer than 2 raters or 2 complete
|
|
125
|
+
* items remain.
|
|
126
|
+
*/
|
|
127
|
+
declare function continuousAgreement(scores: number[][], opts?: ContinuousAgreementOptions): ContinuousAgreement;
|
|
128
|
+
interface ContinuousCalibrationResult extends CalibrationResult {
|
|
129
|
+
/** Cohen's κ_w computed on raw (un-rounded) scores. */
|
|
130
|
+
weightedKappaContinuous: number;
|
|
131
|
+
/** ICC(2,1) treating golden + candidate as two raters. */
|
|
132
|
+
icc: number;
|
|
133
|
+
spearman: number;
|
|
134
|
+
ci: {
|
|
135
|
+
icc: [number, number];
|
|
136
|
+
weightedKappa: [number, number];
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Extends `calibrateJudge` with continuous-value agreement metrics while
|
|
141
|
+
* retaining its base calibration summary.
|
|
142
|
+
*/
|
|
143
|
+
declare function calibrateJudgeContinuous(golden: GoldenItem[], candidate: CandidateScore[], opts?: ContinuousAgreementOptions): ContinuousCalibrationResult;
|
|
144
|
+
/**
|
|
145
|
+
* Inter-rater reliability — Krippendorff's α under the squared-difference
|
|
146
|
+
* metric, pooled across dimensions.
|
|
147
|
+
*
|
|
148
|
+
* Each inner array is one judge's scores. Items are matched by position
|
|
149
|
+
* WITHIN a dimension: the k-th score a judge supplies carrying dimension
|
|
150
|
+
* `d` is item k of `d`, and the ratings compared against each other are
|
|
151
|
+
* the ones different judges gave to the same item. Every judge that scores
|
|
152
|
+
* a dimension at all must supply the same number of scores for it —
|
|
153
|
+
* ragged input cannot be aligned into items and throws rather than
|
|
154
|
+
* comparing mismatched items.
|
|
155
|
+
*
|
|
156
|
+
* α = 1 − D_observed / D_expected: D_observed averages the squared
|
|
157
|
+
* difference over within-item judge pairs, D_expected over every pair of
|
|
158
|
+
* ratings irrespective of item. α = 1 is perfect agreement, 0 is chance,
|
|
159
|
+
* negative is systematic disagreement.
|
|
160
|
+
*/
|
|
161
|
+
declare function interRaterReliability(judgeScores: DimensionJudgeScore[][]): number;
|
|
162
|
+
interface CorpusScoreRecord {
|
|
163
|
+
/** Stable identifier for the rated item (scenario, span, turn, …). */
|
|
164
|
+
itemId: string;
|
|
165
|
+
/** Identifier for the judge that produced this score. */
|
|
166
|
+
judgeName: string;
|
|
167
|
+
/** Dimension name (matches `DimensionJudgeScore.dimension`). */
|
|
168
|
+
dimension: string;
|
|
169
|
+
/** Numeric score; must be finite. */
|
|
170
|
+
score: number;
|
|
171
|
+
}
|
|
172
|
+
interface CorpusAgreementPerDimension extends ContinuousAgreement {
|
|
173
|
+
dimension: string;
|
|
174
|
+
/** Item IDs that contributed to this dimension's matrix (every judge scored them). */
|
|
175
|
+
itemIds: string[];
|
|
176
|
+
/** Judge IDs that contributed to this dimension's matrix. */
|
|
177
|
+
judgeIds: string[];
|
|
178
|
+
}
|
|
179
|
+
interface CorpusAgreementReport {
|
|
180
|
+
/** Per-dimension ICC(2,1) + κ_w + Pearson + Spearman + bootstrap CIs. */
|
|
181
|
+
perDimension: CorpusAgreementPerDimension[];
|
|
182
|
+
/** Mean ICC across dimensions (NaN if no dimension yielded a finite ICC). */
|
|
183
|
+
overallIcc: number;
|
|
184
|
+
/** Mean weighted κ across dimensions (NaN if none finite). */
|
|
185
|
+
overallWeightedKappa: number;
|
|
186
|
+
/** Dimensions evaluated (sorted). */
|
|
187
|
+
dimensions: string[];
|
|
188
|
+
/** Judges seen across the corpus (sorted). */
|
|
189
|
+
judgeIds: string[];
|
|
190
|
+
}
|
|
191
|
+
interface CorpusAgreementOptions extends ContinuousAgreementOptions {
|
|
192
|
+
/** Restrict the audit to these dimensions; default every one in the input. A named absentee throws. */
|
|
193
|
+
dimensions?: string[];
|
|
194
|
+
/** Restrict the audit to these judges; default every one in the input. A named absentee throws. */
|
|
195
|
+
judges?: string[];
|
|
196
|
+
}
|
|
197
|
+
/**
|
|
198
|
+
* Corpus-wide inter-rater agreement across N items × M judges × D dimensions, the
|
|
199
|
+
* question `interRaterReliability`'s within-item α cannot answer.
|
|
200
|
+
*
|
|
201
|
+
* For each dimension, builds the [n_items][n_judges] matrix of scores
|
|
202
|
+
* (keeping only items every judge rated on that dimension), then runs
|
|
203
|
+
* `continuousAgreement` to get ICC(2,1), κ_w, Pearson, Spearman, and
|
|
204
|
+
* bootstrap CIs. Reports a pooled mean across dimensions as a single
|
|
205
|
+
* "is this judge panel reliable on this corpus?" number. Empty input, fewer than 2 judges or
|
|
206
|
+
* items, a judge without scores on a dimension and duplicate records all throw: a silently
|
|
207
|
+
* shrunk matrix would corrupt the overall metric.
|
|
208
|
+
*/
|
|
209
|
+
declare function corpusInterRaterAgreement(records: CorpusScoreRecord[], opts?: CorpusAgreementOptions): CorpusAgreementReport;
|
|
210
|
+
/**
|
|
211
|
+
* Convenience adapter for `DimensionJudgeScore[]` data keyed externally by item.
|
|
212
|
+
*
|
|
213
|
+
* Use when you have per-item arrays of `DimensionJudgeScore[]` (e.g. one
|
|
214
|
+
* `ScenarioResult.judgeScores` per scenario) and want corpus-wide
|
|
215
|
+
* agreement without manually flattening. `itemId` must be unique per
|
|
216
|
+
* row of `itemsScores`.
|
|
217
|
+
*/
|
|
218
|
+
declare function corpusInterRaterAgreementFromJudgeScores(itemsScores: Array<{
|
|
219
|
+
itemId: string;
|
|
220
|
+
scores: DimensionJudgeScore[];
|
|
221
|
+
}>, opts?: CorpusAgreementOptions): CorpusAgreementReport;
|
|
222
|
+
/** Agreement between two raters who each assigned one nominal label per item. */
|
|
223
|
+
interface NominalAgreement {
|
|
224
|
+
/** Items both raters labeled. */
|
|
225
|
+
n: number;
|
|
226
|
+
/** Share of items with identical labels. */
|
|
227
|
+
agreement: number;
|
|
228
|
+
/**
|
|
229
|
+
* Cohen's κ = (pₒ − pₑ) / (1 − pₑ), with pₑ from each rater's own label
|
|
230
|
+
* marginals. NaN when n = 0, or when both raters used one identical label
|
|
231
|
+
* for every item (pₑ = 1): chance and observed agreement are then the same
|
|
232
|
+
* thing, and no number distinguishes skill from luck.
|
|
233
|
+
*/
|
|
234
|
+
kappa: number;
|
|
235
|
+
/** Seeded percentile bootstrap of κ over clusters; NaN bounds when no resample has a finite κ. */
|
|
236
|
+
kappaCi: BootstrapInterval;
|
|
237
|
+
}
|
|
238
|
+
interface NominalAgreementOptions {
|
|
239
|
+
/** Cluster id per pair; pairs sharing one are resampled together. Default: every pair alone. */
|
|
240
|
+
clusters?: readonly string[];
|
|
241
|
+
/** Default 2000. */
|
|
242
|
+
resamples?: number;
|
|
243
|
+
seed?: number;
|
|
244
|
+
}
|
|
245
|
+
/**
|
|
246
|
+
* Unweighted Cohen's κ for paired nominal labels. Labels carry no order, so a
|
|
247
|
+
* near miss counts as a miss; use `calibrateJudge` for ordinal scores.
|
|
248
|
+
*/
|
|
249
|
+
declare function nominalAgreement(pairs: ReadonlyArray<readonly [string, string]>, options?: NominalAgreementOptions): NominalAgreement;
|
|
250
|
+
/** Mann–Whitney ROC area: ties count half; no observed member of either class means unknown. */
|
|
251
|
+
declare function rocAuc(scores: number[], labels: number[]): number | null;
|
|
252
|
+
/**
|
|
253
|
+
* Descriptive statistics: means, bootstrap spread, correlation, and the
|
|
254
|
+
* weighted judge-dimension composite. Nothing here is a significance test.
|
|
255
|
+
*/
|
|
256
|
+
/** Weighted mean — falls back to uniform weights when omitted */
|
|
257
|
+
declare function weightedMean(scores: {
|
|
258
|
+
score: number;
|
|
259
|
+
weight?: number;
|
|
260
|
+
}[]): number;
|
|
261
|
+
/**
|
|
262
|
+
* Percentile bootstrap confidence interval on the mean of `scores`.
|
|
263
|
+
*
|
|
264
|
+
* Descriptive spread. It is not a significance test, and at small n its bounds
|
|
265
|
+
* are anti-conservative in the same way {@link pairedBootstrap}'s are — see
|
|
266
|
+
* {@link BOOTSTRAP_GATE_MIN_N}. With no `seed` the resampling is seeded from
|
|
267
|
+
* the scores themselves, so the interval is reproducible either way.
|
|
268
|
+
*/
|
|
269
|
+
declare function confidenceInterval(scores: number[], confidence?: number, opts?: {
|
|
270
|
+
seed?: number;
|
|
271
|
+
resamples?: number;
|
|
272
|
+
}): {
|
|
273
|
+
mean: number;
|
|
274
|
+
lower: number;
|
|
275
|
+
upper: number;
|
|
276
|
+
};
|
|
277
|
+
/**
|
|
278
|
+
* Two-sided Student-t critical value: the `t` with `P(|T| ≤ t) = confidence`
|
|
279
|
+
* for `df` degrees of freedom. Exact, from {@link studentTQuantile}, so any
|
|
280
|
+
* confidence in (0, 1) and fractional `df` are supported.
|
|
281
|
+
*/
|
|
282
|
+
declare function studentTCritical(df: number, confidence?: number): number;
|
|
283
|
+
/** One-sample Student-t interval on the mean; see {@link meanConfidenceInterval}. */
|
|
284
|
+
interface MeanConfidenceInterval {
|
|
285
|
+
readonly n: number;
|
|
286
|
+
/** Null for an empty sample. */
|
|
287
|
+
readonly mean: number | null;
|
|
288
|
+
/** Sample standard deviation (n − 1 denominator); null below two samples. */
|
|
289
|
+
readonly stdev: number | null;
|
|
290
|
+
readonly low: number | null;
|
|
291
|
+
readonly high: number | null;
|
|
292
|
+
/** Two-sided Student-t critical value for df = n − 1; null below two samples. */
|
|
293
|
+
readonly criticalValue: number | null;
|
|
294
|
+
}
|
|
295
|
+
/**
|
|
296
|
+
* Student-t confidence interval on the mean of one sample:
|
|
297
|
+
* `mean ± t(df = n − 1) · s / √n`. Below two samples there is no spread to
|
|
298
|
+
* estimate, so the bounds are null rather than a zero-width interval that
|
|
299
|
+
* would read as certainty. Unlike {@link confidenceInterval} (a percentile
|
|
300
|
+
* bootstrap), the width keeps the small-n t penalty that z = 1.96 drops.
|
|
301
|
+
*/
|
|
302
|
+
declare function meanConfidenceInterval(samples: readonly number[], confidence?: number): MeanConfidenceInterval;
|
|
303
|
+
/** Partial credit: returns 0-1 ratio of current toward target */
|
|
304
|
+
declare function partialCredit(current: number, target: number): number;
|
|
305
|
+
/** Distribution summary of a number series: count, extremes, quantiles, sum. */
|
|
306
|
+
interface SeriesDistribution {
|
|
307
|
+
readonly n: number;
|
|
308
|
+
readonly min: number;
|
|
309
|
+
readonly p50: number;
|
|
310
|
+
readonly p90: number;
|
|
311
|
+
readonly max: number;
|
|
312
|
+
readonly sum: number;
|
|
313
|
+
readonly percentiles?: Readonly<Record<number, number>>;
|
|
314
|
+
readonly medianInterval?: {
|
|
315
|
+
readonly method: 'normal-order-statistics';
|
|
316
|
+
readonly criticalValue: number;
|
|
317
|
+
readonly sampleSize: number;
|
|
318
|
+
readonly lowRank: number;
|
|
319
|
+
readonly highRank: number;
|
|
320
|
+
readonly low: number;
|
|
321
|
+
readonly high: number;
|
|
322
|
+
};
|
|
323
|
+
}
|
|
324
|
+
interface SeriesDistributionOptions {
|
|
325
|
+
/** Nearest-rank percentiles on a 0–100 scale, including fractional tails. */
|
|
326
|
+
percentiles?: readonly number[];
|
|
327
|
+
/** Descriptive median rank band, not a bootstrap interval or promotion test. */
|
|
328
|
+
medianInterval?: {
|
|
329
|
+
criticalValue: number;
|
|
330
|
+
};
|
|
331
|
+
}
|
|
332
|
+
/**
|
|
333
|
+
* Fold a number series into its distribution summary. Quantiles use the
|
|
334
|
+
* nearest-rank definition — the `ceil(q·n)`-th order statistic — so every
|
|
335
|
+
* reported quantile is a value from the series. Returns `null` for an empty
|
|
336
|
+
* series: an empty series has no distribution, and a zero-filled summary
|
|
337
|
+
* would read as a measured all-zero series.
|
|
338
|
+
*/
|
|
339
|
+
declare function summarizeNumberSeries(values: readonly number[], options?: SeriesDistributionOptions): SeriesDistribution | null;
|
|
340
|
+
/**
|
|
341
|
+
* Average-rank-with-ties transform (1-indexed). Tied values receive the mean
|
|
342
|
+
* of the ranks they span, the standard correction for Spearman's ρ.
|
|
343
|
+
*/
|
|
344
|
+
declare function ranks(xs: number[]): number[];
|
|
345
|
+
/**
|
|
346
|
+
* Pearson product-moment correlation coefficient r ∈ [-1, 1] between two
|
|
347
|
+
* equal-length series. See the edge-case contract above: NaN for n < 2 or
|
|
348
|
+
* unequal lengths, 1 when both series are constant, 0 when exactly one is.
|
|
349
|
+
*/
|
|
350
|
+
declare function pearsonR(a: number[], b: number[]): number;
|
|
351
|
+
/**
|
|
352
|
+
* Spearman's rank correlation ρ — Pearson over the average-rank-with-ties
|
|
353
|
+
* transform of each series. Same edge-case contract as {@link pearsonR}.
|
|
354
|
+
*/
|
|
355
|
+
declare function spearmanR(a: number[], b: number[]): number;
|
|
356
|
+
interface WeightedCompositeInput {
|
|
357
|
+
/** Per-dimension scores (typically 0..1). */
|
|
358
|
+
dims: Record<string, number>;
|
|
359
|
+
/** Weight per dimension. Every weighted dimension MUST be present in
|
|
360
|
+
* `dims` — a weight for an absent dimension is a config error and throws,
|
|
361
|
+
* because silently dropping it would renormalise the composite onto a
|
|
362
|
+
* different denominator than intended. */
|
|
363
|
+
weights: Record<string, number>;
|
|
364
|
+
/** Optional pass threshold; when set, the result reports `pass`. */
|
|
365
|
+
threshold?: number;
|
|
366
|
+
}
|
|
367
|
+
interface WeightedCompositeResult {
|
|
368
|
+
composite: number;
|
|
369
|
+
pass?: boolean;
|
|
370
|
+
}
|
|
371
|
+
/**
|
|
372
|
+
* Weighted composite over judge dimensions: `Σ(score_d · w_d) / Σ(w_d)` across
|
|
373
|
+
* the weighted dimensions. The canonical replacement for the per-consumer
|
|
374
|
+
* hand-rolled composite math (tax/legal/creative/gtm each ship a copy).
|
|
375
|
+
*
|
|
376
|
+
* Fail-loud: throws if a weighted dimension is missing from `dims`, if any
|
|
377
|
+
* weight is negative, or if the weights sum to 0 — none of which can produce
|
|
378
|
+
* a meaningful composite.
|
|
379
|
+
*/
|
|
380
|
+
declare function weightedComposite(input: WeightedCompositeInput): WeightedCompositeResult;
|
|
381
|
+
/**
|
|
382
|
+
* Nearest-rank distribution summary for measurement paths (gates, dashboards,
|
|
383
|
+
* load harnesses): min/p50/p90/p95/p99/max + mean + sample stddev + a 95%
|
|
384
|
+
* Student-t CI on the mean. Consolidates the former @repo/shared
|
|
385
|
+
* distribution-stats: identical nearest-rank percentiles, with the CI now the
|
|
386
|
+
* exact t critical value for any df (the retired implementation capped its
|
|
387
|
+
* table at df 30 and fell back to 1.96).
|
|
388
|
+
*/
|
|
389
|
+
interface SampleDistribution {
|
|
390
|
+
min: number;
|
|
391
|
+
p50: number;
|
|
392
|
+
p90: number;
|
|
393
|
+
p95: number;
|
|
394
|
+
p99: number;
|
|
395
|
+
max: number;
|
|
396
|
+
mean: number;
|
|
397
|
+
stddev?: number;
|
|
398
|
+
ci95Low?: number;
|
|
399
|
+
ci95High?: number;
|
|
400
|
+
n: number;
|
|
401
|
+
values: number[];
|
|
402
|
+
}
|
|
403
|
+
interface DistributionSummaryOptions {
|
|
404
|
+
lowerBound?: number;
|
|
405
|
+
}
|
|
406
|
+
/**
|
|
407
|
+
* Nearest-rank percentile on an arbitrary sample, `percentile` on a 0-100
|
|
408
|
+
* scale — the same rule `summarizeDistribution` applies, exposed for a single
|
|
409
|
+
* percentile the summary struct does not carry (e.g. p99.9). Returns NaN for
|
|
410
|
+
* an empty sample; callers that need a numeric floor guard it.
|
|
411
|
+
*/
|
|
412
|
+
declare function percentileNearestRank(values: number[], percentile: number): number;
|
|
413
|
+
declare function summarizeDistribution(values: number[], options?: DistributionSummaryOptions): SampleDistribution;
|
|
414
|
+
/** Standardized effect sizes; undefined effects remain null. */
|
|
415
|
+
/** Independent Cohen's d, oriented b − a; null below two samples or for an unbounded effect. */
|
|
416
|
+
declare function cohensD(a: number[], b: number[]): number | null;
|
|
417
|
+
/** Paired Cohen's dz; constant nonzero deltas have no finite standardized effect. */
|
|
418
|
+
declare function pairedCohensDz(before: number[], after: number[]): number | null;
|
|
419
|
+
type CliffsMagnitude = 'negligible' | 'small' | 'medium' | 'large';
|
|
420
|
+
/** Cliff's delta, oriented after − before; empty samples return zero. */
|
|
421
|
+
declare function cliffsDelta(before: number[], after: number[]): number;
|
|
422
|
+
/** Romano magnitude thresholds for the absolute Cliff's delta. */
|
|
423
|
+
declare function interpretCliffs(delta: number): CliffsMagnitude;
|
|
424
|
+
/** Two-sided normal tail, evaluated without subtracting a near-one CDF. */
|
|
425
|
+
declare function normalTwoSidedPValue(z: number): number;
|
|
426
|
+
/** Two-sided Student-t tail, preserving small tail probabilities. */
|
|
427
|
+
declare function studentTTwoSidedPValue(t: number, df: number): number;
|
|
428
|
+
/** Pearson statistic for a 2x2 table; empty or fixed margins have statistic zero. */
|
|
429
|
+
declare function chiSquared(a: number, b: number, c: number, d: number): number;
|
|
430
|
+
/** One-degree-of-freedom chi-squared survival probability. */
|
|
431
|
+
declare function chiSquaredPValue(statistic: number): number;
|
|
432
|
+
/** Fisher two-sided probability: sum all fixed-margin tables no likelier than the observed table. */
|
|
433
|
+
declare function fisherExactTwoSided(a: number, b: number, c: number, d: number): number;
|
|
434
|
+
/** Fisher when any expected count is below five; otherwise Pearson's chi-squared tail. */
|
|
435
|
+
declare function twoByTwoPValue(a: number, b: number, c: number, d: number): number;
|
|
436
|
+
interface LinearRegressionResult {
|
|
437
|
+
slope: number;
|
|
438
|
+
rSquared: number;
|
|
439
|
+
slopePValue: number | null;
|
|
440
|
+
}
|
|
441
|
+
/** Centered least-squares slope; null p-value means no slope test was possible. */
|
|
442
|
+
declare function linearRegressionSlope(points: readonly {
|
|
443
|
+
x: number;
|
|
444
|
+
y: number;
|
|
445
|
+
}[]): LinearRegressionResult;
|
|
446
|
+
/**
|
|
447
|
+
* Multiple-comparison corrections: family-wise error (Bonferroni, Holm) and
|
|
448
|
+
* false discovery rate (Benjamini-Hochberg), with inclusive rejection
|
|
449
|
+
* boundaries throughout.
|
|
450
|
+
*/
|
|
451
|
+
/**
|
|
452
|
+
* Bonferroni adjustment: multiply every p-value by the test count, clamp at 1.
|
|
453
|
+
*
|
|
454
|
+
* Rejects at `p_adjusted ≤ alpha` — the boundary is inclusive, matching
|
|
455
|
+
* {@link holm}, which uniformly dominates this correction and must therefore
|
|
456
|
+
* never reject less. Validates its inputs on the same terms.
|
|
457
|
+
*/
|
|
458
|
+
declare function bonferroni(pValues: readonly number[], alpha?: number): {
|
|
459
|
+
adjusted: number[];
|
|
460
|
+
significant: boolean[];
|
|
461
|
+
};
|
|
462
|
+
/**
|
|
463
|
+
* Holm step-down family-wise error adjustment.
|
|
464
|
+
*
|
|
465
|
+
* P-values are sorted from smallest to largest, multiplied by their remaining
|
|
466
|
+
* hypothesis count, and made monotonically non-decreasing before being mapped
|
|
467
|
+
* back to input order. This uniformly dominates plain Bonferroni while keeping
|
|
468
|
+
* strong family-wise error control under arbitrary dependence.
|
|
469
|
+
*/
|
|
470
|
+
declare function holm(pValues: readonly number[], alpha?: number): {
|
|
471
|
+
adjusted: number[];
|
|
472
|
+
significant: boolean[];
|
|
473
|
+
};
|
|
474
|
+
/**
|
|
475
|
+
* Benjamini–Hochberg false discovery rate. Returns adjusted q-values and
|
|
476
|
+
* significance at the target FDR; handles ties and preserves q monotonicity.
|
|
477
|
+
*
|
|
478
|
+
* Rejects at `q ≤ fdr` — the BH rule is inclusive at the boundary, so an
|
|
479
|
+
* exactly-`fdr` q-value is a discovery.
|
|
480
|
+
*/
|
|
481
|
+
declare function benjaminiHochberg(pValues: readonly number[], fdr?: number): {
|
|
482
|
+
qValues: number[];
|
|
483
|
+
significant: boolean[];
|
|
484
|
+
};
|
|
485
|
+
/** A binomial proportion estimate with a confidence interval. */
|
|
486
|
+
interface ProportionInterval {
|
|
487
|
+
/** Point estimate successes / n; 0 is a compatibility placeholder when n = 0. */
|
|
488
|
+
estimate: number;
|
|
489
|
+
/** Lower bound, clamped to [0, 1]. */
|
|
490
|
+
lower: number;
|
|
491
|
+
/** Upper bound, clamped to [0, 1]. */
|
|
492
|
+
upper: number;
|
|
493
|
+
}
|
|
494
|
+
/**
|
|
495
|
+
* Wilson score interval for a binomial proportion. Correct at small n and near
|
|
496
|
+
* 0/1, where the normal (Wald) approximation produces bounds outside [0, 1] and
|
|
497
|
+
* understates coverage. Use this for any pass-rate / hit-rate / realness-rate
|
|
498
|
+
* CI — the continuous `confidenceInterval` assumes the wrong distribution for a
|
|
499
|
+
* proportion. Counts must be finite nonnegative integers, successes cannot exceed
|
|
500
|
+
* n, and confidence must lie strictly between zero and one. With n = 0 the
|
|
501
|
+
* interval is the full [0, 1] support: no rate is ruled out. The estimate stays
|
|
502
|
+
* 0 for compatibility, not as a measured failure rate; retain n alongside it.
|
|
503
|
+
*/
|
|
504
|
+
declare function wilson(successes: number, n: number, confidence?: number): ProportionInterval;
|
|
505
|
+
/** Result of a McNemar paired-binary significance test. */
|
|
506
|
+
interface McNemarResult {
|
|
507
|
+
/** Total paired observations. */
|
|
508
|
+
n: number;
|
|
509
|
+
/** Discordant pairs (b + c) — the only ones that carry signal. */
|
|
510
|
+
nDiscordant: number;
|
|
511
|
+
/** Pairs where treatment succeeded and control failed ("newly correct"). */
|
|
512
|
+
b: number;
|
|
513
|
+
/** Pairs where control succeeded and treatment failed ("newly wrong"). */
|
|
514
|
+
c: number;
|
|
515
|
+
/** Continuity-corrected chi-square statistic (reference; exact p drives the call). */
|
|
516
|
+
statistic: number;
|
|
517
|
+
/** Two-sided p-value. Exact (binomial sign test on discordant pairs). */
|
|
518
|
+
pValue: number;
|
|
519
|
+
}
|
|
520
|
+
/**
|
|
521
|
+
* McNemar's test for paired binary outcomes — the correct significance test for
|
|
522
|
+
* "does treatment change the success rate vs control on the SAME items". Only
|
|
523
|
+
* discordant pairs (one arm right, the other wrong) carry information; concordant
|
|
524
|
+
* pairs are uninformative, so a paired t-test / two-proportion z-test on the raw
|
|
525
|
+
* rates is wrong here. The p-value is exact: under H0 the b "treatment-wins" are
|
|
526
|
+
* Binomial(b + c, 0.5), so the two-sided p is the doubled binomial tail — correct
|
|
527
|
+
* at the small discordant counts typical of eval runs (no continuity-corrected
|
|
528
|
+
* chi-square approximation needed, though it is returned as `statistic` for
|
|
529
|
+
* reference). Inputs are paired 0/1 (or boolean) arrays, control first to match
|
|
530
|
+
* the module's (before, after) convention. Throws on unequal lengths.
|
|
531
|
+
*/
|
|
532
|
+
declare function mcnemar(control: ArrayLike<number | boolean>, treatment: ArrayLike<number | boolean>): McNemarResult;
|
|
533
|
+
/** A paired binary effect size (treatment rate − control rate) with a CI. */
|
|
534
|
+
interface RiskDifferenceResult {
|
|
535
|
+
/** Total paired observations. */
|
|
536
|
+
n: number;
|
|
537
|
+
/** Discordant pairs: treatment-win count. */
|
|
538
|
+
b: number;
|
|
539
|
+
/** Discordant pairs: control-win count. */
|
|
540
|
+
c: number;
|
|
541
|
+
/** Paired risk difference (b − c) / n; 0 is a placeholder when n = 0. */
|
|
542
|
+
riskDifference: number;
|
|
543
|
+
/** Lower bound of the CI, clamped to [-1, 1]. */
|
|
544
|
+
lower: number;
|
|
545
|
+
/** Upper bound of the CI, clamped to [-1, 1]. */
|
|
546
|
+
upper: number;
|
|
547
|
+
/** Confidence level used. */
|
|
548
|
+
confidence: number;
|
|
549
|
+
}
|
|
550
|
+
/**
|
|
551
|
+
* Paired risk difference (the effect-size companion to {@link mcnemar}): the
|
|
552
|
+
* change in success rate p(treatment) − p(control) on matched items, which for
|
|
553
|
+
* paired binary data equals (b − c) / n. The CI uses the paired variance from
|
|
554
|
+
* the discordant counts, not the independent-samples formula (which overstates
|
|
555
|
+
* the interval by ignoring the pairing). Inputs are paired 0/1 (or boolean)
|
|
556
|
+
* arrays, control first. Throws on unequal lengths. With no pairs, returns the
|
|
557
|
+
* full [-1, 1] support and a compatibility point estimate of 0, not equivalence.
|
|
558
|
+
*
|
|
559
|
+
* REPORTING ONLY — do NOT decide a promotion on this interval. The CI is a Wald
|
|
560
|
+
* normal approximation, which badly UNDERCOVERS when only a handful of pairs are
|
|
561
|
+
* discordant: at n = 3 with b = 2, c = 0 it returns [0.133, 1.000], excluding 0,
|
|
562
|
+
* while McNemar's exact test on the same data gives p = 0.50. A gate keying on
|
|
563
|
+
* `lower > 0` would promote noise. Use {@link pairedRiskDifferenceExact}, whose
|
|
564
|
+
* interval is dual to the exact test by construction, for any decision.
|
|
565
|
+
*/
|
|
566
|
+
declare function pairedRiskDifference(control: ArrayLike<number | boolean>, treatment: ArrayLike<number | boolean>, confidence?: number): RiskDifferenceResult;
|
|
567
|
+
/** A paired binary effect size with an EXACT interval and the exact test that
|
|
568
|
+
* bounds it — one object so a caller cannot read the estimate without the
|
|
569
|
+
* significance it is entitled to. */
|
|
570
|
+
interface ExactRiskDifferenceResult {
|
|
571
|
+
/** Total paired observations. */
|
|
572
|
+
n: number;
|
|
573
|
+
/** Discordant pairs: treatment-win count. */
|
|
574
|
+
b: number;
|
|
575
|
+
/** Discordant pairs: control-win count. */
|
|
576
|
+
c: number;
|
|
577
|
+
/** Discordant pairs (b + c) — the only ones carrying information. */
|
|
578
|
+
nDiscordant: number;
|
|
579
|
+
/** Paired risk difference (b − c) / n; 0 is a placeholder when n = 0. */
|
|
580
|
+
riskDifference: number;
|
|
581
|
+
/** Exact conditional CI lower bound: -1 with no pairs; 0 with only concordant pairs. */
|
|
582
|
+
lower: number;
|
|
583
|
+
/** Exact conditional CI upper bound: 1 with no pairs; 0 with only concordant pairs. */
|
|
584
|
+
upper: number;
|
|
585
|
+
/** Confidence level used. */
|
|
586
|
+
confidence: number;
|
|
587
|
+
/** McNemar's exact two-sided p-value on the same discordant counts. */
|
|
588
|
+
pValue: number;
|
|
589
|
+
}
|
|
590
|
+
/**
|
|
591
|
+
* Paired risk difference with the EXACT CONDITIONAL interval — the estimator a
|
|
592
|
+
* promotion gate may decide on.
|
|
593
|
+
*
|
|
594
|
+
* Conditional on the number of discordant pairs m = b + c, the treatment-win
|
|
595
|
+
* count b is Binomial(m, π) with π = P(treatment wins | discordant), and the
|
|
596
|
+
* risk difference is an exact reparameterisation: RD = (2π − 1)·m/n. So a
|
|
597
|
+
* Clopper-Pearson exact interval for π maps straight onto RD. This buys the
|
|
598
|
+
* property the Wald interval in {@link pairedRiskDifference} does not have:
|
|
599
|
+
*
|
|
600
|
+
* **`lower > 0` ⟺ McNemar's exact test rejects at α = 1 − confidence.**
|
|
601
|
+
*
|
|
602
|
+
* Clopper-Pearson excludes π = 0.5 exactly when the two-sided exact binomial
|
|
603
|
+
* test of π = 0.5 rejects, and that test IS {@link mcnemar}'s p-value — so the
|
|
604
|
+
* interval and the test can never disagree, and a gate keyed on `lower` cannot
|
|
605
|
+
* promote what the exact test refuses. The exact p is returned in the same
|
|
606
|
+
* object so the two are impossible to compute apart.
|
|
607
|
+
*
|
|
608
|
+
* The interval is conservative (exact intervals over-cover; conditioning on m
|
|
609
|
+
* discards the concordant pairs' information about m itself). That is the
|
|
610
|
+
* correct direction for a promotion gate: it refuses more often, never less.
|
|
611
|
+
*
|
|
612
|
+
* With n = 0, returns the full [-1, 1] support, p = 1, and a compatibility
|
|
613
|
+
* point estimate of 0. With n > 0 and m = 0, π is not identified: the result
|
|
614
|
+
* is the degenerate [0, 0] with p = 1. That is NOT evidence of equivalence —
|
|
615
|
+
* callers must treat a zero-width interval as "cannot decide", not as "no
|
|
616
|
+
* difference". Inputs are paired 0/1 (or boolean) arrays, control first.
|
|
617
|
+
* Throws on unequal lengths.
|
|
618
|
+
*/
|
|
619
|
+
declare function pairedRiskDifferenceExact(control: ArrayLike<number | boolean>, treatment: ArrayLike<number | boolean>, confidence?: number): ExactRiskDifferenceResult;
|
|
620
|
+
/** A paired binary effect size with an interval that is valid at a NONZERO
|
|
621
|
+
* margin — the estimator a noninferiority decision may be made on. */
|
|
622
|
+
interface ScoreRiskDifferenceResult {
|
|
623
|
+
/** Total paired observations. */
|
|
624
|
+
n: number;
|
|
625
|
+
/** Discordant pairs: treatment-win count. */
|
|
626
|
+
b: number;
|
|
627
|
+
/** Discordant pairs: control-win count. */
|
|
628
|
+
c: number;
|
|
629
|
+
/** Discordant pairs (b + c). */
|
|
630
|
+
nDiscordant: number;
|
|
631
|
+
/** Paired risk difference (b − c) / n; 0 is a placeholder when n = 0. */
|
|
632
|
+
riskDifference: number;
|
|
633
|
+
/** Score-interval lower bound on the population risk difference. */
|
|
634
|
+
lower: number;
|
|
635
|
+
/** Score-interval upper bound on the population risk difference. */
|
|
636
|
+
upper: number;
|
|
637
|
+
/** Confidence level used. */
|
|
638
|
+
confidence: number;
|
|
639
|
+
}
|
|
640
|
+
/**
|
|
641
|
+
* Paired risk difference with TANGO'S (1998) SCORE INTERVAL — the estimator a
|
|
642
|
+
* promotion gate may decide on **at a nonzero margin**.
|
|
643
|
+
*
|
|
644
|
+
* {@link pairedRiskDifferenceExact} conditions on the observed discordant count
|
|
645
|
+
* `m = b + c`, builds a Clopper-Pearson interval for the win share among those
|
|
646
|
+
* `m` pairs, and multiplies by the observed `m/n`. That is exact for testing
|
|
647
|
+
* RD = 0 — it is dual to McNemar — but it is NOT a confidence interval for the
|
|
648
|
+
* population risk difference at a nonzero margin, because the sampling
|
|
649
|
+
* variability of `m/n` itself is discarded. The gap is not academic: with the
|
|
650
|
+
* production caller's `pairedDeltaThreshold: -0.05`, a process whose true risk
|
|
651
|
+
* difference sits exactly on that margin clears a nominal-95 % `lower > margin`
|
|
652
|
+
* check 24.75 % of the time at n = 40 and 43.95 % at n = 76 (2000 replicates
|
|
653
|
+
* each) when the conditional interval decides.
|
|
654
|
+
*
|
|
655
|
+
* Tango's interval inverts the score test of RD = delta, which estimates the
|
|
656
|
+
* nuisance loss rate under each hypothesised delta instead of fixing it at the
|
|
657
|
+
* observed value, so `m` contributes its own uncertainty. It is the method
|
|
658
|
+
* `ratesci::scorepairci` uses for paired risk-difference noninferiority, and it
|
|
659
|
+
* is not conditional, so it stays valid as the margin moves away from zero.
|
|
660
|
+
*
|
|
661
|
+
* The bounds are found by bisecting `tangoScore(delta) = ±z` — the score is
|
|
662
|
+
* monotone decreasing in delta, so each crossing is unique. Inputs are paired
|
|
663
|
+
* 0/1 (or boolean) arrays, control first. Throws on unequal lengths.
|
|
664
|
+
*/
|
|
665
|
+
declare function pairedRiskDifferenceScore(control: ArrayLike<number | boolean>, treatment: ArrayLike<number | boolean>, confidence?: number): ScoreRiskDifferenceResult;
|
|
666
|
+
/**
|
|
667
|
+
* The common positive level `s` such that EVERY value across both paired arms is
|
|
668
|
+
* exactly 0 or `s` — i.e. the outcome is pass/fail, whatever encoding it arrived
|
|
669
|
+
* in. Returns null when the outcomes are not two-point, when the two arms use
|
|
670
|
+
* different levels, or when no positive value was observed at all (all-zero
|
|
671
|
+
* arms: the level is not identified, and there is nothing to decide anyway).
|
|
672
|
+
*
|
|
673
|
+
* This is the scale-aware successor to {@link isBinaryOutcomeVector}, which only
|
|
674
|
+
* recognises literal {0, 1}. Judges in this codebase emit dimensions on 0-100 as
|
|
675
|
+
* well as [0,1] (see `detectScale` in `campaign/gates/statistical-heldout.ts`),
|
|
676
|
+
* so a pass/fail dimension routinely arrives as {0, 100} and a {0,1}-only test
|
|
677
|
+
* silently sends it down the median path that cannot see it. Any positive level
|
|
678
|
+
* is accepted, not just 1 and 100: for a two-point {0, s} outcome the mean paired
|
|
679
|
+
* delta is exactly s·(b − c)/n, so the binary estimators apply after dividing by
|
|
680
|
+
* s and rescaling the result back into the caller's native units.
|
|
681
|
+
*
|
|
682
|
+
* Non-finite values ⇒ null: an unusable outcome must not be classified as a
|
|
683
|
+
* clean pass/fail shape.
|
|
684
|
+
*/
|
|
685
|
+
declare function pairedBinaryScale(before: ArrayLike<number>, after: ArrayLike<number>): number | null;
|
|
686
|
+
/**
|
|
687
|
+
* Unbiased pass@k for code generation (Chen et al. 2021, "Evaluating Large
|
|
688
|
+
* Language Models Trained on Code"). Given `n` independent samples for one
|
|
689
|
+
* problem of which `c` pass, the probability that at least one of a random k of
|
|
690
|
+
* them passes is 1 − C(n−c, k) / C(n, k). Estimating pass@k as "did any of the
|
|
691
|
+
* first k pass" is biased high at small n; this is the variance-reduced estimator
|
|
692
|
+
* averaged implicitly over all k-subsets. Average the per-problem values across
|
|
693
|
+
* the suite for the corpus pass@k. Computed in the numerically stable product
|
|
694
|
+
* form. Requires 1 ≤ k ≤ n and 0 ≤ c ≤ n.
|
|
695
|
+
*/
|
|
696
|
+
declare function passAtK(n: number, c: number, k: number): number;
|
|
697
|
+
/**
|
|
698
|
+
* Paired significance tests on continuous scores: the paired t-test, the
|
|
699
|
+
* promotion-gate paired bootstrap, the exact sign test, and the package-wide
|
|
700
|
+
* paired-delta decision statistic.
|
|
701
|
+
*/
|
|
702
|
+
interface PairedTTestResult {
|
|
703
|
+
/** Null when the statistic is undefined — see {@link pairedTTest}. */
|
|
704
|
+
t: number | null;
|
|
705
|
+
df: number;
|
|
706
|
+
/** Null exactly when `t` is null. */
|
|
707
|
+
p: number | null;
|
|
708
|
+
}
|
|
709
|
+
/**
|
|
710
|
+
* Paired t-test — before/after measurements on the SAME items.
|
|
711
|
+
* Pairing removes inter-item variance, giving tighter significance than
|
|
712
|
+
* an unpaired test when comparing prompt v1 vs prompt v2 on identical
|
|
713
|
+
* scenarios.
|
|
714
|
+
*
|
|
715
|
+
* Returns `t = p = null` where the statistic is undefined: fewer than two
|
|
716
|
+
* pairs, or a non-zero constant delta whose observed variance is zero. A
|
|
717
|
+
* constant shift carries no information about the variance it would have to
|
|
718
|
+
* be compared against, so the honest answer is "undefined", not `p = 0` —
|
|
719
|
+
* three observations cannot buy absolute certainty. This is the same contract
|
|
720
|
+
* {@link pairedCohensDz} states for the same condition. An all-zero delta is
|
|
721
|
+
* different: it is a measured null, and returns `t = 0, p = 1`.
|
|
722
|
+
*/
|
|
723
|
+
declare function pairedTTest(before: number[], after: number[]): PairedTTestResult;
|
|
724
|
+
interface PairedBootstrapResult {
|
|
725
|
+
/** Number of paired observations. */
|
|
726
|
+
n: number;
|
|
727
|
+
/** Median of paired deltas (after − before). */
|
|
728
|
+
median: number;
|
|
729
|
+
/** Mean of paired deltas. */
|
|
730
|
+
mean: number;
|
|
731
|
+
/** Lower bound of the bootstrap CI on the chosen statistic. */
|
|
732
|
+
low: number;
|
|
733
|
+
/** Upper bound of the bootstrap CI on the chosen statistic. */
|
|
734
|
+
high: number;
|
|
735
|
+
/** Confidence level used (e.g. 0.95). */
|
|
736
|
+
confidence: number;
|
|
737
|
+
/** Number of bootstrap resamples used. */
|
|
738
|
+
resamples: number;
|
|
739
|
+
/** False below {@link BOOTSTRAP_GATE_MIN_N}. See {@link pairedBootstrap}. */
|
|
740
|
+
gateEligible: boolean;
|
|
741
|
+
}
|
|
742
|
+
/**
|
|
743
|
+
* Pairs below which a percentile bootstrap interval is descriptive spread only.
|
|
744
|
+
*
|
|
745
|
+
* `P(low > 0)` under a true null, against a nominal 2.5 %, measured over 4000
|
|
746
|
+
* seeded trials: 13.53 % at n = 3, 3.52 % at n = 10, 3.10 % at n = 20 on the
|
|
747
|
+
* median; 13.85 %, 4.90 %, 3.80 % on the mean. This is intrinsic to resampling
|
|
748
|
+
* three points, not an implementation error — scipy's BCa gives 16.0 % on the
|
|
749
|
+
* same n = 3 data — so no change to the estimator moves it. Below this floor
|
|
750
|
+
* the decision belongs to the exact sign test or exact signed-rank test.
|
|
751
|
+
*/
|
|
752
|
+
declare const BOOTSTRAP_GATE_MIN_N = 20;
|
|
753
|
+
interface PairedBootstrapOptions {
|
|
754
|
+
/** Confidence level. Default 0.95. */
|
|
755
|
+
confidence?: number;
|
|
756
|
+
/** Bootstrap resample count. Default 2000. */
|
|
757
|
+
resamples?: number;
|
|
758
|
+
/** Statistic to bootstrap. Default 'median'. */
|
|
759
|
+
statistic?: 'median' | 'mean';
|
|
760
|
+
/** Deterministic seed. If omitted, derived from the deltas so the interval
|
|
761
|
+
* is reproducible regardless. */
|
|
762
|
+
seed?: number;
|
|
763
|
+
}
|
|
764
|
+
/**
|
|
765
|
+
* Paired bootstrap on (after − before) deltas. Returns a CI on the chosen
|
|
766
|
+
* statistic (median by default); pairs are resampled with replacement. Throws
|
|
767
|
+
* on unequal sample sizes.
|
|
768
|
+
*
|
|
769
|
+
* `low > threshold` carries the stated confidence ONLY at `n ≥
|
|
770
|
+
* {@link BOOTSTRAP_GATE_MIN_N}`, which `gateEligible` reports. Below it the
|
|
771
|
+
* check fires under a true null several times more often than nominal, so the
|
|
772
|
+
* interval is descriptive spread and a promotion must not turn on it.
|
|
773
|
+
*/
|
|
774
|
+
declare function pairedBootstrap(before: number[], after: number[], opts?: PairedBootstrapOptions): PairedBootstrapResult;
|
|
775
|
+
/** Pre-registered direction for a one-sided paired sign test. */
|
|
776
|
+
type SignTestAlternative = 'greater' | 'less';
|
|
777
|
+
/** Exact one-sided sign-test result for paired numeric differences. */
|
|
778
|
+
interface PairedSignTestResult {
|
|
779
|
+
/** Total supplied differences, including zero ties. */
|
|
780
|
+
n: number;
|
|
781
|
+
/** Strictly positive differences. */
|
|
782
|
+
positive: number;
|
|
783
|
+
/** Strictly negative differences. */
|
|
784
|
+
negative: number;
|
|
785
|
+
/** Zero differences excluded from the binomial test. */
|
|
786
|
+
ties: number;
|
|
787
|
+
/** Non-zero differences used by the binomial test. */
|
|
788
|
+
nNonTies: number;
|
|
789
|
+
/** Direction of the pre-registered alternative hypothesis. */
|
|
790
|
+
alternative: SignTestAlternative;
|
|
791
|
+
/** Exact one-sided p-value under P(positive) = P(negative) = 0.5. */
|
|
792
|
+
pValue: number;
|
|
793
|
+
}
|
|
794
|
+
/**
|
|
795
|
+
* Exact one-sided sign test over paired differences.
|
|
796
|
+
*
|
|
797
|
+
* Pass `after[i] - before[i]` for each matched item. `alternative = 'greater'`
|
|
798
|
+
* tests whether positive signs are more likely than negative signs and returns
|
|
799
|
+
* `P(Binomial(nNonTies, 0.5) >= positive)`. `alternative = 'less'` treats
|
|
800
|
+
* negative signs as successes instead. With a continuous difference
|
|
801
|
+
* distribution this is the usual directional median test. Exact zero
|
|
802
|
+
* differences are ties and do not enter the binomial denominator. All-tie and
|
|
803
|
+
* empty inputs return p = 1. Every input difference must be finite, and the
|
|
804
|
+
* direction must be chosen explicitly so a caller cannot select it after
|
|
805
|
+
* seeing the signs.
|
|
806
|
+
*/
|
|
807
|
+
declare function pairedSignTest(differences: readonly number[], alternative: SignTestAlternative): PairedSignTestResult;
|
|
808
|
+
/** Fraction of paired observations whose delta is an exact tie (|after − before|
|
|
809
|
+
* < 1e-9). Throws on unequal sample sizes; 0 pairs ⇒ 0. */
|
|
810
|
+
declare function pairedDeltaTieFraction(before: ArrayLike<number>, after: ArrayLike<number>): number;
|
|
811
|
+
/**
|
|
812
|
+
* The paired-delta statistic a DECISION is computed on, package-wide.
|
|
813
|
+
*
|
|
814
|
+
* The mean paired delta is the estimator that answers the question a promotion
|
|
815
|
+
* gate asks — "by how much did the candidate move the score" — in the caller's
|
|
816
|
+
* own units, and it equals the aggregate lift everyone quotes. The MEDIAN
|
|
817
|
+
* answers a different question and loses the answer to this one in every regime
|
|
818
|
+
* eval data actually lands in:
|
|
819
|
+
* - TWO-POINT (pass/fail) outcomes on any encoding: the delta vector lives in
|
|
820
|
+
* {−s, 0, +s} dominated by zeros, so the median and its whole bootstrap CI
|
|
821
|
+
* are pinned at exactly 0 however large the shift. (Decide these on
|
|
822
|
+
* {@link pairedRiskDifferenceExact} instead — same estimand, exact interval.)
|
|
823
|
+
* - TIE-DOMINATED outcomes: at half the pairs tied the sample median is 0 by
|
|
824
|
+
* construction, and `ci.low > threshold` then answers "no" forever at a
|
|
825
|
+
* non-negative threshold and "yes" forever at a negative one.
|
|
826
|
+
* - LOW-CARDINALITY outcomes, even well below half ties: judge dimensions on
|
|
827
|
+
* integer 0-100, and block scores like {⅔, 1} from averaging pass/fail
|
|
828
|
+
* leaves, put the median on a coarse lattice whose bootstrap percentiles
|
|
829
|
+
* land on atoms. Measured: 26 blocks of 3 pass/fail leaves carrying a real
|
|
830
|
+
* +12.8pp lift, only 23% of pairs tied, gives a median CI of [0, 0.333] —
|
|
831
|
+
* lower bound exactly 0, so a gate at threshold 0 refuses a real lift.
|
|
832
|
+
* That last case is why there is no tie-fraction threshold here: any cutoff on
|
|
833
|
+
* ties leaves the lattice case open on the other side of it.
|
|
834
|
+
*
|
|
835
|
+
* `heldoutSignificance` has defaulted to the mean since #316 for the same
|
|
836
|
+
* reason. The median remains available per call site for callers who
|
|
837
|
+
* specifically want outlier robustness and accept the blindness.
|
|
838
|
+
*/
|
|
839
|
+
declare const DECISION_PAIRED_DELTA_STATISTIC: 'mean';
|
|
840
|
+
/**
|
|
841
|
+
* Required N per arm for a two-sample comparison at target effect size,
|
|
842
|
+
* alpha, and power. Normal-approximation formula:
|
|
843
|
+
* n = 2 * ( (z_{1-α/2} + z_{1-β}) / d )^2
|
|
844
|
+
* where d is Cohen's d. Returns Infinity for effect ≤ 0.
|
|
845
|
+
*/
|
|
846
|
+
declare function requiredSampleSize(opts: {
|
|
847
|
+
effect: number;
|
|
848
|
+
alpha?: number;
|
|
849
|
+
power?: number;
|
|
850
|
+
twoSided?: boolean;
|
|
851
|
+
}): number;
|
|
852
|
+
/**
|
|
853
|
+
* Required number of paired observations for a target Cohen's dz.
|
|
854
|
+
* Unlike the independent-groups formula, this has no two-arm factor of two.
|
|
855
|
+
*
|
|
856
|
+
* Normal quantiles with no t correction, so treat the result as a LOWER bound:
|
|
857
|
+
* it returns 32 where the exact t-based answer is 34 at dz = 0.5, and 13 where
|
|
858
|
+
* it is 15 at dz = 0.8 — a 6–13 % shortfall precisely in the range a caller
|
|
859
|
+
* consults to decide whether 3–10 repetitions suffice.
|
|
860
|
+
*/
|
|
861
|
+
declare function requiredPairedSampleSize(opts: {
|
|
862
|
+
effect: number;
|
|
863
|
+
alpha?: number;
|
|
864
|
+
power?: number;
|
|
865
|
+
twoSided?: boolean;
|
|
866
|
+
}): number;
|
|
867
|
+
/**
|
|
868
|
+
* Minimum detectable paired effect (standardised units) for a target paired
|
|
869
|
+
* sample size: d_min = (z_{1-α/2} + z_β) / sqrt(n_paired). Multiply by
|
|
870
|
+
* sd(deltas) for score units; treat as a lower bound — Wilcoxon and bootstrap
|
|
871
|
+
* have asymptotic relative efficiency below 1 vs the t-test on heavy tails.
|
|
872
|
+
*/
|
|
873
|
+
declare function pairedMde(opts: {
|
|
874
|
+
nPaired: number;
|
|
875
|
+
alpha?: number;
|
|
876
|
+
power?: number;
|
|
877
|
+
twoSided?: boolean;
|
|
878
|
+
}): number;
|
|
879
|
+
/**
|
|
880
|
+
* Number of paired observations needed for a McNemar test to reach a target
|
|
881
|
+
* power — the pre-registration companion to {@link mcnemar}. Parametrised by the
|
|
882
|
+
* expected discordant-cell probabilities `p10` (P[treatment wins on a pair]) and
|
|
883
|
+
* `p01` (P[control wins]); concordant pairs carry no information, so the count
|
|
884
|
+
* is driven entirely by the discordant rate. Lachin's (1992) asymptotic normal
|
|
885
|
+
* approximation: with discordant rate `pDisc = p10 + p01` and marginal effect
|
|
886
|
+
* `δ = p10 − p01`,
|
|
887
|
+
* n = ( z_{1-α/2}·√pDisc + z_{1-β}·√(pDisc − δ²) )² / δ².
|
|
888
|
+
* Returns Infinity when there is no effect (p10 === p01). Asymptotic — at the
|
|
889
|
+
* tiny discordant counts where the exact {@link mcnemar} differs from the normal
|
|
890
|
+
* approximation, treat the result as a lower bound and prefer the discordant-pair
|
|
891
|
+
* floor.
|
|
892
|
+
*/
|
|
893
|
+
declare function mcnemarRequiredN(opts: {
|
|
894
|
+
p10: number;
|
|
895
|
+
p01: number;
|
|
896
|
+
alpha?: number;
|
|
897
|
+
power?: number;
|
|
898
|
+
twoSided?: boolean;
|
|
899
|
+
}): number;
|
|
900
|
+
/**
|
|
901
|
+
* Power of a McNemar test at a given number of paired observations, the inverse
|
|
902
|
+
* of {@link mcnemarRequiredN} (same Lachin asymptotic model, same parameters).
|
|
903
|
+
* Returns a value in [0, 1]; equals `alpha` when there is no effect.
|
|
904
|
+
*/
|
|
905
|
+
declare function mcnemarPower(opts: {
|
|
906
|
+
p10: number;
|
|
907
|
+
p01: number;
|
|
908
|
+
nPairs: number;
|
|
909
|
+
alpha?: number;
|
|
910
|
+
twoSided?: boolean;
|
|
911
|
+
}): number;
|
|
912
|
+
/** Tiny seedable PRNG (mulberry32) — deterministic resampling/shuffling, not
|
|
913
|
+
* cryptographic. Exported so e-process shuffles and bootstrap resampling
|
|
914
|
+
* share ONE PRNG implementation. Every distinct 32-bit seed gives a distinct
|
|
915
|
+
* stream, including 0. */
|
|
916
|
+
declare function mulberry32(seed: number): () => number;
|
|
917
|
+
type RankTestMethod = 'exact' | 'permutation' | 'asymptotic';
|
|
918
|
+
/** Auto selects bounded exact enumeration or seeded permutation; asymptotic requires an explicit request. */
|
|
919
|
+
type RankTestMethodRequest = 'auto' | 'exact' | 'asymptotic';
|
|
920
|
+
interface RankTestOptions {
|
|
921
|
+
/** Default `'auto'`. `'asymptotic'` inside the exact-feasible range throws. */
|
|
922
|
+
method?: RankTestMethodRequest;
|
|
923
|
+
/** Resamples on the Monte Carlo permutation path. Default 100000. */
|
|
924
|
+
permutations?: number;
|
|
925
|
+
/** Seed for the permutation path. Omitted ⇒ derived from the data itself, so
|
|
926
|
+
* the result is reproducible either way. */
|
|
927
|
+
seed?: number;
|
|
928
|
+
}
|
|
929
|
+
interface MannWhitneyResult {
|
|
930
|
+
/** `min(U_a, U_b)` — the conventional reported statistic. */
|
|
931
|
+
u: number;
|
|
932
|
+
/** U for sample `a`. Carries the direction of the effect, which `u` discards. */
|
|
933
|
+
uA: number;
|
|
934
|
+
/** Two-sided p-value. */
|
|
935
|
+
p: number;
|
|
936
|
+
/** How `p` was computed. */
|
|
937
|
+
method: RankTestMethod;
|
|
938
|
+
/** Smallest two-sided p this design can produce. `p` can never be below it. */
|
|
939
|
+
pFloor: number;
|
|
940
|
+
}
|
|
941
|
+
/**
|
|
942
|
+
* Mann-Whitney U — two independent samples, no distributional assumption.
|
|
943
|
+
*
|
|
944
|
+
* Exact conditional (permutation) p by default when the dynamic program fits
|
|
945
|
+
* {@link MANN_WHITNEY_EXACT_MAX_STATES} cells and
|
|
946
|
+
* {@link MANN_WHITNEY_EXACT_MAX_WORK} transitions, seeded Monte Carlo
|
|
947
|
+
* permutation above those limits. This keeps imbalanced designs such as 1+24
|
|
948
|
+
* exact without admitting expensive balanced designs merely because they have
|
|
949
|
+
* the same total size. Throws on non-finite input and on `method:
|
|
950
|
+
* 'asymptotic'` where an exact answer is available. Empty input yields `p = 1,
|
|
951
|
+
* pFloor = 1` — no design, no attainable evidence.
|
|
952
|
+
*/
|
|
953
|
+
declare function mannWhitneyU(a: number[], b: number[], opts?: RankTestOptions): MannWhitneyResult;
|
|
954
|
+
interface WilcoxonSignedRankResult {
|
|
955
|
+
/** W⁺, the rank sum of the positive differences. (scipy reports
|
|
956
|
+
* `min(W⁺, W⁻)`; compare statistics only after converting.) */
|
|
957
|
+
w: number;
|
|
958
|
+
/** Two-sided p-value. */
|
|
959
|
+
p: number;
|
|
960
|
+
/** How `p` was computed. */
|
|
961
|
+
method: RankTestMethod;
|
|
962
|
+
/** Smallest two-sided p this design can produce. */
|
|
963
|
+
pFloor: number;
|
|
964
|
+
/** Non-zero differences — zero differences are dropped and carry no rank. */
|
|
965
|
+
nNonZero: number;
|
|
966
|
+
}
|
|
967
|
+
/**
|
|
968
|
+
* Wilcoxon signed-rank — paired, no distributional assumption on the deltas.
|
|
969
|
+
*
|
|
970
|
+
* Exact conditional (sign-flip) p by default at `n ≤
|
|
971
|
+
* {@link WILCOXON_EXACT_MAX_N}` non-zero differences, seeded Monte Carlo
|
|
972
|
+
* permutation above it. Throws on non-finite input and on `method:
|
|
973
|
+
* 'asymptotic'` where an exact answer is available.
|
|
974
|
+
*
|
|
975
|
+
* `n` is the count of NON-ZERO differences: exact ties are dropped before
|
|
976
|
+
* ranking, so a run of tied pairs shrinks the design and raises `pFloor`.
|
|
977
|
+
* All-tied input yields `p = 1, pFloor = 1` — no attainable evidence, which
|
|
978
|
+
* `pFloor` states rather than leaving `p = 1` to be read as a measured null.
|
|
979
|
+
*/
|
|
980
|
+
declare function wilcoxonSignedRank(before: number[], after: number[], opts?: RankTestOptions): WilcoxonSignedRankResult;
|
|
981
|
+
interface SampleSummaryOptions {
|
|
982
|
+
confidence?: number;
|
|
983
|
+
seed?: number;
|
|
984
|
+
resamples?: number;
|
|
985
|
+
}
|
|
986
|
+
interface SampleSummary {
|
|
987
|
+
n: number;
|
|
988
|
+
median: number;
|
|
989
|
+
mean: number;
|
|
990
|
+
stddev: number;
|
|
991
|
+
q1: number;
|
|
992
|
+
q3: number;
|
|
993
|
+
iqr: number;
|
|
994
|
+
min: number;
|
|
995
|
+
max: number;
|
|
996
|
+
range: number;
|
|
997
|
+
cv: number | null;
|
|
998
|
+
skewness: number | null;
|
|
999
|
+
kurtosis: number | null;
|
|
1000
|
+
mad: number;
|
|
1001
|
+
outliers: number[];
|
|
1002
|
+
medianInterval: {
|
|
1003
|
+
low: number;
|
|
1004
|
+
high: number;
|
|
1005
|
+
confidence: number;
|
|
1006
|
+
resamples: number;
|
|
1007
|
+
} | null;
|
|
1008
|
+
}
|
|
1009
|
+
/** Interpolated sample median; an empty population has no measured median. */
|
|
1010
|
+
declare function sampleMedian(values: readonly number[]): number | null;
|
|
1011
|
+
/** Descriptive facts, not stability policy or a significance test. */
|
|
1012
|
+
declare function summarizeSample(values: readonly number[], options?: SampleSummaryOptions): SampleSummary | null;
|
|
1013
|
+
interface EProcessOptions {
|
|
1014
|
+
/** Type-I error budget. The process decides when wealth ≥ 1/alpha
|
|
1015
|
+
* (Ville's inequality). Default 0.05. */
|
|
1016
|
+
alpha?: number;
|
|
1017
|
+
/** Truncation bound on the predictable bet λ ∈ [0, maxBet]. Must satisfy
|
|
1018
|
+
* maxBet < 1/nullMean so every wealth factor stays strictly positive.
|
|
1019
|
+
* Default 0.5. */
|
|
1020
|
+
maxBet?: number;
|
|
1021
|
+
/** The null boundary m₀ for H0: E[x] ≤ m₀ on x ∈ [0,1]. Default 0.5
|
|
1022
|
+
* (the paired-delta encoding x = (d+1)/2 maps "no effect" to 1/2).
|
|
1023
|
+
* A pre-registered minEffect shifts this — see `sequentialPairedGate`. */
|
|
1024
|
+
nullMean?: number;
|
|
1025
|
+
/** Continue a process from a `state()` snapshot taken before a restart.
|
|
1026
|
+
* The snapshot never supplies the parameters: `alpha`, `maxBet`, and
|
|
1027
|
+
* `nullMean` resolve from this options object exactly as for a fresh
|
|
1028
|
+
* process, and a snapshot recorded under different parameters is refused
|
|
1029
|
+
* (`ValidationError`) — re-deciding the same stream under new parameters
|
|
1030
|
+
* would reopen optional stopping. A snapshot whose fields are not
|
|
1031
|
+
* mutually consistent (tampered or truncated) is refused the same way. */
|
|
1032
|
+
resume?: EProcessState;
|
|
1033
|
+
}
|
|
1034
|
+
interface EProcessStep {
|
|
1035
|
+
/** Current wealth W_n — the e-value against H0 after n observations. */
|
|
1036
|
+
wealth: number;
|
|
1037
|
+
/** Observations consumed so far. */
|
|
1038
|
+
n: number;
|
|
1039
|
+
/** True from the first n where W_n ≥ 1/alpha onward (sticky). */
|
|
1040
|
+
decided: boolean;
|
|
1041
|
+
}
|
|
1042
|
+
/**
|
|
1043
|
+
* Complete snapshot of an e-process. Together with the parameters it is
|
|
1044
|
+
* sufficient to continue the process after a restart: `sumX` and `varSum`
|
|
1045
|
+
* are the running sums the next bet is computed from, so a process rebuilt
|
|
1046
|
+
* from a snapshot produces the same wealth sequence and decision as one that
|
|
1047
|
+
* was never interrupted. Plain data: a JSON round-trip preserves it
|
|
1048
|
+
* (`decidedAtN` is undefined, and therefore omitted, until decided).
|
|
1049
|
+
*/
|
|
1050
|
+
interface EProcessState extends EProcessStep {
|
|
1051
|
+
alpha: number;
|
|
1052
|
+
maxBet: number;
|
|
1053
|
+
nullMean: number;
|
|
1054
|
+
/** The decision boundary 1/alpha. */
|
|
1055
|
+
threshold: number;
|
|
1056
|
+
/** Observation count at the first threshold crossing; undefined until decided. */
|
|
1057
|
+
decidedAtN?: number;
|
|
1058
|
+
/** Σ x_i over the n observations consumed. */
|
|
1059
|
+
sumX: number;
|
|
1060
|
+
/** Σ (x_i − μ̂_i)² over the n observations consumed, μ̂_i the shrunk running
|
|
1061
|
+
* mean after observation i. */
|
|
1062
|
+
varSum: number;
|
|
1063
|
+
}
|
|
1064
|
+
interface EProcess {
|
|
1065
|
+
/** Consume one observation x ∈ [0,1]. Throws on non-finite / out-of-range
|
|
1066
|
+
* input — a silent clamp would corrupt the type-I guarantee. */
|
|
1067
|
+
update(x: number): EProcessStep;
|
|
1068
|
+
state(): EProcessState;
|
|
1069
|
+
}
|
|
1070
|
+
/**
|
|
1071
|
+
* Betting test-martingale for bounded observations — the e-process core of
|
|
1072
|
+
* anytime-valid sequential testing (Waudby-Smith & Ramdas, "Estimating means
|
|
1073
|
+
* of bounded random variables by betting", JRSS-B 2024).
|
|
1074
|
+
*
|
|
1075
|
+
* Observations x_i ∈ [0,1]; H0: E[x] ≤ m₀ (`nullMean`, default 1/2). Wealth
|
|
1076
|
+
*
|
|
1077
|
+
* W_t = Π_{i≤t} (1 + λ_i (x_i − m₀)), W_0 = 1
|
|
1078
|
+
*
|
|
1079
|
+
* with the truncated GROW-style plug-in bet computed from PRIOR observations:
|
|
1080
|
+
*
|
|
1081
|
+
* λ_i = clamp((μ̂_{i−1} − m₀) / (σ̂²_{i−1} + (μ̂_{i−1} − m₀)²), 0, maxBet)
|
|
1082
|
+
*
|
|
1083
|
+
* where μ̂/σ̂² are the shrunk running estimates μ̂_t = (1/2 + Σx_i)/(t+1),
|
|
1084
|
+
* σ̂²_t = (1/4 + Σ(x_i − μ̂_i)²)/(t+1).
|
|
1085
|
+
*
|
|
1086
|
+
* PREDICTABILITY INVARIANT (load-bearing): λ_i is a function of x_1..x_{i−1}
|
|
1087
|
+
* ONLY — it may never see x_i. With λ_i ≥ 0 predictable, each factor has
|
|
1088
|
+
* E[1 + λ_i(x_i − m₀) | past] ≤ 1 under H0, so W is a nonnegative
|
|
1089
|
+
* supermartingale and Ville's inequality gives P(∃t: W_t ≥ 1/α) ≤ α — the
|
|
1090
|
+
* type-I guarantee holds at ANY data-dependent stopping time. λ_1 is always 0
|
|
1091
|
+
* (no prior evidence), so the first observation never moves wealth.
|
|
1092
|
+
*
|
|
1093
|
+
* `decided` latches at the first crossing W_t ≥ 1/α and never un-latches;
|
|
1094
|
+
* wealth keeps updating after the crossing (the e-process remains valid), but
|
|
1095
|
+
* the decision time is the first crossing.
|
|
1096
|
+
*/
|
|
1097
|
+
declare function eProcess(opts?: EProcessOptions): EProcess;
|
|
1098
|
+
export { linearRegressionSlope as $, pairedBootstrap as A, CorpusAgreementReport as At, pairedBinaryScale as B, ContinuousAgreement as Bt, BOOTSTRAP_GATE_MIN_N as C, studentTCritical as Ct, PairedSignTestResult as D, rocAuc as Dt, PairedBootstrapResult as E, weightedMean as Et, McNemarResult as F, corpusInterRaterAgreementFromJudgeScores as Ft, wilson as G, SelfPreferenceResult as Gt, pairedRiskDifferenceExact as H, ContinuousCalibrationResult as Ht, ProportionInterval as I, interRaterReliability as It, holm as J, calibrateJudgeContinuous as Jt, benjaminiHochberg as K, VerbosityBiasResult as Kt, RiskDifferenceResult as L, nominalAgreement as Lt, pairedSignTest as M, NominalAgreement as Mt, pairedTTest as N, NominalAgreementOptions as Nt, PairedTTestResult as O, CorpusAgreementOptions as Ot, ExactRiskDifferenceResult as P, corpusInterRaterAgreement as Pt, fisherExactTwoSided as Q, verbosityBias as Qt, ScoreRiskDifferenceResult as R, CalibrationResult as Rt, requiredSampleSize as S, spearmanR as St, PairedBootstrapOptions as T, weightedComposite as Tt, pairedRiskDifferenceScore as U, GoldenItem as Ut, pairedRiskDifference as V, ContinuousAgreementOptions as Vt, passAtK as W, PositionalBiasResult as Wt, chiSquared as X, positionalBias as Xt, LinearRegressionResult as Y, continuousAgreement as Yt, chiSquaredPValue as Z, selfPreference as Zt, mulberry32 as _, confidenceInterval as _t, eProcess as a, cohensD as at, pairedMde as b, pearsonR as bt, sampleMedian as c, DistributionSummaryOptions as ct, RankTestMethod as d, summarizeDistribution as dt, normalTwoSidedPValue as et, RankTestMethodRequest as f, MeanConfidenceInterval as ft, wilcoxonSignedRank as g, WeightedCompositeResult as gt, mannWhitneyU as h, WeightedCompositeInput as ht, EProcessStep as i, cliffsDelta as it, pairedDeltaTieFraction as j, CorpusScoreRecord as jt, SignTestAlternative as k, CorpusAgreementPerDimension as kt, summarizeSample as l, SampleDistribution as lt, WilcoxonSignedRankResult as m, SeriesDistributionOptions as mt, EProcessOptions as n, twoByTwoPValue as nt, SampleSummary as o, interpretCliffs as ot, RankTestOptions as p, SeriesDistribution as pt, bonferroni as q, calibrateJudge as qt, EProcessState as r, CliffsMagnitude as rt, SampleSummaryOptions as s, pairedCohensDz as st, EProcess as t, studentTTwoSidedPValue as tt, MannWhitneyResult as u, percentileNearestRank as ut, mcnemarPower as v, meanConfidenceInterval as vt, DECISION_PAIRED_DELTA_STATISTIC as w, summarizeNumberSeries as wt, requiredPairedSampleSize as x, ranks as xt, mcnemarRequiredN as y, partialCredit as yt, mcnemar as z, CandidateScore as zt };
|
|
1099
|
+
//# sourceMappingURL=index-BdLmBInV.d.ts.map
|