@tangle-network/agent-eval 0.216.0 → 0.216.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -0
- package/dist/adapters/http.d.ts +3 -3
- package/dist/{agent-profile-CIA2Atnt.d.ts → agent-profile-VFcQncbW.d.ts} +5 -5
- package/dist/{agent-profile-CIA2Atnt.d.ts.map → agent-profile-VFcQncbW.d.ts.map} +1 -1
- package/dist/{agent-profile-cell-BYc0bxsL.js → agent-profile-cell-BK8mlqlq.js} +3 -3
- package/dist/{agent-profile-cell-BYc0bxsL.js.map → agent-profile-cell-BK8mlqlq.js.map} +1 -1
- package/dist/{agreement-irr-DsTURxdr.js → agreement-irr-BEWLIF3_.js} +3 -3
- package/dist/{agreement-irr-DsTURxdr.js.map → agreement-irr-BEWLIF3_.js.map} +1 -1
- package/dist/analyst/index.d.ts +14 -14
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +15 -15
- package/dist/{analyze-runs-3xpby3p2.js → analyze-runs-C84irm4i.js} +608 -604
- package/dist/analyze-runs-C84irm4i.js.map +1 -0
- package/dist/{analyze-runs-CSDAh8A7.d.ts → analyze-runs-D68fd-su.d.ts} +17 -15
- package/dist/analyze-runs-D68fd-su.d.ts.map +1 -0
- package/dist/{baseline-Dm8966du.js → baseline-Udi_PLuT.js} +5 -5
- package/dist/{baseline-Dm8966du.js.map → baseline-Udi_PLuT.js.map} +1 -1
- package/dist/{benchmark-BywPeAUo.js → benchmark-CXmVVHH2.js} +2 -2
- package/dist/{benchmark-BywPeAUo.js.map → benchmark-CXmVVHH2.js.map} +1 -1
- package/dist/{benchmark-command-CByl9dxG.js → benchmark-command-By0vh8q7.js} +62 -45
- package/dist/benchmark-command-By0vh8q7.js.map +1 -0
- package/dist/benchmarks/index.d.ts +5 -5
- package/dist/benchmarks/index.js +3 -3
- package/dist/builder-eval/index.js +2 -2
- package/dist/campaign/index.d.ts +7 -7
- package/dist/campaign/index.js +13 -13
- package/dist/{campaign-BQ-Jujuj.js → campaign-BudZ70zo.js} +245 -247
- package/dist/campaign-BudZ70zo.js.map +1 -0
- package/dist/{campaign-evidence-OcUciD-N.js → campaign-evidence-Clrn4hBV.js} +8 -8
- package/dist/campaign-evidence-Clrn4hBV.js.map +1 -0
- package/dist/{canonical-BsZVgBhW.js → canonical-B2RqLofv.js} +2 -2
- package/dist/{canonical-BsZVgBhW.js.map → canonical-B2RqLofv.js.map} +1 -1
- package/dist/{chat-client-DthPOYGd.js → chat-client-l-MVDlDk.js} +2 -2
- package/dist/{chat-client-DthPOYGd.js.map → chat-client-l-MVDlDk.js.map} +1 -1
- package/dist/{chat-json-call-BrKsyspg.js → chat-json-call-puMArmtW.js} +2 -2
- package/dist/{chat-json-call-BrKsyspg.js.map → chat-json-call-puMArmtW.js.map} +1 -1
- package/dist/{claim-BXcZw-jo.js → claim-q4HQhK1G.js} +4 -4
- package/dist/claim-q4HQhK1G.js.map +1 -0
- package/dist/cli.js +9 -9
- package/dist/cli.js.map +1 -1
- package/dist/{cluster-bootstrap-B3JywQQn.js → cluster-bootstrap-qcUWYFl-.js} +3 -3
- package/dist/{cluster-bootstrap-B3JywQQn.js.map → cluster-bootstrap-qcUWYFl-.js.map} +1 -1
- package/dist/contract/index.d.ts +11 -11
- package/dist/contract/index.js +1022 -1078
- package/dist/contract/index.js.map +1 -1
- package/dist/{cost-ledger-D2FnJPBF.js → cost-ledger-BP8V4kUQ.js} +835 -816
- package/dist/cost-ledger-BP8V4kUQ.js.map +1 -0
- package/dist/{cost-ledger-CO7UyEcQ.d.ts → cost-ledger-Jk62OBfF.d.ts} +12 -47
- package/dist/cost-ledger-Jk62OBfF.d.ts.map +1 -0
- package/dist/{counterfactual-DtMWvn7D.js → counterfactual-Dn3veueY.js} +2 -2
- package/dist/{counterfactual-DtMWvn7D.js.map → counterfactual-Dn3veueY.js.map} +1 -1
- package/dist/{default-registry-Cn1LQ2Ve.js → default-registry-COhk9g36.js} +5 -5
- package/dist/{default-registry-Cn1LQ2Ve.js.map → default-registry-COhk9g36.js.map} +1 -1
- package/dist/{default-registry-BrWh62zf.d.ts → default-registry-DzHiGruW.d.ts} +6 -6
- package/dist/{default-registry-BrWh62zf.d.ts.map → default-registry-DzHiGruW.d.ts.map} +1 -1
- package/dist/{define-agent-eval-Ch3MWX6k.js → define-agent-eval-CNTPzIkp.js} +10 -10
- package/dist/define-agent-eval-CNTPzIkp.js.map +1 -0
- package/dist/{define-agent-eval-CEWkX5wZ.d.ts → define-agent-eval-WSck0CZJ.d.ts} +8 -8
- package/dist/{define-agent-eval-CEWkX5wZ.d.ts.map → define-agent-eval-WSck0CZJ.d.ts.map} +1 -1
- package/dist/{descriptive-CnU8Yq29.js → descriptive-DboxPZrG.js} +2 -2
- package/dist/{descriptive-CnU8Yq29.js.map → descriptive-DboxPZrG.js.map} +1 -1
- package/dist/diagnosis/index.d.ts +3 -3
- package/dist/diagnosis/index.js +4 -4
- package/dist/{dspy-rlm-engine-D-KX_e64.js → dspy-rlm-engine-LRCu61m5.js} +3 -3
- package/dist/{dspy-rlm-engine-D-KX_e64.js.map → dspy-rlm-engine-LRCu61m5.js.map} +1 -1
- package/dist/{effect-sizes-DiH8MGOH.js → effect-sizes-BhzEpXMc.js} +2 -2
- package/dist/{effect-sizes-DiH8MGOH.js.map → effect-sizes-BhzEpXMc.js.map} +1 -1
- package/dist/{engine-C_zNWfM8.d.ts → engine-3A_4jOlI.d.ts} +5 -5
- package/dist/{engine-C_zNWfM8.d.ts.map → engine-3A_4jOlI.d.ts.map} +1 -1
- package/dist/{equal-terms-Blx9Zk5I.js → equal-terms-BtdXayCJ.js} +2 -2
- package/dist/{equal-terms-Blx9Zk5I.js.map → equal-terms-BtdXayCJ.js.map} +1 -1
- package/dist/{errors-Dngq5h35.js → errors-DVZiqS7K.js} +6 -2
- package/dist/{errors-Dngq5h35.js.map → errors-DVZiqS7K.js.map} +1 -1
- package/dist/{eval-campaign-4byTtqrf.js → eval-campaign-90jeaCAI.js} +6 -6
- package/dist/{eval-campaign-4byTtqrf.js.map → eval-campaign-90jeaCAI.js.map} +1 -1
- package/dist/{evaluation-Cn2dRZIz.d.ts → evaluation-6Yoi74gM.d.ts} +4 -4
- package/dist/{evaluation-Cn2dRZIz.d.ts.map → evaluation-6Yoi74gM.d.ts.map} +1 -1
- package/dist/evaluation.d.ts +1 -1
- package/dist/evaluation.js +1 -1
- package/dist/{exact-types-CD0lf5Qd.d.ts → exact-types-ClugMJfI.d.ts} +2 -2
- package/dist/{exact-types-CD0lf5Qd.d.ts.map → exact-types-ClugMJfI.d.ts.map} +1 -1
- package/dist/experiment/index.d.ts +404 -3
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +313 -19
- package/dist/experiment/index.js.map +1 -1
- package/dist/{external-optimizer-process-nnsY5D_4.js → external-optimizer-process-BjMGUuds.js} +3 -3
- package/dist/{external-optimizer-process-nnsY5D_4.js.map → external-optimizer-process-BjMGUuds.js.map} +1 -1
- package/dist/{external-optimizer-subprocess-DcerlZSG.js → external-optimizer-subprocess-BqW0hHcI.js} +548 -548
- package/dist/external-optimizer-subprocess-BqW0hHcI.js.map +1 -0
- package/dist/{feedback-trajectory-DkaDj0dj.d.ts → feedback-trajectory-C4EG0nNH.d.ts} +5 -8
- package/dist/feedback-trajectory-C4EG0nNH.d.ts.map +1 -0
- package/dist/fuzz.d.ts +1 -1
- package/dist/fuzz.js +3 -3
- package/dist/hosted/index.js +1 -1
- package/dist/{index-CPJJr98o.d.ts → index-BFRO6chs.d.ts} +3 -2
- package/dist/index-BFRO6chs.d.ts.map +1 -0
- package/dist/{index-Ch9VxCpK.d.ts → index-BO41eGj5.d.ts} +2 -2
- package/dist/{index-Ch9VxCpK.d.ts.map → index-BO41eGj5.d.ts.map} +1 -1
- package/dist/{index-NvGBiO8H.d.ts → index-CyszSg4O.d.ts} +165 -162
- package/dist/index-CyszSg4O.d.ts.map +1 -0
- package/dist/{index-DlH0Y9gI.d.ts → index-DPcngtwt.d.ts} +30 -17
- package/dist/index-DPcngtwt.d.ts.map +1 -0
- package/dist/{index-oQ4aHziK.d.ts → index-DV-jSzxF.d.ts} +8 -8
- package/dist/{index-oQ4aHziK.d.ts.map → index-DV-jSzxF.d.ts.map} +1 -1
- package/dist/index.d.ts +1969 -1871
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +161 -201
- package/dist/index.js.map +1 -1
- package/dist/{integrity-BvnVARVl.js → integrity-BXOsAb3i.js} +2 -2
- package/dist/{integrity-BvnVARVl.js.map → integrity-BXOsAb3i.js.map} +1 -1
- package/dist/{integrity-CCAxGVK6.d.ts → integrity-DAMEDFaC.d.ts} +2 -2
- package/dist/{integrity-CCAxGVK6.d.ts.map → integrity-DAMEDFaC.d.ts.map} +1 -1
- package/dist/{integrity-BV1ZDf42.js → integrity-dLiDe2kD.js} +2 -5
- package/dist/integrity-dLiDe2kD.js.map +1 -0
- package/dist/{internal-CRHwS9iz.js → internal-Epl-YBZW.js} +3 -3
- package/dist/{internal-CRHwS9iz.js.map → internal-Epl-YBZW.js.map} +1 -1
- package/dist/interval-CELpye1n.js +219 -0
- package/dist/interval-CELpye1n.js.map +1 -0
- package/dist/interval-ohuOJurX.d.ts +47 -0
- package/dist/interval-ohuOJurX.d.ts.map +1 -0
- package/dist/{journal-DvKCvjty.js → journal-DfRz9kps.js} +2 -2
- package/dist/{journal-DvKCvjty.js.map → journal-DfRz9kps.js.map} +1 -1
- package/dist/{judge-calibration-N44V04Se.js → judge-calibration-BpCy5XQl.js} +3 -3
- package/dist/{judge-calibration-N44V04Se.js.map → judge-calibration-BpCy5XQl.js.map} +1 -1
- package/dist/{judge-gate-DNcUrRFA.d.ts → judge-gate-CE7CQAgt.d.ts} +3 -3
- package/dist/{judge-gate-DNcUrRFA.d.ts.map → judge-gate-CE7CQAgt.d.ts.map} +1 -1
- package/dist/{judge-gate-B1qYnkpi.js → judge-gate-iY9s4L44.js} +3 -3
- package/dist/judge-gate-iY9s4L44.js.map +1 -0
- package/dist/{kind-factory-CbZ-6tvu.js → kind-factory-BobI5Jy3.js} +4 -4
- package/dist/{kind-factory-CbZ-6tvu.js.map → kind-factory-BobI5Jy3.js.map} +1 -1
- package/dist/ledger-core/index.js +3 -3
- package/dist/{ledger-core-B2hahzrt.js → ledger-core-Gq7yp8lP.js} +4 -4
- package/dist/{ledger-core-B2hahzrt.js.map → ledger-core-Gq7yp8lP.js.map} +1 -1
- package/dist/{llm-client-DQveoxSM.js → llm-client-CyBAoYiX.js} +192 -219
- package/dist/llm-client-CyBAoYiX.js.map +1 -0
- package/dist/{llm-judge-PXV7Xljl.js → llm-judge-CG12SQJq.js} +499 -921
- package/dist/llm-judge-CG12SQJq.js.map +1 -0
- package/dist/matrix/index.d.ts +2 -2
- package/dist/matrix/index.js +1 -1
- package/dist/{matrix-CHMiebFx.d.ts → matrix-DSrSxjjy.d.ts} +4 -4
- package/dist/{matrix-CHMiebFx.d.ts.map → matrix-DSrSxjjy.d.ts.map} +1 -1
- package/dist/{matrix-_BV9-Lk5.js → matrix-Dm0BQW7U.js} +7 -11
- package/dist/matrix-Dm0BQW7U.js.map +1 -0
- package/dist/{paired-tests-jbwPdSz5.js → mean-DHoMiudD.js} +11 -5
- package/dist/mean-DHoMiudD.js.map +1 -0
- package/dist/meta-eval/index.d.ts +3 -3
- package/dist/meta-eval/index.js +9 -9
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/mint-DNU8QPG4.d.ts +81 -0
- package/dist/mint-DNU8QPG4.d.ts.map +1 -0
- package/dist/{mint-Csor1BLJ.js → mint-QaD5cA0u.js} +3 -3
- package/dist/{mint-Csor1BLJ.js.map → mint-QaD5cA0u.js.map} +1 -1
- package/dist/{multiplicity-D9cjmStq.js → multiplicity-DQUFcbW4.js} +2 -2
- package/dist/{multiplicity-D9cjmStq.js.map → multiplicity-DQUFcbW4.js.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/golden/index.d.ts.map +1 -1
- package/dist/multishot/golden/index.js +3 -6
- package/dist/multishot/golden/index.js.map +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/multishot/index.js +1 -1
- package/dist/nonnegative-number-DvdMxg_f.js +9 -0
- package/dist/nonnegative-number-DvdMxg_f.js.map +1 -0
- package/dist/openapi.json +1 -1
- package/dist/{otel-spans-C8YIJWy1.js → otel-spans-B9tUXRHF.js} +3 -3
- package/dist/{otel-spans-C8YIJWy1.js.map → otel-spans-B9tUXRHF.js.map} +1 -1
- package/dist/{paired-arms-DzKn0TfY.js → paired-arms-CwMGJsrS.js} +5 -5
- package/dist/{paired-arms-DzKn0TfY.js.map → paired-arms-CwMGJsrS.js.map} +1 -1
- package/dist/{paired-binary-Q36bhn2R.js → paired-binary-CvpFHfKZ.js} +2 -2
- package/dist/{paired-binary-Q36bhn2R.js.map → paired-binary-CvpFHfKZ.js.map} +1 -1
- package/dist/{paired-promotion-decision-BN6IQTiu.js → paired-promotion-decision-DsAPYGCm.js} +3 -3
- package/dist/{paired-promotion-decision-BN6IQTiu.js.map → paired-promotion-decision-DsAPYGCm.js.map} +1 -1
- package/dist/{search-kernel-BztlERLv.js → pareto-Cm6ndrWN.js} +883 -861
- package/dist/pareto-Cm6ndrWN.js.map +1 -0
- package/dist/pipelines/index.d.ts +2 -2
- package/dist/pipelines/index.js +3 -3
- package/dist/{power-and-mde-fK5Qocwo.js → power-and-mde-Hj2ubb4B.js} +2 -2
- package/dist/{power-and-mde-fK5Qocwo.js.map → power-and-mde-Hj2ubb4B.js.map} +1 -1
- package/dist/predicate-FdKmElA_.js +38 -0
- package/dist/predicate-FdKmElA_.js.map +1 -0
- package/dist/{prime-protocol-DcAK9pC8.js → prime-protocol-CHKgA3a6.js} +2 -2
- package/dist/{prime-protocol-DcAK9pC8.js.map → prime-protocol-CHKgA3a6.js.map} +1 -1
- package/dist/{prime-protocol-CEE4BR9N.d.ts → prime-protocol-HiB9HX7E.d.ts} +3 -3
- package/dist/{prime-protocol-CEE4BR9N.d.ts.map → prime-protocol-HiB9HX7E.d.ts.map} +1 -1
- package/dist/{produced-state-ZCtDBxAE.js → produced-state-BsDmnwxc.js} +8 -8
- package/dist/{produced-state-ZCtDBxAE.js.map → produced-state-BsDmnwxc.js.map} +1 -1
- package/dist/profile-cell.js +1 -1
- package/dist/{release-confidence-BK2QCB5M.d.ts → promotion-gate-BawrdPYr.d.ts} +102 -102
- package/dist/promotion-gate-BawrdPYr.d.ts.map +1 -0
- package/dist/{promotion-policy-CWW35UWq.js → promotion-policy-KO57gbyV.js} +4 -4
- package/dist/{promotion-policy-CWW35UWq.js.map → promotion-policy-KO57gbyV.js.map} +1 -1
- package/dist/{proposal-findings-BZiMTzAF.js → proposal-findings-CY1xKn_a.js} +2 -2
- package/dist/{proposal-findings-BZiMTzAF.js.map → proposal-findings-CY1xKn_a.js.map} +1 -1
- package/dist/{query-B1D1qp2Z.js → query-BUCZCK7F.js} +3 -3
- package/dist/{query-B1D1qp2Z.js.map → query-BUCZCK7F.js.map} +1 -1
- package/dist/{random-Dn5fPWkt.js → random-ByUSnV9w.js} +2 -2
- package/dist/{random-Dn5fPWkt.js.map → random-ByUSnV9w.js.map} +1 -1
- package/dist/{raw-provider-sink-rB2DYxr6.js → raw-provider-sink-DoHz4JNG.js} +2 -2
- package/dist/{raw-provider-sink-rB2DYxr6.js.map → raw-provider-sink-DoHz4JNG.js.map} +1 -1
- package/dist/{redact-Dk1ZdJEm.js → redact-B5xXKqQ3.js} +97 -98
- package/dist/redact-B5xXKqQ3.js.map +1 -0
- package/dist/{redact-Cveoq6fJ.d.ts → redact-mtCCcT8z.d.ts} +4 -20
- package/dist/redact-mtCCcT8z.d.ts.map +1 -0
- package/dist/{registry-JYfPxcVz.d.ts → registry-D7_hm2fK.d.ts} +5 -5
- package/dist/{registry-JYfPxcVz.d.ts.map → registry-D7_hm2fK.d.ts.map} +1 -1
- package/dist/{release-confidence-Dbe9AS_X.js → release-confidence-CpuP65S5.js} +5 -5
- package/dist/{release-confidence-Dbe9AS_X.js.map → release-confidence-CpuP65S5.js.map} +1 -1
- package/dist/{report-command-y8H6RSX8.js → report-command-SC9nqb4M.js} +106 -171
- package/dist/report-command-SC9nqb4M.js.map +1 -0
- package/dist/reporting-html-Kvlc8oMt.js +100 -0
- package/dist/reporting-html-Kvlc8oMt.js.map +1 -0
- package/dist/reporting.d.ts +3 -3
- package/dist/reporting.js +6 -6
- package/dist/{researcher-f6mZhh58.d.ts → researcher-CEcfku-T.d.ts} +5 -5
- package/dist/{researcher-f6mZhh58.d.ts.map → researcher-CEcfku-T.d.ts.map} +1 -1
- package/dist/reward-CpCmAM8L.d.ts +78 -0
- package/dist/reward-CpCmAM8L.d.ts.map +1 -0
- package/dist/{reward-hacking-BFcQgkux.js → reward-hacking-E0d-WVcP.js} +2 -2
- package/dist/{reward-hacking-BFcQgkux.js.map → reward-hacking-E0d-WVcP.js.map} +1 -1
- package/dist/{reward-hacking-DHA7c1l8.d.ts → reward-hacking-RVdZ6Kkt.d.ts} +2 -2
- package/dist/{reward-hacking-DHA7c1l8.d.ts.map → reward-hacking-RVdZ6Kkt.d.ts.map} +1 -1
- package/dist/rl.d.ts +8 -8
- package/dist/rl.js +14 -14
- package/dist/rollout/index.d.ts +3 -2
- package/dist/rollout/index.js +2 -2
- package/dist/{rollout-D9bKKBnZ.js → rollout-DZV1t2Wg.js} +2 -2
- package/dist/{rollout-D9bKKBnZ.js.map → rollout-DZV1t2Wg.js.map} +1 -1
- package/dist/round-decimal-COrKglMh.js +10 -0
- package/dist/round-decimal-COrKglMh.js.map +1 -0
- package/dist/{rubric-predictive-validity-2SgHxoa5.d.ts → rubric-predictive-validity-C03EC7ZE.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-2SgHxoa5.d.ts.map → rubric-predictive-validity-C03EC7ZE.d.ts.map} +1 -1
- package/dist/{rubric-predictive-validity-BsgkBfT0.js → rubric-predictive-validity-DaFW70y9.js} +3 -3
- package/dist/{rubric-predictive-validity-BsgkBfT0.js.map → rubric-predictive-validity-DaFW70y9.js.map} +1 -1
- package/dist/{run-record-DlTZ_lqe.js → run-record-Boc5IUTp.js} +4 -4
- package/dist/{run-record-DlTZ_lqe.js.map → run-record-Boc5IUTp.js.map} +1 -1
- package/dist/{run-record-D1Hw-Jml.d.ts → run-record-C1E24G2A.d.ts} +2 -2
- package/dist/{run-record-D1Hw-Jml.d.ts.map → run-record-C1E24G2A.d.ts.map} +1 -1
- package/dist/{run-record-ES07wfz9.js → run-record-C5e7FjhJ.js} +3 -5
- package/dist/run-record-C5e7FjhJ.js.map +1 -0
- package/dist/search/index.d.ts +46 -32
- package/dist/search/index.d.ts.map +1 -1
- package/dist/search/index.js +3 -3
- package/dist/{search-kernel-CpcKH5Ih.d.ts → search-kernel-DyNEa6IU.d.ts} +14 -10
- package/dist/search-kernel-DyNEa6IU.d.ts.map +1 -0
- package/dist/{search-ledger-d7V8o9Lk.js → search-ledger-CVUGLDZp.js} +189 -199
- package/dist/search-ledger-CVUGLDZp.js.map +1 -0
- package/dist/{search-ledger-file-BqOo-bVo.js → search-ledger-file-DDQUgvCg.js} +4 -4
- package/dist/{search-ledger-file-BqOo-bVo.js.map → search-ledger-file-DDQUgvCg.js.map} +1 -1
- package/dist/search-ledger-types-p2xKfiDJ.d.ts.map +1 -1
- package/dist/{search-shipper-er4pK5_Y.js → search-shipper-BOSD46qv.js} +3 -3
- package/dist/{search-shipper-er4pK5_Y.js.map → search-shipper-BOSD46qv.js.map} +1 -1
- package/dist/{search-summary-CBvz3u_S.js → search-summary-BQCSWf1g.js} +5 -5
- package/dist/{search-summary-CBvz3u_S.js.map → search-summary-BQCSWf1g.js.map} +1 -1
- package/dist/{semantic-concept-judge-zQBG9hHU.js → semantic-concept-judge-BQF8te0T.js} +12 -14
- package/dist/semantic-concept-judge-BQF8te0T.js.map +1 -0
- package/dist/{sequential-D-1PPCmM.js → sequential-Bf_pbE50.js} +5 -5
- package/dist/{sequential-D-1PPCmM.js.map → sequential-Bf_pbE50.js.map} +1 -1
- package/dist/{sequential-eprocess-D1jKoihe.js → sequential-eprocess-DM_qENCl.js} +2 -2
- package/dist/{sequential-eprocess-D1jKoihe.js.map → sequential-eprocess-DM_qENCl.js.map} +1 -1
- package/dist/{server-DbcC7xMl.js → server-e9oWO_bN.js} +4 -4
- package/dist/{server-DbcC7xMl.js.map → server-e9oWO_bN.js.map} +1 -1
- package/dist/{single-run-lock-fVu3MZWH.js → single-run-lock-CV9Eyd_q.js} +2 -2
- package/dist/{single-run-lock-fVu3MZWH.js.map → single-run-lock-CV9Eyd_q.js.map} +1 -1
- package/dist/{skillopt-optimization-method-CCtzralq.js → skillopt-optimization-method-BiwA8nfX.js} +427 -12
- package/dist/skillopt-optimization-method-BiwA8nfX.js.map +1 -0
- package/dist/{spans-D9_ehFuk.d.ts → spans-BHAUXMGN.d.ts} +2 -2
- package/dist/{spans-D9_ehFuk.d.ts.map → spans-BHAUXMGN.d.ts.map} +1 -1
- package/dist/{statistical-heldout-1Gbmex7v.js → statistical-heldout-42NlCNDe.js} +3 -3
- package/dist/{statistical-heldout-1Gbmex7v.js.map → statistical-heldout-42NlCNDe.js.map} +1 -1
- package/dist/{statistical-heldout-BeR4bOdg.d.ts → statistical-heldout-CYpGaoDH.d.ts} +3 -3
- package/dist/{statistical-heldout-BeR4bOdg.d.ts.map → statistical-heldout-CYpGaoDH.d.ts.map} +1 -1
- package/dist/{store-otlp-DtB4Slsr.js → store-otlp-DJ9dtA5x.js} +135 -132
- package/dist/store-otlp-DJ9dtA5x.js.map +1 -0
- package/dist/{store-tool-spans-fZIigrpP.js → store-tool-spans-CIX5arAG.js} +7 -7
- package/dist/{store-tool-spans-fZIigrpP.js.map → store-tool-spans-CIX5arAG.js.map} +1 -1
- package/dist/{store-tool-spans-DR1SrCLK.d.ts → store-tool-spans-DdCSPa7C.d.ts} +162 -161
- package/dist/store-tool-spans-DdCSPa7C.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts.map +1 -1
- package/dist/storyboard/index.js +20 -3
- package/dist/storyboard/index.js.map +1 -1
- package/dist/{student-t-BQeKJx2N.js → student-t-DVPW7CdL.js} +2 -2
- package/dist/{student-t-BQeKJx2N.js.map → student-t-DVPW7CdL.js.map} +1 -1
- package/dist/{summary-report-DtdRucM0.js → summary-report-V_JACR4Y.js} +526 -614
- package/dist/summary-report-V_JACR4Y.js.map +1 -0
- package/dist/{summary-report-C-CtItCN.d.ts → summary-report-bdFDXq0b.d.ts} +10 -5
- package/dist/summary-report-bdFDXq0b.d.ts.map +1 -0
- package/dist/supervisor-run/index.d.ts +153 -149
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +65 -3
- package/dist/supervisor-run/index.js.map +1 -1
- package/dist/{systemone-CpTLAlqb.d.ts → systemone-BomhHr2c.d.ts} +6 -6
- package/dist/{systemone-CpTLAlqb.d.ts.map → systemone-BomhHr2c.d.ts.map} +1 -1
- package/dist/systemone-protocol.js +1 -1
- package/dist/{systemone-review-B_yo1xsd.d.ts → systemone-review-CUVcQJu6.d.ts} +2 -2
- package/dist/{systemone-review-B_yo1xsd.d.ts.map → systemone-review-CUVcQJu6.d.ts.map} +1 -1
- package/dist/{systemone-review-BZWfrtq1.js → systemone-review-D7sXwsqq.js} +2 -2
- package/dist/{systemone-review-BZWfrtq1.js.map → systemone-review-D7sXwsqq.js.map} +1 -1
- package/dist/systemone.d.ts +3 -3
- package/dist/systemone.js +4 -4
- package/dist/{task-failure-attributes-DngaLdE4.js → task-failure-attributes-hEmASjix.js} +2 -2
- package/dist/{task-failure-attributes-DngaLdE4.js.map → task-failure-attributes-hEmASjix.js.map} +1 -1
- package/dist/{terminal-record-BHocwFw1.js → terminal-record-DZyQZ3pz.js} +14 -10
- package/dist/terminal-record-DZyQZ3pz.js.map +1 -0
- package/dist/{tool-groups-mcxOXVlb.d.ts → tool-groups-C3NEcdCM.d.ts} +3 -3
- package/dist/tool-groups-C3NEcdCM.d.ts.map +1 -0
- package/dist/{tool-waste-Wi4g6Ekb.d.ts → tool-waste-BKgvd-TP.d.ts} +30 -30
- package/dist/tool-waste-BKgvd-TP.d.ts.map +1 -0
- package/dist/{tool-waste-D52PqU9Q.js → tool-waste-BlesqTqW.js} +4 -4
- package/dist/{tool-waste-D52PqU9Q.js.map → tool-waste-BlesqTqW.js.map} +1 -1
- package/dist/{trace-questions-DG7nqLQ7.d.ts → trace-questions-CM2CAocU.d.ts} +4 -4
- package/dist/{trace-questions-DG7nqLQ7.d.ts.map → trace-questions-CM2CAocU.d.ts.map} +1 -1
- package/dist/trace-repair/index.d.ts +3 -3
- package/dist/trace-repair/index.d.ts.map +1 -1
- package/dist/trace-repair/index.js +7 -13
- package/dist/trace-repair/index.js.map +1 -1
- package/dist/traces.d.ts +76 -58
- package/dist/traces.d.ts.map +1 -1
- package/dist/traces.js +9 -9
- package/dist/trajectory-replay/index.js +3 -3
- package/dist/{tree-rXLW3M_4.js → tree-C1XYeXvy.js} +2277 -2339
- package/dist/tree-C1XYeXvy.js.map +1 -0
- package/dist/{types-DBZ7keha.d.ts → types-C02qtnG8.d.ts} +4 -46
- package/dist/types-C02qtnG8.d.ts.map +1 -0
- package/dist/{types-oKRhQAde.d.ts → types-C1tnkEgM.d.ts} +4 -4
- package/dist/{types-oKRhQAde.d.ts.map → types-C1tnkEgM.d.ts.map} +1 -1
- package/dist/{types-CfZMIzB_.d.ts → types-Cbe2fWBa.d.ts} +3 -3
- package/dist/{types-CfZMIzB_.d.ts.map → types-Cbe2fWBa.d.ts.map} +1 -1
- package/dist/{types-B4C5nFw6.d.ts → types-Cq8mSbq8.d.ts} +78 -72
- package/dist/types-Cq8mSbq8.d.ts.map +1 -0
- package/dist/{verdict-DhPoXApk.js → verdict-Dl0tiT1X.js} +2 -2
- package/dist/{verdict-DhPoXApk.js.map → verdict-Dl0tiT1X.js.map} +1 -1
- package/dist/{verdict-cache-BQEsVy9n.js → verdict-cache-Bm8EKL2s.js} +2 -2
- package/dist/{verdict-cache-BQEsVy9n.js.map → verdict-cache-Bm8EKL2s.js.map} +1 -1
- package/dist/wire/index.d.ts +3 -3
- package/dist/wire/index.js +1 -1
- package/docs/design/mlbenchmarks-book-review.md +1 -1
- package/docs/design/mlbenchmarks-review/probes.mts +23 -31
- package/docs/experiment.md +1 -1
- package/package.json +1 -1
- package/dist/analyze-runs-3xpby3p2.js.map +0 -1
- package/dist/analyze-runs-CSDAh8A7.d.ts.map +0 -1
- package/dist/ast-DJ6Bc7ab.js +0 -558
- package/dist/ast-DJ6Bc7ab.js.map +0 -1
- package/dist/ast-hI-vjW6J.d.ts +0 -457
- package/dist/ast-hI-vjW6J.d.ts.map +0 -1
- package/dist/benchmark-command-CByl9dxG.js.map +0 -1
- package/dist/campaign-BQ-Jujuj.js.map +0 -1
- package/dist/campaign-evidence-OcUciD-N.js.map +0 -1
- package/dist/claim-BXcZw-jo.js.map +0 -1
- package/dist/cost-ledger-CO7UyEcQ.d.ts.map +0 -1
- package/dist/cost-ledger-D2FnJPBF.js.map +0 -1
- package/dist/define-agent-eval-Ch3MWX6k.js.map +0 -1
- package/dist/external-optimizer-subprocess-DcerlZSG.js.map +0 -1
- package/dist/feedback-trajectory-DkaDj0dj.d.ts.map +0 -1
- package/dist/index-CPJJr98o.d.ts.map +0 -1
- package/dist/index-DlH0Y9gI.d.ts.map +0 -1
- package/dist/index-NvGBiO8H.d.ts.map +0 -1
- package/dist/integrity-BV1ZDf42.js.map +0 -1
- package/dist/judge-gate-B1qYnkpi.js.map +0 -1
- package/dist/llm-client-DQveoxSM.js.map +0 -1
- package/dist/llm-judge-PXV7Xljl.js.map +0 -1
- package/dist/matrix-_BV9-Lk5.js.map +0 -1
- package/dist/mint-DCGoBYhB.d.ts +0 -155
- package/dist/mint-DCGoBYhB.d.ts.map +0 -1
- package/dist/paired-tests-jbwPdSz5.js.map +0 -1
- package/dist/redact-Cveoq6fJ.d.ts.map +0 -1
- package/dist/redact-Dk1ZdJEm.js.map +0 -1
- package/dist/release-confidence-BK2QCB5M.d.ts.map +0 -1
- package/dist/report-command-y8H6RSX8.js.map +0 -1
- package/dist/run-record-ES07wfz9.js.map +0 -1
- package/dist/search-kernel-BztlERLv.js.map +0 -1
- package/dist/search-kernel-CpcKH5Ih.d.ts.map +0 -1
- package/dist/search-ledger-d7V8o9Lk.js.map +0 -1
- package/dist/semantic-concept-judge-zQBG9hHU.js.map +0 -1
- package/dist/skillopt-optimization-method-CCtzralq.js.map +0 -1
- package/dist/store-otlp-DtB4Slsr.js.map +0 -1
- package/dist/store-tool-spans-DR1SrCLK.d.ts.map +0 -1
- package/dist/summary-report-C-CtItCN.d.ts.map +0 -1
- package/dist/summary-report-DtdRucM0.js.map +0 -1
- package/dist/terminal-record-BHocwFw1.js.map +0 -1
- package/dist/tool-groups-mcxOXVlb.d.ts.map +0 -1
- package/dist/tool-waste-Wi4g6Ekb.d.ts.map +0 -1
- package/dist/tree-rXLW3M_4.js.map +0 -1
- package/dist/types-B4C5nFw6.d.ts.map +0 -1
- package/dist/types-DBZ7keha.d.ts.map +0 -1
|
@@ -1,90 +1,17 @@
|
|
|
1
|
-
import { a as hashCanonical, i as compareCodeUnits, r as canonicalString } from "./canonical-
|
|
2
|
-
import { t as mulberry32 } from "./random-
|
|
3
|
-
import { t as confidenceInterval } from "./descriptive-
|
|
4
|
-
import { c as wilson } from "./paired-binary-
|
|
5
|
-
import {
|
|
6
|
-
import { i as pairedDeltaTest, r as minimumPairsForPairedDeltaTest, t as decidePairedPromotion } from "./paired-promotion-decision-
|
|
7
|
-
import { a as SearchLedgerError } from "./search-ledger-file-
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
10
|
-
import { t as acquireSingleRunLock } from "./single-run-lock-
|
|
11
|
-
//#region src/
|
|
12
|
-
/**
|
|
13
|
-
function
|
|
14
|
-
|
|
15
|
-
for (const obj of objectives) {
|
|
16
|
-
const av = obj.value(a);
|
|
17
|
-
const bv = obj.value(b);
|
|
18
|
-
if (!Number.isFinite(av) || !Number.isFinite(bv)) return false;
|
|
19
|
-
const aIsBetter = obj.direction === "maximize" ? av > bv : av < bv;
|
|
20
|
-
if (obj.direction === "maximize" ? av < bv : av > bv) return false;
|
|
21
|
-
if (aIsBetter) strictlyBetter = true;
|
|
22
|
-
}
|
|
23
|
-
return strictlyBetter;
|
|
24
|
-
}
|
|
25
|
-
/**
|
|
26
|
-
* Compute the non-dominated frontier. Candidates with NaN/Infinity on any
|
|
27
|
-
* objective are excluded (can't rank them). A candidate enters the frontier
|
|
28
|
-
* iff no other candidate dominates it.
|
|
29
|
-
*/
|
|
30
|
-
function paretoFrontier(candidates, objectives) {
|
|
31
|
-
if (objectives.length === 0) throw new Error("paretoFrontier: at least 1 objective required");
|
|
32
|
-
const valid = candidates.filter((c) => objectives.every((o) => Number.isFinite(o.value(c))));
|
|
33
|
-
const frontier = [];
|
|
34
|
-
const dominated = [];
|
|
35
|
-
for (const c of valid) if (valid.some((other) => other !== c && dominates(other, c, objectives))) dominated.push(c);
|
|
36
|
-
else frontier.push(c);
|
|
37
|
-
return {
|
|
38
|
-
frontier,
|
|
39
|
-
dominated,
|
|
40
|
-
dominanceMap: frontier.map((d) => ({
|
|
41
|
-
dominator: d,
|
|
42
|
-
dominated: dominated.filter((x) => dominates(d, x, objectives))
|
|
43
|
-
}))
|
|
44
|
-
};
|
|
45
|
-
}
|
|
46
|
-
/**
|
|
47
|
-
* NSGA-II crowding distance — secondary sort for ties on the frontier.
|
|
48
|
-
*
|
|
49
|
-
* When the Pareto front collapses to a single point (or many candidates tie
|
|
50
|
-
* on dominance), naive selection picks arbitrarily and the population
|
|
51
|
-
* degenerates over generations. NSGA-II preserves diversity by preferring
|
|
52
|
-
* candidates with more empty space around them on the frontier.
|
|
53
|
-
*
|
|
54
|
-
* Returns an array of `{ candidate, distance }` in the SAME order as the
|
|
55
|
-
* input. Higher distance = more isolated = should be preferred when
|
|
56
|
-
* preserving diversity.
|
|
57
|
-
*/
|
|
58
|
-
function crowdingDistance(candidates, objectives) {
|
|
59
|
-
const distances = new Map(candidates.map((c) => [c, 0]));
|
|
60
|
-
for (const obj of objectives) {
|
|
61
|
-
const sorted = [...candidates].sort((a, b) => obj.value(a) - obj.value(b));
|
|
62
|
-
const min = obj.value(sorted[0]);
|
|
63
|
-
const range = obj.value(sorted[sorted.length - 1]) - min || 1;
|
|
64
|
-
distances.set(sorted[0], Infinity);
|
|
65
|
-
distances.set(sorted[sorted.length - 1], Infinity);
|
|
66
|
-
for (let i = 1; i < sorted.length - 1; i++) {
|
|
67
|
-
const prev = obj.value(sorted[i - 1]);
|
|
68
|
-
const next = obj.value(sorted[i + 1]);
|
|
69
|
-
const current = distances.get(sorted[i]);
|
|
70
|
-
if (current === Infinity) continue;
|
|
71
|
-
distances.set(sorted[i], current + (next - prev) / range);
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
return candidates.map((c) => ({
|
|
75
|
-
candidate: c,
|
|
76
|
-
distance: distances.get(c) ?? 0
|
|
77
|
-
}));
|
|
78
|
-
}
|
|
79
|
-
/**
|
|
80
|
-
* Pareto frontier with tie-break by crowding distance — the canonical
|
|
81
|
-
* NSGA-II selection step. Returns the frontier sorted by descending crowding
|
|
82
|
-
* distance so callers can `.slice(0, k)` to pick K diverse winners.
|
|
83
|
-
*/
|
|
84
|
-
function paretoFrontierWithCrowding(candidates, objectives) {
|
|
85
|
-
const { frontier } = paretoFrontier(candidates, objectives);
|
|
86
|
-
if (frontier.length === 0) return [];
|
|
87
|
-
return crowdingDistance(frontier, objectives).sort((a, b) => b.distance - a.distance);
|
|
1
|
+
import { a as hashCanonical, i as compareCodeUnits, r as canonicalString } from "./canonical-B2RqLofv.js";
|
|
2
|
+
import { t as mulberry32 } from "./random-ByUSnV9w.js";
|
|
3
|
+
import { t as confidenceInterval } from "./descriptive-DboxPZrG.js";
|
|
4
|
+
import { c as wilson } from "./paired-binary-CvpFHfKZ.js";
|
|
5
|
+
import { r as DECISION_PAIRED_DELTA_STATISTIC } from "./mean-DHoMiudD.js";
|
|
6
|
+
import { i as pairedDeltaTest, r as minimumPairsForPairedDeltaTest, t as decidePairedPromotion } from "./paired-promotion-decision-DsAPYGCm.js";
|
|
7
|
+
import { a as SearchLedgerError } from "./search-ledger-file-DDQUgvCg.js";
|
|
8
|
+
import { c as redactText } from "./redact-B5xXKqQ3.js";
|
|
9
|
+
import { l as searchUnitScores, p as USD_TOLERANCE, t as FileSearchLedger, u as searchCellId } from "./search-ledger-CVUGLDZp.js";
|
|
10
|
+
import { t as acquireSingleRunLock } from "./single-run-lock-CV9Eyd_q.js";
|
|
11
|
+
//#region src/statistics/normalize-signed-zero.ts
|
|
12
|
+
/** Normalize IEEE negative zero before numerical evidence reaches canonical JSON. */
|
|
13
|
+
function normalizeSignedZero(value) {
|
|
14
|
+
return value === 0 ? 0 : value;
|
|
88
15
|
}
|
|
89
16
|
//#endregion
|
|
90
17
|
//#region src/campaign/estimate-node.ts
|
|
@@ -219,8 +146,8 @@ function estimateNodeFromCells(input) {
|
|
|
219
146
|
const spread = (method === "descriptive" || method === "bootstrap") && !indeterminate;
|
|
220
147
|
return {
|
|
221
148
|
...base,
|
|
222
|
-
delta:
|
|
223
|
-
interval: spread ? maximize ? [
|
|
149
|
+
delta: normalizeSignedZero(maximize ? mean : -mean),
|
|
150
|
+
interval: spread ? maximize ? [normalizeSignedZero(low), normalizeSignedZero(high)] : [normalizeSignedZero(-high), normalizeSignedZero(-low)] : null,
|
|
224
151
|
exactSignP: method === "descriptive" && !indeterminate ? test.pValue : null,
|
|
225
152
|
indeterminate
|
|
226
153
|
};
|
|
@@ -283,7 +210,7 @@ function searchPosterior(state, options = {}) {
|
|
|
283
210
|
nodes.push({
|
|
284
211
|
nodeId,
|
|
285
212
|
pairs: gains.length,
|
|
286
|
-
mean: mean === null ? null :
|
|
213
|
+
mean: mean === null ? null : normalizeSignedZero(mean),
|
|
287
214
|
variance: null
|
|
288
215
|
});
|
|
289
216
|
}
|
|
@@ -334,315 +261,110 @@ function sum(values) {
|
|
|
334
261
|
for (const value of values) total += value;
|
|
335
262
|
return total;
|
|
336
263
|
}
|
|
337
|
-
/** Canonical JSON has one zero; never hand it a negative one. */
|
|
338
|
-
function plain$1(value) {
|
|
339
|
-
return value === 0 ? 0 : value;
|
|
340
|
-
}
|
|
341
264
|
//#endregion
|
|
342
|
-
//#region src/
|
|
265
|
+
//#region src/paired-promotion-power.ts
|
|
343
266
|
/**
|
|
344
|
-
*
|
|
267
|
+
* @module
|
|
268
|
+
* Joint power of a configured {@link decidePairedPromotion} call, computed by
|
|
269
|
+
* running that call on data drawn from a registered alternative.
|
|
345
270
|
*
|
|
346
|
-
*
|
|
347
|
-
*
|
|
348
|
-
*
|
|
349
|
-
*
|
|
350
|
-
*
|
|
271
|
+
* Why a simulation and not a formula: `decidePairedPromotion` is not one test.
|
|
272
|
+
* It routes by sample size (exact sign test below `BOOTSTRAP_GATE_MIN_N`,
|
|
273
|
+
* paired bootstrap at or above it) and by shape (Tango's score interval on a
|
|
274
|
+
* two-point outcome), and on top of the deciding interval it applies a
|
|
275
|
+
* sufficiency floor, a zero-width refusal and McNemar's exact veto. A
|
|
276
|
+
* preregistration usually seals two such calls, a continuous primary and a
|
|
277
|
+
* pass/fail veto drawn from the same episodes, and promotes only when both
|
|
278
|
+
* promote. The sample size at which THAT procedure reaches a target power is
|
|
279
|
+
* not the power of any one channel: the sign-test power at n can be 0.8 while
|
|
280
|
+
* the promotion probability is well below it because the veto binds
|
|
281
|
+
* (agent-eval#785). Closed forms exist for single channels only, and each one
|
|
282
|
+
* shipped here under-sized the call it stood in for (`mcnemarRequiredN` is
|
|
283
|
+
* Lachin's normal approximation, 0.65 exact power where it promised 0.8 at 14
|
|
284
|
+
* pairs; `requiredPairedSampleSize` sizes a paired t the procedure never runs).
|
|
285
|
+
* So the only faithful power is the procedure's own promotion rate under the
|
|
286
|
+
* alternative, and that is what this module computes: draw n pairs from the
|
|
287
|
+
* registered law, run every configured call on them exactly as the caller
|
|
288
|
+
* will, repeat, count.
|
|
289
|
+
*
|
|
290
|
+
* The alternative is DECLARATIVE and serializable, never a sampler function,
|
|
291
|
+
* so a preregistration can seal it as bytes: a mixture of cells, each with a
|
|
292
|
+
* probability, an optional pass/fail outcome per arm for the binary calls,
|
|
293
|
+
* and a law for the candidate-minus-baseline delta on the continuous scale.
|
|
294
|
+
* Drawing the cell first and the delta given the cell is what makes the two
|
|
295
|
+
* channels one joint law rather than two marginals.
|
|
296
|
+
*
|
|
297
|
+
* Everything is seeded (`mulberry32`), so a sizing is reproducible from its
|
|
298
|
+
* inputs, and every estimate carries its Monte Carlo standard error and a
|
|
299
|
+
* Wilson interval: a power read at 0.80 from 1,000 simulations is 0.80 ± 0.013.
|
|
351
300
|
*/
|
|
352
|
-
/**
|
|
353
|
-
*
|
|
354
|
-
*
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
}
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
const
|
|
362
|
-
|
|
363
|
-
const
|
|
364
|
-
const
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
301
|
+
/**
|
|
302
|
+
* Joint power of the configured calls at `n` under the registered alternative:
|
|
303
|
+
* the fraction of simulated data sets on which every call promotes.
|
|
304
|
+
*/
|
|
305
|
+
function pairedPromotionPower(options) {
|
|
306
|
+
const n = options.n;
|
|
307
|
+
if (!Number.isInteger(n) || n < 1) throw new Error(`pairedPromotionPower: n must be a positive integer, got ${n}`);
|
|
308
|
+
const simulations = options.simulations ?? 1e3;
|
|
309
|
+
if (!Number.isInteger(simulations) || simulations < 1) throw new Error(`pairedPromotionPower: simulations must be a positive integer, got ${simulations}`);
|
|
310
|
+
const seed = options.seed ?? 1;
|
|
311
|
+
if (!Number.isFinite(seed)) throw new Error(`pairedPromotionPower: seed must be finite, got ${seed}`);
|
|
312
|
+
const calls = validateCalls(options.calls);
|
|
313
|
+
const alternative = validateAlternative(options.alternative, calls);
|
|
314
|
+
const rng = mulberry32(seed);
|
|
315
|
+
let promotions = 0;
|
|
316
|
+
let refusals = 0;
|
|
317
|
+
const perCall = calls.map(() => ({
|
|
318
|
+
promote: 0,
|
|
319
|
+
insufficient: 0,
|
|
320
|
+
indeterminate: 0,
|
|
321
|
+
exactTestVetoes: 0,
|
|
322
|
+
hold: 0,
|
|
323
|
+
methods: /* @__PURE__ */ new Map()
|
|
324
|
+
}));
|
|
325
|
+
for (let s = 0; s < simulations; s++) {
|
|
326
|
+
const pairs = drawPairs(alternative, n, rng);
|
|
327
|
+
let all = true;
|
|
328
|
+
let anyRefusal = false;
|
|
329
|
+
for (let c = 0; c < calls.length; c++) {
|
|
330
|
+
const call = calls[c];
|
|
331
|
+
const decision = runCall(call, pairs, n);
|
|
332
|
+
const tally = perCall[c];
|
|
333
|
+
tally.methods.set(decision.method, (tally.methods.get(decision.method) ?? 0) + 1);
|
|
334
|
+
if (decision.promote) tally.promote++;
|
|
335
|
+
else {
|
|
336
|
+
if (!decision.sufficient) tally.insufficient++;
|
|
337
|
+
if (decision.indeterminate) tally.indeterminate++;
|
|
338
|
+
if (decision.exactTestVetoes) tally.exactTestVetoes++;
|
|
339
|
+
if (decision.sufficient && !decision.indeterminate && !decision.exactTestVetoes) tally.hold++;
|
|
340
|
+
}
|
|
341
|
+
if (!decision.promote) all = false;
|
|
342
|
+
if (!decision.sufficient || decision.indeterminate || decision.exactTestVetoes) anyRefusal = true;
|
|
343
|
+
}
|
|
344
|
+
if (all) promotions++;
|
|
345
|
+
else if (anyRefusal) refusals++;
|
|
346
|
+
}
|
|
347
|
+
const power = promotions / simulations;
|
|
348
|
+
const interval = wilson(promotions, simulations, .95);
|
|
381
349
|
return {
|
|
382
350
|
n,
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
* evidence.
|
|
401
|
-
*
|
|
402
|
-
* "Yield" reuses `searchPosterior` (E3, `../../campaign/estimate-node`): its
|
|
403
|
-
* per-node mean is already the node's improvement over the root, oriented so
|
|
404
|
-
* larger is always better regardless of the objective's direction — this
|
|
405
|
-
* lens adds no second orientation rule of its own. A child's sample is
|
|
406
|
-
* `posteriorMean / screenCostUsd`: the gain over the known cost of producing
|
|
407
|
-
* and screening the node, which is the operator's doing. That cost is the
|
|
408
|
-
* node's share of its proposal (the candidate-generation operation's cost
|
|
409
|
-
* split evenly over every child edge the operation recorded, re-proposals
|
|
410
|
-
* included) plus its cells allocated before its first decision: the first
|
|
411
|
-
* rung and the proposer's train feedback, which the allocator plans together
|
|
412
|
-
* at admission. Cells a later rung or the claim adds come after an
|
|
413
|
-
* `advanced` or `finalist` decision; they are the allocator's and the
|
|
414
|
-
* claim's choice, not the operator's, so an operator whose children advance
|
|
415
|
-
* is not charged for the measurement their success earned. A seed edge
|
|
416
|
-
* proposed nothing, so the root's cost is its cells alone.
|
|
417
|
-
*
|
|
418
|
-
* A child enters the sample only when both numbers are known: the node
|
|
419
|
-
* shares 2 or more units with the root (one shared unit is no estimate,
|
|
420
|
-
* §6.4); the edge names a recorded proposal operation whose cost is known;
|
|
421
|
-
* every screen cell has run and has a known cost (an unpriced or unrecorded
|
|
422
|
-
* part never floors a node to an artificially cheap, inflated yield); and it
|
|
423
|
-
* was not decided `invalid` (a judge integrity or admission failure: its
|
|
424
|
-
* score is not evidence of improvement, and `editCredit` excludes such nodes
|
|
425
|
-
* the same way). A $0-known-cost node is excluded from yield (division by
|
|
426
|
-
* zero is not a yield), not treated as free.
|
|
427
|
-
*/
|
|
428
|
-
/** The operators an expansion may choose (`SearchExpansion.operator` in
|
|
429
|
-
* `search-policy.ts`): every edge operator except `seed` (the search's own
|
|
430
|
-
* start) and `derive` (a cross-search edge a proposer never chooses). */
|
|
431
|
-
const EXPANSION_OPERATORS = [
|
|
432
|
-
"draft",
|
|
433
|
-
"improve",
|
|
434
|
-
"debug",
|
|
435
|
-
"merge"
|
|
436
|
-
];
|
|
437
|
-
/** The bandit's own gate: an operator's weight is used only once it has this
|
|
438
|
-
* many yield-eligible outcomes; below it, the policy falls back to a fixed
|
|
439
|
-
* weight for that operator. Same threshold `NodeEstimate.method` uses for
|
|
440
|
-
* `insufficient` (`minimumPairsForPairedDeltaTest(0.95)` = 6). */
|
|
441
|
-
const MIN_OUTCOMES_FOR_WEIGHT = 6;
|
|
442
|
-
const YIELD_METHOD = "per registered node, searchPosterior's mean gain over the root divided by the known cost of producing and screening the node: its even share of the proposal operation's cost over the operation's child edges, plus its cells allocated before its first decision (first rung and train feedback; later rungs and the claim excluded); no sample when the node was decided invalid, shares fewer than 2 units with the root, or any part of that cost is unknown, unrecorded or zero";
|
|
443
|
-
function operatorYield(state, options = {}) {
|
|
444
|
-
const split = options.split ?? rankingSplit(state);
|
|
445
|
-
const posteriorByNode = new Map(state.header && state.rootNodeId !== null ? searchPosterior(state, { split }).nodes.map((node) => [node.nodeId, node]) : []);
|
|
446
|
-
const grouped = /* @__PURE__ */ new Map();
|
|
447
|
-
const group = (operator) => {
|
|
448
|
-
let existing = grouped.get(operator);
|
|
449
|
-
if (!existing) {
|
|
450
|
-
existing = {
|
|
451
|
-
proposals: 0,
|
|
452
|
-
reproposals: 0,
|
|
453
|
-
outcomes: {
|
|
454
|
-
advanced: 0,
|
|
455
|
-
finalist: 0,
|
|
456
|
-
selected: 0,
|
|
457
|
-
pruned: 0,
|
|
458
|
-
rejected: 0,
|
|
459
|
-
invalid: 0,
|
|
460
|
-
undecided: 0
|
|
461
|
-
},
|
|
462
|
-
excluded: 0,
|
|
463
|
-
samples: []
|
|
464
|
-
};
|
|
465
|
-
grouped.set(operator, existing);
|
|
466
|
-
}
|
|
467
|
-
return existing;
|
|
468
|
-
};
|
|
469
|
-
const edges = state.edges();
|
|
470
|
-
const childEdges = /* @__PURE__ */ new Map();
|
|
471
|
-
for (const edge of edges) {
|
|
472
|
-
const operationId = edge.proposer?.operationId;
|
|
473
|
-
if (operationId) childEdges.set(operationId, (childEdges.get(operationId) ?? 0) + 1);
|
|
474
|
-
}
|
|
475
|
-
for (const edge of edges) {
|
|
476
|
-
const child = state.node(edge.childNodeId);
|
|
477
|
-
if (!child) continue;
|
|
478
|
-
const bucket = group(edge.operator);
|
|
479
|
-
bucket.proposals += 1;
|
|
480
|
-
if (child.edgeIds[0] !== edge.edgeId) {
|
|
481
|
-
bucket.reproposals += 1;
|
|
482
|
-
continue;
|
|
483
|
-
}
|
|
484
|
-
bucket.outcomes[statusKey(child.status)] += 1;
|
|
485
|
-
const posterior = posteriorByNode.get(child.nodeId);
|
|
486
|
-
const costUsd = screenCostUsd(state, child, edge, childEdges);
|
|
487
|
-
if (child.status === "invalid" || !posterior || posterior.mean === null || posterior.pairs < 2 || costUsd === null || costUsd <= 0) {
|
|
488
|
-
bucket.excluded += 1;
|
|
489
|
-
continue;
|
|
490
|
-
}
|
|
491
|
-
bucket.samples.push(posterior.mean / costUsd);
|
|
492
|
-
}
|
|
493
|
-
const rows = [...grouped.entries()].sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0).map(([operator, bucket]) => ({
|
|
494
|
-
operator,
|
|
495
|
-
proposals: bucket.proposals,
|
|
496
|
-
reproposals: bucket.reproposals,
|
|
497
|
-
outcomes: bucket.outcomes,
|
|
498
|
-
excluded: bucket.excluded,
|
|
499
|
-
yield: summarizeSamples(bucket.samples)
|
|
500
|
-
}));
|
|
501
|
-
const signalValue = {};
|
|
502
|
-
for (const operator of EXPANSION_OPERATORS) {
|
|
503
|
-
const row = rows.find((r) => r.operator === operator);
|
|
504
|
-
signalValue[operator] = row && row.yield.n >= 6 && row.yield.mean !== null ? row.yield.mean : null;
|
|
505
|
-
}
|
|
506
|
-
return {
|
|
507
|
-
data: {
|
|
508
|
-
split,
|
|
509
|
-
method: YIELD_METHOD,
|
|
510
|
-
rows
|
|
511
|
-
},
|
|
512
|
-
signal: {
|
|
513
|
-
name: "operatorYield.weights",
|
|
514
|
-
value: signalValue
|
|
515
|
-
}
|
|
516
|
-
};
|
|
517
|
-
}
|
|
518
|
-
/** The known cost of producing and screening `node`, which `edge` registered;
|
|
519
|
-
* null when any part of it is unknown. The screen ends at the node's first
|
|
520
|
-
* decision: the allocator records `advanced` before it allocates a further
|
|
521
|
-
* rung, and the claim decides `finalist` before it allocates test cells. */
|
|
522
|
-
function screenCostUsd(state, node, edge, childEdges) {
|
|
523
|
-
let usd = 0;
|
|
524
|
-
if (edge.proposer !== null) {
|
|
525
|
-
const { operationId } = edge.proposer;
|
|
526
|
-
if (operationId === null) return null;
|
|
527
|
-
const operation = state.operation(operationId);
|
|
528
|
-
if (!operation?.recorded || !operation.costKnown) return null;
|
|
529
|
-
usd += operation.spentUsd / childEdges.get(operationId);
|
|
530
|
-
}
|
|
531
|
-
const screenEnd = node.decisions[0]?.sequence ?? Number.POSITIVE_INFINITY;
|
|
532
|
-
for (const cell of state.cells({ nodeId: node.nodeId })) {
|
|
533
|
-
if (cell.allocatedSequence > screenEnd) continue;
|
|
534
|
-
if (!cell.costKnown || cell.attempts === 0 && cell.cancelled === null) return null;
|
|
535
|
-
usd += cell.spentUsd;
|
|
536
|
-
}
|
|
537
|
-
return usd;
|
|
538
|
-
}
|
|
539
|
-
function statusKey(status) {
|
|
540
|
-
return status ?? "undecided";
|
|
541
|
-
}
|
|
542
|
-
//#endregion
|
|
543
|
-
//#region src/paired-promotion-power.ts
|
|
544
|
-
/**
|
|
545
|
-
* @module
|
|
546
|
-
* Joint power of a configured {@link decidePairedPromotion} call, computed by
|
|
547
|
-
* running that call on data drawn from a registered alternative.
|
|
548
|
-
*
|
|
549
|
-
* Why a simulation and not a formula: `decidePairedPromotion` is not one test.
|
|
550
|
-
* It routes by sample size (exact sign test below `BOOTSTRAP_GATE_MIN_N`,
|
|
551
|
-
* paired bootstrap at or above it) and by shape (Tango's score interval on a
|
|
552
|
-
* two-point outcome), and on top of the deciding interval it applies a
|
|
553
|
-
* sufficiency floor, a zero-width refusal and McNemar's exact veto. A
|
|
554
|
-
* preregistration usually seals two such calls, a continuous primary and a
|
|
555
|
-
* pass/fail veto drawn from the same episodes, and promotes only when both
|
|
556
|
-
* promote. The sample size at which THAT procedure reaches a target power is
|
|
557
|
-
* not the power of any one channel: the sign-test power at n can be 0.8 while
|
|
558
|
-
* the promotion probability is well below it because the veto binds
|
|
559
|
-
* (agent-eval#785). Closed forms exist for single channels only, and each one
|
|
560
|
-
* shipped here under-sized the call it stood in for (`mcnemarRequiredN` is
|
|
561
|
-
* Lachin's normal approximation, 0.65 exact power where it promised 0.8 at 14
|
|
562
|
-
* pairs; `requiredPairedSampleSize` sizes a paired t the procedure never runs).
|
|
563
|
-
* So the only faithful power is the procedure's own promotion rate under the
|
|
564
|
-
* alternative, and that is what this module computes: draw n pairs from the
|
|
565
|
-
* registered law, run every configured call on them exactly as the caller
|
|
566
|
-
* will, repeat, count.
|
|
567
|
-
*
|
|
568
|
-
* The alternative is DECLARATIVE and serializable, never a sampler function,
|
|
569
|
-
* so a preregistration can seal it as bytes: a mixture of cells, each with a
|
|
570
|
-
* probability, an optional pass/fail outcome per arm for the binary calls,
|
|
571
|
-
* and a law for the candidate-minus-baseline delta on the continuous scale.
|
|
572
|
-
* Drawing the cell first and the delta given the cell is what makes the two
|
|
573
|
-
* channels one joint law rather than two marginals.
|
|
574
|
-
*
|
|
575
|
-
* Everything is seeded (`mulberry32`), so a sizing is reproducible from its
|
|
576
|
-
* inputs, and every estimate carries its Monte Carlo standard error and a
|
|
577
|
-
* Wilson interval: a power read at 0.80 from 1,000 simulations is 0.80 ± 0.013.
|
|
578
|
-
*/
|
|
579
|
-
/**
|
|
580
|
-
* Joint power of the configured calls at `n` under the registered alternative:
|
|
581
|
-
* the fraction of simulated data sets on which every call promotes.
|
|
582
|
-
*/
|
|
583
|
-
function pairedPromotionPower(options) {
|
|
584
|
-
const n = options.n;
|
|
585
|
-
if (!Number.isInteger(n) || n < 1) throw new Error(`pairedPromotionPower: n must be a positive integer, got ${n}`);
|
|
586
|
-
const simulations = options.simulations ?? 1e3;
|
|
587
|
-
if (!Number.isInteger(simulations) || simulations < 1) throw new Error(`pairedPromotionPower: simulations must be a positive integer, got ${simulations}`);
|
|
588
|
-
const seed = options.seed ?? 1;
|
|
589
|
-
if (!Number.isFinite(seed)) throw new Error(`pairedPromotionPower: seed must be finite, got ${seed}`);
|
|
590
|
-
const calls = validateCalls(options.calls);
|
|
591
|
-
const alternative = validateAlternative(options.alternative, calls);
|
|
592
|
-
const rng = mulberry32(seed);
|
|
593
|
-
let promotions = 0;
|
|
594
|
-
let refusals = 0;
|
|
595
|
-
const perCall = calls.map(() => ({
|
|
596
|
-
promote: 0,
|
|
597
|
-
insufficient: 0,
|
|
598
|
-
indeterminate: 0,
|
|
599
|
-
exactTestVetoes: 0,
|
|
600
|
-
hold: 0,
|
|
601
|
-
methods: /* @__PURE__ */ new Map()
|
|
602
|
-
}));
|
|
603
|
-
for (let s = 0; s < simulations; s++) {
|
|
604
|
-
const pairs = drawPairs(alternative, n, rng);
|
|
605
|
-
let all = true;
|
|
606
|
-
let anyRefusal = false;
|
|
607
|
-
for (let c = 0; c < calls.length; c++) {
|
|
608
|
-
const call = calls[c];
|
|
609
|
-
const decision = runCall(call, pairs, n);
|
|
610
|
-
const tally = perCall[c];
|
|
611
|
-
tally.methods.set(decision.method, (tally.methods.get(decision.method) ?? 0) + 1);
|
|
612
|
-
if (decision.promote) tally.promote++;
|
|
613
|
-
else {
|
|
614
|
-
if (!decision.sufficient) tally.insufficient++;
|
|
615
|
-
if (decision.indeterminate) tally.indeterminate++;
|
|
616
|
-
if (decision.exactTestVetoes) tally.exactTestVetoes++;
|
|
617
|
-
if (decision.sufficient && !decision.indeterminate && !decision.exactTestVetoes) tally.hold++;
|
|
618
|
-
}
|
|
619
|
-
if (!decision.promote) all = false;
|
|
620
|
-
if (!decision.sufficient || decision.indeterminate || decision.exactTestVetoes) anyRefusal = true;
|
|
621
|
-
}
|
|
622
|
-
if (all) promotions++;
|
|
623
|
-
else if (anyRefusal) refusals++;
|
|
624
|
-
}
|
|
625
|
-
const power = promotions / simulations;
|
|
626
|
-
const interval = wilson(promotions, simulations, .95);
|
|
627
|
-
return {
|
|
628
|
-
n,
|
|
629
|
-
simulations,
|
|
630
|
-
seed,
|
|
631
|
-
power,
|
|
632
|
-
standardError: Math.sqrt(power * (1 - power) / simulations),
|
|
633
|
-
low: interval.lower,
|
|
634
|
-
high: interval.upper,
|
|
635
|
-
refuse: refusals / simulations,
|
|
636
|
-
hold: (simulations - promotions - refusals) / simulations,
|
|
637
|
-
calls: perCall.map((tally, c) => ({
|
|
638
|
-
outcome: calls[c].outcome,
|
|
639
|
-
promote: tally.promote / simulations,
|
|
640
|
-
insufficient: tally.insufficient / simulations,
|
|
641
|
-
indeterminate: tally.indeterminate / simulations,
|
|
642
|
-
exactTestVetoes: tally.exactTestVetoes / simulations,
|
|
643
|
-
hold: tally.hold / simulations,
|
|
644
|
-
methods: Object.fromEntries([...tally.methods.entries()].map(([method, count]) => [method, count / simulations]))
|
|
645
|
-
}))
|
|
351
|
+
simulations,
|
|
352
|
+
seed,
|
|
353
|
+
power,
|
|
354
|
+
standardError: Math.sqrt(power * (1 - power) / simulations),
|
|
355
|
+
low: interval.lower,
|
|
356
|
+
high: interval.upper,
|
|
357
|
+
refuse: refusals / simulations,
|
|
358
|
+
hold: (simulations - promotions - refusals) / simulations,
|
|
359
|
+
calls: perCall.map((tally, c) => ({
|
|
360
|
+
outcome: calls[c].outcome,
|
|
361
|
+
promote: tally.promote / simulations,
|
|
362
|
+
insufficient: tally.insufficient / simulations,
|
|
363
|
+
indeterminate: tally.indeterminate / simulations,
|
|
364
|
+
exactTestVetoes: tally.exactTestVetoes / simulations,
|
|
365
|
+
hold: tally.hold / simulations,
|
|
366
|
+
methods: Object.fromEntries([...tally.methods.entries()].map(([method, count]) => [method, count / simulations]))
|
|
367
|
+
}))
|
|
646
368
|
};
|
|
647
369
|
}
|
|
648
370
|
/**
|
|
@@ -1363,92 +1085,633 @@ function pairedTest(input) {
|
|
|
1363
1085
|
seed: input.seed,
|
|
1364
1086
|
...estimator.kind === "binary" ? { binaryScale: estimator.scale } : { continuous: true }
|
|
1365
1087
|
});
|
|
1366
|
-
const test = {
|
|
1367
|
-
pairs,
|
|
1368
|
-
confidence: input.confidence,
|
|
1369
|
-
method: decision.method,
|
|
1370
|
-
delta:
|
|
1371
|
-
interval: input.maximize ? [
|
|
1372
|
-
};
|
|
1373
|
-
const complete = pairs === input.testUnits.length;
|
|
1374
|
-
const promote = decision.promote && complete;
|
|
1375
|
-
const why = !complete ? `it and the root scored ${pairs} of ${input.testUnits.length} test units, and a claim needs every one` : decision.promote ? "promoted" : !decision.sufficient ? `${pairs} test units are below the ${decision.minimumPairs} its paired test needs` : decision.indeterminate ? `its test interval has no width: ${decision.indeterminateCause}` : decision.exactTestVetoes ? "McNemar's exact test vetoes the improvement" : `its improvement interval [${round(decision.low)}, ${round(decision.high)}] at confidence ${round(input.confidence)} does not exclude 0`;
|
|
1088
|
+
const test = {
|
|
1089
|
+
pairs,
|
|
1090
|
+
confidence: input.confidence,
|
|
1091
|
+
method: decision.method,
|
|
1092
|
+
delta: normalizeSignedZero(input.maximize ? decision.delta : -decision.delta),
|
|
1093
|
+
interval: input.maximize ? [normalizeSignedZero(decision.low), normalizeSignedZero(decision.high)] : [normalizeSignedZero(-decision.high), normalizeSignedZero(-decision.low)]
|
|
1094
|
+
};
|
|
1095
|
+
const complete = pairs === input.testUnits.length;
|
|
1096
|
+
const promote = decision.promote && complete;
|
|
1097
|
+
const why = !complete ? `it and the root scored ${pairs} of ${input.testUnits.length} test units, and a claim needs every one` : decision.promote ? "promoted" : !decision.sufficient ? `${pairs} test units are below the ${decision.minimumPairs} its paired test needs` : decision.indeterminate ? `its test interval has no width: ${decision.indeterminateCause}` : decision.exactTestVetoes ? "McNemar's exact test vetoes the improvement" : `its improvement interval [${round(decision.low)}, ${round(decision.high)}] at confidence ${round(input.confidence)} does not exclude 0`;
|
|
1098
|
+
return {
|
|
1099
|
+
test,
|
|
1100
|
+
decision,
|
|
1101
|
+
promote,
|
|
1102
|
+
improvement: decision.delta,
|
|
1103
|
+
why
|
|
1104
|
+
};
|
|
1105
|
+
}
|
|
1106
|
+
function requireHeader(state) {
|
|
1107
|
+
const header = state.header;
|
|
1108
|
+
if (!header) throw new Error(`search ${state.searchId} has not been opened`);
|
|
1109
|
+
if (state.rootNodeId === null) throw new Error(`search ${state.searchId} has no root node`);
|
|
1110
|
+
return header;
|
|
1111
|
+
}
|
|
1112
|
+
function unitIds(tasks) {
|
|
1113
|
+
return [...new Set(tasks.map((task) => task.unitId))];
|
|
1114
|
+
}
|
|
1115
|
+
function mean(values) {
|
|
1116
|
+
let total = 0;
|
|
1117
|
+
for (const value of values) total += value;
|
|
1118
|
+
return total / values.length;
|
|
1119
|
+
}
|
|
1120
|
+
function round(value) {
|
|
1121
|
+
return Math.round(value * 1e4) / 1e4;
|
|
1122
|
+
}
|
|
1123
|
+
//#endregion
|
|
1124
|
+
//#region src/campaign/search-kernel-close.ts
|
|
1125
|
+
/** Final unclaimed selections and verification of the closed search receipt. */
|
|
1126
|
+
/** Decisions for a search without a held-out claim, derived before its ledger advances. */
|
|
1127
|
+
function unclaimedSearchDecisions(state, view, leader, context) {
|
|
1128
|
+
const { split } = view;
|
|
1129
|
+
const root = state.rootNodeId;
|
|
1130
|
+
const leaderUnits = view.unitScores(leader).map((unit) => unit.unitId);
|
|
1131
|
+
return state.nodes().filter((node) => !isTerminal(node)).map((node) => {
|
|
1132
|
+
const scored = state.scoredCells(node.nodeId, split).length > 0;
|
|
1133
|
+
const lead = node.nodeId === leader && scored;
|
|
1134
|
+
const against = node.nodeId === leader ? root : leader;
|
|
1135
|
+
const basis = node.nodeId === against || !scored ? null : estimateNode(state, node.nodeId, {
|
|
1136
|
+
against,
|
|
1137
|
+
split
|
|
1138
|
+
});
|
|
1139
|
+
const units = new Set(view.unitScores(node.nodeId).map((unit) => unit.unitId));
|
|
1140
|
+
return {
|
|
1141
|
+
nodeId: node.nodeId,
|
|
1142
|
+
decision: { status: lead ? "selected" : "rejected" },
|
|
1143
|
+
basis,
|
|
1144
|
+
rule: context.rule,
|
|
1145
|
+
reason: lead ? `the leader when the search stopped (${context.stopReason}): a budget decision on the ${split} split, which claims nothing` : !context.screened.has(node.nodeId) || !scored ? "the search stopped before this node was measured" : !view.complete(node.nodeId) ? "it left a unit unscored, so it could not lead" : leaderUnits.some((unitId) => !units.has(unitId)) ? "the search stopped before it was measured on every unit the leader was" : "it did not beat the leader on the units they share"
|
|
1146
|
+
};
|
|
1147
|
+
});
|
|
1148
|
+
}
|
|
1149
|
+
function closedSearchResult(state) {
|
|
1150
|
+
const claimVerification = verifySearchClaim(state);
|
|
1151
|
+
if (claimVerification?.status === "mismatch") throw new Error(`runSearch: search ${state.searchId} closed with a claim its ledger does not support: ${claimVerification.differences.join("; ")}`);
|
|
1152
|
+
return {
|
|
1153
|
+
state,
|
|
1154
|
+
leader: state.audit.selectedNodeId ?? state.rootNodeId,
|
|
1155
|
+
reason: state.closed.reason,
|
|
1156
|
+
claim: state.closed.claim,
|
|
1157
|
+
claimVerification
|
|
1158
|
+
};
|
|
1159
|
+
}
|
|
1160
|
+
function isTerminal(node) {
|
|
1161
|
+
return node.status !== null && node.status !== "advanced";
|
|
1162
|
+
}
|
|
1163
|
+
/** The kernel every `runSearch` ledger names as its search implementation. */
|
|
1164
|
+
const SEARCH_KERNEL_SOURCE = {
|
|
1165
|
+
uri: "npm:@tangle-network/agent-eval#runSearch",
|
|
1166
|
+
revision: hashCanonical({
|
|
1167
|
+
name: "tangle.search-kernel.2026-09",
|
|
1168
|
+
dispatch: "claim, then rung and root, then screen, then train; first allocated first within a stage",
|
|
1169
|
+
expansion: "when no cell waits, fewer than twice the lanes capacity run, the cap admits one proposal and its expected screens, and overspend has not taken the search past its cap",
|
|
1170
|
+
parents: "a policy names at least one parent, and every parent finished its screen",
|
|
1171
|
+
reservation: {
|
|
1172
|
+
hard: "the lane per-cell maximum",
|
|
1173
|
+
estimate: "1.5 times the p99 of the lane settled cells once 20 settled, else the lane prior"
|
|
1174
|
+
},
|
|
1175
|
+
retries: "a retryable errored attempt runs again, up to maxAttempts",
|
|
1176
|
+
rungs: "when a node finishes its cells, the allocator names the nodes it advances, deepest rung first; each is recorded advanced and its rung allocated when the cap admits the rung",
|
|
1177
|
+
resume: "an unrecorded operation is recorded failed at an unknown cost with floor 0; a recorded proposal is finished from its stored output; an unsettled cell is offered to adopt before it runs",
|
|
1178
|
+
writers: "one kernel per ledger file on a host, by a pid lock beside the ledger",
|
|
1179
|
+
divergence: "a node whose train mean over shared units rises above its parent while its descriptive or bootstrap selection interval against the parent lies wholly on the worse side is decided invalid when its cells finish",
|
|
1180
|
+
claim: "with a test split: after expansion stops, the claim plan is stored on the claim operation, the finalists are decided, the root and finalist test cells run together, and the claim decides; the kernel refuses to start a claim without a selection split, a minimumEffect, a pinned judge, or (under a cap) a claim reserve for the root and 3 finalists plus a cap for one screening round",
|
|
1181
|
+
close: "the allocator prunes the nodes left waiting on a rung, except the policy leader; then, with a test split, the claim decides; without one, the policy leader is selected when it has a scored cell; every other undecided node is rejected",
|
|
1182
|
+
complete: "a node leads only when it has a scored cell on the ranked split and no cell there ended unscored: a final unscored outcome, or an error with no attempt left"
|
|
1183
|
+
})
|
|
1184
|
+
};
|
|
1185
|
+
/** The operation that holds the claim's plan. */
|
|
1186
|
+
const CLAIM_OPERATION = "claim";
|
|
1187
|
+
const ESTIMATE_MARGIN = 1.5;
|
|
1188
|
+
/** Dispatch order: claim cells first, then rungs and the root, then screens, then train. */
|
|
1189
|
+
const STAGE_PRIORITY = {
|
|
1190
|
+
claim: 0,
|
|
1191
|
+
rung: 1,
|
|
1192
|
+
root: 1,
|
|
1193
|
+
screen: 2,
|
|
1194
|
+
train: 3,
|
|
1195
|
+
external: 3
|
|
1196
|
+
};
|
|
1197
|
+
/** The expansion index a kernel operation id encodes, or null. */
|
|
1198
|
+
function searchExpansionIndex(operationId) {
|
|
1199
|
+
const match = /^expand-(\d+)$/.exec(operationId);
|
|
1200
|
+
return match ? Number(match[1]) : null;
|
|
1201
|
+
}
|
|
1202
|
+
//#endregion
|
|
1203
|
+
//#region src/campaign/search-kernel-claim.ts
|
|
1204
|
+
/** The sealed-test lifecycle: preflight, durable plan, cell allocation, and final claim. */
|
|
1205
|
+
var SearchKernelClaim = class {
|
|
1206
|
+
host;
|
|
1207
|
+
plan = null;
|
|
1208
|
+
constructor(host) {
|
|
1209
|
+
this.host = host;
|
|
1210
|
+
}
|
|
1211
|
+
/** Refuse an unclaimable search before its first paid cell starts. */
|
|
1212
|
+
assertReady() {
|
|
1213
|
+
const header = this.host.state.header;
|
|
1214
|
+
const testTasks = header.splits.test.tasks.length;
|
|
1215
|
+
if (testTasks === 0) return;
|
|
1216
|
+
const refuse = (why) => {
|
|
1217
|
+
throw new Error(`runSearch: search ${this.host.recorder.searchId} has a test split, but ${why}`);
|
|
1218
|
+
};
|
|
1219
|
+
if (header.splits.selection.tasks.length === 0) refuse("no selection split to choose its finalists on");
|
|
1220
|
+
if (header.objective.claim.minimumEffect === void 0) refuse("its claim declares no minimumEffect for the power check");
|
|
1221
|
+
if ("unknown" in header.objective.judge) refuse("its judge is not pinned; a claim with an unknown judge cannot ship, and a resumed search could mix two judges");
|
|
1222
|
+
const { maxUsd, reservedClaimUsd } = header.budget;
|
|
1223
|
+
if (maxUsd === null) return;
|
|
1224
|
+
const cellUsd = this.host.maxCellReservation();
|
|
1225
|
+
const need = searchClaimReserveUsd({
|
|
1226
|
+
testTasks,
|
|
1227
|
+
reps: this.host.options.allocation.reps,
|
|
1228
|
+
cellUsd
|
|
1229
|
+
});
|
|
1230
|
+
if (reservedClaimUsd + 1e-9 < need) refuse(`its claim reserve $${reservedClaimUsd} is below the $${need} the root and 3 finalists need on ${testTasks} test tasks at $${cellUsd} a cell`);
|
|
1231
|
+
const rootCells = this.host.options.allocation.plan(this.host.state, this.host.state.rootNodeId, 0).length;
|
|
1232
|
+
const screens = (this.host.options.proposer.childrenPerProposal ?? 1) * this.host.options.allocation.screenSize(this.host.state);
|
|
1233
|
+
const round = this.host.operationReservation().usd + (rootCells + screens) * cellUsd;
|
|
1234
|
+
if (maxUsd - reservedClaimUsd + 1e-9 < round) refuse(`its cap $${maxUsd} less the claim reserve $${reservedClaimUsd} does not cover the root and one screening round ($${round})`);
|
|
1235
|
+
}
|
|
1236
|
+
/** Recover the exact durable claim design left by an earlier process. */
|
|
1237
|
+
async restore() {
|
|
1238
|
+
const operation = this.host.state.operation(CLAIM_OPERATION);
|
|
1239
|
+
if (!operation) return;
|
|
1240
|
+
const ref = operation.artifacts.find((artifact) => artifact.role === "claim-plan");
|
|
1241
|
+
if (!ref) throw new Error(`runSearch: the ${CLAIM_OPERATION} operation has no stored plan`);
|
|
1242
|
+
const plan = this.host.recorder.readBlob(ref);
|
|
1243
|
+
if (!operation.recorded) await this.recordClaimOperation();
|
|
1244
|
+
this.plan = plan;
|
|
1245
|
+
this.host.stopReason = plan.stopReason;
|
|
1246
|
+
}
|
|
1247
|
+
async begin() {
|
|
1248
|
+
await this.host.refresh();
|
|
1249
|
+
const stopReason = this.host.stopReason ?? "converged";
|
|
1250
|
+
const header = this.host.state.header;
|
|
1251
|
+
const reps = this.host.options.allocation.reps;
|
|
1252
|
+
const plan = planSearchClaim(this.host.state, {
|
|
1253
|
+
stopReason,
|
|
1254
|
+
reps,
|
|
1255
|
+
affordable: (finalists) => {
|
|
1256
|
+
const placed = this.claimCells(finalists, this.host.state.rootNodeId);
|
|
1257
|
+
const hold = placed.reduce((sum, cell) => sum + cell.reservation.usd, 0);
|
|
1258
|
+
const { headroomUsd, claimReserveUsd } = this.host.state.budget;
|
|
1259
|
+
const { maxCells } = header.budget;
|
|
1260
|
+
return (headroomUsd === null || hold <= headroomUsd + claimReserveUsd + 1e-9) && (maxCells === null || this.host.state.audit.cells.allocated + placed.length <= maxCells);
|
|
1261
|
+
}
|
|
1262
|
+
});
|
|
1263
|
+
await this.host.recorder.startOperation({
|
|
1264
|
+
operationId: CLAIM_OPERATION,
|
|
1265
|
+
operationKind: "selection",
|
|
1266
|
+
reservation: null,
|
|
1267
|
+
artifacts: [this.host.recorder.blob("claim-plan", plan)]
|
|
1268
|
+
});
|
|
1269
|
+
await this.recordClaimOperation();
|
|
1270
|
+
this.plan = plan;
|
|
1271
|
+
this.host.stopReason = stopReason;
|
|
1272
|
+
await this.apply(plan);
|
|
1273
|
+
}
|
|
1274
|
+
async recordClaimOperation() {
|
|
1275
|
+
await this.host.recorder.recordOperation({
|
|
1276
|
+
operationId: CLAIM_OPERATION,
|
|
1277
|
+
operationKind: "selection",
|
|
1278
|
+
execution: {
|
|
1279
|
+
kind: "deterministic",
|
|
1280
|
+
source: SEARCH_CLAIM_RULE
|
|
1281
|
+
},
|
|
1282
|
+
outcome: { status: "completed" },
|
|
1283
|
+
accounting: {
|
|
1284
|
+
tokens: {
|
|
1285
|
+
status: "known",
|
|
1286
|
+
inputTokens: 0,
|
|
1287
|
+
outputTokens: 0,
|
|
1288
|
+
cachedTokens: 0
|
|
1289
|
+
},
|
|
1290
|
+
cost: {
|
|
1291
|
+
status: "known",
|
|
1292
|
+
usd: 0,
|
|
1293
|
+
source: "free"
|
|
1294
|
+
}
|
|
1295
|
+
}
|
|
1296
|
+
});
|
|
1297
|
+
await this.host.refresh();
|
|
1298
|
+
}
|
|
1299
|
+
/** Decide finalists and allocate all interleaved test cells before any starts. Idempotent. */
|
|
1300
|
+
async apply(plan) {
|
|
1301
|
+
const root = this.host.state.rootNodeId;
|
|
1302
|
+
const bases = plan.finalists.map((finalist) => estimateNode(this.host.state, finalist.nodeId, {
|
|
1303
|
+
against: root,
|
|
1304
|
+
split: "selection"
|
|
1305
|
+
}));
|
|
1306
|
+
for (const [rank, finalist] of plan.finalists.entries()) await this.host.recorder.decideNode({
|
|
1307
|
+
nodeId: finalist.nodeId,
|
|
1308
|
+
decision: { status: "finalist" },
|
|
1309
|
+
basis: bases[rank],
|
|
1310
|
+
rule: SEARCH_CLAIM_RULE_NAME,
|
|
1311
|
+
reason: `rank ${rank + 1} of ${plan.finalists.length} by selection mean (${finalist.selectionMean}); ${plan.reason}`
|
|
1312
|
+
});
|
|
1313
|
+
await this.host.refresh();
|
|
1314
|
+
if (plan.test !== "run") return;
|
|
1315
|
+
const missing = this.claimCells(plan.finalists.length, root, plan).map((cell) => ({
|
|
1316
|
+
...cell,
|
|
1317
|
+
cellId: searchCellId(this.host.recorder.searchId, cell.nodeId, cell.plan.taskId, "test", cell.plan.rep)
|
|
1318
|
+
})).filter((cell) => !this.host.state.cell(cell.cellId));
|
|
1319
|
+
for (const cell of missing) await this.host.recorder.allocateCell({
|
|
1320
|
+
nodeId: cell.nodeId,
|
|
1321
|
+
taskId: cell.plan.taskId,
|
|
1322
|
+
split: "test",
|
|
1323
|
+
rep: cell.plan.rep,
|
|
1324
|
+
stage: "claim",
|
|
1325
|
+
lane: cell.lane.name,
|
|
1326
|
+
reservation: cell.reservation
|
|
1327
|
+
});
|
|
1328
|
+
await this.host.refresh();
|
|
1329
|
+
for (const cell of missing) this.host.enqueue(this.host.state.cell(cell.cellId));
|
|
1330
|
+
}
|
|
1331
|
+
claimCells(finalists, root, plan) {
|
|
1332
|
+
const header = this.host.state.header;
|
|
1333
|
+
const arms = [root, ...plan ? plan.finalists.map((entry) => entry.nodeId) : []];
|
|
1334
|
+
while (arms.length < finalists + 1) arms.push(root);
|
|
1335
|
+
const cells = [];
|
|
1336
|
+
for (const task of header.splits.test.tasks) for (let rep = 0; rep < this.host.options.allocation.reps; rep++) for (const nodeId of arms) {
|
|
1337
|
+
const cellPlan = {
|
|
1338
|
+
taskId: task.taskId,
|
|
1339
|
+
split: "test",
|
|
1340
|
+
rep,
|
|
1341
|
+
stage: "claim"
|
|
1342
|
+
};
|
|
1343
|
+
const laneName = this.host.options.executor.place({
|
|
1344
|
+
...cellPlan,
|
|
1345
|
+
nodeId
|
|
1346
|
+
});
|
|
1347
|
+
const lane = this.host.lanes.get(laneName);
|
|
1348
|
+
if (!lane) throw new Error(`runSearch: the executor placed a cell on unknown lane ${laneName}`);
|
|
1349
|
+
cells.push({
|
|
1350
|
+
nodeId,
|
|
1351
|
+
plan: cellPlan,
|
|
1352
|
+
lane,
|
|
1353
|
+
reservation: this.host.cellReservation(lane)
|
|
1354
|
+
});
|
|
1355
|
+
}
|
|
1356
|
+
return cells;
|
|
1357
|
+
}
|
|
1358
|
+
async close(plan) {
|
|
1359
|
+
const { claim, decisions } = decideSearchClaim(this.host.state, plan);
|
|
1360
|
+
const decided = new Set(decisions.map((decision) => decision.nodeId));
|
|
1361
|
+
const root = this.host.state.rootNodeId;
|
|
1362
|
+
const rest = this.host.state.nodes().filter((node) => !decided.has(node.nodeId) && !isTerminal(node)).map((node) => {
|
|
1363
|
+
const scored = this.host.state.scoredCells(node.nodeId, "selection").length > 0;
|
|
1364
|
+
return {
|
|
1365
|
+
nodeId: node.nodeId,
|
|
1366
|
+
decision: { status: "rejected" },
|
|
1367
|
+
basis: scored ? estimateNode(this.host.state, node.nodeId, {
|
|
1368
|
+
against: root,
|
|
1369
|
+
split: "selection"
|
|
1370
|
+
}) : null,
|
|
1371
|
+
rule: SEARCH_CLAIM_RULE_NAME,
|
|
1372
|
+
reason: scored ? "not a finalist: the claim tests at most 3 nodes that scored every selection unit and beat the root, best selection mean first" : "the search stopped before this node was measured on the selection split"
|
|
1373
|
+
};
|
|
1374
|
+
});
|
|
1375
|
+
for (const decision of [...decisions, ...rest]) await this.host.recorder.decideNode(decision);
|
|
1376
|
+
await this.host.recorder.close({
|
|
1377
|
+
reason: plan.stopReason,
|
|
1378
|
+
claim
|
|
1379
|
+
});
|
|
1380
|
+
await this.host.refresh();
|
|
1381
|
+
}
|
|
1382
|
+
};
|
|
1383
|
+
//#endregion
|
|
1384
|
+
//#region src/search/lenses/shared.ts
|
|
1385
|
+
/**
|
|
1386
|
+
* Shared, pure helpers for `src/search/lenses/*`.
|
|
1387
|
+
*
|
|
1388
|
+
* Every lens is a pure function of a `SearchStateView` (search-tree-design
|
|
1389
|
+
* §12): it reads no other record type and does no I/O. These helpers exist so
|
|
1390
|
+
* each lens states its own numbers instead of re-deriving thresholds; they
|
|
1391
|
+
* reuse agent-eval's own statistics and estimator revisions rather than
|
|
1392
|
+
* inventing new ones.
|
|
1393
|
+
*/
|
|
1394
|
+
/** The search's own ranking split: selection when the search declares one,
|
|
1395
|
+
* else train. Matches `searchPolicyView` and `agent-eval search show`, so a
|
|
1396
|
+
* lens ranks nodes on exactly the split the search itself ranks them on. */
|
|
1397
|
+
function rankingSplit(state) {
|
|
1398
|
+
const header = state.header;
|
|
1399
|
+
return header && header.splits.selection.tasks.length > 0 ? "selection" : "train";
|
|
1400
|
+
}
|
|
1401
|
+
/** The smallest sample an exact one-sided sign test can resolve at 95%: the
|
|
1402
|
+
* same gate `NodeEstimate.method` uses for `insufficient`. */
|
|
1403
|
+
const INSUFFICIENT_FROM = minimumPairsForPairedDeltaTest(.95);
|
|
1404
|
+
function summarizeSamples(samples, seed) {
|
|
1405
|
+
const n = samples.length;
|
|
1406
|
+
const method = searchEstimateMethod(n);
|
|
1407
|
+
if (method === "none") return {
|
|
1408
|
+
n,
|
|
1409
|
+
mean: null,
|
|
1410
|
+
method,
|
|
1411
|
+
interval: null,
|
|
1412
|
+
intervalMethod: null
|
|
1413
|
+
};
|
|
1414
|
+
const mean = samples.reduce((a, b) => a + b, 0) / n;
|
|
1415
|
+
if (method === "insufficient") return {
|
|
1416
|
+
n,
|
|
1417
|
+
mean,
|
|
1418
|
+
method,
|
|
1419
|
+
interval: null,
|
|
1420
|
+
intervalMethod: null
|
|
1421
|
+
};
|
|
1422
|
+
const { lower, upper } = confidenceInterval([...samples], .95, seed === void 0 ? {} : { seed });
|
|
1423
|
+
return {
|
|
1424
|
+
n,
|
|
1425
|
+
mean,
|
|
1426
|
+
method,
|
|
1427
|
+
interval: [lower, upper],
|
|
1428
|
+
intervalMethod: "percentile-bootstrap-of-mean"
|
|
1429
|
+
};
|
|
1430
|
+
}
|
|
1431
|
+
//#endregion
|
|
1432
|
+
//#region src/search/lenses/operator-yield.ts
|
|
1433
|
+
/**
|
|
1434
|
+
* `operatorYield(state)`: outcome counts and yield per known dollar, by the
|
|
1435
|
+
* operator each edge names (search-tree-design §12).
|
|
1436
|
+
*
|
|
1437
|
+
* A node counts once, under the operator of the edge that registered it. A
|
|
1438
|
+
* re-proposal (an edge into a node an earlier edge registered) adds a
|
|
1439
|
+
* proposal and a `reproposals` count to its operator, but no outcome and no
|
|
1440
|
+
* yield sample: it produced no new artifact and no new measurement, so
|
|
1441
|
+
* counting its node again would repeat one measurement as independent
|
|
1442
|
+
* evidence.
|
|
1443
|
+
*
|
|
1444
|
+
* "Yield" reuses `searchPosterior` (E3, `../../campaign/estimate-node`): its
|
|
1445
|
+
* per-node mean is already the node's improvement over the root, oriented so
|
|
1446
|
+
* larger is always better regardless of the objective's direction — this
|
|
1447
|
+
* lens adds no second orientation rule of its own. A child's sample is
|
|
1448
|
+
* `posteriorMean / screenCostUsd`: the gain over the known cost of producing
|
|
1449
|
+
* and screening the node, which is the operator's doing. That cost is the
|
|
1450
|
+
* node's share of its proposal (the candidate-generation operation's cost
|
|
1451
|
+
* split evenly over every child edge the operation recorded, re-proposals
|
|
1452
|
+
* included) plus its cells allocated before its first decision: the first
|
|
1453
|
+
* rung and the proposer's train feedback, which the allocator plans together
|
|
1454
|
+
* at admission. Cells a later rung or the claim adds come after an
|
|
1455
|
+
* `advanced` or `finalist` decision; they are the allocator's and the
|
|
1456
|
+
* claim's choice, not the operator's, so an operator whose children advance
|
|
1457
|
+
* is not charged for the measurement their success earned. A seed edge
|
|
1458
|
+
* proposed nothing, so the root's cost is its cells alone.
|
|
1459
|
+
*
|
|
1460
|
+
* A child enters the sample only when both numbers are known: the node
|
|
1461
|
+
* shares 2 or more units with the root (one shared unit is no estimate,
|
|
1462
|
+
* §6.4); the edge names a recorded proposal operation whose cost is known;
|
|
1463
|
+
* every screen cell has run and has a known cost (an unpriced or unrecorded
|
|
1464
|
+
* part never floors a node to an artificially cheap, inflated yield); and it
|
|
1465
|
+
* was not decided `invalid` (a judge integrity or admission failure: its
|
|
1466
|
+
* score is not evidence of improvement, and `editCredit` excludes such nodes
|
|
1467
|
+
* the same way). A $0-known-cost node is excluded from yield (division by
|
|
1468
|
+
* zero is not a yield), not treated as free.
|
|
1469
|
+
*/
|
|
1470
|
+
/** The operators an expansion may choose (`SearchExpansion.operator` in
|
|
1471
|
+
* `search-policy.ts`): every edge operator except `seed` (the search's own
|
|
1472
|
+
* start) and `derive` (a cross-search edge a proposer never chooses). */
|
|
1473
|
+
const EXPANSION_OPERATORS = [
|
|
1474
|
+
"draft",
|
|
1475
|
+
"improve",
|
|
1476
|
+
"debug",
|
|
1477
|
+
"merge"
|
|
1478
|
+
];
|
|
1479
|
+
/** The bandit's own gate: an operator's weight is used only once it has this
|
|
1480
|
+
* many yield-eligible outcomes; below it, the policy falls back to a fixed
|
|
1481
|
+
* weight for that operator. Same threshold `NodeEstimate.method` uses for
|
|
1482
|
+
* `insufficient` (`minimumPairsForPairedDeltaTest(0.95)` = 6). */
|
|
1483
|
+
const MIN_OUTCOMES_FOR_WEIGHT = 6;
|
|
1484
|
+
const YIELD_METHOD = "per registered node, searchPosterior's mean gain over the root divided by the known cost of producing and screening the node: its even share of the proposal operation's cost over the operation's child edges, plus its cells allocated before its first decision (first rung and train feedback; later rungs and the claim excluded); no sample when the node was decided invalid, shares fewer than 2 units with the root, or any part of that cost is unknown, unrecorded or zero";
|
|
1485
|
+
function operatorYield(state, options = {}) {
|
|
1486
|
+
const split = options.split ?? rankingSplit(state);
|
|
1487
|
+
const posteriorByNode = new Map(state.header && state.rootNodeId !== null ? searchPosterior(state, { split }).nodes.map((node) => [node.nodeId, node]) : []);
|
|
1488
|
+
const grouped = /* @__PURE__ */ new Map();
|
|
1489
|
+
const group = (operator) => {
|
|
1490
|
+
let existing = grouped.get(operator);
|
|
1491
|
+
if (!existing) {
|
|
1492
|
+
existing = {
|
|
1493
|
+
proposals: 0,
|
|
1494
|
+
reproposals: 0,
|
|
1495
|
+
outcomes: {
|
|
1496
|
+
advanced: 0,
|
|
1497
|
+
finalist: 0,
|
|
1498
|
+
selected: 0,
|
|
1499
|
+
pruned: 0,
|
|
1500
|
+
rejected: 0,
|
|
1501
|
+
invalid: 0,
|
|
1502
|
+
undecided: 0
|
|
1503
|
+
},
|
|
1504
|
+
excluded: 0,
|
|
1505
|
+
samples: []
|
|
1506
|
+
};
|
|
1507
|
+
grouped.set(operator, existing);
|
|
1508
|
+
}
|
|
1509
|
+
return existing;
|
|
1510
|
+
};
|
|
1511
|
+
const edges = state.edges();
|
|
1512
|
+
const childEdges = /* @__PURE__ */ new Map();
|
|
1513
|
+
for (const edge of edges) {
|
|
1514
|
+
const operationId = edge.proposer?.operationId;
|
|
1515
|
+
if (operationId) childEdges.set(operationId, (childEdges.get(operationId) ?? 0) + 1);
|
|
1516
|
+
}
|
|
1517
|
+
for (const edge of edges) {
|
|
1518
|
+
const child = state.node(edge.childNodeId);
|
|
1519
|
+
if (!child) continue;
|
|
1520
|
+
const bucket = group(edge.operator);
|
|
1521
|
+
bucket.proposals += 1;
|
|
1522
|
+
if (child.edgeIds[0] !== edge.edgeId) {
|
|
1523
|
+
bucket.reproposals += 1;
|
|
1524
|
+
continue;
|
|
1525
|
+
}
|
|
1526
|
+
bucket.outcomes[statusKey(child.status)] += 1;
|
|
1527
|
+
const posterior = posteriorByNode.get(child.nodeId);
|
|
1528
|
+
const costUsd = screenCostUsd(state, child, edge, childEdges);
|
|
1529
|
+
if (child.status === "invalid" || !posterior || posterior.mean === null || posterior.pairs < 2 || costUsd === null || costUsd <= 0) {
|
|
1530
|
+
bucket.excluded += 1;
|
|
1531
|
+
continue;
|
|
1532
|
+
}
|
|
1533
|
+
bucket.samples.push(posterior.mean / costUsd);
|
|
1534
|
+
}
|
|
1535
|
+
const rows = [...grouped.entries()].sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0).map(([operator, bucket]) => ({
|
|
1536
|
+
operator,
|
|
1537
|
+
proposals: bucket.proposals,
|
|
1538
|
+
reproposals: bucket.reproposals,
|
|
1539
|
+
outcomes: bucket.outcomes,
|
|
1540
|
+
excluded: bucket.excluded,
|
|
1541
|
+
yield: summarizeSamples(bucket.samples)
|
|
1542
|
+
}));
|
|
1543
|
+
const signalValue = {};
|
|
1544
|
+
for (const operator of EXPANSION_OPERATORS) {
|
|
1545
|
+
const row = rows.find((r) => r.operator === operator);
|
|
1546
|
+
signalValue[operator] = row && row.yield.n >= 6 && row.yield.mean !== null ? row.yield.mean : null;
|
|
1547
|
+
}
|
|
1548
|
+
return {
|
|
1549
|
+
data: {
|
|
1550
|
+
split,
|
|
1551
|
+
method: YIELD_METHOD,
|
|
1552
|
+
rows
|
|
1553
|
+
},
|
|
1554
|
+
signal: {
|
|
1555
|
+
name: "operatorYield.weights",
|
|
1556
|
+
value: signalValue
|
|
1557
|
+
}
|
|
1558
|
+
};
|
|
1559
|
+
}
|
|
1560
|
+
/** The known cost of producing and screening `node`, which `edge` registered;
|
|
1561
|
+
* null when any part of it is unknown. The screen ends at the node's first
|
|
1562
|
+
* decision: the allocator records `advanced` before it allocates a further
|
|
1563
|
+
* rung, and the claim decides `finalist` before it allocates test cells. */
|
|
1564
|
+
function screenCostUsd(state, node, edge, childEdges) {
|
|
1565
|
+
let usd = 0;
|
|
1566
|
+
if (edge.proposer !== null) {
|
|
1567
|
+
const { operationId } = edge.proposer;
|
|
1568
|
+
if (operationId === null) return null;
|
|
1569
|
+
const operation = state.operation(operationId);
|
|
1570
|
+
if (!operation?.recorded || !operation.costKnown) return null;
|
|
1571
|
+
usd += operation.spentUsd / childEdges.get(operationId);
|
|
1572
|
+
}
|
|
1573
|
+
const screenEnd = node.decisions[0]?.sequence ?? Number.POSITIVE_INFINITY;
|
|
1574
|
+
for (const cell of state.cells({ nodeId: node.nodeId })) {
|
|
1575
|
+
if (cell.allocatedSequence > screenEnd) continue;
|
|
1576
|
+
if (!cell.costKnown || cell.attempts === 0 && cell.cancelled === null) return null;
|
|
1577
|
+
usd += cell.spentUsd;
|
|
1578
|
+
}
|
|
1579
|
+
return usd;
|
|
1580
|
+
}
|
|
1581
|
+
function statusKey(status) {
|
|
1582
|
+
return status ?? "undecided";
|
|
1583
|
+
}
|
|
1584
|
+
//#endregion
|
|
1585
|
+
//#region src/campaign/search-kernel-policy-view.ts
|
|
1586
|
+
/** Policy observations and divergence decisions from train and selection evidence. */
|
|
1587
|
+
/**
|
|
1588
|
+
* A policy's read of a search: `screened` lists the admitted nodes whose screen
|
|
1589
|
+
* finished, in registration order; `screening` counts the admitted nodes still
|
|
1590
|
+
* being screened. Nodes are ranked on the selection split, or on train when
|
|
1591
|
+
* the search declares no selection split. The view reads no test cell.
|
|
1592
|
+
* `maxAttempts` (default 3, the kernel's) says when an errored cell has no
|
|
1593
|
+
* attempt left, which makes its unit a dodge.
|
|
1594
|
+
*/
|
|
1595
|
+
function searchPolicyView(state, input) {
|
|
1596
|
+
const maxAttempts = input.maxAttempts ?? 3;
|
|
1597
|
+
const header = state.header;
|
|
1598
|
+
if (!header) throw new Error(`search ${state.searchId} has not been opened`);
|
|
1599
|
+
const split = header.splits.selection.tasks.length > 0 ? "selection" : "train";
|
|
1600
|
+
let nodes;
|
|
1601
|
+
let operatorWeights;
|
|
1602
|
+
return {
|
|
1603
|
+
searchId: state.searchId,
|
|
1604
|
+
seed: header.policy.seed,
|
|
1605
|
+
direction: header.objective.direction,
|
|
1606
|
+
split,
|
|
1607
|
+
rootNodeId: state.rootNodeId,
|
|
1608
|
+
expansions: input.expansions,
|
|
1609
|
+
screened: [...input.screened],
|
|
1610
|
+
screening: input.screening,
|
|
1611
|
+
complete: (nodeId) => {
|
|
1612
|
+
let scored = false;
|
|
1613
|
+
for (const cell of state.cells({ nodeId })) {
|
|
1614
|
+
if (cell.split !== split) continue;
|
|
1615
|
+
if (cell.score !== null) scored = true;
|
|
1616
|
+
else if (cell.final || cell.outcome === "errored" && cell.attempts >= maxAttempts) return false;
|
|
1617
|
+
}
|
|
1618
|
+
return scored;
|
|
1619
|
+
},
|
|
1620
|
+
unitScores: (nodeId) => state.unitScores(nodeId, split),
|
|
1621
|
+
estimate: (nodeId, against) => estimateNode(state, nodeId, {
|
|
1622
|
+
against,
|
|
1623
|
+
split
|
|
1624
|
+
}),
|
|
1625
|
+
nodes: () => {
|
|
1626
|
+
nodes ??= policyNodes(state);
|
|
1627
|
+
return nodes;
|
|
1628
|
+
},
|
|
1629
|
+
get posterior() {
|
|
1630
|
+
let posterior = POSTERIORS.get(state);
|
|
1631
|
+
if (!posterior) {
|
|
1632
|
+
posterior = searchPosterior(state, { split });
|
|
1633
|
+
POSTERIORS.set(state, posterior);
|
|
1634
|
+
}
|
|
1635
|
+
return posterior;
|
|
1636
|
+
},
|
|
1637
|
+
get operatorWeights() {
|
|
1638
|
+
operatorWeights ??= operatorYield(state, { split }).signal.value;
|
|
1639
|
+
return operatorWeights;
|
|
1640
|
+
}
|
|
1641
|
+
};
|
|
1642
|
+
}
|
|
1643
|
+
/** One posterior per ledger state: a state view is retired when the ledger
|
|
1644
|
+
* moves on, so the view it was computed from names the sequence. */
|
|
1645
|
+
const POSTERIORS = /* @__PURE__ */ new WeakMap();
|
|
1646
|
+
/** Every node whose edge is recorded, with its lineage and its defects outside
|
|
1647
|
+
* the test split. */
|
|
1648
|
+
function policyNodes(state) {
|
|
1649
|
+
const nodes = [];
|
|
1650
|
+
for (const node of state.nodes()) {
|
|
1651
|
+
const edgeId = node.edgeIds[0];
|
|
1652
|
+
if (edgeId === void 0) continue;
|
|
1653
|
+
let outcomes = 0;
|
|
1654
|
+
let defects = 0;
|
|
1655
|
+
for (const cell of state.cells({ nodeId: node.nodeId })) {
|
|
1656
|
+
if (cell.split === "test" || !cell.final) continue;
|
|
1657
|
+
outcomes += 1;
|
|
1658
|
+
if (cell.outcome === "failed") defects += 1;
|
|
1659
|
+
}
|
|
1660
|
+
const edge = state.edge(edgeId);
|
|
1661
|
+
nodes.push({
|
|
1662
|
+
nodeId: node.nodeId,
|
|
1663
|
+
ordinal: node.ordinal,
|
|
1664
|
+
parent: node.primaryParentId,
|
|
1665
|
+
operator: edge.operator,
|
|
1666
|
+
rule: edge.selection?.rule ?? null,
|
|
1667
|
+
children: node.children,
|
|
1668
|
+
status: node.status,
|
|
1669
|
+
outcomes,
|
|
1670
|
+
defects
|
|
1671
|
+
});
|
|
1672
|
+
}
|
|
1673
|
+
return nodes;
|
|
1674
|
+
}
|
|
1675
|
+
/**
|
|
1676
|
+
* The divergence rule: a node whose train mean rose over its parent's on the
|
|
1677
|
+
* train units they share while its selection interval against the parent (6
|
|
1678
|
+
* or more units, not indeterminate) lies wholly on the worse side. Train is
|
|
1679
|
+
* what the proposer reads; a gain there that selection contradicts is the
|
|
1680
|
+
* signature of fitting the feedback instead of the task.
|
|
1681
|
+
*/
|
|
1682
|
+
function searchDivergence(state, nodeId) {
|
|
1683
|
+
const header = state.header;
|
|
1684
|
+
const parent = state.node(nodeId)?.primaryParentId ?? null;
|
|
1685
|
+
if (!header || parent === null || header.splits.selection.tasks.length === 0) return null;
|
|
1686
|
+
const selection = estimateNode(state, nodeId, {
|
|
1687
|
+
against: parent,
|
|
1688
|
+
split: "selection"
|
|
1689
|
+
});
|
|
1690
|
+
if (selection.interval === null) return null;
|
|
1691
|
+
const maximize = header.objective.direction === "maximize";
|
|
1692
|
+
const [low, high] = selection.interval;
|
|
1693
|
+
if (maximize ? high >= 0 : low <= 0) return null;
|
|
1694
|
+
const parentTrain = new Map(state.unitScores(parent, "train").map((unit) => [unit.unitId, unit.mean]));
|
|
1695
|
+
let gain = 0;
|
|
1696
|
+
let shared = 0;
|
|
1697
|
+
for (const unit of state.unitScores(nodeId, "train")) {
|
|
1698
|
+
const before = parentTrain.get(unit.unitId);
|
|
1699
|
+
if (before === void 0) continue;
|
|
1700
|
+
gain += maximize ? unit.mean - before : before - unit.mean;
|
|
1701
|
+
shared += 1;
|
|
1702
|
+
}
|
|
1703
|
+
if (shared === 0 || gain <= 0) return null;
|
|
1376
1704
|
return {
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
promote,
|
|
1380
|
-
improvement: decision.delta,
|
|
1381
|
-
why
|
|
1705
|
+
basis: selection,
|
|
1706
|
+
reason: `its train mean rose ${Math.round(gain / shared * 1e4) / 1e4} over its parent's on ${shared} shared train unit${shared === 1 ? "" : "s"}, while its selection interval against the parent, [${round4(low)}, ${round4(high)}] on ${selection.pairs} units, lies wholly on the worse side`
|
|
1382
1707
|
};
|
|
1383
1708
|
}
|
|
1384
|
-
function
|
|
1385
|
-
const header = state.header;
|
|
1386
|
-
if (!header) throw new Error(`search ${state.searchId} has not been opened`);
|
|
1387
|
-
if (state.rootNodeId === null) throw new Error(`search ${state.searchId} has no root node`);
|
|
1388
|
-
return header;
|
|
1389
|
-
}
|
|
1390
|
-
function unitIds(tasks) {
|
|
1391
|
-
return [...new Set(tasks.map((task) => task.unitId))];
|
|
1392
|
-
}
|
|
1393
|
-
function mean(values) {
|
|
1394
|
-
let total = 0;
|
|
1395
|
-
for (const value of values) total += value;
|
|
1396
|
-
return total / values.length;
|
|
1397
|
-
}
|
|
1398
|
-
function round(value) {
|
|
1709
|
+
function round4(value) {
|
|
1399
1710
|
return Math.round(value * 1e4) / 1e4;
|
|
1400
1711
|
}
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
}
|
|
1405
|
-
/** The kernel every `runSearch` ledger names as its search implementation. */
|
|
1406
|
-
const SEARCH_KERNEL_SOURCE = {
|
|
1407
|
-
uri: "npm:@tangle-network/agent-eval#runSearch",
|
|
1408
|
-
revision: hashCanonical({
|
|
1409
|
-
name: "tangle.search-kernel.2026-09",
|
|
1410
|
-
dispatch: "claim, then rung and root, then screen, then train; first allocated first within a stage",
|
|
1411
|
-
expansion: "when no cell waits, fewer than twice the lanes capacity run, the cap admits one proposal and its expected screens, and overspend has not taken the search past its cap",
|
|
1412
|
-
parents: "a policy names at least one parent, and every parent finished its screen",
|
|
1413
|
-
reservation: {
|
|
1414
|
-
hard: "the lane per-cell maximum",
|
|
1415
|
-
estimate: "1.5 times the p99 of the lane settled cells once 20 settled, else the lane prior"
|
|
1416
|
-
},
|
|
1417
|
-
retries: "a retryable errored attempt runs again, up to maxAttempts",
|
|
1418
|
-
rungs: "when a node finishes its cells, the allocator names the nodes it advances, deepest rung first; each is recorded advanced and its rung allocated when the cap admits the rung",
|
|
1419
|
-
resume: "an unrecorded operation is recorded failed at an unknown cost with floor 0; a recorded proposal is finished from its stored output; an unsettled cell is offered to adopt before it runs",
|
|
1420
|
-
writers: "one kernel per ledger file on a host, by a pid lock beside the ledger",
|
|
1421
|
-
divergence: "a node whose train mean over shared units rises above its parent while its descriptive or bootstrap selection interval against the parent lies wholly on the worse side is decided invalid when its cells finish",
|
|
1422
|
-
claim: "with a test split: after expansion stops, the claim plan is stored on the claim operation, the finalists are decided, the root and finalist test cells run together, and the claim decides; the kernel refuses to start a claim without a selection split, a minimumEffect, a pinned judge, or (under a cap) a claim reserve for the root and 3 finalists plus a cap for one screening round",
|
|
1423
|
-
close: "the allocator prunes the nodes left waiting on a rung, except the policy leader; then, with a test split, the claim decides; without one, the policy leader is selected when it has a scored cell; every other undecided node is rejected",
|
|
1424
|
-
complete: "a node leads only when it has a scored cell on the ranked split and no cell there ended unscored: a final unscored outcome, or an error with no attempt left"
|
|
1425
|
-
})
|
|
1426
|
-
};
|
|
1427
|
-
const USD_TOLERANCE = 1e-9;
|
|
1428
|
-
/** The operation that holds the claim's plan. */
|
|
1429
|
-
const CLAIM_OPERATION = "claim";
|
|
1430
|
-
/** Settled cells a lane needs before its own cost distribution sets its estimate. */
|
|
1431
|
-
const ESTIMATE_FROM_CELLS = 20;
|
|
1432
|
-
const ESTIMATE_MARGIN = 1.5;
|
|
1433
|
-
const DEFAULT_MAX_ATTEMPTS = 3;
|
|
1434
|
-
/** Dispatch order: claim cells first, then rungs and the root, then screens, then train. */
|
|
1435
|
-
const STAGE_PRIORITY = {
|
|
1436
|
-
claim: 0,
|
|
1437
|
-
rung: 1,
|
|
1438
|
-
root: 1,
|
|
1439
|
-
screen: 2,
|
|
1440
|
-
train: 3,
|
|
1441
|
-
external: 3
|
|
1442
|
-
};
|
|
1443
|
-
/** Run a search to its close, or continue one from its ledger. */
|
|
1444
|
-
async function runSearch(options) {
|
|
1445
|
-
return new SearchKernel(options).run();
|
|
1446
|
-
}
|
|
1447
|
-
/** The expansion index a kernel operation id encodes, or null. */
|
|
1448
|
-
function searchExpansionIndex(operationId) {
|
|
1449
|
-
const match = /^expand-(\d+)$/.exec(operationId);
|
|
1450
|
-
return match ? Number(match[1]) : null;
|
|
1451
|
-
}
|
|
1712
|
+
//#endregion
|
|
1713
|
+
//#region src/campaign/search-kernel-runner.ts
|
|
1714
|
+
/** Event loop and mutable scheduling state for one ledger-backed search. */
|
|
1452
1715
|
var SearchKernel = class {
|
|
1453
1716
|
options;
|
|
1454
1717
|
recorder;
|
|
@@ -1483,11 +1746,11 @@ var SearchKernel = class {
|
|
|
1483
1746
|
stopReason = null;
|
|
1484
1747
|
failure = null;
|
|
1485
1748
|
/** The claim's design once expansion stopped; the search then only claims. */
|
|
1486
|
-
|
|
1749
|
+
claim;
|
|
1487
1750
|
constructor(options) {
|
|
1488
1751
|
this.options = options;
|
|
1489
1752
|
this.recorder = options.recorder;
|
|
1490
|
-
this.maxAttempts = options.maxAttempts ??
|
|
1753
|
+
this.maxAttempts = options.maxAttempts ?? 3;
|
|
1491
1754
|
this.now = options.now ?? Date.now;
|
|
1492
1755
|
if (!Number.isSafeInteger(this.maxAttempts) || this.maxAttempts < 1) throw new TypeError("runSearch: maxAttempts must be a positive integer");
|
|
1493
1756
|
const { maxExpansions } = options;
|
|
@@ -1509,6 +1772,26 @@ var SearchKernel = class {
|
|
|
1509
1772
|
});
|
|
1510
1773
|
}
|
|
1511
1774
|
if (this.lanes.size === 0) throw new TypeError("runSearch: the executor declares no lane");
|
|
1775
|
+
const kernel = this;
|
|
1776
|
+
this.claim = new SearchKernelClaim({
|
|
1777
|
+
get state() {
|
|
1778
|
+
return kernel.state;
|
|
1779
|
+
},
|
|
1780
|
+
get stopReason() {
|
|
1781
|
+
return kernel.stopReason;
|
|
1782
|
+
},
|
|
1783
|
+
set stopReason(reason) {
|
|
1784
|
+
kernel.stopReason = reason;
|
|
1785
|
+
},
|
|
1786
|
+
recorder: this.recorder,
|
|
1787
|
+
options,
|
|
1788
|
+
lanes: this.lanes,
|
|
1789
|
+
refresh: () => this.refresh(),
|
|
1790
|
+
cellReservation: (lane) => this.cellReservation(this.lanes.get(lane.name)),
|
|
1791
|
+
maxCellReservation: () => this.maxCellReservation(),
|
|
1792
|
+
operationReservation: () => this.operationReservation(),
|
|
1793
|
+
enqueue: (cell) => this.enqueue(cell)
|
|
1794
|
+
});
|
|
1512
1795
|
}
|
|
1513
1796
|
async run() {
|
|
1514
1797
|
const { ledger } = this.recorder;
|
|
@@ -1524,7 +1807,7 @@ var SearchKernel = class {
|
|
|
1524
1807
|
const header = this.state.header;
|
|
1525
1808
|
if (!header) throw new Error(`runSearch: search ${this.recorder.searchId} has not been opened`);
|
|
1526
1809
|
if (header.policy.expansion !== this.options.policy.name || header.policy.allocation !== this.options.allocation.name) throw new Error(`runSearch: search ${this.recorder.searchId} was opened with policy ${header.policy.expansion} and allocation ${header.policy.allocation}, not ${this.options.policy.name} and ${this.options.allocation.name}`);
|
|
1527
|
-
if (this.state.closed) return this.
|
|
1810
|
+
if (this.state.closed) return closedSearchResult(this.state);
|
|
1528
1811
|
await this.restore();
|
|
1529
1812
|
const { signal } = this.options;
|
|
1530
1813
|
const onAbort = () => {
|
|
@@ -1534,10 +1817,10 @@ var SearchKernel = class {
|
|
|
1534
1817
|
};
|
|
1535
1818
|
signal?.addEventListener("abort", onAbort, { once: true });
|
|
1536
1819
|
try {
|
|
1537
|
-
if (this.
|
|
1820
|
+
if (this.claim.plan) await this.claim.apply(this.claim.plan);
|
|
1538
1821
|
await this.loop();
|
|
1539
|
-
if (!this.failure && !this.
|
|
1540
|
-
await this.
|
|
1822
|
+
if (!this.failure && !this.claim.plan && this.state.header.splits.test.tasks.length > 0) {
|
|
1823
|
+
await this.claim.begin();
|
|
1541
1824
|
await this.loop();
|
|
1542
1825
|
}
|
|
1543
1826
|
} finally {
|
|
@@ -1545,7 +1828,7 @@ var SearchKernel = class {
|
|
|
1545
1828
|
}
|
|
1546
1829
|
if (this.failure) throw this.failure.error;
|
|
1547
1830
|
await this.close();
|
|
1548
|
-
return this.
|
|
1831
|
+
return closedSearchResult(this.state);
|
|
1549
1832
|
}
|
|
1550
1833
|
async restore() {
|
|
1551
1834
|
const rootInput = this.options.codec.node(this.recorder, this.options.root);
|
|
@@ -1568,8 +1851,8 @@ var SearchKernel = class {
|
|
|
1568
1851
|
const root = this.state.rootNodeId;
|
|
1569
1852
|
this.artifacts.set(root, this.options.root);
|
|
1570
1853
|
this.expansionOf.set(root, -1);
|
|
1571
|
-
if (this.state.audit.cells.allocated === 0) this.
|
|
1572
|
-
await this.
|
|
1854
|
+
if (this.state.audit.cells.allocated === 0) this.claim.assertReady();
|
|
1855
|
+
await this.claim.restore();
|
|
1573
1856
|
const recorded = [];
|
|
1574
1857
|
for (let expansion = 0;; expansion++) {
|
|
1575
1858
|
const operationId = `expand-${expansion}`;
|
|
@@ -1613,7 +1896,7 @@ var SearchKernel = class {
|
|
|
1613
1896
|
this.enqueue(cell);
|
|
1614
1897
|
if (cell.stage !== "claim") this.pending.set(cell.nodeId, (this.pending.get(cell.nodeId) ?? 0) + 1);
|
|
1615
1898
|
}
|
|
1616
|
-
if (this.
|
|
1899
|
+
if (this.claim.plan) return;
|
|
1617
1900
|
for (const expansion of recorded) await this.finishProposal(expansion);
|
|
1618
1901
|
for (const node of this.state.nodes()) if (this.admitted.has(node.nodeId) && !isTerminal(node)) await this.allocateFor(node.nodeId);
|
|
1619
1902
|
for (const nodeId of this.state.nodeIds()) {
|
|
@@ -1623,49 +1906,6 @@ var SearchKernel = class {
|
|
|
1623
1906
|
}
|
|
1624
1907
|
await this.advance();
|
|
1625
1908
|
}
|
|
1626
|
-
/**
|
|
1627
|
-
* A search that will claim must be able to: finalists come from a selection
|
|
1628
|
-
* split, the power check needs the claim's minimum effect, a ship needs a
|
|
1629
|
-
* pinned judge, and under a cap the claim reserve must cover the root and 3
|
|
1630
|
-
* finalists on every test task while the rest of the cap covers one
|
|
1631
|
-
* screening round. Checked before the first cell, so a search that cannot
|
|
1632
|
-
* claim spends nothing.
|
|
1633
|
-
*/
|
|
1634
|
-
assertClaimReady() {
|
|
1635
|
-
const header = this.state.header;
|
|
1636
|
-
const testTasks = header.splits.test.tasks.length;
|
|
1637
|
-
if (testTasks === 0) return;
|
|
1638
|
-
const refuse = (why) => {
|
|
1639
|
-
throw new Error(`runSearch: search ${this.recorder.searchId} has a test split, but ${why}`);
|
|
1640
|
-
};
|
|
1641
|
-
if (header.splits.selection.tasks.length === 0) refuse("no selection split to choose its finalists on");
|
|
1642
|
-
if (header.objective.claim.minimumEffect === void 0) refuse("its claim declares no minimumEffect for the power check");
|
|
1643
|
-
if ("unknown" in header.objective.judge) refuse("its judge is not pinned; a claim with an unknown judge cannot ship, and a resumed search could mix two judges");
|
|
1644
|
-
const { maxUsd, reservedClaimUsd } = header.budget;
|
|
1645
|
-
if (maxUsd === null) return;
|
|
1646
|
-
const cellUsd = this.maxCellReservation();
|
|
1647
|
-
const need = searchClaimReserveUsd({
|
|
1648
|
-
testTasks,
|
|
1649
|
-
reps: this.options.allocation.reps,
|
|
1650
|
-
cellUsd
|
|
1651
|
-
});
|
|
1652
|
-
if (reservedClaimUsd + USD_TOLERANCE < need) refuse(`its claim reserve $${reservedClaimUsd} is below the $${need} the root and 3 finalists need on ${testTasks} test tasks at $${cellUsd} a cell`);
|
|
1653
|
-
const rootCells = this.options.allocation.plan(this.state, this.state.rootNodeId, 0).length;
|
|
1654
|
-
const screens = (this.options.proposer.childrenPerProposal ?? 1) * this.options.allocation.screenSize(this.state);
|
|
1655
|
-
const round = this.operationReservation().usd + (rootCells + screens) * cellUsd;
|
|
1656
|
-
if (maxUsd - reservedClaimUsd + USD_TOLERANCE < round) refuse(`its cap $${maxUsd} less the claim reserve $${reservedClaimUsd} does not cover the root and one screening round ($${round})`);
|
|
1657
|
-
}
|
|
1658
|
-
/** A claim an earlier process began: continue it from its stored plan. */
|
|
1659
|
-
async restoreClaimPlan() {
|
|
1660
|
-
const operation = this.state.operation(CLAIM_OPERATION);
|
|
1661
|
-
if (!operation) return;
|
|
1662
|
-
const ref = operation.artifacts.find((artifact) => artifact.role === "claim-plan");
|
|
1663
|
-
if (!ref) throw new Error(`runSearch: the ${CLAIM_OPERATION} operation has no stored plan`);
|
|
1664
|
-
const plan = this.recorder.readBlob(ref);
|
|
1665
|
-
if (!operation.recorded) await this.recordClaimOperation();
|
|
1666
|
-
this.claimPlan = plan;
|
|
1667
|
-
this.stopReason = plan.stopReason;
|
|
1668
|
-
}
|
|
1669
1909
|
async loop() {
|
|
1670
1910
|
for (;;) {
|
|
1671
1911
|
if (this.failure || this.options.signal?.aborted) {
|
|
@@ -1801,7 +2041,7 @@ var SearchKernel = class {
|
|
|
1801
2041
|
* search fails or is aborted, no rung opens; a search that stopped
|
|
1802
2042
|
* expanding still measures the nodes that earned it. */
|
|
1803
2043
|
async advance() {
|
|
1804
|
-
if (this.
|
|
2044
|
+
if (this.claim.plan || this.failure || this.options.signal?.aborted || this.pastDeadline()) return;
|
|
1805
2045
|
for (const decision of this.options.allocation.advance(this.allocationView())) await this.allocateFor(decision.nodeId, decision);
|
|
1806
2046
|
}
|
|
1807
2047
|
/**
|
|
@@ -1811,7 +2051,7 @@ var SearchKernel = class {
|
|
|
1811
2051
|
* becomes a parent; any other node is screened. Returns whether it was.
|
|
1812
2052
|
*/
|
|
1813
2053
|
async screenDone(nodeId) {
|
|
1814
|
-
if (!this.
|
|
2054
|
+
if (!this.claim.plan && !this.screenedSet.has(nodeId) && this.state.node(nodeId).status !== "advanced") {
|
|
1815
2055
|
const divergence = searchDivergence(this.state, nodeId);
|
|
1816
2056
|
if (divergence) {
|
|
1817
2057
|
await this.recorder.decideNode({
|
|
@@ -1869,7 +2109,7 @@ var SearchKernel = class {
|
|
|
1869
2109
|
});
|
|
1870
2110
|
const hold = placed.reduce((sum, { reservation }) => sum + reservation.usd, 0);
|
|
1871
2111
|
const { headroomUsd } = this.state.budget;
|
|
1872
|
-
if (headroomUsd !== null && hold > headroomUsd +
|
|
2112
|
+
if (headroomUsd !== null && hold > headroomUsd + 1e-9) return false;
|
|
1873
2113
|
const { maxCells } = this.state.header.budget;
|
|
1874
2114
|
if (maxCells !== null && this.state.audit.cells.allocated + placed.length > maxCells) return false;
|
|
1875
2115
|
if (decision) await this.decide(decision);
|
|
@@ -1883,143 +2123,25 @@ var SearchKernel = class {
|
|
|
1883
2123
|
reservation
|
|
1884
2124
|
});
|
|
1885
2125
|
await this.refresh();
|
|
1886
|
-
for (const { plan } of placed) {
|
|
1887
|
-
const cellId = searchCellId(this.recorder.searchId, nodeId, plan.taskId, plan.split, plan.rep);
|
|
1888
|
-
this.enqueue(this.state.cell(cellId));
|
|
1889
|
-
}
|
|
1890
|
-
this.pending.set(nodeId, (this.pending.get(nodeId) ?? 0) + placed.length);
|
|
1891
|
-
this.dispatch();
|
|
1892
|
-
return true;
|
|
1893
|
-
}
|
|
1894
|
-
async decide(decision) {
|
|
1895
|
-
await this.recorder.decideNode(decision);
|
|
1896
|
-
await this.refresh();
|
|
1897
|
-
}
|
|
1898
|
-
enqueue(cell) {
|
|
1899
|
-
this.lanes.get(cell.lane).queues[STAGE_PRIORITY[cell.stage]].push(cell.cellId);
|
|
1900
|
-
}
|
|
1901
|
-
queuedCount() {
|
|
1902
|
-
let count = 0;
|
|
1903
|
-
for (const lane of this.lanes.values()) for (const queue of lane.queues) count += queue.length;
|
|
1904
|
-
return count;
|
|
1905
|
-
}
|
|
1906
|
-
/** Fix the claim's design, store it, and start its test cells. */
|
|
1907
|
-
async beginClaim() {
|
|
1908
|
-
await this.refresh();
|
|
1909
|
-
const stopReason = this.stopReason ?? "converged";
|
|
1910
|
-
const header = this.state.header;
|
|
1911
|
-
const reps = this.options.allocation.reps;
|
|
1912
|
-
const plan = planSearchClaim(this.state, {
|
|
1913
|
-
stopReason,
|
|
1914
|
-
reps,
|
|
1915
|
-
affordable: (finalists) => {
|
|
1916
|
-
const placed = this.claimCells(finalists, this.state.rootNodeId);
|
|
1917
|
-
const hold = placed.reduce((sum, cell) => sum + cell.reservation.usd, 0);
|
|
1918
|
-
const { headroomUsd, claimReserveUsd } = this.state.budget;
|
|
1919
|
-
const { maxCells } = header.budget;
|
|
1920
|
-
return (headroomUsd === null || hold <= headroomUsd + claimReserveUsd + USD_TOLERANCE) && (maxCells === null || this.state.audit.cells.allocated + placed.length <= maxCells);
|
|
1921
|
-
}
|
|
1922
|
-
});
|
|
1923
|
-
await this.recorder.startOperation({
|
|
1924
|
-
operationId: CLAIM_OPERATION,
|
|
1925
|
-
operationKind: "selection",
|
|
1926
|
-
reservation: null,
|
|
1927
|
-
artifacts: [this.recorder.blob("claim-plan", plan)]
|
|
1928
|
-
});
|
|
1929
|
-
await this.recordClaimOperation();
|
|
1930
|
-
this.claimPlan = plan;
|
|
1931
|
-
this.stopReason = stopReason;
|
|
1932
|
-
await this.applyClaimPlan(plan);
|
|
1933
|
-
}
|
|
1934
|
-
async recordClaimOperation() {
|
|
1935
|
-
await this.recorder.recordOperation({
|
|
1936
|
-
operationId: CLAIM_OPERATION,
|
|
1937
|
-
operationKind: "selection",
|
|
1938
|
-
execution: {
|
|
1939
|
-
kind: "deterministic",
|
|
1940
|
-
source: SEARCH_CLAIM_RULE
|
|
1941
|
-
},
|
|
1942
|
-
outcome: { status: "completed" },
|
|
1943
|
-
accounting: {
|
|
1944
|
-
tokens: {
|
|
1945
|
-
status: "known",
|
|
1946
|
-
inputTokens: 0,
|
|
1947
|
-
outputTokens: 0,
|
|
1948
|
-
cachedTokens: 0
|
|
1949
|
-
},
|
|
1950
|
-
cost: {
|
|
1951
|
-
status: "known",
|
|
1952
|
-
usd: 0,
|
|
1953
|
-
source: "free"
|
|
1954
|
-
}
|
|
1955
|
-
}
|
|
1956
|
-
});
|
|
1957
|
-
await this.refresh();
|
|
1958
|
-
}
|
|
1959
|
-
/**
|
|
1960
|
-
* Decide the plan's finalists and allocate the test cells of the root and
|
|
1961
|
-
* the finalists, interleaved by task so the arms of one task run side by
|
|
1962
|
-
* side. Allocation finishes before any claim cell starts. Idempotent.
|
|
1963
|
-
*/
|
|
1964
|
-
async applyClaimPlan(plan) {
|
|
1965
|
-
const root = this.state.rootNodeId;
|
|
1966
|
-
const bases = plan.finalists.map((finalist) => estimateNode(this.state, finalist.nodeId, {
|
|
1967
|
-
against: root,
|
|
1968
|
-
split: "selection"
|
|
1969
|
-
}));
|
|
1970
|
-
for (const [rank, finalist] of plan.finalists.entries()) await this.recorder.decideNode({
|
|
1971
|
-
nodeId: finalist.nodeId,
|
|
1972
|
-
decision: { status: "finalist" },
|
|
1973
|
-
basis: bases[rank],
|
|
1974
|
-
rule: SEARCH_CLAIM_RULE_NAME,
|
|
1975
|
-
reason: `rank ${rank + 1} of ${plan.finalists.length} by selection mean (${finalist.selectionMean}); ${plan.reason}`
|
|
1976
|
-
});
|
|
1977
|
-
await this.refresh();
|
|
1978
|
-
if (plan.test !== "run") return;
|
|
1979
|
-
const missing = this.claimCells(plan.finalists.length, root, plan).map((cell) => ({
|
|
1980
|
-
...cell,
|
|
1981
|
-
cellId: searchCellId(this.recorder.searchId, cell.nodeId, cell.plan.taskId, "test", cell.plan.rep)
|
|
1982
|
-
})).filter((cell) => !this.state.cell(cell.cellId));
|
|
1983
|
-
for (const cell of missing) await this.recorder.allocateCell({
|
|
1984
|
-
nodeId: cell.nodeId,
|
|
1985
|
-
taskId: cell.plan.taskId,
|
|
1986
|
-
split: "test",
|
|
1987
|
-
rep: cell.plan.rep,
|
|
1988
|
-
stage: "claim",
|
|
1989
|
-
lane: cell.lane.name,
|
|
1990
|
-
reservation: cell.reservation
|
|
1991
|
-
});
|
|
1992
|
-
await this.refresh();
|
|
1993
|
-
for (const cell of missing) this.enqueue(this.state.cell(cell.cellId));
|
|
1994
|
-
}
|
|
1995
|
-
/** The test cells of the root and the first `finalists` of the plan (or of
|
|
1996
|
-
* the finalist count being priced), task by task, with their lanes and holds. */
|
|
1997
|
-
claimCells(finalists, root, plan) {
|
|
1998
|
-
const header = this.state.header;
|
|
1999
|
-
const arms = [root, ...plan ? plan.finalists.map((entry) => entry.nodeId) : []];
|
|
2000
|
-
while (arms.length < finalists + 1) arms.push(root);
|
|
2001
|
-
const cells = [];
|
|
2002
|
-
for (const task of header.splits.test.tasks) for (let rep = 0; rep < this.options.allocation.reps; rep++) for (const nodeId of arms) {
|
|
2003
|
-
const cellPlan = {
|
|
2004
|
-
taskId: task.taskId,
|
|
2005
|
-
split: "test",
|
|
2006
|
-
rep,
|
|
2007
|
-
stage: "claim"
|
|
2008
|
-
};
|
|
2009
|
-
const laneName = this.options.executor.place({
|
|
2010
|
-
...cellPlan,
|
|
2011
|
-
nodeId
|
|
2012
|
-
});
|
|
2013
|
-
const lane = this.lanes.get(laneName);
|
|
2014
|
-
if (!lane) throw new Error(`runSearch: the executor placed a cell on unknown lane ${laneName}`);
|
|
2015
|
-
cells.push({
|
|
2016
|
-
nodeId,
|
|
2017
|
-
plan: cellPlan,
|
|
2018
|
-
lane,
|
|
2019
|
-
reservation: this.cellReservation(lane)
|
|
2020
|
-
});
|
|
2126
|
+
for (const { plan } of placed) {
|
|
2127
|
+
const cellId = searchCellId(this.recorder.searchId, nodeId, plan.taskId, plan.split, plan.rep);
|
|
2128
|
+
this.enqueue(this.state.cell(cellId));
|
|
2021
2129
|
}
|
|
2022
|
-
|
|
2130
|
+
this.pending.set(nodeId, (this.pending.get(nodeId) ?? 0) + placed.length);
|
|
2131
|
+
this.dispatch();
|
|
2132
|
+
return true;
|
|
2133
|
+
}
|
|
2134
|
+
async decide(decision) {
|
|
2135
|
+
await this.recorder.decideNode(decision);
|
|
2136
|
+
await this.refresh();
|
|
2137
|
+
}
|
|
2138
|
+
enqueue(cell) {
|
|
2139
|
+
this.lanes.get(cell.lane).queues[STAGE_PRIORITY[cell.stage]].push(cell.cellId);
|
|
2140
|
+
}
|
|
2141
|
+
queuedCount() {
|
|
2142
|
+
let count = 0;
|
|
2143
|
+
for (const lane of this.lanes.values()) for (const queue of lane.queues) count += queue.length;
|
|
2144
|
+
return count;
|
|
2023
2145
|
}
|
|
2024
2146
|
async cancelQueued(reason) {
|
|
2025
2147
|
for (const lane of this.lanes.values()) for (const [priority, queue] of lane.queues.entries()) {
|
|
@@ -2061,7 +2183,7 @@ var SearchKernel = class {
|
|
|
2061
2183
|
const children = this.options.proposer.childrenPerProposal ?? 1;
|
|
2062
2184
|
const need = operation.usd + children * screen * this.maxCellReservation();
|
|
2063
2185
|
const { headroomUsd } = this.state.budget;
|
|
2064
|
-
if (headroomUsd !== null && (need > headroomUsd +
|
|
2186
|
+
if (headroomUsd !== null && (need > headroomUsd + 1e-9 || this.overCap())) return "budget";
|
|
2065
2187
|
if (maxCells !== null && this.state.audit.cells.allocated + children * screen > maxCells) return "budget";
|
|
2066
2188
|
const expansion = policy.expand(view);
|
|
2067
2189
|
if (expansion === null) return "converged";
|
|
@@ -2218,32 +2340,16 @@ var SearchKernel = class {
|
|
|
2218
2340
|
await this.refresh();
|
|
2219
2341
|
const keep = this.options.policy.leader(this.policyView());
|
|
2220
2342
|
for (const decision of this.options.allocation.prune(this.allocationView(), keep)) await this.decide(decision);
|
|
2221
|
-
if (this.
|
|
2222
|
-
await this.
|
|
2343
|
+
if (this.claim.plan) {
|
|
2344
|
+
await this.claim.close(this.claim.plan);
|
|
2223
2345
|
return;
|
|
2224
2346
|
}
|
|
2225
2347
|
const view = this.policyView();
|
|
2226
2348
|
const leader = this.options.policy.leader(view);
|
|
2227
|
-
const
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
2231
|
-
const decisions = this.state.nodes().filter((node) => !isTerminal(node)).map((node) => {
|
|
2232
|
-
const scored = this.state.scoredCells(node.nodeId, split).length > 0;
|
|
2233
|
-
const lead = node.nodeId === leader && scored;
|
|
2234
|
-
const against = node.nodeId === leader ? root : leader;
|
|
2235
|
-
const basis = node.nodeId === against || !scored ? null : estimateNode(this.state, node.nodeId, {
|
|
2236
|
-
against,
|
|
2237
|
-
split
|
|
2238
|
-
});
|
|
2239
|
-
const units = new Set(view.unitScores(node.nodeId).map((unit) => unit.unitId));
|
|
2240
|
-
return {
|
|
2241
|
-
nodeId: node.nodeId,
|
|
2242
|
-
decision: { status: lead ? "selected" : "rejected" },
|
|
2243
|
-
basis,
|
|
2244
|
-
rule,
|
|
2245
|
-
reason: lead ? `the leader when the search stopped (${this.stopReason}): a budget decision on the ${split} split, which claims nothing` : !this.screenedSet.has(node.nodeId) || !scored ? "the search stopped before this node was measured" : !view.complete(node.nodeId) ? "it left a unit unscored, so it could not lead" : leaderUnits.some((unitId) => !units.has(unitId)) ? "the search stopped before it was measured on every unit the leader was" : "it did not beat the leader on the units they share"
|
|
2246
|
-
};
|
|
2349
|
+
const decisions = unclaimedSearchDecisions(this.state, view, leader, {
|
|
2350
|
+
rule: this.options.policy.name,
|
|
2351
|
+
stopReason: this.stopReason,
|
|
2352
|
+
screened: this.screenedSet
|
|
2247
2353
|
});
|
|
2248
2354
|
for (const decision of decisions) await this.recorder.decideNode(decision);
|
|
2249
2355
|
await this.recorder.close({
|
|
@@ -2252,47 +2358,6 @@ var SearchKernel = class {
|
|
|
2252
2358
|
});
|
|
2253
2359
|
await this.refresh();
|
|
2254
2360
|
}
|
|
2255
|
-
/** Close with the claim: the claim's decisions, then every other open node
|
|
2256
|
-
* rejected with its selection estimate against the root. */
|
|
2257
|
-
async closeWithClaim(plan) {
|
|
2258
|
-
const { claim, decisions } = decideSearchClaim(this.state, plan);
|
|
2259
|
-
const decided = new Set(decisions.map((decision) => decision.nodeId));
|
|
2260
|
-
const root = this.state.rootNodeId;
|
|
2261
|
-
const rest = this.state.nodes().filter((node) => !decided.has(node.nodeId) && !isTerminal(node)).map((node) => {
|
|
2262
|
-
const scored = this.state.scoredCells(node.nodeId, "selection").length > 0;
|
|
2263
|
-
return {
|
|
2264
|
-
nodeId: node.nodeId,
|
|
2265
|
-
decision: { status: "rejected" },
|
|
2266
|
-
basis: scored ? estimateNode(this.state, node.nodeId, {
|
|
2267
|
-
against: root,
|
|
2268
|
-
split: "selection"
|
|
2269
|
-
}) : null,
|
|
2270
|
-
rule: SEARCH_CLAIM_RULE_NAME,
|
|
2271
|
-
reason: scored ? "not a finalist: the claim tests at most 3 nodes that scored every selection unit and beat the root, best selection mean first" : "the search stopped before this node was measured on the selection split"
|
|
2272
|
-
};
|
|
2273
|
-
});
|
|
2274
|
-
for (const decision of [...decisions, ...rest]) await this.recorder.decideNode(decision);
|
|
2275
|
-
await this.recorder.close({
|
|
2276
|
-
reason: plan.stopReason,
|
|
2277
|
-
claim
|
|
2278
|
-
});
|
|
2279
|
-
await this.refresh();
|
|
2280
|
-
}
|
|
2281
|
-
/** The closed search, with its claim made again from the ledger alone. A
|
|
2282
|
-
* claim the ledger does not support throws: a tampered ledger, or a
|
|
2283
|
-
* producer and verifier that disagree. */
|
|
2284
|
-
closedResult() {
|
|
2285
|
-
const state = this.state;
|
|
2286
|
-
const claimVerification = verifySearchClaim(state);
|
|
2287
|
-
if (claimVerification?.status === "mismatch") throw new Error(`runSearch: search ${state.searchId} closed with a claim its ledger does not support: ${claimVerification.differences.join("; ")}`);
|
|
2288
|
-
return {
|
|
2289
|
-
state,
|
|
2290
|
-
leader: state.audit.selectedNodeId ?? state.rootNodeId,
|
|
2291
|
-
reason: state.closed.reason,
|
|
2292
|
-
claim: state.closed.claim,
|
|
2293
|
-
claimVerification
|
|
2294
|
-
};
|
|
2295
|
-
}
|
|
2296
2361
|
policyView() {
|
|
2297
2362
|
return searchPolicyView(this.state, {
|
|
2298
2363
|
screened: this.screened,
|
|
@@ -2359,136 +2424,8 @@ var SearchKernel = class {
|
|
|
2359
2424
|
this.state = await this.recorder.state();
|
|
2360
2425
|
}
|
|
2361
2426
|
};
|
|
2362
|
-
/**
|
|
2363
|
-
* A policy's read of a search: `screened` lists the admitted nodes whose screen
|
|
2364
|
-
* finished, in registration order; `screening` counts the admitted nodes still
|
|
2365
|
-
* being screened. Nodes are ranked on the selection split, or on train when
|
|
2366
|
-
* the search declares no selection split. The view reads no test cell.
|
|
2367
|
-
* `maxAttempts` (default 3, the kernel's) says when an errored cell has no
|
|
2368
|
-
* attempt left, which makes its unit a dodge.
|
|
2369
|
-
*/
|
|
2370
|
-
function searchPolicyView(state, input) {
|
|
2371
|
-
const maxAttempts = input.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
|
|
2372
|
-
const header = state.header;
|
|
2373
|
-
if (!header) throw new Error(`search ${state.searchId} has not been opened`);
|
|
2374
|
-
const split = header.splits.selection.tasks.length > 0 ? "selection" : "train";
|
|
2375
|
-
let nodes;
|
|
2376
|
-
let operatorWeights;
|
|
2377
|
-
return {
|
|
2378
|
-
searchId: state.searchId,
|
|
2379
|
-
seed: header.policy.seed,
|
|
2380
|
-
direction: header.objective.direction,
|
|
2381
|
-
split,
|
|
2382
|
-
rootNodeId: state.rootNodeId,
|
|
2383
|
-
expansions: input.expansions,
|
|
2384
|
-
screened: [...input.screened],
|
|
2385
|
-
screening: input.screening,
|
|
2386
|
-
complete: (nodeId) => {
|
|
2387
|
-
let scored = false;
|
|
2388
|
-
for (const cell of state.cells({ nodeId })) {
|
|
2389
|
-
if (cell.split !== split) continue;
|
|
2390
|
-
if (cell.score !== null) scored = true;
|
|
2391
|
-
else if (cell.final || cell.outcome === "errored" && cell.attempts >= maxAttempts) return false;
|
|
2392
|
-
}
|
|
2393
|
-
return scored;
|
|
2394
|
-
},
|
|
2395
|
-
unitScores: (nodeId) => state.unitScores(nodeId, split),
|
|
2396
|
-
estimate: (nodeId, against) => estimateNode(state, nodeId, {
|
|
2397
|
-
against,
|
|
2398
|
-
split
|
|
2399
|
-
}),
|
|
2400
|
-
nodes: () => {
|
|
2401
|
-
nodes ??= policyNodes(state);
|
|
2402
|
-
return nodes;
|
|
2403
|
-
},
|
|
2404
|
-
get posterior() {
|
|
2405
|
-
let posterior = POSTERIORS.get(state);
|
|
2406
|
-
if (!posterior) {
|
|
2407
|
-
posterior = searchPosterior(state, { split });
|
|
2408
|
-
POSTERIORS.set(state, posterior);
|
|
2409
|
-
}
|
|
2410
|
-
return posterior;
|
|
2411
|
-
},
|
|
2412
|
-
get operatorWeights() {
|
|
2413
|
-
operatorWeights ??= operatorYield(state, { split }).signal.value;
|
|
2414
|
-
return operatorWeights;
|
|
2415
|
-
}
|
|
2416
|
-
};
|
|
2417
|
-
}
|
|
2418
|
-
/** One posterior per ledger state: a state view is retired when the ledger
|
|
2419
|
-
* moves on, so the view it was computed from names the sequence. */
|
|
2420
|
-
const POSTERIORS = /* @__PURE__ */ new WeakMap();
|
|
2421
|
-
/** Every node whose edge is recorded, with its lineage and its defects outside
|
|
2422
|
-
* the test split. */
|
|
2423
|
-
function policyNodes(state) {
|
|
2424
|
-
const nodes = [];
|
|
2425
|
-
for (const node of state.nodes()) {
|
|
2426
|
-
const edgeId = node.edgeIds[0];
|
|
2427
|
-
if (edgeId === void 0) continue;
|
|
2428
|
-
let outcomes = 0;
|
|
2429
|
-
let defects = 0;
|
|
2430
|
-
for (const cell of state.cells({ nodeId: node.nodeId })) {
|
|
2431
|
-
if (cell.split === "test" || !cell.final) continue;
|
|
2432
|
-
outcomes += 1;
|
|
2433
|
-
if (cell.outcome === "failed") defects += 1;
|
|
2434
|
-
}
|
|
2435
|
-
const edge = state.edge(edgeId);
|
|
2436
|
-
nodes.push({
|
|
2437
|
-
nodeId: node.nodeId,
|
|
2438
|
-
ordinal: node.ordinal,
|
|
2439
|
-
parent: node.primaryParentId,
|
|
2440
|
-
operator: edge.operator,
|
|
2441
|
-
rule: edge.selection?.rule ?? null,
|
|
2442
|
-
children: node.children,
|
|
2443
|
-
status: node.status,
|
|
2444
|
-
outcomes,
|
|
2445
|
-
defects
|
|
2446
|
-
});
|
|
2447
|
-
}
|
|
2448
|
-
return nodes;
|
|
2449
|
-
}
|
|
2450
|
-
/**
|
|
2451
|
-
* The divergence rule: a node whose train mean rose over its parent's on the
|
|
2452
|
-
* train units they share while its selection interval against the parent (6
|
|
2453
|
-
* or more units, not indeterminate) lies wholly on the worse side. Train is
|
|
2454
|
-
* what the proposer reads; a gain there that selection contradicts is the
|
|
2455
|
-
* signature of fitting the feedback instead of the task.
|
|
2456
|
-
*/
|
|
2457
|
-
function searchDivergence(state, nodeId) {
|
|
2458
|
-
const header = state.header;
|
|
2459
|
-
const parent = state.node(nodeId)?.primaryParentId ?? null;
|
|
2460
|
-
if (!header || parent === null || header.splits.selection.tasks.length === 0) return null;
|
|
2461
|
-
const selection = estimateNode(state, nodeId, {
|
|
2462
|
-
against: parent,
|
|
2463
|
-
split: "selection"
|
|
2464
|
-
});
|
|
2465
|
-
if (selection.interval === null) return null;
|
|
2466
|
-
const maximize = header.objective.direction === "maximize";
|
|
2467
|
-
const [low, high] = selection.interval;
|
|
2468
|
-
if (maximize ? high >= 0 : low <= 0) return null;
|
|
2469
|
-
const parentTrain = new Map(state.unitScores(parent, "train").map((unit) => [unit.unitId, unit.mean]));
|
|
2470
|
-
let gain = 0;
|
|
2471
|
-
let shared = 0;
|
|
2472
|
-
for (const unit of state.unitScores(nodeId, "train")) {
|
|
2473
|
-
const before = parentTrain.get(unit.unitId);
|
|
2474
|
-
if (before === void 0) continue;
|
|
2475
|
-
gain += maximize ? unit.mean - before : before - unit.mean;
|
|
2476
|
-
shared += 1;
|
|
2477
|
-
}
|
|
2478
|
-
if (shared === 0 || gain <= 0) return null;
|
|
2479
|
-
return {
|
|
2480
|
-
basis: selection,
|
|
2481
|
-
reason: `its train mean rose ${Math.round(gain / shared * 1e4) / 1e4} over its parent's on ${shared} shared train unit${shared === 1 ? "" : "s"}, while its selection interval against the parent, [${round4(low)}, ${round4(high)}] on ${selection.pairs} units, lies wholly on the worse side`
|
|
2482
|
-
};
|
|
2483
|
-
}
|
|
2484
|
-
function round4(value) {
|
|
2485
|
-
return Math.round(value * 1e4) / 1e4;
|
|
2486
|
-
}
|
|
2487
|
-
function isTerminal(node) {
|
|
2488
|
-
return node.status !== null && node.status !== "advanced";
|
|
2489
|
-
}
|
|
2490
2427
|
function estimate(costs, prior) {
|
|
2491
|
-
if (costs.length <
|
|
2428
|
+
if (costs.length < 20) return prior;
|
|
2492
2429
|
const sorted = [...costs].sort((a, b) => a - b);
|
|
2493
2430
|
const p99 = sorted[Math.min(sorted.length - 1, Math.ceil(.99 * sorted.length) - 1)];
|
|
2494
2431
|
return Math.round(ESTIMATE_MARGIN * p99 * 1e9) / 1e9;
|
|
@@ -2512,6 +2449,91 @@ function errorMessage(error) {
|
|
|
2512
2449
|
return text.trim().length > 0 ? text : "the proposer failed without a message";
|
|
2513
2450
|
}
|
|
2514
2451
|
//#endregion
|
|
2515
|
-
|
|
2452
|
+
//#region src/campaign/search-kernel.ts
|
|
2453
|
+
/** Run a search to its close, or continue one from its ledger. */
|
|
2454
|
+
async function runSearch(options) {
|
|
2455
|
+
return new SearchKernel(options).run();
|
|
2456
|
+
}
|
|
2457
|
+
//#endregion
|
|
2458
|
+
//#region src/pareto.ts
|
|
2459
|
+
/** Does candidate A weakly dominate B — strictly better on at least one objective and no worse on any? */
|
|
2460
|
+
function dominates(a, b, objectives) {
|
|
2461
|
+
let strictlyBetter = false;
|
|
2462
|
+
for (const obj of objectives) {
|
|
2463
|
+
const av = obj.value(a);
|
|
2464
|
+
const bv = obj.value(b);
|
|
2465
|
+
if (!Number.isFinite(av) || !Number.isFinite(bv)) return false;
|
|
2466
|
+
const aIsBetter = obj.direction === "maximize" ? av > bv : av < bv;
|
|
2467
|
+
if (obj.direction === "maximize" ? av < bv : av > bv) return false;
|
|
2468
|
+
if (aIsBetter) strictlyBetter = true;
|
|
2469
|
+
}
|
|
2470
|
+
return strictlyBetter;
|
|
2471
|
+
}
|
|
2472
|
+
/**
|
|
2473
|
+
* Compute the non-dominated frontier. Candidates with NaN/Infinity on any
|
|
2474
|
+
* objective are excluded (can't rank them). A candidate enters the frontier
|
|
2475
|
+
* iff no other candidate dominates it.
|
|
2476
|
+
*/
|
|
2477
|
+
function paretoFrontier(candidates, objectives) {
|
|
2478
|
+
if (objectives.length === 0) throw new Error("paretoFrontier: at least 1 objective required");
|
|
2479
|
+
const valid = candidates.filter((c) => objectives.every((o) => Number.isFinite(o.value(c))));
|
|
2480
|
+
const frontier = [];
|
|
2481
|
+
const dominated = [];
|
|
2482
|
+
for (const c of valid) if (valid.some((other) => other !== c && dominates(other, c, objectives))) dominated.push(c);
|
|
2483
|
+
else frontier.push(c);
|
|
2484
|
+
return {
|
|
2485
|
+
frontier,
|
|
2486
|
+
dominated,
|
|
2487
|
+
dominanceMap: frontier.map((d) => ({
|
|
2488
|
+
dominator: d,
|
|
2489
|
+
dominated: dominated.filter((x) => dominates(d, x, objectives))
|
|
2490
|
+
}))
|
|
2491
|
+
};
|
|
2492
|
+
}
|
|
2493
|
+
/**
|
|
2494
|
+
* NSGA-II crowding distance — secondary sort for ties on the frontier.
|
|
2495
|
+
*
|
|
2496
|
+
* When the Pareto front collapses to a single point (or many candidates tie
|
|
2497
|
+
* on dominance), naive selection picks arbitrarily and the population
|
|
2498
|
+
* degenerates over generations. NSGA-II preserves diversity by preferring
|
|
2499
|
+
* candidates with more empty space around them on the frontier.
|
|
2500
|
+
*
|
|
2501
|
+
* Returns an array of `{ candidate, distance }` in the SAME order as the
|
|
2502
|
+
* input. Higher distance = more isolated = should be preferred when
|
|
2503
|
+
* preserving diversity.
|
|
2504
|
+
*/
|
|
2505
|
+
function crowdingDistance(candidates, objectives) {
|
|
2506
|
+
const distances = new Map(candidates.map((c) => [c, 0]));
|
|
2507
|
+
for (const obj of objectives) {
|
|
2508
|
+
const sorted = [...candidates].sort((a, b) => obj.value(a) - obj.value(b));
|
|
2509
|
+
const min = obj.value(sorted[0]);
|
|
2510
|
+
const range = obj.value(sorted[sorted.length - 1]) - min || 1;
|
|
2511
|
+
distances.set(sorted[0], Infinity);
|
|
2512
|
+
distances.set(sorted[sorted.length - 1], Infinity);
|
|
2513
|
+
for (let i = 1; i < sorted.length - 1; i++) {
|
|
2514
|
+
const prev = obj.value(sorted[i - 1]);
|
|
2515
|
+
const next = obj.value(sorted[i + 1]);
|
|
2516
|
+
const current = distances.get(sorted[i]);
|
|
2517
|
+
if (current === Infinity) continue;
|
|
2518
|
+
distances.set(sorted[i], current + (next - prev) / range);
|
|
2519
|
+
}
|
|
2520
|
+
}
|
|
2521
|
+
return candidates.map((c) => ({
|
|
2522
|
+
candidate: c,
|
|
2523
|
+
distance: distances.get(c) ?? 0
|
|
2524
|
+
}));
|
|
2525
|
+
}
|
|
2526
|
+
/**
|
|
2527
|
+
* Pareto frontier with tie-break by crowding distance — the canonical
|
|
2528
|
+
* NSGA-II selection step. Returns the frontier sorted by descending crowding
|
|
2529
|
+
* distance so callers can `.slice(0, k)` to pick K diverse winners.
|
|
2530
|
+
*/
|
|
2531
|
+
function paretoFrontierWithCrowding(candidates, objectives) {
|
|
2532
|
+
const { frontier } = paretoFrontier(candidates, objectives);
|
|
2533
|
+
if (frontier.length === 0) return [];
|
|
2534
|
+
return crowdingDistance(frontier, objectives).sort((a, b) => b.distance - a.distance);
|
|
2535
|
+
}
|
|
2536
|
+
//#endregion
|
|
2537
|
+
export { normalizeSignedZero as A, SEARCH_ESTIMATOR as C, searchEstimateMethod as D, searchCellSetDigest as E, searchPosterior as O, requiredPairsForPairedPromotion as S, estimateNodeFromCells as T, planSearchClaim as _, searchDivergence as a, verifySearchClaim as b, MIN_OUTCOMES_FOR_WEIGHT as c, rankingSplit as d, summarizeSamples as f, decideSearchClaim as g, SEARCH_CLAIM_RULE as h, runSearch as i, seedFromDigest as k, operatorYield as l, searchExpansionIndex as m, paretoFrontier as n, searchPolicyView as o, SEARCH_KERNEL_SOURCE as p, paretoFrontierWithCrowding as r, EXPANSION_OPERATORS as s, dominates as t, INSUFFICIENT_FROM as u, searchClaimDecision as v, estimateNode as w, pairedPromotionPower as x, searchClaimReserveUsd as y };
|
|
2516
2538
|
|
|
2517
|
-
//# sourceMappingURL=
|
|
2539
|
+
//# sourceMappingURL=pareto-Cm6ndrWN.js.map
|