@tangle-network/agent-eval 0.215.1 → 0.216.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +51 -0
- package/dist/adapters/http.d.ts +3 -3
- package/dist/{agent-profile-DmzL8KaJ.d.ts → agent-profile-VFcQncbW.d.ts} +5 -5
- package/dist/{agent-profile-DmzL8KaJ.d.ts.map → agent-profile-VFcQncbW.d.ts.map} +1 -1
- package/dist/{agent-profile-cell-BYc0bxsL.js → agent-profile-cell-BK8mlqlq.js} +3 -3
- package/dist/{agent-profile-cell-BYc0bxsL.js.map → agent-profile-cell-BK8mlqlq.js.map} +1 -1
- package/dist/agreement-irr-BEWLIF3_.js +186 -0
- package/dist/agreement-irr-BEWLIF3_.js.map +1 -0
- package/dist/analyst/index.d.ts +487 -14
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +1547 -12
- package/dist/analyst/index.js.map +1 -1
- package/dist/{analyze-runs-3xpby3p2.js → analyze-runs-C84irm4i.js} +608 -604
- package/dist/analyze-runs-C84irm4i.js.map +1 -0
- package/dist/{analyze-runs-BhFLOztS.d.ts → analyze-runs-D68fd-su.d.ts} +17 -15
- package/dist/analyze-runs-D68fd-su.d.ts.map +1 -0
- package/dist/{baseline-Dm8966du.js → baseline-Udi_PLuT.js} +5 -5
- package/dist/{baseline-Dm8966du.js.map → baseline-Udi_PLuT.js.map} +1 -1
- package/dist/{benchmark-BywPeAUo.js → benchmark-CXmVVHH2.js} +2 -2
- package/dist/{benchmark-BywPeAUo.js.map → benchmark-CXmVVHH2.js.map} +1 -1
- package/dist/{benchmark-command-CgpGi7vY.js → benchmark-command-By0vh8q7.js} +71 -70
- package/dist/benchmark-command-By0vh8q7.js.map +1 -0
- package/dist/benchmarks/index.d.ts +5 -5
- package/dist/benchmarks/index.js +3 -3
- package/dist/builder-eval/index.js +2 -2
- package/dist/campaign/index.d.ts +7 -7
- package/dist/campaign/index.js +14 -14
- package/dist/{campaign-CpnsCgfz.js → campaign-BudZ70zo.js} +245 -247
- package/dist/campaign-BudZ70zo.js.map +1 -0
- package/dist/{campaign-evidence-OcUciD-N.js → campaign-evidence-Clrn4hBV.js} +8 -8
- package/dist/campaign-evidence-Clrn4hBV.js.map +1 -0
- package/dist/{canonical-BsZVgBhW.js → canonical-B2RqLofv.js} +2 -2
- package/dist/{canonical-BsZVgBhW.js.map → canonical-B2RqLofv.js.map} +1 -1
- package/dist/{chat-client-DthPOYGd.js → chat-client-l-MVDlDk.js} +2 -2
- package/dist/{chat-client-DthPOYGd.js.map → chat-client-l-MVDlDk.js.map} +1 -1
- package/dist/{chat-json-call-BrKsyspg.js → chat-json-call-puMArmtW.js} +2 -2
- package/dist/{chat-json-call-BrKsyspg.js.map → chat-json-call-puMArmtW.js.map} +1 -1
- package/dist/{claim-BXcZw-jo.js → claim-q4HQhK1G.js} +4 -4
- package/dist/claim-q4HQhK1G.js.map +1 -0
- package/dist/cli.js +9 -9
- package/dist/cli.js.map +1 -1
- package/dist/cluster-bootstrap-qcUWYFl-.js +66 -0
- package/dist/cluster-bootstrap-qcUWYFl-.js.map +1 -0
- package/dist/contract/index.d.ts +11 -11
- package/dist/contract/index.js +1023 -1079
- package/dist/contract/index.js.map +1 -1
- package/dist/{cost-ledger-D2FnJPBF.js → cost-ledger-BP8V4kUQ.js} +835 -816
- package/dist/cost-ledger-BP8V4kUQ.js.map +1 -0
- package/dist/{cost-ledger-CO7UyEcQ.d.ts → cost-ledger-Jk62OBfF.d.ts} +12 -47
- package/dist/cost-ledger-Jk62OBfF.d.ts.map +1 -0
- package/dist/{counterfactual-DtMWvn7D.js → counterfactual-Dn3veueY.js} +2 -2
- package/dist/{counterfactual-DtMWvn7D.js.map → counterfactual-Dn3veueY.js.map} +1 -1
- package/dist/{default-registry-KP4-Tolz.js → default-registry-COhk9g36.js} +744 -759
- package/dist/default-registry-COhk9g36.js.map +1 -0
- package/dist/{default-registry-B4B7GMvI.d.ts → default-registry-DzHiGruW.d.ts} +6 -6
- package/dist/{default-registry-B4B7GMvI.d.ts.map → default-registry-DzHiGruW.d.ts.map} +1 -1
- package/dist/{define-agent-eval-BdfnqXeP.js → define-agent-eval-CNTPzIkp.js} +10 -10
- package/dist/define-agent-eval-CNTPzIkp.js.map +1 -0
- package/dist/{define-agent-eval-DfXvNho5.d.ts → define-agent-eval-WSck0CZJ.d.ts} +8 -8
- package/dist/{define-agent-eval-DfXvNho5.d.ts.map → define-agent-eval-WSck0CZJ.d.ts.map} +1 -1
- package/dist/{descriptive-CnU8Yq29.js → descriptive-DboxPZrG.js} +2 -2
- package/dist/{descriptive-CnU8Yq29.js.map → descriptive-DboxPZrG.js.map} +1 -1
- package/dist/diagnosis/index.d.ts +3 -3
- package/dist/diagnosis/index.js +4 -4
- package/dist/{dspy-rlm-engine-DZfHM7kT.js → dspy-rlm-engine-LRCu61m5.js} +3 -3
- package/dist/{dspy-rlm-engine-DZfHM7kT.js.map → dspy-rlm-engine-LRCu61m5.js.map} +1 -1
- package/dist/{effect-sizes-DiH8MGOH.js → effect-sizes-BhzEpXMc.js} +2 -2
- package/dist/{effect-sizes-DiH8MGOH.js.map → effect-sizes-BhzEpXMc.js.map} +1 -1
- package/dist/{engine-C37HS2kr.d.ts → engine-3A_4jOlI.d.ts} +5 -5
- package/dist/{engine-C37HS2kr.d.ts.map → engine-3A_4jOlI.d.ts.map} +1 -1
- package/dist/{equal-terms-Blx9Zk5I.js → equal-terms-BtdXayCJ.js} +2 -2
- package/dist/{equal-terms-Blx9Zk5I.js.map → equal-terms-BtdXayCJ.js.map} +1 -1
- package/dist/{errors-Dngq5h35.js → errors-DVZiqS7K.js} +6 -2
- package/dist/{errors-Dngq5h35.js.map → errors-DVZiqS7K.js.map} +1 -1
- package/dist/{eval-campaign-4byTtqrf.js → eval-campaign-90jeaCAI.js} +6 -6
- package/dist/{eval-campaign-4byTtqrf.js.map → eval-campaign-90jeaCAI.js.map} +1 -1
- package/dist/evaluation-6Yoi74gM.d.ts +75 -0
- package/dist/evaluation-6Yoi74gM.d.ts.map +1 -0
- package/dist/evaluation.d.ts +2 -74
- package/dist/evaluation.js +1 -1
- package/dist/{exact-types-Dlb73wN4.d.ts → exact-types-ClugMJfI.d.ts} +2 -2
- package/dist/{exact-types-Dlb73wN4.d.ts.map → exact-types-ClugMJfI.d.ts.map} +1 -1
- package/dist/experiment/index.d.ts +404 -3
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +313 -19
- package/dist/experiment/index.js.map +1 -1
- package/dist/{external-optimizer-process-GngofQML.js → external-optimizer-process-BjMGUuds.js} +3 -3
- package/dist/{external-optimizer-process-GngofQML.js.map → external-optimizer-process-BjMGUuds.js.map} +1 -1
- package/dist/{external-optimizer-subprocess-BUxifb8g.js → external-optimizer-subprocess-BqW0hHcI.js} +791 -532
- package/dist/external-optimizer-subprocess-BqW0hHcI.js.map +1 -0
- package/dist/{feedback-trajectory-BLN_6fEs.d.ts → feedback-trajectory-C4EG0nNH.d.ts} +5 -8
- package/dist/feedback-trajectory-C4EG0nNH.d.ts.map +1 -0
- package/dist/fuzz.d.ts +1 -1
- package/dist/fuzz.js +3 -3
- package/dist/hosted/index.js +1 -1
- package/dist/{index-CPJJr98o.d.ts → index-BFRO6chs.d.ts} +3 -2
- package/dist/index-BFRO6chs.d.ts.map +1 -0
- package/dist/{index-Ch9VxCpK.d.ts → index-BO41eGj5.d.ts} +2 -2
- package/dist/{index-Ch9VxCpK.d.ts.map → index-BO41eGj5.d.ts.map} +1 -1
- package/dist/{index-B3WBm953.d.ts → index-CyszSg4O.d.ts} +165 -162
- package/dist/index-CyszSg4O.d.ts.map +1 -0
- package/dist/{index-B7m2uzJL.d.ts → index-DPcngtwt.d.ts} +30 -17
- package/dist/index-DPcngtwt.d.ts.map +1 -0
- package/dist/{index-tYLUvk_w.d.ts → index-DV-jSzxF.d.ts} +9 -9
- package/dist/{index-tYLUvk_w.d.ts.map → index-DV-jSzxF.d.ts.map} +1 -1
- package/dist/index.d.ts +1969 -1871
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +162 -201
- package/dist/index.js.map +1 -1
- package/dist/{integrity-BvnVARVl.js → integrity-BXOsAb3i.js} +2 -2
- package/dist/{integrity-BvnVARVl.js.map → integrity-BXOsAb3i.js.map} +1 -1
- package/dist/{integrity-CCAxGVK6.d.ts → integrity-DAMEDFaC.d.ts} +2 -2
- package/dist/{integrity-CCAxGVK6.d.ts.map → integrity-DAMEDFaC.d.ts.map} +1 -1
- package/dist/{integrity-BV1ZDf42.js → integrity-dLiDe2kD.js} +2 -5
- package/dist/integrity-dLiDe2kD.js.map +1 -0
- package/dist/{internal-CRHwS9iz.js → internal-Epl-YBZW.js} +3 -3
- package/dist/{internal-CRHwS9iz.js.map → internal-Epl-YBZW.js.map} +1 -1
- package/dist/interval-CELpye1n.js +219 -0
- package/dist/interval-CELpye1n.js.map +1 -0
- package/dist/interval-ohuOJurX.d.ts +47 -0
- package/dist/interval-ohuOJurX.d.ts.map +1 -0
- package/dist/{journal-DvKCvjty.js → journal-DfRz9kps.js} +2 -2
- package/dist/{journal-DvKCvjty.js.map → journal-DfRz9kps.js.map} +1 -1
- package/dist/{judge-calibration-N44V04Se.js → judge-calibration-BpCy5XQl.js} +3 -3
- package/dist/{judge-calibration-N44V04Se.js.map → judge-calibration-BpCy5XQl.js.map} +1 -1
- package/dist/{judge-gate-DNcUrRFA.d.ts → judge-gate-CE7CQAgt.d.ts} +3 -3
- package/dist/{judge-gate-DNcUrRFA.d.ts.map → judge-gate-CE7CQAgt.d.ts.map} +1 -1
- package/dist/{judge-gate-B1qYnkpi.js → judge-gate-iY9s4L44.js} +3 -3
- package/dist/judge-gate-iY9s4L44.js.map +1 -0
- package/dist/{kind-factory-AaEhUADH.js → kind-factory-BobI5Jy3.js} +5 -5
- package/dist/{kind-factory-AaEhUADH.js.map → kind-factory-BobI5Jy3.js.map} +1 -1
- package/dist/ledger-core/index.js +3 -3
- package/dist/{ledger-core-B2hahzrt.js → ledger-core-Gq7yp8lP.js} +4 -4
- package/dist/{ledger-core-B2hahzrt.js.map → ledger-core-Gq7yp8lP.js.map} +1 -1
- package/dist/{llm-client-DQveoxSM.js → llm-client-CyBAoYiX.js} +192 -219
- package/dist/llm-client-CyBAoYiX.js.map +1 -0
- package/dist/{llm-judge-Cd5s5Ei6.js → llm-judge-CG12SQJq.js} +499 -921
- package/dist/llm-judge-CG12SQJq.js.map +1 -0
- package/dist/matrix/index.d.ts +2 -2
- package/dist/matrix/index.js +1 -1
- package/dist/{matrix-DXVAJMLD.d.ts → matrix-DSrSxjjy.d.ts} +4 -4
- package/dist/{matrix-DXVAJMLD.d.ts.map → matrix-DSrSxjjy.d.ts.map} +1 -1
- package/dist/{matrix-_BV9-Lk5.js → matrix-Dm0BQW7U.js} +7 -11
- package/dist/matrix-Dm0BQW7U.js.map +1 -0
- package/dist/{paired-tests-jbwPdSz5.js → mean-DHoMiudD.js} +11 -5
- package/dist/mean-DHoMiudD.js.map +1 -0
- package/dist/meta-eval/index.d.ts +3 -3
- package/dist/meta-eval/index.js +9 -70
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/mint-DNU8QPG4.d.ts +81 -0
- package/dist/mint-DNU8QPG4.d.ts.map +1 -0
- package/dist/{mint-Csor1BLJ.js → mint-QaD5cA0u.js} +3 -3
- package/dist/{mint-Csor1BLJ.js.map → mint-QaD5cA0u.js.map} +1 -1
- package/dist/{multiplicity-D9cjmStq.js → multiplicity-DQUFcbW4.js} +2 -2
- package/dist/{multiplicity-D9cjmStq.js.map → multiplicity-DQUFcbW4.js.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/golden/index.d.ts.map +1 -1
- package/dist/multishot/golden/index.js +3 -6
- package/dist/multishot/golden/index.js.map +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/multishot/index.js +1 -1
- package/dist/nonnegative-number-DvdMxg_f.js +9 -0
- package/dist/nonnegative-number-DvdMxg_f.js.map +1 -0
- package/dist/openapi.json +1 -1
- package/dist/{otel-spans-C8YIJWy1.js → otel-spans-B9tUXRHF.js} +3 -3
- package/dist/{otel-spans-C8YIJWy1.js.map → otel-spans-B9tUXRHF.js.map} +1 -1
- package/dist/{paired-arms-DzKn0TfY.js → paired-arms-CwMGJsrS.js} +5 -5
- package/dist/{paired-arms-DzKn0TfY.js.map → paired-arms-CwMGJsrS.js.map} +1 -1
- package/dist/{paired-binary-Q36bhn2R.js → paired-binary-CvpFHfKZ.js} +2 -2
- package/dist/{paired-binary-Q36bhn2R.js.map → paired-binary-CvpFHfKZ.js.map} +1 -1
- package/dist/{paired-promotion-decision-BN6IQTiu.js → paired-promotion-decision-DsAPYGCm.js} +3 -3
- package/dist/{paired-promotion-decision-BN6IQTiu.js.map → paired-promotion-decision-DsAPYGCm.js.map} +1 -1
- package/dist/{search-kernel-BztlERLv.js → pareto-Cm6ndrWN.js} +883 -861
- package/dist/pareto-Cm6ndrWN.js.map +1 -0
- package/dist/pipelines/index.d.ts +2 -2
- package/dist/pipelines/index.js +3 -3
- package/dist/{power-and-mde-fK5Qocwo.js → power-and-mde-Hj2ubb4B.js} +2 -2
- package/dist/{power-and-mde-fK5Qocwo.js.map → power-and-mde-Hj2ubb4B.js.map} +1 -1
- package/dist/predicate-FdKmElA_.js +38 -0
- package/dist/predicate-FdKmElA_.js.map +1 -0
- package/dist/{prime-protocol-DcAK9pC8.js → prime-protocol-CHKgA3a6.js} +2 -2
- package/dist/{prime-protocol-DcAK9pC8.js.map → prime-protocol-CHKgA3a6.js.map} +1 -1
- package/dist/{prime-protocol-BHia9vDz.d.ts → prime-protocol-HiB9HX7E.d.ts} +3 -3
- package/dist/{prime-protocol-BHia9vDz.d.ts.map → prime-protocol-HiB9HX7E.d.ts.map} +1 -1
- package/dist/{produced-state-BGp4BX8t.js → produced-state-BsDmnwxc.js} +8 -8
- package/dist/{produced-state-BGp4BX8t.js.map → produced-state-BsDmnwxc.js.map} +1 -1
- package/dist/profile-cell.js +1 -1
- package/dist/{release-confidence-BK2QCB5M.d.ts → promotion-gate-BawrdPYr.d.ts} +102 -102
- package/dist/promotion-gate-BawrdPYr.d.ts.map +1 -0
- package/dist/{promotion-policy-CWW35UWq.js → promotion-policy-KO57gbyV.js} +4 -4
- package/dist/{promotion-policy-CWW35UWq.js.map → promotion-policy-KO57gbyV.js.map} +1 -1
- package/dist/{proposal-findings-CXblsskH.js → proposal-findings-CY1xKn_a.js} +2 -36
- package/dist/proposal-findings-CY1xKn_a.js.map +1 -0
- package/dist/{query-B1D1qp2Z.js → query-BUCZCK7F.js} +3 -3
- package/dist/{query-B1D1qp2Z.js.map → query-BUCZCK7F.js.map} +1 -1
- package/dist/{random-Dn5fPWkt.js → random-ByUSnV9w.js} +2 -2
- package/dist/{random-Dn5fPWkt.js.map → random-ByUSnV9w.js.map} +1 -1
- package/dist/{raw-provider-sink-rB2DYxr6.js → raw-provider-sink-DoHz4JNG.js} +2 -2
- package/dist/{raw-provider-sink-rB2DYxr6.js.map → raw-provider-sink-DoHz4JNG.js.map} +1 -1
- package/dist/{redact-Dk1ZdJEm.js → redact-B5xXKqQ3.js} +97 -98
- package/dist/redact-B5xXKqQ3.js.map +1 -0
- package/dist/{redact-Cveoq6fJ.d.ts → redact-mtCCcT8z.d.ts} +4 -20
- package/dist/redact-mtCCcT8z.d.ts.map +1 -0
- package/dist/{registry-CMYIaDgS.d.ts → registry-D7_hm2fK.d.ts} +11 -13
- package/dist/registry-D7_hm2fK.d.ts.map +1 -0
- package/dist/{release-confidence-Dbe9AS_X.js → release-confidence-CpuP65S5.js} +5 -5
- package/dist/{release-confidence-Dbe9AS_X.js.map → release-confidence-CpuP65S5.js.map} +1 -1
- package/dist/{report-command-CtIIsQKP.js → report-command-SC9nqb4M.js} +128 -177
- package/dist/report-command-SC9nqb4M.js.map +1 -0
- package/dist/reporting-html-Kvlc8oMt.js +100 -0
- package/dist/reporting-html-Kvlc8oMt.js.map +1 -0
- package/dist/reporting.d.ts +3 -3
- package/dist/reporting.js +6 -6
- package/dist/{researcher-f6mZhh58.d.ts → researcher-CEcfku-T.d.ts} +5 -5
- package/dist/{researcher-f6mZhh58.d.ts.map → researcher-CEcfku-T.d.ts.map} +1 -1
- package/dist/reward-CpCmAM8L.d.ts +78 -0
- package/dist/reward-CpCmAM8L.d.ts.map +1 -0
- package/dist/{reward-hacking-BFcQgkux.js → reward-hacking-E0d-WVcP.js} +2 -2
- package/dist/{reward-hacking-BFcQgkux.js.map → reward-hacking-E0d-WVcP.js.map} +1 -1
- package/dist/{reward-hacking-DHA7c1l8.d.ts → reward-hacking-RVdZ6Kkt.d.ts} +2 -2
- package/dist/{reward-hacking-DHA7c1l8.d.ts.map → reward-hacking-RVdZ6Kkt.d.ts.map} +1 -1
- package/dist/rl.d.ts +8 -8
- package/dist/rl.js +14 -14
- package/dist/rollout/index.d.ts +3 -2
- package/dist/rollout/index.js +2 -2
- package/dist/{rollout-D9bKKBnZ.js → rollout-DZV1t2Wg.js} +2 -2
- package/dist/{rollout-D9bKKBnZ.js.map → rollout-DZV1t2Wg.js.map} +1 -1
- package/dist/round-decimal-COrKglMh.js +10 -0
- package/dist/round-decimal-COrKglMh.js.map +1 -0
- package/dist/{rubric-predictive-validity-2SgHxoa5.d.ts → rubric-predictive-validity-C03EC7ZE.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-2SgHxoa5.d.ts.map → rubric-predictive-validity-C03EC7ZE.d.ts.map} +1 -1
- package/dist/{rubric-predictive-validity-BsgkBfT0.js → rubric-predictive-validity-DaFW70y9.js} +3 -3
- package/dist/{rubric-predictive-validity-BsgkBfT0.js.map → rubric-predictive-validity-DaFW70y9.js.map} +1 -1
- package/dist/{run-record-DlTZ_lqe.js → run-record-Boc5IUTp.js} +4 -4
- package/dist/{run-record-DlTZ_lqe.js.map → run-record-Boc5IUTp.js.map} +1 -1
- package/dist/{run-record-D1Hw-Jml.d.ts → run-record-C1E24G2A.d.ts} +2 -2
- package/dist/{run-record-D1Hw-Jml.d.ts.map → run-record-C1E24G2A.d.ts.map} +1 -1
- package/dist/{run-record-ES07wfz9.js → run-record-C5e7FjhJ.js} +3 -5
- package/dist/run-record-C5e7FjhJ.js.map +1 -0
- package/dist/search/index.d.ts +46 -32
- package/dist/search/index.d.ts.map +1 -1
- package/dist/search/index.js +3 -3
- package/dist/{search-kernel-BqyBZ9Wm.d.ts → search-kernel-DyNEa6IU.d.ts} +14 -10
- package/dist/search-kernel-DyNEa6IU.d.ts.map +1 -0
- package/dist/{search-ledger-d7V8o9Lk.js → search-ledger-CVUGLDZp.js} +189 -199
- package/dist/search-ledger-CVUGLDZp.js.map +1 -0
- package/dist/{search-ledger-file-BqOo-bVo.js → search-ledger-file-DDQUgvCg.js} +4 -4
- package/dist/{search-ledger-file-BqOo-bVo.js.map → search-ledger-file-DDQUgvCg.js.map} +1 -1
- package/dist/search-ledger-types-p2xKfiDJ.d.ts.map +1 -1
- package/dist/{search-shipper-er4pK5_Y.js → search-shipper-BOSD46qv.js} +3 -3
- package/dist/{search-shipper-er4pK5_Y.js.map → search-shipper-BOSD46qv.js.map} +1 -1
- package/dist/{search-summary-CBvz3u_S.js → search-summary-BQCSWf1g.js} +5 -5
- package/dist/{search-summary-CBvz3u_S.js.map → search-summary-BQCSWf1g.js.map} +1 -1
- package/dist/{semantic-concept-judge-BPqbwIVs.js → semantic-concept-judge-BQF8te0T.js} +12 -14
- package/dist/semantic-concept-judge-BQF8te0T.js.map +1 -0
- package/dist/{sequential-D-1PPCmM.js → sequential-Bf_pbE50.js} +5 -5
- package/dist/{sequential-D-1PPCmM.js.map → sequential-Bf_pbE50.js.map} +1 -1
- package/dist/{sequential-eprocess-D1jKoihe.js → sequential-eprocess-DM_qENCl.js} +2 -2
- package/dist/{sequential-eprocess-D1jKoihe.js.map → sequential-eprocess-DM_qENCl.js.map} +1 -1
- package/dist/{server-DbcC7xMl.js → server-e9oWO_bN.js} +4 -4
- package/dist/{server-DbcC7xMl.js.map → server-e9oWO_bN.js.map} +1 -1
- package/dist/{single-run-lock-fVu3MZWH.js → single-run-lock-CV9Eyd_q.js} +2 -2
- package/dist/{single-run-lock-fVu3MZWH.js.map → single-run-lock-CV9Eyd_q.js.map} +1 -1
- package/dist/{skillopt-optimization-method-Dcepr6QL.js → skillopt-optimization-method-BiwA8nfX.js} +427 -12
- package/dist/skillopt-optimization-method-BiwA8nfX.js.map +1 -0
- package/dist/{spans-D9_ehFuk.d.ts → spans-BHAUXMGN.d.ts} +2 -2
- package/dist/{spans-D9_ehFuk.d.ts.map → spans-BHAUXMGN.d.ts.map} +1 -1
- package/dist/{statistical-heldout-1Gbmex7v.js → statistical-heldout-42NlCNDe.js} +3 -3
- package/dist/{statistical-heldout-1Gbmex7v.js.map → statistical-heldout-42NlCNDe.js.map} +1 -1
- package/dist/{statistical-heldout-CJTj2EoS.d.ts → statistical-heldout-CYpGaoDH.d.ts} +3 -3
- package/dist/{statistical-heldout-CJTj2EoS.d.ts.map → statistical-heldout-CYpGaoDH.d.ts.map} +1 -1
- package/dist/{store-otlp-DWVhUgEj.js → store-otlp-DJ9dtA5x.js} +135 -132
- package/dist/store-otlp-DJ9dtA5x.js.map +1 -0
- package/dist/{store-tool-spans-BNICrhsb.js → store-tool-spans-CIX5arAG.js} +7 -7
- package/dist/{store-tool-spans-BNICrhsb.js.map → store-tool-spans-CIX5arAG.js.map} +1 -1
- package/dist/{store-tool-spans-BNzls3OD.d.ts → store-tool-spans-DdCSPa7C.d.ts} +162 -161
- package/dist/store-tool-spans-DdCSPa7C.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts.map +1 -1
- package/dist/storyboard/index.js +20 -3
- package/dist/storyboard/index.js.map +1 -1
- package/dist/{student-t-BQeKJx2N.js → student-t-DVPW7CdL.js} +2 -2
- package/dist/{student-t-BQeKJx2N.js.map → student-t-DVPW7CdL.js.map} +1 -1
- package/dist/{summary-report-DtdRucM0.js → summary-report-V_JACR4Y.js} +526 -614
- package/dist/summary-report-V_JACR4Y.js.map +1 -0
- package/dist/{summary-report-C-CtItCN.d.ts → summary-report-bdFDXq0b.d.ts} +10 -5
- package/dist/summary-report-bdFDXq0b.d.ts.map +1 -0
- package/dist/supervisor-run/index.d.ts +154 -150
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +65 -3
- package/dist/supervisor-run/index.js.map +1 -1
- package/dist/systemone-BomhHr2c.d.ts +135 -0
- package/dist/systemone-BomhHr2c.d.ts.map +1 -0
- package/dist/systemone-protocol.js +1 -1
- package/dist/{systemone-review-C4dZgrgN.d.ts → systemone-review-CUVcQJu6.d.ts} +2 -2
- package/dist/{systemone-review-C4dZgrgN.d.ts.map → systemone-review-CUVcQJu6.d.ts.map} +1 -1
- package/dist/{systemone-review-DTClwgmE.js → systemone-review-D7sXwsqq.js} +3 -3
- package/dist/{systemone-review-DTClwgmE.js.map → systemone-review-D7sXwsqq.js.map} +1 -1
- package/dist/systemone.d.ts +4 -134
- package/dist/systemone.js +4 -4
- package/dist/{task-failure-attributes-DngaLdE4.js → task-failure-attributes-hEmASjix.js} +2 -2
- package/dist/{task-failure-attributes-DngaLdE4.js.map → task-failure-attributes-hEmASjix.js.map} +1 -1
- package/dist/{terminal-record-BHocwFw1.js → terminal-record-DZyQZ3pz.js} +14 -10
- package/dist/terminal-record-DZyQZ3pz.js.map +1 -0
- package/dist/{tool-groups-BKRrtqy4.d.ts → tool-groups-C3NEcdCM.d.ts} +3 -3
- package/dist/tool-groups-C3NEcdCM.d.ts.map +1 -0
- package/dist/{tool-waste-Wi4g6Ekb.d.ts → tool-waste-BKgvd-TP.d.ts} +30 -30
- package/dist/tool-waste-BKgvd-TP.d.ts.map +1 -0
- package/dist/{tool-waste-BYt5rdt8.js → tool-waste-BlesqTqW.js} +5 -158
- package/dist/tool-waste-BlesqTqW.js.map +1 -0
- package/dist/{trace-questions-CLx8vt6t.d.ts → trace-questions-CM2CAocU.d.ts} +4 -4
- package/dist/{trace-questions-CLx8vt6t.d.ts.map → trace-questions-CM2CAocU.d.ts.map} +1 -1
- package/dist/trace-repair/index.d.ts +3 -3
- package/dist/trace-repair/index.d.ts.map +1 -1
- package/dist/trace-repair/index.js +7 -13
- package/dist/trace-repair/index.js.map +1 -1
- package/dist/traces.d.ts +76 -58
- package/dist/traces.d.ts.map +1 -1
- package/dist/traces.js +9 -9
- package/dist/trajectory-replay/index.js +3 -3
- package/dist/{tree-rXLW3M_4.js → tree-C1XYeXvy.js} +2277 -2339
- package/dist/tree-C1XYeXvy.js.map +1 -0
- package/dist/{types-DjOMwGY_.d.ts → types-BqkVmHfv.d.ts} +7 -1
- package/dist/types-BqkVmHfv.d.ts.map +1 -0
- package/dist/{types-DBZ7keha.d.ts → types-C02qtnG8.d.ts} +4 -46
- package/dist/types-C02qtnG8.d.ts.map +1 -0
- package/dist/{types-DHb5TI36.d.ts → types-C1tnkEgM.d.ts} +347 -6
- package/dist/types-C1tnkEgM.d.ts.map +1 -0
- package/dist/{types-DinHtRJp.d.ts → types-Cbe2fWBa.d.ts} +3 -3
- package/dist/{types-DinHtRJp.d.ts.map → types-Cbe2fWBa.d.ts.map} +1 -1
- package/dist/{types-CGEmVYOk.d.ts → types-Cq8mSbq8.d.ts} +78 -72
- package/dist/types-Cq8mSbq8.d.ts.map +1 -0
- package/dist/{types-CW5b90JH.js → types-IXD2HJzu.js} +16 -3
- package/dist/types-IXD2HJzu.js.map +1 -0
- package/dist/{verdict-DhPoXApk.js → verdict-Dl0tiT1X.js} +2 -2
- package/dist/{verdict-DhPoXApk.js.map → verdict-Dl0tiT1X.js.map} +1 -1
- package/dist/{verdict-cache-BQEsVy9n.js → verdict-cache-Bm8EKL2s.js} +2 -2
- package/dist/{verdict-cache-BQEsVy9n.js.map → verdict-cache-Bm8EKL2s.js.map} +1 -1
- package/dist/wire/index.d.ts +3 -3
- package/dist/wire/index.js +1 -1
- package/docs/concepts.md +1 -0
- package/docs/design/mlbenchmarks-book-review.md +1 -1
- package/docs/design/mlbenchmarks-review/probes.mts +23 -31
- package/docs/experiment.md +1 -1
- package/docs/finding-layers.md +69 -0
- package/package.json +1 -1
- package/dist/analyze-runs-3xpby3p2.js.map +0 -1
- package/dist/analyze-runs-BhFLOztS.d.ts.map +0 -1
- package/dist/ast-DJ6Bc7ab.js +0 -558
- package/dist/ast-DJ6Bc7ab.js.map +0 -1
- package/dist/ast-hI-vjW6J.d.ts +0 -457
- package/dist/ast-hI-vjW6J.d.ts.map +0 -1
- package/dist/benchmark-command-CgpGi7vY.js.map +0 -1
- package/dist/campaign-CpnsCgfz.js.map +0 -1
- package/dist/campaign-evidence-OcUciD-N.js.map +0 -1
- package/dist/claim-BXcZw-jo.js.map +0 -1
- package/dist/cost-ledger-CO7UyEcQ.d.ts.map +0 -1
- package/dist/cost-ledger-D2FnJPBF.js.map +0 -1
- package/dist/default-registry-KP4-Tolz.js.map +0 -1
- package/dist/define-agent-eval-BdfnqXeP.js.map +0 -1
- package/dist/evaluation.d.ts.map +0 -1
- package/dist/external-optimizer-subprocess-BUxifb8g.js.map +0 -1
- package/dist/feedback-trajectory-BLN_6fEs.d.ts.map +0 -1
- package/dist/index-B3WBm953.d.ts.map +0 -1
- package/dist/index-B7m2uzJL.d.ts.map +0 -1
- package/dist/index-CPJJr98o.d.ts.map +0 -1
- package/dist/integrity-BV1ZDf42.js.map +0 -1
- package/dist/judge-gate-B1qYnkpi.js.map +0 -1
- package/dist/llm-client-DQveoxSM.js.map +0 -1
- package/dist/llm-judge-Cd5s5Ei6.js.map +0 -1
- package/dist/matrix-_BV9-Lk5.js.map +0 -1
- package/dist/mint-DCGoBYhB.d.ts +0 -155
- package/dist/mint-DCGoBYhB.d.ts.map +0 -1
- package/dist/paired-tests-jbwPdSz5.js.map +0 -1
- package/dist/proposal-findings-CXblsskH.js.map +0 -1
- package/dist/redact-Cveoq6fJ.d.ts.map +0 -1
- package/dist/redact-Dk1ZdJEm.js.map +0 -1
- package/dist/registry-CMYIaDgS.d.ts.map +0 -1
- package/dist/release-confidence-BK2QCB5M.d.ts.map +0 -1
- package/dist/report-command-CtIIsQKP.js.map +0 -1
- package/dist/run-record-ES07wfz9.js.map +0 -1
- package/dist/search-kernel-BqyBZ9Wm.d.ts.map +0 -1
- package/dist/search-kernel-BztlERLv.js.map +0 -1
- package/dist/search-ledger-d7V8o9Lk.js.map +0 -1
- package/dist/semantic-concept-judge-BPqbwIVs.js.map +0 -1
- package/dist/skillopt-optimization-method-Dcepr6QL.js.map +0 -1
- package/dist/store-otlp-DWVhUgEj.js.map +0 -1
- package/dist/store-tool-spans-BNzls3OD.d.ts.map +0 -1
- package/dist/summary-report-C-CtItCN.d.ts.map +0 -1
- package/dist/summary-report-DtdRucM0.js.map +0 -1
- package/dist/systemone.d.ts.map +0 -1
- package/dist/terminal-record-BHocwFw1.js.map +0 -1
- package/dist/tool-groups-BKRrtqy4.d.ts.map +0 -1
- package/dist/tool-waste-BYt5rdt8.js.map +0 -1
- package/dist/tool-waste-Wi4g6Ekb.d.ts.map +0 -1
- package/dist/tree-rXLW3M_4.js.map +0 -1
- package/dist/types-CGEmVYOk.d.ts.map +0 -1
- package/dist/types-CW5b90JH.js.map +0 -1
- package/dist/types-DBZ7keha.d.ts.map +0 -1
- package/dist/types-DHb5TI36.d.ts.map +0 -1
- package/dist/types-DjOMwGY_.d.ts.map +0 -1
|
@@ -1,25 +1,24 @@
|
|
|
1
|
-
import { i as JudgeError, s as ValidationError } from "./errors-
|
|
2
|
-
import { a as hashCanonical, i as compareCodeUnits, r as canonicalString, t as LEDGER_HASH_PATTERN } from "./canonical-
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import { i as
|
|
8
|
-
import { n as contentHash } from "./verdict-cache-
|
|
9
|
-
import { r as projectCampaignCellQuality, t as campaignCellCostProvenance } from "./run-record-
|
|
10
|
-
import {
|
|
11
|
-
import { B as assertCampaignDesign, D as FinalEvidenceError, E as FinalEvidenceConflictError, F as surfaceHash, G as campaignSplitDigest, H as assertCompleteCampaign, I as surfaceHashMatches, L as BackendIntegrityError, M as renderSurfaceDiff, N as surfaceContentHash, P as surfaceDispatchRef, U as campaignCoverage, V as assertCampaignSplitIdentity, W as campaignScenarioIdentity, b as campaignMeanCompositeOrNull, q as formatCoverageFailures,
|
|
12
|
-
import { m as mapConcurrent } from "./journal-DvKCvjty.js";
|
|
1
|
+
import { i as JudgeError, s as ValidationError } from "./errors-DVZiqS7K.js";
|
|
2
|
+
import { a as hashCanonical, i as compareCodeUnits, r as canonicalString, t as LEDGER_HASH_PATTERN } from "./canonical-B2RqLofv.js";
|
|
3
|
+
import { i as modelHasSnapshot, s as searchCellRunId } from "./run-record-Boc5IUTp.js";
|
|
4
|
+
import { t as mulberry32 } from "./random-ByUSnV9w.js";
|
|
5
|
+
import { o as summarizeNumberSeries, s as weightedComposite, t as confidenceInterval } from "./descriptive-DboxPZrG.js";
|
|
6
|
+
import { r as minimumPairsForPairedDeltaTest } from "./paired-promotion-decision-DsAPYGCm.js";
|
|
7
|
+
import { a as heldoutSignificance, i as dimensionRegressions, o as pairHoldout } from "./statistical-heldout-42NlCNDe.js";
|
|
8
|
+
import { n as contentHash } from "./verdict-cache-Bm8EKL2s.js";
|
|
9
|
+
import { r as projectCampaignCellQuality, t as campaignCellCostProvenance } from "./run-record-C5e7FjhJ.js";
|
|
10
|
+
import { n as CostAccountingIncompleteError, t as CostLedger } from "./cost-ledger-BP8V4kUQ.js";
|
|
11
|
+
import { B as assertCampaignDesign, D as FinalEvidenceError, E as FinalEvidenceConflictError, F as surfaceHash, G as campaignSplitDigest, H as assertCompleteCampaign, I as surfaceHashMatches, L as BackendIntegrityError, M as renderSurfaceDiff, N as surfaceContentHash, P as surfaceDispatchRef, U as campaignCoverage, V as assertCampaignSplitIdentity, W as campaignScenarioIdentity, b as campaignMeanCompositeOrNull, q as formatCoverageFailures, v as campaignBreakdown, y as campaignMeanComposite } from "./campaign-evidence-Clrn4hBV.js";
|
|
13
12
|
import { t as deepFreezeCanonicalJson } from "./deep-freeze-CurcOQg3.js";
|
|
14
|
-
import {
|
|
15
|
-
import { r as summarizeEvaluationUnits, t as defineEvaluationClaim } from "./claim-
|
|
16
|
-
import {
|
|
17
|
-
import { t as detectRewardHacking } from "./reward-hacking-
|
|
18
|
-
import { t as assertProposalFindings } from "./proposal-findings-
|
|
19
|
-
import {
|
|
20
|
-
import { a as validateSearchLedgerEvent, d as
|
|
21
|
-
import { n as searchProposerView, t as renderSearchSummary } from "./search-summary-
|
|
22
|
-
import {
|
|
13
|
+
import { N as createRunCostLedger, P as fsCampaignStorage, g as isRecord, h as isExternalTextCandidate } from "./external-optimizer-subprocess-BqW0hHcI.js";
|
|
14
|
+
import { r as summarizeEvaluationUnits, t as defineEvaluationClaim } from "./claim-q4HQhK1G.js";
|
|
15
|
+
import { c as redactText, d as detectCredential, f as detectPersonalData, n as REDACTION_VERSION } from "./redact-B5xXKqQ3.js";
|
|
16
|
+
import { t as detectRewardHacking } from "./reward-hacking-E0d-WVcP.js";
|
|
17
|
+
import { t as assertProposalFindings } from "./proposal-findings-CY1xKn_a.js";
|
|
18
|
+
import { i as runSearch, m as searchExpansionIndex, n as paretoFrontier, o as searchPolicyView, p as SEARCH_KERNEL_SOURCE, r as paretoFrontierWithCrowding, s as EXPANSION_OPERATORS, w as estimateNode } from "./pareto-Cm6ndrWN.js";
|
|
19
|
+
import { a as validateSearchLedgerEvent, d as searchNodeId, f as searchTaskSetDigest, i as replaySearchLedgerText, n as openSearchLedger, u as searchCellId } from "./search-ledger-CVUGLDZp.js";
|
|
20
|
+
import { n as searchProposerView, t as renderSearchSummary } from "./search-summary-BQCSWf1g.js";
|
|
21
|
+
import { a as maximumChargeForLlmRequest, i as costReceiptFromLlmError, r as costReceiptFromLlm, x as stripFencedJson } from "./llm-client-CyBAoYiX.js";
|
|
23
22
|
import { createHash } from "node:crypto";
|
|
24
23
|
import { writeFileSync } from "node:fs";
|
|
25
24
|
import { z } from "zod";
|
|
@@ -1200,8 +1199,8 @@ function detectCalibrationDrift(runs, opts) {
|
|
|
1200
1199
|
alpha,
|
|
1201
1200
|
recentN: recent.length,
|
|
1202
1201
|
historyN: historical.length,
|
|
1203
|
-
recentMean: mean$
|
|
1204
|
-
historyMean: mean$
|
|
1202
|
+
recentMean: mean$1(recent),
|
|
1203
|
+
historyMean: mean$1(historical)
|
|
1205
1204
|
}
|
|
1206
1205
|
}],
|
|
1207
1206
|
evaluation: {
|
|
@@ -1438,7 +1437,7 @@ function chiSquareCritical(df, alpha) {
|
|
|
1438
1437
|
}
|
|
1439
1438
|
return TABLE[10][idx];
|
|
1440
1439
|
}
|
|
1441
|
-
function mean$
|
|
1440
|
+
function mean$1(xs) {
|
|
1442
1441
|
if (xs.length === 0) return 0;
|
|
1443
1442
|
return xs.reduce((s, x) => s + x, 0) / xs.length;
|
|
1444
1443
|
}
|
|
@@ -3190,7 +3189,6 @@ function nonEmpty(value, label) {
|
|
|
3190
3189
|
* and a fact already in the ledger is not appended again, so a resumed search
|
|
3191
3190
|
* continues its ledger instead of conflicting with its own history.
|
|
3192
3191
|
*/
|
|
3193
|
-
const MAX_LABEL_CHARS = 200;
|
|
3194
3192
|
/** The default claim: development numbers over exactly the supplied tasks. */
|
|
3195
3193
|
function developmentClaim(populationId) {
|
|
3196
3194
|
return {
|
|
@@ -3554,7 +3552,7 @@ function declaredSurfaces(surface, artifact) {
|
|
|
3554
3552
|
}
|
|
3555
3553
|
function boundedLabel(label) {
|
|
3556
3554
|
const redacted = redactText(label.trim(), { profile: "share" });
|
|
3557
|
-
return redacted.length <=
|
|
3555
|
+
return redacted.length <= 200 ? redacted : `${redacted.slice(0, 199)}…`;
|
|
3558
3556
|
}
|
|
3559
3557
|
/** The header without its stamp, for comparing a resumed opening to the stored one. */
|
|
3560
3558
|
function headerMaterial(header) {
|
|
@@ -3968,7 +3966,7 @@ function hillClimb(spec) {
|
|
|
3968
3966
|
evidence: {
|
|
3969
3967
|
...chosen.evidence,
|
|
3970
3968
|
units: units.length,
|
|
3971
|
-
...units.length > 0 ? { mean: mean
|
|
3969
|
+
...units.length > 0 ? { mean: mean(units.map((unit) => unit.mean)) } : {},
|
|
3972
3970
|
isLeader: chosen.nodeId === kept ? 1 : 0
|
|
3973
3971
|
}
|
|
3974
3972
|
}
|
|
@@ -4024,7 +4022,7 @@ function paretoMembers(view) {
|
|
|
4024
4022
|
members.push({
|
|
4025
4023
|
nodeId,
|
|
4026
4024
|
ordinal,
|
|
4027
|
-
mean: sign * mean
|
|
4025
|
+
mean: sign * mean([...objectives.values()]),
|
|
4028
4026
|
objectives,
|
|
4029
4027
|
distance: 0
|
|
4030
4028
|
});
|
|
@@ -4046,7 +4044,7 @@ function compareCrowded(a, b) {
|
|
|
4046
4044
|
if (a.mean !== b.mean) return a.mean > b.mean ? -1 : 1;
|
|
4047
4045
|
return a.ordinal - b.ordinal;
|
|
4048
4046
|
}
|
|
4049
|
-
function mean
|
|
4047
|
+
function mean(values) {
|
|
4050
4048
|
let total = 0;
|
|
4051
4049
|
for (const value of values) total += value;
|
|
4052
4050
|
return total / values.length;
|
|
@@ -4068,275 +4066,8 @@ function labelTrustRank(trust) {
|
|
|
4068
4066
|
return LABEL_TRUST_RANK[trust ?? "unverified"];
|
|
4069
4067
|
}
|
|
4070
4068
|
//#endregion
|
|
4071
|
-
//#region src/campaign/presets/
|
|
4072
|
-
/**
|
|
4073
|
-
* `runOptimization` improves a mutable surface with a caller-owned proposer,
|
|
4074
|
-
* as a search on the one kernel (`runSearch`).
|
|
4075
|
-
*
|
|
4076
|
-
* The baseline is the seeded root. The scenarios are the train split: the
|
|
4077
|
-
* proposer reads their scores, and the policy ranks on them because the loop
|
|
4078
|
-
* has no private selection split. A "generation" is one proposal of
|
|
4079
|
-
* `populationSize` candidates from the node the policy chose (by default the
|
|
4080
|
-
* incumbent, once every earlier candidate is measured). Every candidate is a
|
|
4081
|
-
* node measured on every scenario at every repeat, one cell at a time on the
|
|
4082
|
-
* run's lanes. The kept surface is the policy's leader: a budget decision on
|
|
4083
|
-
* the train split that claims nothing; `runImprovementLoop` compares it with
|
|
4084
|
-
* the baseline on separate cases.
|
|
4085
|
-
*
|
|
4086
|
-
* The search ledger is always written, beside the run (`<runDir>/search/`) or
|
|
4087
|
-
* where `searchLedger` puts it, and it is the run's only checkpoint: running
|
|
4088
|
-
* again on the same run directory and options continues an interrupted run
|
|
4089
|
-
* instead of starting over, and reruns nothing that settled.
|
|
4090
|
-
*/
|
|
4091
|
-
/** Improve a surface as a search on the kernel; see the module comment. */
|
|
4092
|
-
async function runOptimization(opts) {
|
|
4093
|
-
const candidateConcurrency = opts.candidateConcurrency ?? 1;
|
|
4094
|
-
if (typeof opts.runDir !== "string" || opts.runDir.trim().length === 0) throw new Error("runOptimization: runDir is required and must be a non-empty string");
|
|
4095
|
-
if (!Number.isInteger(candidateConcurrency) || candidateConcurrency < 1) throw new Error("runOptimization: candidateConcurrency must be a positive integer");
|
|
4096
|
-
for (const [name, value] of [["populationSize", opts.populationSize], ["maxGenerations", opts.maxGenerations]]) if (!Number.isSafeInteger(value) || value < 0) throw new Error(`runOptimization: ${name} must be a non-negative integer`);
|
|
4097
|
-
const initialFindings = immutableProposalSnapshot(assertProposalFindings(opts.findings ?? [], "runOptimization initial proposal findings"), "initial findings");
|
|
4098
|
-
const baselineSurface = immutableProposalSnapshot(opts.baselineSurface, "baseline surface");
|
|
4099
|
-
const runDir = resolveRunDir(opts.runDir, opts.repo);
|
|
4100
|
-
const storage = opts.storage ?? fsCampaignStorage();
|
|
4101
|
-
const costLedger = opts.costLedger ?? createRunCostLedger({
|
|
4102
|
-
storage,
|
|
4103
|
-
runDir,
|
|
4104
|
-
costCeilingUsd: opts.costCeiling
|
|
4105
|
-
});
|
|
4106
|
-
const reps = opts.reps ?? 1;
|
|
4107
|
-
const seed = opts.seed ?? 42;
|
|
4108
|
-
const requireJudgeScore = (opts.judges?.length ?? 0) > 0;
|
|
4109
|
-
const premeasured = opts.premeasuredBaseline ? validatedPremeasuredBaseline({
|
|
4110
|
-
input: opts.premeasuredBaseline,
|
|
4111
|
-
baselineSurface,
|
|
4112
|
-
scenarios: opts.scenarios,
|
|
4113
|
-
reps,
|
|
4114
|
-
seed,
|
|
4115
|
-
judges: opts.judges ?? [],
|
|
4116
|
-
dispatchRef: surfaceDispatchRef(baselineSurface, opts.dispatchRef),
|
|
4117
|
-
requireJudgeScore
|
|
4118
|
-
}) : void 0;
|
|
4119
|
-
const splitDigest = campaignSplitDigest(opts.scenarios, reps);
|
|
4120
|
-
const binding = opts.searchLedger ?? await defaultBinding(opts, runDir, storage);
|
|
4121
|
-
const { identity } = binding;
|
|
4122
|
-
const execution = {
|
|
4123
|
-
model: identity.model,
|
|
4124
|
-
agent: identity.agent,
|
|
4125
|
-
benchmark: {
|
|
4126
|
-
uri: "campaign://scenarios",
|
|
4127
|
-
revision: splitDigest
|
|
4128
|
-
}
|
|
4129
|
-
};
|
|
4130
|
-
const policy = opts.policy ?? incumbent();
|
|
4131
|
-
const allocation = uniform({ reps });
|
|
4132
|
-
const recorder = await SearchRecorder.open({
|
|
4133
|
-
ledger: binding.ledger,
|
|
4134
|
-
storage
|
|
4135
|
-
}, {
|
|
4136
|
-
subject: identity.subject ?? opts.proposer.kind,
|
|
4137
|
-
process: {
|
|
4138
|
-
name: "runOptimization",
|
|
4139
|
-
executionRef: identity.search
|
|
4140
|
-
},
|
|
4141
|
-
artifactKind: artifactKindOf(baselineSurface),
|
|
4142
|
-
objective: {
|
|
4143
|
-
metric: "composite",
|
|
4144
|
-
direction: "maximize",
|
|
4145
|
-
judge: identity.judge ?? { unknown: "runOptimization receives its judges as functions without a pinned source" },
|
|
4146
|
-
claim: identity.claim ?? developmentClaim(splitDigest)
|
|
4147
|
-
},
|
|
4148
|
-
splits: {
|
|
4149
|
-
train: opts.scenarios.map((scenario) => ({
|
|
4150
|
-
taskId: scenario.id,
|
|
4151
|
-
unitId: scenario.id,
|
|
4152
|
-
source: {
|
|
4153
|
-
uri: `scenario://${scenario.id}`,
|
|
4154
|
-
revision: hashCanonical(scenario)
|
|
4155
|
-
}
|
|
4156
|
-
})),
|
|
4157
|
-
selection: [],
|
|
4158
|
-
test: [],
|
|
4159
|
-
heldOutUnits: true
|
|
4160
|
-
},
|
|
4161
|
-
policy: {
|
|
4162
|
-
expansion: policy.name,
|
|
4163
|
-
allocation: allocation.name,
|
|
4164
|
-
seed
|
|
4165
|
-
},
|
|
4166
|
-
budget: {
|
|
4167
|
-
maxUsd: opts.costCeiling ?? null,
|
|
4168
|
-
maxCells: null,
|
|
4169
|
-
maxNodes: 1 + opts.populationSize * opts.maxGenerations,
|
|
4170
|
-
deadline: null,
|
|
4171
|
-
maxConcurrency: null,
|
|
4172
|
-
reservedClaimUsd: 0
|
|
4173
|
-
},
|
|
4174
|
-
containment: null,
|
|
4175
|
-
derivedFrom: null,
|
|
4176
|
-
identity: execution
|
|
4177
|
-
});
|
|
4178
|
-
const nodes = new SurfaceNodes({
|
|
4179
|
-
opts,
|
|
4180
|
-
recorder,
|
|
4181
|
-
runDir,
|
|
4182
|
-
costLedger,
|
|
4183
|
-
execution,
|
|
4184
|
-
premeasured
|
|
4185
|
-
});
|
|
4186
|
-
const scenarioById = new Map(opts.scenarios.map((scenario) => [scenario.id, scenario]));
|
|
4187
|
-
let findings = initialFindings;
|
|
4188
|
-
const executor = {
|
|
4189
|
-
lanes: () => [{
|
|
4190
|
-
name: "campaign",
|
|
4191
|
-
capacity: candidateConcurrency * (opts.maxConcurrency ?? 2),
|
|
4192
|
-
costCap: "estimate",
|
|
4193
|
-
cellUsd: 0
|
|
4194
|
-
}],
|
|
4195
|
-
place: () => "campaign",
|
|
4196
|
-
run: (work) => nodes.runCell(work, scenarioById),
|
|
4197
|
-
adopt: (work) => nodes.adoptCell(work)
|
|
4198
|
-
};
|
|
4199
|
-
const proposer = opts.proposer;
|
|
4200
|
-
const proposerSource = identity.proposer.source;
|
|
4201
|
-
const port = {
|
|
4202
|
-
name: proposer.kind,
|
|
4203
|
-
kind: "optimizer",
|
|
4204
|
-
source: proposerSource,
|
|
4205
|
-
execution: identity.proposer,
|
|
4206
|
-
childrenPerProposal: opts.populationSize,
|
|
4207
|
-
async propose(request) {
|
|
4208
|
-
const phases = [
|
|
4209
|
-
"search.proposal",
|
|
4210
|
-
"analysis.baseline",
|
|
4211
|
-
"analysis.generation"
|
|
4212
|
-
];
|
|
4213
|
-
const before = new Set(phases.flatMap((phase) => costLedger.list({ phase })).map(receiptKey));
|
|
4214
|
-
const accounting = () => {
|
|
4215
|
-
const fresh = phases.flatMap((phase) => costLedger.list({ phase })).filter((receipt) => !before.has(receiptKey(receipt)));
|
|
4216
|
-
return {
|
|
4217
|
-
accounting: searchReceiptAccounting(fresh),
|
|
4218
|
-
execution: opts.searchLedger ? identity.proposer : searchProposalExecution(fresh, identity.model.provider, proposerSource)
|
|
4219
|
-
};
|
|
4220
|
-
};
|
|
4221
|
-
const earlier = await recorder.state();
|
|
4222
|
-
const root = earlier.rootNodeId;
|
|
4223
|
-
const rootCampaign = nodes.campaign(earlier, root);
|
|
4224
|
-
const rootCoverage = campaignCoverage(rootCampaign.cells, opts.scenarios, reps, requireJudgeScore);
|
|
4225
|
-
if (!rootCoverage.complete) throw new Error(`runOptimization: ${premeasured ? "premeasured baseline" : "baseline"} is incomplete (${rootCoverage.scorableCellIds.length}/${rootCoverage.expectedCellIds.length} designed cells scorable) — ${formatCoverageFailures(rootCoverage)}. Refusing to optimize against an incomplete incumbent.`);
|
|
4226
|
-
const generation = nodes.generations(earlier).length;
|
|
4227
|
-
if (opts.analyzeGeneration) {
|
|
4228
|
-
const previous = generation - 1;
|
|
4229
|
-
const candidates = (previous < 0 ? [root] : nodes.expansionNodes(earlier, nodes.generations(earlier)[previous])).map((nodeId) => {
|
|
4230
|
-
const campaign = nodes.campaign(earlier, nodeId);
|
|
4231
|
-
return {
|
|
4232
|
-
surfaceHash: surfaceHash(nodes.surface(earlier, nodeId)),
|
|
4233
|
-
campaign,
|
|
4234
|
-
composite: campaignMeanCompositeOrNull(campaign)
|
|
4235
|
-
};
|
|
4236
|
-
});
|
|
4237
|
-
const history = nodes.history(earlier, policy);
|
|
4238
|
-
if (candidates.some(({ campaign }) => campaign.cells.length > 0)) {
|
|
4239
|
-
const fresh = await opts.analyzeGeneration({
|
|
4240
|
-
generation: previous,
|
|
4241
|
-
runDir: previous < 0 ? rootCampaign.runDir : `${runDir}/candidates`,
|
|
4242
|
-
candidates,
|
|
4243
|
-
history,
|
|
4244
|
-
costLedger,
|
|
4245
|
-
costPhase: previous < 0 ? "analysis.baseline" : "analysis.generation"
|
|
4246
|
-
});
|
|
4247
|
-
if (!Array.isArray(fresh)) throw new TypeError("runOptimization: analyzeGeneration must return an array");
|
|
4248
|
-
findings = immutableProposalSnapshot(assertProposalFindings(fresh, "runOptimization analysis findings"), "analysis findings");
|
|
4249
|
-
}
|
|
4250
|
-
}
|
|
4251
|
-
const state = await recorder.state();
|
|
4252
|
-
const proposalHistory = immutableProposalSnapshot(nodes.history(state, policy), "history");
|
|
4253
|
-
const parent = request.parents[0];
|
|
4254
|
-
const context = Object.freeze({
|
|
4255
|
-
currentSurface: immutableProposalSnapshot(parent.artifact, "current surface"),
|
|
4256
|
-
operator: request.operator,
|
|
4257
|
-
parents: immutableProposalSnapshot(request.parents.map(({ nodeId, artifact }) => ({
|
|
4258
|
-
nodeId,
|
|
4259
|
-
artifact: immutableProposalSnapshot(artifact, "parent surface")
|
|
4260
|
-
})), "parents"),
|
|
4261
|
-
train: searchProposerView(state),
|
|
4262
|
-
summary: renderSearchSummary(state, { split: "train" }),
|
|
4263
|
-
history: proposalHistory,
|
|
4264
|
-
findings: immutableProposalSnapshot(assertProposalFindings(findings, "runOptimization proposal findings"), "findings"),
|
|
4265
|
-
populationSize: opts.populationSize,
|
|
4266
|
-
generation,
|
|
4267
|
-
signal: request.signal,
|
|
4268
|
-
baselineOutcome: immutableProposalSnapshot(nodes.outcome(state, root), "baseline"),
|
|
4269
|
-
incumbentOutcome: immutableProposalSnapshot(nodes.outcome(state, request.leader), "incumbent outcome"),
|
|
4270
|
-
parentOutcome: immutableProposalSnapshot(nodes.outcome(state, parent.nodeId), "parent outcome"),
|
|
4271
|
-
maxImprovementShots: opts.maxImprovementShots,
|
|
4272
|
-
paretoParents: immutableProposalSnapshot(nodes.frontier(state), "Pareto parents"),
|
|
4273
|
-
costLedger,
|
|
4274
|
-
costPhase: "search.proposal"
|
|
4275
|
-
});
|
|
4276
|
-
if (proposer.decide?.({ history: proposalHistory }).stop) return {
|
|
4277
|
-
children: [],
|
|
4278
|
-
stop: "the proposer decided to stop",
|
|
4279
|
-
...accounting()
|
|
4280
|
-
};
|
|
4281
|
-
const proposed = await proposer.propose(context);
|
|
4282
|
-
if (!Array.isArray(proposed)) throw new TypeError("runOptimization: proposer must return an array");
|
|
4283
|
-
return {
|
|
4284
|
-
children: immutableProposalSnapshot(proposed, "candidate outputs").slice(0, opts.populationSize).map((candidate) => isProposedCandidate(candidate) ? {
|
|
4285
|
-
artifact: candidate.surface,
|
|
4286
|
-
label: candidate.label,
|
|
4287
|
-
rationale: candidate.rationale,
|
|
4288
|
-
...candidate.attribution ? { attribution: candidate.attribution } : {}
|
|
4289
|
-
} : {
|
|
4290
|
-
artifact: candidate,
|
|
4291
|
-
label: "",
|
|
4292
|
-
rationale: ""
|
|
4293
|
-
}),
|
|
4294
|
-
...accounting()
|
|
4295
|
-
};
|
|
4296
|
-
}
|
|
4297
|
-
};
|
|
4298
|
-
const result = await runSearch({
|
|
4299
|
-
recorder,
|
|
4300
|
-
root: baselineSurface,
|
|
4301
|
-
codec: surfaceCodec,
|
|
4302
|
-
policy,
|
|
4303
|
-
allocation,
|
|
4304
|
-
proposer: port,
|
|
4305
|
-
executor,
|
|
4306
|
-
maxExpansions: opts.maxGenerations,
|
|
4307
|
-
signal: opts.signal
|
|
4308
|
-
});
|
|
4309
|
-
const state = result.state;
|
|
4310
|
-
const winnerSurface = nodes.surface(state, result.leader);
|
|
4311
|
-
const firstEdge = state.edge(state.node(result.leader).edgeIds[0]);
|
|
4312
|
-
const winnerRationale = firstEdge.operator === "seed" || !("sha256" in firstEdge.rationale) ? void 0 : recorder.readBlob(firstEdge.rationale).text;
|
|
4313
|
-
const expansions = nodes.generations(state);
|
|
4314
|
-
return {
|
|
4315
|
-
generations: nodes.history(state, policy).map((record) => ({
|
|
4316
|
-
record,
|
|
4317
|
-
surfaces: nodes.expansionNodes(state, expansions[record.generationIndex]).map((nodeId) => {
|
|
4318
|
-
const surface = nodes.surface(state, nodeId);
|
|
4319
|
-
return {
|
|
4320
|
-
surfaceHash: surfaceHash(surface),
|
|
4321
|
-
surface,
|
|
4322
|
-
campaign: nodes.campaign(state, nodeId)
|
|
4323
|
-
};
|
|
4324
|
-
})
|
|
4325
|
-
})),
|
|
4326
|
-
baselineSurface,
|
|
4327
|
-
winnerSurface,
|
|
4328
|
-
winnerSurfaceHash: surfaceHash(winnerSurface),
|
|
4329
|
-
...firstEdge.operator !== "seed" && firstEdge.label ? { winnerLabel: firstEdge.label } : {},
|
|
4330
|
-
...winnerRationale ? { winnerRationale } : {},
|
|
4331
|
-
baselineCampaign: nodes.campaign(state, state.rootNodeId),
|
|
4332
|
-
cost: costLedger.summary(),
|
|
4333
|
-
searchHistory: recorder.receipt({
|
|
4334
|
-
producerId: proposer.kind,
|
|
4335
|
-
runId: runDir
|
|
4336
|
-
}),
|
|
4337
|
-
paretoFrontier: nodes.frontier(state)
|
|
4338
|
-
};
|
|
4339
|
-
}
|
|
4069
|
+
//#region src/campaign/presets/optimization-surface-nodes.ts
|
|
4070
|
+
/** Surface-node campaigns, cached cell adoption, and proposer history from the search ledger. */
|
|
4340
4071
|
/** A mutable surface as a search artifact. */
|
|
4341
4072
|
const surfaceCodec = {
|
|
4342
4073
|
node: surfaceNode,
|
|
@@ -4364,14 +4095,14 @@ var SurfaceNodes = class {
|
|
|
4364
4095
|
proposals = /* @__PURE__ */ new Map();
|
|
4365
4096
|
/** Cells this process ran or adopted, by node, keyed `scenarioId:rep`. */
|
|
4366
4097
|
cells = /* @__PURE__ */ new Map();
|
|
4367
|
-
constructor(
|
|
4368
|
-
this.opts =
|
|
4369
|
-
this.recorder =
|
|
4370
|
-
this.runDir =
|
|
4371
|
-
this.costLedger =
|
|
4372
|
-
this.execution =
|
|
4373
|
-
this.premeasured =
|
|
4374
|
-
this.storage =
|
|
4098
|
+
constructor(opts, recorder, runDir, costLedger, execution, premeasured) {
|
|
4099
|
+
this.opts = opts;
|
|
4100
|
+
this.recorder = recorder;
|
|
4101
|
+
this.runDir = runDir;
|
|
4102
|
+
this.costLedger = costLedger;
|
|
4103
|
+
this.execution = execution;
|
|
4104
|
+
this.premeasured = premeasured;
|
|
4105
|
+
this.storage = opts.storage ?? fsCampaignStorage();
|
|
4375
4106
|
}
|
|
4376
4107
|
/** `<runDir>/baseline` for the root, `<runDir>/candidates/<surfaceHash>`
|
|
4377
4108
|
* for every other node: content-addressed, so a cell's directory needs no
|
|
@@ -4406,7 +4137,10 @@ var SurfaceNodes = class {
|
|
|
4406
4137
|
const cell = cached === void 0 ? void 0 : JSON.parse(cached);
|
|
4407
4138
|
if (cell?.manifestHash !== this.manifestFor(work.artifact)) return null;
|
|
4408
4139
|
this.remember(work.nodeId, cell);
|
|
4409
|
-
return
|
|
4140
|
+
return campaignCellSearchResult(cell, {
|
|
4141
|
+
execution: this.execution,
|
|
4142
|
+
lane: work.lane
|
|
4143
|
+
});
|
|
4410
4144
|
}
|
|
4411
4145
|
/** Run one cell as a one-cell campaign in its node's directory; a cached
|
|
4412
4146
|
* cell is read back, not dispatched again. */
|
|
@@ -4421,7 +4155,10 @@ var SurfaceNodes = class {
|
|
|
4421
4155
|
})).cells[0];
|
|
4422
4156
|
if (!cell) throw new Error(`runOptimization: cell ${work.cellId} produced no campaign cell`);
|
|
4423
4157
|
this.remember(work.nodeId, cell);
|
|
4424
|
-
return
|
|
4158
|
+
return campaignCellSearchResult(cell, {
|
|
4159
|
+
execution: this.execution,
|
|
4160
|
+
lane: work.lane
|
|
4161
|
+
});
|
|
4425
4162
|
}
|
|
4426
4163
|
/** The node's campaign over the cells the ledger settled: the cells this
|
|
4427
4164
|
* process ran, else their cached results or failure receipts. */
|
|
@@ -4589,7 +4326,10 @@ var SurfaceNodes = class {
|
|
|
4589
4326
|
premeasuredCell(work) {
|
|
4590
4327
|
const cell = this.premeasured.cells.find((candidate) => candidate.scenarioId === work.taskId && candidate.rep === work.rep);
|
|
4591
4328
|
if (!cell) throw new Error(`runOptimization: premeasured baseline lacks ${work.taskId}:${work.rep}`);
|
|
4592
|
-
return
|
|
4329
|
+
return campaignCellSearchResult(cell, {
|
|
4330
|
+
execution: this.execution,
|
|
4331
|
+
lane: work.lane
|
|
4332
|
+
});
|
|
4593
4333
|
}
|
|
4594
4334
|
/** A cell's result as its campaign stored it for this surface: the cache of
|
|
4595
4335
|
* a scored cell, the final failure receipt of a failed one. A result another
|
|
@@ -4606,34 +4346,334 @@ var SurfaceNodes = class {
|
|
|
4606
4346
|
const cell = result !== void 0 ? JSON.parse(result) : receipt !== void 0 ? JSON.parse(receipt).cell : void 0;
|
|
4607
4347
|
return cell?.manifestHash === manifestHash ? cell : void 0;
|
|
4608
4348
|
}
|
|
4609
|
-
const cached = this.storage.read(cellCachePath(dir, cellId));
|
|
4610
|
-
const scored = cached === void 0 ? void 0 : JSON.parse(cached);
|
|
4611
|
-
if (scored?.manifestHash === manifestHash) return scored;
|
|
4612
|
-
const receipt = this.storage.read(join(cellDirectory(dir, cellId), "failure-receipt.json"));
|
|
4613
|
-
const failed = receipt === void 0 ? void 0 : JSON.parse(receipt).cell;
|
|
4614
|
-
return failed?.manifestHash === manifestHash ? failed : void 0;
|
|
4615
|
-
}
|
|
4616
|
-
manifestFor(surface) {
|
|
4617
|
-
return computeManifestHash({
|
|
4618
|
-
scenarios: this.opts.scenarios,
|
|
4619
|
-
judges: this.opts.judges ?? [],
|
|
4620
|
-
dispatchRef: surfaceDispatchRef(surface, this.opts.dispatchRef),
|
|
4621
|
-
seed: this.opts.seed ?? 42,
|
|
4622
|
-
reps: this.opts.reps ?? 1
|
|
4623
|
-
});
|
|
4624
|
-
}
|
|
4625
|
-
remember(nodeId, cell) {
|
|
4626
|
-
const cells = this.cells.get(nodeId) ?? /* @__PURE__ */ new Map();
|
|
4627
|
-
cells.set(`${cell.scenarioId}:${cell.rep}`, cell);
|
|
4628
|
-
this.cells.set(nodeId, cells);
|
|
4629
|
-
}
|
|
4630
|
-
|
|
4631
|
-
|
|
4632
|
-
|
|
4633
|
-
|
|
4634
|
-
|
|
4635
|
-
|
|
4636
|
-
|
|
4349
|
+
const cached = this.storage.read(cellCachePath(dir, cellId));
|
|
4350
|
+
const scored = cached === void 0 ? void 0 : JSON.parse(cached);
|
|
4351
|
+
if (scored?.manifestHash === manifestHash) return scored;
|
|
4352
|
+
const receipt = this.storage.read(join(cellDirectory(dir, cellId), "failure-receipt.json"));
|
|
4353
|
+
const failed = receipt === void 0 ? void 0 : JSON.parse(receipt).cell;
|
|
4354
|
+
return failed?.manifestHash === manifestHash ? failed : void 0;
|
|
4355
|
+
}
|
|
4356
|
+
manifestFor(surface) {
|
|
4357
|
+
return computeManifestHash({
|
|
4358
|
+
scenarios: this.opts.scenarios,
|
|
4359
|
+
judges: this.opts.judges ?? [],
|
|
4360
|
+
dispatchRef: surfaceDispatchRef(surface, this.opts.dispatchRef),
|
|
4361
|
+
seed: this.opts.seed ?? 42,
|
|
4362
|
+
reps: this.opts.reps ?? 1
|
|
4363
|
+
});
|
|
4364
|
+
}
|
|
4365
|
+
remember(nodeId, cell) {
|
|
4366
|
+
const cells = this.cells.get(nodeId) ?? /* @__PURE__ */ new Map();
|
|
4367
|
+
cells.set(`${cell.scenarioId}:${cell.rep}`, cell);
|
|
4368
|
+
this.cells.set(nodeId, cells);
|
|
4369
|
+
}
|
|
4370
|
+
};
|
|
4371
|
+
/** A scored campaign as a frontier member: per-scenario composite objectives. */
|
|
4372
|
+
function toParetoParent(surface, campaign, generation, label) {
|
|
4373
|
+
const objectives = {};
|
|
4374
|
+
for (const { scenarioId, composite } of campaignBreakdown(campaign).scenarios) objectives[scenarioId] = composite;
|
|
4375
|
+
const parent = {
|
|
4376
|
+
surface,
|
|
4377
|
+
surfaceHash: surfaceHash(surface),
|
|
4378
|
+
objectives,
|
|
4379
|
+
composite: campaignMeanComposite(campaign),
|
|
4380
|
+
generation
|
|
4381
|
+
};
|
|
4382
|
+
if (label) parent.label = label;
|
|
4383
|
+
return parent;
|
|
4384
|
+
}
|
|
4385
|
+
/** The non-dominated set over the per-scenario objective vectors. */
|
|
4386
|
+
function computeParetoFrontier(scored) {
|
|
4387
|
+
if (scored.length <= 1) return [...scored];
|
|
4388
|
+
const ids = /* @__PURE__ */ new Set();
|
|
4389
|
+
for (const p of scored) for (const id of Object.keys(p.objectives)) ids.add(id);
|
|
4390
|
+
if (ids.size === 0) return [...scored];
|
|
4391
|
+
const objectives = [...ids].map((id) => ({
|
|
4392
|
+
name: id,
|
|
4393
|
+
direction: "maximize",
|
|
4394
|
+
value: (p) => p.objectives[id] ?? Number.NEGATIVE_INFINITY
|
|
4395
|
+
}));
|
|
4396
|
+
return paretoFrontier(scored, objectives).frontier;
|
|
4397
|
+
}
|
|
4398
|
+
function toScoredSurfaceOutcome(hash, campaign, coverage, generation) {
|
|
4399
|
+
const breakdown = campaignBreakdown(campaign);
|
|
4400
|
+
return {
|
|
4401
|
+
split: "search",
|
|
4402
|
+
generation,
|
|
4403
|
+
surfaceHash: hash,
|
|
4404
|
+
composite: campaignMeanComposite(campaign),
|
|
4405
|
+
dimensions: breakdown.dimensions,
|
|
4406
|
+
scenarios: breakdown.scenarios,
|
|
4407
|
+
coverage: {
|
|
4408
|
+
expectedCells: coverage.expectedCellIds.length,
|
|
4409
|
+
scorableCells: coverage.scorableCellIds.length
|
|
4410
|
+
}
|
|
4411
|
+
};
|
|
4412
|
+
}
|
|
4413
|
+
//#endregion
|
|
4414
|
+
//#region src/campaign/presets/run-optimization.ts
|
|
4415
|
+
/**
|
|
4416
|
+
* `runOptimization` improves a mutable surface with a caller-owned proposer,
|
|
4417
|
+
* as a search on the one kernel (`runSearch`).
|
|
4418
|
+
*
|
|
4419
|
+
* The baseline is the seeded root. The scenarios are the train split: the
|
|
4420
|
+
* proposer reads their scores, and the policy ranks on them because the loop
|
|
4421
|
+
* has no private selection split. A "generation" is one proposal of
|
|
4422
|
+
* `populationSize` candidates from the node the policy chose (by default the
|
|
4423
|
+
* incumbent, once every earlier candidate is measured). Every candidate is a
|
|
4424
|
+
* node measured on every scenario at every repeat, one cell at a time on the
|
|
4425
|
+
* run's lanes. The kept surface is the policy's leader: a budget decision on
|
|
4426
|
+
* the train split that claims nothing; `runImprovementLoop` compares it with
|
|
4427
|
+
* the baseline on separate cases.
|
|
4428
|
+
*
|
|
4429
|
+
* The search ledger is always written, beside the run (`<runDir>/search/`) or
|
|
4430
|
+
* where `searchLedger` puts it, and it is the run's only checkpoint: running
|
|
4431
|
+
* again on the same run directory and options continues an interrupted run
|
|
4432
|
+
* instead of starting over, and reruns nothing that settled.
|
|
4433
|
+
*/
|
|
4434
|
+
/** Compose the optimization proposer and cell executor on the shared search kernel. */
|
|
4435
|
+
/** Improve a surface as a search on the kernel; see the module comment. */
|
|
4436
|
+
async function runOptimization(opts) {
|
|
4437
|
+
const candidateConcurrency = opts.candidateConcurrency ?? 1;
|
|
4438
|
+
if (typeof opts.runDir !== "string" || opts.runDir.trim().length === 0) throw new Error("runOptimization: runDir is required and must be a non-empty string");
|
|
4439
|
+
if (!Number.isInteger(candidateConcurrency) || candidateConcurrency < 1) throw new Error("runOptimization: candidateConcurrency must be a positive integer");
|
|
4440
|
+
for (const [name, value] of [["populationSize", opts.populationSize], ["maxGenerations", opts.maxGenerations]]) if (!Number.isSafeInteger(value) || value < 0) throw new Error(`runOptimization: ${name} must be a non-negative integer`);
|
|
4441
|
+
const initialFindings = immutableProposalSnapshot(assertProposalFindings(opts.findings ?? [], "runOptimization initial proposal findings"), "initial findings");
|
|
4442
|
+
const baselineSurface = immutableProposalSnapshot(opts.baselineSurface, "baseline surface");
|
|
4443
|
+
const runDir = resolveRunDir(opts.runDir, opts.repo);
|
|
4444
|
+
const storage = opts.storage ?? fsCampaignStorage();
|
|
4445
|
+
const costLedger = opts.costLedger ?? createRunCostLedger({
|
|
4446
|
+
storage,
|
|
4447
|
+
runDir,
|
|
4448
|
+
costCeilingUsd: opts.costCeiling
|
|
4449
|
+
});
|
|
4450
|
+
const reps = opts.reps ?? 1;
|
|
4451
|
+
const seed = opts.seed ?? 42;
|
|
4452
|
+
const requireJudgeScore = (opts.judges?.length ?? 0) > 0;
|
|
4453
|
+
const premeasured = opts.premeasuredBaseline ? validatedPremeasuredBaseline({
|
|
4454
|
+
input: opts.premeasuredBaseline,
|
|
4455
|
+
baselineSurface,
|
|
4456
|
+
scenarios: opts.scenarios,
|
|
4457
|
+
reps,
|
|
4458
|
+
seed,
|
|
4459
|
+
judges: opts.judges ?? [],
|
|
4460
|
+
dispatchRef: surfaceDispatchRef(baselineSurface, opts.dispatchRef),
|
|
4461
|
+
requireJudgeScore
|
|
4462
|
+
}) : void 0;
|
|
4463
|
+
const splitDigest = campaignSplitDigest(opts.scenarios, reps);
|
|
4464
|
+
const binding = opts.searchLedger ?? await defaultBinding(opts, runDir, storage);
|
|
4465
|
+
const { identity } = binding;
|
|
4466
|
+
const execution = {
|
|
4467
|
+
model: identity.model,
|
|
4468
|
+
agent: identity.agent,
|
|
4469
|
+
benchmark: {
|
|
4470
|
+
uri: "campaign://scenarios",
|
|
4471
|
+
revision: splitDigest
|
|
4472
|
+
}
|
|
4473
|
+
};
|
|
4474
|
+
const policy = opts.policy ?? incumbent();
|
|
4475
|
+
const allocation = uniform({ reps });
|
|
4476
|
+
const recorder = await SearchRecorder.open({
|
|
4477
|
+
ledger: binding.ledger,
|
|
4478
|
+
storage
|
|
4479
|
+
}, {
|
|
4480
|
+
subject: identity.subject ?? opts.proposer.kind,
|
|
4481
|
+
process: {
|
|
4482
|
+
name: "runOptimization",
|
|
4483
|
+
executionRef: identity.search
|
|
4484
|
+
},
|
|
4485
|
+
artifactKind: typeof baselineSurface === "object" && baselineSurface.kind === "code" ? "code" : "prompt",
|
|
4486
|
+
objective: {
|
|
4487
|
+
metric: "composite",
|
|
4488
|
+
direction: "maximize",
|
|
4489
|
+
judge: identity.judge ?? { unknown: "runOptimization receives its judges as functions without a pinned source" },
|
|
4490
|
+
claim: identity.claim ?? developmentClaim(splitDigest)
|
|
4491
|
+
},
|
|
4492
|
+
splits: {
|
|
4493
|
+
train: opts.scenarios.map((scenario) => ({
|
|
4494
|
+
taskId: scenario.id,
|
|
4495
|
+
unitId: scenario.id,
|
|
4496
|
+
source: {
|
|
4497
|
+
uri: `scenario://${scenario.id}`,
|
|
4498
|
+
revision: hashCanonical(scenario)
|
|
4499
|
+
}
|
|
4500
|
+
})),
|
|
4501
|
+
selection: [],
|
|
4502
|
+
test: [],
|
|
4503
|
+
heldOutUnits: true
|
|
4504
|
+
},
|
|
4505
|
+
policy: {
|
|
4506
|
+
expansion: policy.name,
|
|
4507
|
+
allocation: allocation.name,
|
|
4508
|
+
seed
|
|
4509
|
+
},
|
|
4510
|
+
budget: {
|
|
4511
|
+
maxUsd: opts.costCeiling ?? null,
|
|
4512
|
+
maxCells: null,
|
|
4513
|
+
maxNodes: 1 + opts.populationSize * opts.maxGenerations,
|
|
4514
|
+
deadline: null,
|
|
4515
|
+
maxConcurrency: null,
|
|
4516
|
+
reservedClaimUsd: 0
|
|
4517
|
+
},
|
|
4518
|
+
containment: null,
|
|
4519
|
+
derivedFrom: null,
|
|
4520
|
+
identity: execution
|
|
4521
|
+
});
|
|
4522
|
+
const nodes = new SurfaceNodes(opts, recorder, runDir, costLedger, execution, premeasured);
|
|
4523
|
+
const scenarioById = new Map(opts.scenarios.map((scenario) => [scenario.id, scenario]));
|
|
4524
|
+
let findings = initialFindings;
|
|
4525
|
+
const executor = {
|
|
4526
|
+
lanes: () => [{
|
|
4527
|
+
name: "campaign",
|
|
4528
|
+
capacity: candidateConcurrency * (opts.maxConcurrency ?? 2),
|
|
4529
|
+
costCap: "estimate",
|
|
4530
|
+
cellUsd: 0
|
|
4531
|
+
}],
|
|
4532
|
+
place: () => "campaign",
|
|
4533
|
+
run: (work) => nodes.runCell(work, scenarioById),
|
|
4534
|
+
adopt: (work) => nodes.adoptCell(work)
|
|
4535
|
+
};
|
|
4536
|
+
const proposer = opts.proposer;
|
|
4537
|
+
const proposerSource = identity.proposer.source;
|
|
4538
|
+
const port = {
|
|
4539
|
+
name: proposer.kind,
|
|
4540
|
+
kind: "optimizer",
|
|
4541
|
+
source: proposerSource,
|
|
4542
|
+
execution: identity.proposer,
|
|
4543
|
+
childrenPerProposal: opts.populationSize,
|
|
4544
|
+
async propose(request) {
|
|
4545
|
+
const phases = [
|
|
4546
|
+
"search.proposal",
|
|
4547
|
+
"analysis.baseline",
|
|
4548
|
+
"analysis.generation"
|
|
4549
|
+
];
|
|
4550
|
+
const before = new Set(phases.flatMap((phase) => costLedger.list({ phase })).map((receipt) => receipt.callId));
|
|
4551
|
+
const accounting = () => {
|
|
4552
|
+
const fresh = phases.flatMap((phase) => costLedger.list({ phase })).filter((receipt) => !before.has(receipt.callId));
|
|
4553
|
+
return {
|
|
4554
|
+
accounting: searchReceiptAccounting(fresh),
|
|
4555
|
+
execution: opts.searchLedger ? identity.proposer : searchProposalExecution(fresh, identity.model.provider, proposerSource)
|
|
4556
|
+
};
|
|
4557
|
+
};
|
|
4558
|
+
const earlier = await recorder.state();
|
|
4559
|
+
const root = earlier.rootNodeId;
|
|
4560
|
+
const rootCampaign = nodes.campaign(earlier, root);
|
|
4561
|
+
const rootCoverage = campaignCoverage(rootCampaign.cells, opts.scenarios, reps, requireJudgeScore);
|
|
4562
|
+
if (!rootCoverage.complete) throw new Error(`runOptimization: ${premeasured ? "premeasured baseline" : "baseline"} is incomplete (${rootCoverage.scorableCellIds.length}/${rootCoverage.expectedCellIds.length} designed cells scorable) — ${formatCoverageFailures(rootCoverage)}. Refusing to optimize against an incomplete incumbent.`);
|
|
4563
|
+
const generation = nodes.generations(earlier).length;
|
|
4564
|
+
if (opts.analyzeGeneration) {
|
|
4565
|
+
const previous = generation - 1;
|
|
4566
|
+
const candidates = (previous < 0 ? [root] : nodes.expansionNodes(earlier, nodes.generations(earlier)[previous])).map((nodeId) => {
|
|
4567
|
+
const campaign = nodes.campaign(earlier, nodeId);
|
|
4568
|
+
return {
|
|
4569
|
+
surfaceHash: surfaceHash(nodes.surface(earlier, nodeId)),
|
|
4570
|
+
campaign,
|
|
4571
|
+
composite: campaignMeanCompositeOrNull(campaign)
|
|
4572
|
+
};
|
|
4573
|
+
});
|
|
4574
|
+
const history = nodes.history(earlier, policy);
|
|
4575
|
+
if (candidates.some(({ campaign }) => campaign.cells.length > 0)) {
|
|
4576
|
+
const fresh = await opts.analyzeGeneration({
|
|
4577
|
+
generation: previous,
|
|
4578
|
+
runDir: previous < 0 ? rootCampaign.runDir : `${runDir}/candidates`,
|
|
4579
|
+
candidates,
|
|
4580
|
+
history,
|
|
4581
|
+
costLedger,
|
|
4582
|
+
costPhase: previous < 0 ? "analysis.baseline" : "analysis.generation"
|
|
4583
|
+
});
|
|
4584
|
+
if (!Array.isArray(fresh)) throw new TypeError("runOptimization: analyzeGeneration must return an array");
|
|
4585
|
+
findings = immutableProposalSnapshot(assertProposalFindings(fresh, "runOptimization analysis findings"), "analysis findings");
|
|
4586
|
+
}
|
|
4587
|
+
}
|
|
4588
|
+
const state = await recorder.state();
|
|
4589
|
+
const proposalHistory = immutableProposalSnapshot(nodes.history(state, policy), "history");
|
|
4590
|
+
const parent = request.parents[0];
|
|
4591
|
+
const context = Object.freeze({
|
|
4592
|
+
currentSurface: immutableProposalSnapshot(parent.artifact, "current surface"),
|
|
4593
|
+
operator: request.operator,
|
|
4594
|
+
parents: immutableProposalSnapshot(request.parents.map(({ nodeId, artifact }) => ({
|
|
4595
|
+
nodeId,
|
|
4596
|
+
artifact: immutableProposalSnapshot(artifact, "parent surface")
|
|
4597
|
+
})), "parents"),
|
|
4598
|
+
train: searchProposerView(state),
|
|
4599
|
+
summary: renderSearchSummary(state, { split: "train" }),
|
|
4600
|
+
history: proposalHistory,
|
|
4601
|
+
findings: immutableProposalSnapshot(assertProposalFindings(findings, "runOptimization proposal findings"), "findings"),
|
|
4602
|
+
populationSize: opts.populationSize,
|
|
4603
|
+
generation,
|
|
4604
|
+
signal: request.signal,
|
|
4605
|
+
baselineOutcome: immutableProposalSnapshot(nodes.outcome(state, root), "baseline"),
|
|
4606
|
+
incumbentOutcome: immutableProposalSnapshot(nodes.outcome(state, request.leader), "incumbent outcome"),
|
|
4607
|
+
parentOutcome: immutableProposalSnapshot(nodes.outcome(state, parent.nodeId), "parent outcome"),
|
|
4608
|
+
maxImprovementShots: opts.maxImprovementShots,
|
|
4609
|
+
paretoParents: immutableProposalSnapshot(nodes.frontier(state), "Pareto parents"),
|
|
4610
|
+
costLedger,
|
|
4611
|
+
costPhase: "search.proposal"
|
|
4612
|
+
});
|
|
4613
|
+
if (proposer.decide?.({ history: proposalHistory }).stop) return {
|
|
4614
|
+
children: [],
|
|
4615
|
+
stop: "the proposer decided to stop",
|
|
4616
|
+
...accounting()
|
|
4617
|
+
};
|
|
4618
|
+
const proposed = await proposer.propose(context);
|
|
4619
|
+
if (!Array.isArray(proposed)) throw new TypeError("runOptimization: proposer must return an array");
|
|
4620
|
+
return {
|
|
4621
|
+
children: immutableProposalSnapshot(proposed, "candidate outputs").slice(0, opts.populationSize).map((candidate) => isProposedCandidate(candidate) ? {
|
|
4622
|
+
artifact: candidate.surface,
|
|
4623
|
+
label: candidate.label,
|
|
4624
|
+
rationale: candidate.rationale,
|
|
4625
|
+
...candidate.attribution ? { attribution: candidate.attribution } : {}
|
|
4626
|
+
} : {
|
|
4627
|
+
artifact: candidate,
|
|
4628
|
+
label: "",
|
|
4629
|
+
rationale: ""
|
|
4630
|
+
}),
|
|
4631
|
+
...accounting()
|
|
4632
|
+
};
|
|
4633
|
+
}
|
|
4634
|
+
};
|
|
4635
|
+
const result = await runSearch({
|
|
4636
|
+
recorder,
|
|
4637
|
+
root: baselineSurface,
|
|
4638
|
+
codec: surfaceCodec,
|
|
4639
|
+
policy,
|
|
4640
|
+
allocation,
|
|
4641
|
+
proposer: port,
|
|
4642
|
+
executor,
|
|
4643
|
+
maxExpansions: opts.maxGenerations,
|
|
4644
|
+
signal: opts.signal
|
|
4645
|
+
});
|
|
4646
|
+
const state = result.state;
|
|
4647
|
+
const winnerSurface = nodes.surface(state, result.leader);
|
|
4648
|
+
const firstEdge = state.edge(state.node(result.leader).edgeIds[0]);
|
|
4649
|
+
const winnerRationale = firstEdge.operator === "seed" || !("sha256" in firstEdge.rationale) ? void 0 : recorder.readBlob(firstEdge.rationale).text;
|
|
4650
|
+
const expansions = nodes.generations(state);
|
|
4651
|
+
return {
|
|
4652
|
+
generations: nodes.history(state, policy).map((record) => ({
|
|
4653
|
+
record,
|
|
4654
|
+
surfaces: nodes.expansionNodes(state, expansions[record.generationIndex]).map((nodeId) => {
|
|
4655
|
+
const surface = nodes.surface(state, nodeId);
|
|
4656
|
+
return {
|
|
4657
|
+
surfaceHash: surfaceHash(surface),
|
|
4658
|
+
surface,
|
|
4659
|
+
campaign: nodes.campaign(state, nodeId)
|
|
4660
|
+
};
|
|
4661
|
+
})
|
|
4662
|
+
})),
|
|
4663
|
+
baselineSurface,
|
|
4664
|
+
winnerSurface,
|
|
4665
|
+
winnerSurfaceHash: surfaceHash(winnerSurface),
|
|
4666
|
+
...firstEdge.operator !== "seed" && firstEdge.label ? { winnerLabel: firstEdge.label } : {},
|
|
4667
|
+
...winnerRationale ? { winnerRationale } : {},
|
|
4668
|
+
baselineCampaign: nodes.campaign(state, state.rootNodeId),
|
|
4669
|
+
cost: costLedger.summary(),
|
|
4670
|
+
searchHistory: recorder.receipt({
|
|
4671
|
+
producerId: proposer.kind,
|
|
4672
|
+
runId: runDir
|
|
4673
|
+
}),
|
|
4674
|
+
paretoFrontier: nodes.frontier(state)
|
|
4675
|
+
};
|
|
4676
|
+
}
|
|
4637
4677
|
/** The default ledger and identities when the caller binds none. Revisions are
|
|
4638
4678
|
* content digests of what the caller declared, and each uri names what its
|
|
4639
4679
|
* digest covers: the dispatch ref, the proposer kind, the kernel. */
|
|
@@ -4676,12 +4716,6 @@ async function defaultBinding(opts, runDir, storage) {
|
|
|
4676
4716
|
};
|
|
4677
4717
|
}
|
|
4678
4718
|
}
|
|
4679
|
-
function artifactKindOf(surface) {
|
|
4680
|
-
return typeof surface === "object" && surface.kind === "code" ? "code" : "prompt";
|
|
4681
|
-
}
|
|
4682
|
-
function receiptKey(receipt) {
|
|
4683
|
-
return receipt.callId;
|
|
4684
|
-
}
|
|
4685
4719
|
function immutableProposalSnapshot(value, label) {
|
|
4686
4720
|
try {
|
|
4687
4721
|
return deepFreeze(structuredClone(value));
|
|
@@ -4719,48 +4753,6 @@ function validatedPremeasuredBaseline(args) {
|
|
|
4719
4753
|
if (!coverage.complete) throw new Error(`runOptimization: premeasured baseline is incomplete (${coverage.scorableCellIds.length}/${coverage.expectedCellIds.length} designed cells scorable) — ${formatCoverageFailures(coverage)}. Refusing to optimize against an incomplete incumbent.`);
|
|
4720
4754
|
return campaign;
|
|
4721
4755
|
}
|
|
4722
|
-
/** A scored campaign as a frontier member: per-scenario composite objectives. */
|
|
4723
|
-
function toParetoParent(surface, campaign, generation, label) {
|
|
4724
|
-
const objectives = {};
|
|
4725
|
-
for (const { scenarioId, composite } of campaignBreakdown(campaign).scenarios) objectives[scenarioId] = composite;
|
|
4726
|
-
const parent = {
|
|
4727
|
-
surface,
|
|
4728
|
-
surfaceHash: surfaceHash(surface),
|
|
4729
|
-
objectives,
|
|
4730
|
-
composite: campaignMeanComposite(campaign),
|
|
4731
|
-
generation
|
|
4732
|
-
};
|
|
4733
|
-
if (label) parent.label = label;
|
|
4734
|
-
return parent;
|
|
4735
|
-
}
|
|
4736
|
-
/** The non-dominated set over the per-scenario objective vectors. */
|
|
4737
|
-
function computeParetoFrontier(scored) {
|
|
4738
|
-
if (scored.length <= 1) return [...scored];
|
|
4739
|
-
const ids = /* @__PURE__ */ new Set();
|
|
4740
|
-
for (const p of scored) for (const id of Object.keys(p.objectives)) ids.add(id);
|
|
4741
|
-
if (ids.size === 0) return [...scored];
|
|
4742
|
-
const objectives = [...ids].map((id) => ({
|
|
4743
|
-
name: id,
|
|
4744
|
-
direction: "maximize",
|
|
4745
|
-
value: (p) => p.objectives[id] ?? Number.NEGATIVE_INFINITY
|
|
4746
|
-
}));
|
|
4747
|
-
return paretoFrontier(scored, objectives).frontier;
|
|
4748
|
-
}
|
|
4749
|
-
function toScoredSurfaceOutcome(hash, campaign, coverage, generation) {
|
|
4750
|
-
const breakdown = campaignBreakdown(campaign);
|
|
4751
|
-
return {
|
|
4752
|
-
split: "search",
|
|
4753
|
-
generation,
|
|
4754
|
-
surfaceHash: hash,
|
|
4755
|
-
composite: campaignMeanComposite(campaign),
|
|
4756
|
-
dimensions: breakdown.dimensions,
|
|
4757
|
-
scenarios: breakdown.scenarios,
|
|
4758
|
-
coverage: {
|
|
4759
|
-
expectedCells: coverage.expectedCellIds.length,
|
|
4760
|
-
scorableCells: coverage.scorableCellIds.length
|
|
4761
|
-
}
|
|
4762
|
-
};
|
|
4763
|
-
}
|
|
4764
4756
|
//#endregion
|
|
4765
4757
|
//#region src/campaign/presets/run-improvement-loop.ts
|
|
4766
4758
|
/**
|
|
@@ -4830,10 +4822,126 @@ async function runImprovementLoop(opts) {
|
|
|
4830
4822
|
ghRepo: opts.ghRepo
|
|
4831
4823
|
});
|
|
4832
4824
|
return {
|
|
4833
|
-
...optimization,
|
|
4834
|
-
...comparison,
|
|
4835
|
-
prResult,
|
|
4836
|
-
cost: costLedger.summary()
|
|
4825
|
+
...optimization,
|
|
4826
|
+
...comparison,
|
|
4827
|
+
prResult,
|
|
4828
|
+
cost: costLedger.summary()
|
|
4829
|
+
};
|
|
4830
|
+
}
|
|
4831
|
+
//#endregion
|
|
4832
|
+
//#region src/campaign/optimization-cost.ts
|
|
4833
|
+
/** Attribute method calls while retaining the shared account's admission and read behavior. */
|
|
4834
|
+
function createMethodCostScope(account, methodName) {
|
|
4835
|
+
const tags = { [`optimizationAttempt:${crypto.randomUUID()}`]: methodName };
|
|
4836
|
+
return {
|
|
4837
|
+
ledger: Object.freeze({
|
|
4838
|
+
costCeilingUsd: account.costCeilingUsd,
|
|
4839
|
+
runPaidCall: (input) => account.runPaidCall({
|
|
4840
|
+
...input,
|
|
4841
|
+
tags: {
|
|
4842
|
+
...input.tags,
|
|
4843
|
+
...tags
|
|
4844
|
+
}
|
|
4845
|
+
}),
|
|
4846
|
+
summary: account.summary.bind(account),
|
|
4847
|
+
list: account.list.bind(account),
|
|
4848
|
+
reconcile: account.reconcile.bind(account),
|
|
4849
|
+
markCompleted: account.markCompleted.bind(account),
|
|
4850
|
+
costPerCompletedTask: account.costPerCompletedTask.bind(account),
|
|
4851
|
+
...account.listPending ? { listPending: account.listPending.bind(account) } : {},
|
|
4852
|
+
...account.waitForIdle ? { waitForIdle: account.waitForIdle.bind(account) } : {}
|
|
4853
|
+
}),
|
|
4854
|
+
reconcile(reported) {
|
|
4855
|
+
const summary = account.summary({ tags });
|
|
4856
|
+
if (summary.pendingCalls > 0) throw new Error(`optimization method '${methodName}' returned with ${summary.pendingCalls} pending paid call(s)`);
|
|
4857
|
+
const recorded = costFromLedgerSummary(summary);
|
|
4858
|
+
const totalCostUsd = Math.max(reported.totalCostUsd, recorded.totalCostUsd);
|
|
4859
|
+
const roundingToleranceUsd = Number.EPSILON * Math.max(1, totalCostUsd) * (summary.totalCalls + 1);
|
|
4860
|
+
const combined = combineComparisonCosts([{
|
|
4861
|
+
label: "reported",
|
|
4862
|
+
cost: reported
|
|
4863
|
+
}, {
|
|
4864
|
+
label: "recorded",
|
|
4865
|
+
cost: recorded
|
|
4866
|
+
}]);
|
|
4867
|
+
const incompleteReasons = [
|
|
4868
|
+
...reported.incompleteReasons,
|
|
4869
|
+
...recorded.incompleteReasons.map((reason) => `recorded: ${reason}`),
|
|
4870
|
+
...recorded.totalCostUsd - reported.totalCostUsd > roundingToleranceUsd ? [`reported ${reported.totalCostUsd} USD below recorded ${recorded.totalCostUsd} USD`] : []
|
|
4871
|
+
];
|
|
4872
|
+
return {
|
|
4873
|
+
totalCostUsd,
|
|
4874
|
+
costProvenance: combined.costProvenance.kind === "uncaptured" ? combined.costProvenance : {
|
|
4875
|
+
kind: combined.costProvenance.kind,
|
|
4876
|
+
usd: totalCostUsd
|
|
4877
|
+
},
|
|
4878
|
+
accountingComplete: incompleteReasons.length === 0,
|
|
4879
|
+
incompleteReasons
|
|
4880
|
+
};
|
|
4881
|
+
}
|
|
4882
|
+
};
|
|
4883
|
+
}
|
|
4884
|
+
/** Keep the cost fields a custom optimization method must report. */
|
|
4885
|
+
function costFromLedgerSummary(summary) {
|
|
4886
|
+
const cost = {
|
|
4887
|
+
totalCostUsd: summary.totalCostUsd,
|
|
4888
|
+
costProvenance: structuredClone(summary.costProvenance),
|
|
4889
|
+
accountingComplete: summary.accountingComplete,
|
|
4890
|
+
incompleteReasons: [...summary.incompleteReasons]
|
|
4891
|
+
};
|
|
4892
|
+
assertComparisonCost(cost, "cost ledger");
|
|
4893
|
+
return cost;
|
|
4894
|
+
}
|
|
4895
|
+
/** Combine method costs without turning one unknown bill into a known total. */
|
|
4896
|
+
function combineComparisonCosts(entries) {
|
|
4897
|
+
const totalCostUsd = entries.reduce((total, entry) => total + entry.cost.totalCostUsd, 0);
|
|
4898
|
+
const cost = {
|
|
4899
|
+
totalCostUsd,
|
|
4900
|
+
costProvenance: entries.some((entry) => entry.cost.costProvenance.kind === "uncaptured") ? {
|
|
4901
|
+
kind: "uncaptured",
|
|
4902
|
+
usd: null
|
|
4903
|
+
} : entries.every((entry) => entry.cost.costProvenance.kind === "observed") ? {
|
|
4904
|
+
kind: "observed",
|
|
4905
|
+
usd: totalCostUsd
|
|
4906
|
+
} : {
|
|
4907
|
+
kind: "estimated",
|
|
4908
|
+
usd: totalCostUsd
|
|
4909
|
+
},
|
|
4910
|
+
accountingComplete: entries.every((entry) => entry.cost.accountingComplete),
|
|
4911
|
+
incompleteReasons: entries.flatMap((entry) => entry.cost.incompleteReasons.map((reason) => `${entry.label}: ${reason}`))
|
|
4912
|
+
};
|
|
4913
|
+
assertComparisonCost(cost, "combined cost");
|
|
4914
|
+
return cost;
|
|
4915
|
+
}
|
|
4916
|
+
function assertComparisonCost(cost, label) {
|
|
4917
|
+
if (!cost || typeof cost !== "object") throw new Error(`compareOptimizationMethods: ${label} returned no cost`);
|
|
4918
|
+
if (!Number.isFinite(cost.totalCostUsd) || cost.totalCostUsd < 0) throw new Error(`compareOptimizationMethods: ${label} returned an invalid totalCostUsd`);
|
|
4919
|
+
const provenance = cost.costProvenance;
|
|
4920
|
+
if (!provenance || typeof provenance !== "object" || provenance.kind !== "observed" && provenance.kind !== "estimated" && provenance.kind !== "uncaptured" || (provenance.kind === "uncaptured" ? provenance.usd !== null : !Number.isFinite(provenance.usd) || provenance.usd < 0)) throw new Error(`compareOptimizationMethods: ${label} returned invalid costProvenance`);
|
|
4921
|
+
if (provenance.kind !== "uncaptured" && provenance.usd !== cost.totalCostUsd) throw new Error(`compareOptimizationMethods: ${label} returned costProvenance inconsistent with totalCostUsd`);
|
|
4922
|
+
if (typeof cost.accountingComplete !== "boolean") throw new Error(`compareOptimizationMethods: ${label} returned invalid accountingComplete`);
|
|
4923
|
+
if (!Array.isArray(cost.incompleteReasons) || cost.incompleteReasons.some((reason) => typeof reason !== "string" || reason.trim().length === 0)) throw new Error(`compareOptimizationMethods: ${label} returned invalid incompleteReasons`);
|
|
4924
|
+
if (cost.accountingComplete !== (cost.incompleteReasons.length === 0)) throw new Error(`compareOptimizationMethods: ${label} returned inconsistent cost completeness and reasons`);
|
|
4925
|
+
if (cost.accountingComplete && provenance.kind === "uncaptured") throw new Error(`compareOptimizationMethods: ${label} cannot mark uncaptured cost as complete`);
|
|
4926
|
+
}
|
|
4927
|
+
/** Preserve every optimizer token class while keeping total input and output explicit. */
|
|
4928
|
+
function optimizationTokenUsageFromSummary(summary, receipts) {
|
|
4929
|
+
if (!summary.usageComplete) return void 0;
|
|
4930
|
+
if (receipts.length !== summary.totalCalls) throw new Error("optimization token usage receipt count does not match the cost summary");
|
|
4931
|
+
const cachedInputTokens = summary.cachedTokens;
|
|
4932
|
+
const cacheWriteInputTokens = summary.cacheWriteTokens ?? 0;
|
|
4933
|
+
const inputTokens = summary.inputTokens + cachedInputTokens + cacheWriteInputTokens;
|
|
4934
|
+
const cacheReadComplete = receipts.length === 0 || receipts.every((receipt) => receipt.cachedTokens !== void 0);
|
|
4935
|
+
const cacheWriteComplete = receipts.length === 0 || receipts.every((receipt) => receipt.cacheWriteTokens !== void 0);
|
|
4936
|
+
const reasoningComplete = receipts.length === 0 || receipts.every((receipt) => receipt.reasoningTokens !== void 0);
|
|
4937
|
+
return {
|
|
4938
|
+
inputTokens,
|
|
4939
|
+
...cacheReadComplete ? { cachedInputTokens } : {},
|
|
4940
|
+
...cacheWriteComplete ? { cacheWriteInputTokens } : {},
|
|
4941
|
+
outputTokens: summary.outputTokens,
|
|
4942
|
+
...reasoningComplete ? { reasoningTokens: summary.reasoningTokens ?? 0 } : {},
|
|
4943
|
+
totalTokens: inputTokens + summary.outputTokens,
|
|
4944
|
+
calls: summary.totalCalls
|
|
4837
4945
|
};
|
|
4838
4946
|
}
|
|
4839
4947
|
//#endregion
|
|
@@ -5022,102 +5130,6 @@ function assertPositiveSafeInteger(value, label) {
|
|
|
5022
5130
|
if (!Number.isSafeInteger(value) || value <= 0) throw new Error(`GEPA candidate population ${label} must be a positive safe integer`);
|
|
5023
5131
|
}
|
|
5024
5132
|
//#endregion
|
|
5025
|
-
//#region src/campaign/optimization-cost.ts
|
|
5026
|
-
/** Attribute method calls while retaining the shared account's admission and read behavior. */
|
|
5027
|
-
function createMethodCostScope(account, methodName) {
|
|
5028
|
-
const tags = { [`optimizationAttempt:${crypto.randomUUID()}`]: methodName };
|
|
5029
|
-
return {
|
|
5030
|
-
ledger: Object.freeze({
|
|
5031
|
-
costCeilingUsd: account.costCeilingUsd,
|
|
5032
|
-
runPaidCall: (input) => account.runPaidCall({
|
|
5033
|
-
...input,
|
|
5034
|
-
tags: {
|
|
5035
|
-
...input.tags,
|
|
5036
|
-
...tags
|
|
5037
|
-
}
|
|
5038
|
-
}),
|
|
5039
|
-
summary: account.summary.bind(account),
|
|
5040
|
-
list: account.list.bind(account),
|
|
5041
|
-
reconcile: account.reconcile.bind(account),
|
|
5042
|
-
markCompleted: account.markCompleted.bind(account),
|
|
5043
|
-
costPerCompletedTask: account.costPerCompletedTask.bind(account),
|
|
5044
|
-
...account.listPending ? { listPending: account.listPending.bind(account) } : {},
|
|
5045
|
-
...account.waitForIdle ? { waitForIdle: account.waitForIdle.bind(account) } : {}
|
|
5046
|
-
}),
|
|
5047
|
-
reconcile(reported) {
|
|
5048
|
-
const summary = account.summary({ tags });
|
|
5049
|
-
if (summary.pendingCalls > 0) throw new Error(`optimization method '${methodName}' returned with ${summary.pendingCalls} pending paid call(s)`);
|
|
5050
|
-
const recorded = costFromLedgerSummary(summary);
|
|
5051
|
-
const totalCostUsd = Math.max(reported.totalCostUsd, recorded.totalCostUsd);
|
|
5052
|
-
const roundingToleranceUsd = Number.EPSILON * Math.max(1, totalCostUsd) * (summary.totalCalls + 1);
|
|
5053
|
-
const combined = combineComparisonCosts([{
|
|
5054
|
-
label: "reported",
|
|
5055
|
-
cost: reported
|
|
5056
|
-
}, {
|
|
5057
|
-
label: "recorded",
|
|
5058
|
-
cost: recorded
|
|
5059
|
-
}]);
|
|
5060
|
-
const incompleteReasons = [
|
|
5061
|
-
...reported.incompleteReasons,
|
|
5062
|
-
...recorded.incompleteReasons.map((reason) => `recorded: ${reason}`),
|
|
5063
|
-
...recorded.totalCostUsd - reported.totalCostUsd > roundingToleranceUsd ? [`reported ${reported.totalCostUsd} USD below recorded ${recorded.totalCostUsd} USD`] : []
|
|
5064
|
-
];
|
|
5065
|
-
return {
|
|
5066
|
-
totalCostUsd,
|
|
5067
|
-
costProvenance: combined.costProvenance.kind === "uncaptured" ? combined.costProvenance : {
|
|
5068
|
-
kind: combined.costProvenance.kind,
|
|
5069
|
-
usd: totalCostUsd
|
|
5070
|
-
},
|
|
5071
|
-
accountingComplete: incompleteReasons.length === 0,
|
|
5072
|
-
incompleteReasons
|
|
5073
|
-
};
|
|
5074
|
-
}
|
|
5075
|
-
};
|
|
5076
|
-
}
|
|
5077
|
-
/** Keep the cost fields a custom optimization method must report. */
|
|
5078
|
-
function costFromLedgerSummary(summary) {
|
|
5079
|
-
const cost = {
|
|
5080
|
-
totalCostUsd: summary.totalCostUsd,
|
|
5081
|
-
costProvenance: structuredClone(summary.costProvenance),
|
|
5082
|
-
accountingComplete: summary.accountingComplete,
|
|
5083
|
-
incompleteReasons: [...summary.incompleteReasons]
|
|
5084
|
-
};
|
|
5085
|
-
assertComparisonCost(cost, "cost ledger");
|
|
5086
|
-
return cost;
|
|
5087
|
-
}
|
|
5088
|
-
/** Combine method costs without turning one unknown bill into a known total. */
|
|
5089
|
-
function combineComparisonCosts(entries) {
|
|
5090
|
-
const totalCostUsd = entries.reduce((total, entry) => total + entry.cost.totalCostUsd, 0);
|
|
5091
|
-
const cost = {
|
|
5092
|
-
totalCostUsd,
|
|
5093
|
-
costProvenance: entries.some((entry) => entry.cost.costProvenance.kind === "uncaptured") ? {
|
|
5094
|
-
kind: "uncaptured",
|
|
5095
|
-
usd: null
|
|
5096
|
-
} : entries.every((entry) => entry.cost.costProvenance.kind === "observed") ? {
|
|
5097
|
-
kind: "observed",
|
|
5098
|
-
usd: totalCostUsd
|
|
5099
|
-
} : {
|
|
5100
|
-
kind: "estimated",
|
|
5101
|
-
usd: totalCostUsd
|
|
5102
|
-
},
|
|
5103
|
-
accountingComplete: entries.every((entry) => entry.cost.accountingComplete),
|
|
5104
|
-
incompleteReasons: entries.flatMap((entry) => entry.cost.incompleteReasons.map((reason) => `${entry.label}: ${reason}`))
|
|
5105
|
-
};
|
|
5106
|
-
assertComparisonCost(cost, "combined cost");
|
|
5107
|
-
return cost;
|
|
5108
|
-
}
|
|
5109
|
-
function assertComparisonCost(cost, label) {
|
|
5110
|
-
if (!cost || typeof cost !== "object") throw new Error(`compareOptimizationMethods: ${label} returned no cost`);
|
|
5111
|
-
if (!Number.isFinite(cost.totalCostUsd) || cost.totalCostUsd < 0) throw new Error(`compareOptimizationMethods: ${label} returned an invalid totalCostUsd`);
|
|
5112
|
-
const provenance = cost.costProvenance;
|
|
5113
|
-
if (!provenance || typeof provenance !== "object" || provenance.kind !== "observed" && provenance.kind !== "estimated" && provenance.kind !== "uncaptured" || (provenance.kind === "uncaptured" ? provenance.usd !== null : !Number.isFinite(provenance.usd) || provenance.usd < 0)) throw new Error(`compareOptimizationMethods: ${label} returned invalid costProvenance`);
|
|
5114
|
-
if (provenance.kind !== "uncaptured" && provenance.usd !== cost.totalCostUsd) throw new Error(`compareOptimizationMethods: ${label} returned costProvenance inconsistent with totalCostUsd`);
|
|
5115
|
-
if (typeof cost.accountingComplete !== "boolean") throw new Error(`compareOptimizationMethods: ${label} returned invalid accountingComplete`);
|
|
5116
|
-
if (!Array.isArray(cost.incompleteReasons) || cost.incompleteReasons.some((reason) => typeof reason !== "string" || reason.trim().length === 0)) throw new Error(`compareOptimizationMethods: ${label} returned invalid incompleteReasons`);
|
|
5117
|
-
if (cost.accountingComplete !== (cost.incompleteReasons.length === 0)) throw new Error(`compareOptimizationMethods: ${label} returned inconsistent cost completeness and reasons`);
|
|
5118
|
-
if (cost.accountingComplete && provenance.kind === "uncaptured") throw new Error(`compareOptimizationMethods: ${label} cannot mark uncaptured cost as complete`);
|
|
5119
|
-
}
|
|
5120
|
-
//#endregion
|
|
5121
5133
|
//#region src/campaign/optimization-method.ts
|
|
5122
5134
|
/** Both complete-method workflows use the same detached inputs, accounting, and evidence admission. */
|
|
5123
5135
|
async function executeOptimizationMethod(options) {
|
|
@@ -5266,440 +5278,6 @@ function assertOptimizationProvenance(methodName, value) {
|
|
|
5266
5278
|
if (value.optimizerModel === void 0 !== (value.optimizerCallRef === void 0) || value.optimizerModel === void 0 !== (value.modelExecutions === void 0)) fail("optimizerModel execution provenance");
|
|
5267
5279
|
}
|
|
5268
5280
|
//#endregion
|
|
5269
|
-
//#region src/campaign/presets/compare-optimization-methods.ts
|
|
5270
|
-
/**
|
|
5271
|
-
* Compare optimization methods on shared train, selection, and test data.
|
|
5272
|
-
* Optimizers receive only train and selection data. After every optimizer
|
|
5273
|
-
* finishes, their selected surfaces are measured on the same untouched test
|
|
5274
|
-
* data and compared with paired confidence intervals.
|
|
5275
|
-
*/
|
|
5276
|
-
/**
|
|
5277
|
-
* Compare complete optimization methods on disjoint train, selection, and final test data.
|
|
5278
|
-
*/
|
|
5279
|
-
async function compareOptimizationMethods(opts) {
|
|
5280
|
-
opts = {
|
|
5281
|
-
...opts,
|
|
5282
|
-
methods: opts.methods.map((method) => ({ ...method })),
|
|
5283
|
-
trainScenarios: structuredClone(opts.trainScenarios),
|
|
5284
|
-
selectionScenarios: structuredClone(opts.selectionScenarios),
|
|
5285
|
-
testScenarios: structuredClone(opts.testScenarios),
|
|
5286
|
-
judges: opts.judges.map(captureJudge),
|
|
5287
|
-
claim: opts.claim && defineEvaluationClaim(opts.claim),
|
|
5288
|
-
finalEvidence: opts.finalEvidence && captureFinalEvidencePolicy(opts.finalEvidence)
|
|
5289
|
-
};
|
|
5290
|
-
assertOptimizationMethods(opts.methods);
|
|
5291
|
-
assertComparisonPartitions(opts);
|
|
5292
|
-
const searchHistoryPolicy = opts.searchHistoryPolicy ?? "allow-missing";
|
|
5293
|
-
assertSearchHistoryAdmissionOptions(opts);
|
|
5294
|
-
const seed = opts.seed ?? 42;
|
|
5295
|
-
const confidence = opts.confidence ?? .95;
|
|
5296
|
-
assertConfidence(confidence);
|
|
5297
|
-
const optimizationConcurrency = opts.optimizationConcurrency ?? 1;
|
|
5298
|
-
const comparisonCount = opts.methods.length * (opts.methods.length + 1) / 2;
|
|
5299
|
-
const intervalConfidence = 1 - (1 - confidence) / comparisonCount;
|
|
5300
|
-
const minimumResamples = minimumBootstrapResamples(confidence, comparisonCount);
|
|
5301
|
-
const resamples = opts.resamples ?? Math.max(2e3, minimumResamples);
|
|
5302
|
-
assertComparisonControls(opts, seed, resamples, confidence);
|
|
5303
|
-
const evidence = opts.evidence === void 0 ? void 0 : structuredClone(opts.evidence);
|
|
5304
|
-
const evidenceBySurface = /* @__PURE__ */ new Map();
|
|
5305
|
-
const storage = opts.storage ?? fsCampaignStorage();
|
|
5306
|
-
const resolvedRunDir = resolveRunDir(opts.runDir, opts.repo);
|
|
5307
|
-
const baselineSurface = structuredClone(opts.baselineSurface);
|
|
5308
|
-
const costLedger = opts.costLedger ?? createRunCostLedger({
|
|
5309
|
-
storage,
|
|
5310
|
-
runDir: `${resolvedRunDir}/cost`,
|
|
5311
|
-
costCeilingUsd: opts.costCeiling
|
|
5312
|
-
});
|
|
5313
|
-
const scoreOnTest = async (surface, tag, costPhase) => {
|
|
5314
|
-
const measuredSurface = structuredClone(surface);
|
|
5315
|
-
const campaign = await runCampaign({
|
|
5316
|
-
...opts,
|
|
5317
|
-
storage,
|
|
5318
|
-
costLedger,
|
|
5319
|
-
costPhase,
|
|
5320
|
-
scenarios: opts.testScenarios.map((scenario) => structuredClone(scenario)),
|
|
5321
|
-
dispatch: (scenario, ctx) => opts.dispatchWithSurface(structuredClone(measuredSurface), scenario, ctx),
|
|
5322
|
-
dispatchRef: finalDispatchRef(opts, measuredSurface),
|
|
5323
|
-
runDir: `${resolvedRunDir}/${tag}`
|
|
5324
|
-
});
|
|
5325
|
-
assertCompleteCampaign(campaign, opts.testScenarios, opts.reps ?? 1, true, `compareOptimizationMethods: ${tag} final comparison`);
|
|
5326
|
-
if (evidence) evidenceBySurface.set(surfaceContentHash(measuredSurface), createCampaignEvidenceReceipt({
|
|
5327
|
-
campaign,
|
|
5328
|
-
surface: measuredSurface,
|
|
5329
|
-
context: evidence
|
|
5330
|
-
}));
|
|
5331
|
-
const byScenario = {};
|
|
5332
|
-
for (const { scenarioId, composite } of campaignBreakdown(campaign).scenarios) byScenario[scenarioId] = composite;
|
|
5333
|
-
return byScenario;
|
|
5334
|
-
};
|
|
5335
|
-
const scenarioIds = opts.testScenarios.map((s) => s.id).sort();
|
|
5336
|
-
const unitByScenario = opts.claim ? evaluationUnitMap(opts.claim, opts.testScenarios) : new Map(scenarioIds.map((id) => [id, id]));
|
|
5337
|
-
const units = opts.claim ? summarizeEvaluationUnits(opts.claim, opts.testScenarios) : {
|
|
5338
|
-
observations: scenarioIds.length,
|
|
5339
|
-
independentUnits: scenarioIds.length,
|
|
5340
|
-
units: scenarioIds.map((id) => ({
|
|
5341
|
-
id,
|
|
5342
|
-
observations: 1
|
|
5343
|
-
}))
|
|
5344
|
-
};
|
|
5345
|
-
const aggregateUnits = (arr) => {
|
|
5346
|
-
const values = /* @__PURE__ */ new Map();
|
|
5347
|
-
scenarioIds.forEach((id, index) => {
|
|
5348
|
-
const unitId = unitByScenario.get(id);
|
|
5349
|
-
const bucket = values.get(unitId) ?? [];
|
|
5350
|
-
bucket.push(arr[index]);
|
|
5351
|
-
values.set(unitId, bucket);
|
|
5352
|
-
});
|
|
5353
|
-
return units.units.map((unit) => mean(values.get(unit.id)));
|
|
5354
|
-
};
|
|
5355
|
-
const decisionOptions = {
|
|
5356
|
-
seed,
|
|
5357
|
-
resamples,
|
|
5358
|
-
confidence: intervalConfidence,
|
|
5359
|
-
threshold: opts.claim?.minimumEffect ?? 0
|
|
5360
|
-
};
|
|
5361
|
-
const align = (byScenario, label) => {
|
|
5362
|
-
const missing = scenarioIds.filter((id) => !(id in byScenario));
|
|
5363
|
-
if (missing.length > 0) throw new Error(`compareOptimizationMethods: ${label} produced no test score for scenario(s) [${missing.join(", ")}]. A cell failed or its judges returned nothing. Fix the dispatch or judge; the comparison will not replace missing scores with zero.`);
|
|
5364
|
-
return scenarioIds.map((id) => byScenario[id]);
|
|
5365
|
-
};
|
|
5366
|
-
if (opts.claim?.generalization === "new-units") assertIndependentEvaluationSplit(opts.claim, opts.testScenarios, [...opts.trainScenarios, ...opts.selectionScenarios]);
|
|
5367
|
-
if (opts.finalEvidence) await reserveFinalEvidence(opts.finalEvidence, opts.claim, opts.testScenarios, [...opts.trainScenarios, ...opts.selectionScenarios]);
|
|
5368
|
-
const optimizationOwner = new AbortController();
|
|
5369
|
-
const optimized = await mapConcurrent(opts.methods, optimizationConcurrency, async (method) => {
|
|
5370
|
-
try {
|
|
5371
|
-
const { selected: out, cost, history: searchHistoryCoverage } = await executeOptimizationMethod({
|
|
5372
|
-
method,
|
|
5373
|
-
input: createOptimizationMethodInput(opts, method.name, resolvedRunDir, seed, baselineSurface, costLedger, optimizationOwner.signal),
|
|
5374
|
-
storage,
|
|
5375
|
-
searchHistoryPolicy: opts.searchHistoryPolicy,
|
|
5376
|
-
searchHistoryVerification: opts.searchHistoryVerification
|
|
5377
|
-
});
|
|
5378
|
-
const winnerSurface = out.winnerSurface;
|
|
5379
|
-
return {
|
|
5380
|
-
name: method.name,
|
|
5381
|
-
winnerSurface,
|
|
5382
|
-
cost,
|
|
5383
|
-
...out.durationMs === void 0 ? {} : { durationMs: out.durationMs },
|
|
5384
|
-
...out.composition === void 0 ? {} : { composition: out.composition },
|
|
5385
|
-
...out.provenance === void 0 ? {} : { provenance: out.provenance },
|
|
5386
|
-
searchHistoryCoverage
|
|
5387
|
-
};
|
|
5388
|
-
} catch (error) {
|
|
5389
|
-
if (!optimizationOwner.signal.aborted) optimizationOwner.abort(error);
|
|
5390
|
-
throw error;
|
|
5391
|
-
}
|
|
5392
|
-
});
|
|
5393
|
-
assertReportedCostWithinCeiling(combineComparisonCosts(optimized.map((result) => ({
|
|
5394
|
-
label: `method '${result.name}'`,
|
|
5395
|
-
cost: result.cost
|
|
5396
|
-
}))).totalCostUsd, opts.costCeiling, "optimization");
|
|
5397
|
-
const testCostPhase = finalCostPhase(opts, baselineSurface, optimized, seed);
|
|
5398
|
-
const finalEvidence = opts.finalEvidence ? await exposeFinalEvidence(opts.finalEvidence, opts.claim, opts.testScenarios, [baselineSurface, ...optimized.map((method) => method.winnerSurface)]) : void 0;
|
|
5399
|
-
const baselineArr = align(await scoreOnTest(baselineSurface, "test/baseline", testCostPhase), "baseline");
|
|
5400
|
-
const testScoresBySurface = /* @__PURE__ */ new Map([[surfaceContentHash(baselineSurface), baselineArr]]);
|
|
5401
|
-
const winners = [];
|
|
5402
|
-
for (const winner of optimized) {
|
|
5403
|
-
const surfaceKey = surfaceContentHash(winner.winnerSurface);
|
|
5404
|
-
let arr = testScoresBySurface.get(surfaceKey);
|
|
5405
|
-
if (!arr) {
|
|
5406
|
-
arr = align(await scoreOnTest(winner.winnerSurface, `test/methods/${slug(winner.name)}`, testCostPhase), `method "${winner.name}"`);
|
|
5407
|
-
testScoresBySurface.set(surfaceKey, arr);
|
|
5408
|
-
}
|
|
5409
|
-
winners.push({
|
|
5410
|
-
...winner,
|
|
5411
|
-
arr
|
|
5412
|
-
});
|
|
5413
|
-
}
|
|
5414
|
-
const scores = winners.map((w) => {
|
|
5415
|
-
const baselineUnits = aggregateUnits(baselineArr);
|
|
5416
|
-
const winnerUnits = aggregateUnits(w.arr);
|
|
5417
|
-
const decision = decidePairedPromotion(baselineUnits, winnerUnits, decisionOptions);
|
|
5418
|
-
const score = {
|
|
5419
|
-
name: w.name,
|
|
5420
|
-
baselineComposite: mean(baselineUnits),
|
|
5421
|
-
winnerComposite: mean(winnerUnits),
|
|
5422
|
-
lift: decision.delta,
|
|
5423
|
-
liftCi: {
|
|
5424
|
-
low: decision.low,
|
|
5425
|
-
high: decision.high
|
|
5426
|
-
},
|
|
5427
|
-
decision,
|
|
5428
|
-
optimizationCost: w.cost,
|
|
5429
|
-
scenarioScores: scenarioIds.map((scenarioId, index) => ({
|
|
5430
|
-
scenarioId,
|
|
5431
|
-
baselineComposite: baselineArr[index],
|
|
5432
|
-
winnerComposite: w.arr[index],
|
|
5433
|
-
lift: w.arr[index] - baselineArr[index]
|
|
5434
|
-
})),
|
|
5435
|
-
unitScores: units.units.map((unit, index) => ({
|
|
5436
|
-
unitId: unit.id,
|
|
5437
|
-
scenarios: unit.observations,
|
|
5438
|
-
baselineComposite: baselineUnits[index],
|
|
5439
|
-
winnerComposite: winnerUnits[index],
|
|
5440
|
-
lift: winnerUnits[index] - baselineUnits[index]
|
|
5441
|
-
})),
|
|
5442
|
-
winnerSurface: structuredClone(w.winnerSurface),
|
|
5443
|
-
rank: 0
|
|
5444
|
-
};
|
|
5445
|
-
if (evidence) {
|
|
5446
|
-
const baseline = evidenceBySurface.get(surfaceContentHash(baselineSurface));
|
|
5447
|
-
const winner = evidenceBySurface.get(surfaceContentHash(w.winnerSurface));
|
|
5448
|
-
if (!baseline || !winner) throw new Error("final measurement evidence is missing");
|
|
5449
|
-
score.evidence = {
|
|
5450
|
-
baseline,
|
|
5451
|
-
winner
|
|
5452
|
-
};
|
|
5453
|
-
}
|
|
5454
|
-
if (w.composition !== void 0) score.composition = structuredClone(w.composition);
|
|
5455
|
-
if (w.durationMs !== void 0) score.durationMs = w.durationMs;
|
|
5456
|
-
if (w.provenance !== void 0) score.provenance = structuredClone(w.provenance);
|
|
5457
|
-
return score;
|
|
5458
|
-
});
|
|
5459
|
-
scores.sort((a, b) => b.lift - a.lift);
|
|
5460
|
-
for (let start = 0; start < scores.length;) {
|
|
5461
|
-
let end = start + 1;
|
|
5462
|
-
while (end < scores.length && scores[end].lift === scores[start].lift) end += 1;
|
|
5463
|
-
const tied = scores.slice(start, end);
|
|
5464
|
-
if (tied.every((score) => score.optimizationCost.accountingComplete)) {
|
|
5465
|
-
tied.sort((a, b) => a.optimizationCost.totalCostUsd - b.optimizationCost.totalCostUsd);
|
|
5466
|
-
scores.splice(start, tied.length, ...tied);
|
|
5467
|
-
}
|
|
5468
|
-
start = end;
|
|
5469
|
-
}
|
|
5470
|
-
scores.forEach((s, i) => {
|
|
5471
|
-
s.rank = i + 1;
|
|
5472
|
-
});
|
|
5473
|
-
const best = scores[0];
|
|
5474
|
-
const byName = new Map(winners.map((w) => [w.name, w]));
|
|
5475
|
-
const bestArr = aggregateUnits(byName.get(best.name).arr);
|
|
5476
|
-
const pairwise = scores.slice(1).map((other) => {
|
|
5477
|
-
const otherArr = aggregateUnits(byName.get(other.name).arr);
|
|
5478
|
-
const decision = decidePairedPromotion(otherArr, bestArr, decisionOptions);
|
|
5479
|
-
const reverse = decidePairedPromotion(bestArr, otherArr, decisionOptions);
|
|
5480
|
-
const favored = decision.promote ? best.name : reverse.promote ? other.name : null;
|
|
5481
|
-
return {
|
|
5482
|
-
a: best.name,
|
|
5483
|
-
b: other.name,
|
|
5484
|
-
deltaMean: decision.delta,
|
|
5485
|
-
low: decision.low,
|
|
5486
|
-
high: decision.high,
|
|
5487
|
-
favored,
|
|
5488
|
-
decision
|
|
5489
|
-
};
|
|
5490
|
-
});
|
|
5491
|
-
const optimizationCost = combineComparisonCosts(scores.map((score) => ({
|
|
5492
|
-
label: `method '${score.name}'`,
|
|
5493
|
-
cost: score.optimizationCost
|
|
5494
|
-
})));
|
|
5495
|
-
const testCost = costFromLedgerSummary(costLedger.summary({ phase: testCostPhase }));
|
|
5496
|
-
const totalCost = combineComparisonCosts([{
|
|
5497
|
-
label: "optimization",
|
|
5498
|
-
cost: optimizationCost
|
|
5499
|
-
}, {
|
|
5500
|
-
label: "final test",
|
|
5501
|
-
cost: testCost
|
|
5502
|
-
}]);
|
|
5503
|
-
assertReportedCostWithinCeiling(totalCost.totalCostUsd, opts.costCeiling, "total");
|
|
5504
|
-
return {
|
|
5505
|
-
scores,
|
|
5506
|
-
best,
|
|
5507
|
-
pairwise,
|
|
5508
|
-
testScenarioIds: scenarioIds,
|
|
5509
|
-
units,
|
|
5510
|
-
observationUnit: opts.claim ? "registered" : "scenario",
|
|
5511
|
-
pairedCellN: scenarioIds.length * (opts.reps ?? 1),
|
|
5512
|
-
...opts.claim ? { claim: opts.claim } : {},
|
|
5513
|
-
...finalEvidence ? { finalEvidence } : {},
|
|
5514
|
-
optimizationCost,
|
|
5515
|
-
testCost,
|
|
5516
|
-
totalCost,
|
|
5517
|
-
confidence,
|
|
5518
|
-
intervalConfidence,
|
|
5519
|
-
comparisonCount,
|
|
5520
|
-
seed,
|
|
5521
|
-
resamples,
|
|
5522
|
-
reps: opts.reps ?? 1,
|
|
5523
|
-
searchHistory: Object.freeze({
|
|
5524
|
-
policy: searchHistoryPolicy,
|
|
5525
|
-
allComplete: optimized.every((result) => result.searchHistoryCoverage.status === "complete"),
|
|
5526
|
-
producers: Object.freeze(optimized.map((result) => result.searchHistoryCoverage))
|
|
5527
|
-
})
|
|
5528
|
-
};
|
|
5529
|
-
}
|
|
5530
|
-
function assertReportedCostWithinCeiling(totalCostUsd, costCeiling, phase) {
|
|
5531
|
-
const tolerance = Number.EPSILON * Math.max(1, Math.abs(totalCostUsd), Math.abs(costCeiling ?? 0)) * 8;
|
|
5532
|
-
if (costCeiling !== void 0 && totalCostUsd > costCeiling + tolerance) throw new Error(`compareOptimizationMethods: reported ${phase} cost ${totalCostUsd} exceeds costCeiling ${costCeiling}`);
|
|
5533
|
-
}
|
|
5534
|
-
function assertOptimizationMethods(methods) {
|
|
5535
|
-
if (!Array.isArray(methods) || methods.length === 0) throw new Error("compareOptimizationMethods: no methods to compare");
|
|
5536
|
-
const names = /* @__PURE__ */ new Set();
|
|
5537
|
-
const pathOwners = /* @__PURE__ */ new Map();
|
|
5538
|
-
for (const method of methods) {
|
|
5539
|
-
if (!method || typeof method !== "object" || typeof method.optimize !== "function") throw new Error("compareOptimizationMethods: every method must provide optimize(input)");
|
|
5540
|
-
if (!method.name || method.name.trim() !== method.name) throw new Error("compareOptimizationMethods: method names must be trimmed and non-empty");
|
|
5541
|
-
if (names.has(method.name)) throw new Error(`compareOptimizationMethods: duplicate method name '${method.name}'`);
|
|
5542
|
-
names.add(method.name);
|
|
5543
|
-
const pathKey = slug(method.name);
|
|
5544
|
-
const prior = pathOwners.get(pathKey);
|
|
5545
|
-
if (prior) throw new Error(`compareOptimizationMethods: method names '${prior}' and '${method.name}' map to the same run path '${pathKey}'`);
|
|
5546
|
-
pathOwners.set(pathKey, method.name);
|
|
5547
|
-
}
|
|
5548
|
-
}
|
|
5549
|
-
function assertComparisonControls(opts, seed, resamples, confidence) {
|
|
5550
|
-
if (opts.optimizationRunOptions && "costCeiling" in opts.optimizationRunOptions) throw new Error("compareOptimizationMethods: optimizationRunOptions.costCeiling is not supported; costCeiling covers optimization and final scoring");
|
|
5551
|
-
if (!opts.judges || opts.judges.length === 0) throw new Error("compareOptimizationMethods: at least one judge is required");
|
|
5552
|
-
if (typeof opts.dispatchWithSurface !== "function") throw new Error("compareOptimizationMethods: dispatchWithSurface must be a function");
|
|
5553
|
-
if (opts.dispatchRef !== void 0 && (typeof opts.dispatchRef !== "string" || opts.dispatchRef.trim().length === 0 || opts.dispatchRef.trim() !== opts.dispatchRef)) throw new Error("compareOptimizationMethods: dispatchRef must be trimmed and non-empty when provided");
|
|
5554
|
-
const optimizationDispatchRef = opts.optimizationRunOptions?.dispatchRef;
|
|
5555
|
-
if (optimizationDispatchRef !== void 0 && (typeof optimizationDispatchRef !== "string" || optimizationDispatchRef.trim().length === 0 || optimizationDispatchRef.trim() !== optimizationDispatchRef)) throw new Error("compareOptimizationMethods: optimizationRunOptions.dispatchRef must be trimmed and non-empty when provided");
|
|
5556
|
-
if (opts.dispatchRef !== void 0 && optimizationDispatchRef !== void 0 && opts.dispatchRef !== optimizationDispatchRef) throw new Error("compareOptimizationMethods: dispatchRef must match optimizationRunOptions.dispatchRef when both are provided");
|
|
5557
|
-
try {
|
|
5558
|
-
surfaceContentHash(opts.baselineSurface);
|
|
5559
|
-
} catch (cause) {
|
|
5560
|
-
throw new Error("compareOptimizationMethods: baselineSurface is invalid", { cause });
|
|
5561
|
-
}
|
|
5562
|
-
const judgeNames = /* @__PURE__ */ new Set();
|
|
5563
|
-
for (const judge of opts.judges) {
|
|
5564
|
-
if (!judge || typeof judge !== "object" || typeof judge.name !== "string" || judge.name.trim().length === 0 || judge.name.trim() !== judge.name || typeof judge.score !== "function" || !Array.isArray(judge.dimensions) || judge.dimensions.length === 0) throw new Error("compareOptimizationMethods: every judge needs a trimmed name, at least one dimension, and score(input)");
|
|
5565
|
-
if (judgeNames.has(judge.name)) throw new Error(`compareOptimizationMethods: duplicate judge name '${judge.name}'`);
|
|
5566
|
-
judgeNames.add(judge.name);
|
|
5567
|
-
const dimensionKeys = /* @__PURE__ */ new Set();
|
|
5568
|
-
for (const dimension of judge.dimensions) {
|
|
5569
|
-
if (!dimension || typeof dimension.key !== "string" || dimension.key.trim().length === 0 || dimension.key.trim() !== dimension.key || typeof dimension.description !== "string" || dimension.description.trim().length === 0) throw new Error(`compareOptimizationMethods: judge '${judge.name}' has an invalid dimension`);
|
|
5570
|
-
if (dimensionKeys.has(dimension.key)) throw new Error(`compareOptimizationMethods: judge '${judge.name}' has duplicate dimension '${dimension.key}'`);
|
|
5571
|
-
dimensionKeys.add(dimension.key);
|
|
5572
|
-
}
|
|
5573
|
-
}
|
|
5574
|
-
if (typeof opts.runDir !== "string" || opts.runDir.trim().length === 0) throw new Error("compareOptimizationMethods: runDir must be a non-empty string");
|
|
5575
|
-
if (!Number.isSafeInteger(seed)) throw new Error(`compareOptimizationMethods: seed must be a safe integer, got ${String(seed)}`);
|
|
5576
|
-
if (!Number.isSafeInteger(resamples) || resamples <= 0 || resamples > 1e6) throw new Error(`compareOptimizationMethods: resamples must be a positive safe integer no greater than 1000000, got ${String(resamples)}`);
|
|
5577
|
-
if (!Number.isFinite(confidence) || confidence <= 0 || confidence >= 1) throw new Error(`compareOptimizationMethods: confidence must be a finite number in (0,1), got ${String(confidence)}`);
|
|
5578
|
-
const minimumResamples = minimumBootstrapResamples(confidence, opts.methods.length * (opts.methods.length + 1) / 2);
|
|
5579
|
-
if (resamples < minimumResamples) throw new Error(`compareOptimizationMethods: resamples must be at least ${minimumResamples} for simultaneous confidence ${confidence} across ${opts.methods.length} methods, got ${resamples}`);
|
|
5580
|
-
if (opts.optimizationConcurrency !== void 0 && (!Number.isSafeInteger(opts.optimizationConcurrency) || opts.optimizationConcurrency <= 0)) throw new Error("compareOptimizationMethods: optimizationConcurrency must be a positive safe integer");
|
|
5581
|
-
if (opts.maxConcurrency !== void 0 && (!Number.isSafeInteger(opts.maxConcurrency) || opts.maxConcurrency <= 0)) throw new Error("compareOptimizationMethods: maxConcurrency must be a positive safe integer");
|
|
5582
|
-
if (opts.dispatchTimeoutMs !== void 0 && (!Number.isSafeInteger(opts.dispatchTimeoutMs) || opts.dispatchTimeoutMs < 0 || opts.dispatchTimeoutMs > 2147483647)) throw new Error("compareOptimizationMethods: dispatchTimeoutMs must be a non-negative safe integer no greater than 2147483647");
|
|
5583
|
-
if (opts.costCeiling !== void 0 && (!Number.isFinite(opts.costCeiling) || opts.costCeiling < 0)) throw new Error("compareOptimizationMethods: costCeiling must be a finite number greater than or equal to 0");
|
|
5584
|
-
if (opts.costCeiling !== void 0 && opts.costLedger !== void 0 && opts.costLedger.costCeilingUsd !== opts.costCeiling) throw new Error("compareOptimizationMethods: costCeiling must match the shared CostLedger ceiling");
|
|
5585
|
-
}
|
|
5586
|
-
function assertComparisonPartitions(opts) {
|
|
5587
|
-
const partitions = [
|
|
5588
|
-
{
|
|
5589
|
-
name: "trainScenarios",
|
|
5590
|
-
scenarios: opts.trainScenarios
|
|
5591
|
-
},
|
|
5592
|
-
{
|
|
5593
|
-
name: "selectionScenarios",
|
|
5594
|
-
scenarios: opts.selectionScenarios
|
|
5595
|
-
},
|
|
5596
|
-
{
|
|
5597
|
-
name: "testScenarios",
|
|
5598
|
-
scenarios: opts.testScenarios
|
|
5599
|
-
}
|
|
5600
|
-
];
|
|
5601
|
-
const owner = /* @__PURE__ */ new Map();
|
|
5602
|
-
for (const partition of partitions) {
|
|
5603
|
-
if (!Array.isArray(partition.scenarios) || partition.scenarios.length === 0) throw new Error(`compareOptimizationMethods: ${partition.name} is empty`);
|
|
5604
|
-
if (partition.name === "testScenarios" && partition.scenarios.length < 2) throw new Error("compareOptimizationMethods: testScenarios requires at least 2 scenarios to estimate uncertainty");
|
|
5605
|
-
const local = /* @__PURE__ */ new Set();
|
|
5606
|
-
const duplicates = /* @__PURE__ */ new Set();
|
|
5607
|
-
const overlaps = /* @__PURE__ */ new Map();
|
|
5608
|
-
for (const scenario of partition.scenarios) {
|
|
5609
|
-
if (local.has(scenario.id)) duplicates.add(scenario.id);
|
|
5610
|
-
local.add(scenario.id);
|
|
5611
|
-
const prior = owner.get(scenario.id);
|
|
5612
|
-
if (prior !== void 0 && prior !== partition.name) overlaps.set(scenario.id, prior);
|
|
5613
|
-
}
|
|
5614
|
-
if (duplicates.size > 0) throw new Error(`compareOptimizationMethods: ${partition.name} contains duplicate scenario id(s) [${[...duplicates].join(", ")}]`);
|
|
5615
|
-
if (overlaps.size > 0) {
|
|
5616
|
-
const detail = [...overlaps].map(([id, prior]) => `${id} (${prior} ∩ ${partition.name})`).join(", ");
|
|
5617
|
-
throw new Error(`compareOptimizationMethods: trainScenarios, selectionScenarios, and testScenarios must be pairwise disjoint; overlap: [${detail}]`);
|
|
5618
|
-
}
|
|
5619
|
-
assertCampaignDesign(partition.scenarios, opts.reps ?? 1);
|
|
5620
|
-
for (const id of local) owner.set(id, partition.name);
|
|
5621
|
-
}
|
|
5622
|
-
}
|
|
5623
|
-
function mean(xs) {
|
|
5624
|
-
return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length;
|
|
5625
|
-
}
|
|
5626
|
-
function slug(name) {
|
|
5627
|
-
return name.replace(/[^a-z0-9]+/gi, "-").replace(/^-|-$/g, "").toLowerCase() || "method";
|
|
5628
|
-
}
|
|
5629
|
-
function assertConfidence(confidence) {
|
|
5630
|
-
if (!Number.isFinite(confidence) || confidence <= 0 || confidence >= 1) throw new Error(`compareOptimizationMethods: confidence must be a finite number in (0,1), got ${String(confidence)}`);
|
|
5631
|
-
}
|
|
5632
|
-
function minimumBootstrapResamples(confidence, comparisonCount) {
|
|
5633
|
-
const exact = 2 * comparisonCount / (1 - confidence);
|
|
5634
|
-
return Math.ceil(exact - Number.EPSILON * Math.max(1, exact) * 32);
|
|
5635
|
-
}
|
|
5636
|
-
function createOptimizationMethodInput(opts, methodName, resolvedRunDir, seed, baselineSurface, costLedger, optimizationSignal) {
|
|
5637
|
-
const methodRunDir = `${resolvedRunDir}/optimization/${slug(methodName)}`;
|
|
5638
|
-
const signal = combineAbortSignals(opts.signal, opts.optimizationRunOptions?.signal, optimizationSignal);
|
|
5639
|
-
return Object.freeze({
|
|
5640
|
-
baselineSurface: structuredClone(baselineSurface),
|
|
5641
|
-
trainScenarios: opts.trainScenarios,
|
|
5642
|
-
selectionScenarios: opts.selectionScenarios,
|
|
5643
|
-
dispatchWithSurface: opts.dispatchWithSurface,
|
|
5644
|
-
judges: opts.judges,
|
|
5645
|
-
runDir: methodRunDir,
|
|
5646
|
-
seed,
|
|
5647
|
-
runOptions: Object.freeze({
|
|
5648
|
-
...opts.optimizationRunOptions ?? {},
|
|
5649
|
-
...opts.optimizationRunOptions?.dispatchRef === void 0 && opts.dispatchRef !== void 0 ? { dispatchRef: opts.dispatchRef } : {},
|
|
5650
|
-
...signal ? { signal } : {}
|
|
5651
|
-
}),
|
|
5652
|
-
costLedger
|
|
5653
|
-
});
|
|
5654
|
-
}
|
|
5655
|
-
function finalCostPhase(opts, baselineSurface, winners, seed) {
|
|
5656
|
-
return `compareOptimizationMethods.test:${contentHash({
|
|
5657
|
-
baseline: surfaceContentHash(baselineSurface),
|
|
5658
|
-
winners: winners.map((winner) => ({
|
|
5659
|
-
name: winner.name,
|
|
5660
|
-
surface: surfaceContentHash(winner.winnerSurface)
|
|
5661
|
-
})),
|
|
5662
|
-
testScenarios: opts.testScenarios,
|
|
5663
|
-
judges: opts.judges.map((judge) => ({
|
|
5664
|
-
name: judge.name,
|
|
5665
|
-
dimensions: judge.dimensions,
|
|
5666
|
-
version: judge.judgeVersion ?? contentHash({
|
|
5667
|
-
score: judge.score.toString(),
|
|
5668
|
-
appliesTo: judge.appliesTo?.toString() ?? null
|
|
5669
|
-
})
|
|
5670
|
-
})),
|
|
5671
|
-
dispatch: callerDispatchRef(opts),
|
|
5672
|
-
seed,
|
|
5673
|
-
reps: opts.reps ?? 1
|
|
5674
|
-
})}`;
|
|
5675
|
-
}
|
|
5676
|
-
function finalDispatchRef(opts, surface) {
|
|
5677
|
-
return `compareOptimizationMethods:${callerDispatchRef(opts)}:${surfaceContentHash(surface)}`;
|
|
5678
|
-
}
|
|
5679
|
-
function callerDispatchRef(opts) {
|
|
5680
|
-
return (opts.dispatchRef ?? opts.dispatchWithSurface.name) || "anonymous";
|
|
5681
|
-
}
|
|
5682
|
-
/** Preserve every optimizer token class while keeping total input and output explicit. */
|
|
5683
|
-
function optimizationTokenUsageFromSummary(summary, receipts) {
|
|
5684
|
-
if (!summary.usageComplete) return void 0;
|
|
5685
|
-
if (receipts.length !== summary.totalCalls) throw new Error("optimization token usage receipt count does not match the cost summary");
|
|
5686
|
-
const cachedInputTokens = summary.cachedTokens;
|
|
5687
|
-
const cacheWriteInputTokens = summary.cacheWriteTokens ?? 0;
|
|
5688
|
-
const inputTokens = summary.inputTokens + cachedInputTokens + cacheWriteInputTokens;
|
|
5689
|
-
const cacheReadComplete = receipts.length === 0 || receipts.every((receipt) => receipt.cachedTokens !== void 0);
|
|
5690
|
-
const cacheWriteComplete = receipts.length === 0 || receipts.every((receipt) => receipt.cacheWriteTokens !== void 0);
|
|
5691
|
-
const reasoningComplete = receipts.length === 0 || receipts.every((receipt) => receipt.reasoningTokens !== void 0);
|
|
5692
|
-
return {
|
|
5693
|
-
inputTokens,
|
|
5694
|
-
...cacheReadComplete ? { cachedInputTokens } : {},
|
|
5695
|
-
...cacheWriteComplete ? { cacheWriteInputTokens } : {},
|
|
5696
|
-
outputTokens: summary.outputTokens,
|
|
5697
|
-
...reasoningComplete ? { reasoningTokens: summary.reasoningTokens ?? 0 } : {},
|
|
5698
|
-
totalTokens: inputTokens + summary.outputTokens,
|
|
5699
|
-
calls: summary.totalCalls
|
|
5700
|
-
};
|
|
5701
|
-
}
|
|
5702
|
-
//#endregion
|
|
5703
5281
|
//#region src/run-score.ts
|
|
5704
5282
|
const DEFAULT_RUN_SCORE_WEIGHTS = {
|
|
5705
5283
|
success: 4,
|
|
@@ -6172,6 +5750,6 @@ function firstString(value) {
|
|
|
6172
5750
|
return typeof value === "string" && value.trim() ? value : void 0;
|
|
6173
5751
|
}
|
|
6174
5752
|
//#endregion
|
|
6175
|
-
export {
|
|
5753
|
+
export { evaluationUnitMap as $, assertSearchHistoryAdmissionOptions as A, asha as B, SearchRecorder as C, surfaceNode as D, surfaceDiff as E, verifySearchHistoryReceipt as F, defaultProductionGate as G, runFinalComparison as H, campaignCellSearchResult as I, redTeamReport as J, DEFAULT_RED_TEAM_CORPUS as K, searchModelIdentity as L, createSearchHistoryReceipt as M, searchHistoryCoverageRow as N, SearchHistoryRequiredError as O, verifySearchHistoryArtifact as P, captureFinalEvidencePolicy as Q, searchProposalExecution as R, incumbentWithOperatorBandit as S, searchEdgeId as T, openAutoPr as U, uniform as V, captureJudge as W, runCanaries as X, scoreRedTeamOutput as Y, assertIndependentEvaluationSplit as Z, labelTrustRank as _, transientDispatchFailure as a, resolveRunDir as at, crowdedFrontierParent as b, executeOptimizationMethod as c, buildCellSchedule as ct, combineComparisonCosts as d, exposeFinalEvidence as et, costFromLedgerSummary as f, isProposedCandidate as g, runOptimization as h, quotaExhaustedUntil as i, planCampaignRun as it, assertSearchHistoryMatchesState as j, assertCompleteSearchHistory as k, assertGepaCandidatePopulationSummary as l, cellCachePath as lt, runImprovementLoop as m, JudgeParseError as n, runEval as nt, aggregateRunScore as o, tangleTracesRoot as ot, optimizationTokenUsageFromSummary as p, redTeamDataset as q, isTransientTransportFailure as r, runCampaign as rt, clamp01 as s, readCachedCell as st, llmJudge as t, reserveFinalEvidence as tt, readGepaCandidatePopulationArtifact as u, computeManifestHash as ut, aide as v, developmentClaim as w, incumbent as x, beam as y, searchReceiptAccounting as z };
|
|
6176
5754
|
|
|
6177
|
-
//# sourceMappingURL=llm-judge-
|
|
5755
|
+
//# sourceMappingURL=llm-judge-CG12SQJq.js.map
|