@tangle-network/agent-eval 0.216.0 → 0.216.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (384) hide show
  1. package/CHANGELOG.md +39 -0
  2. package/dist/adapters/http.d.ts +3 -3
  3. package/dist/{agent-profile-CIA2Atnt.d.ts → agent-profile-VFcQncbW.d.ts} +5 -5
  4. package/dist/{agent-profile-CIA2Atnt.d.ts.map → agent-profile-VFcQncbW.d.ts.map} +1 -1
  5. package/dist/{agent-profile-cell-BYc0bxsL.js → agent-profile-cell-BK8mlqlq.js} +3 -3
  6. package/dist/{agent-profile-cell-BYc0bxsL.js.map → agent-profile-cell-BK8mlqlq.js.map} +1 -1
  7. package/dist/{agreement-irr-DsTURxdr.js → agreement-irr-BEWLIF3_.js} +3 -3
  8. package/dist/{agreement-irr-DsTURxdr.js.map → agreement-irr-BEWLIF3_.js.map} +1 -1
  9. package/dist/analyst/index.d.ts +14 -14
  10. package/dist/analyst/index.d.ts.map +1 -1
  11. package/dist/analyst/index.js +15 -15
  12. package/dist/{analyze-runs-3xpby3p2.js → analyze-runs-C84irm4i.js} +608 -604
  13. package/dist/analyze-runs-C84irm4i.js.map +1 -0
  14. package/dist/{analyze-runs-CSDAh8A7.d.ts → analyze-runs-D68fd-su.d.ts} +17 -15
  15. package/dist/analyze-runs-D68fd-su.d.ts.map +1 -0
  16. package/dist/{baseline-Dm8966du.js → baseline-Udi_PLuT.js} +5 -5
  17. package/dist/{baseline-Dm8966du.js.map → baseline-Udi_PLuT.js.map} +1 -1
  18. package/dist/{benchmark-BywPeAUo.js → benchmark-CXmVVHH2.js} +2 -2
  19. package/dist/{benchmark-BywPeAUo.js.map → benchmark-CXmVVHH2.js.map} +1 -1
  20. package/dist/{benchmark-command-CByl9dxG.js → benchmark-command-By0vh8q7.js} +62 -45
  21. package/dist/benchmark-command-By0vh8q7.js.map +1 -0
  22. package/dist/benchmarks/index.d.ts +5 -5
  23. package/dist/benchmarks/index.js +3 -3
  24. package/dist/builder-eval/index.js +2 -2
  25. package/dist/campaign/index.d.ts +7 -7
  26. package/dist/campaign/index.js +13 -13
  27. package/dist/{campaign-BQ-Jujuj.js → campaign-BudZ70zo.js} +245 -247
  28. package/dist/campaign-BudZ70zo.js.map +1 -0
  29. package/dist/{campaign-evidence-OcUciD-N.js → campaign-evidence-Clrn4hBV.js} +8 -8
  30. package/dist/campaign-evidence-Clrn4hBV.js.map +1 -0
  31. package/dist/{canonical-BsZVgBhW.js → canonical-B2RqLofv.js} +2 -2
  32. package/dist/{canonical-BsZVgBhW.js.map → canonical-B2RqLofv.js.map} +1 -1
  33. package/dist/{chat-client-DthPOYGd.js → chat-client-l-MVDlDk.js} +2 -2
  34. package/dist/{chat-client-DthPOYGd.js.map → chat-client-l-MVDlDk.js.map} +1 -1
  35. package/dist/{chat-json-call-BrKsyspg.js → chat-json-call-puMArmtW.js} +2 -2
  36. package/dist/{chat-json-call-BrKsyspg.js.map → chat-json-call-puMArmtW.js.map} +1 -1
  37. package/dist/{claim-BXcZw-jo.js → claim-q4HQhK1G.js} +4 -4
  38. package/dist/claim-q4HQhK1G.js.map +1 -0
  39. package/dist/cli.js +9 -9
  40. package/dist/cli.js.map +1 -1
  41. package/dist/{cluster-bootstrap-B3JywQQn.js → cluster-bootstrap-qcUWYFl-.js} +3 -3
  42. package/dist/{cluster-bootstrap-B3JywQQn.js.map → cluster-bootstrap-qcUWYFl-.js.map} +1 -1
  43. package/dist/contract/index.d.ts +11 -11
  44. package/dist/contract/index.js +1022 -1078
  45. package/dist/contract/index.js.map +1 -1
  46. package/dist/{cost-ledger-D2FnJPBF.js → cost-ledger-BP8V4kUQ.js} +835 -816
  47. package/dist/cost-ledger-BP8V4kUQ.js.map +1 -0
  48. package/dist/{cost-ledger-CO7UyEcQ.d.ts → cost-ledger-Jk62OBfF.d.ts} +12 -47
  49. package/dist/cost-ledger-Jk62OBfF.d.ts.map +1 -0
  50. package/dist/{counterfactual-DtMWvn7D.js → counterfactual-Dn3veueY.js} +2 -2
  51. package/dist/{counterfactual-DtMWvn7D.js.map → counterfactual-Dn3veueY.js.map} +1 -1
  52. package/dist/{default-registry-Cn1LQ2Ve.js → default-registry-COhk9g36.js} +5 -5
  53. package/dist/{default-registry-Cn1LQ2Ve.js.map → default-registry-COhk9g36.js.map} +1 -1
  54. package/dist/{default-registry-BrWh62zf.d.ts → default-registry-DzHiGruW.d.ts} +6 -6
  55. package/dist/{default-registry-BrWh62zf.d.ts.map → default-registry-DzHiGruW.d.ts.map} +1 -1
  56. package/dist/{define-agent-eval-Ch3MWX6k.js → define-agent-eval-CNTPzIkp.js} +10 -10
  57. package/dist/define-agent-eval-CNTPzIkp.js.map +1 -0
  58. package/dist/{define-agent-eval-CEWkX5wZ.d.ts → define-agent-eval-WSck0CZJ.d.ts} +8 -8
  59. package/dist/{define-agent-eval-CEWkX5wZ.d.ts.map → define-agent-eval-WSck0CZJ.d.ts.map} +1 -1
  60. package/dist/{descriptive-CnU8Yq29.js → descriptive-DboxPZrG.js} +2 -2
  61. package/dist/{descriptive-CnU8Yq29.js.map → descriptive-DboxPZrG.js.map} +1 -1
  62. package/dist/diagnosis/index.d.ts +3 -3
  63. package/dist/diagnosis/index.js +4 -4
  64. package/dist/{dspy-rlm-engine-D-KX_e64.js → dspy-rlm-engine-LRCu61m5.js} +3 -3
  65. package/dist/{dspy-rlm-engine-D-KX_e64.js.map → dspy-rlm-engine-LRCu61m5.js.map} +1 -1
  66. package/dist/{effect-sizes-DiH8MGOH.js → effect-sizes-BhzEpXMc.js} +2 -2
  67. package/dist/{effect-sizes-DiH8MGOH.js.map → effect-sizes-BhzEpXMc.js.map} +1 -1
  68. package/dist/{engine-C_zNWfM8.d.ts → engine-3A_4jOlI.d.ts} +5 -5
  69. package/dist/{engine-C_zNWfM8.d.ts.map → engine-3A_4jOlI.d.ts.map} +1 -1
  70. package/dist/{equal-terms-Blx9Zk5I.js → equal-terms-BtdXayCJ.js} +2 -2
  71. package/dist/{equal-terms-Blx9Zk5I.js.map → equal-terms-BtdXayCJ.js.map} +1 -1
  72. package/dist/{errors-Dngq5h35.js → errors-DVZiqS7K.js} +6 -2
  73. package/dist/{errors-Dngq5h35.js.map → errors-DVZiqS7K.js.map} +1 -1
  74. package/dist/{eval-campaign-4byTtqrf.js → eval-campaign-90jeaCAI.js} +6 -6
  75. package/dist/{eval-campaign-4byTtqrf.js.map → eval-campaign-90jeaCAI.js.map} +1 -1
  76. package/dist/{evaluation-Cn2dRZIz.d.ts → evaluation-6Yoi74gM.d.ts} +4 -4
  77. package/dist/{evaluation-Cn2dRZIz.d.ts.map → evaluation-6Yoi74gM.d.ts.map} +1 -1
  78. package/dist/evaluation.d.ts +1 -1
  79. package/dist/evaluation.js +1 -1
  80. package/dist/{exact-types-CD0lf5Qd.d.ts → exact-types-ClugMJfI.d.ts} +2 -2
  81. package/dist/{exact-types-CD0lf5Qd.d.ts.map → exact-types-ClugMJfI.d.ts.map} +1 -1
  82. package/dist/experiment/index.d.ts +404 -3
  83. package/dist/experiment/index.d.ts.map +1 -1
  84. package/dist/experiment/index.js +313 -19
  85. package/dist/experiment/index.js.map +1 -1
  86. package/dist/{external-optimizer-process-nnsY5D_4.js → external-optimizer-process-BjMGUuds.js} +3 -3
  87. package/dist/{external-optimizer-process-nnsY5D_4.js.map → external-optimizer-process-BjMGUuds.js.map} +1 -1
  88. package/dist/{external-optimizer-subprocess-DcerlZSG.js → external-optimizer-subprocess-BqW0hHcI.js} +548 -548
  89. package/dist/external-optimizer-subprocess-BqW0hHcI.js.map +1 -0
  90. package/dist/{feedback-trajectory-DkaDj0dj.d.ts → feedback-trajectory-C4EG0nNH.d.ts} +5 -8
  91. package/dist/feedback-trajectory-C4EG0nNH.d.ts.map +1 -0
  92. package/dist/fuzz.d.ts +1 -1
  93. package/dist/fuzz.js +3 -3
  94. package/dist/hosted/index.js +1 -1
  95. package/dist/{index-CPJJr98o.d.ts → index-BFRO6chs.d.ts} +3 -2
  96. package/dist/index-BFRO6chs.d.ts.map +1 -0
  97. package/dist/{index-Ch9VxCpK.d.ts → index-BO41eGj5.d.ts} +2 -2
  98. package/dist/{index-Ch9VxCpK.d.ts.map → index-BO41eGj5.d.ts.map} +1 -1
  99. package/dist/{index-NvGBiO8H.d.ts → index-CyszSg4O.d.ts} +165 -162
  100. package/dist/index-CyszSg4O.d.ts.map +1 -0
  101. package/dist/{index-DlH0Y9gI.d.ts → index-DPcngtwt.d.ts} +30 -17
  102. package/dist/index-DPcngtwt.d.ts.map +1 -0
  103. package/dist/{index-oQ4aHziK.d.ts → index-DV-jSzxF.d.ts} +8 -8
  104. package/dist/{index-oQ4aHziK.d.ts.map → index-DV-jSzxF.d.ts.map} +1 -1
  105. package/dist/index.d.ts +1969 -1871
  106. package/dist/index.d.ts.map +1 -1
  107. package/dist/index.js +161 -201
  108. package/dist/index.js.map +1 -1
  109. package/dist/{integrity-BvnVARVl.js → integrity-BXOsAb3i.js} +2 -2
  110. package/dist/{integrity-BvnVARVl.js.map → integrity-BXOsAb3i.js.map} +1 -1
  111. package/dist/{integrity-CCAxGVK6.d.ts → integrity-DAMEDFaC.d.ts} +2 -2
  112. package/dist/{integrity-CCAxGVK6.d.ts.map → integrity-DAMEDFaC.d.ts.map} +1 -1
  113. package/dist/{integrity-BV1ZDf42.js → integrity-dLiDe2kD.js} +2 -5
  114. package/dist/integrity-dLiDe2kD.js.map +1 -0
  115. package/dist/{internal-CRHwS9iz.js → internal-Epl-YBZW.js} +3 -3
  116. package/dist/{internal-CRHwS9iz.js.map → internal-Epl-YBZW.js.map} +1 -1
  117. package/dist/interval-CELpye1n.js +219 -0
  118. package/dist/interval-CELpye1n.js.map +1 -0
  119. package/dist/interval-ohuOJurX.d.ts +47 -0
  120. package/dist/interval-ohuOJurX.d.ts.map +1 -0
  121. package/dist/{journal-DvKCvjty.js → journal-DfRz9kps.js} +2 -2
  122. package/dist/{journal-DvKCvjty.js.map → journal-DfRz9kps.js.map} +1 -1
  123. package/dist/{judge-calibration-N44V04Se.js → judge-calibration-BpCy5XQl.js} +3 -3
  124. package/dist/{judge-calibration-N44V04Se.js.map → judge-calibration-BpCy5XQl.js.map} +1 -1
  125. package/dist/{judge-gate-DNcUrRFA.d.ts → judge-gate-CE7CQAgt.d.ts} +3 -3
  126. package/dist/{judge-gate-DNcUrRFA.d.ts.map → judge-gate-CE7CQAgt.d.ts.map} +1 -1
  127. package/dist/{judge-gate-B1qYnkpi.js → judge-gate-iY9s4L44.js} +3 -3
  128. package/dist/judge-gate-iY9s4L44.js.map +1 -0
  129. package/dist/{kind-factory-CbZ-6tvu.js → kind-factory-BobI5Jy3.js} +4 -4
  130. package/dist/{kind-factory-CbZ-6tvu.js.map → kind-factory-BobI5Jy3.js.map} +1 -1
  131. package/dist/ledger-core/index.js +3 -3
  132. package/dist/{ledger-core-B2hahzrt.js → ledger-core-Gq7yp8lP.js} +4 -4
  133. package/dist/{ledger-core-B2hahzrt.js.map → ledger-core-Gq7yp8lP.js.map} +1 -1
  134. package/dist/{llm-client-DQveoxSM.js → llm-client-CyBAoYiX.js} +192 -219
  135. package/dist/llm-client-CyBAoYiX.js.map +1 -0
  136. package/dist/{llm-judge-PXV7Xljl.js → llm-judge-CG12SQJq.js} +499 -921
  137. package/dist/llm-judge-CG12SQJq.js.map +1 -0
  138. package/dist/matrix/index.d.ts +2 -2
  139. package/dist/matrix/index.js +1 -1
  140. package/dist/{matrix-CHMiebFx.d.ts → matrix-DSrSxjjy.d.ts} +4 -4
  141. package/dist/{matrix-CHMiebFx.d.ts.map → matrix-DSrSxjjy.d.ts.map} +1 -1
  142. package/dist/{matrix-_BV9-Lk5.js → matrix-Dm0BQW7U.js} +7 -11
  143. package/dist/matrix-Dm0BQW7U.js.map +1 -0
  144. package/dist/{paired-tests-jbwPdSz5.js → mean-DHoMiudD.js} +11 -5
  145. package/dist/mean-DHoMiudD.js.map +1 -0
  146. package/dist/meta-eval/index.d.ts +3 -3
  147. package/dist/meta-eval/index.js +9 -9
  148. package/dist/meta-eval/index.js.map +1 -1
  149. package/dist/mint-DNU8QPG4.d.ts +81 -0
  150. package/dist/mint-DNU8QPG4.d.ts.map +1 -0
  151. package/dist/{mint-Csor1BLJ.js → mint-QaD5cA0u.js} +3 -3
  152. package/dist/{mint-Csor1BLJ.js.map → mint-QaD5cA0u.js.map} +1 -1
  153. package/dist/{multiplicity-D9cjmStq.js → multiplicity-DQUFcbW4.js} +2 -2
  154. package/dist/{multiplicity-D9cjmStq.js.map → multiplicity-DQUFcbW4.js.map} +1 -1
  155. package/dist/multishot/golden/index.d.ts +1 -1
  156. package/dist/multishot/golden/index.d.ts.map +1 -1
  157. package/dist/multishot/golden/index.js +3 -6
  158. package/dist/multishot/golden/index.js.map +1 -1
  159. package/dist/multishot/index.d.ts +2 -2
  160. package/dist/multishot/index.js +1 -1
  161. package/dist/nonnegative-number-DvdMxg_f.js +9 -0
  162. package/dist/nonnegative-number-DvdMxg_f.js.map +1 -0
  163. package/dist/openapi.json +1 -1
  164. package/dist/{otel-spans-C8YIJWy1.js → otel-spans-B9tUXRHF.js} +3 -3
  165. package/dist/{otel-spans-C8YIJWy1.js.map → otel-spans-B9tUXRHF.js.map} +1 -1
  166. package/dist/{paired-arms-DzKn0TfY.js → paired-arms-CwMGJsrS.js} +5 -5
  167. package/dist/{paired-arms-DzKn0TfY.js.map → paired-arms-CwMGJsrS.js.map} +1 -1
  168. package/dist/{paired-binary-Q36bhn2R.js → paired-binary-CvpFHfKZ.js} +2 -2
  169. package/dist/{paired-binary-Q36bhn2R.js.map → paired-binary-CvpFHfKZ.js.map} +1 -1
  170. package/dist/{paired-promotion-decision-BN6IQTiu.js → paired-promotion-decision-DsAPYGCm.js} +3 -3
  171. package/dist/{paired-promotion-decision-BN6IQTiu.js.map → paired-promotion-decision-DsAPYGCm.js.map} +1 -1
  172. package/dist/{search-kernel-BztlERLv.js → pareto-Cm6ndrWN.js} +883 -861
  173. package/dist/pareto-Cm6ndrWN.js.map +1 -0
  174. package/dist/pipelines/index.d.ts +2 -2
  175. package/dist/pipelines/index.js +3 -3
  176. package/dist/{power-and-mde-fK5Qocwo.js → power-and-mde-Hj2ubb4B.js} +2 -2
  177. package/dist/{power-and-mde-fK5Qocwo.js.map → power-and-mde-Hj2ubb4B.js.map} +1 -1
  178. package/dist/predicate-FdKmElA_.js +38 -0
  179. package/dist/predicate-FdKmElA_.js.map +1 -0
  180. package/dist/{prime-protocol-DcAK9pC8.js → prime-protocol-CHKgA3a6.js} +2 -2
  181. package/dist/{prime-protocol-DcAK9pC8.js.map → prime-protocol-CHKgA3a6.js.map} +1 -1
  182. package/dist/{prime-protocol-CEE4BR9N.d.ts → prime-protocol-HiB9HX7E.d.ts} +3 -3
  183. package/dist/{prime-protocol-CEE4BR9N.d.ts.map → prime-protocol-HiB9HX7E.d.ts.map} +1 -1
  184. package/dist/{produced-state-ZCtDBxAE.js → produced-state-BsDmnwxc.js} +8 -8
  185. package/dist/{produced-state-ZCtDBxAE.js.map → produced-state-BsDmnwxc.js.map} +1 -1
  186. package/dist/profile-cell.js +1 -1
  187. package/dist/{release-confidence-BK2QCB5M.d.ts → promotion-gate-BawrdPYr.d.ts} +102 -102
  188. package/dist/promotion-gate-BawrdPYr.d.ts.map +1 -0
  189. package/dist/{promotion-policy-CWW35UWq.js → promotion-policy-KO57gbyV.js} +4 -4
  190. package/dist/{promotion-policy-CWW35UWq.js.map → promotion-policy-KO57gbyV.js.map} +1 -1
  191. package/dist/{proposal-findings-BZiMTzAF.js → proposal-findings-CY1xKn_a.js} +2 -2
  192. package/dist/{proposal-findings-BZiMTzAF.js.map → proposal-findings-CY1xKn_a.js.map} +1 -1
  193. package/dist/{query-B1D1qp2Z.js → query-BUCZCK7F.js} +3 -3
  194. package/dist/{query-B1D1qp2Z.js.map → query-BUCZCK7F.js.map} +1 -1
  195. package/dist/{random-Dn5fPWkt.js → random-ByUSnV9w.js} +2 -2
  196. package/dist/{random-Dn5fPWkt.js.map → random-ByUSnV9w.js.map} +1 -1
  197. package/dist/{raw-provider-sink-rB2DYxr6.js → raw-provider-sink-DoHz4JNG.js} +2 -2
  198. package/dist/{raw-provider-sink-rB2DYxr6.js.map → raw-provider-sink-DoHz4JNG.js.map} +1 -1
  199. package/dist/{redact-Dk1ZdJEm.js → redact-B5xXKqQ3.js} +97 -98
  200. package/dist/redact-B5xXKqQ3.js.map +1 -0
  201. package/dist/{redact-Cveoq6fJ.d.ts → redact-mtCCcT8z.d.ts} +4 -20
  202. package/dist/redact-mtCCcT8z.d.ts.map +1 -0
  203. package/dist/{registry-JYfPxcVz.d.ts → registry-D7_hm2fK.d.ts} +5 -5
  204. package/dist/{registry-JYfPxcVz.d.ts.map → registry-D7_hm2fK.d.ts.map} +1 -1
  205. package/dist/{release-confidence-Dbe9AS_X.js → release-confidence-CpuP65S5.js} +5 -5
  206. package/dist/{release-confidence-Dbe9AS_X.js.map → release-confidence-CpuP65S5.js.map} +1 -1
  207. package/dist/{report-command-y8H6RSX8.js → report-command-SC9nqb4M.js} +106 -171
  208. package/dist/report-command-SC9nqb4M.js.map +1 -0
  209. package/dist/reporting-html-Kvlc8oMt.js +100 -0
  210. package/dist/reporting-html-Kvlc8oMt.js.map +1 -0
  211. package/dist/reporting.d.ts +3 -3
  212. package/dist/reporting.js +6 -6
  213. package/dist/{researcher-f6mZhh58.d.ts → researcher-CEcfku-T.d.ts} +5 -5
  214. package/dist/{researcher-f6mZhh58.d.ts.map → researcher-CEcfku-T.d.ts.map} +1 -1
  215. package/dist/reward-CpCmAM8L.d.ts +78 -0
  216. package/dist/reward-CpCmAM8L.d.ts.map +1 -0
  217. package/dist/{reward-hacking-BFcQgkux.js → reward-hacking-E0d-WVcP.js} +2 -2
  218. package/dist/{reward-hacking-BFcQgkux.js.map → reward-hacking-E0d-WVcP.js.map} +1 -1
  219. package/dist/{reward-hacking-DHA7c1l8.d.ts → reward-hacking-RVdZ6Kkt.d.ts} +2 -2
  220. package/dist/{reward-hacking-DHA7c1l8.d.ts.map → reward-hacking-RVdZ6Kkt.d.ts.map} +1 -1
  221. package/dist/rl.d.ts +8 -8
  222. package/dist/rl.js +14 -14
  223. package/dist/rollout/index.d.ts +3 -2
  224. package/dist/rollout/index.js +2 -2
  225. package/dist/{rollout-D9bKKBnZ.js → rollout-DZV1t2Wg.js} +2 -2
  226. package/dist/{rollout-D9bKKBnZ.js.map → rollout-DZV1t2Wg.js.map} +1 -1
  227. package/dist/round-decimal-COrKglMh.js +10 -0
  228. package/dist/round-decimal-COrKglMh.js.map +1 -0
  229. package/dist/{rubric-predictive-validity-2SgHxoa5.d.ts → rubric-predictive-validity-C03EC7ZE.d.ts} +2 -2
  230. package/dist/{rubric-predictive-validity-2SgHxoa5.d.ts.map → rubric-predictive-validity-C03EC7ZE.d.ts.map} +1 -1
  231. package/dist/{rubric-predictive-validity-BsgkBfT0.js → rubric-predictive-validity-DaFW70y9.js} +3 -3
  232. package/dist/{rubric-predictive-validity-BsgkBfT0.js.map → rubric-predictive-validity-DaFW70y9.js.map} +1 -1
  233. package/dist/{run-record-DlTZ_lqe.js → run-record-Boc5IUTp.js} +4 -4
  234. package/dist/{run-record-DlTZ_lqe.js.map → run-record-Boc5IUTp.js.map} +1 -1
  235. package/dist/{run-record-D1Hw-Jml.d.ts → run-record-C1E24G2A.d.ts} +2 -2
  236. package/dist/{run-record-D1Hw-Jml.d.ts.map → run-record-C1E24G2A.d.ts.map} +1 -1
  237. package/dist/{run-record-ES07wfz9.js → run-record-C5e7FjhJ.js} +3 -5
  238. package/dist/run-record-C5e7FjhJ.js.map +1 -0
  239. package/dist/search/index.d.ts +46 -32
  240. package/dist/search/index.d.ts.map +1 -1
  241. package/dist/search/index.js +3 -3
  242. package/dist/{search-kernel-CpcKH5Ih.d.ts → search-kernel-DyNEa6IU.d.ts} +14 -10
  243. package/dist/search-kernel-DyNEa6IU.d.ts.map +1 -0
  244. package/dist/{search-ledger-d7V8o9Lk.js → search-ledger-CVUGLDZp.js} +189 -199
  245. package/dist/search-ledger-CVUGLDZp.js.map +1 -0
  246. package/dist/{search-ledger-file-BqOo-bVo.js → search-ledger-file-DDQUgvCg.js} +4 -4
  247. package/dist/{search-ledger-file-BqOo-bVo.js.map → search-ledger-file-DDQUgvCg.js.map} +1 -1
  248. package/dist/search-ledger-types-p2xKfiDJ.d.ts.map +1 -1
  249. package/dist/{search-shipper-er4pK5_Y.js → search-shipper-BOSD46qv.js} +3 -3
  250. package/dist/{search-shipper-er4pK5_Y.js.map → search-shipper-BOSD46qv.js.map} +1 -1
  251. package/dist/{search-summary-CBvz3u_S.js → search-summary-BQCSWf1g.js} +5 -5
  252. package/dist/{search-summary-CBvz3u_S.js.map → search-summary-BQCSWf1g.js.map} +1 -1
  253. package/dist/{semantic-concept-judge-zQBG9hHU.js → semantic-concept-judge-BQF8te0T.js} +12 -14
  254. package/dist/semantic-concept-judge-BQF8te0T.js.map +1 -0
  255. package/dist/{sequential-D-1PPCmM.js → sequential-Bf_pbE50.js} +5 -5
  256. package/dist/{sequential-D-1PPCmM.js.map → sequential-Bf_pbE50.js.map} +1 -1
  257. package/dist/{sequential-eprocess-D1jKoihe.js → sequential-eprocess-DM_qENCl.js} +2 -2
  258. package/dist/{sequential-eprocess-D1jKoihe.js.map → sequential-eprocess-DM_qENCl.js.map} +1 -1
  259. package/dist/{server-DbcC7xMl.js → server-e9oWO_bN.js} +4 -4
  260. package/dist/{server-DbcC7xMl.js.map → server-e9oWO_bN.js.map} +1 -1
  261. package/dist/{single-run-lock-fVu3MZWH.js → single-run-lock-CV9Eyd_q.js} +2 -2
  262. package/dist/{single-run-lock-fVu3MZWH.js.map → single-run-lock-CV9Eyd_q.js.map} +1 -1
  263. package/dist/{skillopt-optimization-method-CCtzralq.js → skillopt-optimization-method-BiwA8nfX.js} +427 -12
  264. package/dist/skillopt-optimization-method-BiwA8nfX.js.map +1 -0
  265. package/dist/{spans-D9_ehFuk.d.ts → spans-BHAUXMGN.d.ts} +2 -2
  266. package/dist/{spans-D9_ehFuk.d.ts.map → spans-BHAUXMGN.d.ts.map} +1 -1
  267. package/dist/{statistical-heldout-1Gbmex7v.js → statistical-heldout-42NlCNDe.js} +3 -3
  268. package/dist/{statistical-heldout-1Gbmex7v.js.map → statistical-heldout-42NlCNDe.js.map} +1 -1
  269. package/dist/{statistical-heldout-BeR4bOdg.d.ts → statistical-heldout-CYpGaoDH.d.ts} +3 -3
  270. package/dist/{statistical-heldout-BeR4bOdg.d.ts.map → statistical-heldout-CYpGaoDH.d.ts.map} +1 -1
  271. package/dist/{store-otlp-DtB4Slsr.js → store-otlp-DJ9dtA5x.js} +135 -132
  272. package/dist/store-otlp-DJ9dtA5x.js.map +1 -0
  273. package/dist/{store-tool-spans-fZIigrpP.js → store-tool-spans-CIX5arAG.js} +7 -7
  274. package/dist/{store-tool-spans-fZIigrpP.js.map → store-tool-spans-CIX5arAG.js.map} +1 -1
  275. package/dist/{store-tool-spans-DR1SrCLK.d.ts → store-tool-spans-DdCSPa7C.d.ts} +162 -161
  276. package/dist/store-tool-spans-DdCSPa7C.d.ts.map +1 -0
  277. package/dist/storyboard/index.d.ts.map +1 -1
  278. package/dist/storyboard/index.js +20 -3
  279. package/dist/storyboard/index.js.map +1 -1
  280. package/dist/{student-t-BQeKJx2N.js → student-t-DVPW7CdL.js} +2 -2
  281. package/dist/{student-t-BQeKJx2N.js.map → student-t-DVPW7CdL.js.map} +1 -1
  282. package/dist/{summary-report-DtdRucM0.js → summary-report-V_JACR4Y.js} +526 -614
  283. package/dist/summary-report-V_JACR4Y.js.map +1 -0
  284. package/dist/{summary-report-C-CtItCN.d.ts → summary-report-bdFDXq0b.d.ts} +10 -5
  285. package/dist/summary-report-bdFDXq0b.d.ts.map +1 -0
  286. package/dist/supervisor-run/index.d.ts +153 -149
  287. package/dist/supervisor-run/index.d.ts.map +1 -1
  288. package/dist/supervisor-run/index.js +65 -3
  289. package/dist/supervisor-run/index.js.map +1 -1
  290. package/dist/{systemone-CpTLAlqb.d.ts → systemone-BomhHr2c.d.ts} +6 -6
  291. package/dist/{systemone-CpTLAlqb.d.ts.map → systemone-BomhHr2c.d.ts.map} +1 -1
  292. package/dist/systemone-protocol.js +1 -1
  293. package/dist/{systemone-review-B_yo1xsd.d.ts → systemone-review-CUVcQJu6.d.ts} +2 -2
  294. package/dist/{systemone-review-B_yo1xsd.d.ts.map → systemone-review-CUVcQJu6.d.ts.map} +1 -1
  295. package/dist/{systemone-review-BZWfrtq1.js → systemone-review-D7sXwsqq.js} +2 -2
  296. package/dist/{systemone-review-BZWfrtq1.js.map → systemone-review-D7sXwsqq.js.map} +1 -1
  297. package/dist/systemone.d.ts +3 -3
  298. package/dist/systemone.js +4 -4
  299. package/dist/{task-failure-attributes-DngaLdE4.js → task-failure-attributes-hEmASjix.js} +2 -2
  300. package/dist/{task-failure-attributes-DngaLdE4.js.map → task-failure-attributes-hEmASjix.js.map} +1 -1
  301. package/dist/{terminal-record-BHocwFw1.js → terminal-record-DZyQZ3pz.js} +14 -10
  302. package/dist/terminal-record-DZyQZ3pz.js.map +1 -0
  303. package/dist/{tool-groups-mcxOXVlb.d.ts → tool-groups-C3NEcdCM.d.ts} +3 -3
  304. package/dist/tool-groups-C3NEcdCM.d.ts.map +1 -0
  305. package/dist/{tool-waste-Wi4g6Ekb.d.ts → tool-waste-BKgvd-TP.d.ts} +30 -30
  306. package/dist/tool-waste-BKgvd-TP.d.ts.map +1 -0
  307. package/dist/{tool-waste-D52PqU9Q.js → tool-waste-BlesqTqW.js} +4 -4
  308. package/dist/{tool-waste-D52PqU9Q.js.map → tool-waste-BlesqTqW.js.map} +1 -1
  309. package/dist/{trace-questions-DG7nqLQ7.d.ts → trace-questions-CM2CAocU.d.ts} +4 -4
  310. package/dist/{trace-questions-DG7nqLQ7.d.ts.map → trace-questions-CM2CAocU.d.ts.map} +1 -1
  311. package/dist/trace-repair/index.d.ts +3 -3
  312. package/dist/trace-repair/index.d.ts.map +1 -1
  313. package/dist/trace-repair/index.js +7 -13
  314. package/dist/trace-repair/index.js.map +1 -1
  315. package/dist/traces.d.ts +76 -58
  316. package/dist/traces.d.ts.map +1 -1
  317. package/dist/traces.js +9 -9
  318. package/dist/trajectory-replay/index.js +3 -3
  319. package/dist/{tree-rXLW3M_4.js → tree-C1XYeXvy.js} +2277 -2339
  320. package/dist/tree-C1XYeXvy.js.map +1 -0
  321. package/dist/{types-DBZ7keha.d.ts → types-C02qtnG8.d.ts} +4 -46
  322. package/dist/types-C02qtnG8.d.ts.map +1 -0
  323. package/dist/{types-oKRhQAde.d.ts → types-C1tnkEgM.d.ts} +4 -4
  324. package/dist/{types-oKRhQAde.d.ts.map → types-C1tnkEgM.d.ts.map} +1 -1
  325. package/dist/{types-CfZMIzB_.d.ts → types-Cbe2fWBa.d.ts} +3 -3
  326. package/dist/{types-CfZMIzB_.d.ts.map → types-Cbe2fWBa.d.ts.map} +1 -1
  327. package/dist/{types-B4C5nFw6.d.ts → types-Cq8mSbq8.d.ts} +78 -72
  328. package/dist/types-Cq8mSbq8.d.ts.map +1 -0
  329. package/dist/{verdict-DhPoXApk.js → verdict-Dl0tiT1X.js} +2 -2
  330. package/dist/{verdict-DhPoXApk.js.map → verdict-Dl0tiT1X.js.map} +1 -1
  331. package/dist/{verdict-cache-BQEsVy9n.js → verdict-cache-Bm8EKL2s.js} +2 -2
  332. package/dist/{verdict-cache-BQEsVy9n.js.map → verdict-cache-Bm8EKL2s.js.map} +1 -1
  333. package/dist/wire/index.d.ts +3 -3
  334. package/dist/wire/index.js +1 -1
  335. package/docs/design/mlbenchmarks-book-review.md +1 -1
  336. package/docs/design/mlbenchmarks-review/probes.mts +23 -31
  337. package/docs/experiment.md +1 -1
  338. package/package.json +1 -1
  339. package/dist/analyze-runs-3xpby3p2.js.map +0 -1
  340. package/dist/analyze-runs-CSDAh8A7.d.ts.map +0 -1
  341. package/dist/ast-DJ6Bc7ab.js +0 -558
  342. package/dist/ast-DJ6Bc7ab.js.map +0 -1
  343. package/dist/ast-hI-vjW6J.d.ts +0 -457
  344. package/dist/ast-hI-vjW6J.d.ts.map +0 -1
  345. package/dist/benchmark-command-CByl9dxG.js.map +0 -1
  346. package/dist/campaign-BQ-Jujuj.js.map +0 -1
  347. package/dist/campaign-evidence-OcUciD-N.js.map +0 -1
  348. package/dist/claim-BXcZw-jo.js.map +0 -1
  349. package/dist/cost-ledger-CO7UyEcQ.d.ts.map +0 -1
  350. package/dist/cost-ledger-D2FnJPBF.js.map +0 -1
  351. package/dist/define-agent-eval-Ch3MWX6k.js.map +0 -1
  352. package/dist/external-optimizer-subprocess-DcerlZSG.js.map +0 -1
  353. package/dist/feedback-trajectory-DkaDj0dj.d.ts.map +0 -1
  354. package/dist/index-CPJJr98o.d.ts.map +0 -1
  355. package/dist/index-DlH0Y9gI.d.ts.map +0 -1
  356. package/dist/index-NvGBiO8H.d.ts.map +0 -1
  357. package/dist/integrity-BV1ZDf42.js.map +0 -1
  358. package/dist/judge-gate-B1qYnkpi.js.map +0 -1
  359. package/dist/llm-client-DQveoxSM.js.map +0 -1
  360. package/dist/llm-judge-PXV7Xljl.js.map +0 -1
  361. package/dist/matrix-_BV9-Lk5.js.map +0 -1
  362. package/dist/mint-DCGoBYhB.d.ts +0 -155
  363. package/dist/mint-DCGoBYhB.d.ts.map +0 -1
  364. package/dist/paired-tests-jbwPdSz5.js.map +0 -1
  365. package/dist/redact-Cveoq6fJ.d.ts.map +0 -1
  366. package/dist/redact-Dk1ZdJEm.js.map +0 -1
  367. package/dist/release-confidence-BK2QCB5M.d.ts.map +0 -1
  368. package/dist/report-command-y8H6RSX8.js.map +0 -1
  369. package/dist/run-record-ES07wfz9.js.map +0 -1
  370. package/dist/search-kernel-BztlERLv.js.map +0 -1
  371. package/dist/search-kernel-CpcKH5Ih.d.ts.map +0 -1
  372. package/dist/search-ledger-d7V8o9Lk.js.map +0 -1
  373. package/dist/semantic-concept-judge-zQBG9hHU.js.map +0 -1
  374. package/dist/skillopt-optimization-method-CCtzralq.js.map +0 -1
  375. package/dist/store-otlp-DtB4Slsr.js.map +0 -1
  376. package/dist/store-tool-spans-DR1SrCLK.d.ts.map +0 -1
  377. package/dist/summary-report-C-CtItCN.d.ts.map +0 -1
  378. package/dist/summary-report-DtdRucM0.js.map +0 -1
  379. package/dist/terminal-record-BHocwFw1.js.map +0 -1
  380. package/dist/tool-groups-mcxOXVlb.d.ts.map +0 -1
  381. package/dist/tool-waste-Wi4g6Ekb.d.ts.map +0 -1
  382. package/dist/tree-rXLW3M_4.js.map +0 -1
  383. package/dist/types-B4C5nFw6.d.ts.map +0 -1
  384. package/dist/types-DBZ7keha.d.ts.map +0 -1
@@ -1,25 +1,24 @@
1
- import { i as JudgeError, s as ValidationError } from "./errors-Dngq5h35.js";
2
- import { a as hashCanonical, i as compareCodeUnits, r as canonicalString, t as LEDGER_HASH_PATTERN } from "./canonical-BsZVgBhW.js";
3
- import { t as mulberry32 } from "./random-Dn5fPWkt.js";
4
- import { o as summarizeNumberSeries, s as weightedComposite, t as confidenceInterval } from "./descriptive-CnU8Yq29.js";
5
- import { r as minimumPairsForPairedDeltaTest, t as decidePairedPromotion } from "./paired-promotion-decision-BN6IQTiu.js";
6
- import { a as heldoutSignificance, i as dimensionRegressions, o as pairHoldout } from "./statistical-heldout-1Gbmex7v.js";
7
- import { i as modelHasSnapshot, s as searchCellRunId } from "./run-record-DlTZ_lqe.js";
8
- import { n as contentHash } from "./verdict-cache-BQEsVy9n.js";
9
- import { r as projectCampaignCellQuality, t as campaignCellCostProvenance } from "./run-record-ES07wfz9.js";
10
- import { i as CostLedger, t as CostAccountingIncompleteError } from "./cost-ledger-D2FnJPBF.js";
11
- import { B as assertCampaignDesign, D as FinalEvidenceError, E as FinalEvidenceConflictError, F as surfaceHash, G as campaignSplitDigest, H as assertCompleteCampaign, I as surfaceHashMatches, L as BackendIntegrityError, M as renderSurfaceDiff, N as surfaceContentHash, P as surfaceDispatchRef, U as campaignCoverage, V as assertCampaignSplitIdentity, W as campaignScenarioIdentity, b as campaignMeanCompositeOrNull, q as formatCoverageFailures, t as createCampaignEvidenceReceipt, v as campaignBreakdown, y as campaignMeanComposite } from "./campaign-evidence-OcUciD-N.js";
12
- import { m as mapConcurrent } from "./journal-DvKCvjty.js";
1
+ import { i as JudgeError, s as ValidationError } from "./errors-DVZiqS7K.js";
2
+ import { a as hashCanonical, i as compareCodeUnits, r as canonicalString, t as LEDGER_HASH_PATTERN } from "./canonical-B2RqLofv.js";
3
+ import { i as modelHasSnapshot, s as searchCellRunId } from "./run-record-Boc5IUTp.js";
4
+ import { t as mulberry32 } from "./random-ByUSnV9w.js";
5
+ import { o as summarizeNumberSeries, s as weightedComposite, t as confidenceInterval } from "./descriptive-DboxPZrG.js";
6
+ import { r as minimumPairsForPairedDeltaTest } from "./paired-promotion-decision-DsAPYGCm.js";
7
+ import { a as heldoutSignificance, i as dimensionRegressions, o as pairHoldout } from "./statistical-heldout-42NlCNDe.js";
8
+ import { n as contentHash } from "./verdict-cache-Bm8EKL2s.js";
9
+ import { r as projectCampaignCellQuality, t as campaignCellCostProvenance } from "./run-record-C5e7FjhJ.js";
10
+ import { n as CostAccountingIncompleteError, t as CostLedger } from "./cost-ledger-BP8V4kUQ.js";
11
+ import { B as assertCampaignDesign, D as FinalEvidenceError, E as FinalEvidenceConflictError, F as surfaceHash, G as campaignSplitDigest, H as assertCompleteCampaign, I as surfaceHashMatches, L as BackendIntegrityError, M as renderSurfaceDiff, N as surfaceContentHash, P as surfaceDispatchRef, U as campaignCoverage, V as assertCampaignSplitIdentity, W as campaignScenarioIdentity, b as campaignMeanCompositeOrNull, q as formatCoverageFailures, v as campaignBreakdown, y as campaignMeanComposite } from "./campaign-evidence-Clrn4hBV.js";
13
12
  import { t as deepFreezeCanonicalJson } from "./deep-freeze-CurcOQg3.js";
14
- import { N as createRunCostLedger, P as fsCampaignStorage, g as isRecord, h as isExternalTextCandidate } from "./external-optimizer-subprocess-DcerlZSG.js";
15
- import { r as summarizeEvaluationUnits, t as defineEvaluationClaim } from "./claim-BXcZw-jo.js";
16
- import { f as redactText, n as REDACTION_VERSION, o as detectCredential, s as detectPersonalData } from "./redact-Dk1ZdJEm.js";
17
- import { t as detectRewardHacking } from "./reward-hacking-BFcQgkux.js";
18
- import { t as assertProposalFindings } from "./proposal-findings-BZiMTzAF.js";
19
- import { O as paretoFrontier, a as searchPolicyView, i as searchExpansionIndex, k as paretoFrontierWithCrowding, m as EXPANSION_OPERATORS, n as runSearch, t as SEARCH_KERNEL_SOURCE, x as estimateNode } from "./search-kernel-BztlERLv.js";
20
- import { a as validateSearchLedgerEvent, d as searchTaskSetDigest, i as replaySearchLedgerText, l as searchCellId, n as openSearchLedger, u as searchNodeId } from "./search-ledger-d7V8o9Lk.js";
21
- import { n as searchProposerView, t as renderSearchSummary } from "./search-summary-CBvz3u_S.js";
22
- import { C as combineAbortSignals, d as stripFencedJson, o as costReceiptFromLlm, s as costReceiptFromLlmError, u as maximumChargeForLlmRequest } from "./llm-client-DQveoxSM.js";
13
+ import { N as createRunCostLedger, P as fsCampaignStorage, g as isRecord, h as isExternalTextCandidate } from "./external-optimizer-subprocess-BqW0hHcI.js";
14
+ import { r as summarizeEvaluationUnits, t as defineEvaluationClaim } from "./claim-q4HQhK1G.js";
15
+ import { c as redactText, d as detectCredential, f as detectPersonalData, n as REDACTION_VERSION } from "./redact-B5xXKqQ3.js";
16
+ import { t as detectRewardHacking } from "./reward-hacking-E0d-WVcP.js";
17
+ import { t as assertProposalFindings } from "./proposal-findings-CY1xKn_a.js";
18
+ import { i as runSearch, m as searchExpansionIndex, n as paretoFrontier, o as searchPolicyView, p as SEARCH_KERNEL_SOURCE, r as paretoFrontierWithCrowding, s as EXPANSION_OPERATORS, w as estimateNode } from "./pareto-Cm6ndrWN.js";
19
+ import { a as validateSearchLedgerEvent, d as searchNodeId, f as searchTaskSetDigest, i as replaySearchLedgerText, n as openSearchLedger, u as searchCellId } from "./search-ledger-CVUGLDZp.js";
20
+ import { n as searchProposerView, t as renderSearchSummary } from "./search-summary-BQCSWf1g.js";
21
+ import { a as maximumChargeForLlmRequest, i as costReceiptFromLlmError, r as costReceiptFromLlm, x as stripFencedJson } from "./llm-client-CyBAoYiX.js";
23
22
  import { createHash } from "node:crypto";
24
23
  import { writeFileSync } from "node:fs";
25
24
  import { z } from "zod";
@@ -1200,8 +1199,8 @@ function detectCalibrationDrift(runs, opts) {
1200
1199
  alpha,
1201
1200
  recentN: recent.length,
1202
1201
  historyN: historical.length,
1203
- recentMean: mean$2(recent),
1204
- historyMean: mean$2(historical)
1202
+ recentMean: mean$1(recent),
1203
+ historyMean: mean$1(historical)
1205
1204
  }
1206
1205
  }],
1207
1206
  evaluation: {
@@ -1438,7 +1437,7 @@ function chiSquareCritical(df, alpha) {
1438
1437
  }
1439
1438
  return TABLE[10][idx];
1440
1439
  }
1441
- function mean$2(xs) {
1440
+ function mean$1(xs) {
1442
1441
  if (xs.length === 0) return 0;
1443
1442
  return xs.reduce((s, x) => s + x, 0) / xs.length;
1444
1443
  }
@@ -3190,7 +3189,6 @@ function nonEmpty(value, label) {
3190
3189
  * and a fact already in the ledger is not appended again, so a resumed search
3191
3190
  * continues its ledger instead of conflicting with its own history.
3192
3191
  */
3193
- const MAX_LABEL_CHARS = 200;
3194
3192
  /** The default claim: development numbers over exactly the supplied tasks. */
3195
3193
  function developmentClaim(populationId) {
3196
3194
  return {
@@ -3554,7 +3552,7 @@ function declaredSurfaces(surface, artifact) {
3554
3552
  }
3555
3553
  function boundedLabel(label) {
3556
3554
  const redacted = redactText(label.trim(), { profile: "share" });
3557
- return redacted.length <= MAX_LABEL_CHARS ? redacted : `${redacted.slice(0, 199)}…`;
3555
+ return redacted.length <= 200 ? redacted : `${redacted.slice(0, 199)}…`;
3558
3556
  }
3559
3557
  /** The header without its stamp, for comparing a resumed opening to the stored one. */
3560
3558
  function headerMaterial(header) {
@@ -3968,7 +3966,7 @@ function hillClimb(spec) {
3968
3966
  evidence: {
3969
3967
  ...chosen.evidence,
3970
3968
  units: units.length,
3971
- ...units.length > 0 ? { mean: mean$1(units.map((unit) => unit.mean)) } : {},
3969
+ ...units.length > 0 ? { mean: mean(units.map((unit) => unit.mean)) } : {},
3972
3970
  isLeader: chosen.nodeId === kept ? 1 : 0
3973
3971
  }
3974
3972
  }
@@ -4024,7 +4022,7 @@ function paretoMembers(view) {
4024
4022
  members.push({
4025
4023
  nodeId,
4026
4024
  ordinal,
4027
- mean: sign * mean$1([...objectives.values()]),
4025
+ mean: sign * mean([...objectives.values()]),
4028
4026
  objectives,
4029
4027
  distance: 0
4030
4028
  });
@@ -4046,7 +4044,7 @@ function compareCrowded(a, b) {
4046
4044
  if (a.mean !== b.mean) return a.mean > b.mean ? -1 : 1;
4047
4045
  return a.ordinal - b.ordinal;
4048
4046
  }
4049
- function mean$1(values) {
4047
+ function mean(values) {
4050
4048
  let total = 0;
4051
4049
  for (const value of values) total += value;
4052
4050
  return total / values.length;
@@ -4068,275 +4066,8 @@ function labelTrustRank(trust) {
4068
4066
  return LABEL_TRUST_RANK[trust ?? "unverified"];
4069
4067
  }
4070
4068
  //#endregion
4071
- //#region src/campaign/presets/run-optimization.ts
4072
- /**
4073
- * `runOptimization` improves a mutable surface with a caller-owned proposer,
4074
- * as a search on the one kernel (`runSearch`).
4075
- *
4076
- * The baseline is the seeded root. The scenarios are the train split: the
4077
- * proposer reads their scores, and the policy ranks on them because the loop
4078
- * has no private selection split. A "generation" is one proposal of
4079
- * `populationSize` candidates from the node the policy chose (by default the
4080
- * incumbent, once every earlier candidate is measured). Every candidate is a
4081
- * node measured on every scenario at every repeat, one cell at a time on the
4082
- * run's lanes. The kept surface is the policy's leader: a budget decision on
4083
- * the train split that claims nothing; `runImprovementLoop` compares it with
4084
- * the baseline on separate cases.
4085
- *
4086
- * The search ledger is always written, beside the run (`<runDir>/search/`) or
4087
- * where `searchLedger` puts it, and it is the run's only checkpoint: running
4088
- * again on the same run directory and options continues an interrupted run
4089
- * instead of starting over, and reruns nothing that settled.
4090
- */
4091
- /** Improve a surface as a search on the kernel; see the module comment. */
4092
- async function runOptimization(opts) {
4093
- const candidateConcurrency = opts.candidateConcurrency ?? 1;
4094
- if (typeof opts.runDir !== "string" || opts.runDir.trim().length === 0) throw new Error("runOptimization: runDir is required and must be a non-empty string");
4095
- if (!Number.isInteger(candidateConcurrency) || candidateConcurrency < 1) throw new Error("runOptimization: candidateConcurrency must be a positive integer");
4096
- for (const [name, value] of [["populationSize", opts.populationSize], ["maxGenerations", opts.maxGenerations]]) if (!Number.isSafeInteger(value) || value < 0) throw new Error(`runOptimization: ${name} must be a non-negative integer`);
4097
- const initialFindings = immutableProposalSnapshot(assertProposalFindings(opts.findings ?? [], "runOptimization initial proposal findings"), "initial findings");
4098
- const baselineSurface = immutableProposalSnapshot(opts.baselineSurface, "baseline surface");
4099
- const runDir = resolveRunDir(opts.runDir, opts.repo);
4100
- const storage = opts.storage ?? fsCampaignStorage();
4101
- const costLedger = opts.costLedger ?? createRunCostLedger({
4102
- storage,
4103
- runDir,
4104
- costCeilingUsd: opts.costCeiling
4105
- });
4106
- const reps = opts.reps ?? 1;
4107
- const seed = opts.seed ?? 42;
4108
- const requireJudgeScore = (opts.judges?.length ?? 0) > 0;
4109
- const premeasured = opts.premeasuredBaseline ? validatedPremeasuredBaseline({
4110
- input: opts.premeasuredBaseline,
4111
- baselineSurface,
4112
- scenarios: opts.scenarios,
4113
- reps,
4114
- seed,
4115
- judges: opts.judges ?? [],
4116
- dispatchRef: surfaceDispatchRef(baselineSurface, opts.dispatchRef),
4117
- requireJudgeScore
4118
- }) : void 0;
4119
- const splitDigest = campaignSplitDigest(opts.scenarios, reps);
4120
- const binding = opts.searchLedger ?? await defaultBinding(opts, runDir, storage);
4121
- const { identity } = binding;
4122
- const execution = {
4123
- model: identity.model,
4124
- agent: identity.agent,
4125
- benchmark: {
4126
- uri: "campaign://scenarios",
4127
- revision: splitDigest
4128
- }
4129
- };
4130
- const policy = opts.policy ?? incumbent();
4131
- const allocation = uniform({ reps });
4132
- const recorder = await SearchRecorder.open({
4133
- ledger: binding.ledger,
4134
- storage
4135
- }, {
4136
- subject: identity.subject ?? opts.proposer.kind,
4137
- process: {
4138
- name: "runOptimization",
4139
- executionRef: identity.search
4140
- },
4141
- artifactKind: artifactKindOf(baselineSurface),
4142
- objective: {
4143
- metric: "composite",
4144
- direction: "maximize",
4145
- judge: identity.judge ?? { unknown: "runOptimization receives its judges as functions without a pinned source" },
4146
- claim: identity.claim ?? developmentClaim(splitDigest)
4147
- },
4148
- splits: {
4149
- train: opts.scenarios.map((scenario) => ({
4150
- taskId: scenario.id,
4151
- unitId: scenario.id,
4152
- source: {
4153
- uri: `scenario://${scenario.id}`,
4154
- revision: hashCanonical(scenario)
4155
- }
4156
- })),
4157
- selection: [],
4158
- test: [],
4159
- heldOutUnits: true
4160
- },
4161
- policy: {
4162
- expansion: policy.name,
4163
- allocation: allocation.name,
4164
- seed
4165
- },
4166
- budget: {
4167
- maxUsd: opts.costCeiling ?? null,
4168
- maxCells: null,
4169
- maxNodes: 1 + opts.populationSize * opts.maxGenerations,
4170
- deadline: null,
4171
- maxConcurrency: null,
4172
- reservedClaimUsd: 0
4173
- },
4174
- containment: null,
4175
- derivedFrom: null,
4176
- identity: execution
4177
- });
4178
- const nodes = new SurfaceNodes({
4179
- opts,
4180
- recorder,
4181
- runDir,
4182
- costLedger,
4183
- execution,
4184
- premeasured
4185
- });
4186
- const scenarioById = new Map(opts.scenarios.map((scenario) => [scenario.id, scenario]));
4187
- let findings = initialFindings;
4188
- const executor = {
4189
- lanes: () => [{
4190
- name: "campaign",
4191
- capacity: candidateConcurrency * (opts.maxConcurrency ?? 2),
4192
- costCap: "estimate",
4193
- cellUsd: 0
4194
- }],
4195
- place: () => "campaign",
4196
- run: (work) => nodes.runCell(work, scenarioById),
4197
- adopt: (work) => nodes.adoptCell(work)
4198
- };
4199
- const proposer = opts.proposer;
4200
- const proposerSource = identity.proposer.source;
4201
- const port = {
4202
- name: proposer.kind,
4203
- kind: "optimizer",
4204
- source: proposerSource,
4205
- execution: identity.proposer,
4206
- childrenPerProposal: opts.populationSize,
4207
- async propose(request) {
4208
- const phases = [
4209
- "search.proposal",
4210
- "analysis.baseline",
4211
- "analysis.generation"
4212
- ];
4213
- const before = new Set(phases.flatMap((phase) => costLedger.list({ phase })).map(receiptKey));
4214
- const accounting = () => {
4215
- const fresh = phases.flatMap((phase) => costLedger.list({ phase })).filter((receipt) => !before.has(receiptKey(receipt)));
4216
- return {
4217
- accounting: searchReceiptAccounting(fresh),
4218
- execution: opts.searchLedger ? identity.proposer : searchProposalExecution(fresh, identity.model.provider, proposerSource)
4219
- };
4220
- };
4221
- const earlier = await recorder.state();
4222
- const root = earlier.rootNodeId;
4223
- const rootCampaign = nodes.campaign(earlier, root);
4224
- const rootCoverage = campaignCoverage(rootCampaign.cells, opts.scenarios, reps, requireJudgeScore);
4225
- if (!rootCoverage.complete) throw new Error(`runOptimization: ${premeasured ? "premeasured baseline" : "baseline"} is incomplete (${rootCoverage.scorableCellIds.length}/${rootCoverage.expectedCellIds.length} designed cells scorable) — ${formatCoverageFailures(rootCoverage)}. Refusing to optimize against an incomplete incumbent.`);
4226
- const generation = nodes.generations(earlier).length;
4227
- if (opts.analyzeGeneration) {
4228
- const previous = generation - 1;
4229
- const candidates = (previous < 0 ? [root] : nodes.expansionNodes(earlier, nodes.generations(earlier)[previous])).map((nodeId) => {
4230
- const campaign = nodes.campaign(earlier, nodeId);
4231
- return {
4232
- surfaceHash: surfaceHash(nodes.surface(earlier, nodeId)),
4233
- campaign,
4234
- composite: campaignMeanCompositeOrNull(campaign)
4235
- };
4236
- });
4237
- const history = nodes.history(earlier, policy);
4238
- if (candidates.some(({ campaign }) => campaign.cells.length > 0)) {
4239
- const fresh = await opts.analyzeGeneration({
4240
- generation: previous,
4241
- runDir: previous < 0 ? rootCampaign.runDir : `${runDir}/candidates`,
4242
- candidates,
4243
- history,
4244
- costLedger,
4245
- costPhase: previous < 0 ? "analysis.baseline" : "analysis.generation"
4246
- });
4247
- if (!Array.isArray(fresh)) throw new TypeError("runOptimization: analyzeGeneration must return an array");
4248
- findings = immutableProposalSnapshot(assertProposalFindings(fresh, "runOptimization analysis findings"), "analysis findings");
4249
- }
4250
- }
4251
- const state = await recorder.state();
4252
- const proposalHistory = immutableProposalSnapshot(nodes.history(state, policy), "history");
4253
- const parent = request.parents[0];
4254
- const context = Object.freeze({
4255
- currentSurface: immutableProposalSnapshot(parent.artifact, "current surface"),
4256
- operator: request.operator,
4257
- parents: immutableProposalSnapshot(request.parents.map(({ nodeId, artifact }) => ({
4258
- nodeId,
4259
- artifact: immutableProposalSnapshot(artifact, "parent surface")
4260
- })), "parents"),
4261
- train: searchProposerView(state),
4262
- summary: renderSearchSummary(state, { split: "train" }),
4263
- history: proposalHistory,
4264
- findings: immutableProposalSnapshot(assertProposalFindings(findings, "runOptimization proposal findings"), "findings"),
4265
- populationSize: opts.populationSize,
4266
- generation,
4267
- signal: request.signal,
4268
- baselineOutcome: immutableProposalSnapshot(nodes.outcome(state, root), "baseline"),
4269
- incumbentOutcome: immutableProposalSnapshot(nodes.outcome(state, request.leader), "incumbent outcome"),
4270
- parentOutcome: immutableProposalSnapshot(nodes.outcome(state, parent.nodeId), "parent outcome"),
4271
- maxImprovementShots: opts.maxImprovementShots,
4272
- paretoParents: immutableProposalSnapshot(nodes.frontier(state), "Pareto parents"),
4273
- costLedger,
4274
- costPhase: "search.proposal"
4275
- });
4276
- if (proposer.decide?.({ history: proposalHistory }).stop) return {
4277
- children: [],
4278
- stop: "the proposer decided to stop",
4279
- ...accounting()
4280
- };
4281
- const proposed = await proposer.propose(context);
4282
- if (!Array.isArray(proposed)) throw new TypeError("runOptimization: proposer must return an array");
4283
- return {
4284
- children: immutableProposalSnapshot(proposed, "candidate outputs").slice(0, opts.populationSize).map((candidate) => isProposedCandidate(candidate) ? {
4285
- artifact: candidate.surface,
4286
- label: candidate.label,
4287
- rationale: candidate.rationale,
4288
- ...candidate.attribution ? { attribution: candidate.attribution } : {}
4289
- } : {
4290
- artifact: candidate,
4291
- label: "",
4292
- rationale: ""
4293
- }),
4294
- ...accounting()
4295
- };
4296
- }
4297
- };
4298
- const result = await runSearch({
4299
- recorder,
4300
- root: baselineSurface,
4301
- codec: surfaceCodec,
4302
- policy,
4303
- allocation,
4304
- proposer: port,
4305
- executor,
4306
- maxExpansions: opts.maxGenerations,
4307
- signal: opts.signal
4308
- });
4309
- const state = result.state;
4310
- const winnerSurface = nodes.surface(state, result.leader);
4311
- const firstEdge = state.edge(state.node(result.leader).edgeIds[0]);
4312
- const winnerRationale = firstEdge.operator === "seed" || !("sha256" in firstEdge.rationale) ? void 0 : recorder.readBlob(firstEdge.rationale).text;
4313
- const expansions = nodes.generations(state);
4314
- return {
4315
- generations: nodes.history(state, policy).map((record) => ({
4316
- record,
4317
- surfaces: nodes.expansionNodes(state, expansions[record.generationIndex]).map((nodeId) => {
4318
- const surface = nodes.surface(state, nodeId);
4319
- return {
4320
- surfaceHash: surfaceHash(surface),
4321
- surface,
4322
- campaign: nodes.campaign(state, nodeId)
4323
- };
4324
- })
4325
- })),
4326
- baselineSurface,
4327
- winnerSurface,
4328
- winnerSurfaceHash: surfaceHash(winnerSurface),
4329
- ...firstEdge.operator !== "seed" && firstEdge.label ? { winnerLabel: firstEdge.label } : {},
4330
- ...winnerRationale ? { winnerRationale } : {},
4331
- baselineCampaign: nodes.campaign(state, state.rootNodeId),
4332
- cost: costLedger.summary(),
4333
- searchHistory: recorder.receipt({
4334
- producerId: proposer.kind,
4335
- runId: runDir
4336
- }),
4337
- paretoFrontier: nodes.frontier(state)
4338
- };
4339
- }
4069
+ //#region src/campaign/presets/optimization-surface-nodes.ts
4070
+ /** Surface-node campaigns, cached cell adoption, and proposer history from the search ledger. */
4340
4071
  /** A mutable surface as a search artifact. */
4341
4072
  const surfaceCodec = {
4342
4073
  node: surfaceNode,
@@ -4364,14 +4095,14 @@ var SurfaceNodes = class {
4364
4095
  proposals = /* @__PURE__ */ new Map();
4365
4096
  /** Cells this process ran or adopted, by node, keyed `scenarioId:rep`. */
4366
4097
  cells = /* @__PURE__ */ new Map();
4367
- constructor(input) {
4368
- this.opts = input.opts;
4369
- this.recorder = input.recorder;
4370
- this.runDir = input.runDir;
4371
- this.costLedger = input.costLedger;
4372
- this.execution = input.execution;
4373
- this.premeasured = input.premeasured;
4374
- this.storage = input.opts.storage ?? fsCampaignStorage();
4098
+ constructor(opts, recorder, runDir, costLedger, execution, premeasured) {
4099
+ this.opts = opts;
4100
+ this.recorder = recorder;
4101
+ this.runDir = runDir;
4102
+ this.costLedger = costLedger;
4103
+ this.execution = execution;
4104
+ this.premeasured = premeasured;
4105
+ this.storage = opts.storage ?? fsCampaignStorage();
4375
4106
  }
4376
4107
  /** `<runDir>/baseline` for the root, `<runDir>/candidates/<surfaceHash>`
4377
4108
  * for every other node: content-addressed, so a cell's directory needs no
@@ -4406,7 +4137,10 @@ var SurfaceNodes = class {
4406
4137
  const cell = cached === void 0 ? void 0 : JSON.parse(cached);
4407
4138
  if (cell?.manifestHash !== this.manifestFor(work.artifact)) return null;
4408
4139
  this.remember(work.nodeId, cell);
4409
- return this.cellResult(cell, work);
4140
+ return campaignCellSearchResult(cell, {
4141
+ execution: this.execution,
4142
+ lane: work.lane
4143
+ });
4410
4144
  }
4411
4145
  /** Run one cell as a one-cell campaign in its node's directory; a cached
4412
4146
  * cell is read back, not dispatched again. */
@@ -4421,7 +4155,10 @@ var SurfaceNodes = class {
4421
4155
  })).cells[0];
4422
4156
  if (!cell) throw new Error(`runOptimization: cell ${work.cellId} produced no campaign cell`);
4423
4157
  this.remember(work.nodeId, cell);
4424
- return this.cellResult(cell, work);
4158
+ return campaignCellSearchResult(cell, {
4159
+ execution: this.execution,
4160
+ lane: work.lane
4161
+ });
4425
4162
  }
4426
4163
  /** The node's campaign over the cells the ledger settled: the cells this
4427
4164
  * process ran, else their cached results or failure receipts. */
@@ -4589,7 +4326,10 @@ var SurfaceNodes = class {
4589
4326
  premeasuredCell(work) {
4590
4327
  const cell = this.premeasured.cells.find((candidate) => candidate.scenarioId === work.taskId && candidate.rep === work.rep);
4591
4328
  if (!cell) throw new Error(`runOptimization: premeasured baseline lacks ${work.taskId}:${work.rep}`);
4592
- return this.cellResult(cell, work);
4329
+ return campaignCellSearchResult(cell, {
4330
+ execution: this.execution,
4331
+ lane: work.lane
4332
+ });
4593
4333
  }
4594
4334
  /** A cell's result as its campaign stored it for this surface: the cache of
4595
4335
  * a scored cell, the final failure receipt of a failed one. A result another
@@ -4606,34 +4346,334 @@ var SurfaceNodes = class {
4606
4346
  const cell = result !== void 0 ? JSON.parse(result) : receipt !== void 0 ? JSON.parse(receipt).cell : void 0;
4607
4347
  return cell?.manifestHash === manifestHash ? cell : void 0;
4608
4348
  }
4609
- const cached = this.storage.read(cellCachePath(dir, cellId));
4610
- const scored = cached === void 0 ? void 0 : JSON.parse(cached);
4611
- if (scored?.manifestHash === manifestHash) return scored;
4612
- const receipt = this.storage.read(join(cellDirectory(dir, cellId), "failure-receipt.json"));
4613
- const failed = receipt === void 0 ? void 0 : JSON.parse(receipt).cell;
4614
- return failed?.manifestHash === manifestHash ? failed : void 0;
4615
- }
4616
- manifestFor(surface) {
4617
- return computeManifestHash({
4618
- scenarios: this.opts.scenarios,
4619
- judges: this.opts.judges ?? [],
4620
- dispatchRef: surfaceDispatchRef(surface, this.opts.dispatchRef),
4621
- seed: this.opts.seed ?? 42,
4622
- reps: this.opts.reps ?? 1
4623
- });
4624
- }
4625
- remember(nodeId, cell) {
4626
- const cells = this.cells.get(nodeId) ?? /* @__PURE__ */ new Map();
4627
- cells.set(`${cell.scenarioId}:${cell.rep}`, cell);
4628
- this.cells.set(nodeId, cells);
4629
- }
4630
- cellResult(cell, work) {
4631
- return campaignCellSearchResult(cell, {
4632
- execution: this.execution,
4633
- lane: work.lane
4634
- });
4635
- }
4636
- };
4349
+ const cached = this.storage.read(cellCachePath(dir, cellId));
4350
+ const scored = cached === void 0 ? void 0 : JSON.parse(cached);
4351
+ if (scored?.manifestHash === manifestHash) return scored;
4352
+ const receipt = this.storage.read(join(cellDirectory(dir, cellId), "failure-receipt.json"));
4353
+ const failed = receipt === void 0 ? void 0 : JSON.parse(receipt).cell;
4354
+ return failed?.manifestHash === manifestHash ? failed : void 0;
4355
+ }
4356
+ manifestFor(surface) {
4357
+ return computeManifestHash({
4358
+ scenarios: this.opts.scenarios,
4359
+ judges: this.opts.judges ?? [],
4360
+ dispatchRef: surfaceDispatchRef(surface, this.opts.dispatchRef),
4361
+ seed: this.opts.seed ?? 42,
4362
+ reps: this.opts.reps ?? 1
4363
+ });
4364
+ }
4365
+ remember(nodeId, cell) {
4366
+ const cells = this.cells.get(nodeId) ?? /* @__PURE__ */ new Map();
4367
+ cells.set(`${cell.scenarioId}:${cell.rep}`, cell);
4368
+ this.cells.set(nodeId, cells);
4369
+ }
4370
+ };
4371
+ /** A scored campaign as a frontier member: per-scenario composite objectives. */
4372
+ function toParetoParent(surface, campaign, generation, label) {
4373
+ const objectives = {};
4374
+ for (const { scenarioId, composite } of campaignBreakdown(campaign).scenarios) objectives[scenarioId] = composite;
4375
+ const parent = {
4376
+ surface,
4377
+ surfaceHash: surfaceHash(surface),
4378
+ objectives,
4379
+ composite: campaignMeanComposite(campaign),
4380
+ generation
4381
+ };
4382
+ if (label) parent.label = label;
4383
+ return parent;
4384
+ }
4385
+ /** The non-dominated set over the per-scenario objective vectors. */
4386
+ function computeParetoFrontier(scored) {
4387
+ if (scored.length <= 1) return [...scored];
4388
+ const ids = /* @__PURE__ */ new Set();
4389
+ for (const p of scored) for (const id of Object.keys(p.objectives)) ids.add(id);
4390
+ if (ids.size === 0) return [...scored];
4391
+ const objectives = [...ids].map((id) => ({
4392
+ name: id,
4393
+ direction: "maximize",
4394
+ value: (p) => p.objectives[id] ?? Number.NEGATIVE_INFINITY
4395
+ }));
4396
+ return paretoFrontier(scored, objectives).frontier;
4397
+ }
4398
+ function toScoredSurfaceOutcome(hash, campaign, coverage, generation) {
4399
+ const breakdown = campaignBreakdown(campaign);
4400
+ return {
4401
+ split: "search",
4402
+ generation,
4403
+ surfaceHash: hash,
4404
+ composite: campaignMeanComposite(campaign),
4405
+ dimensions: breakdown.dimensions,
4406
+ scenarios: breakdown.scenarios,
4407
+ coverage: {
4408
+ expectedCells: coverage.expectedCellIds.length,
4409
+ scorableCells: coverage.scorableCellIds.length
4410
+ }
4411
+ };
4412
+ }
4413
+ //#endregion
4414
+ //#region src/campaign/presets/run-optimization.ts
4415
+ /**
4416
+ * `runOptimization` improves a mutable surface with a caller-owned proposer,
4417
+ * as a search on the one kernel (`runSearch`).
4418
+ *
4419
+ * The baseline is the seeded root. The scenarios are the train split: the
4420
+ * proposer reads their scores, and the policy ranks on them because the loop
4421
+ * has no private selection split. A "generation" is one proposal of
4422
+ * `populationSize` candidates from the node the policy chose (by default the
4423
+ * incumbent, once every earlier candidate is measured). Every candidate is a
4424
+ * node measured on every scenario at every repeat, one cell at a time on the
4425
+ * run's lanes. The kept surface is the policy's leader: a budget decision on
4426
+ * the train split that claims nothing; `runImprovementLoop` compares it with
4427
+ * the baseline on separate cases.
4428
+ *
4429
+ * The search ledger is always written, beside the run (`<runDir>/search/`) or
4430
+ * where `searchLedger` puts it, and it is the run's only checkpoint: running
4431
+ * again on the same run directory and options continues an interrupted run
4432
+ * instead of starting over, and reruns nothing that settled.
4433
+ */
4434
+ /** Compose the optimization proposer and cell executor on the shared search kernel. */
4435
+ /** Improve a surface as a search on the kernel; see the module comment. */
4436
+ async function runOptimization(opts) {
4437
+ const candidateConcurrency = opts.candidateConcurrency ?? 1;
4438
+ if (typeof opts.runDir !== "string" || opts.runDir.trim().length === 0) throw new Error("runOptimization: runDir is required and must be a non-empty string");
4439
+ if (!Number.isInteger(candidateConcurrency) || candidateConcurrency < 1) throw new Error("runOptimization: candidateConcurrency must be a positive integer");
4440
+ for (const [name, value] of [["populationSize", opts.populationSize], ["maxGenerations", opts.maxGenerations]]) if (!Number.isSafeInteger(value) || value < 0) throw new Error(`runOptimization: ${name} must be a non-negative integer`);
4441
+ const initialFindings = immutableProposalSnapshot(assertProposalFindings(opts.findings ?? [], "runOptimization initial proposal findings"), "initial findings");
4442
+ const baselineSurface = immutableProposalSnapshot(opts.baselineSurface, "baseline surface");
4443
+ const runDir = resolveRunDir(opts.runDir, opts.repo);
4444
+ const storage = opts.storage ?? fsCampaignStorage();
4445
+ const costLedger = opts.costLedger ?? createRunCostLedger({
4446
+ storage,
4447
+ runDir,
4448
+ costCeilingUsd: opts.costCeiling
4449
+ });
4450
+ const reps = opts.reps ?? 1;
4451
+ const seed = opts.seed ?? 42;
4452
+ const requireJudgeScore = (opts.judges?.length ?? 0) > 0;
4453
+ const premeasured = opts.premeasuredBaseline ? validatedPremeasuredBaseline({
4454
+ input: opts.premeasuredBaseline,
4455
+ baselineSurface,
4456
+ scenarios: opts.scenarios,
4457
+ reps,
4458
+ seed,
4459
+ judges: opts.judges ?? [],
4460
+ dispatchRef: surfaceDispatchRef(baselineSurface, opts.dispatchRef),
4461
+ requireJudgeScore
4462
+ }) : void 0;
4463
+ const splitDigest = campaignSplitDigest(opts.scenarios, reps);
4464
+ const binding = opts.searchLedger ?? await defaultBinding(opts, runDir, storage);
4465
+ const { identity } = binding;
4466
+ const execution = {
4467
+ model: identity.model,
4468
+ agent: identity.agent,
4469
+ benchmark: {
4470
+ uri: "campaign://scenarios",
4471
+ revision: splitDigest
4472
+ }
4473
+ };
4474
+ const policy = opts.policy ?? incumbent();
4475
+ const allocation = uniform({ reps });
4476
+ const recorder = await SearchRecorder.open({
4477
+ ledger: binding.ledger,
4478
+ storage
4479
+ }, {
4480
+ subject: identity.subject ?? opts.proposer.kind,
4481
+ process: {
4482
+ name: "runOptimization",
4483
+ executionRef: identity.search
4484
+ },
4485
+ artifactKind: typeof baselineSurface === "object" && baselineSurface.kind === "code" ? "code" : "prompt",
4486
+ objective: {
4487
+ metric: "composite",
4488
+ direction: "maximize",
4489
+ judge: identity.judge ?? { unknown: "runOptimization receives its judges as functions without a pinned source" },
4490
+ claim: identity.claim ?? developmentClaim(splitDigest)
4491
+ },
4492
+ splits: {
4493
+ train: opts.scenarios.map((scenario) => ({
4494
+ taskId: scenario.id,
4495
+ unitId: scenario.id,
4496
+ source: {
4497
+ uri: `scenario://${scenario.id}`,
4498
+ revision: hashCanonical(scenario)
4499
+ }
4500
+ })),
4501
+ selection: [],
4502
+ test: [],
4503
+ heldOutUnits: true
4504
+ },
4505
+ policy: {
4506
+ expansion: policy.name,
4507
+ allocation: allocation.name,
4508
+ seed
4509
+ },
4510
+ budget: {
4511
+ maxUsd: opts.costCeiling ?? null,
4512
+ maxCells: null,
4513
+ maxNodes: 1 + opts.populationSize * opts.maxGenerations,
4514
+ deadline: null,
4515
+ maxConcurrency: null,
4516
+ reservedClaimUsd: 0
4517
+ },
4518
+ containment: null,
4519
+ derivedFrom: null,
4520
+ identity: execution
4521
+ });
4522
+ const nodes = new SurfaceNodes(opts, recorder, runDir, costLedger, execution, premeasured);
4523
+ const scenarioById = new Map(opts.scenarios.map((scenario) => [scenario.id, scenario]));
4524
+ let findings = initialFindings;
4525
+ const executor = {
4526
+ lanes: () => [{
4527
+ name: "campaign",
4528
+ capacity: candidateConcurrency * (opts.maxConcurrency ?? 2),
4529
+ costCap: "estimate",
4530
+ cellUsd: 0
4531
+ }],
4532
+ place: () => "campaign",
4533
+ run: (work) => nodes.runCell(work, scenarioById),
4534
+ adopt: (work) => nodes.adoptCell(work)
4535
+ };
4536
+ const proposer = opts.proposer;
4537
+ const proposerSource = identity.proposer.source;
4538
+ const port = {
4539
+ name: proposer.kind,
4540
+ kind: "optimizer",
4541
+ source: proposerSource,
4542
+ execution: identity.proposer,
4543
+ childrenPerProposal: opts.populationSize,
4544
+ async propose(request) {
4545
+ const phases = [
4546
+ "search.proposal",
4547
+ "analysis.baseline",
4548
+ "analysis.generation"
4549
+ ];
4550
+ const before = new Set(phases.flatMap((phase) => costLedger.list({ phase })).map((receipt) => receipt.callId));
4551
+ const accounting = () => {
4552
+ const fresh = phases.flatMap((phase) => costLedger.list({ phase })).filter((receipt) => !before.has(receipt.callId));
4553
+ return {
4554
+ accounting: searchReceiptAccounting(fresh),
4555
+ execution: opts.searchLedger ? identity.proposer : searchProposalExecution(fresh, identity.model.provider, proposerSource)
4556
+ };
4557
+ };
4558
+ const earlier = await recorder.state();
4559
+ const root = earlier.rootNodeId;
4560
+ const rootCampaign = nodes.campaign(earlier, root);
4561
+ const rootCoverage = campaignCoverage(rootCampaign.cells, opts.scenarios, reps, requireJudgeScore);
4562
+ if (!rootCoverage.complete) throw new Error(`runOptimization: ${premeasured ? "premeasured baseline" : "baseline"} is incomplete (${rootCoverage.scorableCellIds.length}/${rootCoverage.expectedCellIds.length} designed cells scorable) — ${formatCoverageFailures(rootCoverage)}. Refusing to optimize against an incomplete incumbent.`);
4563
+ const generation = nodes.generations(earlier).length;
4564
+ if (opts.analyzeGeneration) {
4565
+ const previous = generation - 1;
4566
+ const candidates = (previous < 0 ? [root] : nodes.expansionNodes(earlier, nodes.generations(earlier)[previous])).map((nodeId) => {
4567
+ const campaign = nodes.campaign(earlier, nodeId);
4568
+ return {
4569
+ surfaceHash: surfaceHash(nodes.surface(earlier, nodeId)),
4570
+ campaign,
4571
+ composite: campaignMeanCompositeOrNull(campaign)
4572
+ };
4573
+ });
4574
+ const history = nodes.history(earlier, policy);
4575
+ if (candidates.some(({ campaign }) => campaign.cells.length > 0)) {
4576
+ const fresh = await opts.analyzeGeneration({
4577
+ generation: previous,
4578
+ runDir: previous < 0 ? rootCampaign.runDir : `${runDir}/candidates`,
4579
+ candidates,
4580
+ history,
4581
+ costLedger,
4582
+ costPhase: previous < 0 ? "analysis.baseline" : "analysis.generation"
4583
+ });
4584
+ if (!Array.isArray(fresh)) throw new TypeError("runOptimization: analyzeGeneration must return an array");
4585
+ findings = immutableProposalSnapshot(assertProposalFindings(fresh, "runOptimization analysis findings"), "analysis findings");
4586
+ }
4587
+ }
4588
+ const state = await recorder.state();
4589
+ const proposalHistory = immutableProposalSnapshot(nodes.history(state, policy), "history");
4590
+ const parent = request.parents[0];
4591
+ const context = Object.freeze({
4592
+ currentSurface: immutableProposalSnapshot(parent.artifact, "current surface"),
4593
+ operator: request.operator,
4594
+ parents: immutableProposalSnapshot(request.parents.map(({ nodeId, artifact }) => ({
4595
+ nodeId,
4596
+ artifact: immutableProposalSnapshot(artifact, "parent surface")
4597
+ })), "parents"),
4598
+ train: searchProposerView(state),
4599
+ summary: renderSearchSummary(state, { split: "train" }),
4600
+ history: proposalHistory,
4601
+ findings: immutableProposalSnapshot(assertProposalFindings(findings, "runOptimization proposal findings"), "findings"),
4602
+ populationSize: opts.populationSize,
4603
+ generation,
4604
+ signal: request.signal,
4605
+ baselineOutcome: immutableProposalSnapshot(nodes.outcome(state, root), "baseline"),
4606
+ incumbentOutcome: immutableProposalSnapshot(nodes.outcome(state, request.leader), "incumbent outcome"),
4607
+ parentOutcome: immutableProposalSnapshot(nodes.outcome(state, parent.nodeId), "parent outcome"),
4608
+ maxImprovementShots: opts.maxImprovementShots,
4609
+ paretoParents: immutableProposalSnapshot(nodes.frontier(state), "Pareto parents"),
4610
+ costLedger,
4611
+ costPhase: "search.proposal"
4612
+ });
4613
+ if (proposer.decide?.({ history: proposalHistory }).stop) return {
4614
+ children: [],
4615
+ stop: "the proposer decided to stop",
4616
+ ...accounting()
4617
+ };
4618
+ const proposed = await proposer.propose(context);
4619
+ if (!Array.isArray(proposed)) throw new TypeError("runOptimization: proposer must return an array");
4620
+ return {
4621
+ children: immutableProposalSnapshot(proposed, "candidate outputs").slice(0, opts.populationSize).map((candidate) => isProposedCandidate(candidate) ? {
4622
+ artifact: candidate.surface,
4623
+ label: candidate.label,
4624
+ rationale: candidate.rationale,
4625
+ ...candidate.attribution ? { attribution: candidate.attribution } : {}
4626
+ } : {
4627
+ artifact: candidate,
4628
+ label: "",
4629
+ rationale: ""
4630
+ }),
4631
+ ...accounting()
4632
+ };
4633
+ }
4634
+ };
4635
+ const result = await runSearch({
4636
+ recorder,
4637
+ root: baselineSurface,
4638
+ codec: surfaceCodec,
4639
+ policy,
4640
+ allocation,
4641
+ proposer: port,
4642
+ executor,
4643
+ maxExpansions: opts.maxGenerations,
4644
+ signal: opts.signal
4645
+ });
4646
+ const state = result.state;
4647
+ const winnerSurface = nodes.surface(state, result.leader);
4648
+ const firstEdge = state.edge(state.node(result.leader).edgeIds[0]);
4649
+ const winnerRationale = firstEdge.operator === "seed" || !("sha256" in firstEdge.rationale) ? void 0 : recorder.readBlob(firstEdge.rationale).text;
4650
+ const expansions = nodes.generations(state);
4651
+ return {
4652
+ generations: nodes.history(state, policy).map((record) => ({
4653
+ record,
4654
+ surfaces: nodes.expansionNodes(state, expansions[record.generationIndex]).map((nodeId) => {
4655
+ const surface = nodes.surface(state, nodeId);
4656
+ return {
4657
+ surfaceHash: surfaceHash(surface),
4658
+ surface,
4659
+ campaign: nodes.campaign(state, nodeId)
4660
+ };
4661
+ })
4662
+ })),
4663
+ baselineSurface,
4664
+ winnerSurface,
4665
+ winnerSurfaceHash: surfaceHash(winnerSurface),
4666
+ ...firstEdge.operator !== "seed" && firstEdge.label ? { winnerLabel: firstEdge.label } : {},
4667
+ ...winnerRationale ? { winnerRationale } : {},
4668
+ baselineCampaign: nodes.campaign(state, state.rootNodeId),
4669
+ cost: costLedger.summary(),
4670
+ searchHistory: recorder.receipt({
4671
+ producerId: proposer.kind,
4672
+ runId: runDir
4673
+ }),
4674
+ paretoFrontier: nodes.frontier(state)
4675
+ };
4676
+ }
4637
4677
  /** The default ledger and identities when the caller binds none. Revisions are
4638
4678
  * content digests of what the caller declared, and each uri names what its
4639
4679
  * digest covers: the dispatch ref, the proposer kind, the kernel. */
@@ -4676,12 +4716,6 @@ async function defaultBinding(opts, runDir, storage) {
4676
4716
  };
4677
4717
  }
4678
4718
  }
4679
- function artifactKindOf(surface) {
4680
- return typeof surface === "object" && surface.kind === "code" ? "code" : "prompt";
4681
- }
4682
- function receiptKey(receipt) {
4683
- return receipt.callId;
4684
- }
4685
4719
  function immutableProposalSnapshot(value, label) {
4686
4720
  try {
4687
4721
  return deepFreeze(structuredClone(value));
@@ -4719,48 +4753,6 @@ function validatedPremeasuredBaseline(args) {
4719
4753
  if (!coverage.complete) throw new Error(`runOptimization: premeasured baseline is incomplete (${coverage.scorableCellIds.length}/${coverage.expectedCellIds.length} designed cells scorable) — ${formatCoverageFailures(coverage)}. Refusing to optimize against an incomplete incumbent.`);
4720
4754
  return campaign;
4721
4755
  }
4722
- /** A scored campaign as a frontier member: per-scenario composite objectives. */
4723
- function toParetoParent(surface, campaign, generation, label) {
4724
- const objectives = {};
4725
- for (const { scenarioId, composite } of campaignBreakdown(campaign).scenarios) objectives[scenarioId] = composite;
4726
- const parent = {
4727
- surface,
4728
- surfaceHash: surfaceHash(surface),
4729
- objectives,
4730
- composite: campaignMeanComposite(campaign),
4731
- generation
4732
- };
4733
- if (label) parent.label = label;
4734
- return parent;
4735
- }
4736
- /** The non-dominated set over the per-scenario objective vectors. */
4737
- function computeParetoFrontier(scored) {
4738
- if (scored.length <= 1) return [...scored];
4739
- const ids = /* @__PURE__ */ new Set();
4740
- for (const p of scored) for (const id of Object.keys(p.objectives)) ids.add(id);
4741
- if (ids.size === 0) return [...scored];
4742
- const objectives = [...ids].map((id) => ({
4743
- name: id,
4744
- direction: "maximize",
4745
- value: (p) => p.objectives[id] ?? Number.NEGATIVE_INFINITY
4746
- }));
4747
- return paretoFrontier(scored, objectives).frontier;
4748
- }
4749
- function toScoredSurfaceOutcome(hash, campaign, coverage, generation) {
4750
- const breakdown = campaignBreakdown(campaign);
4751
- return {
4752
- split: "search",
4753
- generation,
4754
- surfaceHash: hash,
4755
- composite: campaignMeanComposite(campaign),
4756
- dimensions: breakdown.dimensions,
4757
- scenarios: breakdown.scenarios,
4758
- coverage: {
4759
- expectedCells: coverage.expectedCellIds.length,
4760
- scorableCells: coverage.scorableCellIds.length
4761
- }
4762
- };
4763
- }
4764
4756
  //#endregion
4765
4757
  //#region src/campaign/presets/run-improvement-loop.ts
4766
4758
  /**
@@ -4830,10 +4822,126 @@ async function runImprovementLoop(opts) {
4830
4822
  ghRepo: opts.ghRepo
4831
4823
  });
4832
4824
  return {
4833
- ...optimization,
4834
- ...comparison,
4835
- prResult,
4836
- cost: costLedger.summary()
4825
+ ...optimization,
4826
+ ...comparison,
4827
+ prResult,
4828
+ cost: costLedger.summary()
4829
+ };
4830
+ }
4831
+ //#endregion
4832
+ //#region src/campaign/optimization-cost.ts
4833
+ /** Attribute method calls while retaining the shared account's admission and read behavior. */
4834
+ function createMethodCostScope(account, methodName) {
4835
+ const tags = { [`optimizationAttempt:${crypto.randomUUID()}`]: methodName };
4836
+ return {
4837
+ ledger: Object.freeze({
4838
+ costCeilingUsd: account.costCeilingUsd,
4839
+ runPaidCall: (input) => account.runPaidCall({
4840
+ ...input,
4841
+ tags: {
4842
+ ...input.tags,
4843
+ ...tags
4844
+ }
4845
+ }),
4846
+ summary: account.summary.bind(account),
4847
+ list: account.list.bind(account),
4848
+ reconcile: account.reconcile.bind(account),
4849
+ markCompleted: account.markCompleted.bind(account),
4850
+ costPerCompletedTask: account.costPerCompletedTask.bind(account),
4851
+ ...account.listPending ? { listPending: account.listPending.bind(account) } : {},
4852
+ ...account.waitForIdle ? { waitForIdle: account.waitForIdle.bind(account) } : {}
4853
+ }),
4854
+ reconcile(reported) {
4855
+ const summary = account.summary({ tags });
4856
+ if (summary.pendingCalls > 0) throw new Error(`optimization method '${methodName}' returned with ${summary.pendingCalls} pending paid call(s)`);
4857
+ const recorded = costFromLedgerSummary(summary);
4858
+ const totalCostUsd = Math.max(reported.totalCostUsd, recorded.totalCostUsd);
4859
+ const roundingToleranceUsd = Number.EPSILON * Math.max(1, totalCostUsd) * (summary.totalCalls + 1);
4860
+ const combined = combineComparisonCosts([{
4861
+ label: "reported",
4862
+ cost: reported
4863
+ }, {
4864
+ label: "recorded",
4865
+ cost: recorded
4866
+ }]);
4867
+ const incompleteReasons = [
4868
+ ...reported.incompleteReasons,
4869
+ ...recorded.incompleteReasons.map((reason) => `recorded: ${reason}`),
4870
+ ...recorded.totalCostUsd - reported.totalCostUsd > roundingToleranceUsd ? [`reported ${reported.totalCostUsd} USD below recorded ${recorded.totalCostUsd} USD`] : []
4871
+ ];
4872
+ return {
4873
+ totalCostUsd,
4874
+ costProvenance: combined.costProvenance.kind === "uncaptured" ? combined.costProvenance : {
4875
+ kind: combined.costProvenance.kind,
4876
+ usd: totalCostUsd
4877
+ },
4878
+ accountingComplete: incompleteReasons.length === 0,
4879
+ incompleteReasons
4880
+ };
4881
+ }
4882
+ };
4883
+ }
4884
+ /** Keep the cost fields a custom optimization method must report. */
4885
+ function costFromLedgerSummary(summary) {
4886
+ const cost = {
4887
+ totalCostUsd: summary.totalCostUsd,
4888
+ costProvenance: structuredClone(summary.costProvenance),
4889
+ accountingComplete: summary.accountingComplete,
4890
+ incompleteReasons: [...summary.incompleteReasons]
4891
+ };
4892
+ assertComparisonCost(cost, "cost ledger");
4893
+ return cost;
4894
+ }
4895
+ /** Combine method costs without turning one unknown bill into a known total. */
4896
+ function combineComparisonCosts(entries) {
4897
+ const totalCostUsd = entries.reduce((total, entry) => total + entry.cost.totalCostUsd, 0);
4898
+ const cost = {
4899
+ totalCostUsd,
4900
+ costProvenance: entries.some((entry) => entry.cost.costProvenance.kind === "uncaptured") ? {
4901
+ kind: "uncaptured",
4902
+ usd: null
4903
+ } : entries.every((entry) => entry.cost.costProvenance.kind === "observed") ? {
4904
+ kind: "observed",
4905
+ usd: totalCostUsd
4906
+ } : {
4907
+ kind: "estimated",
4908
+ usd: totalCostUsd
4909
+ },
4910
+ accountingComplete: entries.every((entry) => entry.cost.accountingComplete),
4911
+ incompleteReasons: entries.flatMap((entry) => entry.cost.incompleteReasons.map((reason) => `${entry.label}: ${reason}`))
4912
+ };
4913
+ assertComparisonCost(cost, "combined cost");
4914
+ return cost;
4915
+ }
4916
+ function assertComparisonCost(cost, label) {
4917
+ if (!cost || typeof cost !== "object") throw new Error(`compareOptimizationMethods: ${label} returned no cost`);
4918
+ if (!Number.isFinite(cost.totalCostUsd) || cost.totalCostUsd < 0) throw new Error(`compareOptimizationMethods: ${label} returned an invalid totalCostUsd`);
4919
+ const provenance = cost.costProvenance;
4920
+ if (!provenance || typeof provenance !== "object" || provenance.kind !== "observed" && provenance.kind !== "estimated" && provenance.kind !== "uncaptured" || (provenance.kind === "uncaptured" ? provenance.usd !== null : !Number.isFinite(provenance.usd) || provenance.usd < 0)) throw new Error(`compareOptimizationMethods: ${label} returned invalid costProvenance`);
4921
+ if (provenance.kind !== "uncaptured" && provenance.usd !== cost.totalCostUsd) throw new Error(`compareOptimizationMethods: ${label} returned costProvenance inconsistent with totalCostUsd`);
4922
+ if (typeof cost.accountingComplete !== "boolean") throw new Error(`compareOptimizationMethods: ${label} returned invalid accountingComplete`);
4923
+ if (!Array.isArray(cost.incompleteReasons) || cost.incompleteReasons.some((reason) => typeof reason !== "string" || reason.trim().length === 0)) throw new Error(`compareOptimizationMethods: ${label} returned invalid incompleteReasons`);
4924
+ if (cost.accountingComplete !== (cost.incompleteReasons.length === 0)) throw new Error(`compareOptimizationMethods: ${label} returned inconsistent cost completeness and reasons`);
4925
+ if (cost.accountingComplete && provenance.kind === "uncaptured") throw new Error(`compareOptimizationMethods: ${label} cannot mark uncaptured cost as complete`);
4926
+ }
4927
+ /** Preserve every optimizer token class while keeping total input and output explicit. */
4928
+ function optimizationTokenUsageFromSummary(summary, receipts) {
4929
+ if (!summary.usageComplete) return void 0;
4930
+ if (receipts.length !== summary.totalCalls) throw new Error("optimization token usage receipt count does not match the cost summary");
4931
+ const cachedInputTokens = summary.cachedTokens;
4932
+ const cacheWriteInputTokens = summary.cacheWriteTokens ?? 0;
4933
+ const inputTokens = summary.inputTokens + cachedInputTokens + cacheWriteInputTokens;
4934
+ const cacheReadComplete = receipts.length === 0 || receipts.every((receipt) => receipt.cachedTokens !== void 0);
4935
+ const cacheWriteComplete = receipts.length === 0 || receipts.every((receipt) => receipt.cacheWriteTokens !== void 0);
4936
+ const reasoningComplete = receipts.length === 0 || receipts.every((receipt) => receipt.reasoningTokens !== void 0);
4937
+ return {
4938
+ inputTokens,
4939
+ ...cacheReadComplete ? { cachedInputTokens } : {},
4940
+ ...cacheWriteComplete ? { cacheWriteInputTokens } : {},
4941
+ outputTokens: summary.outputTokens,
4942
+ ...reasoningComplete ? { reasoningTokens: summary.reasoningTokens ?? 0 } : {},
4943
+ totalTokens: inputTokens + summary.outputTokens,
4944
+ calls: summary.totalCalls
4837
4945
  };
4838
4946
  }
4839
4947
  //#endregion
@@ -5022,102 +5130,6 @@ function assertPositiveSafeInteger(value, label) {
5022
5130
  if (!Number.isSafeInteger(value) || value <= 0) throw new Error(`GEPA candidate population ${label} must be a positive safe integer`);
5023
5131
  }
5024
5132
  //#endregion
5025
- //#region src/campaign/optimization-cost.ts
5026
- /** Attribute method calls while retaining the shared account's admission and read behavior. */
5027
- function createMethodCostScope(account, methodName) {
5028
- const tags = { [`optimizationAttempt:${crypto.randomUUID()}`]: methodName };
5029
- return {
5030
- ledger: Object.freeze({
5031
- costCeilingUsd: account.costCeilingUsd,
5032
- runPaidCall: (input) => account.runPaidCall({
5033
- ...input,
5034
- tags: {
5035
- ...input.tags,
5036
- ...tags
5037
- }
5038
- }),
5039
- summary: account.summary.bind(account),
5040
- list: account.list.bind(account),
5041
- reconcile: account.reconcile.bind(account),
5042
- markCompleted: account.markCompleted.bind(account),
5043
- costPerCompletedTask: account.costPerCompletedTask.bind(account),
5044
- ...account.listPending ? { listPending: account.listPending.bind(account) } : {},
5045
- ...account.waitForIdle ? { waitForIdle: account.waitForIdle.bind(account) } : {}
5046
- }),
5047
- reconcile(reported) {
5048
- const summary = account.summary({ tags });
5049
- if (summary.pendingCalls > 0) throw new Error(`optimization method '${methodName}' returned with ${summary.pendingCalls} pending paid call(s)`);
5050
- const recorded = costFromLedgerSummary(summary);
5051
- const totalCostUsd = Math.max(reported.totalCostUsd, recorded.totalCostUsd);
5052
- const roundingToleranceUsd = Number.EPSILON * Math.max(1, totalCostUsd) * (summary.totalCalls + 1);
5053
- const combined = combineComparisonCosts([{
5054
- label: "reported",
5055
- cost: reported
5056
- }, {
5057
- label: "recorded",
5058
- cost: recorded
5059
- }]);
5060
- const incompleteReasons = [
5061
- ...reported.incompleteReasons,
5062
- ...recorded.incompleteReasons.map((reason) => `recorded: ${reason}`),
5063
- ...recorded.totalCostUsd - reported.totalCostUsd > roundingToleranceUsd ? [`reported ${reported.totalCostUsd} USD below recorded ${recorded.totalCostUsd} USD`] : []
5064
- ];
5065
- return {
5066
- totalCostUsd,
5067
- costProvenance: combined.costProvenance.kind === "uncaptured" ? combined.costProvenance : {
5068
- kind: combined.costProvenance.kind,
5069
- usd: totalCostUsd
5070
- },
5071
- accountingComplete: incompleteReasons.length === 0,
5072
- incompleteReasons
5073
- };
5074
- }
5075
- };
5076
- }
5077
- /** Keep the cost fields a custom optimization method must report. */
5078
- function costFromLedgerSummary(summary) {
5079
- const cost = {
5080
- totalCostUsd: summary.totalCostUsd,
5081
- costProvenance: structuredClone(summary.costProvenance),
5082
- accountingComplete: summary.accountingComplete,
5083
- incompleteReasons: [...summary.incompleteReasons]
5084
- };
5085
- assertComparisonCost(cost, "cost ledger");
5086
- return cost;
5087
- }
5088
- /** Combine method costs without turning one unknown bill into a known total. */
5089
- function combineComparisonCosts(entries) {
5090
- const totalCostUsd = entries.reduce((total, entry) => total + entry.cost.totalCostUsd, 0);
5091
- const cost = {
5092
- totalCostUsd,
5093
- costProvenance: entries.some((entry) => entry.cost.costProvenance.kind === "uncaptured") ? {
5094
- kind: "uncaptured",
5095
- usd: null
5096
- } : entries.every((entry) => entry.cost.costProvenance.kind === "observed") ? {
5097
- kind: "observed",
5098
- usd: totalCostUsd
5099
- } : {
5100
- kind: "estimated",
5101
- usd: totalCostUsd
5102
- },
5103
- accountingComplete: entries.every((entry) => entry.cost.accountingComplete),
5104
- incompleteReasons: entries.flatMap((entry) => entry.cost.incompleteReasons.map((reason) => `${entry.label}: ${reason}`))
5105
- };
5106
- assertComparisonCost(cost, "combined cost");
5107
- return cost;
5108
- }
5109
- function assertComparisonCost(cost, label) {
5110
- if (!cost || typeof cost !== "object") throw new Error(`compareOptimizationMethods: ${label} returned no cost`);
5111
- if (!Number.isFinite(cost.totalCostUsd) || cost.totalCostUsd < 0) throw new Error(`compareOptimizationMethods: ${label} returned an invalid totalCostUsd`);
5112
- const provenance = cost.costProvenance;
5113
- if (!provenance || typeof provenance !== "object" || provenance.kind !== "observed" && provenance.kind !== "estimated" && provenance.kind !== "uncaptured" || (provenance.kind === "uncaptured" ? provenance.usd !== null : !Number.isFinite(provenance.usd) || provenance.usd < 0)) throw new Error(`compareOptimizationMethods: ${label} returned invalid costProvenance`);
5114
- if (provenance.kind !== "uncaptured" && provenance.usd !== cost.totalCostUsd) throw new Error(`compareOptimizationMethods: ${label} returned costProvenance inconsistent with totalCostUsd`);
5115
- if (typeof cost.accountingComplete !== "boolean") throw new Error(`compareOptimizationMethods: ${label} returned invalid accountingComplete`);
5116
- if (!Array.isArray(cost.incompleteReasons) || cost.incompleteReasons.some((reason) => typeof reason !== "string" || reason.trim().length === 0)) throw new Error(`compareOptimizationMethods: ${label} returned invalid incompleteReasons`);
5117
- if (cost.accountingComplete !== (cost.incompleteReasons.length === 0)) throw new Error(`compareOptimizationMethods: ${label} returned inconsistent cost completeness and reasons`);
5118
- if (cost.accountingComplete && provenance.kind === "uncaptured") throw new Error(`compareOptimizationMethods: ${label} cannot mark uncaptured cost as complete`);
5119
- }
5120
- //#endregion
5121
5133
  //#region src/campaign/optimization-method.ts
5122
5134
  /** Both complete-method workflows use the same detached inputs, accounting, and evidence admission. */
5123
5135
  async function executeOptimizationMethod(options) {
@@ -5266,440 +5278,6 @@ function assertOptimizationProvenance(methodName, value) {
5266
5278
  if (value.optimizerModel === void 0 !== (value.optimizerCallRef === void 0) || value.optimizerModel === void 0 !== (value.modelExecutions === void 0)) fail("optimizerModel execution provenance");
5267
5279
  }
5268
5280
  //#endregion
5269
- //#region src/campaign/presets/compare-optimization-methods.ts
5270
- /**
5271
- * Compare optimization methods on shared train, selection, and test data.
5272
- * Optimizers receive only train and selection data. After every optimizer
5273
- * finishes, their selected surfaces are measured on the same untouched test
5274
- * data and compared with paired confidence intervals.
5275
- */
5276
- /**
5277
- * Compare complete optimization methods on disjoint train, selection, and final test data.
5278
- */
5279
- async function compareOptimizationMethods(opts) {
5280
- opts = {
5281
- ...opts,
5282
- methods: opts.methods.map((method) => ({ ...method })),
5283
- trainScenarios: structuredClone(opts.trainScenarios),
5284
- selectionScenarios: structuredClone(opts.selectionScenarios),
5285
- testScenarios: structuredClone(opts.testScenarios),
5286
- judges: opts.judges.map(captureJudge),
5287
- claim: opts.claim && defineEvaluationClaim(opts.claim),
5288
- finalEvidence: opts.finalEvidence && captureFinalEvidencePolicy(opts.finalEvidence)
5289
- };
5290
- assertOptimizationMethods(opts.methods);
5291
- assertComparisonPartitions(opts);
5292
- const searchHistoryPolicy = opts.searchHistoryPolicy ?? "allow-missing";
5293
- assertSearchHistoryAdmissionOptions(opts);
5294
- const seed = opts.seed ?? 42;
5295
- const confidence = opts.confidence ?? .95;
5296
- assertConfidence(confidence);
5297
- const optimizationConcurrency = opts.optimizationConcurrency ?? 1;
5298
- const comparisonCount = opts.methods.length * (opts.methods.length + 1) / 2;
5299
- const intervalConfidence = 1 - (1 - confidence) / comparisonCount;
5300
- const minimumResamples = minimumBootstrapResamples(confidence, comparisonCount);
5301
- const resamples = opts.resamples ?? Math.max(2e3, minimumResamples);
5302
- assertComparisonControls(opts, seed, resamples, confidence);
5303
- const evidence = opts.evidence === void 0 ? void 0 : structuredClone(opts.evidence);
5304
- const evidenceBySurface = /* @__PURE__ */ new Map();
5305
- const storage = opts.storage ?? fsCampaignStorage();
5306
- const resolvedRunDir = resolveRunDir(opts.runDir, opts.repo);
5307
- const baselineSurface = structuredClone(opts.baselineSurface);
5308
- const costLedger = opts.costLedger ?? createRunCostLedger({
5309
- storage,
5310
- runDir: `${resolvedRunDir}/cost`,
5311
- costCeilingUsd: opts.costCeiling
5312
- });
5313
- const scoreOnTest = async (surface, tag, costPhase) => {
5314
- const measuredSurface = structuredClone(surface);
5315
- const campaign = await runCampaign({
5316
- ...opts,
5317
- storage,
5318
- costLedger,
5319
- costPhase,
5320
- scenarios: opts.testScenarios.map((scenario) => structuredClone(scenario)),
5321
- dispatch: (scenario, ctx) => opts.dispatchWithSurface(structuredClone(measuredSurface), scenario, ctx),
5322
- dispatchRef: finalDispatchRef(opts, measuredSurface),
5323
- runDir: `${resolvedRunDir}/${tag}`
5324
- });
5325
- assertCompleteCampaign(campaign, opts.testScenarios, opts.reps ?? 1, true, `compareOptimizationMethods: ${tag} final comparison`);
5326
- if (evidence) evidenceBySurface.set(surfaceContentHash(measuredSurface), createCampaignEvidenceReceipt({
5327
- campaign,
5328
- surface: measuredSurface,
5329
- context: evidence
5330
- }));
5331
- const byScenario = {};
5332
- for (const { scenarioId, composite } of campaignBreakdown(campaign).scenarios) byScenario[scenarioId] = composite;
5333
- return byScenario;
5334
- };
5335
- const scenarioIds = opts.testScenarios.map((s) => s.id).sort();
5336
- const unitByScenario = opts.claim ? evaluationUnitMap(opts.claim, opts.testScenarios) : new Map(scenarioIds.map((id) => [id, id]));
5337
- const units = opts.claim ? summarizeEvaluationUnits(opts.claim, opts.testScenarios) : {
5338
- observations: scenarioIds.length,
5339
- independentUnits: scenarioIds.length,
5340
- units: scenarioIds.map((id) => ({
5341
- id,
5342
- observations: 1
5343
- }))
5344
- };
5345
- const aggregateUnits = (arr) => {
5346
- const values = /* @__PURE__ */ new Map();
5347
- scenarioIds.forEach((id, index) => {
5348
- const unitId = unitByScenario.get(id);
5349
- const bucket = values.get(unitId) ?? [];
5350
- bucket.push(arr[index]);
5351
- values.set(unitId, bucket);
5352
- });
5353
- return units.units.map((unit) => mean(values.get(unit.id)));
5354
- };
5355
- const decisionOptions = {
5356
- seed,
5357
- resamples,
5358
- confidence: intervalConfidence,
5359
- threshold: opts.claim?.minimumEffect ?? 0
5360
- };
5361
- const align = (byScenario, label) => {
5362
- const missing = scenarioIds.filter((id) => !(id in byScenario));
5363
- if (missing.length > 0) throw new Error(`compareOptimizationMethods: ${label} produced no test score for scenario(s) [${missing.join(", ")}]. A cell failed or its judges returned nothing. Fix the dispatch or judge; the comparison will not replace missing scores with zero.`);
5364
- return scenarioIds.map((id) => byScenario[id]);
5365
- };
5366
- if (opts.claim?.generalization === "new-units") assertIndependentEvaluationSplit(opts.claim, opts.testScenarios, [...opts.trainScenarios, ...opts.selectionScenarios]);
5367
- if (opts.finalEvidence) await reserveFinalEvidence(opts.finalEvidence, opts.claim, opts.testScenarios, [...opts.trainScenarios, ...opts.selectionScenarios]);
5368
- const optimizationOwner = new AbortController();
5369
- const optimized = await mapConcurrent(opts.methods, optimizationConcurrency, async (method) => {
5370
- try {
5371
- const { selected: out, cost, history: searchHistoryCoverage } = await executeOptimizationMethod({
5372
- method,
5373
- input: createOptimizationMethodInput(opts, method.name, resolvedRunDir, seed, baselineSurface, costLedger, optimizationOwner.signal),
5374
- storage,
5375
- searchHistoryPolicy: opts.searchHistoryPolicy,
5376
- searchHistoryVerification: opts.searchHistoryVerification
5377
- });
5378
- const winnerSurface = out.winnerSurface;
5379
- return {
5380
- name: method.name,
5381
- winnerSurface,
5382
- cost,
5383
- ...out.durationMs === void 0 ? {} : { durationMs: out.durationMs },
5384
- ...out.composition === void 0 ? {} : { composition: out.composition },
5385
- ...out.provenance === void 0 ? {} : { provenance: out.provenance },
5386
- searchHistoryCoverage
5387
- };
5388
- } catch (error) {
5389
- if (!optimizationOwner.signal.aborted) optimizationOwner.abort(error);
5390
- throw error;
5391
- }
5392
- });
5393
- assertReportedCostWithinCeiling(combineComparisonCosts(optimized.map((result) => ({
5394
- label: `method '${result.name}'`,
5395
- cost: result.cost
5396
- }))).totalCostUsd, opts.costCeiling, "optimization");
5397
- const testCostPhase = finalCostPhase(opts, baselineSurface, optimized, seed);
5398
- const finalEvidence = opts.finalEvidence ? await exposeFinalEvidence(opts.finalEvidence, opts.claim, opts.testScenarios, [baselineSurface, ...optimized.map((method) => method.winnerSurface)]) : void 0;
5399
- const baselineArr = align(await scoreOnTest(baselineSurface, "test/baseline", testCostPhase), "baseline");
5400
- const testScoresBySurface = /* @__PURE__ */ new Map([[surfaceContentHash(baselineSurface), baselineArr]]);
5401
- const winners = [];
5402
- for (const winner of optimized) {
5403
- const surfaceKey = surfaceContentHash(winner.winnerSurface);
5404
- let arr = testScoresBySurface.get(surfaceKey);
5405
- if (!arr) {
5406
- arr = align(await scoreOnTest(winner.winnerSurface, `test/methods/${slug(winner.name)}`, testCostPhase), `method "${winner.name}"`);
5407
- testScoresBySurface.set(surfaceKey, arr);
5408
- }
5409
- winners.push({
5410
- ...winner,
5411
- arr
5412
- });
5413
- }
5414
- const scores = winners.map((w) => {
5415
- const baselineUnits = aggregateUnits(baselineArr);
5416
- const winnerUnits = aggregateUnits(w.arr);
5417
- const decision = decidePairedPromotion(baselineUnits, winnerUnits, decisionOptions);
5418
- const score = {
5419
- name: w.name,
5420
- baselineComposite: mean(baselineUnits),
5421
- winnerComposite: mean(winnerUnits),
5422
- lift: decision.delta,
5423
- liftCi: {
5424
- low: decision.low,
5425
- high: decision.high
5426
- },
5427
- decision,
5428
- optimizationCost: w.cost,
5429
- scenarioScores: scenarioIds.map((scenarioId, index) => ({
5430
- scenarioId,
5431
- baselineComposite: baselineArr[index],
5432
- winnerComposite: w.arr[index],
5433
- lift: w.arr[index] - baselineArr[index]
5434
- })),
5435
- unitScores: units.units.map((unit, index) => ({
5436
- unitId: unit.id,
5437
- scenarios: unit.observations,
5438
- baselineComposite: baselineUnits[index],
5439
- winnerComposite: winnerUnits[index],
5440
- lift: winnerUnits[index] - baselineUnits[index]
5441
- })),
5442
- winnerSurface: structuredClone(w.winnerSurface),
5443
- rank: 0
5444
- };
5445
- if (evidence) {
5446
- const baseline = evidenceBySurface.get(surfaceContentHash(baselineSurface));
5447
- const winner = evidenceBySurface.get(surfaceContentHash(w.winnerSurface));
5448
- if (!baseline || !winner) throw new Error("final measurement evidence is missing");
5449
- score.evidence = {
5450
- baseline,
5451
- winner
5452
- };
5453
- }
5454
- if (w.composition !== void 0) score.composition = structuredClone(w.composition);
5455
- if (w.durationMs !== void 0) score.durationMs = w.durationMs;
5456
- if (w.provenance !== void 0) score.provenance = structuredClone(w.provenance);
5457
- return score;
5458
- });
5459
- scores.sort((a, b) => b.lift - a.lift);
5460
- for (let start = 0; start < scores.length;) {
5461
- let end = start + 1;
5462
- while (end < scores.length && scores[end].lift === scores[start].lift) end += 1;
5463
- const tied = scores.slice(start, end);
5464
- if (tied.every((score) => score.optimizationCost.accountingComplete)) {
5465
- tied.sort((a, b) => a.optimizationCost.totalCostUsd - b.optimizationCost.totalCostUsd);
5466
- scores.splice(start, tied.length, ...tied);
5467
- }
5468
- start = end;
5469
- }
5470
- scores.forEach((s, i) => {
5471
- s.rank = i + 1;
5472
- });
5473
- const best = scores[0];
5474
- const byName = new Map(winners.map((w) => [w.name, w]));
5475
- const bestArr = aggregateUnits(byName.get(best.name).arr);
5476
- const pairwise = scores.slice(1).map((other) => {
5477
- const otherArr = aggregateUnits(byName.get(other.name).arr);
5478
- const decision = decidePairedPromotion(otherArr, bestArr, decisionOptions);
5479
- const reverse = decidePairedPromotion(bestArr, otherArr, decisionOptions);
5480
- const favored = decision.promote ? best.name : reverse.promote ? other.name : null;
5481
- return {
5482
- a: best.name,
5483
- b: other.name,
5484
- deltaMean: decision.delta,
5485
- low: decision.low,
5486
- high: decision.high,
5487
- favored,
5488
- decision
5489
- };
5490
- });
5491
- const optimizationCost = combineComparisonCosts(scores.map((score) => ({
5492
- label: `method '${score.name}'`,
5493
- cost: score.optimizationCost
5494
- })));
5495
- const testCost = costFromLedgerSummary(costLedger.summary({ phase: testCostPhase }));
5496
- const totalCost = combineComparisonCosts([{
5497
- label: "optimization",
5498
- cost: optimizationCost
5499
- }, {
5500
- label: "final test",
5501
- cost: testCost
5502
- }]);
5503
- assertReportedCostWithinCeiling(totalCost.totalCostUsd, opts.costCeiling, "total");
5504
- return {
5505
- scores,
5506
- best,
5507
- pairwise,
5508
- testScenarioIds: scenarioIds,
5509
- units,
5510
- observationUnit: opts.claim ? "registered" : "scenario",
5511
- pairedCellN: scenarioIds.length * (opts.reps ?? 1),
5512
- ...opts.claim ? { claim: opts.claim } : {},
5513
- ...finalEvidence ? { finalEvidence } : {},
5514
- optimizationCost,
5515
- testCost,
5516
- totalCost,
5517
- confidence,
5518
- intervalConfidence,
5519
- comparisonCount,
5520
- seed,
5521
- resamples,
5522
- reps: opts.reps ?? 1,
5523
- searchHistory: Object.freeze({
5524
- policy: searchHistoryPolicy,
5525
- allComplete: optimized.every((result) => result.searchHistoryCoverage.status === "complete"),
5526
- producers: Object.freeze(optimized.map((result) => result.searchHistoryCoverage))
5527
- })
5528
- };
5529
- }
5530
- function assertReportedCostWithinCeiling(totalCostUsd, costCeiling, phase) {
5531
- const tolerance = Number.EPSILON * Math.max(1, Math.abs(totalCostUsd), Math.abs(costCeiling ?? 0)) * 8;
5532
- if (costCeiling !== void 0 && totalCostUsd > costCeiling + tolerance) throw new Error(`compareOptimizationMethods: reported ${phase} cost ${totalCostUsd} exceeds costCeiling ${costCeiling}`);
5533
- }
5534
- function assertOptimizationMethods(methods) {
5535
- if (!Array.isArray(methods) || methods.length === 0) throw new Error("compareOptimizationMethods: no methods to compare");
5536
- const names = /* @__PURE__ */ new Set();
5537
- const pathOwners = /* @__PURE__ */ new Map();
5538
- for (const method of methods) {
5539
- if (!method || typeof method !== "object" || typeof method.optimize !== "function") throw new Error("compareOptimizationMethods: every method must provide optimize(input)");
5540
- if (!method.name || method.name.trim() !== method.name) throw new Error("compareOptimizationMethods: method names must be trimmed and non-empty");
5541
- if (names.has(method.name)) throw new Error(`compareOptimizationMethods: duplicate method name '${method.name}'`);
5542
- names.add(method.name);
5543
- const pathKey = slug(method.name);
5544
- const prior = pathOwners.get(pathKey);
5545
- if (prior) throw new Error(`compareOptimizationMethods: method names '${prior}' and '${method.name}' map to the same run path '${pathKey}'`);
5546
- pathOwners.set(pathKey, method.name);
5547
- }
5548
- }
5549
- function assertComparisonControls(opts, seed, resamples, confidence) {
5550
- if (opts.optimizationRunOptions && "costCeiling" in opts.optimizationRunOptions) throw new Error("compareOptimizationMethods: optimizationRunOptions.costCeiling is not supported; costCeiling covers optimization and final scoring");
5551
- if (!opts.judges || opts.judges.length === 0) throw new Error("compareOptimizationMethods: at least one judge is required");
5552
- if (typeof opts.dispatchWithSurface !== "function") throw new Error("compareOptimizationMethods: dispatchWithSurface must be a function");
5553
- if (opts.dispatchRef !== void 0 && (typeof opts.dispatchRef !== "string" || opts.dispatchRef.trim().length === 0 || opts.dispatchRef.trim() !== opts.dispatchRef)) throw new Error("compareOptimizationMethods: dispatchRef must be trimmed and non-empty when provided");
5554
- const optimizationDispatchRef = opts.optimizationRunOptions?.dispatchRef;
5555
- if (optimizationDispatchRef !== void 0 && (typeof optimizationDispatchRef !== "string" || optimizationDispatchRef.trim().length === 0 || optimizationDispatchRef.trim() !== optimizationDispatchRef)) throw new Error("compareOptimizationMethods: optimizationRunOptions.dispatchRef must be trimmed and non-empty when provided");
5556
- if (opts.dispatchRef !== void 0 && optimizationDispatchRef !== void 0 && opts.dispatchRef !== optimizationDispatchRef) throw new Error("compareOptimizationMethods: dispatchRef must match optimizationRunOptions.dispatchRef when both are provided");
5557
- try {
5558
- surfaceContentHash(opts.baselineSurface);
5559
- } catch (cause) {
5560
- throw new Error("compareOptimizationMethods: baselineSurface is invalid", { cause });
5561
- }
5562
- const judgeNames = /* @__PURE__ */ new Set();
5563
- for (const judge of opts.judges) {
5564
- if (!judge || typeof judge !== "object" || typeof judge.name !== "string" || judge.name.trim().length === 0 || judge.name.trim() !== judge.name || typeof judge.score !== "function" || !Array.isArray(judge.dimensions) || judge.dimensions.length === 0) throw new Error("compareOptimizationMethods: every judge needs a trimmed name, at least one dimension, and score(input)");
5565
- if (judgeNames.has(judge.name)) throw new Error(`compareOptimizationMethods: duplicate judge name '${judge.name}'`);
5566
- judgeNames.add(judge.name);
5567
- const dimensionKeys = /* @__PURE__ */ new Set();
5568
- for (const dimension of judge.dimensions) {
5569
- if (!dimension || typeof dimension.key !== "string" || dimension.key.trim().length === 0 || dimension.key.trim() !== dimension.key || typeof dimension.description !== "string" || dimension.description.trim().length === 0) throw new Error(`compareOptimizationMethods: judge '${judge.name}' has an invalid dimension`);
5570
- if (dimensionKeys.has(dimension.key)) throw new Error(`compareOptimizationMethods: judge '${judge.name}' has duplicate dimension '${dimension.key}'`);
5571
- dimensionKeys.add(dimension.key);
5572
- }
5573
- }
5574
- if (typeof opts.runDir !== "string" || opts.runDir.trim().length === 0) throw new Error("compareOptimizationMethods: runDir must be a non-empty string");
5575
- if (!Number.isSafeInteger(seed)) throw new Error(`compareOptimizationMethods: seed must be a safe integer, got ${String(seed)}`);
5576
- if (!Number.isSafeInteger(resamples) || resamples <= 0 || resamples > 1e6) throw new Error(`compareOptimizationMethods: resamples must be a positive safe integer no greater than 1000000, got ${String(resamples)}`);
5577
- if (!Number.isFinite(confidence) || confidence <= 0 || confidence >= 1) throw new Error(`compareOptimizationMethods: confidence must be a finite number in (0,1), got ${String(confidence)}`);
5578
- const minimumResamples = minimumBootstrapResamples(confidence, opts.methods.length * (opts.methods.length + 1) / 2);
5579
- if (resamples < minimumResamples) throw new Error(`compareOptimizationMethods: resamples must be at least ${minimumResamples} for simultaneous confidence ${confidence} across ${opts.methods.length} methods, got ${resamples}`);
5580
- if (opts.optimizationConcurrency !== void 0 && (!Number.isSafeInteger(opts.optimizationConcurrency) || opts.optimizationConcurrency <= 0)) throw new Error("compareOptimizationMethods: optimizationConcurrency must be a positive safe integer");
5581
- if (opts.maxConcurrency !== void 0 && (!Number.isSafeInteger(opts.maxConcurrency) || opts.maxConcurrency <= 0)) throw new Error("compareOptimizationMethods: maxConcurrency must be a positive safe integer");
5582
- if (opts.dispatchTimeoutMs !== void 0 && (!Number.isSafeInteger(opts.dispatchTimeoutMs) || opts.dispatchTimeoutMs < 0 || opts.dispatchTimeoutMs > 2147483647)) throw new Error("compareOptimizationMethods: dispatchTimeoutMs must be a non-negative safe integer no greater than 2147483647");
5583
- if (opts.costCeiling !== void 0 && (!Number.isFinite(opts.costCeiling) || opts.costCeiling < 0)) throw new Error("compareOptimizationMethods: costCeiling must be a finite number greater than or equal to 0");
5584
- if (opts.costCeiling !== void 0 && opts.costLedger !== void 0 && opts.costLedger.costCeilingUsd !== opts.costCeiling) throw new Error("compareOptimizationMethods: costCeiling must match the shared CostLedger ceiling");
5585
- }
5586
- function assertComparisonPartitions(opts) {
5587
- const partitions = [
5588
- {
5589
- name: "trainScenarios",
5590
- scenarios: opts.trainScenarios
5591
- },
5592
- {
5593
- name: "selectionScenarios",
5594
- scenarios: opts.selectionScenarios
5595
- },
5596
- {
5597
- name: "testScenarios",
5598
- scenarios: opts.testScenarios
5599
- }
5600
- ];
5601
- const owner = /* @__PURE__ */ new Map();
5602
- for (const partition of partitions) {
5603
- if (!Array.isArray(partition.scenarios) || partition.scenarios.length === 0) throw new Error(`compareOptimizationMethods: ${partition.name} is empty`);
5604
- if (partition.name === "testScenarios" && partition.scenarios.length < 2) throw new Error("compareOptimizationMethods: testScenarios requires at least 2 scenarios to estimate uncertainty");
5605
- const local = /* @__PURE__ */ new Set();
5606
- const duplicates = /* @__PURE__ */ new Set();
5607
- const overlaps = /* @__PURE__ */ new Map();
5608
- for (const scenario of partition.scenarios) {
5609
- if (local.has(scenario.id)) duplicates.add(scenario.id);
5610
- local.add(scenario.id);
5611
- const prior = owner.get(scenario.id);
5612
- if (prior !== void 0 && prior !== partition.name) overlaps.set(scenario.id, prior);
5613
- }
5614
- if (duplicates.size > 0) throw new Error(`compareOptimizationMethods: ${partition.name} contains duplicate scenario id(s) [${[...duplicates].join(", ")}]`);
5615
- if (overlaps.size > 0) {
5616
- const detail = [...overlaps].map(([id, prior]) => `${id} (${prior} ∩ ${partition.name})`).join(", ");
5617
- throw new Error(`compareOptimizationMethods: trainScenarios, selectionScenarios, and testScenarios must be pairwise disjoint; overlap: [${detail}]`);
5618
- }
5619
- assertCampaignDesign(partition.scenarios, opts.reps ?? 1);
5620
- for (const id of local) owner.set(id, partition.name);
5621
- }
5622
- }
5623
- function mean(xs) {
5624
- return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length;
5625
- }
5626
- function slug(name) {
5627
- return name.replace(/[^a-z0-9]+/gi, "-").replace(/^-|-$/g, "").toLowerCase() || "method";
5628
- }
5629
- function assertConfidence(confidence) {
5630
- if (!Number.isFinite(confidence) || confidence <= 0 || confidence >= 1) throw new Error(`compareOptimizationMethods: confidence must be a finite number in (0,1), got ${String(confidence)}`);
5631
- }
5632
- function minimumBootstrapResamples(confidence, comparisonCount) {
5633
- const exact = 2 * comparisonCount / (1 - confidence);
5634
- return Math.ceil(exact - Number.EPSILON * Math.max(1, exact) * 32);
5635
- }
5636
- function createOptimizationMethodInput(opts, methodName, resolvedRunDir, seed, baselineSurface, costLedger, optimizationSignal) {
5637
- const methodRunDir = `${resolvedRunDir}/optimization/${slug(methodName)}`;
5638
- const signal = combineAbortSignals(opts.signal, opts.optimizationRunOptions?.signal, optimizationSignal);
5639
- return Object.freeze({
5640
- baselineSurface: structuredClone(baselineSurface),
5641
- trainScenarios: opts.trainScenarios,
5642
- selectionScenarios: opts.selectionScenarios,
5643
- dispatchWithSurface: opts.dispatchWithSurface,
5644
- judges: opts.judges,
5645
- runDir: methodRunDir,
5646
- seed,
5647
- runOptions: Object.freeze({
5648
- ...opts.optimizationRunOptions ?? {},
5649
- ...opts.optimizationRunOptions?.dispatchRef === void 0 && opts.dispatchRef !== void 0 ? { dispatchRef: opts.dispatchRef } : {},
5650
- ...signal ? { signal } : {}
5651
- }),
5652
- costLedger
5653
- });
5654
- }
5655
- function finalCostPhase(opts, baselineSurface, winners, seed) {
5656
- return `compareOptimizationMethods.test:${contentHash({
5657
- baseline: surfaceContentHash(baselineSurface),
5658
- winners: winners.map((winner) => ({
5659
- name: winner.name,
5660
- surface: surfaceContentHash(winner.winnerSurface)
5661
- })),
5662
- testScenarios: opts.testScenarios,
5663
- judges: opts.judges.map((judge) => ({
5664
- name: judge.name,
5665
- dimensions: judge.dimensions,
5666
- version: judge.judgeVersion ?? contentHash({
5667
- score: judge.score.toString(),
5668
- appliesTo: judge.appliesTo?.toString() ?? null
5669
- })
5670
- })),
5671
- dispatch: callerDispatchRef(opts),
5672
- seed,
5673
- reps: opts.reps ?? 1
5674
- })}`;
5675
- }
5676
- function finalDispatchRef(opts, surface) {
5677
- return `compareOptimizationMethods:${callerDispatchRef(opts)}:${surfaceContentHash(surface)}`;
5678
- }
5679
- function callerDispatchRef(opts) {
5680
- return (opts.dispatchRef ?? opts.dispatchWithSurface.name) || "anonymous";
5681
- }
5682
- /** Preserve every optimizer token class while keeping total input and output explicit. */
5683
- function optimizationTokenUsageFromSummary(summary, receipts) {
5684
- if (!summary.usageComplete) return void 0;
5685
- if (receipts.length !== summary.totalCalls) throw new Error("optimization token usage receipt count does not match the cost summary");
5686
- const cachedInputTokens = summary.cachedTokens;
5687
- const cacheWriteInputTokens = summary.cacheWriteTokens ?? 0;
5688
- const inputTokens = summary.inputTokens + cachedInputTokens + cacheWriteInputTokens;
5689
- const cacheReadComplete = receipts.length === 0 || receipts.every((receipt) => receipt.cachedTokens !== void 0);
5690
- const cacheWriteComplete = receipts.length === 0 || receipts.every((receipt) => receipt.cacheWriteTokens !== void 0);
5691
- const reasoningComplete = receipts.length === 0 || receipts.every((receipt) => receipt.reasoningTokens !== void 0);
5692
- return {
5693
- inputTokens,
5694
- ...cacheReadComplete ? { cachedInputTokens } : {},
5695
- ...cacheWriteComplete ? { cacheWriteInputTokens } : {},
5696
- outputTokens: summary.outputTokens,
5697
- ...reasoningComplete ? { reasoningTokens: summary.reasoningTokens ?? 0 } : {},
5698
- totalTokens: inputTokens + summary.outputTokens,
5699
- calls: summary.totalCalls
5700
- };
5701
- }
5702
- //#endregion
5703
5281
  //#region src/run-score.ts
5704
5282
  const DEFAULT_RUN_SCORE_WEIGHTS = {
5705
5283
  success: 4,
@@ -6172,6 +5750,6 @@ function firstString(value) {
6172
5750
  return typeof value === "string" && value.trim() ? value : void 0;
6173
5751
  }
6174
5752
  //#endregion
6175
- export { captureFinalEvidencePolicy as $, assertCompleteSearchHistory as A, searchReceiptAccounting as B, incumbentWithOperatorBandit as C, surfaceDiff as D, searchEdgeId as E, verifySearchHistoryArtifact as F, captureJudge as G, uniform as H, verifySearchHistoryReceipt as I, redTeamDataset as J, defaultProductionGate as K, campaignCellSearchResult as L, assertSearchHistoryMatchesState as M, createSearchHistoryReceipt as N, surfaceNode as O, searchHistoryCoverageRow as P, assertIndependentEvaluationSplit as Q, searchModelIdentity as R, incumbent as S, developmentClaim as T, runFinalComparison as U, asha as V, openAutoPr as W, scoreRedTeamOutput as X, redTeamReport as Y, runCanaries as Z, isProposedCandidate as _, transientDispatchFailure as a, resolveRunDir as at, beam as b, compareOptimizationMethods as c, buildCellSchedule as ct, combineComparisonCosts as d, evaluationUnitMap as et, costFromLedgerSummary as f, runOptimization as g, runImprovementLoop as h, quotaExhaustedUntil as i, planCampaignRun as it, assertSearchHistoryAdmissionOptions as j, SearchHistoryRequiredError as k, optimizationTokenUsageFromSummary as l, cellCachePath as lt, readGepaCandidatePopulationArtifact as m, JudgeParseError as n, runEval as nt, aggregateRunScore as o, tangleTracesRoot as ot, assertGepaCandidatePopulationSummary as p, DEFAULT_RED_TEAM_CORPUS as q, isTransientTransportFailure as r, runCampaign as rt, clamp01 as s, readCachedCell as st, llmJudge as t, reserveFinalEvidence as tt, executeOptimizationMethod as u, computeManifestHash as ut, labelTrustRank as v, SearchRecorder as w, crowdedFrontierParent as x, aide as y, searchProposalExecution as z };
5753
+ export { evaluationUnitMap as $, assertSearchHistoryAdmissionOptions as A, asha as B, SearchRecorder as C, surfaceNode as D, surfaceDiff as E, verifySearchHistoryReceipt as F, defaultProductionGate as G, runFinalComparison as H, campaignCellSearchResult as I, redTeamReport as J, DEFAULT_RED_TEAM_CORPUS as K, searchModelIdentity as L, createSearchHistoryReceipt as M, searchHistoryCoverageRow as N, SearchHistoryRequiredError as O, verifySearchHistoryArtifact as P, captureFinalEvidencePolicy as Q, searchProposalExecution as R, incumbentWithOperatorBandit as S, searchEdgeId as T, openAutoPr as U, uniform as V, captureJudge as W, runCanaries as X, scoreRedTeamOutput as Y, assertIndependentEvaluationSplit as Z, labelTrustRank as _, transientDispatchFailure as a, resolveRunDir as at, crowdedFrontierParent as b, executeOptimizationMethod as c, buildCellSchedule as ct, combineComparisonCosts as d, exposeFinalEvidence as et, costFromLedgerSummary as f, isProposedCandidate as g, runOptimization as h, quotaExhaustedUntil as i, planCampaignRun as it, assertSearchHistoryMatchesState as j, assertCompleteSearchHistory as k, assertGepaCandidatePopulationSummary as l, cellCachePath as lt, runImprovementLoop as m, JudgeParseError as n, runEval as nt, aggregateRunScore as o, tangleTracesRoot as ot, optimizationTokenUsageFromSummary as p, redTeamDataset as q, isTransientTransportFailure as r, runCampaign as rt, clamp01 as s, readCachedCell as st, llmJudge as t, reserveFinalEvidence as tt, readGepaCandidatePopulationArtifact as u, computeManifestHash as ut, aide as v, developmentClaim as w, incumbent as x, beam as y, searchReceiptAccounting as z };
6176
5754
 
6177
- //# sourceMappingURL=llm-judge-PXV7Xljl.js.map
5755
+ //# sourceMappingURL=llm-judge-CG12SQJq.js.map