@tangle-network/agent-eval 0.216.0 → 0.217.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (553) hide show
  1. package/CHANGELOG.md +141 -57
  2. package/README.md +2 -2
  3. package/dist/adapters/http.d.ts +3 -3
  4. package/dist/{agent-profile-CIA2Atnt.d.ts → agent-profile-DTt8uGP9.d.ts} +5 -18
  5. package/dist/agent-profile-DTt8uGP9.d.ts.map +1 -0
  6. package/dist/{agent-profile-cell-BYc0bxsL.js → agent-profile-cell-B1V1gUZs.js} +4 -53
  7. package/dist/agent-profile-cell-B1V1gUZs.js.map +1 -0
  8. package/dist/{agent-profile-cell-BNp-Js9F.d.ts → agent-profile-cell-BcpFbNd5.d.ts} +2 -22
  9. package/dist/agent-profile-cell-BcpFbNd5.d.ts.map +1 -0
  10. package/dist/{agreement-irr-DsTURxdr.js → agreement-irr-BhAqjluA.js} +5 -5
  11. package/dist/agreement-irr-BhAqjluA.js.map +1 -0
  12. package/dist/analyst/index.d.ts +37 -170
  13. package/dist/analyst/index.d.ts.map +1 -1
  14. package/dist/analyst/index.js +17 -192
  15. package/dist/analyst/index.js.map +1 -1
  16. package/dist/{analyze-runs-3xpby3p2.js → analyze-runs-DqPJaJ4X.js} +763 -555
  17. package/dist/analyze-runs-DqPJaJ4X.js.map +1 -0
  18. package/dist/{analyze-runs-CSDAh8A7.d.ts → analyze-runs-DyeNcenW.d.ts} +18 -16
  19. package/dist/analyze-runs-DyeNcenW.d.ts.map +1 -0
  20. package/dist/authenticity/index.d.ts +0 -23
  21. package/dist/authenticity/index.d.ts.map +1 -1
  22. package/dist/authenticity/index.js +1 -34
  23. package/dist/authenticity/index.js.map +1 -1
  24. package/dist/{benchmark-command-CByl9dxG.js → benchmark-command-xQK8nOI6.js} +591 -109
  25. package/dist/benchmark-command-xQK8nOI6.js.map +1 -0
  26. package/dist/benchmarks/index.d.ts +5 -6
  27. package/dist/benchmarks/index.d.ts.map +1 -1
  28. package/dist/benchmarks/index.js +4 -4
  29. package/dist/benchmarks/index.js.map +1 -1
  30. package/dist/builder-eval/index.d.ts +3 -3
  31. package/dist/builder-eval/index.js +2 -2
  32. package/dist/builder-eval/index.js.map +1 -1
  33. package/dist/campaign/index.d.ts +8 -8
  34. package/dist/campaign/index.js +15 -15
  35. package/dist/{campaign-BQ-Jujuj.js → campaign-CJGkxAQ6.js} +256 -765
  36. package/dist/campaign-CJGkxAQ6.js.map +1 -0
  37. package/dist/{campaign-evidence-OcUciD-N.js → campaign-evidence-CQYw8_0H.js} +10 -18
  38. package/dist/{campaign-evidence-OcUciD-N.js.map → campaign-evidence-CQYw8_0H.js.map} +1 -1
  39. package/dist/{canonical-BsZVgBhW.js → canonical-Dq2hOQ8L.js} +2 -2
  40. package/dist/{canonical-BsZVgBhW.js.map → canonical-Dq2hOQ8L.js.map} +1 -1
  41. package/dist/cell-spend-DEWR3nj-.js +90 -0
  42. package/dist/cell-spend-DEWR3nj-.js.map +1 -0
  43. package/dist/{chat-client-DthPOYGd.js → chat-client-ldqxYYTN.js} +2 -2
  44. package/dist/{chat-client-DthPOYGd.js.map → chat-client-ldqxYYTN.js.map} +1 -1
  45. package/dist/{chat-json-call-BrKsyspg.js → chat-json-call-DavzexSp.js} +3 -2
  46. package/dist/chat-json-call-DavzexSp.js.map +1 -0
  47. package/dist/{claim-BXcZw-jo.js → claim-BPVuyHuy.js} +4 -4
  48. package/dist/claim-BPVuyHuy.js.map +1 -0
  49. package/dist/cli.js +113 -12
  50. package/dist/cli.js.map +1 -1
  51. package/dist/{client-BzL9niqi.d.ts → client-hwo3cJjR.d.ts} +3 -16
  52. package/dist/client-hwo3cJjR.d.ts.map +1 -0
  53. package/dist/{cluster-bootstrap-B3JywQQn.js → cluster-bootstrap-DEwwhl0C.js} +3 -3
  54. package/dist/{cluster-bootstrap-B3JywQQn.js.map → cluster-bootstrap-DEwwhl0C.js.map} +1 -1
  55. package/dist/{produced-state-ZCtDBxAE.js → completion-verifier-B_jw6C9p.js} +9 -64
  56. package/dist/completion-verifier-B_jw6C9p.js.map +1 -0
  57. package/dist/contract/index.d.ts +13 -13
  58. package/dist/contract/index.js +1041 -1473
  59. package/dist/contract/index.js.map +1 -1
  60. package/dist/{cost-ledger-D2FnJPBF.js → cost-ledger-D0352Z4d.js} +832 -817
  61. package/dist/cost-ledger-D0352Z4d.js.map +1 -0
  62. package/dist/{cost-ledger-CO7UyEcQ.d.ts → cost-ledger-Jk62OBfF.d.ts} +12 -47
  63. package/dist/cost-ledger-Jk62OBfF.d.ts.map +1 -0
  64. package/dist/dataset-DQqhOCPt.d.ts.map +1 -1
  65. package/dist/{default-registry-Cn1LQ2Ve.js → default-registry-ClYjq3QE.js} +8 -11
  66. package/dist/default-registry-ClYjq3QE.js.map +1 -0
  67. package/dist/{default-registry-BrWh62zf.d.ts → default-registry-DXgLmoDI.d.ts} +7 -50
  68. package/dist/default-registry-DXgLmoDI.d.ts.map +1 -0
  69. package/dist/{define-agent-eval-Ch3MWX6k.js → define-agent-eval-BDxNlP3D.js} +10 -10
  70. package/dist/define-agent-eval-BDxNlP3D.js.map +1 -0
  71. package/dist/{define-agent-eval-CEWkX5wZ.d.ts → define-agent-eval-BIkH_gAR.d.ts} +9 -9
  72. package/dist/define-agent-eval-BIkH_gAR.d.ts.map +1 -0
  73. package/dist/{descriptive-CnU8Yq29.js → descriptive-CVNy48rR.js} +2 -2
  74. package/dist/{descriptive-CnU8Yq29.js.map → descriptive-CVNy48rR.js.map} +1 -1
  75. package/dist/diagnosis/index.d.ts +4 -14
  76. package/dist/diagnosis/index.d.ts.map +1 -1
  77. package/dist/diagnosis/index.js +67 -28
  78. package/dist/diagnosis/index.js.map +1 -1
  79. package/dist/{dspy-rlm-engine-D-KX_e64.js → dspy-rlm-engine-DhdCr1Dc.js} +5 -10
  80. package/dist/dspy-rlm-engine-DhdCr1Dc.js.map +1 -0
  81. package/dist/{emitter-HcCmFQl_.d.ts → emitter-DABYYTcH.d.ts} +5 -5
  82. package/dist/emitter-DABYYTcH.d.ts.map +1 -0
  83. package/dist/emitter-DbMXUIVP.js.map +1 -1
  84. package/dist/{engine-C_zNWfM8.d.ts → engine-BvZ0R-36.d.ts} +18 -22
  85. package/dist/engine-BvZ0R-36.d.ts.map +1 -0
  86. package/dist/{equal-terms-Blx9Zk5I.js → equal-terms-DbPGcVtI.js} +2 -2
  87. package/dist/equal-terms-DbPGcVtI.js.map +1 -0
  88. package/dist/{errors-Dngq5h35.js → errors-BcOs5MKW.js} +6 -8
  89. package/dist/{errors-Dngq5h35.js.map → errors-BcOs5MKW.js.map} +1 -1
  90. package/dist/{eval-campaign-4byTtqrf.js → eval-campaign-BLEfBGbA.js} +6 -6
  91. package/dist/{eval-campaign-4byTtqrf.js.map → eval-campaign-BLEfBGbA.js.map} +1 -1
  92. package/dist/{evaluation-Cn2dRZIz.d.ts → evaluation-jTjC_QQD.d.ts} +8 -8
  93. package/dist/evaluation-jTjC_QQD.d.ts.map +1 -0
  94. package/dist/evaluation.d.ts +2 -2
  95. package/dist/evaluation.js +1 -1
  96. package/dist/evaluation.js.map +1 -1
  97. package/dist/{exact-types-CD0lf5Qd.d.ts → exact-types-CflbJ9f3.d.ts} +2 -2
  98. package/dist/{exact-types-CD0lf5Qd.d.ts.map → exact-types-CflbJ9f3.d.ts.map} +1 -1
  99. package/dist/{exec-ChJ6d5if.js → exec-CpbtOXVt.js} +2 -24
  100. package/dist/{exec-ChJ6d5if.js.map → exec-CpbtOXVt.js.map} +1 -1
  101. package/dist/execution-tracks-CpgFPpS5.js.map +1 -1
  102. package/dist/experiment/index.d.ts +331 -33
  103. package/dist/experiment/index.d.ts.map +1 -1
  104. package/dist/experiment/index.js +303 -101
  105. package/dist/experiment/index.js.map +1 -1
  106. package/dist/{exporters-X4xwla8A.js → exporters-Cl6IeNAu.js} +3 -3
  107. package/dist/exporters-Cl6IeNAu.js.map +1 -0
  108. package/dist/{exporters-ChTnR90l.d.ts → exporters-Dk3oEbK_.d.ts} +2 -4
  109. package/dist/{exporters-ChTnR90l.d.ts.map → exporters-Dk3oEbK_.d.ts.map} +1 -1
  110. package/dist/{external-optimizer-process-nnsY5D_4.js → external-optimizer-process-KVFGvYX1.js} +4 -3
  111. package/dist/external-optimizer-process-KVFGvYX1.js.map +1 -0
  112. package/dist/{external-optimizer-subprocess-DcerlZSG.js → external-optimizer-subprocess-XIHBOSKt.js} +551 -553
  113. package/dist/external-optimizer-subprocess-XIHBOSKt.js.map +1 -0
  114. package/dist/{extract-usage-BrnzRJ6H.js → extract-usage-BZLU01kR.js} +2 -23
  115. package/dist/{extract-usage-BrnzRJ6H.js.map → extract-usage-BZLU01kR.js.map} +1 -1
  116. package/dist/{failure-cluster-CbgEDcFJ.d.ts → failure-cluster-DN0t63Z0.d.ts} +4 -43
  117. package/dist/failure-cluster-DN0t63Z0.d.ts.map +1 -0
  118. package/dist/{failure-taxonomy-CjabTATj.js → failure-taxonomy-Bx5oWqaW.js} +2 -19
  119. package/dist/{failure-taxonomy-CjabTATj.js.map → failure-taxonomy-Bx5oWqaW.js.map} +1 -1
  120. package/dist/{feedback-trajectory-DkaDj0dj.d.ts → feedback-trajectory-BWmhRBGu.d.ts} +8 -13
  121. package/dist/feedback-trajectory-BWmhRBGu.d.ts.map +1 -0
  122. package/dist/{semantic-concept-judge-zQBG9hHU.js → findings-store-BMVQtEsm.js} +13 -255
  123. package/dist/findings-store-BMVQtEsm.js.map +1 -0
  124. package/dist/fuzz.d.ts +20 -63
  125. package/dist/fuzz.d.ts.map +1 -1
  126. package/dist/fuzz.js +62 -108
  127. package/dist/fuzz.js.map +1 -1
  128. package/dist/{hf-dataset-BR2yhuNn.js → hf-dataset-D9f7-oX7.js} +4 -24
  129. package/dist/hf-dataset-D9f7-oX7.js.map +1 -0
  130. package/dist/hosted/index.d.ts +2 -2
  131. package/dist/hosted/index.js +2 -2
  132. package/dist/{index-DHKEAv3s.d.ts → index-9jJ2fKSX.d.ts} +2 -64
  133. package/dist/index-9jJ2fKSX.d.ts.map +1 -0
  134. package/dist/{index-oQ4aHziK.d.ts → index-BLGqZ81N.d.ts} +11 -16
  135. package/dist/{index-oQ4aHziK.d.ts.map → index-BLGqZ81N.d.ts.map} +1 -1
  136. package/dist/{index-DlH0Y9gI.d.ts → index-BiT6RtNH.d.ts} +31 -43
  137. package/dist/index-BiT6RtNH.d.ts.map +1 -0
  138. package/dist/{index-Ch9VxCpK.d.ts → index-CX-GAUpw.d.ts} +4 -8
  139. package/dist/index-CX-GAUpw.d.ts.map +1 -0
  140. package/dist/{index-CPJJr98o.d.ts → index-CiRwLv2R.d.ts} +5 -68
  141. package/dist/index-CiRwLv2R.d.ts.map +1 -0
  142. package/dist/{index-NvGBiO8H.d.ts → index-ZuWx52WG.d.ts} +266 -495
  143. package/dist/index-ZuWx52WG.d.ts.map +1 -0
  144. package/dist/index.d.ts +2256 -2165
  145. package/dist/index.d.ts.map +1 -1
  146. package/dist/index.js +514 -1192
  147. package/dist/index.js.map +1 -1
  148. package/dist/{integrity-BV1ZDf42.js → integrity-DOd9Q86O.js} +7 -12
  149. package/dist/integrity-DOd9Q86O.js.map +1 -0
  150. package/dist/{integrity-BvnVARVl.js → integrity-W4CADcFc.js} +3 -7
  151. package/dist/{integrity-BvnVARVl.js.map → integrity-W4CADcFc.js.map} +1 -1
  152. package/dist/{integrity-CCAxGVK6.d.ts → integrity-_qgZLA5p.d.ts} +4 -6
  153. package/dist/{integrity-CCAxGVK6.d.ts.map → integrity-_qgZLA5p.d.ts.map} +1 -1
  154. package/dist/{internal-CRHwS9iz.js → internal-BTK91Bqt.js} +3 -3
  155. package/dist/{internal-CRHwS9iz.js.map → internal-BTK91Bqt.js.map} +1 -1
  156. package/dist/interval-Dq9gt7jx.js +219 -0
  157. package/dist/interval-Dq9gt7jx.js.map +1 -0
  158. package/dist/interval-ohuOJurX.d.ts +47 -0
  159. package/dist/interval-ohuOJurX.d.ts.map +1 -0
  160. package/dist/{journal-DvKCvjty.js → journal-CR4QikYp.js} +110 -110
  161. package/dist/journal-CR4QikYp.js.map +1 -0
  162. package/dist/json-BhdsJaOQ.js +111 -0
  163. package/dist/json-BhdsJaOQ.js.map +1 -0
  164. package/dist/{judge-calibration-N44V04Se.js → judge-calibration-BkXMIWA4.js} +3 -3
  165. package/dist/{judge-calibration-N44V04Se.js.map → judge-calibration-BkXMIWA4.js.map} +1 -1
  166. package/dist/{judge-gate-B1qYnkpi.js → judge-gate-Co5tuh57.js} +4 -27
  167. package/dist/judge-gate-Co5tuh57.js.map +1 -0
  168. package/dist/{judge-gate-DNcUrRFA.d.ts → judge-gate-DsL0yFcb.d.ts} +9 -10
  169. package/dist/judge-gate-DsL0yFcb.d.ts.map +1 -0
  170. package/dist/{kind-factory-CbZ-6tvu.js → kind-factory-Gy4fXixd.js} +88 -17
  171. package/dist/kind-factory-Gy4fXixd.js.map +1 -0
  172. package/dist/ledger-core/index.d.ts +2 -2
  173. package/dist/ledger-core/index.js +4 -4
  174. package/dist/{ledger-core-B2hahzrt.js → ledger-core-DrJhbms9.js} +5 -84
  175. package/dist/ledger-core-DrJhbms9.js.map +1 -0
  176. package/dist/{llm-client-DQveoxSM.js → llm-client-BshCREn6.js} +90 -219
  177. package/dist/llm-client-BshCREn6.js.map +1 -0
  178. package/dist/{llm-judge-PXV7Xljl.js → llm-judge-C8jMTNNG.js} +501 -931
  179. package/dist/llm-judge-C8jMTNNG.js.map +1 -0
  180. package/dist/matrix/index.d.ts +4 -4
  181. package/dist/matrix/index.js +3 -2
  182. package/dist/{matrix-_BV9-Lk5.js → matrix-B8l-Nsxd.js} +7 -96
  183. package/dist/matrix-B8l-Nsxd.js.map +1 -0
  184. package/dist/{matrix-CHMiebFx.d.ts → matrix-DHe-dwVJ.d.ts} +10 -19
  185. package/dist/matrix-DHe-dwVJ.d.ts.map +1 -0
  186. package/dist/{paired-tests-jbwPdSz5.js → mean-BbA_JCzW.js} +45 -5
  187. package/dist/mean-BbA_JCzW.js.map +1 -0
  188. package/dist/meta-eval/index.d.ts +6 -6
  189. package/dist/meta-eval/index.js +10 -10
  190. package/dist/meta-eval/index.js.map +1 -1
  191. package/dist/metrics-BKaPBtIG.js.map +1 -1
  192. package/dist/mint-BRZXuwRi.d.ts +81 -0
  193. package/dist/mint-BRZXuwRi.d.ts.map +1 -0
  194. package/dist/{mint-Csor1BLJ.js → mint-zszyYlpA.js} +4 -4
  195. package/dist/{mint-Csor1BLJ.js.map → mint-zszyYlpA.js.map} +1 -1
  196. package/dist/{multi-layer-verifier-BUaQ4C17.d.ts → multi-layer-verifier-DEgMUk6v.d.ts} +2 -2
  197. package/dist/{multi-layer-verifier-BUaQ4C17.d.ts.map → multi-layer-verifier-DEgMUk6v.d.ts.map} +1 -1
  198. package/dist/{multiplicity-D9cjmStq.js → multiplicity-Sg0U7h8j.js} +2 -2
  199. package/dist/{multiplicity-D9cjmStq.js.map → multiplicity-Sg0U7h8j.js.map} +1 -1
  200. package/dist/multishot/golden/index.d.ts +1 -23
  201. package/dist/multishot/golden/index.d.ts.map +1 -1
  202. package/dist/multishot/golden/index.js +5 -8
  203. package/dist/multishot/golden/index.js.map +1 -1
  204. package/dist/multishot/index.d.ts +3 -12
  205. package/dist/multishot/index.d.ts.map +1 -1
  206. package/dist/multishot/index.js +4 -3
  207. package/dist/multishot/index.js.map +1 -1
  208. package/dist/nonnegative-number-DvdMxg_f.js +9 -0
  209. package/dist/nonnegative-number-DvdMxg_f.js.map +1 -0
  210. package/dist/openapi.json +114 -10
  211. package/dist/{otel-spans-C8YIJWy1.js → otel-spans-CdN909k1.js} +3 -3
  212. package/dist/{otel-spans-C8YIJWy1.js.map → otel-spans-CdN909k1.js.map} +1 -1
  213. package/dist/{paired-arms-DzKn0TfY.js → paired-arms-Ct4LTKc1.js} +32 -9
  214. package/dist/paired-arms-Ct4LTKc1.js.map +1 -0
  215. package/dist/{paired-binary-Q36bhn2R.js → paired-binary-B_nejA44.js} +3 -36
  216. package/dist/paired-binary-B_nejA44.js.map +1 -0
  217. package/dist/{paired-promotion-decision-DPKz8ZnF.d.ts → paired-promotion-decision-CQMJ5xGN.d.ts} +2 -34
  218. package/dist/paired-promotion-decision-CQMJ5xGN.d.ts.map +1 -0
  219. package/dist/{paired-promotion-decision-BN6IQTiu.js → paired-promotion-decision-D0SHb-ZL.js} +3 -3
  220. package/dist/{paired-promotion-decision-BN6IQTiu.js.map → paired-promotion-decision-D0SHb-ZL.js.map} +1 -1
  221. package/dist/{search-kernel-BztlERLv.js → pareto-BzsxidFD.js} +883 -861
  222. package/dist/pareto-BzsxidFD.js.map +1 -0
  223. package/dist/pipelines/index.d.ts +16 -19
  224. package/dist/pipelines/index.d.ts.map +1 -1
  225. package/dist/pipelines/index.js +4 -68
  226. package/dist/pipelines/index.js.map +1 -1
  227. package/dist/{power-and-mde-fK5Qocwo.js → power-and-mde-N5nU4_Ly.js} +3 -3
  228. package/dist/{power-and-mde-fK5Qocwo.js.map → power-and-mde-N5nU4_Ly.js.map} +1 -1
  229. package/dist/predicate-FdKmElA_.js +38 -0
  230. package/dist/predicate-FdKmElA_.js.map +1 -0
  231. package/dist/{prime-protocol-DcAK9pC8.js → prime-protocol-BFgGrEMP.js} +3 -3
  232. package/dist/prime-protocol-BFgGrEMP.js.map +1 -0
  233. package/dist/{prime-protocol-CEE4BR9N.d.ts → prime-protocol-BWG2DQIk.d.ts} +4 -22
  234. package/dist/prime-protocol-BWG2DQIk.d.ts.map +1 -0
  235. package/dist/profile-cell.d.ts +2 -2
  236. package/dist/profile-cell.js +2 -2
  237. package/dist/{release-confidence-BK2QCB5M.d.ts → promotion-gate-DaY95yU6.d.ts} +105 -106
  238. package/dist/promotion-gate-DaY95yU6.d.ts.map +1 -0
  239. package/dist/{promotion-policy-CWW35UWq.js → promotion-policy-6A8osGr_.js} +4 -4
  240. package/dist/{promotion-policy-CWW35UWq.js.map → promotion-policy-6A8osGr_.js.map} +1 -1
  241. package/dist/{proposal-findings-BZiMTzAF.js → proposal-findings-BbWidvUk.js} +3 -3
  242. package/dist/proposal-findings-BbWidvUk.js.map +1 -0
  243. package/dist/{query-B1D1qp2Z.js → query-B_RZ36an.js} +5 -5
  244. package/dist/query-B_RZ36an.js.map +1 -0
  245. package/dist/{query-DK1nWcmp.d.ts → query-Cmv_U8vf.d.ts} +4 -24
  246. package/dist/{query-DK1nWcmp.d.ts.map → query-Cmv_U8vf.d.ts.map} +1 -1
  247. package/dist/{random-Dn5fPWkt.js → random-DDDcC7YU.js} +2 -2
  248. package/dist/{random-Dn5fPWkt.js.map → random-DDDcC7YU.js.map} +1 -1
  249. package/dist/{raw-provider-sink-rB2DYxr6.js → raw-provider-sink-75w4FWBj.js} +2 -2
  250. package/dist/{raw-provider-sink-rB2DYxr6.js.map → raw-provider-sink-75w4FWBj.js.map} +1 -1
  251. package/dist/{redact-Cveoq6fJ.d.ts → redact-DXdlFb_U.d.ts} +4 -21
  252. package/dist/redact-DXdlFb_U.d.ts.map +1 -0
  253. package/dist/{redact-Dk1ZdJEm.js → redact-mtkTiJVk.js} +92 -98
  254. package/dist/redact-mtkTiJVk.js.map +1 -0
  255. package/dist/{registry-JYfPxcVz.d.ts → registry-ssNoHzwD.d.ts} +5 -5
  256. package/dist/{registry-JYfPxcVz.d.ts.map → registry-ssNoHzwD.d.ts.map} +1 -1
  257. package/dist/{release-confidence-Dbe9AS_X.js → release-confidence-CYIFH-fp.js} +6 -11
  258. package/dist/release-confidence-CYIFH-fp.js.map +1 -0
  259. package/dist/{report-command-y8H6RSX8.js → render-C-w2EZ6p.js} +331 -554
  260. package/dist/render-C-w2EZ6p.js.map +1 -0
  261. package/dist/reporting-html-Kvlc8oMt.js +100 -0
  262. package/dist/reporting-html-Kvlc8oMt.js.map +1 -0
  263. package/dist/reporting.d.ts +6 -6
  264. package/dist/reporting.js +8 -8
  265. package/dist/{researcher-f6mZhh58.d.ts → researcher-0ABVIdGa.d.ts} +9 -9
  266. package/dist/{researcher-f6mZhh58.d.ts.map → researcher-0ABVIdGa.d.ts.map} +1 -1
  267. package/dist/reward-DrZ71vnf.d.ts +55 -0
  268. package/dist/reward-DrZ71vnf.d.ts.map +1 -0
  269. package/dist/{reward-hacking-BFcQgkux.js → reward-hacking-ByuETaIb.js} +3 -3
  270. package/dist/{reward-hacking-BFcQgkux.js.map → reward-hacking-ByuETaIb.js.map} +1 -1
  271. package/dist/{reward-hacking-DHA7c1l8.d.ts → reward-hacking-E52wq86n.d.ts} +5 -18
  272. package/dist/reward-hacking-E52wq86n.d.ts.map +1 -0
  273. package/dist/reward-nw2xZGZG.js.map +1 -1
  274. package/dist/rl.d.ts +117 -295
  275. package/dist/rl.d.ts.map +1 -1
  276. package/dist/rl.js +27 -747
  277. package/dist/rl.js.map +1 -1
  278. package/dist/rollout/index.d.ts +5 -4
  279. package/dist/rollout/index.js +7 -8
  280. package/dist/{rollout-D9bKKBnZ.js → rollout-D6C50mfA.js} +88 -13
  281. package/dist/rollout-D6C50mfA.js.map +1 -0
  282. package/dist/round-decimal-COrKglMh.js +10 -0
  283. package/dist/round-decimal-COrKglMh.js.map +1 -0
  284. package/dist/{rubric-predictive-validity-BsgkBfT0.js → rubric-predictive-validity-D3_oI72N.js} +3 -3
  285. package/dist/{rubric-predictive-validity-BsgkBfT0.js.map → rubric-predictive-validity-D3_oI72N.js.map} +1 -1
  286. package/dist/{rubric-predictive-validity-2SgHxoa5.d.ts → rubric-predictive-validity-DiwT8PbL.d.ts} +2 -2
  287. package/dist/{rubric-predictive-validity-2SgHxoa5.d.ts.map → rubric-predictive-validity-DiwT8PbL.d.ts.map} +1 -1
  288. package/dist/{run-record-D1Hw-Jml.d.ts → run-record-CaxbLq6Y.d.ts} +5 -11
  289. package/dist/run-record-CaxbLq6Y.d.ts.map +1 -0
  290. package/dist/{run-record-ES07wfz9.js → run-record-CysKLB93.js} +3 -5
  291. package/dist/run-record-CysKLB93.js.map +1 -0
  292. package/dist/{run-record-DlTZ_lqe.js → run-record-DhEJgoPz.js} +4 -4
  293. package/dist/{run-record-DlTZ_lqe.js.map → run-record-DhEJgoPz.js.map} +1 -1
  294. package/dist/sandbox-harness-BlSOu4LX.d.ts.map +1 -1
  295. package/dist/{schema-CSf6qWgZ.js → schema-Bt2vLFm3.js} +2 -14
  296. package/dist/schema-Bt2vLFm3.js.map +1 -0
  297. package/dist/schema-C-ziyJD7.d.ts.map +1 -1
  298. package/dist/{schema-2g7trUwH.d.ts → schema-D6EVKzTB.d.ts} +4 -5
  299. package/dist/schema-D6EVKzTB.d.ts.map +1 -0
  300. package/dist/{schema-BuiujZ-g.js → schema-DxcKTzbi.js} +2 -2
  301. package/dist/schema-DxcKTzbi.js.map +1 -0
  302. package/dist/search/index.d.ts +39 -53
  303. package/dist/search/index.d.ts.map +1 -1
  304. package/dist/search/index.js +4 -4
  305. package/dist/{search-kernel-CpcKH5Ih.d.ts → search-kernel-DxqjBxA-.d.ts} +18 -31
  306. package/dist/search-kernel-DxqjBxA-.d.ts.map +1 -0
  307. package/dist/{search-ledger-d7V8o9Lk.js → search-ledger-TUvB5gWH.js} +189 -199
  308. package/dist/search-ledger-TUvB5gWH.js.map +1 -0
  309. package/dist/{search-ledger-file-BqOo-bVo.js → search-ledger-file-UKCO2r8a.js} +4 -4
  310. package/dist/search-ledger-file-UKCO2r8a.js.map +1 -0
  311. package/dist/search-ledger-types-p2xKfiDJ.d.ts.map +1 -1
  312. package/dist/{search-shipper-er4pK5_Y.js → search-shipper-Cw2z3-td.js} +4 -4
  313. package/dist/search-shipper-Cw2z3-td.js.map +1 -0
  314. package/dist/{search-summary-CBvz3u_S.js → search-summary-DQIvJsSB.js} +5 -5
  315. package/dist/{search-summary-CBvz3u_S.js.map → search-summary-DQIvJsSB.js.map} +1 -1
  316. package/dist/{sequential-CzK5DarL.js → sequential-BVv1sjsw.js} +2 -32
  317. package/dist/{sequential-CzK5DarL.js.map → sequential-BVv1sjsw.js.map} +1 -1
  318. package/dist/{sequential-D-1PPCmM.js → sequential-CimQrbjv.js} +5 -5
  319. package/dist/sequential-CimQrbjv.js.map +1 -0
  320. package/dist/{sequential-BhsrMupG.d.ts → sequential-KwGspfeR.d.ts} +2 -9
  321. package/dist/{sequential-BhsrMupG.d.ts.map → sequential-KwGspfeR.d.ts.map} +1 -1
  322. package/dist/{sequential-eprocess-D1jKoihe.js → sequential-eprocess-DodI6RoF.js} +2 -2
  323. package/dist/{sequential-eprocess-D1jKoihe.js.map → sequential-eprocess-DodI6RoF.js.map} +1 -1
  324. package/dist/{server-DbcC7xMl.js → server-DoVbqqtJ.js} +101 -101
  325. package/dist/server-DoVbqqtJ.js.map +1 -0
  326. package/dist/{single-run-lock-fVu3MZWH.js → single-run-lock-U58yf8N6.js} +2 -2
  327. package/dist/{single-run-lock-fVu3MZWH.js.map → single-run-lock-U58yf8N6.js.map} +1 -1
  328. package/dist/{skillopt-optimization-method-CCtzralq.js → skillopt-optimization-method-kyL94-ri.js} +428 -91
  329. package/dist/skillopt-optimization-method-kyL94-ri.js.map +1 -0
  330. package/dist/{spans-D9_ehFuk.d.ts → spans-CVngGij3.d.ts} +2 -2
  331. package/dist/{spans-D9_ehFuk.d.ts.map → spans-CVngGij3.d.ts.map} +1 -1
  332. package/dist/{statistical-heldout-1Gbmex7v.js → statistical-heldout-Bksv92Th.js} +3 -3
  333. package/dist/statistical-heldout-Bksv92Th.js.map +1 -0
  334. package/dist/{statistical-heldout-BeR4bOdg.d.ts → statistical-heldout-C9AbgREN.d.ts} +8 -67
  335. package/dist/statistical-heldout-C9AbgREN.d.ts.map +1 -0
  336. package/dist/{steps-Bg5-KVx1.d.ts → steps-DlepJPwD.d.ts} +2 -56
  337. package/dist/steps-DlepJPwD.d.ts.map +1 -0
  338. package/dist/{store-C4avFjIi.d.ts → store-B3nWRy3B.d.ts} +2 -2
  339. package/dist/{store-C4avFjIi.d.ts.map → store-B3nWRy3B.d.ts.map} +1 -1
  340. package/dist/{store-otlp-DtB4Slsr.js → store-otlp-BteDAbHK.js} +135 -132
  341. package/dist/store-otlp-BteDAbHK.js.map +1 -0
  342. package/dist/{store-tool-spans-fZIigrpP.js → store-tool-spans-D5l_OfTJ.js} +9 -9
  343. package/dist/store-tool-spans-D5l_OfTJ.js.map +1 -0
  344. package/dist/{store-tool-spans-DR1SrCLK.d.ts → store-tool-spans-DZS0HWVf.d.ts} +157 -165
  345. package/dist/store-tool-spans-DZS0HWVf.d.ts.map +1 -0
  346. package/dist/storyboard/index.d.ts +4 -11
  347. package/dist/storyboard/index.d.ts.map +1 -1
  348. package/dist/storyboard/index.js +21 -19
  349. package/dist/storyboard/index.js.map +1 -1
  350. package/dist/{summary-report-DtdRucM0.js → summary-report-BsXJoYX5.js} +588 -598
  351. package/dist/summary-report-BsXJoYX5.js.map +1 -0
  352. package/dist/{summary-report-C-CtItCN.d.ts → summary-report-DM7tlml1.d.ts} +16 -20
  353. package/dist/summary-report-DM7tlml1.d.ts.map +1 -0
  354. package/dist/supervisor-run/index.d.ts +189 -153
  355. package/dist/supervisor-run/index.d.ts.map +1 -1
  356. package/dist/supervisor-run/index.js +58 -450
  357. package/dist/supervisor-run/index.js.map +1 -1
  358. package/dist/{systemone-CpTLAlqb.d.ts → systemone-Bp_Ac0p1.d.ts} +6 -6
  359. package/dist/{systemone-CpTLAlqb.d.ts.map → systemone-Bp_Ac0p1.d.ts.map} +1 -1
  360. package/dist/systemone-protocol.js +1 -1
  361. package/dist/{systemone-review-B_yo1xsd.d.ts → systemone-review-CwE6JGIp.d.ts} +2 -2
  362. package/dist/{systemone-review-B_yo1xsd.d.ts.map → systemone-review-CwE6JGIp.d.ts.map} +1 -1
  363. package/dist/{systemone-review-BZWfrtq1.js → systemone-review-DzdOJ8I6.js} +3 -3
  364. package/dist/{systemone-review-BZWfrtq1.js.map → systemone-review-DzdOJ8I6.js.map} +1 -1
  365. package/dist/systemone.d.ts +5 -5
  366. package/dist/systemone.js +5 -5
  367. package/dist/systemone.js.map +1 -1
  368. package/dist/{task-failure-attributes-DngaLdE4.js → task-failure-attributes-BMsblaDh.js} +3 -3
  369. package/dist/task-failure-attributes-BMsblaDh.js.map +1 -0
  370. package/dist/{terminal-record-BHocwFw1.js → terminal-record-78g6E4vh.js} +17 -13
  371. package/dist/terminal-record-78g6E4vh.js.map +1 -0
  372. package/dist/{tool-groups-mcxOXVlb.d.ts → tool-groups-DyNz2JZ_.d.ts} +3 -3
  373. package/dist/tool-groups-DyNz2JZ_.d.ts.map +1 -0
  374. package/dist/{tool-waste-Wi4g6Ekb.d.ts → tool-waste-CkjzWmeY.d.ts} +53 -56
  375. package/dist/tool-waste-CkjzWmeY.d.ts.map +1 -0
  376. package/dist/{tool-waste-D52PqU9Q.js → tool-waste-DLsjS_p-.js} +5 -5
  377. package/dist/{tool-waste-D52PqU9Q.js.map → tool-waste-DLsjS_p-.js.map} +1 -1
  378. package/dist/{trace-questions-DG7nqLQ7.d.ts → trace-questions-xxwgsuMi.d.ts} +5 -10
  379. package/dist/{trace-questions-DG7nqLQ7.d.ts.map → trace-questions-xxwgsuMi.d.ts.map} +1 -1
  380. package/dist/trace-repair/index.d.ts +15 -212
  381. package/dist/trace-repair/index.d.ts.map +1 -1
  382. package/dist/trace-repair/index.js +865 -987
  383. package/dist/trace-repair/index.js.map +1 -1
  384. package/dist/traces.d.ts +76 -86
  385. package/dist/traces.d.ts.map +1 -1
  386. package/dist/traces.js +13 -13
  387. package/dist/traces.js.map +1 -1
  388. package/dist/trajectory-replay/index.d.ts +3 -117
  389. package/dist/trajectory-replay/index.d.ts.map +1 -1
  390. package/dist/trajectory-replay/index.js +98 -56
  391. package/dist/trajectory-replay/index.js.map +1 -1
  392. package/dist/{tree-rXLW3M_4.js → tree-BGRglkKd.js} +2355 -2414
  393. package/dist/tree-BGRglkKd.js.map +1 -0
  394. package/dist/{types-B3jzCp0p.js → types-BXgASBCL.js} +2 -2
  395. package/dist/{types-B3jzCp0p.js.map → types-BXgASBCL.js.map} +1 -1
  396. package/dist/{types-oKRhQAde.d.ts → types-CW8ewATZ.d.ts} +5 -114
  397. package/dist/types-CW8ewATZ.d.ts.map +1 -0
  398. package/dist/{types-BqkVmHfv.d.ts → types-CmXY0PQf.d.ts} +2 -4
  399. package/dist/types-CmXY0PQf.d.ts.map +1 -0
  400. package/dist/{types-DBZ7keha.d.ts → types-D-mgRzlh.d.ts} +9 -72
  401. package/dist/types-D-mgRzlh.d.ts.map +1 -0
  402. package/dist/{types-B4C5nFw6.d.ts → types-D8GhJoHk.d.ts} +79 -75
  403. package/dist/types-D8GhJoHk.d.ts.map +1 -0
  404. package/dist/{types-IXD2HJzu.js → types-DqFMNTgK.js} +2 -4
  405. package/dist/types-DqFMNTgK.js.map +1 -0
  406. package/dist/{types-CfZMIzB_.d.ts → types-DtMbFw6U.d.ts} +3 -3
  407. package/dist/{types-CfZMIzB_.d.ts.map → types-DtMbFw6U.d.ts.map} +1 -1
  408. package/dist/{verdict-DhPoXApk.js → verdict-BGQmxQ8A.js} +3 -3
  409. package/dist/verdict-BGQmxQ8A.js.map +1 -0
  410. package/dist/{verdict-E4eRNf7-.d.ts → verdict-C24NELtN.d.ts} +2 -15
  411. package/dist/{verdict-E4eRNf7-.d.ts.map → verdict-C24NELtN.d.ts.map} +1 -1
  412. package/dist/{verdict-cache-BQEsVy9n.js → verdict-cache-CL_7tePD.js} +2 -2
  413. package/dist/verdict-cache-CL_7tePD.js.map +1 -0
  414. package/dist/wire/index.d.ts +32 -411
  415. package/dist/wire/index.d.ts.map +1 -1
  416. package/dist/wire/index.js +2 -2
  417. package/docs/changes/jev-evidence-review.md +1 -1
  418. package/docs/changes/jev-product-adoption.md +1 -1
  419. package/docs/changes/jev-recorded-evidence.md +1 -1
  420. package/docs/design/mlbenchmarks-book-review.md +1 -1
  421. package/docs/design/mlbenchmarks-review/probes.mts +23 -31
  422. package/docs/design/statistics-decisions.md +17 -17
  423. package/docs/design.md +1 -1
  424. package/docs/evaluation-integrity.md +1 -1
  425. package/docs/experiment.md +1 -1
  426. package/docs/proofs/campaign-attempt-retention-20261001/README.md +2 -2
  427. package/docs/public-api.md +138 -529
  428. package/docs/rollout.md +1 -1
  429. package/docs/trace-repair-admission.md +1 -1
  430. package/docs/wire-protocol.md +1 -1
  431. package/package.json +1 -1
  432. package/dist/active-curriculum-B_Lc9kfO.js +0 -61
  433. package/dist/active-curriculum-B_Lc9kfO.js.map +0 -1
  434. package/dist/agent-profile-CIA2Atnt.d.ts.map +0 -1
  435. package/dist/agent-profile-cell-BNp-Js9F.d.ts.map +0 -1
  436. package/dist/agent-profile-cell-BYc0bxsL.js.map +0 -1
  437. package/dist/agreement-irr-DsTURxdr.js.map +0 -1
  438. package/dist/analyze-runs-3xpby3p2.js.map +0 -1
  439. package/dist/analyze-runs-CSDAh8A7.d.ts.map +0 -1
  440. package/dist/ast-DJ6Bc7ab.js +0 -558
  441. package/dist/ast-DJ6Bc7ab.js.map +0 -1
  442. package/dist/ast-hI-vjW6J.d.ts +0 -457
  443. package/dist/ast-hI-vjW6J.d.ts.map +0 -1
  444. package/dist/baseline-Dm8966du.js +0 -259
  445. package/dist/baseline-Dm8966du.js.map +0 -1
  446. package/dist/benchmark-BywPeAUo.js +0 -562
  447. package/dist/benchmark-BywPeAUo.js.map +0 -1
  448. package/dist/benchmark-command-CByl9dxG.js.map +0 -1
  449. package/dist/campaign-BQ-Jujuj.js.map +0 -1
  450. package/dist/chat-json-call-BrKsyspg.js.map +0 -1
  451. package/dist/claim-BXcZw-jo.js.map +0 -1
  452. package/dist/claude-jsonl-dZir12m6.js +0 -86
  453. package/dist/claude-jsonl-dZir12m6.js.map +0 -1
  454. package/dist/client-BzL9niqi.d.ts.map +0 -1
  455. package/dist/cost-ledger-CO7UyEcQ.d.ts.map +0 -1
  456. package/dist/cost-ledger-D2FnJPBF.js.map +0 -1
  457. package/dist/counterfactual-CPaWN99-.d.ts +0 -60
  458. package/dist/counterfactual-CPaWN99-.d.ts.map +0 -1
  459. package/dist/counterfactual-DtMWvn7D.js +0 -98
  460. package/dist/counterfactual-DtMWvn7D.js.map +0 -1
  461. package/dist/default-registry-BrWh62zf.d.ts.map +0 -1
  462. package/dist/default-registry-Cn1LQ2Ve.js.map +0 -1
  463. package/dist/define-agent-eval-CEWkX5wZ.d.ts.map +0 -1
  464. package/dist/define-agent-eval-Ch3MWX6k.js.map +0 -1
  465. package/dist/dspy-rlm-engine-D-KX_e64.js.map +0 -1
  466. package/dist/effect-sizes-DiH8MGOH.js +0 -82
  467. package/dist/effect-sizes-DiH8MGOH.js.map +0 -1
  468. package/dist/emitter-HcCmFQl_.d.ts.map +0 -1
  469. package/dist/engine-C_zNWfM8.d.ts.map +0 -1
  470. package/dist/equal-terms-Blx9Zk5I.js.map +0 -1
  471. package/dist/evaluation-Cn2dRZIz.d.ts.map +0 -1
  472. package/dist/exporters-X4xwla8A.js.map +0 -1
  473. package/dist/external-optimizer-process-nnsY5D_4.js.map +0 -1
  474. package/dist/external-optimizer-subprocess-DcerlZSG.js.map +0 -1
  475. package/dist/failure-cluster-CbgEDcFJ.d.ts.map +0 -1
  476. package/dist/feedback-trajectory-DkaDj0dj.d.ts.map +0 -1
  477. package/dist/hf-dataset-BR2yhuNn.js.map +0 -1
  478. package/dist/index-CPJJr98o.d.ts.map +0 -1
  479. package/dist/index-Ch9VxCpK.d.ts.map +0 -1
  480. package/dist/index-DHKEAv3s.d.ts.map +0 -1
  481. package/dist/index-DlH0Y9gI.d.ts.map +0 -1
  482. package/dist/index-NvGBiO8H.d.ts.map +0 -1
  483. package/dist/integrity-BV1ZDf42.js.map +0 -1
  484. package/dist/journal-DvKCvjty.js.map +0 -1
  485. package/dist/judge-gate-B1qYnkpi.js.map +0 -1
  486. package/dist/judge-gate-DNcUrRFA.d.ts.map +0 -1
  487. package/dist/kind-factory-CbZ-6tvu.js.map +0 -1
  488. package/dist/ledger-core-B2hahzrt.js.map +0 -1
  489. package/dist/llm-client-DQveoxSM.js.map +0 -1
  490. package/dist/llm-judge-PXV7Xljl.js.map +0 -1
  491. package/dist/matrix-CHMiebFx.d.ts.map +0 -1
  492. package/dist/matrix-_BV9-Lk5.js.map +0 -1
  493. package/dist/mint-DCGoBYhB.d.ts +0 -155
  494. package/dist/mint-DCGoBYhB.d.ts.map +0 -1
  495. package/dist/normal-DsJZtrJc.js +0 -25
  496. package/dist/normal-DsJZtrJc.js.map +0 -1
  497. package/dist/paired-arms-DzKn0TfY.js.map +0 -1
  498. package/dist/paired-binary-Q36bhn2R.js.map +0 -1
  499. package/dist/paired-promotion-decision-DPKz8ZnF.d.ts.map +0 -1
  500. package/dist/paired-tests-jbwPdSz5.js.map +0 -1
  501. package/dist/prime-protocol-CEE4BR9N.d.ts.map +0 -1
  502. package/dist/prime-protocol-DcAK9pC8.js.map +0 -1
  503. package/dist/produced-state-ZCtDBxAE.js.map +0 -1
  504. package/dist/proposal-findings-BZiMTzAF.js.map +0 -1
  505. package/dist/query-B1D1qp2Z.js.map +0 -1
  506. package/dist/redact-Cveoq6fJ.d.ts.map +0 -1
  507. package/dist/redact-Dk1ZdJEm.js.map +0 -1
  508. package/dist/release-confidence-BK2QCB5M.d.ts.map +0 -1
  509. package/dist/release-confidence-Dbe9AS_X.js.map +0 -1
  510. package/dist/report-command-y8H6RSX8.js.map +0 -1
  511. package/dist/reward-hacking-DHA7c1l8.d.ts.map +0 -1
  512. package/dist/rollout-D9bKKBnZ.js.map +0 -1
  513. package/dist/run-record-D1Hw-Jml.d.ts.map +0 -1
  514. package/dist/run-record-ES07wfz9.js.map +0 -1
  515. package/dist/schema-2g7trUwH.d.ts.map +0 -1
  516. package/dist/schema-BuiujZ-g.js.map +0 -1
  517. package/dist/schema-CSf6qWgZ.js.map +0 -1
  518. package/dist/search-kernel-BztlERLv.js.map +0 -1
  519. package/dist/search-kernel-CpcKH5Ih.d.ts.map +0 -1
  520. package/dist/search-ledger-d7V8o9Lk.js.map +0 -1
  521. package/dist/search-ledger-file-BqOo-bVo.js.map +0 -1
  522. package/dist/search-shipper-er4pK5_Y.js.map +0 -1
  523. package/dist/semantic-concept-judge-zQBG9hHU.js.map +0 -1
  524. package/dist/sequential-D-1PPCmM.js.map +0 -1
  525. package/dist/server-DbcC7xMl.js.map +0 -1
  526. package/dist/skillopt-optimization-method-CCtzralq.js.map +0 -1
  527. package/dist/statistical-heldout-1Gbmex7v.js.map +0 -1
  528. package/dist/statistical-heldout-BeR4bOdg.d.ts.map +0 -1
  529. package/dist/steps-Bg5-KVx1.d.ts.map +0 -1
  530. package/dist/store-otlp-DtB4Slsr.js.map +0 -1
  531. package/dist/store-tool-spans-DR1SrCLK.d.ts.map +0 -1
  532. package/dist/store-tool-spans-fZIigrpP.js.map +0 -1
  533. package/dist/student-t-BQeKJx2N.js +0 -39
  534. package/dist/student-t-BQeKJx2N.js.map +0 -1
  535. package/dist/summary-report-C-CtItCN.d.ts.map +0 -1
  536. package/dist/summary-report-DtdRucM0.js.map +0 -1
  537. package/dist/task-failure-attributes-DngaLdE4.js.map +0 -1
  538. package/dist/terminal-record-BHocwFw1.js.map +0 -1
  539. package/dist/tool-groups-mcxOXVlb.d.ts.map +0 -1
  540. package/dist/tool-waste-Wi4g6Ekb.d.ts.map +0 -1
  541. package/dist/trajectory-Y1fmdlw7.d.ts +0 -24
  542. package/dist/trajectory-Y1fmdlw7.d.ts.map +0 -1
  543. package/dist/tree-rXLW3M_4.js.map +0 -1
  544. package/dist/types-B4C5nFw6.d.ts.map +0 -1
  545. package/dist/types-BqkVmHfv.d.ts.map +0 -1
  546. package/dist/types-DBZ7keha.d.ts.map +0 -1
  547. package/dist/types-IXD2HJzu.js.map +0 -1
  548. package/dist/types-oKRhQAde.d.ts.map +0 -1
  549. package/dist/usage-receipt-Cx3xNeyB.js +0 -76
  550. package/dist/usage-receipt-Cx3xNeyB.js.map +0 -1
  551. package/dist/verdict-DhPoXApk.js.map +0 -1
  552. package/dist/verdict-cache-BQEsVy9n.js.map +0 -1
  553. package/docs/design/self-improvement-evidence-audit.md +0 -263
@@ -1,25 +1,25 @@
1
- import { i as JudgeError, s as ValidationError } from "./errors-Dngq5h35.js";
2
- import { a as hashCanonical, i as compareCodeUnits, r as canonicalString, t as LEDGER_HASH_PATTERN } from "./canonical-BsZVgBhW.js";
3
- import { t as mulberry32 } from "./random-Dn5fPWkt.js";
4
- import { o as summarizeNumberSeries, s as weightedComposite, t as confidenceInterval } from "./descriptive-CnU8Yq29.js";
5
- import { r as minimumPairsForPairedDeltaTest, t as decidePairedPromotion } from "./paired-promotion-decision-BN6IQTiu.js";
6
- import { a as heldoutSignificance, i as dimensionRegressions, o as pairHoldout } from "./statistical-heldout-1Gbmex7v.js";
7
- import { i as modelHasSnapshot, s as searchCellRunId } from "./run-record-DlTZ_lqe.js";
8
- import { n as contentHash } from "./verdict-cache-BQEsVy9n.js";
9
- import { r as projectCampaignCellQuality, t as campaignCellCostProvenance } from "./run-record-ES07wfz9.js";
10
- import { i as CostLedger, t as CostAccountingIncompleteError } from "./cost-ledger-D2FnJPBF.js";
11
- import { B as assertCampaignDesign, D as FinalEvidenceError, E as FinalEvidenceConflictError, F as surfaceHash, G as campaignSplitDigest, H as assertCompleteCampaign, I as surfaceHashMatches, L as BackendIntegrityError, M as renderSurfaceDiff, N as surfaceContentHash, P as surfaceDispatchRef, U as campaignCoverage, V as assertCampaignSplitIdentity, W as campaignScenarioIdentity, b as campaignMeanCompositeOrNull, q as formatCoverageFailures, t as createCampaignEvidenceReceipt, v as campaignBreakdown, y as campaignMeanComposite } from "./campaign-evidence-OcUciD-N.js";
12
- import { m as mapConcurrent } from "./journal-DvKCvjty.js";
1
+ import { i as JudgeError, s as ValidationError } from "./errors-BcOs5MKW.js";
2
+ import { a as hashCanonical, i as compareCodeUnits, r as canonicalString, t as LEDGER_HASH_PATTERN } from "./canonical-Dq2hOQ8L.js";
3
+ import { i as modelHasSnapshot, s as searchCellRunId } from "./run-record-DhEJgoPz.js";
4
+ import { t as mulberry32 } from "./random-DDDcC7YU.js";
5
+ import { o as summarizeNumberSeries, s as weightedComposite, t as confidenceInterval } from "./descriptive-CVNy48rR.js";
6
+ import { r as minimumPairsForPairedDeltaTest } from "./paired-promotion-decision-D0SHb-ZL.js";
7
+ import { a as heldoutSignificance, i as dimensionRegressions, o as pairHoldout } from "./statistical-heldout-Bksv92Th.js";
8
+ import { n as contentHash } from "./verdict-cache-CL_7tePD.js";
9
+ import { r as projectCampaignCellQuality, t as campaignCellCostProvenance } from "./run-record-CysKLB93.js";
10
+ import { n as CostAccountingIncompleteError, t as CostLedger } from "./cost-ledger-D0352Z4d.js";
11
+ import { A as surfaceHashMatches, D as surfaceContentHash, E as renderSurfaceDiff, F as assertCampaignSplitIdentity, I as assertCompleteCampaign, L as campaignCoverage, O as surfaceDispatchRef, P as assertCampaignDesign, R as campaignScenarioIdentity, V as formatCoverageFailures, b as FinalEvidenceConflictError, f as campaignBreakdown, j as BackendIntegrityError, k as surfaceHash, m as campaignMeanCompositeOrNull, p as campaignMeanComposite, x as FinalEvidenceError, z as campaignSplitDigest } from "./campaign-evidence-CQYw8_0H.js";
13
12
  import { t as deepFreezeCanonicalJson } from "./deep-freeze-CurcOQg3.js";
14
- import { N as createRunCostLedger, P as fsCampaignStorage, g as isRecord, h as isExternalTextCandidate } from "./external-optimizer-subprocess-DcerlZSG.js";
15
- import { r as summarizeEvaluationUnits, t as defineEvaluationClaim } from "./claim-BXcZw-jo.js";
16
- import { f as redactText, n as REDACTION_VERSION, o as detectCredential, s as detectPersonalData } from "./redact-Dk1ZdJEm.js";
17
- import { t as detectRewardHacking } from "./reward-hacking-BFcQgkux.js";
18
- import { t as assertProposalFindings } from "./proposal-findings-BZiMTzAF.js";
19
- import { O as paretoFrontier, a as searchPolicyView, i as searchExpansionIndex, k as paretoFrontierWithCrowding, m as EXPANSION_OPERATORS, n as runSearch, t as SEARCH_KERNEL_SOURCE, x as estimateNode } from "./search-kernel-BztlERLv.js";
20
- import { a as validateSearchLedgerEvent, d as searchTaskSetDigest, i as replaySearchLedgerText, l as searchCellId, n as openSearchLedger, u as searchNodeId } from "./search-ledger-d7V8o9Lk.js";
21
- import { n as searchProposerView, t as renderSearchSummary } from "./search-summary-CBvz3u_S.js";
22
- import { C as combineAbortSignals, d as stripFencedJson, o as costReceiptFromLlm, s as costReceiptFromLlmError, u as maximumChargeForLlmRequest } from "./llm-client-DQveoxSM.js";
13
+ import { M as fsCampaignStorage, h as isExternalTextCandidate, j as createRunCostLedger } from "./external-optimizer-subprocess-XIHBOSKt.js";
14
+ import { r as summarizeEvaluationUnits, t as defineEvaluationClaim } from "./claim-BPVuyHuy.js";
15
+ import { d as detectPersonalData, s as redactText, t as REDACTION_VERSION, u as detectCredential } from "./redact-mtkTiJVk.js";
16
+ import { t as detectRewardHacking } from "./reward-hacking-ByuETaIb.js";
17
+ import { t as assertProposalFindings } from "./proposal-findings-BbWidvUk.js";
18
+ import { i as runSearch, m as searchExpansionIndex, n as paretoFrontier, o as searchPolicyView, p as SEARCH_KERNEL_SOURCE, r as paretoFrontierWithCrowding, s as EXPANSION_OPERATORS, w as estimateNode } from "./pareto-BzsxidFD.js";
19
+ import { a as validateSearchLedgerEvent, d as searchNodeId, f as searchTaskSetDigest, i as replaySearchLedgerText, n as openSearchLedger, u as searchCellId } from "./search-ledger-TUvB5gWH.js";
20
+ import { n as searchProposerView, t as renderSearchSummary } from "./search-summary-DQIvJsSB.js";
21
+ import { i as stripFencedJson, n as isRecord } from "./json-BhdsJaOQ.js";
22
+ import { a as maximumChargeForLlmRequest, i as costReceiptFromLlmError, r as costReceiptFromLlm } from "./llm-client-BshCREn6.js";
23
23
  import { createHash } from "node:crypto";
24
24
  import { writeFileSync } from "node:fs";
25
25
  import { z } from "zod";
@@ -1200,8 +1200,8 @@ function detectCalibrationDrift(runs, opts) {
1200
1200
  alpha,
1201
1201
  recentN: recent.length,
1202
1202
  historyN: historical.length,
1203
- recentMean: mean$2(recent),
1204
- historyMean: mean$2(historical)
1203
+ recentMean: mean$1(recent),
1204
+ historyMean: mean$1(historical)
1205
1205
  }
1206
1206
  }],
1207
1207
  evaluation: {
@@ -1438,7 +1438,7 @@ function chiSquareCritical(df, alpha) {
1438
1438
  }
1439
1439
  return TABLE[10][idx];
1440
1440
  }
1441
- function mean$2(xs) {
1441
+ function mean$1(xs) {
1442
1442
  if (xs.length === 0) return 0;
1443
1443
  return xs.reduce((s, x) => s + x, 0) / xs.length;
1444
1444
  }
@@ -3190,7 +3190,6 @@ function nonEmpty(value, label) {
3190
3190
  * and a fact already in the ledger is not appended again, so a resumed search
3191
3191
  * continues its ledger instead of conflicting with its own history.
3192
3192
  */
3193
- const MAX_LABEL_CHARS = 200;
3194
3193
  /** The default claim: development numbers over exactly the supplied tasks. */
3195
3194
  function developmentClaim(populationId) {
3196
3195
  return {
@@ -3554,7 +3553,7 @@ function declaredSurfaces(surface, artifact) {
3554
3553
  }
3555
3554
  function boundedLabel(label) {
3556
3555
  const redacted = redactText(label.trim(), { profile: "share" });
3557
- return redacted.length <= MAX_LABEL_CHARS ? redacted : `${redacted.slice(0, 199)}…`;
3556
+ return redacted.length <= 200 ? redacted : `${redacted.slice(0, 199)}…`;
3558
3557
  }
3559
3558
  /** The header without its stamp, for comparing a resumed opening to the stored one. */
3560
3559
  function headerMaterial(header) {
@@ -3709,7 +3708,7 @@ function crowdedFrontierParent(options) {
3709
3708
  });
3710
3709
  }
3711
3710
  /**
3712
- * AIDE's search policy (`aide/agent.py`, `search_policy`) with three changes
3711
+ * AIDE's search_policy with three changes
3713
3712
  * for noisy scores.
3714
3713
  *
3715
3714
  * 1. **Draft** while fewer than `drafts` nodes were drafted: the root is the
@@ -3968,7 +3967,7 @@ function hillClimb(spec) {
3968
3967
  evidence: {
3969
3968
  ...chosen.evidence,
3970
3969
  units: units.length,
3971
- ...units.length > 0 ? { mean: mean$1(units.map((unit) => unit.mean)) } : {},
3970
+ ...units.length > 0 ? { mean: mean(units.map((unit) => unit.mean)) } : {},
3972
3971
  isLeader: chosen.nodeId === kept ? 1 : 0
3973
3972
  }
3974
3973
  }
@@ -4024,7 +4023,7 @@ function paretoMembers(view) {
4024
4023
  members.push({
4025
4024
  nodeId,
4026
4025
  ordinal,
4027
- mean: sign * mean$1([...objectives.values()]),
4026
+ mean: sign * mean([...objectives.values()]),
4028
4027
  objectives,
4029
4028
  distance: 0
4030
4029
  });
@@ -4046,7 +4045,7 @@ function compareCrowded(a, b) {
4046
4045
  if (a.mean !== b.mean) return a.mean > b.mean ? -1 : 1;
4047
4046
  return a.ordinal - b.ordinal;
4048
4047
  }
4049
- function mean$1(values) {
4048
+ function mean(values) {
4050
4049
  let total = 0;
4051
4050
  for (const value of values) total += value;
4052
4051
  return total / values.length;
@@ -4058,285 +4057,9 @@ function mean$1(values) {
4058
4057
  function isProposedCandidate(value) {
4059
4058
  return typeof value === "object" && value !== null && "surface" in value && "label" in value && "rationale" in value;
4060
4059
  }
4061
- const LABEL_TRUST_RANK = {
4062
- unverified: 0,
4063
- "verified-signal": 1,
4064
- "human-rated": 2
4065
- };
4066
- /** Ordinal rank for a label-trust tier; absent ⇒ `unverified` (rank 0). */
4067
- function labelTrustRank(trust) {
4068
- return LABEL_TRUST_RANK[trust ?? "unverified"];
4069
- }
4070
4060
  //#endregion
4071
- //#region src/campaign/presets/run-optimization.ts
4072
- /**
4073
- * `runOptimization` improves a mutable surface with a caller-owned proposer,
4074
- * as a search on the one kernel (`runSearch`).
4075
- *
4076
- * The baseline is the seeded root. The scenarios are the train split: the
4077
- * proposer reads their scores, and the policy ranks on them because the loop
4078
- * has no private selection split. A "generation" is one proposal of
4079
- * `populationSize` candidates from the node the policy chose (by default the
4080
- * incumbent, once every earlier candidate is measured). Every candidate is a
4081
- * node measured on every scenario at every repeat, one cell at a time on the
4082
- * run's lanes. The kept surface is the policy's leader: a budget decision on
4083
- * the train split that claims nothing; `runImprovementLoop` compares it with
4084
- * the baseline on separate cases.
4085
- *
4086
- * The search ledger is always written, beside the run (`<runDir>/search/`) or
4087
- * where `searchLedger` puts it, and it is the run's only checkpoint: running
4088
- * again on the same run directory and options continues an interrupted run
4089
- * instead of starting over, and reruns nothing that settled.
4090
- */
4091
- /** Improve a surface as a search on the kernel; see the module comment. */
4092
- async function runOptimization(opts) {
4093
- const candidateConcurrency = opts.candidateConcurrency ?? 1;
4094
- if (typeof opts.runDir !== "string" || opts.runDir.trim().length === 0) throw new Error("runOptimization: runDir is required and must be a non-empty string");
4095
- if (!Number.isInteger(candidateConcurrency) || candidateConcurrency < 1) throw new Error("runOptimization: candidateConcurrency must be a positive integer");
4096
- for (const [name, value] of [["populationSize", opts.populationSize], ["maxGenerations", opts.maxGenerations]]) if (!Number.isSafeInteger(value) || value < 0) throw new Error(`runOptimization: ${name} must be a non-negative integer`);
4097
- const initialFindings = immutableProposalSnapshot(assertProposalFindings(opts.findings ?? [], "runOptimization initial proposal findings"), "initial findings");
4098
- const baselineSurface = immutableProposalSnapshot(opts.baselineSurface, "baseline surface");
4099
- const runDir = resolveRunDir(opts.runDir, opts.repo);
4100
- const storage = opts.storage ?? fsCampaignStorage();
4101
- const costLedger = opts.costLedger ?? createRunCostLedger({
4102
- storage,
4103
- runDir,
4104
- costCeilingUsd: opts.costCeiling
4105
- });
4106
- const reps = opts.reps ?? 1;
4107
- const seed = opts.seed ?? 42;
4108
- const requireJudgeScore = (opts.judges?.length ?? 0) > 0;
4109
- const premeasured = opts.premeasuredBaseline ? validatedPremeasuredBaseline({
4110
- input: opts.premeasuredBaseline,
4111
- baselineSurface,
4112
- scenarios: opts.scenarios,
4113
- reps,
4114
- seed,
4115
- judges: opts.judges ?? [],
4116
- dispatchRef: surfaceDispatchRef(baselineSurface, opts.dispatchRef),
4117
- requireJudgeScore
4118
- }) : void 0;
4119
- const splitDigest = campaignSplitDigest(opts.scenarios, reps);
4120
- const binding = opts.searchLedger ?? await defaultBinding(opts, runDir, storage);
4121
- const { identity } = binding;
4122
- const execution = {
4123
- model: identity.model,
4124
- agent: identity.agent,
4125
- benchmark: {
4126
- uri: "campaign://scenarios",
4127
- revision: splitDigest
4128
- }
4129
- };
4130
- const policy = opts.policy ?? incumbent();
4131
- const allocation = uniform({ reps });
4132
- const recorder = await SearchRecorder.open({
4133
- ledger: binding.ledger,
4134
- storage
4135
- }, {
4136
- subject: identity.subject ?? opts.proposer.kind,
4137
- process: {
4138
- name: "runOptimization",
4139
- executionRef: identity.search
4140
- },
4141
- artifactKind: artifactKindOf(baselineSurface),
4142
- objective: {
4143
- metric: "composite",
4144
- direction: "maximize",
4145
- judge: identity.judge ?? { unknown: "runOptimization receives its judges as functions without a pinned source" },
4146
- claim: identity.claim ?? developmentClaim(splitDigest)
4147
- },
4148
- splits: {
4149
- train: opts.scenarios.map((scenario) => ({
4150
- taskId: scenario.id,
4151
- unitId: scenario.id,
4152
- source: {
4153
- uri: `scenario://${scenario.id}`,
4154
- revision: hashCanonical(scenario)
4155
- }
4156
- })),
4157
- selection: [],
4158
- test: [],
4159
- heldOutUnits: true
4160
- },
4161
- policy: {
4162
- expansion: policy.name,
4163
- allocation: allocation.name,
4164
- seed
4165
- },
4166
- budget: {
4167
- maxUsd: opts.costCeiling ?? null,
4168
- maxCells: null,
4169
- maxNodes: 1 + opts.populationSize * opts.maxGenerations,
4170
- deadline: null,
4171
- maxConcurrency: null,
4172
- reservedClaimUsd: 0
4173
- },
4174
- containment: null,
4175
- derivedFrom: null,
4176
- identity: execution
4177
- });
4178
- const nodes = new SurfaceNodes({
4179
- opts,
4180
- recorder,
4181
- runDir,
4182
- costLedger,
4183
- execution,
4184
- premeasured
4185
- });
4186
- const scenarioById = new Map(opts.scenarios.map((scenario) => [scenario.id, scenario]));
4187
- let findings = initialFindings;
4188
- const executor = {
4189
- lanes: () => [{
4190
- name: "campaign",
4191
- capacity: candidateConcurrency * (opts.maxConcurrency ?? 2),
4192
- costCap: "estimate",
4193
- cellUsd: 0
4194
- }],
4195
- place: () => "campaign",
4196
- run: (work) => nodes.runCell(work, scenarioById),
4197
- adopt: (work) => nodes.adoptCell(work)
4198
- };
4199
- const proposer = opts.proposer;
4200
- const proposerSource = identity.proposer.source;
4201
- const port = {
4202
- name: proposer.kind,
4203
- kind: "optimizer",
4204
- source: proposerSource,
4205
- execution: identity.proposer,
4206
- childrenPerProposal: opts.populationSize,
4207
- async propose(request) {
4208
- const phases = [
4209
- "search.proposal",
4210
- "analysis.baseline",
4211
- "analysis.generation"
4212
- ];
4213
- const before = new Set(phases.flatMap((phase) => costLedger.list({ phase })).map(receiptKey));
4214
- const accounting = () => {
4215
- const fresh = phases.flatMap((phase) => costLedger.list({ phase })).filter((receipt) => !before.has(receiptKey(receipt)));
4216
- return {
4217
- accounting: searchReceiptAccounting(fresh),
4218
- execution: opts.searchLedger ? identity.proposer : searchProposalExecution(fresh, identity.model.provider, proposerSource)
4219
- };
4220
- };
4221
- const earlier = await recorder.state();
4222
- const root = earlier.rootNodeId;
4223
- const rootCampaign = nodes.campaign(earlier, root);
4224
- const rootCoverage = campaignCoverage(rootCampaign.cells, opts.scenarios, reps, requireJudgeScore);
4225
- if (!rootCoverage.complete) throw new Error(`runOptimization: ${premeasured ? "premeasured baseline" : "baseline"} is incomplete (${rootCoverage.scorableCellIds.length}/${rootCoverage.expectedCellIds.length} designed cells scorable) — ${formatCoverageFailures(rootCoverage)}. Refusing to optimize against an incomplete incumbent.`);
4226
- const generation = nodes.generations(earlier).length;
4227
- if (opts.analyzeGeneration) {
4228
- const previous = generation - 1;
4229
- const candidates = (previous < 0 ? [root] : nodes.expansionNodes(earlier, nodes.generations(earlier)[previous])).map((nodeId) => {
4230
- const campaign = nodes.campaign(earlier, nodeId);
4231
- return {
4232
- surfaceHash: surfaceHash(nodes.surface(earlier, nodeId)),
4233
- campaign,
4234
- composite: campaignMeanCompositeOrNull(campaign)
4235
- };
4236
- });
4237
- const history = nodes.history(earlier, policy);
4238
- if (candidates.some(({ campaign }) => campaign.cells.length > 0)) {
4239
- const fresh = await opts.analyzeGeneration({
4240
- generation: previous,
4241
- runDir: previous < 0 ? rootCampaign.runDir : `${runDir}/candidates`,
4242
- candidates,
4243
- history,
4244
- costLedger,
4245
- costPhase: previous < 0 ? "analysis.baseline" : "analysis.generation"
4246
- });
4247
- if (!Array.isArray(fresh)) throw new TypeError("runOptimization: analyzeGeneration must return an array");
4248
- findings = immutableProposalSnapshot(assertProposalFindings(fresh, "runOptimization analysis findings"), "analysis findings");
4249
- }
4250
- }
4251
- const state = await recorder.state();
4252
- const proposalHistory = immutableProposalSnapshot(nodes.history(state, policy), "history");
4253
- const parent = request.parents[0];
4254
- const context = Object.freeze({
4255
- currentSurface: immutableProposalSnapshot(parent.artifact, "current surface"),
4256
- operator: request.operator,
4257
- parents: immutableProposalSnapshot(request.parents.map(({ nodeId, artifact }) => ({
4258
- nodeId,
4259
- artifact: immutableProposalSnapshot(artifact, "parent surface")
4260
- })), "parents"),
4261
- train: searchProposerView(state),
4262
- summary: renderSearchSummary(state, { split: "train" }),
4263
- history: proposalHistory,
4264
- findings: immutableProposalSnapshot(assertProposalFindings(findings, "runOptimization proposal findings"), "findings"),
4265
- populationSize: opts.populationSize,
4266
- generation,
4267
- signal: request.signal,
4268
- baselineOutcome: immutableProposalSnapshot(nodes.outcome(state, root), "baseline"),
4269
- incumbentOutcome: immutableProposalSnapshot(nodes.outcome(state, request.leader), "incumbent outcome"),
4270
- parentOutcome: immutableProposalSnapshot(nodes.outcome(state, parent.nodeId), "parent outcome"),
4271
- maxImprovementShots: opts.maxImprovementShots,
4272
- paretoParents: immutableProposalSnapshot(nodes.frontier(state), "Pareto parents"),
4273
- costLedger,
4274
- costPhase: "search.proposal"
4275
- });
4276
- if (proposer.decide?.({ history: proposalHistory }).stop) return {
4277
- children: [],
4278
- stop: "the proposer decided to stop",
4279
- ...accounting()
4280
- };
4281
- const proposed = await proposer.propose(context);
4282
- if (!Array.isArray(proposed)) throw new TypeError("runOptimization: proposer must return an array");
4283
- return {
4284
- children: immutableProposalSnapshot(proposed, "candidate outputs").slice(0, opts.populationSize).map((candidate) => isProposedCandidate(candidate) ? {
4285
- artifact: candidate.surface,
4286
- label: candidate.label,
4287
- rationale: candidate.rationale,
4288
- ...candidate.attribution ? { attribution: candidate.attribution } : {}
4289
- } : {
4290
- artifact: candidate,
4291
- label: "",
4292
- rationale: ""
4293
- }),
4294
- ...accounting()
4295
- };
4296
- }
4297
- };
4298
- const result = await runSearch({
4299
- recorder,
4300
- root: baselineSurface,
4301
- codec: surfaceCodec,
4302
- policy,
4303
- allocation,
4304
- proposer: port,
4305
- executor,
4306
- maxExpansions: opts.maxGenerations,
4307
- signal: opts.signal
4308
- });
4309
- const state = result.state;
4310
- const winnerSurface = nodes.surface(state, result.leader);
4311
- const firstEdge = state.edge(state.node(result.leader).edgeIds[0]);
4312
- const winnerRationale = firstEdge.operator === "seed" || !("sha256" in firstEdge.rationale) ? void 0 : recorder.readBlob(firstEdge.rationale).text;
4313
- const expansions = nodes.generations(state);
4314
- return {
4315
- generations: nodes.history(state, policy).map((record) => ({
4316
- record,
4317
- surfaces: nodes.expansionNodes(state, expansions[record.generationIndex]).map((nodeId) => {
4318
- const surface = nodes.surface(state, nodeId);
4319
- return {
4320
- surfaceHash: surfaceHash(surface),
4321
- surface,
4322
- campaign: nodes.campaign(state, nodeId)
4323
- };
4324
- })
4325
- })),
4326
- baselineSurface,
4327
- winnerSurface,
4328
- winnerSurfaceHash: surfaceHash(winnerSurface),
4329
- ...firstEdge.operator !== "seed" && firstEdge.label ? { winnerLabel: firstEdge.label } : {},
4330
- ...winnerRationale ? { winnerRationale } : {},
4331
- baselineCampaign: nodes.campaign(state, state.rootNodeId),
4332
- cost: costLedger.summary(),
4333
- searchHistory: recorder.receipt({
4334
- producerId: proposer.kind,
4335
- runId: runDir
4336
- }),
4337
- paretoFrontier: nodes.frontier(state)
4338
- };
4339
- }
4061
+ //#region src/campaign/presets/optimization-surface-nodes.ts
4062
+ /** Surface-node campaigns, cached cell adoption, and proposer history from the search ledger. */
4340
4063
  /** A mutable surface as a search artifact. */
4341
4064
  const surfaceCodec = {
4342
4065
  node: surfaceNode,
@@ -4364,14 +4087,14 @@ var SurfaceNodes = class {
4364
4087
  proposals = /* @__PURE__ */ new Map();
4365
4088
  /** Cells this process ran or adopted, by node, keyed `scenarioId:rep`. */
4366
4089
  cells = /* @__PURE__ */ new Map();
4367
- constructor(input) {
4368
- this.opts = input.opts;
4369
- this.recorder = input.recorder;
4370
- this.runDir = input.runDir;
4371
- this.costLedger = input.costLedger;
4372
- this.execution = input.execution;
4373
- this.premeasured = input.premeasured;
4374
- this.storage = input.opts.storage ?? fsCampaignStorage();
4090
+ constructor(opts, recorder, runDir, costLedger, execution, premeasured) {
4091
+ this.opts = opts;
4092
+ this.recorder = recorder;
4093
+ this.runDir = runDir;
4094
+ this.costLedger = costLedger;
4095
+ this.execution = execution;
4096
+ this.premeasured = premeasured;
4097
+ this.storage = opts.storage ?? fsCampaignStorage();
4375
4098
  }
4376
4099
  /** `<runDir>/baseline` for the root, `<runDir>/candidates/<surfaceHash>`
4377
4100
  * for every other node: content-addressed, so a cell's directory needs no
@@ -4406,7 +4129,10 @@ var SurfaceNodes = class {
4406
4129
  const cell = cached === void 0 ? void 0 : JSON.parse(cached);
4407
4130
  if (cell?.manifestHash !== this.manifestFor(work.artifact)) return null;
4408
4131
  this.remember(work.nodeId, cell);
4409
- return this.cellResult(cell, work);
4132
+ return campaignCellSearchResult(cell, {
4133
+ execution: this.execution,
4134
+ lane: work.lane
4135
+ });
4410
4136
  }
4411
4137
  /** Run one cell as a one-cell campaign in its node's directory; a cached
4412
4138
  * cell is read back, not dispatched again. */
@@ -4421,7 +4147,10 @@ var SurfaceNodes = class {
4421
4147
  })).cells[0];
4422
4148
  if (!cell) throw new Error(`runOptimization: cell ${work.cellId} produced no campaign cell`);
4423
4149
  this.remember(work.nodeId, cell);
4424
- return this.cellResult(cell, work);
4150
+ return campaignCellSearchResult(cell, {
4151
+ execution: this.execution,
4152
+ lane: work.lane
4153
+ });
4425
4154
  }
4426
4155
  /** The node's campaign over the cells the ledger settled: the cells this
4427
4156
  * process ran, else their cached results or failure receipts. */
@@ -4589,7 +4318,10 @@ var SurfaceNodes = class {
4589
4318
  premeasuredCell(work) {
4590
4319
  const cell = this.premeasured.cells.find((candidate) => candidate.scenarioId === work.taskId && candidate.rep === work.rep);
4591
4320
  if (!cell) throw new Error(`runOptimization: premeasured baseline lacks ${work.taskId}:${work.rep}`);
4592
- return this.cellResult(cell, work);
4321
+ return campaignCellSearchResult(cell, {
4322
+ execution: this.execution,
4323
+ lane: work.lane
4324
+ });
4593
4325
  }
4594
4326
  /** A cell's result as its campaign stored it for this surface: the cache of
4595
4327
  * a scored cell, the final failure receipt of a failed one. A result another
@@ -4606,34 +4338,334 @@ var SurfaceNodes = class {
4606
4338
  const cell = result !== void 0 ? JSON.parse(result) : receipt !== void 0 ? JSON.parse(receipt).cell : void 0;
4607
4339
  return cell?.manifestHash === manifestHash ? cell : void 0;
4608
4340
  }
4609
- const cached = this.storage.read(cellCachePath(dir, cellId));
4610
- const scored = cached === void 0 ? void 0 : JSON.parse(cached);
4611
- if (scored?.manifestHash === manifestHash) return scored;
4612
- const receipt = this.storage.read(join(cellDirectory(dir, cellId), "failure-receipt.json"));
4613
- const failed = receipt === void 0 ? void 0 : JSON.parse(receipt).cell;
4614
- return failed?.manifestHash === manifestHash ? failed : void 0;
4615
- }
4616
- manifestFor(surface) {
4617
- return computeManifestHash({
4618
- scenarios: this.opts.scenarios,
4619
- judges: this.opts.judges ?? [],
4620
- dispatchRef: surfaceDispatchRef(surface, this.opts.dispatchRef),
4621
- seed: this.opts.seed ?? 42,
4622
- reps: this.opts.reps ?? 1
4623
- });
4624
- }
4625
- remember(nodeId, cell) {
4626
- const cells = this.cells.get(nodeId) ?? /* @__PURE__ */ new Map();
4627
- cells.set(`${cell.scenarioId}:${cell.rep}`, cell);
4628
- this.cells.set(nodeId, cells);
4629
- }
4630
- cellResult(cell, work) {
4631
- return campaignCellSearchResult(cell, {
4632
- execution: this.execution,
4633
- lane: work.lane
4634
- });
4635
- }
4636
- };
4341
+ const cached = this.storage.read(cellCachePath(dir, cellId));
4342
+ const scored = cached === void 0 ? void 0 : JSON.parse(cached);
4343
+ if (scored?.manifestHash === manifestHash) return scored;
4344
+ const receipt = this.storage.read(join(cellDirectory(dir, cellId), "failure-receipt.json"));
4345
+ const failed = receipt === void 0 ? void 0 : JSON.parse(receipt).cell;
4346
+ return failed?.manifestHash === manifestHash ? failed : void 0;
4347
+ }
4348
+ manifestFor(surface) {
4349
+ return computeManifestHash({
4350
+ scenarios: this.opts.scenarios,
4351
+ judges: this.opts.judges ?? [],
4352
+ dispatchRef: surfaceDispatchRef(surface, this.opts.dispatchRef),
4353
+ seed: this.opts.seed ?? 42,
4354
+ reps: this.opts.reps ?? 1
4355
+ });
4356
+ }
4357
+ remember(nodeId, cell) {
4358
+ const cells = this.cells.get(nodeId) ?? /* @__PURE__ */ new Map();
4359
+ cells.set(`${cell.scenarioId}:${cell.rep}`, cell);
4360
+ this.cells.set(nodeId, cells);
4361
+ }
4362
+ };
4363
+ /** A scored campaign as a frontier member: per-scenario composite objectives. */
4364
+ function toParetoParent(surface, campaign, generation, label) {
4365
+ const objectives = {};
4366
+ for (const { scenarioId, composite } of campaignBreakdown(campaign).scenarios) objectives[scenarioId] = composite;
4367
+ const parent = {
4368
+ surface,
4369
+ surfaceHash: surfaceHash(surface),
4370
+ objectives,
4371
+ composite: campaignMeanComposite(campaign),
4372
+ generation
4373
+ };
4374
+ if (label) parent.label = label;
4375
+ return parent;
4376
+ }
4377
+ /** The non-dominated set over the per-scenario objective vectors. */
4378
+ function computeParetoFrontier(scored) {
4379
+ if (scored.length <= 1) return [...scored];
4380
+ const ids = /* @__PURE__ */ new Set();
4381
+ for (const p of scored) for (const id of Object.keys(p.objectives)) ids.add(id);
4382
+ if (ids.size === 0) return [...scored];
4383
+ const objectives = [...ids].map((id) => ({
4384
+ name: id,
4385
+ direction: "maximize",
4386
+ value: (p) => p.objectives[id] ?? Number.NEGATIVE_INFINITY
4387
+ }));
4388
+ return paretoFrontier(scored, objectives).frontier;
4389
+ }
4390
+ function toScoredSurfaceOutcome(hash, campaign, coverage, generation) {
4391
+ const breakdown = campaignBreakdown(campaign);
4392
+ return {
4393
+ split: "search",
4394
+ generation,
4395
+ surfaceHash: hash,
4396
+ composite: campaignMeanComposite(campaign),
4397
+ dimensions: breakdown.dimensions,
4398
+ scenarios: breakdown.scenarios,
4399
+ coverage: {
4400
+ expectedCells: coverage.expectedCellIds.length,
4401
+ scorableCells: coverage.scorableCellIds.length
4402
+ }
4403
+ };
4404
+ }
4405
+ //#endregion
4406
+ //#region src/campaign/presets/run-optimization.ts
4407
+ /**
4408
+ * `runOptimization` improves a mutable surface with a caller-owned proposer,
4409
+ * as a search on the one kernel (`runSearch`).
4410
+ *
4411
+ * The baseline is the seeded root. The scenarios are the train split: the
4412
+ * proposer reads their scores, and the policy ranks on them because the loop
4413
+ * has no private selection split. A "generation" is one proposal of
4414
+ * `populationSize` candidates from the node the policy chose (by default the
4415
+ * incumbent, once every earlier candidate is measured). Every candidate is a
4416
+ * node measured on every scenario at every repeat, one cell at a time on the
4417
+ * run's lanes. The kept surface is the policy's leader: a budget decision on
4418
+ * the train split that claims nothing; `runImprovementLoop` compares it with
4419
+ * the baseline on separate cases.
4420
+ *
4421
+ * The search ledger is always written, beside the run (`<runDir>/search/`) or
4422
+ * where `searchLedger` puts it, and it is the run's only checkpoint: running
4423
+ * again on the same run directory and options continues an interrupted run
4424
+ * instead of starting over, and reruns nothing that settled.
4425
+ */
4426
+ /** Compose the optimization proposer and cell executor on the shared search kernel. */
4427
+ /** Improve a surface as a search on the kernel; see the module comment. */
4428
+ async function runOptimization(opts) {
4429
+ const candidateConcurrency = opts.candidateConcurrency ?? 1;
4430
+ if (typeof opts.runDir !== "string" || opts.runDir.trim().length === 0) throw new Error("runOptimization: runDir is required and must be a non-empty string");
4431
+ if (!Number.isInteger(candidateConcurrency) || candidateConcurrency < 1) throw new Error("runOptimization: candidateConcurrency must be a positive integer");
4432
+ for (const [name, value] of [["populationSize", opts.populationSize], ["maxGenerations", opts.maxGenerations]]) if (!Number.isSafeInteger(value) || value < 0) throw new Error(`runOptimization: ${name} must be a non-negative integer`);
4433
+ const initialFindings = immutableProposalSnapshot(assertProposalFindings(opts.findings ?? [], "runOptimization initial proposal findings"), "initial findings");
4434
+ const baselineSurface = immutableProposalSnapshot(opts.baselineSurface, "baseline surface");
4435
+ const runDir = resolveRunDir(opts.runDir, opts.repo);
4436
+ const storage = opts.storage ?? fsCampaignStorage();
4437
+ const costLedger = opts.costLedger ?? createRunCostLedger({
4438
+ storage,
4439
+ runDir,
4440
+ costCeilingUsd: opts.costCeiling
4441
+ });
4442
+ const reps = opts.reps ?? 1;
4443
+ const seed = opts.seed ?? 42;
4444
+ const requireJudgeScore = (opts.judges?.length ?? 0) > 0;
4445
+ const premeasured = opts.premeasuredBaseline ? validatedPremeasuredBaseline({
4446
+ input: opts.premeasuredBaseline,
4447
+ baselineSurface,
4448
+ scenarios: opts.scenarios,
4449
+ reps,
4450
+ seed,
4451
+ judges: opts.judges ?? [],
4452
+ dispatchRef: surfaceDispatchRef(baselineSurface, opts.dispatchRef),
4453
+ requireJudgeScore
4454
+ }) : void 0;
4455
+ const splitDigest = campaignSplitDigest(opts.scenarios, reps);
4456
+ const binding = opts.searchLedger ?? await defaultBinding(opts, runDir, storage);
4457
+ const { identity } = binding;
4458
+ const execution = {
4459
+ model: identity.model,
4460
+ agent: identity.agent,
4461
+ benchmark: {
4462
+ uri: "campaign://scenarios",
4463
+ revision: splitDigest
4464
+ }
4465
+ };
4466
+ const policy = opts.policy ?? incumbent();
4467
+ const allocation = uniform({ reps });
4468
+ const recorder = await SearchRecorder.open({
4469
+ ledger: binding.ledger,
4470
+ storage
4471
+ }, {
4472
+ subject: identity.subject ?? opts.proposer.kind,
4473
+ process: {
4474
+ name: "runOptimization",
4475
+ executionRef: identity.search
4476
+ },
4477
+ artifactKind: typeof baselineSurface === "object" && baselineSurface.kind === "code" ? "code" : "prompt",
4478
+ objective: {
4479
+ metric: "composite",
4480
+ direction: "maximize",
4481
+ judge: identity.judge ?? { unknown: "runOptimization receives its judges as functions without a pinned source" },
4482
+ claim: identity.claim ?? developmentClaim(splitDigest)
4483
+ },
4484
+ splits: {
4485
+ train: opts.scenarios.map((scenario) => ({
4486
+ taskId: scenario.id,
4487
+ unitId: scenario.id,
4488
+ source: {
4489
+ uri: `scenario://${scenario.id}`,
4490
+ revision: hashCanonical(scenario)
4491
+ }
4492
+ })),
4493
+ selection: [],
4494
+ test: [],
4495
+ heldOutUnits: true
4496
+ },
4497
+ policy: {
4498
+ expansion: policy.name,
4499
+ allocation: allocation.name,
4500
+ seed
4501
+ },
4502
+ budget: {
4503
+ maxUsd: opts.costCeiling ?? null,
4504
+ maxCells: null,
4505
+ maxNodes: 1 + opts.populationSize * opts.maxGenerations,
4506
+ deadline: null,
4507
+ maxConcurrency: null,
4508
+ reservedClaimUsd: 0
4509
+ },
4510
+ containment: null,
4511
+ derivedFrom: null,
4512
+ identity: execution
4513
+ });
4514
+ const nodes = new SurfaceNodes(opts, recorder, runDir, costLedger, execution, premeasured);
4515
+ const scenarioById = new Map(opts.scenarios.map((scenario) => [scenario.id, scenario]));
4516
+ let findings = initialFindings;
4517
+ const executor = {
4518
+ lanes: () => [{
4519
+ name: "campaign",
4520
+ capacity: candidateConcurrency * (opts.maxConcurrency ?? 2),
4521
+ costCap: "estimate",
4522
+ cellUsd: 0
4523
+ }],
4524
+ place: () => "campaign",
4525
+ run: (work) => nodes.runCell(work, scenarioById),
4526
+ adopt: (work) => nodes.adoptCell(work)
4527
+ };
4528
+ const proposer = opts.proposer;
4529
+ const proposerSource = identity.proposer.source;
4530
+ const port = {
4531
+ name: proposer.kind,
4532
+ kind: "optimizer",
4533
+ source: proposerSource,
4534
+ execution: identity.proposer,
4535
+ childrenPerProposal: opts.populationSize,
4536
+ async propose(request) {
4537
+ const phases = [
4538
+ "search.proposal",
4539
+ "analysis.baseline",
4540
+ "analysis.generation"
4541
+ ];
4542
+ const before = new Set(phases.flatMap((phase) => costLedger.list({ phase })).map((receipt) => receipt.callId));
4543
+ const accounting = () => {
4544
+ const fresh = phases.flatMap((phase) => costLedger.list({ phase })).filter((receipt) => !before.has(receipt.callId));
4545
+ return {
4546
+ accounting: searchReceiptAccounting(fresh),
4547
+ execution: opts.searchLedger ? identity.proposer : searchProposalExecution(fresh, identity.model.provider, proposerSource)
4548
+ };
4549
+ };
4550
+ const earlier = await recorder.state();
4551
+ const root = earlier.rootNodeId;
4552
+ const rootCampaign = nodes.campaign(earlier, root);
4553
+ const rootCoverage = campaignCoverage(rootCampaign.cells, opts.scenarios, reps, requireJudgeScore);
4554
+ if (!rootCoverage.complete) throw new Error(`runOptimization: ${premeasured ? "premeasured baseline" : "baseline"} is incomplete (${rootCoverage.scorableCellIds.length}/${rootCoverage.expectedCellIds.length} designed cells scorable) — ${formatCoverageFailures(rootCoverage)}. Refusing to optimize against an incomplete incumbent.`);
4555
+ const generation = nodes.generations(earlier).length;
4556
+ if (opts.analyzeGeneration) {
4557
+ const previous = generation - 1;
4558
+ const candidates = (previous < 0 ? [root] : nodes.expansionNodes(earlier, nodes.generations(earlier)[previous])).map((nodeId) => {
4559
+ const campaign = nodes.campaign(earlier, nodeId);
4560
+ return {
4561
+ surfaceHash: surfaceHash(nodes.surface(earlier, nodeId)),
4562
+ campaign,
4563
+ composite: campaignMeanCompositeOrNull(campaign)
4564
+ };
4565
+ });
4566
+ const history = nodes.history(earlier, policy);
4567
+ if (candidates.some(({ campaign }) => campaign.cells.length > 0)) {
4568
+ const fresh = await opts.analyzeGeneration({
4569
+ generation: previous,
4570
+ runDir: previous < 0 ? rootCampaign.runDir : `${runDir}/candidates`,
4571
+ candidates,
4572
+ history,
4573
+ costLedger,
4574
+ costPhase: previous < 0 ? "analysis.baseline" : "analysis.generation"
4575
+ });
4576
+ if (!Array.isArray(fresh)) throw new TypeError("runOptimization: analyzeGeneration must return an array");
4577
+ findings = immutableProposalSnapshot(assertProposalFindings(fresh, "runOptimization analysis findings"), "analysis findings");
4578
+ }
4579
+ }
4580
+ const state = await recorder.state();
4581
+ const proposalHistory = immutableProposalSnapshot(nodes.history(state, policy), "history");
4582
+ const parent = request.parents[0];
4583
+ const context = Object.freeze({
4584
+ currentSurface: immutableProposalSnapshot(parent.artifact, "current surface"),
4585
+ operator: request.operator,
4586
+ parents: immutableProposalSnapshot(request.parents.map(({ nodeId, artifact }) => ({
4587
+ nodeId,
4588
+ artifact: immutableProposalSnapshot(artifact, "parent surface")
4589
+ })), "parents"),
4590
+ train: searchProposerView(state),
4591
+ summary: renderSearchSummary(state, { split: "train" }),
4592
+ history: proposalHistory,
4593
+ findings: immutableProposalSnapshot(assertProposalFindings(findings, "runOptimization proposal findings"), "findings"),
4594
+ populationSize: opts.populationSize,
4595
+ generation,
4596
+ signal: request.signal,
4597
+ baselineOutcome: immutableProposalSnapshot(nodes.outcome(state, root), "baseline"),
4598
+ incumbentOutcome: immutableProposalSnapshot(nodes.outcome(state, request.leader), "incumbent outcome"),
4599
+ parentOutcome: immutableProposalSnapshot(nodes.outcome(state, parent.nodeId), "parent outcome"),
4600
+ maxImprovementShots: opts.maxImprovementShots,
4601
+ paretoParents: immutableProposalSnapshot(nodes.frontier(state), "Pareto parents"),
4602
+ costLedger,
4603
+ costPhase: "search.proposal"
4604
+ });
4605
+ if (proposer.decide?.({ history: proposalHistory }).stop) return {
4606
+ children: [],
4607
+ stop: "the proposer decided to stop",
4608
+ ...accounting()
4609
+ };
4610
+ const proposed = await proposer.propose(context);
4611
+ if (!Array.isArray(proposed)) throw new TypeError("runOptimization: proposer must return an array");
4612
+ return {
4613
+ children: immutableProposalSnapshot(proposed, "candidate outputs").slice(0, opts.populationSize).map((candidate) => isProposedCandidate(candidate) ? {
4614
+ artifact: candidate.surface,
4615
+ label: candidate.label,
4616
+ rationale: candidate.rationale,
4617
+ ...candidate.attribution ? { attribution: candidate.attribution } : {}
4618
+ } : {
4619
+ artifact: candidate,
4620
+ label: "",
4621
+ rationale: ""
4622
+ }),
4623
+ ...accounting()
4624
+ };
4625
+ }
4626
+ };
4627
+ const result = await runSearch({
4628
+ recorder,
4629
+ root: baselineSurface,
4630
+ codec: surfaceCodec,
4631
+ policy,
4632
+ allocation,
4633
+ proposer: port,
4634
+ executor,
4635
+ maxExpansions: opts.maxGenerations,
4636
+ signal: opts.signal
4637
+ });
4638
+ const state = result.state;
4639
+ const winnerSurface = nodes.surface(state, result.leader);
4640
+ const firstEdge = state.edge(state.node(result.leader).edgeIds[0]);
4641
+ const winnerRationale = firstEdge.operator === "seed" || !("sha256" in firstEdge.rationale) ? void 0 : recorder.readBlob(firstEdge.rationale).text;
4642
+ const expansions = nodes.generations(state);
4643
+ return {
4644
+ generations: nodes.history(state, policy).map((record) => ({
4645
+ record,
4646
+ surfaces: nodes.expansionNodes(state, expansions[record.generationIndex]).map((nodeId) => {
4647
+ const surface = nodes.surface(state, nodeId);
4648
+ return {
4649
+ surfaceHash: surfaceHash(surface),
4650
+ surface,
4651
+ campaign: nodes.campaign(state, nodeId)
4652
+ };
4653
+ })
4654
+ })),
4655
+ baselineSurface,
4656
+ winnerSurface,
4657
+ winnerSurfaceHash: surfaceHash(winnerSurface),
4658
+ ...firstEdge.operator !== "seed" && firstEdge.label ? { winnerLabel: firstEdge.label } : {},
4659
+ ...winnerRationale ? { winnerRationale } : {},
4660
+ baselineCampaign: nodes.campaign(state, state.rootNodeId),
4661
+ cost: costLedger.summary(),
4662
+ searchHistory: recorder.receipt({
4663
+ producerId: proposer.kind,
4664
+ runId: runDir
4665
+ }),
4666
+ paretoFrontier: nodes.frontier(state)
4667
+ };
4668
+ }
4637
4669
  /** The default ledger and identities when the caller binds none. Revisions are
4638
4670
  * content digests of what the caller declared, and each uri names what its
4639
4671
  * digest covers: the dispatch ref, the proposer kind, the kernel. */
@@ -4676,12 +4708,6 @@ async function defaultBinding(opts, runDir, storage) {
4676
4708
  };
4677
4709
  }
4678
4710
  }
4679
- function artifactKindOf(surface) {
4680
- return typeof surface === "object" && surface.kind === "code" ? "code" : "prompt";
4681
- }
4682
- function receiptKey(receipt) {
4683
- return receipt.callId;
4684
- }
4685
4711
  function immutableProposalSnapshot(value, label) {
4686
4712
  try {
4687
4713
  return deepFreeze(structuredClone(value));
@@ -4719,48 +4745,6 @@ function validatedPremeasuredBaseline(args) {
4719
4745
  if (!coverage.complete) throw new Error(`runOptimization: premeasured baseline is incomplete (${coverage.scorableCellIds.length}/${coverage.expectedCellIds.length} designed cells scorable) — ${formatCoverageFailures(coverage)}. Refusing to optimize against an incomplete incumbent.`);
4720
4746
  return campaign;
4721
4747
  }
4722
- /** A scored campaign as a frontier member: per-scenario composite objectives. */
4723
- function toParetoParent(surface, campaign, generation, label) {
4724
- const objectives = {};
4725
- for (const { scenarioId, composite } of campaignBreakdown(campaign).scenarios) objectives[scenarioId] = composite;
4726
- const parent = {
4727
- surface,
4728
- surfaceHash: surfaceHash(surface),
4729
- objectives,
4730
- composite: campaignMeanComposite(campaign),
4731
- generation
4732
- };
4733
- if (label) parent.label = label;
4734
- return parent;
4735
- }
4736
- /** The non-dominated set over the per-scenario objective vectors. */
4737
- function computeParetoFrontier(scored) {
4738
- if (scored.length <= 1) return [...scored];
4739
- const ids = /* @__PURE__ */ new Set();
4740
- for (const p of scored) for (const id of Object.keys(p.objectives)) ids.add(id);
4741
- if (ids.size === 0) return [...scored];
4742
- const objectives = [...ids].map((id) => ({
4743
- name: id,
4744
- direction: "maximize",
4745
- value: (p) => p.objectives[id] ?? Number.NEGATIVE_INFINITY
4746
- }));
4747
- return paretoFrontier(scored, objectives).frontier;
4748
- }
4749
- function toScoredSurfaceOutcome(hash, campaign, coverage, generation) {
4750
- const breakdown = campaignBreakdown(campaign);
4751
- return {
4752
- split: "search",
4753
- generation,
4754
- surfaceHash: hash,
4755
- composite: campaignMeanComposite(campaign),
4756
- dimensions: breakdown.dimensions,
4757
- scenarios: breakdown.scenarios,
4758
- coverage: {
4759
- expectedCells: coverage.expectedCellIds.length,
4760
- scorableCells: coverage.scorableCellIds.length
4761
- }
4762
- };
4763
- }
4764
4748
  //#endregion
4765
4749
  //#region src/campaign/presets/run-improvement-loop.ts
4766
4750
  /**
@@ -4830,10 +4814,126 @@ async function runImprovementLoop(opts) {
4830
4814
  ghRepo: opts.ghRepo
4831
4815
  });
4832
4816
  return {
4833
- ...optimization,
4834
- ...comparison,
4835
- prResult,
4836
- cost: costLedger.summary()
4817
+ ...optimization,
4818
+ ...comparison,
4819
+ prResult,
4820
+ cost: costLedger.summary()
4821
+ };
4822
+ }
4823
+ //#endregion
4824
+ //#region src/campaign/optimization-cost.ts
4825
+ /** Attribute method calls while retaining the shared account's admission and read behavior. */
4826
+ function createMethodCostScope(account, methodName) {
4827
+ const tags = { [`optimizationAttempt:${crypto.randomUUID()}`]: methodName };
4828
+ return {
4829
+ ledger: Object.freeze({
4830
+ costCeilingUsd: account.costCeilingUsd,
4831
+ runPaidCall: (input) => account.runPaidCall({
4832
+ ...input,
4833
+ tags: {
4834
+ ...input.tags,
4835
+ ...tags
4836
+ }
4837
+ }),
4838
+ summary: account.summary.bind(account),
4839
+ list: account.list.bind(account),
4840
+ reconcile: account.reconcile.bind(account),
4841
+ markCompleted: account.markCompleted.bind(account),
4842
+ costPerCompletedTask: account.costPerCompletedTask.bind(account),
4843
+ ...account.listPending ? { listPending: account.listPending.bind(account) } : {},
4844
+ ...account.waitForIdle ? { waitForIdle: account.waitForIdle.bind(account) } : {}
4845
+ }),
4846
+ reconcile(reported) {
4847
+ const summary = account.summary({ tags });
4848
+ if (summary.pendingCalls > 0) throw new Error(`optimization method '${methodName}' returned with ${summary.pendingCalls} pending paid call(s)`);
4849
+ const recorded = costFromLedgerSummary(summary);
4850
+ const totalCostUsd = Math.max(reported.totalCostUsd, recorded.totalCostUsd);
4851
+ const roundingToleranceUsd = Number.EPSILON * Math.max(1, totalCostUsd) * (summary.totalCalls + 1);
4852
+ const combined = combineComparisonCosts([{
4853
+ label: "reported",
4854
+ cost: reported
4855
+ }, {
4856
+ label: "recorded",
4857
+ cost: recorded
4858
+ }]);
4859
+ const incompleteReasons = [
4860
+ ...reported.incompleteReasons,
4861
+ ...recorded.incompleteReasons.map((reason) => `recorded: ${reason}`),
4862
+ ...recorded.totalCostUsd - reported.totalCostUsd > roundingToleranceUsd ? [`reported ${reported.totalCostUsd} USD below recorded ${recorded.totalCostUsd} USD`] : []
4863
+ ];
4864
+ return {
4865
+ totalCostUsd,
4866
+ costProvenance: combined.costProvenance.kind === "uncaptured" ? combined.costProvenance : {
4867
+ kind: combined.costProvenance.kind,
4868
+ usd: totalCostUsd
4869
+ },
4870
+ accountingComplete: incompleteReasons.length === 0,
4871
+ incompleteReasons
4872
+ };
4873
+ }
4874
+ };
4875
+ }
4876
+ /** Keep the cost fields a custom optimization method must report. */
4877
+ function costFromLedgerSummary(summary) {
4878
+ const cost = {
4879
+ totalCostUsd: summary.totalCostUsd,
4880
+ costProvenance: structuredClone(summary.costProvenance),
4881
+ accountingComplete: summary.accountingComplete,
4882
+ incompleteReasons: [...summary.incompleteReasons]
4883
+ };
4884
+ assertComparisonCost(cost, "cost ledger");
4885
+ return cost;
4886
+ }
4887
+ /** Combine method costs without turning one unknown bill into a known total. */
4888
+ function combineComparisonCosts(entries) {
4889
+ const totalCostUsd = entries.reduce((total, entry) => total + entry.cost.totalCostUsd, 0);
4890
+ const cost = {
4891
+ totalCostUsd,
4892
+ costProvenance: entries.some((entry) => entry.cost.costProvenance.kind === "uncaptured") ? {
4893
+ kind: "uncaptured",
4894
+ usd: null
4895
+ } : entries.every((entry) => entry.cost.costProvenance.kind === "observed") ? {
4896
+ kind: "observed",
4897
+ usd: totalCostUsd
4898
+ } : {
4899
+ kind: "estimated",
4900
+ usd: totalCostUsd
4901
+ },
4902
+ accountingComplete: entries.every((entry) => entry.cost.accountingComplete),
4903
+ incompleteReasons: entries.flatMap((entry) => entry.cost.incompleteReasons.map((reason) => `${entry.label}: ${reason}`))
4904
+ };
4905
+ assertComparisonCost(cost, "combined cost");
4906
+ return cost;
4907
+ }
4908
+ function assertComparisonCost(cost, label) {
4909
+ if (!cost || typeof cost !== "object") throw new Error(`compareOptimizationMethods: ${label} returned no cost`);
4910
+ if (!Number.isFinite(cost.totalCostUsd) || cost.totalCostUsd < 0) throw new Error(`compareOptimizationMethods: ${label} returned an invalid totalCostUsd`);
4911
+ const provenance = cost.costProvenance;
4912
+ if (!provenance || typeof provenance !== "object" || provenance.kind !== "observed" && provenance.kind !== "estimated" && provenance.kind !== "uncaptured" || (provenance.kind === "uncaptured" ? provenance.usd !== null : !Number.isFinite(provenance.usd) || provenance.usd < 0)) throw new Error(`compareOptimizationMethods: ${label} returned invalid costProvenance`);
4913
+ if (provenance.kind !== "uncaptured" && provenance.usd !== cost.totalCostUsd) throw new Error(`compareOptimizationMethods: ${label} returned costProvenance inconsistent with totalCostUsd`);
4914
+ if (typeof cost.accountingComplete !== "boolean") throw new Error(`compareOptimizationMethods: ${label} returned invalid accountingComplete`);
4915
+ if (!Array.isArray(cost.incompleteReasons) || cost.incompleteReasons.some((reason) => typeof reason !== "string" || reason.trim().length === 0)) throw new Error(`compareOptimizationMethods: ${label} returned invalid incompleteReasons`);
4916
+ if (cost.accountingComplete !== (cost.incompleteReasons.length === 0)) throw new Error(`compareOptimizationMethods: ${label} returned inconsistent cost completeness and reasons`);
4917
+ if (cost.accountingComplete && provenance.kind === "uncaptured") throw new Error(`compareOptimizationMethods: ${label} cannot mark uncaptured cost as complete`);
4918
+ }
4919
+ /** Preserve every optimizer token class while keeping total input and output explicit. */
4920
+ function optimizationTokenUsageFromSummary(summary, receipts) {
4921
+ if (!summary.usageComplete) return void 0;
4922
+ if (receipts.length !== summary.totalCalls) throw new Error("optimization token usage receipt count does not match the cost summary");
4923
+ const cachedInputTokens = summary.cachedTokens;
4924
+ const cacheWriteInputTokens = summary.cacheWriteTokens ?? 0;
4925
+ const inputTokens = summary.inputTokens + cachedInputTokens + cacheWriteInputTokens;
4926
+ const cacheReadComplete = receipts.length === 0 || receipts.every((receipt) => receipt.cachedTokens !== void 0);
4927
+ const cacheWriteComplete = receipts.length === 0 || receipts.every((receipt) => receipt.cacheWriteTokens !== void 0);
4928
+ const reasoningComplete = receipts.length === 0 || receipts.every((receipt) => receipt.reasoningTokens !== void 0);
4929
+ return {
4930
+ inputTokens,
4931
+ ...cacheReadComplete ? { cachedInputTokens } : {},
4932
+ ...cacheWriteComplete ? { cacheWriteInputTokens } : {},
4933
+ outputTokens: summary.outputTokens,
4934
+ ...reasoningComplete ? { reasoningTokens: summary.reasoningTokens ?? 0 } : {},
4935
+ totalTokens: inputTokens + summary.outputTokens,
4936
+ calls: summary.totalCalls
4837
4937
  };
4838
4938
  }
4839
4939
  //#endregion
@@ -5022,102 +5122,6 @@ function assertPositiveSafeInteger(value, label) {
5022
5122
  if (!Number.isSafeInteger(value) || value <= 0) throw new Error(`GEPA candidate population ${label} must be a positive safe integer`);
5023
5123
  }
5024
5124
  //#endregion
5025
- //#region src/campaign/optimization-cost.ts
5026
- /** Attribute method calls while retaining the shared account's admission and read behavior. */
5027
- function createMethodCostScope(account, methodName) {
5028
- const tags = { [`optimizationAttempt:${crypto.randomUUID()}`]: methodName };
5029
- return {
5030
- ledger: Object.freeze({
5031
- costCeilingUsd: account.costCeilingUsd,
5032
- runPaidCall: (input) => account.runPaidCall({
5033
- ...input,
5034
- tags: {
5035
- ...input.tags,
5036
- ...tags
5037
- }
5038
- }),
5039
- summary: account.summary.bind(account),
5040
- list: account.list.bind(account),
5041
- reconcile: account.reconcile.bind(account),
5042
- markCompleted: account.markCompleted.bind(account),
5043
- costPerCompletedTask: account.costPerCompletedTask.bind(account),
5044
- ...account.listPending ? { listPending: account.listPending.bind(account) } : {},
5045
- ...account.waitForIdle ? { waitForIdle: account.waitForIdle.bind(account) } : {}
5046
- }),
5047
- reconcile(reported) {
5048
- const summary = account.summary({ tags });
5049
- if (summary.pendingCalls > 0) throw new Error(`optimization method '${methodName}' returned with ${summary.pendingCalls} pending paid call(s)`);
5050
- const recorded = costFromLedgerSummary(summary);
5051
- const totalCostUsd = Math.max(reported.totalCostUsd, recorded.totalCostUsd);
5052
- const roundingToleranceUsd = Number.EPSILON * Math.max(1, totalCostUsd) * (summary.totalCalls + 1);
5053
- const combined = combineComparisonCosts([{
5054
- label: "reported",
5055
- cost: reported
5056
- }, {
5057
- label: "recorded",
5058
- cost: recorded
5059
- }]);
5060
- const incompleteReasons = [
5061
- ...reported.incompleteReasons,
5062
- ...recorded.incompleteReasons.map((reason) => `recorded: ${reason}`),
5063
- ...recorded.totalCostUsd - reported.totalCostUsd > roundingToleranceUsd ? [`reported ${reported.totalCostUsd} USD below recorded ${recorded.totalCostUsd} USD`] : []
5064
- ];
5065
- return {
5066
- totalCostUsd,
5067
- costProvenance: combined.costProvenance.kind === "uncaptured" ? combined.costProvenance : {
5068
- kind: combined.costProvenance.kind,
5069
- usd: totalCostUsd
5070
- },
5071
- accountingComplete: incompleteReasons.length === 0,
5072
- incompleteReasons
5073
- };
5074
- }
5075
- };
5076
- }
5077
- /** Keep the cost fields a custom optimization method must report. */
5078
- function costFromLedgerSummary(summary) {
5079
- const cost = {
5080
- totalCostUsd: summary.totalCostUsd,
5081
- costProvenance: structuredClone(summary.costProvenance),
5082
- accountingComplete: summary.accountingComplete,
5083
- incompleteReasons: [...summary.incompleteReasons]
5084
- };
5085
- assertComparisonCost(cost, "cost ledger");
5086
- return cost;
5087
- }
5088
- /** Combine method costs without turning one unknown bill into a known total. */
5089
- function combineComparisonCosts(entries) {
5090
- const totalCostUsd = entries.reduce((total, entry) => total + entry.cost.totalCostUsd, 0);
5091
- const cost = {
5092
- totalCostUsd,
5093
- costProvenance: entries.some((entry) => entry.cost.costProvenance.kind === "uncaptured") ? {
5094
- kind: "uncaptured",
5095
- usd: null
5096
- } : entries.every((entry) => entry.cost.costProvenance.kind === "observed") ? {
5097
- kind: "observed",
5098
- usd: totalCostUsd
5099
- } : {
5100
- kind: "estimated",
5101
- usd: totalCostUsd
5102
- },
5103
- accountingComplete: entries.every((entry) => entry.cost.accountingComplete),
5104
- incompleteReasons: entries.flatMap((entry) => entry.cost.incompleteReasons.map((reason) => `${entry.label}: ${reason}`))
5105
- };
5106
- assertComparisonCost(cost, "combined cost");
5107
- return cost;
5108
- }
5109
- function assertComparisonCost(cost, label) {
5110
- if (!cost || typeof cost !== "object") throw new Error(`compareOptimizationMethods: ${label} returned no cost`);
5111
- if (!Number.isFinite(cost.totalCostUsd) || cost.totalCostUsd < 0) throw new Error(`compareOptimizationMethods: ${label} returned an invalid totalCostUsd`);
5112
- const provenance = cost.costProvenance;
5113
- if (!provenance || typeof provenance !== "object" || provenance.kind !== "observed" && provenance.kind !== "estimated" && provenance.kind !== "uncaptured" || (provenance.kind === "uncaptured" ? provenance.usd !== null : !Number.isFinite(provenance.usd) || provenance.usd < 0)) throw new Error(`compareOptimizationMethods: ${label} returned invalid costProvenance`);
5114
- if (provenance.kind !== "uncaptured" && provenance.usd !== cost.totalCostUsd) throw new Error(`compareOptimizationMethods: ${label} returned costProvenance inconsistent with totalCostUsd`);
5115
- if (typeof cost.accountingComplete !== "boolean") throw new Error(`compareOptimizationMethods: ${label} returned invalid accountingComplete`);
5116
- if (!Array.isArray(cost.incompleteReasons) || cost.incompleteReasons.some((reason) => typeof reason !== "string" || reason.trim().length === 0)) throw new Error(`compareOptimizationMethods: ${label} returned invalid incompleteReasons`);
5117
- if (cost.accountingComplete !== (cost.incompleteReasons.length === 0)) throw new Error(`compareOptimizationMethods: ${label} returned inconsistent cost completeness and reasons`);
5118
- if (cost.accountingComplete && provenance.kind === "uncaptured") throw new Error(`compareOptimizationMethods: ${label} cannot mark uncaptured cost as complete`);
5119
- }
5120
- //#endregion
5121
5125
  //#region src/campaign/optimization-method.ts
5122
5126
  /** Both complete-method workflows use the same detached inputs, accounting, and evidence admission. */
5123
5127
  async function executeOptimizationMethod(options) {
@@ -5266,440 +5270,6 @@ function assertOptimizationProvenance(methodName, value) {
5266
5270
  if (value.optimizerModel === void 0 !== (value.optimizerCallRef === void 0) || value.optimizerModel === void 0 !== (value.modelExecutions === void 0)) fail("optimizerModel execution provenance");
5267
5271
  }
5268
5272
  //#endregion
5269
- //#region src/campaign/presets/compare-optimization-methods.ts
5270
- /**
5271
- * Compare optimization methods on shared train, selection, and test data.
5272
- * Optimizers receive only train and selection data. After every optimizer
5273
- * finishes, their selected surfaces are measured on the same untouched test
5274
- * data and compared with paired confidence intervals.
5275
- */
5276
- /**
5277
- * Compare complete optimization methods on disjoint train, selection, and final test data.
5278
- */
5279
- async function compareOptimizationMethods(opts) {
5280
- opts = {
5281
- ...opts,
5282
- methods: opts.methods.map((method) => ({ ...method })),
5283
- trainScenarios: structuredClone(opts.trainScenarios),
5284
- selectionScenarios: structuredClone(opts.selectionScenarios),
5285
- testScenarios: structuredClone(opts.testScenarios),
5286
- judges: opts.judges.map(captureJudge),
5287
- claim: opts.claim && defineEvaluationClaim(opts.claim),
5288
- finalEvidence: opts.finalEvidence && captureFinalEvidencePolicy(opts.finalEvidence)
5289
- };
5290
- assertOptimizationMethods(opts.methods);
5291
- assertComparisonPartitions(opts);
5292
- const searchHistoryPolicy = opts.searchHistoryPolicy ?? "allow-missing";
5293
- assertSearchHistoryAdmissionOptions(opts);
5294
- const seed = opts.seed ?? 42;
5295
- const confidence = opts.confidence ?? .95;
5296
- assertConfidence(confidence);
5297
- const optimizationConcurrency = opts.optimizationConcurrency ?? 1;
5298
- const comparisonCount = opts.methods.length * (opts.methods.length + 1) / 2;
5299
- const intervalConfidence = 1 - (1 - confidence) / comparisonCount;
5300
- const minimumResamples = minimumBootstrapResamples(confidence, comparisonCount);
5301
- const resamples = opts.resamples ?? Math.max(2e3, minimumResamples);
5302
- assertComparisonControls(opts, seed, resamples, confidence);
5303
- const evidence = opts.evidence === void 0 ? void 0 : structuredClone(opts.evidence);
5304
- const evidenceBySurface = /* @__PURE__ */ new Map();
5305
- const storage = opts.storage ?? fsCampaignStorage();
5306
- const resolvedRunDir = resolveRunDir(opts.runDir, opts.repo);
5307
- const baselineSurface = structuredClone(opts.baselineSurface);
5308
- const costLedger = opts.costLedger ?? createRunCostLedger({
5309
- storage,
5310
- runDir: `${resolvedRunDir}/cost`,
5311
- costCeilingUsd: opts.costCeiling
5312
- });
5313
- const scoreOnTest = async (surface, tag, costPhase) => {
5314
- const measuredSurface = structuredClone(surface);
5315
- const campaign = await runCampaign({
5316
- ...opts,
5317
- storage,
5318
- costLedger,
5319
- costPhase,
5320
- scenarios: opts.testScenarios.map((scenario) => structuredClone(scenario)),
5321
- dispatch: (scenario, ctx) => opts.dispatchWithSurface(structuredClone(measuredSurface), scenario, ctx),
5322
- dispatchRef: finalDispatchRef(opts, measuredSurface),
5323
- runDir: `${resolvedRunDir}/${tag}`
5324
- });
5325
- assertCompleteCampaign(campaign, opts.testScenarios, opts.reps ?? 1, true, `compareOptimizationMethods: ${tag} final comparison`);
5326
- if (evidence) evidenceBySurface.set(surfaceContentHash(measuredSurface), createCampaignEvidenceReceipt({
5327
- campaign,
5328
- surface: measuredSurface,
5329
- context: evidence
5330
- }));
5331
- const byScenario = {};
5332
- for (const { scenarioId, composite } of campaignBreakdown(campaign).scenarios) byScenario[scenarioId] = composite;
5333
- return byScenario;
5334
- };
5335
- const scenarioIds = opts.testScenarios.map((s) => s.id).sort();
5336
- const unitByScenario = opts.claim ? evaluationUnitMap(opts.claim, opts.testScenarios) : new Map(scenarioIds.map((id) => [id, id]));
5337
- const units = opts.claim ? summarizeEvaluationUnits(opts.claim, opts.testScenarios) : {
5338
- observations: scenarioIds.length,
5339
- independentUnits: scenarioIds.length,
5340
- units: scenarioIds.map((id) => ({
5341
- id,
5342
- observations: 1
5343
- }))
5344
- };
5345
- const aggregateUnits = (arr) => {
5346
- const values = /* @__PURE__ */ new Map();
5347
- scenarioIds.forEach((id, index) => {
5348
- const unitId = unitByScenario.get(id);
5349
- const bucket = values.get(unitId) ?? [];
5350
- bucket.push(arr[index]);
5351
- values.set(unitId, bucket);
5352
- });
5353
- return units.units.map((unit) => mean(values.get(unit.id)));
5354
- };
5355
- const decisionOptions = {
5356
- seed,
5357
- resamples,
5358
- confidence: intervalConfidence,
5359
- threshold: opts.claim?.minimumEffect ?? 0
5360
- };
5361
- const align = (byScenario, label) => {
5362
- const missing = scenarioIds.filter((id) => !(id in byScenario));
5363
- if (missing.length > 0) throw new Error(`compareOptimizationMethods: ${label} produced no test score for scenario(s) [${missing.join(", ")}]. A cell failed or its judges returned nothing. Fix the dispatch or judge; the comparison will not replace missing scores with zero.`);
5364
- return scenarioIds.map((id) => byScenario[id]);
5365
- };
5366
- if (opts.claim?.generalization === "new-units") assertIndependentEvaluationSplit(opts.claim, opts.testScenarios, [...opts.trainScenarios, ...opts.selectionScenarios]);
5367
- if (opts.finalEvidence) await reserveFinalEvidence(opts.finalEvidence, opts.claim, opts.testScenarios, [...opts.trainScenarios, ...opts.selectionScenarios]);
5368
- const optimizationOwner = new AbortController();
5369
- const optimized = await mapConcurrent(opts.methods, optimizationConcurrency, async (method) => {
5370
- try {
5371
- const { selected: out, cost, history: searchHistoryCoverage } = await executeOptimizationMethod({
5372
- method,
5373
- input: createOptimizationMethodInput(opts, method.name, resolvedRunDir, seed, baselineSurface, costLedger, optimizationOwner.signal),
5374
- storage,
5375
- searchHistoryPolicy: opts.searchHistoryPolicy,
5376
- searchHistoryVerification: opts.searchHistoryVerification
5377
- });
5378
- const winnerSurface = out.winnerSurface;
5379
- return {
5380
- name: method.name,
5381
- winnerSurface,
5382
- cost,
5383
- ...out.durationMs === void 0 ? {} : { durationMs: out.durationMs },
5384
- ...out.composition === void 0 ? {} : { composition: out.composition },
5385
- ...out.provenance === void 0 ? {} : { provenance: out.provenance },
5386
- searchHistoryCoverage
5387
- };
5388
- } catch (error) {
5389
- if (!optimizationOwner.signal.aborted) optimizationOwner.abort(error);
5390
- throw error;
5391
- }
5392
- });
5393
- assertReportedCostWithinCeiling(combineComparisonCosts(optimized.map((result) => ({
5394
- label: `method '${result.name}'`,
5395
- cost: result.cost
5396
- }))).totalCostUsd, opts.costCeiling, "optimization");
5397
- const testCostPhase = finalCostPhase(opts, baselineSurface, optimized, seed);
5398
- const finalEvidence = opts.finalEvidence ? await exposeFinalEvidence(opts.finalEvidence, opts.claim, opts.testScenarios, [baselineSurface, ...optimized.map((method) => method.winnerSurface)]) : void 0;
5399
- const baselineArr = align(await scoreOnTest(baselineSurface, "test/baseline", testCostPhase), "baseline");
5400
- const testScoresBySurface = /* @__PURE__ */ new Map([[surfaceContentHash(baselineSurface), baselineArr]]);
5401
- const winners = [];
5402
- for (const winner of optimized) {
5403
- const surfaceKey = surfaceContentHash(winner.winnerSurface);
5404
- let arr = testScoresBySurface.get(surfaceKey);
5405
- if (!arr) {
5406
- arr = align(await scoreOnTest(winner.winnerSurface, `test/methods/${slug(winner.name)}`, testCostPhase), `method "${winner.name}"`);
5407
- testScoresBySurface.set(surfaceKey, arr);
5408
- }
5409
- winners.push({
5410
- ...winner,
5411
- arr
5412
- });
5413
- }
5414
- const scores = winners.map((w) => {
5415
- const baselineUnits = aggregateUnits(baselineArr);
5416
- const winnerUnits = aggregateUnits(w.arr);
5417
- const decision = decidePairedPromotion(baselineUnits, winnerUnits, decisionOptions);
5418
- const score = {
5419
- name: w.name,
5420
- baselineComposite: mean(baselineUnits),
5421
- winnerComposite: mean(winnerUnits),
5422
- lift: decision.delta,
5423
- liftCi: {
5424
- low: decision.low,
5425
- high: decision.high
5426
- },
5427
- decision,
5428
- optimizationCost: w.cost,
5429
- scenarioScores: scenarioIds.map((scenarioId, index) => ({
5430
- scenarioId,
5431
- baselineComposite: baselineArr[index],
5432
- winnerComposite: w.arr[index],
5433
- lift: w.arr[index] - baselineArr[index]
5434
- })),
5435
- unitScores: units.units.map((unit, index) => ({
5436
- unitId: unit.id,
5437
- scenarios: unit.observations,
5438
- baselineComposite: baselineUnits[index],
5439
- winnerComposite: winnerUnits[index],
5440
- lift: winnerUnits[index] - baselineUnits[index]
5441
- })),
5442
- winnerSurface: structuredClone(w.winnerSurface),
5443
- rank: 0
5444
- };
5445
- if (evidence) {
5446
- const baseline = evidenceBySurface.get(surfaceContentHash(baselineSurface));
5447
- const winner = evidenceBySurface.get(surfaceContentHash(w.winnerSurface));
5448
- if (!baseline || !winner) throw new Error("final measurement evidence is missing");
5449
- score.evidence = {
5450
- baseline,
5451
- winner
5452
- };
5453
- }
5454
- if (w.composition !== void 0) score.composition = structuredClone(w.composition);
5455
- if (w.durationMs !== void 0) score.durationMs = w.durationMs;
5456
- if (w.provenance !== void 0) score.provenance = structuredClone(w.provenance);
5457
- return score;
5458
- });
5459
- scores.sort((a, b) => b.lift - a.lift);
5460
- for (let start = 0; start < scores.length;) {
5461
- let end = start + 1;
5462
- while (end < scores.length && scores[end].lift === scores[start].lift) end += 1;
5463
- const tied = scores.slice(start, end);
5464
- if (tied.every((score) => score.optimizationCost.accountingComplete)) {
5465
- tied.sort((a, b) => a.optimizationCost.totalCostUsd - b.optimizationCost.totalCostUsd);
5466
- scores.splice(start, tied.length, ...tied);
5467
- }
5468
- start = end;
5469
- }
5470
- scores.forEach((s, i) => {
5471
- s.rank = i + 1;
5472
- });
5473
- const best = scores[0];
5474
- const byName = new Map(winners.map((w) => [w.name, w]));
5475
- const bestArr = aggregateUnits(byName.get(best.name).arr);
5476
- const pairwise = scores.slice(1).map((other) => {
5477
- const otherArr = aggregateUnits(byName.get(other.name).arr);
5478
- const decision = decidePairedPromotion(otherArr, bestArr, decisionOptions);
5479
- const reverse = decidePairedPromotion(bestArr, otherArr, decisionOptions);
5480
- const favored = decision.promote ? best.name : reverse.promote ? other.name : null;
5481
- return {
5482
- a: best.name,
5483
- b: other.name,
5484
- deltaMean: decision.delta,
5485
- low: decision.low,
5486
- high: decision.high,
5487
- favored,
5488
- decision
5489
- };
5490
- });
5491
- const optimizationCost = combineComparisonCosts(scores.map((score) => ({
5492
- label: `method '${score.name}'`,
5493
- cost: score.optimizationCost
5494
- })));
5495
- const testCost = costFromLedgerSummary(costLedger.summary({ phase: testCostPhase }));
5496
- const totalCost = combineComparisonCosts([{
5497
- label: "optimization",
5498
- cost: optimizationCost
5499
- }, {
5500
- label: "final test",
5501
- cost: testCost
5502
- }]);
5503
- assertReportedCostWithinCeiling(totalCost.totalCostUsd, opts.costCeiling, "total");
5504
- return {
5505
- scores,
5506
- best,
5507
- pairwise,
5508
- testScenarioIds: scenarioIds,
5509
- units,
5510
- observationUnit: opts.claim ? "registered" : "scenario",
5511
- pairedCellN: scenarioIds.length * (opts.reps ?? 1),
5512
- ...opts.claim ? { claim: opts.claim } : {},
5513
- ...finalEvidence ? { finalEvidence } : {},
5514
- optimizationCost,
5515
- testCost,
5516
- totalCost,
5517
- confidence,
5518
- intervalConfidence,
5519
- comparisonCount,
5520
- seed,
5521
- resamples,
5522
- reps: opts.reps ?? 1,
5523
- searchHistory: Object.freeze({
5524
- policy: searchHistoryPolicy,
5525
- allComplete: optimized.every((result) => result.searchHistoryCoverage.status === "complete"),
5526
- producers: Object.freeze(optimized.map((result) => result.searchHistoryCoverage))
5527
- })
5528
- };
5529
- }
5530
- function assertReportedCostWithinCeiling(totalCostUsd, costCeiling, phase) {
5531
- const tolerance = Number.EPSILON * Math.max(1, Math.abs(totalCostUsd), Math.abs(costCeiling ?? 0)) * 8;
5532
- if (costCeiling !== void 0 && totalCostUsd > costCeiling + tolerance) throw new Error(`compareOptimizationMethods: reported ${phase} cost ${totalCostUsd} exceeds costCeiling ${costCeiling}`);
5533
- }
5534
- function assertOptimizationMethods(methods) {
5535
- if (!Array.isArray(methods) || methods.length === 0) throw new Error("compareOptimizationMethods: no methods to compare");
5536
- const names = /* @__PURE__ */ new Set();
5537
- const pathOwners = /* @__PURE__ */ new Map();
5538
- for (const method of methods) {
5539
- if (!method || typeof method !== "object" || typeof method.optimize !== "function") throw new Error("compareOptimizationMethods: every method must provide optimize(input)");
5540
- if (!method.name || method.name.trim() !== method.name) throw new Error("compareOptimizationMethods: method names must be trimmed and non-empty");
5541
- if (names.has(method.name)) throw new Error(`compareOptimizationMethods: duplicate method name '${method.name}'`);
5542
- names.add(method.name);
5543
- const pathKey = slug(method.name);
5544
- const prior = pathOwners.get(pathKey);
5545
- if (prior) throw new Error(`compareOptimizationMethods: method names '${prior}' and '${method.name}' map to the same run path '${pathKey}'`);
5546
- pathOwners.set(pathKey, method.name);
5547
- }
5548
- }
5549
- function assertComparisonControls(opts, seed, resamples, confidence) {
5550
- if (opts.optimizationRunOptions && "costCeiling" in opts.optimizationRunOptions) throw new Error("compareOptimizationMethods: optimizationRunOptions.costCeiling is not supported; costCeiling covers optimization and final scoring");
5551
- if (!opts.judges || opts.judges.length === 0) throw new Error("compareOptimizationMethods: at least one judge is required");
5552
- if (typeof opts.dispatchWithSurface !== "function") throw new Error("compareOptimizationMethods: dispatchWithSurface must be a function");
5553
- if (opts.dispatchRef !== void 0 && (typeof opts.dispatchRef !== "string" || opts.dispatchRef.trim().length === 0 || opts.dispatchRef.trim() !== opts.dispatchRef)) throw new Error("compareOptimizationMethods: dispatchRef must be trimmed and non-empty when provided");
5554
- const optimizationDispatchRef = opts.optimizationRunOptions?.dispatchRef;
5555
- if (optimizationDispatchRef !== void 0 && (typeof optimizationDispatchRef !== "string" || optimizationDispatchRef.trim().length === 0 || optimizationDispatchRef.trim() !== optimizationDispatchRef)) throw new Error("compareOptimizationMethods: optimizationRunOptions.dispatchRef must be trimmed and non-empty when provided");
5556
- if (opts.dispatchRef !== void 0 && optimizationDispatchRef !== void 0 && opts.dispatchRef !== optimizationDispatchRef) throw new Error("compareOptimizationMethods: dispatchRef must match optimizationRunOptions.dispatchRef when both are provided");
5557
- try {
5558
- surfaceContentHash(opts.baselineSurface);
5559
- } catch (cause) {
5560
- throw new Error("compareOptimizationMethods: baselineSurface is invalid", { cause });
5561
- }
5562
- const judgeNames = /* @__PURE__ */ new Set();
5563
- for (const judge of opts.judges) {
5564
- if (!judge || typeof judge !== "object" || typeof judge.name !== "string" || judge.name.trim().length === 0 || judge.name.trim() !== judge.name || typeof judge.score !== "function" || !Array.isArray(judge.dimensions) || judge.dimensions.length === 0) throw new Error("compareOptimizationMethods: every judge needs a trimmed name, at least one dimension, and score(input)");
5565
- if (judgeNames.has(judge.name)) throw new Error(`compareOptimizationMethods: duplicate judge name '${judge.name}'`);
5566
- judgeNames.add(judge.name);
5567
- const dimensionKeys = /* @__PURE__ */ new Set();
5568
- for (const dimension of judge.dimensions) {
5569
- if (!dimension || typeof dimension.key !== "string" || dimension.key.trim().length === 0 || dimension.key.trim() !== dimension.key || typeof dimension.description !== "string" || dimension.description.trim().length === 0) throw new Error(`compareOptimizationMethods: judge '${judge.name}' has an invalid dimension`);
5570
- if (dimensionKeys.has(dimension.key)) throw new Error(`compareOptimizationMethods: judge '${judge.name}' has duplicate dimension '${dimension.key}'`);
5571
- dimensionKeys.add(dimension.key);
5572
- }
5573
- }
5574
- if (typeof opts.runDir !== "string" || opts.runDir.trim().length === 0) throw new Error("compareOptimizationMethods: runDir must be a non-empty string");
5575
- if (!Number.isSafeInteger(seed)) throw new Error(`compareOptimizationMethods: seed must be a safe integer, got ${String(seed)}`);
5576
- if (!Number.isSafeInteger(resamples) || resamples <= 0 || resamples > 1e6) throw new Error(`compareOptimizationMethods: resamples must be a positive safe integer no greater than 1000000, got ${String(resamples)}`);
5577
- if (!Number.isFinite(confidence) || confidence <= 0 || confidence >= 1) throw new Error(`compareOptimizationMethods: confidence must be a finite number in (0,1), got ${String(confidence)}`);
5578
- const minimumResamples = minimumBootstrapResamples(confidence, opts.methods.length * (opts.methods.length + 1) / 2);
5579
- if (resamples < minimumResamples) throw new Error(`compareOptimizationMethods: resamples must be at least ${minimumResamples} for simultaneous confidence ${confidence} across ${opts.methods.length} methods, got ${resamples}`);
5580
- if (opts.optimizationConcurrency !== void 0 && (!Number.isSafeInteger(opts.optimizationConcurrency) || opts.optimizationConcurrency <= 0)) throw new Error("compareOptimizationMethods: optimizationConcurrency must be a positive safe integer");
5581
- if (opts.maxConcurrency !== void 0 && (!Number.isSafeInteger(opts.maxConcurrency) || opts.maxConcurrency <= 0)) throw new Error("compareOptimizationMethods: maxConcurrency must be a positive safe integer");
5582
- if (opts.dispatchTimeoutMs !== void 0 && (!Number.isSafeInteger(opts.dispatchTimeoutMs) || opts.dispatchTimeoutMs < 0 || opts.dispatchTimeoutMs > 2147483647)) throw new Error("compareOptimizationMethods: dispatchTimeoutMs must be a non-negative safe integer no greater than 2147483647");
5583
- if (opts.costCeiling !== void 0 && (!Number.isFinite(opts.costCeiling) || opts.costCeiling < 0)) throw new Error("compareOptimizationMethods: costCeiling must be a finite number greater than or equal to 0");
5584
- if (opts.costCeiling !== void 0 && opts.costLedger !== void 0 && opts.costLedger.costCeilingUsd !== opts.costCeiling) throw new Error("compareOptimizationMethods: costCeiling must match the shared CostLedger ceiling");
5585
- }
5586
- function assertComparisonPartitions(opts) {
5587
- const partitions = [
5588
- {
5589
- name: "trainScenarios",
5590
- scenarios: opts.trainScenarios
5591
- },
5592
- {
5593
- name: "selectionScenarios",
5594
- scenarios: opts.selectionScenarios
5595
- },
5596
- {
5597
- name: "testScenarios",
5598
- scenarios: opts.testScenarios
5599
- }
5600
- ];
5601
- const owner = /* @__PURE__ */ new Map();
5602
- for (const partition of partitions) {
5603
- if (!Array.isArray(partition.scenarios) || partition.scenarios.length === 0) throw new Error(`compareOptimizationMethods: ${partition.name} is empty`);
5604
- if (partition.name === "testScenarios" && partition.scenarios.length < 2) throw new Error("compareOptimizationMethods: testScenarios requires at least 2 scenarios to estimate uncertainty");
5605
- const local = /* @__PURE__ */ new Set();
5606
- const duplicates = /* @__PURE__ */ new Set();
5607
- const overlaps = /* @__PURE__ */ new Map();
5608
- for (const scenario of partition.scenarios) {
5609
- if (local.has(scenario.id)) duplicates.add(scenario.id);
5610
- local.add(scenario.id);
5611
- const prior = owner.get(scenario.id);
5612
- if (prior !== void 0 && prior !== partition.name) overlaps.set(scenario.id, prior);
5613
- }
5614
- if (duplicates.size > 0) throw new Error(`compareOptimizationMethods: ${partition.name} contains duplicate scenario id(s) [${[...duplicates].join(", ")}]`);
5615
- if (overlaps.size > 0) {
5616
- const detail = [...overlaps].map(([id, prior]) => `${id} (${prior} ∩ ${partition.name})`).join(", ");
5617
- throw new Error(`compareOptimizationMethods: trainScenarios, selectionScenarios, and testScenarios must be pairwise disjoint; overlap: [${detail}]`);
5618
- }
5619
- assertCampaignDesign(partition.scenarios, opts.reps ?? 1);
5620
- for (const id of local) owner.set(id, partition.name);
5621
- }
5622
- }
5623
- function mean(xs) {
5624
- return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length;
5625
- }
5626
- function slug(name) {
5627
- return name.replace(/[^a-z0-9]+/gi, "-").replace(/^-|-$/g, "").toLowerCase() || "method";
5628
- }
5629
- function assertConfidence(confidence) {
5630
- if (!Number.isFinite(confidence) || confidence <= 0 || confidence >= 1) throw new Error(`compareOptimizationMethods: confidence must be a finite number in (0,1), got ${String(confidence)}`);
5631
- }
5632
- function minimumBootstrapResamples(confidence, comparisonCount) {
5633
- const exact = 2 * comparisonCount / (1 - confidence);
5634
- return Math.ceil(exact - Number.EPSILON * Math.max(1, exact) * 32);
5635
- }
5636
- function createOptimizationMethodInput(opts, methodName, resolvedRunDir, seed, baselineSurface, costLedger, optimizationSignal) {
5637
- const methodRunDir = `${resolvedRunDir}/optimization/${slug(methodName)}`;
5638
- const signal = combineAbortSignals(opts.signal, opts.optimizationRunOptions?.signal, optimizationSignal);
5639
- return Object.freeze({
5640
- baselineSurface: structuredClone(baselineSurface),
5641
- trainScenarios: opts.trainScenarios,
5642
- selectionScenarios: opts.selectionScenarios,
5643
- dispatchWithSurface: opts.dispatchWithSurface,
5644
- judges: opts.judges,
5645
- runDir: methodRunDir,
5646
- seed,
5647
- runOptions: Object.freeze({
5648
- ...opts.optimizationRunOptions ?? {},
5649
- ...opts.optimizationRunOptions?.dispatchRef === void 0 && opts.dispatchRef !== void 0 ? { dispatchRef: opts.dispatchRef } : {},
5650
- ...signal ? { signal } : {}
5651
- }),
5652
- costLedger
5653
- });
5654
- }
5655
- function finalCostPhase(opts, baselineSurface, winners, seed) {
5656
- return `compareOptimizationMethods.test:${contentHash({
5657
- baseline: surfaceContentHash(baselineSurface),
5658
- winners: winners.map((winner) => ({
5659
- name: winner.name,
5660
- surface: surfaceContentHash(winner.winnerSurface)
5661
- })),
5662
- testScenarios: opts.testScenarios,
5663
- judges: opts.judges.map((judge) => ({
5664
- name: judge.name,
5665
- dimensions: judge.dimensions,
5666
- version: judge.judgeVersion ?? contentHash({
5667
- score: judge.score.toString(),
5668
- appliesTo: judge.appliesTo?.toString() ?? null
5669
- })
5670
- })),
5671
- dispatch: callerDispatchRef(opts),
5672
- seed,
5673
- reps: opts.reps ?? 1
5674
- })}`;
5675
- }
5676
- function finalDispatchRef(opts, surface) {
5677
- return `compareOptimizationMethods:${callerDispatchRef(opts)}:${surfaceContentHash(surface)}`;
5678
- }
5679
- function callerDispatchRef(opts) {
5680
- return (opts.dispatchRef ?? opts.dispatchWithSurface.name) || "anonymous";
5681
- }
5682
- /** Preserve every optimizer token class while keeping total input and output explicit. */
5683
- function optimizationTokenUsageFromSummary(summary, receipts) {
5684
- if (!summary.usageComplete) return void 0;
5685
- if (receipts.length !== summary.totalCalls) throw new Error("optimization token usage receipt count does not match the cost summary");
5686
- const cachedInputTokens = summary.cachedTokens;
5687
- const cacheWriteInputTokens = summary.cacheWriteTokens ?? 0;
5688
- const inputTokens = summary.inputTokens + cachedInputTokens + cacheWriteInputTokens;
5689
- const cacheReadComplete = receipts.length === 0 || receipts.every((receipt) => receipt.cachedTokens !== void 0);
5690
- const cacheWriteComplete = receipts.length === 0 || receipts.every((receipt) => receipt.cacheWriteTokens !== void 0);
5691
- const reasoningComplete = receipts.length === 0 || receipts.every((receipt) => receipt.reasoningTokens !== void 0);
5692
- return {
5693
- inputTokens,
5694
- ...cacheReadComplete ? { cachedInputTokens } : {},
5695
- ...cacheWriteComplete ? { cacheWriteInputTokens } : {},
5696
- outputTokens: summary.outputTokens,
5697
- ...reasoningComplete ? { reasoningTokens: summary.reasoningTokens ?? 0 } : {},
5698
- totalTokens: inputTokens + summary.outputTokens,
5699
- calls: summary.totalCalls
5700
- };
5701
- }
5702
- //#endregion
5703
5273
  //#region src/run-score.ts
5704
5274
  const DEFAULT_RUN_SCORE_WEIGHTS = {
5705
5275
  success: 4,
@@ -6172,6 +5742,6 @@ function firstString(value) {
6172
5742
  return typeof value === "string" && value.trim() ? value : void 0;
6173
5743
  }
6174
5744
  //#endregion
6175
- export { captureFinalEvidencePolicy as $, assertCompleteSearchHistory as A, searchReceiptAccounting as B, incumbentWithOperatorBandit as C, surfaceDiff as D, searchEdgeId as E, verifySearchHistoryArtifact as F, captureJudge as G, uniform as H, verifySearchHistoryReceipt as I, redTeamDataset as J, defaultProductionGate as K, campaignCellSearchResult as L, assertSearchHistoryMatchesState as M, createSearchHistoryReceipt as N, surfaceNode as O, searchHistoryCoverageRow as P, assertIndependentEvaluationSplit as Q, searchModelIdentity as R, incumbent as S, developmentClaim as T, runFinalComparison as U, asha as V, openAutoPr as W, scoreRedTeamOutput as X, redTeamReport as Y, runCanaries as Z, isProposedCandidate as _, transientDispatchFailure as a, resolveRunDir as at, beam as b, compareOptimizationMethods as c, buildCellSchedule as ct, combineComparisonCosts as d, evaluationUnitMap as et, costFromLedgerSummary as f, runOptimization as g, runImprovementLoop as h, quotaExhaustedUntil as i, planCampaignRun as it, assertSearchHistoryAdmissionOptions as j, SearchHistoryRequiredError as k, optimizationTokenUsageFromSummary as l, cellCachePath as lt, readGepaCandidatePopulationArtifact as m, JudgeParseError as n, runEval as nt, aggregateRunScore as o, tangleTracesRoot as ot, assertGepaCandidatePopulationSummary as p, DEFAULT_RED_TEAM_CORPUS as q, isTransientTransportFailure as r, runCampaign as rt, clamp01 as s, readCachedCell as st, llmJudge as t, reserveFinalEvidence as tt, executeOptimizationMethod as u, computeManifestHash as ut, labelTrustRank as v, SearchRecorder as w, crowdedFrontierParent as x, aide as y, searchProposalExecution as z };
5745
+ export { planCampaignRun as $, verifySearchHistoryReceipt as A, defaultProductionGate as B, searchEdgeId as C, assertSearchHistoryMatchesState as D, assertSearchHistoryAdmissionOptions as E, asha as F, runCanaries as G, redTeamDataset as H, uniform as I, evaluationUnitMap as J, assertIndependentEvaluationSplit as K, runFinalComparison as L, searchModelIdentity as M, searchProposalExecution as N, createSearchHistoryReceipt as O, searchReceiptAccounting as P, runCampaign as Q, openAutoPr as R, developmentClaim as S, surfaceNode as T, redTeamReport as U, DEFAULT_RED_TEAM_CORPUS as V, scoreRedTeamOutput as W, reserveFinalEvidence as X, exposeFinalEvidence as Y, runEval as Z, beam as _, aggregateRunScore as a, incumbentWithOperatorBandit as b, assertGepaCandidatePopulationSummary as c, costFromLedgerSummary as d, resolveRunDir as et, optimizationTokenUsageFromSummary as f, aide as g, isProposedCandidate as h, transientDispatchFailure as i, campaignCellSearchResult as j, verifySearchHistoryArtifact as k, readGepaCandidatePopulationArtifact as l, runOptimization as m, JudgeParseError as n, cellCachePath as nt, clamp01 as o, runImprovementLoop as p, captureFinalEvidencePolicy as q, isTransientTransportFailure as r, computeManifestHash as rt, executeOptimizationMethod as s, llmJudge as t, buildCellSchedule as tt, combineComparisonCosts as u, crowdedFrontierParent as v, surfaceDiff as w, SearchRecorder as x, incumbent as y, captureJudge as z };
6176
5746
 
6177
- //# sourceMappingURL=llm-judge-PXV7Xljl.js.map
5747
+ //# sourceMappingURL=llm-judge-C8jMTNNG.js.map