@tangle-network/agent-eval 0.216.1 → 0.218.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (532) hide show
  1. package/CHANGELOG.md +137 -57
  2. package/README.md +2 -2
  3. package/dist/adapters/http.d.ts +2 -2
  4. package/dist/{agent-profile-VFcQncbW.d.ts → agent-profile-DTt8uGP9.d.ts} +4 -17
  5. package/dist/agent-profile-DTt8uGP9.d.ts.map +1 -0
  6. package/dist/{agent-profile-cell-BK8mlqlq.js → agent-profile-cell-B1V1gUZs.js} +4 -53
  7. package/dist/agent-profile-cell-B1V1gUZs.js.map +1 -0
  8. package/dist/{agent-profile-cell-BNp-Js9F.d.ts → agent-profile-cell-BcpFbNd5.d.ts} +2 -22
  9. package/dist/agent-profile-cell-BcpFbNd5.d.ts.map +1 -0
  10. package/dist/{agreement-irr-BEWLIF3_.js → agreement-irr-BhAqjluA.js} +5 -5
  11. package/dist/agreement-irr-BhAqjluA.js.map +1 -0
  12. package/dist/analyst/index.d.ts +36 -169
  13. package/dist/analyst/index.d.ts.map +1 -1
  14. package/dist/analyst/index.js +17 -192
  15. package/dist/analyst/index.js.map +1 -1
  16. package/dist/{analyze-runs-D68fd-su.d.ts → analyze-runs-DjlZOVXx.d.ts} +5 -5
  17. package/dist/{analyze-runs-D68fd-su.d.ts.map → analyze-runs-DjlZOVXx.d.ts.map} +1 -1
  18. package/dist/{analyze-runs-C84irm4i.js → analyze-runs-DqPJaJ4X.js} +225 -21
  19. package/dist/analyze-runs-DqPJaJ4X.js.map +1 -0
  20. package/dist/authenticity/index.d.ts +0 -23
  21. package/dist/authenticity/index.d.ts.map +1 -1
  22. package/dist/authenticity/index.js +1 -34
  23. package/dist/authenticity/index.js.map +1 -1
  24. package/dist/{benchmark-command-By0vh8q7.js → benchmark-command-BupdHRyf.js} +1225 -661
  25. package/dist/benchmark-command-BupdHRyf.js.map +1 -0
  26. package/dist/benchmarks/index.d.ts +5 -6
  27. package/dist/benchmarks/index.d.ts.map +1 -1
  28. package/dist/benchmarks/index.js +4 -4
  29. package/dist/benchmarks/index.js.map +1 -1
  30. package/dist/builder-eval/index.d.ts +3 -3
  31. package/dist/builder-eval/index.js +2 -2
  32. package/dist/builder-eval/index.js.map +1 -1
  33. package/dist/campaign/index.d.ts +7 -7
  34. package/dist/campaign/index.js +15 -15
  35. package/dist/{campaign-BudZ70zo.js → campaign-CGvUTnmc.js} +324 -800
  36. package/dist/campaign-CGvUTnmc.js.map +1 -0
  37. package/dist/{campaign-evidence-Clrn4hBV.js → campaign-evidence-DunzcJGv.js} +10 -18
  38. package/dist/{campaign-evidence-Clrn4hBV.js.map → campaign-evidence-DunzcJGv.js.map} +1 -1
  39. package/dist/{canonical-B2RqLofv.js → canonical-Dq2hOQ8L.js} +2 -2
  40. package/dist/{canonical-B2RqLofv.js.map → canonical-Dq2hOQ8L.js.map} +1 -1
  41. package/dist/cell-spend-DEWR3nj-.js +90 -0
  42. package/dist/cell-spend-DEWR3nj-.js.map +1 -0
  43. package/dist/{chat-client-l-MVDlDk.js → chat-client-B2pXlPdx.js} +2 -2
  44. package/dist/{chat-client-l-MVDlDk.js.map → chat-client-B2pXlPdx.js.map} +1 -1
  45. package/dist/{chat-json-call-puMArmtW.js → chat-json-call-Dm9c-b41.js} +3 -2
  46. package/dist/chat-json-call-Dm9c-b41.js.map +1 -0
  47. package/dist/{claim-q4HQhK1G.js → claim-BPVuyHuy.js} +3 -3
  48. package/dist/{claim-q4HQhK1G.js.map → claim-BPVuyHuy.js.map} +1 -1
  49. package/dist/cli.js +113 -12
  50. package/dist/cli.js.map +1 -1
  51. package/dist/{client-BzL9niqi.d.ts → client-hwo3cJjR.d.ts} +3 -16
  52. package/dist/client-hwo3cJjR.d.ts.map +1 -0
  53. package/dist/{cluster-bootstrap-qcUWYFl-.js → cluster-bootstrap-DEwwhl0C.js} +3 -3
  54. package/dist/{cluster-bootstrap-qcUWYFl-.js.map → cluster-bootstrap-DEwwhl0C.js.map} +1 -1
  55. package/dist/{produced-state-BsDmnwxc.js → completion-verifier-DbLVf3i8.js} +9 -64
  56. package/dist/completion-verifier-DbLVf3i8.js.map +1 -0
  57. package/dist/contract/index.d.ts +12 -12
  58. package/dist/contract/index.js +107 -403
  59. package/dist/contract/index.js.map +1 -1
  60. package/dist/{cost-ledger-BP8V4kUQ.js → cost-ledger-D0352Z4d.js} +4 -8
  61. package/dist/cost-ledger-D0352Z4d.js.map +1 -0
  62. package/dist/dataset-DQqhOCPt.d.ts.map +1 -1
  63. package/dist/{default-registry-COhk9g36.js → default-registry-BJ-Pq6yL.js} +314 -285
  64. package/dist/default-registry-BJ-Pq6yL.js.map +1 -0
  65. package/dist/{default-registry-DzHiGruW.d.ts → default-registry-DXgLmoDI.d.ts} +7 -50
  66. package/dist/default-registry-DXgLmoDI.d.ts.map +1 -0
  67. package/dist/{define-agent-eval-CNTPzIkp.js → define-agent-eval-BGdFKlrc.js} +170 -110
  68. package/dist/define-agent-eval-BGdFKlrc.js.map +1 -0
  69. package/dist/{define-agent-eval-WSck0CZJ.d.ts → define-agent-eval-BlH10zEw.d.ts} +8 -8
  70. package/dist/define-agent-eval-BlH10zEw.d.ts.map +1 -0
  71. package/dist/{descriptive-DboxPZrG.js → descriptive-CVNy48rR.js} +2 -2
  72. package/dist/{descriptive-DboxPZrG.js.map → descriptive-CVNy48rR.js.map} +1 -1
  73. package/dist/diagnosis/index.d.ts +4 -14
  74. package/dist/diagnosis/index.d.ts.map +1 -1
  75. package/dist/diagnosis/index.js +67 -28
  76. package/dist/diagnosis/index.js.map +1 -1
  77. package/dist/{dspy-rlm-engine-LRCu61m5.js → dspy-rlm-engine-CSMwudLO.js} +5 -10
  78. package/dist/dspy-rlm-engine-CSMwudLO.js.map +1 -0
  79. package/dist/{emitter-HcCmFQl_.d.ts → emitter-DABYYTcH.d.ts} +5 -5
  80. package/dist/emitter-DABYYTcH.d.ts.map +1 -0
  81. package/dist/emitter-DbMXUIVP.js.map +1 -1
  82. package/dist/{engine-3A_4jOlI.d.ts → engine-BvZ0R-36.d.ts} +17 -21
  83. package/dist/engine-BvZ0R-36.d.ts.map +1 -0
  84. package/dist/{equal-terms-BtdXayCJ.js → equal-terms-DbPGcVtI.js} +2 -2
  85. package/dist/equal-terms-DbPGcVtI.js.map +1 -0
  86. package/dist/{errors-DVZiqS7K.js → errors-BcOs5MKW.js} +2 -8
  87. package/dist/{errors-DVZiqS7K.js.map → errors-BcOs5MKW.js.map} +1 -1
  88. package/dist/eval-campaign-BUMIxKTE.js +393 -0
  89. package/dist/eval-campaign-BUMIxKTE.js.map +1 -0
  90. package/dist/{evaluation-6Yoi74gM.d.ts → evaluation-BOZ38vOH.d.ts} +7 -7
  91. package/dist/evaluation-BOZ38vOH.d.ts.map +1 -0
  92. package/dist/evaluation.d.ts +2 -2
  93. package/dist/evaluation.js +1 -1
  94. package/dist/evaluation.js.map +1 -1
  95. package/dist/{exact-types-ClugMJfI.d.ts → exact-types-CflbJ9f3.d.ts} +2 -2
  96. package/dist/{exact-types-ClugMJfI.d.ts.map → exact-types-CflbJ9f3.d.ts.map} +1 -1
  97. package/dist/{exec-ChJ6d5if.js → exec-CpbtOXVt.js} +2 -24
  98. package/dist/{exec-ChJ6d5if.js.map → exec-CpbtOXVt.js.map} +1 -1
  99. package/dist/execution-tracks-CpgFPpS5.js.map +1 -1
  100. package/dist/experiment/index.d.ts +9 -112
  101. package/dist/experiment/index.d.ts.map +1 -1
  102. package/dist/experiment/index.js +258 -311
  103. package/dist/experiment/index.js.map +1 -1
  104. package/dist/{exporters-X4xwla8A.js → exporters-DWvM16LE.js} +3 -3
  105. package/dist/exporters-DWvM16LE.js.map +1 -0
  106. package/dist/{exporters-ChTnR90l.d.ts → exporters-Dk3oEbK_.d.ts} +2 -4
  107. package/dist/{exporters-ChTnR90l.d.ts.map → exporters-Dk3oEbK_.d.ts.map} +1 -1
  108. package/dist/{external-optimizer-process-BjMGUuds.js → external-optimizer-process-C3Wm7Vhn.js} +4 -3
  109. package/dist/external-optimizer-process-C3Wm7Vhn.js.map +1 -0
  110. package/dist/{external-optimizer-subprocess-BqW0hHcI.js → external-optimizer-subprocess-B5gdLRZv.js} +9 -11
  111. package/dist/external-optimizer-subprocess-B5gdLRZv.js.map +1 -0
  112. package/dist/{extract-usage-BrnzRJ6H.js → extract-usage-BZLU01kR.js} +2 -23
  113. package/dist/{extract-usage-BrnzRJ6H.js.map → extract-usage-BZLU01kR.js.map} +1 -1
  114. package/dist/{failure-cluster-CbgEDcFJ.d.ts → failure-cluster-DN0t63Z0.d.ts} +4 -43
  115. package/dist/failure-cluster-DN0t63Z0.d.ts.map +1 -0
  116. package/dist/{failure-taxonomy-CjabTATj.js → failure-taxonomy-Bx5oWqaW.js} +2 -19
  117. package/dist/{failure-taxonomy-CjabTATj.js.map → failure-taxonomy-Bx5oWqaW.js.map} +1 -1
  118. package/dist/{feedback-trajectory-C4EG0nNH.d.ts → feedback-trajectory-BWmhRBGu.d.ts} +7 -9
  119. package/dist/feedback-trajectory-BWmhRBGu.d.ts.map +1 -0
  120. package/dist/{semantic-concept-judge-BQF8te0T.js → findings-store-BtTzPL0d.js} +9 -249
  121. package/dist/findings-store-BtTzPL0d.js.map +1 -0
  122. package/dist/fuzz.d.ts +19 -62
  123. package/dist/fuzz.d.ts.map +1 -1
  124. package/dist/fuzz.js +62 -108
  125. package/dist/fuzz.js.map +1 -1
  126. package/dist/{hf-dataset-BR2yhuNn.js → hf-dataset-CKxIJQba.js} +4 -24
  127. package/dist/hf-dataset-CKxIJQba.js.map +1 -0
  128. package/dist/hosted/index.d.ts +2 -2
  129. package/dist/hosted/index.js +2 -2
  130. package/dist/{index-CyszSg4O.d.ts → index-B16FNdW5.d.ts} +145 -360
  131. package/dist/index-B16FNdW5.d.ts.map +1 -0
  132. package/dist/{index-DV-jSzxF.d.ts → index-BLGqZ81N.d.ts} +10 -15
  133. package/dist/{index-DV-jSzxF.d.ts.map → index-BLGqZ81N.d.ts.map} +1 -1
  134. package/dist/{index-BO41eGj5.d.ts → index-CX-GAUpw.d.ts} +3 -7
  135. package/dist/index-CX-GAUpw.d.ts.map +1 -0
  136. package/dist/{index-DHKEAv3s.d.ts → index-CalAsXMR.d.ts} +8 -68
  137. package/dist/index-CalAsXMR.d.ts.map +1 -0
  138. package/dist/{index-BFRO6chs.d.ts → index-CiRwLv2R.d.ts} +5 -69
  139. package/dist/index-CiRwLv2R.d.ts.map +1 -0
  140. package/dist/{index-DPcngtwt.d.ts → index-DbMOXYnP.d.ts} +11 -36
  141. package/dist/index-DbMOXYnP.d.ts.map +1 -0
  142. package/dist/index.d.ts +211 -218
  143. package/dist/index.d.ts.map +1 -1
  144. package/dist/index.js +999 -1714
  145. package/dist/index.js.map +1 -1
  146. package/dist/{integrity-dLiDe2kD.js → integrity-C8oUeB0g.js} +326 -203
  147. package/dist/integrity-C8oUeB0g.js.map +1 -0
  148. package/dist/{integrity-BXOsAb3i.js → integrity-W4CADcFc.js} +3 -7
  149. package/dist/{integrity-BXOsAb3i.js.map → integrity-W4CADcFc.js.map} +1 -1
  150. package/dist/{integrity-DAMEDFaC.d.ts → integrity-_qgZLA5p.d.ts} +4 -6
  151. package/dist/{integrity-DAMEDFaC.d.ts.map → integrity-_qgZLA5p.d.ts.map} +1 -1
  152. package/dist/{internal-Epl-YBZW.js → internal-BTK91Bqt.js} +3 -3
  153. package/dist/{internal-Epl-YBZW.js.map → internal-BTK91Bqt.js.map} +1 -1
  154. package/dist/{interval-CELpye1n.js → interval-Dq9gt7jx.js} +3 -3
  155. package/dist/{interval-CELpye1n.js.map → interval-Dq9gt7jx.js.map} +1 -1
  156. package/dist/{journal-DfRz9kps.js → journal-C7om8Gum.js} +157 -147
  157. package/dist/journal-C7om8Gum.js.map +1 -0
  158. package/dist/json-BhdsJaOQ.js +111 -0
  159. package/dist/json-BhdsJaOQ.js.map +1 -0
  160. package/dist/{judge-calibration-BpCy5XQl.js → judge-calibration-BkXMIWA4.js} +3 -3
  161. package/dist/{judge-calibration-BpCy5XQl.js.map → judge-calibration-BkXMIWA4.js.map} +1 -1
  162. package/dist/{judge-gate-iY9s4L44.js → judge-gate-Co5tuh57.js} +4 -27
  163. package/dist/{judge-gate-iY9s4L44.js.map → judge-gate-Co5tuh57.js.map} +1 -1
  164. package/dist/{judge-gate-CE7CQAgt.d.ts → judge-gate-DsL0yFcb.d.ts} +8 -9
  165. package/dist/judge-gate-DsL0yFcb.d.ts.map +1 -0
  166. package/dist/{kind-factory-BobI5Jy3.js → kind-factory-Gy4fXixd.js} +88 -17
  167. package/dist/kind-factory-Gy4fXixd.js.map +1 -0
  168. package/dist/ledger-core/index.d.ts +2 -2
  169. package/dist/ledger-core/index.js +4 -4
  170. package/dist/ledger-core/lock-holder.d.ts +2 -0
  171. package/dist/ledger-core/lock-holder.js +181 -0
  172. package/dist/ledger-core/lock-holder.js.map +1 -0
  173. package/dist/{ledger-core-Gq7yp8lP.js → ledger-core-CAxUdDk0.js} +5 -84
  174. package/dist/ledger-core-CAxUdDk0.js.map +1 -0
  175. package/dist/{llm-client-CyBAoYiX.js → llm-client-_Y6ic-tf.js} +243 -355
  176. package/dist/llm-client-_Y6ic-tf.js.map +1 -0
  177. package/dist/{llm-judge-CG12SQJq.js → llm-judge-Djxh-Uhs.js} +662 -482
  178. package/dist/llm-judge-Djxh-Uhs.js.map +1 -0
  179. package/dist/lock-holder-DaJbfpci.d.ts +86 -0
  180. package/dist/lock-holder-DaJbfpci.d.ts.map +1 -0
  181. package/dist/matrix/index.d.ts +3 -3
  182. package/dist/matrix/index.js +3 -2
  183. package/dist/{matrix-Dm0BQW7U.js → matrix-B8l-Nsxd.js} +3 -88
  184. package/dist/matrix-B8l-Nsxd.js.map +1 -0
  185. package/dist/{matrix-DSrSxjjy.d.ts → matrix-CCSBna4q.d.ts} +9 -18
  186. package/dist/matrix-CCSBna4q.d.ts.map +1 -0
  187. package/dist/{mean-DHoMiudD.js → mean-BbA_JCzW.js} +39 -5
  188. package/dist/mean-BbA_JCzW.js.map +1 -0
  189. package/dist/meta-eval/index.d.ts +5 -5
  190. package/dist/meta-eval/index.js +10 -10
  191. package/dist/metrics-BKaPBtIG.js.map +1 -1
  192. package/dist/{mint-DNU8QPG4.d.ts → mint-BRZXuwRi.d.ts} +4 -4
  193. package/dist/{mint-DNU8QPG4.d.ts.map → mint-BRZXuwRi.d.ts.map} +1 -1
  194. package/dist/{mint-QaD5cA0u.js → mint-CbdwjTPW.js} +4 -4
  195. package/dist/{mint-QaD5cA0u.js.map → mint-CbdwjTPW.js.map} +1 -1
  196. package/dist/{multi-layer-verifier-BUaQ4C17.d.ts → multi-layer-verifier-DEgMUk6v.d.ts} +2 -2
  197. package/dist/{multi-layer-verifier-BUaQ4C17.d.ts.map → multi-layer-verifier-DEgMUk6v.d.ts.map} +1 -1
  198. package/dist/{multiplicity-DQUFcbW4.js → multiplicity-Sg0U7h8j.js} +2 -2
  199. package/dist/{multiplicity-DQUFcbW4.js.map → multiplicity-Sg0U7h8j.js.map} +1 -1
  200. package/dist/multishot/golden/index.d.ts +1 -23
  201. package/dist/multishot/golden/index.d.ts.map +1 -1
  202. package/dist/multishot/golden/index.js +3 -3
  203. package/dist/multishot/golden/index.js.map +1 -1
  204. package/dist/multishot/index.d.ts +3 -12
  205. package/dist/multishot/index.d.ts.map +1 -1
  206. package/dist/multishot/index.js +4 -3
  207. package/dist/multishot/index.js.map +1 -1
  208. package/dist/openapi.json +114 -10
  209. package/dist/{otel-spans-B9tUXRHF.js → otel-spans-CdN909k1.js} +3 -3
  210. package/dist/{otel-spans-B9tUXRHF.js.map → otel-spans-CdN909k1.js.map} +1 -1
  211. package/dist/{paired-arms-CwMGJsrS.js → paired-arms-Ct4LTKc1.js} +32 -9
  212. package/dist/paired-arms-Ct4LTKc1.js.map +1 -0
  213. package/dist/{paired-binary-CvpFHfKZ.js → paired-binary-B_nejA44.js} +3 -36
  214. package/dist/paired-binary-B_nejA44.js.map +1 -0
  215. package/dist/{paired-promotion-decision-DPKz8ZnF.d.ts → paired-promotion-decision-CQMJ5xGN.d.ts} +2 -34
  216. package/dist/paired-promotion-decision-CQMJ5xGN.d.ts.map +1 -0
  217. package/dist/{paired-promotion-decision-DsAPYGCm.js → paired-promotion-decision-D0SHb-ZL.js} +3 -3
  218. package/dist/{paired-promotion-decision-DsAPYGCm.js.map → paired-promotion-decision-D0SHb-ZL.js.map} +1 -1
  219. package/dist/{pareto-Cm6ndrWN.js → pareto-CTv8vozZ.js} +11 -11
  220. package/dist/pareto-CTv8vozZ.js.map +1 -0
  221. package/dist/pipelines/index.d.ts +16 -19
  222. package/dist/pipelines/index.d.ts.map +1 -1
  223. package/dist/pipelines/index.js +4 -68
  224. package/dist/pipelines/index.js.map +1 -1
  225. package/dist/{power-and-mde-Hj2ubb4B.js → power-and-mde-N5nU4_Ly.js} +3 -3
  226. package/dist/{power-and-mde-Hj2ubb4B.js.map → power-and-mde-N5nU4_Ly.js.map} +1 -1
  227. package/dist/{prime-protocol-CHKgA3a6.js → prime-protocol-BFgGrEMP.js} +3 -3
  228. package/dist/prime-protocol-BFgGrEMP.js.map +1 -0
  229. package/dist/{prime-protocol-HiB9HX7E.d.ts → prime-protocol-BWG2DQIk.d.ts} +3 -21
  230. package/dist/prime-protocol-BWG2DQIk.d.ts.map +1 -0
  231. package/dist/profile-cell.d.ts +2 -2
  232. package/dist/profile-cell.js +2 -2
  233. package/dist/{promotion-gate-BawrdPYr.d.ts → promotion-gate-CxEHRsZW.d.ts} +7 -8
  234. package/dist/{promotion-gate-BawrdPYr.d.ts.map → promotion-gate-CxEHRsZW.d.ts.map} +1 -1
  235. package/dist/{promotion-policy-KO57gbyV.js → promotion-policy-6A8osGr_.js} +4 -4
  236. package/dist/{promotion-policy-KO57gbyV.js.map → promotion-policy-6A8osGr_.js.map} +1 -1
  237. package/dist/{proposal-findings-CY1xKn_a.js → proposal-findings-n-5WBbEx.js} +3 -3
  238. package/dist/proposal-findings-n-5WBbEx.js.map +1 -0
  239. package/dist/{query-BUCZCK7F.js → query-B_RZ36an.js} +5 -5
  240. package/dist/query-B_RZ36an.js.map +1 -0
  241. package/dist/{query-DK1nWcmp.d.ts → query-Cmv_U8vf.d.ts} +4 -24
  242. package/dist/{query-DK1nWcmp.d.ts.map → query-Cmv_U8vf.d.ts.map} +1 -1
  243. package/dist/{random-ByUSnV9w.js → random-DDDcC7YU.js} +2 -2
  244. package/dist/{random-ByUSnV9w.js.map → random-DDDcC7YU.js.map} +1 -1
  245. package/dist/{raw-provider-sink-DoHz4JNG.js → raw-provider-sink-75w4FWBj.js} +2 -2
  246. package/dist/{raw-provider-sink-DoHz4JNG.js.map → raw-provider-sink-75w4FWBj.js.map} +1 -1
  247. package/dist/{redact-mtCCcT8z.d.ts → redact-DXdlFb_U.d.ts} +2 -3
  248. package/dist/redact-DXdlFb_U.d.ts.map +1 -0
  249. package/dist/{redact-B5xXKqQ3.js → redact-mtkTiJVk.js} +2 -7
  250. package/dist/{redact-B5xXKqQ3.js.map → redact-mtkTiJVk.js.map} +1 -1
  251. package/dist/{registry-D7_hm2fK.d.ts → registry-ssNoHzwD.d.ts} +4 -4
  252. package/dist/{registry-D7_hm2fK.d.ts.map → registry-ssNoHzwD.d.ts.map} +1 -1
  253. package/dist/{release-confidence-CpuP65S5.js → release-confidence-CYIFH-fp.js} +6 -11
  254. package/dist/release-confidence-CYIFH-fp.js.map +1 -0
  255. package/dist/{report-command-SC9nqb4M.js → render-Cs34y0iX.js} +625 -587
  256. package/dist/render-Cs34y0iX.js.map +1 -0
  257. package/dist/reporting.d.ts +6 -6
  258. package/dist/reporting.js +8 -8
  259. package/dist/{researcher-CEcfku-T.d.ts → researcher-GU1tV4dn.d.ts} +9 -9
  260. package/dist/{researcher-CEcfku-T.d.ts.map → researcher-GU1tV4dn.d.ts.map} +1 -1
  261. package/dist/{reward-CpCmAM8L.d.ts → reward-DrZ71vnf.d.ts} +3 -26
  262. package/dist/reward-DrZ71vnf.d.ts.map +1 -0
  263. package/dist/{reward-hacking-E0d-WVcP.js → reward-hacking-ByuETaIb.js} +3 -3
  264. package/dist/{reward-hacking-E0d-WVcP.js.map → reward-hacking-ByuETaIb.js.map} +1 -1
  265. package/dist/{reward-hacking-RVdZ6Kkt.d.ts → reward-hacking-E52wq86n.d.ts} +5 -18
  266. package/dist/reward-hacking-E52wq86n.d.ts.map +1 -0
  267. package/dist/reward-nw2xZGZG.js.map +1 -1
  268. package/dist/rl.d.ts +117 -295
  269. package/dist/rl.d.ts.map +1 -1
  270. package/dist/rl.js +27 -747
  271. package/dist/rl.js.map +1 -1
  272. package/dist/rollout/index.d.ts +5 -5
  273. package/dist/rollout/index.js +7 -8
  274. package/dist/{rollout-DZV1t2Wg.js → rollout-Dd1ucT_M.js} +88 -13
  275. package/dist/rollout-Dd1ucT_M.js.map +1 -0
  276. package/dist/{rubric-predictive-validity-DaFW70y9.js → rubric-predictive-validity-D3_oI72N.js} +3 -3
  277. package/dist/{rubric-predictive-validity-DaFW70y9.js.map → rubric-predictive-validity-D3_oI72N.js.map} +1 -1
  278. package/dist/{rubric-predictive-validity-C03EC7ZE.d.ts → rubric-predictive-validity-DiwT8PbL.d.ts} +2 -2
  279. package/dist/{rubric-predictive-validity-C03EC7ZE.d.ts.map → rubric-predictive-validity-DiwT8PbL.d.ts.map} +1 -1
  280. package/dist/{run-record-C1E24G2A.d.ts → run-record-CaxbLq6Y.d.ts} +4 -10
  281. package/dist/run-record-CaxbLq6Y.d.ts.map +1 -0
  282. package/dist/{run-record-C5e7FjhJ.js → run-record-CysKLB93.js} +3 -3
  283. package/dist/run-record-CysKLB93.js.map +1 -0
  284. package/dist/{run-record-Boc5IUTp.js → run-record-DhEJgoPz.js} +4 -4
  285. package/dist/{run-record-Boc5IUTp.js.map → run-record-DhEJgoPz.js.map} +1 -1
  286. package/dist/sandbox-harness-BlSOu4LX.d.ts.map +1 -1
  287. package/dist/{schema-CSf6qWgZ.js → schema-Bt2vLFm3.js} +2 -14
  288. package/dist/schema-Bt2vLFm3.js.map +1 -0
  289. package/dist/schema-C-ziyJD7.d.ts.map +1 -1
  290. package/dist/{schema-BuiujZ-g.js → schema-CvhfLjtZ.js} +78 -48
  291. package/dist/schema-CvhfLjtZ.js.map +1 -0
  292. package/dist/{schema-2g7trUwH.d.ts → schema-D6EVKzTB.d.ts} +4 -5
  293. package/dist/schema-D6EVKzTB.d.ts.map +1 -0
  294. package/dist/search/index.d.ts +3 -31
  295. package/dist/search/index.d.ts.map +1 -1
  296. package/dist/search/index.js +4 -4
  297. package/dist/{search-kernel-DyNEa6IU.d.ts → search-kernel-p9JB3SmL.d.ts} +10 -27
  298. package/dist/search-kernel-p9JB3SmL.d.ts.map +1 -0
  299. package/dist/{search-ledger-CVUGLDZp.js → search-ledger-ClpN4I3e.js} +7 -7
  300. package/dist/{search-ledger-CVUGLDZp.js.map → search-ledger-ClpN4I3e.js.map} +1 -1
  301. package/dist/{search-ledger-file-DDQUgvCg.js → search-ledger-file-Bd5Tioq6.js} +4 -4
  302. package/dist/search-ledger-file-Bd5Tioq6.js.map +1 -0
  303. package/dist/{search-shipper-BOSD46qv.js → search-shipper-Bg2HW3Uv.js} +4 -4
  304. package/dist/search-shipper-Bg2HW3Uv.js.map +1 -0
  305. package/dist/{search-summary-BQCSWf1g.js → search-summary-CrBuvVnf.js} +5 -5
  306. package/dist/{search-summary-BQCSWf1g.js.map → search-summary-CrBuvVnf.js.map} +1 -1
  307. package/dist/{sequential-CzK5DarL.js → sequential-BVv1sjsw.js} +2 -32
  308. package/dist/{sequential-CzK5DarL.js.map → sequential-BVv1sjsw.js.map} +1 -1
  309. package/dist/{sequential-Bf_pbE50.js → sequential-CimQrbjv.js} +5 -5
  310. package/dist/sequential-CimQrbjv.js.map +1 -0
  311. package/dist/{sequential-BhsrMupG.d.ts → sequential-KwGspfeR.d.ts} +2 -9
  312. package/dist/{sequential-BhsrMupG.d.ts.map → sequential-KwGspfeR.d.ts.map} +1 -1
  313. package/dist/{sequential-eprocess-DM_qENCl.js → sequential-eprocess-DodI6RoF.js} +2 -2
  314. package/dist/{sequential-eprocess-DM_qENCl.js.map → sequential-eprocess-DodI6RoF.js.map} +1 -1
  315. package/dist/{server-e9oWO_bN.js → server-C6M0qR8h.js} +101 -101
  316. package/dist/server-C6M0qR8h.js.map +1 -0
  317. package/dist/{single-run-lock-CV9Eyd_q.js → single-run-lock-Dekd8_Wv.js} +7 -7
  318. package/dist/single-run-lock-Dekd8_Wv.js.map +1 -0
  319. package/dist/{skillopt-optimization-method-BiwA8nfX.js → skillopt-optimization-method-5dlex7MJ.js} +1386 -1184
  320. package/dist/skillopt-optimization-method-5dlex7MJ.js.map +1 -0
  321. package/dist/{spans-BHAUXMGN.d.ts → spans-CVngGij3.d.ts} +2 -2
  322. package/dist/{spans-BHAUXMGN.d.ts.map → spans-CVngGij3.d.ts.map} +1 -1
  323. package/dist/{statistical-heldout-CYpGaoDH.d.ts → statistical-heldout-BHfLZHrm.d.ts} +8 -67
  324. package/dist/statistical-heldout-BHfLZHrm.d.ts.map +1 -0
  325. package/dist/{statistical-heldout-42NlCNDe.js → statistical-heldout-Bksv92Th.js} +3 -3
  326. package/dist/statistical-heldout-Bksv92Th.js.map +1 -0
  327. package/dist/{steps-Bg5-KVx1.d.ts → steps-DlepJPwD.d.ts} +2 -56
  328. package/dist/steps-DlepJPwD.d.ts.map +1 -0
  329. package/dist/{store-C4avFjIi.d.ts → store-B3nWRy3B.d.ts} +2 -2
  330. package/dist/{store-C4avFjIi.d.ts.map → store-B3nWRy3B.d.ts.map} +1 -1
  331. package/dist/{store-otlp-DJ9dtA5x.js → store-otlp-BteDAbHK.js} +3 -3
  332. package/dist/{store-otlp-DJ9dtA5x.js.map → store-otlp-BteDAbHK.js.map} +1 -1
  333. package/dist/{store-tool-spans-CIX5arAG.js → store-tool-spans-D5l_OfTJ.js} +8 -8
  334. package/dist/store-tool-spans-D5l_OfTJ.js.map +1 -0
  335. package/dist/{store-tool-spans-DdCSPa7C.d.ts → store-tool-spans-DZS0HWVf.d.ts} +9 -18
  336. package/dist/store-tool-spans-DZS0HWVf.d.ts.map +1 -0
  337. package/dist/storyboard/index.d.ts +4 -11
  338. package/dist/storyboard/index.d.ts.map +1 -1
  339. package/dist/storyboard/index.js +1 -16
  340. package/dist/storyboard/index.js.map +1 -1
  341. package/dist/{summary-report-V_JACR4Y.js → summary-report-BsXJoYX5.js} +88 -10
  342. package/dist/summary-report-BsXJoYX5.js.map +1 -0
  343. package/dist/{summary-report-bdFDXq0b.d.ts → summary-report-CkbRcSuu.d.ts} +25 -19
  344. package/dist/summary-report-CkbRcSuu.d.ts.map +1 -0
  345. package/dist/supervisor-run/index.d.ts +60 -28
  346. package/dist/supervisor-run/index.d.ts.map +1 -1
  347. package/dist/supervisor-run/index.js +4 -458
  348. package/dist/supervisor-run/index.js.map +1 -1
  349. package/dist/{systemone-BomhHr2c.d.ts → systemone-BMzDJb1i.d.ts} +5 -5
  350. package/dist/{systemone-BomhHr2c.d.ts.map → systemone-BMzDJb1i.d.ts.map} +1 -1
  351. package/dist/systemone-protocol.js +1 -1
  352. package/dist/{systemone-review-CUVcQJu6.d.ts → systemone-review-CwE6JGIp.d.ts} +2 -2
  353. package/dist/{systemone-review-CUVcQJu6.d.ts.map → systemone-review-CwE6JGIp.d.ts.map} +1 -1
  354. package/dist/{systemone-review-D7sXwsqq.js → systemone-review-DzdOJ8I6.js} +3 -3
  355. package/dist/{systemone-review-D7sXwsqq.js.map → systemone-review-DzdOJ8I6.js.map} +1 -1
  356. package/dist/systemone.d.ts +5 -5
  357. package/dist/systemone.js +5 -5
  358. package/dist/systemone.js.map +1 -1
  359. package/dist/{task-failure-attributes-hEmASjix.js → task-failure-attributes-BMsblaDh.js} +3 -3
  360. package/dist/task-failure-attributes-BMsblaDh.js.map +1 -0
  361. package/dist/{terminal-record-DZyQZ3pz.js → terminal-record-D4Rv-j1W.js} +244 -196
  362. package/dist/terminal-record-D4Rv-j1W.js.map +1 -0
  363. package/dist/{tool-groups-C3NEcdCM.d.ts → tool-groups-DyNz2JZ_.d.ts} +3 -3
  364. package/dist/tool-groups-DyNz2JZ_.d.ts.map +1 -0
  365. package/dist/{tool-waste-BKgvd-TP.d.ts → tool-waste-CkjzWmeY.d.ts} +25 -28
  366. package/dist/tool-waste-CkjzWmeY.d.ts.map +1 -0
  367. package/dist/{tool-waste-BlesqTqW.js → tool-waste-DLsjS_p-.js} +5 -5
  368. package/dist/{tool-waste-BlesqTqW.js.map → tool-waste-DLsjS_p-.js.map} +1 -1
  369. package/dist/{trace-questions-CM2CAocU.d.ts → trace-questions-xxwgsuMi.d.ts} +4 -9
  370. package/dist/{trace-questions-CM2CAocU.d.ts.map → trace-questions-xxwgsuMi.d.ts.map} +1 -1
  371. package/dist/trace-repair/index.d.ts +14 -211
  372. package/dist/trace-repair/index.d.ts.map +1 -1
  373. package/dist/trace-repair/index.js +865 -981
  374. package/dist/trace-repair/index.js.map +1 -1
  375. package/dist/traces.d.ts +18 -46
  376. package/dist/traces.d.ts.map +1 -1
  377. package/dist/traces.js +13 -13
  378. package/dist/traces.js.map +1 -1
  379. package/dist/trajectory-replay/index.d.ts +7 -118
  380. package/dist/trajectory-replay/index.d.ts.map +1 -1
  381. package/dist/trajectory-replay/index.js +921 -869
  382. package/dist/trajectory-replay/index.js.map +1 -1
  383. package/dist/{tree-C1XYeXvy.js → tree-B5e1uAeL.js} +132 -21
  384. package/dist/tree-B5e1uAeL.js.map +1 -0
  385. package/dist/{types-Cbe2fWBa.d.ts → types-BKCzzmit.d.ts} +3 -3
  386. package/dist/{types-Cbe2fWBa.d.ts.map → types-BKCzzmit.d.ts.map} +1 -1
  387. package/dist/{types-B3jzCp0p.js → types-BXgASBCL.js} +2 -2
  388. package/dist/{types-B3jzCp0p.js.map → types-BXgASBCL.js.map} +1 -1
  389. package/dist/{types-C1tnkEgM.d.ts → types-CW8ewATZ.d.ts} +4 -113
  390. package/dist/types-CW8ewATZ.d.ts.map +1 -0
  391. package/dist/{types-BqkVmHfv.d.ts → types-CmXY0PQf.d.ts} +2 -4
  392. package/dist/types-CmXY0PQf.d.ts.map +1 -0
  393. package/dist/{types-C02qtnG8.d.ts → types-D-mgRzlh.d.ts} +7 -28
  394. package/dist/types-D-mgRzlh.d.ts.map +1 -0
  395. package/dist/{types-IXD2HJzu.js → types-DqFMNTgK.js} +2 -4
  396. package/dist/types-DqFMNTgK.js.map +1 -0
  397. package/dist/{types-Cq8mSbq8.d.ts → types-DtkO1G9L.d.ts} +7 -9
  398. package/dist/{types-Cq8mSbq8.d.ts.map → types-DtkO1G9L.d.ts.map} +1 -1
  399. package/dist/{verdict-Dl0tiT1X.js → verdict-BGQmxQ8A.js} +3 -3
  400. package/dist/verdict-BGQmxQ8A.js.map +1 -0
  401. package/dist/{verdict-E4eRNf7-.d.ts → verdict-C24NELtN.d.ts} +2 -15
  402. package/dist/{verdict-E4eRNf7-.d.ts.map → verdict-C24NELtN.d.ts.map} +1 -1
  403. package/dist/{verdict-cache-Bm8EKL2s.js → verdict-cache-CL_7tePD.js} +2 -2
  404. package/dist/verdict-cache-CL_7tePD.js.map +1 -0
  405. package/dist/wire/index.d.ts +31 -410
  406. package/dist/wire/index.d.ts.map +1 -1
  407. package/dist/wire/index.js +2 -2
  408. package/docs/changes/jev-evidence-review.md +1 -1
  409. package/docs/changes/jev-product-adoption.md +1 -1
  410. package/docs/changes/jev-recorded-evidence.md +1 -1
  411. package/docs/design/statistics-decisions.md +18 -18
  412. package/docs/design.md +1 -1
  413. package/docs/eval-surface-map.md +1 -0
  414. package/docs/evaluation-integrity.md +1 -1
  415. package/docs/profile-json-identity.md +1 -2
  416. package/docs/proofs/campaign-attempt-retention-20261001/README.md +2 -2
  417. package/docs/public-api.md +158 -538
  418. package/docs/rollout.md +1 -1
  419. package/docs/trace-repair-admission.md +1 -1
  420. package/docs/wire-protocol.md +1 -1
  421. package/package.json +7 -2
  422. package/dist/active-curriculum-B_Lc9kfO.js +0 -61
  423. package/dist/active-curriculum-B_Lc9kfO.js.map +0 -1
  424. package/dist/agent-profile-VFcQncbW.d.ts.map +0 -1
  425. package/dist/agent-profile-cell-BK8mlqlq.js.map +0 -1
  426. package/dist/agent-profile-cell-BNp-Js9F.d.ts.map +0 -1
  427. package/dist/agreement-irr-BEWLIF3_.js.map +0 -1
  428. package/dist/analyze-runs-C84irm4i.js.map +0 -1
  429. package/dist/baseline-Udi_PLuT.js +0 -259
  430. package/dist/baseline-Udi_PLuT.js.map +0 -1
  431. package/dist/benchmark-CXmVVHH2.js +0 -562
  432. package/dist/benchmark-CXmVVHH2.js.map +0 -1
  433. package/dist/benchmark-command-By0vh8q7.js.map +0 -1
  434. package/dist/campaign-BudZ70zo.js.map +0 -1
  435. package/dist/chat-json-call-puMArmtW.js.map +0 -1
  436. package/dist/claude-jsonl-dZir12m6.js +0 -86
  437. package/dist/claude-jsonl-dZir12m6.js.map +0 -1
  438. package/dist/client-BzL9niqi.d.ts.map +0 -1
  439. package/dist/cost-ledger-BP8V4kUQ.js.map +0 -1
  440. package/dist/counterfactual-CPaWN99-.d.ts +0 -60
  441. package/dist/counterfactual-CPaWN99-.d.ts.map +0 -1
  442. package/dist/counterfactual-Dn3veueY.js +0 -98
  443. package/dist/counterfactual-Dn3veueY.js.map +0 -1
  444. package/dist/default-registry-COhk9g36.js.map +0 -1
  445. package/dist/default-registry-DzHiGruW.d.ts.map +0 -1
  446. package/dist/define-agent-eval-CNTPzIkp.js.map +0 -1
  447. package/dist/define-agent-eval-WSck0CZJ.d.ts.map +0 -1
  448. package/dist/dspy-rlm-engine-LRCu61m5.js.map +0 -1
  449. package/dist/effect-sizes-BhzEpXMc.js +0 -82
  450. package/dist/effect-sizes-BhzEpXMc.js.map +0 -1
  451. package/dist/emitter-HcCmFQl_.d.ts.map +0 -1
  452. package/dist/engine-3A_4jOlI.d.ts.map +0 -1
  453. package/dist/equal-terms-BtdXayCJ.js.map +0 -1
  454. package/dist/eval-campaign-90jeaCAI.js +0 -343
  455. package/dist/eval-campaign-90jeaCAI.js.map +0 -1
  456. package/dist/evaluation-6Yoi74gM.d.ts.map +0 -1
  457. package/dist/exporters-X4xwla8A.js.map +0 -1
  458. package/dist/external-optimizer-process-BjMGUuds.js.map +0 -1
  459. package/dist/external-optimizer-subprocess-BqW0hHcI.js.map +0 -1
  460. package/dist/failure-cluster-CbgEDcFJ.d.ts.map +0 -1
  461. package/dist/feedback-trajectory-C4EG0nNH.d.ts.map +0 -1
  462. package/dist/hf-dataset-BR2yhuNn.js.map +0 -1
  463. package/dist/index-BFRO6chs.d.ts.map +0 -1
  464. package/dist/index-BO41eGj5.d.ts.map +0 -1
  465. package/dist/index-CyszSg4O.d.ts.map +0 -1
  466. package/dist/index-DHKEAv3s.d.ts.map +0 -1
  467. package/dist/index-DPcngtwt.d.ts.map +0 -1
  468. package/dist/integrity-dLiDe2kD.js.map +0 -1
  469. package/dist/journal-DfRz9kps.js.map +0 -1
  470. package/dist/judge-gate-CE7CQAgt.d.ts.map +0 -1
  471. package/dist/kind-factory-BobI5Jy3.js.map +0 -1
  472. package/dist/ledger-core-Gq7yp8lP.js.map +0 -1
  473. package/dist/llm-client-CyBAoYiX.js.map +0 -1
  474. package/dist/llm-judge-CG12SQJq.js.map +0 -1
  475. package/dist/matrix-DSrSxjjy.d.ts.map +0 -1
  476. package/dist/matrix-Dm0BQW7U.js.map +0 -1
  477. package/dist/mean-DHoMiudD.js.map +0 -1
  478. package/dist/normal-DsJZtrJc.js +0 -25
  479. package/dist/normal-DsJZtrJc.js.map +0 -1
  480. package/dist/paired-arms-CwMGJsrS.js.map +0 -1
  481. package/dist/paired-binary-CvpFHfKZ.js.map +0 -1
  482. package/dist/paired-promotion-decision-DPKz8ZnF.d.ts.map +0 -1
  483. package/dist/pareto-Cm6ndrWN.js.map +0 -1
  484. package/dist/prime-protocol-CHKgA3a6.js.map +0 -1
  485. package/dist/prime-protocol-HiB9HX7E.d.ts.map +0 -1
  486. package/dist/produced-state-BsDmnwxc.js.map +0 -1
  487. package/dist/proposal-findings-CY1xKn_a.js.map +0 -1
  488. package/dist/query-BUCZCK7F.js.map +0 -1
  489. package/dist/redact-mtCCcT8z.d.ts.map +0 -1
  490. package/dist/release-confidence-CpuP65S5.js.map +0 -1
  491. package/dist/report-command-SC9nqb4M.js.map +0 -1
  492. package/dist/reward-CpCmAM8L.d.ts.map +0 -1
  493. package/dist/reward-hacking-RVdZ6Kkt.d.ts.map +0 -1
  494. package/dist/rollout-DZV1t2Wg.js.map +0 -1
  495. package/dist/run-record-C1E24G2A.d.ts.map +0 -1
  496. package/dist/run-record-C5e7FjhJ.js.map +0 -1
  497. package/dist/schema-2g7trUwH.d.ts.map +0 -1
  498. package/dist/schema-BuiujZ-g.js.map +0 -1
  499. package/dist/schema-CSf6qWgZ.js.map +0 -1
  500. package/dist/search-kernel-DyNEa6IU.d.ts.map +0 -1
  501. package/dist/search-ledger-file-DDQUgvCg.js.map +0 -1
  502. package/dist/search-shipper-BOSD46qv.js.map +0 -1
  503. package/dist/semantic-concept-judge-BQF8te0T.js.map +0 -1
  504. package/dist/sequential-Bf_pbE50.js.map +0 -1
  505. package/dist/server-e9oWO_bN.js.map +0 -1
  506. package/dist/single-run-lock-CV9Eyd_q.js.map +0 -1
  507. package/dist/skillopt-optimization-method-BiwA8nfX.js.map +0 -1
  508. package/dist/statistical-heldout-42NlCNDe.js.map +0 -1
  509. package/dist/statistical-heldout-CYpGaoDH.d.ts.map +0 -1
  510. package/dist/steps-Bg5-KVx1.d.ts.map +0 -1
  511. package/dist/store-tool-spans-CIX5arAG.js.map +0 -1
  512. package/dist/store-tool-spans-DdCSPa7C.d.ts.map +0 -1
  513. package/dist/student-t-DVPW7CdL.js +0 -39
  514. package/dist/student-t-DVPW7CdL.js.map +0 -1
  515. package/dist/summary-report-V_JACR4Y.js.map +0 -1
  516. package/dist/summary-report-bdFDXq0b.d.ts.map +0 -1
  517. package/dist/task-failure-attributes-hEmASjix.js.map +0 -1
  518. package/dist/terminal-record-DZyQZ3pz.js.map +0 -1
  519. package/dist/tool-groups-C3NEcdCM.d.ts.map +0 -1
  520. package/dist/tool-waste-BKgvd-TP.d.ts.map +0 -1
  521. package/dist/trajectory-Y1fmdlw7.d.ts +0 -24
  522. package/dist/trajectory-Y1fmdlw7.d.ts.map +0 -1
  523. package/dist/tree-C1XYeXvy.js.map +0 -1
  524. package/dist/types-BqkVmHfv.d.ts.map +0 -1
  525. package/dist/types-C02qtnG8.d.ts.map +0 -1
  526. package/dist/types-C1tnkEgM.d.ts.map +0 -1
  527. package/dist/types-IXD2HJzu.js.map +0 -1
  528. package/dist/usage-receipt-Cx3xNeyB.js +0 -76
  529. package/dist/usage-receipt-Cx3xNeyB.js.map +0 -1
  530. package/dist/verdict-Dl0tiT1X.js.map +0 -1
  531. package/dist/verdict-cache-Bm8EKL2s.js.map +0 -1
  532. package/docs/design/self-improvement-evidence-audit.md +0 -263
@@ -1,24 +1,25 @@
1
- import { i as JudgeError, s as ValidationError } from "./errors-DVZiqS7K.js";
2
- import { a as hashCanonical, i as compareCodeUnits, r as canonicalString, t as LEDGER_HASH_PATTERN } from "./canonical-B2RqLofv.js";
3
- import { i as modelHasSnapshot, s as searchCellRunId } from "./run-record-Boc5IUTp.js";
4
- import { t as mulberry32 } from "./random-ByUSnV9w.js";
5
- import { o as summarizeNumberSeries, s as weightedComposite, t as confidenceInterval } from "./descriptive-DboxPZrG.js";
6
- import { r as minimumPairsForPairedDeltaTest } from "./paired-promotion-decision-DsAPYGCm.js";
7
- import { a as heldoutSignificance, i as dimensionRegressions, o as pairHoldout } from "./statistical-heldout-42NlCNDe.js";
8
- import { n as contentHash } from "./verdict-cache-Bm8EKL2s.js";
9
- import { r as projectCampaignCellQuality, t as campaignCellCostProvenance } from "./run-record-C5e7FjhJ.js";
10
- import { n as CostAccountingIncompleteError, t as CostLedger } from "./cost-ledger-BP8V4kUQ.js";
11
- import { B as assertCampaignDesign, D as FinalEvidenceError, E as FinalEvidenceConflictError, F as surfaceHash, G as campaignSplitDigest, H as assertCompleteCampaign, I as surfaceHashMatches, L as BackendIntegrityError, M as renderSurfaceDiff, N as surfaceContentHash, P as surfaceDispatchRef, U as campaignCoverage, V as assertCampaignSplitIdentity, W as campaignScenarioIdentity, b as campaignMeanCompositeOrNull, q as formatCoverageFailures, v as campaignBreakdown, y as campaignMeanComposite } from "./campaign-evidence-Clrn4hBV.js";
1
+ import { i as JudgeError, s as ValidationError } from "./errors-BcOs5MKW.js";
2
+ import { a as hashCanonical, i as compareCodeUnits, r as canonicalString, t as LEDGER_HASH_PATTERN } from "./canonical-Dq2hOQ8L.js";
3
+ import { i as modelHasSnapshot, s as searchCellRunId } from "./run-record-DhEJgoPz.js";
4
+ import { t as mulberry32 } from "./random-DDDcC7YU.js";
5
+ import { o as summarizeNumberSeries, s as weightedComposite, t as confidenceInterval } from "./descriptive-CVNy48rR.js";
6
+ import { r as minimumPairsForPairedDeltaTest } from "./paired-promotion-decision-D0SHb-ZL.js";
7
+ import { a as heldoutSignificance, i as dimensionRegressions, o as pairHoldout } from "./statistical-heldout-Bksv92Th.js";
8
+ import { n as contentHash } from "./verdict-cache-CL_7tePD.js";
9
+ import { r as projectCampaignCellQuality, t as campaignCellCostProvenance } from "./run-record-CysKLB93.js";
10
+ import { n as CostAccountingIncompleteError, t as CostLedger } from "./cost-ledger-D0352Z4d.js";
11
+ import { A as surfaceHashMatches, D as surfaceContentHash, E as renderSurfaceDiff, F as assertCampaignSplitIdentity, I as assertCompleteCampaign, L as campaignCoverage, O as surfaceDispatchRef, P as assertCampaignDesign, R as campaignScenarioIdentity, V as formatCoverageFailures, b as FinalEvidenceConflictError, f as campaignBreakdown, j as BackendIntegrityError, k as surfaceHash, m as campaignMeanCompositeOrNull, p as campaignMeanComposite, x as FinalEvidenceError, z as campaignSplitDigest } from "./campaign-evidence-DunzcJGv.js";
12
12
  import { t as deepFreezeCanonicalJson } from "./deep-freeze-CurcOQg3.js";
13
- import { N as createRunCostLedger, P as fsCampaignStorage, g as isRecord, h as isExternalTextCandidate } from "./external-optimizer-subprocess-BqW0hHcI.js";
14
- import { r as summarizeEvaluationUnits, t as defineEvaluationClaim } from "./claim-q4HQhK1G.js";
15
- import { c as redactText, d as detectCredential, f as detectPersonalData, n as REDACTION_VERSION } from "./redact-B5xXKqQ3.js";
16
- import { t as detectRewardHacking } from "./reward-hacking-E0d-WVcP.js";
17
- import { t as assertProposalFindings } from "./proposal-findings-CY1xKn_a.js";
18
- import { i as runSearch, m as searchExpansionIndex, n as paretoFrontier, o as searchPolicyView, p as SEARCH_KERNEL_SOURCE, r as paretoFrontierWithCrowding, s as EXPANSION_OPERATORS, w as estimateNode } from "./pareto-Cm6ndrWN.js";
19
- import { a as validateSearchLedgerEvent, d as searchNodeId, f as searchTaskSetDigest, i as replaySearchLedgerText, n as openSearchLedger, u as searchCellId } from "./search-ledger-CVUGLDZp.js";
20
- import { n as searchProposerView, t as renderSearchSummary } from "./search-summary-BQCSWf1g.js";
21
- import { a as maximumChargeForLlmRequest, i as costReceiptFromLlmError, r as costReceiptFromLlm, x as stripFencedJson } from "./llm-client-CyBAoYiX.js";
13
+ import { M as fsCampaignStorage, h as isExternalTextCandidate, j as createRunCostLedger } from "./external-optimizer-subprocess-B5gdLRZv.js";
14
+ import { r as summarizeEvaluationUnits, t as defineEvaluationClaim } from "./claim-BPVuyHuy.js";
15
+ import { d as detectPersonalData, s as redactText, t as REDACTION_VERSION, u as detectCredential } from "./redact-mtkTiJVk.js";
16
+ import { t as detectRewardHacking } from "./reward-hacking-ByuETaIb.js";
17
+ import { t as assertProposalFindings } from "./proposal-findings-n-5WBbEx.js";
18
+ import { i as runSearch, m as searchExpansionIndex, n as paretoFrontier, o as searchPolicyView, p as SEARCH_KERNEL_SOURCE, r as paretoFrontierWithCrowding, s as EXPANSION_OPERATORS, w as estimateNode } from "./pareto-CTv8vozZ.js";
19
+ import { a as validateSearchLedgerEvent, d as searchNodeId, f as searchTaskSetDigest, i as replaySearchLedgerText, n as openSearchLedger, u as searchCellId } from "./search-ledger-ClpN4I3e.js";
20
+ import { n as searchProposerView, t as renderSearchSummary } from "./search-summary-CrBuvVnf.js";
21
+ import { i as stripFencedJson, n as isRecord } from "./json-BhdsJaOQ.js";
22
+ import { a as maximumChargeForLlmRequest, i as costReceiptFromLlmError, r as costReceiptFromLlm } from "./llm-client-_Y6ic-tf.js";
22
23
  import { createHash } from "node:crypto";
23
24
  import { writeFileSync } from "node:fs";
24
25
  import { z } from "zod";
@@ -336,52 +337,41 @@ function hasLlmCall(value) {
336
337
  * persisted cell result. `runCampaign` owns the schedule; this module owns
337
338
  * everything that happens inside a single cell.
338
339
  */
339
- async function executeCell(args) {
340
- const storage = args.storage;
341
- const cellDir = cellDirectory(args.opts.runDir, args.slot.cellId);
342
- storage.ensureDir(cellDir);
343
- const stableCostTags = stableCostTagsFor(args.opts, args.slot);
344
- const costTags = {
345
- ...stableCostTags,
346
- runAttemptId: args.runAttemptId
347
- };
348
- const attemptCostTags = {
349
- ...costTags,
350
- attemptNumber: String(args.attempt)
351
- };
352
- const cachePath = join(cellDir, "cached-result.json");
353
- if (args.resumable) {
354
- const cached = readCachedCell({
355
- storage,
356
- cachePath,
357
- cellId: args.slot.cellId,
358
- manifestHash: args.manifestHash
359
- });
360
- if (cached.status === "miss" && cacheIssueRequiresExplicitRerun(cached.reason) && !args.opts.rerunInvalidCachedCells) throw invalidCachedCellsError([{
340
+ /** A cached cell is reused only when its receipts still match the ledger; invalid cache entries need an explicit rerun. */
341
+ function reuseCachedCell(args, cachePath, stableCostTags) {
342
+ const cached = readCachedCell({
343
+ storage: args.storage,
344
+ cachePath,
345
+ cellId: args.slot.cellId,
346
+ manifestHash: args.manifestHash
347
+ });
348
+ if (cached.status === "miss" && cacheIssueRequiresExplicitRerun(cached.reason) && !args.opts.rerunInvalidCachedCells) throw invalidCachedCellsError([{
349
+ cellId: args.slot.cellId,
350
+ reason: cached.reason
351
+ }]);
352
+ if (cached.status === "hit" && (!cached.cell.error || args.opts.reuseFailedCells)) {
353
+ const receiptProblem = cachedCellReceiptProblem(cached.cell, args.costLedger, stableCostTags);
354
+ if (receiptProblem === void 0) {
355
+ const cell = withCurrentAgentModelEvidence(cached.cell, args.costLedger, stableCostTags);
356
+ enforceDispatchUsage(cell, args.opts.expectUsage ?? "warn");
357
+ return {
358
+ cell: {
359
+ ...cell,
360
+ cached: true
361
+ },
362
+ artifactsByPath: {}
363
+ };
364
+ }
365
+ if (!args.opts.rerunInvalidCachedCells) throw invalidCachedCellsError([{
361
366
  cellId: args.slot.cellId,
362
- reason: cached.reason
367
+ reason: "invalid-cost-receipts",
368
+ detail: receiptProblem
363
369
  }]);
364
- if (cached.status === "hit" && (!cached.cell.error || args.opts.reuseFailedCells)) {
365
- const receiptProblem = cachedCellReceiptProblem(cached.cell, args.costLedger, stableCostTags);
366
- if (receiptProblem === void 0) {
367
- const cell = withCurrentAgentModelEvidence(cached.cell, args.costLedger, stableCostTags);
368
- enforceDispatchUsage(cell, args.opts.expectUsage ?? "warn");
369
- return {
370
- cell: {
371
- ...cell,
372
- cached: true
373
- },
374
- artifactsByPath: {}
375
- };
376
- }
377
- if (!args.opts.rerunInvalidCachedCells) throw invalidCachedCellsError([{
378
- cellId: args.slot.cellId,
379
- reason: "invalid-cost-receipts",
380
- detail: receiptProblem
381
- }]);
382
- }
383
370
  }
384
- const startMs = Date.now();
371
+ }
372
+ /** The directories, writers, abort chain and dispatch context of one execution attempt. */
373
+ function openAttempt(args, cellDir, attemptCostTags) {
374
+ const { storage } = args;
385
375
  const attempt = {
386
376
  runAttemptId: args.runAttemptId,
387
377
  number: args.attempt
@@ -406,7 +396,7 @@ async function executeCell(args) {
406
396
  artifactsByPath[`${args.slot.cellId}/${attemptPrefix}/identity.json`] = identityPath;
407
397
  storage.write(join(cellDir, "latest-attempt.json"), JSON.stringify(attempt));
408
398
  const trace = args.buildTraceWriter(args.slot.cellId, traceDir);
409
- let paidCallStarted = false;
399
+ const paid = { started: false };
410
400
  const artifacts = {
411
401
  async write(path, content) {
412
402
  const fullPath = resolve(artifactDir, path);
@@ -421,8 +411,10 @@ async function executeCell(args) {
421
411
  return artifacts.write(path, JSON.stringify(value, null, 2));
422
412
  }
423
413
  };
414
+ const cellAbort = new AbortController();
415
+ const onCampaignAbort = () => cellAbort.abort(args.signal.reason);
424
416
  const cost = { async runPaidCall(input) {
425
- paidCallStarted = true;
417
+ paid.started = true;
426
418
  const result = await args.costLedger.runPaidCall({
427
419
  ...input,
428
420
  channel: input.channel ?? "agent",
@@ -438,25 +430,36 @@ async function executeCell(args) {
438
430
  scenario: args.slot.scenario,
439
431
  rep: args.slot.rep
440
432
  });
441
- const cellAbort = new AbortController();
442
- const onCampaignAbort = () => cellAbort.abort(args.signal.reason);
443
433
  if (args.signal.aborted) cellAbort.abort(args.signal.reason);
444
434
  else args.signal.addEventListener("abort", onCampaignAbort, { once: true });
445
- const ctx = {
446
- cellId: args.slot.cellId,
447
- runAttemptId: args.runAttemptId,
448
- rep: args.slot.rep,
449
- seed: args.slot.cellSeed,
450
- signal: cellAbort.signal,
435
+ return {
436
+ attempt,
437
+ attemptDir,
438
+ attemptPrefix,
439
+ artifactsByPath,
451
440
  trace,
452
- artifacts,
453
- cost,
454
- placement
441
+ paid,
442
+ cellAbort,
443
+ onCampaignAbort,
444
+ ctx: {
445
+ cellId: args.slot.cellId,
446
+ runAttemptId: args.runAttemptId,
447
+ rep: args.slot.rep,
448
+ seed: args.slot.cellSeed,
449
+ signal: cellAbort.signal,
450
+ trace,
451
+ artifacts,
452
+ cost,
453
+ placement
454
+ }
455
455
  };
456
+ }
457
+ /** Dispatch the scenario under the per-cell abort and the dispatch deadline. A failure is recorded, not thrown. */
458
+ async function dispatchCell(args, run) {
459
+ const { cellAbort, onCampaignAbort, ctx } = run;
456
460
  let artifact;
457
461
  let errorMessage;
458
462
  let failure;
459
- let fatalCellError;
460
463
  let dispatched;
461
464
  const timeoutMs = args.dispatchTimeoutMs;
462
465
  let timeoutTimer;
@@ -499,29 +502,41 @@ async function executeCell(args) {
499
502
  removeAbortListener();
500
503
  args.signal.removeEventListener("abort", onCampaignAbort);
501
504
  }
505
+ return {
506
+ artifact,
507
+ errorMessage,
508
+ failure,
509
+ dispatched
510
+ };
511
+ }
512
+ /** A dispatch that ignored cancellation, or paid calls that never settled, mean no trustworthy cell result exists. */
513
+ async function awaitCellShutdown(args, run, dispatched, costTags) {
514
+ const { trace, paid } = run;
502
515
  if (dispatched) {
503
516
  if (!await settlesWithin(dispatched, args.dispatchShutdownTimeoutMs)) {
504
517
  await trace.flush();
505
518
  throw new CostAccountingIncompleteError(`dispatch for cell '${args.slot.cellId}' ignored cancellation and did not stop within ${args.dispatchShutdownTimeoutMs}ms; no campaign result was produced`);
506
519
  }
507
520
  }
508
- if (paidCallStarted) {
509
- if (typeof args.costLedger.waitForIdle !== "function") {
510
- await trace.flush();
511
- throw new CostAccountingIncompleteError(`cost ledger for cell '${args.slot.cellId}' cannot prove that paid calls stopped`);
512
- }
513
- if (!await args.costLedger.waitForIdle({
514
- timeoutMs: args.dispatchShutdownTimeoutMs,
515
- filter: {
516
- channel: "agent",
517
- phase: args.costPhase,
518
- tags: costTags
519
- }
520
- })) {
521
- await trace.flush();
522
- throw new CostAccountingIncompleteError(`paid calls for cell '${args.slot.cellId}' did not settle within ${args.dispatchShutdownTimeoutMs}ms; no campaign result was produced`);
521
+ if (!paid.started) return;
522
+ if (typeof args.costLedger.waitForIdle !== "function") {
523
+ await trace.flush();
524
+ throw new CostAccountingIncompleteError(`cost ledger for cell '${args.slot.cellId}' cannot prove that paid calls stopped`);
525
+ }
526
+ if (!await args.costLedger.waitForIdle({
527
+ timeoutMs: args.dispatchShutdownTimeoutMs,
528
+ filter: {
529
+ channel: "agent",
530
+ phase: args.costPhase,
531
+ tags: costTags
523
532
  }
533
+ })) {
534
+ await trace.flush();
535
+ throw new CostAccountingIncompleteError(`paid calls for cell '${args.slot.cellId}' did not settle within ${args.dispatchShutdownTimeoutMs}ms; no campaign result was produced`);
524
536
  }
537
+ }
538
+ /** The agent spend of this attempt: cost, token usage and the models that served it. */
539
+ function measureAgentUsage(args, costTags) {
525
540
  const agentReceipts = args.costLedger.list({
526
541
  channel: "agent",
527
542
  tags: costTags
@@ -530,29 +545,29 @@ async function executeCell(args) {
530
545
  channel: "agent",
531
546
  tags: costTags
532
547
  });
533
- const tokenUsage = {
534
- input: agentCost.inputTokens,
535
- output: agentCost.outputTokens,
536
- ...agentCost.usageComplete ? {} : { tokensKnown: false },
537
- ...agentCost.reasoningTokens !== void 0 && agentCost.reasoningTokens > 0 ? { reasoning: agentCost.reasoningTokens } : {},
538
- ...agentCost.cachedTokens > 0 ? { cached: agentCost.cachedTokens } : {},
539
- ...agentCost.cacheWriteTokens !== void 0 && agentCost.cacheWriteTokens > 0 ? { cacheWrite: agentCost.cacheWriteTokens } : {}
540
- };
541
- const agentModelEvidence = modelEvidenceFromReceipts(agentReceipts);
542
- const dispatchResult = {
543
- cellId: args.slot.cellId,
544
- artifact,
545
- error: errorMessage,
546
- costUsd: agentCost.totalCostUsd,
547
- tokenUsage
548
+ return {
549
+ agentCost,
550
+ tokenUsage: {
551
+ input: agentCost.inputTokens,
552
+ output: agentCost.outputTokens,
553
+ ...agentCost.usageComplete ? {} : { tokensKnown: false },
554
+ ...agentCost.reasoningTokens !== void 0 && agentCost.reasoningTokens > 0 ? { reasoning: agentCost.reasoningTokens } : {},
555
+ ...agentCost.cachedTokens > 0 ? { cached: agentCost.cachedTokens } : {},
556
+ ...agentCost.cacheWriteTokens !== void 0 && agentCost.cacheWriteTokens > 0 ? { cacheWrite: agentCost.cacheWriteTokens } : {}
557
+ },
558
+ agentModelEvidence: modelEvidenceFromReceipts(agentReceipts)
548
559
  };
549
- try {
550
- enforceDispatchUsage(dispatchResult, args.opts.expectUsage ?? "warn");
551
- } catch (error) {
552
- await trace.flush();
553
- throw error;
554
- }
560
+ }
561
+ /**
562
+ * Run judges (only if we have an artifact). A judge that throws invalidates
563
+ * the cell — recorded as `error`, NOT folded into a fake composite:0 (a fake
564
+ * zero is indistinguishable from a real zero and poisons every aggregate).
565
+ */
566
+ async function judgeArtifact(args, artifact, attemptCostTags) {
555
567
  const judgeScores = {};
568
+ let errorMessage;
569
+ let failure;
570
+ let fatalCellError;
556
571
  if (artifact !== void 0) for (const judge of args.opts.judges ?? []) {
557
572
  if (judge.appliesTo && !judge.appliesTo(args.slot.scenario)) continue;
558
573
  try {
@@ -576,6 +591,79 @@ async function executeCell(args) {
576
591
  break;
577
592
  }
578
593
  }
594
+ return {
595
+ judgeScores,
596
+ errorMessage,
597
+ failure,
598
+ fatalCellError
599
+ };
600
+ }
601
+ /** Write the failure receipt and decide whether `cellRetry` dispatches this slot again. */
602
+ function recordCellFailure(args, run, failure, cell, costTags, fatalCellError) {
603
+ const { attemptDir, attemptPrefix, artifactsByPath } = run;
604
+ const receipt = {
605
+ schemaVersion: 1,
606
+ runAttemptId: args.runAttemptId,
607
+ recordedAt: args.now().toISOString(),
608
+ failure: {
609
+ stage: failure.stage,
610
+ ...failure.judge ? { judge: failure.judge } : {},
611
+ error: serializeCellError(failure.cause)
612
+ },
613
+ cell,
614
+ cost: args.costLedger.summary({
615
+ phase: args.costPhase,
616
+ tags: costTags
617
+ })
618
+ };
619
+ const retry = args.cellRetry !== void 0 && args.attempt < args.cellRetry.attempts && !args.signal.aborted && fatalCellError === void 0 && args.cellRetry.retryable(receipt.failure);
620
+ const failurePath = join(attemptDir, "failure-receipt.json");
621
+ args.storage.write(failurePath, JSON.stringify(receipt, null, 2));
622
+ artifactsByPath[`${args.slot.cellId}/${attemptPrefix}/failure-receipt.json`] = failurePath;
623
+ if (!retry) args.onFailure?.(failure);
624
+ return retry;
625
+ }
626
+ async function executeCell(args) {
627
+ const storage = args.storage;
628
+ const cellDir = cellDirectory(args.opts.runDir, args.slot.cellId);
629
+ storage.ensureDir(cellDir);
630
+ const stableCostTags = stableCostTagsFor(args.opts, args.slot);
631
+ const costTags = {
632
+ ...stableCostTags,
633
+ runAttemptId: args.runAttemptId
634
+ };
635
+ const attemptCostTags = {
636
+ ...costTags,
637
+ attemptNumber: String(args.attempt)
638
+ };
639
+ const cachePath = join(cellDir, "cached-result.json");
640
+ if (args.resumable) {
641
+ const reused = reuseCachedCell(args, cachePath, stableCostTags);
642
+ if (reused) return reused;
643
+ }
644
+ const startMs = Date.now();
645
+ const run = openAttempt(args, cellDir, attemptCostTags);
646
+ const { attempt, attemptDir, attemptPrefix, artifactsByPath, trace } = run;
647
+ const dispatch = await dispatchCell(args, run);
648
+ await awaitCellShutdown(args, run, dispatch.dispatched, costTags);
649
+ const { artifact } = dispatch;
650
+ const { agentCost, tokenUsage, agentModelEvidence } = measureAgentUsage(args, costTags);
651
+ try {
652
+ enforceDispatchUsage({
653
+ cellId: args.slot.cellId,
654
+ artifact,
655
+ error: dispatch.errorMessage,
656
+ costUsd: agentCost.totalCostUsd,
657
+ tokenUsage
658
+ }, args.opts.expectUsage ?? "warn");
659
+ } catch (error) {
660
+ await trace.flush();
661
+ throw error;
662
+ }
663
+ const judged = await judgeArtifact(args, artifact, attemptCostTags);
664
+ const failure = judged.failure ?? dispatch.failure;
665
+ const errorMessage = judged.failure ? judged.errorMessage : dispatch.errorMessage;
666
+ const { judgeScores, fatalCellError } = judged;
579
667
  if (failure) await waitForFailedCellCostSettlement({
580
668
  costLedger: args.costLedger,
581
669
  costPhase: args.costPhase,
@@ -605,29 +693,7 @@ async function executeCell(args) {
605
693
  ...failure?.judge ? { errorJudge: failure.judge } : {},
606
694
  error: errorMessage
607
695
  };
608
- let retry = false;
609
- if (failure) {
610
- const receipt = {
611
- schemaVersion: 1,
612
- runAttemptId: args.runAttemptId,
613
- recordedAt: args.now().toISOString(),
614
- failure: {
615
- stage: failure.stage,
616
- ...failure.judge ? { judge: failure.judge } : {},
617
- error: serializeCellError(failure.cause)
618
- },
619
- cell,
620
- cost: args.costLedger.summary({
621
- phase: args.costPhase,
622
- tags: costTags
623
- })
624
- };
625
- retry = args.cellRetry !== void 0 && args.attempt < args.cellRetry.attempts && !args.signal.aborted && fatalCellError === void 0 && args.cellRetry.retryable(receipt.failure);
626
- const failurePath = join(attemptDir, "failure-receipt.json");
627
- storage.write(failurePath, JSON.stringify(receipt, null, 2));
628
- artifactsByPath[`${args.slot.cellId}/${attemptPrefix}/failure-receipt.json`] = failurePath;
629
- if (!retry) args.onFailure?.(failure);
630
- }
696
+ const retry = failure ? recordCellFailure(args, run, failure, cell, costTags, fatalCellError) : false;
631
697
  await trace.flush();
632
698
  const resultPath = join(attemptDir, "result.json");
633
699
  storage.write(resultPath, JSON.stringify(cell));
@@ -1889,261 +1955,269 @@ function excerptAt(source, at, needleLength) {
1889
1955
  return (start > 0 ? "…" : "") + source.slice(start, end) + (end < source.length ? "…" : "");
1890
1956
  }
1891
1957
  //#endregion
1892
- //#region src/campaign/gates/default-production-gate.ts
1893
- /**
1894
- * Opinionated production gate composing held-out significance, red-team, reward-hacking, and canary checks into a single `Gate.decide` decision.
1895
- */
1896
- function defaultProductionGate(options) {
1897
- const deltaThreshold = options.deltaThreshold ?? 0;
1898
- const confidence = options.confidence ?? .95;
1899
- const resamples = options.bootstrapResamples ?? 2e3;
1900
- const seed = options.bootstrapSeed ?? 1337;
1901
- const minProductiveRuns = options.minProductiveRuns ?? 3;
1902
- const heldoutStatistic = options.heldoutStatistic ?? "mean";
1903
- const explicitlyRequired = new Set(options.requiredChecks ?? []);
1904
- const scenarioIds = new Set(options.holdoutScenarios.map((scenario) => scenario.id));
1905
- const independentUnitByScenarioId = options.independentUnitByScenarioId === void 0 ? void 0 : new Map(options.independentUnitByScenarioId);
1906
- if (independentUnitByScenarioId !== void 0) for (const id of scenarioIds) {
1907
- const unit = independentUnitByScenarioId.get(id);
1908
- if (typeof unit !== "string" || unit.length === 0 || unit.trim() !== unit) throw new Error(`defaultProductionGate: missing independent unit for holdout scenario '${id}'`);
1909
- }
1958
+ //#region src/campaign/gates/default-production-checks.ts
1959
+ function newFindings() {
1960
+ const reasons = [];
1961
+ const contributing = [];
1962
+ const requiredUnavailable = /* @__PURE__ */ new Set();
1963
+ const unavailable = (name, required, reason, detail = {}) => {
1964
+ contributing.push({
1965
+ name,
1966
+ status: "not_evaluated",
1967
+ detail: {
1968
+ reason,
1969
+ required,
1970
+ ...detail
1971
+ }
1972
+ });
1973
+ if (required) {
1974
+ requiredUnavailable.add(name);
1975
+ reasons.push(`${name}: ${reason}`);
1976
+ }
1977
+ };
1910
1978
  return {
1911
- name: "defaultProductionGate",
1912
- async decide(ctx) {
1913
- const reasons = [];
1914
- const contributing = [];
1915
- const requiredUnavailable = /* @__PURE__ */ new Set();
1916
- const unavailable = (name, required, reason, detail = {}) => {
1917
- contributing.push({
1918
- name,
1919
- status: "not_evaluated",
1920
- detail: {
1921
- reason,
1922
- required,
1923
- ...detail
1924
- }
1925
- });
1926
- if (required) {
1927
- requiredUnavailable.add(name);
1928
- reasons.push(`${name}: ${reason}`);
1929
- }
1930
- };
1931
- let delta;
1932
- if (!ctx.baselineJudgeScores) unavailable("heldout-significance", true, "baselineJudgeScores is required for a held-out comparison");
1933
- else {
1934
- const sig = heldoutSignificance(pairHoldout(ctx.judgeScores, ctx.baselineJudgeScores, scenarioIds, (s) => s.composite), {
1935
- deltaThreshold,
1936
- minProductiveRuns,
1937
- confidence,
1938
- resamples,
1939
- seed,
1940
- statistic: heldoutStatistic,
1941
- independentUnitByScenarioId
1942
- });
1943
- const dec = sig.decision;
1944
- delta = dec.delta;
1945
- const heldoutPass = sig.significant;
1946
- contributing.push({
1947
- name: "heldout-significance",
1948
- status: sig.fewRuns ? "not_evaluated" : heldoutPass ? "pass" : "fail",
1949
- detail: {
1950
- n: sig.n,
1951
- pairedCellN: sig.pairedCellN,
1952
- observationUnit: sig.observationUnit,
1953
- unitIds: sig.unitIds,
1954
- delta,
1955
- decisionStatistic: sig.decisionStatistic,
1956
- decisionMethod: sig.decisionMethod,
1957
- binaryScale: dec.binaryScale,
1958
- mcnemar: dec.mcnemar,
1959
- indeterminate: dec.indeterminate,
1960
- deltaMean: sig.bootstrap.mean,
1961
- deltaMedianDiagnostic: sig.medianBootstrap.median,
1962
- deltaMedian: sig.medianBootstrap.median,
1963
- tieFraction: sig.tieFraction,
1964
- ciLow: dec.low,
1965
- ciHigh: dec.high,
1966
- bootstrapCiLow: sig.bootstrap.low,
1967
- bootstrapCiHigh: sig.bootstrap.high,
1968
- confidence: dec.confidence,
1969
- deltaThreshold,
1970
- fewRuns: sig.fewRuns
1971
- }
1972
- });
1973
- if (sig.fewRuns) requiredUnavailable.add("heldout-significance");
1974
- if (!heldoutPass) {
1975
- const tieNote = sig.tieFraction >= .4 ? `; ${(sig.tieFraction * 100).toFixed(0)}% tied observation units` : "";
1976
- const ci = `${(dec.confidence * 100).toFixed(0)}% CI [${dec.low.toFixed(3)}, ${dec.high.toFixed(3)}]`;
1977
- reasons.push(sig.fewRuns ? `held-out: only ${sig.n} paired observation units (< ${sig.minimumRequired}) — too few to claim significance` : dec.indeterminate ? `held-out: ${dec.indeterminateCause}, so the paired CI is ${ci} and carries no direction — it cannot clear threshold ${deltaThreshold} on evidence${tieNote}` : dec.exactTestVetoes ? `held-out: McNemar exact p=${dec.mcnemar?.pValue.toExponential(2)} does not reject at α=${(1 - dec.confidence).toFixed(4)} (${dec.label} Δ ${delta.toFixed(3)}, ${ci}${tieNote})` : dec.method === "exact-sign" ? `held-out: exact one-sided sign test p=${dec.pValue} does not reject at α=${((1 - dec.confidence) / 2).toFixed(4)} (${dec.label} Δ ${delta.toFixed(3)}, diagnostic ${ci}${tieNote})` : `held-out CI.low ${dec.low.toFixed(3)} ≤ threshold ${deltaThreshold} (${dec.label} Δ ${delta.toFixed(3)}, ${ci}${tieNote})`);
1978
- }
1979
+ reasons,
1980
+ contributing,
1981
+ requiredUnavailable,
1982
+ unavailable
1983
+ };
1984
+ }
1985
+ /** Held-out significance of the candidate over the baseline, paired by cell before any averaging. */
1986
+ function checkHeldout(g, cfg, ctx) {
1987
+ const { reasons, contributing, requiredUnavailable, unavailable } = g;
1988
+ const { deltaThreshold, confidence, resamples, seed, minProductiveRuns, heldoutStatistic, scenarioIds, independentUnitByScenarioId } = cfg;
1989
+ let delta;
1990
+ if (!ctx.baselineJudgeScores) unavailable("heldout-significance", true, "baselineJudgeScores is required for a held-out comparison");
1991
+ else {
1992
+ const sig = heldoutSignificance(pairHoldout(ctx.judgeScores, ctx.baselineJudgeScores, scenarioIds, (s) => s.composite), {
1993
+ deltaThreshold,
1994
+ minProductiveRuns,
1995
+ confidence,
1996
+ resamples,
1997
+ seed,
1998
+ statistic: heldoutStatistic,
1999
+ independentUnitByScenarioId
2000
+ });
2001
+ const dec = sig.decision;
2002
+ delta = dec.delta;
2003
+ const heldoutPass = sig.significant;
2004
+ contributing.push({
2005
+ name: "heldout-significance",
2006
+ status: sig.fewRuns ? "not_evaluated" : heldoutPass ? "pass" : "fail",
2007
+ detail: {
2008
+ n: sig.n,
2009
+ pairedCellN: sig.pairedCellN,
2010
+ observationUnit: sig.observationUnit,
2011
+ unitIds: sig.unitIds,
2012
+ delta,
2013
+ decisionStatistic: sig.decisionStatistic,
2014
+ decisionMethod: sig.decisionMethod,
2015
+ binaryScale: dec.binaryScale,
2016
+ mcnemar: dec.mcnemar,
2017
+ indeterminate: dec.indeterminate,
2018
+ deltaMean: sig.bootstrap.mean,
2019
+ deltaMedianDiagnostic: sig.medianBootstrap.median,
2020
+ deltaMedian: sig.medianBootstrap.median,
2021
+ tieFraction: sig.tieFraction,
2022
+ ciLow: dec.low,
2023
+ ciHigh: dec.high,
2024
+ bootstrapCiLow: sig.bootstrap.low,
2025
+ bootstrapCiHigh: sig.bootstrap.high,
2026
+ confidence: dec.confidence,
2027
+ deltaThreshold,
2028
+ fewRuns: sig.fewRuns
1979
2029
  }
1980
- const dimensionsProvided = options.criticalDimensions !== void 0;
1981
- const criticalDimensions = [...new Set(options.criticalDimensions ?? [])];
1982
- const dimensionsRequired = dimensionsProvided || explicitlyRequired.has("dimension-regression");
1983
- if (!dimensionsProvided) unavailable("dimension-regression", dimensionsRequired, "criticalDimensions is not configured");
1984
- else if (criticalDimensions.length === 0) unavailable("dimension-regression", true, "criticalDimensions must contain at least one dimension");
1985
- else if (!ctx.baselineJudgeScores) unavailable("dimension-regression", true, "baselineJudgeScores is required to measure dimension regressions", { guarded: criticalDimensions });
1986
- else {
1987
- const dimRegs = dimensionRegressions(ctx.judgeScores, ctx.baselineJudgeScores, scenarioIds, criticalDimensions, {
1988
- tolerance: options.regressionTolerance,
1989
- confidence,
1990
- resamples,
1991
- seed,
1992
- independentUnitByScenarioId,
1993
- minProductiveRuns
1994
- });
1995
- const measured = new Set(dimRegs.map((result) => result.dimension));
1996
- const missingDimensions = criticalDimensions.filter((dimension) => !measured.has(dimension));
1997
- const regressed = dimRegs.filter((result) => result.regressed);
1998
- const incompleteDimensions = dimRegs.filter((result) => result.fewRuns || result.missingCellIds.length > 0 || result.missingScenarioIds.length > 0);
1999
- const dimensionStatus = regressed.length > 0 ? "fail" : missingDimensions.length > 0 || incompleteDimensions.length > 0 ? "not_evaluated" : "pass";
2000
- contributing.push({
2001
- name: "dimension-regression",
2002
- status: dimensionStatus,
2003
- detail: {
2004
- guarded: criticalDimensions,
2005
- missingDimensions,
2006
- incompleteDimensions: incompleteDimensions.map((result) => result.dimension),
2007
- regressions: dimRegs.map((result) => ({
2008
- dimension: result.dimension,
2009
- ciLow: result.ci.low,
2010
- ciHigh: result.ci.high,
2011
- decisionStatistic: result.decisionStatistic,
2012
- indeterminate: result.indeterminate,
2013
- bootstrapCiLow: result.bootstrap.low,
2014
- median: result.bootstrap.median,
2015
- tolerance: result.tolerance,
2016
- n: result.n,
2017
- pairedCellN: result.pairedCellN,
2018
- observationUnit: result.observationUnit,
2019
- minimumRequired: result.minimumRequired,
2020
- fewRuns: result.fewRuns,
2021
- missingCellIds: result.missingCellIds,
2022
- missingScenarioIds: result.missingScenarioIds,
2023
- regressed: result.regressed
2024
- }))
2025
- }
2026
- });
2027
- if (missingDimensions.length > 0) {
2028
- requiredUnavailable.add("dimension-regression");
2029
- reasons.push(`critical dimension(s) were not scored: ${missingDimensions.join(", ")}`);
2030
- }
2031
- if (incompleteDimensions.length > 0) {
2032
- requiredUnavailable.add("dimension-regression");
2033
- reasons.push(`critical dimension evidence is incomplete: ${incompleteDimensions.map((result) => `${result.dimension} has ${result.n} paired observation units (minimum ${result.minimumRequired}), ${result.missingCellIds.length} unscored cells, ${result.missingScenarioIds.length} unscored scenarios`).join("; ")}`);
2034
- }
2035
- if (regressed.length > 0) reasons.push(`critical dimension(s) regressed: ${regressed.map((result) => result.bootstrap.low < -result.tolerance ? `${result.dimension} bootstrap CI.low ${result.bootstrap.low.toFixed(3)} < -${result.tolerance}` : `${result.dimension} paired test supports a drop greater than ${result.tolerance}`).join("; ")}`);
2030
+ });
2031
+ if (sig.fewRuns) requiredUnavailable.add("heldout-significance");
2032
+ if (!heldoutPass) {
2033
+ const tieNote = sig.tieFraction >= .4 ? `; ${(sig.tieFraction * 100).toFixed(0)}% tied observation units` : "";
2034
+ const ci = `${(dec.confidence * 100).toFixed(0)}% CI [${dec.low.toFixed(3)}, ${dec.high.toFixed(3)}]`;
2035
+ reasons.push(sig.fewRuns ? `held-out: only ${sig.n} paired observation units (< ${sig.minimumRequired}) — too few to claim significance` : dec.indeterminate ? `held-out: ${dec.indeterminateCause}, so the paired CI is ${ci} and carries no direction — it cannot clear threshold ${deltaThreshold} on evidence${tieNote}` : dec.exactTestVetoes ? `held-out: McNemar exact p=${dec.mcnemar?.pValue.toExponential(2)} does not reject at α=${(1 - dec.confidence).toFixed(4)} (${dec.label} Δ ${delta.toFixed(3)}, ${ci}${tieNote})` : dec.method === "exact-sign" ? `held-out: exact one-sided sign test p=${dec.pValue} does not reject at α=${((1 - dec.confidence) / 2).toFixed(4)} (${dec.label} Δ ${delta.toFixed(3)}, diagnostic ${ci}${tieNote})` : `held-out CI.low ${dec.low.toFixed(3)} ≤ threshold ${deltaThreshold} (${dec.label} Δ ${delta.toFixed(3)}, ${ci}${tieNote})`);
2036
+ }
2037
+ }
2038
+ return delta;
2039
+ }
2040
+ /** Anti-Goodhart guard: a net composite gain must not hide a regression on a critical dimension. */
2041
+ function checkDimensions(g, cfg, ctx) {
2042
+ const { reasons, contributing, requiredUnavailable, unavailable } = g;
2043
+ const { options, confidence, resamples, seed, minProductiveRuns, explicitlyRequired, scenarioIds, independentUnitByScenarioId } = cfg;
2044
+ const dimensionsProvided = options.criticalDimensions !== void 0;
2045
+ const criticalDimensions = [...new Set(options.criticalDimensions ?? [])];
2046
+ const dimensionsRequired = dimensionsProvided || explicitlyRequired.has("dimension-regression");
2047
+ if (!dimensionsProvided) unavailable("dimension-regression", dimensionsRequired, "criticalDimensions is not configured");
2048
+ else if (criticalDimensions.length === 0) unavailable("dimension-regression", true, "criticalDimensions must contain at least one dimension");
2049
+ else if (!ctx.baselineJudgeScores) unavailable("dimension-regression", true, "baselineJudgeScores is required to measure dimension regressions", { guarded: criticalDimensions });
2050
+ else {
2051
+ const dimRegs = dimensionRegressions(ctx.judgeScores, ctx.baselineJudgeScores, scenarioIds, criticalDimensions, {
2052
+ tolerance: options.regressionTolerance,
2053
+ confidence,
2054
+ resamples,
2055
+ seed,
2056
+ independentUnitByScenarioId,
2057
+ minProductiveRuns
2058
+ });
2059
+ const measured = new Set(dimRegs.map((result) => result.dimension));
2060
+ const missingDimensions = criticalDimensions.filter((dimension) => !measured.has(dimension));
2061
+ const regressed = dimRegs.filter((result) => result.regressed);
2062
+ const incompleteDimensions = dimRegs.filter((result) => result.fewRuns || result.missingCellIds.length > 0 || result.missingScenarioIds.length > 0);
2063
+ const dimensionStatus = regressed.length > 0 ? "fail" : missingDimensions.length > 0 || incompleteDimensions.length > 0 ? "not_evaluated" : "pass";
2064
+ contributing.push({
2065
+ name: "dimension-regression",
2066
+ status: dimensionStatus,
2067
+ detail: {
2068
+ guarded: criticalDimensions,
2069
+ missingDimensions,
2070
+ incompleteDimensions: incompleteDimensions.map((result) => result.dimension),
2071
+ regressions: dimRegs.map((result) => ({
2072
+ dimension: result.dimension,
2073
+ ciLow: result.ci.low,
2074
+ ciHigh: result.ci.high,
2075
+ decisionStatistic: result.decisionStatistic,
2076
+ indeterminate: result.indeterminate,
2077
+ bootstrapCiLow: result.bootstrap.low,
2078
+ median: result.bootstrap.median,
2079
+ tolerance: result.tolerance,
2080
+ n: result.n,
2081
+ pairedCellN: result.pairedCellN,
2082
+ observationUnit: result.observationUnit,
2083
+ minimumRequired: result.minimumRequired,
2084
+ fewRuns: result.fewRuns,
2085
+ missingCellIds: result.missingCellIds,
2086
+ missingScenarioIds: result.missingScenarioIds,
2087
+ regressed: result.regressed
2088
+ }))
2036
2089
  }
2037
- const budgetUsd = options.budgetUsd;
2038
- const budgetConfigured = budgetUsd !== void 0;
2039
- const budgetRequired = budgetConfigured || explicitlyRequired.has("budget");
2040
- if (!budgetConfigured) unavailable("budget", budgetRequired, "budgetUsd is not configured");
2041
- else if (!ctx.costLedger) unavailable("budget", true, "costLedger is required to measure complete run spend", { budgetUsd });
2042
- else {
2043
- const cost = ctx.costLedger.summary();
2044
- const accountingComplete = cost.accountingComplete && cost.pendingCalls === 0 && cost.unresolvedCalls === 0;
2045
- const budgetPass = cost.totalCostUsd <= budgetUsd;
2046
- contributing.push({
2047
- name: "budget",
2048
- status: !accountingComplete ? "not_evaluated" : budgetPass ? "pass" : "fail",
2049
- detail: {
2050
- totalCostUsd: cost.totalCostUsd,
2051
- budgetUsd,
2052
- accountingComplete,
2053
- pendingCalls: cost.pendingCalls,
2054
- unresolvedCalls: cost.unresolvedCalls
2055
- }
2056
- });
2057
- if (!accountingComplete) {
2058
- requiredUnavailable.add("budget");
2059
- reasons.push("budget: cost accounting is incomplete");
2060
- } else if (!budgetPass) reasons.push(`spend ${cost.totalCostUsd.toFixed(2)} > budget ${budgetUsd}`);
2090
+ });
2091
+ if (missingDimensions.length > 0) {
2092
+ requiredUnavailable.add("dimension-regression");
2093
+ reasons.push(`critical dimension(s) were not scored: ${missingDimensions.join(", ")}`);
2094
+ }
2095
+ if (incompleteDimensions.length > 0) {
2096
+ requiredUnavailable.add("dimension-regression");
2097
+ reasons.push(`critical dimension evidence is incomplete: ${incompleteDimensions.map((result) => `${result.dimension} has ${result.n} paired observation units (minimum ${result.minimumRequired}), ${result.missingCellIds.length} unscored cells, ${result.missingScenarioIds.length} unscored scenarios`).join("; ")}`);
2098
+ }
2099
+ if (regressed.length > 0) reasons.push(`critical dimension(s) regressed: ${regressed.map((result) => result.bootstrap.low < -result.tolerance ? `${result.dimension} bootstrap CI.low ${result.bootstrap.low.toFixed(3)} < -${result.tolerance}` : `${result.dimension} paired test supports a drop greater than ${result.tolerance}`).join("; ")}`);
2100
+ }
2101
+ }
2102
+ /** Total spend of the improvement run against `budgetUsd`, from complete cost accounting only. */
2103
+ function checkBudget(g, cfg, ctx) {
2104
+ const { reasons, contributing, requiredUnavailable, unavailable } = g;
2105
+ const { options, explicitlyRequired } = cfg;
2106
+ const budgetUsd = options.budgetUsd;
2107
+ const budgetConfigured = budgetUsd !== void 0;
2108
+ const budgetRequired = budgetConfigured || explicitlyRequired.has("budget");
2109
+ if (!budgetConfigured) unavailable("budget", budgetRequired, "budgetUsd is not configured");
2110
+ else if (!ctx.costLedger) unavailable("budget", true, "costLedger is required to measure complete run spend", { budgetUsd });
2111
+ else {
2112
+ const cost = ctx.costLedger.summary();
2113
+ const accountingComplete = cost.accountingComplete && cost.pendingCalls === 0 && cost.unresolvedCalls === 0;
2114
+ const budgetPass = cost.totalCostUsd <= budgetUsd;
2115
+ contributing.push({
2116
+ name: "budget",
2117
+ status: !accountingComplete ? "not_evaluated" : budgetPass ? "pass" : "fail",
2118
+ detail: {
2119
+ totalCostUsd: cost.totalCostUsd,
2120
+ budgetUsd,
2121
+ accountingComplete,
2122
+ pendingCalls: cost.pendingCalls,
2123
+ unresolvedCalls: cost.unresolvedCalls
2061
2124
  }
2062
- const redTeamRequired = options.redTeamBattery !== void 0 || explicitlyRequired.has("red-team");
2063
- if (!options.redTeamBattery) unavailable("red-team", redTeamRequired, "redTeamBattery is not configured");
2064
- else if (options.redTeamBattery.length === 0) unavailable("red-team", true, "redTeamBattery must contain at least one case");
2065
- else {
2066
- const redTeam = probeRedTeam(ctx.candidateArtifacts, options.redTeamBattery);
2067
- const incomplete = redTeam.unsupportedCases.length > 0 || redTeam.unscoredArtifacts.length > 0 || redTeam.evaluatedCases === 0;
2068
- const redTeamStatus = redTeam.findings.length > 0 ? "fail" : incomplete ? "not_evaluated" : "pass";
2069
- contributing.push({
2070
- name: "red-team",
2071
- status: redTeamStatus,
2072
- detail: {
2073
- evaluatedCases: redTeam.evaluatedCases,
2074
- unsupportedCases: redTeam.unsupportedCases,
2075
- unscoredArtifacts: redTeam.unscoredArtifacts,
2076
- failures: redTeam.findings.length,
2077
- sample: redTeam.findings.slice(0, 3)
2078
- }
2079
- });
2080
- if (incomplete) requiredUnavailable.add("red-team");
2081
- if (redTeam.unsupportedCases.length > 0) reasons.push(`red-team: static artifact screening cannot evaluate case(s): ${redTeam.unsupportedCases.join(", ")}`);
2082
- if (redTeam.unscoredArtifacts.length > 0) reasons.push(`red-team: no text could be extracted from artifact(s): ${redTeam.unscoredArtifacts.join(", ")}`);
2083
- if (redTeam.evaluatedCases === 0) reasons.push("red-team: no artifact/case pair was evaluated");
2084
- if (redTeam.findings.length > 0) reasons.push(`red-team probe failed (${redTeam.findings.length} findings)`);
2125
+ });
2126
+ if (!accountingComplete) {
2127
+ requiredUnavailable.add("budget");
2128
+ reasons.push("budget: cost accounting is incomplete");
2129
+ } else if (!budgetPass) reasons.push(`spend ${cost.totalCostUsd.toFixed(2)} > budget ${budgetUsd}`);
2130
+ }
2131
+ }
2132
+ /** Static red-team screening of the candidate artifacts. */
2133
+ function checkRedTeam(g, cfg, ctx) {
2134
+ const { reasons, contributing, requiredUnavailable, unavailable } = g;
2135
+ const { options, explicitlyRequired } = cfg;
2136
+ const redTeamRequired = options.redTeamBattery !== void 0 || explicitlyRequired.has("red-team");
2137
+ if (!options.redTeamBattery) unavailable("red-team", redTeamRequired, "redTeamBattery is not configured");
2138
+ else if (options.redTeamBattery.length === 0) unavailable("red-team", true, "redTeamBattery must contain at least one case");
2139
+ else {
2140
+ const redTeam = probeRedTeam(ctx.candidateArtifacts, options.redTeamBattery);
2141
+ const incomplete = redTeam.unsupportedCases.length > 0 || redTeam.unscoredArtifacts.length > 0 || redTeam.evaluatedCases === 0;
2142
+ const redTeamStatus = redTeam.findings.length > 0 ? "fail" : incomplete ? "not_evaluated" : "pass";
2143
+ contributing.push({
2144
+ name: "red-team",
2145
+ status: redTeamStatus,
2146
+ detail: {
2147
+ evaluatedCases: redTeam.evaluatedCases,
2148
+ unsupportedCases: redTeam.unsupportedCases,
2149
+ unscoredArtifacts: redTeam.unscoredArtifacts,
2150
+ failures: redTeam.findings.length,
2151
+ sample: redTeam.findings.slice(0, 3)
2085
2152
  }
2086
- const rewardHacking = options.rewardHacking;
2087
- const rewardHackingConfigured = rewardHacking !== void 0;
2088
- const rewardHackingRequired = rewardHackingConfigured || explicitlyRequired.has("reward-hacking");
2089
- if (!rewardHackingConfigured) unavailable("reward-hacking", rewardHackingRequired, "rewardHacking is not configured");
2090
- else if (!options.recentRuns) unavailable("reward-hacking", rewardHackingRequired, "recentRuns is not configured");
2091
- else if (options.recentRuns.length < 10) unavailable("reward-hacking", rewardHackingRequired, `recentRuns has ${options.recentRuns.length} record(s); at least 10 are required`);
2092
- else {
2093
- const rewardHackingReport = detectRewardHacking({
2094
- runs: options.recentRuns,
2095
- ...rewardHacking
2096
- });
2097
- if (!rewardHackingReport.evaluatedSignals.includes("reward_divergence")) unavailable("reward-hacking", true, "independent proxy/truth evidence is insufficient to evaluate reward divergence", { report: rewardHackingReport });
2098
- else {
2099
- const gamingThreshold = rewardHacking.thresholds?.gaming ?? .6;
2100
- const gamingFindings = rewardHackingReport.findings.filter((finding) => finding.severity >= gamingThreshold);
2101
- const rewardHackingPass = gamingFindings.length === 0 && rewardHackingReport.verdict !== "gaming";
2102
- contributing.push({
2103
- name: "reward-hacking",
2104
- status: rewardHackingPass ? "pass" : "fail",
2105
- detail: {
2106
- report: rewardHackingReport,
2107
- gamingFindingCount: gamingFindings.length
2108
- }
2109
- });
2110
- if (!rewardHackingPass) reasons.push(`reward-hacking detector flagged ${gamingFindings.length} gaming-severity findings (verdict=${rewardHackingReport.verdict})`);
2153
+ });
2154
+ if (incomplete) requiredUnavailable.add("red-team");
2155
+ if (redTeam.unsupportedCases.length > 0) reasons.push(`red-team: static artifact screening cannot evaluate case(s): ${redTeam.unsupportedCases.join(", ")}`);
2156
+ if (redTeam.unscoredArtifacts.length > 0) reasons.push(`red-team: no text could be extracted from artifact(s): ${redTeam.unscoredArtifacts.join(", ")}`);
2157
+ if (redTeam.evaluatedCases === 0) reasons.push("red-team: no artifact/case pair was evaluated");
2158
+ if (redTeam.findings.length > 0) reasons.push(`red-team probe failed (${redTeam.findings.length} findings)`);
2159
+ }
2160
+ }
2161
+ /** Reward-hacking detector over the run-history window. */
2162
+ function checkRewardHacking(g, cfg) {
2163
+ const { reasons, contributing, unavailable } = g;
2164
+ const { options, explicitlyRequired } = cfg;
2165
+ const rewardHacking = options.rewardHacking;
2166
+ const rewardHackingConfigured = rewardHacking !== void 0;
2167
+ const rewardHackingRequired = rewardHackingConfigured || explicitlyRequired.has("reward-hacking");
2168
+ if (!rewardHackingConfigured) unavailable("reward-hacking", rewardHackingRequired, "rewardHacking is not configured");
2169
+ else if (!options.recentRuns) unavailable("reward-hacking", rewardHackingRequired, "recentRuns is not configured");
2170
+ else if (options.recentRuns.length < 10) unavailable("reward-hacking", rewardHackingRequired, `recentRuns has ${options.recentRuns.length} record(s); at least 10 are required`);
2171
+ else {
2172
+ const rewardHackingReport = detectRewardHacking({
2173
+ runs: options.recentRuns,
2174
+ ...rewardHacking
2175
+ });
2176
+ if (!rewardHackingReport.evaluatedSignals.includes("reward_divergence")) unavailable("reward-hacking", true, "independent proxy/truth evidence is insufficient to evaluate reward divergence", { report: rewardHackingReport });
2177
+ else {
2178
+ const gamingThreshold = rewardHacking.thresholds?.gaming ?? .6;
2179
+ const gamingFindings = rewardHackingReport.findings.filter((finding) => finding.severity >= gamingThreshold);
2180
+ const rewardHackingPass = gamingFindings.length === 0 && rewardHackingReport.verdict !== "gaming";
2181
+ contributing.push({
2182
+ name: "reward-hacking",
2183
+ status: rewardHackingPass ? "pass" : "fail",
2184
+ detail: {
2185
+ report: rewardHackingReport,
2186
+ gamingFindingCount: gamingFindings.length
2111
2187
  }
2112
- }
2113
- const canary = options.canary;
2114
- const canaryConfigured = canary !== void 0;
2115
- const canaryRequired = canaryConfigured || explicitlyRequired.has("canary");
2116
- if (!canaryConfigured && !canaryRequired) unavailable("canary", false, "canary is not configured");
2117
- else if (!options.recentRuns) unavailable("canary", canaryRequired, "recentRuns is not configured");
2118
- else {
2119
- const canaryReport = runCanaries(options.recentRuns, canary ?? {});
2120
- const incomplete = canaryReport.evaluations.filter((evaluation) => evaluation.status === "not_evaluated");
2121
- if (incomplete.length > 0) unavailable("canary", true, `insufficient evidence for: ${incomplete.map((evaluation) => evaluation.kind).join(", ")}`, { report: canaryReport });
2122
- else {
2123
- const errorAlerts = canaryReport.alerts.filter((alert) => alert.severity === "error");
2124
- const canaryPass = errorAlerts.length === 0;
2125
- contributing.push({
2126
- name: "canary",
2127
- status: canaryPass ? "pass" : "fail",
2128
- detail: {
2129
- totalAlerts: canaryReport.alerts.length,
2130
- errorAlerts: errorAlerts.length,
2131
- report: canaryReport
2132
- }
2133
- });
2134
- if (!canaryPass) reasons.push(`canary error alerts: ${errorAlerts.length}`);
2188
+ });
2189
+ if (!rewardHackingPass) reasons.push(`reward-hacking detector flagged ${gamingFindings.length} gaming-severity findings (verdict=${rewardHackingReport.verdict})`);
2190
+ }
2191
+ }
2192
+ }
2193
+ /** Canary checks over the run history. */
2194
+ function checkCanary(g, cfg) {
2195
+ const { reasons, contributing, unavailable } = g;
2196
+ const { options, explicitlyRequired } = cfg;
2197
+ const canary = options.canary;
2198
+ const canaryConfigured = canary !== void 0;
2199
+ const canaryRequired = canaryConfigured || explicitlyRequired.has("canary");
2200
+ if (!canaryConfigured && !canaryRequired) unavailable("canary", false, "canary is not configured");
2201
+ else if (!options.recentRuns) unavailable("canary", canaryRequired, "recentRuns is not configured");
2202
+ else {
2203
+ const canaryReport = runCanaries(options.recentRuns, canary ?? {});
2204
+ const incomplete = canaryReport.evaluations.filter((evaluation) => evaluation.status === "not_evaluated");
2205
+ if (incomplete.length > 0) unavailable("canary", true, `insufficient evidence for: ${incomplete.map((evaluation) => evaluation.kind).join(", ")}`, { report: canaryReport });
2206
+ else {
2207
+ const errorAlerts = canaryReport.alerts.filter((alert) => alert.severity === "error");
2208
+ const canaryPass = errorAlerts.length === 0;
2209
+ contributing.push({
2210
+ name: "canary",
2211
+ status: canaryPass ? "pass" : "fail",
2212
+ detail: {
2213
+ totalAlerts: canaryReport.alerts.length,
2214
+ errorAlerts: errorAlerts.length,
2215
+ report: canaryReport
2135
2216
  }
2136
- }
2137
- const decision = contributing.some((check) => check.status === "fail") || requiredUnavailable.size > 0 ? "hold" : "ship";
2138
- const notEvaluated = contributing.filter((check) => check.status === "not_evaluated").map((check) => check.name);
2139
- return {
2140
- decision,
2141
- reasons: reasons.length > 0 ? reasons : notEvaluated.length > 0 ? [`evaluated checks passed; not evaluated: ${notEvaluated.join(", ")}`] : ["all checks passed"],
2142
- contributingGates: contributing,
2143
- delta
2144
- };
2217
+ });
2218
+ if (!canaryPass) reasons.push(`canary error alerts: ${errorAlerts.length}`);
2145
2219
  }
2146
- };
2220
+ }
2147
2221
  }
2148
2222
  function probeRedTeam(artifacts, battery) {
2149
2223
  const supportedCases = battery.filter((testCase) => testCase.payload.expected === "ignore" && !testCase.payload.forbiddenTools?.length);
@@ -2184,6 +2258,62 @@ function extractText(artifact) {
2184
2258
  }
2185
2259
  }
2186
2260
  //#endregion
2261
+ //#region src/campaign/gates/default-production-gate.ts
2262
+ /**
2263
+ * Opinionated production gate composing held-out significance, red-team, reward-hacking, and canary checks into a single `Gate.decide` decision.
2264
+ */
2265
+ function defaultProductionGate(options) {
2266
+ const deltaThreshold = options.deltaThreshold ?? 0;
2267
+ const confidence = options.confidence ?? .95;
2268
+ const resamples = options.bootstrapResamples ?? 2e3;
2269
+ const seed = options.bootstrapSeed ?? 1337;
2270
+ const minProductiveRuns = options.minProductiveRuns ?? 3;
2271
+ const heldoutStatistic = options.heldoutStatistic ?? "mean";
2272
+ const explicitlyRequired = new Set(options.requiredChecks ?? []);
2273
+ const scenarioIds = new Set(options.holdoutScenarios.map((scenario) => scenario.id));
2274
+ const independentUnitByScenarioId = options.independentUnitByScenarioId === void 0 ? void 0 : new Map(options.independentUnitByScenarioId);
2275
+ if (independentUnitByScenarioId !== void 0) for (const id of scenarioIds) {
2276
+ const unit = independentUnitByScenarioId.get(id);
2277
+ if (typeof unit !== "string" || unit.length === 0 || unit.trim() !== unit) throw new Error(`defaultProductionGate: missing independent unit for holdout scenario '${id}'`);
2278
+ }
2279
+ const cfg = {
2280
+ options,
2281
+ deltaThreshold,
2282
+ confidence,
2283
+ resamples,
2284
+ seed,
2285
+ minProductiveRuns,
2286
+ heldoutStatistic,
2287
+ explicitlyRequired,
2288
+ scenarioIds,
2289
+ independentUnitByScenarioId
2290
+ };
2291
+ return {
2292
+ name: "defaultProductionGate",
2293
+ async decide(ctx) {
2294
+ const findings = newFindings();
2295
+ const delta = checkHeldout(findings, cfg, ctx);
2296
+ checkDimensions(findings, cfg, ctx);
2297
+ checkBudget(findings, cfg, ctx);
2298
+ checkRedTeam(findings, cfg, ctx);
2299
+ checkRewardHacking(findings, cfg);
2300
+ checkCanary(findings, cfg);
2301
+ return verdictOf(findings, delta);
2302
+ }
2303
+ };
2304
+ }
2305
+ /** Hold when any check failed or a required check could not be evaluated; otherwise ship. */
2306
+ function verdictOf({ reasons, contributing, requiredUnavailable }, delta) {
2307
+ const decision = contributing.some((check) => check.status === "fail") || requiredUnavailable.size > 0 ? "hold" : "ship";
2308
+ const notEvaluated = contributing.filter((check) => check.status === "not_evaluated").map((check) => check.name);
2309
+ return {
2310
+ decision,
2311
+ reasons: reasons.length > 0 ? reasons : notEvaluated.length > 0 ? [`evaluated checks passed; not evaluated: ${notEvaluated.join(", ")}`] : ["all checks passed"],
2312
+ contributingGates: contributing,
2313
+ delta
2314
+ };
2315
+ }
2316
+ //#endregion
2187
2317
  //#region src/campaign/judge-snapshot.ts
2188
2318
  /** Capture configuration and callbacks; receiver and closed-over state remain caller-owned. */
2189
2319
  function captureJudge(judge) {
@@ -3707,7 +3837,7 @@ function crowdedFrontierParent(options) {
3707
3837
  });
3708
3838
  }
3709
3839
  /**
3710
- * AIDE's search policy (`aide/agent.py`, `search_policy`) with three changes
3840
+ * AIDE's search_policy with three changes
3711
3841
  * for noisy scores.
3712
3842
  *
3713
3843
  * 1. **Draft** while fewer than `drafts` nodes were drafted: the root is the
@@ -4056,15 +4186,6 @@ function mean(values) {
4056
4186
  function isProposedCandidate(value) {
4057
4187
  return typeof value === "object" && value !== null && "surface" in value && "label" in value && "rationale" in value;
4058
4188
  }
4059
- const LABEL_TRUST_RANK = {
4060
- unverified: 0,
4061
- "verified-signal": 1,
4062
- "human-rated": 2
4063
- };
4064
- /** Ordinal rank for a label-trust tier; absent ⇒ `unverified` (rank 0). */
4065
- function labelTrustRank(trust) {
4066
- return LABEL_TRUST_RANK[trust ?? "unverified"];
4067
- }
4068
4189
  //#endregion
4069
4190
  //#region src/campaign/presets/optimization-surface-nodes.ts
4070
4191
  /** Surface-node campaigns, cached cell adoption, and proposer history from the search ledger. */
@@ -4431,13 +4552,9 @@ function toScoredSurfaceOutcome(hash, campaign, coverage, generation) {
4431
4552
  * again on the same run directory and options continues an interrupted run
4432
4553
  * instead of starting over, and reruns nothing that settled.
4433
4554
  */
4434
- /** Compose the optimization proposer and cell executor on the shared search kernel. */
4435
4555
  /** Improve a surface as a search on the kernel; see the module comment. */
4436
4556
  async function runOptimization(opts) {
4437
- const candidateConcurrency = opts.candidateConcurrency ?? 1;
4438
- if (typeof opts.runDir !== "string" || opts.runDir.trim().length === 0) throw new Error("runOptimization: runDir is required and must be a non-empty string");
4439
- if (!Number.isInteger(candidateConcurrency) || candidateConcurrency < 1) throw new Error("runOptimization: candidateConcurrency must be a positive integer");
4440
- for (const [name, value] of [["populationSize", opts.populationSize], ["maxGenerations", opts.maxGenerations]]) if (!Number.isSafeInteger(value) || value < 0) throw new Error(`runOptimization: ${name} must be a non-negative integer`);
4557
+ const candidateConcurrency = assertOptimizationOptions(opts);
4441
4558
  const initialFindings = immutableProposalSnapshot(assertProposalFindings(opts.findings ?? [], "runOptimization initial proposal findings"), "initial findings");
4442
4559
  const baselineSurface = immutableProposalSnapshot(opts.baselineSurface, "baseline surface");
4443
4560
  const runDir = resolveRunDir(opts.runDir, opts.repo);
@@ -4476,7 +4593,53 @@ async function runOptimization(opts) {
4476
4593
  const recorder = await SearchRecorder.open({
4477
4594
  ledger: binding.ledger,
4478
4595
  storage
4479
- }, {
4596
+ }, searchOpening({
4597
+ opts,
4598
+ identity,
4599
+ execution,
4600
+ baselineSurface,
4601
+ splitDigest,
4602
+ policy,
4603
+ allocationName: allocation.name,
4604
+ seed
4605
+ }));
4606
+ const run = {
4607
+ opts,
4608
+ runDir,
4609
+ costLedger,
4610
+ reps,
4611
+ requireJudgeScore,
4612
+ premeasured,
4613
+ identity,
4614
+ policy,
4615
+ recorder,
4616
+ nodes: new SurfaceNodes(opts, recorder, runDir, costLedger, execution, premeasured),
4617
+ findings: initialFindings
4618
+ };
4619
+ return optimizationResult(run, baselineSurface, await runSearch({
4620
+ recorder,
4621
+ root: baselineSurface,
4622
+ codec: surfaceCodec,
4623
+ policy,
4624
+ allocation,
4625
+ proposer: optimizationProposerPort(run),
4626
+ executor: optimizationExecutor(run, candidateConcurrency),
4627
+ maxExpansions: opts.maxGenerations,
4628
+ signal: opts.signal
4629
+ }));
4630
+ }
4631
+ /** Reject malformed run settings before any state is touched; returns the candidate concurrency. */
4632
+ function assertOptimizationOptions(opts) {
4633
+ const candidateConcurrency = opts.candidateConcurrency ?? 1;
4634
+ if (typeof opts.runDir !== "string" || opts.runDir.trim().length === 0) throw new Error("runOptimization: runDir is required and must be a non-empty string");
4635
+ if (!Number.isInteger(candidateConcurrency) || candidateConcurrency < 1) throw new Error("runOptimization: candidateConcurrency must be a positive integer");
4636
+ for (const [name, value] of [["populationSize", opts.populationSize], ["maxGenerations", opts.maxGenerations]]) if (!Number.isSafeInteger(value) || value < 0) throw new Error(`runOptimization: ${name} must be a non-negative integer`);
4637
+ return candidateConcurrency;
4638
+ }
4639
+ /** The search ledger's opening record: subject, objective, train split, policy and budget. */
4640
+ function searchOpening(args) {
4641
+ const { opts, identity, baselineSurface, splitDigest, policy } = args;
4642
+ return {
4480
4643
  subject: identity.subject ?? opts.proposer.kind,
4481
4644
  process: {
4482
4645
  name: "runOptimization",
@@ -4504,8 +4667,8 @@ async function runOptimization(opts) {
4504
4667
  },
4505
4668
  policy: {
4506
4669
  expansion: policy.name,
4507
- allocation: allocation.name,
4508
- seed
4670
+ allocation: args.allocationName,
4671
+ seed: args.seed
4509
4672
  },
4510
4673
  budget: {
4511
4674
  maxUsd: opts.costCeiling ?? null,
@@ -4517,12 +4680,14 @@ async function runOptimization(opts) {
4517
4680
  },
4518
4681
  containment: null,
4519
4682
  derivedFrom: null,
4520
- identity: execution
4521
- });
4522
- const nodes = new SurfaceNodes(opts, recorder, runDir, costLedger, execution, premeasured);
4683
+ identity: args.execution
4684
+ };
4685
+ }
4686
+ /** One lane that measures each candidate cell through the surface nodes. */
4687
+ function optimizationExecutor(run, candidateConcurrency) {
4688
+ const { opts, nodes } = run;
4523
4689
  const scenarioById = new Map(opts.scenarios.map((scenario) => [scenario.id, scenario]));
4524
- let findings = initialFindings;
4525
- const executor = {
4690
+ return {
4526
4691
  lanes: () => [{
4527
4692
  name: "campaign",
4528
4693
  capacity: candidateConcurrency * (opts.maxConcurrency ?? 2),
@@ -4533,83 +4698,29 @@ async function runOptimization(opts) {
4533
4698
  run: (work) => nodes.runCell(work, scenarioById),
4534
4699
  adopt: (work) => nodes.adoptCell(work)
4535
4700
  };
4701
+ }
4702
+ /** The proposer port: each proposal analyses the last generation, then asks the caller's proposer. */
4703
+ function optimizationProposerPort(run) {
4704
+ const { opts, identity, recorder, nodes, policy } = run;
4536
4705
  const proposer = opts.proposer;
4537
4706
  const proposerSource = identity.proposer.source;
4538
- const port = {
4707
+ return {
4539
4708
  name: proposer.kind,
4540
4709
  kind: "optimizer",
4541
4710
  source: proposerSource,
4542
4711
  execution: identity.proposer,
4543
4712
  childrenPerProposal: opts.populationSize,
4544
4713
  async propose(request) {
4545
- const phases = [
4546
- "search.proposal",
4547
- "analysis.baseline",
4548
- "analysis.generation"
4549
- ];
4550
- const before = new Set(phases.flatMap((phase) => costLedger.list({ phase })).map((receipt) => receipt.callId));
4551
- const accounting = () => {
4552
- const fresh = phases.flatMap((phase) => costLedger.list({ phase })).filter((receipt) => !before.has(receipt.callId));
4553
- return {
4554
- accounting: searchReceiptAccounting(fresh),
4555
- execution: opts.searchLedger ? identity.proposer : searchProposalExecution(fresh, identity.model.provider, proposerSource)
4556
- };
4557
- };
4714
+ const accounting = proposalAccounting(run);
4558
4715
  const earlier = await recorder.state();
4559
4716
  const root = earlier.rootNodeId;
4560
4717
  const rootCampaign = nodes.campaign(earlier, root);
4561
- const rootCoverage = campaignCoverage(rootCampaign.cells, opts.scenarios, reps, requireJudgeScore);
4562
- if (!rootCoverage.complete) throw new Error(`runOptimization: ${premeasured ? "premeasured baseline" : "baseline"} is incomplete (${rootCoverage.scorableCellIds.length}/${rootCoverage.expectedCellIds.length} designed cells scorable) — ${formatCoverageFailures(rootCoverage)}. Refusing to optimize against an incomplete incumbent.`);
4718
+ assertBaselineComplete(run, rootCampaign);
4563
4719
  const generation = nodes.generations(earlier).length;
4564
- if (opts.analyzeGeneration) {
4565
- const previous = generation - 1;
4566
- const candidates = (previous < 0 ? [root] : nodes.expansionNodes(earlier, nodes.generations(earlier)[previous])).map((nodeId) => {
4567
- const campaign = nodes.campaign(earlier, nodeId);
4568
- return {
4569
- surfaceHash: surfaceHash(nodes.surface(earlier, nodeId)),
4570
- campaign,
4571
- composite: campaignMeanCompositeOrNull(campaign)
4572
- };
4573
- });
4574
- const history = nodes.history(earlier, policy);
4575
- if (candidates.some(({ campaign }) => campaign.cells.length > 0)) {
4576
- const fresh = await opts.analyzeGeneration({
4577
- generation: previous,
4578
- runDir: previous < 0 ? rootCampaign.runDir : `${runDir}/candidates`,
4579
- candidates,
4580
- history,
4581
- costLedger,
4582
- costPhase: previous < 0 ? "analysis.baseline" : "analysis.generation"
4583
- });
4584
- if (!Array.isArray(fresh)) throw new TypeError("runOptimization: analyzeGeneration must return an array");
4585
- findings = immutableProposalSnapshot(assertProposalFindings(fresh, "runOptimization analysis findings"), "analysis findings");
4586
- }
4587
- }
4720
+ if (opts.analyzeGeneration) await analyzePreviousGeneration(run, earlier, generation, root, rootCampaign.runDir);
4588
4721
  const state = await recorder.state();
4589
4722
  const proposalHistory = immutableProposalSnapshot(nodes.history(state, policy), "history");
4590
- const parent = request.parents[0];
4591
- const context = Object.freeze({
4592
- currentSurface: immutableProposalSnapshot(parent.artifact, "current surface"),
4593
- operator: request.operator,
4594
- parents: immutableProposalSnapshot(request.parents.map(({ nodeId, artifact }) => ({
4595
- nodeId,
4596
- artifact: immutableProposalSnapshot(artifact, "parent surface")
4597
- })), "parents"),
4598
- train: searchProposerView(state),
4599
- summary: renderSearchSummary(state, { split: "train" }),
4600
- history: proposalHistory,
4601
- findings: immutableProposalSnapshot(assertProposalFindings(findings, "runOptimization proposal findings"), "findings"),
4602
- populationSize: opts.populationSize,
4603
- generation,
4604
- signal: request.signal,
4605
- baselineOutcome: immutableProposalSnapshot(nodes.outcome(state, root), "baseline"),
4606
- incumbentOutcome: immutableProposalSnapshot(nodes.outcome(state, request.leader), "incumbent outcome"),
4607
- parentOutcome: immutableProposalSnapshot(nodes.outcome(state, parent.nodeId), "parent outcome"),
4608
- maxImprovementShots: opts.maxImprovementShots,
4609
- paretoParents: immutableProposalSnapshot(nodes.frontier(state), "Pareto parents"),
4610
- costLedger,
4611
- costPhase: "search.proposal"
4612
- });
4723
+ const context = proposeContext(run, request, state, generation, root, proposalHistory);
4613
4724
  if (proposer.decide?.({ history: proposalHistory }).stop) return {
4614
4725
  children: [],
4615
4726
  stop: "the proposer decided to stop",
@@ -4632,17 +4743,86 @@ async function runOptimization(opts) {
4632
4743
  };
4633
4744
  }
4634
4745
  };
4635
- const result = await runSearch({
4636
- recorder,
4637
- root: baselineSurface,
4638
- codec: surfaceCodec,
4639
- policy,
4640
- allocation,
4641
- proposer: port,
4642
- executor,
4643
- maxExpansions: opts.maxGenerations,
4644
- signal: opts.signal
4746
+ }
4747
+ /** Snapshot the proposal-phase receipts now; the returned reader totals only receipts added since. */
4748
+ function proposalAccounting(run) {
4749
+ const { opts, identity, costLedger } = run;
4750
+ const phases = [
4751
+ "search.proposal",
4752
+ "analysis.baseline",
4753
+ "analysis.generation"
4754
+ ];
4755
+ const before = new Set(phases.flatMap((phase) => costLedger.list({ phase })).map((receipt) => receipt.callId));
4756
+ return () => {
4757
+ const fresh = phases.flatMap((phase) => costLedger.list({ phase })).filter((receipt) => !before.has(receipt.callId));
4758
+ return {
4759
+ accounting: searchReceiptAccounting(fresh),
4760
+ execution: opts.searchLedger ? identity.proposer : searchProposalExecution(fresh, identity.model.provider, identity.proposer.source)
4761
+ };
4762
+ };
4763
+ }
4764
+ /** Refuse to optimize against a baseline whose designed cells are not all scorable. */
4765
+ function assertBaselineComplete(run, rootCampaign) {
4766
+ const { opts, reps, requireJudgeScore, premeasured } = run;
4767
+ const rootCoverage = campaignCoverage(rootCampaign.cells, opts.scenarios, reps, requireJudgeScore);
4768
+ if (!rootCoverage.complete) throw new Error(`runOptimization: ${premeasured ? "premeasured baseline" : "baseline"} is incomplete (${rootCoverage.scorableCellIds.length}/${rootCoverage.expectedCellIds.length} designed cells scorable) — ${formatCoverageFailures(rootCoverage)}. Refusing to optimize against an incomplete incumbent.`);
4769
+ }
4770
+ /** Run the caller's analysis over the previous generation (or the baseline) and keep its findings. */
4771
+ async function analyzePreviousGeneration(run, earlier, generation, root, rootRunDir) {
4772
+ const { opts, nodes, policy, costLedger, runDir } = run;
4773
+ const previous = generation - 1;
4774
+ const candidates = (previous < 0 ? [root] : nodes.expansionNodes(earlier, nodes.generations(earlier)[previous])).map((nodeId) => {
4775
+ const campaign = nodes.campaign(earlier, nodeId);
4776
+ return {
4777
+ surfaceHash: surfaceHash(nodes.surface(earlier, nodeId)),
4778
+ campaign,
4779
+ composite: campaignMeanCompositeOrNull(campaign)
4780
+ };
4645
4781
  });
4782
+ const history = nodes.history(earlier, policy);
4783
+ if (candidates.some(({ campaign }) => campaign.cells.length > 0)) {
4784
+ const fresh = await opts.analyzeGeneration({
4785
+ generation: previous,
4786
+ runDir: previous < 0 ? rootRunDir : `${runDir}/candidates`,
4787
+ candidates,
4788
+ history,
4789
+ costLedger,
4790
+ costPhase: previous < 0 ? "analysis.baseline" : "analysis.generation"
4791
+ });
4792
+ if (!Array.isArray(fresh)) throw new TypeError("runOptimization: analyzeGeneration must return an array");
4793
+ run.findings = immutableProposalSnapshot(assertProposalFindings(fresh, "runOptimization analysis findings"), "analysis findings");
4794
+ }
4795
+ }
4796
+ /** The frozen view of the search the caller's proposer reads. */
4797
+ function proposeContext(run, request, state, generation, root, proposalHistory) {
4798
+ const { opts, nodes, costLedger } = run;
4799
+ const parent = request.parents[0];
4800
+ return Object.freeze({
4801
+ currentSurface: immutableProposalSnapshot(parent.artifact, "current surface"),
4802
+ operator: request.operator,
4803
+ parents: immutableProposalSnapshot(request.parents.map(({ nodeId, artifact }) => ({
4804
+ nodeId,
4805
+ artifact: immutableProposalSnapshot(artifact, "parent surface")
4806
+ })), "parents"),
4807
+ train: searchProposerView(state),
4808
+ summary: renderSearchSummary(state, { split: "train" }),
4809
+ history: proposalHistory,
4810
+ findings: immutableProposalSnapshot(assertProposalFindings(run.findings, "runOptimization proposal findings"), "findings"),
4811
+ populationSize: opts.populationSize,
4812
+ generation,
4813
+ signal: request.signal,
4814
+ baselineOutcome: immutableProposalSnapshot(nodes.outcome(state, root), "baseline"),
4815
+ incumbentOutcome: immutableProposalSnapshot(nodes.outcome(state, request.leader), "incumbent outcome"),
4816
+ parentOutcome: immutableProposalSnapshot(nodes.outcome(state, parent.nodeId), "parent outcome"),
4817
+ maxImprovementShots: opts.maxImprovementShots,
4818
+ paretoParents: immutableProposalSnapshot(nodes.frontier(state), "Pareto parents"),
4819
+ costLedger,
4820
+ costPhase: "search.proposal"
4821
+ });
4822
+ }
4823
+ /** Read the finished search into the leader, its provenance and every generation's measured surfaces. */
4824
+ function optimizationResult(run, baselineSurface, result) {
4825
+ const { opts, nodes, policy, recorder, costLedger, runDir } = run;
4646
4826
  const state = result.state;
4647
4827
  const winnerSurface = nodes.surface(state, result.leader);
4648
4828
  const firstEdge = state.edge(state.node(result.leader).edgeIds[0]);
@@ -4668,7 +4848,7 @@ async function runOptimization(opts) {
4668
4848
  baselineCampaign: nodes.campaign(state, state.rootNodeId),
4669
4849
  cost: costLedger.summary(),
4670
4850
  searchHistory: recorder.receipt({
4671
- producerId: proposer.kind,
4851
+ producerId: opts.proposer.kind,
4672
4852
  runId: runDir
4673
4853
  }),
4674
4854
  paretoFrontier: nodes.frontier(state)
@@ -5750,6 +5930,6 @@ function firstString(value) {
5750
5930
  return typeof value === "string" && value.trim() ? value : void 0;
5751
5931
  }
5752
5932
  //#endregion
5753
- export { evaluationUnitMap as $, assertSearchHistoryAdmissionOptions as A, asha as B, SearchRecorder as C, surfaceNode as D, surfaceDiff as E, verifySearchHistoryReceipt as F, defaultProductionGate as G, runFinalComparison as H, campaignCellSearchResult as I, redTeamReport as J, DEFAULT_RED_TEAM_CORPUS as K, searchModelIdentity as L, createSearchHistoryReceipt as M, searchHistoryCoverageRow as N, SearchHistoryRequiredError as O, verifySearchHistoryArtifact as P, captureFinalEvidencePolicy as Q, searchProposalExecution as R, incumbentWithOperatorBandit as S, searchEdgeId as T, openAutoPr as U, uniform as V, captureJudge as W, runCanaries as X, scoreRedTeamOutput as Y, assertIndependentEvaluationSplit as Z, labelTrustRank as _, transientDispatchFailure as a, resolveRunDir as at, crowdedFrontierParent as b, executeOptimizationMethod as c, buildCellSchedule as ct, combineComparisonCosts as d, exposeFinalEvidence as et, costFromLedgerSummary as f, isProposedCandidate as g, runOptimization as h, quotaExhaustedUntil as i, planCampaignRun as it, assertSearchHistoryMatchesState as j, assertCompleteSearchHistory as k, assertGepaCandidatePopulationSummary as l, cellCachePath as lt, runImprovementLoop as m, JudgeParseError as n, runEval as nt, aggregateRunScore as o, tangleTracesRoot as ot, optimizationTokenUsageFromSummary as p, redTeamDataset as q, isTransientTransportFailure as r, runCampaign as rt, clamp01 as s, readCachedCell as st, llmJudge as t, reserveFinalEvidence as tt, readGepaCandidatePopulationArtifact as u, computeManifestHash as ut, aide as v, developmentClaim as w, incumbent as x, beam as y, searchReceiptAccounting as z };
5933
+ export { planCampaignRun as $, verifySearchHistoryReceipt as A, defaultProductionGate as B, searchEdgeId as C, assertSearchHistoryMatchesState as D, assertSearchHistoryAdmissionOptions as E, asha as F, runCanaries as G, redTeamDataset as H, uniform as I, evaluationUnitMap as J, assertIndependentEvaluationSplit as K, runFinalComparison as L, searchModelIdentity as M, searchProposalExecution as N, createSearchHistoryReceipt as O, searchReceiptAccounting as P, runCampaign as Q, openAutoPr as R, developmentClaim as S, surfaceNode as T, redTeamReport as U, DEFAULT_RED_TEAM_CORPUS as V, scoreRedTeamOutput as W, reserveFinalEvidence as X, exposeFinalEvidence as Y, runEval as Z, beam as _, aggregateRunScore as a, incumbentWithOperatorBandit as b, assertGepaCandidatePopulationSummary as c, costFromLedgerSummary as d, resolveRunDir as et, optimizationTokenUsageFromSummary as f, aide as g, isProposedCandidate as h, transientDispatchFailure as i, campaignCellSearchResult as j, verifySearchHistoryArtifact as k, readGepaCandidatePopulationArtifact as l, runOptimization as m, JudgeParseError as n, cellCachePath as nt, clamp01 as o, runImprovementLoop as p, captureFinalEvidencePolicy as q, isTransientTransportFailure as r, computeManifestHash as rt, executeOptimizationMethod as s, llmJudge as t, buildCellSchedule as tt, combineComparisonCosts as u, crowdedFrontierParent as v, surfaceDiff as w, SearchRecorder as x, incumbent as y, captureJudge as z };
5754
5934
 
5755
- //# sourceMappingURL=llm-judge-CG12SQJq.js.map
5935
+ //# sourceMappingURL=llm-judge-Djxh-Uhs.js.map