@tangle-network/agent-eval 0.150.2 → 0.161.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (342) hide show
  1. package/CHANGELOG.md +158 -1
  2. package/README.md +7 -3
  3. package/dist/{active-curriculum-C4mk67HP.js → active-curriculum-CD5TU2yW.js} +3 -13
  4. package/dist/active-curriculum-CD5TU2yW.js.map +1 -0
  5. package/dist/{agent-profile-cell-BkcRDikH.d.ts → agent-profile-cell-CTOZJUuE.d.ts} +4 -2
  6. package/dist/agent-profile-cell-CTOZJUuE.d.ts.map +1 -0
  7. package/dist/analyst/index.d.ts +19 -36
  8. package/dist/analyst/index.d.ts.map +1 -1
  9. package/dist/analyst/index.js +8 -8
  10. package/dist/analyst/index.js.map +1 -1
  11. package/dist/{backend-integrity-DOCa_QrR.d.ts → backend-integrity-DxuQCu_A.d.ts} +4 -3
  12. package/dist/backend-integrity-DxuQCu_A.d.ts.map +1 -0
  13. package/dist/{benchmark-BtAWA8nT.d.ts → benchmark-CGPp-kDC.d.ts} +3 -3
  14. package/dist/{benchmark-BtAWA8nT.d.ts.map → benchmark-CGPp-kDC.d.ts.map} +1 -1
  15. package/dist/{benchmark-command-BU1Las59.js → benchmark-command-BDC3Gocz.js} +26 -31
  16. package/dist/benchmark-command-BDC3Gocz.js.map +1 -0
  17. package/dist/benchmarks/index.d.ts +6 -19
  18. package/dist/benchmarks/index.d.ts.map +1 -1
  19. package/dist/benchmarks/index.js +4 -4
  20. package/dist/benchmarks/index.js.map +1 -1
  21. package/dist/builder-eval/index.d.ts +3 -3
  22. package/dist/builder-eval/index.js +2 -2
  23. package/dist/campaign/index.d.ts +9 -9
  24. package/dist/campaign/index.js +7 -7
  25. package/dist/{campaign-la-gEYNz.js → campaign-BSmOwskD.js} +77 -795
  26. package/dist/campaign-BSmOwskD.js.map +1 -0
  27. package/dist/{canonical-D-XsTQ6_.js → canonical-IL-Bu-14.js} +26 -2
  28. package/dist/canonical-IL-Bu-14.js.map +1 -0
  29. package/dist/{capture-fetch-BBVFzhkk.d.ts → capture-fetch-CqwsJkkG.d.ts} +3 -3
  30. package/dist/{capture-fetch-BBVFzhkk.d.ts.map → capture-fetch-CqwsJkkG.d.ts.map} +1 -1
  31. package/dist/{chat-client-Bvmxedyv.js → chat-client-DlMlAeYI.js} +5 -55
  32. package/dist/{chat-client-Bvmxedyv.js.map → chat-client-DlMlAeYI.js.map} +1 -1
  33. package/dist/chat-json-call-6g5sJobJ.js +53 -0
  34. package/dist/chat-json-call-6g5sJobJ.js.map +1 -0
  35. package/dist/cli.js +54 -19
  36. package/dist/cli.js.map +1 -1
  37. package/dist/{client-LIuo-KPv.js → client-CX7KqIdB.js} +3 -3
  38. package/dist/client-CX7KqIdB.js.map +1 -0
  39. package/dist/{client-kPQYT_56.d.ts → client-L9VVPkim.d.ts} +4 -4
  40. package/dist/{client-kPQYT_56.d.ts.map → client-L9VVPkim.d.ts.map} +1 -1
  41. package/dist/contract/index.d.ts +13 -27
  42. package/dist/contract/index.d.ts.map +1 -1
  43. package/dist/contract/index.js +14 -17
  44. package/dist/contract/index.js.map +1 -1
  45. package/dist/{counterfactual--bpysZF0.d.ts → counterfactual-BaFUWK3H.d.ts} +4 -4
  46. package/dist/{counterfactual--bpysZF0.d.ts.map → counterfactual-BaFUWK3H.d.ts.map} +1 -1
  47. package/dist/{counterfactual-lDfCx0Uz.js → counterfactual-D_VWavVm.js} +2 -2
  48. package/dist/{counterfactual-lDfCx0Uz.js.map → counterfactual-D_VWavVm.js.map} +1 -1
  49. package/dist/{dataset-CJjKqQfA.d.ts → dataset-DQqhOCPt.d.ts} +5 -4
  50. package/dist/{dataset-CJjKqQfA.d.ts.map → dataset-DQqhOCPt.d.ts.map} +1 -1
  51. package/dist/{default-registry-Cw0Ohdoj.d.ts → default-registry-G9CKMNkc.d.ts} +7 -8
  52. package/dist/{default-registry-Cw0Ohdoj.d.ts.map → default-registry-G9CKMNkc.d.ts.map} +1 -1
  53. package/dist/{define-agent-eval-CEQWL9Hy.d.ts → define-agent-eval-Dx1JnPEa.d.ts} +26 -6
  54. package/dist/define-agent-eval-Dx1JnPEa.d.ts.map +1 -0
  55. package/dist/{define-agent-eval-C8V8sMqP.js → define-agent-eval-h-s-sI-v.js} +15 -9
  56. package/dist/define-agent-eval-h-s-sI-v.js.map +1 -0
  57. package/dist/{dspy-rlm-engine-CBYlPvNy.js → dspy-rlm-engine-DptEII26.js} +95 -15
  58. package/dist/dspy-rlm-engine-DptEII26.js.map +1 -0
  59. package/dist/{emitter-CPBAhxum.js → emitter-BpYFQPj4.js} +2 -18
  60. package/dist/emitter-BpYFQPj4.js.map +1 -0
  61. package/dist/{emitter-DGQGoLyj.d.ts → emitter-D_jYSGRd.d.ts} +4 -20
  62. package/dist/{emitter-DGQGoLyj.d.ts.map → emitter-D_jYSGRd.d.ts.map} +1 -1
  63. package/dist/{engine-BLzhNzoY.d.ts → engine-Cu5qD5Fc.d.ts} +9 -11
  64. package/dist/{engine-BLzhNzoY.d.ts.map → engine-Cu5qD5Fc.d.ts.map} +1 -1
  65. package/dist/{eval-campaign-CQuZrLR_.js → eval-campaign-BsXWL2-2.js} +17 -28
  66. package/dist/eval-campaign-BsXWL2-2.js.map +1 -0
  67. package/dist/{exact-types-ccQAyut1.d.ts → exact-types-qnexxJ1Z.d.ts} +2 -2
  68. package/dist/{exact-types-ccQAyut1.d.ts.map → exact-types-qnexxJ1Z.d.ts.map} +1 -1
  69. package/dist/{exec-y-DCLqK7.js → exec-D9WpA2p-.js} +2 -2
  70. package/dist/exec-D9WpA2p-.js.map +1 -0
  71. package/dist/experiment/index.d.ts +45 -11
  72. package/dist/experiment/index.d.ts.map +1 -1
  73. package/dist/experiment/index.js +30 -11
  74. package/dist/experiment/index.js.map +1 -1
  75. package/dist/{experiment-tracker-0MhuPArU.d.ts → experiment-tracker-DCO6Cz4s.d.ts} +2 -2
  76. package/dist/{experiment-tracker-0MhuPArU.d.ts.map → experiment-tracker-DCO6Cz4s.d.ts.map} +1 -1
  77. package/dist/{exporters-q9iL-2Jf.js → exporters-Df7TgHFv.js} +3 -3
  78. package/dist/exporters-Df7TgHFv.js.map +1 -0
  79. package/dist/{external-optimizer-contracts-DbLsm4Po.d.ts → external-optimizer-contracts-szBJ_1vh.d.ts} +2 -2
  80. package/dist/{external-optimizer-contracts-DbLsm4Po.d.ts.map → external-optimizer-contracts-szBJ_1vh.d.ts.map} +1 -1
  81. package/dist/{external-optimizer-process-x9oEXKsU.js → external-optimizer-process-WosTBChy.js} +4 -4
  82. package/dist/{external-optimizer-process-x9oEXKsU.js.map → external-optimizer-process-WosTBChy.js.map} +1 -1
  83. package/dist/{external-optimizer-subprocess-CKNb42oM.js → external-optimizer-subprocess-BIWbHpgD.js} +4 -4
  84. package/dist/{external-optimizer-subprocess-CKNb42oM.js.map → external-optimizer-subprocess-BIWbHpgD.js.map} +1 -1
  85. package/dist/{failure-cluster-BLURuWG4.d.ts → failure-cluster-CXL8NbEw.d.ts} +3 -3
  86. package/dist/{failure-cluster-BLURuWG4.d.ts.map → failure-cluster-CXL8NbEw.d.ts.map} +1 -1
  87. package/dist/{feedback-trajectory-DpTTjo0q.d.ts → feedback-trajectory-B3ZHaHV_.d.ts} +7 -7
  88. package/dist/{feedback-trajectory-DpTTjo0q.d.ts.map → feedback-trajectory-B3ZHaHV_.d.ts.map} +1 -1
  89. package/dist/fuzz.js +1 -1
  90. package/dist/{hf-dataset-XggBupCr.js → hf-dataset-D8_RNIis.js} +4 -4
  91. package/dist/{hf-dataset-XggBupCr.js.map → hf-dataset-D8_RNIis.js.map} +1 -1
  92. package/dist/hosted/index.d.ts +3 -14
  93. package/dist/hosted/index.d.ts.map +1 -1
  94. package/dist/hosted/index.js +2 -2
  95. package/dist/{index-Aj3WO3_a.d.ts → index-CGtH1piv.d.ts} +48 -26
  96. package/dist/index-CGtH1piv.d.ts.map +1 -0
  97. package/dist/{index-BNPtkBPf.d.ts → index-D-IiQIBB.d.ts} +5 -10
  98. package/dist/index-D-IiQIBB.d.ts.map +1 -0
  99. package/dist/{index-IQccV3Ou.d.ts → index-D-V8gCs_.d.ts} +13 -90
  100. package/dist/index-D-V8gCs_.d.ts.map +1 -0
  101. package/dist/{index-B8Ui1mr1.d.ts → index-lfaSeKSD.d.ts} +18 -2
  102. package/dist/index-lfaSeKSD.d.ts.map +1 -0
  103. package/dist/index-vrJugRal.d.ts +1 -0
  104. package/dist/index.d.ts +68 -56
  105. package/dist/index.d.ts.map +1 -1
  106. package/dist/index.js +59 -110
  107. package/dist/index.js.map +1 -1
  108. package/dist/{insight-report-BeT8KCgI.d.ts → insight-report-DRe8LB6d.d.ts} +4 -4
  109. package/dist/{insight-report-BeT8KCgI.d.ts.map → insight-report-DRe8LB6d.d.ts.map} +1 -1
  110. package/dist/{integrity-DL91tucI.js → integrity-CyWSSoQS.js} +2 -2
  111. package/dist/{integrity-DL91tucI.js.map → integrity-CyWSSoQS.js.map} +1 -1
  112. package/dist/{integrity-B0dZ96EO.d.ts → integrity-DUNX9Fao.d.ts} +3 -3
  113. package/dist/{integrity-B0dZ96EO.d.ts.map → integrity-DUNX9Fao.d.ts.map} +1 -1
  114. package/dist/{kind-factory-DmAa0h3K.js → kind-factory-DY8FdoXf.js} +3 -71
  115. package/dist/kind-factory-DY8FdoXf.js.map +1 -0
  116. package/dist/ledger-core/index.d.ts +2 -2
  117. package/dist/ledger-core/index.js +3 -3
  118. package/dist/{ledger-core-DTae9rv_.js → ledger-core-BOzlRygb.js} +2 -2
  119. package/dist/{ledger-core-DTae9rv_.js.map → ledger-core-BOzlRygb.js.map} +1 -1
  120. package/dist/{llm-client-Bg32RW0j.js → llm-client-hgDieDNN.js} +53 -98
  121. package/dist/llm-client-hgDieDNN.js.map +1 -0
  122. package/dist/{llm-judge-BqqMS8t7.js → llm-judge-BhasIPFT.js} +1135 -77
  123. package/dist/llm-judge-BhasIPFT.js.map +1 -0
  124. package/dist/{matrix-DrVnRp4G.d.ts → matrix-eXKRMHnL.d.ts} +74 -72
  125. package/dist/matrix-eXKRMHnL.d.ts.map +1 -0
  126. package/dist/meta-eval/index.d.ts +8 -6
  127. package/dist/meta-eval/index.d.ts.map +1 -1
  128. package/dist/meta-eval/index.js +8 -6
  129. package/dist/meta-eval/index.js.map +1 -1
  130. package/dist/{mint-BV6tLVWl.js → mint-DfODW1KW.js} +3 -3
  131. package/dist/{mint-BV6tLVWl.js.map → mint-DfODW1KW.js.map} +1 -1
  132. package/dist/multishot/golden/index.d.ts +2 -8
  133. package/dist/multishot/golden/index.d.ts.map +1 -1
  134. package/dist/multishot/golden/index.js +56 -86
  135. package/dist/multishot/golden/index.js.map +1 -1
  136. package/dist/multishot/index.d.ts +11 -46
  137. package/dist/multishot/index.d.ts.map +1 -1
  138. package/dist/multishot/index.js +30 -83
  139. package/dist/multishot/index.js.map +1 -1
  140. package/dist/openapi.json +1 -1
  141. package/dist/{opencode-sqlite-DJWAXLms.js → opencode-sqlite-eK6HW6dr.js} +2 -6
  142. package/dist/{opencode-sqlite-DJWAXLms.js.map → opencode-sqlite-eK6HW6dr.js.map} +1 -1
  143. package/dist/pipelines/index.d.ts +5 -5
  144. package/dist/pipelines/index.js +3 -3
  145. package/dist/{pre-registration-zFSLEiFU.d.ts → pre-registration-CzFCcwYk.d.ts} +55 -40
  146. package/dist/pre-registration-CzFCcwYk.d.ts.map +1 -0
  147. package/dist/pre-registration-KN9jkh58.js +110 -0
  148. package/dist/pre-registration-KN9jkh58.js.map +1 -0
  149. package/dist/{produced-state-Bnq4FaDO.js → produced-state-DZ89riy5.js} +8 -8
  150. package/dist/produced-state-DZ89riy5.js.map +1 -0
  151. package/dist/profile-cell.d.ts +1 -1
  152. package/dist/profile-cell.js +31 -5
  153. package/dist/profile-cell.js.map +1 -1
  154. package/dist/{promotion-policy-DLOUkYhI.d.ts → promotion-policy-DtnOIZvk.d.ts} +2 -2
  155. package/dist/{promotion-policy-DLOUkYhI.d.ts.map → promotion-policy-DtnOIZvk.d.ts.map} +1 -1
  156. package/dist/{query-Di7eEQ79.js → query-CHmMP42p.js} +20 -11
  157. package/dist/query-CHmMP42p.js.map +1 -0
  158. package/dist/{query-CJ_DX8vl.d.ts → query-DxPYqpmT.d.ts} +10 -4
  159. package/dist/query-DxPYqpmT.d.ts.map +1 -0
  160. package/dist/raw-provider-sink-BU29Sh8h.d.ts +134 -0
  161. package/dist/raw-provider-sink-BU29Sh8h.d.ts.map +1 -0
  162. package/dist/{registry-BQwrSYpC.d.ts → registry-8You7OK1.d.ts} +5 -7
  163. package/dist/{registry-BQwrSYpC.d.ts.map → registry-8You7OK1.d.ts.map} +1 -1
  164. package/dist/{release-confidence-BknrpBnO.js → release-confidence-DKfD2RYU.js} +28 -14
  165. package/dist/release-confidence-DKfD2RYU.js.map +1 -0
  166. package/dist/{release-confidence-4XrqlpFD.d.ts → release-confidence-Dqt0NFep.d.ts} +7 -6
  167. package/dist/release-confidence-Dqt0NFep.d.ts.map +1 -0
  168. package/dist/reporting.d.ts +3 -3
  169. package/dist/reporting.js +3 -3
  170. package/dist/{researcher-DJnoUE8c.d.ts → researcher-Cz565b7D.d.ts} +34 -21
  171. package/dist/researcher-Cz565b7D.d.ts.map +1 -0
  172. package/dist/{reward-hacking-62tojkQd.d.ts → reward-hacking-MBf7qpSB.d.ts} +2 -2
  173. package/dist/{reward-hacking-62tojkQd.d.ts.map → reward-hacking-MBf7qpSB.d.ts.map} +1 -1
  174. package/dist/{reward-hacking-C0x0xihA.js → reward-hacking-t4lB1yt8.js} +2 -2
  175. package/dist/{reward-hacking-C0x0xihA.js.map → reward-hacking-t4lB1yt8.js.map} +1 -1
  176. package/dist/rl.d.ts +11 -42
  177. package/dist/rl.d.ts.map +1 -1
  178. package/dist/rl.js +41 -24
  179. package/dist/rl.js.map +1 -1
  180. package/dist/rollout/index.d.ts +3 -3
  181. package/dist/rollout/index.js +7 -7
  182. package/dist/{rollout-ytVQ7WT8.js → rollout-Dm2tSdiQ.js} +6 -6
  183. package/dist/{rollout-ytVQ7WT8.js.map → rollout-Dm2tSdiQ.js.map} +1 -1
  184. package/dist/{rubric-predictive-validity-CzxLoZge.js → rubric-predictive-validity-CK8SCOg-.js} +5 -15
  185. package/dist/rubric-predictive-validity-CK8SCOg-.js.map +1 -0
  186. package/dist/{rubric-predictive-validity-C7LnNvF2.d.ts → rubric-predictive-validity-CxycqzX5.d.ts} +4 -3
  187. package/dist/rubric-predictive-validity-CxycqzX5.d.ts.map +1 -0
  188. package/dist/{run-record-D2lDdSAz.js → run-record-BC0ebuRP.js} +2 -2
  189. package/dist/{run-record-D2lDdSAz.js.map → run-record-BC0ebuRP.js.map} +1 -1
  190. package/dist/{run-record-DVV82Gwh.d.ts → run-record-VVy4T9OW.d.ts} +3 -3
  191. package/dist/{run-record-DVV82Gwh.d.ts.map → run-record-VVy4T9OW.d.ts.map} +1 -1
  192. package/dist/{schema-BtVldJ3T.d.ts → schema-Bjgdsn73.d.ts} +2 -4
  193. package/dist/{schema-BtVldJ3T.d.ts.map → schema-Bjgdsn73.d.ts.map} +1 -1
  194. package/dist/{schema-Cef2cFmb.d.ts → schema-BzWDXhOR.d.ts} +2 -5
  195. package/dist/schema-BzWDXhOR.d.ts.map +1 -0
  196. package/dist/{schema-C6DW4ZHR.js → schema-C1aaAxTf.js} +2 -2
  197. package/dist/schema-C1aaAxTf.js.map +1 -0
  198. package/dist/{schema-CRhEY1SO.js → schema-k6ZBftVv.js} +2 -8
  199. package/dist/{schema-CRhEY1SO.js.map → schema-k6ZBftVv.js.map} +1 -1
  200. package/dist/{semantic-concept-judge-laMCnTLn.js → semantic-concept-judge-BSkKKHeq.js} +14 -38
  201. package/dist/semantic-concept-judge-BSkKKHeq.js.map +1 -0
  202. package/dist/{sequential-eprocess-CbUt2htw.js → sequential-eprocess-D1jKoihe.js} +49 -2
  203. package/dist/sequential-eprocess-D1jKoihe.js.map +1 -0
  204. package/dist/{sequential-C458DXNf.js → sequential-rYW-Ophm.js} +41 -16
  205. package/dist/sequential-rYW-Ophm.js.map +1 -0
  206. package/dist/{series-convergence-BxKEgBwA.d.ts → series-convergence-D9WgpXGi.d.ts} +2 -2
  207. package/dist/{series-convergence-BxKEgBwA.d.ts.map → series-convergence-D9WgpXGi.d.ts.map} +1 -1
  208. package/dist/{server-dIWwF3j_.js → server-BtFd4uzB.js} +19 -42
  209. package/dist/server-BtFd4uzB.js.map +1 -0
  210. package/dist/{skillopt-optimization-method-CPBlTcj5.js → skillopt-optimization-method-DbaekMcn.js} +794 -8
  211. package/dist/skillopt-optimization-method-DbaekMcn.js.map +1 -0
  212. package/dist/{skillopt-optimization-method-BDD_o1xE.d.ts → skillopt-optimization-method-x7TTF23P.d.ts} +20 -7
  213. package/dist/{skillopt-optimization-method-BDD_o1xE.d.ts.map → skillopt-optimization-method-x7TTF23P.d.ts.map} +1 -1
  214. package/dist/{statistical-heldout-_woZ9q9j.d.ts → statistical-heldout-Cy3EhjlC.d.ts} +21 -8
  215. package/dist/statistical-heldout-Cy3EhjlC.d.ts.map +1 -0
  216. package/dist/{steps-AmkT-GIM.d.ts → steps-CiNVJry_.d.ts} +2 -17
  217. package/dist/steps-CiNVJry_.d.ts.map +1 -0
  218. package/dist/{store-CT9YIIve.d.ts → store-B06JdC56.d.ts} +2 -2
  219. package/dist/{store-CT9YIIve.d.ts.map → store-B06JdC56.d.ts.map} +1 -1
  220. package/dist/{store-otlp-CDYWW_8N.js → store-otlp-C_Rq5I4D.js} +2 -2
  221. package/dist/{store-otlp-CDYWW_8N.js.map → store-otlp-C_Rq5I4D.js.map} +1 -1
  222. package/dist/{store-tool-spans-Br2_IUhm.d.ts → store-tool-spans-DPUG7UUY.d.ts} +6 -6
  223. package/dist/{store-tool-spans-Br2_IUhm.d.ts.map → store-tool-spans-DPUG7UUY.d.ts.map} +1 -1
  224. package/dist/{store-tool-spans-CykkbOlv.js → store-tool-spans-Dlh9vkFK.js} +3 -3
  225. package/dist/{store-tool-spans-CykkbOlv.js.map → store-tool-spans-Dlh9vkFK.js.map} +1 -1
  226. package/dist/storyboard/index.d.ts +1 -1
  227. package/dist/{summary-report-DW2bEpdB.js → summary-report-BI5hUtvK.js} +6 -16
  228. package/dist/summary-report-BI5hUtvK.js.map +1 -0
  229. package/dist/{summary-report-B__Y5ub3.d.ts → summary-report-CC07PhEL.d.ts} +6 -5
  230. package/dist/summary-report-CC07PhEL.d.ts.map +1 -0
  231. package/dist/supervisor-run/index.d.ts +3 -16
  232. package/dist/supervisor-run/index.d.ts.map +1 -1
  233. package/dist/supervisor-run/index.js +3 -3
  234. package/dist/supervisor-run/index.js.map +1 -1
  235. package/dist/{task-failure-attributes--ZTP3tYO.js → task-failure-attributes-DTl-7-Kw.js} +3 -3
  236. package/dist/{task-failure-attributes--ZTP3tYO.js.map → task-failure-attributes-DTl-7-Kw.js.map} +1 -1
  237. package/dist/{tool-groups-B4tqh8jB.d.ts → tool-groups-Ci8i9ErB.d.ts} +3 -3
  238. package/dist/tool-groups-Ci8i9ErB.d.ts.map +1 -0
  239. package/dist/{tool-waste-C-VHSRwF.js → tool-waste-BqzmVdJk.js} +2 -2
  240. package/dist/{tool-waste-C-VHSRwF.js.map → tool-waste-BqzmVdJk.js.map} +1 -1
  241. package/dist/{tool-waste-DjRDEsuI.d.ts → tool-waste-Dro0gJi3.d.ts} +4 -4
  242. package/dist/{tool-waste-DjRDEsuI.d.ts.map → tool-waste-Dro0gJi3.d.ts.map} +1 -1
  243. package/dist/trace-repair/index.d.ts +4 -77
  244. package/dist/trace-repair/index.d.ts.map +1 -1
  245. package/dist/trace-repair/index.js +5 -15
  246. package/dist/trace-repair/index.js.map +1 -1
  247. package/dist/traces.d.ts +13 -23
  248. package/dist/traces.d.ts.map +1 -1
  249. package/dist/traces.js +9 -20
  250. package/dist/traces.js.map +1 -1
  251. package/dist/{trajectory-YC15QDYQ.d.ts → trajectory-Bi157Gun.d.ts} +3 -3
  252. package/dist/{trajectory-YC15QDYQ.d.ts.map → trajectory-Bi157Gun.d.ts.map} +1 -1
  253. package/dist/trajectory-replay/index.d.ts +5 -5
  254. package/dist/trajectory-replay/index.js +5 -5
  255. package/dist/{provenance-oA4-zUqm.d.ts → transient-failure-DKF5Mofa.d.ts} +468 -13
  256. package/dist/transient-failure-DKF5Mofa.d.ts.map +1 -0
  257. package/dist/types-B3jzCp0p.js.map +1 -1
  258. package/dist/{types-CLAwnY-L.d.ts → types-BPb2Kf_C2.d.ts} +3 -3
  259. package/dist/types-BPb2Kf_C2.d.ts.map +1 -0
  260. package/dist/types-Bfk0uxRj.d.ts +443 -0
  261. package/dist/types-Bfk0uxRj.d.ts.map +1 -0
  262. package/dist/{types-DdFNuyxQ.d.ts → types-D4s7Z6nq.d.ts} +30 -6
  263. package/dist/types-D4s7Z6nq.d.ts.map +1 -0
  264. package/dist/{types-B2NsbrNy.d.ts → types-D9ssmxKL.d.ts} +3 -3
  265. package/dist/{types-B2NsbrNy.d.ts.map → types-D9ssmxKL.d.ts.map} +1 -1
  266. package/dist/{types-I5WwVzQ7.d.ts → types-DeIUdzNd.d.ts} +2 -2
  267. package/dist/{types-I5WwVzQ7.d.ts.map → types-DeIUdzNd.d.ts.map} +1 -1
  268. package/dist/{verdict-BndeTAh_.js → verdict-B0xltqu6.js} +2 -2
  269. package/dist/{verdict-BndeTAh_.js.map → verdict-B0xltqu6.js.map} +1 -1
  270. package/dist/verdict-cache-CdVVTVmn.js +88 -0
  271. package/dist/verdict-cache-CdVVTVmn.js.map +1 -0
  272. package/dist/wire/index.d.ts +21 -111
  273. package/dist/wire/index.d.ts.map +1 -1
  274. package/dist/wire/index.js +2 -2
  275. package/docs/building-doctrine.md +3 -3
  276. package/docs/campaign-proposers.md +41 -10
  277. package/docs/design/statistics-decisions.md +89 -1
  278. package/docs/eval-surface-map.md +14 -0
  279. package/docs/experiment.md +19 -2
  280. package/docs/feedback-trajectories.md +1 -1
  281. package/docs/multishot-golden-records.md +4 -4
  282. package/docs/public-api.md +1616 -0
  283. package/docs/research-report-methodology.md +1 -1
  284. package/docs/search-history-receipts.md +39 -1
  285. package/docs/trace-analysis.md +1 -1
  286. package/docs/trace-repair-admission.md +1 -1
  287. package/docs/trace-repair-continuation.md +1 -1
  288. package/docs/verdicts.md +24 -0
  289. package/package.json +7 -3
  290. package/dist/active-curriculum-C4mk67HP.js.map +0 -1
  291. package/dist/agent-profile-cell-BkcRDikH.d.ts.map +0 -1
  292. package/dist/backend-integrity-DOCa_QrR.d.ts.map +0 -1
  293. package/dist/benchmark-command-BU1Las59.js.map +0 -1
  294. package/dist/campaign-la-gEYNz.js.map +0 -1
  295. package/dist/canonical-D-XsTQ6_.js.map +0 -1
  296. package/dist/client-LIuo-KPv.js.map +0 -1
  297. package/dist/define-agent-eval-C8V8sMqP.js.map +0 -1
  298. package/dist/define-agent-eval-CEQWL9Hy.d.ts.map +0 -1
  299. package/dist/dspy-rlm-engine-CBYlPvNy.js.map +0 -1
  300. package/dist/emitter-CPBAhxum.js.map +0 -1
  301. package/dist/eval-campaign-CQuZrLR_.js.map +0 -1
  302. package/dist/exec-y-DCLqK7.js.map +0 -1
  303. package/dist/exporters-q9iL-2Jf.js.map +0 -1
  304. package/dist/index-Aj3WO3_a.d.ts.map +0 -1
  305. package/dist/index-B8Ui1mr1.d.ts.map +0 -1
  306. package/dist/index-BNPtkBPf.d.ts.map +0 -1
  307. package/dist/index-C1ravkGA.d.ts +0 -1
  308. package/dist/index-IQccV3Ou.d.ts.map +0 -1
  309. package/dist/kind-factory-DmAa0h3K.js.map +0 -1
  310. package/dist/llm-client-Bg32RW0j.js.map +0 -1
  311. package/dist/llm-judge-BqqMS8t7.js.map +0 -1
  312. package/dist/matrix-DrVnRp4G.d.ts.map +0 -1
  313. package/dist/pre-registration-DakwTRXk.js +0 -96
  314. package/dist/pre-registration-DakwTRXk.js.map +0 -1
  315. package/dist/pre-registration-zFSLEiFU.d.ts.map +0 -1
  316. package/dist/produced-state-Bnq4FaDO.js.map +0 -1
  317. package/dist/provenance-oA4-zUqm.d.ts.map +0 -1
  318. package/dist/query-CJ_DX8vl.d.ts.map +0 -1
  319. package/dist/query-Di7eEQ79.js.map +0 -1
  320. package/dist/release-confidence-4XrqlpFD.d.ts.map +0 -1
  321. package/dist/release-confidence-BknrpBnO.js.map +0 -1
  322. package/dist/researcher-DJnoUE8c.d.ts.map +0 -1
  323. package/dist/rubric-predictive-validity-C7LnNvF2.d.ts.map +0 -1
  324. package/dist/rubric-predictive-validity-CzxLoZge.js.map +0 -1
  325. package/dist/schema-C6DW4ZHR.js.map +0 -1
  326. package/dist/schema-Cef2cFmb.d.ts.map +0 -1
  327. package/dist/semantic-concept-judge-laMCnTLn.js.map +0 -1
  328. package/dist/sequential-C458DXNf.js.map +0 -1
  329. package/dist/sequential-eprocess-CbUt2htw.js.map +0 -1
  330. package/dist/server-dIWwF3j_.js.map +0 -1
  331. package/dist/skillopt-optimization-method-CPBlTcj5.js.map +0 -1
  332. package/dist/statistical-heldout-_woZ9q9j.d.ts.map +0 -1
  333. package/dist/steps-AmkT-GIM.d.ts.map +0 -1
  334. package/dist/summary-report-B__Y5ub3.d.ts.map +0 -1
  335. package/dist/summary-report-DW2bEpdB.js.map +0 -1
  336. package/dist/tool-groups-B4tqh8jB.d.ts.map +0 -1
  337. package/dist/types-CLAwnY-L.d.ts.map +0 -1
  338. package/dist/types-DdFNuyxQ.d.ts.map +0 -1
  339. package/dist/types-jUBXJ7Iz.d.ts +0 -884
  340. package/dist/types-jUBXJ7Iz.d.ts.map +0 -1
  341. package/dist/verdict-cache-mZf5FEiY.js +0 -107
  342. package/dist/verdict-cache-mZf5FEiY.js.map +0 -1
@@ -1,6 +1,6 @@
1
- import { s as TraceStore } from "./store-CT9YIIve.js";
2
- import { i as TraceEmitter } from "./emitter-DGQGoLyj.js";
3
- import { n as TrajectoryStep, t as Trajectory } from "./trajectory-YC15QDYQ.js";
1
+ import { s as TraceStore } from "./store-B06JdC56.js";
2
+ import { i as TraceEmitter } from "./emitter-D_jYSGRd.js";
3
+ import { n as TrajectoryStep, t as Trajectory } from "./trajectory-Bi157Gun.js";
4
4
  //#region src/counterfactual.d.ts
5
5
  type CounterfactualMutation = {
6
6
  kind: 'swap-model';
@@ -57,4 +57,4 @@ interface CounterfactualRunner {
57
57
  declare function runCounterfactual(store: TraceStore, originalRunId: string, mutation: CounterfactualMutation, runner: CounterfactualRunner): Promise<CounterfactualResult>;
58
58
  //#endregion
59
59
  export { runCounterfactual as a, CounterfactualRunner as i, CounterfactualMutation as n, CounterfactualResult as r, CounterfactualContext as t };
60
- //# sourceMappingURL=counterfactual--bpysZF0.d.ts.map
60
+ //# sourceMappingURL=counterfactual-BaFUWK3H.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"counterfactual--bpysZF0.d.ts","names":[],"sources":["../src/counterfactual.ts"],"mappings":";;;;KAoBY;EACN;EAAoB;EAAY;;EAChC;EAA0B;EAAY;;EACtC;EAAwB;;EACxB;EAA+B;EAAY;;EAE3C;EACA;EACA;EACA,QAAQ,MAAM,mBAAmB;;UAGtB;EACf;EACA,oBAAoB;;;EAGpB,QAAQ;EACR,UAAU;;;;EAIV,aAAa;;UAGE;EACf;EACA;EACA,UAAU;;EAEV;IACE;IACA;IACA;;;UAIa;;;;;;EAMf,cAAc,KAAK,uBAAuB,SAAS,iBAAiB;;iBAGhD,kBACpB,OAAO,YACP,uBACA,UAAU,wBACV,QAAQ,uBACP,QAAQ"}
1
+ {"version":3,"file":"counterfactual-BaFUWK3H.d.ts","names":[],"sources":["../src/counterfactual.ts"],"mappings":";;;;KAoBY;EACN;EAAoB;EAAY;;EAChC;EAA0B;EAAY;;EACtC;EAAwB;;EACxB;EAA+B;EAAY;;EAE3C;EACA;EACA;EACA,QAAQ,MAAM,mBAAmB;;UAGtB;EACf;EACA,oBAAoB;;;EAGpB,QAAQ;EACR,UAAU;;;;EAIV,aAAa;;UAGE;EACf;EACA;EACA,UAAU;;EAEV;IACE;IACA;IACA;;;UAIa;;;;;;EAMf,cAAc,KAAK,uBAAuB,SAAS,iBAAiB;;iBAGhD,kBACpB,OAAO,YACP,uBACA,UAAU,wBACV,QAAQ,uBACP,QAAQ"}
@@ -1,5 +1,5 @@
1
1
  import { o as NotFoundError, s as ValidationError } from "./errors-Dngq5h35.js";
2
- import { t as TraceEmitter } from "./emitter-CPBAhxum.js";
2
+ import { t as TraceEmitter } from "./emitter-BpYFQPj4.js";
3
3
  import { t as buildTrajectory } from "./trajectory-D_7rLrvE.js";
4
4
  //#region src/counterfactual.ts
5
5
  /**
@@ -95,4 +95,4 @@ function applyMutation(step, mutation) {
95
95
  //#endregion
96
96
  export { runCounterfactual as t };
97
97
 
98
- //# sourceMappingURL=counterfactual-lDfCx0Uz.js.map
98
+ //# sourceMappingURL=counterfactual-D_VWavVm.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"counterfactual-lDfCx0Uz.js","names":[],"sources":["../src/counterfactual.ts"],"sourcesContent":["/**\n * Counterfactual replay — \"what would have happened if we'd changed\n * exactly one thing at turn N?\"\n *\n * The framework does NOT drive the agent — it sets up the replay\n * context (prior spans, prior state, mutation spec) and records the\n * resulting divergence. Consumers supply an `executeFrom(ctx)` callback\n * that runs their agent starting from turn N with the mutation applied.\n *\n * Counterfactual runs are recorded as a new Run with `layer='meta'` and\n * `parentRunId = originalRunId`, so downstream diff + correlation\n * pipelines see them natively.\n */\n\nimport { NotFoundError, ValidationError } from './errors'\nimport { TraceEmitter } from './trace/emitter'\nimport type { LlmSpan, Span, ToolSpan } from './trace/schema'\nimport type { TraceStore } from './trace/store'\nimport { buildTrajectory, type Trajectory, type TrajectoryStep } from './trajectory'\n\nexport type CounterfactualMutation =\n | { kind: 'swap-model'; at: number; newModel: string }\n | { kind: 'swap-tool-result'; at: number; newResult: unknown }\n | { kind: 'truncate-after'; at: number }\n | { kind: 'inject-system-message'; at: number; content: string }\n | {\n kind: 'custom'\n at: number\n describe: string\n apply: (step: TrajectoryStep) => TrajectoryStep\n }\n\nexport interface CounterfactualContext {\n originalRunId: string\n originalTrajectory: Trajectory\n /** Steps up to (but not including) the mutation point — the prefix the\n * replayed agent inherits as its prior conversation/tool history. */\n prefix: TrajectoryStep[]\n mutation: CounterfactualMutation\n /** Pre-applied mutation on the step at `mutation.at`. Consumers use this\n * as the FIRST step the replayed agent emits (they decide whether to\n * re-emit it or continue from there). */\n mutatedStep: TrajectoryStep\n}\n\nexport interface CounterfactualResult {\n counterfactualRunId: string\n originalRunId: string\n mutation: CounterfactualMutation\n /** Structured delta summary — caller can extend via scoring. */\n delta: {\n originalOutcomeScore: number | null\n counterfactualOutcomeScore: number | null\n deltaScore: number | null\n }\n}\n\nexport interface CounterfactualRunner {\n /**\n * Execute the agent from `ctx.prefix` with the mutation applied.\n * MUST emit spans into the provided emitter so they become part of\n * the counterfactual run. MUST call emitter.endRun() with a verdict.\n */\n executeFrom: (ctx: CounterfactualContext, emitter: TraceEmitter) => Promise<void>\n}\n\nexport async function runCounterfactual(\n store: TraceStore,\n originalRunId: string,\n mutation: CounterfactualMutation,\n runner: CounterfactualRunner,\n): Promise<CounterfactualResult> {\n const originalRun = await store.getRun(originalRunId)\n if (!originalRun) throw new NotFoundError(`counterfactual: run ${originalRunId} not found`)\n const trajectory = await buildTrajectory(store, originalRunId)\n if (mutation.at < 0 || mutation.at >= trajectory.steps.length) {\n throw new ValidationError(\n `counterfactual: mutation.at=${mutation.at} out of range [0, ${trajectory.steps.length})`,\n )\n }\n const targetStep = trajectory.steps[mutation.at]!\n const mutatedStep = applyMutation(targetStep, mutation)\n\n const cfEmitter = new TraceEmitter(store)\n await cfEmitter.startRun({\n scenarioId: originalRun.scenarioId,\n variantId: originalRun.variantId\n ? `${originalRun.variantId}+cf:${mutation.kind}@${mutation.at}`\n : `cf:${mutation.kind}@${mutation.at}`,\n projectId: originalRun.projectId,\n parentRunId: originalRunId,\n layer: 'meta',\n tags: { counterfactual: 'true', mutationKind: mutation.kind, mutationAt: String(mutation.at) },\n })\n\n await runner.executeFrom(\n {\n originalRunId,\n originalTrajectory: trajectory,\n prefix: trajectory.steps.slice(0, mutation.at),\n mutation,\n mutatedStep,\n },\n cfEmitter,\n )\n\n const counterfactual = await store.getRun(cfEmitter.runId)\n const delta = {\n originalOutcomeScore: originalRun.outcome?.score ?? null,\n counterfactualOutcomeScore: counterfactual?.outcome?.score ?? null,\n deltaScore:\n originalRun.outcome?.score !== undefined && counterfactual?.outcome?.score !== undefined\n ? counterfactual.outcome.score - originalRun.outcome.score\n : null,\n }\n return { counterfactualRunId: cfEmitter.runId, originalRunId, mutation, delta }\n}\n\nfunction applyMutation(step: TrajectoryStep, mutation: CounterfactualMutation): TrajectoryStep {\n if (mutation.kind === 'swap-model' && step.span.kind === 'llm') {\n const llm = step.span as LlmSpan\n return { ...step, span: { ...llm, model: mutation.newModel } }\n }\n if (mutation.kind === 'swap-tool-result' && step.span.kind === 'tool') {\n const tool = step.span as ToolSpan\n return { ...step, span: { ...tool, result: mutation.newResult } }\n }\n if (mutation.kind === 'inject-system-message' && step.span.kind === 'llm') {\n const llm = step.span as LlmSpan\n return {\n ...step,\n span: {\n ...llm,\n messages: [{ role: 'system', content: mutation.content }, ...llm.messages],\n },\n }\n }\n if (mutation.kind === 'custom') return mutation.apply(step)\n // swap-tool-result on non-tool span / swap-model on non-llm / truncate-after: no step-level change.\n return step\n}\n\n/**\n * Aggregate a batch of counterfactuals into a simple attribution table:\n * which mutation kinds move outcomes most? (Useful when you run a grid\n * over the same trajectory — swap-model at every llm span, swap-tool\n * at every tool span — and want a ranked summary.)\n */\nexport function attributeCounterfactuals(results: CounterfactualResult[]): Array<{\n mutationKind: CounterfactualMutation['kind']\n n: number\n meanAbsDelta: number\n meanSignedDelta: number\n}> {\n const grouped = new Map<string, CounterfactualResult[]>()\n for (const r of results) {\n const arr = grouped.get(r.mutation.kind) ?? []\n arr.push(r)\n grouped.set(r.mutation.kind, arr)\n }\n const out: Array<{\n mutationKind: CounterfactualMutation['kind']\n n: number\n meanAbsDelta: number\n meanSignedDelta: number\n }> = []\n for (const [kind, items] of grouped) {\n const deltas = items\n .map((i) => i.delta.deltaScore)\n .filter((d): d is number => typeof d === 'number')\n if (deltas.length === 0) continue\n const meanAbs = deltas.reduce((a, b) => a + Math.abs(b), 0) / deltas.length\n const meanSigned = deltas.reduce((a, b) => a + b, 0) / deltas.length\n out.push({\n mutationKind: kind as CounterfactualMutation['kind'],\n n: deltas.length,\n meanAbsDelta: meanAbs,\n meanSignedDelta: meanSigned,\n })\n }\n return out.sort((a, b) => b.meanAbsDelta - a.meanAbsDelta)\n}\n\n// Re-export Span type for consumer ergonomics.\nexport type { Span }\n"],"mappings":";;;;;;;;;;;;;;;;;AAkEA,eAAsB,kBACpB,OACA,eACA,UACA,QAC+B;CAC/B,MAAM,cAAc,MAAM,MAAM,OAAO,aAAa;CACpD,IAAI,CAAC,aAAa,MAAM,IAAI,cAAc,uBAAuB,cAAc,WAAW;CAC1F,MAAM,aAAa,MAAM,gBAAgB,OAAO,aAAa;CAC7D,IAAI,SAAS,KAAK,KAAK,SAAS,MAAM,WAAW,MAAM,QACrD,MAAM,IAAI,gBACR,+BAA+B,SAAS,GAAG,oBAAoB,WAAW,MAAM,OAAO,EACzF;CAEF,MAAM,aAAa,WAAW,MAAM,SAAS;CAC7C,MAAM,cAAc,cAAc,YAAY,QAAQ;CAEtD,MAAM,YAAY,IAAI,aAAa,KAAK;CACxC,MAAM,UAAU,SAAS;EACvB,YAAY,YAAY;EACxB,WAAW,YAAY,YACnB,GAAG,YAAY,UAAU,MAAM,SAAS,KAAK,GAAG,SAAS,OACzD,MAAM,SAAS,KAAK,GAAG,SAAS;EACpC,WAAW,YAAY;EACvB,aAAa;EACb,OAAO;EACP,MAAM;GAAE,gBAAgB;GAAQ,cAAc,SAAS;GAAM,YAAY,OAAO,SAAS,EAAE;EAAE;CAC/F,CAAC;CAED,MAAM,OAAO,YACX;EACE;EACA,oBAAoB;EACpB,QAAQ,WAAW,MAAM,MAAM,GAAG,SAAS,EAAE;EAC7C;EACA;CACF,GACA,SACF;CAEA,MAAM,iBAAiB,MAAM,MAAM,OAAO,UAAU,KAAK;CACzD,MAAM,QAAQ;EACZ,sBAAsB,YAAY,SAAS,SAAS;EACpD,4BAA4B,gBAAgB,SAAS,SAAS;EAC9D,YACE,YAAY,SAAS,UAAU,KAAA,KAAa,gBAAgB,SAAS,UAAU,KAAA,IAC3E,eAAe,QAAQ,QAAQ,YAAY,QAAQ,QACnD;CACR;CACA,OAAO;EAAE,qBAAqB,UAAU;EAAO;EAAe;EAAU;CAAM;AAChF;AAEA,SAAS,cAAc,MAAsB,UAAkD;CAC7F,IAAI,SAAS,SAAS,gBAAgB,KAAK,KAAK,SAAS,OAAO;EAC9D,MAAM,MAAM,KAAK;EACjB,OAAO;GAAE,GAAG;GAAM,MAAM;IAAE,GAAG;IAAK,OAAO,SAAS;GAAS;EAAE;CAC/D;CACA,IAAI,SAAS,SAAS,sBAAsB,KAAK,KAAK,SAAS,QAAQ;EACrE,MAAM,OAAO,KAAK;EAClB,OAAO;GAAE,GAAG;GAAM,MAAM;IAAE,GAAG;IAAM,QAAQ,SAAS;GAAU;EAAE;CAClE;CACA,IAAI,SAAS,SAAS,2BAA2B,KAAK,KAAK,SAAS,OAAO;EACzE,MAAM,MAAM,KAAK;EACjB,OAAO;GACL,GAAG;GACH,MAAM;IACJ,GAAG;IACH,UAAU,CAAC;KAAE,MAAM;KAAU,SAAS,SAAS;IAAQ,GAAG,GAAG,IAAI,QAAQ;GAC3E;EACF;CACF;CACA,IAAI,SAAS,SAAS,UAAU,OAAO,SAAS,MAAM,IAAI;CAE1D,OAAO;AACT"}
1
+ {"version":3,"file":"counterfactual-D_VWavVm.js","names":[],"sources":["../src/counterfactual.ts"],"sourcesContent":["/**\n * Counterfactual replay — \"what would have happened if we'd changed\n * exactly one thing at turn N?\"\n *\n * The framework does NOT drive the agent — it sets up the replay\n * context (prior spans, prior state, mutation spec) and records the\n * resulting divergence. Consumers supply an `executeFrom(ctx)` callback\n * that runs their agent starting from turn N with the mutation applied.\n *\n * Counterfactual runs are recorded as a new Run with `layer='meta'` and\n * `parentRunId = originalRunId`, so downstream diff + correlation\n * pipelines see them natively.\n */\n\nimport { NotFoundError, ValidationError } from './errors'\nimport { TraceEmitter } from './trace/emitter'\nimport type { LlmSpan, Span, ToolSpan } from './trace/schema'\nimport type { TraceStore } from './trace/store'\nimport { buildTrajectory, type Trajectory, type TrajectoryStep } from './trajectory'\n\nexport type CounterfactualMutation =\n | { kind: 'swap-model'; at: number; newModel: string }\n | { kind: 'swap-tool-result'; at: number; newResult: unknown }\n | { kind: 'truncate-after'; at: number }\n | { kind: 'inject-system-message'; at: number; content: string }\n | {\n kind: 'custom'\n at: number\n describe: string\n apply: (step: TrajectoryStep) => TrajectoryStep\n }\n\nexport interface CounterfactualContext {\n originalRunId: string\n originalTrajectory: Trajectory\n /** Steps up to (but not including) the mutation point — the prefix the\n * replayed agent inherits as its prior conversation/tool history. */\n prefix: TrajectoryStep[]\n mutation: CounterfactualMutation\n /** Pre-applied mutation on the step at `mutation.at`. Consumers use this\n * as the FIRST step the replayed agent emits (they decide whether to\n * re-emit it or continue from there). */\n mutatedStep: TrajectoryStep\n}\n\nexport interface CounterfactualResult {\n counterfactualRunId: string\n originalRunId: string\n mutation: CounterfactualMutation\n /** Structured delta summary — caller can extend via scoring. */\n delta: {\n originalOutcomeScore: number | null\n counterfactualOutcomeScore: number | null\n deltaScore: number | null\n }\n}\n\nexport interface CounterfactualRunner {\n /**\n * Execute the agent from `ctx.prefix` with the mutation applied.\n * MUST emit spans into the provided emitter so they become part of\n * the counterfactual run. MUST call emitter.endRun() with a verdict.\n */\n executeFrom: (ctx: CounterfactualContext, emitter: TraceEmitter) => Promise<void>\n}\n\nexport async function runCounterfactual(\n store: TraceStore,\n originalRunId: string,\n mutation: CounterfactualMutation,\n runner: CounterfactualRunner,\n): Promise<CounterfactualResult> {\n const originalRun = await store.getRun(originalRunId)\n if (!originalRun) throw new NotFoundError(`counterfactual: run ${originalRunId} not found`)\n const trajectory = await buildTrajectory(store, originalRunId)\n if (mutation.at < 0 || mutation.at >= trajectory.steps.length) {\n throw new ValidationError(\n `counterfactual: mutation.at=${mutation.at} out of range [0, ${trajectory.steps.length})`,\n )\n }\n const targetStep = trajectory.steps[mutation.at]!\n const mutatedStep = applyMutation(targetStep, mutation)\n\n const cfEmitter = new TraceEmitter(store)\n await cfEmitter.startRun({\n scenarioId: originalRun.scenarioId,\n variantId: originalRun.variantId\n ? `${originalRun.variantId}+cf:${mutation.kind}@${mutation.at}`\n : `cf:${mutation.kind}@${mutation.at}`,\n projectId: originalRun.projectId,\n parentRunId: originalRunId,\n layer: 'meta',\n tags: { counterfactual: 'true', mutationKind: mutation.kind, mutationAt: String(mutation.at) },\n })\n\n await runner.executeFrom(\n {\n originalRunId,\n originalTrajectory: trajectory,\n prefix: trajectory.steps.slice(0, mutation.at),\n mutation,\n mutatedStep,\n },\n cfEmitter,\n )\n\n const counterfactual = await store.getRun(cfEmitter.runId)\n const delta = {\n originalOutcomeScore: originalRun.outcome?.score ?? null,\n counterfactualOutcomeScore: counterfactual?.outcome?.score ?? null,\n deltaScore:\n originalRun.outcome?.score !== undefined && counterfactual?.outcome?.score !== undefined\n ? counterfactual.outcome.score - originalRun.outcome.score\n : null,\n }\n return { counterfactualRunId: cfEmitter.runId, originalRunId, mutation, delta }\n}\n\nfunction applyMutation(step: TrajectoryStep, mutation: CounterfactualMutation): TrajectoryStep {\n if (mutation.kind === 'swap-model' && step.span.kind === 'llm') {\n const llm = step.span as LlmSpan\n return { ...step, span: { ...llm, model: mutation.newModel } }\n }\n if (mutation.kind === 'swap-tool-result' && step.span.kind === 'tool') {\n const tool = step.span as ToolSpan\n return { ...step, span: { ...tool, result: mutation.newResult } }\n }\n if (mutation.kind === 'inject-system-message' && step.span.kind === 'llm') {\n const llm = step.span as LlmSpan\n return {\n ...step,\n span: {\n ...llm,\n messages: [{ role: 'system', content: mutation.content }, ...llm.messages],\n },\n }\n }\n if (mutation.kind === 'custom') return mutation.apply(step)\n // swap-tool-result on non-tool span / swap-model on non-llm / truncate-after: no step-level change.\n return step\n}\n\n/**\n * Aggregate a batch of counterfactuals into a simple attribution table:\n * which mutation kinds move outcomes most? (Useful when you run a grid\n * over the same trajectory — swap-model at every llm span, swap-tool\n * at every tool span — and want a ranked summary.)\n */\nexport function attributeCounterfactuals(results: CounterfactualResult[]): Array<{\n mutationKind: CounterfactualMutation['kind']\n n: number\n meanAbsDelta: number\n meanSignedDelta: number\n}> {\n const grouped = new Map<string, CounterfactualResult[]>()\n for (const r of results) {\n const arr = grouped.get(r.mutation.kind) ?? []\n arr.push(r)\n grouped.set(r.mutation.kind, arr)\n }\n const out: Array<{\n mutationKind: CounterfactualMutation['kind']\n n: number\n meanAbsDelta: number\n meanSignedDelta: number\n }> = []\n for (const [kind, items] of grouped) {\n const deltas = items\n .map((i) => i.delta.deltaScore)\n .filter((d): d is number => typeof d === 'number')\n if (deltas.length === 0) continue\n const meanAbs = deltas.reduce((a, b) => a + Math.abs(b), 0) / deltas.length\n const meanSigned = deltas.reduce((a, b) => a + b, 0) / deltas.length\n out.push({\n mutationKind: kind as CounterfactualMutation['kind'],\n n: deltas.length,\n meanAbsDelta: meanAbs,\n meanSignedDelta: meanSigned,\n })\n }\n return out.sort((a, b) => b.meanAbsDelta - a.meanAbsDelta)\n}\n\n// Re-export Span type for consumer ergonomics.\nexport type { Span }\n"],"mappings":";;;;;;;;;;;;;;;;;AAkEA,eAAsB,kBACpB,OACA,eACA,UACA,QAC+B;CAC/B,MAAM,cAAc,MAAM,MAAM,OAAO,aAAa;CACpD,IAAI,CAAC,aAAa,MAAM,IAAI,cAAc,uBAAuB,cAAc,WAAW;CAC1F,MAAM,aAAa,MAAM,gBAAgB,OAAO,aAAa;CAC7D,IAAI,SAAS,KAAK,KAAK,SAAS,MAAM,WAAW,MAAM,QACrD,MAAM,IAAI,gBACR,+BAA+B,SAAS,GAAG,oBAAoB,WAAW,MAAM,OAAO,EACzF;CAEF,MAAM,aAAa,WAAW,MAAM,SAAS;CAC7C,MAAM,cAAc,cAAc,YAAY,QAAQ;CAEtD,MAAM,YAAY,IAAI,aAAa,KAAK;CACxC,MAAM,UAAU,SAAS;EACvB,YAAY,YAAY;EACxB,WAAW,YAAY,YACnB,GAAG,YAAY,UAAU,MAAM,SAAS,KAAK,GAAG,SAAS,OACzD,MAAM,SAAS,KAAK,GAAG,SAAS;EACpC,WAAW,YAAY;EACvB,aAAa;EACb,OAAO;EACP,MAAM;GAAE,gBAAgB;GAAQ,cAAc,SAAS;GAAM,YAAY,OAAO,SAAS,EAAE;EAAE;CAC/F,CAAC;CAED,MAAM,OAAO,YACX;EACE;EACA,oBAAoB;EACpB,QAAQ,WAAW,MAAM,MAAM,GAAG,SAAS,EAAE;EAC7C;EACA;CACF,GACA,SACF;CAEA,MAAM,iBAAiB,MAAM,MAAM,OAAO,UAAU,KAAK;CACzD,MAAM,QAAQ;EACZ,sBAAsB,YAAY,SAAS,SAAS;EACpD,4BAA4B,gBAAgB,SAAS,SAAS;EAC9D,YACE,YAAY,SAAS,UAAU,KAAA,KAAa,gBAAgB,SAAS,UAAU,KAAA,IAC3E,eAAe,QAAQ,QAAQ,YAAY,QAAQ,QACnD;CACR;CACA,OAAO;EAAE,qBAAqB,UAAU;EAAO;EAAe;EAAU;CAAM;AAChF;AAEA,SAAS,cAAc,MAAsB,UAAkD;CAC7F,IAAI,SAAS,SAAS,gBAAgB,KAAK,KAAK,SAAS,OAAO;EAC9D,MAAM,MAAM,KAAK;EACjB,OAAO;GAAE,GAAG;GAAM,MAAM;IAAE,GAAG;IAAK,OAAO,SAAS;GAAS;EAAE;CAC/D;CACA,IAAI,SAAS,SAAS,sBAAsB,KAAK,KAAK,SAAS,QAAQ;EACrE,MAAM,OAAO,KAAK;EAClB,OAAO;GAAE,GAAG;GAAM,MAAM;IAAE,GAAG;IAAM,QAAQ,SAAS;GAAU;EAAE;CAClE;CACA,IAAI,SAAS,SAAS,2BAA2B,KAAK,KAAK,SAAS,OAAO;EACzE,MAAM,MAAM,KAAK;EACjB,OAAO;GACL,GAAG;GACH,MAAM;IACJ,GAAG;IACH,UAAU,CAAC;KAAE,MAAM;KAAU,SAAS,SAAS;IAAQ,GAAG,GAAG,IAAI,QAAQ;GAC3E;EACF;CACF;CACA,IAAI,SAAS,SAAS,UAAU,OAAO,SAAS,MAAM,IAAI;CAE1D,OAAO;AACT"}
@@ -49,7 +49,7 @@ interface DatasetProvenance {
49
49
  interface DatasetManifest {
50
50
  name: string;
51
51
  provenance: DatasetProvenance;
52
- /** sha256 hex over canonicalized scenarios. */
52
+ /** sha256 hex over the RFC 8785 canonical JSON of the id-sorted scenarios. */
53
53
  contentHash: string;
54
54
  scenarioCount: number;
55
55
  splitCounts: Record<DatasetSplit, number>;
@@ -98,12 +98,13 @@ declare class Dataset {
98
98
  add(scenario: DatasetScenario): void;
99
99
  remove(scenarioId: string): void;
100
100
  /**
101
- * Stable JSON-Lines serialization — deterministic byte-for-byte.
102
- * Write to disk for contamination-verifiable archives.
101
+ * Stable JSON-Lines serialization — one RFC 8785 canonical row per scenario,
102
+ * sorted by id, deterministic byte-for-byte. Write to disk for
103
+ * contamination-verifiable archives.
103
104
  */
104
105
  toJsonl(): string;
105
106
  static fromJsonl(jsonl: string, manifest: Omit<DatasetManifest, 'contentHash' | 'scenarioCount' | 'splitCounts'>): Dataset;
106
107
  }
107
108
  //#endregion
108
109
  export { DatasetSplit as i, DatasetManifest as n, DatasetScenario as r, Dataset as t };
109
- //# sourceMappingURL=dataset-CJjKqQfA.d.ts.map
110
+ //# sourceMappingURL=dataset-DQqhOCPt.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"dataset-CJjKqQfA.d.ts","names":[],"sources":["../src/dataset.ts"],"mappings":";;;;;;;;;;;;;;;;;KAiBY;KACA;UAEK;EACf;;EAEA;EACA,QAAQ;EACR,aAAa;;EAEb;;;;;;;;;;EAUA;EACA,OAAO;;UAGQ;EACf;EACA;EACA;EACA;EACA;;EAEA;;UAGe;EACf;EACA,YAAY;;EAEZ;EACA;EACA,aAAa,OAAO;;UAGL;EACf,QAAQ;EACR,aAAa;;EAEb;EACA;;EAEA,UAAU,UAAU;;EAEpB;;cAcW;WACF;WACA,YAAY;UACb;UACA;EAER,YAAY;IACV;IACA,YAAY;IACZ,WAAW;IACX;;;EASF,gBAAgB;MAIZ;;;;;EAQJ,MAAM,UAAS,eAAoB;;;;;EAqB7B,YAAY,QAAQ;;EAgB1B,MAAM,YAAW;IAAU;IAAc;OAA0B;EAWnE;EAIA,IAAI,UAAU;EAQd,OAAO;;;;;EAWP;SAQO,UACL,eACA,UAAU,KAAK,oEACd"}
1
+ {"version":3,"file":"dataset-DQqhOCPt.d.ts","names":[],"sources":["../src/dataset.ts"],"mappings":";;;;;;;;;;;;;;;;;KAiBY;KACA;UAEK;EACf;;EAEA;EACA,QAAQ;EACR,aAAa;;EAEb;;;;;;;;;;EAUA;EACA,OAAO;;UAGQ;EACf;EACA;EACA;EACA;EACA;;EAEA;;UAGe;EACf;EACA,YAAY;;EAEZ;EACA;EACA,aAAa,OAAO;;UAGL;EACf,QAAQ;EACR,aAAa;;EAEb;EACA;;EAEA,UAAU,UAAU;;EAEpB;;cAeW;WACF;WACA,YAAY;UACb;UACA;EAER,YAAY;IACV;IACA,YAAY;IACZ,WAAW;IACX;;;EASF,gBAAgB;MAIZ;;;;;EAQJ,MAAM,UAAS,eAAoB;;;;;EAqB7B,YAAY,QAAQ;;EAgB1B,MAAM,YAAW;IAAU;IAAc;OAA0B;EAWnE;EAIA,IAAI,UAAU;EAQd,OAAO;;;;;;EAYP;SAQO,UACL,eACA,UAAU,KAAK,oEACd"}
@@ -1,8 +1,8 @@
1
- import { i as AnalystFinding, n as AnalystContext, w as TraceAnalysisStore } from "./types-B2NsbrNy.js";
2
- import { l as ExactCapableAnalyst } from "./exact-types-ccQAyut1.js";
3
- import { n as AnalystRegistry, r as AnalystRegistryOptions } from "./registry-BQwrSYpC.js";
4
- import { a as TraceAnalystLimits, d as RawAnalystFinding, i as TraceAnalysisEngineResult, n as TraceAnalysisEngine } from "./engine-BLzhNzoY.js";
5
- import { t as TraceToolGroupName } from "./tool-groups-B4tqh8jB.js";
1
+ import { i as AnalystFinding, n as AnalystContext, w as TraceAnalysisStore } from "./types-D9ssmxKL.js";
2
+ import { l as ExactCapableAnalyst } from "./exact-types-qnexxJ1Z.js";
3
+ import { n as AnalystRegistry, r as AnalystRegistryOptions } from "./registry-8You7OK1.js";
4
+ import { c as RawAnalystFinding, i as TraceAnalystLimits, r as TraceAnalysisEngineResult, t as TraceAnalysisEngine } from "./engine-Cu5qD5Fc.js";
5
+ import { t as TraceToolGroupName } from "./tool-groups-Ci8i9ErB.js";
6
6
  //#region src/trace-analyst/behavioral-metrics.d.ts
7
7
  type SuboptimalCode = 'monotonic-input-growth' | 'output-length-decay' | 'single-tool-dependency' | 'no-self-verification';
8
8
  interface SuboptimalSignal {
@@ -92,7 +92,6 @@ declare function runTraceAnalyst(args: {
92
92
  /** Adapt a research definition to the common Analyst registry contract. */
93
93
  declare function createTraceAnalyst(definition: TraceAnalystDefinition, options: CreateTraceAnalystOptions): ExactCapableAnalyst<TraceAnalysisStore>;
94
94
  declare function renderPriorFindings(prior: AnalystContext['priorFindings']): string;
95
- declare function renderUpstreamFindings(upstream: AnalystContext['upstreamFindings']): string;
96
95
  //#endregion
97
96
  //#region src/analyst/default-registry.d.ts
98
97
  interface DefaultAnalystRegistryOptions {
@@ -110,5 +109,5 @@ interface DefaultAnalystRegistryOptions {
110
109
  }
111
110
  declare function buildDefaultAnalystRegistry(options?: DefaultAnalystRegistryOptions): AnalystRegistry;
112
111
  //#endregion
113
- export { createTraceAnalyst as a, runTraceAnalyst as c, deriveEfficiencyFindings as d, TraceAnalystDefinition as i, BehavioralAnalystOptions as l, buildDefaultAnalystRegistry as n, renderPriorFindings as o, CreateTraceAnalystOptions as r, renderUpstreamFindings as s, DefaultAnalystRegistryOptions as t, behavioralAnalyst as u };
114
- //# sourceMappingURL=default-registry-Cw0Ohdoj.d.ts.map
112
+ export { createTraceAnalyst as a, BehavioralAnalystOptions as c, TraceAnalystDefinition as i, behavioralAnalyst as l, buildDefaultAnalystRegistry as n, renderPriorFindings as o, CreateTraceAnalystOptions as r, runTraceAnalyst as s, DefaultAnalystRegistryOptions as t, deriveEfficiencyFindings as u };
113
+ //# sourceMappingURL=default-registry-G9CKMNkc.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"default-registry-Cw0Ohdoj.d.ts","names":[],"sources":["../src/trace-analyst/behavioral-metrics.ts","../src/analyst/behavioral-analyst.ts","../src/analyst/kind-factory.ts","../src/analyst/default-registry.ts"],"mappings":";;;;;;KAoBY;UAMK;EACf,MAAM;EACN;;EAEA;;EAEA,UAAU;;UAGK;;EAEf;EACA;;EAEA,gBAAgB;;EAEhB;EACA;EACA,eAAe;EACf;EACA;;EAEA;EACA;EACA,SAAS;;UAGM;EACf;EACA;EACA,sBAAsB;EACtB,uBAAuB;;;;UC1BR;;EAEf;;EAEA;;;;;;iBA+Dc,yBACd,SAAS,mBACT;EAAQ;EAAoB;IAC3B;;iBAkCa,kBACd,UAAS,2BACR,oBAAoB;;;;UCrHN;EACf;EACA;EACA;EACA;;EAEA,sBAAsB,SAAS;;EAE/B;;EAEA,WAAW;;EAEX,kBACE,OAAO,oBACP,SAAS,wCACe;EAC1B,SAAS,QAAQ;EACjB,eAAe,KAAK,mBAAmB,SAAS,mBAAmB;EACnE;EACA;;UAGe;EACf,QAAQ;;EAER;EACA;;;iBAIoB,gBAAgB;EACpC,YAAY;EACZ,QAAQ;EACR,OAAO;EACP,SAAS;EACT;IACE,QAAQ;;iBAgFI,mBACd,YAAY,wBACZ,SAAS,4BACR,oBAAoB;iBA+QP,oBAAoB,OAAO;iBAe3B,uBAAuB,UAAU;;;UCnahC;;;;;;EAMf,SAAS;EACT,uBAAuB;;EAEvB;EACA,aAAa;EACb,WAAW;;iBAGG,4BACd,UAAS,gCACR"}
1
+ {"version":3,"file":"default-registry-G9CKMNkc.d.ts","names":[],"sources":["../src/trace-analyst/behavioral-metrics.ts","../src/analyst/behavioral-analyst.ts","../src/analyst/kind-factory.ts","../src/analyst/default-registry.ts"],"mappings":";;;;;;KAoBY;UAMK;EACf,MAAM;EACN;;EAEA;;EAEA,UAAU;;UAGK;;EAEf;EACA;;EAEA,gBAAgB;;EAEhB;EACA;EACA,eAAe;EACf;EACA;;EAEA;EACA;EACA,SAAS;;UAGM;EACf;EACA;EACA,sBAAsB;EACtB,uBAAuB;;;;UC1BR;;EAEf;;EAEA;;;;;;iBA+Dc,yBACd,SAAS,mBACT;EAAQ;EAAoB;IAC3B;;iBAkCa,kBACd,UAAS,2BACR,oBAAoB;;;;UCrHN;EACf;EACA;EACA;EACA;;EAEA,sBAAsB,SAAS;;EAE/B;;EAEA,WAAW;;EAEX,kBACE,OAAO,oBACP,SAAS,wCACe;EAC1B,SAAS,QAAQ;EACjB,eAAe,KAAK,mBAAmB,SAAS,mBAAmB;EACnE;EACA;;UAGe;EACf,QAAQ;;EAER;EACA;;;iBAIoB,gBAAgB;EACpC,YAAY;EACZ,QAAQ;EACR,OAAO;EACP,SAAS;EACT;IACE,QAAQ;;iBAgFI,mBACd,YAAY,wBACZ,SAAS,4BACR,oBAAoB;iBA+QP,oBAAoB,OAAO;;;UCpZ1B;;;;;;EAMf,SAAS;EACT,uBAAuB;;EAEvB;EACA,aAAa;EACb,WAAW;;iBAGG,4BACd,UAAS,gCACR"}
@@ -1,10 +1,10 @@
1
1
  import { c as CostLedgerHandle, f as CostLedgerSummary, m as CostReceipt } from "./cost-ledger-DbQdN3nO.js";
2
- import { _ as ProposalFinding } from "./types-B2NsbrNy.js";
3
- import { H as SurfaceProposer, R as Scenario, S as JudgeConfig, a as CampaignResult, d as DispatchContext, j as MutableSurface, k as LabeledScenarioStore, p as Gate } from "./types-DdFNuyxQ.js";
4
- import { o as InsightReport } from "./insight-report-BeT8KCgI.js";
5
- import { C as RunOptimizationOptions, Cn as CampaignStorage, E as RunEvalOptions, J as OptimizationMethodProvenance, S as PremeasuredOptimizationBaseline, W as OptimizationMethod, Y as OptimizationMethodResult, b as RunImprovementLoopResult, l as LoopProvenanceRecord } from "./provenance-oA4-zUqm.js";
2
+ import { An as CampaignCellRetryPolicy, C as RunImprovementLoopResult, E as RunOptimizationOptions, Ln as CampaignStorage, T as PremeasuredOptimizationBaseline, at as OptimizationMethod, f as LoopProvenanceRecord, k as RunEvalOptions, lt as OptimizationMethodProvenance, ut as OptimizationMethodResult, wt as SearchHistoryReceipt } from "./transient-failure-DKF5Mofa.js";
3
+ import { _ as ProposalFinding } from "./types-D9ssmxKL.js";
4
+ import { H as SurfaceProposer, R as Scenario, S as JudgeConfig, a as CampaignResult, d as DispatchContext, j as MutableSurface, k as LabeledScenarioStore, p as Gate } from "./types-D4s7Z6nq.js";
5
+ import { o as InsightReport } from "./insight-report-DRe8LB6d.js";
6
6
  import { o as PowerPreflight } from "./pareto-BqNW3LJR.js";
7
- import { n as HostedTenant } from "./client-kPQYT_56.js";
7
+ import { n as HostedTenant } from "./client-L9VVPkim.js";
8
8
  //#region src/contract/self-improve.d.ts
9
9
  interface SelfImproveBudget {
10
10
  /** Hard spend cap across the full run. Each paid call reserves its enforced
@@ -166,6 +166,13 @@ interface SelfImproveOptions<TScenario extends Scenario, TArtifact> {
166
166
  /** Per-cell agent dispatch deadline, applied to baseline, candidate, and
167
167
  * held-out campaigns. Default 600_000 ms. Set 0 to disable. */
168
168
  dispatchTimeoutMs?: number;
169
+ /** Bounded in-run retry of failed cells — same as
170
+ * `RunCampaignOptions.cellRetry`, applied to baseline, candidate, and
171
+ * held-out campaigns. Pair with `transientDispatchFailure()` so a
172
+ * transport hiccup (a router 503, a dropped stream) is re-dispatched in
173
+ * the same slot instead of leaving holdout coverage incomplete. Absent by
174
+ * default: a failed cell is final. */
175
+ cellRetry?: CampaignCellRetryPolicy;
169
176
  /** Streaming hook — fires on baseline + each generation + gate decision.
170
177
  * Consumer routes events wherever (UI, dashboard, logs). */
171
178
  onProgress?: (event: SelfImproveProgressEvent) => void;
@@ -225,6 +232,16 @@ interface SelfImproveOptions<TScenario extends Scenario, TArtifact> {
225
232
  * winner-selection and the ship-gate rank on the identical metric and cannot
226
233
  * invert. See `RunOptimizationOptions.selectionRankKey`. */
227
234
  selectionRankKey?: RunOptimizationOptions<TScenario, TArtifact>['selectionRankKey'];
235
+ /** Override which scored surface each generation MUTATES. Defaults to the
236
+ * global incumbent (hill-climb; the recorded lineage is a chain). Pass
237
+ * `crowdedFrontierParent({ seed })` or a custom policy to draw the parent
238
+ * from the Pareto frontier; promotion still compares against the incumbent.
239
+ * Proposer mode only. See `RunOptimizationOptions.selectParent`. */
240
+ selectParent?: RunOptimizationOptions<TScenario, TArtifact>['selectParent'];
241
+ /** Record this run's candidate search into a durable `SearchLedger` and
242
+ * return the bounded receipt on `searchHistory`. See
243
+ * `RunOptimizationOptions.searchLedger`. */
244
+ searchLedger?: RunOptimizationOptions<TScenario, TArtifact>['searchLedger'];
228
245
  }
229
246
  interface SelfImproveResult<TScenario extends Scenario, TArtifact> {
230
247
  /** Composite mean across all scenarios, baseline run. When
@@ -275,6 +292,9 @@ interface SelfImproveResult<TScenario extends Scenario, TArtifact> {
275
292
  /** Run-wide receipts across proposal, search, holdout, judging, analysis,
276
293
  * and promotion work, with phase and actor attribution. */
277
294
  receipts: CostReceipt[];
295
+ /** Bounded proof envelope over this run's canonical search ledger. Present
296
+ * only when `searchLedger` was supplied. */
297
+ searchHistory?: SearchHistoryReceipt;
278
298
  /** Exact external method and source identity, when `method` was used. */
279
299
  optimization?: {
280
300
  name: string;
@@ -384,4 +404,4 @@ interface DefinedAgentEval<TScenario extends Scenario, TArtifact> {
384
404
  declare function defineAgentEval<TScenario extends Scenario, TArtifact>(defaults: DefineAgentEvalOptions<TScenario, TArtifact>): DefinedAgentEval<TScenario, TArtifact>;
385
405
  //#endregion
386
406
  export { DefinedAgentEval as a, SelfImproveOptions as c, SelfImproveRunError as d, selfImprove as f, DefineAgentEvalOptions as i, SelfImproveProgressEvent as l, AgentEvalEvaluateOptions as n, defineAgentEval as o, AgentEvalImproveOptions as r, SelfImproveBudget as s, AgentEvalAgent as t, SelfImproveResult as u };
387
- //# sourceMappingURL=define-agent-eval-CEQWL9Hy.d.ts.map
407
+ //# sourceMappingURL=define-agent-eval-Dx1JnPEa.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"define-agent-eval-Dx1JnPEa.d.ts","names":[],"sources":["../src/contract/self-improve.ts","../src/contract/define-agent-eval.ts"],"mappings":";;;;;;;;UAiEiB;;;EAGf;;;;EAIA;;EAEA;;EAEA;;;EAGA;;;EAGA;;;;EAIA;;EAEA,mBAAmB;;;;;;;;;EASnB;;EAEA;;;;;EAKA;;KAGU;EACN;EAA0B;;EAC1B;EAA4B;EAAuB;;EACnD;EAA4B;EAAe;;EAC3C;EAA8B;EAAe;EAAuB;;EAGpE;EAAsB;EAAkB;;EACxC;EAAyB;EAAW;EAAY;EAAa;;UAElD,mBAAmB,kBAAkB,UAAU;;;;;;;;;;;;;;EAc9D,QAAQ,SAAS,gBAAgB,UAAU,WAAW,KAAK,oBAAoB,QAAQ;;;;;;;EAQvF;;;EAIA,WAAW;;EAGX,OAAO,YAAY,WAAW;;;EAI9B,iBAAiB;;EAGjB,SAAS;;;;;;;;;;;;EAaT,sBAAsB,gCAAgC,WAAW;;;;;EAMjE,WAAW,gBAAgB;;;;;;EAO3B,SAAS,mBAAmB,WAAW;;;EAIvC,qBAAqB;;;EAIrB,OAAO,KAAK,WAAW;;;;;;;;EASvB,cAAc,eAAe,gBAAgB,iBAAiB,mBAAmB;;;;EAKjF,UAAU;;;;EAKV;;;EAIA,gBAAgB,QAAQ;;;;;EAMxB,iBAAiB;IACf,UAAU;IACV;IACA;;;;EAKF;;;;;;;EAQA,YAAY;;;EAIZ,cAAc,OAAO;;;;EAKrB;EACA;EACA;;;;;;;;;;;;;;EAeA,eAAe;;;EAIf,eAAe;;;;EAKf,eAAe;;EAGf;;;;;;;;EASA;;;;;;;;;EAUA,oBAAoB,uBAAuB,WAAW;;;EAItD,WAAW;;;;;;;EAQX,mBAAmB,uBAAuB,WAAW;;;;;;EAOrD,eAAe,uBAAuB,WAAW;;;;EAKjD,eAAe,uBAAuB,WAAW;;UAGlC,kBAAkB,kBAAkB,UAAU;;;;EAI7D;IACE;IACA,aAAa;;;;;EAKf;IACE;IACA,aAAa;IACb,SAAS;;;IAGT;;;;IAIA;;;;;;EAMF;;;EAGA;;;;EAIA,YAAY;;EAEZ;;;EAGA;;EAEA;;EAEA;;EAEA,MAAM;;;EAGN,UAAU;;;EAGV,gBAAgB;;EAEhB;IACE;IACA,MAAM;IACN;IACA,aAAa;;;;;;;;EAQf,SAAS;;;;;EAKT,QAAQ;;;;;;EAMR,KAAK,yBAAyB,WAAW;;;cAI9B,4BAA4B;WAC9B,MAAM;WACN,UAAU;EAEnB,YAAY,gBAAgB,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAgMhB,YAAY,kBAAkB,UAAU,WAC5D,MAAM,mBAAmB,WAAW,aACnC,QAAQ,kBAAkB,WAAW;;;KChkB5B,eAAe,kBAAkB,UAAU,cACrD,SAAS,gBACT,UAAU,WACV,KAAK,oBACF,QAAQ;KAED,uBAAuB,kBAAkB,UAAU,aAAa,mBAC1E,WACA;UAGe,yBAAyB,kBAAkB,UAAU,mBAC5D,KACN,eAAe,WAAW;;EAI5B,YAAY;;EAEZ,UAAU;;EAEV,QAAQ,eAAe,WAAW;;EAElC,QAAQ,YAAY,WAAW;;EAE/B,SAAS,YAAY,WAAW;;EAEhC;;KAGU,wBAAwB,kBAAkB,UAAU,aAAa,KAC3E,QAAQ,mBAAmB,WAAW;EAGtC,SAAS,QAAQ;EACjB,eAAe,QAAQ;;UAGR,iBAAiB,kBAAkB,UAAU;;WAEnD,oBAAoB;;WAEpB,iBAAiB;;;;;EAK1B,SACE,OAAO,yBAAyB,WAAW,aAC1C,QAAQ,eAAe,WAAW;;;;;;;EAOrC,QACE,OAAO,wBAAwB,WAAW,aACzC,QAAQ,kBAAkB,WAAW;;;;;;;;;iBAU1B,gBAAgB,kBAAkB,UAAU,WAC1D,UAAU,uBAAuB,WAAW,aAC3C,iBAAiB,WAAW"}
@@ -6,14 +6,15 @@ import { i as requiredPairedSampleSize, r as pairedMde } from "./power-and-mde-C
6
6
  import { r as pairRunRecords } from "./paired-arms-D-XRF_fy.js";
7
7
  import { o as pairedTTest, r as pairedBootstrap } from "./paired-tests-BHIhYVdu.js";
8
8
  import { r as welchsTTest } from "./baseline-BhPRQBVn.js";
9
+ import "./query-CHmMP42p.js";
9
10
  import { r as observedSplitScore } from "./reward-nw2xZGZG.js";
10
- import { c as validateRunRecord, i as modelHasSnapshot } from "./run-record-D2lDdSAz.js";
11
- import { r as paretoChart } from "./summary-report-DW2bEpdB.js";
12
- import { $ as runEval, A as surfaceHash, f as runImprovementLoop, k as surfaceContentHash, nt as resolveRunDir, o as emitLoopProvenance, q as defaultProductionGate, s as loopProvenanceArgsFromResult, y as assertOptimizationResult } from "./llm-judge-BqqMS8t7.js";
13
- import { c as campaignCellTaskScore, l as campaignCellToRunRecord, o as campaignCellExecutionEvidence, s as campaignCellJudgeDimensions } from "./reward-hacking-C0x0xihA.js";
14
- import { S as inMemoryCampaignStorage, b as createRunCostLedger, x as fsCampaignStorage } from "./external-optimizer-subprocess-CKNb42oM.js";
11
+ import { c as validateRunRecord, i as modelHasSnapshot } from "./run-record-BC0ebuRP.js";
12
+ import { r as paretoChart } from "./summary-report-BI5hUtvK.js";
13
+ import { N as surfaceContentHash, P as surfaceHash, Y as defaultProductionGate, c as emitLoopProvenance, it as resolveRunDir, l as loopProvenanceArgsFromResult, m as runImprovementLoop, tt as runEval, w as assertOptimizationResult } from "./llm-judge-BhasIPFT.js";
14
+ import { c as campaignCellTaskScore, l as campaignCellToRunRecord, o as campaignCellExecutionEvidence, s as campaignCellJudgeDimensions } from "./reward-hacking-t4lB1yt8.js";
15
+ import { S as inMemoryCampaignStorage, b as createRunCostLedger, x as fsCampaignStorage } from "./external-optimizer-subprocess-BIWbHpgD.js";
15
16
  import { t as powerPreflight } from "./power-preflight-DEw-uC7q.js";
16
- import { t as createHostedClient } from "./client-LIuo-KPv.js";
17
+ import { t as createHostedClient } from "./client-CX7KqIdB.js";
17
18
  //#region src/contamination-guard.ts
18
19
  function checkCanaries(output, scenarios) {
19
20
  const leaks = [];
@@ -987,7 +988,7 @@ function assertSelfImproveSearchMode(opts) {
987
988
  const budget = opts.budget;
988
989
  if (budget?.generations !== void 0 && budget.generations !== 1) throw new Error("selfImprove: method owns its rounds; budget.generations must be 1 when set");
989
990
  if (budget?.populationSize !== void 0 && budget.populationSize !== 1) throw new Error("selfImprove: method owns its candidates; budget.populationSize must be 1 when set");
990
- if (budget?.candidateConcurrency !== void 0 || budget?.maxImprovementShots !== void 0 || opts.analyzeGeneration !== void 0 || opts.findings !== void 0) throw new Error("selfImprove: candidateConcurrency, maxImprovementShots, analyzeGeneration, and findings apply only to proposer mode");
991
+ if (budget?.candidateConcurrency !== void 0 || budget?.maxImprovementShots !== void 0 || opts.analyzeGeneration !== void 0 || opts.findings !== void 0 || opts.selectParent !== void 0) throw new Error("selfImprove: candidateConcurrency, maxImprovementShots, analyzeGeneration, findings, and selectParent apply only to proposer mode");
991
992
  }
992
993
  function splitMethodPartitions(searchScenarios, explicitSelection, fraction) {
993
994
  if (!Number.isFinite(fraction) || fraction <= 0 || fraction >= 1) throw new Error("selfImprove: budget.selectionFraction must be in (0, 1)");
@@ -1147,6 +1148,7 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
1147
1148
  maxConcurrency,
1148
1149
  reps: budget.reps,
1149
1150
  dispatchTimeoutMs: opts.dispatchTimeoutMs,
1151
+ cellRetry: opts.cellRetry,
1150
1152
  expectUsage,
1151
1153
  costCeiling: budget.dollars
1152
1154
  }),
@@ -1196,13 +1198,16 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
1196
1198
  maxConcurrency,
1197
1199
  cellPlacement: opts.cellPlacement,
1198
1200
  dispatchTimeoutMs: opts.dispatchTimeoutMs,
1201
+ cellRetry: opts.cellRetry,
1199
1202
  costLedger,
1200
1203
  expectUsage,
1201
1204
  labeledStore: opts.labeledStore,
1202
1205
  captureSource: opts.captureSource,
1203
1206
  analyzeGeneration: opts.analyzeGeneration,
1204
1207
  findings: opts.findings,
1205
- selectionRankKey: opts.selectionRankKey
1208
+ selectionRankKey: opts.selectionRankKey,
1209
+ selectParent: opts.selectParent,
1210
+ searchLedger: opts.searchLedger
1206
1211
  });
1207
1212
  const reportSplit = holdoutDeferred ? "search" : "holdout";
1208
1213
  const reportBaselineCampaign = holdoutDeferred ? result.baselineCampaign : result.baselineOnHoldout;
@@ -1292,6 +1297,7 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
1292
1297
  ...optimizationResult.durationMs === void 0 ? {} : { durationMs: optimizationResult.durationMs },
1293
1298
  ...optimizationResult.provenance === void 0 ? {} : { provenance: structuredClone(optimizationResult.provenance) }
1294
1299
  } } : {},
1300
+ ...result.searchHistory ? { searchHistory: result.searchHistory } : {},
1295
1301
  insight,
1296
1302
  ...power ? { power } : {},
1297
1303
  raw: result
@@ -1491,4 +1497,4 @@ function requirePositiveInteger(value, field) {
1491
1497
  //#endregion
1492
1498
  export { summarizeExecution as a, analyzeRuns as i, SelfImproveRunError as n, checkCanaries as o, selfImprove as r, defineAgentEval as t };
1493
1499
 
1494
- //# sourceMappingURL=define-agent-eval-C8V8sMqP.js.map
1500
+ //# sourceMappingURL=define-agent-eval-h-s-sI-v.js.map