@tangle-network/agent-eval 0.129.0 → 0.130.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (428) hide show
  1. package/CHANGELOG.md +21 -0
  2. package/README.md +2 -0
  3. package/dist/active-curriculum-C4mk67HP.js +214 -0
  4. package/dist/active-curriculum-C4mk67HP.js.map +1 -0
  5. package/dist/adversarial-smnADNFS.d.ts +21 -0
  6. package/dist/adversarial-smnADNFS.d.ts.map +1 -0
  7. package/dist/analyst/index.d.ts +81 -2872
  8. package/dist/analyst/index.d.ts.map +1 -0
  9. package/dist/analyst/index.js +319 -360
  10. package/dist/analyst/index.js.map +1 -1
  11. package/dist/analyst-BkTS3C58.d.ts +89 -0
  12. package/dist/analyst-BkTS3C58.d.ts.map +1 -0
  13. package/dist/analyst-LsnNpSkm.js +152 -0
  14. package/dist/analyst-LsnNpSkm.js.map +1 -0
  15. package/dist/analyze-runs-C1CavBMk.js +1067 -0
  16. package/dist/analyze-runs-C1CavBMk.js.map +1 -0
  17. package/dist/analyze-runs-FsgYCinh.d.ts +72 -0
  18. package/dist/analyze-runs-FsgYCinh.d.ts.map +1 -0
  19. package/dist/attribute-vocabulary-DLJ6303h.d.ts +54 -0
  20. package/dist/attribute-vocabulary-DLJ6303h.d.ts.map +1 -0
  21. package/dist/authenticity/index.d.ts +81 -79
  22. package/dist/authenticity/index.d.ts.map +1 -0
  23. package/dist/authenticity/index.js +209 -193
  24. package/dist/authenticity/index.js.map +1 -1
  25. package/dist/baseline-HsBvw_dk.js +550 -0
  26. package/dist/baseline-HsBvw_dk.js.map +1 -0
  27. package/dist/baseline-hG3K85h4.d.ts +125 -0
  28. package/dist/baseline-hG3K85h4.d.ts.map +1 -0
  29. package/dist/belief-state/index.d.ts +448 -1188
  30. package/dist/belief-state/index.d.ts.map +1 -0
  31. package/dist/belief-state/index.js +1617 -1709
  32. package/dist/belief-state/index.js.map +1 -1
  33. package/dist/benchmarks/index.d.ts +2 -891
  34. package/dist/benchmarks/index.js +2 -60
  35. package/dist/benchmarks-BJgDGkAD.js +754 -0
  36. package/dist/benchmarks-BJgDGkAD.js.map +1 -0
  37. package/dist/builder-eval/index.d.ts +150 -662
  38. package/dist/builder-eval/index.d.ts.map +1 -0
  39. package/dist/builder-eval/index.js +356 -345
  40. package/dist/builder-eval/index.js.map +1 -1
  41. package/dist/calibration-CNWWA6K8.js +94 -0
  42. package/dist/calibration-CNWWA6K8.js.map +1 -0
  43. package/dist/campaign/index.d.ts +5 -6381
  44. package/dist/campaign/index.js +3 -213
  45. package/dist/campaign-aKJt6emI.js +3886 -0
  46. package/dist/campaign-aKJt6emI.js.map +1 -0
  47. package/dist/cli.d.ts +1 -1
  48. package/dist/cli.js +164 -175
  49. package/dist/cli.js.map +1 -1
  50. package/dist/client-C97NMzqi.d.ts +581 -0
  51. package/dist/client-C97NMzqi.d.ts.map +1 -0
  52. package/dist/client-CYzbdJOZ.js +637 -0
  53. package/dist/client-CYzbdJOZ.js.map +1 -0
  54. package/dist/code-agent-session-BjkMTQ7H.js +1390 -0
  55. package/dist/code-agent-session-BjkMTQ7H.js.map +1 -0
  56. package/dist/code-agent-session-DqqgOJaz.d.ts +143 -0
  57. package/dist/code-agent-session-DqqgOJaz.d.ts.map +1 -0
  58. package/dist/concurrency-DIxRZF_J.js +85 -0
  59. package/dist/concurrency-DIxRZF_J.js.map +1 -0
  60. package/dist/contract/index.d.ts +645 -5565
  61. package/dist/contract/index.d.ts.map +1 -0
  62. package/dist/contract/index.js +1730 -1938
  63. package/dist/contract/index.js.map +1 -1
  64. package/dist/control.d.ts +3 -1030
  65. package/dist/control.js +2 -33
  66. package/dist/cost-ledger-DIgQUFZZ.js +801 -0
  67. package/dist/cost-ledger-DIgQUFZZ.js.map +1 -0
  68. package/dist/cost-ledger-Dye6jCgg.d.ts +236 -0
  69. package/dist/cost-ledger-Dye6jCgg.d.ts.map +1 -0
  70. package/dist/dataset-BvtnC8Dc.d.ts +115 -0
  71. package/dist/dataset-BvtnC8Dc.d.ts.map +1 -0
  72. package/dist/default-registry-C-vFCSEc.js +2579 -0
  73. package/dist/default-registry-C-vFCSEc.js.map +1 -0
  74. package/dist/default-registry-CNPo-Vsb.d.ts +540 -0
  75. package/dist/default-registry-CNPo-Vsb.d.ts.map +1 -0
  76. package/dist/emitter-CPBAhxum.js +266 -0
  77. package/dist/emitter-CPBAhxum.js.map +1 -0
  78. package/dist/emitter-DGQGoLyj.d.ts +113 -0
  79. package/dist/emitter-DGQGoLyj.d.ts.map +1 -0
  80. package/dist/errors-8YnH8WlF.js +64 -0
  81. package/dist/errors-8YnH8WlF.js.map +1 -0
  82. package/dist/errors-CEk209JS.d.ts +76 -0
  83. package/dist/errors-CEk209JS.d.ts.map +1 -0
  84. package/dist/eval-campaign-DEm6c8ru.js +349 -0
  85. package/dist/eval-campaign-DEm6c8ru.js.map +1 -0
  86. package/dist/execution-tracks-CpgFPpS5.js +93 -0
  87. package/dist/execution-tracks-CpgFPpS5.js.map +1 -0
  88. package/dist/exporters-q9iL-2Jf.js +148 -0
  89. package/dist/exporters-q9iL-2Jf.js.map +1 -0
  90. package/dist/extract-usage-BrQ8mCLX.js +155 -0
  91. package/dist/extract-usage-BrQ8mCLX.js.map +1 -0
  92. package/dist/failure-cluster-CqcvCcdR.d.ts +59 -0
  93. package/dist/failure-cluster-CqcvCcdR.d.ts.map +1 -0
  94. package/dist/feedback-trajectory-CVaeREXV.d.ts +340 -0
  95. package/dist/feedback-trajectory-CVaeREXV.d.ts.map +1 -0
  96. package/dist/fuzz.d.ts +320 -646
  97. package/dist/fuzz.d.ts.map +1 -0
  98. package/dist/fuzz.js +670 -618
  99. package/dist/fuzz.js.map +1 -1
  100. package/dist/hf-dataset-DBJXXoY1.js +763 -0
  101. package/dist/hf-dataset-DBJXXoY1.js.map +1 -0
  102. package/dist/hosted/index.d.ts +11 -831
  103. package/dist/hosted/index.d.ts.map +1 -0
  104. package/dist/hosted/index.js +2 -37
  105. package/dist/index-2JJSA6-r2.d.ts +926 -0
  106. package/dist/index-2JJSA6-r2.d.ts.map +1 -0
  107. package/dist/index-6N0aYmpW.d.ts +217 -0
  108. package/dist/index-6N0aYmpW.d.ts.map +1 -0
  109. package/dist/index-BAvgST_9.d.ts +131 -0
  110. package/dist/index-BAvgST_9.d.ts.map +1 -0
  111. package/dist/index-BvnJuTGD.d.ts +68 -0
  112. package/dist/index-BvnJuTGD.d.ts.map +1 -0
  113. package/dist/index-C61Wi7yg.d.ts +547 -0
  114. package/dist/index-C61Wi7yg.d.ts.map +1 -0
  115. package/dist/index-CD_WZ_Xr.d.ts +2250 -0
  116. package/dist/index-CD_WZ_Xr.d.ts.map +1 -0
  117. package/dist/index-DSC51roc.d.ts +102 -0
  118. package/dist/index-DSC51roc.d.ts.map +1 -0
  119. package/dist/index-Em67JBjs.d.ts +335 -0
  120. package/dist/index-Em67JBjs.d.ts.map +1 -0
  121. package/dist/index.d.ts +3755 -15555
  122. package/dist/index.d.ts.map +1 -0
  123. package/dist/index.js +11182 -11216
  124. package/dist/index.js.map +1 -1
  125. package/dist/integrity-BzRbCHzi.js +424 -0
  126. package/dist/integrity-BzRbCHzi.js.map +1 -0
  127. package/dist/integrity-rmVhXWA7.d.ts +61 -0
  128. package/dist/integrity-rmVhXWA7.d.ts.map +1 -0
  129. package/dist/judge-calibration-DFtEMlde.d.ts +146 -0
  130. package/dist/judge-calibration-DFtEMlde.d.ts.map +1 -0
  131. package/dist/ledger-core/index.d.ts +2 -0
  132. package/dist/ledger-core/index.js +2 -0
  133. package/dist/ledger-core-DtZz1RG0.js +388 -0
  134. package/dist/ledger-core-DtZz1RG0.js.map +1 -0
  135. package/dist/llm-client--GR4JbZE.js +687 -0
  136. package/dist/llm-client--GR4JbZE.js.map +1 -0
  137. package/dist/llm-client-B_nIBlYo.d.ts +290 -0
  138. package/dist/llm-client-B_nIBlYo.d.ts.map +1 -0
  139. package/dist/matrix/index.d.ts +3 -155
  140. package/dist/matrix/index.js +2 -8
  141. package/dist/matrix-BzQnu2S6.js +270 -0
  142. package/dist/matrix-BzQnu2S6.js.map +1 -0
  143. package/dist/meta-eval/index.d.ts +4 -1027
  144. package/dist/meta-eval/index.js +393 -390
  145. package/dist/meta-eval/index.js.map +1 -1
  146. package/dist/metrics-C9YY1OcL.js +239 -0
  147. package/dist/metrics-C9YY1OcL.js.map +1 -0
  148. package/dist/mint-yN2M2eh0.js +201 -0
  149. package/dist/mint-yN2M2eh0.js.map +1 -0
  150. package/dist/multi-layer-verifier-BHY1gWAc.d.ts +138 -0
  151. package/dist/multi-layer-verifier-BHY1gWAc.d.ts.map +1 -0
  152. package/dist/multishot/index.d.ts +273 -480
  153. package/dist/multishot/index.d.ts.map +1 -0
  154. package/dist/multishot/index.js +595 -548
  155. package/dist/multishot/index.js.map +1 -1
  156. package/dist/off-policy-DvgzvtIx.js +220 -0
  157. package/dist/off-policy-DvgzvtIx.js.map +1 -0
  158. package/dist/off-policy-mskQw8Mb.d.ts +153 -0
  159. package/dist/off-policy-mskQw8Mb.d.ts.map +1 -0
  160. package/dist/openapi.json +1 -1
  161. package/dist/opencode-sqlite-BGrHeDu3.js +318 -0
  162. package/dist/opencode-sqlite-BGrHeDu3.js.map +1 -0
  163. package/dist/outcome-store-BYHIuO0e.d.ts +65 -0
  164. package/dist/outcome-store-BYHIuO0e.d.ts.map +1 -0
  165. package/dist/outcome-store-ChBKlTd_.js +75 -0
  166. package/dist/outcome-store-ChBKlTd_.js.map +1 -0
  167. package/dist/paired-arms-D9D0wXj2.js +260 -0
  168. package/dist/paired-arms-D9D0wXj2.js.map +1 -0
  169. package/dist/pipelines/index.d.ts +95 -532
  170. package/dist/pipelines/index.d.ts.map +1 -0
  171. package/dist/pipelines/index.js +497 -478
  172. package/dist/pipelines/index.js.map +1 -1
  173. package/dist/pre-registration-DakwTRXk.js +96 -0
  174. package/dist/pre-registration-DakwTRXk.js.map +1 -0
  175. package/dist/propose-review-control-LhIWGzHl.js +1458 -0
  176. package/dist/propose-review-control-LhIWGzHl.js.map +1 -0
  177. package/dist/query-CJ_DX8vl.d.ts +28 -0
  178. package/dist/query-CJ_DX8vl.d.ts.map +1 -0
  179. package/dist/query-Di7eEQ79.js +83 -0
  180. package/dist/query-Di7eEQ79.js.map +1 -0
  181. package/dist/raw-provider-sink-BQd7mzyT.js +201 -0
  182. package/dist/raw-provider-sink-BQd7mzyT.js.map +1 -0
  183. package/dist/raw-provider-sink-BU29Sh8h.d.ts +134 -0
  184. package/dist/raw-provider-sink-BU29Sh8h.d.ts.map +1 -0
  185. package/dist/redact-7Aq1ukl-.js +107 -0
  186. package/dist/redact-7Aq1ukl-.js.map +1 -0
  187. package/dist/release-report-Cz9NKH39.js +603 -0
  188. package/dist/release-report-Cz9NKH39.js.map +1 -0
  189. package/dist/release-report-mvB2G4_J.d.ts +244 -0
  190. package/dist/release-report-mvB2G4_J.d.ts.map +1 -0
  191. package/dist/replay-GnyotH0J.js +1741 -0
  192. package/dist/replay-GnyotH0J.js.map +1 -0
  193. package/dist/replay-OoidtG1E.d.ts +749 -0
  194. package/dist/replay-OoidtG1E.d.ts.map +1 -0
  195. package/dist/reporting.d.ts +6 -1312
  196. package/dist/reporting.js +6 -51
  197. package/dist/researcher-CwTdwXG1.d.ts +314 -0
  198. package/dist/researcher-CwTdwXG1.d.ts.map +1 -0
  199. package/dist/reward-hacking-eAnOsynk.d.ts +224 -0
  200. package/dist/reward-hacking-eAnOsynk.d.ts.map +1 -0
  201. package/dist/reward-hacking-qipEpKvY.js +596 -0
  202. package/dist/reward-hacking-qipEpKvY.js.map +1 -0
  203. package/dist/reward-nw2xZGZG.js +137 -0
  204. package/dist/reward-nw2xZGZG.js.map +1 -0
  205. package/dist/rl.d.ts +760 -4010
  206. package/dist/rl.d.ts.map +1 -0
  207. package/dist/rl.js +2325 -1958
  208. package/dist/rl.js.map +1 -1
  209. package/dist/rolldown-runtime-8H4AJuhK.js +14 -0
  210. package/dist/rollout/index.d.ts +3 -2087
  211. package/dist/rollout/index.js +8 -168
  212. package/dist/rollout-BOYjemfR.js +624 -0
  213. package/dist/rollout-BOYjemfR.js.map +1 -0
  214. package/dist/rubric-predictive-validity-B3xmbmS1.js +141 -0
  215. package/dist/rubric-predictive-validity-B3xmbmS1.js.map +1 -0
  216. package/dist/rubric-predictive-validity-Ku_clp_1.d.ts +74 -0
  217. package/dist/rubric-predictive-validity-Ku_clp_1.d.ts.map +1 -0
  218. package/dist/run-evidence-ClRX_8A9.d.ts +225 -0
  219. package/dist/run-evidence-ClRX_8A9.d.ts.map +1 -0
  220. package/dist/run-record-BuoE80Dq.js +467 -0
  221. package/dist/{chunk-56TAVBOK.js.map → run-record-BuoE80Dq.js.map} +1 -1
  222. package/dist/run-record-CnZu_gjl.d.ts +357 -0
  223. package/dist/run-record-CnZu_gjl.d.ts.map +1 -0
  224. package/dist/run-score-iEEAWiBY.js +41 -0
  225. package/dist/run-score-iEEAWiBY.js.map +1 -0
  226. package/dist/runtime-trajectory-1gyaTOoC.js +93 -0
  227. package/dist/runtime-trajectory-1gyaTOoC.js.map +1 -0
  228. package/dist/runtime-trajectory-BvSZcCHD.d.ts +50 -0
  229. package/dist/runtime-trajectory-BvSZcCHD.d.ts.map +1 -0
  230. package/dist/schema-BtVldJ3T.d.ts +206 -0
  231. package/dist/schema-BtVldJ3T.d.ts.map +1 -0
  232. package/dist/schema-C6DW4ZHR.js +821 -0
  233. package/dist/schema-C6DW4ZHR.js.map +1 -0
  234. package/dist/schema-CRhEY1SO.js +69 -0
  235. package/dist/schema-CRhEY1SO.js.map +1 -0
  236. package/dist/schema-Cef2cFmb.d.ts +408 -0
  237. package/dist/schema-Cef2cFmb.d.ts.map +1 -0
  238. package/dist/semantic-concept-judge-B6cWNJ2K.js +725 -0
  239. package/dist/semantic-concept-judge-B6cWNJ2K.js.map +1 -0
  240. package/dist/sequential-Br0mAPHA.js +148 -0
  241. package/dist/sequential-Br0mAPHA.js.map +1 -0
  242. package/dist/sequential-CYwq6Ff_.d.ts +141 -0
  243. package/dist/sequential-CYwq6Ff_.d.ts.map +1 -0
  244. package/dist/series-convergence-CjO2QdRW.js +43 -0
  245. package/dist/series-convergence-CjO2QdRW.js.map +1 -0
  246. package/dist/series-convergence-ofsqPWhs.d.ts +35 -0
  247. package/dist/series-convergence-ofsqPWhs.d.ts.map +1 -0
  248. package/dist/server-m5D9cvnG.js +1040 -0
  249. package/dist/server-m5D9cvnG.js.map +1 -0
  250. package/dist/skill-usage-C_pXm7lP.d.ts +534 -0
  251. package/dist/skill-usage-C_pXm7lP.d.ts.map +1 -0
  252. package/dist/skillopt-optimization-method-CUmHkGbI.js +7718 -0
  253. package/dist/skillopt-optimization-method-CUmHkGbI.js.map +1 -0
  254. package/dist/skillopt-optimization-method-CWKVTnks.d.ts +1740 -0
  255. package/dist/skillopt-optimization-method-CWKVTnks.d.ts.map +1 -0
  256. package/dist/statistics-Cmj6nynr.d.ts +514 -0
  257. package/dist/statistics-Cmj6nynr.d.ts.map +1 -0
  258. package/dist/statistics-CnnxdpOg.js +1437 -0
  259. package/dist/statistics-CnnxdpOg.js.map +1 -0
  260. package/dist/store-CT9YIIve.d.ts +117 -0
  261. package/dist/store-CT9YIIve.d.ts.map +1 -0
  262. package/dist/store-CxJry_cs.d.ts +229 -0
  263. package/dist/store-CxJry_cs.d.ts.map +1 -0
  264. package/dist/storyboard/index.d.ts +84 -203
  265. package/dist/storyboard/index.d.ts.map +1 -0
  266. package/dist/storyboard/index.js +609 -542
  267. package/dist/storyboard/index.js.map +1 -1
  268. package/dist/summary-report-BNs5nmXI.js +862 -0
  269. package/dist/summary-report-BNs5nmXI.js.map +1 -0
  270. package/dist/summary-report-Cj9gdw4i.d.ts +407 -0
  271. package/dist/summary-report-Cj9gdw4i.d.ts.map +1 -0
  272. package/dist/supervisor-run/index.d.ts +2 -959
  273. package/dist/supervisor-run/index.js +2 -65
  274. package/dist/supervisor-run-_lnTLM3z.js +1679 -0
  275. package/dist/supervisor-run-_lnTLM3z.js.map +1 -0
  276. package/dist/task-failure-attributes-CQZlB3et.js +311 -0
  277. package/dist/task-failure-attributes-CQZlB3et.js.map +1 -0
  278. package/dist/test-graded-scenario-BsqWLmPt.js +318 -0
  279. package/dist/test-graded-scenario-BsqWLmPt.js.map +1 -0
  280. package/dist/test-graded-scenario-D1TaI2va.d.ts +141 -0
  281. package/dist/test-graded-scenario-D1TaI2va.d.ts.map +1 -0
  282. package/dist/tools-BmuN627J.js +1085 -0
  283. package/dist/tools-BmuN627J.js.map +1 -0
  284. package/dist/trace-attributes.d.ts +2 -52
  285. package/dist/trace-attributes.js +131 -61
  286. package/dist/trace-attributes.js.map +1 -1
  287. package/dist/traces.d.ts +12 -2365
  288. package/dist/traces.js +12 -252
  289. package/dist/trajectory-D_7rLrvE.js +56 -0
  290. package/dist/trajectory-D_7rLrvE.js.map +1 -0
  291. package/dist/types-DGsxbAEd.d.ts +387 -0
  292. package/dist/types-DGsxbAEd.d.ts.map +1 -0
  293. package/dist/types-k9tZGKUg.d.ts +640 -0
  294. package/dist/types-k9tZGKUg.d.ts.map +1 -0
  295. package/dist/verdict-Dps8_okt.d.ts +37 -0
  296. package/dist/verdict-Dps8_okt.d.ts.map +1 -0
  297. package/dist/wire/index.d.ts +702 -1173
  298. package/dist/wire/index.d.ts.map +1 -0
  299. package/dist/wire/index.js +2 -81
  300. package/docs/campaign-proposers.md +1 -0
  301. package/package.json +17 -9
  302. package/dist/benchmarks/index.js.map +0 -1
  303. package/dist/campaign/index.js.map +0 -1
  304. package/dist/chunk-2QU3YOPR.js +0 -7374
  305. package/dist/chunk-2QU3YOPR.js.map +0 -1
  306. package/dist/chunk-3OCR4R5I.js +0 -728
  307. package/dist/chunk-3OCR4R5I.js.map +0 -1
  308. package/dist/chunk-3RF76KTD.js +0 -84
  309. package/dist/chunk-3RF76KTD.js.map +0 -1
  310. package/dist/chunk-56TAVBOK.js +0 -698
  311. package/dist/chunk-5DTSBUL2.js +0 -159
  312. package/dist/chunk-5DTSBUL2.js.map +0 -1
  313. package/dist/chunk-7FO3TNPI.js +0 -232
  314. package/dist/chunk-7FO3TNPI.js.map +0 -1
  315. package/dist/chunk-7ZZMD7UK.js +0 -386
  316. package/dist/chunk-7ZZMD7UK.js.map +0 -1
  317. package/dist/chunk-BOD4O7OF.js +0 -40
  318. package/dist/chunk-BOD4O7OF.js.map +0 -1
  319. package/dist/chunk-BSO5JDQH.js +0 -2335
  320. package/dist/chunk-BSO5JDQH.js.map +0 -1
  321. package/dist/chunk-C6LXANRU.js +0 -1550
  322. package/dist/chunk-C6LXANRU.js.map +0 -1
  323. package/dist/chunk-DODXQREJ.js +0 -752
  324. package/dist/chunk-DODXQREJ.js.map +0 -1
  325. package/dist/chunk-DRYIUNWY.js +0 -622
  326. package/dist/chunk-DRYIUNWY.js.map +0 -1
  327. package/dist/chunk-E7QXT7SX.js +0 -183
  328. package/dist/chunk-E7QXT7SX.js.map +0 -1
  329. package/dist/chunk-EG66UGL4.js +0 -341
  330. package/dist/chunk-EG66UGL4.js.map +0 -1
  331. package/dist/chunk-FXTVJPYD.js +0 -576
  332. package/dist/chunk-FXTVJPYD.js.map +0 -1
  333. package/dist/chunk-G7MGMCZD.js +0 -153
  334. package/dist/chunk-G7MGMCZD.js.map +0 -1
  335. package/dist/chunk-GGE4NNQT.js +0 -65
  336. package/dist/chunk-GGE4NNQT.js.map +0 -1
  337. package/dist/chunk-H23X7XKK.js +0 -181
  338. package/dist/chunk-H23X7XKK.js.map +0 -1
  339. package/dist/chunk-HHWE3POT.js +0 -94
  340. package/dist/chunk-HHWE3POT.js.map +0 -1
  341. package/dist/chunk-HPWUNB47.js +0 -289
  342. package/dist/chunk-HPWUNB47.js.map +0 -1
  343. package/dist/chunk-IYCLP2N2.js +0 -766
  344. package/dist/chunk-IYCLP2N2.js.map +0 -1
  345. package/dist/chunk-JHCHEVET.js +0 -274
  346. package/dist/chunk-JHCHEVET.js.map +0 -1
  347. package/dist/chunk-JQSF5DQT.js +0 -701
  348. package/dist/chunk-JQSF5DQT.js.map +0 -1
  349. package/dist/chunk-K4DBDHLK.js +0 -158
  350. package/dist/chunk-K4DBDHLK.js.map +0 -1
  351. package/dist/chunk-K6N6XJJX.js +0 -306
  352. package/dist/chunk-K6N6XJJX.js.map +0 -1
  353. package/dist/chunk-M4YBQKIJ.js +0 -1040
  354. package/dist/chunk-M4YBQKIJ.js.map +0 -1
  355. package/dist/chunk-MA6HLL3S.js +0 -65
  356. package/dist/chunk-MA6HLL3S.js.map +0 -1
  357. package/dist/chunk-MAZ26DC7.js +0 -99
  358. package/dist/chunk-MAZ26DC7.js.map +0 -1
  359. package/dist/chunk-NPCTHQIO.js +0 -91
  360. package/dist/chunk-NPCTHQIO.js.map +0 -1
  361. package/dist/chunk-NY44NC4A.js +0 -1056
  362. package/dist/chunk-NY44NC4A.js.map +0 -1
  363. package/dist/chunk-OIUOT4QD.js +0 -44
  364. package/dist/chunk-OIUOT4QD.js.map +0 -1
  365. package/dist/chunk-ONWEPEDO.js +0 -57
  366. package/dist/chunk-ONWEPEDO.js.map +0 -1
  367. package/dist/chunk-OWN5NPMC.js +0 -152
  368. package/dist/chunk-OWN5NPMC.js.map +0 -1
  369. package/dist/chunk-P6FYH6K4.js +0 -1161
  370. package/dist/chunk-P6FYH6K4.js.map +0 -1
  371. package/dist/chunk-PC4UYEBM.js +0 -166
  372. package/dist/chunk-PC4UYEBM.js.map +0 -1
  373. package/dist/chunk-PC5DOSM7.js +0 -579
  374. package/dist/chunk-PC5DOSM7.js.map +0 -1
  375. package/dist/chunk-PXE2VKMX.js +0 -140
  376. package/dist/chunk-PXE2VKMX.js.map +0 -1
  377. package/dist/chunk-PZ5AY32C.js +0 -10
  378. package/dist/chunk-PZ5AY32C.js.map +0 -1
  379. package/dist/chunk-QB6BDBP2.js +0 -4464
  380. package/dist/chunk-QB6BDBP2.js.map +0 -1
  381. package/dist/chunk-RXHCETDZ.js +0 -536
  382. package/dist/chunk-RXHCETDZ.js.map +0 -1
  383. package/dist/chunk-RZTMDUO7.js +0 -49
  384. package/dist/chunk-RZTMDUO7.js.map +0 -1
  385. package/dist/chunk-SFLLL76A.js +0 -669
  386. package/dist/chunk-SFLLL76A.js.map +0 -1
  387. package/dist/chunk-SZLVEKMJ.js +0 -1446
  388. package/dist/chunk-SZLVEKMJ.js.map +0 -1
  389. package/dist/chunk-T4SQEITX.js +0 -95
  390. package/dist/chunk-T4SQEITX.js.map +0 -1
  391. package/dist/chunk-T6RLYGAD.js +0 -158
  392. package/dist/chunk-T6RLYGAD.js.map +0 -1
  393. package/dist/chunk-TJVT4QFF.js +0 -911
  394. package/dist/chunk-TJVT4QFF.js.map +0 -1
  395. package/dist/chunk-TQ7LNKZ3.js +0 -136
  396. package/dist/chunk-TQ7LNKZ3.js.map +0 -1
  397. package/dist/chunk-U4L7JRPZ.js +0 -1706
  398. package/dist/chunk-U4L7JRPZ.js.map +0 -1
  399. package/dist/chunk-U4PHLT2N.js +0 -419
  400. package/dist/chunk-U4PHLT2N.js.map +0 -1
  401. package/dist/chunk-VCZ5FQYW.js +0 -928
  402. package/dist/chunk-VCZ5FQYW.js.map +0 -1
  403. package/dist/chunk-VI2UW6B6.js +0 -162
  404. package/dist/chunk-VI2UW6B6.js.map +0 -1
  405. package/dist/chunk-VQMK5FMP.js +0 -247
  406. package/dist/chunk-VQMK5FMP.js.map +0 -1
  407. package/dist/chunk-WGXIEX7P.js +0 -116
  408. package/dist/chunk-WGXIEX7P.js.map +0 -1
  409. package/dist/chunk-WVATSFCP.js +0 -1553
  410. package/dist/chunk-WVATSFCP.js.map +0 -1
  411. package/dist/chunk-X4YIBDER.js +0 -1662
  412. package/dist/chunk-X4YIBDER.js.map +0 -1
  413. package/dist/chunk-YQN4ICPP.js +0 -355
  414. package/dist/chunk-YQN4ICPP.js.map +0 -1
  415. package/dist/chunk-ZET2UAYW.js +0 -89
  416. package/dist/chunk-ZET2UAYW.js.map +0 -1
  417. package/dist/chunk-ZHTZ4EYI.js +0 -1212
  418. package/dist/chunk-ZHTZ4EYI.js.map +0 -1
  419. package/dist/control.js.map +0 -1
  420. package/dist/hosted/index.js.map +0 -1
  421. package/dist/matrix/index.js.map +0 -1
  422. package/dist/reporting.js.map +0 -1
  423. package/dist/rollout/index.js.map +0 -1
  424. package/dist/run-campaign-OJJ7CZF4.js +0 -18
  425. package/dist/run-campaign-OJJ7CZF4.js.map +0 -1
  426. package/dist/supervisor-run/index.js.map +0 -1
  427. package/dist/traces.js.map +0 -1
  428. package/dist/wire/index.js.map +0 -1
@@ -1,4464 +0,0 @@
1
- import {
2
- JudgeParseError,
3
- assertCodeSurfaceIdentity,
4
- labelTrustRank,
5
- pairHoldout,
6
- recoverTruncatedJson,
7
- surfaceContentHash
8
- } from "./chunk-2QU3YOPR.js";
9
- import {
10
- SearchLedgerConflictError,
11
- SearchLedgerError,
12
- SearchLedgerIntegrityError,
13
- appendSearchLedgerLine,
14
- assertRealBackend,
15
- contentHash,
16
- planCampaignRun,
17
- runCampaign,
18
- summarizeBackendIntegrity,
19
- withSearchLedgerFileLock
20
- } from "./chunk-C6LXANRU.js";
21
- import {
22
- Mutex
23
- } from "./chunk-WGXIEX7P.js";
24
- import {
25
- campaignCellToRunRecord
26
- } from "./chunk-E7QXT7SX.js";
27
- import {
28
- costReceiptFromLlm,
29
- costReceiptFromLlmError,
30
- maximumChargeForLlmRequest
31
- } from "./chunk-SFLLL76A.js";
32
- import {
33
- comparePairedArms
34
- } from "./chunk-7FO3TNPI.js";
35
- import {
36
- eProcess,
37
- mulberry32,
38
- pairedBootstrap
39
- } from "./chunk-ZHTZ4EYI.js";
40
- import {
41
- CostLedger
42
- } from "./chunk-VCZ5FQYW.js";
43
- import {
44
- analyzeTraces
45
- } from "./chunk-5DTSBUL2.js";
46
- import {
47
- buildAgentProfileCell,
48
- canonicalize,
49
- modelHasSnapshot,
50
- runTaskScore,
51
- validateRunRecord
52
- } from "./chunk-56TAVBOK.js";
53
- import {
54
- AgentEvalError,
55
- ValidationError
56
- } from "./chunk-ONWEPEDO.js";
57
-
58
- // src/campaign/analyst-surface.ts
59
- function surfaceToText(surface) {
60
- if (typeof surface === "string") return surface;
61
- throw new Error(
62
- `buildAnalystSurfaceDispatch: the analyst surface must be a string actorDescription, got a ${surface.kind}-tier surface. The analyst prompt is prompt-tier.`
63
- );
64
- }
65
- function buildAnalystSurfaceDispatch(opts) {
66
- const analyze = opts.analyze ?? analyzeTraces;
67
- return async (surface, scenario, _ctx) => {
68
- const actorDescription = surfaceToText(surface);
69
- const res = await analyze(
70
- { question: scenario.question },
71
- { ...opts.analystOptions, actorDescription, source: scenario.source }
72
- );
73
- return {
74
- answer: res.answer,
75
- findings: res.findings,
76
- actorPromptVersion: res.actorPromptVersion
77
- };
78
- };
79
- }
80
- function failureModeRecallJudge(opts = {}) {
81
- const recallWeight = opts.recallWeight ?? 0.5;
82
- return {
83
- name: "failure-mode-recall",
84
- dimensions: [
85
- { key: "recall", description: "fraction of ground-truth failure modes the analyst surfaced" },
86
- {
87
- key: "precision",
88
- description: "1 \u2212 share of findings that named a failure/tool/error absent from this corpus"
89
- }
90
- ],
91
- appliesTo: (s) => s.kind === "analyst-surface",
92
- score({ artifact, scenario }) {
93
- const modes = scenario.expectedFailureModes;
94
- if (modes.length === 0) {
95
- throw new Error(
96
- `failureModeRecallJudge: scenario '${scenario.id}' has no expectedFailureModes \u2014 refusing to score (a vacuous 1.0 would corrupt the comparison)`
97
- );
98
- }
99
- const hay = artifact.findings.join("\n").toLowerCase();
100
- const matched = modes.filter((m) => m.cues.some((c) => hay.includes(c.toLowerCase())));
101
- const recall = matched.length / modes.length;
102
- const forbidden = (scenario.forbiddenCues ?? []).map((c) => c.toLowerCase());
103
- let precision = 1;
104
- let hallucinated = 0;
105
- if (forbidden.length > 0) {
106
- const denom = Math.max(1, artifact.findings.length);
107
- hallucinated = artifact.findings.filter(
108
- (f) => forbidden.some((c) => f.toLowerCase().includes(c))
109
- ).length;
110
- precision = 1 - hallucinated / denom;
111
- }
112
- const composite = forbidden.length > 0 ? recallWeight * recall + (1 - recallWeight) * precision : recall;
113
- const missed = modes.filter((m) => !matched.includes(m)).map((m) => m.id);
114
- const notes = `matched ${matched.length}/${modes.length} failure modes` + (missed.length ? `; missed [${missed.join(", ")}]` : "") + (hallucinated ? `; ${hallucinated} out-of-corpus finding(s)` : "");
115
- return { dimensions: { recall, precision }, composite, notes };
116
- }
117
- };
118
- }
119
-
120
- // src/campaign/cross-surface-context.ts
121
- function validateCrossSurfaceInput(input) {
122
- assertNonEmptyUnique(input.taskOrder, "taskOrder");
123
- assertNonEmptyUnique(input.componentOrder, "componentOrder");
124
- if (input.componentOrder.length < 2) {
125
- throw new ValidationError(
126
- "analyzeCrossSurfaceInteractions: componentOrder must contain at least two surfaces"
127
- );
128
- }
129
- assertNonEmptyUnique(input.candidateOrder, "candidateOrder");
130
- assertNonEmptyUnique(input.costMetricOrder, "costMetricOrder");
131
- if (input.costMetricOrder.includes("score")) {
132
- throw new ValidationError(
133
- `analyzeCrossSurfaceInteractions: costMetricOrder cannot contain reserved metric 'score'`
134
- );
135
- }
136
- validateBootstrap(input);
137
- validateSelection(input);
138
- const componentById = indexUnique(
139
- input.components,
140
- (component) => component.componentId,
141
- "component"
142
- );
143
- assertExactSet(input.componentOrder, componentById.keys(), "componentOrder", "components");
144
- const surfaces = /* @__PURE__ */ new Set();
145
- for (const component of componentById.values()) {
146
- assertNonEmpty(component.componentId, "componentId");
147
- assertNonEmpty(component.surfaceId, `surfaceId for component '${component.componentId}'`);
148
- if (surfaces.has(component.surfaceId)) {
149
- throw new ValidationError(
150
- `analyzeCrossSurfaceInteractions: surfaceId '${component.surfaceId}' has more than one component; the interaction stage requires one independently selected finalist per surface`
151
- );
152
- }
153
- surfaces.add(component.surfaceId);
154
- if (typeof component.bestSingleEligible !== "boolean") {
155
- throw new ValidationError(
156
- `analyzeCrossSurfaceInteractions: component '${component.componentId}' must declare bestSingleEligible`
157
- );
158
- }
159
- }
160
- const componentIndex = new Map(input.componentOrder.map((id, index) => [id, index]));
161
- const candidateById = indexUnique(
162
- input.candidates,
163
- (candidate) => candidate.candidateId,
164
- "candidate"
165
- );
166
- assertExactSet(input.candidateOrder, candidateById.keys(), "candidateOrder", "candidates");
167
- if (!candidateById.has(input.baselineCandidateId)) {
168
- throw new ValidationError(
169
- `analyzeCrossSurfaceInteractions: unknown baselineCandidateId '${input.baselineCandidateId}'`
170
- );
171
- }
172
- const candidateByComponents = /* @__PURE__ */ new Map();
173
- const singleByComponent = /* @__PURE__ */ new Map();
174
- for (const candidate of candidateById.values()) {
175
- validateCandidate(candidate, componentById, componentIndex, input.baselineCandidateId);
176
- const key = crossSurfaceComponentSetKey(candidate.componentIds);
177
- const duplicate = candidateByComponents.get(key);
178
- if (duplicate) {
179
- throw new ValidationError(
180
- `analyzeCrossSurfaceInteractions: candidates '${duplicate.candidateId}' and '${candidate.candidateId}' materialize the same component set`
181
- );
182
- }
183
- candidateByComponents.set(key, candidate);
184
- if (candidate.componentIds.length === 1) {
185
- singleByComponent.set(candidate.componentIds[0], candidate);
186
- }
187
- }
188
- const baseline = candidateById.get(input.baselineCandidateId);
189
- if (baseline.componentIds.length !== 0) {
190
- throw new ValidationError(
191
- `analyzeCrossSurfaceInteractions: baseline '${baseline.candidateId}' must have zero components`
192
- );
193
- }
194
- for (const componentId of input.componentOrder) {
195
- if (!singleByComponent.has(componentId)) {
196
- throw new ValidationError(
197
- `analyzeCrossSurfaceInteractions: no single-surface candidate for component '${componentId}'`
198
- );
199
- }
200
- }
201
- for (let left = 0; left < input.componentOrder.length; left++) {
202
- for (let right = left + 1; right < input.componentOrder.length; right++) {
203
- const ids = [input.componentOrder[left], input.componentOrder[right]];
204
- if (!candidateByComponents.has(crossSurfaceComponentSetKey(ids))) {
205
- throw new ValidationError(
206
- `analyzeCrossSurfaceInteractions: missing pair candidate for components [${ids.join(", ")}]`
207
- );
208
- }
209
- }
210
- }
211
- const rowsByCandidate = /* @__PURE__ */ new Map();
212
- const taskIds = new Set(input.taskOrder);
213
- for (const row of input.rows) {
214
- const candidate = candidateById.get(row.candidateId);
215
- if (!candidate) {
216
- throw new ValidationError(
217
- `analyzeCrossSurfaceInteractions: row for task '${row.taskId}' names unknown candidate '${row.candidateId}'`
218
- );
219
- }
220
- if (!taskIds.has(row.taskId)) {
221
- throw new ValidationError(
222
- `analyzeCrossSurfaceInteractions: row for candidate '${row.candidateId}' names task '${row.taskId}' outside the declared taskOrder`
223
- );
224
- }
225
- validateRow(input, row, candidate);
226
- const byTask = rowsByCandidate.get(row.candidateId) ?? /* @__PURE__ */ new Map();
227
- if (byTask.has(row.taskId)) {
228
- throw new ValidationError(
229
- `analyzeCrossSurfaceInteractions: duplicate row for candidate '${row.candidateId}' and task '${row.taskId}'`
230
- );
231
- }
232
- byTask.set(row.taskId, row);
233
- rowsByCandidate.set(row.candidateId, byTask);
234
- }
235
- for (const candidateId of input.candidateOrder) {
236
- const byTask = rowsByCandidate.get(candidateId);
237
- const missing = input.taskOrder.filter((taskId) => !byTask?.has(taskId));
238
- if (missing.length > 0) {
239
- throw new ValidationError(
240
- `analyzeCrossSurfaceInteractions: candidate '${candidateId}' is missing declared task row(s) [${missing.join(", ")}]; encode failed attempts explicitly instead of changing the task axis`
241
- );
242
- }
243
- }
244
- const expectedRows = input.candidateOrder.length * input.taskOrder.length;
245
- if (input.rows.length !== expectedRows) {
246
- throw new ValidationError(
247
- `analyzeCrossSurfaceInteractions: expected exactly ${expectedRows} candidate \xD7 task rows, got ${input.rows.length}`
248
- );
249
- }
250
- return {
251
- input,
252
- components: input.componentOrder.map((id) => componentById.get(id)),
253
- candidates: input.candidateOrder.map((id) => candidateById.get(id)),
254
- componentById,
255
- candidateById,
256
- candidateByComponents,
257
- singleByComponent,
258
- rowsByCandidate,
259
- componentIndex,
260
- candidateIndex: new Map(input.candidateOrder.map((id, index) => [id, index]))
261
- };
262
- }
263
- function crossSurfaceRowsFor(context, candidateId) {
264
- return context.input.taskOrder.map((taskId) => crossSurfaceRowFor(context, candidateId, taskId));
265
- }
266
- function crossSurfaceRowFor(context, candidateId, taskId) {
267
- return context.rowsByCandidate.get(candidateId).get(taskId);
268
- }
269
- function canonicalCrossSurfaceComponents(context, componentIds) {
270
- return [...componentIds].sort(
271
- (left, right) => context.componentIndex.get(left) - context.componentIndex.get(right)
272
- );
273
- }
274
- function crossSurfaceComponentSetKey(componentIds) {
275
- return JSON.stringify(componentIds);
276
- }
277
- function validateBootstrap(input) {
278
- const { seed, resamples, confidence } = input.bootstrap;
279
- if (!Number.isInteger(seed)) {
280
- throw new ValidationError(`analyzeCrossSurfaceInteractions: bootstrap.seed must be an integer`);
281
- }
282
- if (!Number.isInteger(resamples) || resamples <= 0) {
283
- throw new ValidationError(
284
- `analyzeCrossSurfaceInteractions: bootstrap.resamples must be a positive integer`
285
- );
286
- }
287
- if (!Number.isFinite(confidence) || confidence <= 0 || confidence >= 1) {
288
- throw new ValidationError(
289
- `analyzeCrossSurfaceInteractions: bootstrap.confidence must be in (0,1)`
290
- );
291
- }
292
- }
293
- function validateSelection(input) {
294
- const policy = input.selection;
295
- for (const [name, value] of [
296
- ["minimumFiringTasks", policy.minimumFiringTasks],
297
- ["minimumEffectTasks", policy.minimumEffectTasks]
298
- ]) {
299
- if (!Number.isInteger(value) || value < 0 || value > input.taskOrder.length) {
300
- throw new ValidationError(
301
- `analyzeCrossSurfaceInteractions: selection.${name} must be an integer in [0,${input.taskOrder.length}]`
302
- );
303
- }
304
- }
305
- if (!Number.isInteger(policy.minimumBundleComponents) || policy.minimumBundleComponents < 2 || policy.minimumBundleComponents > input.componentOrder.length) {
306
- throw new ValidationError(
307
- `analyzeCrossSurfaceInteractions: selection.minimumBundleComponents must be an integer in [2,${input.componentOrder.length}]`
308
- );
309
- }
310
- for (const [metric, limit] of Object.entries(policy.maximumMedianCostRatioToBaseline)) {
311
- if (!input.costMetricOrder.includes(metric)) {
312
- throw new ValidationError(
313
- `analyzeCrossSurfaceInteractions: cost limit names undeclared metric '${metric}'`
314
- );
315
- }
316
- if (!Number.isFinite(limit) || limit <= 0) {
317
- throw new ValidationError(
318
- `analyzeCrossSurfaceInteractions: cost ratio limit for '${metric}' must be positive and finite`
319
- );
320
- }
321
- }
322
- }
323
- function validateCandidate(candidate, componentById, componentIndex, baselineCandidateId) {
324
- assertNonEmpty(candidate.candidateId, "candidateId");
325
- assertNonEmpty(candidate.contentHash, `contentHash for candidate '${candidate.candidateId}'`);
326
- if (!Number.isInteger(candidate.artifactBytes) || candidate.artifactBytes < 0) {
327
- throw new ValidationError(
328
- `analyzeCrossSurfaceInteractions: artifactBytes for candidate '${candidate.candidateId}' must be a non-negative integer`
329
- );
330
- }
331
- assertUnique(candidate.componentIds, `componentIds for candidate '${candidate.candidateId}'`);
332
- for (const componentId of candidate.componentIds) {
333
- if (!componentById.has(componentId)) {
334
- throw new ValidationError(
335
- `analyzeCrossSurfaceInteractions: candidate '${candidate.candidateId}' names unknown component '${componentId}'`
336
- );
337
- }
338
- }
339
- const canonical = [...candidate.componentIds].sort(
340
- (left, right) => componentIndex.get(left) - componentIndex.get(right)
341
- );
342
- if (!arraysEqual(candidate.componentIds, canonical)) {
343
- throw new ValidationError(
344
- `analyzeCrossSurfaceInteractions: candidate '${candidate.candidateId}' components must follow componentOrder`
345
- );
346
- }
347
- if (candidate.candidateId !== baselineCandidateId && candidate.componentIds.length === 0) {
348
- throw new ValidationError(
349
- `analyzeCrossSurfaceInteractions: only baseline '${baselineCandidateId}' may have zero components`
350
- );
351
- }
352
- }
353
- function validateRow(input, row, candidate) {
354
- if (!arraysEqual(row.componentIds, candidate.componentIds)) {
355
- throw new ValidationError(
356
- `analyzeCrossSurfaceInteractions: row '${row.candidateId}/${row.taskId}' componentIds do not match its candidate`
357
- );
358
- }
359
- if (!["complete", "missing", "invalid"].includes(row.completeness)) {
360
- throw new ValidationError(
361
- `analyzeCrossSurfaceInteractions: row '${row.candidateId}/${row.taskId}' has unknown completeness '${String(row.completeness)}'`
362
- );
363
- }
364
- if (row.completeness === "complete") {
365
- if (typeof row.pass !== "boolean" || !Number.isFinite(row.score)) {
366
- throw new ValidationError(
367
- `analyzeCrossSurfaceInteractions: complete row '${row.candidateId}/${row.taskId}' requires boolean pass and finite score`
368
- );
369
- }
370
- if (row.rejectReason !== null) {
371
- throw new ValidationError(
372
- `analyzeCrossSurfaceInteractions: complete row '${row.candidateId}/${row.taskId}' cannot carry rejectReason`
373
- );
374
- }
375
- } else {
376
- if (row.pass !== null || row.score !== null) {
377
- throw new ValidationError(
378
- `analyzeCrossSurfaceInteractions: ${row.completeness} row '${row.candidateId}/${row.taskId}' must use null pass and score`
379
- );
380
- }
381
- if (typeof row.rejectReason !== "string" || row.rejectReason.trim() === "") {
382
- throw new ValidationError(
383
- `analyzeCrossSurfaceInteractions: ${row.completeness} row '${row.candidateId}/${row.taskId}' requires rejectReason`
384
- );
385
- }
386
- }
387
- assertExactSet(
388
- input.costMetricOrder,
389
- Object.keys(row.cost),
390
- `cost keys for row '${row.candidateId}/${row.taskId}'`,
391
- "costMetricOrder"
392
- );
393
- for (const metric of input.costMetricOrder) {
394
- const value = row.cost[metric];
395
- if (value === null || value === void 0 || !Number.isFinite(value) || value < 0) {
396
- throw new ValidationError(
397
- `analyzeCrossSurfaceInteractions: unknown or invalid cost '${metric}' on row '${row.candidateId}/${row.taskId}'; every attempt must report a non-negative finite value`
398
- );
399
- }
400
- }
401
- const evidenceByComponent = indexUnique(
402
- row.componentEvidence,
403
- (evidence) => evidence.componentId,
404
- `componentEvidence on row '${row.candidateId}/${row.taskId}'`
405
- );
406
- assertExactSet(
407
- candidate.componentIds,
408
- evidenceByComponent.keys(),
409
- `componentEvidence on row '${row.candidateId}/${row.taskId}'`,
410
- "candidate components"
411
- );
412
- for (const evidence of evidenceByComponent.values()) {
413
- assertTriState(evidence.fired, "fired", row);
414
- assertTriState(evidence.effectObserved, "effectObserved", row);
415
- if (evidence.effectObserved === true && evidence.fired !== true) {
416
- throw new ValidationError(
417
- `analyzeCrossSurfaceInteractions: effectObserved=true requires fired=true for component '${evidence.componentId}' on row '${row.candidateId}/${row.taskId}'`
418
- );
419
- }
420
- }
421
- }
422
- function assertTriState(value, field, row) {
423
- if (value !== true && value !== false && value !== null) {
424
- throw new ValidationError(
425
- `analyzeCrossSurfaceInteractions: ${field} on row '${row.candidateId}/${row.taskId}' must be boolean or null`
426
- );
427
- }
428
- }
429
- function indexUnique(items, id, label) {
430
- const result = /* @__PURE__ */ new Map();
431
- for (const item of items) {
432
- const key = id(item);
433
- assertNonEmpty(key, `${label} id`);
434
- if (result.has(key)) {
435
- throw new ValidationError(`analyzeCrossSurfaceInteractions: duplicate ${label} id '${key}'`);
436
- }
437
- result.set(key, item);
438
- }
439
- return result;
440
- }
441
- function assertNonEmptyUnique(values, label) {
442
- if (values.length === 0) {
443
- throw new ValidationError(`analyzeCrossSurfaceInteractions: ${label} is empty`);
444
- }
445
- assertUnique(values, label);
446
- for (const value of values) assertNonEmpty(value, label);
447
- }
448
- function assertUnique(values, label) {
449
- if (new Set(values).size !== values.length) {
450
- throw new ValidationError(`analyzeCrossSurfaceInteractions: ${label} contains duplicates`);
451
- }
452
- }
453
- function assertNonEmpty(value, label) {
454
- if (typeof value !== "string" || value.trim() === "") {
455
- throw new ValidationError(
456
- `analyzeCrossSurfaceInteractions: ${label} must be a non-empty string`
457
- );
458
- }
459
- }
460
- function assertExactSet(expected, actualIterable, actualLabel, expectedLabel) {
461
- const actual = [...actualIterable];
462
- const expectedSet = new Set(expected);
463
- const actualSet = new Set(actual);
464
- const missing = expected.filter((value) => !actualSet.has(value));
465
- const extra = actual.filter((value) => !expectedSet.has(value));
466
- if (missing.length > 0 || extra.length > 0 || actual.length !== expected.length) {
467
- throw new ValidationError(
468
- `analyzeCrossSurfaceInteractions: ${actualLabel} does not match ${expectedLabel}; missing=[${missing.join(", ")}], extra=[${extra.join(", ")}]`
469
- );
470
- }
471
- }
472
- function arraysEqual(left, right) {
473
- return left.length === right.length && left.every((value, index) => value === right[index]);
474
- }
475
-
476
- // src/campaign/cross-surface-interaction.ts
477
- function analyzeCrossSurfaceInteractions(input) {
478
- const context = validateCrossSurfaceInput(input);
479
- const baseline = context.candidateById.get(input.baselineCandidateId);
480
- const preliminary = context.candidates.map(
481
- (candidate) => summarizeCandidate(context, candidate, baseline)
482
- );
483
- const baselineSummary = preliminary.find(
484
- (summary) => summary.candidate.candidateId === input.baselineCandidateId
485
- );
486
- const summaries = preliminary.map(
487
- (summary) => summary.candidate.candidateId === input.baselineCandidateId ? summary : {
488
- ...summary,
489
- eligibility: candidateEligibility(context, summary, baselineSummary)
490
- }
491
- );
492
- const summaryById = new Map(summaries.map((summary) => [summary.candidate.candidateId, summary]));
493
- const eligibleSingles = rankedEligibleSingles(context, summaryById);
494
- const interactionReadySingles = rankedInteractionReadySingles(context, summaryById);
495
- const pairwise = buildPairwise(context, summaryById, interactionReadySingles, baselineSummary);
496
- const selections = selectCandidates(
497
- context,
498
- summaryById,
499
- eligibleSingles,
500
- interactionReadySingles,
501
- pairwise
502
- );
503
- const rows = context.candidates.flatMap(
504
- (candidate) => input.taskOrder.map(
505
- (taskId) => context.rowsByCandidate.get(candidate.candidateId).get(taskId)
506
- )
507
- );
508
- return {
509
- taskIds: [...input.taskOrder],
510
- componentIds: [...input.componentOrder],
511
- candidateIds: [...input.candidateOrder],
512
- costMetrics: [...input.costMetricOrder],
513
- rows,
514
- missingAttempts: rows.filter((row) => row.completeness === "missing"),
515
- invalidAttempts: rows.filter((row) => row.completeness === "invalid"),
516
- candidates: summaries,
517
- pairwise,
518
- selections
519
- };
520
- }
521
- function summarizeCandidate(context, candidate, baseline) {
522
- const rows = crossSurfaceRowsFor(context, candidate.candidateId);
523
- const baselineRows = crossSurfaceRowsFor(context, baseline.candidateId);
524
- const outcome = summarizeOutcome(
525
- context,
526
- rows,
527
- baselineRows,
528
- candidate.candidateId === baseline.candidateId
529
- );
530
- const completeScores = rows.filter((row) => row.completeness === "complete").map((row) => row.score);
531
- const costs = Object.fromEntries(
532
- context.input.costMetricOrder.map((metric) => [
533
- metric,
534
- distribution(rows.map((row) => row.cost[metric]))
535
- ])
536
- );
537
- return {
538
- candidate,
539
- outcome,
540
- score: completeScores.length === 0 ? null : distribution(completeScores),
541
- costs,
542
- firing: summarizeCandidateEvidence(rows, candidate.componentIds, "fired"),
543
- effect: summarizeCandidateEvidence(rows, candidate.componentIds, "effectObserved"),
544
- comparisonToBaseline: candidate.candidateId === baseline.candidateId ? null : pairedComparison(context, baseline.candidateId, candidate.candidateId),
545
- eligibility: null
546
- };
547
- }
548
- function summarizeOutcome(context, rows, baselineRows, isBaseline) {
549
- const resolvedTaskIds = [];
550
- const failedTaskIds = [];
551
- const missingTaskIds = [];
552
- const invalidTaskIds = [];
553
- const benefitTaskIds = [];
554
- const regressionTaskIds = [];
555
- const comparisonMissingTaskIds = [];
556
- for (let index = 0; index < context.input.taskOrder.length; index++) {
557
- const row = rows[index];
558
- const baseline = baselineRows[index];
559
- if (row.completeness === "missing") missingTaskIds.push(row.taskId);
560
- else if (row.completeness === "invalid") invalidTaskIds.push(row.taskId);
561
- else if (row.pass) resolvedTaskIds.push(row.taskId);
562
- else failedTaskIds.push(row.taskId);
563
- if (isBaseline) continue;
564
- if (row.completeness !== "complete" || baseline.completeness !== "complete") {
565
- comparisonMissingTaskIds.push(row.taskId);
566
- } else if (row.pass && !baseline.pass) benefitTaskIds.push(row.taskId);
567
- else if (!row.pass && baseline.pass) regressionTaskIds.push(row.taskId);
568
- }
569
- return {
570
- resolvedTaskIds,
571
- failedTaskIds,
572
- missingTaskIds,
573
- invalidTaskIds,
574
- benefitTaskIds,
575
- regressionTaskIds,
576
- comparisonMissingTaskIds,
577
- netBenefit: benefitTaskIds.length - regressionTaskIds.length
578
- };
579
- }
580
- function summarizeCandidateEvidence(rows, componentIds, field) {
581
- if (componentIds.length === 0) {
582
- return {
583
- byComponent: [],
584
- allObservedTaskIds: [],
585
- someObservedTaskIds: [],
586
- noneObservedTaskIds: [],
587
- unobservedTaskIds: []
588
- };
589
- }
590
- const byComponent = componentIds.map((componentId) => {
591
- const observedTaskIds = [];
592
- const notObservedTaskIds = [];
593
- const unobservedTaskIds2 = [];
594
- for (const row of rows) {
595
- const value = evidenceValue(row, componentId, field);
596
- if (value === true) observedTaskIds.push(row.taskId);
597
- else if (value === false) notObservedTaskIds.push(row.taskId);
598
- else unobservedTaskIds2.push(row.taskId);
599
- }
600
- return { componentId, observedTaskIds, notObservedTaskIds, unobservedTaskIds: unobservedTaskIds2 };
601
- });
602
- const allObservedTaskIds = [];
603
- const someObservedTaskIds = [];
604
- const noneObservedTaskIds = [];
605
- const unobservedTaskIds = [];
606
- for (const row of rows) {
607
- const values = componentIds.map((componentId) => evidenceValue(row, componentId, field));
608
- if (values.some((value) => value === null)) unobservedTaskIds.push(row.taskId);
609
- else if (values.every(Boolean)) allObservedTaskIds.push(row.taskId);
610
- else if (values.some(Boolean)) someObservedTaskIds.push(row.taskId);
611
- else noneObservedTaskIds.push(row.taskId);
612
- }
613
- return {
614
- byComponent,
615
- allObservedTaskIds,
616
- someObservedTaskIds,
617
- noneObservedTaskIds,
618
- unobservedTaskIds
619
- };
620
- }
621
- function candidateEligibility(context, summary, baseline) {
622
- const reasons = [];
623
- if (summary.outcome.missingTaskIds.length > 0) reasons.push("missing_attempt");
624
- if (summary.outcome.invalidTaskIds.length > 0) reasons.push("invalid_attempt");
625
- if (summary.outcome.comparisonMissingTaskIds.length > 0) reasons.push("baseline_outcome_missing");
626
- if (summary.outcome.benefitTaskIds.length <= summary.outcome.regressionTaskIds.length) {
627
- reasons.push("benefit_not_greater_than_regression");
628
- }
629
- appendEvidenceEligibilityReasons(
630
- reasons,
631
- summary.firing,
632
- context.input.selection.minimumFiringTasks,
633
- context.input.selection.requireObservedFiring,
634
- "firing_below_minimum",
635
- "firing_unobserved"
636
- );
637
- appendEvidenceEligibilityReasons(
638
- reasons,
639
- summary.effect,
640
- context.input.selection.minimumEffectTasks,
641
- context.input.selection.requireObservedEffect,
642
- "effect_below_minimum",
643
- "effect_unobserved"
644
- );
645
- if (!withinCostLimits(context, summary, baseline)) reasons.push("cost_limit_exceeded");
646
- return { eligible: reasons.length === 0, reasons: unique(reasons) };
647
- }
648
- function appendEvidenceEligibilityReasons(reasons, evidence, minimum, requireObserved, belowReason, unobservedReason) {
649
- if (evidence.byComponent.some((component) => component.observedTaskIds.length < minimum)) {
650
- reasons.push(belowReason);
651
- }
652
- if (requireObserved && evidence.byComponent.some((component) => component.unobservedTaskIds.length > 0)) {
653
- reasons.push(unobservedReason);
654
- }
655
- }
656
- function rankedEligibleSingles(context, summaryById) {
657
- return [...context.singleByComponent.values()].map((candidate) => summaryById.get(candidate.candidateId)).filter((summary) => summary.eligibility?.eligible).sort((left, right) => compareSingleSummaries(context, left, right));
658
- }
659
- function rankedInteractionReadySingles(context, summaryById) {
660
- return [...context.singleByComponent.values()].map((candidate) => summaryById.get(candidate.candidateId)).filter(
661
- (summary) => summary.outcome.regressionTaskIds.length === 0 && summary.eligibility?.reasons.every(
662
- (reason) => reason === "benefit_not_greater_than_regression"
663
- )
664
- ).sort((left, right) => compareSingleSummaries(context, left, right));
665
- }
666
- function compareSingleSummaries(context, left, right) {
667
- const byNetBenefit = right.outcome.netBenefit - left.outcome.netBenefit;
668
- if (byNetBenefit !== 0) return byNetBenefit;
669
- const byBenefit = right.outcome.benefitTaskIds.length - left.outcome.benefitTaskIds.length;
670
- if (byBenefit !== 0) return byBenefit;
671
- const byRegression = left.outcome.regressionTaskIds.length - right.outcome.regressionTaskIds.length;
672
- if (byRegression !== 0) return byRegression;
673
- for (const metric of context.input.costMetricOrder) {
674
- const byCost = left.costs[metric].median - right.costs[metric].median;
675
- if (byCost !== 0) return byCost;
676
- }
677
- const byBytes = left.candidate.artifactBytes - right.candidate.artifactBytes;
678
- if (byBytes !== 0) return byBytes;
679
- return context.candidateIndex.get(left.candidate.candidateId) - context.candidateIndex.get(right.candidate.candidateId);
680
- }
681
- function buildPairwise(context, summaryById, interactionReadySingles, baseline) {
682
- const interactionReadyIds = new Set(
683
- interactionReadySingles.map((summary) => summary.candidate.candidateId)
684
- );
685
- const entries = [];
686
- for (let leftIndex = 0; leftIndex < context.components.length; leftIndex++) {
687
- for (let rightIndex = leftIndex + 1; rightIndex < context.components.length; rightIndex++) {
688
- const leftComponent = context.components[leftIndex];
689
- const rightComponent = context.components[rightIndex];
690
- const leftSingle = context.singleByComponent.get(leftComponent.componentId);
691
- const rightSingle = context.singleByComponent.get(rightComponent.componentId);
692
- const pair = context.candidateByComponents.get(
693
- crossSurfaceComponentSetKey([leftComponent.componentId, rightComponent.componentId])
694
- );
695
- const pairSummary = summaryById.get(pair.candidateId);
696
- const leftSummary = summaryById.get(leftSingle.candidateId);
697
- const rightSummary = summaryById.get(rightSingle.candidateId);
698
- const synergyTaskIds = [];
699
- const interferenceTaskIds = [];
700
- for (const taskId of context.input.taskOrder) {
701
- const pairRow = crossSurfaceRowFor(context, pair.candidateId, taskId);
702
- const leftRow = crossSurfaceRowFor(context, leftSingle.candidateId, taskId);
703
- const rightRow = crossSurfaceRowFor(context, rightSingle.candidateId, taskId);
704
- if (![pairRow, leftRow, rightRow].every((row) => row.completeness === "complete")) continue;
705
- if (pairRow.pass && !leftRow.pass && !rightRow.pass) synergyTaskIds.push(taskId);
706
- if (!pairRow.pass && (leftRow.pass || rightRow.pass)) interferenceTaskIds.push(taskId);
707
- }
708
- const incrementalVsConstituents = [
709
- compareCandidates(context, summaryById, leftSingle.candidateId, pair.candidateId),
710
- compareCandidates(context, summaryById, rightSingle.candidateId, pair.candidateId)
711
- ];
712
- const betterSingle = compareSingleSummaries(context, leftSummary, rightSummary) <= 0 ? leftSummary : rightSummary;
713
- const firing = summarizePairEvidence(
714
- crossSurfaceRowsFor(context, pair.candidateId),
715
- leftComponent.componentId,
716
- rightComponent.componentId,
717
- "fired"
718
- );
719
- const effect = summarizePairEvidence(
720
- crossSurfaceRowsFor(context, pair.candidateId),
721
- leftComponent.componentId,
722
- rightComponent.componentId,
723
- "effectObserved"
724
- );
725
- const compatibility = pairCompatibility(
726
- context,
727
- pairSummary,
728
- baseline,
729
- betterSingle,
730
- incrementalVsConstituents,
731
- interactionReadyIds.has(leftSingle.candidateId) && interactionReadyIds.has(rightSingle.candidateId),
732
- interferenceTaskIds,
733
- firing,
734
- effect
735
- );
736
- entries.push({
737
- componentIds: [leftComponent.componentId, rightComponent.componentId],
738
- singleCandidateIds: [leftSingle.candidateId, rightSingle.candidateId],
739
- compositionCandidateId: pair.candidateId,
740
- benefitTaskIds: [...pairSummary.outcome.benefitTaskIds],
741
- regressionTaskIds: [...pairSummary.outcome.regressionTaskIds],
742
- synergyTaskIds,
743
- interferenceTaskIds,
744
- incrementalVsConstituents,
745
- relativeCostToBaseline: relativeCosts(context, pairSummary, baseline),
746
- firing,
747
- effect,
748
- interaction: interactionEffect(
749
- context,
750
- baseline.candidate,
751
- [leftSingle, rightSingle],
752
- pair
753
- ),
754
- compatibility
755
- });
756
- }
757
- }
758
- return entries;
759
- }
760
- function pairCompatibility(context, pair, baseline, betterSingle, comparisons, constituentsReady, interferenceTaskIds, firing, effect) {
761
- const reasons = [];
762
- if (!constituentsReady) reasons.push("constituent_not_ready");
763
- if (pair.outcome.missingTaskIds.length > 0 || pair.outcome.invalidTaskIds.length > 0 || pair.outcome.comparisonMissingTaskIds.length > 0) {
764
- reasons.push("pair_incomplete");
765
- }
766
- if (pair.outcome.regressionTaskIds.length > 0) reasons.push("baseline_regression");
767
- if (interferenceTaskIds.length > 0) reasons.push("interference");
768
- const betterComparison = comparisons.find(
769
- (comparison) => comparison.comparatorCandidateId === betterSingle.candidate.candidateId
770
- );
771
- if (betterComparison.winsTaskIds.length === 0) reasons.push("no_incremental_resolution");
772
- appendPairEvidenceReasons(
773
- reasons,
774
- firing,
775
- context.input.selection.minimumFiringTasks,
776
- context.input.selection.requireObservedFiring,
777
- "firing_below_minimum",
778
- "firing_unobserved"
779
- );
780
- appendPairEvidenceReasons(
781
- reasons,
782
- effect,
783
- context.input.selection.minimumEffectTasks,
784
- context.input.selection.requireObservedEffect,
785
- "effect_below_minimum",
786
- "effect_unobserved"
787
- );
788
- if (!withinCostLimits(context, pair, baseline)) reasons.push("cost_limit_exceeded");
789
- return {
790
- compatible: reasons.length === 0,
791
- reasons: unique(reasons),
792
- betterSingleCandidateId: betterSingle.candidate.candidateId
793
- };
794
- }
795
- function appendPairEvidenceReasons(reasons, evidence, minimum, requireObserved, belowReason, unobservedReason) {
796
- const leftObserved = evidence.bothTaskIds.length + evidence.leftOnlyTaskIds.length;
797
- const rightObserved = evidence.bothTaskIds.length + evidence.rightOnlyTaskIds.length;
798
- if (leftObserved < minimum || rightObserved < minimum) reasons.push(belowReason);
799
- if (requireObserved && evidence.unobservedTaskIds.length > 0) reasons.push(unobservedReason);
800
- }
801
- function interactionEffect(context, baseline, singles, composition) {
802
- const perTask = context.input.taskOrder.map((taskId) => {
803
- const baselineRow = crossSurfaceRowFor(context, baseline.candidateId, taskId);
804
- const singleRows = singles.map((single) => crossSurfaceRowFor(context, single.candidateId, taskId));
805
- const compositionRow = crossSurfaceRowFor(context, composition.candidateId, taskId);
806
- const rows = [baselineRow, ...singleRows, compositionRow];
807
- if (!rows.every((row) => row.completeness === "complete")) {
808
- return { taskId, passInteraction: null, scoreInteraction: null };
809
- }
810
- const additivePass = singleRows.reduce((sum, row) => sum + Number(row.pass), 0) - (singles.length - 1) * Number(baselineRow.pass);
811
- const additiveScore = singleRows.reduce((sum, row) => sum + row.score, 0) - (singles.length - 1) * baselineRow.score;
812
- return {
813
- taskId,
814
- passInteraction: Number(compositionRow.pass) - additivePass,
815
- scoreInteraction: compositionRow.score - additiveScore
816
- };
817
- });
818
- const complete = perTask.filter(
819
- (row) => row.passInteraction !== null && row.scoreInteraction !== null
820
- );
821
- const passInteractions = complete.map((row) => row.passInteraction);
822
- const scoreInteractions = complete.map((row) => row.scoreInteraction);
823
- return {
824
- perTask,
825
- n: complete.length,
826
- nMissing: perTask.length - complete.length,
827
- meanPassInteraction: meanOrNull(passInteractions),
828
- meanScoreInteraction: meanOrNull(scoreInteractions),
829
- passBootstrap: bootstrapInteraction(passInteractions, context),
830
- scoreBootstrap: bootstrapInteraction(scoreInteractions, context)
831
- };
832
- }
833
- function bootstrapInteraction(interactions, context) {
834
- if (interactions.length === 0) return null;
835
- return pairedBootstrap(new Array(interactions.length).fill(0), interactions, {
836
- ...context.input.bootstrap,
837
- statistic: "mean"
838
- });
839
- }
840
- function selectCandidates(context, summaryById, eligibleSingles, interactionReadySingles, pairwise) {
841
- const bestSingleRanking = eligibleSingles.filter((summary) => {
842
- const componentId = summary.candidate.componentIds[0];
843
- return context.componentById.get(componentId).bestSingleEligible;
844
- });
845
- const ranking = bestSingleRanking.map((summary, index) => ({
846
- rank: index + 1,
847
- candidateId: summary.candidate.candidateId,
848
- componentId: summary.candidate.componentIds[0]
849
- }));
850
- const best = bestSingleRanking[0];
851
- const bestSingle = best ? {
852
- candidateId: best.candidate.candidateId,
853
- componentId: best.candidate.componentIds[0],
854
- ranking
855
- } : null;
856
- const naiveStack = selectNaiveStack(context, eligibleSingles);
857
- const interactionAware = selectInteractionAware(
858
- context,
859
- summaryById,
860
- interactionReadySingles,
861
- pairwise
862
- );
863
- return { bestSingle, naiveStack, interactionAware };
864
- }
865
- function selectNaiveStack(context, eligibleSingles) {
866
- if (eligibleSingles.length === 0) return null;
867
- const componentIds = eligibleSingles.map((summary) => summary.candidate.componentIds[0]).sort((left, right) => context.componentIndex.get(left) - context.componentIndex.get(right));
868
- const candidate = context.candidateByComponents.get(crossSurfaceComponentSetKey(componentIds));
869
- if (!candidate) {
870
- throw new ValidationError(
871
- `analyzeCrossSurfaceInteractions: naive stack [${componentIds.join(", ")}] was not evaluated`
872
- );
873
- }
874
- return { candidateId: candidate.candidateId, componentIds };
875
- }
876
- function selectInteractionAware(context, summaryById, interactionReadySingles, pairwise) {
877
- const baseline = summaryById.get(context.input.baselineCandidateId);
878
- const pairByComponents = new Map(
879
- pairwise.map((entry) => [crossSurfaceComponentSetKey(entry.componentIds), entry])
880
- );
881
- const paths = pairwise.filter((entry) => entry.compatibility.compatible).map(
882
- (entry) => growInteractionPath(
883
- context,
884
- summaryById,
885
- interactionReadySingles,
886
- pairByComponents,
887
- baseline,
888
- summaryById.get(entry.compositionCandidateId)
889
- )
890
- );
891
- if (paths.length === 0) return null;
892
- const qualified = paths.filter((path) => path.qualified).sort((left, right) => compareInteractionPaths(context, summaryById, left, right));
893
- const winning = qualified[0] ?? [...paths].sort((left, right) => compareInteractionPaths(context, summaryById, left, right))[0];
894
- return {
895
- seedCandidateId: winning.seedCandidateId,
896
- terminalCandidateId: winning.terminalCandidateId,
897
- terminalComponentIds: [...winning.terminalComponentIds],
898
- selectedCandidateId: qualified.length > 0 ? winning.terminalCandidateId : null,
899
- qualified: qualified.length > 0,
900
- evaluatedPaths: paths,
901
- steps: winning.steps
902
- };
903
- }
904
- function growInteractionPath(context, summaryById, interactionReadySingles, pairByComponents, baseline, seed) {
905
- let current = seed;
906
- let retained = [...seed.candidate.componentIds];
907
- let remaining = interactionReadySingles.filter(
908
- (summary) => !retained.includes(summary.candidate.componentIds[0])
909
- );
910
- const steps = [];
911
- while (remaining.length > 0) {
912
- const decisions = remaining.map(
913
- (addition) => evaluateAddition(
914
- context,
915
- summaryById,
916
- pairByComponents,
917
- baseline,
918
- current,
919
- retained,
920
- addition
921
- )
922
- );
923
- const eligible = decisions.filter((decision) => decision.eligible).sort((left, right) => compareAdditionDecisions(context, summaryById, left, right));
924
- const selected = eligible[0];
925
- if (selected) selected.selected = true;
926
- steps.push({
927
- fromCandidateId: current.candidate.candidateId,
928
- retainedComponentIds: [...retained],
929
- considered: decisions.sort(
930
- (left, right) => context.candidateIndex.get(left.additionCandidateId) - context.candidateIndex.get(right.additionCandidateId)
931
- ),
932
- selectedCandidateId: selected?.bundleCandidateId ?? null
933
- });
934
- if (!selected?.bundleCandidateId) break;
935
- current = summaryById.get(selected.bundleCandidateId);
936
- retained = [...current.candidate.componentIds];
937
- remaining = remaining.filter(
938
- (summary) => summary.candidate.candidateId !== selected.additionCandidateId
939
- );
940
- }
941
- const qualified = retained.length >= context.input.selection.minimumBundleComponents;
942
- return {
943
- seedCandidateId: seed.candidate.candidateId,
944
- terminalCandidateId: current.candidate.candidateId,
945
- terminalComponentIds: retained,
946
- qualified,
947
- steps
948
- };
949
- }
950
- function compareInteractionPaths(context, summaryById, left, right) {
951
- const leftSummary = summaryById.get(left.terminalCandidateId);
952
- const rightSummary = summaryById.get(right.terminalCandidateId);
953
- const byNetBenefit = rightSummary.outcome.netBenefit - leftSummary.outcome.netBenefit;
954
- if (byNetBenefit !== 0) return byNetBenefit;
955
- const byBenefit = rightSummary.outcome.benefitTaskIds.length - leftSummary.outcome.benefitTaskIds.length;
956
- if (byBenefit !== 0) return byBenefit;
957
- const byRegression = leftSummary.outcome.regressionTaskIds.length - rightSummary.outcome.regressionTaskIds.length;
958
- if (byRegression !== 0) return byRegression;
959
- for (const metric of context.input.costMetricOrder) {
960
- const byCost = leftSummary.costs[metric].median - rightSummary.costs[metric].median;
961
- if (byCost !== 0) return byCost;
962
- }
963
- const byBytes = leftSummary.candidate.artifactBytes - rightSummary.candidate.artifactBytes;
964
- if (byBytes !== 0) return byBytes;
965
- const byCandidateOrder = context.candidateIndex.get(left.terminalCandidateId) - context.candidateIndex.get(right.terminalCandidateId);
966
- if (byCandidateOrder !== 0) return byCandidateOrder;
967
- return context.candidateIndex.get(left.seedCandidateId) - context.candidateIndex.get(right.seedCandidateId);
968
- }
969
- function evaluateAddition(context, summaryById, pairByComponents, baseline, current, retained, addition) {
970
- const additionComponentId = addition.candidate.componentIds[0];
971
- const reasons = [];
972
- for (const retainedComponentId of retained) {
973
- const pair = pairByComponents.get(
974
- crossSurfaceComponentSetKey(canonicalCrossSurfaceComponents(context, [retainedComponentId, additionComponentId]))
975
- );
976
- if (!pair?.compatibility.compatible) reasons.push("pair_incompatible");
977
- }
978
- const bundleComponents = canonicalCrossSurfaceComponents(context, [...retained, additionComponentId]);
979
- const bundle = context.candidateByComponents.get(crossSurfaceComponentSetKey(bundleComponents));
980
- if (!bundle) {
981
- reasons.push("full_bundle_not_evaluated");
982
- return emptyAdditionDecision(addition, additionComponentId, reasons);
983
- }
984
- const bundleSummary = summaryById.get(bundle.candidateId);
985
- if (bundleSummary.outcome.missingTaskIds.length > 0 || bundleSummary.outcome.invalidTaskIds.length > 0 || bundleSummary.outcome.comparisonMissingTaskIds.length > 0) {
986
- reasons.push("bundle_incomplete");
987
- }
988
- if (bundleSummary.outcome.regressionTaskIds.length > 0) reasons.push("baseline_regression");
989
- const comparison = compareCandidates(
990
- context,
991
- summaryById,
992
- current.candidate.candidateId,
993
- bundle.candidateId
994
- );
995
- if (comparison.winsTaskIds.length === 0) reasons.push("no_incremental_resolution");
996
- if (comparison.regressionTaskIds.length > 0) reasons.push("incremental_regression");
997
- appendBundleEvidenceReasons(
998
- reasons,
999
- bundleSummary.firing,
1000
- context.input.selection.minimumFiringTasks,
1001
- context.input.selection.requireObservedFiring,
1002
- "firing_below_minimum",
1003
- "firing_unobserved"
1004
- );
1005
- appendBundleEvidenceReasons(
1006
- reasons,
1007
- bundleSummary.effect,
1008
- context.input.selection.minimumEffectTasks,
1009
- context.input.selection.requireObservedEffect,
1010
- "effect_below_minimum",
1011
- "effect_unobserved"
1012
- );
1013
- if (!withinCostLimits(context, bundleSummary, baseline)) reasons.push("cost_limit_exceeded");
1014
- return {
1015
- additionCandidateId: addition.candidate.candidateId,
1016
- additionComponentId,
1017
- bundleCandidateId: bundle.candidateId,
1018
- incrementalResolutionTaskIds: comparison.winsTaskIds,
1019
- incrementalRegressionTaskIds: comparison.regressionTaskIds,
1020
- incrementalMedianCost: Object.fromEntries(
1021
- context.input.costMetricOrder.map((metric) => [
1022
- metric,
1023
- bundleSummary.costs[metric].median - current.costs[metric].median
1024
- ])
1025
- ),
1026
- eligible: reasons.length === 0,
1027
- selected: false,
1028
- reasons: unique(reasons)
1029
- };
1030
- }
1031
- function emptyAdditionDecision(addition, additionComponentId, reasons) {
1032
- return {
1033
- additionCandidateId: addition.candidate.candidateId,
1034
- additionComponentId,
1035
- bundleCandidateId: null,
1036
- incrementalResolutionTaskIds: [],
1037
- incrementalRegressionTaskIds: [],
1038
- incrementalMedianCost: null,
1039
- eligible: false,
1040
- selected: false,
1041
- reasons: unique(reasons)
1042
- };
1043
- }
1044
- function appendBundleEvidenceReasons(reasons, evidence, minimum, requireObserved, belowReason, unobservedReason) {
1045
- if (evidence.byComponent.some((component) => component.observedTaskIds.length < minimum)) {
1046
- reasons.push(belowReason);
1047
- }
1048
- if (requireObserved && evidence.byComponent.some((component) => component.unobservedTaskIds.length > 0)) {
1049
- reasons.push(unobservedReason);
1050
- }
1051
- }
1052
- function compareAdditionDecisions(context, summaryById, left, right) {
1053
- const byWins = right.incrementalResolutionTaskIds.length - left.incrementalResolutionTaskIds.length;
1054
- if (byWins !== 0) return byWins;
1055
- for (const metric of context.input.costMetricOrder) {
1056
- const byCost = left.incrementalMedianCost[metric] - right.incrementalMedianCost[metric];
1057
- if (byCost !== 0) return byCost;
1058
- }
1059
- const leftBytes = summaryById.get(left.additionCandidateId).candidate.artifactBytes;
1060
- const rightBytes = summaryById.get(right.additionCandidateId).candidate.artifactBytes;
1061
- const byBytes = leftBytes - rightBytes;
1062
- if (byBytes !== 0) return byBytes;
1063
- return context.candidateIndex.get(left.additionCandidateId) - context.candidateIndex.get(right.additionCandidateId);
1064
- }
1065
- function compareCandidates(context, summaryById, comparatorCandidateId, treatmentCandidateId) {
1066
- const winsTaskIds = [];
1067
- const regressionTaskIds = [];
1068
- const missingTaskIds = [];
1069
- for (const taskId of context.input.taskOrder) {
1070
- const comparator = crossSurfaceRowFor(context, comparatorCandidateId, taskId);
1071
- const treatment = crossSurfaceRowFor(context, treatmentCandidateId, taskId);
1072
- if (comparator.completeness !== "complete" || treatment.completeness !== "complete") {
1073
- missingTaskIds.push(taskId);
1074
- } else if (treatment.pass && !comparator.pass) winsTaskIds.push(taskId);
1075
- else if (!treatment.pass && comparator.pass) regressionTaskIds.push(taskId);
1076
- }
1077
- return {
1078
- comparatorCandidateId,
1079
- treatmentCandidateId,
1080
- winsTaskIds,
1081
- regressionTaskIds,
1082
- missingTaskIds,
1083
- paired: pairedComparison(context, comparatorCandidateId, treatmentCandidateId),
1084
- relativeCost: relativeCosts(
1085
- context,
1086
- summaryById.get(treatmentCandidateId),
1087
- summaryById.get(comparatorCandidateId)
1088
- )
1089
- };
1090
- }
1091
- function pairedComparison(context, comparatorCandidateId, treatmentCandidateId) {
1092
- const rows = [];
1093
- for (const taskId of context.input.taskOrder) {
1094
- rows.push(toPairedArmRow(context, crossSurfaceRowFor(context, comparatorCandidateId, taskId)));
1095
- rows.push(toPairedArmRow(context, crossSurfaceRowFor(context, treatmentCandidateId, taskId)));
1096
- }
1097
- return comparePairedArms(rows, {
1098
- baselineArm: comparatorCandidateId,
1099
- treatmentArm: treatmentCandidateId,
1100
- metricNames: ["score", ...context.input.costMetricOrder],
1101
- bootstrap: { ...context.input.bootstrap, statistic: "mean" }
1102
- });
1103
- }
1104
- function toPairedArmRow(context, row) {
1105
- const metrics = Object.fromEntries(
1106
- context.input.costMetricOrder.map((metric) => [metric, row.cost[metric]])
1107
- );
1108
- if (row.completeness === "complete") metrics.score = row.score;
1109
- return {
1110
- pairKey: row.taskId,
1111
- arm: row.candidateId,
1112
- ...row.completeness === "complete" ? { pass: row.pass } : {},
1113
- metrics
1114
- };
1115
- }
1116
- function summarizePairEvidence(rows, leftComponentId, rightComponentId, field) {
1117
- const bothTaskIds = [];
1118
- const leftOnlyTaskIds = [];
1119
- const rightOnlyTaskIds = [];
1120
- const neitherTaskIds = [];
1121
- const unobservedTaskIds = [];
1122
- for (const row of rows) {
1123
- const left = evidenceValue(row, leftComponentId, field);
1124
- const right = evidenceValue(row, rightComponentId, field);
1125
- if (left === null || right === null) unobservedTaskIds.push(row.taskId);
1126
- else if (left && right) bothTaskIds.push(row.taskId);
1127
- else if (left) leftOnlyTaskIds.push(row.taskId);
1128
- else if (right) rightOnlyTaskIds.push(row.taskId);
1129
- else neitherTaskIds.push(row.taskId);
1130
- }
1131
- return {
1132
- bothTaskIds,
1133
- leftOnlyTaskIds,
1134
- rightOnlyTaskIds,
1135
- neitherTaskIds,
1136
- unobservedTaskIds
1137
- };
1138
- }
1139
- function relativeCosts(context, treatment, comparator) {
1140
- return Object.fromEntries(
1141
- context.input.costMetricOrder.map((metric) => {
1142
- const treatmentMedian = treatment.costs[metric].median;
1143
- const comparatorMedian = comparator.costs[metric].median;
1144
- return [
1145
- metric,
1146
- {
1147
- treatmentMedian,
1148
- comparatorMedian,
1149
- medianDelta: treatmentMedian - comparatorMedian,
1150
- medianRatio: comparatorMedian === 0 ? treatmentMedian === 0 ? 1 : null : treatmentMedian / comparatorMedian
1151
- }
1152
- ];
1153
- })
1154
- );
1155
- }
1156
- function withinCostLimits(context, treatment, baseline) {
1157
- const relative3 = relativeCosts(context, treatment, baseline);
1158
- return Object.entries(context.input.selection.maximumMedianCostRatioToBaseline).every(
1159
- ([metric, limit]) => {
1160
- const ratio = relative3[metric].medianRatio;
1161
- return ratio !== null && ratio <= limit;
1162
- }
1163
- );
1164
- }
1165
- function distribution(values) {
1166
- const sorted = [...values].sort((left, right) => left - right);
1167
- const total = sorted.reduce((sum, value) => sum + value, 0);
1168
- const middle = Math.floor(sorted.length / 2);
1169
- const median = sorted.length % 2 === 1 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2;
1170
- return {
1171
- n: sorted.length,
1172
- min: sorted[0],
1173
- median,
1174
- mean: total / sorted.length,
1175
- max: sorted[sorted.length - 1],
1176
- total
1177
- };
1178
- }
1179
- function evidenceValue(row, componentId, field) {
1180
- return row.componentEvidence.find((evidence) => evidence.componentId === componentId)[field];
1181
- }
1182
- function meanOrNull(values) {
1183
- return values.length === 0 ? null : values.reduce((sum, value) => sum + value, 0) / values.length;
1184
- }
1185
- function unique(values) {
1186
- return [...new Set(values)];
1187
- }
1188
-
1189
- // src/campaign/fixtures.ts
1190
- import { createHash } from "crypto";
1191
- import { existsSync, readdirSync, readFileSync, statSync } from "fs";
1192
- import { isAbsolute, join, relative, resolve } from "path";
1193
- function discoverEvalFixtures(evalsDir) {
1194
- const root = resolve(evalsDir);
1195
- if (!existsSync(root)) throw new Error(`discoverEvalFixtures: evalsDir not found: ${root}`);
1196
- const fixtures = [];
1197
- const walk = (dir, base = "") => {
1198
- for (const entry of readdirSync(dir).sort()) {
1199
- if (entry.startsWith(".") || entry === "node_modules") continue;
1200
- const fullPath = join(dir, entry);
1201
- if (!statSync(fullPath).isDirectory()) continue;
1202
- const name = base ? `${base}/${entry}` : entry;
1203
- if (existsWithExactCase(fullPath, "PROMPT.md")) {
1204
- fixtures.push(name);
1205
- } else {
1206
- walk(fullPath, name);
1207
- }
1208
- }
1209
- };
1210
- walk(root);
1211
- return fixtures;
1212
- }
1213
- function loadEvalFixture(evalsDir, name, options = {}) {
1214
- const validation = options.validation ?? "vitest";
1215
- const root = resolve(evalsDir);
1216
- const fixturePath = resolve(root, name);
1217
- assertInside(root, fixturePath, name);
1218
- if (!existsSync(fixturePath) || !statSync(fixturePath).isDirectory()) {
1219
- throw new Error(`loadEvalFixture: fixture not found: ${name}`);
1220
- }
1221
- if (!existsWithExactCase(fixturePath, "PROMPT.md")) {
1222
- throw new Error(`loadEvalFixture: ${name} is missing exact-case PROMPT.md`);
1223
- }
1224
- const promptPath = join(fixturePath, "PROMPT.md");
1225
- const evalPath = resolveEvalPath(fixturePath);
1226
- const packageJsonPath = existsWithExactCase(fixturePath, "package.json") ? join(fixturePath, "package.json") : void 0;
1227
- if (validation !== "none") {
1228
- if (!evalPath)
1229
- throw new Error(`loadEvalFixture: ${name} is missing exact-case EVAL.ts or EVAL.tsx`);
1230
- if (!packageJsonPath)
1231
- throw new Error(`loadEvalFixture: ${name} is missing exact-case package.json`);
1232
- assertModulePackage(packageJsonPath, name);
1233
- }
1234
- const files = collectFixtureFiles(fixturePath);
1235
- return {
1236
- name,
1237
- path: fixturePath,
1238
- promptPath,
1239
- evalPath,
1240
- packageJsonPath,
1241
- prompt: readFileSync(promptPath, "utf8"),
1242
- files,
1243
- fingerprint: contentHash({
1244
- files,
1245
- config: options.fingerprintConfig ?? null
1246
- })
1247
- };
1248
- }
1249
- function loadEvalFixtureScenarios(evalsDir, options = {}) {
1250
- const names = options.names ?? discoverEvalFixtures(evalsDir);
1251
- return names.map((name) => {
1252
- const fixture = loadEvalFixture(evalsDir, name, options);
1253
- return {
1254
- id: fixture.name,
1255
- kind: "eval-fixture",
1256
- tags: ["eval-fixture"],
1257
- fixtureName: fixture.name,
1258
- fixturePath: fixture.path,
1259
- promptPath: fixture.promptPath,
1260
- evalPath: fixture.evalPath,
1261
- packageJsonPath: fixture.packageJsonPath,
1262
- prompt: fixture.prompt,
1263
- fingerprint: fixture.fingerprint
1264
- };
1265
- });
1266
- }
1267
- function planEvalFixtureRun(options) {
1268
- const scenarios = loadEvalFixtureScenarios(options.evalsDir, {
1269
- names: options.names,
1270
- validation: options.validation,
1271
- fingerprintConfig: options.fingerprintConfig
1272
- });
1273
- const plan = planCampaignRun({
1274
- scenarios,
1275
- dispatchRef: options.dispatchRef ?? "eval-fixture-dispatch",
1276
- judges: options.judges,
1277
- seed: options.seed,
1278
- reps: options.reps,
1279
- resumable: options.resumable,
1280
- runDir: options.runDir,
1281
- storage: options.storage
1282
- });
1283
- return {
1284
- ...plan,
1285
- fixtures: scenarios.map((scenario) => ({
1286
- fixtureName: scenario.fixtureName,
1287
- fixturePath: scenario.fixturePath,
1288
- fingerprint: scenario.fingerprint
1289
- }))
1290
- };
1291
- }
1292
- function existsWithExactCase(dirPath, fileName) {
1293
- try {
1294
- return readdirSync(dirPath).includes(fileName);
1295
- } catch {
1296
- return false;
1297
- }
1298
- }
1299
- function resolveEvalPath(fixturePath) {
1300
- if (existsWithExactCase(fixturePath, "EVAL.ts")) return join(fixturePath, "EVAL.ts");
1301
- if (existsWithExactCase(fixturePath, "EVAL.tsx")) return join(fixturePath, "EVAL.tsx");
1302
- return void 0;
1303
- }
1304
- function assertModulePackage(packageJsonPath, name) {
1305
- let parsed;
1306
- try {
1307
- parsed = JSON.parse(readFileSync(packageJsonPath, "utf8"));
1308
- } catch (err) {
1309
- throw new Error(
1310
- `loadEvalFixture: ${name} package.json is invalid JSON: ${err instanceof Error ? err.message : String(err)}`
1311
- );
1312
- }
1313
- if (typeof parsed !== "object" || parsed === null || parsed.type !== "module") {
1314
- throw new Error(`loadEvalFixture: ${name} package.json must set "type": "module"`);
1315
- }
1316
- }
1317
- function collectFixtureFiles(fixturePath, base = "") {
1318
- const files = [];
1319
- for (const entry of readdirSync(join(fixturePath, base)).sort()) {
1320
- if (entry === "node_modules" || entry === ".git") continue;
1321
- const relativePath = base ? `${base}/${entry}` : entry;
1322
- const fullPath = join(fixturePath, relativePath);
1323
- const stat = statSync(fullPath);
1324
- if (stat.isDirectory()) {
1325
- files.push(...collectFixtureFiles(fixturePath, relativePath));
1326
- continue;
1327
- }
1328
- const bytes = readFileSync(fullPath);
1329
- files.push({
1330
- path: relativePath,
1331
- sha256: createHash("sha256").update(bytes).digest("hex"),
1332
- bytes: bytes.byteLength
1333
- });
1334
- }
1335
- return files;
1336
- }
1337
- function assertInside(root, target, label) {
1338
- const rel = relative(root, target);
1339
- if (rel === "" || !rel.startsWith("..") && !isAbsolute(rel)) return;
1340
- throw new Error(`loadEvalFixture: fixture path escapes evalsDir: ${label}`);
1341
- }
1342
-
1343
- // src/campaign/gates/neutralization-gate.ts
1344
- function mean(xs) {
1345
- return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length;
1346
- }
1347
- function pairedLift(arm, baseline, scenarioIds) {
1348
- const paired = pairHoldout(arm, baseline, scenarioIds, (s) => s.composite);
1349
- const deltas = paired.after.map((a, i) => a - (paired.before[i] ?? 0));
1350
- return { lift: mean(deltas), n: deltas.length };
1351
- }
1352
- function neutralizationGate(options) {
1353
- const maxDecorativeFraction = options.maxDecorativeFraction ?? 0.5;
1354
- return {
1355
- name: "neutralizationGate",
1356
- async decide(ctx) {
1357
- if (!ctx.baselineJudgeScores) {
1358
- throw new Error(
1359
- "neutralizationGate: ctx.baselineJudgeScores is required \u2014 the placebo control measures lift OVER baseline."
1360
- );
1361
- }
1362
- if (!ctx.neutralizedJudgeScores) {
1363
- throw new Error(
1364
- "neutralizationGate: ctx.neutralizedJudgeScores is required. It is populated by runImprovementLoop only when a `neutralize` function is supplied \u2014 composing this gate without that wiring would pass an unproven candidate."
1365
- );
1366
- }
1367
- const scenarioIds = new Set(options.scenarios.map((s) => s.id));
1368
- const cand = pairedLift(
1369
- ctx.judgeScores,
1370
- ctx.baselineJudgeScores,
1371
- scenarioIds
1372
- );
1373
- const neut = pairedLift(
1374
- ctx.neutralizedJudgeScores,
1375
- ctx.baselineJudgeScores,
1376
- scenarioIds
1377
- );
1378
- if (cand.lift <= 0) {
1379
- return {
1380
- decision: "hold",
1381
- reasons: [
1382
- `neutralization: candidate held-out lift ${cand.lift.toFixed(3)} \u2264 0 \u2014 no positive lift to attribute to content`
1383
- ],
1384
- contributingGates: [
1385
- {
1386
- name: "neutralizationGate",
1387
- status: "fail",
1388
- detail: { candidateLift: cand.lift, neutralizedLift: neut.lift, n: cand.n }
1389
- }
1390
- ],
1391
- delta: cand.lift
1392
- };
1393
- }
1394
- const decorativeFraction = neut.lift / cand.lift;
1395
- const passed = decorativeFraction < maxDecorativeFraction;
1396
- const pct = (decorativeFraction * 100).toFixed(0);
1397
- return {
1398
- decision: passed ? "ship" : "hold",
1399
- reasons: passed ? [
1400
- `neutralization: content is causal \u2014 blanked variant reproduces ${pct}% of the lift (< ${(maxDecorativeFraction * 100).toFixed(0)}%); candidate \u0394 ${cand.lift.toFixed(3)}, neutralized \u0394 ${neut.lift.toFixed(3)}`
1401
- ] : [
1402
- `neutralization: lift is DECORATIVE \u2014 blanking the content (footprint-matched) reproduces ${pct}% of the lift (\u2265 ${(maxDecorativeFraction * 100).toFixed(0)}%); candidate \u0394 ${cand.lift.toFixed(3)}, neutralized \u0394 ${neut.lift.toFixed(3)}`
1403
- ],
1404
- contributingGates: [
1405
- {
1406
- name: "neutralizationGate",
1407
- status: passed ? "pass" : "fail",
1408
- detail: {
1409
- candidateLift: cand.lift,
1410
- neutralizedLift: neut.lift,
1411
- decorativeFraction,
1412
- maxDecorativeFraction,
1413
- n: cand.n
1414
- }
1415
- }
1416
- ],
1417
- delta: cand.lift
1418
- };
1419
- }
1420
- };
1421
- }
1422
-
1423
- // src/campaign/gates/sequential.ts
1424
- import { createHash as createHash2 } from "crypto";
1425
- function verifyManifestSync(m) {
1426
- if (m.algo !== void 0 && m.algo !== "sha256-content") {
1427
- throw new Error(`sequentialPairedGate: unrecognized manifest hash algo '${m.algo}'`);
1428
- }
1429
- const { contentHash: contentHash2, algo: _algo, ...rest } = m;
1430
- void _algo;
1431
- const bytes = JSON.stringify(canonicalize(rest));
1432
- const hash = createHash2("sha256").update(bytes, "utf8").digest("hex");
1433
- return hash === contentHash2;
1434
- }
1435
- function resolveConfig(opts) {
1436
- const m = opts.preRegistration;
1437
- let alpha;
1438
- let maxN;
1439
- let direction;
1440
- let minEffect;
1441
- if (m) {
1442
- if (!verifyManifestSync(m)) {
1443
- throw new Error(
1444
- `sequentialPairedGate: pre-registration manifest '${m.id}' content hash mismatch (tampered)`
1445
- );
1446
- }
1447
- if (opts.alpha !== void 0 && opts.alpha !== m.alpha) {
1448
- throw new Error(
1449
- `sequentialPairedGate: alpha ${opts.alpha} conflicts with pre-registered alpha ${m.alpha} \u2014 the registered statistic is the only one anytime validity covers`
1450
- );
1451
- }
1452
- if (opts.maxN !== void 0 && opts.maxN !== m.preRegisteredN) {
1453
- throw new Error(
1454
- `sequentialPairedGate: maxN ${opts.maxN} conflicts with pre-registered N ${m.preRegisteredN}`
1455
- );
1456
- }
1457
- alpha = m.alpha;
1458
- maxN = m.preRegisteredN;
1459
- direction = m.direction;
1460
- minEffect = m.minEffect;
1461
- } else {
1462
- alpha = opts.alpha ?? 0.05;
1463
- if (opts.maxN === void 0) {
1464
- throw new Error(
1465
- "sequentialPairedGate: maxN is required (or bind a preRegistration manifest whose preRegisteredN is the budget) \u2014 an unbounded stream has no pre-registered budget"
1466
- );
1467
- }
1468
- maxN = opts.maxN;
1469
- direction = "increase";
1470
- minEffect = 0;
1471
- }
1472
- const scale = opts.scale ?? 1;
1473
- if (!Number.isFinite(scale) || scale <= 0) {
1474
- throw new Error(`sequentialPairedGate: scale must be > 0, got ${scale}`);
1475
- }
1476
- if (!Number.isInteger(maxN) || maxN < 1) {
1477
- throw new Error(`sequentialPairedGate: maxN must be a positive integer, got ${maxN}`);
1478
- }
1479
- const minN = opts.minN ?? 5;
1480
- if (!Number.isInteger(minN) || minN < 1 || minN > maxN) {
1481
- throw new Error(
1482
- `sequentialPairedGate: minN must be an integer in [1, maxN=${maxN}], got ${minN}`
1483
- );
1484
- }
1485
- if (!Number.isFinite(minEffect) || minEffect < 0 || minEffect >= scale) {
1486
- throw new Error(
1487
- `sequentialPairedGate: minEffect must be in [0, scale=${scale}), got ${minEffect}`
1488
- );
1489
- }
1490
- const nullMean = 0.5 + minEffect / (2 * scale);
1491
- return {
1492
- alpha,
1493
- minN,
1494
- maxN,
1495
- maxBet: opts.maxBet ?? 0.5,
1496
- scale,
1497
- shuffleSeed: opts.shuffleSeed ?? 1337,
1498
- direction,
1499
- nullMean,
1500
- minEffect
1501
- };
1502
- }
1503
- function makeStream(cfg) {
1504
- const proc = eProcess({ alpha: cfg.alpha, maxBet: cfg.maxBet, nullMean: cfg.nullMean });
1505
- const threshold = 1 / cfg.alpha;
1506
- let terminal;
1507
- return {
1508
- observe(delta) {
1509
- if (terminal === "undecided-at-maxN") {
1510
- throw new Error(
1511
- `sequentialPairedGate: pre-registered maxN=${cfg.maxN} exhausted \u2014 extending the stream after seeing the result reopens optional stopping; start a NEW pre-registered test`
1512
- );
1513
- }
1514
- if (!Number.isFinite(delta) || Math.abs(delta) > cfg.scale) {
1515
- throw new Error(
1516
- `sequentialPairedGate: delta ${delta} outside \xB1scale=${cfg.scale} \u2014 pass the judge's native scale explicitly (detectScale helps pick 1 vs 100)`
1517
- );
1518
- }
1519
- const d = cfg.direction === "decrease" ? -delta : delta;
1520
- const step = proc.update((d / cfg.scale + 1) / 2);
1521
- if (terminal === void 0 && step.n >= cfg.minN && step.wealth >= threshold) {
1522
- terminal = "promote";
1523
- }
1524
- if (terminal === "promote") {
1525
- return {
1526
- decision: "promote",
1527
- eValue: step.wealth,
1528
- n: step.n,
1529
- reason: `e-value ${step.wealth.toFixed(2)} \u2265 1/\u03B1=${threshold.toFixed(2)} at n=${step.n} (minN=${cfg.minN}): the paired improvement exceeds ${cfg.minEffect} at anytime-valid level \u03B1=${cfg.alpha}`
1530
- };
1531
- }
1532
- if (step.n >= cfg.maxN) {
1533
- terminal = "undecided-at-maxN";
1534
- return {
1535
- decision: "undecided-at-maxN",
1536
- eValue: step.wealth,
1537
- n: step.n,
1538
- reason: `undecided at pre-registered maxN=${cfg.maxN} (e-value ${step.wealth.toFixed(2)} < 1/\u03B1=${threshold.toFixed(2)}). This is NOT evidence of no effect \u2014 the effect may be real but smaller than this budget can detect; re-register with a larger N to test that`
1539
- };
1540
- }
1541
- return {
1542
- decision: "continue",
1543
- eValue: step.wealth,
1544
- n: step.n,
1545
- reason: `e-value ${step.wealth.toFixed(2)} < 1/\u03B1=${threshold.toFixed(2)} at n=${step.n}/${cfg.maxN} \u2014 keep observing`
1546
- };
1547
- },
1548
- state() {
1549
- return { ...proc.state(), decision: terminal ?? "continue" };
1550
- }
1551
- };
1552
- }
1553
- function seededShuffle(items, seed) {
1554
- const rng = mulberry32(seed);
1555
- for (let i = items.length - 1; i > 0; i--) {
1556
- const j = Math.floor(rng() * (i + 1));
1557
- const tmp = items[i];
1558
- items[i] = items[j];
1559
- items[j] = tmp;
1560
- }
1561
- return items;
1562
- }
1563
- function sequentialPairedGate(options) {
1564
- const cfg = resolveConfig(options);
1565
- const name = options.name ?? "sequentialPairedGate";
1566
- const manifest = options.preRegistration;
1567
- const observeStream = makeStream(cfg);
1568
- return {
1569
- name,
1570
- observe(delta) {
1571
- return observeStream.observe(delta);
1572
- },
1573
- state() {
1574
- return observeStream.state();
1575
- },
1576
- async decide(ctx) {
1577
- if (!ctx.baselineJudgeScores) {
1578
- throw new Error(
1579
- `${name}: ctx.baselineJudgeScores is required \u2014 falling back to the candidate's own scores would compare the candidate against itself (delta 0, silent no-op)`
1580
- );
1581
- }
1582
- const scenarioIds = new Set(ctx.scenarios.map((s) => s.id));
1583
- const paired = pairHoldout(
1584
- ctx.judgeScores,
1585
- ctx.baselineJudgeScores,
1586
- scenarioIds,
1587
- (s) => s.composite
1588
- );
1589
- const deltas = paired.after.map((a, i) => a - paired.before[i]);
1590
- seededShuffle(deltas, cfg.shuffleSeed);
1591
- const stream = makeStream(cfg);
1592
- let last;
1593
- for (const d of deltas) {
1594
- last = stream.observe(d);
1595
- if (last.decision !== "continue") break;
1596
- }
1597
- const detail = {
1598
- ...stream.state(),
1599
- minN: cfg.minN,
1600
- maxN: cfg.maxN,
1601
- scale: cfg.scale,
1602
- shuffleSeed: cfg.shuffleSeed,
1603
- direction: cfg.direction,
1604
- minEffect: cfg.minEffect,
1605
- pairedN: deltas.length,
1606
- ...manifest ? { preRegisteredId: manifest.id, metric: manifest.metric } : {}
1607
- };
1608
- const meanDelta = deltas.length === 0 ? void 0 : deltas.reduce((s, d) => s + d, 0) / deltas.length;
1609
- if (last === void 0) {
1610
- return {
1611
- decision: "need_more_work",
1612
- reasons: [`${name}: no paired holdout observations \u2014 nothing to test`],
1613
- contributingGates: [{ name, status: "not_evaluated", detail }]
1614
- };
1615
- }
1616
- const decision = last.decision === "promote" ? "ship" : last.decision === "continue" ? "need_more_work" : "hold";
1617
- return {
1618
- decision,
1619
- reasons: [`${name}: ${last.reason}`],
1620
- contributingGates: [
1621
- {
1622
- name,
1623
- status: decision === "ship" ? "pass" : decision === "need_more_work" ? "not_evaluated" : "fail",
1624
- detail
1625
- }
1626
- ],
1627
- delta: meanDelta
1628
- };
1629
- }
1630
- };
1631
- }
1632
- function sequentialDecide(options = {}) {
1633
- const alpha = options.alpha ?? 0.05;
1634
- const minN = options.minN ?? 5;
1635
- const scale = options.scale ?? 1;
1636
- if (!Number.isFinite(scale) || scale <= 0) {
1637
- throw new Error(`sequentialDecide: scale must be > 0, got ${scale}`);
1638
- }
1639
- const proc = eProcess({ alpha, maxBet: options.maxBet ?? 0.5, nullMean: 0.5 });
1640
- const threshold = 1 / alpha;
1641
- let processedGenerations = 0;
1642
- let reference;
1643
- let stopped;
1644
- const topCandidate = (record) => {
1645
- if (record.candidates.length === 0) {
1646
- throw new Error(
1647
- `sequentialDecide: generation ${record.generationIndex} has no candidates \u2014 cannot extract a top candidate`
1648
- );
1649
- }
1650
- return record.candidates.reduce((best, candidate) => {
1651
- if (best.composite === null) return candidate;
1652
- if (candidate.composite === null) return best;
1653
- return candidate.composite > best.composite ? candidate : best;
1654
- });
1655
- };
1656
- const decide = ({ history }) => {
1657
- if (stopped) return stopped;
1658
- if (history.length === 0) return { stop: false };
1659
- if (reference === void 0) {
1660
- reference = new Map(
1661
- topCandidate(history[0]).scenarios.map((s) => [s.scenarioId, s.composite])
1662
- );
1663
- }
1664
- for (let g = Math.max(1, processedGenerations); g < history.length; g++) {
1665
- const top = topCandidate(history[g]);
1666
- const byScenario = new Map(top.scenarios.map((s) => [s.scenarioId, s.composite]));
1667
- for (const [scenarioId, refComposite] of reference) {
1668
- const candComposite = byScenario.get(scenarioId);
1669
- if (candComposite === void 0) {
1670
- throw new Error(
1671
- `sequentialDecide: generation ${history[g].generationIndex} top candidate is missing scenario '${scenarioId}' \u2014 generations must score the same scenario set to pair`
1672
- );
1673
- }
1674
- const delta = candComposite - refComposite;
1675
- if (!Number.isFinite(delta) || Math.abs(delta) > scale) {
1676
- throw new Error(
1677
- `sequentialDecide: paired delta ${delta} outside \xB1scale=${scale} on scenario '${scenarioId}' \u2014 pass the composite scale explicitly`
1678
- );
1679
- }
1680
- const step = proc.update((delta / scale + 1) / 2);
1681
- if (step.n >= minN && step.wealth >= threshold) {
1682
- stopped = {
1683
- stop: true,
1684
- reason: `sequential e-process decided at generation ${history[g].generationIndex}: e-value ${step.wealth.toFixed(2)} \u2265 1/\u03B1=${threshold.toFixed(2)} after n=${step.n} paired deltas vs the generation-0 incumbent \u2014 the improvement is real at \u03B1=${alpha}; stop exploring and promote via the gate`
1685
- };
1686
- processedGenerations = history.length;
1687
- return stopped;
1688
- }
1689
- }
1690
- }
1691
- processedGenerations = history.length;
1692
- return { stop: false };
1693
- };
1694
- decide.state = () => proc.state();
1695
- return decide;
1696
- }
1697
-
1698
- // src/campaign/grounded-reflection.ts
1699
- function rolloutArgumentDiff(rollouts, opts = {}) {
1700
- const passThreshold = opts.passThreshold ?? 1;
1701
- const maxValues = opts.maxValuesPerField ?? 4;
1702
- const collect = (pass) => {
1703
- const byField = /* @__PURE__ */ new Map();
1704
- for (const r of rollouts) {
1705
- if (pass !== r.score >= passThreshold) continue;
1706
- for (const c of r.calls) {
1707
- for (const [k, v] of Object.entries(c.args)) {
1708
- const set = byField.get(k) ?? /* @__PURE__ */ new Set();
1709
- set.add(String(v));
1710
- byField.set(k, set);
1711
- }
1712
- }
1713
- }
1714
- return byField;
1715
- };
1716
- const passing = collect(true);
1717
- const failing = collect(false);
1718
- const lines = [];
1719
- for (const field of /* @__PURE__ */ new Set([...passing.keys(), ...failing.keys()])) {
1720
- const pv = [...passing.get(field) ?? []].slice(0, maxValues);
1721
- const fv = [...failing.get(field) ?? []].slice(0, maxValues);
1722
- const render = (vals) => vals.length ? JSON.stringify(vals) : "NOT SET (omitted)";
1723
- lines.push(` ${field}: passing runs -> ${render(pv)} | failing runs -> ${render(fv)}`);
1724
- }
1725
- const lower = (m) => {
1726
- const out = /* @__PURE__ */ new Set();
1727
- for (const vals of m.values()) for (const v of vals) out.add(v.toLowerCase());
1728
- return out;
1729
- };
1730
- return {
1731
- text: lines.join("\n") || " (no calls observed)",
1732
- passingValues: lower(passing),
1733
- failingValues: lower(failing)
1734
- };
1735
- }
1736
- function classifyUngroundedLiterals(text, diff) {
1737
- const ungrounded = /* @__PURE__ */ new Set();
1738
- for (const m of text.matchAll(/'([a-z][a-z_-]{1,19})'/gi)) {
1739
- const w = m[1].toLowerCase();
1740
- if (!diff.passingValues.has(w)) ungrounded.add(w);
1741
- }
1742
- const all = [...ungrounded];
1743
- return {
1744
- ungrounded: all,
1745
- harmful: all.filter((w) => diff.failingValues.has(w))
1746
- };
1747
- }
1748
-
1749
- // src/campaign/labeled-store/fs-adapter.ts
1750
- import { createHash as createHash3 } from "crypto";
1751
- import { existsSync as existsSync2, mkdirSync, readFileSync as readFileSync2, writeFileSync } from "fs";
1752
- import { join as join2 } from "path";
1753
- var LabeledScenarioStoreError = class extends Error {
1754
- constructor(code, message) {
1755
- super(message);
1756
- this.code = code;
1757
- this.name = "LabeledScenarioStoreError";
1758
- }
1759
- code;
1760
- };
1761
- var FsLabeledScenarioStore = class {
1762
- constructor(options) {
1763
- this.options = options;
1764
- if (!existsSync2(options.root)) mkdirSync(options.root, { recursive: true });
1765
- this.now = options.now ?? Date.now;
1766
- }
1767
- options;
1768
- now;
1769
- rateLimits = /* @__PURE__ */ new Map();
1770
- async observe(write) {
1771
- this.assertProvenance(write);
1772
- this.assertRateLimit(write);
1773
- const record = this.toRecord(write);
1774
- const path = this.pathForSource(write.source);
1775
- const line = `${JSON.stringify(record)}
1776
- `;
1777
- appendLine(path, line);
1778
- }
1779
- async sample(args) {
1780
- if (!args.split) {
1781
- throw new LabeledScenarioStoreError(
1782
- "split_required",
1783
- "sample() requires an explicit `split` (train | test) \u2014 substrate refuses ambiguous reads"
1784
- );
1785
- }
1786
- if (!args.capturedBefore) {
1787
- throw new LabeledScenarioStoreError(
1788
- "capturedBefore_required",
1789
- "sample() requires an explicit `capturedBefore` timestamp for temporal-split discipline"
1790
- );
1791
- }
1792
- const all = [];
1793
- for (const source of ALL_SOURCES) {
1794
- if (args.split === "train" && source === "production-trace") {
1795
- const explicit = sourceFilterContains(args.filter?.source, "production-trace");
1796
- if (!explicit) continue;
1797
- }
1798
- const path = this.pathForSource(source);
1799
- if (!existsSync2(path)) continue;
1800
- const lines = readFileSync2(path, "utf8").split("\n").filter(Boolean);
1801
- for (const line of lines) {
1802
- let record;
1803
- try {
1804
- record = JSON.parse(line);
1805
- } catch {
1806
- continue;
1807
- }
1808
- if (!matchesFilter(record, args, source)) continue;
1809
- all.push(record);
1810
- }
1811
- }
1812
- all.sort((a, b) => {
1813
- if (a.capturedAt !== b.capturedAt) return a.capturedAt.localeCompare(b.capturedAt);
1814
- return a.recordHash.localeCompare(b.recordHash);
1815
- });
1816
- return all.slice(0, args.count);
1817
- }
1818
- async size() {
1819
- const bySource = {};
1820
- const byTrust = {
1821
- unverified: 0,
1822
- "verified-signal": 0,
1823
- "human-rated": 0
1824
- };
1825
- let total = 0;
1826
- for (const source of ALL_SOURCES) {
1827
- const path = this.pathForSource(source);
1828
- if (!existsSync2(path)) {
1829
- bySource[source] = 0;
1830
- continue;
1831
- }
1832
- const lines = readFileSync2(path, "utf8").split("\n").filter(Boolean);
1833
- bySource[source] = lines.length;
1834
- total += lines.length;
1835
- for (const line of lines) {
1836
- let trust = "unverified";
1837
- try {
1838
- trust = JSON.parse(line).labelTrust ?? "unverified";
1839
- } catch {
1840
- }
1841
- byTrust[trust] += 1;
1842
- }
1843
- }
1844
- return { train: total, test: total, bySource, byTrust };
1845
- }
1846
- assertProvenance(write) {
1847
- if (!write.source) {
1848
- throw new LabeledScenarioStoreError(
1849
- "missing_source",
1850
- "LabeledScenarioWrite requires `source`"
1851
- );
1852
- }
1853
- if (!write.sourceVersionHash || write.sourceVersionHash.length === 0) {
1854
- throw new LabeledScenarioStoreError(
1855
- "missing_source_version",
1856
- "LabeledScenarioWrite requires `sourceVersionHash` (git sha or substrate version)"
1857
- );
1858
- }
1859
- if (!write.capturedAt) {
1860
- throw new LabeledScenarioStoreError(
1861
- "missing_captured_at",
1862
- "LabeledScenarioWrite requires `capturedAt` ISO timestamp"
1863
- );
1864
- }
1865
- if (!write.redactionStatus) {
1866
- throw new LabeledScenarioStoreError(
1867
- "missing_redaction_status",
1868
- "LabeledScenarioWrite requires explicit `redactionStatus` \u2014 raw / redacted-pii / redacted-secrets / fully-redacted"
1869
- );
1870
- }
1871
- if (!ALL_SOURCES.includes(write.source)) {
1872
- throw new LabeledScenarioStoreError(
1873
- "unknown_source",
1874
- `LabeledScenarioWrite.source must be one of: ${ALL_SOURCES.join(", ")}`
1875
- );
1876
- }
1877
- }
1878
- assertRateLimit(write) {
1879
- const cap = this.options.maxWritesPerMinutePerBucket;
1880
- if (!cap || !write.rateLimitBucket) return;
1881
- const now = this.now();
1882
- const windowMs = 6e4;
1883
- let state = this.rateLimits.get(write.rateLimitBucket);
1884
- if (!state || now - state.windowStartMs >= windowMs) {
1885
- state = { bucket: write.rateLimitBucket, windowStartMs: now, count: 0 };
1886
- this.rateLimits.set(write.rateLimitBucket, state);
1887
- }
1888
- if (state.count >= cap) {
1889
- throw new LabeledScenarioStoreError(
1890
- "rate_limit_exceeded",
1891
- `LabeledScenarioStore: bucket ${write.rateLimitBucket} exceeded ${cap} writes/min`
1892
- );
1893
- }
1894
- state.count += 1;
1895
- }
1896
- toRecord(write) {
1897
- const recordHash = sha256(
1898
- JSON.stringify({
1899
- id: write.scenario.id,
1900
- src: write.source,
1901
- at: write.capturedAt,
1902
- ver: write.sourceVersionHash
1903
- })
1904
- );
1905
- return {
1906
- ...write,
1907
- recordHash,
1908
- split: "train"
1909
- };
1910
- }
1911
- pathForSource(source) {
1912
- return join2(this.options.root, `${source}.jsonl`);
1913
- }
1914
- };
1915
- var ALL_SOURCES = [
1916
- "production-trace",
1917
- "eval-run",
1918
- "manual",
1919
- "red-team",
1920
- "synthetic"
1921
- ];
1922
- function sourceFilterContains(filter, needle) {
1923
- if (!filter) return false;
1924
- if (Array.isArray(filter)) return filter.includes(needle);
1925
- return filter === needle;
1926
- }
1927
- function matchesFilter(record, args, source) {
1928
- if (args.split === "train" && record.capturedAt >= args.capturedBefore) return false;
1929
- if (args.split === "test" && record.capturedAt < args.capturedBefore) return false;
1930
- const f = args.filter;
1931
- if (!f) return true;
1932
- if (f.kind && record.scenario.kind !== f.kind) return false;
1933
- if (f.source) {
1934
- const sources = Array.isArray(f.source) ? f.source : [f.source];
1935
- if (!sources.includes(source)) return false;
1936
- }
1937
- if (f.minComposite !== void 0 || f.maxComposite !== void 0) {
1938
- const composites = Object.values(record.judgeScores).map((s) => s.composite);
1939
- const max = composites.length === 0 ? 0 : Math.max(...composites);
1940
- if (f.minComposite !== void 0 && max < f.minComposite) return false;
1941
- if (f.maxComposite !== void 0 && max > f.maxComposite) return false;
1942
- }
1943
- if (f.minTrust !== void 0 && labelTrustRank(record.labelTrust) < labelTrustRank(f.minTrust)) {
1944
- return false;
1945
- }
1946
- return true;
1947
- }
1948
- function sha256(input) {
1949
- return createHash3("sha256").update(input).digest("hex").slice(0, 16);
1950
- }
1951
- function appendLine(path, line) {
1952
- if (existsSync2(path)) {
1953
- const existing = readFileSync2(path, "utf8");
1954
- writeFileSync(path, existing + line);
1955
- } else {
1956
- writeFileSync(path, line);
1957
- }
1958
- }
1959
-
1960
- // src/campaign/neutralize.ts
1961
- var FILLER = "#";
1962
- function neutralizeText(content) {
1963
- return content.replace(/\S/g, FILLER);
1964
- }
1965
-
1966
- // src/completion-verifier.ts
1967
- import { randomUUID } from "crypto";
1968
- function completionVerdict(input) {
1969
- if (input.requirements.length === 0) {
1970
- throw new Error(
1971
- `completionVerdict: task '${input.taskId}' has no requirement checks \u2014 nothing to derive a verdict from`
1972
- );
1973
- }
1974
- const measurable = input.requirements.filter((r) => !r.unmeasured);
1975
- const unmeasuredCount = input.requirements.length - measurable.length;
1976
- if (measurable.length === 0) {
1977
- throw new Error(
1978
- `completionVerdict: task '${input.taskId}' has no measurable requirements \u2014 all ${input.requirements.length} correctness checks failed (${input.requirements[0]?.unmeasuredReason ?? "unknown reason"})`
1979
- );
1980
- }
1981
- const satisfiedCount = measurable.filter((r) => r.satisfied).length;
1982
- const completionRate = satisfiedCount / measurable.length;
1983
- const fullyComplete = unmeasuredCount === 0 && satisfiedCount === measurable.length;
1984
- return {
1985
- taskId: input.taskId,
1986
- requirements: input.requirements,
1987
- completionRate,
1988
- fullyComplete,
1989
- unmeasuredCount,
1990
- valid: fullyComplete,
1991
- score: completionRate
1992
- };
1993
- }
1994
- var STOPWORDS = /* @__PURE__ */ new Set([
1995
- "the",
1996
- "a",
1997
- "an",
1998
- "of",
1999
- "for",
2000
- "and",
2001
- "or",
2002
- "to",
2003
- "in",
2004
- "on",
2005
- "with",
2006
- "by"
2007
- ]);
2008
- var REQUIREMENT_FORM_STOPWORDS = /* @__PURE__ */ new Set([
2009
- "generated",
2010
- "generate",
2011
- "view",
2012
- "render",
2013
- "rendered",
2014
- "persisted",
2015
- "persist",
2016
- "artifact",
2017
- "file",
2018
- "document",
2019
- "note",
2020
- "proposal",
2021
- "deliverable",
2022
- "output",
2023
- "created",
2024
- "create",
2025
- "produce",
2026
- "produced",
2027
- "flag"
2028
- ]);
2029
- var MATCH_THRESHOLD = 0.5;
2030
- var MIN_CONTENT_CHARS = 50;
2031
- function tokens(s, extraStop) {
2032
- return new Set(
2033
- s.toLowerCase().split(/[^a-z0-9]+/).filter((t) => t.length > 1 && !STOPWORDS.has(t) && !extraStop?.has(t))
2034
- );
2035
- }
2036
- function tokenRecall(requirementText, candidateText) {
2037
- const req = tokens(requirementText, REQUIREMENT_FORM_STOPWORDS);
2038
- if (req.size === 0) return 0;
2039
- const cand = tokens(candidateText);
2040
- let hit = 0;
2041
- for (const t of req) if (cand.has(t)) hit++;
2042
- return hit / req.size;
2043
- }
2044
- function artifactCandidates(req, reqIndex, artifacts) {
2045
- const reqText = `${req.title} ${req.category ?? ""}`;
2046
- const out = [];
2047
- artifacts.forEach((a, i) => {
2048
- if ((a.content ?? "").trim().length < MIN_CONTENT_CHARS) return;
2049
- let score = tokenRecall(
2050
- reqText,
2051
- `${a.path ?? ""} ${a.kind} ${(a.content ?? "").slice(0, 4e3)}`
2052
- );
2053
- if (req.category && a.kind && req.category.toLowerCase() === a.kind.toLowerCase()) {
2054
- score = Math.max(score, 1);
2055
- }
2056
- if (score < MATCH_THRESHOLD) return;
2057
- out.push({
2058
- reqIndex,
2059
- itemKey: `artifact:${i}`,
2060
- score,
2061
- evidence: `artifact '${a.path ?? a.kind}' matched (token recall ${score.toFixed(2)})`,
2062
- content: a.content ?? null
2063
- });
2064
- });
2065
- return out;
2066
- }
2067
- function proposalCandidates(req, reqIndex, proposals) {
2068
- const reqText = `${req.title} ${req.category ?? ""}`;
2069
- const out = [];
2070
- for (const p of proposals) {
2071
- if (p.status !== "approved") continue;
2072
- const body = (p.content ?? "").trim();
2073
- if (body.length < MIN_CONTENT_CHARS) continue;
2074
- const score = tokenRecall(reqText, `${p.title} ${body}`);
2075
- if (score < MATCH_THRESHOLD) continue;
2076
- out.push({
2077
- reqIndex,
2078
- itemKey: `proposal:${p.id}`,
2079
- score,
2080
- evidence: `approved proposal '${p.title}' matched (token recall ${score.toFixed(2)})`,
2081
- content: body
2082
- });
2083
- }
2084
- return out;
2085
- }
2086
- function toolCallCandidates(req, reqIndex, toolCalls) {
2087
- const out = [];
2088
- toolCalls.forEach((name, i) => {
2089
- const score = tokenRecall(req.title, name);
2090
- if (score < MATCH_THRESHOLD) return;
2091
- out.push({
2092
- reqIndex,
2093
- itemKey: `tool:${i}`,
2094
- score,
2095
- evidence: `tool call '${name}' matched (token recall ${score.toFixed(2)})`,
2096
- content: null
2097
- });
2098
- });
2099
- return out;
2100
- }
2101
- async function verifyCompletion(gold, state, checkCorrectness) {
2102
- if (gold.requirements.length === 0) {
2103
- throw new Error(
2104
- `verifyCompletion: task '${gold.taskId}' has no requirements \u2014 malformed gold spec`
2105
- );
2106
- }
2107
- const candidates = [];
2108
- gold.requirements.forEach((req, i) => {
2109
- const by = req.satisfiedBy ?? "any";
2110
- if (by === "artifact" || by === "any") {
2111
- candidates.push(...artifactCandidates(req, i, state.artifacts));
2112
- }
2113
- if (by === "proposal" || by === "any") {
2114
- candidates.push(...proposalCandidates(req, i, state.proposals));
2115
- }
2116
- if (by === "tool-call" || by === "any") {
2117
- candidates.push(...toolCallCandidates(req, i, state.toolCalls));
2118
- }
2119
- });
2120
- candidates.sort((a, b) => b.score - a.score);
2121
- const assigned = /* @__PURE__ */ new Map();
2122
- const itemTaken = /* @__PURE__ */ new Set();
2123
- for (const c of candidates) {
2124
- if (assigned.has(c.reqIndex) || itemTaken.has(c.itemKey)) continue;
2125
- assigned.set(c.reqIndex, c);
2126
- itemTaken.add(c.itemKey);
2127
- }
2128
- const requirements = [];
2129
- for (let i = 0; i < gold.requirements.length; i++) {
2130
- const req = gold.requirements[i];
2131
- const match = assigned.get(i);
2132
- const evidence = [];
2133
- let correct = null;
2134
- let unmeasuredReason;
2135
- if (match) {
2136
- evidence.push(match.evidence);
2137
- if (match.content !== null) {
2138
- try {
2139
- const r = await checkCorrectness(req, match.content);
2140
- correct = r.correct;
2141
- evidence.push(`correctness: ${r.correct ? "pass" : "fail"} \u2014 ${r.reason}`);
2142
- } catch (err) {
2143
- unmeasuredReason = err instanceof JudgeParseError ? `checker response unparseable after retry: ${err.raw.slice(0, 200)}` : `checker call failed: ${err instanceof Error ? err.message : String(err)}`;
2144
- evidence.push(`correctness: UNMEASURED \u2014 ${unmeasuredReason}`);
2145
- }
2146
- } else {
2147
- evidence.push("correctness: not assessed \u2014 matched item carries no content");
2148
- }
2149
- } else {
2150
- const by = req.satisfiedBy ?? "any";
2151
- const kind = by === "any" ? "artifact/proposal/tool-call" : by;
2152
- evidence.push(`no produced ${kind} matched this requirement`);
2153
- }
2154
- const structurallyPresent = match !== void 0;
2155
- const unmeasured = unmeasuredReason !== void 0;
2156
- const satisfied = structurallyPresent && !unmeasured && correct !== false;
2157
- requirements.push({
2158
- reqId: req.reqId,
2159
- title: req.title,
2160
- structurallyPresent,
2161
- correct,
2162
- satisfied,
2163
- ...unmeasured ? { unmeasured: true, unmeasuredReason } : {},
2164
- evidence
2165
- });
2166
- }
2167
- return completionVerdict({ taskId: gold.taskId, requirements });
2168
- }
2169
- function parseCorrectnessResponse(raw) {
2170
- const readVerdict = (candidate) => {
2171
- if (candidate === null || typeof candidate !== "object") return null;
2172
- const { correct, reason } = candidate;
2173
- if (typeof correct !== "boolean") return null;
2174
- return { correct, reason: typeof reason === "string" ? reason : "" };
2175
- };
2176
- const match = raw.match(/\{[\s\S]*\}/);
2177
- if (match) {
2178
- try {
2179
- const strict = readVerdict(JSON.parse(match[0]));
2180
- if (strict) return strict;
2181
- } catch {
2182
- }
2183
- }
2184
- const start = raw.indexOf("{");
2185
- if (start !== -1) {
2186
- const recovered = readVerdict(recoverTruncatedJson(raw.slice(start)));
2187
- if (recovered) return recovered;
2188
- }
2189
- throw new JudgeParseError("correctness-checker", raw);
2190
- }
2191
- function createLlmCorrectnessChecker(chat, opts = {}) {
2192
- const model = opts.model ?? "claude-sonnet-4-6";
2193
- const maxContentChars = opts.maxContentChars ?? 8e3;
2194
- const maxAttempts = opts.maxAttempts ?? 2;
2195
- const costLedger = opts.costLedger ?? new CostLedger();
2196
- const sink = opts.rawSink;
2197
- const record = async (event) => {
2198
- try {
2199
- await sink?.record(event);
2200
- } catch {
2201
- }
2202
- };
2203
- return async (requirement, content) => {
2204
- const request = {
2205
- model,
2206
- messages: [
2207
- {
2208
- role: "system",
2209
- content: 'You verify whether a produced work artifact actually fulfils a stated requirement. Judge fulfilment only \u2014 is the deliverable substantively present and on-point \u2014 not polish. A plan to do it later, a vague gesture, or a description of what should be done does NOT fulfil a requirement; the artifact must BE the deliverable. Respond with a single JSON object: {"correct": boolean, "reason": string (<= 30 words)}.'
2210
- },
2211
- {
2212
- role: "user",
2213
- content: `Requirement: ${requirement.title}
2214
- ${requirement.category ? `Category: ${requirement.category}
2215
- ` : ""}
2216
- Produced artifact:
2217
- ${content.slice(0, maxContentChars)}`
2218
- }
2219
- ],
2220
- temperature: 0,
2221
- maxTokens: 200
2222
- };
2223
- let lastErr;
2224
- for (let attempt = 0; attempt < maxAttempts; attempt++) {
2225
- const started = Date.now();
2226
- await record({
2227
- eventId: randomUUID(),
2228
- provider: chat.transport,
2229
- model,
2230
- endpoint: "/chat",
2231
- baseUrl: "",
2232
- attemptIndex: attempt,
2233
- direction: "request",
2234
- timestamp: started,
2235
- requestBody: request,
2236
- redactedFields: []
2237
- });
2238
- try {
2239
- const paid = await costLedger.runPaidCall({
2240
- channel: "verifier",
2241
- phase: opts.costPhase ?? "completion.correctness",
2242
- actor: "correctness-checker",
2243
- model,
2244
- maximumCharge: chat.maximumAttempts === void 0 ? void 0 : maximumChargeForLlmRequest(request, { maximumAttempts: chat.maximumAttempts }),
2245
- tags: {
2246
- ...opts.costTags,
2247
- requirementId: requirement.reqId,
2248
- attempt: String(attempt)
2249
- },
2250
- signal: opts.signal,
2251
- execute: (signal, callId) => chat.chat(request, { signal, idempotencyKey: callId }),
2252
- receipt: costReceiptFromLlm,
2253
- receiptFromError: costReceiptFromLlmError
2254
- });
2255
- if (!paid.succeeded) throw paid.error;
2256
- const resp = paid.value;
2257
- const raw = resp.content;
2258
- await record({
2259
- eventId: randomUUID(),
2260
- provider: chat.transport,
2261
- model,
2262
- endpoint: "/chat",
2263
- baseUrl: "",
2264
- attemptIndex: attempt,
2265
- direction: "response",
2266
- timestamp: Date.now(),
2267
- durationMs: Date.now() - started,
2268
- responseBody: resp,
2269
- redactedFields: []
2270
- });
2271
- return parseCorrectnessResponse(raw);
2272
- } catch (err) {
2273
- lastErr = err;
2274
- await record({
2275
- eventId: randomUUID(),
2276
- provider: chat.transport,
2277
- model,
2278
- endpoint: "/chat",
2279
- baseUrl: "",
2280
- attemptIndex: attempt,
2281
- direction: "error",
2282
- timestamp: Date.now(),
2283
- durationMs: Date.now() - started,
2284
- errorMessage: err instanceof Error ? err.message : String(err),
2285
- redactedFields: []
2286
- });
2287
- }
2288
- }
2289
- throw lastErr instanceof Error ? lastErr : new Error(String(lastErr));
2290
- };
2291
- }
2292
- var TITLE_STOPWORDS = /* @__PURE__ */ new Set([
2293
- "the",
2294
- "a",
2295
- "an",
2296
- "and",
2297
- "or",
2298
- "for",
2299
- "to",
2300
- "of",
2301
- "in",
2302
- "on",
2303
- "with",
2304
- "review",
2305
- "update",
2306
- "new",
2307
- "proposed"
2308
- ]);
2309
- function createTokenRecallChecker(opts = {}) {
2310
- const minRecall = opts.minRecall ?? 0.5;
2311
- const minLen = opts.minContentLength ?? 120;
2312
- return async (requirement, content) => {
2313
- const body = content.trim();
2314
- if (body.length < minLen)
2315
- return {
2316
- correct: false,
2317
- reason: `content too thin (${body.length} chars) to be the deliverable`
2318
- };
2319
- const titleTokens = requirement.title.toLowerCase().split(/[^a-z0-9]+/).filter((t) => t.length > 2 && !TITLE_STOPWORDS.has(t));
2320
- if (titleTokens.length === 0)
2321
- return {
2322
- correct: true,
2323
- reason: "requirement title has no significant tokens \u2014 structural match accepted"
2324
- };
2325
- const lower = body.toLowerCase();
2326
- const hits = titleTokens.filter((t) => lower.includes(t)).length;
2327
- const recall = hits / titleTokens.length;
2328
- return recall >= minRecall ? {
2329
- correct: true,
2330
- reason: `content recalls ${hits}/${titleTokens.length} requirement tokens`
2331
- } : {
2332
- correct: false,
2333
- reason: `content recalls only ${hits}/${titleTokens.length} requirement tokens`
2334
- };
2335
- };
2336
- }
2337
-
2338
- // src/produced-state.ts
2339
- function artifactKind(mimeType) {
2340
- if (!mimeType) return "file";
2341
- if (mimeType.includes("json")) return "json";
2342
- if (mimeType.startsWith("text/")) return "text";
2343
- return "file";
2344
- }
2345
- function extractProducedState(events) {
2346
- const artifacts = [];
2347
- const proposals = [];
2348
- const toolCalls = [];
2349
- const seenTools = /* @__PURE__ */ new Set();
2350
- for (const ev of events) {
2351
- if (ev.type === "tool_call") {
2352
- const name = ev.toolName;
2353
- if (name && !seenTools.has(name)) {
2354
- seenTools.add(name);
2355
- toolCalls.push(name);
2356
- }
2357
- } else if (ev.type === "artifact") {
2358
- const a = ev;
2359
- artifacts.push({
2360
- kind: artifactKind(a.mimeType),
2361
- path: a.name ?? a.uri ?? a.artifactId,
2362
- content: a.content ?? ""
2363
- });
2364
- } else if (ev.type === "proposal_created") {
2365
- const p = ev;
2366
- proposals.push({
2367
- id: p.proposalId,
2368
- title: p.title,
2369
- status: p.status ?? "pending",
2370
- ...p.content !== void 0 ? { content: p.content } : {}
2371
- });
2372
- }
2373
- }
2374
- return { artifacts, proposals, toolCalls };
2375
- }
2376
-
2377
- // src/campaign/presets/playback.ts
2378
- function makePlaybackDispatch(driver) {
2379
- return async (profile, scenario, ctx) => {
2380
- const events = await driver.run(scenario, { ...ctx, profile });
2381
- return extractProducedState(events);
2382
- };
2383
- }
2384
- async function scoreUserStory(story, state, checkCorrectness) {
2385
- const verdict = await verifyCompletion(
2386
- { taskId: story.id, requirements: story.requirements },
2387
- state,
2388
- checkCorrectness
2389
- );
2390
- return { ...verdict, title: story.title };
2391
- }
2392
- function userStoryScoreboard(verdicts) {
2393
- const rows = [];
2394
- for (const v of verdicts) {
2395
- for (const r of v.requirements) {
2396
- rows.push({
2397
- storyId: v.taskId,
2398
- storyTitle: v.title,
2399
- reqId: r.reqId,
2400
- reqTitle: r.title,
2401
- status: r.satisfied ? "PASS" : "FAIL",
2402
- evidence: r.evidence
2403
- });
2404
- }
2405
- }
2406
- return rows;
2407
- }
2408
- function scoreboardSummary(rows) {
2409
- const byStory = /* @__PURE__ */ new Map();
2410
- let passed = 0;
2411
- for (const r of rows) {
2412
- const s = byStory.get(r.storyId) ?? { total: 0, passed: 0 };
2413
- s.total++;
2414
- if (r.status === "PASS") {
2415
- s.passed++;
2416
- passed++;
2417
- }
2418
- byStory.set(r.storyId, s);
2419
- }
2420
- let storiesFullyComplete = 0;
2421
- for (const s of byStory.values()) if (s.total > 0 && s.passed === s.total) storiesFullyComplete++;
2422
- return {
2423
- stories: byStory.size,
2424
- storiesFullyComplete,
2425
- requirements: rows.length,
2426
- passed,
2427
- failed: rows.length - passed,
2428
- passRate: rows.length === 0 ? 0 : passed / rows.length
2429
- };
2430
- }
2431
- function escapeCell(s) {
2432
- return s.replace(/\|/g, "\\|").replace(/\r?\n/g, " ");
2433
- }
2434
- function truncate(s, max) {
2435
- return s.length <= max ? s : `${s.slice(0, Math.max(0, max - 1))}\u2026`;
2436
- }
2437
- function renderScoreboardMarkdown(rows, opts = {}) {
2438
- const maxEv = opts.maxEvidenceChars ?? 160;
2439
- const sum = scoreboardSummary(rows);
2440
- const pct = (n) => `${Math.round(n * 100)}%`;
2441
- const ev = (e) => escapeCell(truncate(e.join("; "), maxEv)) || "\u2014";
2442
- const out = [`# ${opts.title ?? "Product-flow playback scoreboard"}`, ""];
2443
- if (opts.meta) {
2444
- for (const [k, v] of Object.entries(opts.meta)) out.push(`- **${k}:** ${v}`);
2445
- out.push("");
2446
- }
2447
- out.push(
2448
- `**${sum.storiesFullyComplete}/${sum.stories}** user stories fully shipped \xB7 **${sum.passed}/${sum.requirements}** requirements passing (${pct(sum.passRate)}) \xB7 **${sum.failed}** open`,
2449
- ""
2450
- );
2451
- const fails = rows.filter((r) => r.status === "FAIL");
2452
- if (fails.length > 0) {
2453
- out.push("## Open tickets", "", "| Story | Requirement | Evidence |", "| --- | --- | --- |");
2454
- for (const r of fails) {
2455
- out.push(`| ${escapeCell(r.storyTitle)} | ${escapeCell(r.reqTitle)} | ${ev(r.evidence)} |`);
2456
- }
2457
- out.push("");
2458
- } else {
2459
- out.push("_All requirements passing \u2014 no open tickets._", "");
2460
- }
2461
- out.push("## Per-story tick-off", "");
2462
- for (const storyId of [...new Set(rows.map((r) => r.storyId))]) {
2463
- const storyRows = rows.filter((r) => r.storyId === storyId);
2464
- const passed = storyRows.filter((r) => r.status === "PASS").length;
2465
- const mark = passed === storyRows.length ? "\u2705" : "\u26A0\uFE0F";
2466
- out.push(
2467
- `### ${escapeCell(storyRows[0].storyTitle)} \u2014 ${passed}/${storyRows.length} ${mark}`,
2468
- "",
2469
- "| Requirement | Status | Evidence |",
2470
- "| --- | --- | --- |"
2471
- );
2472
- for (const r of storyRows) {
2473
- out.push(
2474
- `| ${escapeCell(r.reqTitle)} | ${r.status === "PASS" ? "\u2705 PASS" : "\u274C FAIL"} | ${ev(r.evidence)} |`
2475
- );
2476
- }
2477
- out.push("");
2478
- }
2479
- return out.join("\n");
2480
- }
2481
-
2482
- // src/campaign/presets/run-profile-matrix.ts
2483
- import { createHash as createHash5 } from "crypto";
2484
- import { join as join3 } from "path";
2485
-
2486
- // src/agent-profile.ts
2487
- import { createHash as createHash4 } from "crypto";
2488
- import { harnessSupportsModel } from "@tangle-network/agent-interface";
2489
- var CODING_HARNESSES = [
2490
- "opencode",
2491
- "claude-code",
2492
- "codex",
2493
- "kimi-code"
2494
- ];
2495
- var HARNESS_NATIVE_MODEL = "default";
2496
- function expandProfileAxes(spec) {
2497
- const harnesses = spec.harnesses ?? CODING_HARNESSES;
2498
- if (harnesses.length === 0) throw new ValidationError("expandProfileAxes: no harnesses to sweep");
2499
- const baseModel = spec.base.model?.default;
2500
- const models = spec.models ?? (baseModel ? [baseModel] : []);
2501
- if (models.length === 0) {
2502
- throw new ValidationError(
2503
- "expandProfileAxes: no models to sweep \u2014 base profile has no model.default and none were supplied"
2504
- );
2505
- }
2506
- const out = [];
2507
- const seen = /* @__PURE__ */ new Set();
2508
- for (const harness of harnesses) {
2509
- const supported = spec.keepIncompatible ? models : models.filter((model) => harnessSupportsModel(harness, model));
2510
- const effective = supported.length > 0 ? supported : [HARNESS_NATIVE_MODEL];
2511
- for (const model of effective) {
2512
- const profile = {
2513
- ...spec.base,
2514
- name: `${spec.base.name ?? "agent"}/${harness}/${model}`,
2515
- model: { ...spec.base.model, default: model },
2516
- metadata: { ...spec.base.metadata ?? {}, harness, harnessModel: model }
2517
- };
2518
- const id = agentProfileId(profile);
2519
- if (seen.has(id)) continue;
2520
- seen.add(id);
2521
- out.push(profile);
2522
- }
2523
- }
2524
- if (out.length === 0) {
2525
- throw new ValidationError(
2526
- `expandProfileAxes: produced no profiles (harnesses=[${harnesses.join(", ")}], models=[${models.join(", ")}]).`
2527
- );
2528
- }
2529
- return out;
2530
- }
2531
- function harnessAxisOf(profile) {
2532
- const m = profile.metadata;
2533
- const harness = m?.harness;
2534
- const model = m?.harnessModel;
2535
- if (typeof harness === "string" && typeof model === "string") {
2536
- return { harness, model };
2537
- }
2538
- return void 0;
2539
- }
2540
- function agentProfileId(profile) {
2541
- const label = pathSafeProfileLabel(agentProfileDisplayLabel(profile)) ?? "profile";
2542
- return `${label}-${agentProfileHash(profile).slice(0, 16)}`;
2543
- }
2544
- function agentProfileModelId(profile) {
2545
- const model = profile.model?.default?.trim();
2546
- if (!model) {
2547
- const label = agentProfileDisplayLabel(profile) ?? "unnamed profile";
2548
- throw new ValidationError(
2549
- `AgentProfile "${label}" has no model.default \u2014 cannot record eval run`
2550
- );
2551
- }
2552
- return model;
2553
- }
2554
- function agentProfileDisplayLabel(profile) {
2555
- return profile.name?.trim() || profile.version?.trim() || void 0;
2556
- }
2557
- function pathSafeProfileLabel(label) {
2558
- const safe = label?.trim().replace(/[^A-Za-z0-9._-]+/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "");
2559
- return safe || void 0;
2560
- }
2561
- function compact(input) {
2562
- const out = {};
2563
- for (const [key, value] of Object.entries(input)) {
2564
- if (value !== void 0) out[key] = value;
2565
- }
2566
- return out;
2567
- }
2568
- function agentProfileHash(profile) {
2569
- const model = agentProfileModelId(profile);
2570
- const behaviour = {
2571
- ...profile,
2572
- name: void 0,
2573
- description: void 0,
2574
- tags: profile.tags ? [...profile.tags].sort() : void 0,
2575
- model: compact({ ...profile.model, default: model })
2576
- };
2577
- return createHash4("sha256").update(JSON.stringify(canonicalize(behaviour))).digest("hex");
2578
- }
2579
-
2580
- // src/campaign/presets/run-profile-matrix.ts
2581
- var ProfileMatrixError = class extends AgentEvalError {
2582
- constructor(message) {
2583
- super("profile_matrix", message);
2584
- }
2585
- };
2586
- function sanitize(id) {
2587
- return id.replace(/[^a-zA-Z0-9_-]/g, "_");
2588
- }
2589
- function sha(input) {
2590
- return createHash5("sha256").update(JSON.stringify(input)).digest("hex");
2591
- }
2592
- function mean2(xs) {
2593
- return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length;
2594
- }
2595
- function requireResolvedModel(cell, profileId) {
2596
- const resolved = cell.resolvedModel?.trim();
2597
- if (!resolved) {
2598
- throw new ProfileMatrixError(
2599
- `profile '${profileId}' declared the '${HARNESS_NATIVE_MODEL}' runtime-resolved model but its dispatch reported no resolved model for cell '${cell.cellId}' \u2014 return it in the ctx.cost.runPaidCall receipt so the RunRecord pins the real model (never records '${HARNESS_NATIVE_MODEL}')`
2600
- );
2601
- }
2602
- if (!modelHasSnapshot(resolved)) {
2603
- throw new ProfileMatrixError(
2604
- `profile '${profileId}' resolved to model '${resolved}' for cell '${cell.cellId}', which lacks a snapshot version \u2014 pin it (name@YYYY-MM-DD or name-YYYYMMDD) in the paid-call receipt`
2605
- );
2606
- }
2607
- return resolved;
2608
- }
2609
- function buildRunRecord(args) {
2610
- const { cell, profile, profileHash, configHash, experimentId, splitTag, commitSha, matrixId } = args;
2611
- const profileId = agentProfileId(profile);
2612
- const declaredModel = agentProfileModelId(profile);
2613
- const model = declaredModel === HARNESS_NATIVE_MODEL ? requireResolvedModel(cell, profileId) : declaredModel;
2614
- const record = campaignCellToRunRecord(cell, {
2615
- runId: `${matrixId}:${profileId}:${cell.cellId}`,
2616
- experimentId,
2617
- candidateId: profileId,
2618
- model,
2619
- promptHash: profileHash,
2620
- configHash,
2621
- commitSha,
2622
- splitTag,
2623
- agentProfile: args.agentProfileCell
2624
- });
2625
- if (args.corpusText && args.scenario) {
2626
- try {
2627
- const text = args.corpusText(cell.artifact, args.scenario);
2628
- if (text && typeof text.prompt === "string" && typeof text.completion === "string") {
2629
- record.prompt = text.prompt;
2630
- record.completion = text.completion;
2631
- }
2632
- } catch {
2633
- }
2634
- }
2635
- return record;
2636
- }
2637
- async function runProfileMatrix(opts) {
2638
- if (opts.profiles.length === 0) throw new ProfileMatrixError("profiles must not be empty");
2639
- if (opts.scenarios.length === 0) throw new ProfileMatrixError("scenarios must not be empty");
2640
- const splitTag = opts.splitTag ?? "search";
2641
- const seed = opts.seed ?? 42;
2642
- const validate = opts.validate ?? true;
2643
- const integrityMode = opts.integrity ?? "assert";
2644
- const profileIds = opts.profiles.map(agentProfileId);
2645
- const experimentId = opts.experimentId ?? `pm_${sha({ profileIds, scenarios: opts.scenarios.map((s) => s.id) }).slice(0, 16)}`;
2646
- const matrixId = `mtx_${sha({ experimentId, profileIds, seed, splitTag }).slice(0, 16)}`;
2647
- const scenarioById = new Map(opts.scenarios.map((s) => [s.id, s]));
2648
- for (const profile of opts.profiles) {
2649
- const profileHash = agentProfileHash(profile);
2650
- const profileId = agentProfileId(profile);
2651
- const declaredModel = agentProfileModelId(profile);
2652
- const model = declaredModel === HARNESS_NATIVE_MODEL ? `${HARNESS_NATIVE_MODEL}@runtime-resolved` : declaredModel;
2653
- try {
2654
- validateRunRecord({
2655
- runId: `${matrixId}:${profileId}:probe`,
2656
- experimentId,
2657
- candidateId: profileId,
2658
- seed,
2659
- model,
2660
- promptHash: profileHash,
2661
- configHash: profileHash,
2662
- commitSha: opts.commitSha,
2663
- wallMs: 0,
2664
- costUsd: null,
2665
- costProvenance: { kind: "uncaptured", usd: null },
2666
- tokenUsage: { input: 0, output: 0 },
2667
- terminalOutcome: "succeeded",
2668
- outcome: { raw: { execution_error_count: 0 } },
2669
- splitTag,
2670
- scenarioId: "recordability-probe"
2671
- });
2672
- } catch (err) {
2673
- throw new ProfileMatrixError(
2674
- `profile '${profileId}' is not recordable: ${err instanceof Error ? err.message : String(err)}`
2675
- );
2676
- }
2677
- }
2678
- const records = [];
2679
- const campaigns = {};
2680
- const byProfile = {};
2681
- for (const profile of opts.profiles) {
2682
- const profileHash = agentProfileHash(profile);
2683
- const profileId = agentProfileId(profile);
2684
- const declaredModel = agentProfileModelId(profile);
2685
- const configHash = sha({
2686
- profile: profileHash,
2687
- judges: (opts.judges ?? []).map((j) => j.name),
2688
- seed,
2689
- splitTag
2690
- });
2691
- const dispatch = (scenario, ctx) => opts.dispatch(profile, scenario, ctx);
2692
- Object.defineProperty(dispatch, "name", { value: `profile_${sanitize(profileId)}` });
2693
- const campaign = await runCampaign({
2694
- scenarios: opts.scenarios,
2695
- dispatch,
2696
- judges: opts.judges,
2697
- seed,
2698
- reps: opts.reps,
2699
- maxConcurrency: opts.maxConcurrency,
2700
- costCeiling: opts.costCeiling,
2701
- labeledStore: opts.labeledStore,
2702
- captureSource: opts.captureSource,
2703
- storage: opts.storage,
2704
- now: opts.now,
2705
- runDir: join3(opts.runDir, sanitize(profileId))
2706
- });
2707
- const axis = harnessAxisOf(profile);
2708
- const buildCellIdentity = (cellModel) => buildAgentProfileCell({
2709
- profileId,
2710
- sourceProfile: { kind: "agent-interface-profile", hash: profileHash },
2711
- model: cellModel,
2712
- ...axis ? { harness: { id: axis.harness } } : {}
2713
- });
2714
- const sharedCellIdentity = declaredModel === HARNESS_NATIVE_MODEL ? void 0 : await buildCellIdentity(declaredModel);
2715
- const profileRecords = [];
2716
- for (const cell of campaign.cells) {
2717
- const agentProfileCell = sharedCellIdentity ?? await buildCellIdentity(requireResolvedModel(cell, profileId));
2718
- const record = buildRunRecord({
2719
- cell,
2720
- profile,
2721
- profileHash,
2722
- configHash,
2723
- experimentId,
2724
- splitTag,
2725
- commitSha: opts.commitSha,
2726
- matrixId,
2727
- agentProfileCell,
2728
- scenario: scenarioById.get(cell.scenarioId),
2729
- corpusText: opts.corpusText
2730
- });
2731
- if (validate) validateRunRecord(record);
2732
- profileRecords.push(record);
2733
- records.push(record);
2734
- }
2735
- const totalCostUsd = campaign.aggregates.cost.totalCostUsd;
2736
- campaigns[profileId] = campaign;
2737
- byProfile[profileId] = {
2738
- profileId,
2739
- profileHash,
2740
- // The declared model, unless it snapped to the sentinel — then the
2741
- // resolved model the cells actually ran on (all cells of a profile share
2742
- // one harness, so the first record's model is representative).
2743
- model: declaredModel === HARNESS_NATIVE_MODEL ? profileRecords[0]?.model ?? declaredModel : declaredModel,
2744
- records: profileRecords.length,
2745
- meanComposite: meanOrNull2(
2746
- profileRecords.map(scoreOf).filter((score) => score !== void 0)
2747
- ),
2748
- totalCostUsd,
2749
- integrity: summarizeBackendIntegrity(profileRecords)
2750
- };
2751
- }
2752
- const integrity = summarizeBackendIntegrity(records);
2753
- if (integrityMode === "assert") {
2754
- assertRealBackend(records, { allowMixed: opts.allowMixed ?? true });
2755
- } else if (integrityMode === "warn" && integrity.verdict !== "real") {
2756
- console.warn(
2757
- `[runProfileMatrix] backend integrity: ${integrity.verdict} \u2014 ${integrity.diagnosis}`
2758
- );
2759
- }
2760
- const byScenario = rollup(records, (r) => r.scenarioId);
2761
- const byPersona = opts.personaOf ? rollupByPersona(records, opts.scenarios, opts.personaOf) : void 0;
2762
- return { matrixId, experimentId, records, byProfile, byScenario, byPersona, integrity, campaigns };
2763
- }
2764
- function scoreOf(r) {
2765
- return runTaskScore(r);
2766
- }
2767
- function meanOrNull2(values) {
2768
- return values.length === 0 ? null : mean2(values);
2769
- }
2770
- function rollup(records, keyOf) {
2771
- const groups = /* @__PURE__ */ new Map();
2772
- for (const r of records) {
2773
- const key = keyOf(r);
2774
- if (key === void 0) continue;
2775
- const score = scoreOf(r);
2776
- if (score === void 0) continue;
2777
- const arr = groups.get(key) ?? [];
2778
- arr.push(score);
2779
- groups.set(key, arr);
2780
- }
2781
- const out = {};
2782
- for (const [key, xs] of groups) out[key] = { meanComposite: mean2(xs), n: xs.length };
2783
- return out;
2784
- }
2785
- function rollupByPersona(records, scenarios, personaOf) {
2786
- const personaByScenarioId = /* @__PURE__ */ new Map();
2787
- for (const s of scenarios) personaByScenarioId.set(s.id, personaOf(s));
2788
- return rollup(records, (r) => r.scenarioId ? personaByScenarioId.get(r.scenarioId) : void 0);
2789
- }
2790
-
2791
- // src/campaign/scenario-selection.ts
2792
- var DEFAULT_SATURATION_CEILING = 0.999;
2793
- var VARIANCE_EPSILON = 1e-9;
2794
- function moments(scores) {
2795
- const n = scores.length;
2796
- if (n === 0) return { meanScore: 0, variance: 0 };
2797
- let sum = 0;
2798
- for (const s of scores) sum += s;
2799
- const meanScore = sum / n;
2800
- let sqDev = 0;
2801
- for (const s of scores) {
2802
- const d = s - meanScore;
2803
- sqDev += d * d;
2804
- }
2805
- return { meanScore, variance: sqDev / n };
2806
- }
2807
- function compareDiscrimination(a, b) {
2808
- if (b.discrimination !== a.discrimination) return b.discrimination - a.discrimination;
2809
- if (a.meanScore !== b.meanScore) return a.meanScore - b.meanScore;
2810
- return a.scenarioId < b.scenarioId ? -1 : a.scenarioId > b.scenarioId ? 1 : 0;
2811
- }
2812
- function scoreDiscrimination(signals, opts) {
2813
- const saturationCeiling = opts?.saturationCeiling ?? DEFAULT_SATURATION_CEILING;
2814
- const scored = signals.map((signal) => {
2815
- const { meanScore, variance } = moments(signal.scores);
2816
- const tied = variance < VARIANCE_EPSILON && meanScore >= saturationCeiling;
2817
- return {
2818
- scenarioId: signal.scenarioId,
2819
- discrimination: variance,
2820
- meanScore,
2821
- variance,
2822
- tied
2823
- };
2824
- });
2825
- return scored.sort(compareDiscrimination);
2826
- }
2827
- function selectDiscriminative(signals, k, opts) {
2828
- if (k < 1) throw new Error(`selectDiscriminative: k must be >= 1 (got ${k})`);
2829
- const ranked = scoreDiscrimination(signals, opts);
2830
- if (ranked.length <= k) return ranked.map((s) => s.scenarioId);
2831
- const nonTied = ranked.filter((s) => !s.tied);
2832
- if (nonTied.length >= k) return nonTied.slice(0, k).map((s) => s.scenarioId);
2833
- const tied = ranked.filter((s) => s.tied);
2834
- const fill = tied.slice(0, k - nonTied.length);
2835
- return [...nonTied, ...fill].map((s) => s.scenarioId);
2836
- }
2837
-
2838
- // src/campaign/search-ledger.ts
2839
- import { createHash as createHash6 } from "crypto";
2840
- import { existsSync as existsSync3, readFileSync as readFileSync3 } from "fs";
2841
- import { resolve as resolve2 } from "path";
2842
- import { z } from "zod";
2843
- var SEARCH_LEDGER_SCHEMA = "tangle.search-ledger.v1";
2844
- var NON_EMPTY = z.string().min(1).refine((value) => value.trim() === value, "must not contain surrounding whitespace");
2845
- var HASH = z.string().regex(/^sha256:[a-f0-9]{64}$/);
2846
- var LINEAGE_NODE_ID = z.string().regex(/^[a-f0-9]{16}$/);
2847
- var IMMUTABLE_REVISION = z.string().regex(/^(?:[a-f0-9]{40}|[a-f0-9]{64}|sha256:[a-f0-9]{64}|sha512:[A-Za-z0-9+/=]+)$/);
2848
- var ISO_TIMESTAMP = z.string().regex(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{3})?Z$/).refine((value) => Number.isFinite(Date.parse(value)), "invalid timestamp");
2849
- var NON_NEGATIVE_INT = z.number().int().nonnegative().safe();
2850
- var FINITE_NUMBER = z.number().finite();
2851
- var ArtifactRefSchema = z.object({
2852
- role: NON_EMPTY,
2853
- uri: NON_EMPTY,
2854
- sha256: HASH,
2855
- byteLength: NON_NEGATIVE_INT
2856
- }).strict();
2857
- var SourceRefSchema = z.object({
2858
- uri: NON_EMPTY,
2859
- revision: IMMUTABLE_REVISION
2860
- }).strict();
2861
- var FailureReasonSchema = z.object({
2862
- code: NON_EMPTY,
2863
- message: NON_EMPTY
2864
- }).strict();
2865
- var EventBaseShape = {
2866
- eventId: NON_EMPTY,
2867
- occurredAt: ISO_TIMESTAMP,
2868
- artifacts: z.array(ArtifactRefSchema).min(1)
2869
- };
2870
- var OperationKindSchema = z.enum([
2871
- "candidate-generation",
2872
- "analysis",
2873
- "selection",
2874
- "judge",
2875
- "other"
2876
- ]);
2877
- var SearchPlannedSchema = z.object({
2878
- ...EventBaseShape,
2879
- kind: z.literal("search-planned"),
2880
- plan: z.object({
2881
- candidateSlots: z.array(
2882
- z.object({
2883
- slotId: NON_EMPTY,
2884
- generationOperationId: NON_EMPTY
2885
- }).strict()
2886
- ).min(1),
2887
- tasks: z.array(
2888
- z.object({
2889
- taskId: NON_EMPTY,
2890
- source: SourceRefSchema,
2891
- benchmark: SourceRefSchema,
2892
- maxAttempts: z.number().int().positive().safe()
2893
- }).strict()
2894
- ).min(1),
2895
- operations: z.array(
2896
- z.object({
2897
- operationId: NON_EMPTY,
2898
- kind: OperationKindSchema
2899
- }).strict()
2900
- ).min(1)
2901
- }).strict()
2902
- }).strict();
2903
- var CandidateRegisteredSchema = z.object({
2904
- ...EventBaseShape,
2905
- kind: z.literal("candidate-registered"),
2906
- slotId: NON_EMPTY,
2907
- generationOperationId: NON_EMPTY,
2908
- candidateId: NON_EMPTY,
2909
- lineage: z.object({
2910
- lineageNodeId: LINEAGE_NODE_ID,
2911
- parentCandidateIds: z.array(NON_EMPTY),
2912
- generation: NON_NEGATIVE_INT,
2913
- proposer: NON_EMPTY,
2914
- proposerSource: SourceRefSchema
2915
- }).strict(),
2916
- surfaces: z.array(
2917
- z.object({
2918
- surfaceId: NON_EMPTY,
2919
- kind: z.enum([
2920
- "prompt",
2921
- "tool-contract",
2922
- "runtime-config",
2923
- "memory",
2924
- "knowledge",
2925
- "agent-profile",
2926
- "code",
2927
- "deployment"
2928
- ]),
2929
- artifact: ArtifactRefSchema
2930
- }).strict()
2931
- ).min(1)
2932
- }).strict();
2933
- var CandidateSlotClosedSchema = z.object({
2934
- ...EventBaseShape,
2935
- kind: z.literal("candidate-slot-closed"),
2936
- slotId: NON_EMPTY,
2937
- generationOperationId: NON_EMPTY,
2938
- reason: FailureReasonSchema
2939
- }).strict();
2940
- var KnownTokensSchema = z.object({
2941
- status: z.literal("known"),
2942
- inputTokens: NON_NEGATIVE_INT,
2943
- outputTokens: NON_NEGATIVE_INT,
2944
- cachedTokens: NON_NEGATIVE_INT
2945
- }).strict();
2946
- var UnknownSchema = z.object({
2947
- status: z.literal("unknown"),
2948
- reason: NON_EMPTY
2949
- }).strict();
2950
- var KnownCostSchema = z.object({
2951
- status: z.literal("known"),
2952
- usd: z.number().finite().nonnegative(),
2953
- source: z.enum(["provider", "pricing-table", "free"])
2954
- }).strict().superRefine((cost, ctx) => {
2955
- if (cost.source === "free" && cost.usd !== 0) {
2956
- ctx.addIssue({ code: "custom", message: "free cost source must have usd 0" });
2957
- }
2958
- });
2959
- var UnknownCostSchema = z.object({
2960
- status: z.literal("unknown"),
2961
- knownLowerBoundUsd: z.number().finite().nonnegative(),
2962
- reason: NON_EMPTY
2963
- }).strict();
2964
- var AccountingSchema = z.object({
2965
- tokens: z.discriminatedUnion("status", [KnownTokensSchema, UnknownSchema]),
2966
- cost: z.discriminatedUnion("status", [KnownCostSchema, UnknownCostSchema])
2967
- }).strict();
2968
- var MetricsSchema = z.record(NON_EMPTY, FINITE_NUMBER).superRefine((metrics, ctx) => {
2969
- for (const key of Object.keys(metrics)) {
2970
- if (key === "__proto__" || key === "constructor" || key === "prototype") {
2971
- ctx.addIssue({ code: "custom", message: `unsafe metric key ${key}` });
2972
- }
2973
- }
2974
- });
2975
- var OutcomeSchema = z.discriminatedUnion("status", [
2976
- z.object({
2977
- status: z.literal("passed"),
2978
- score: FINITE_NUMBER,
2979
- metrics: MetricsSchema
2980
- }).strict(),
2981
- z.object({
2982
- status: z.literal("failed"),
2983
- score: FINITE_NUMBER,
2984
- metrics: MetricsSchema,
2985
- failure: FailureReasonSchema
2986
- }).strict(),
2987
- z.object({
2988
- status: z.literal("errored"),
2989
- metrics: MetricsSchema,
2990
- error: FailureReasonSchema.extend({ retryable: z.boolean() }).strict()
2991
- }).strict()
2992
- ]);
2993
- var EffectSchema = z.discriminatedUnion("status", [
2994
- z.object({
2995
- status: z.literal("measured"),
2996
- metric: NON_EMPTY,
2997
- baselineValue: FINITE_NUMBER,
2998
- candidateValue: FINITE_NUMBER,
2999
- delta: FINITE_NUMBER
3000
- }).strict().superRefine((effect, ctx) => {
3001
- const expected = effect.candidateValue - effect.baselineValue;
3002
- const tolerance = Number.EPSILON * Math.max(1, Math.abs(expected), Math.abs(effect.delta)) * 8;
3003
- if (Math.abs(effect.delta - expected) > tolerance) {
3004
- ctx.addIssue({ code: "custom", message: "delta must equal candidateValue - baselineValue" });
3005
- }
3006
- }),
3007
- z.object({
3008
- status: z.literal("not-measured"),
3009
- reason: NON_EMPTY
3010
- }).strict()
3011
- ]);
3012
- var SurfaceEvidenceSchema = z.object({
3013
- surfaceId: NON_EMPTY,
3014
- fired: z.boolean(),
3015
- firingCount: NON_NEGATIVE_INT,
3016
- effect: EffectSchema,
3017
- evidence: z.array(ArtifactRefSchema).min(1)
3018
- }).strict().superRefine((evidence, ctx) => {
3019
- if (evidence.fired && evidence.firingCount === 0) {
3020
- ctx.addIssue({ code: "custom", message: "a fired surface must have firingCount >= 1" });
3021
- }
3022
- if (!evidence.fired && evidence.firingCount !== 0) {
3023
- ctx.addIssue({
3024
- code: "custom",
3025
- message: "a surface that did not fire must have firingCount 0"
3026
- });
3027
- }
3028
- if (!evidence.fired && evidence.effect.status === "measured" && evidence.effect.delta !== 0) {
3029
- ctx.addIssue({
3030
- code: "custom",
3031
- message: "a surface that did not fire cannot claim non-zero effect"
3032
- });
3033
- }
3034
- });
3035
- var TaskAttemptedSchema = z.object({
3036
- ...EventBaseShape,
3037
- kind: z.literal("task-attempted"),
3038
- candidateId: NON_EMPTY,
3039
- runId: NON_EMPTY,
3040
- attemptIndex: NON_NEGATIVE_INT,
3041
- task: z.object({ taskId: NON_EMPTY, source: SourceRefSchema }).strict(),
3042
- identity: z.object({
3043
- model: z.object({
3044
- provider: NON_EMPTY,
3045
- snapshot: NON_EMPTY.refine(
3046
- modelHasSnapshot,
3047
- "model must include an immutable snapshot"
3048
- )
3049
- }).strict(),
3050
- agent: SourceRefSchema,
3051
- benchmark: SourceRefSchema
3052
- }).strict(),
3053
- outcome: OutcomeSchema,
3054
- accounting: AccountingSchema,
3055
- surfaceEvidence: z.array(SurfaceEvidenceSchema).min(1)
3056
- }).strict();
3057
- var SearchOperationRecordedSchema = z.object({
3058
- ...EventBaseShape,
3059
- kind: z.literal("search-operation-recorded"),
3060
- operationId: NON_EMPTY,
3061
- operationKind: OperationKindSchema,
3062
- execution: z.discriminatedUnion("kind", [
3063
- z.object({
3064
- kind: z.literal("model"),
3065
- model: z.object({
3066
- provider: NON_EMPTY,
3067
- snapshot: NON_EMPTY.refine(
3068
- modelHasSnapshot,
3069
- "model must include an immutable snapshot"
3070
- )
3071
- }).strict(),
3072
- source: SourceRefSchema
3073
- }).strict(),
3074
- z.object({
3075
- kind: z.literal("deterministic"),
3076
- source: SourceRefSchema
3077
- }).strict()
3078
- ]),
3079
- outcome: z.discriminatedUnion("status", [
3080
- z.object({ status: z.literal("completed") }).strict(),
3081
- z.object({
3082
- status: z.literal("partial"),
3083
- failure: FailureReasonSchema
3084
- }).strict(),
3085
- z.object({
3086
- status: z.literal("failed"),
3087
- failure: FailureReasonSchema
3088
- }).strict()
3089
- ]),
3090
- accounting: AccountingSchema
3091
- }).strict();
3092
- var CandidateDecidedSchema = z.object({
3093
- ...EventBaseShape,
3094
- kind: z.literal("candidate-decided"),
3095
- candidateId: NON_EMPTY,
3096
- decision: z.discriminatedUnion("status", [
3097
- z.object({ status: z.literal("selected") }).strict(),
3098
- z.object({
3099
- status: z.literal("rejected"),
3100
- reason: FailureReasonSchema
3101
- }).strict()
3102
- ])
3103
- }).strict();
3104
- var SearchCompletedSchema = z.object({
3105
- ...EventBaseShape,
3106
- kind: z.literal("search-completed"),
3107
- result: z.discriminatedUnion("status", [
3108
- z.object({
3109
- status: z.literal("selected"),
3110
- candidateId: NON_EMPTY
3111
- }).strict(),
3112
- z.object({
3113
- status: z.literal("all-rejected"),
3114
- reason: FailureReasonSchema
3115
- }).strict()
3116
- ])
3117
- }).strict();
3118
- var EventSchema = z.discriminatedUnion("kind", [
3119
- SearchPlannedSchema,
3120
- CandidateRegisteredSchema,
3121
- CandidateSlotClosedSchema,
3122
- TaskAttemptedSchema,
3123
- SearchOperationRecordedSchema,
3124
- CandidateDecidedSchema,
3125
- SearchCompletedSchema
3126
- ]);
3127
- var EntrySchema = z.object({
3128
- schema: z.literal(SEARCH_LEDGER_SCHEMA),
3129
- campaignId: NON_EMPTY,
3130
- sequence: NON_NEGATIVE_INT,
3131
- previousHash: z.union([HASH, z.null()]),
3132
- event: EventSchema,
3133
- entryHash: HASH
3134
- }).strict();
3135
- function validateSearchLedgerEvent(input) {
3136
- const parsed = EventSchema.safeParse(input);
3137
- if (!parsed.success) {
3138
- throw new SearchLedgerError(`invalid search ledger event: ${formatZodError(parsed.error)}`);
3139
- }
3140
- return normalizeEvent(parsed.data);
3141
- }
3142
- function openSearchLedger(options) {
3143
- if (options.path.trim().length === 0) throw new SearchLedgerError("ledger path is empty");
3144
- return new FileSearchLedger(options.path, options.campaignId);
3145
- }
3146
- var ledgerMutexes = /* @__PURE__ */ new Map();
3147
- function mutexFor(path) {
3148
- const existing = ledgerMutexes.get(path);
3149
- if (existing) return existing;
3150
- const mutex = new Mutex();
3151
- ledgerMutexes.set(path, mutex);
3152
- return mutex;
3153
- }
3154
- var FileSearchLedger = class {
3155
- path;
3156
- campaignId;
3157
- mutex;
3158
- constructor(path, campaignId) {
3159
- if (path.trim().length === 0) throw new SearchLedgerError("ledger path is empty");
3160
- if (campaignId.length === 0) throw new SearchLedgerError("campaignId is empty");
3161
- if (campaignId.trim() !== campaignId) {
3162
- throw new SearchLedgerError("campaignId must not contain surrounding whitespace");
3163
- }
3164
- this.path = resolve2(path);
3165
- this.campaignId = campaignId;
3166
- this.mutex = mutexFor(this.path);
3167
- }
3168
- async replay() {
3169
- return this.mutex.runExclusive(
3170
- () => withSearchLedgerFileLock(this.path, () => replayFile(this.path, this.campaignId))
3171
- );
3172
- }
3173
- async append(input) {
3174
- const event = validateSearchLedgerEvent(input);
3175
- return this.mutex.runExclusive(
3176
- () => withSearchLedgerFileLock(this.path, () => {
3177
- const before = replayFile(this.path, this.campaignId);
3178
- const existing = before.entries.find((entry2) => entry2.event.eventId === event.eventId);
3179
- if (existing) {
3180
- if (canonicalString(existing.event) !== canonicalString(event)) {
3181
- throw new SearchLedgerConflictError(
3182
- `eventId ${event.eventId} already exists with different content`
3183
- );
3184
- }
3185
- return { entry: existing, appended: false, replay: before };
3186
- }
3187
- const previousHash = before.audit.headHash;
3188
- const material = {
3189
- schema: SEARCH_LEDGER_SCHEMA,
3190
- campaignId: this.campaignId,
3191
- sequence: before.entries.length,
3192
- previousHash,
3193
- event
3194
- };
3195
- const entry = {
3196
- ...material,
3197
- entryHash: hashCanonical(material)
3198
- };
3199
- const replay = replayEntries([...before.entries, entry], this.campaignId);
3200
- appendSearchLedgerLine(this.path, `${canonicalString(entry)}
3201
- `);
3202
- return { entry, appended: true, replay };
3203
- })
3204
- );
3205
- }
3206
- };
3207
- function replayFile(path, campaignId) {
3208
- if (!existsSync3(path)) return replayEntries([], campaignId);
3209
- const text = readFileSync3(path, "utf8");
3210
- if (text.length === 0) return replayEntries([], campaignId);
3211
- if (!text.endsWith("\n")) {
3212
- throw new SearchLedgerIntegrityError(
3213
- `search ledger ${path} has a truncated final record (missing newline)`
3214
- );
3215
- }
3216
- const lines = text.slice(0, -1).split("\n");
3217
- const entries = [];
3218
- for (let index = 0; index < lines.length; index += 1) {
3219
- const line = lines[index];
3220
- if (line.length === 0) {
3221
- throw new SearchLedgerIntegrityError(
3222
- `search ledger ${path} has a blank row at line ${index + 1}`
3223
- );
3224
- }
3225
- let raw;
3226
- try {
3227
- raw = JSON.parse(line);
3228
- } catch (error) {
3229
- throw new SearchLedgerIntegrityError(
3230
- `search ledger ${path} has invalid JSON at line ${index + 1}`,
3231
- { cause: error }
3232
- );
3233
- }
3234
- const parsed = EntrySchema.safeParse(raw);
3235
- if (!parsed.success) {
3236
- throw new SearchLedgerIntegrityError(
3237
- `search ledger ${path} has a malformed entry at line ${index + 1}: ${formatZodError(parsed.error)}`
3238
- );
3239
- }
3240
- const entry = parsed.data;
3241
- const normalizedEvent = validateSearchLedgerEvent(entry.event);
3242
- if (canonicalString(normalizedEvent) !== canonicalString(entry.event)) {
3243
- throw new SearchLedgerIntegrityError(
3244
- `search ledger ${path} has non-canonical event ordering at line ${index + 1}`
3245
- );
3246
- }
3247
- if (line !== canonicalString(entry)) {
3248
- throw new SearchLedgerIntegrityError(
3249
- `search ledger ${path} has non-canonical bytes at line ${index + 1}`
3250
- );
3251
- }
3252
- entries.push(entry);
3253
- }
3254
- return replayEntries(entries, campaignId);
3255
- }
3256
- function replayEntries(entries, campaignId) {
3257
- const candidates = /* @__PURE__ */ new Map();
3258
- const candidateBySlot = /* @__PURE__ */ new Map();
3259
- const closedSlots = /* @__PURE__ */ new Map();
3260
- const lineageNodes = /* @__PURE__ */ new Map();
3261
- const eventIds = /* @__PURE__ */ new Set();
3262
- const runIds = /* @__PURE__ */ new Set();
3263
- const attemptKeys = /* @__PURE__ */ new Set();
3264
- const candidateEvents = [];
3265
- const closedSlotEvents = [];
3266
- const attempts = [];
3267
- const operationEvents = [];
3268
- const operationsById = /* @__PURE__ */ new Map();
3269
- const decisions = [];
3270
- let planEvent = null;
3271
- let completion = null;
3272
- let expectedPrevious = null;
3273
- let previousOccurredAt = Number.NEGATIVE_INFINITY;
3274
- for (let index = 0; index < entries.length; index += 1) {
3275
- const entry = entries[index];
3276
- if (entry.campaignId !== campaignId) {
3277
- throw new SearchLedgerIntegrityError(
3278
- `entry ${index} belongs to campaign ${entry.campaignId}, expected ${campaignId}`
3279
- );
3280
- }
3281
- if (entry.sequence !== index) {
3282
- throw new SearchLedgerIntegrityError(
3283
- `entry ${entry.event.eventId} has sequence ${entry.sequence}, expected ${index}`
3284
- );
3285
- }
3286
- if (entry.previousHash !== expectedPrevious) {
3287
- throw new SearchLedgerIntegrityError(
3288
- `entry ${entry.event.eventId} does not extend the previous hash`
3289
- );
3290
- }
3291
- const { entryHash: _entryHash, ...material } = entry;
3292
- const expectedHash = hashCanonical(material);
3293
- if (entry.entryHash !== expectedHash) {
3294
- throw new SearchLedgerIntegrityError(
3295
- `entry ${entry.event.eventId} hash mismatch: expected ${expectedHash}, got ${entry.entryHash}`
3296
- );
3297
- }
3298
- expectedPrevious = entry.entryHash;
3299
- const event = entry.event;
3300
- if (eventIds.has(event.eventId)) {
3301
- throw new SearchLedgerIntegrityError(`duplicate eventId ${event.eventId} in durable ledger`);
3302
- }
3303
- eventIds.add(event.eventId);
3304
- if (completion) {
3305
- throw new SearchLedgerIntegrityError(
3306
- `event ${event.eventId} appears after terminal event ${completion.eventId}`
3307
- );
3308
- }
3309
- const occurredAt = Date.parse(event.occurredAt);
3310
- if (occurredAt < previousOccurredAt) {
3311
- throw new SearchLedgerIntegrityError(
3312
- `event ${event.eventId} occurred before the preceding durable event`
3313
- );
3314
- }
3315
- previousOccurredAt = occurredAt;
3316
- assertUnique2(event.artifacts.map(artifactKey), "artifact receipt", event.eventId);
3317
- if (event.kind === "search-planned") {
3318
- if (index !== 0 || planEvent) {
3319
- throw new SearchLedgerIntegrityError("search plan must be the first and only plan event");
3320
- }
3321
- assertUnique2(
3322
- event.plan.candidateSlots.map((slot) => slot.slotId),
3323
- "candidate slot",
3324
- event.eventId
3325
- );
3326
- assertUnique2(
3327
- event.plan.tasks.map((task) => task.taskId),
3328
- "planned taskId",
3329
- event.eventId
3330
- );
3331
- assertUnique2(
3332
- event.plan.operations.map((operation) => operation.operationId),
3333
- "planned operationId",
3334
- event.eventId
3335
- );
3336
- for (const slot of event.plan.candidateSlots) {
3337
- const generationOperation = event.plan.operations.find(
3338
- (operation) => operation.operationId === slot.generationOperationId
3339
- );
3340
- if (generationOperation?.kind !== "candidate-generation") {
3341
- throw new SearchLedgerIntegrityError(
3342
- `candidate slot ${slot.slotId} references unplanned candidate-generation operation ${slot.generationOperationId}`
3343
- );
3344
- }
3345
- }
3346
- planEvent = event;
3347
- continue;
3348
- }
3349
- if (!planEvent) {
3350
- throw new SearchLedgerIntegrityError(
3351
- `event ${event.eventId} appears before the required search plan`
3352
- );
3353
- }
3354
- if (event.kind === "candidate-registered") {
3355
- if (candidates.has(event.candidateId)) {
3356
- throw new SearchLedgerIntegrityError(`candidate ${event.candidateId} was registered twice`);
3357
- }
3358
- const plannedSlot = planEvent.plan.candidateSlots.find((slot) => slot.slotId === event.slotId);
3359
- if (!plannedSlot) {
3360
- throw new SearchLedgerIntegrityError(
3361
- `candidate ${event.candidateId} binds unknown slot ${event.slotId}`
3362
- );
3363
- }
3364
- if (candidateBySlot.has(event.slotId)) {
3365
- throw new SearchLedgerIntegrityError(`candidate slot ${event.slotId} was bound twice`);
3366
- }
3367
- if (closedSlots.has(event.slotId)) {
3368
- throw new SearchLedgerIntegrityError(`candidate slot ${event.slotId} was already closed`);
3369
- }
3370
- if (event.generationOperationId !== plannedSlot.generationOperationId) {
3371
- throw new SearchLedgerIntegrityError(
3372
- `candidate ${event.candidateId} generation operation ${event.generationOperationId} does not match slot ${event.slotId} plan ${plannedSlot.generationOperationId}`
3373
- );
3374
- }
3375
- const generationOperation = operationsById.get(event.generationOperationId);
3376
- if (!generationOperation) {
3377
- throw new SearchLedgerIntegrityError(
3378
- `candidate ${event.candidateId} precedes generation operation ${event.generationOperationId}`
3379
- );
3380
- }
3381
- if (generationOperation.outcome.status === "failed") {
3382
- throw new SearchLedgerIntegrityError(
3383
- `candidate ${event.candidateId} cannot bind failed generation operation ${event.generationOperationId}`
3384
- );
3385
- }
3386
- const previousCandidate = lineageNodes.get(event.lineage.lineageNodeId);
3387
- if (previousCandidate) {
3388
- throw new SearchLedgerIntegrityError(
3389
- `lineage node ${event.lineage.lineageNodeId} is already bound to ${previousCandidate}`
3390
- );
3391
- }
3392
- assertUnique2(event.lineage.parentCandidateIds, "parentCandidateId", event.eventId);
3393
- const parents = event.lineage.parentCandidateIds.map((id) => {
3394
- const parent = candidates.get(id);
3395
- if (!parent) {
3396
- throw new SearchLedgerIntegrityError(
3397
- `candidate ${event.candidateId} references unknown parent ${id}`
3398
- );
3399
- }
3400
- return parent;
3401
- });
3402
- const expectedGeneration = parents.length === 0 ? 0 : Math.max(...parents.map((parent) => parent.registered.lineage.generation)) + 1;
3403
- if (event.lineage.generation !== expectedGeneration) {
3404
- throw new SearchLedgerIntegrityError(
3405
- `candidate ${event.candidateId} generation ${event.lineage.generation} does not follow its parents (expected ${expectedGeneration})`
3406
- );
3407
- }
3408
- assertUnique2(
3409
- event.surfaces.map((surface) => surface.surfaceId),
3410
- "surfaceId",
3411
- event.eventId
3412
- );
3413
- candidates.set(event.candidateId, { registered: event, attempts: [], decision: null });
3414
- candidateBySlot.set(event.slotId, event.candidateId);
3415
- lineageNodes.set(event.lineage.lineageNodeId, event.candidateId);
3416
- candidateEvents.push(event);
3417
- continue;
3418
- }
3419
- if (event.kind === "task-attempted") {
3420
- const candidate = candidates.get(event.candidateId);
3421
- if (!candidate) {
3422
- throw new SearchLedgerIntegrityError(
3423
- `attempt ${event.eventId} references unknown candidate ${event.candidateId}`
3424
- );
3425
- }
3426
- if (candidate.decision) {
3427
- throw new SearchLedgerIntegrityError(
3428
- `attempt ${event.eventId} appears after candidate ${event.candidateId} was decided`
3429
- );
3430
- }
3431
- const plannedTask = planEvent.plan.tasks.find((task) => task.taskId === event.task.taskId);
3432
- if (!plannedTask) {
3433
- throw new SearchLedgerIntegrityError(
3434
- `attempt ${event.eventId} references unplanned task ${event.task.taskId}`
3435
- );
3436
- }
3437
- if (canonicalString(plannedTask.source) !== canonicalString(event.task.source) || canonicalString(plannedTask.benchmark) !== canonicalString(event.identity.benchmark)) {
3438
- throw new SearchLedgerIntegrityError(
3439
- `task ${event.task.taskId} does not match its planned source identity`
3440
- );
3441
- }
3442
- if (event.attemptIndex >= plannedTask.maxAttempts) {
3443
- throw new SearchLedgerIntegrityError(
3444
- `task ${event.task.taskId} attempt ${event.attemptIndex} exceeds planned maxAttempts ${plannedTask.maxAttempts}`
3445
- );
3446
- }
3447
- if (runIds.has(event.runId)) {
3448
- throw new SearchLedgerIntegrityError(`runId ${event.runId} was recorded twice`);
3449
- }
3450
- runIds.add(event.runId);
3451
- const attemptKey = canonicalString([event.candidateId, event.task.taskId, event.attemptIndex]);
3452
- if (attemptKeys.has(attemptKey)) {
3453
- throw new SearchLedgerIntegrityError(
3454
- `candidate ${event.candidateId} task ${event.task.taskId} attempt ${event.attemptIndex} was recorded twice`
3455
- );
3456
- }
3457
- const expectedAttemptIndex = candidate.attempts.filter(
3458
- (attempt) => attempt.task.taskId === event.task.taskId
3459
- ).length;
3460
- if (event.attemptIndex !== expectedAttemptIndex) {
3461
- throw new SearchLedgerIntegrityError(
3462
- `candidate ${event.candidateId} task ${event.task.taskId} attempt index ${event.attemptIndex} is not contiguous (expected ${expectedAttemptIndex})`
3463
- );
3464
- }
3465
- const previousAttempt = candidate.attempts.find(
3466
- (attempt) => attempt.task.taskId === event.task.taskId
3467
- );
3468
- if (previousAttempt?.outcome.status !== void 0 && previousAttempt.outcome.status !== "errored") {
3469
- throw new SearchLedgerIntegrityError(
3470
- `task ${event.task.taskId} was retried after a measured outcome`
3471
- );
3472
- }
3473
- if (previousAttempt && canonicalString({ task: previousAttempt.task, identity: previousAttempt.identity }) !== canonicalString({ task: event.task, identity: event.identity })) {
3474
- throw new SearchLedgerIntegrityError(
3475
- `candidate ${event.candidateId} task ${event.task.taskId} changed immutable execution identity between attempts`
3476
- );
3477
- }
3478
- attemptKeys.add(attemptKey);
3479
- const declared = candidate.registered.surfaces.map((surface) => surface.surfaceId).sort();
3480
- const observed = event.surfaceEvidence.map((surface) => surface.surfaceId).sort();
3481
- assertUnique2(observed, "surface evidence", event.eventId);
3482
- for (const evidence of event.surfaceEvidence) {
3483
- assertUnique2(evidence.evidence.map(artifactKey), "surface evidence receipt", event.eventId);
3484
- }
3485
- if (canonicalString(declared) !== canonicalString(observed)) {
3486
- throw new SearchLedgerIntegrityError(
3487
- `attempt ${event.eventId} surface evidence does not exactly cover candidate ${event.candidateId}`
3488
- );
3489
- }
3490
- candidate.attempts.push(event);
3491
- attempts.push(event);
3492
- continue;
3493
- }
3494
- if (event.kind === "search-operation-recorded") {
3495
- const plannedOperation = planEvent.plan.operations.find(
3496
- (operation) => operation.operationId === event.operationId
3497
- );
3498
- if (!plannedOperation) {
3499
- throw new SearchLedgerIntegrityError(
3500
- `operation ${event.operationId} was not declared in the search plan`
3501
- );
3502
- }
3503
- if (plannedOperation.kind !== event.operationKind) {
3504
- throw new SearchLedgerIntegrityError(
3505
- `operation ${event.operationId} kind ${event.operationKind} does not match planned ${plannedOperation.kind}`
3506
- );
3507
- }
3508
- if (operationsById.has(event.operationId)) {
3509
- throw new SearchLedgerIntegrityError(`operation ${event.operationId} was recorded twice`);
3510
- }
3511
- operationsById.set(event.operationId, event);
3512
- operationEvents.push(event);
3513
- continue;
3514
- }
3515
- if (event.kind === "candidate-slot-closed") {
3516
- const plannedSlot = planEvent.plan.candidateSlots.find((slot) => slot.slotId === event.slotId);
3517
- if (!plannedSlot) {
3518
- throw new SearchLedgerIntegrityError(
3519
- `candidate slot closure ${event.eventId} references unknown slot ${event.slotId}`
3520
- );
3521
- }
3522
- if (event.generationOperationId !== plannedSlot.generationOperationId) {
3523
- throw new SearchLedgerIntegrityError(
3524
- `candidate slot closure ${event.eventId} generation operation ${event.generationOperationId} does not match slot ${event.slotId} plan ${plannedSlot.generationOperationId}`
3525
- );
3526
- }
3527
- if (candidateBySlot.has(event.slotId)) {
3528
- throw new SearchLedgerIntegrityError(
3529
- `candidate slot ${event.slotId} was already bound to a candidate`
3530
- );
3531
- }
3532
- if (closedSlots.has(event.slotId)) {
3533
- throw new SearchLedgerIntegrityError(`candidate slot ${event.slotId} was closed twice`);
3534
- }
3535
- const operation = operationsById.get(event.generationOperationId);
3536
- if (!operation) {
3537
- throw new SearchLedgerIntegrityError(
3538
- `candidate slot closure ${event.eventId} precedes operation ${event.generationOperationId}`
3539
- );
3540
- }
3541
- if (operation.outcome.status === "completed") {
3542
- throw new SearchLedgerIntegrityError(
3543
- `candidate slot ${event.slotId} cannot close from completed operation ${event.generationOperationId}`
3544
- );
3545
- }
3546
- closedSlots.set(event.slotId, event);
3547
- closedSlotEvents.push(event);
3548
- continue;
3549
- }
3550
- if (event.kind === "candidate-decided") {
3551
- const candidate = candidates.get(event.candidateId);
3552
- if (!candidate) {
3553
- throw new SearchLedgerIntegrityError(
3554
- `decision ${event.eventId} references unknown candidate ${event.candidateId}`
3555
- );
3556
- }
3557
- if (candidate.decision) {
3558
- throw new SearchLedgerIntegrityError(`candidate ${event.candidateId} was decided twice`);
3559
- }
3560
- if (event.decision.status === "selected") {
3561
- if (!candidate.attempts.some((attempt) => attempt.outcome.status !== "errored")) {
3562
- throw new SearchLedgerIntegrityError(
3563
- `candidate ${event.candidateId} cannot be selected without a measured task outcome`
3564
- );
3565
- }
3566
- if (decisions.some((decision) => decision.decision.status === "selected")) {
3567
- throw new SearchLedgerIntegrityError("more than one candidate was selected");
3568
- }
3569
- }
3570
- candidate.decision = event;
3571
- decisions.push(event);
3572
- continue;
3573
- }
3574
- const missingCandidateSlots2 = planEvent.plan.candidateSlots.filter((slot) => !candidateBySlot.has(slot.slotId) && !closedSlots.has(slot.slotId)).map((slot) => slot.slotId);
3575
- if (missingCandidateSlots2.length > 0) {
3576
- throw new SearchLedgerIntegrityError(
3577
- `search completed with missing candidate slots: ${missingCandidateSlots2.join(", ")}`
3578
- );
3579
- }
3580
- const missingTaskOutcomes2 = plannedTaskOutcomeKeys(planEvent, candidates);
3581
- if (missingTaskOutcomes2.length > 0) {
3582
- throw new SearchLedgerIntegrityError(
3583
- `search completed with missing task outcomes: ${missingTaskOutcomes2.join(", ")}`
3584
- );
3585
- }
3586
- const missingOperations2 = planEvent.plan.operations.filter((operation) => !operationsById.has(operation.operationId)).map((operation) => operation.operationId);
3587
- if (missingOperations2.length > 0) {
3588
- throw new SearchLedgerIntegrityError(
3589
- `search completed with missing search operations: ${missingOperations2.join(", ")}`
3590
- );
3591
- }
3592
- for (const operation of planEvent.plan.operations) {
3593
- if (operation.kind !== "candidate-generation") continue;
3594
- const generationOutcome = operationsById.get(operation.operationId).outcome.status;
3595
- const slots = planEvent.plan.candidateSlots.filter(
3596
- (slot) => slot.generationOperationId === operation.operationId
3597
- );
3598
- if (slots.length === 0) continue;
3599
- const registeredCount = slots.filter((slot) => candidateBySlot.has(slot.slotId)).length;
3600
- const closedCount = slots.filter((slot) => closedSlots.has(slot.slotId)).length;
3601
- if (generationOutcome === "completed" && closedCount > 0) {
3602
- throw new SearchLedgerIntegrityError(
3603
- `completed generation operation ${operation.operationId} contains ${closedCount} closed slot(s)`
3604
- );
3605
- }
3606
- if (generationOutcome === "failed" && registeredCount > 0) {
3607
- throw new SearchLedgerIntegrityError(
3608
- `failed generation operation ${operation.operationId} contains ${registeredCount} registered candidate(s)`
3609
- );
3610
- }
3611
- if (generationOutcome === "partial" && (registeredCount === 0 || closedCount === 0)) {
3612
- throw new SearchLedgerIntegrityError(
3613
- `partial generation operation ${operation.operationId} must contain both a registered candidate and a closed slot`
3614
- );
3615
- }
3616
- }
3617
- const pending = [...candidates.values()].filter((candidate) => candidate.decision === null);
3618
- if (pending.length > 0) {
3619
- throw new SearchLedgerIntegrityError(
3620
- `search completed with ${pending.length} candidate decision(s) missing`
3621
- );
3622
- }
3623
- const selected = decisions.filter((decision) => decision.decision.status === "selected");
3624
- if (event.result.status === "selected") {
3625
- if (selected.length !== 1 || selected[0].candidateId !== event.result.candidateId) {
3626
- throw new SearchLedgerIntegrityError(
3627
- `search completion winner ${event.result.candidateId} does not match candidate decisions`
3628
- );
3629
- }
3630
- } else if (selected.length !== 0) {
3631
- throw new SearchLedgerIntegrityError("all-rejected completion contains a selected candidate");
3632
- }
3633
- completion = event;
3634
- }
3635
- const selectedDecisions = decisions.filter((decision) => decision.decision.status === "selected");
3636
- const rejectedDecisions = decisions.filter((decision) => decision.decision.status === "rejected");
3637
- const outcomeCounts = { passed: 0, failed: 0, errored: 0 };
3638
- const operationOutcomeCounts = { completed: 0, partial: 0, failed: 0 };
3639
- let inputTokens = 0;
3640
- let outputTokens = 0;
3641
- let cachedTokens = 0;
3642
- let costUsd = 0;
3643
- const unknownTokenEventIds = [];
3644
- const unknownCostEventIds = [];
3645
- for (const attempt of attempts) {
3646
- outcomeCounts[attempt.outcome.status] += 1;
3647
- }
3648
- for (const operation of operationEvents) {
3649
- operationOutcomeCounts[operation.outcome.status] += 1;
3650
- }
3651
- for (const costedEvent of [...attempts, ...operationEvents]) {
3652
- if (costedEvent.accounting.tokens.status === "known") {
3653
- inputTokens += costedEvent.accounting.tokens.inputTokens;
3654
- outputTokens += costedEvent.accounting.tokens.outputTokens;
3655
- cachedTokens += costedEvent.accounting.tokens.cachedTokens;
3656
- } else {
3657
- unknownTokenEventIds.push(costedEvent.eventId);
3658
- }
3659
- if (costedEvent.accounting.cost.status === "known") {
3660
- costUsd += costedEvent.accounting.cost.usd;
3661
- } else {
3662
- costUsd += costedEvent.accounting.cost.knownLowerBoundUsd;
3663
- unknownCostEventIds.push(costedEvent.eventId);
3664
- }
3665
- }
3666
- const accounting = unknownTokenEventIds.length === 0 && unknownCostEventIds.length === 0 ? {
3667
- status: "known",
3668
- inputTokens,
3669
- outputTokens,
3670
- cachedTokens,
3671
- costUsd
3672
- } : {
3673
- status: "partial",
3674
- knownInputTokens: inputTokens,
3675
- knownOutputTokens: outputTokens,
3676
- knownCachedTokens: cachedTokens,
3677
- knownCostUsd: costUsd,
3678
- unknownTokenEventIds,
3679
- unknownCostEventIds
3680
- };
3681
- const selectedCandidateId = completion?.result.status === "selected" ? completion.result.candidateId : null;
3682
- const status = completion?.result.status === "selected" ? "selected" : completion?.result.status === "all-rejected" ? "all-rejected" : "in-progress";
3683
- const missingCandidateSlots = planEvent?.plan.candidateSlots.filter((slot) => !candidateBySlot.has(slot.slotId) && !closedSlots.has(slot.slotId)).map((slot) => slot.slotId) ?? [];
3684
- const missingTaskOutcomes = planEvent ? plannedTaskOutcomeKeys(planEvent, candidates) : [];
3685
- const missingOperations = planEvent?.plan.operations.filter((operation) => !operationsById.has(operation.operationId)).map((operation) => operation.operationId) ?? [];
3686
- return {
3687
- entries: [...entries],
3688
- plan: planEvent,
3689
- candidates: candidateEvents,
3690
- closedCandidateSlots: closedSlotEvents,
3691
- attempts,
3692
- operations: operationEvents,
3693
- decisions,
3694
- completion,
3695
- audit: {
3696
- campaignId,
3697
- eventCount: entries.length,
3698
- candidateCount: candidates.size,
3699
- closedCandidateSlotCount: closedSlots.size,
3700
- attemptCount: attempts.length,
3701
- operationCount: operationEvents.length,
3702
- outcomes: outcomeCounts,
3703
- operationOutcomes: operationOutcomeCounts,
3704
- decisions: {
3705
- selected: selectedDecisions.length,
3706
- rejected: rejectedDecisions.length,
3707
- pending: candidates.size - decisions.length
3708
- },
3709
- expected: {
3710
- candidateSlots: planEvent?.plan.candidateSlots.length ?? 0,
3711
- taskOutcomes: candidates.size * (planEvent?.plan.tasks.length ?? 0),
3712
- operations: planEvent?.plan.operations.length ?? 0,
3713
- missingCandidateSlots,
3714
- missingTaskOutcomes,
3715
- missingOperations
3716
- },
3717
- status,
3718
- selectedCandidateId,
3719
- accounting,
3720
- headHash: entries.at(-1)?.entryHash ?? null
3721
- }
3722
- };
3723
- }
3724
- function plannedTaskOutcomeKeys(planEvent, candidates) {
3725
- const missing = [];
3726
- const registeredCandidates = [...candidates.values()].sort(
3727
- (a, b) => compareStrings(a.registered.slotId, b.registered.slotId)
3728
- );
3729
- for (const candidate of registeredCandidates) {
3730
- const slotId = candidate.registered.slotId;
3731
- for (const task of planEvent.plan.tasks) {
3732
- const measured = candidate.attempts.some(
3733
- (attempt) => attempt.task.taskId === task.taskId && attempt.outcome.status !== "errored"
3734
- );
3735
- if (!measured) missing.push(`${slotId}/${task.taskId}`);
3736
- }
3737
- }
3738
- return missing;
3739
- }
3740
- function normalizeEvent(event) {
3741
- const artifacts = sortArtifacts(event.artifacts);
3742
- if (event.kind === "search-planned") {
3743
- return {
3744
- ...event,
3745
- artifacts,
3746
- plan: {
3747
- candidateSlots: [...event.plan.candidateSlots].sort(
3748
- (a, b) => compareStrings(a.slotId, b.slotId)
3749
- ),
3750
- tasks: [...event.plan.tasks].sort((a, b) => compareStrings(a.taskId, b.taskId)),
3751
- operations: [...event.plan.operations].sort(
3752
- (a, b) => compareStrings(a.operationId, b.operationId)
3753
- )
3754
- }
3755
- };
3756
- }
3757
- if (event.kind === "candidate-registered") {
3758
- return {
3759
- ...event,
3760
- artifacts,
3761
- lineage: {
3762
- ...event.lineage,
3763
- parentCandidateIds: sortedStrings(event.lineage.parentCandidateIds)
3764
- },
3765
- surfaces: [...event.surfaces].map((surface) => ({ ...surface, artifact: { ...surface.artifact } })).sort((a, b) => compareStrings(a.surfaceId, b.surfaceId))
3766
- };
3767
- }
3768
- if (event.kind === "task-attempted") {
3769
- return {
3770
- ...event,
3771
- artifacts,
3772
- surfaceEvidence: [...event.surfaceEvidence].map((evidence) => ({ ...evidence, evidence: sortArtifacts(evidence.evidence) })).sort((a, b) => compareStrings(a.surfaceId, b.surfaceId))
3773
- };
3774
- }
3775
- return { ...event, artifacts };
3776
- }
3777
- function sortArtifacts(artifacts) {
3778
- return [...artifacts].map((artifact) => ({ ...artifact })).sort((a, b) => compareStrings(artifactKey(a), artifactKey(b)));
3779
- }
3780
- function artifactKey(artifact) {
3781
- return canonicalString(artifact);
3782
- }
3783
- function compareStrings(a, b) {
3784
- return a < b ? -1 : a > b ? 1 : 0;
3785
- }
3786
- function sortedStrings(values) {
3787
- return [...values].sort();
3788
- }
3789
- function assertUnique2(values, label, eventId) {
3790
- if (new Set(values).size !== values.length) {
3791
- throw new SearchLedgerIntegrityError(`event ${eventId} contains duplicate ${label} values`);
3792
- }
3793
- }
3794
- function canonicalString(value) {
3795
- return JSON.stringify(canonicalize(value));
3796
- }
3797
- function hashCanonical(value) {
3798
- return `sha256:${createHash6("sha256").update(canonicalString(value)).digest("hex")}`;
3799
- }
3800
- function formatZodError(error) {
3801
- return error.issues.map((issue) => `${issue.path.length > 0 ? issue.path.join(".") : "<root>"}: ${issue.message}`).join("; ");
3802
- }
3803
-
3804
- // src/campaign/transient-failure.ts
3805
- var BASE_TRANSIENT = /\b50[234]\b|no stream output|produced no stream|admission timed out|admission_rejected|queue_timeout|fetch failed|ECONNRESET|This operation was aborted/i;
3806
- var TIMEOUT_PATTERN = /timeout after \d+ ?ms|cli-bridge timeout/i;
3807
- function isTransientTransportFailure(message, opts = {}) {
3808
- if (!message) return false;
3809
- if (BASE_TRANSIENT.test(message)) return true;
3810
- if ((opts.retryFullDurationTimeouts ?? false) && TIMEOUT_PATTERN.test(message)) return true;
3811
- for (const p of opts.extraPatterns ?? []) if (p.test(message)) return true;
3812
- return false;
3813
- }
3814
-
3815
- // src/campaign/worktree/index.ts
3816
- import { execFileSync } from "child_process";
3817
- import { createHash as createHash7 } from "crypto";
3818
- import {
3819
- closeSync,
3820
- existsSync as existsSync4,
3821
- constants as fsConstants,
3822
- fstatSync,
3823
- lstatSync,
3824
- mkdirSync as mkdirSync2,
3825
- mkdtempSync,
3826
- openSync,
3827
- readlinkSync,
3828
- readSync,
3829
- realpathSync,
3830
- rmSync
3831
- } from "fs";
3832
- import { devNull, tmpdir } from "os";
3833
- import { basename, dirname, isAbsolute as isAbsolute2, join as join4, relative as relative2, resolve as resolve3, sep } from "path";
3834
- var MAX_GIT_OUTPUT_BYTES = 256 * 1024 * 1024;
3835
- var FILE_HASH_CHUNK_BYTES = 1024 * 1024;
3836
- var GIT_REPOSITORY_ENV = /* @__PURE__ */ new Set([
3837
- "GIT_ALTERNATE_OBJECT_DIRECTORIES",
3838
- "GIT_COMMON_DIR",
3839
- "GIT_DIR",
3840
- "GIT_INDEX_FILE",
3841
- "GIT_NAMESPACE",
3842
- "GIT_OBJECT_DIRECTORY",
3843
- "GIT_PREFIX",
3844
- "GIT_QUARANTINE_PATH",
3845
- "GIT_WORK_TREE"
3846
- ]);
3847
- var WorktreeAdapterError = class extends Error {
3848
- constructor(message, cause) {
3849
- super(message);
3850
- this.cause = cause;
3851
- this.name = "WorktreeAdapterError";
3852
- }
3853
- cause;
3854
- };
3855
- function defaultGit(args, cwd, overrides) {
3856
- try {
3857
- const env = { ...process.env };
3858
- for (const key of Object.keys(env)) {
3859
- if (GIT_REPOSITORY_ENV.has(key) || key === "GIT_CONFIG" || key.startsWith("GIT_CONFIG_") || key.startsWith("GIT_ATTR_") || key === "GIT_DIFF_OPTS" || key === "GIT_EXTERNAL_DIFF") {
3860
- delete env[key];
3861
- }
3862
- }
3863
- Object.assign(env, overrides);
3864
- env.GIT_NO_REPLACE_OBJECTS = "1";
3865
- env.LC_ALL = "C";
3866
- env.LANG = "C";
3867
- return execFileSync("git", args, { cwd, env, maxBuffer: MAX_GIT_OUTPUT_BYTES });
3868
- } catch (err) {
3869
- const stderr = err && typeof err === "object" && "stderr" in err ? String(err.stderr) : "";
3870
- throw new WorktreeAdapterError(`git ${args.join(" ")} failed: ${stderr || String(err)}`, err);
3871
- }
3872
- }
3873
- function gitBytes(git, args, cwd, env) {
3874
- try {
3875
- const output = git(args, cwd, env);
3876
- return typeof output === "string" ? Buffer.from(output, "utf8") : Buffer.from(output);
3877
- } catch (err) {
3878
- if (err instanceof WorktreeAdapterError) throw err;
3879
- throw new WorktreeAdapterError(`git ${args.join(" ")} failed: ${String(err)}`, err);
3880
- }
3881
- }
3882
- function gitText(git, args, cwd, env) {
3883
- return gitBytes(git, args, cwd, env).toString("utf8").trim();
3884
- }
3885
- function hasRegisteredWorktree(git, repoRoot, path) {
3886
- const expected = Buffer.from(`worktree ${resolve3(path)}`, "utf8");
3887
- const records = gitBytes(git, ["worktree", "list", "--porcelain", "-z"], repoRoot);
3888
- let start = 0;
3889
- while (start < records.length) {
3890
- const end = records.indexOf(0, start);
3891
- if (end < 0) {
3892
- throw new WorktreeAdapterError("Git worktree list output was not NUL-terminated");
3893
- }
3894
- if (records.subarray(start, end).equals(expected)) return true;
3895
- start = end + 1;
3896
- }
3897
- return false;
3898
- }
3899
- function hasLocalBranch(git, repoRoot, branch) {
3900
- const ref = `refs/heads/${branch}`;
3901
- return gitText(git, ["for-each-ref", "--format=%(refname)", "--", ref], repoRoot).split("\n").some((candidate) => candidate === ref);
3902
- }
3903
- function reconcileAbsent(exists, remove) {
3904
- try {
3905
- if (!exists()) return void 0;
3906
- } catch (err) {
3907
- return err;
3908
- }
3909
- try {
3910
- remove();
3911
- return void 0;
3912
- } catch (removeError) {
3913
- try {
3914
- if (!exists()) return void 0;
3915
- } catch (recheckError) {
3916
- return new AggregateError(
3917
- [removeError, recheckError],
3918
- "Removal failed and the resulting resource state could not be checked"
3919
- );
3920
- }
3921
- return removeError;
3922
- }
3923
- }
3924
- function sha2562(bytes) {
3925
- return `sha256:${createHash7("sha256").update(bytes).digest("hex")}`;
3926
- }
3927
- var PATCH_ARGS = [
3928
- // Git binary patches embed a zlib stream. Pin no-compression at command
3929
- // scope so repository/global config and compressor heuristics cannot change
3930
- // the candidate identity for the same base and final tree.
3931
- "-c",
3932
- "core.compression=0",
3933
- "-c",
3934
- `core.attributesFile=${devNull}`,
3935
- "-c",
3936
- "core.quotePath=true",
3937
- "-c",
3938
- "diff.suppressBlankEmpty=false",
3939
- "diff",
3940
- "--unified=3",
3941
- "--inter-hunk-context=0",
3942
- `-O${devNull}`,
3943
- "--binary",
3944
- "--full-index",
3945
- "--no-color",
3946
- "--no-ext-diff",
3947
- "--no-textconv",
3948
- "--no-renames",
3949
- "--diff-algorithm=myers",
3950
- "--no-indent-heuristic",
3951
- "--src-prefix=a/",
3952
- "--dst-prefix=b/"
3953
- ];
3954
- var CANONICAL_GIT_ENV = {
3955
- GIT_ATTR_GLOBAL: devNull,
3956
- GIT_ATTR_NOSYSTEM: "1",
3957
- GIT_CONFIG_GLOBAL: devNull,
3958
- GIT_CONFIG_NOSYSTEM: "1",
3959
- GIT_CONFIG_SYSTEM: devNull
3960
- };
3961
- function patchBytes(git, cwd, baseCommit, candidateCommit) {
3962
- const scratch = mkdtempSync(join4(tmpdir(), "agent-eval-patch-"));
3963
- const bareRepo = join4(scratch, "repo.git");
3964
- const emptyTemplate = join4(scratch, "empty-template");
3965
- mkdirSync2(emptyTemplate);
3966
- try {
3967
- const objectFormat = gitObjectHashAlgorithm(candidateCommit);
3968
- const sourceObjects = realpathSync(gitText(git, ["rev-parse", "--git-path", "objects"], cwd));
3969
- gitText(
3970
- git,
3971
- [
3972
- "init",
3973
- "--bare",
3974
- "--quiet",
3975
- ...objectFormat === "sha256" ? ["--object-format=sha256"] : [],
3976
- `--template=${emptyTemplate}`,
3977
- bareRepo
3978
- ],
3979
- scratch,
3980
- CANONICAL_GIT_ENV
3981
- );
3982
- return gitBytes(
3983
- git,
3984
- [`--git-dir=${bareRepo}`, ...PATCH_ARGS, baseCommit, candidateCommit, "--"],
3985
- scratch,
3986
- {
3987
- ...CANONICAL_GIT_ENV,
3988
- GIT_ALTERNATE_OBJECT_DIRECTORIES: sourceObjects
3989
- }
3990
- );
3991
- } finally {
3992
- rmSync(scratch, { recursive: true, force: true });
3993
- }
3994
- }
3995
- function resolveCommit(git, cwd, ref) {
3996
- return gitText(git, ["rev-parse", "--verify", `${ref}^{commit}`], cwd);
3997
- }
3998
- function unresolvedWorktreePath(surface, worktreeDir) {
3999
- if (isAbsolute2(surface.worktreeRef)) return surface.worktreeRef;
4000
- if (worktreeDir) return join4(worktreeDir, basename(surface.worktreeRef));
4001
- return surface.worktreeRef;
4002
- }
4003
- function displayGitPath(path) {
4004
- return JSON.stringify(path);
4005
- }
4006
- function parseGitTreeEntries(bytes) {
4007
- const input = Buffer.from(bytes);
4008
- const entries = [];
4009
- let start = 0;
4010
- while (start < input.length) {
4011
- const end = input.indexOf(0, start);
4012
- if (end < 0) throw new WorktreeAdapterError("Git tree output was not NUL-terminated");
4013
- const record = input.subarray(start, end);
4014
- start = end + 1;
4015
- if (record.length === 0) continue;
4016
- const tab = record.indexOf(9);
4017
- if (tab < 0) throw new WorktreeAdapterError("Git tree entry did not contain a path");
4018
- const metadata = record.subarray(0, tab).toString("ascii").split(" ");
4019
- if (metadata.length !== 3) throw new WorktreeAdapterError("Git tree entry was malformed");
4020
- const mode = metadata[0];
4021
- const type = metadata[1];
4022
- const objectId = metadata[2];
4023
- if (!mode || !type || !objectId) {
4024
- throw new WorktreeAdapterError("Git tree entry was malformed");
4025
- }
4026
- const pathBytes = record.subarray(tab + 1);
4027
- const path = pathBytes.toString("utf8");
4028
- if (!Buffer.from(path, "utf8").equals(pathBytes)) {
4029
- throw new WorktreeAdapterError("CodeSurface paths must be valid UTF-8");
4030
- }
4031
- if (type !== "blob" && type !== "commit") {
4032
- throw new WorktreeAdapterError(
4033
- `CodeSurface contains unsupported Git object type ${type} at ${displayGitPath(path)}`
4034
- );
4035
- }
4036
- entries.push({ mode, objectId, path });
4037
- }
4038
- return entries;
4039
- }
4040
- function parseGitPathList(bytes) {
4041
- const input = Buffer.from(bytes);
4042
- const paths = [];
4043
- let start = 0;
4044
- while (start < input.length) {
4045
- const end = input.indexOf(0, start);
4046
- if (end < 0) throw new WorktreeAdapterError("Git path output was not NUL-terminated");
4047
- const pathBytes = input.subarray(start, end);
4048
- start = end + 1;
4049
- if (pathBytes.length === 0) continue;
4050
- const path = pathBytes.toString("utf8");
4051
- if (!Buffer.from(path, "utf8").equals(pathBytes)) {
4052
- throw new WorktreeAdapterError("CodeSurface paths must be valid UTF-8");
4053
- }
4054
- paths.push(path);
4055
- }
4056
- return paths;
4057
- }
4058
- function isWithinRoot(root, candidate) {
4059
- const rel = relative2(root, candidate);
4060
- return rel === "" || !isAbsolute2(rel) && rel !== ".." && !rel.startsWith(`..${sep}`);
4061
- }
4062
- function assertSafeRelativePath(root, path) {
4063
- if (path.length === 0 || isAbsolute2(path)) {
4064
- throw new WorktreeAdapterError(`CodeSurface contains unsafe path ${displayGitPath(path)}`);
4065
- }
4066
- const absolutePath = resolve3(root, path);
4067
- if (!isWithinRoot(root, absolutePath) || absolutePath === root) {
4068
- throw new WorktreeAdapterError(`CodeSurface path escapes its worktree: ${displayGitPath(path)}`);
4069
- }
4070
- const segments = path.split(/[\\/]/u);
4071
- let parent = root;
4072
- for (const segment of segments.slice(0, -1)) {
4073
- if (segment.length === 0 || segment === "." || segment === "..") {
4074
- throw new WorktreeAdapterError(`CodeSurface contains unsafe path ${displayGitPath(path)}`);
4075
- }
4076
- parent = join4(parent, segment);
4077
- const stat = lstatSync(parent);
4078
- if (!stat.isDirectory() || stat.isSymbolicLink()) {
4079
- throw new WorktreeAdapterError(
4080
- `CodeSurface path traverses a non-directory or symbolic link: ${displayGitPath(path)}`
4081
- );
4082
- }
4083
- }
4084
- return absolutePath;
4085
- }
4086
- function gitObjectHashAlgorithm(objectId) {
4087
- if (/^[a-f0-9]{40}$/.test(objectId)) return "sha1";
4088
- if (/^[a-f0-9]{64}$/.test(objectId)) return "sha256";
4089
- throw new WorktreeAdapterError(`Unsupported Git object id: ${objectId}`);
4090
- }
4091
- function hashGitBlobBytes(bytes, objectId) {
4092
- const hash = createHash7(gitObjectHashAlgorithm(objectId));
4093
- hash.update(`blob ${bytes.byteLength}\0`);
4094
- hash.update(bytes);
4095
- return hash.digest("hex");
4096
- }
4097
- function hashGitBlobFile(path, objectId) {
4098
- const before = lstatSync(path);
4099
- if (!before.isFile() || before.isSymbolicLink()) {
4100
- throw new WorktreeAdapterError(`CodeSurface expected a regular file at ${displayGitPath(path)}`);
4101
- }
4102
- const noFollow = process.platform === "win32" ? 0 : fsConstants.O_NOFOLLOW;
4103
- const fd = openSync(path, fsConstants.O_RDONLY | noFollow);
4104
- try {
4105
- const opened = fstatSync(fd);
4106
- if (!opened.isFile() || opened.dev !== before.dev || opened.ino !== before.ino || opened.size !== before.size) {
4107
- throw new WorktreeAdapterError(
4108
- `CodeSurface file changed while it was being verified: ${displayGitPath(path)}`
4109
- );
4110
- }
4111
- const hash = createHash7(gitObjectHashAlgorithm(objectId));
4112
- hash.update(`blob ${opened.size}\0`);
4113
- const chunk = Buffer.allocUnsafe(FILE_HASH_CHUNK_BYTES);
4114
- let total = 0;
4115
- while (true) {
4116
- const count = readSync(fd, chunk, 0, chunk.length, null);
4117
- if (count === 0) break;
4118
- total += count;
4119
- hash.update(chunk.subarray(0, count));
4120
- }
4121
- if (total !== opened.size) {
4122
- throw new WorktreeAdapterError(
4123
- `CodeSurface file changed while it was being verified: ${displayGitPath(path)}`
4124
- );
4125
- }
4126
- return { hash: hash.digest("hex"), executable: (opened.mode & 73) !== 0 };
4127
- } finally {
4128
- closeSync(fd);
4129
- }
4130
- }
4131
- function assertSymlinkTargetIsBound(root, linkPath, trackedPaths) {
4132
- const targetBytes = readlinkSync(linkPath, { encoding: "buffer" });
4133
- const target = targetBytes.toString("utf8");
4134
- if (!Buffer.from(target, "utf8").equals(targetBytes)) {
4135
- throw new WorktreeAdapterError(
4136
- `CodeSurface symbolic link has an unsafe target at ${displayGitPath(linkPath)}`
4137
- );
4138
- }
4139
- if (isAbsolute2(target)) {
4140
- throw new WorktreeAdapterError(
4141
- `CodeSurface symbolic link escapes its worktree at ${displayGitPath(linkPath)}`
4142
- );
4143
- }
4144
- const lexicalTarget = resolve3(dirname(linkPath), target);
4145
- if (!isWithinRoot(root, lexicalTarget)) {
4146
- throw new WorktreeAdapterError(
4147
- `CodeSurface symbolic link escapes its worktree at ${displayGitPath(linkPath)}`
4148
- );
4149
- }
4150
- let resolvedTarget;
4151
- try {
4152
- resolvedTarget = realpathSync(linkPath);
4153
- } catch (err) {
4154
- throw new WorktreeAdapterError(
4155
- `CodeSurface symbolic link target is missing or cyclic at ${displayGitPath(linkPath)}`,
4156
- err
4157
- );
4158
- }
4159
- if (!isWithinRoot(root, resolvedTarget)) {
4160
- throw new WorktreeAdapterError(
4161
- `CodeSurface symbolic link escapes its worktree at ${displayGitPath(linkPath)}`
4162
- );
4163
- }
4164
- const targetRelative = relative2(root, resolvedTarget).split(sep).join("/");
4165
- const targetIsTracked = trackedPaths.has(targetRelative) || [...trackedPaths].some((trackedPath) => trackedPath.startsWith(`${targetRelative}/`));
4166
- if (!targetIsTracked) {
4167
- throw new WorktreeAdapterError(
4168
- `CodeSurface symbolic link resolves to untracked content at ${displayGitPath(linkPath)}`
4169
- );
4170
- }
4171
- }
4172
- function assertRawTreeMatchesWorktree(git, root, candidateCommit) {
4173
- const entries = parseGitTreeEntries(
4174
- gitBytes(git, ["ls-tree", "-r", "-z", "--full-tree", candidateCommit], root)
4175
- );
4176
- const trackedPaths = new Set(entries.map((entry) => entry.path));
4177
- for (const entry of entries) {
4178
- const absolutePath = assertSafeRelativePath(root, entry.path);
4179
- if (entry.mode === "160000" || entry.mode === "040000") {
4180
- throw new WorktreeAdapterError(
4181
- `CodeSurface contains a Git submodule whose executable bytes are not bound: ${displayGitPath(entry.path)}`
4182
- );
4183
- }
4184
- if (entry.mode === "120000") {
4185
- const stat = lstatSync(absolutePath);
4186
- if (!stat.isSymbolicLink()) {
4187
- throw new WorktreeAdapterError(
4188
- `CodeSurface expected a symbolic link at ${displayGitPath(entry.path)}`
4189
- );
4190
- }
4191
- const targetBytes = readlinkSync(absolutePath, { encoding: "buffer" });
4192
- const actualObjectId = hashGitBlobBytes(targetBytes, entry.objectId);
4193
- if (actualObjectId !== entry.objectId) {
4194
- throw new WorktreeAdapterError(
4195
- `CodeSurface raw symbolic-link bytes differ from the candidate tree at ${displayGitPath(entry.path)}`
4196
- );
4197
- }
4198
- assertSymlinkTargetIsBound(root, absolutePath, trackedPaths);
4199
- continue;
4200
- }
4201
- if (entry.mode !== "100644" && entry.mode !== "100755") {
4202
- throw new WorktreeAdapterError(
4203
- `CodeSurface contains unsupported Git mode ${entry.mode} at ${displayGitPath(entry.path)}`
4204
- );
4205
- }
4206
- const actual = hashGitBlobFile(absolutePath, entry.objectId);
4207
- if (actual.hash !== entry.objectId) {
4208
- throw new WorktreeAdapterError(
4209
- `CodeSurface raw tracked bytes differ from the candidate tree because the worktree changed after finalization at ${displayGitPath(entry.path)}`
4210
- );
4211
- }
4212
- if (process.platform !== "win32" && actual.executable !== (entry.mode === "100755")) {
4213
- throw new WorktreeAdapterError(
4214
- `CodeSurface executable mode differs from the candidate tree at ${displayGitPath(entry.path)}`
4215
- );
4216
- }
4217
- }
4218
- }
4219
- function verifyCodeSurfaceWithGit(surface, path, git) {
4220
- assertCodeSurfaceIdentity(surface);
4221
- if (!existsSync4(path)) {
4222
- throw new WorktreeAdapterError(`CodeSurface worktree does not exist: ${path}`);
4223
- }
4224
- const lexicalRoot = resolve3(path);
4225
- const canonicalRoot = realpathSync(path);
4226
- if (lstatSync(lexicalRoot).isSymbolicLink() || process.platform !== "win32" && lexicalRoot !== canonicalRoot) {
4227
- throw new WorktreeAdapterError(
4228
- `CodeSurface worktree locator must not contain a symbolic link: ${path}`
4229
- );
4230
- }
4231
- const repoRoot = gitText(git, ["rev-parse", "--show-toplevel"], path);
4232
- const canonicalRepoRoot = realpathSync(repoRoot);
4233
- if (canonicalRepoRoot !== canonicalRoot) {
4234
- throw new WorktreeAdapterError(
4235
- `CodeSurface worktree locator is not the repository root: expected ${repoRoot}, got ${path}`
4236
- );
4237
- }
4238
- const indexFlags = gitText(git, ["ls-files", "-v"], path).split("\n").filter((line) => line.length > 0 && (line.startsWith("S ") || /^[a-z]/.test(line)));
4239
- if (indexFlags.length > 0) {
4240
- throw new WorktreeAdapterError(
4241
- `CodeSurface worktree uses hidden index entries that cannot be verified: ${path}
4242
- ${indexFlags.join("\n")}`
4243
- );
4244
- }
4245
- const extraPaths = [
4246
- ...parseGitPathList(gitBytes(git, ["ls-files", "--others", "--exclude-standard", "-z"], path)),
4247
- ...parseGitPathList(
4248
- gitBytes(git, ["ls-files", "--others", "--ignored", "--exclude-standard", "-z"], path)
4249
- )
4250
- ];
4251
- if (extraPaths.length > 0) {
4252
- throw new WorktreeAdapterError(
4253
- `CodeSurface worktree changed after finalization: ${path}
4254
- ${extraPaths.join("\n")}`
4255
- );
4256
- }
4257
- const candidateCommit = resolveCommit(git, path, "HEAD");
4258
- if (candidateCommit !== surface.candidateCommit) {
4259
- throw new WorktreeAdapterError(
4260
- `CodeSurface candidate commit mismatch: expected ${surface.candidateCommit}, got ${candidateCommit}`
4261
- );
4262
- }
4263
- const baseCommit = resolveCommit(git, path, surface.baseCommit);
4264
- if (baseCommit !== surface.baseCommit) {
4265
- throw new WorktreeAdapterError(
4266
- `CodeSurface base commit mismatch: expected ${surface.baseCommit}, got ${baseCommit}`
4267
- );
4268
- }
4269
- const baseTree = gitText(git, ["rev-parse", "--verify", `${surface.baseCommit}^{tree}`], path);
4270
- if (baseTree !== surface.baseTree) {
4271
- throw new WorktreeAdapterError(
4272
- `CodeSurface base tree mismatch: expected ${surface.baseTree}, got ${baseTree}`
4273
- );
4274
- }
4275
- const candidateTree = gitText(
4276
- git,
4277
- ["rev-parse", "--verify", `${surface.candidateCommit}^{tree}`],
4278
- path
4279
- );
4280
- if (candidateTree !== surface.candidateTree) {
4281
- throw new WorktreeAdapterError(
4282
- `CodeSurface tree mismatch: expected ${surface.candidateTree}, got ${candidateTree}`
4283
- );
4284
- }
4285
- try {
4286
- gitText(
4287
- git,
4288
- [
4289
- "diff-index",
4290
- "--cached",
4291
- "--quiet",
4292
- "--no-ext-diff",
4293
- "--no-textconv",
4294
- surface.candidateCommit,
4295
- "--"
4296
- ],
4297
- path
4298
- );
4299
- } catch (err) {
4300
- throw new WorktreeAdapterError(
4301
- `CodeSurface index differs from finalized candidate: ${path}`,
4302
- err
4303
- );
4304
- }
4305
- try {
4306
- assertRawTreeMatchesWorktree(git, canonicalRepoRoot, surface.candidateCommit);
4307
- } catch (err) {
4308
- if (err instanceof WorktreeAdapterError) throw err;
4309
- throw new WorktreeAdapterError(`CodeSurface raw tree verification failed: ${path}`, err);
4310
- }
4311
- const mergeBase = gitText(git, ["merge-base", surface.baseCommit, surface.candidateCommit], path);
4312
- if (mergeBase !== surface.baseCommit) {
4313
- throw new WorktreeAdapterError(
4314
- `CodeSurface candidate ${surface.candidateCommit} does not descend from base ${surface.baseCommit}`
4315
- );
4316
- }
4317
- const patch = patchBytes(git, path, surface.baseCommit, surface.candidateCommit);
4318
- const actualPatchHash = sha2562(patch);
4319
- if (actualPatchHash !== surface.patch.sha256 || patch.byteLength !== surface.patch.byteLength) {
4320
- throw new WorktreeAdapterError(
4321
- `CodeSurface patch mismatch: expected ${surface.patch.sha256}/${surface.patch.byteLength} bytes, got ${actualPatchHash}/${patch.byteLength} bytes`
4322
- );
4323
- }
4324
- return {
4325
- path,
4326
- repoRoot: canonicalRepoRoot,
4327
- contentHash: surfaceContentHash(surface),
4328
- patchBytes: new Uint8Array(patch)
4329
- };
4330
- }
4331
- function slug(label) {
4332
- return label.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 48) || "candidate";
4333
- }
4334
- function gitWorktreeAdapter(opts) {
4335
- const git = opts.git ?? defaultGit;
4336
- const worktreeDir = opts.worktreeDir ?? join4(opts.repoRoot, ".worktrees");
4337
- const branchPrefix = opts.branchPrefix ?? "improve";
4338
- return {
4339
- async create({ baseRef, label }) {
4340
- const id = `${slug(label)}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`;
4341
- const branch = `${branchPrefix}/${id}`;
4342
- const path = join4(worktreeDir, id);
4343
- const baseCommit = resolveCommit(git, opts.repoRoot, baseRef);
4344
- const baseTree = gitText(
4345
- git,
4346
- ["rev-parse", "--verify", `${baseCommit}^{tree}`],
4347
- opts.repoRoot
4348
- );
4349
- gitText(git, ["worktree", "add", "-b", branch, path, baseCommit], opts.repoRoot);
4350
- return { path, branch, baseRef, baseCommit, baseTree };
4351
- },
4352
- async finalize(worktree, summary) {
4353
- const status = gitText(
4354
- git,
4355
- ["status", "--porcelain=v1", "--untracked-files=all"],
4356
- worktree.path
4357
- );
4358
- if (status.length > 0) {
4359
- gitText(git, ["add", "-A"], worktree.path);
4360
- gitText(git, ["commit", "-m", summary], worktree.path);
4361
- }
4362
- const candidateCommit = resolveCommit(git, worktree.path, "HEAD");
4363
- const candidateTree = gitText(
4364
- git,
4365
- ["rev-parse", "--verify", `${candidateCommit}^{tree}`],
4366
- worktree.path
4367
- );
4368
- const patch = patchBytes(git, worktree.path, worktree.baseCommit, candidateCommit);
4369
- const surface = {
4370
- kind: "code",
4371
- worktreeRef: worktree.path,
4372
- baseRef: worktree.baseRef,
4373
- baseCommit: worktree.baseCommit,
4374
- baseTree: worktree.baseTree,
4375
- candidateCommit,
4376
- candidateTree,
4377
- patch: {
4378
- format: "git-diff-binary",
4379
- sha256: sha2562(patch),
4380
- byteLength: patch.byteLength
4381
- },
4382
- summary
4383
- };
4384
- verifyCodeSurfaceWithGit(surface, worktree.path, git);
4385
- return surface;
4386
- },
4387
- async discard(worktree) {
4388
- const failures = [
4389
- reconcileAbsent(
4390
- () => hasRegisteredWorktree(git, opts.repoRoot, worktree.path),
4391
- () => gitText(git, ["worktree", "remove", "--force", "--", worktree.path], opts.repoRoot)
4392
- ),
4393
- reconcileAbsent(
4394
- () => hasLocalBranch(git, opts.repoRoot, worktree.branch),
4395
- () => gitText(git, ["branch", "-D", "--", worktree.branch], opts.repoRoot)
4396
- )
4397
- ].filter((failure) => failure !== void 0);
4398
- if (failures.length > 0) {
4399
- const cause = failures.length === 1 ? failures[0] : new AggregateError(failures, "Multiple Git resources could not be removed");
4400
- throw new WorktreeAdapterError(
4401
- `Failed to discard worktree ${worktree.path} and branch ${worktree.branch}`,
4402
- cause
4403
- );
4404
- }
4405
- }
4406
- };
4407
- }
4408
- function verifyCodeSurface(surface, worktreeDir) {
4409
- const path = unresolvedWorktreePath(surface, worktreeDir);
4410
- return verifyCodeSurfaceWithGit(surface, path, defaultGit);
4411
- }
4412
- function resolveWorktreePath(surface, worktreeDir) {
4413
- return verifyCodeSurface(surface, worktreeDir).path;
4414
- }
4415
-
4416
- export {
4417
- completionVerdict,
4418
- verifyCompletion,
4419
- parseCorrectnessResponse,
4420
- createLlmCorrectnessChecker,
4421
- createTokenRecallChecker,
4422
- extractProducedState,
4423
- CODING_HARNESSES,
4424
- HARNESS_NATIVE_MODEL,
4425
- expandProfileAxes,
4426
- harnessAxisOf,
4427
- agentProfileId,
4428
- agentProfileModelId,
4429
- agentProfileHash,
4430
- buildAnalystSurfaceDispatch,
4431
- failureModeRecallJudge,
4432
- analyzeCrossSurfaceInteractions,
4433
- discoverEvalFixtures,
4434
- loadEvalFixture,
4435
- loadEvalFixtureScenarios,
4436
- planEvalFixtureRun,
4437
- neutralizationGate,
4438
- sequentialPairedGate,
4439
- sequentialDecide,
4440
- rolloutArgumentDiff,
4441
- classifyUngroundedLiterals,
4442
- LabeledScenarioStoreError,
4443
- FsLabeledScenarioStore,
4444
- neutralizeText,
4445
- makePlaybackDispatch,
4446
- scoreUserStory,
4447
- userStoryScoreboard,
4448
- scoreboardSummary,
4449
- renderScoreboardMarkdown,
4450
- ProfileMatrixError,
4451
- runProfileMatrix,
4452
- scoreDiscrimination,
4453
- selectDiscriminative,
4454
- SEARCH_LEDGER_SCHEMA,
4455
- validateSearchLedgerEvent,
4456
- openSearchLedger,
4457
- FileSearchLedger,
4458
- isTransientTransportFailure,
4459
- WorktreeAdapterError,
4460
- gitWorktreeAdapter,
4461
- verifyCodeSurface,
4462
- resolveWorktreePath
4463
- };
4464
- //# sourceMappingURL=chunk-QB6BDBP2.js.map