@tangle-network/agent-eval 0.128.2 → 0.130.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (424) hide show
  1. package/CHANGELOG.md +279 -0
  2. package/README.md +19 -0
  3. package/dist/active-curriculum-C4mk67HP.js +214 -0
  4. package/dist/active-curriculum-C4mk67HP.js.map +1 -0
  5. package/dist/adversarial-smnADNFS.d.ts +21 -0
  6. package/dist/adversarial-smnADNFS.d.ts.map +1 -0
  7. package/dist/analyst/index.d.ts +83 -2932
  8. package/dist/analyst/index.d.ts.map +1 -0
  9. package/dist/analyst/index.js +319 -364
  10. package/dist/analyst/index.js.map +1 -1
  11. package/dist/analyst-BkTS3C58.d.ts +89 -0
  12. package/dist/analyst-BkTS3C58.d.ts.map +1 -0
  13. package/dist/analyst-LsnNpSkm.js +152 -0
  14. package/dist/analyst-LsnNpSkm.js.map +1 -0
  15. package/dist/analyze-runs-C1CavBMk.js +1067 -0
  16. package/dist/analyze-runs-C1CavBMk.js.map +1 -0
  17. package/dist/analyze-runs-FsgYCinh.d.ts +72 -0
  18. package/dist/analyze-runs-FsgYCinh.d.ts.map +1 -0
  19. package/dist/attribute-vocabulary-DLJ6303h.d.ts +54 -0
  20. package/dist/attribute-vocabulary-DLJ6303h.d.ts.map +1 -0
  21. package/dist/authenticity/index.d.ts +81 -79
  22. package/dist/authenticity/index.d.ts.map +1 -0
  23. package/dist/authenticity/index.js +209 -193
  24. package/dist/authenticity/index.js.map +1 -1
  25. package/dist/baseline-HsBvw_dk.js +550 -0
  26. package/dist/baseline-HsBvw_dk.js.map +1 -0
  27. package/dist/baseline-hG3K85h4.d.ts +125 -0
  28. package/dist/baseline-hG3K85h4.d.ts.map +1 -0
  29. package/dist/belief-state/index.d.ts +448 -1205
  30. package/dist/belief-state/index.d.ts.map +1 -0
  31. package/dist/belief-state/index.js +1617 -1710
  32. package/dist/belief-state/index.js.map +1 -1
  33. package/dist/benchmarks/index.d.ts +2 -894
  34. package/dist/benchmarks/index.js +2 -59
  35. package/dist/benchmarks-DviOvUNr.js +754 -0
  36. package/dist/benchmarks-DviOvUNr.js.map +1 -0
  37. package/dist/builder-eval/index.d.ts +150 -662
  38. package/dist/builder-eval/index.d.ts.map +1 -0
  39. package/dist/builder-eval/index.js +356 -345
  40. package/dist/builder-eval/index.js.map +1 -1
  41. package/dist/calibration-CNWWA6K8.js +94 -0
  42. package/dist/calibration-CNWWA6K8.js.map +1 -0
  43. package/dist/campaign/index.d.ts +5 -6390
  44. package/dist/campaign/index.js +3 -212
  45. package/dist/campaign-CBKZvQ1H.js +3885 -0
  46. package/dist/campaign-CBKZvQ1H.js.map +1 -0
  47. package/dist/cli.d.ts +1 -1
  48. package/dist/cli.js +164 -174
  49. package/dist/cli.js.map +1 -1
  50. package/dist/client-C97NMzqi.d.ts +581 -0
  51. package/dist/client-C97NMzqi.d.ts.map +1 -0
  52. package/dist/client-CYzbdJOZ.js +637 -0
  53. package/dist/client-CYzbdJOZ.js.map +1 -0
  54. package/dist/code-agent-session-BjkMTQ7H.js +1390 -0
  55. package/dist/code-agent-session-BjkMTQ7H.js.map +1 -0
  56. package/dist/code-agent-session-DqqgOJaz.d.ts +143 -0
  57. package/dist/code-agent-session-DqqgOJaz.d.ts.map +1 -0
  58. package/dist/concurrency-DIxRZF_J.js +85 -0
  59. package/dist/concurrency-DIxRZF_J.js.map +1 -0
  60. package/dist/contract/index.d.ts +645 -5605
  61. package/dist/contract/index.d.ts.map +1 -0
  62. package/dist/contract/index.js +1730 -1937
  63. package/dist/contract/index.js.map +1 -1
  64. package/dist/control.d.ts +3 -1030
  65. package/dist/control.js +2 -32
  66. package/dist/cost-ledger-DIgQUFZZ.js +801 -0
  67. package/dist/cost-ledger-DIgQUFZZ.js.map +1 -0
  68. package/dist/cost-ledger-Dye6jCgg.d.ts +236 -0
  69. package/dist/cost-ledger-Dye6jCgg.d.ts.map +1 -0
  70. package/dist/dataset-BvtnC8Dc.d.ts +115 -0
  71. package/dist/dataset-BvtnC8Dc.d.ts.map +1 -0
  72. package/dist/default-registry-C-vFCSEc.js +2579 -0
  73. package/dist/default-registry-C-vFCSEc.js.map +1 -0
  74. package/dist/default-registry-CNPo-Vsb.d.ts +540 -0
  75. package/dist/default-registry-CNPo-Vsb.d.ts.map +1 -0
  76. package/dist/emitter-CPBAhxum.js +266 -0
  77. package/dist/emitter-CPBAhxum.js.map +1 -0
  78. package/dist/emitter-DGQGoLyj.d.ts +113 -0
  79. package/dist/emitter-DGQGoLyj.d.ts.map +1 -0
  80. package/dist/errors-8YnH8WlF.js +64 -0
  81. package/dist/errors-8YnH8WlF.js.map +1 -0
  82. package/dist/errors-CEk209JS.d.ts +76 -0
  83. package/dist/errors-CEk209JS.d.ts.map +1 -0
  84. package/dist/eval-campaign-DEm6c8ru.js +349 -0
  85. package/dist/eval-campaign-DEm6c8ru.js.map +1 -0
  86. package/dist/execution-tracks-CpgFPpS5.js +93 -0
  87. package/dist/execution-tracks-CpgFPpS5.js.map +1 -0
  88. package/dist/exporters-q9iL-2Jf.js +148 -0
  89. package/dist/exporters-q9iL-2Jf.js.map +1 -0
  90. package/dist/extract-usage-BrQ8mCLX.js +155 -0
  91. package/dist/extract-usage-BrQ8mCLX.js.map +1 -0
  92. package/dist/failure-cluster-CqcvCcdR.d.ts +59 -0
  93. package/dist/failure-cluster-CqcvCcdR.d.ts.map +1 -0
  94. package/dist/feedback-trajectory-CVaeREXV.d.ts +340 -0
  95. package/dist/feedback-trajectory-CVaeREXV.d.ts.map +1 -0
  96. package/dist/fuzz.d.ts +320 -646
  97. package/dist/fuzz.d.ts.map +1 -0
  98. package/dist/fuzz.js +670 -617
  99. package/dist/fuzz.js.map +1 -1
  100. package/dist/hf-dataset-DBJXXoY1.js +763 -0
  101. package/dist/hf-dataset-DBJXXoY1.js.map +1 -0
  102. package/dist/hosted/index.d.ts +11 -831
  103. package/dist/hosted/index.d.ts.map +1 -0
  104. package/dist/hosted/index.js +2 -37
  105. package/dist/index-2JJSA6-r2.d.ts +926 -0
  106. package/dist/index-2JJSA6-r2.d.ts.map +1 -0
  107. package/dist/index-6N0aYmpW.d.ts +217 -0
  108. package/dist/index-6N0aYmpW.d.ts.map +1 -0
  109. package/dist/index-BAvgST_9.d.ts +131 -0
  110. package/dist/index-BAvgST_9.d.ts.map +1 -0
  111. package/dist/index-BvnJuTGD.d.ts +68 -0
  112. package/dist/index-BvnJuTGD.d.ts.map +1 -0
  113. package/dist/index-C61Wi7yg.d.ts +547 -0
  114. package/dist/index-C61Wi7yg.d.ts.map +1 -0
  115. package/dist/index-CAPUUKaM.d.ts +335 -0
  116. package/dist/index-CAPUUKaM.d.ts.map +1 -0
  117. package/dist/index-DE5fb3EC.d.ts +2244 -0
  118. package/dist/index-DE5fb3EC.d.ts.map +1 -0
  119. package/dist/index-DSC51roc.d.ts +102 -0
  120. package/dist/index-DSC51roc.d.ts.map +1 -0
  121. package/dist/index.d.ts +3776 -15120
  122. package/dist/index.d.ts.map +1 -0
  123. package/dist/index.js +11185 -11191
  124. package/dist/index.js.map +1 -1
  125. package/dist/integrity-BzRbCHzi.js +424 -0
  126. package/dist/integrity-BzRbCHzi.js.map +1 -0
  127. package/dist/integrity-rmVhXWA7.d.ts +61 -0
  128. package/dist/integrity-rmVhXWA7.d.ts.map +1 -0
  129. package/dist/judge-calibration-DFtEMlde.d.ts +146 -0
  130. package/dist/judge-calibration-DFtEMlde.d.ts.map +1 -0
  131. package/dist/ledger-core/index.d.ts +2 -0
  132. package/dist/ledger-core/index.js +2 -0
  133. package/dist/ledger-core-DtZz1RG0.js +388 -0
  134. package/dist/ledger-core-DtZz1RG0.js.map +1 -0
  135. package/dist/llm-client--GR4JbZE.js +687 -0
  136. package/dist/llm-client--GR4JbZE.js.map +1 -0
  137. package/dist/llm-client-B_nIBlYo.d.ts +290 -0
  138. package/dist/llm-client-B_nIBlYo.d.ts.map +1 -0
  139. package/dist/matrix/index.d.ts +3 -155
  140. package/dist/matrix/index.js +2 -8
  141. package/dist/matrix-BzQnu2S6.js +270 -0
  142. package/dist/matrix-BzQnu2S6.js.map +1 -0
  143. package/dist/meta-eval/index.d.ts +4 -1027
  144. package/dist/meta-eval/index.js +393 -390
  145. package/dist/meta-eval/index.js.map +1 -1
  146. package/dist/metrics-C9YY1OcL.js +239 -0
  147. package/dist/metrics-C9YY1OcL.js.map +1 -0
  148. package/dist/mint-yN2M2eh0.js +201 -0
  149. package/dist/mint-yN2M2eh0.js.map +1 -0
  150. package/dist/multi-layer-verifier-BHY1gWAc.d.ts +138 -0
  151. package/dist/multi-layer-verifier-BHY1gWAc.d.ts.map +1 -0
  152. package/dist/multishot/index.d.ts +273 -481
  153. package/dist/multishot/index.d.ts.map +1 -0
  154. package/dist/multishot/index.js +595 -548
  155. package/dist/multishot/index.js.map +1 -1
  156. package/dist/off-policy-DvgzvtIx.js +220 -0
  157. package/dist/off-policy-DvgzvtIx.js.map +1 -0
  158. package/dist/off-policy-mskQw8Mb.d.ts +153 -0
  159. package/dist/off-policy-mskQw8Mb.d.ts.map +1 -0
  160. package/dist/openapi.json +1 -1
  161. package/dist/opencode-sqlite-BGrHeDu3.js +318 -0
  162. package/dist/opencode-sqlite-BGrHeDu3.js.map +1 -0
  163. package/dist/outcome-store-BYHIuO0e.d.ts +65 -0
  164. package/dist/outcome-store-BYHIuO0e.d.ts.map +1 -0
  165. package/dist/outcome-store-ChBKlTd_.js +75 -0
  166. package/dist/outcome-store-ChBKlTd_.js.map +1 -0
  167. package/dist/paired-arms-D9D0wXj2.js +260 -0
  168. package/dist/paired-arms-D9D0wXj2.js.map +1 -0
  169. package/dist/pipelines/index.d.ts +95 -532
  170. package/dist/pipelines/index.d.ts.map +1 -0
  171. package/dist/pipelines/index.js +497 -478
  172. package/dist/pipelines/index.js.map +1 -1
  173. package/dist/pre-registration-DakwTRXk.js +96 -0
  174. package/dist/pre-registration-DakwTRXk.js.map +1 -0
  175. package/dist/propose-review-control-LhIWGzHl.js +1458 -0
  176. package/dist/propose-review-control-LhIWGzHl.js.map +1 -0
  177. package/dist/query-CJ_DX8vl.d.ts +28 -0
  178. package/dist/query-CJ_DX8vl.d.ts.map +1 -0
  179. package/dist/query-Di7eEQ79.js +83 -0
  180. package/dist/query-Di7eEQ79.js.map +1 -0
  181. package/dist/raw-provider-sink-BQd7mzyT.js +201 -0
  182. package/dist/raw-provider-sink-BQd7mzyT.js.map +1 -0
  183. package/dist/raw-provider-sink-BU29Sh8h.d.ts +134 -0
  184. package/dist/raw-provider-sink-BU29Sh8h.d.ts.map +1 -0
  185. package/dist/redact-7Aq1ukl-.js +107 -0
  186. package/dist/redact-7Aq1ukl-.js.map +1 -0
  187. package/dist/release-report-Cz9NKH39.js +603 -0
  188. package/dist/release-report-Cz9NKH39.js.map +1 -0
  189. package/dist/release-report-mvB2G4_J.d.ts +244 -0
  190. package/dist/release-report-mvB2G4_J.d.ts.map +1 -0
  191. package/dist/replay-GnyotH0J.js +1741 -0
  192. package/dist/replay-GnyotH0J.js.map +1 -0
  193. package/dist/replay-OoidtG1E.d.ts +749 -0
  194. package/dist/replay-OoidtG1E.d.ts.map +1 -0
  195. package/dist/reporting.d.ts +6 -1298
  196. package/dist/reporting.js +6 -50
  197. package/dist/researcher-CwTdwXG1.d.ts +314 -0
  198. package/dist/researcher-CwTdwXG1.d.ts.map +1 -0
  199. package/dist/reward-hacking-eAnOsynk.d.ts +224 -0
  200. package/dist/reward-hacking-eAnOsynk.d.ts.map +1 -0
  201. package/dist/reward-hacking-qipEpKvY.js +596 -0
  202. package/dist/reward-hacking-qipEpKvY.js.map +1 -0
  203. package/dist/reward-nw2xZGZG.js +137 -0
  204. package/dist/reward-nw2xZGZG.js.map +1 -0
  205. package/dist/rl.d.ts +916 -3596
  206. package/dist/rl.d.ts.map +1 -0
  207. package/dist/rl.js +2362 -1751
  208. package/dist/rl.js.map +1 -1
  209. package/dist/rolldown-runtime-8H4AJuhK.js +14 -0
  210. package/dist/rollout/index.d.ts +3 -1048
  211. package/dist/rollout/index.js +8 -110
  212. package/dist/rollout-BOYjemfR.js +624 -0
  213. package/dist/rollout-BOYjemfR.js.map +1 -0
  214. package/dist/rubric-predictive-validity-B3xmbmS1.js +141 -0
  215. package/dist/rubric-predictive-validity-B3xmbmS1.js.map +1 -0
  216. package/dist/rubric-predictive-validity-Ku_clp_1.d.ts +74 -0
  217. package/dist/rubric-predictive-validity-Ku_clp_1.d.ts.map +1 -0
  218. package/dist/run-evidence-ClRX_8A9.d.ts +225 -0
  219. package/dist/run-evidence-ClRX_8A9.d.ts.map +1 -0
  220. package/dist/run-record-BuoE80Dq.js +467 -0
  221. package/dist/run-record-BuoE80Dq.js.map +1 -0
  222. package/dist/run-record-CnZu_gjl.d.ts +357 -0
  223. package/dist/run-record-CnZu_gjl.d.ts.map +1 -0
  224. package/dist/run-score-iEEAWiBY.js +41 -0
  225. package/dist/run-score-iEEAWiBY.js.map +1 -0
  226. package/dist/runtime-trajectory-1gyaTOoC.js +93 -0
  227. package/dist/runtime-trajectory-1gyaTOoC.js.map +1 -0
  228. package/dist/runtime-trajectory-BvSZcCHD.d.ts +50 -0
  229. package/dist/runtime-trajectory-BvSZcCHD.d.ts.map +1 -0
  230. package/dist/schema-BtVldJ3T.d.ts +206 -0
  231. package/dist/schema-BtVldJ3T.d.ts.map +1 -0
  232. package/dist/schema-C6DW4ZHR.js +821 -0
  233. package/dist/schema-C6DW4ZHR.js.map +1 -0
  234. package/dist/schema-CRhEY1SO.js +69 -0
  235. package/dist/schema-CRhEY1SO.js.map +1 -0
  236. package/dist/schema-Cef2cFmb.d.ts +408 -0
  237. package/dist/schema-Cef2cFmb.d.ts.map +1 -0
  238. package/dist/semantic-concept-judge-B6cWNJ2K.js +725 -0
  239. package/dist/semantic-concept-judge-B6cWNJ2K.js.map +1 -0
  240. package/dist/sequential-Br0mAPHA.js +148 -0
  241. package/dist/sequential-Br0mAPHA.js.map +1 -0
  242. package/dist/sequential-CYwq6Ff_.d.ts +141 -0
  243. package/dist/sequential-CYwq6Ff_.d.ts.map +1 -0
  244. package/dist/series-convergence-CjO2QdRW.js +43 -0
  245. package/dist/series-convergence-CjO2QdRW.js.map +1 -0
  246. package/dist/series-convergence-ofsqPWhs.d.ts +35 -0
  247. package/dist/series-convergence-ofsqPWhs.d.ts.map +1 -0
  248. package/dist/server-m5D9cvnG.js +1040 -0
  249. package/dist/server-m5D9cvnG.js.map +1 -0
  250. package/dist/skill-usage-C_pXm7lP.d.ts +534 -0
  251. package/dist/skill-usage-C_pXm7lP.d.ts.map +1 -0
  252. package/dist/skillopt-optimization-method-B7wX7XkF.d.ts +1739 -0
  253. package/dist/skillopt-optimization-method-B7wX7XkF.d.ts.map +1 -0
  254. package/dist/skillopt-optimization-method-D4ODwFVV.js +7712 -0
  255. package/dist/skillopt-optimization-method-D4ODwFVV.js.map +1 -0
  256. package/dist/statistics-Cmj6nynr.d.ts +514 -0
  257. package/dist/statistics-Cmj6nynr.d.ts.map +1 -0
  258. package/dist/statistics-CnnxdpOg.js +1437 -0
  259. package/dist/statistics-CnnxdpOg.js.map +1 -0
  260. package/dist/store-CT9YIIve.d.ts +117 -0
  261. package/dist/store-CT9YIIve.d.ts.map +1 -0
  262. package/dist/store-CxJry_cs.d.ts +229 -0
  263. package/dist/store-CxJry_cs.d.ts.map +1 -0
  264. package/dist/storyboard/index.d.ts +84 -203
  265. package/dist/storyboard/index.d.ts.map +1 -0
  266. package/dist/storyboard/index.js +609 -542
  267. package/dist/storyboard/index.js.map +1 -1
  268. package/dist/summary-report-BNs5nmXI.js +862 -0
  269. package/dist/summary-report-BNs5nmXI.js.map +1 -0
  270. package/dist/summary-report-Cj9gdw4i.d.ts +407 -0
  271. package/dist/summary-report-Cj9gdw4i.d.ts.map +1 -0
  272. package/dist/supervisor-run/index.d.ts +2 -849
  273. package/dist/supervisor-run/index.js +2 -64
  274. package/dist/supervisor-run-_lnTLM3z.js +1679 -0
  275. package/dist/supervisor-run-_lnTLM3z.js.map +1 -0
  276. package/dist/task-failure-attributes-CQZlB3et.js +311 -0
  277. package/dist/task-failure-attributes-CQZlB3et.js.map +1 -0
  278. package/dist/test-graded-scenario-BsqWLmPt.js +318 -0
  279. package/dist/test-graded-scenario-BsqWLmPt.js.map +1 -0
  280. package/dist/test-graded-scenario-D1TaI2va.d.ts +141 -0
  281. package/dist/test-graded-scenario-D1TaI2va.d.ts.map +1 -0
  282. package/dist/tools-BmuN627J.js +1085 -0
  283. package/dist/tools-BmuN627J.js.map +1 -0
  284. package/dist/trace-attributes.d.ts +2 -52
  285. package/dist/trace-attributes.js +131 -61
  286. package/dist/trace-attributes.js.map +1 -1
  287. package/dist/traces.d.ts +12 -2365
  288. package/dist/traces.js +12 -251
  289. package/dist/trajectory-D_7rLrvE.js +56 -0
  290. package/dist/trajectory-D_7rLrvE.js.map +1 -0
  291. package/dist/types-DGsxbAEd.d.ts +387 -0
  292. package/dist/types-DGsxbAEd.d.ts.map +1 -0
  293. package/dist/types-k9tZGKUg.d.ts +640 -0
  294. package/dist/types-k9tZGKUg.d.ts.map +1 -0
  295. package/dist/verdict-Dps8_okt.d.ts +37 -0
  296. package/dist/verdict-Dps8_okt.d.ts.map +1 -0
  297. package/dist/wire/index.d.ts +702 -1174
  298. package/dist/wire/index.d.ts.map +1 -0
  299. package/dist/wire/index.js +2 -81
  300. package/docs/feature-guide.md +1 -1
  301. package/docs/rollout.md +116 -2
  302. package/package.json +18 -10
  303. package/dist/benchmarks/index.js.map +0 -1
  304. package/dist/campaign/index.js.map +0 -1
  305. package/dist/chunk-2JX3CFMB.js +0 -695
  306. package/dist/chunk-2JX3CFMB.js.map +0 -1
  307. package/dist/chunk-2MKQIFS4.js +0 -183
  308. package/dist/chunk-2MKQIFS4.js.map +0 -1
  309. package/dist/chunk-3RF76KTD.js +0 -84
  310. package/dist/chunk-3RF76KTD.js.map +0 -1
  311. package/dist/chunk-5DTSBUL2.js +0 -159
  312. package/dist/chunk-5DTSBUL2.js.map +0 -1
  313. package/dist/chunk-7ZZMD7UK.js +0 -386
  314. package/dist/chunk-7ZZMD7UK.js.map +0 -1
  315. package/dist/chunk-BOD4O7OF.js +0 -40
  316. package/dist/chunk-BOD4O7OF.js.map +0 -1
  317. package/dist/chunk-BYT7ELPS.js +0 -1553
  318. package/dist/chunk-BYT7ELPS.js.map +0 -1
  319. package/dist/chunk-DJKY2TSY.js +0 -2428
  320. package/dist/chunk-DJKY2TSY.js.map +0 -1
  321. package/dist/chunk-DPUHNQLN.js +0 -232
  322. package/dist/chunk-DPUHNQLN.js.map +0 -1
  323. package/dist/chunk-DRYIUNWY.js +0 -622
  324. package/dist/chunk-DRYIUNWY.js.map +0 -1
  325. package/dist/chunk-EJGRPCO3.js +0 -617
  326. package/dist/chunk-EJGRPCO3.js.map +0 -1
  327. package/dist/chunk-EOSZT7PL.js +0 -2001
  328. package/dist/chunk-EOSZT7PL.js.map +0 -1
  329. package/dist/chunk-EZJEIH2R.js +0 -1559
  330. package/dist/chunk-EZJEIH2R.js.map +0 -1
  331. package/dist/chunk-GGE4NNQT.js +0 -65
  332. package/dist/chunk-GGE4NNQT.js.map +0 -1
  333. package/dist/chunk-HHWE3POT.js +0 -94
  334. package/dist/chunk-HHWE3POT.js.map +0 -1
  335. package/dist/chunk-IHQDPH7D.js +0 -171
  336. package/dist/chunk-IHQDPH7D.js.map +0 -1
  337. package/dist/chunk-JHCHEVET.js +0 -274
  338. package/dist/chunk-JHCHEVET.js.map +0 -1
  339. package/dist/chunk-K4DBDHLK.js +0 -158
  340. package/dist/chunk-K4DBDHLK.js.map +0 -1
  341. package/dist/chunk-K6N6XJJX.js +0 -306
  342. package/dist/chunk-K6N6XJJX.js.map +0 -1
  343. package/dist/chunk-MA6HLL3S.js +0 -65
  344. package/dist/chunk-MA6HLL3S.js.map +0 -1
  345. package/dist/chunk-MAZ26DC7.js +0 -99
  346. package/dist/chunk-MAZ26DC7.js.map +0 -1
  347. package/dist/chunk-MHELPNRP.js +0 -1212
  348. package/dist/chunk-MHELPNRP.js.map +0 -1
  349. package/dist/chunk-NACAGYSY.js +0 -1040
  350. package/dist/chunk-NACAGYSY.js.map +0 -1
  351. package/dist/chunk-NKAGIDE2.js +0 -7633
  352. package/dist/chunk-NKAGIDE2.js.map +0 -1
  353. package/dist/chunk-NPCTHQIO.js +0 -91
  354. package/dist/chunk-NPCTHQIO.js.map +0 -1
  355. package/dist/chunk-NYLOYM6N.js +0 -332
  356. package/dist/chunk-NYLOYM6N.js.map +0 -1
  357. package/dist/chunk-ONWEPEDO.js +0 -57
  358. package/dist/chunk-ONWEPEDO.js.map +0 -1
  359. package/dist/chunk-P5W7RQKK.js +0 -576
  360. package/dist/chunk-P5W7RQKK.js.map +0 -1
  361. package/dist/chunk-P6FYH6K4.js +0 -1161
  362. package/dist/chunk-P6FYH6K4.js.map +0 -1
  363. package/dist/chunk-PBE2LOSS.js +0 -669
  364. package/dist/chunk-PBE2LOSS.js.map +0 -1
  365. package/dist/chunk-PC4UYEBM.js +0 -166
  366. package/dist/chunk-PC4UYEBM.js.map +0 -1
  367. package/dist/chunk-PXE2VKMX.js +0 -140
  368. package/dist/chunk-PXE2VKMX.js.map +0 -1
  369. package/dist/chunk-PZ5AY32C.js +0 -10
  370. package/dist/chunk-PZ5AY32C.js.map +0 -1
  371. package/dist/chunk-RZTMDUO7.js +0 -49
  372. package/dist/chunk-RZTMDUO7.js.map +0 -1
  373. package/dist/chunk-S5YLIBFX.js +0 -136
  374. package/dist/chunk-S5YLIBFX.js.map +0 -1
  375. package/dist/chunk-SZLVEKMJ.js +0 -1446
  376. package/dist/chunk-SZLVEKMJ.js.map +0 -1
  377. package/dist/chunk-T4SQEITX.js +0 -95
  378. package/dist/chunk-T4SQEITX.js.map +0 -1
  379. package/dist/chunk-TBL77AUT.js +0 -355
  380. package/dist/chunk-TBL77AUT.js.map +0 -1
  381. package/dist/chunk-TSN7JT6D.js +0 -1646
  382. package/dist/chunk-TSN7JT6D.js.map +0 -1
  383. package/dist/chunk-TT4KNT67.js +0 -124
  384. package/dist/chunk-TT4KNT67.js.map +0 -1
  385. package/dist/chunk-UB2LOJ6Q.js +0 -4461
  386. package/dist/chunk-UB2LOJ6Q.js.map +0 -1
  387. package/dist/chunk-UWZZKKU7.js +0 -237
  388. package/dist/chunk-UWZZKKU7.js.map +0 -1
  389. package/dist/chunk-VBQ3CRKH.js +0 -291
  390. package/dist/chunk-VBQ3CRKH.js.map +0 -1
  391. package/dist/chunk-VGRCHJON.js +0 -163
  392. package/dist/chunk-VGRCHJON.js.map +0 -1
  393. package/dist/chunk-VI2UW6B6.js +0 -162
  394. package/dist/chunk-VI2UW6B6.js.map +0 -1
  395. package/dist/chunk-VLOATJQ2.js +0 -908
  396. package/dist/chunk-VLOATJQ2.js.map +0 -1
  397. package/dist/chunk-VQMK5FMP.js +0 -247
  398. package/dist/chunk-VQMK5FMP.js.map +0 -1
  399. package/dist/chunk-VZSRQ272.js +0 -149
  400. package/dist/chunk-VZSRQ272.js.map +0 -1
  401. package/dist/chunk-WGXIEX7P.js +0 -116
  402. package/dist/chunk-WGXIEX7P.js.map +0 -1
  403. package/dist/chunk-WS3NZZQQ.js +0 -929
  404. package/dist/chunk-WS3NZZQQ.js.map +0 -1
  405. package/dist/chunk-XDWDC2MP.js +0 -695
  406. package/dist/chunk-XDWDC2MP.js.map +0 -1
  407. package/dist/chunk-XPRT64IE.js +0 -766
  408. package/dist/chunk-XPRT64IE.js.map +0 -1
  409. package/dist/chunk-YJBNWCAA.js +0 -1056
  410. package/dist/chunk-YJBNWCAA.js.map +0 -1
  411. package/dist/chunk-ZET2UAYW.js +0 -89
  412. package/dist/chunk-ZET2UAYW.js.map +0 -1
  413. package/dist/chunk-ZUUWPZCV.js +0 -752
  414. package/dist/chunk-ZUUWPZCV.js.map +0 -1
  415. package/dist/control.js.map +0 -1
  416. package/dist/hosted/index.js.map +0 -1
  417. package/dist/matrix/index.js.map +0 -1
  418. package/dist/reporting.js.map +0 -1
  419. package/dist/rollout/index.js.map +0 -1
  420. package/dist/run-campaign-ISHFZ7FJ.js +0 -17
  421. package/dist/run-campaign-ISHFZ7FJ.js.map +0 -1
  422. package/dist/supervisor-run/index.js.map +0 -1
  423. package/dist/traces.js.map +0 -1
  424. package/dist/wire/index.js.map +0 -1
@@ -1,4461 +0,0 @@
1
- import {
2
- JudgeParseError,
3
- assertCodeSurfaceIdentity,
4
- costReceiptFromTCloud,
5
- labelTrustRank,
6
- maximumChargeForTCloudRequest,
7
- pairHoldout,
8
- recoverTruncatedJson,
9
- surfaceContentHash
10
- } from "./chunk-NKAGIDE2.js";
11
- import {
12
- SearchLedgerConflictError,
13
- SearchLedgerError,
14
- SearchLedgerIntegrityError,
15
- appendSearchLedgerLine,
16
- assertRealBackend,
17
- contentHash,
18
- planCampaignRun,
19
- runCampaign,
20
- summarizeBackendIntegrity,
21
- withSearchLedgerFileLock
22
- } from "./chunk-EZJEIH2R.js";
23
- import {
24
- Mutex
25
- } from "./chunk-WGXIEX7P.js";
26
- import {
27
- campaignCellToRunRecord
28
- } from "./chunk-2MKQIFS4.js";
29
- import {
30
- comparePairedArms
31
- } from "./chunk-DPUHNQLN.js";
32
- import {
33
- eProcess,
34
- mulberry32,
35
- pairedBootstrap
36
- } from "./chunk-MHELPNRP.js";
37
- import {
38
- CostLedger
39
- } from "./chunk-WS3NZZQQ.js";
40
- import {
41
- analyzeTraces
42
- } from "./chunk-5DTSBUL2.js";
43
- import {
44
- buildAgentProfileCell,
45
- canonicalize,
46
- modelHasSnapshot,
47
- runTaskScore,
48
- validateRunRecord
49
- } from "./chunk-2JX3CFMB.js";
50
- import {
51
- AgentEvalError,
52
- ValidationError
53
- } from "./chunk-ONWEPEDO.js";
54
-
55
- // src/campaign/analyst-surface.ts
56
- function surfaceToText(surface) {
57
- if (typeof surface === "string") return surface;
58
- throw new Error(
59
- `buildAnalystSurfaceDispatch: the analyst surface must be a string actorDescription, got a ${surface.kind}-tier surface. The analyst prompt is prompt-tier.`
60
- );
61
- }
62
- function buildAnalystSurfaceDispatch(opts) {
63
- const analyze = opts.analyze ?? analyzeTraces;
64
- return async (surface, scenario, _ctx) => {
65
- const actorDescription = surfaceToText(surface);
66
- const res = await analyze(
67
- { question: scenario.question },
68
- { ...opts.analystOptions, actorDescription, source: scenario.source }
69
- );
70
- return {
71
- answer: res.answer,
72
- findings: res.findings,
73
- actorPromptVersion: res.actorPromptVersion
74
- };
75
- };
76
- }
77
- function failureModeRecallJudge(opts = {}) {
78
- const recallWeight = opts.recallWeight ?? 0.5;
79
- return {
80
- name: "failure-mode-recall",
81
- dimensions: [
82
- { key: "recall", description: "fraction of ground-truth failure modes the analyst surfaced" },
83
- {
84
- key: "precision",
85
- description: "1 \u2212 share of findings that named a failure/tool/error absent from this corpus"
86
- }
87
- ],
88
- appliesTo: (s) => s.kind === "analyst-surface",
89
- score({ artifact, scenario }) {
90
- const modes = scenario.expectedFailureModes;
91
- if (modes.length === 0) {
92
- throw new Error(
93
- `failureModeRecallJudge: scenario '${scenario.id}' has no expectedFailureModes \u2014 refusing to score (a vacuous 1.0 would corrupt the comparison)`
94
- );
95
- }
96
- const hay = artifact.findings.join("\n").toLowerCase();
97
- const matched = modes.filter((m) => m.cues.some((c) => hay.includes(c.toLowerCase())));
98
- const recall = matched.length / modes.length;
99
- const forbidden = (scenario.forbiddenCues ?? []).map((c) => c.toLowerCase());
100
- let precision = 1;
101
- let hallucinated = 0;
102
- if (forbidden.length > 0) {
103
- const denom = Math.max(1, artifact.findings.length);
104
- hallucinated = artifact.findings.filter(
105
- (f) => forbidden.some((c) => f.toLowerCase().includes(c))
106
- ).length;
107
- precision = 1 - hallucinated / denom;
108
- }
109
- const composite = forbidden.length > 0 ? recallWeight * recall + (1 - recallWeight) * precision : recall;
110
- const missed = modes.filter((m) => !matched.includes(m)).map((m) => m.id);
111
- const notes = `matched ${matched.length}/${modes.length} failure modes` + (missed.length ? `; missed [${missed.join(", ")}]` : "") + (hallucinated ? `; ${hallucinated} out-of-corpus finding(s)` : "");
112
- return { dimensions: { recall, precision }, composite, notes };
113
- }
114
- };
115
- }
116
-
117
- // src/campaign/cross-surface-context.ts
118
- function validateCrossSurfaceInput(input) {
119
- assertNonEmptyUnique(input.taskOrder, "taskOrder");
120
- assertNonEmptyUnique(input.componentOrder, "componentOrder");
121
- if (input.componentOrder.length < 2) {
122
- throw new ValidationError(
123
- "analyzeCrossSurfaceInteractions: componentOrder must contain at least two surfaces"
124
- );
125
- }
126
- assertNonEmptyUnique(input.candidateOrder, "candidateOrder");
127
- assertNonEmptyUnique(input.costMetricOrder, "costMetricOrder");
128
- if (input.costMetricOrder.includes("score")) {
129
- throw new ValidationError(
130
- `analyzeCrossSurfaceInteractions: costMetricOrder cannot contain reserved metric 'score'`
131
- );
132
- }
133
- validateBootstrap(input);
134
- validateSelection(input);
135
- const componentById = indexUnique(
136
- input.components,
137
- (component) => component.componentId,
138
- "component"
139
- );
140
- assertExactSet(input.componentOrder, componentById.keys(), "componentOrder", "components");
141
- const surfaces = /* @__PURE__ */ new Set();
142
- for (const component of componentById.values()) {
143
- assertNonEmpty(component.componentId, "componentId");
144
- assertNonEmpty(component.surfaceId, `surfaceId for component '${component.componentId}'`);
145
- if (surfaces.has(component.surfaceId)) {
146
- throw new ValidationError(
147
- `analyzeCrossSurfaceInteractions: surfaceId '${component.surfaceId}' has more than one component; the interaction stage requires one independently selected finalist per surface`
148
- );
149
- }
150
- surfaces.add(component.surfaceId);
151
- if (typeof component.bestSingleEligible !== "boolean") {
152
- throw new ValidationError(
153
- `analyzeCrossSurfaceInteractions: component '${component.componentId}' must declare bestSingleEligible`
154
- );
155
- }
156
- }
157
- const componentIndex = new Map(input.componentOrder.map((id, index) => [id, index]));
158
- const candidateById = indexUnique(
159
- input.candidates,
160
- (candidate) => candidate.candidateId,
161
- "candidate"
162
- );
163
- assertExactSet(input.candidateOrder, candidateById.keys(), "candidateOrder", "candidates");
164
- if (!candidateById.has(input.baselineCandidateId)) {
165
- throw new ValidationError(
166
- `analyzeCrossSurfaceInteractions: unknown baselineCandidateId '${input.baselineCandidateId}'`
167
- );
168
- }
169
- const candidateByComponents = /* @__PURE__ */ new Map();
170
- const singleByComponent = /* @__PURE__ */ new Map();
171
- for (const candidate of candidateById.values()) {
172
- validateCandidate(candidate, componentById, componentIndex, input.baselineCandidateId);
173
- const key = crossSurfaceComponentSetKey(candidate.componentIds);
174
- const duplicate = candidateByComponents.get(key);
175
- if (duplicate) {
176
- throw new ValidationError(
177
- `analyzeCrossSurfaceInteractions: candidates '${duplicate.candidateId}' and '${candidate.candidateId}' materialize the same component set`
178
- );
179
- }
180
- candidateByComponents.set(key, candidate);
181
- if (candidate.componentIds.length === 1) {
182
- singleByComponent.set(candidate.componentIds[0], candidate);
183
- }
184
- }
185
- const baseline = candidateById.get(input.baselineCandidateId);
186
- if (baseline.componentIds.length !== 0) {
187
- throw new ValidationError(
188
- `analyzeCrossSurfaceInteractions: baseline '${baseline.candidateId}' must have zero components`
189
- );
190
- }
191
- for (const componentId of input.componentOrder) {
192
- if (!singleByComponent.has(componentId)) {
193
- throw new ValidationError(
194
- `analyzeCrossSurfaceInteractions: no single-surface candidate for component '${componentId}'`
195
- );
196
- }
197
- }
198
- for (let left = 0; left < input.componentOrder.length; left++) {
199
- for (let right = left + 1; right < input.componentOrder.length; right++) {
200
- const ids = [input.componentOrder[left], input.componentOrder[right]];
201
- if (!candidateByComponents.has(crossSurfaceComponentSetKey(ids))) {
202
- throw new ValidationError(
203
- `analyzeCrossSurfaceInteractions: missing pair candidate for components [${ids.join(", ")}]`
204
- );
205
- }
206
- }
207
- }
208
- const rowsByCandidate = /* @__PURE__ */ new Map();
209
- const taskIds = new Set(input.taskOrder);
210
- for (const row of input.rows) {
211
- const candidate = candidateById.get(row.candidateId);
212
- if (!candidate) {
213
- throw new ValidationError(
214
- `analyzeCrossSurfaceInteractions: row for task '${row.taskId}' names unknown candidate '${row.candidateId}'`
215
- );
216
- }
217
- if (!taskIds.has(row.taskId)) {
218
- throw new ValidationError(
219
- `analyzeCrossSurfaceInteractions: row for candidate '${row.candidateId}' names task '${row.taskId}' outside the declared taskOrder`
220
- );
221
- }
222
- validateRow(input, row, candidate);
223
- const byTask = rowsByCandidate.get(row.candidateId) ?? /* @__PURE__ */ new Map();
224
- if (byTask.has(row.taskId)) {
225
- throw new ValidationError(
226
- `analyzeCrossSurfaceInteractions: duplicate row for candidate '${row.candidateId}' and task '${row.taskId}'`
227
- );
228
- }
229
- byTask.set(row.taskId, row);
230
- rowsByCandidate.set(row.candidateId, byTask);
231
- }
232
- for (const candidateId of input.candidateOrder) {
233
- const byTask = rowsByCandidate.get(candidateId);
234
- const missing = input.taskOrder.filter((taskId) => !byTask?.has(taskId));
235
- if (missing.length > 0) {
236
- throw new ValidationError(
237
- `analyzeCrossSurfaceInteractions: candidate '${candidateId}' is missing declared task row(s) [${missing.join(", ")}]; encode failed attempts explicitly instead of changing the task axis`
238
- );
239
- }
240
- }
241
- const expectedRows = input.candidateOrder.length * input.taskOrder.length;
242
- if (input.rows.length !== expectedRows) {
243
- throw new ValidationError(
244
- `analyzeCrossSurfaceInteractions: expected exactly ${expectedRows} candidate \xD7 task rows, got ${input.rows.length}`
245
- );
246
- }
247
- return {
248
- input,
249
- components: input.componentOrder.map((id) => componentById.get(id)),
250
- candidates: input.candidateOrder.map((id) => candidateById.get(id)),
251
- componentById,
252
- candidateById,
253
- candidateByComponents,
254
- singleByComponent,
255
- rowsByCandidate,
256
- componentIndex,
257
- candidateIndex: new Map(input.candidateOrder.map((id, index) => [id, index]))
258
- };
259
- }
260
- function crossSurfaceRowsFor(context, candidateId) {
261
- return context.input.taskOrder.map((taskId) => crossSurfaceRowFor(context, candidateId, taskId));
262
- }
263
- function crossSurfaceRowFor(context, candidateId, taskId) {
264
- return context.rowsByCandidate.get(candidateId).get(taskId);
265
- }
266
- function canonicalCrossSurfaceComponents(context, componentIds) {
267
- return [...componentIds].sort(
268
- (left, right) => context.componentIndex.get(left) - context.componentIndex.get(right)
269
- );
270
- }
271
- function crossSurfaceComponentSetKey(componentIds) {
272
- return JSON.stringify(componentIds);
273
- }
274
- function validateBootstrap(input) {
275
- const { seed, resamples, confidence } = input.bootstrap;
276
- if (!Number.isInteger(seed)) {
277
- throw new ValidationError(`analyzeCrossSurfaceInteractions: bootstrap.seed must be an integer`);
278
- }
279
- if (!Number.isInteger(resamples) || resamples <= 0) {
280
- throw new ValidationError(
281
- `analyzeCrossSurfaceInteractions: bootstrap.resamples must be a positive integer`
282
- );
283
- }
284
- if (!Number.isFinite(confidence) || confidence <= 0 || confidence >= 1) {
285
- throw new ValidationError(
286
- `analyzeCrossSurfaceInteractions: bootstrap.confidence must be in (0,1)`
287
- );
288
- }
289
- }
290
- function validateSelection(input) {
291
- const policy = input.selection;
292
- for (const [name, value] of [
293
- ["minimumFiringTasks", policy.minimumFiringTasks],
294
- ["minimumEffectTasks", policy.minimumEffectTasks]
295
- ]) {
296
- if (!Number.isInteger(value) || value < 0 || value > input.taskOrder.length) {
297
- throw new ValidationError(
298
- `analyzeCrossSurfaceInteractions: selection.${name} must be an integer in [0,${input.taskOrder.length}]`
299
- );
300
- }
301
- }
302
- if (!Number.isInteger(policy.minimumBundleComponents) || policy.minimumBundleComponents < 2 || policy.minimumBundleComponents > input.componentOrder.length) {
303
- throw new ValidationError(
304
- `analyzeCrossSurfaceInteractions: selection.minimumBundleComponents must be an integer in [2,${input.componentOrder.length}]`
305
- );
306
- }
307
- for (const [metric, limit] of Object.entries(policy.maximumMedianCostRatioToBaseline)) {
308
- if (!input.costMetricOrder.includes(metric)) {
309
- throw new ValidationError(
310
- `analyzeCrossSurfaceInteractions: cost limit names undeclared metric '${metric}'`
311
- );
312
- }
313
- if (!Number.isFinite(limit) || limit <= 0) {
314
- throw new ValidationError(
315
- `analyzeCrossSurfaceInteractions: cost ratio limit for '${metric}' must be positive and finite`
316
- );
317
- }
318
- }
319
- }
320
- function validateCandidate(candidate, componentById, componentIndex, baselineCandidateId) {
321
- assertNonEmpty(candidate.candidateId, "candidateId");
322
- assertNonEmpty(candidate.contentHash, `contentHash for candidate '${candidate.candidateId}'`);
323
- if (!Number.isInteger(candidate.artifactBytes) || candidate.artifactBytes < 0) {
324
- throw new ValidationError(
325
- `analyzeCrossSurfaceInteractions: artifactBytes for candidate '${candidate.candidateId}' must be a non-negative integer`
326
- );
327
- }
328
- assertUnique(candidate.componentIds, `componentIds for candidate '${candidate.candidateId}'`);
329
- for (const componentId of candidate.componentIds) {
330
- if (!componentById.has(componentId)) {
331
- throw new ValidationError(
332
- `analyzeCrossSurfaceInteractions: candidate '${candidate.candidateId}' names unknown component '${componentId}'`
333
- );
334
- }
335
- }
336
- const canonical = [...candidate.componentIds].sort(
337
- (left, right) => componentIndex.get(left) - componentIndex.get(right)
338
- );
339
- if (!arraysEqual(candidate.componentIds, canonical)) {
340
- throw new ValidationError(
341
- `analyzeCrossSurfaceInteractions: candidate '${candidate.candidateId}' components must follow componentOrder`
342
- );
343
- }
344
- if (candidate.candidateId !== baselineCandidateId && candidate.componentIds.length === 0) {
345
- throw new ValidationError(
346
- `analyzeCrossSurfaceInteractions: only baseline '${baselineCandidateId}' may have zero components`
347
- );
348
- }
349
- }
350
- function validateRow(input, row, candidate) {
351
- if (!arraysEqual(row.componentIds, candidate.componentIds)) {
352
- throw new ValidationError(
353
- `analyzeCrossSurfaceInteractions: row '${row.candidateId}/${row.taskId}' componentIds do not match its candidate`
354
- );
355
- }
356
- if (!["complete", "missing", "invalid"].includes(row.completeness)) {
357
- throw new ValidationError(
358
- `analyzeCrossSurfaceInteractions: row '${row.candidateId}/${row.taskId}' has unknown completeness '${String(row.completeness)}'`
359
- );
360
- }
361
- if (row.completeness === "complete") {
362
- if (typeof row.pass !== "boolean" || !Number.isFinite(row.score)) {
363
- throw new ValidationError(
364
- `analyzeCrossSurfaceInteractions: complete row '${row.candidateId}/${row.taskId}' requires boolean pass and finite score`
365
- );
366
- }
367
- if (row.rejectReason !== null) {
368
- throw new ValidationError(
369
- `analyzeCrossSurfaceInteractions: complete row '${row.candidateId}/${row.taskId}' cannot carry rejectReason`
370
- );
371
- }
372
- } else {
373
- if (row.pass !== null || row.score !== null) {
374
- throw new ValidationError(
375
- `analyzeCrossSurfaceInteractions: ${row.completeness} row '${row.candidateId}/${row.taskId}' must use null pass and score`
376
- );
377
- }
378
- if (typeof row.rejectReason !== "string" || row.rejectReason.trim() === "") {
379
- throw new ValidationError(
380
- `analyzeCrossSurfaceInteractions: ${row.completeness} row '${row.candidateId}/${row.taskId}' requires rejectReason`
381
- );
382
- }
383
- }
384
- assertExactSet(
385
- input.costMetricOrder,
386
- Object.keys(row.cost),
387
- `cost keys for row '${row.candidateId}/${row.taskId}'`,
388
- "costMetricOrder"
389
- );
390
- for (const metric of input.costMetricOrder) {
391
- const value = row.cost[metric];
392
- if (value === null || value === void 0 || !Number.isFinite(value) || value < 0) {
393
- throw new ValidationError(
394
- `analyzeCrossSurfaceInteractions: unknown or invalid cost '${metric}' on row '${row.candidateId}/${row.taskId}'; every attempt must report a non-negative finite value`
395
- );
396
- }
397
- }
398
- const evidenceByComponent = indexUnique(
399
- row.componentEvidence,
400
- (evidence) => evidence.componentId,
401
- `componentEvidence on row '${row.candidateId}/${row.taskId}'`
402
- );
403
- assertExactSet(
404
- candidate.componentIds,
405
- evidenceByComponent.keys(),
406
- `componentEvidence on row '${row.candidateId}/${row.taskId}'`,
407
- "candidate components"
408
- );
409
- for (const evidence of evidenceByComponent.values()) {
410
- assertTriState(evidence.fired, "fired", row);
411
- assertTriState(evidence.effectObserved, "effectObserved", row);
412
- if (evidence.effectObserved === true && evidence.fired !== true) {
413
- throw new ValidationError(
414
- `analyzeCrossSurfaceInteractions: effectObserved=true requires fired=true for component '${evidence.componentId}' on row '${row.candidateId}/${row.taskId}'`
415
- );
416
- }
417
- }
418
- }
419
- function assertTriState(value, field, row) {
420
- if (value !== true && value !== false && value !== null) {
421
- throw new ValidationError(
422
- `analyzeCrossSurfaceInteractions: ${field} on row '${row.candidateId}/${row.taskId}' must be boolean or null`
423
- );
424
- }
425
- }
426
- function indexUnique(items, id, label) {
427
- const result = /* @__PURE__ */ new Map();
428
- for (const item of items) {
429
- const key = id(item);
430
- assertNonEmpty(key, `${label} id`);
431
- if (result.has(key)) {
432
- throw new ValidationError(`analyzeCrossSurfaceInteractions: duplicate ${label} id '${key}'`);
433
- }
434
- result.set(key, item);
435
- }
436
- return result;
437
- }
438
- function assertNonEmptyUnique(values, label) {
439
- if (values.length === 0) {
440
- throw new ValidationError(`analyzeCrossSurfaceInteractions: ${label} is empty`);
441
- }
442
- assertUnique(values, label);
443
- for (const value of values) assertNonEmpty(value, label);
444
- }
445
- function assertUnique(values, label) {
446
- if (new Set(values).size !== values.length) {
447
- throw new ValidationError(`analyzeCrossSurfaceInteractions: ${label} contains duplicates`);
448
- }
449
- }
450
- function assertNonEmpty(value, label) {
451
- if (typeof value !== "string" || value.trim() === "") {
452
- throw new ValidationError(
453
- `analyzeCrossSurfaceInteractions: ${label} must be a non-empty string`
454
- );
455
- }
456
- }
457
- function assertExactSet(expected, actualIterable, actualLabel, expectedLabel) {
458
- const actual = [...actualIterable];
459
- const expectedSet = new Set(expected);
460
- const actualSet = new Set(actual);
461
- const missing = expected.filter((value) => !actualSet.has(value));
462
- const extra = actual.filter((value) => !expectedSet.has(value));
463
- if (missing.length > 0 || extra.length > 0 || actual.length !== expected.length) {
464
- throw new ValidationError(
465
- `analyzeCrossSurfaceInteractions: ${actualLabel} does not match ${expectedLabel}; missing=[${missing.join(", ")}], extra=[${extra.join(", ")}]`
466
- );
467
- }
468
- }
469
- function arraysEqual(left, right) {
470
- return left.length === right.length && left.every((value, index) => value === right[index]);
471
- }
472
-
473
- // src/campaign/cross-surface-interaction.ts
474
- function analyzeCrossSurfaceInteractions(input) {
475
- const context = validateCrossSurfaceInput(input);
476
- const baseline = context.candidateById.get(input.baselineCandidateId);
477
- const preliminary = context.candidates.map(
478
- (candidate) => summarizeCandidate(context, candidate, baseline)
479
- );
480
- const baselineSummary = preliminary.find(
481
- (summary) => summary.candidate.candidateId === input.baselineCandidateId
482
- );
483
- const summaries = preliminary.map(
484
- (summary) => summary.candidate.candidateId === input.baselineCandidateId ? summary : {
485
- ...summary,
486
- eligibility: candidateEligibility(context, summary, baselineSummary)
487
- }
488
- );
489
- const summaryById = new Map(summaries.map((summary) => [summary.candidate.candidateId, summary]));
490
- const eligibleSingles = rankedEligibleSingles(context, summaryById);
491
- const interactionReadySingles = rankedInteractionReadySingles(context, summaryById);
492
- const pairwise = buildPairwise(context, summaryById, interactionReadySingles, baselineSummary);
493
- const selections = selectCandidates(
494
- context,
495
- summaryById,
496
- eligibleSingles,
497
- interactionReadySingles,
498
- pairwise
499
- );
500
- const rows = context.candidates.flatMap(
501
- (candidate) => input.taskOrder.map(
502
- (taskId) => context.rowsByCandidate.get(candidate.candidateId).get(taskId)
503
- )
504
- );
505
- return {
506
- taskIds: [...input.taskOrder],
507
- componentIds: [...input.componentOrder],
508
- candidateIds: [...input.candidateOrder],
509
- costMetrics: [...input.costMetricOrder],
510
- rows,
511
- missingAttempts: rows.filter((row) => row.completeness === "missing"),
512
- invalidAttempts: rows.filter((row) => row.completeness === "invalid"),
513
- candidates: summaries,
514
- pairwise,
515
- selections
516
- };
517
- }
518
- function summarizeCandidate(context, candidate, baseline) {
519
- const rows = crossSurfaceRowsFor(context, candidate.candidateId);
520
- const baselineRows = crossSurfaceRowsFor(context, baseline.candidateId);
521
- const outcome = summarizeOutcome(
522
- context,
523
- rows,
524
- baselineRows,
525
- candidate.candidateId === baseline.candidateId
526
- );
527
- const completeScores = rows.filter((row) => row.completeness === "complete").map((row) => row.score);
528
- const costs = Object.fromEntries(
529
- context.input.costMetricOrder.map((metric) => [
530
- metric,
531
- distribution(rows.map((row) => row.cost[metric]))
532
- ])
533
- );
534
- return {
535
- candidate,
536
- outcome,
537
- score: completeScores.length === 0 ? null : distribution(completeScores),
538
- costs,
539
- firing: summarizeCandidateEvidence(rows, candidate.componentIds, "fired"),
540
- effect: summarizeCandidateEvidence(rows, candidate.componentIds, "effectObserved"),
541
- comparisonToBaseline: candidate.candidateId === baseline.candidateId ? null : pairedComparison(context, baseline.candidateId, candidate.candidateId),
542
- eligibility: null
543
- };
544
- }
545
- function summarizeOutcome(context, rows, baselineRows, isBaseline) {
546
- const resolvedTaskIds = [];
547
- const failedTaskIds = [];
548
- const missingTaskIds = [];
549
- const invalidTaskIds = [];
550
- const benefitTaskIds = [];
551
- const regressionTaskIds = [];
552
- const comparisonMissingTaskIds = [];
553
- for (let index = 0; index < context.input.taskOrder.length; index++) {
554
- const row = rows[index];
555
- const baseline = baselineRows[index];
556
- if (row.completeness === "missing") missingTaskIds.push(row.taskId);
557
- else if (row.completeness === "invalid") invalidTaskIds.push(row.taskId);
558
- else if (row.pass) resolvedTaskIds.push(row.taskId);
559
- else failedTaskIds.push(row.taskId);
560
- if (isBaseline) continue;
561
- if (row.completeness !== "complete" || baseline.completeness !== "complete") {
562
- comparisonMissingTaskIds.push(row.taskId);
563
- } else if (row.pass && !baseline.pass) benefitTaskIds.push(row.taskId);
564
- else if (!row.pass && baseline.pass) regressionTaskIds.push(row.taskId);
565
- }
566
- return {
567
- resolvedTaskIds,
568
- failedTaskIds,
569
- missingTaskIds,
570
- invalidTaskIds,
571
- benefitTaskIds,
572
- regressionTaskIds,
573
- comparisonMissingTaskIds,
574
- netBenefit: benefitTaskIds.length - regressionTaskIds.length
575
- };
576
- }
577
- function summarizeCandidateEvidence(rows, componentIds, field) {
578
- if (componentIds.length === 0) {
579
- return {
580
- byComponent: [],
581
- allObservedTaskIds: [],
582
- someObservedTaskIds: [],
583
- noneObservedTaskIds: [],
584
- unobservedTaskIds: []
585
- };
586
- }
587
- const byComponent = componentIds.map((componentId) => {
588
- const observedTaskIds = [];
589
- const notObservedTaskIds = [];
590
- const unobservedTaskIds2 = [];
591
- for (const row of rows) {
592
- const value = evidenceValue(row, componentId, field);
593
- if (value === true) observedTaskIds.push(row.taskId);
594
- else if (value === false) notObservedTaskIds.push(row.taskId);
595
- else unobservedTaskIds2.push(row.taskId);
596
- }
597
- return { componentId, observedTaskIds, notObservedTaskIds, unobservedTaskIds: unobservedTaskIds2 };
598
- });
599
- const allObservedTaskIds = [];
600
- const someObservedTaskIds = [];
601
- const noneObservedTaskIds = [];
602
- const unobservedTaskIds = [];
603
- for (const row of rows) {
604
- const values = componentIds.map((componentId) => evidenceValue(row, componentId, field));
605
- if (values.some((value) => value === null)) unobservedTaskIds.push(row.taskId);
606
- else if (values.every(Boolean)) allObservedTaskIds.push(row.taskId);
607
- else if (values.some(Boolean)) someObservedTaskIds.push(row.taskId);
608
- else noneObservedTaskIds.push(row.taskId);
609
- }
610
- return {
611
- byComponent,
612
- allObservedTaskIds,
613
- someObservedTaskIds,
614
- noneObservedTaskIds,
615
- unobservedTaskIds
616
- };
617
- }
618
- function candidateEligibility(context, summary, baseline) {
619
- const reasons = [];
620
- if (summary.outcome.missingTaskIds.length > 0) reasons.push("missing_attempt");
621
- if (summary.outcome.invalidTaskIds.length > 0) reasons.push("invalid_attempt");
622
- if (summary.outcome.comparisonMissingTaskIds.length > 0) reasons.push("baseline_outcome_missing");
623
- if (summary.outcome.benefitTaskIds.length <= summary.outcome.regressionTaskIds.length) {
624
- reasons.push("benefit_not_greater_than_regression");
625
- }
626
- appendEvidenceEligibilityReasons(
627
- reasons,
628
- summary.firing,
629
- context.input.selection.minimumFiringTasks,
630
- context.input.selection.requireObservedFiring,
631
- "firing_below_minimum",
632
- "firing_unobserved"
633
- );
634
- appendEvidenceEligibilityReasons(
635
- reasons,
636
- summary.effect,
637
- context.input.selection.minimumEffectTasks,
638
- context.input.selection.requireObservedEffect,
639
- "effect_below_minimum",
640
- "effect_unobserved"
641
- );
642
- if (!withinCostLimits(context, summary, baseline)) reasons.push("cost_limit_exceeded");
643
- return { eligible: reasons.length === 0, reasons: unique(reasons) };
644
- }
645
- function appendEvidenceEligibilityReasons(reasons, evidence, minimum, requireObserved, belowReason, unobservedReason) {
646
- if (evidence.byComponent.some((component) => component.observedTaskIds.length < minimum)) {
647
- reasons.push(belowReason);
648
- }
649
- if (requireObserved && evidence.byComponent.some((component) => component.unobservedTaskIds.length > 0)) {
650
- reasons.push(unobservedReason);
651
- }
652
- }
653
- function rankedEligibleSingles(context, summaryById) {
654
- return [...context.singleByComponent.values()].map((candidate) => summaryById.get(candidate.candidateId)).filter((summary) => summary.eligibility?.eligible).sort((left, right) => compareSingleSummaries(context, left, right));
655
- }
656
- function rankedInteractionReadySingles(context, summaryById) {
657
- return [...context.singleByComponent.values()].map((candidate) => summaryById.get(candidate.candidateId)).filter(
658
- (summary) => summary.outcome.regressionTaskIds.length === 0 && summary.eligibility?.reasons.every(
659
- (reason) => reason === "benefit_not_greater_than_regression"
660
- )
661
- ).sort((left, right) => compareSingleSummaries(context, left, right));
662
- }
663
- function compareSingleSummaries(context, left, right) {
664
- const byNetBenefit = right.outcome.netBenefit - left.outcome.netBenefit;
665
- if (byNetBenefit !== 0) return byNetBenefit;
666
- const byBenefit = right.outcome.benefitTaskIds.length - left.outcome.benefitTaskIds.length;
667
- if (byBenefit !== 0) return byBenefit;
668
- const byRegression = left.outcome.regressionTaskIds.length - right.outcome.regressionTaskIds.length;
669
- if (byRegression !== 0) return byRegression;
670
- for (const metric of context.input.costMetricOrder) {
671
- const byCost = left.costs[metric].median - right.costs[metric].median;
672
- if (byCost !== 0) return byCost;
673
- }
674
- const byBytes = left.candidate.artifactBytes - right.candidate.artifactBytes;
675
- if (byBytes !== 0) return byBytes;
676
- return context.candidateIndex.get(left.candidate.candidateId) - context.candidateIndex.get(right.candidate.candidateId);
677
- }
678
- function buildPairwise(context, summaryById, interactionReadySingles, baseline) {
679
- const interactionReadyIds = new Set(
680
- interactionReadySingles.map((summary) => summary.candidate.candidateId)
681
- );
682
- const entries = [];
683
- for (let leftIndex = 0; leftIndex < context.components.length; leftIndex++) {
684
- for (let rightIndex = leftIndex + 1; rightIndex < context.components.length; rightIndex++) {
685
- const leftComponent = context.components[leftIndex];
686
- const rightComponent = context.components[rightIndex];
687
- const leftSingle = context.singleByComponent.get(leftComponent.componentId);
688
- const rightSingle = context.singleByComponent.get(rightComponent.componentId);
689
- const pair = context.candidateByComponents.get(
690
- crossSurfaceComponentSetKey([leftComponent.componentId, rightComponent.componentId])
691
- );
692
- const pairSummary = summaryById.get(pair.candidateId);
693
- const leftSummary = summaryById.get(leftSingle.candidateId);
694
- const rightSummary = summaryById.get(rightSingle.candidateId);
695
- const synergyTaskIds = [];
696
- const interferenceTaskIds = [];
697
- for (const taskId of context.input.taskOrder) {
698
- const pairRow = crossSurfaceRowFor(context, pair.candidateId, taskId);
699
- const leftRow = crossSurfaceRowFor(context, leftSingle.candidateId, taskId);
700
- const rightRow = crossSurfaceRowFor(context, rightSingle.candidateId, taskId);
701
- if (![pairRow, leftRow, rightRow].every((row) => row.completeness === "complete")) continue;
702
- if (pairRow.pass && !leftRow.pass && !rightRow.pass) synergyTaskIds.push(taskId);
703
- if (!pairRow.pass && (leftRow.pass || rightRow.pass)) interferenceTaskIds.push(taskId);
704
- }
705
- const incrementalVsConstituents = [
706
- compareCandidates(context, summaryById, leftSingle.candidateId, pair.candidateId),
707
- compareCandidates(context, summaryById, rightSingle.candidateId, pair.candidateId)
708
- ];
709
- const betterSingle = compareSingleSummaries(context, leftSummary, rightSummary) <= 0 ? leftSummary : rightSummary;
710
- const firing = summarizePairEvidence(
711
- crossSurfaceRowsFor(context, pair.candidateId),
712
- leftComponent.componentId,
713
- rightComponent.componentId,
714
- "fired"
715
- );
716
- const effect = summarizePairEvidence(
717
- crossSurfaceRowsFor(context, pair.candidateId),
718
- leftComponent.componentId,
719
- rightComponent.componentId,
720
- "effectObserved"
721
- );
722
- const compatibility = pairCompatibility(
723
- context,
724
- pairSummary,
725
- baseline,
726
- betterSingle,
727
- incrementalVsConstituents,
728
- interactionReadyIds.has(leftSingle.candidateId) && interactionReadyIds.has(rightSingle.candidateId),
729
- interferenceTaskIds,
730
- firing,
731
- effect
732
- );
733
- entries.push({
734
- componentIds: [leftComponent.componentId, rightComponent.componentId],
735
- singleCandidateIds: [leftSingle.candidateId, rightSingle.candidateId],
736
- compositionCandidateId: pair.candidateId,
737
- benefitTaskIds: [...pairSummary.outcome.benefitTaskIds],
738
- regressionTaskIds: [...pairSummary.outcome.regressionTaskIds],
739
- synergyTaskIds,
740
- interferenceTaskIds,
741
- incrementalVsConstituents,
742
- relativeCostToBaseline: relativeCosts(context, pairSummary, baseline),
743
- firing,
744
- effect,
745
- interaction: interactionEffect(
746
- context,
747
- baseline.candidate,
748
- [leftSingle, rightSingle],
749
- pair
750
- ),
751
- compatibility
752
- });
753
- }
754
- }
755
- return entries;
756
- }
757
- function pairCompatibility(context, pair, baseline, betterSingle, comparisons, constituentsReady, interferenceTaskIds, firing, effect) {
758
- const reasons = [];
759
- if (!constituentsReady) reasons.push("constituent_not_ready");
760
- if (pair.outcome.missingTaskIds.length > 0 || pair.outcome.invalidTaskIds.length > 0 || pair.outcome.comparisonMissingTaskIds.length > 0) {
761
- reasons.push("pair_incomplete");
762
- }
763
- if (pair.outcome.regressionTaskIds.length > 0) reasons.push("baseline_regression");
764
- if (interferenceTaskIds.length > 0) reasons.push("interference");
765
- const betterComparison = comparisons.find(
766
- (comparison) => comparison.comparatorCandidateId === betterSingle.candidate.candidateId
767
- );
768
- if (betterComparison.winsTaskIds.length === 0) reasons.push("no_incremental_resolution");
769
- appendPairEvidenceReasons(
770
- reasons,
771
- firing,
772
- context.input.selection.minimumFiringTasks,
773
- context.input.selection.requireObservedFiring,
774
- "firing_below_minimum",
775
- "firing_unobserved"
776
- );
777
- appendPairEvidenceReasons(
778
- reasons,
779
- effect,
780
- context.input.selection.minimumEffectTasks,
781
- context.input.selection.requireObservedEffect,
782
- "effect_below_minimum",
783
- "effect_unobserved"
784
- );
785
- if (!withinCostLimits(context, pair, baseline)) reasons.push("cost_limit_exceeded");
786
- return {
787
- compatible: reasons.length === 0,
788
- reasons: unique(reasons),
789
- betterSingleCandidateId: betterSingle.candidate.candidateId
790
- };
791
- }
792
- function appendPairEvidenceReasons(reasons, evidence, minimum, requireObserved, belowReason, unobservedReason) {
793
- const leftObserved = evidence.bothTaskIds.length + evidence.leftOnlyTaskIds.length;
794
- const rightObserved = evidence.bothTaskIds.length + evidence.rightOnlyTaskIds.length;
795
- if (leftObserved < minimum || rightObserved < minimum) reasons.push(belowReason);
796
- if (requireObserved && evidence.unobservedTaskIds.length > 0) reasons.push(unobservedReason);
797
- }
798
- function interactionEffect(context, baseline, singles, composition) {
799
- const perTask = context.input.taskOrder.map((taskId) => {
800
- const baselineRow = crossSurfaceRowFor(context, baseline.candidateId, taskId);
801
- const singleRows = singles.map((single) => crossSurfaceRowFor(context, single.candidateId, taskId));
802
- const compositionRow = crossSurfaceRowFor(context, composition.candidateId, taskId);
803
- const rows = [baselineRow, ...singleRows, compositionRow];
804
- if (!rows.every((row) => row.completeness === "complete")) {
805
- return { taskId, passInteraction: null, scoreInteraction: null };
806
- }
807
- const additivePass = singleRows.reduce((sum, row) => sum + Number(row.pass), 0) - (singles.length - 1) * Number(baselineRow.pass);
808
- const additiveScore = singleRows.reduce((sum, row) => sum + row.score, 0) - (singles.length - 1) * baselineRow.score;
809
- return {
810
- taskId,
811
- passInteraction: Number(compositionRow.pass) - additivePass,
812
- scoreInteraction: compositionRow.score - additiveScore
813
- };
814
- });
815
- const complete = perTask.filter(
816
- (row) => row.passInteraction !== null && row.scoreInteraction !== null
817
- );
818
- const passInteractions = complete.map((row) => row.passInteraction);
819
- const scoreInteractions = complete.map((row) => row.scoreInteraction);
820
- return {
821
- perTask,
822
- n: complete.length,
823
- nMissing: perTask.length - complete.length,
824
- meanPassInteraction: meanOrNull(passInteractions),
825
- meanScoreInteraction: meanOrNull(scoreInteractions),
826
- passBootstrap: bootstrapInteraction(passInteractions, context),
827
- scoreBootstrap: bootstrapInteraction(scoreInteractions, context)
828
- };
829
- }
830
- function bootstrapInteraction(interactions, context) {
831
- if (interactions.length === 0) return null;
832
- return pairedBootstrap(new Array(interactions.length).fill(0), interactions, {
833
- ...context.input.bootstrap,
834
- statistic: "mean"
835
- });
836
- }
837
- function selectCandidates(context, summaryById, eligibleSingles, interactionReadySingles, pairwise) {
838
- const bestSingleRanking = eligibleSingles.filter((summary) => {
839
- const componentId = summary.candidate.componentIds[0];
840
- return context.componentById.get(componentId).bestSingleEligible;
841
- });
842
- const ranking = bestSingleRanking.map((summary, index) => ({
843
- rank: index + 1,
844
- candidateId: summary.candidate.candidateId,
845
- componentId: summary.candidate.componentIds[0]
846
- }));
847
- const best = bestSingleRanking[0];
848
- const bestSingle = best ? {
849
- candidateId: best.candidate.candidateId,
850
- componentId: best.candidate.componentIds[0],
851
- ranking
852
- } : null;
853
- const naiveStack = selectNaiveStack(context, eligibleSingles);
854
- const interactionAware = selectInteractionAware(
855
- context,
856
- summaryById,
857
- interactionReadySingles,
858
- pairwise
859
- );
860
- return { bestSingle, naiveStack, interactionAware };
861
- }
862
- function selectNaiveStack(context, eligibleSingles) {
863
- if (eligibleSingles.length === 0) return null;
864
- const componentIds = eligibleSingles.map((summary) => summary.candidate.componentIds[0]).sort((left, right) => context.componentIndex.get(left) - context.componentIndex.get(right));
865
- const candidate = context.candidateByComponents.get(crossSurfaceComponentSetKey(componentIds));
866
- if (!candidate) {
867
- throw new ValidationError(
868
- `analyzeCrossSurfaceInteractions: naive stack [${componentIds.join(", ")}] was not evaluated`
869
- );
870
- }
871
- return { candidateId: candidate.candidateId, componentIds };
872
- }
873
- function selectInteractionAware(context, summaryById, interactionReadySingles, pairwise) {
874
- const baseline = summaryById.get(context.input.baselineCandidateId);
875
- const pairByComponents = new Map(
876
- pairwise.map((entry) => [crossSurfaceComponentSetKey(entry.componentIds), entry])
877
- );
878
- const paths = pairwise.filter((entry) => entry.compatibility.compatible).map(
879
- (entry) => growInteractionPath(
880
- context,
881
- summaryById,
882
- interactionReadySingles,
883
- pairByComponents,
884
- baseline,
885
- summaryById.get(entry.compositionCandidateId)
886
- )
887
- );
888
- if (paths.length === 0) return null;
889
- const qualified = paths.filter((path) => path.qualified).sort((left, right) => compareInteractionPaths(context, summaryById, left, right));
890
- const winning = qualified[0] ?? [...paths].sort((left, right) => compareInteractionPaths(context, summaryById, left, right))[0];
891
- return {
892
- seedCandidateId: winning.seedCandidateId,
893
- terminalCandidateId: winning.terminalCandidateId,
894
- terminalComponentIds: [...winning.terminalComponentIds],
895
- selectedCandidateId: qualified.length > 0 ? winning.terminalCandidateId : null,
896
- qualified: qualified.length > 0,
897
- evaluatedPaths: paths,
898
- steps: winning.steps
899
- };
900
- }
901
- function growInteractionPath(context, summaryById, interactionReadySingles, pairByComponents, baseline, seed) {
902
- let current = seed;
903
- let retained = [...seed.candidate.componentIds];
904
- let remaining = interactionReadySingles.filter(
905
- (summary) => !retained.includes(summary.candidate.componentIds[0])
906
- );
907
- const steps = [];
908
- while (remaining.length > 0) {
909
- const decisions = remaining.map(
910
- (addition) => evaluateAddition(
911
- context,
912
- summaryById,
913
- pairByComponents,
914
- baseline,
915
- current,
916
- retained,
917
- addition
918
- )
919
- );
920
- const eligible = decisions.filter((decision) => decision.eligible).sort((left, right) => compareAdditionDecisions(context, summaryById, left, right));
921
- const selected = eligible[0];
922
- if (selected) selected.selected = true;
923
- steps.push({
924
- fromCandidateId: current.candidate.candidateId,
925
- retainedComponentIds: [...retained],
926
- considered: decisions.sort(
927
- (left, right) => context.candidateIndex.get(left.additionCandidateId) - context.candidateIndex.get(right.additionCandidateId)
928
- ),
929
- selectedCandidateId: selected?.bundleCandidateId ?? null
930
- });
931
- if (!selected?.bundleCandidateId) break;
932
- current = summaryById.get(selected.bundleCandidateId);
933
- retained = [...current.candidate.componentIds];
934
- remaining = remaining.filter(
935
- (summary) => summary.candidate.candidateId !== selected.additionCandidateId
936
- );
937
- }
938
- const qualified = retained.length >= context.input.selection.minimumBundleComponents;
939
- return {
940
- seedCandidateId: seed.candidate.candidateId,
941
- terminalCandidateId: current.candidate.candidateId,
942
- terminalComponentIds: retained,
943
- qualified,
944
- steps
945
- };
946
- }
947
- function compareInteractionPaths(context, summaryById, left, right) {
948
- const leftSummary = summaryById.get(left.terminalCandidateId);
949
- const rightSummary = summaryById.get(right.terminalCandidateId);
950
- const byNetBenefit = rightSummary.outcome.netBenefit - leftSummary.outcome.netBenefit;
951
- if (byNetBenefit !== 0) return byNetBenefit;
952
- const byBenefit = rightSummary.outcome.benefitTaskIds.length - leftSummary.outcome.benefitTaskIds.length;
953
- if (byBenefit !== 0) return byBenefit;
954
- const byRegression = leftSummary.outcome.regressionTaskIds.length - rightSummary.outcome.regressionTaskIds.length;
955
- if (byRegression !== 0) return byRegression;
956
- for (const metric of context.input.costMetricOrder) {
957
- const byCost = leftSummary.costs[metric].median - rightSummary.costs[metric].median;
958
- if (byCost !== 0) return byCost;
959
- }
960
- const byBytes = leftSummary.candidate.artifactBytes - rightSummary.candidate.artifactBytes;
961
- if (byBytes !== 0) return byBytes;
962
- const byCandidateOrder = context.candidateIndex.get(left.terminalCandidateId) - context.candidateIndex.get(right.terminalCandidateId);
963
- if (byCandidateOrder !== 0) return byCandidateOrder;
964
- return context.candidateIndex.get(left.seedCandidateId) - context.candidateIndex.get(right.seedCandidateId);
965
- }
966
- function evaluateAddition(context, summaryById, pairByComponents, baseline, current, retained, addition) {
967
- const additionComponentId = addition.candidate.componentIds[0];
968
- const reasons = [];
969
- for (const retainedComponentId of retained) {
970
- const pair = pairByComponents.get(
971
- crossSurfaceComponentSetKey(canonicalCrossSurfaceComponents(context, [retainedComponentId, additionComponentId]))
972
- );
973
- if (!pair?.compatibility.compatible) reasons.push("pair_incompatible");
974
- }
975
- const bundleComponents = canonicalCrossSurfaceComponents(context, [...retained, additionComponentId]);
976
- const bundle = context.candidateByComponents.get(crossSurfaceComponentSetKey(bundleComponents));
977
- if (!bundle) {
978
- reasons.push("full_bundle_not_evaluated");
979
- return emptyAdditionDecision(addition, additionComponentId, reasons);
980
- }
981
- const bundleSummary = summaryById.get(bundle.candidateId);
982
- if (bundleSummary.outcome.missingTaskIds.length > 0 || bundleSummary.outcome.invalidTaskIds.length > 0 || bundleSummary.outcome.comparisonMissingTaskIds.length > 0) {
983
- reasons.push("bundle_incomplete");
984
- }
985
- if (bundleSummary.outcome.regressionTaskIds.length > 0) reasons.push("baseline_regression");
986
- const comparison = compareCandidates(
987
- context,
988
- summaryById,
989
- current.candidate.candidateId,
990
- bundle.candidateId
991
- );
992
- if (comparison.winsTaskIds.length === 0) reasons.push("no_incremental_resolution");
993
- if (comparison.regressionTaskIds.length > 0) reasons.push("incremental_regression");
994
- appendBundleEvidenceReasons(
995
- reasons,
996
- bundleSummary.firing,
997
- context.input.selection.minimumFiringTasks,
998
- context.input.selection.requireObservedFiring,
999
- "firing_below_minimum",
1000
- "firing_unobserved"
1001
- );
1002
- appendBundleEvidenceReasons(
1003
- reasons,
1004
- bundleSummary.effect,
1005
- context.input.selection.minimumEffectTasks,
1006
- context.input.selection.requireObservedEffect,
1007
- "effect_below_minimum",
1008
- "effect_unobserved"
1009
- );
1010
- if (!withinCostLimits(context, bundleSummary, baseline)) reasons.push("cost_limit_exceeded");
1011
- return {
1012
- additionCandidateId: addition.candidate.candidateId,
1013
- additionComponentId,
1014
- bundleCandidateId: bundle.candidateId,
1015
- incrementalResolutionTaskIds: comparison.winsTaskIds,
1016
- incrementalRegressionTaskIds: comparison.regressionTaskIds,
1017
- incrementalMedianCost: Object.fromEntries(
1018
- context.input.costMetricOrder.map((metric) => [
1019
- metric,
1020
- bundleSummary.costs[metric].median - current.costs[metric].median
1021
- ])
1022
- ),
1023
- eligible: reasons.length === 0,
1024
- selected: false,
1025
- reasons: unique(reasons)
1026
- };
1027
- }
1028
- function emptyAdditionDecision(addition, additionComponentId, reasons) {
1029
- return {
1030
- additionCandidateId: addition.candidate.candidateId,
1031
- additionComponentId,
1032
- bundleCandidateId: null,
1033
- incrementalResolutionTaskIds: [],
1034
- incrementalRegressionTaskIds: [],
1035
- incrementalMedianCost: null,
1036
- eligible: false,
1037
- selected: false,
1038
- reasons: unique(reasons)
1039
- };
1040
- }
1041
- function appendBundleEvidenceReasons(reasons, evidence, minimum, requireObserved, belowReason, unobservedReason) {
1042
- if (evidence.byComponent.some((component) => component.observedTaskIds.length < minimum)) {
1043
- reasons.push(belowReason);
1044
- }
1045
- if (requireObserved && evidence.byComponent.some((component) => component.unobservedTaskIds.length > 0)) {
1046
- reasons.push(unobservedReason);
1047
- }
1048
- }
1049
- function compareAdditionDecisions(context, summaryById, left, right) {
1050
- const byWins = right.incrementalResolutionTaskIds.length - left.incrementalResolutionTaskIds.length;
1051
- if (byWins !== 0) return byWins;
1052
- for (const metric of context.input.costMetricOrder) {
1053
- const byCost = left.incrementalMedianCost[metric] - right.incrementalMedianCost[metric];
1054
- if (byCost !== 0) return byCost;
1055
- }
1056
- const leftBytes = summaryById.get(left.additionCandidateId).candidate.artifactBytes;
1057
- const rightBytes = summaryById.get(right.additionCandidateId).candidate.artifactBytes;
1058
- const byBytes = leftBytes - rightBytes;
1059
- if (byBytes !== 0) return byBytes;
1060
- return context.candidateIndex.get(left.additionCandidateId) - context.candidateIndex.get(right.additionCandidateId);
1061
- }
1062
- function compareCandidates(context, summaryById, comparatorCandidateId, treatmentCandidateId) {
1063
- const winsTaskIds = [];
1064
- const regressionTaskIds = [];
1065
- const missingTaskIds = [];
1066
- for (const taskId of context.input.taskOrder) {
1067
- const comparator = crossSurfaceRowFor(context, comparatorCandidateId, taskId);
1068
- const treatment = crossSurfaceRowFor(context, treatmentCandidateId, taskId);
1069
- if (comparator.completeness !== "complete" || treatment.completeness !== "complete") {
1070
- missingTaskIds.push(taskId);
1071
- } else if (treatment.pass && !comparator.pass) winsTaskIds.push(taskId);
1072
- else if (!treatment.pass && comparator.pass) regressionTaskIds.push(taskId);
1073
- }
1074
- return {
1075
- comparatorCandidateId,
1076
- treatmentCandidateId,
1077
- winsTaskIds,
1078
- regressionTaskIds,
1079
- missingTaskIds,
1080
- paired: pairedComparison(context, comparatorCandidateId, treatmentCandidateId),
1081
- relativeCost: relativeCosts(
1082
- context,
1083
- summaryById.get(treatmentCandidateId),
1084
- summaryById.get(comparatorCandidateId)
1085
- )
1086
- };
1087
- }
1088
- function pairedComparison(context, comparatorCandidateId, treatmentCandidateId) {
1089
- const rows = [];
1090
- for (const taskId of context.input.taskOrder) {
1091
- rows.push(toPairedArmRow(context, crossSurfaceRowFor(context, comparatorCandidateId, taskId)));
1092
- rows.push(toPairedArmRow(context, crossSurfaceRowFor(context, treatmentCandidateId, taskId)));
1093
- }
1094
- return comparePairedArms(rows, {
1095
- baselineArm: comparatorCandidateId,
1096
- treatmentArm: treatmentCandidateId,
1097
- metricNames: ["score", ...context.input.costMetricOrder],
1098
- bootstrap: { ...context.input.bootstrap, statistic: "mean" }
1099
- });
1100
- }
1101
- function toPairedArmRow(context, row) {
1102
- const metrics = Object.fromEntries(
1103
- context.input.costMetricOrder.map((metric) => [metric, row.cost[metric]])
1104
- );
1105
- if (row.completeness === "complete") metrics.score = row.score;
1106
- return {
1107
- pairKey: row.taskId,
1108
- arm: row.candidateId,
1109
- ...row.completeness === "complete" ? { pass: row.pass } : {},
1110
- metrics
1111
- };
1112
- }
1113
- function summarizePairEvidence(rows, leftComponentId, rightComponentId, field) {
1114
- const bothTaskIds = [];
1115
- const leftOnlyTaskIds = [];
1116
- const rightOnlyTaskIds = [];
1117
- const neitherTaskIds = [];
1118
- const unobservedTaskIds = [];
1119
- for (const row of rows) {
1120
- const left = evidenceValue(row, leftComponentId, field);
1121
- const right = evidenceValue(row, rightComponentId, field);
1122
- if (left === null || right === null) unobservedTaskIds.push(row.taskId);
1123
- else if (left && right) bothTaskIds.push(row.taskId);
1124
- else if (left) leftOnlyTaskIds.push(row.taskId);
1125
- else if (right) rightOnlyTaskIds.push(row.taskId);
1126
- else neitherTaskIds.push(row.taskId);
1127
- }
1128
- return {
1129
- bothTaskIds,
1130
- leftOnlyTaskIds,
1131
- rightOnlyTaskIds,
1132
- neitherTaskIds,
1133
- unobservedTaskIds
1134
- };
1135
- }
1136
- function relativeCosts(context, treatment, comparator) {
1137
- return Object.fromEntries(
1138
- context.input.costMetricOrder.map((metric) => {
1139
- const treatmentMedian = treatment.costs[metric].median;
1140
- const comparatorMedian = comparator.costs[metric].median;
1141
- return [
1142
- metric,
1143
- {
1144
- treatmentMedian,
1145
- comparatorMedian,
1146
- medianDelta: treatmentMedian - comparatorMedian,
1147
- medianRatio: comparatorMedian === 0 ? treatmentMedian === 0 ? 1 : null : treatmentMedian / comparatorMedian
1148
- }
1149
- ];
1150
- })
1151
- );
1152
- }
1153
- function withinCostLimits(context, treatment, baseline) {
1154
- const relative3 = relativeCosts(context, treatment, baseline);
1155
- return Object.entries(context.input.selection.maximumMedianCostRatioToBaseline).every(
1156
- ([metric, limit]) => {
1157
- const ratio = relative3[metric].medianRatio;
1158
- return ratio !== null && ratio <= limit;
1159
- }
1160
- );
1161
- }
1162
- function distribution(values) {
1163
- const sorted = [...values].sort((left, right) => left - right);
1164
- const total = sorted.reduce((sum, value) => sum + value, 0);
1165
- const middle = Math.floor(sorted.length / 2);
1166
- const median = sorted.length % 2 === 1 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2;
1167
- return {
1168
- n: sorted.length,
1169
- min: sorted[0],
1170
- median,
1171
- mean: total / sorted.length,
1172
- max: sorted[sorted.length - 1],
1173
- total
1174
- };
1175
- }
1176
- function evidenceValue(row, componentId, field) {
1177
- return row.componentEvidence.find((evidence) => evidence.componentId === componentId)[field];
1178
- }
1179
- function meanOrNull(values) {
1180
- return values.length === 0 ? null : values.reduce((sum, value) => sum + value, 0) / values.length;
1181
- }
1182
- function unique(values) {
1183
- return [...new Set(values)];
1184
- }
1185
-
1186
- // src/campaign/fixtures.ts
1187
- import { createHash } from "crypto";
1188
- import { existsSync, readdirSync, readFileSync, statSync } from "fs";
1189
- import { isAbsolute, join, relative, resolve } from "path";
1190
- function discoverEvalFixtures(evalsDir) {
1191
- const root = resolve(evalsDir);
1192
- if (!existsSync(root)) throw new Error(`discoverEvalFixtures: evalsDir not found: ${root}`);
1193
- const fixtures = [];
1194
- const walk = (dir, base = "") => {
1195
- for (const entry of readdirSync(dir).sort()) {
1196
- if (entry.startsWith(".") || entry === "node_modules") continue;
1197
- const fullPath = join(dir, entry);
1198
- if (!statSync(fullPath).isDirectory()) continue;
1199
- const name = base ? `${base}/${entry}` : entry;
1200
- if (existsWithExactCase(fullPath, "PROMPT.md")) {
1201
- fixtures.push(name);
1202
- } else {
1203
- walk(fullPath, name);
1204
- }
1205
- }
1206
- };
1207
- walk(root);
1208
- return fixtures;
1209
- }
1210
- function loadEvalFixture(evalsDir, name, options = {}) {
1211
- const validation = options.validation ?? "vitest";
1212
- const root = resolve(evalsDir);
1213
- const fixturePath = resolve(root, name);
1214
- assertInside(root, fixturePath, name);
1215
- if (!existsSync(fixturePath) || !statSync(fixturePath).isDirectory()) {
1216
- throw new Error(`loadEvalFixture: fixture not found: ${name}`);
1217
- }
1218
- if (!existsWithExactCase(fixturePath, "PROMPT.md")) {
1219
- throw new Error(`loadEvalFixture: ${name} is missing exact-case PROMPT.md`);
1220
- }
1221
- const promptPath = join(fixturePath, "PROMPT.md");
1222
- const evalPath = resolveEvalPath(fixturePath);
1223
- const packageJsonPath = existsWithExactCase(fixturePath, "package.json") ? join(fixturePath, "package.json") : void 0;
1224
- if (validation !== "none") {
1225
- if (!evalPath)
1226
- throw new Error(`loadEvalFixture: ${name} is missing exact-case EVAL.ts or EVAL.tsx`);
1227
- if (!packageJsonPath)
1228
- throw new Error(`loadEvalFixture: ${name} is missing exact-case package.json`);
1229
- assertModulePackage(packageJsonPath, name);
1230
- }
1231
- const files = collectFixtureFiles(fixturePath);
1232
- return {
1233
- name,
1234
- path: fixturePath,
1235
- promptPath,
1236
- evalPath,
1237
- packageJsonPath,
1238
- prompt: readFileSync(promptPath, "utf8"),
1239
- files,
1240
- fingerprint: contentHash({
1241
- files,
1242
- config: options.fingerprintConfig ?? null
1243
- })
1244
- };
1245
- }
1246
- function loadEvalFixtureScenarios(evalsDir, options = {}) {
1247
- const names = options.names ?? discoverEvalFixtures(evalsDir);
1248
- return names.map((name) => {
1249
- const fixture = loadEvalFixture(evalsDir, name, options);
1250
- return {
1251
- id: fixture.name,
1252
- kind: "eval-fixture",
1253
- tags: ["eval-fixture"],
1254
- fixtureName: fixture.name,
1255
- fixturePath: fixture.path,
1256
- promptPath: fixture.promptPath,
1257
- evalPath: fixture.evalPath,
1258
- packageJsonPath: fixture.packageJsonPath,
1259
- prompt: fixture.prompt,
1260
- fingerprint: fixture.fingerprint
1261
- };
1262
- });
1263
- }
1264
- function planEvalFixtureRun(options) {
1265
- const scenarios = loadEvalFixtureScenarios(options.evalsDir, {
1266
- names: options.names,
1267
- validation: options.validation,
1268
- fingerprintConfig: options.fingerprintConfig
1269
- });
1270
- const plan = planCampaignRun({
1271
- scenarios,
1272
- dispatchRef: options.dispatchRef ?? "eval-fixture-dispatch",
1273
- judges: options.judges,
1274
- seed: options.seed,
1275
- reps: options.reps,
1276
- resumable: options.resumable,
1277
- runDir: options.runDir,
1278
- storage: options.storage
1279
- });
1280
- return {
1281
- ...plan,
1282
- fixtures: scenarios.map((scenario) => ({
1283
- fixtureName: scenario.fixtureName,
1284
- fixturePath: scenario.fixturePath,
1285
- fingerprint: scenario.fingerprint
1286
- }))
1287
- };
1288
- }
1289
- function existsWithExactCase(dirPath, fileName) {
1290
- try {
1291
- return readdirSync(dirPath).includes(fileName);
1292
- } catch {
1293
- return false;
1294
- }
1295
- }
1296
- function resolveEvalPath(fixturePath) {
1297
- if (existsWithExactCase(fixturePath, "EVAL.ts")) return join(fixturePath, "EVAL.ts");
1298
- if (existsWithExactCase(fixturePath, "EVAL.tsx")) return join(fixturePath, "EVAL.tsx");
1299
- return void 0;
1300
- }
1301
- function assertModulePackage(packageJsonPath, name) {
1302
- let parsed;
1303
- try {
1304
- parsed = JSON.parse(readFileSync(packageJsonPath, "utf8"));
1305
- } catch (err) {
1306
- throw new Error(
1307
- `loadEvalFixture: ${name} package.json is invalid JSON: ${err instanceof Error ? err.message : String(err)}`
1308
- );
1309
- }
1310
- if (typeof parsed !== "object" || parsed === null || parsed.type !== "module") {
1311
- throw new Error(`loadEvalFixture: ${name} package.json must set "type": "module"`);
1312
- }
1313
- }
1314
- function collectFixtureFiles(fixturePath, base = "") {
1315
- const files = [];
1316
- for (const entry of readdirSync(join(fixturePath, base)).sort()) {
1317
- if (entry === "node_modules" || entry === ".git") continue;
1318
- const relativePath = base ? `${base}/${entry}` : entry;
1319
- const fullPath = join(fixturePath, relativePath);
1320
- const stat = statSync(fullPath);
1321
- if (stat.isDirectory()) {
1322
- files.push(...collectFixtureFiles(fixturePath, relativePath));
1323
- continue;
1324
- }
1325
- const bytes = readFileSync(fullPath);
1326
- files.push({
1327
- path: relativePath,
1328
- sha256: createHash("sha256").update(bytes).digest("hex"),
1329
- bytes: bytes.byteLength
1330
- });
1331
- }
1332
- return files;
1333
- }
1334
- function assertInside(root, target, label) {
1335
- const rel = relative(root, target);
1336
- if (rel === "" || !rel.startsWith("..") && !isAbsolute(rel)) return;
1337
- throw new Error(`loadEvalFixture: fixture path escapes evalsDir: ${label}`);
1338
- }
1339
-
1340
- // src/campaign/gates/neutralization-gate.ts
1341
- function mean(xs) {
1342
- return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length;
1343
- }
1344
- function pairedLift(arm, baseline, scenarioIds) {
1345
- const paired = pairHoldout(arm, baseline, scenarioIds, (s) => s.composite);
1346
- const deltas = paired.after.map((a, i) => a - (paired.before[i] ?? 0));
1347
- return { lift: mean(deltas), n: deltas.length };
1348
- }
1349
- function neutralizationGate(options) {
1350
- const maxDecorativeFraction = options.maxDecorativeFraction ?? 0.5;
1351
- return {
1352
- name: "neutralizationGate",
1353
- async decide(ctx) {
1354
- if (!ctx.baselineJudgeScores) {
1355
- throw new Error(
1356
- "neutralizationGate: ctx.baselineJudgeScores is required \u2014 the placebo control measures lift OVER baseline."
1357
- );
1358
- }
1359
- if (!ctx.neutralizedJudgeScores) {
1360
- throw new Error(
1361
- "neutralizationGate: ctx.neutralizedJudgeScores is required. It is populated by runImprovementLoop only when a `neutralize` function is supplied \u2014 composing this gate without that wiring would pass an unproven candidate."
1362
- );
1363
- }
1364
- const scenarioIds = new Set(options.scenarios.map((s) => s.id));
1365
- const cand = pairedLift(
1366
- ctx.judgeScores,
1367
- ctx.baselineJudgeScores,
1368
- scenarioIds
1369
- );
1370
- const neut = pairedLift(
1371
- ctx.neutralizedJudgeScores,
1372
- ctx.baselineJudgeScores,
1373
- scenarioIds
1374
- );
1375
- if (cand.lift <= 0) {
1376
- return {
1377
- decision: "hold",
1378
- reasons: [
1379
- `neutralization: candidate held-out lift ${cand.lift.toFixed(3)} \u2264 0 \u2014 no positive lift to attribute to content`
1380
- ],
1381
- contributingGates: [
1382
- {
1383
- name: "neutralizationGate",
1384
- status: "fail",
1385
- detail: { candidateLift: cand.lift, neutralizedLift: neut.lift, n: cand.n }
1386
- }
1387
- ],
1388
- delta: cand.lift
1389
- };
1390
- }
1391
- const decorativeFraction = neut.lift / cand.lift;
1392
- const passed = decorativeFraction < maxDecorativeFraction;
1393
- const pct = (decorativeFraction * 100).toFixed(0);
1394
- return {
1395
- decision: passed ? "ship" : "hold",
1396
- reasons: passed ? [
1397
- `neutralization: content is causal \u2014 blanked variant reproduces ${pct}% of the lift (< ${(maxDecorativeFraction * 100).toFixed(0)}%); candidate \u0394 ${cand.lift.toFixed(3)}, neutralized \u0394 ${neut.lift.toFixed(3)}`
1398
- ] : [
1399
- `neutralization: lift is DECORATIVE \u2014 blanking the content (footprint-matched) reproduces ${pct}% of the lift (\u2265 ${(maxDecorativeFraction * 100).toFixed(0)}%); candidate \u0394 ${cand.lift.toFixed(3)}, neutralized \u0394 ${neut.lift.toFixed(3)}`
1400
- ],
1401
- contributingGates: [
1402
- {
1403
- name: "neutralizationGate",
1404
- status: passed ? "pass" : "fail",
1405
- detail: {
1406
- candidateLift: cand.lift,
1407
- neutralizedLift: neut.lift,
1408
- decorativeFraction,
1409
- maxDecorativeFraction,
1410
- n: cand.n
1411
- }
1412
- }
1413
- ],
1414
- delta: cand.lift
1415
- };
1416
- }
1417
- };
1418
- }
1419
-
1420
- // src/campaign/gates/sequential.ts
1421
- import { createHash as createHash2 } from "crypto";
1422
- function verifyManifestSync(m) {
1423
- if (m.algo !== void 0 && m.algo !== "sha256-content") {
1424
- throw new Error(`sequentialPairedGate: unrecognized manifest hash algo '${m.algo}'`);
1425
- }
1426
- const { contentHash: contentHash2, algo: _algo, ...rest } = m;
1427
- void _algo;
1428
- const bytes = JSON.stringify(canonicalize(rest));
1429
- const hash = createHash2("sha256").update(bytes, "utf8").digest("hex");
1430
- return hash === contentHash2;
1431
- }
1432
- function resolveConfig(opts) {
1433
- const m = opts.preRegistration;
1434
- let alpha;
1435
- let maxN;
1436
- let direction;
1437
- let minEffect;
1438
- if (m) {
1439
- if (!verifyManifestSync(m)) {
1440
- throw new Error(
1441
- `sequentialPairedGate: pre-registration manifest '${m.id}' content hash mismatch (tampered)`
1442
- );
1443
- }
1444
- if (opts.alpha !== void 0 && opts.alpha !== m.alpha) {
1445
- throw new Error(
1446
- `sequentialPairedGate: alpha ${opts.alpha} conflicts with pre-registered alpha ${m.alpha} \u2014 the registered statistic is the only one anytime validity covers`
1447
- );
1448
- }
1449
- if (opts.maxN !== void 0 && opts.maxN !== m.preRegisteredN) {
1450
- throw new Error(
1451
- `sequentialPairedGate: maxN ${opts.maxN} conflicts with pre-registered N ${m.preRegisteredN}`
1452
- );
1453
- }
1454
- alpha = m.alpha;
1455
- maxN = m.preRegisteredN;
1456
- direction = m.direction;
1457
- minEffect = m.minEffect;
1458
- } else {
1459
- alpha = opts.alpha ?? 0.05;
1460
- if (opts.maxN === void 0) {
1461
- throw new Error(
1462
- "sequentialPairedGate: maxN is required (or bind a preRegistration manifest whose preRegisteredN is the budget) \u2014 an unbounded stream has no pre-registered budget"
1463
- );
1464
- }
1465
- maxN = opts.maxN;
1466
- direction = "increase";
1467
- minEffect = 0;
1468
- }
1469
- const scale = opts.scale ?? 1;
1470
- if (!Number.isFinite(scale) || scale <= 0) {
1471
- throw new Error(`sequentialPairedGate: scale must be > 0, got ${scale}`);
1472
- }
1473
- if (!Number.isInteger(maxN) || maxN < 1) {
1474
- throw new Error(`sequentialPairedGate: maxN must be a positive integer, got ${maxN}`);
1475
- }
1476
- const minN = opts.minN ?? 5;
1477
- if (!Number.isInteger(minN) || minN < 1 || minN > maxN) {
1478
- throw new Error(
1479
- `sequentialPairedGate: minN must be an integer in [1, maxN=${maxN}], got ${minN}`
1480
- );
1481
- }
1482
- if (!Number.isFinite(minEffect) || minEffect < 0 || minEffect >= scale) {
1483
- throw new Error(
1484
- `sequentialPairedGate: minEffect must be in [0, scale=${scale}), got ${minEffect}`
1485
- );
1486
- }
1487
- const nullMean = 0.5 + minEffect / (2 * scale);
1488
- return {
1489
- alpha,
1490
- minN,
1491
- maxN,
1492
- maxBet: opts.maxBet ?? 0.5,
1493
- scale,
1494
- shuffleSeed: opts.shuffleSeed ?? 1337,
1495
- direction,
1496
- nullMean,
1497
- minEffect
1498
- };
1499
- }
1500
- function makeStream(cfg) {
1501
- const proc = eProcess({ alpha: cfg.alpha, maxBet: cfg.maxBet, nullMean: cfg.nullMean });
1502
- const threshold = 1 / cfg.alpha;
1503
- let terminal;
1504
- return {
1505
- observe(delta) {
1506
- if (terminal === "undecided-at-maxN") {
1507
- throw new Error(
1508
- `sequentialPairedGate: pre-registered maxN=${cfg.maxN} exhausted \u2014 extending the stream after seeing the result reopens optional stopping; start a NEW pre-registered test`
1509
- );
1510
- }
1511
- if (!Number.isFinite(delta) || Math.abs(delta) > cfg.scale) {
1512
- throw new Error(
1513
- `sequentialPairedGate: delta ${delta} outside \xB1scale=${cfg.scale} \u2014 pass the judge's native scale explicitly (detectScale helps pick 1 vs 100)`
1514
- );
1515
- }
1516
- const d = cfg.direction === "decrease" ? -delta : delta;
1517
- const step = proc.update((d / cfg.scale + 1) / 2);
1518
- if (terminal === void 0 && step.n >= cfg.minN && step.wealth >= threshold) {
1519
- terminal = "promote";
1520
- }
1521
- if (terminal === "promote") {
1522
- return {
1523
- decision: "promote",
1524
- eValue: step.wealth,
1525
- n: step.n,
1526
- reason: `e-value ${step.wealth.toFixed(2)} \u2265 1/\u03B1=${threshold.toFixed(2)} at n=${step.n} (minN=${cfg.minN}): the paired improvement exceeds ${cfg.minEffect} at anytime-valid level \u03B1=${cfg.alpha}`
1527
- };
1528
- }
1529
- if (step.n >= cfg.maxN) {
1530
- terminal = "undecided-at-maxN";
1531
- return {
1532
- decision: "undecided-at-maxN",
1533
- eValue: step.wealth,
1534
- n: step.n,
1535
- reason: `undecided at pre-registered maxN=${cfg.maxN} (e-value ${step.wealth.toFixed(2)} < 1/\u03B1=${threshold.toFixed(2)}). This is NOT evidence of no effect \u2014 the effect may be real but smaller than this budget can detect; re-register with a larger N to test that`
1536
- };
1537
- }
1538
- return {
1539
- decision: "continue",
1540
- eValue: step.wealth,
1541
- n: step.n,
1542
- reason: `e-value ${step.wealth.toFixed(2)} < 1/\u03B1=${threshold.toFixed(2)} at n=${step.n}/${cfg.maxN} \u2014 keep observing`
1543
- };
1544
- },
1545
- state() {
1546
- return { ...proc.state(), decision: terminal ?? "continue" };
1547
- }
1548
- };
1549
- }
1550
- function seededShuffle(items, seed) {
1551
- const rng = mulberry32(seed);
1552
- for (let i = items.length - 1; i > 0; i--) {
1553
- const j = Math.floor(rng() * (i + 1));
1554
- const tmp = items[i];
1555
- items[i] = items[j];
1556
- items[j] = tmp;
1557
- }
1558
- return items;
1559
- }
1560
- function sequentialPairedGate(options) {
1561
- const cfg = resolveConfig(options);
1562
- const name = options.name ?? "sequentialPairedGate";
1563
- const manifest = options.preRegistration;
1564
- const observeStream = makeStream(cfg);
1565
- return {
1566
- name,
1567
- observe(delta) {
1568
- return observeStream.observe(delta);
1569
- },
1570
- state() {
1571
- return observeStream.state();
1572
- },
1573
- async decide(ctx) {
1574
- if (!ctx.baselineJudgeScores) {
1575
- throw new Error(
1576
- `${name}: ctx.baselineJudgeScores is required \u2014 falling back to the candidate's own scores would compare the candidate against itself (delta 0, silent no-op)`
1577
- );
1578
- }
1579
- const scenarioIds = new Set(ctx.scenarios.map((s) => s.id));
1580
- const paired = pairHoldout(
1581
- ctx.judgeScores,
1582
- ctx.baselineJudgeScores,
1583
- scenarioIds,
1584
- (s) => s.composite
1585
- );
1586
- const deltas = paired.after.map((a, i) => a - paired.before[i]);
1587
- seededShuffle(deltas, cfg.shuffleSeed);
1588
- const stream = makeStream(cfg);
1589
- let last;
1590
- for (const d of deltas) {
1591
- last = stream.observe(d);
1592
- if (last.decision !== "continue") break;
1593
- }
1594
- const detail = {
1595
- ...stream.state(),
1596
- minN: cfg.minN,
1597
- maxN: cfg.maxN,
1598
- scale: cfg.scale,
1599
- shuffleSeed: cfg.shuffleSeed,
1600
- direction: cfg.direction,
1601
- minEffect: cfg.minEffect,
1602
- pairedN: deltas.length,
1603
- ...manifest ? { preRegisteredId: manifest.id, metric: manifest.metric } : {}
1604
- };
1605
- const meanDelta = deltas.length === 0 ? void 0 : deltas.reduce((s, d) => s + d, 0) / deltas.length;
1606
- if (last === void 0) {
1607
- return {
1608
- decision: "need_more_work",
1609
- reasons: [`${name}: no paired holdout observations \u2014 nothing to test`],
1610
- contributingGates: [{ name, status: "not_evaluated", detail }]
1611
- };
1612
- }
1613
- const decision = last.decision === "promote" ? "ship" : last.decision === "continue" ? "need_more_work" : "hold";
1614
- return {
1615
- decision,
1616
- reasons: [`${name}: ${last.reason}`],
1617
- contributingGates: [
1618
- {
1619
- name,
1620
- status: decision === "ship" ? "pass" : decision === "need_more_work" ? "not_evaluated" : "fail",
1621
- detail
1622
- }
1623
- ],
1624
- delta: meanDelta
1625
- };
1626
- }
1627
- };
1628
- }
1629
- function sequentialDecide(options = {}) {
1630
- const alpha = options.alpha ?? 0.05;
1631
- const minN = options.minN ?? 5;
1632
- const scale = options.scale ?? 1;
1633
- if (!Number.isFinite(scale) || scale <= 0) {
1634
- throw new Error(`sequentialDecide: scale must be > 0, got ${scale}`);
1635
- }
1636
- const proc = eProcess({ alpha, maxBet: options.maxBet ?? 0.5, nullMean: 0.5 });
1637
- const threshold = 1 / alpha;
1638
- let processedGenerations = 0;
1639
- let reference;
1640
- let stopped;
1641
- const topCandidate = (record) => {
1642
- if (record.candidates.length === 0) {
1643
- throw new Error(
1644
- `sequentialDecide: generation ${record.generationIndex} has no candidates \u2014 cannot extract a top candidate`
1645
- );
1646
- }
1647
- return record.candidates.reduce((best, candidate) => {
1648
- if (best.composite === null) return candidate;
1649
- if (candidate.composite === null) return best;
1650
- return candidate.composite > best.composite ? candidate : best;
1651
- });
1652
- };
1653
- const decide = ({ history }) => {
1654
- if (stopped) return stopped;
1655
- if (history.length === 0) return { stop: false };
1656
- if (reference === void 0) {
1657
- reference = new Map(
1658
- topCandidate(history[0]).scenarios.map((s) => [s.scenarioId, s.composite])
1659
- );
1660
- }
1661
- for (let g = Math.max(1, processedGenerations); g < history.length; g++) {
1662
- const top = topCandidate(history[g]);
1663
- const byScenario = new Map(top.scenarios.map((s) => [s.scenarioId, s.composite]));
1664
- for (const [scenarioId, refComposite] of reference) {
1665
- const candComposite = byScenario.get(scenarioId);
1666
- if (candComposite === void 0) {
1667
- throw new Error(
1668
- `sequentialDecide: generation ${history[g].generationIndex} top candidate is missing scenario '${scenarioId}' \u2014 generations must score the same scenario set to pair`
1669
- );
1670
- }
1671
- const delta = candComposite - refComposite;
1672
- if (!Number.isFinite(delta) || Math.abs(delta) > scale) {
1673
- throw new Error(
1674
- `sequentialDecide: paired delta ${delta} outside \xB1scale=${scale} on scenario '${scenarioId}' \u2014 pass the composite scale explicitly`
1675
- );
1676
- }
1677
- const step = proc.update((delta / scale + 1) / 2);
1678
- if (step.n >= minN && step.wealth >= threshold) {
1679
- stopped = {
1680
- stop: true,
1681
- reason: `sequential e-process decided at generation ${history[g].generationIndex}: e-value ${step.wealth.toFixed(2)} \u2265 1/\u03B1=${threshold.toFixed(2)} after n=${step.n} paired deltas vs the generation-0 incumbent \u2014 the improvement is real at \u03B1=${alpha}; stop exploring and promote via the gate`
1682
- };
1683
- processedGenerations = history.length;
1684
- return stopped;
1685
- }
1686
- }
1687
- }
1688
- processedGenerations = history.length;
1689
- return { stop: false };
1690
- };
1691
- decide.state = () => proc.state();
1692
- return decide;
1693
- }
1694
-
1695
- // src/campaign/grounded-reflection.ts
1696
- function rolloutArgumentDiff(rollouts, opts = {}) {
1697
- const passThreshold = opts.passThreshold ?? 1;
1698
- const maxValues = opts.maxValuesPerField ?? 4;
1699
- const collect = (pass) => {
1700
- const byField = /* @__PURE__ */ new Map();
1701
- for (const r of rollouts) {
1702
- if (pass !== r.score >= passThreshold) continue;
1703
- for (const c of r.calls) {
1704
- for (const [k, v] of Object.entries(c.args)) {
1705
- const set = byField.get(k) ?? /* @__PURE__ */ new Set();
1706
- set.add(String(v));
1707
- byField.set(k, set);
1708
- }
1709
- }
1710
- }
1711
- return byField;
1712
- };
1713
- const passing = collect(true);
1714
- const failing = collect(false);
1715
- const lines = [];
1716
- for (const field of /* @__PURE__ */ new Set([...passing.keys(), ...failing.keys()])) {
1717
- const pv = [...passing.get(field) ?? []].slice(0, maxValues);
1718
- const fv = [...failing.get(field) ?? []].slice(0, maxValues);
1719
- const render = (vals) => vals.length ? JSON.stringify(vals) : "NOT SET (omitted)";
1720
- lines.push(` ${field}: passing runs -> ${render(pv)} | failing runs -> ${render(fv)}`);
1721
- }
1722
- const lower = (m) => {
1723
- const out = /* @__PURE__ */ new Set();
1724
- for (const vals of m.values()) for (const v of vals) out.add(v.toLowerCase());
1725
- return out;
1726
- };
1727
- return {
1728
- text: lines.join("\n") || " (no calls observed)",
1729
- passingValues: lower(passing),
1730
- failingValues: lower(failing)
1731
- };
1732
- }
1733
- function classifyUngroundedLiterals(text, diff) {
1734
- const ungrounded = /* @__PURE__ */ new Set();
1735
- for (const m of text.matchAll(/'([a-z][a-z_-]{1,19})'/gi)) {
1736
- const w = m[1].toLowerCase();
1737
- if (!diff.passingValues.has(w)) ungrounded.add(w);
1738
- }
1739
- const all = [...ungrounded];
1740
- return {
1741
- ungrounded: all,
1742
- harmful: all.filter((w) => diff.failingValues.has(w))
1743
- };
1744
- }
1745
-
1746
- // src/campaign/labeled-store/fs-adapter.ts
1747
- import { createHash as createHash3 } from "crypto";
1748
- import { existsSync as existsSync2, mkdirSync, readFileSync as readFileSync2, writeFileSync } from "fs";
1749
- import { join as join2 } from "path";
1750
- var LabeledScenarioStoreError = class extends Error {
1751
- constructor(code, message) {
1752
- super(message);
1753
- this.code = code;
1754
- this.name = "LabeledScenarioStoreError";
1755
- }
1756
- code;
1757
- };
1758
- var FsLabeledScenarioStore = class {
1759
- constructor(options) {
1760
- this.options = options;
1761
- if (!existsSync2(options.root)) mkdirSync(options.root, { recursive: true });
1762
- this.now = options.now ?? Date.now;
1763
- }
1764
- options;
1765
- now;
1766
- rateLimits = /* @__PURE__ */ new Map();
1767
- async observe(write) {
1768
- this.assertProvenance(write);
1769
- this.assertRateLimit(write);
1770
- const record = this.toRecord(write);
1771
- const path = this.pathForSource(write.source);
1772
- const line = `${JSON.stringify(record)}
1773
- `;
1774
- appendLine(path, line);
1775
- }
1776
- async sample(args) {
1777
- if (!args.split) {
1778
- throw new LabeledScenarioStoreError(
1779
- "split_required",
1780
- "sample() requires an explicit `split` (train | test) \u2014 substrate refuses ambiguous reads"
1781
- );
1782
- }
1783
- if (!args.capturedBefore) {
1784
- throw new LabeledScenarioStoreError(
1785
- "capturedBefore_required",
1786
- "sample() requires an explicit `capturedBefore` timestamp for temporal-split discipline"
1787
- );
1788
- }
1789
- const all = [];
1790
- for (const source of ALL_SOURCES) {
1791
- if (args.split === "train" && source === "production-trace") {
1792
- const explicit = sourceFilterContains(args.filter?.source, "production-trace");
1793
- if (!explicit) continue;
1794
- }
1795
- const path = this.pathForSource(source);
1796
- if (!existsSync2(path)) continue;
1797
- const lines = readFileSync2(path, "utf8").split("\n").filter(Boolean);
1798
- for (const line of lines) {
1799
- let record;
1800
- try {
1801
- record = JSON.parse(line);
1802
- } catch {
1803
- continue;
1804
- }
1805
- if (!matchesFilter(record, args, source)) continue;
1806
- all.push(record);
1807
- }
1808
- }
1809
- all.sort((a, b) => {
1810
- if (a.capturedAt !== b.capturedAt) return a.capturedAt.localeCompare(b.capturedAt);
1811
- return a.recordHash.localeCompare(b.recordHash);
1812
- });
1813
- return all.slice(0, args.count);
1814
- }
1815
- async size() {
1816
- const bySource = {};
1817
- const byTrust = {
1818
- unverified: 0,
1819
- "verified-signal": 0,
1820
- "human-rated": 0
1821
- };
1822
- let total = 0;
1823
- for (const source of ALL_SOURCES) {
1824
- const path = this.pathForSource(source);
1825
- if (!existsSync2(path)) {
1826
- bySource[source] = 0;
1827
- continue;
1828
- }
1829
- const lines = readFileSync2(path, "utf8").split("\n").filter(Boolean);
1830
- bySource[source] = lines.length;
1831
- total += lines.length;
1832
- for (const line of lines) {
1833
- let trust = "unverified";
1834
- try {
1835
- trust = JSON.parse(line).labelTrust ?? "unverified";
1836
- } catch {
1837
- }
1838
- byTrust[trust] += 1;
1839
- }
1840
- }
1841
- return { train: total, test: total, bySource, byTrust };
1842
- }
1843
- assertProvenance(write) {
1844
- if (!write.source) {
1845
- throw new LabeledScenarioStoreError(
1846
- "missing_source",
1847
- "LabeledScenarioWrite requires `source`"
1848
- );
1849
- }
1850
- if (!write.sourceVersionHash || write.sourceVersionHash.length === 0) {
1851
- throw new LabeledScenarioStoreError(
1852
- "missing_source_version",
1853
- "LabeledScenarioWrite requires `sourceVersionHash` (git sha or substrate version)"
1854
- );
1855
- }
1856
- if (!write.capturedAt) {
1857
- throw new LabeledScenarioStoreError(
1858
- "missing_captured_at",
1859
- "LabeledScenarioWrite requires `capturedAt` ISO timestamp"
1860
- );
1861
- }
1862
- if (!write.redactionStatus) {
1863
- throw new LabeledScenarioStoreError(
1864
- "missing_redaction_status",
1865
- "LabeledScenarioWrite requires explicit `redactionStatus` \u2014 raw / redacted-pii / redacted-secrets / fully-redacted"
1866
- );
1867
- }
1868
- if (!ALL_SOURCES.includes(write.source)) {
1869
- throw new LabeledScenarioStoreError(
1870
- "unknown_source",
1871
- `LabeledScenarioWrite.source must be one of: ${ALL_SOURCES.join(", ")}`
1872
- );
1873
- }
1874
- }
1875
- assertRateLimit(write) {
1876
- const cap = this.options.maxWritesPerMinutePerBucket;
1877
- if (!cap || !write.rateLimitBucket) return;
1878
- const now = this.now();
1879
- const windowMs = 6e4;
1880
- let state = this.rateLimits.get(write.rateLimitBucket);
1881
- if (!state || now - state.windowStartMs >= windowMs) {
1882
- state = { bucket: write.rateLimitBucket, windowStartMs: now, count: 0 };
1883
- this.rateLimits.set(write.rateLimitBucket, state);
1884
- }
1885
- if (state.count >= cap) {
1886
- throw new LabeledScenarioStoreError(
1887
- "rate_limit_exceeded",
1888
- `LabeledScenarioStore: bucket ${write.rateLimitBucket} exceeded ${cap} writes/min`
1889
- );
1890
- }
1891
- state.count += 1;
1892
- }
1893
- toRecord(write) {
1894
- const recordHash = sha256(
1895
- JSON.stringify({
1896
- id: write.scenario.id,
1897
- src: write.source,
1898
- at: write.capturedAt,
1899
- ver: write.sourceVersionHash
1900
- })
1901
- );
1902
- return {
1903
- ...write,
1904
- recordHash,
1905
- split: "train"
1906
- };
1907
- }
1908
- pathForSource(source) {
1909
- return join2(this.options.root, `${source}.jsonl`);
1910
- }
1911
- };
1912
- var ALL_SOURCES = [
1913
- "production-trace",
1914
- "eval-run",
1915
- "manual",
1916
- "red-team",
1917
- "synthetic"
1918
- ];
1919
- function sourceFilterContains(filter, needle) {
1920
- if (!filter) return false;
1921
- if (Array.isArray(filter)) return filter.includes(needle);
1922
- return filter === needle;
1923
- }
1924
- function matchesFilter(record, args, source) {
1925
- if (args.split === "train" && record.capturedAt >= args.capturedBefore) return false;
1926
- if (args.split === "test" && record.capturedAt < args.capturedBefore) return false;
1927
- const f = args.filter;
1928
- if (!f) return true;
1929
- if (f.kind && record.scenario.kind !== f.kind) return false;
1930
- if (f.source) {
1931
- const sources = Array.isArray(f.source) ? f.source : [f.source];
1932
- if (!sources.includes(source)) return false;
1933
- }
1934
- if (f.minComposite !== void 0 || f.maxComposite !== void 0) {
1935
- const composites = Object.values(record.judgeScores).map((s) => s.composite);
1936
- const max = composites.length === 0 ? 0 : Math.max(...composites);
1937
- if (f.minComposite !== void 0 && max < f.minComposite) return false;
1938
- if (f.maxComposite !== void 0 && max > f.maxComposite) return false;
1939
- }
1940
- if (f.minTrust !== void 0 && labelTrustRank(record.labelTrust) < labelTrustRank(f.minTrust)) {
1941
- return false;
1942
- }
1943
- return true;
1944
- }
1945
- function sha256(input) {
1946
- return createHash3("sha256").update(input).digest("hex").slice(0, 16);
1947
- }
1948
- function appendLine(path, line) {
1949
- if (existsSync2(path)) {
1950
- const existing = readFileSync2(path, "utf8");
1951
- writeFileSync(path, existing + line);
1952
- } else {
1953
- writeFileSync(path, line);
1954
- }
1955
- }
1956
-
1957
- // src/campaign/neutralize.ts
1958
- var FILLER = "#";
1959
- function neutralizeText(content) {
1960
- return content.replace(/\S/g, FILLER);
1961
- }
1962
-
1963
- // src/completion-verifier.ts
1964
- import { randomUUID } from "crypto";
1965
- function completionVerdict(input) {
1966
- if (input.requirements.length === 0) {
1967
- throw new Error(
1968
- `completionVerdict: task '${input.taskId}' has no requirement checks \u2014 nothing to derive a verdict from`
1969
- );
1970
- }
1971
- const measurable = input.requirements.filter((r) => !r.unmeasured);
1972
- const unmeasuredCount = input.requirements.length - measurable.length;
1973
- if (measurable.length === 0) {
1974
- throw new Error(
1975
- `completionVerdict: task '${input.taskId}' has no measurable requirements \u2014 all ${input.requirements.length} correctness checks failed (${input.requirements[0]?.unmeasuredReason ?? "unknown reason"})`
1976
- );
1977
- }
1978
- const satisfiedCount = measurable.filter((r) => r.satisfied).length;
1979
- const completionRate = satisfiedCount / measurable.length;
1980
- const fullyComplete = unmeasuredCount === 0 && satisfiedCount === measurable.length;
1981
- return {
1982
- taskId: input.taskId,
1983
- requirements: input.requirements,
1984
- completionRate,
1985
- fullyComplete,
1986
- unmeasuredCount,
1987
- valid: fullyComplete,
1988
- score: completionRate
1989
- };
1990
- }
1991
- var STOPWORDS = /* @__PURE__ */ new Set([
1992
- "the",
1993
- "a",
1994
- "an",
1995
- "of",
1996
- "for",
1997
- "and",
1998
- "or",
1999
- "to",
2000
- "in",
2001
- "on",
2002
- "with",
2003
- "by"
2004
- ]);
2005
- var REQUIREMENT_FORM_STOPWORDS = /* @__PURE__ */ new Set([
2006
- "generated",
2007
- "generate",
2008
- "view",
2009
- "render",
2010
- "rendered",
2011
- "persisted",
2012
- "persist",
2013
- "artifact",
2014
- "file",
2015
- "document",
2016
- "note",
2017
- "proposal",
2018
- "deliverable",
2019
- "output",
2020
- "created",
2021
- "create",
2022
- "produce",
2023
- "produced",
2024
- "flag"
2025
- ]);
2026
- var MATCH_THRESHOLD = 0.5;
2027
- var MIN_CONTENT_CHARS = 50;
2028
- function tokens(s, extraStop) {
2029
- return new Set(
2030
- s.toLowerCase().split(/[^a-z0-9]+/).filter((t) => t.length > 1 && !STOPWORDS.has(t) && !extraStop?.has(t))
2031
- );
2032
- }
2033
- function tokenRecall(requirementText, candidateText) {
2034
- const req = tokens(requirementText, REQUIREMENT_FORM_STOPWORDS);
2035
- if (req.size === 0) return 0;
2036
- const cand = tokens(candidateText);
2037
- let hit = 0;
2038
- for (const t of req) if (cand.has(t)) hit++;
2039
- return hit / req.size;
2040
- }
2041
- function artifactCandidates(req, reqIndex, artifacts) {
2042
- const reqText = `${req.title} ${req.category ?? ""}`;
2043
- const out = [];
2044
- artifacts.forEach((a, i) => {
2045
- if ((a.content ?? "").trim().length < MIN_CONTENT_CHARS) return;
2046
- let score = tokenRecall(
2047
- reqText,
2048
- `${a.path ?? ""} ${a.kind} ${(a.content ?? "").slice(0, 4e3)}`
2049
- );
2050
- if (req.category && a.kind && req.category.toLowerCase() === a.kind.toLowerCase()) {
2051
- score = Math.max(score, 1);
2052
- }
2053
- if (score < MATCH_THRESHOLD) return;
2054
- out.push({
2055
- reqIndex,
2056
- itemKey: `artifact:${i}`,
2057
- score,
2058
- evidence: `artifact '${a.path ?? a.kind}' matched (token recall ${score.toFixed(2)})`,
2059
- content: a.content ?? null
2060
- });
2061
- });
2062
- return out;
2063
- }
2064
- function proposalCandidates(req, reqIndex, proposals) {
2065
- const reqText = `${req.title} ${req.category ?? ""}`;
2066
- const out = [];
2067
- for (const p of proposals) {
2068
- if (p.status !== "approved") continue;
2069
- const body = (p.content ?? "").trim();
2070
- if (body.length < MIN_CONTENT_CHARS) continue;
2071
- const score = tokenRecall(reqText, `${p.title} ${body}`);
2072
- if (score < MATCH_THRESHOLD) continue;
2073
- out.push({
2074
- reqIndex,
2075
- itemKey: `proposal:${p.id}`,
2076
- score,
2077
- evidence: `approved proposal '${p.title}' matched (token recall ${score.toFixed(2)})`,
2078
- content: body
2079
- });
2080
- }
2081
- return out;
2082
- }
2083
- function toolCallCandidates(req, reqIndex, toolCalls) {
2084
- const out = [];
2085
- toolCalls.forEach((name, i) => {
2086
- const score = tokenRecall(req.title, name);
2087
- if (score < MATCH_THRESHOLD) return;
2088
- out.push({
2089
- reqIndex,
2090
- itemKey: `tool:${i}`,
2091
- score,
2092
- evidence: `tool call '${name}' matched (token recall ${score.toFixed(2)})`,
2093
- content: null
2094
- });
2095
- });
2096
- return out;
2097
- }
2098
- async function verifyCompletion(gold, state, checkCorrectness) {
2099
- if (gold.requirements.length === 0) {
2100
- throw new Error(
2101
- `verifyCompletion: task '${gold.taskId}' has no requirements \u2014 malformed gold spec`
2102
- );
2103
- }
2104
- const candidates = [];
2105
- gold.requirements.forEach((req, i) => {
2106
- const by = req.satisfiedBy ?? "any";
2107
- if (by === "artifact" || by === "any") {
2108
- candidates.push(...artifactCandidates(req, i, state.artifacts));
2109
- }
2110
- if (by === "proposal" || by === "any") {
2111
- candidates.push(...proposalCandidates(req, i, state.proposals));
2112
- }
2113
- if (by === "tool-call" || by === "any") {
2114
- candidates.push(...toolCallCandidates(req, i, state.toolCalls));
2115
- }
2116
- });
2117
- candidates.sort((a, b) => b.score - a.score);
2118
- const assigned = /* @__PURE__ */ new Map();
2119
- const itemTaken = /* @__PURE__ */ new Set();
2120
- for (const c of candidates) {
2121
- if (assigned.has(c.reqIndex) || itemTaken.has(c.itemKey)) continue;
2122
- assigned.set(c.reqIndex, c);
2123
- itemTaken.add(c.itemKey);
2124
- }
2125
- const requirements = [];
2126
- for (let i = 0; i < gold.requirements.length; i++) {
2127
- const req = gold.requirements[i];
2128
- const match = assigned.get(i);
2129
- const evidence = [];
2130
- let correct = null;
2131
- let unmeasuredReason;
2132
- if (match) {
2133
- evidence.push(match.evidence);
2134
- if (match.content !== null) {
2135
- try {
2136
- const r = await checkCorrectness(req, match.content);
2137
- correct = r.correct;
2138
- evidence.push(`correctness: ${r.correct ? "pass" : "fail"} \u2014 ${r.reason}`);
2139
- } catch (err) {
2140
- unmeasuredReason = err instanceof JudgeParseError ? `checker response unparseable after retry: ${err.raw.slice(0, 200)}` : `checker call failed: ${err instanceof Error ? err.message : String(err)}`;
2141
- evidence.push(`correctness: UNMEASURED \u2014 ${unmeasuredReason}`);
2142
- }
2143
- } else {
2144
- evidence.push("correctness: not assessed \u2014 matched item carries no content");
2145
- }
2146
- } else {
2147
- const by = req.satisfiedBy ?? "any";
2148
- const kind = by === "any" ? "artifact/proposal/tool-call" : by;
2149
- evidence.push(`no produced ${kind} matched this requirement`);
2150
- }
2151
- const structurallyPresent = match !== void 0;
2152
- const unmeasured = unmeasuredReason !== void 0;
2153
- const satisfied = structurallyPresent && !unmeasured && correct !== false;
2154
- requirements.push({
2155
- reqId: req.reqId,
2156
- title: req.title,
2157
- structurallyPresent,
2158
- correct,
2159
- satisfied,
2160
- ...unmeasured ? { unmeasured: true, unmeasuredReason } : {},
2161
- evidence
2162
- });
2163
- }
2164
- return completionVerdict({ taskId: gold.taskId, requirements });
2165
- }
2166
- function parseCorrectnessResponse(raw) {
2167
- const readVerdict = (candidate) => {
2168
- if (candidate === null || typeof candidate !== "object") return null;
2169
- const { correct, reason } = candidate;
2170
- if (typeof correct !== "boolean") return null;
2171
- return { correct, reason: typeof reason === "string" ? reason : "" };
2172
- };
2173
- const match = raw.match(/\{[\s\S]*\}/);
2174
- if (match) {
2175
- try {
2176
- const strict = readVerdict(JSON.parse(match[0]));
2177
- if (strict) return strict;
2178
- } catch {
2179
- }
2180
- }
2181
- const start = raw.indexOf("{");
2182
- if (start !== -1) {
2183
- const recovered = readVerdict(recoverTruncatedJson(raw.slice(start)));
2184
- if (recovered) return recovered;
2185
- }
2186
- throw new JudgeParseError("correctness-checker", raw);
2187
- }
2188
- function createLlmCorrectnessChecker(tc, opts = {}) {
2189
- const model = opts.model ?? "claude-sonnet-4-6";
2190
- const maxContentChars = opts.maxContentChars ?? 8e3;
2191
- const maxAttempts = opts.maxAttempts ?? 2;
2192
- const costLedger = opts.costLedger ?? new CostLedger();
2193
- const sink = opts.rawSink;
2194
- const record = async (event) => {
2195
- try {
2196
- await sink?.record(event);
2197
- } catch {
2198
- }
2199
- };
2200
- return async (requirement, content) => {
2201
- const request = {
2202
- model,
2203
- messages: [
2204
- {
2205
- role: "system",
2206
- content: 'You verify whether a produced work artifact actually fulfils a stated requirement. Judge fulfilment only \u2014 is the deliverable substantively present and on-point \u2014 not polish. A plan to do it later, a vague gesture, or a description of what should be done does NOT fulfil a requirement; the artifact must BE the deliverable. Respond with a single JSON object: {"correct": boolean, "reason": string (<= 30 words)}.'
2207
- },
2208
- {
2209
- role: "user",
2210
- content: `Requirement: ${requirement.title}
2211
- ${requirement.category ? `Category: ${requirement.category}
2212
- ` : ""}
2213
- Produced artifact:
2214
- ${content.slice(0, maxContentChars)}`
2215
- }
2216
- ],
2217
- temperature: 0,
2218
- maxTokens: 200
2219
- };
2220
- let lastErr;
2221
- for (let attempt = 0; attempt < maxAttempts; attempt++) {
2222
- const started = Date.now();
2223
- await record({
2224
- eventId: randomUUID(),
2225
- provider: "correctness-checker",
2226
- model,
2227
- endpoint: "/chat",
2228
- baseUrl: "",
2229
- attemptIndex: attempt,
2230
- direction: "request",
2231
- timestamp: started,
2232
- requestBody: request,
2233
- redactedFields: []
2234
- });
2235
- try {
2236
- const paid = await costLedger.runPaidCall({
2237
- channel: "verifier",
2238
- phase: opts.costPhase ?? "completion.correctness",
2239
- actor: "correctness-checker",
2240
- model,
2241
- maximumCharge: maximumChargeForTCloudRequest(request, opts.tcloudMaximumAttempts),
2242
- tags: {
2243
- ...opts.costTags,
2244
- requirementId: requirement.reqId,
2245
- attempt: String(attempt)
2246
- },
2247
- signal: opts.signal,
2248
- execute: () => tc.chat(request),
2249
- receipt: (response) => costReceiptFromTCloud(response, model),
2250
- receiptFromError: (error) => opts.receiptFromError?.(error, attempt)
2251
- });
2252
- if (!paid.succeeded) throw paid.error;
2253
- const resp = paid.value;
2254
- const raw = resp.choices?.[0]?.message?.content ?? "";
2255
- await record({
2256
- eventId: randomUUID(),
2257
- provider: "correctness-checker",
2258
- model,
2259
- endpoint: "/chat",
2260
- baseUrl: "",
2261
- attemptIndex: attempt,
2262
- direction: "response",
2263
- timestamp: Date.now(),
2264
- durationMs: Date.now() - started,
2265
- responseBody: resp,
2266
- redactedFields: []
2267
- });
2268
- return parseCorrectnessResponse(raw);
2269
- } catch (err) {
2270
- lastErr = err;
2271
- await record({
2272
- eventId: randomUUID(),
2273
- provider: "correctness-checker",
2274
- model,
2275
- endpoint: "/chat",
2276
- baseUrl: "",
2277
- attemptIndex: attempt,
2278
- direction: "error",
2279
- timestamp: Date.now(),
2280
- durationMs: Date.now() - started,
2281
- errorMessage: err instanceof Error ? err.message : String(err),
2282
- redactedFields: []
2283
- });
2284
- }
2285
- }
2286
- throw lastErr instanceof Error ? lastErr : new Error(String(lastErr));
2287
- };
2288
- }
2289
- var TITLE_STOPWORDS = /* @__PURE__ */ new Set([
2290
- "the",
2291
- "a",
2292
- "an",
2293
- "and",
2294
- "or",
2295
- "for",
2296
- "to",
2297
- "of",
2298
- "in",
2299
- "on",
2300
- "with",
2301
- "review",
2302
- "update",
2303
- "new",
2304
- "proposed"
2305
- ]);
2306
- function createTokenRecallChecker(opts = {}) {
2307
- const minRecall = opts.minRecall ?? 0.5;
2308
- const minLen = opts.minContentLength ?? 120;
2309
- return async (requirement, content) => {
2310
- const body = content.trim();
2311
- if (body.length < minLen)
2312
- return {
2313
- correct: false,
2314
- reason: `content too thin (${body.length} chars) to be the deliverable`
2315
- };
2316
- const titleTokens = requirement.title.toLowerCase().split(/[^a-z0-9]+/).filter((t) => t.length > 2 && !TITLE_STOPWORDS.has(t));
2317
- if (titleTokens.length === 0)
2318
- return {
2319
- correct: true,
2320
- reason: "requirement title has no significant tokens \u2014 structural match accepted"
2321
- };
2322
- const lower = body.toLowerCase();
2323
- const hits = titleTokens.filter((t) => lower.includes(t)).length;
2324
- const recall = hits / titleTokens.length;
2325
- return recall >= minRecall ? {
2326
- correct: true,
2327
- reason: `content recalls ${hits}/${titleTokens.length} requirement tokens`
2328
- } : {
2329
- correct: false,
2330
- reason: `content recalls only ${hits}/${titleTokens.length} requirement tokens`
2331
- };
2332
- };
2333
- }
2334
-
2335
- // src/produced-state.ts
2336
- function artifactKind(mimeType) {
2337
- if (!mimeType) return "file";
2338
- if (mimeType.includes("json")) return "json";
2339
- if (mimeType.startsWith("text/")) return "text";
2340
- return "file";
2341
- }
2342
- function extractProducedState(events) {
2343
- const artifacts = [];
2344
- const proposals = [];
2345
- const toolCalls = [];
2346
- const seenTools = /* @__PURE__ */ new Set();
2347
- for (const ev of events) {
2348
- if (ev.type === "tool_call") {
2349
- const name = ev.toolName;
2350
- if (name && !seenTools.has(name)) {
2351
- seenTools.add(name);
2352
- toolCalls.push(name);
2353
- }
2354
- } else if (ev.type === "artifact") {
2355
- const a = ev;
2356
- artifacts.push({
2357
- kind: artifactKind(a.mimeType),
2358
- path: a.name ?? a.uri ?? a.artifactId,
2359
- content: a.content ?? ""
2360
- });
2361
- } else if (ev.type === "proposal_created") {
2362
- const p = ev;
2363
- proposals.push({
2364
- id: p.proposalId,
2365
- title: p.title,
2366
- status: p.status ?? "pending",
2367
- ...p.content !== void 0 ? { content: p.content } : {}
2368
- });
2369
- }
2370
- }
2371
- return { artifacts, proposals, toolCalls };
2372
- }
2373
-
2374
- // src/campaign/presets/playback.ts
2375
- function makePlaybackDispatch(driver) {
2376
- return async (profile, scenario, ctx) => {
2377
- const events = await driver.run(scenario, { ...ctx, profile });
2378
- return extractProducedState(events);
2379
- };
2380
- }
2381
- async function scoreUserStory(story, state, checkCorrectness) {
2382
- const verdict = await verifyCompletion(
2383
- { taskId: story.id, requirements: story.requirements },
2384
- state,
2385
- checkCorrectness
2386
- );
2387
- return { ...verdict, title: story.title };
2388
- }
2389
- function userStoryScoreboard(verdicts) {
2390
- const rows = [];
2391
- for (const v of verdicts) {
2392
- for (const r of v.requirements) {
2393
- rows.push({
2394
- storyId: v.taskId,
2395
- storyTitle: v.title,
2396
- reqId: r.reqId,
2397
- reqTitle: r.title,
2398
- status: r.satisfied ? "PASS" : "FAIL",
2399
- evidence: r.evidence
2400
- });
2401
- }
2402
- }
2403
- return rows;
2404
- }
2405
- function scoreboardSummary(rows) {
2406
- const byStory = /* @__PURE__ */ new Map();
2407
- let passed = 0;
2408
- for (const r of rows) {
2409
- const s = byStory.get(r.storyId) ?? { total: 0, passed: 0 };
2410
- s.total++;
2411
- if (r.status === "PASS") {
2412
- s.passed++;
2413
- passed++;
2414
- }
2415
- byStory.set(r.storyId, s);
2416
- }
2417
- let storiesFullyComplete = 0;
2418
- for (const s of byStory.values()) if (s.total > 0 && s.passed === s.total) storiesFullyComplete++;
2419
- return {
2420
- stories: byStory.size,
2421
- storiesFullyComplete,
2422
- requirements: rows.length,
2423
- passed,
2424
- failed: rows.length - passed,
2425
- passRate: rows.length === 0 ? 0 : passed / rows.length
2426
- };
2427
- }
2428
- function escapeCell(s) {
2429
- return s.replace(/\|/g, "\\|").replace(/\r?\n/g, " ");
2430
- }
2431
- function truncate(s, max) {
2432
- return s.length <= max ? s : `${s.slice(0, Math.max(0, max - 1))}\u2026`;
2433
- }
2434
- function renderScoreboardMarkdown(rows, opts = {}) {
2435
- const maxEv = opts.maxEvidenceChars ?? 160;
2436
- const sum = scoreboardSummary(rows);
2437
- const pct = (n) => `${Math.round(n * 100)}%`;
2438
- const ev = (e) => escapeCell(truncate(e.join("; "), maxEv)) || "\u2014";
2439
- const out = [`# ${opts.title ?? "Product-flow playback scoreboard"}`, ""];
2440
- if (opts.meta) {
2441
- for (const [k, v] of Object.entries(opts.meta)) out.push(`- **${k}:** ${v}`);
2442
- out.push("");
2443
- }
2444
- out.push(
2445
- `**${sum.storiesFullyComplete}/${sum.stories}** user stories fully shipped \xB7 **${sum.passed}/${sum.requirements}** requirements passing (${pct(sum.passRate)}) \xB7 **${sum.failed}** open`,
2446
- ""
2447
- );
2448
- const fails = rows.filter((r) => r.status === "FAIL");
2449
- if (fails.length > 0) {
2450
- out.push("## Open tickets", "", "| Story | Requirement | Evidence |", "| --- | --- | --- |");
2451
- for (const r of fails) {
2452
- out.push(`| ${escapeCell(r.storyTitle)} | ${escapeCell(r.reqTitle)} | ${ev(r.evidence)} |`);
2453
- }
2454
- out.push("");
2455
- } else {
2456
- out.push("_All requirements passing \u2014 no open tickets._", "");
2457
- }
2458
- out.push("## Per-story tick-off", "");
2459
- for (const storyId of [...new Set(rows.map((r) => r.storyId))]) {
2460
- const storyRows = rows.filter((r) => r.storyId === storyId);
2461
- const passed = storyRows.filter((r) => r.status === "PASS").length;
2462
- const mark = passed === storyRows.length ? "\u2705" : "\u26A0\uFE0F";
2463
- out.push(
2464
- `### ${escapeCell(storyRows[0].storyTitle)} \u2014 ${passed}/${storyRows.length} ${mark}`,
2465
- "",
2466
- "| Requirement | Status | Evidence |",
2467
- "| --- | --- | --- |"
2468
- );
2469
- for (const r of storyRows) {
2470
- out.push(
2471
- `| ${escapeCell(r.reqTitle)} | ${r.status === "PASS" ? "\u2705 PASS" : "\u274C FAIL"} | ${ev(r.evidence)} |`
2472
- );
2473
- }
2474
- out.push("");
2475
- }
2476
- return out.join("\n");
2477
- }
2478
-
2479
- // src/campaign/presets/run-profile-matrix.ts
2480
- import { createHash as createHash5 } from "crypto";
2481
- import { join as join3 } from "path";
2482
-
2483
- // src/agent-profile.ts
2484
- import { createHash as createHash4 } from "crypto";
2485
- import { harnessSupportsModel } from "@tangle-network/agent-interface";
2486
- var CODING_HARNESSES = [
2487
- "opencode",
2488
- "claude-code",
2489
- "codex",
2490
- "kimi-code"
2491
- ];
2492
- var HARNESS_NATIVE_MODEL = "default";
2493
- function expandProfileAxes(spec) {
2494
- const harnesses = spec.harnesses ?? CODING_HARNESSES;
2495
- if (harnesses.length === 0) throw new ValidationError("expandProfileAxes: no harnesses to sweep");
2496
- const baseModel = spec.base.model?.default;
2497
- const models = spec.models ?? (baseModel ? [baseModel] : []);
2498
- if (models.length === 0) {
2499
- throw new ValidationError(
2500
- "expandProfileAxes: no models to sweep \u2014 base profile has no model.default and none were supplied"
2501
- );
2502
- }
2503
- const out = [];
2504
- const seen = /* @__PURE__ */ new Set();
2505
- for (const harness of harnesses) {
2506
- const supported = spec.keepIncompatible ? models : models.filter((model) => harnessSupportsModel(harness, model));
2507
- const effective = supported.length > 0 ? supported : [HARNESS_NATIVE_MODEL];
2508
- for (const model of effective) {
2509
- const profile = {
2510
- ...spec.base,
2511
- name: `${spec.base.name ?? "agent"}/${harness}/${model}`,
2512
- model: { ...spec.base.model, default: model },
2513
- metadata: { ...spec.base.metadata ?? {}, harness, harnessModel: model }
2514
- };
2515
- const id = agentProfileId(profile);
2516
- if (seen.has(id)) continue;
2517
- seen.add(id);
2518
- out.push(profile);
2519
- }
2520
- }
2521
- if (out.length === 0) {
2522
- throw new ValidationError(
2523
- `expandProfileAxes: produced no profiles (harnesses=[${harnesses.join(", ")}], models=[${models.join(", ")}]).`
2524
- );
2525
- }
2526
- return out;
2527
- }
2528
- function harnessAxisOf(profile) {
2529
- const m = profile.metadata;
2530
- const harness = m?.harness;
2531
- const model = m?.harnessModel;
2532
- if (typeof harness === "string" && typeof model === "string") {
2533
- return { harness, model };
2534
- }
2535
- return void 0;
2536
- }
2537
- function agentProfileId(profile) {
2538
- const label = pathSafeProfileLabel(agentProfileDisplayLabel(profile)) ?? "profile";
2539
- return `${label}-${agentProfileHash(profile).slice(0, 16)}`;
2540
- }
2541
- function agentProfileModelId(profile) {
2542
- const model = profile.model?.default?.trim();
2543
- if (!model) {
2544
- const label = agentProfileDisplayLabel(profile) ?? "unnamed profile";
2545
- throw new ValidationError(
2546
- `AgentProfile "${label}" has no model.default \u2014 cannot record eval run`
2547
- );
2548
- }
2549
- return model;
2550
- }
2551
- function agentProfileDisplayLabel(profile) {
2552
- return profile.name?.trim() || profile.version?.trim() || void 0;
2553
- }
2554
- function pathSafeProfileLabel(label) {
2555
- const safe = label?.trim().replace(/[^A-Za-z0-9._-]+/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "");
2556
- return safe || void 0;
2557
- }
2558
- function compact(input) {
2559
- const out = {};
2560
- for (const [key, value] of Object.entries(input)) {
2561
- if (value !== void 0) out[key] = value;
2562
- }
2563
- return out;
2564
- }
2565
- function agentProfileHash(profile) {
2566
- const model = agentProfileModelId(profile);
2567
- const behaviour = {
2568
- ...profile,
2569
- name: void 0,
2570
- description: void 0,
2571
- tags: profile.tags ? [...profile.tags].sort() : void 0,
2572
- model: compact({ ...profile.model, default: model })
2573
- };
2574
- return createHash4("sha256").update(JSON.stringify(canonicalize(behaviour))).digest("hex");
2575
- }
2576
-
2577
- // src/campaign/presets/run-profile-matrix.ts
2578
- var ProfileMatrixError = class extends AgentEvalError {
2579
- constructor(message) {
2580
- super("profile_matrix", message);
2581
- }
2582
- };
2583
- function sanitize(id) {
2584
- return id.replace(/[^a-zA-Z0-9_-]/g, "_");
2585
- }
2586
- function sha(input) {
2587
- return createHash5("sha256").update(JSON.stringify(input)).digest("hex");
2588
- }
2589
- function mean2(xs) {
2590
- return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length;
2591
- }
2592
- function requireResolvedModel(cell, profileId) {
2593
- const resolved = cell.resolvedModel?.trim();
2594
- if (!resolved) {
2595
- throw new ProfileMatrixError(
2596
- `profile '${profileId}' declared the '${HARNESS_NATIVE_MODEL}' runtime-resolved model but its dispatch reported no resolved model for cell '${cell.cellId}' \u2014 return it in the ctx.cost.runPaidCall receipt so the RunRecord pins the real model (never records '${HARNESS_NATIVE_MODEL}')`
2597
- );
2598
- }
2599
- if (!modelHasSnapshot(resolved)) {
2600
- throw new ProfileMatrixError(
2601
- `profile '${profileId}' resolved to model '${resolved}' for cell '${cell.cellId}', which lacks a snapshot version \u2014 pin it (name@YYYY-MM-DD or name-YYYYMMDD) in the paid-call receipt`
2602
- );
2603
- }
2604
- return resolved;
2605
- }
2606
- function buildRunRecord(args) {
2607
- const { cell, profile, profileHash, configHash, experimentId, splitTag, commitSha, matrixId } = args;
2608
- const profileId = agentProfileId(profile);
2609
- const declaredModel = agentProfileModelId(profile);
2610
- const model = declaredModel === HARNESS_NATIVE_MODEL ? requireResolvedModel(cell, profileId) : declaredModel;
2611
- const record = campaignCellToRunRecord(cell, {
2612
- runId: `${matrixId}:${profileId}:${cell.cellId}`,
2613
- experimentId,
2614
- candidateId: profileId,
2615
- model,
2616
- promptHash: profileHash,
2617
- configHash,
2618
- commitSha,
2619
- splitTag,
2620
- agentProfile: args.agentProfileCell
2621
- });
2622
- if (args.corpusText && args.scenario) {
2623
- try {
2624
- const text = args.corpusText(cell.artifact, args.scenario);
2625
- if (text && typeof text.prompt === "string" && typeof text.completion === "string") {
2626
- record.prompt = text.prompt;
2627
- record.completion = text.completion;
2628
- }
2629
- } catch {
2630
- }
2631
- }
2632
- return record;
2633
- }
2634
- async function runProfileMatrix(opts) {
2635
- if (opts.profiles.length === 0) throw new ProfileMatrixError("profiles must not be empty");
2636
- if (opts.scenarios.length === 0) throw new ProfileMatrixError("scenarios must not be empty");
2637
- const splitTag = opts.splitTag ?? "search";
2638
- const seed = opts.seed ?? 42;
2639
- const validate = opts.validate ?? true;
2640
- const integrityMode = opts.integrity ?? "assert";
2641
- const profileIds = opts.profiles.map(agentProfileId);
2642
- const experimentId = opts.experimentId ?? `pm_${sha({ profileIds, scenarios: opts.scenarios.map((s) => s.id) }).slice(0, 16)}`;
2643
- const matrixId = `mtx_${sha({ experimentId, profileIds, seed, splitTag }).slice(0, 16)}`;
2644
- const scenarioById = new Map(opts.scenarios.map((s) => [s.id, s]));
2645
- for (const profile of opts.profiles) {
2646
- const profileHash = agentProfileHash(profile);
2647
- const profileId = agentProfileId(profile);
2648
- const declaredModel = agentProfileModelId(profile);
2649
- const model = declaredModel === HARNESS_NATIVE_MODEL ? `${HARNESS_NATIVE_MODEL}@runtime-resolved` : declaredModel;
2650
- try {
2651
- validateRunRecord({
2652
- runId: `${matrixId}:${profileId}:probe`,
2653
- experimentId,
2654
- candidateId: profileId,
2655
- seed,
2656
- model,
2657
- promptHash: profileHash,
2658
- configHash: profileHash,
2659
- commitSha: opts.commitSha,
2660
- wallMs: 0,
2661
- costUsd: null,
2662
- costProvenance: { kind: "uncaptured", usd: null },
2663
- tokenUsage: { input: 0, output: 0 },
2664
- terminalOutcome: "succeeded",
2665
- outcome: { raw: { execution_error_count: 0 } },
2666
- splitTag,
2667
- scenarioId: "recordability-probe"
2668
- });
2669
- } catch (err) {
2670
- throw new ProfileMatrixError(
2671
- `profile '${profileId}' is not recordable: ${err instanceof Error ? err.message : String(err)}`
2672
- );
2673
- }
2674
- }
2675
- const records = [];
2676
- const campaigns = {};
2677
- const byProfile = {};
2678
- for (const profile of opts.profiles) {
2679
- const profileHash = agentProfileHash(profile);
2680
- const profileId = agentProfileId(profile);
2681
- const declaredModel = agentProfileModelId(profile);
2682
- const configHash = sha({
2683
- profile: profileHash,
2684
- judges: (opts.judges ?? []).map((j) => j.name),
2685
- seed,
2686
- splitTag
2687
- });
2688
- const dispatch = (scenario, ctx) => opts.dispatch(profile, scenario, ctx);
2689
- Object.defineProperty(dispatch, "name", { value: `profile_${sanitize(profileId)}` });
2690
- const campaign = await runCampaign({
2691
- scenarios: opts.scenarios,
2692
- dispatch,
2693
- judges: opts.judges,
2694
- seed,
2695
- reps: opts.reps,
2696
- maxConcurrency: opts.maxConcurrency,
2697
- costCeiling: opts.costCeiling,
2698
- labeledStore: opts.labeledStore,
2699
- captureSource: opts.captureSource,
2700
- storage: opts.storage,
2701
- now: opts.now,
2702
- runDir: join3(opts.runDir, sanitize(profileId))
2703
- });
2704
- const axis = harnessAxisOf(profile);
2705
- const buildCellIdentity = (cellModel) => buildAgentProfileCell({
2706
- profileId,
2707
- sourceProfile: { kind: "agent-interface-profile", hash: profileHash },
2708
- model: cellModel,
2709
- ...axis ? { harness: { id: axis.harness } } : {}
2710
- });
2711
- const sharedCellIdentity = declaredModel === HARNESS_NATIVE_MODEL ? void 0 : await buildCellIdentity(declaredModel);
2712
- const profileRecords = [];
2713
- for (const cell of campaign.cells) {
2714
- const agentProfileCell = sharedCellIdentity ?? await buildCellIdentity(requireResolvedModel(cell, profileId));
2715
- const record = buildRunRecord({
2716
- cell,
2717
- profile,
2718
- profileHash,
2719
- configHash,
2720
- experimentId,
2721
- splitTag,
2722
- commitSha: opts.commitSha,
2723
- matrixId,
2724
- agentProfileCell,
2725
- scenario: scenarioById.get(cell.scenarioId),
2726
- corpusText: opts.corpusText
2727
- });
2728
- if (validate) validateRunRecord(record);
2729
- profileRecords.push(record);
2730
- records.push(record);
2731
- }
2732
- const totalCostUsd = campaign.aggregates.totalCostUsd;
2733
- campaigns[profileId] = campaign;
2734
- byProfile[profileId] = {
2735
- profileId,
2736
- profileHash,
2737
- // The declared model, unless it snapped to the sentinel — then the
2738
- // resolved model the cells actually ran on (all cells of a profile share
2739
- // one harness, so the first record's model is representative).
2740
- model: declaredModel === HARNESS_NATIVE_MODEL ? profileRecords[0]?.model ?? declaredModel : declaredModel,
2741
- records: profileRecords.length,
2742
- meanComposite: meanOrNull2(
2743
- profileRecords.map(scoreOf).filter((score) => score !== void 0)
2744
- ),
2745
- totalCostUsd,
2746
- integrity: summarizeBackendIntegrity(profileRecords)
2747
- };
2748
- }
2749
- const integrity = summarizeBackendIntegrity(records);
2750
- if (integrityMode === "assert") {
2751
- assertRealBackend(records, { allowMixed: opts.allowMixed ?? true });
2752
- } else if (integrityMode === "warn" && integrity.verdict !== "real") {
2753
- console.warn(
2754
- `[runProfileMatrix] backend integrity: ${integrity.verdict} \u2014 ${integrity.diagnosis}`
2755
- );
2756
- }
2757
- const byScenario = rollup(records, (r) => r.scenarioId);
2758
- const byPersona = opts.personaOf ? rollupByPersona(records, opts.scenarios, opts.personaOf) : void 0;
2759
- return { matrixId, experimentId, records, byProfile, byScenario, byPersona, integrity, campaigns };
2760
- }
2761
- function scoreOf(r) {
2762
- return runTaskScore(r);
2763
- }
2764
- function meanOrNull2(values) {
2765
- return values.length === 0 ? null : mean2(values);
2766
- }
2767
- function rollup(records, keyOf) {
2768
- const groups = /* @__PURE__ */ new Map();
2769
- for (const r of records) {
2770
- const key = keyOf(r);
2771
- if (key === void 0) continue;
2772
- const score = scoreOf(r);
2773
- if (score === void 0) continue;
2774
- const arr = groups.get(key) ?? [];
2775
- arr.push(score);
2776
- groups.set(key, arr);
2777
- }
2778
- const out = {};
2779
- for (const [key, xs] of groups) out[key] = { meanComposite: mean2(xs), n: xs.length };
2780
- return out;
2781
- }
2782
- function rollupByPersona(records, scenarios, personaOf) {
2783
- const personaByScenarioId = /* @__PURE__ */ new Map();
2784
- for (const s of scenarios) personaByScenarioId.set(s.id, personaOf(s));
2785
- return rollup(records, (r) => r.scenarioId ? personaByScenarioId.get(r.scenarioId) : void 0);
2786
- }
2787
-
2788
- // src/campaign/scenario-selection.ts
2789
- var DEFAULT_SATURATION_CEILING = 0.999;
2790
- var VARIANCE_EPSILON = 1e-9;
2791
- function moments(scores) {
2792
- const n = scores.length;
2793
- if (n === 0) return { meanScore: 0, variance: 0 };
2794
- let sum = 0;
2795
- for (const s of scores) sum += s;
2796
- const meanScore = sum / n;
2797
- let sqDev = 0;
2798
- for (const s of scores) {
2799
- const d = s - meanScore;
2800
- sqDev += d * d;
2801
- }
2802
- return { meanScore, variance: sqDev / n };
2803
- }
2804
- function compareDiscrimination(a, b) {
2805
- if (b.discrimination !== a.discrimination) return b.discrimination - a.discrimination;
2806
- if (a.meanScore !== b.meanScore) return a.meanScore - b.meanScore;
2807
- return a.scenarioId < b.scenarioId ? -1 : a.scenarioId > b.scenarioId ? 1 : 0;
2808
- }
2809
- function scoreDiscrimination(signals, opts) {
2810
- const saturationCeiling = opts?.saturationCeiling ?? DEFAULT_SATURATION_CEILING;
2811
- const scored = signals.map((signal) => {
2812
- const { meanScore, variance } = moments(signal.scores);
2813
- const tied = variance < VARIANCE_EPSILON && meanScore >= saturationCeiling;
2814
- return {
2815
- scenarioId: signal.scenarioId,
2816
- discrimination: variance,
2817
- meanScore,
2818
- variance,
2819
- tied
2820
- };
2821
- });
2822
- return scored.sort(compareDiscrimination);
2823
- }
2824
- function selectDiscriminative(signals, k, opts) {
2825
- if (k < 1) throw new Error(`selectDiscriminative: k must be >= 1 (got ${k})`);
2826
- const ranked = scoreDiscrimination(signals, opts);
2827
- if (ranked.length <= k) return ranked.map((s) => s.scenarioId);
2828
- const nonTied = ranked.filter((s) => !s.tied);
2829
- if (nonTied.length >= k) return nonTied.slice(0, k).map((s) => s.scenarioId);
2830
- const tied = ranked.filter((s) => s.tied);
2831
- const fill = tied.slice(0, k - nonTied.length);
2832
- return [...nonTied, ...fill].map((s) => s.scenarioId);
2833
- }
2834
-
2835
- // src/campaign/search-ledger.ts
2836
- import { createHash as createHash6 } from "crypto";
2837
- import { existsSync as existsSync3, readFileSync as readFileSync3 } from "fs";
2838
- import { resolve as resolve2 } from "path";
2839
- import { z } from "zod";
2840
- var SEARCH_LEDGER_SCHEMA = "tangle.search-ledger.v1";
2841
- var NON_EMPTY = z.string().min(1).refine((value) => value.trim() === value, "must not contain surrounding whitespace");
2842
- var HASH = z.string().regex(/^sha256:[a-f0-9]{64}$/);
2843
- var LINEAGE_NODE_ID = z.string().regex(/^[a-f0-9]{16}$/);
2844
- var IMMUTABLE_REVISION = z.string().regex(/^(?:[a-f0-9]{40}|[a-f0-9]{64}|sha256:[a-f0-9]{64}|sha512:[A-Za-z0-9+/=]+)$/);
2845
- var ISO_TIMESTAMP = z.string().regex(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{3})?Z$/).refine((value) => Number.isFinite(Date.parse(value)), "invalid timestamp");
2846
- var NON_NEGATIVE_INT = z.number().int().nonnegative().safe();
2847
- var FINITE_NUMBER = z.number().finite();
2848
- var ArtifactRefSchema = z.object({
2849
- role: NON_EMPTY,
2850
- uri: NON_EMPTY,
2851
- sha256: HASH,
2852
- byteLength: NON_NEGATIVE_INT
2853
- }).strict();
2854
- var SourceRefSchema = z.object({
2855
- uri: NON_EMPTY,
2856
- revision: IMMUTABLE_REVISION
2857
- }).strict();
2858
- var FailureReasonSchema = z.object({
2859
- code: NON_EMPTY,
2860
- message: NON_EMPTY
2861
- }).strict();
2862
- var EventBaseShape = {
2863
- eventId: NON_EMPTY,
2864
- occurredAt: ISO_TIMESTAMP,
2865
- artifacts: z.array(ArtifactRefSchema).min(1)
2866
- };
2867
- var OperationKindSchema = z.enum([
2868
- "candidate-generation",
2869
- "analysis",
2870
- "selection",
2871
- "judge",
2872
- "other"
2873
- ]);
2874
- var SearchPlannedSchema = z.object({
2875
- ...EventBaseShape,
2876
- kind: z.literal("search-planned"),
2877
- plan: z.object({
2878
- candidateSlots: z.array(
2879
- z.object({
2880
- slotId: NON_EMPTY,
2881
- generationOperationId: NON_EMPTY
2882
- }).strict()
2883
- ).min(1),
2884
- tasks: z.array(
2885
- z.object({
2886
- taskId: NON_EMPTY,
2887
- source: SourceRefSchema,
2888
- benchmark: SourceRefSchema,
2889
- maxAttempts: z.number().int().positive().safe()
2890
- }).strict()
2891
- ).min(1),
2892
- operations: z.array(
2893
- z.object({
2894
- operationId: NON_EMPTY,
2895
- kind: OperationKindSchema
2896
- }).strict()
2897
- ).min(1)
2898
- }).strict()
2899
- }).strict();
2900
- var CandidateRegisteredSchema = z.object({
2901
- ...EventBaseShape,
2902
- kind: z.literal("candidate-registered"),
2903
- slotId: NON_EMPTY,
2904
- generationOperationId: NON_EMPTY,
2905
- candidateId: NON_EMPTY,
2906
- lineage: z.object({
2907
- lineageNodeId: LINEAGE_NODE_ID,
2908
- parentCandidateIds: z.array(NON_EMPTY),
2909
- generation: NON_NEGATIVE_INT,
2910
- proposer: NON_EMPTY,
2911
- proposerSource: SourceRefSchema
2912
- }).strict(),
2913
- surfaces: z.array(
2914
- z.object({
2915
- surfaceId: NON_EMPTY,
2916
- kind: z.enum([
2917
- "prompt",
2918
- "tool-contract",
2919
- "runtime-config",
2920
- "memory",
2921
- "knowledge",
2922
- "agent-profile",
2923
- "code",
2924
- "deployment"
2925
- ]),
2926
- artifact: ArtifactRefSchema
2927
- }).strict()
2928
- ).min(1)
2929
- }).strict();
2930
- var CandidateSlotClosedSchema = z.object({
2931
- ...EventBaseShape,
2932
- kind: z.literal("candidate-slot-closed"),
2933
- slotId: NON_EMPTY,
2934
- generationOperationId: NON_EMPTY,
2935
- reason: FailureReasonSchema
2936
- }).strict();
2937
- var KnownTokensSchema = z.object({
2938
- status: z.literal("known"),
2939
- inputTokens: NON_NEGATIVE_INT,
2940
- outputTokens: NON_NEGATIVE_INT,
2941
- cachedTokens: NON_NEGATIVE_INT
2942
- }).strict();
2943
- var UnknownSchema = z.object({
2944
- status: z.literal("unknown"),
2945
- reason: NON_EMPTY
2946
- }).strict();
2947
- var KnownCostSchema = z.object({
2948
- status: z.literal("known"),
2949
- usd: z.number().finite().nonnegative(),
2950
- source: z.enum(["provider", "pricing-table", "free"])
2951
- }).strict().superRefine((cost, ctx) => {
2952
- if (cost.source === "free" && cost.usd !== 0) {
2953
- ctx.addIssue({ code: "custom", message: "free cost source must have usd 0" });
2954
- }
2955
- });
2956
- var UnknownCostSchema = z.object({
2957
- status: z.literal("unknown"),
2958
- knownLowerBoundUsd: z.number().finite().nonnegative(),
2959
- reason: NON_EMPTY
2960
- }).strict();
2961
- var AccountingSchema = z.object({
2962
- tokens: z.discriminatedUnion("status", [KnownTokensSchema, UnknownSchema]),
2963
- cost: z.discriminatedUnion("status", [KnownCostSchema, UnknownCostSchema])
2964
- }).strict();
2965
- var MetricsSchema = z.record(NON_EMPTY, FINITE_NUMBER).superRefine((metrics, ctx) => {
2966
- for (const key of Object.keys(metrics)) {
2967
- if (key === "__proto__" || key === "constructor" || key === "prototype") {
2968
- ctx.addIssue({ code: "custom", message: `unsafe metric key ${key}` });
2969
- }
2970
- }
2971
- });
2972
- var OutcomeSchema = z.discriminatedUnion("status", [
2973
- z.object({
2974
- status: z.literal("passed"),
2975
- score: FINITE_NUMBER,
2976
- metrics: MetricsSchema
2977
- }).strict(),
2978
- z.object({
2979
- status: z.literal("failed"),
2980
- score: FINITE_NUMBER,
2981
- metrics: MetricsSchema,
2982
- failure: FailureReasonSchema
2983
- }).strict(),
2984
- z.object({
2985
- status: z.literal("errored"),
2986
- metrics: MetricsSchema,
2987
- error: FailureReasonSchema.extend({ retryable: z.boolean() }).strict()
2988
- }).strict()
2989
- ]);
2990
- var EffectSchema = z.discriminatedUnion("status", [
2991
- z.object({
2992
- status: z.literal("measured"),
2993
- metric: NON_EMPTY,
2994
- baselineValue: FINITE_NUMBER,
2995
- candidateValue: FINITE_NUMBER,
2996
- delta: FINITE_NUMBER
2997
- }).strict().superRefine((effect, ctx) => {
2998
- const expected = effect.candidateValue - effect.baselineValue;
2999
- const tolerance = Number.EPSILON * Math.max(1, Math.abs(expected), Math.abs(effect.delta)) * 8;
3000
- if (Math.abs(effect.delta - expected) > tolerance) {
3001
- ctx.addIssue({ code: "custom", message: "delta must equal candidateValue - baselineValue" });
3002
- }
3003
- }),
3004
- z.object({
3005
- status: z.literal("not-measured"),
3006
- reason: NON_EMPTY
3007
- }).strict()
3008
- ]);
3009
- var SurfaceEvidenceSchema = z.object({
3010
- surfaceId: NON_EMPTY,
3011
- fired: z.boolean(),
3012
- firingCount: NON_NEGATIVE_INT,
3013
- effect: EffectSchema,
3014
- evidence: z.array(ArtifactRefSchema).min(1)
3015
- }).strict().superRefine((evidence, ctx) => {
3016
- if (evidence.fired && evidence.firingCount === 0) {
3017
- ctx.addIssue({ code: "custom", message: "a fired surface must have firingCount >= 1" });
3018
- }
3019
- if (!evidence.fired && evidence.firingCount !== 0) {
3020
- ctx.addIssue({
3021
- code: "custom",
3022
- message: "a surface that did not fire must have firingCount 0"
3023
- });
3024
- }
3025
- if (!evidence.fired && evidence.effect.status === "measured" && evidence.effect.delta !== 0) {
3026
- ctx.addIssue({
3027
- code: "custom",
3028
- message: "a surface that did not fire cannot claim non-zero effect"
3029
- });
3030
- }
3031
- });
3032
- var TaskAttemptedSchema = z.object({
3033
- ...EventBaseShape,
3034
- kind: z.literal("task-attempted"),
3035
- candidateId: NON_EMPTY,
3036
- runId: NON_EMPTY,
3037
- attemptIndex: NON_NEGATIVE_INT,
3038
- task: z.object({ taskId: NON_EMPTY, source: SourceRefSchema }).strict(),
3039
- identity: z.object({
3040
- model: z.object({
3041
- provider: NON_EMPTY,
3042
- snapshot: NON_EMPTY.refine(
3043
- modelHasSnapshot,
3044
- "model must include an immutable snapshot"
3045
- )
3046
- }).strict(),
3047
- agent: SourceRefSchema,
3048
- benchmark: SourceRefSchema
3049
- }).strict(),
3050
- outcome: OutcomeSchema,
3051
- accounting: AccountingSchema,
3052
- surfaceEvidence: z.array(SurfaceEvidenceSchema).min(1)
3053
- }).strict();
3054
- var SearchOperationRecordedSchema = z.object({
3055
- ...EventBaseShape,
3056
- kind: z.literal("search-operation-recorded"),
3057
- operationId: NON_EMPTY,
3058
- operationKind: OperationKindSchema,
3059
- execution: z.discriminatedUnion("kind", [
3060
- z.object({
3061
- kind: z.literal("model"),
3062
- model: z.object({
3063
- provider: NON_EMPTY,
3064
- snapshot: NON_EMPTY.refine(
3065
- modelHasSnapshot,
3066
- "model must include an immutable snapshot"
3067
- )
3068
- }).strict(),
3069
- source: SourceRefSchema
3070
- }).strict(),
3071
- z.object({
3072
- kind: z.literal("deterministic"),
3073
- source: SourceRefSchema
3074
- }).strict()
3075
- ]),
3076
- outcome: z.discriminatedUnion("status", [
3077
- z.object({ status: z.literal("completed") }).strict(),
3078
- z.object({
3079
- status: z.literal("partial"),
3080
- failure: FailureReasonSchema
3081
- }).strict(),
3082
- z.object({
3083
- status: z.literal("failed"),
3084
- failure: FailureReasonSchema
3085
- }).strict()
3086
- ]),
3087
- accounting: AccountingSchema
3088
- }).strict();
3089
- var CandidateDecidedSchema = z.object({
3090
- ...EventBaseShape,
3091
- kind: z.literal("candidate-decided"),
3092
- candidateId: NON_EMPTY,
3093
- decision: z.discriminatedUnion("status", [
3094
- z.object({ status: z.literal("selected") }).strict(),
3095
- z.object({
3096
- status: z.literal("rejected"),
3097
- reason: FailureReasonSchema
3098
- }).strict()
3099
- ])
3100
- }).strict();
3101
- var SearchCompletedSchema = z.object({
3102
- ...EventBaseShape,
3103
- kind: z.literal("search-completed"),
3104
- result: z.discriminatedUnion("status", [
3105
- z.object({
3106
- status: z.literal("selected"),
3107
- candidateId: NON_EMPTY
3108
- }).strict(),
3109
- z.object({
3110
- status: z.literal("all-rejected"),
3111
- reason: FailureReasonSchema
3112
- }).strict()
3113
- ])
3114
- }).strict();
3115
- var EventSchema = z.discriminatedUnion("kind", [
3116
- SearchPlannedSchema,
3117
- CandidateRegisteredSchema,
3118
- CandidateSlotClosedSchema,
3119
- TaskAttemptedSchema,
3120
- SearchOperationRecordedSchema,
3121
- CandidateDecidedSchema,
3122
- SearchCompletedSchema
3123
- ]);
3124
- var EntrySchema = z.object({
3125
- schema: z.literal(SEARCH_LEDGER_SCHEMA),
3126
- campaignId: NON_EMPTY,
3127
- sequence: NON_NEGATIVE_INT,
3128
- previousHash: z.union([HASH, z.null()]),
3129
- event: EventSchema,
3130
- entryHash: HASH
3131
- }).strict();
3132
- function validateSearchLedgerEvent(input) {
3133
- const parsed = EventSchema.safeParse(input);
3134
- if (!parsed.success) {
3135
- throw new SearchLedgerError(`invalid search ledger event: ${formatZodError(parsed.error)}`);
3136
- }
3137
- return normalizeEvent(parsed.data);
3138
- }
3139
- function openSearchLedger(options) {
3140
- if (options.path.trim().length === 0) throw new SearchLedgerError("ledger path is empty");
3141
- return new FileSearchLedger(options.path, options.campaignId);
3142
- }
3143
- var ledgerMutexes = /* @__PURE__ */ new Map();
3144
- function mutexFor(path) {
3145
- const existing = ledgerMutexes.get(path);
3146
- if (existing) return existing;
3147
- const mutex = new Mutex();
3148
- ledgerMutexes.set(path, mutex);
3149
- return mutex;
3150
- }
3151
- var FileSearchLedger = class {
3152
- path;
3153
- campaignId;
3154
- mutex;
3155
- constructor(path, campaignId) {
3156
- if (path.trim().length === 0) throw new SearchLedgerError("ledger path is empty");
3157
- if (campaignId.length === 0) throw new SearchLedgerError("campaignId is empty");
3158
- if (campaignId.trim() !== campaignId) {
3159
- throw new SearchLedgerError("campaignId must not contain surrounding whitespace");
3160
- }
3161
- this.path = resolve2(path);
3162
- this.campaignId = campaignId;
3163
- this.mutex = mutexFor(this.path);
3164
- }
3165
- async replay() {
3166
- return this.mutex.runExclusive(
3167
- () => withSearchLedgerFileLock(this.path, () => replayFile(this.path, this.campaignId))
3168
- );
3169
- }
3170
- async append(input) {
3171
- const event = validateSearchLedgerEvent(input);
3172
- return this.mutex.runExclusive(
3173
- () => withSearchLedgerFileLock(this.path, () => {
3174
- const before = replayFile(this.path, this.campaignId);
3175
- const existing = before.entries.find((entry2) => entry2.event.eventId === event.eventId);
3176
- if (existing) {
3177
- if (canonicalString(existing.event) !== canonicalString(event)) {
3178
- throw new SearchLedgerConflictError(
3179
- `eventId ${event.eventId} already exists with different content`
3180
- );
3181
- }
3182
- return { entry: existing, appended: false, replay: before };
3183
- }
3184
- const previousHash = before.audit.headHash;
3185
- const material = {
3186
- schema: SEARCH_LEDGER_SCHEMA,
3187
- campaignId: this.campaignId,
3188
- sequence: before.entries.length,
3189
- previousHash,
3190
- event
3191
- };
3192
- const entry = {
3193
- ...material,
3194
- entryHash: hashCanonical(material)
3195
- };
3196
- const replay = replayEntries([...before.entries, entry], this.campaignId);
3197
- appendSearchLedgerLine(this.path, `${canonicalString(entry)}
3198
- `);
3199
- return { entry, appended: true, replay };
3200
- })
3201
- );
3202
- }
3203
- };
3204
- function replayFile(path, campaignId) {
3205
- if (!existsSync3(path)) return replayEntries([], campaignId);
3206
- const text = readFileSync3(path, "utf8");
3207
- if (text.length === 0) return replayEntries([], campaignId);
3208
- if (!text.endsWith("\n")) {
3209
- throw new SearchLedgerIntegrityError(
3210
- `search ledger ${path} has a truncated final record (missing newline)`
3211
- );
3212
- }
3213
- const lines = text.slice(0, -1).split("\n");
3214
- const entries = [];
3215
- for (let index = 0; index < lines.length; index += 1) {
3216
- const line = lines[index];
3217
- if (line.length === 0) {
3218
- throw new SearchLedgerIntegrityError(
3219
- `search ledger ${path} has a blank row at line ${index + 1}`
3220
- );
3221
- }
3222
- let raw;
3223
- try {
3224
- raw = JSON.parse(line);
3225
- } catch (error) {
3226
- throw new SearchLedgerIntegrityError(
3227
- `search ledger ${path} has invalid JSON at line ${index + 1}`,
3228
- { cause: error }
3229
- );
3230
- }
3231
- const parsed = EntrySchema.safeParse(raw);
3232
- if (!parsed.success) {
3233
- throw new SearchLedgerIntegrityError(
3234
- `search ledger ${path} has a malformed entry at line ${index + 1}: ${formatZodError(parsed.error)}`
3235
- );
3236
- }
3237
- const entry = parsed.data;
3238
- const normalizedEvent = validateSearchLedgerEvent(entry.event);
3239
- if (canonicalString(normalizedEvent) !== canonicalString(entry.event)) {
3240
- throw new SearchLedgerIntegrityError(
3241
- `search ledger ${path} has non-canonical event ordering at line ${index + 1}`
3242
- );
3243
- }
3244
- if (line !== canonicalString(entry)) {
3245
- throw new SearchLedgerIntegrityError(
3246
- `search ledger ${path} has non-canonical bytes at line ${index + 1}`
3247
- );
3248
- }
3249
- entries.push(entry);
3250
- }
3251
- return replayEntries(entries, campaignId);
3252
- }
3253
- function replayEntries(entries, campaignId) {
3254
- const candidates = /* @__PURE__ */ new Map();
3255
- const candidateBySlot = /* @__PURE__ */ new Map();
3256
- const closedSlots = /* @__PURE__ */ new Map();
3257
- const lineageNodes = /* @__PURE__ */ new Map();
3258
- const eventIds = /* @__PURE__ */ new Set();
3259
- const runIds = /* @__PURE__ */ new Set();
3260
- const attemptKeys = /* @__PURE__ */ new Set();
3261
- const candidateEvents = [];
3262
- const closedSlotEvents = [];
3263
- const attempts = [];
3264
- const operationEvents = [];
3265
- const operationsById = /* @__PURE__ */ new Map();
3266
- const decisions = [];
3267
- let planEvent = null;
3268
- let completion = null;
3269
- let expectedPrevious = null;
3270
- let previousOccurredAt = Number.NEGATIVE_INFINITY;
3271
- for (let index = 0; index < entries.length; index += 1) {
3272
- const entry = entries[index];
3273
- if (entry.campaignId !== campaignId) {
3274
- throw new SearchLedgerIntegrityError(
3275
- `entry ${index} belongs to campaign ${entry.campaignId}, expected ${campaignId}`
3276
- );
3277
- }
3278
- if (entry.sequence !== index) {
3279
- throw new SearchLedgerIntegrityError(
3280
- `entry ${entry.event.eventId} has sequence ${entry.sequence}, expected ${index}`
3281
- );
3282
- }
3283
- if (entry.previousHash !== expectedPrevious) {
3284
- throw new SearchLedgerIntegrityError(
3285
- `entry ${entry.event.eventId} does not extend the previous hash`
3286
- );
3287
- }
3288
- const { entryHash: _entryHash, ...material } = entry;
3289
- const expectedHash = hashCanonical(material);
3290
- if (entry.entryHash !== expectedHash) {
3291
- throw new SearchLedgerIntegrityError(
3292
- `entry ${entry.event.eventId} hash mismatch: expected ${expectedHash}, got ${entry.entryHash}`
3293
- );
3294
- }
3295
- expectedPrevious = entry.entryHash;
3296
- const event = entry.event;
3297
- if (eventIds.has(event.eventId)) {
3298
- throw new SearchLedgerIntegrityError(`duplicate eventId ${event.eventId} in durable ledger`);
3299
- }
3300
- eventIds.add(event.eventId);
3301
- if (completion) {
3302
- throw new SearchLedgerIntegrityError(
3303
- `event ${event.eventId} appears after terminal event ${completion.eventId}`
3304
- );
3305
- }
3306
- const occurredAt = Date.parse(event.occurredAt);
3307
- if (occurredAt < previousOccurredAt) {
3308
- throw new SearchLedgerIntegrityError(
3309
- `event ${event.eventId} occurred before the preceding durable event`
3310
- );
3311
- }
3312
- previousOccurredAt = occurredAt;
3313
- assertUnique2(event.artifacts.map(artifactKey), "artifact receipt", event.eventId);
3314
- if (event.kind === "search-planned") {
3315
- if (index !== 0 || planEvent) {
3316
- throw new SearchLedgerIntegrityError("search plan must be the first and only plan event");
3317
- }
3318
- assertUnique2(
3319
- event.plan.candidateSlots.map((slot) => slot.slotId),
3320
- "candidate slot",
3321
- event.eventId
3322
- );
3323
- assertUnique2(
3324
- event.plan.tasks.map((task) => task.taskId),
3325
- "planned taskId",
3326
- event.eventId
3327
- );
3328
- assertUnique2(
3329
- event.plan.operations.map((operation) => operation.operationId),
3330
- "planned operationId",
3331
- event.eventId
3332
- );
3333
- for (const slot of event.plan.candidateSlots) {
3334
- const generationOperation = event.plan.operations.find(
3335
- (operation) => operation.operationId === slot.generationOperationId
3336
- );
3337
- if (generationOperation?.kind !== "candidate-generation") {
3338
- throw new SearchLedgerIntegrityError(
3339
- `candidate slot ${slot.slotId} references unplanned candidate-generation operation ${slot.generationOperationId}`
3340
- );
3341
- }
3342
- }
3343
- planEvent = event;
3344
- continue;
3345
- }
3346
- if (!planEvent) {
3347
- throw new SearchLedgerIntegrityError(
3348
- `event ${event.eventId} appears before the required search plan`
3349
- );
3350
- }
3351
- if (event.kind === "candidate-registered") {
3352
- if (candidates.has(event.candidateId)) {
3353
- throw new SearchLedgerIntegrityError(`candidate ${event.candidateId} was registered twice`);
3354
- }
3355
- const plannedSlot = planEvent.plan.candidateSlots.find((slot) => slot.slotId === event.slotId);
3356
- if (!plannedSlot) {
3357
- throw new SearchLedgerIntegrityError(
3358
- `candidate ${event.candidateId} binds unknown slot ${event.slotId}`
3359
- );
3360
- }
3361
- if (candidateBySlot.has(event.slotId)) {
3362
- throw new SearchLedgerIntegrityError(`candidate slot ${event.slotId} was bound twice`);
3363
- }
3364
- if (closedSlots.has(event.slotId)) {
3365
- throw new SearchLedgerIntegrityError(`candidate slot ${event.slotId} was already closed`);
3366
- }
3367
- if (event.generationOperationId !== plannedSlot.generationOperationId) {
3368
- throw new SearchLedgerIntegrityError(
3369
- `candidate ${event.candidateId} generation operation ${event.generationOperationId} does not match slot ${event.slotId} plan ${plannedSlot.generationOperationId}`
3370
- );
3371
- }
3372
- const generationOperation = operationsById.get(event.generationOperationId);
3373
- if (!generationOperation) {
3374
- throw new SearchLedgerIntegrityError(
3375
- `candidate ${event.candidateId} precedes generation operation ${event.generationOperationId}`
3376
- );
3377
- }
3378
- if (generationOperation.outcome.status === "failed") {
3379
- throw new SearchLedgerIntegrityError(
3380
- `candidate ${event.candidateId} cannot bind failed generation operation ${event.generationOperationId}`
3381
- );
3382
- }
3383
- const previousCandidate = lineageNodes.get(event.lineage.lineageNodeId);
3384
- if (previousCandidate) {
3385
- throw new SearchLedgerIntegrityError(
3386
- `lineage node ${event.lineage.lineageNodeId} is already bound to ${previousCandidate}`
3387
- );
3388
- }
3389
- assertUnique2(event.lineage.parentCandidateIds, "parentCandidateId", event.eventId);
3390
- const parents = event.lineage.parentCandidateIds.map((id) => {
3391
- const parent = candidates.get(id);
3392
- if (!parent) {
3393
- throw new SearchLedgerIntegrityError(
3394
- `candidate ${event.candidateId} references unknown parent ${id}`
3395
- );
3396
- }
3397
- return parent;
3398
- });
3399
- const expectedGeneration = parents.length === 0 ? 0 : Math.max(...parents.map((parent) => parent.registered.lineage.generation)) + 1;
3400
- if (event.lineage.generation !== expectedGeneration) {
3401
- throw new SearchLedgerIntegrityError(
3402
- `candidate ${event.candidateId} generation ${event.lineage.generation} does not follow its parents (expected ${expectedGeneration})`
3403
- );
3404
- }
3405
- assertUnique2(
3406
- event.surfaces.map((surface) => surface.surfaceId),
3407
- "surfaceId",
3408
- event.eventId
3409
- );
3410
- candidates.set(event.candidateId, { registered: event, attempts: [], decision: null });
3411
- candidateBySlot.set(event.slotId, event.candidateId);
3412
- lineageNodes.set(event.lineage.lineageNodeId, event.candidateId);
3413
- candidateEvents.push(event);
3414
- continue;
3415
- }
3416
- if (event.kind === "task-attempted") {
3417
- const candidate = candidates.get(event.candidateId);
3418
- if (!candidate) {
3419
- throw new SearchLedgerIntegrityError(
3420
- `attempt ${event.eventId} references unknown candidate ${event.candidateId}`
3421
- );
3422
- }
3423
- if (candidate.decision) {
3424
- throw new SearchLedgerIntegrityError(
3425
- `attempt ${event.eventId} appears after candidate ${event.candidateId} was decided`
3426
- );
3427
- }
3428
- const plannedTask = planEvent.plan.tasks.find((task) => task.taskId === event.task.taskId);
3429
- if (!plannedTask) {
3430
- throw new SearchLedgerIntegrityError(
3431
- `attempt ${event.eventId} references unplanned task ${event.task.taskId}`
3432
- );
3433
- }
3434
- if (canonicalString(plannedTask.source) !== canonicalString(event.task.source) || canonicalString(plannedTask.benchmark) !== canonicalString(event.identity.benchmark)) {
3435
- throw new SearchLedgerIntegrityError(
3436
- `task ${event.task.taskId} does not match its planned source identity`
3437
- );
3438
- }
3439
- if (event.attemptIndex >= plannedTask.maxAttempts) {
3440
- throw new SearchLedgerIntegrityError(
3441
- `task ${event.task.taskId} attempt ${event.attemptIndex} exceeds planned maxAttempts ${plannedTask.maxAttempts}`
3442
- );
3443
- }
3444
- if (runIds.has(event.runId)) {
3445
- throw new SearchLedgerIntegrityError(`runId ${event.runId} was recorded twice`);
3446
- }
3447
- runIds.add(event.runId);
3448
- const attemptKey = canonicalString([event.candidateId, event.task.taskId, event.attemptIndex]);
3449
- if (attemptKeys.has(attemptKey)) {
3450
- throw new SearchLedgerIntegrityError(
3451
- `candidate ${event.candidateId} task ${event.task.taskId} attempt ${event.attemptIndex} was recorded twice`
3452
- );
3453
- }
3454
- const expectedAttemptIndex = candidate.attempts.filter(
3455
- (attempt) => attempt.task.taskId === event.task.taskId
3456
- ).length;
3457
- if (event.attemptIndex !== expectedAttemptIndex) {
3458
- throw new SearchLedgerIntegrityError(
3459
- `candidate ${event.candidateId} task ${event.task.taskId} attempt index ${event.attemptIndex} is not contiguous (expected ${expectedAttemptIndex})`
3460
- );
3461
- }
3462
- const previousAttempt = candidate.attempts.find(
3463
- (attempt) => attempt.task.taskId === event.task.taskId
3464
- );
3465
- if (previousAttempt?.outcome.status !== void 0 && previousAttempt.outcome.status !== "errored") {
3466
- throw new SearchLedgerIntegrityError(
3467
- `task ${event.task.taskId} was retried after a measured outcome`
3468
- );
3469
- }
3470
- if (previousAttempt && canonicalString({ task: previousAttempt.task, identity: previousAttempt.identity }) !== canonicalString({ task: event.task, identity: event.identity })) {
3471
- throw new SearchLedgerIntegrityError(
3472
- `candidate ${event.candidateId} task ${event.task.taskId} changed immutable execution identity between attempts`
3473
- );
3474
- }
3475
- attemptKeys.add(attemptKey);
3476
- const declared = candidate.registered.surfaces.map((surface) => surface.surfaceId).sort();
3477
- const observed = event.surfaceEvidence.map((surface) => surface.surfaceId).sort();
3478
- assertUnique2(observed, "surface evidence", event.eventId);
3479
- for (const evidence of event.surfaceEvidence) {
3480
- assertUnique2(evidence.evidence.map(artifactKey), "surface evidence receipt", event.eventId);
3481
- }
3482
- if (canonicalString(declared) !== canonicalString(observed)) {
3483
- throw new SearchLedgerIntegrityError(
3484
- `attempt ${event.eventId} surface evidence does not exactly cover candidate ${event.candidateId}`
3485
- );
3486
- }
3487
- candidate.attempts.push(event);
3488
- attempts.push(event);
3489
- continue;
3490
- }
3491
- if (event.kind === "search-operation-recorded") {
3492
- const plannedOperation = planEvent.plan.operations.find(
3493
- (operation) => operation.operationId === event.operationId
3494
- );
3495
- if (!plannedOperation) {
3496
- throw new SearchLedgerIntegrityError(
3497
- `operation ${event.operationId} was not declared in the search plan`
3498
- );
3499
- }
3500
- if (plannedOperation.kind !== event.operationKind) {
3501
- throw new SearchLedgerIntegrityError(
3502
- `operation ${event.operationId} kind ${event.operationKind} does not match planned ${plannedOperation.kind}`
3503
- );
3504
- }
3505
- if (operationsById.has(event.operationId)) {
3506
- throw new SearchLedgerIntegrityError(`operation ${event.operationId} was recorded twice`);
3507
- }
3508
- operationsById.set(event.operationId, event);
3509
- operationEvents.push(event);
3510
- continue;
3511
- }
3512
- if (event.kind === "candidate-slot-closed") {
3513
- const plannedSlot = planEvent.plan.candidateSlots.find((slot) => slot.slotId === event.slotId);
3514
- if (!plannedSlot) {
3515
- throw new SearchLedgerIntegrityError(
3516
- `candidate slot closure ${event.eventId} references unknown slot ${event.slotId}`
3517
- );
3518
- }
3519
- if (event.generationOperationId !== plannedSlot.generationOperationId) {
3520
- throw new SearchLedgerIntegrityError(
3521
- `candidate slot closure ${event.eventId} generation operation ${event.generationOperationId} does not match slot ${event.slotId} plan ${plannedSlot.generationOperationId}`
3522
- );
3523
- }
3524
- if (candidateBySlot.has(event.slotId)) {
3525
- throw new SearchLedgerIntegrityError(
3526
- `candidate slot ${event.slotId} was already bound to a candidate`
3527
- );
3528
- }
3529
- if (closedSlots.has(event.slotId)) {
3530
- throw new SearchLedgerIntegrityError(`candidate slot ${event.slotId} was closed twice`);
3531
- }
3532
- const operation = operationsById.get(event.generationOperationId);
3533
- if (!operation) {
3534
- throw new SearchLedgerIntegrityError(
3535
- `candidate slot closure ${event.eventId} precedes operation ${event.generationOperationId}`
3536
- );
3537
- }
3538
- if (operation.outcome.status === "completed") {
3539
- throw new SearchLedgerIntegrityError(
3540
- `candidate slot ${event.slotId} cannot close from completed operation ${event.generationOperationId}`
3541
- );
3542
- }
3543
- closedSlots.set(event.slotId, event);
3544
- closedSlotEvents.push(event);
3545
- continue;
3546
- }
3547
- if (event.kind === "candidate-decided") {
3548
- const candidate = candidates.get(event.candidateId);
3549
- if (!candidate) {
3550
- throw new SearchLedgerIntegrityError(
3551
- `decision ${event.eventId} references unknown candidate ${event.candidateId}`
3552
- );
3553
- }
3554
- if (candidate.decision) {
3555
- throw new SearchLedgerIntegrityError(`candidate ${event.candidateId} was decided twice`);
3556
- }
3557
- if (event.decision.status === "selected") {
3558
- if (!candidate.attempts.some((attempt) => attempt.outcome.status !== "errored")) {
3559
- throw new SearchLedgerIntegrityError(
3560
- `candidate ${event.candidateId} cannot be selected without a measured task outcome`
3561
- );
3562
- }
3563
- if (decisions.some((decision) => decision.decision.status === "selected")) {
3564
- throw new SearchLedgerIntegrityError("more than one candidate was selected");
3565
- }
3566
- }
3567
- candidate.decision = event;
3568
- decisions.push(event);
3569
- continue;
3570
- }
3571
- const missingCandidateSlots2 = planEvent.plan.candidateSlots.filter((slot) => !candidateBySlot.has(slot.slotId) && !closedSlots.has(slot.slotId)).map((slot) => slot.slotId);
3572
- if (missingCandidateSlots2.length > 0) {
3573
- throw new SearchLedgerIntegrityError(
3574
- `search completed with missing candidate slots: ${missingCandidateSlots2.join(", ")}`
3575
- );
3576
- }
3577
- const missingTaskOutcomes2 = plannedTaskOutcomeKeys(planEvent, candidates);
3578
- if (missingTaskOutcomes2.length > 0) {
3579
- throw new SearchLedgerIntegrityError(
3580
- `search completed with missing task outcomes: ${missingTaskOutcomes2.join(", ")}`
3581
- );
3582
- }
3583
- const missingOperations2 = planEvent.plan.operations.filter((operation) => !operationsById.has(operation.operationId)).map((operation) => operation.operationId);
3584
- if (missingOperations2.length > 0) {
3585
- throw new SearchLedgerIntegrityError(
3586
- `search completed with missing search operations: ${missingOperations2.join(", ")}`
3587
- );
3588
- }
3589
- for (const operation of planEvent.plan.operations) {
3590
- if (operation.kind !== "candidate-generation") continue;
3591
- const generationOutcome = operationsById.get(operation.operationId).outcome.status;
3592
- const slots = planEvent.plan.candidateSlots.filter(
3593
- (slot) => slot.generationOperationId === operation.operationId
3594
- );
3595
- if (slots.length === 0) continue;
3596
- const registeredCount = slots.filter((slot) => candidateBySlot.has(slot.slotId)).length;
3597
- const closedCount = slots.filter((slot) => closedSlots.has(slot.slotId)).length;
3598
- if (generationOutcome === "completed" && closedCount > 0) {
3599
- throw new SearchLedgerIntegrityError(
3600
- `completed generation operation ${operation.operationId} contains ${closedCount} closed slot(s)`
3601
- );
3602
- }
3603
- if (generationOutcome === "failed" && registeredCount > 0) {
3604
- throw new SearchLedgerIntegrityError(
3605
- `failed generation operation ${operation.operationId} contains ${registeredCount} registered candidate(s)`
3606
- );
3607
- }
3608
- if (generationOutcome === "partial" && (registeredCount === 0 || closedCount === 0)) {
3609
- throw new SearchLedgerIntegrityError(
3610
- `partial generation operation ${operation.operationId} must contain both a registered candidate and a closed slot`
3611
- );
3612
- }
3613
- }
3614
- const pending = [...candidates.values()].filter((candidate) => candidate.decision === null);
3615
- if (pending.length > 0) {
3616
- throw new SearchLedgerIntegrityError(
3617
- `search completed with ${pending.length} candidate decision(s) missing`
3618
- );
3619
- }
3620
- const selected = decisions.filter((decision) => decision.decision.status === "selected");
3621
- if (event.result.status === "selected") {
3622
- if (selected.length !== 1 || selected[0].candidateId !== event.result.candidateId) {
3623
- throw new SearchLedgerIntegrityError(
3624
- `search completion winner ${event.result.candidateId} does not match candidate decisions`
3625
- );
3626
- }
3627
- } else if (selected.length !== 0) {
3628
- throw new SearchLedgerIntegrityError("all-rejected completion contains a selected candidate");
3629
- }
3630
- completion = event;
3631
- }
3632
- const selectedDecisions = decisions.filter((decision) => decision.decision.status === "selected");
3633
- const rejectedDecisions = decisions.filter((decision) => decision.decision.status === "rejected");
3634
- const outcomeCounts = { passed: 0, failed: 0, errored: 0 };
3635
- const operationOutcomeCounts = { completed: 0, partial: 0, failed: 0 };
3636
- let inputTokens = 0;
3637
- let outputTokens = 0;
3638
- let cachedTokens = 0;
3639
- let costUsd = 0;
3640
- const unknownTokenEventIds = [];
3641
- const unknownCostEventIds = [];
3642
- for (const attempt of attempts) {
3643
- outcomeCounts[attempt.outcome.status] += 1;
3644
- }
3645
- for (const operation of operationEvents) {
3646
- operationOutcomeCounts[operation.outcome.status] += 1;
3647
- }
3648
- for (const costedEvent of [...attempts, ...operationEvents]) {
3649
- if (costedEvent.accounting.tokens.status === "known") {
3650
- inputTokens += costedEvent.accounting.tokens.inputTokens;
3651
- outputTokens += costedEvent.accounting.tokens.outputTokens;
3652
- cachedTokens += costedEvent.accounting.tokens.cachedTokens;
3653
- } else {
3654
- unknownTokenEventIds.push(costedEvent.eventId);
3655
- }
3656
- if (costedEvent.accounting.cost.status === "known") {
3657
- costUsd += costedEvent.accounting.cost.usd;
3658
- } else {
3659
- costUsd += costedEvent.accounting.cost.knownLowerBoundUsd;
3660
- unknownCostEventIds.push(costedEvent.eventId);
3661
- }
3662
- }
3663
- const accounting = unknownTokenEventIds.length === 0 && unknownCostEventIds.length === 0 ? {
3664
- status: "known",
3665
- inputTokens,
3666
- outputTokens,
3667
- cachedTokens,
3668
- costUsd
3669
- } : {
3670
- status: "partial",
3671
- knownInputTokens: inputTokens,
3672
- knownOutputTokens: outputTokens,
3673
- knownCachedTokens: cachedTokens,
3674
- knownCostUsd: costUsd,
3675
- unknownTokenEventIds,
3676
- unknownCostEventIds
3677
- };
3678
- const selectedCandidateId = completion?.result.status === "selected" ? completion.result.candidateId : null;
3679
- const status = completion?.result.status === "selected" ? "selected" : completion?.result.status === "all-rejected" ? "all-rejected" : "in-progress";
3680
- const missingCandidateSlots = planEvent?.plan.candidateSlots.filter((slot) => !candidateBySlot.has(slot.slotId) && !closedSlots.has(slot.slotId)).map((slot) => slot.slotId) ?? [];
3681
- const missingTaskOutcomes = planEvent ? plannedTaskOutcomeKeys(planEvent, candidates) : [];
3682
- const missingOperations = planEvent?.plan.operations.filter((operation) => !operationsById.has(operation.operationId)).map((operation) => operation.operationId) ?? [];
3683
- return {
3684
- entries: [...entries],
3685
- plan: planEvent,
3686
- candidates: candidateEvents,
3687
- closedCandidateSlots: closedSlotEvents,
3688
- attempts,
3689
- operations: operationEvents,
3690
- decisions,
3691
- completion,
3692
- audit: {
3693
- campaignId,
3694
- eventCount: entries.length,
3695
- candidateCount: candidates.size,
3696
- closedCandidateSlotCount: closedSlots.size,
3697
- attemptCount: attempts.length,
3698
- operationCount: operationEvents.length,
3699
- outcomes: outcomeCounts,
3700
- operationOutcomes: operationOutcomeCounts,
3701
- decisions: {
3702
- selected: selectedDecisions.length,
3703
- rejected: rejectedDecisions.length,
3704
- pending: candidates.size - decisions.length
3705
- },
3706
- expected: {
3707
- candidateSlots: planEvent?.plan.candidateSlots.length ?? 0,
3708
- taskOutcomes: candidates.size * (planEvent?.plan.tasks.length ?? 0),
3709
- operations: planEvent?.plan.operations.length ?? 0,
3710
- missingCandidateSlots,
3711
- missingTaskOutcomes,
3712
- missingOperations
3713
- },
3714
- status,
3715
- selectedCandidateId,
3716
- accounting,
3717
- headHash: entries.at(-1)?.entryHash ?? null
3718
- }
3719
- };
3720
- }
3721
- function plannedTaskOutcomeKeys(planEvent, candidates) {
3722
- const missing = [];
3723
- const registeredCandidates = [...candidates.values()].sort(
3724
- (a, b) => compareStrings(a.registered.slotId, b.registered.slotId)
3725
- );
3726
- for (const candidate of registeredCandidates) {
3727
- const slotId = candidate.registered.slotId;
3728
- for (const task of planEvent.plan.tasks) {
3729
- const measured = candidate.attempts.some(
3730
- (attempt) => attempt.task.taskId === task.taskId && attempt.outcome.status !== "errored"
3731
- );
3732
- if (!measured) missing.push(`${slotId}/${task.taskId}`);
3733
- }
3734
- }
3735
- return missing;
3736
- }
3737
- function normalizeEvent(event) {
3738
- const artifacts = sortArtifacts(event.artifacts);
3739
- if (event.kind === "search-planned") {
3740
- return {
3741
- ...event,
3742
- artifacts,
3743
- plan: {
3744
- candidateSlots: [...event.plan.candidateSlots].sort(
3745
- (a, b) => compareStrings(a.slotId, b.slotId)
3746
- ),
3747
- tasks: [...event.plan.tasks].sort((a, b) => compareStrings(a.taskId, b.taskId)),
3748
- operations: [...event.plan.operations].sort(
3749
- (a, b) => compareStrings(a.operationId, b.operationId)
3750
- )
3751
- }
3752
- };
3753
- }
3754
- if (event.kind === "candidate-registered") {
3755
- return {
3756
- ...event,
3757
- artifacts,
3758
- lineage: {
3759
- ...event.lineage,
3760
- parentCandidateIds: sortedStrings(event.lineage.parentCandidateIds)
3761
- },
3762
- surfaces: [...event.surfaces].map((surface) => ({ ...surface, artifact: { ...surface.artifact } })).sort((a, b) => compareStrings(a.surfaceId, b.surfaceId))
3763
- };
3764
- }
3765
- if (event.kind === "task-attempted") {
3766
- return {
3767
- ...event,
3768
- artifacts,
3769
- surfaceEvidence: [...event.surfaceEvidence].map((evidence) => ({ ...evidence, evidence: sortArtifacts(evidence.evidence) })).sort((a, b) => compareStrings(a.surfaceId, b.surfaceId))
3770
- };
3771
- }
3772
- return { ...event, artifacts };
3773
- }
3774
- function sortArtifacts(artifacts) {
3775
- return [...artifacts].map((artifact) => ({ ...artifact })).sort((a, b) => compareStrings(artifactKey(a), artifactKey(b)));
3776
- }
3777
- function artifactKey(artifact) {
3778
- return canonicalString(artifact);
3779
- }
3780
- function compareStrings(a, b) {
3781
- return a < b ? -1 : a > b ? 1 : 0;
3782
- }
3783
- function sortedStrings(values) {
3784
- return [...values].sort();
3785
- }
3786
- function assertUnique2(values, label, eventId) {
3787
- if (new Set(values).size !== values.length) {
3788
- throw new SearchLedgerIntegrityError(`event ${eventId} contains duplicate ${label} values`);
3789
- }
3790
- }
3791
- function canonicalString(value) {
3792
- return JSON.stringify(canonicalize(value));
3793
- }
3794
- function hashCanonical(value) {
3795
- return `sha256:${createHash6("sha256").update(canonicalString(value)).digest("hex")}`;
3796
- }
3797
- function formatZodError(error) {
3798
- return error.issues.map((issue) => `${issue.path.length > 0 ? issue.path.join(".") : "<root>"}: ${issue.message}`).join("; ");
3799
- }
3800
-
3801
- // src/campaign/transient-failure.ts
3802
- var BASE_TRANSIENT = /\b50[234]\b|no stream output|produced no stream|admission timed out|admission_rejected|queue_timeout|fetch failed|ECONNRESET|This operation was aborted/i;
3803
- var TIMEOUT_PATTERN = /timeout after \d+ ?ms|cli-bridge timeout/i;
3804
- function isTransientTransportFailure(message, opts = {}) {
3805
- if (!message) return false;
3806
- if (BASE_TRANSIENT.test(message)) return true;
3807
- if ((opts.retryFullDurationTimeouts ?? false) && TIMEOUT_PATTERN.test(message)) return true;
3808
- for (const p of opts.extraPatterns ?? []) if (p.test(message)) return true;
3809
- return false;
3810
- }
3811
-
3812
- // src/campaign/worktree/index.ts
3813
- import { execFileSync } from "child_process";
3814
- import { createHash as createHash7 } from "crypto";
3815
- import {
3816
- closeSync,
3817
- existsSync as existsSync4,
3818
- constants as fsConstants,
3819
- fstatSync,
3820
- lstatSync,
3821
- mkdirSync as mkdirSync2,
3822
- mkdtempSync,
3823
- openSync,
3824
- readlinkSync,
3825
- readSync,
3826
- realpathSync,
3827
- rmSync
3828
- } from "fs";
3829
- import { devNull, tmpdir } from "os";
3830
- import { basename, dirname, isAbsolute as isAbsolute2, join as join4, relative as relative2, resolve as resolve3, sep } from "path";
3831
- var MAX_GIT_OUTPUT_BYTES = 256 * 1024 * 1024;
3832
- var FILE_HASH_CHUNK_BYTES = 1024 * 1024;
3833
- var GIT_REPOSITORY_ENV = /* @__PURE__ */ new Set([
3834
- "GIT_ALTERNATE_OBJECT_DIRECTORIES",
3835
- "GIT_COMMON_DIR",
3836
- "GIT_DIR",
3837
- "GIT_INDEX_FILE",
3838
- "GIT_NAMESPACE",
3839
- "GIT_OBJECT_DIRECTORY",
3840
- "GIT_PREFIX",
3841
- "GIT_QUARANTINE_PATH",
3842
- "GIT_WORK_TREE"
3843
- ]);
3844
- var WorktreeAdapterError = class extends Error {
3845
- constructor(message, cause) {
3846
- super(message);
3847
- this.cause = cause;
3848
- this.name = "WorktreeAdapterError";
3849
- }
3850
- cause;
3851
- };
3852
- function defaultGit(args, cwd, overrides) {
3853
- try {
3854
- const env = { ...process.env };
3855
- for (const key of Object.keys(env)) {
3856
- if (GIT_REPOSITORY_ENV.has(key) || key === "GIT_CONFIG" || key.startsWith("GIT_CONFIG_") || key.startsWith("GIT_ATTR_") || key === "GIT_DIFF_OPTS" || key === "GIT_EXTERNAL_DIFF") {
3857
- delete env[key];
3858
- }
3859
- }
3860
- Object.assign(env, overrides);
3861
- env.GIT_NO_REPLACE_OBJECTS = "1";
3862
- env.LC_ALL = "C";
3863
- env.LANG = "C";
3864
- return execFileSync("git", args, { cwd, env, maxBuffer: MAX_GIT_OUTPUT_BYTES });
3865
- } catch (err) {
3866
- const stderr = err && typeof err === "object" && "stderr" in err ? String(err.stderr) : "";
3867
- throw new WorktreeAdapterError(`git ${args.join(" ")} failed: ${stderr || String(err)}`, err);
3868
- }
3869
- }
3870
- function gitBytes(git, args, cwd, env) {
3871
- try {
3872
- const output = git(args, cwd, env);
3873
- return typeof output === "string" ? Buffer.from(output, "utf8") : Buffer.from(output);
3874
- } catch (err) {
3875
- if (err instanceof WorktreeAdapterError) throw err;
3876
- throw new WorktreeAdapterError(`git ${args.join(" ")} failed: ${String(err)}`, err);
3877
- }
3878
- }
3879
- function gitText(git, args, cwd, env) {
3880
- return gitBytes(git, args, cwd, env).toString("utf8").trim();
3881
- }
3882
- function hasRegisteredWorktree(git, repoRoot, path) {
3883
- const expected = Buffer.from(`worktree ${resolve3(path)}`, "utf8");
3884
- const records = gitBytes(git, ["worktree", "list", "--porcelain", "-z"], repoRoot);
3885
- let start = 0;
3886
- while (start < records.length) {
3887
- const end = records.indexOf(0, start);
3888
- if (end < 0) {
3889
- throw new WorktreeAdapterError("Git worktree list output was not NUL-terminated");
3890
- }
3891
- if (records.subarray(start, end).equals(expected)) return true;
3892
- start = end + 1;
3893
- }
3894
- return false;
3895
- }
3896
- function hasLocalBranch(git, repoRoot, branch) {
3897
- const ref = `refs/heads/${branch}`;
3898
- return gitText(git, ["for-each-ref", "--format=%(refname)", "--", ref], repoRoot).split("\n").some((candidate) => candidate === ref);
3899
- }
3900
- function reconcileAbsent(exists, remove) {
3901
- try {
3902
- if (!exists()) return void 0;
3903
- } catch (err) {
3904
- return err;
3905
- }
3906
- try {
3907
- remove();
3908
- return void 0;
3909
- } catch (removeError) {
3910
- try {
3911
- if (!exists()) return void 0;
3912
- } catch (recheckError) {
3913
- return new AggregateError(
3914
- [removeError, recheckError],
3915
- "Removal failed and the resulting resource state could not be checked"
3916
- );
3917
- }
3918
- return removeError;
3919
- }
3920
- }
3921
- function sha2562(bytes) {
3922
- return `sha256:${createHash7("sha256").update(bytes).digest("hex")}`;
3923
- }
3924
- var PATCH_ARGS = [
3925
- // Git binary patches embed a zlib stream. Pin no-compression at command
3926
- // scope so repository/global config and compressor heuristics cannot change
3927
- // the candidate identity for the same base and final tree.
3928
- "-c",
3929
- "core.compression=0",
3930
- "-c",
3931
- `core.attributesFile=${devNull}`,
3932
- "-c",
3933
- "core.quotePath=true",
3934
- "-c",
3935
- "diff.suppressBlankEmpty=false",
3936
- "diff",
3937
- "--unified=3",
3938
- "--inter-hunk-context=0",
3939
- `-O${devNull}`,
3940
- "--binary",
3941
- "--full-index",
3942
- "--no-color",
3943
- "--no-ext-diff",
3944
- "--no-textconv",
3945
- "--no-renames",
3946
- "--diff-algorithm=myers",
3947
- "--no-indent-heuristic",
3948
- "--src-prefix=a/",
3949
- "--dst-prefix=b/"
3950
- ];
3951
- var CANONICAL_GIT_ENV = {
3952
- GIT_ATTR_GLOBAL: devNull,
3953
- GIT_ATTR_NOSYSTEM: "1",
3954
- GIT_CONFIG_GLOBAL: devNull,
3955
- GIT_CONFIG_NOSYSTEM: "1",
3956
- GIT_CONFIG_SYSTEM: devNull
3957
- };
3958
- function patchBytes(git, cwd, baseCommit, candidateCommit) {
3959
- const scratch = mkdtempSync(join4(tmpdir(), "agent-eval-patch-"));
3960
- const bareRepo = join4(scratch, "repo.git");
3961
- const emptyTemplate = join4(scratch, "empty-template");
3962
- mkdirSync2(emptyTemplate);
3963
- try {
3964
- const objectFormat = gitObjectHashAlgorithm(candidateCommit);
3965
- const sourceObjects = realpathSync(gitText(git, ["rev-parse", "--git-path", "objects"], cwd));
3966
- gitText(
3967
- git,
3968
- [
3969
- "init",
3970
- "--bare",
3971
- "--quiet",
3972
- ...objectFormat === "sha256" ? ["--object-format=sha256"] : [],
3973
- `--template=${emptyTemplate}`,
3974
- bareRepo
3975
- ],
3976
- scratch,
3977
- CANONICAL_GIT_ENV
3978
- );
3979
- return gitBytes(
3980
- git,
3981
- [`--git-dir=${bareRepo}`, ...PATCH_ARGS, baseCommit, candidateCommit, "--"],
3982
- scratch,
3983
- {
3984
- ...CANONICAL_GIT_ENV,
3985
- GIT_ALTERNATE_OBJECT_DIRECTORIES: sourceObjects
3986
- }
3987
- );
3988
- } finally {
3989
- rmSync(scratch, { recursive: true, force: true });
3990
- }
3991
- }
3992
- function resolveCommit(git, cwd, ref) {
3993
- return gitText(git, ["rev-parse", "--verify", `${ref}^{commit}`], cwd);
3994
- }
3995
- function unresolvedWorktreePath(surface, worktreeDir) {
3996
- if (isAbsolute2(surface.worktreeRef)) return surface.worktreeRef;
3997
- if (worktreeDir) return join4(worktreeDir, basename(surface.worktreeRef));
3998
- return surface.worktreeRef;
3999
- }
4000
- function displayGitPath(path) {
4001
- return JSON.stringify(path);
4002
- }
4003
- function parseGitTreeEntries(bytes) {
4004
- const input = Buffer.from(bytes);
4005
- const entries = [];
4006
- let start = 0;
4007
- while (start < input.length) {
4008
- const end = input.indexOf(0, start);
4009
- if (end < 0) throw new WorktreeAdapterError("Git tree output was not NUL-terminated");
4010
- const record = input.subarray(start, end);
4011
- start = end + 1;
4012
- if (record.length === 0) continue;
4013
- const tab = record.indexOf(9);
4014
- if (tab < 0) throw new WorktreeAdapterError("Git tree entry did not contain a path");
4015
- const metadata = record.subarray(0, tab).toString("ascii").split(" ");
4016
- if (metadata.length !== 3) throw new WorktreeAdapterError("Git tree entry was malformed");
4017
- const mode = metadata[0];
4018
- const type = metadata[1];
4019
- const objectId = metadata[2];
4020
- if (!mode || !type || !objectId) {
4021
- throw new WorktreeAdapterError("Git tree entry was malformed");
4022
- }
4023
- const pathBytes = record.subarray(tab + 1);
4024
- const path = pathBytes.toString("utf8");
4025
- if (!Buffer.from(path, "utf8").equals(pathBytes)) {
4026
- throw new WorktreeAdapterError("CodeSurface paths must be valid UTF-8");
4027
- }
4028
- if (type !== "blob" && type !== "commit") {
4029
- throw new WorktreeAdapterError(
4030
- `CodeSurface contains unsupported Git object type ${type} at ${displayGitPath(path)}`
4031
- );
4032
- }
4033
- entries.push({ mode, objectId, path });
4034
- }
4035
- return entries;
4036
- }
4037
- function parseGitPathList(bytes) {
4038
- const input = Buffer.from(bytes);
4039
- const paths = [];
4040
- let start = 0;
4041
- while (start < input.length) {
4042
- const end = input.indexOf(0, start);
4043
- if (end < 0) throw new WorktreeAdapterError("Git path output was not NUL-terminated");
4044
- const pathBytes = input.subarray(start, end);
4045
- start = end + 1;
4046
- if (pathBytes.length === 0) continue;
4047
- const path = pathBytes.toString("utf8");
4048
- if (!Buffer.from(path, "utf8").equals(pathBytes)) {
4049
- throw new WorktreeAdapterError("CodeSurface paths must be valid UTF-8");
4050
- }
4051
- paths.push(path);
4052
- }
4053
- return paths;
4054
- }
4055
- function isWithinRoot(root, candidate) {
4056
- const rel = relative2(root, candidate);
4057
- return rel === "" || !isAbsolute2(rel) && rel !== ".." && !rel.startsWith(`..${sep}`);
4058
- }
4059
- function assertSafeRelativePath(root, path) {
4060
- if (path.length === 0 || isAbsolute2(path)) {
4061
- throw new WorktreeAdapterError(`CodeSurface contains unsafe path ${displayGitPath(path)}`);
4062
- }
4063
- const absolutePath = resolve3(root, path);
4064
- if (!isWithinRoot(root, absolutePath) || absolutePath === root) {
4065
- throw new WorktreeAdapterError(`CodeSurface path escapes its worktree: ${displayGitPath(path)}`);
4066
- }
4067
- const segments = path.split(/[\\/]/u);
4068
- let parent = root;
4069
- for (const segment of segments.slice(0, -1)) {
4070
- if (segment.length === 0 || segment === "." || segment === "..") {
4071
- throw new WorktreeAdapterError(`CodeSurface contains unsafe path ${displayGitPath(path)}`);
4072
- }
4073
- parent = join4(parent, segment);
4074
- const stat = lstatSync(parent);
4075
- if (!stat.isDirectory() || stat.isSymbolicLink()) {
4076
- throw new WorktreeAdapterError(
4077
- `CodeSurface path traverses a non-directory or symbolic link: ${displayGitPath(path)}`
4078
- );
4079
- }
4080
- }
4081
- return absolutePath;
4082
- }
4083
- function gitObjectHashAlgorithm(objectId) {
4084
- if (/^[a-f0-9]{40}$/.test(objectId)) return "sha1";
4085
- if (/^[a-f0-9]{64}$/.test(objectId)) return "sha256";
4086
- throw new WorktreeAdapterError(`Unsupported Git object id: ${objectId}`);
4087
- }
4088
- function hashGitBlobBytes(bytes, objectId) {
4089
- const hash = createHash7(gitObjectHashAlgorithm(objectId));
4090
- hash.update(`blob ${bytes.byteLength}\0`);
4091
- hash.update(bytes);
4092
- return hash.digest("hex");
4093
- }
4094
- function hashGitBlobFile(path, objectId) {
4095
- const before = lstatSync(path);
4096
- if (!before.isFile() || before.isSymbolicLink()) {
4097
- throw new WorktreeAdapterError(`CodeSurface expected a regular file at ${displayGitPath(path)}`);
4098
- }
4099
- const noFollow = process.platform === "win32" ? 0 : fsConstants.O_NOFOLLOW;
4100
- const fd = openSync(path, fsConstants.O_RDONLY | noFollow);
4101
- try {
4102
- const opened = fstatSync(fd);
4103
- if (!opened.isFile() || opened.dev !== before.dev || opened.ino !== before.ino || opened.size !== before.size) {
4104
- throw new WorktreeAdapterError(
4105
- `CodeSurface file changed while it was being verified: ${displayGitPath(path)}`
4106
- );
4107
- }
4108
- const hash = createHash7(gitObjectHashAlgorithm(objectId));
4109
- hash.update(`blob ${opened.size}\0`);
4110
- const chunk = Buffer.allocUnsafe(FILE_HASH_CHUNK_BYTES);
4111
- let total = 0;
4112
- while (true) {
4113
- const count = readSync(fd, chunk, 0, chunk.length, null);
4114
- if (count === 0) break;
4115
- total += count;
4116
- hash.update(chunk.subarray(0, count));
4117
- }
4118
- if (total !== opened.size) {
4119
- throw new WorktreeAdapterError(
4120
- `CodeSurface file changed while it was being verified: ${displayGitPath(path)}`
4121
- );
4122
- }
4123
- return { hash: hash.digest("hex"), executable: (opened.mode & 73) !== 0 };
4124
- } finally {
4125
- closeSync(fd);
4126
- }
4127
- }
4128
- function assertSymlinkTargetIsBound(root, linkPath, trackedPaths) {
4129
- const targetBytes = readlinkSync(linkPath, { encoding: "buffer" });
4130
- const target = targetBytes.toString("utf8");
4131
- if (!Buffer.from(target, "utf8").equals(targetBytes)) {
4132
- throw new WorktreeAdapterError(
4133
- `CodeSurface symbolic link has an unsafe target at ${displayGitPath(linkPath)}`
4134
- );
4135
- }
4136
- if (isAbsolute2(target)) {
4137
- throw new WorktreeAdapterError(
4138
- `CodeSurface symbolic link escapes its worktree at ${displayGitPath(linkPath)}`
4139
- );
4140
- }
4141
- const lexicalTarget = resolve3(dirname(linkPath), target);
4142
- if (!isWithinRoot(root, lexicalTarget)) {
4143
- throw new WorktreeAdapterError(
4144
- `CodeSurface symbolic link escapes its worktree at ${displayGitPath(linkPath)}`
4145
- );
4146
- }
4147
- let resolvedTarget;
4148
- try {
4149
- resolvedTarget = realpathSync(linkPath);
4150
- } catch (err) {
4151
- throw new WorktreeAdapterError(
4152
- `CodeSurface symbolic link target is missing or cyclic at ${displayGitPath(linkPath)}`,
4153
- err
4154
- );
4155
- }
4156
- if (!isWithinRoot(root, resolvedTarget)) {
4157
- throw new WorktreeAdapterError(
4158
- `CodeSurface symbolic link escapes its worktree at ${displayGitPath(linkPath)}`
4159
- );
4160
- }
4161
- const targetRelative = relative2(root, resolvedTarget).split(sep).join("/");
4162
- const targetIsTracked = trackedPaths.has(targetRelative) || [...trackedPaths].some((trackedPath) => trackedPath.startsWith(`${targetRelative}/`));
4163
- if (!targetIsTracked) {
4164
- throw new WorktreeAdapterError(
4165
- `CodeSurface symbolic link resolves to untracked content at ${displayGitPath(linkPath)}`
4166
- );
4167
- }
4168
- }
4169
- function assertRawTreeMatchesWorktree(git, root, candidateCommit) {
4170
- const entries = parseGitTreeEntries(
4171
- gitBytes(git, ["ls-tree", "-r", "-z", "--full-tree", candidateCommit], root)
4172
- );
4173
- const trackedPaths = new Set(entries.map((entry) => entry.path));
4174
- for (const entry of entries) {
4175
- const absolutePath = assertSafeRelativePath(root, entry.path);
4176
- if (entry.mode === "160000" || entry.mode === "040000") {
4177
- throw new WorktreeAdapterError(
4178
- `CodeSurface contains a Git submodule whose executable bytes are not bound: ${displayGitPath(entry.path)}`
4179
- );
4180
- }
4181
- if (entry.mode === "120000") {
4182
- const stat = lstatSync(absolutePath);
4183
- if (!stat.isSymbolicLink()) {
4184
- throw new WorktreeAdapterError(
4185
- `CodeSurface expected a symbolic link at ${displayGitPath(entry.path)}`
4186
- );
4187
- }
4188
- const targetBytes = readlinkSync(absolutePath, { encoding: "buffer" });
4189
- const actualObjectId = hashGitBlobBytes(targetBytes, entry.objectId);
4190
- if (actualObjectId !== entry.objectId) {
4191
- throw new WorktreeAdapterError(
4192
- `CodeSurface raw symbolic-link bytes differ from the candidate tree at ${displayGitPath(entry.path)}`
4193
- );
4194
- }
4195
- assertSymlinkTargetIsBound(root, absolutePath, trackedPaths);
4196
- continue;
4197
- }
4198
- if (entry.mode !== "100644" && entry.mode !== "100755") {
4199
- throw new WorktreeAdapterError(
4200
- `CodeSurface contains unsupported Git mode ${entry.mode} at ${displayGitPath(entry.path)}`
4201
- );
4202
- }
4203
- const actual = hashGitBlobFile(absolutePath, entry.objectId);
4204
- if (actual.hash !== entry.objectId) {
4205
- throw new WorktreeAdapterError(
4206
- `CodeSurface raw tracked bytes differ from the candidate tree because the worktree changed after finalization at ${displayGitPath(entry.path)}`
4207
- );
4208
- }
4209
- if (process.platform !== "win32" && actual.executable !== (entry.mode === "100755")) {
4210
- throw new WorktreeAdapterError(
4211
- `CodeSurface executable mode differs from the candidate tree at ${displayGitPath(entry.path)}`
4212
- );
4213
- }
4214
- }
4215
- }
4216
- function verifyCodeSurfaceWithGit(surface, path, git) {
4217
- assertCodeSurfaceIdentity(surface);
4218
- if (!existsSync4(path)) {
4219
- throw new WorktreeAdapterError(`CodeSurface worktree does not exist: ${path}`);
4220
- }
4221
- const lexicalRoot = resolve3(path);
4222
- const canonicalRoot = realpathSync(path);
4223
- if (lstatSync(lexicalRoot).isSymbolicLink() || process.platform !== "win32" && lexicalRoot !== canonicalRoot) {
4224
- throw new WorktreeAdapterError(
4225
- `CodeSurface worktree locator must not contain a symbolic link: ${path}`
4226
- );
4227
- }
4228
- const repoRoot = gitText(git, ["rev-parse", "--show-toplevel"], path);
4229
- const canonicalRepoRoot = realpathSync(repoRoot);
4230
- if (canonicalRepoRoot !== canonicalRoot) {
4231
- throw new WorktreeAdapterError(
4232
- `CodeSurface worktree locator is not the repository root: expected ${repoRoot}, got ${path}`
4233
- );
4234
- }
4235
- const indexFlags = gitText(git, ["ls-files", "-v"], path).split("\n").filter((line) => line.length > 0 && (line.startsWith("S ") || /^[a-z]/.test(line)));
4236
- if (indexFlags.length > 0) {
4237
- throw new WorktreeAdapterError(
4238
- `CodeSurface worktree uses hidden index entries that cannot be verified: ${path}
4239
- ${indexFlags.join("\n")}`
4240
- );
4241
- }
4242
- const extraPaths = [
4243
- ...parseGitPathList(gitBytes(git, ["ls-files", "--others", "--exclude-standard", "-z"], path)),
4244
- ...parseGitPathList(
4245
- gitBytes(git, ["ls-files", "--others", "--ignored", "--exclude-standard", "-z"], path)
4246
- )
4247
- ];
4248
- if (extraPaths.length > 0) {
4249
- throw new WorktreeAdapterError(
4250
- `CodeSurface worktree changed after finalization: ${path}
4251
- ${extraPaths.join("\n")}`
4252
- );
4253
- }
4254
- const candidateCommit = resolveCommit(git, path, "HEAD");
4255
- if (candidateCommit !== surface.candidateCommit) {
4256
- throw new WorktreeAdapterError(
4257
- `CodeSurface candidate commit mismatch: expected ${surface.candidateCommit}, got ${candidateCommit}`
4258
- );
4259
- }
4260
- const baseCommit = resolveCommit(git, path, surface.baseCommit);
4261
- if (baseCommit !== surface.baseCommit) {
4262
- throw new WorktreeAdapterError(
4263
- `CodeSurface base commit mismatch: expected ${surface.baseCommit}, got ${baseCommit}`
4264
- );
4265
- }
4266
- const baseTree = gitText(git, ["rev-parse", "--verify", `${surface.baseCommit}^{tree}`], path);
4267
- if (baseTree !== surface.baseTree) {
4268
- throw new WorktreeAdapterError(
4269
- `CodeSurface base tree mismatch: expected ${surface.baseTree}, got ${baseTree}`
4270
- );
4271
- }
4272
- const candidateTree = gitText(
4273
- git,
4274
- ["rev-parse", "--verify", `${surface.candidateCommit}^{tree}`],
4275
- path
4276
- );
4277
- if (candidateTree !== surface.candidateTree) {
4278
- throw new WorktreeAdapterError(
4279
- `CodeSurface tree mismatch: expected ${surface.candidateTree}, got ${candidateTree}`
4280
- );
4281
- }
4282
- try {
4283
- gitText(
4284
- git,
4285
- [
4286
- "diff-index",
4287
- "--cached",
4288
- "--quiet",
4289
- "--no-ext-diff",
4290
- "--no-textconv",
4291
- surface.candidateCommit,
4292
- "--"
4293
- ],
4294
- path
4295
- );
4296
- } catch (err) {
4297
- throw new WorktreeAdapterError(
4298
- `CodeSurface index differs from finalized candidate: ${path}`,
4299
- err
4300
- );
4301
- }
4302
- try {
4303
- assertRawTreeMatchesWorktree(git, canonicalRepoRoot, surface.candidateCommit);
4304
- } catch (err) {
4305
- if (err instanceof WorktreeAdapterError) throw err;
4306
- throw new WorktreeAdapterError(`CodeSurface raw tree verification failed: ${path}`, err);
4307
- }
4308
- const mergeBase = gitText(git, ["merge-base", surface.baseCommit, surface.candidateCommit], path);
4309
- if (mergeBase !== surface.baseCommit) {
4310
- throw new WorktreeAdapterError(
4311
- `CodeSurface candidate ${surface.candidateCommit} does not descend from base ${surface.baseCommit}`
4312
- );
4313
- }
4314
- const patch = patchBytes(git, path, surface.baseCommit, surface.candidateCommit);
4315
- const actualPatchHash = sha2562(patch);
4316
- if (actualPatchHash !== surface.patch.sha256 || patch.byteLength !== surface.patch.byteLength) {
4317
- throw new WorktreeAdapterError(
4318
- `CodeSurface patch mismatch: expected ${surface.patch.sha256}/${surface.patch.byteLength} bytes, got ${actualPatchHash}/${patch.byteLength} bytes`
4319
- );
4320
- }
4321
- return {
4322
- path,
4323
- repoRoot: canonicalRepoRoot,
4324
- contentHash: surfaceContentHash(surface),
4325
- patchBytes: new Uint8Array(patch)
4326
- };
4327
- }
4328
- function slug(label) {
4329
- return label.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 48) || "candidate";
4330
- }
4331
- function gitWorktreeAdapter(opts) {
4332
- const git = opts.git ?? defaultGit;
4333
- const worktreeDir = opts.worktreeDir ?? join4(opts.repoRoot, ".worktrees");
4334
- const branchPrefix = opts.branchPrefix ?? "improve";
4335
- return {
4336
- async create({ baseRef, label }) {
4337
- const id = `${slug(label)}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`;
4338
- const branch = `${branchPrefix}/${id}`;
4339
- const path = join4(worktreeDir, id);
4340
- const baseCommit = resolveCommit(git, opts.repoRoot, baseRef);
4341
- const baseTree = gitText(
4342
- git,
4343
- ["rev-parse", "--verify", `${baseCommit}^{tree}`],
4344
- opts.repoRoot
4345
- );
4346
- gitText(git, ["worktree", "add", "-b", branch, path, baseCommit], opts.repoRoot);
4347
- return { path, branch, baseRef, baseCommit, baseTree };
4348
- },
4349
- async finalize(worktree, summary) {
4350
- const status = gitText(
4351
- git,
4352
- ["status", "--porcelain=v1", "--untracked-files=all"],
4353
- worktree.path
4354
- );
4355
- if (status.length > 0) {
4356
- gitText(git, ["add", "-A"], worktree.path);
4357
- gitText(git, ["commit", "-m", summary], worktree.path);
4358
- }
4359
- const candidateCommit = resolveCommit(git, worktree.path, "HEAD");
4360
- const candidateTree = gitText(
4361
- git,
4362
- ["rev-parse", "--verify", `${candidateCommit}^{tree}`],
4363
- worktree.path
4364
- );
4365
- const patch = patchBytes(git, worktree.path, worktree.baseCommit, candidateCommit);
4366
- const surface = {
4367
- kind: "code",
4368
- worktreeRef: worktree.path,
4369
- baseRef: worktree.baseRef,
4370
- baseCommit: worktree.baseCommit,
4371
- baseTree: worktree.baseTree,
4372
- candidateCommit,
4373
- candidateTree,
4374
- patch: {
4375
- format: "git-diff-binary",
4376
- sha256: sha2562(patch),
4377
- byteLength: patch.byteLength
4378
- },
4379
- summary
4380
- };
4381
- verifyCodeSurfaceWithGit(surface, worktree.path, git);
4382
- return surface;
4383
- },
4384
- async discard(worktree) {
4385
- const failures = [
4386
- reconcileAbsent(
4387
- () => hasRegisteredWorktree(git, opts.repoRoot, worktree.path),
4388
- () => gitText(git, ["worktree", "remove", "--force", "--", worktree.path], opts.repoRoot)
4389
- ),
4390
- reconcileAbsent(
4391
- () => hasLocalBranch(git, opts.repoRoot, worktree.branch),
4392
- () => gitText(git, ["branch", "-D", "--", worktree.branch], opts.repoRoot)
4393
- )
4394
- ].filter((failure) => failure !== void 0);
4395
- if (failures.length > 0) {
4396
- const cause = failures.length === 1 ? failures[0] : new AggregateError(failures, "Multiple Git resources could not be removed");
4397
- throw new WorktreeAdapterError(
4398
- `Failed to discard worktree ${worktree.path} and branch ${worktree.branch}`,
4399
- cause
4400
- );
4401
- }
4402
- }
4403
- };
4404
- }
4405
- function verifyCodeSurface(surface, worktreeDir) {
4406
- const path = unresolvedWorktreePath(surface, worktreeDir);
4407
- return verifyCodeSurfaceWithGit(surface, path, defaultGit);
4408
- }
4409
- function resolveWorktreePath(surface, worktreeDir) {
4410
- return verifyCodeSurface(surface, worktreeDir).path;
4411
- }
4412
-
4413
- export {
4414
- completionVerdict,
4415
- verifyCompletion,
4416
- parseCorrectnessResponse,
4417
- createLlmCorrectnessChecker,
4418
- createTokenRecallChecker,
4419
- extractProducedState,
4420
- CODING_HARNESSES,
4421
- HARNESS_NATIVE_MODEL,
4422
- expandProfileAxes,
4423
- harnessAxisOf,
4424
- agentProfileId,
4425
- agentProfileModelId,
4426
- agentProfileHash,
4427
- buildAnalystSurfaceDispatch,
4428
- failureModeRecallJudge,
4429
- analyzeCrossSurfaceInteractions,
4430
- discoverEvalFixtures,
4431
- loadEvalFixture,
4432
- loadEvalFixtureScenarios,
4433
- planEvalFixtureRun,
4434
- neutralizationGate,
4435
- sequentialPairedGate,
4436
- sequentialDecide,
4437
- rolloutArgumentDiff,
4438
- classifyUngroundedLiterals,
4439
- LabeledScenarioStoreError,
4440
- FsLabeledScenarioStore,
4441
- neutralizeText,
4442
- makePlaybackDispatch,
4443
- scoreUserStory,
4444
- userStoryScoreboard,
4445
- scoreboardSummary,
4446
- renderScoreboardMarkdown,
4447
- ProfileMatrixError,
4448
- runProfileMatrix,
4449
- scoreDiscrimination,
4450
- selectDiscriminative,
4451
- SEARCH_LEDGER_SCHEMA,
4452
- validateSearchLedgerEvent,
4453
- openSearchLedger,
4454
- FileSearchLedger,
4455
- isTransientTransportFailure,
4456
- WorktreeAdapterError,
4457
- gitWorktreeAdapter,
4458
- verifyCodeSurface,
4459
- resolveWorktreePath
4460
- };
4461
- //# sourceMappingURL=chunk-UB2LOJ6Q.js.map