@tangle-network/agent-eval 0.128.2 → 0.130.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (424) hide show
  1. package/CHANGELOG.md +279 -0
  2. package/README.md +19 -0
  3. package/dist/active-curriculum-C4mk67HP.js +214 -0
  4. package/dist/active-curriculum-C4mk67HP.js.map +1 -0
  5. package/dist/adversarial-smnADNFS.d.ts +21 -0
  6. package/dist/adversarial-smnADNFS.d.ts.map +1 -0
  7. package/dist/analyst/index.d.ts +83 -2932
  8. package/dist/analyst/index.d.ts.map +1 -0
  9. package/dist/analyst/index.js +319 -364
  10. package/dist/analyst/index.js.map +1 -1
  11. package/dist/analyst-BkTS3C58.d.ts +89 -0
  12. package/dist/analyst-BkTS3C58.d.ts.map +1 -0
  13. package/dist/analyst-LsnNpSkm.js +152 -0
  14. package/dist/analyst-LsnNpSkm.js.map +1 -0
  15. package/dist/analyze-runs-C1CavBMk.js +1067 -0
  16. package/dist/analyze-runs-C1CavBMk.js.map +1 -0
  17. package/dist/analyze-runs-FsgYCinh.d.ts +72 -0
  18. package/dist/analyze-runs-FsgYCinh.d.ts.map +1 -0
  19. package/dist/attribute-vocabulary-DLJ6303h.d.ts +54 -0
  20. package/dist/attribute-vocabulary-DLJ6303h.d.ts.map +1 -0
  21. package/dist/authenticity/index.d.ts +81 -79
  22. package/dist/authenticity/index.d.ts.map +1 -0
  23. package/dist/authenticity/index.js +209 -193
  24. package/dist/authenticity/index.js.map +1 -1
  25. package/dist/baseline-HsBvw_dk.js +550 -0
  26. package/dist/baseline-HsBvw_dk.js.map +1 -0
  27. package/dist/baseline-hG3K85h4.d.ts +125 -0
  28. package/dist/baseline-hG3K85h4.d.ts.map +1 -0
  29. package/dist/belief-state/index.d.ts +448 -1205
  30. package/dist/belief-state/index.d.ts.map +1 -0
  31. package/dist/belief-state/index.js +1617 -1710
  32. package/dist/belief-state/index.js.map +1 -1
  33. package/dist/benchmarks/index.d.ts +2 -894
  34. package/dist/benchmarks/index.js +2 -59
  35. package/dist/benchmarks-DviOvUNr.js +754 -0
  36. package/dist/benchmarks-DviOvUNr.js.map +1 -0
  37. package/dist/builder-eval/index.d.ts +150 -662
  38. package/dist/builder-eval/index.d.ts.map +1 -0
  39. package/dist/builder-eval/index.js +356 -345
  40. package/dist/builder-eval/index.js.map +1 -1
  41. package/dist/calibration-CNWWA6K8.js +94 -0
  42. package/dist/calibration-CNWWA6K8.js.map +1 -0
  43. package/dist/campaign/index.d.ts +5 -6390
  44. package/dist/campaign/index.js +3 -212
  45. package/dist/campaign-CBKZvQ1H.js +3885 -0
  46. package/dist/campaign-CBKZvQ1H.js.map +1 -0
  47. package/dist/cli.d.ts +1 -1
  48. package/dist/cli.js +164 -174
  49. package/dist/cli.js.map +1 -1
  50. package/dist/client-C97NMzqi.d.ts +581 -0
  51. package/dist/client-C97NMzqi.d.ts.map +1 -0
  52. package/dist/client-CYzbdJOZ.js +637 -0
  53. package/dist/client-CYzbdJOZ.js.map +1 -0
  54. package/dist/code-agent-session-BjkMTQ7H.js +1390 -0
  55. package/dist/code-agent-session-BjkMTQ7H.js.map +1 -0
  56. package/dist/code-agent-session-DqqgOJaz.d.ts +143 -0
  57. package/dist/code-agent-session-DqqgOJaz.d.ts.map +1 -0
  58. package/dist/concurrency-DIxRZF_J.js +85 -0
  59. package/dist/concurrency-DIxRZF_J.js.map +1 -0
  60. package/dist/contract/index.d.ts +645 -5605
  61. package/dist/contract/index.d.ts.map +1 -0
  62. package/dist/contract/index.js +1730 -1937
  63. package/dist/contract/index.js.map +1 -1
  64. package/dist/control.d.ts +3 -1030
  65. package/dist/control.js +2 -32
  66. package/dist/cost-ledger-DIgQUFZZ.js +801 -0
  67. package/dist/cost-ledger-DIgQUFZZ.js.map +1 -0
  68. package/dist/cost-ledger-Dye6jCgg.d.ts +236 -0
  69. package/dist/cost-ledger-Dye6jCgg.d.ts.map +1 -0
  70. package/dist/dataset-BvtnC8Dc.d.ts +115 -0
  71. package/dist/dataset-BvtnC8Dc.d.ts.map +1 -0
  72. package/dist/default-registry-C-vFCSEc.js +2579 -0
  73. package/dist/default-registry-C-vFCSEc.js.map +1 -0
  74. package/dist/default-registry-CNPo-Vsb.d.ts +540 -0
  75. package/dist/default-registry-CNPo-Vsb.d.ts.map +1 -0
  76. package/dist/emitter-CPBAhxum.js +266 -0
  77. package/dist/emitter-CPBAhxum.js.map +1 -0
  78. package/dist/emitter-DGQGoLyj.d.ts +113 -0
  79. package/dist/emitter-DGQGoLyj.d.ts.map +1 -0
  80. package/dist/errors-8YnH8WlF.js +64 -0
  81. package/dist/errors-8YnH8WlF.js.map +1 -0
  82. package/dist/errors-CEk209JS.d.ts +76 -0
  83. package/dist/errors-CEk209JS.d.ts.map +1 -0
  84. package/dist/eval-campaign-DEm6c8ru.js +349 -0
  85. package/dist/eval-campaign-DEm6c8ru.js.map +1 -0
  86. package/dist/execution-tracks-CpgFPpS5.js +93 -0
  87. package/dist/execution-tracks-CpgFPpS5.js.map +1 -0
  88. package/dist/exporters-q9iL-2Jf.js +148 -0
  89. package/dist/exporters-q9iL-2Jf.js.map +1 -0
  90. package/dist/extract-usage-BrQ8mCLX.js +155 -0
  91. package/dist/extract-usage-BrQ8mCLX.js.map +1 -0
  92. package/dist/failure-cluster-CqcvCcdR.d.ts +59 -0
  93. package/dist/failure-cluster-CqcvCcdR.d.ts.map +1 -0
  94. package/dist/feedback-trajectory-CVaeREXV.d.ts +340 -0
  95. package/dist/feedback-trajectory-CVaeREXV.d.ts.map +1 -0
  96. package/dist/fuzz.d.ts +320 -646
  97. package/dist/fuzz.d.ts.map +1 -0
  98. package/dist/fuzz.js +670 -617
  99. package/dist/fuzz.js.map +1 -1
  100. package/dist/hf-dataset-DBJXXoY1.js +763 -0
  101. package/dist/hf-dataset-DBJXXoY1.js.map +1 -0
  102. package/dist/hosted/index.d.ts +11 -831
  103. package/dist/hosted/index.d.ts.map +1 -0
  104. package/dist/hosted/index.js +2 -37
  105. package/dist/index-2JJSA6-r2.d.ts +926 -0
  106. package/dist/index-2JJSA6-r2.d.ts.map +1 -0
  107. package/dist/index-6N0aYmpW.d.ts +217 -0
  108. package/dist/index-6N0aYmpW.d.ts.map +1 -0
  109. package/dist/index-BAvgST_9.d.ts +131 -0
  110. package/dist/index-BAvgST_9.d.ts.map +1 -0
  111. package/dist/index-BvnJuTGD.d.ts +68 -0
  112. package/dist/index-BvnJuTGD.d.ts.map +1 -0
  113. package/dist/index-C61Wi7yg.d.ts +547 -0
  114. package/dist/index-C61Wi7yg.d.ts.map +1 -0
  115. package/dist/index-CAPUUKaM.d.ts +335 -0
  116. package/dist/index-CAPUUKaM.d.ts.map +1 -0
  117. package/dist/index-DE5fb3EC.d.ts +2244 -0
  118. package/dist/index-DE5fb3EC.d.ts.map +1 -0
  119. package/dist/index-DSC51roc.d.ts +102 -0
  120. package/dist/index-DSC51roc.d.ts.map +1 -0
  121. package/dist/index.d.ts +3776 -15120
  122. package/dist/index.d.ts.map +1 -0
  123. package/dist/index.js +11185 -11191
  124. package/dist/index.js.map +1 -1
  125. package/dist/integrity-BzRbCHzi.js +424 -0
  126. package/dist/integrity-BzRbCHzi.js.map +1 -0
  127. package/dist/integrity-rmVhXWA7.d.ts +61 -0
  128. package/dist/integrity-rmVhXWA7.d.ts.map +1 -0
  129. package/dist/judge-calibration-DFtEMlde.d.ts +146 -0
  130. package/dist/judge-calibration-DFtEMlde.d.ts.map +1 -0
  131. package/dist/ledger-core/index.d.ts +2 -0
  132. package/dist/ledger-core/index.js +2 -0
  133. package/dist/ledger-core-DtZz1RG0.js +388 -0
  134. package/dist/ledger-core-DtZz1RG0.js.map +1 -0
  135. package/dist/llm-client--GR4JbZE.js +687 -0
  136. package/dist/llm-client--GR4JbZE.js.map +1 -0
  137. package/dist/llm-client-B_nIBlYo.d.ts +290 -0
  138. package/dist/llm-client-B_nIBlYo.d.ts.map +1 -0
  139. package/dist/matrix/index.d.ts +3 -155
  140. package/dist/matrix/index.js +2 -8
  141. package/dist/matrix-BzQnu2S6.js +270 -0
  142. package/dist/matrix-BzQnu2S6.js.map +1 -0
  143. package/dist/meta-eval/index.d.ts +4 -1027
  144. package/dist/meta-eval/index.js +393 -390
  145. package/dist/meta-eval/index.js.map +1 -1
  146. package/dist/metrics-C9YY1OcL.js +239 -0
  147. package/dist/metrics-C9YY1OcL.js.map +1 -0
  148. package/dist/mint-yN2M2eh0.js +201 -0
  149. package/dist/mint-yN2M2eh0.js.map +1 -0
  150. package/dist/multi-layer-verifier-BHY1gWAc.d.ts +138 -0
  151. package/dist/multi-layer-verifier-BHY1gWAc.d.ts.map +1 -0
  152. package/dist/multishot/index.d.ts +273 -481
  153. package/dist/multishot/index.d.ts.map +1 -0
  154. package/dist/multishot/index.js +595 -548
  155. package/dist/multishot/index.js.map +1 -1
  156. package/dist/off-policy-DvgzvtIx.js +220 -0
  157. package/dist/off-policy-DvgzvtIx.js.map +1 -0
  158. package/dist/off-policy-mskQw8Mb.d.ts +153 -0
  159. package/dist/off-policy-mskQw8Mb.d.ts.map +1 -0
  160. package/dist/openapi.json +1 -1
  161. package/dist/opencode-sqlite-BGrHeDu3.js +318 -0
  162. package/dist/opencode-sqlite-BGrHeDu3.js.map +1 -0
  163. package/dist/outcome-store-BYHIuO0e.d.ts +65 -0
  164. package/dist/outcome-store-BYHIuO0e.d.ts.map +1 -0
  165. package/dist/outcome-store-ChBKlTd_.js +75 -0
  166. package/dist/outcome-store-ChBKlTd_.js.map +1 -0
  167. package/dist/paired-arms-D9D0wXj2.js +260 -0
  168. package/dist/paired-arms-D9D0wXj2.js.map +1 -0
  169. package/dist/pipelines/index.d.ts +95 -532
  170. package/dist/pipelines/index.d.ts.map +1 -0
  171. package/dist/pipelines/index.js +497 -478
  172. package/dist/pipelines/index.js.map +1 -1
  173. package/dist/pre-registration-DakwTRXk.js +96 -0
  174. package/dist/pre-registration-DakwTRXk.js.map +1 -0
  175. package/dist/propose-review-control-LhIWGzHl.js +1458 -0
  176. package/dist/propose-review-control-LhIWGzHl.js.map +1 -0
  177. package/dist/query-CJ_DX8vl.d.ts +28 -0
  178. package/dist/query-CJ_DX8vl.d.ts.map +1 -0
  179. package/dist/query-Di7eEQ79.js +83 -0
  180. package/dist/query-Di7eEQ79.js.map +1 -0
  181. package/dist/raw-provider-sink-BQd7mzyT.js +201 -0
  182. package/dist/raw-provider-sink-BQd7mzyT.js.map +1 -0
  183. package/dist/raw-provider-sink-BU29Sh8h.d.ts +134 -0
  184. package/dist/raw-provider-sink-BU29Sh8h.d.ts.map +1 -0
  185. package/dist/redact-7Aq1ukl-.js +107 -0
  186. package/dist/redact-7Aq1ukl-.js.map +1 -0
  187. package/dist/release-report-Cz9NKH39.js +603 -0
  188. package/dist/release-report-Cz9NKH39.js.map +1 -0
  189. package/dist/release-report-mvB2G4_J.d.ts +244 -0
  190. package/dist/release-report-mvB2G4_J.d.ts.map +1 -0
  191. package/dist/replay-GnyotH0J.js +1741 -0
  192. package/dist/replay-GnyotH0J.js.map +1 -0
  193. package/dist/replay-OoidtG1E.d.ts +749 -0
  194. package/dist/replay-OoidtG1E.d.ts.map +1 -0
  195. package/dist/reporting.d.ts +6 -1298
  196. package/dist/reporting.js +6 -50
  197. package/dist/researcher-CwTdwXG1.d.ts +314 -0
  198. package/dist/researcher-CwTdwXG1.d.ts.map +1 -0
  199. package/dist/reward-hacking-eAnOsynk.d.ts +224 -0
  200. package/dist/reward-hacking-eAnOsynk.d.ts.map +1 -0
  201. package/dist/reward-hacking-qipEpKvY.js +596 -0
  202. package/dist/reward-hacking-qipEpKvY.js.map +1 -0
  203. package/dist/reward-nw2xZGZG.js +137 -0
  204. package/dist/reward-nw2xZGZG.js.map +1 -0
  205. package/dist/rl.d.ts +916 -3596
  206. package/dist/rl.d.ts.map +1 -0
  207. package/dist/rl.js +2362 -1751
  208. package/dist/rl.js.map +1 -1
  209. package/dist/rolldown-runtime-8H4AJuhK.js +14 -0
  210. package/dist/rollout/index.d.ts +3 -1048
  211. package/dist/rollout/index.js +8 -110
  212. package/dist/rollout-BOYjemfR.js +624 -0
  213. package/dist/rollout-BOYjemfR.js.map +1 -0
  214. package/dist/rubric-predictive-validity-B3xmbmS1.js +141 -0
  215. package/dist/rubric-predictive-validity-B3xmbmS1.js.map +1 -0
  216. package/dist/rubric-predictive-validity-Ku_clp_1.d.ts +74 -0
  217. package/dist/rubric-predictive-validity-Ku_clp_1.d.ts.map +1 -0
  218. package/dist/run-evidence-ClRX_8A9.d.ts +225 -0
  219. package/dist/run-evidence-ClRX_8A9.d.ts.map +1 -0
  220. package/dist/run-record-BuoE80Dq.js +467 -0
  221. package/dist/run-record-BuoE80Dq.js.map +1 -0
  222. package/dist/run-record-CnZu_gjl.d.ts +357 -0
  223. package/dist/run-record-CnZu_gjl.d.ts.map +1 -0
  224. package/dist/run-score-iEEAWiBY.js +41 -0
  225. package/dist/run-score-iEEAWiBY.js.map +1 -0
  226. package/dist/runtime-trajectory-1gyaTOoC.js +93 -0
  227. package/dist/runtime-trajectory-1gyaTOoC.js.map +1 -0
  228. package/dist/runtime-trajectory-BvSZcCHD.d.ts +50 -0
  229. package/dist/runtime-trajectory-BvSZcCHD.d.ts.map +1 -0
  230. package/dist/schema-BtVldJ3T.d.ts +206 -0
  231. package/dist/schema-BtVldJ3T.d.ts.map +1 -0
  232. package/dist/schema-C6DW4ZHR.js +821 -0
  233. package/dist/schema-C6DW4ZHR.js.map +1 -0
  234. package/dist/schema-CRhEY1SO.js +69 -0
  235. package/dist/schema-CRhEY1SO.js.map +1 -0
  236. package/dist/schema-Cef2cFmb.d.ts +408 -0
  237. package/dist/schema-Cef2cFmb.d.ts.map +1 -0
  238. package/dist/semantic-concept-judge-B6cWNJ2K.js +725 -0
  239. package/dist/semantic-concept-judge-B6cWNJ2K.js.map +1 -0
  240. package/dist/sequential-Br0mAPHA.js +148 -0
  241. package/dist/sequential-Br0mAPHA.js.map +1 -0
  242. package/dist/sequential-CYwq6Ff_.d.ts +141 -0
  243. package/dist/sequential-CYwq6Ff_.d.ts.map +1 -0
  244. package/dist/series-convergence-CjO2QdRW.js +43 -0
  245. package/dist/series-convergence-CjO2QdRW.js.map +1 -0
  246. package/dist/series-convergence-ofsqPWhs.d.ts +35 -0
  247. package/dist/series-convergence-ofsqPWhs.d.ts.map +1 -0
  248. package/dist/server-m5D9cvnG.js +1040 -0
  249. package/dist/server-m5D9cvnG.js.map +1 -0
  250. package/dist/skill-usage-C_pXm7lP.d.ts +534 -0
  251. package/dist/skill-usage-C_pXm7lP.d.ts.map +1 -0
  252. package/dist/skillopt-optimization-method-B7wX7XkF.d.ts +1739 -0
  253. package/dist/skillopt-optimization-method-B7wX7XkF.d.ts.map +1 -0
  254. package/dist/skillopt-optimization-method-D4ODwFVV.js +7712 -0
  255. package/dist/skillopt-optimization-method-D4ODwFVV.js.map +1 -0
  256. package/dist/statistics-Cmj6nynr.d.ts +514 -0
  257. package/dist/statistics-Cmj6nynr.d.ts.map +1 -0
  258. package/dist/statistics-CnnxdpOg.js +1437 -0
  259. package/dist/statistics-CnnxdpOg.js.map +1 -0
  260. package/dist/store-CT9YIIve.d.ts +117 -0
  261. package/dist/store-CT9YIIve.d.ts.map +1 -0
  262. package/dist/store-CxJry_cs.d.ts +229 -0
  263. package/dist/store-CxJry_cs.d.ts.map +1 -0
  264. package/dist/storyboard/index.d.ts +84 -203
  265. package/dist/storyboard/index.d.ts.map +1 -0
  266. package/dist/storyboard/index.js +609 -542
  267. package/dist/storyboard/index.js.map +1 -1
  268. package/dist/summary-report-BNs5nmXI.js +862 -0
  269. package/dist/summary-report-BNs5nmXI.js.map +1 -0
  270. package/dist/summary-report-Cj9gdw4i.d.ts +407 -0
  271. package/dist/summary-report-Cj9gdw4i.d.ts.map +1 -0
  272. package/dist/supervisor-run/index.d.ts +2 -849
  273. package/dist/supervisor-run/index.js +2 -64
  274. package/dist/supervisor-run-_lnTLM3z.js +1679 -0
  275. package/dist/supervisor-run-_lnTLM3z.js.map +1 -0
  276. package/dist/task-failure-attributes-CQZlB3et.js +311 -0
  277. package/dist/task-failure-attributes-CQZlB3et.js.map +1 -0
  278. package/dist/test-graded-scenario-BsqWLmPt.js +318 -0
  279. package/dist/test-graded-scenario-BsqWLmPt.js.map +1 -0
  280. package/dist/test-graded-scenario-D1TaI2va.d.ts +141 -0
  281. package/dist/test-graded-scenario-D1TaI2va.d.ts.map +1 -0
  282. package/dist/tools-BmuN627J.js +1085 -0
  283. package/dist/tools-BmuN627J.js.map +1 -0
  284. package/dist/trace-attributes.d.ts +2 -52
  285. package/dist/trace-attributes.js +131 -61
  286. package/dist/trace-attributes.js.map +1 -1
  287. package/dist/traces.d.ts +12 -2365
  288. package/dist/traces.js +12 -251
  289. package/dist/trajectory-D_7rLrvE.js +56 -0
  290. package/dist/trajectory-D_7rLrvE.js.map +1 -0
  291. package/dist/types-DGsxbAEd.d.ts +387 -0
  292. package/dist/types-DGsxbAEd.d.ts.map +1 -0
  293. package/dist/types-k9tZGKUg.d.ts +640 -0
  294. package/dist/types-k9tZGKUg.d.ts.map +1 -0
  295. package/dist/verdict-Dps8_okt.d.ts +37 -0
  296. package/dist/verdict-Dps8_okt.d.ts.map +1 -0
  297. package/dist/wire/index.d.ts +702 -1174
  298. package/dist/wire/index.d.ts.map +1 -0
  299. package/dist/wire/index.js +2 -81
  300. package/docs/feature-guide.md +1 -1
  301. package/docs/rollout.md +116 -2
  302. package/package.json +18 -10
  303. package/dist/benchmarks/index.js.map +0 -1
  304. package/dist/campaign/index.js.map +0 -1
  305. package/dist/chunk-2JX3CFMB.js +0 -695
  306. package/dist/chunk-2JX3CFMB.js.map +0 -1
  307. package/dist/chunk-2MKQIFS4.js +0 -183
  308. package/dist/chunk-2MKQIFS4.js.map +0 -1
  309. package/dist/chunk-3RF76KTD.js +0 -84
  310. package/dist/chunk-3RF76KTD.js.map +0 -1
  311. package/dist/chunk-5DTSBUL2.js +0 -159
  312. package/dist/chunk-5DTSBUL2.js.map +0 -1
  313. package/dist/chunk-7ZZMD7UK.js +0 -386
  314. package/dist/chunk-7ZZMD7UK.js.map +0 -1
  315. package/dist/chunk-BOD4O7OF.js +0 -40
  316. package/dist/chunk-BOD4O7OF.js.map +0 -1
  317. package/dist/chunk-BYT7ELPS.js +0 -1553
  318. package/dist/chunk-BYT7ELPS.js.map +0 -1
  319. package/dist/chunk-DJKY2TSY.js +0 -2428
  320. package/dist/chunk-DJKY2TSY.js.map +0 -1
  321. package/dist/chunk-DPUHNQLN.js +0 -232
  322. package/dist/chunk-DPUHNQLN.js.map +0 -1
  323. package/dist/chunk-DRYIUNWY.js +0 -622
  324. package/dist/chunk-DRYIUNWY.js.map +0 -1
  325. package/dist/chunk-EJGRPCO3.js +0 -617
  326. package/dist/chunk-EJGRPCO3.js.map +0 -1
  327. package/dist/chunk-EOSZT7PL.js +0 -2001
  328. package/dist/chunk-EOSZT7PL.js.map +0 -1
  329. package/dist/chunk-EZJEIH2R.js +0 -1559
  330. package/dist/chunk-EZJEIH2R.js.map +0 -1
  331. package/dist/chunk-GGE4NNQT.js +0 -65
  332. package/dist/chunk-GGE4NNQT.js.map +0 -1
  333. package/dist/chunk-HHWE3POT.js +0 -94
  334. package/dist/chunk-HHWE3POT.js.map +0 -1
  335. package/dist/chunk-IHQDPH7D.js +0 -171
  336. package/dist/chunk-IHQDPH7D.js.map +0 -1
  337. package/dist/chunk-JHCHEVET.js +0 -274
  338. package/dist/chunk-JHCHEVET.js.map +0 -1
  339. package/dist/chunk-K4DBDHLK.js +0 -158
  340. package/dist/chunk-K4DBDHLK.js.map +0 -1
  341. package/dist/chunk-K6N6XJJX.js +0 -306
  342. package/dist/chunk-K6N6XJJX.js.map +0 -1
  343. package/dist/chunk-MA6HLL3S.js +0 -65
  344. package/dist/chunk-MA6HLL3S.js.map +0 -1
  345. package/dist/chunk-MAZ26DC7.js +0 -99
  346. package/dist/chunk-MAZ26DC7.js.map +0 -1
  347. package/dist/chunk-MHELPNRP.js +0 -1212
  348. package/dist/chunk-MHELPNRP.js.map +0 -1
  349. package/dist/chunk-NACAGYSY.js +0 -1040
  350. package/dist/chunk-NACAGYSY.js.map +0 -1
  351. package/dist/chunk-NKAGIDE2.js +0 -7633
  352. package/dist/chunk-NKAGIDE2.js.map +0 -1
  353. package/dist/chunk-NPCTHQIO.js +0 -91
  354. package/dist/chunk-NPCTHQIO.js.map +0 -1
  355. package/dist/chunk-NYLOYM6N.js +0 -332
  356. package/dist/chunk-NYLOYM6N.js.map +0 -1
  357. package/dist/chunk-ONWEPEDO.js +0 -57
  358. package/dist/chunk-ONWEPEDO.js.map +0 -1
  359. package/dist/chunk-P5W7RQKK.js +0 -576
  360. package/dist/chunk-P5W7RQKK.js.map +0 -1
  361. package/dist/chunk-P6FYH6K4.js +0 -1161
  362. package/dist/chunk-P6FYH6K4.js.map +0 -1
  363. package/dist/chunk-PBE2LOSS.js +0 -669
  364. package/dist/chunk-PBE2LOSS.js.map +0 -1
  365. package/dist/chunk-PC4UYEBM.js +0 -166
  366. package/dist/chunk-PC4UYEBM.js.map +0 -1
  367. package/dist/chunk-PXE2VKMX.js +0 -140
  368. package/dist/chunk-PXE2VKMX.js.map +0 -1
  369. package/dist/chunk-PZ5AY32C.js +0 -10
  370. package/dist/chunk-PZ5AY32C.js.map +0 -1
  371. package/dist/chunk-RZTMDUO7.js +0 -49
  372. package/dist/chunk-RZTMDUO7.js.map +0 -1
  373. package/dist/chunk-S5YLIBFX.js +0 -136
  374. package/dist/chunk-S5YLIBFX.js.map +0 -1
  375. package/dist/chunk-SZLVEKMJ.js +0 -1446
  376. package/dist/chunk-SZLVEKMJ.js.map +0 -1
  377. package/dist/chunk-T4SQEITX.js +0 -95
  378. package/dist/chunk-T4SQEITX.js.map +0 -1
  379. package/dist/chunk-TBL77AUT.js +0 -355
  380. package/dist/chunk-TBL77AUT.js.map +0 -1
  381. package/dist/chunk-TSN7JT6D.js +0 -1646
  382. package/dist/chunk-TSN7JT6D.js.map +0 -1
  383. package/dist/chunk-TT4KNT67.js +0 -124
  384. package/dist/chunk-TT4KNT67.js.map +0 -1
  385. package/dist/chunk-UB2LOJ6Q.js +0 -4461
  386. package/dist/chunk-UB2LOJ6Q.js.map +0 -1
  387. package/dist/chunk-UWZZKKU7.js +0 -237
  388. package/dist/chunk-UWZZKKU7.js.map +0 -1
  389. package/dist/chunk-VBQ3CRKH.js +0 -291
  390. package/dist/chunk-VBQ3CRKH.js.map +0 -1
  391. package/dist/chunk-VGRCHJON.js +0 -163
  392. package/dist/chunk-VGRCHJON.js.map +0 -1
  393. package/dist/chunk-VI2UW6B6.js +0 -162
  394. package/dist/chunk-VI2UW6B6.js.map +0 -1
  395. package/dist/chunk-VLOATJQ2.js +0 -908
  396. package/dist/chunk-VLOATJQ2.js.map +0 -1
  397. package/dist/chunk-VQMK5FMP.js +0 -247
  398. package/dist/chunk-VQMK5FMP.js.map +0 -1
  399. package/dist/chunk-VZSRQ272.js +0 -149
  400. package/dist/chunk-VZSRQ272.js.map +0 -1
  401. package/dist/chunk-WGXIEX7P.js +0 -116
  402. package/dist/chunk-WGXIEX7P.js.map +0 -1
  403. package/dist/chunk-WS3NZZQQ.js +0 -929
  404. package/dist/chunk-WS3NZZQQ.js.map +0 -1
  405. package/dist/chunk-XDWDC2MP.js +0 -695
  406. package/dist/chunk-XDWDC2MP.js.map +0 -1
  407. package/dist/chunk-XPRT64IE.js +0 -766
  408. package/dist/chunk-XPRT64IE.js.map +0 -1
  409. package/dist/chunk-YJBNWCAA.js +0 -1056
  410. package/dist/chunk-YJBNWCAA.js.map +0 -1
  411. package/dist/chunk-ZET2UAYW.js +0 -89
  412. package/dist/chunk-ZET2UAYW.js.map +0 -1
  413. package/dist/chunk-ZUUWPZCV.js +0 -752
  414. package/dist/chunk-ZUUWPZCV.js.map +0 -1
  415. package/dist/control.js.map +0 -1
  416. package/dist/hosted/index.js.map +0 -1
  417. package/dist/matrix/index.js.map +0 -1
  418. package/dist/reporting.js.map +0 -1
  419. package/dist/rollout/index.js.map +0 -1
  420. package/dist/run-campaign-ISHFZ7FJ.js +0 -17
  421. package/dist/run-campaign-ISHFZ7FJ.js.map +0 -1
  422. package/dist/supervisor-run/index.js.map +0 -1
  423. package/dist/traces.js.map +0 -1
  424. package/dist/wire/index.js.map +0 -1
@@ -1,1559 +0,0 @@
1
- import {
2
- projectCampaignCellQuality
3
- } from "./chunk-2MKQIFS4.js";
4
- import {
5
- confidenceInterval
6
- } from "./chunk-MHELPNRP.js";
7
- import {
8
- CostAccountingIncompleteError,
9
- CostLedger
10
- } from "./chunk-WS3NZZQQ.js";
11
- import {
12
- AgentEvalError,
13
- ValidationError
14
- } from "./chunk-ONWEPEDO.js";
15
-
16
- // src/campaign/run-campaign.ts
17
- import { join as join3 } from "path";
18
-
19
- // src/integrity/backend-integrity.ts
20
- var BackendIntegrityError = class extends AgentEvalError {
21
- constructor(message, report) {
22
- super("backend_integrity", message);
23
- this.report = report;
24
- }
25
- report;
26
- };
27
- function summarizeBackendIntegrity(records) {
28
- return summarizeBackendUsage(
29
- records.map((record) => ({
30
- inputTokens: record.tokenUsage.input,
31
- outputTokens: record.tokenUsage.output,
32
- costUsd: record.costUsd
33
- }))
34
- );
35
- }
36
- function summarizeAgentReceiptIntegrity(receipts) {
37
- return summarizeBackendUsage(
38
- receipts.filter((receipt) => receipt.channel === "agent").map((receipt) => ({
39
- inputTokens: receipt.inputTokens,
40
- outputTokens: receipt.outputTokens,
41
- costUsd: receipt.costUsd
42
- }))
43
- );
44
- }
45
- function summarizeBackendUsage(records) {
46
- const totalRecords = records.length;
47
- let stubRecords = 0;
48
- let realRecords = 0;
49
- let uncostedRecords = 0;
50
- let totalInputTokens = 0;
51
- let totalOutputTokens = 0;
52
- let totalCostUsd = 0;
53
- for (const rec of records) {
54
- totalInputTokens += rec.inputTokens;
55
- totalOutputTokens += rec.outputTokens;
56
- totalCostUsd += rec.costUsd ?? 0;
57
- if (rec.inputTokens === 0 && rec.outputTokens === 0) stubRecords++;
58
- else realRecords++;
59
- if (rec.outputTokens > 0 && (rec.costUsd === null || rec.costUsd === 0)) uncostedRecords++;
60
- }
61
- const verdict = totalRecords === 0 ? "stub" : stubRecords === totalRecords ? "stub" : stubRecords === 0 ? "real" : "mixed";
62
- const diagnosis = buildDiagnosis({
63
- totalRecords,
64
- stubRecords,
65
- realRecords,
66
- uncostedRecords,
67
- totalInputTokens,
68
- totalOutputTokens,
69
- totalCostUsd,
70
- verdict
71
- });
72
- return {
73
- totalRecords,
74
- stubRecords,
75
- realRecords,
76
- uncostedRecords,
77
- totalInputTokens,
78
- totalOutputTokens,
79
- totalCostUsd,
80
- verdict,
81
- diagnosis
82
- };
83
- }
84
- function buildDiagnosis(r) {
85
- if (r.totalRecords === 0) {
86
- return "no records \u2014 eval produced zero runs; backend likely failed before first turn";
87
- }
88
- if (r.verdict === "stub") {
89
- return [
90
- `all ${r.totalRecords} records have zero token usage \u2014 the LLM backend was never called.`,
91
- "common causes: --backend sandbox without a sandbox bridge running; stub model returning hard-coded strings;",
92
- "auth misconfigured so requests were silently dropped before the LLM. Re-run with --backend tcloud and TANGLE_API_KEY set,",
93
- "or boot the cli-bridge / sandbox before invoking the eval."
94
- ].join(" ");
95
- }
96
- if (r.verdict === "mixed") {
97
- const pct = (r.stubRecords / r.totalRecords * 100).toFixed(0);
98
- return [
99
- `${r.stubRecords}/${r.totalRecords} records (${pct}%) have zero token usage \u2014 the backend partially failed.`,
100
- "common causes: rate-limit cascade (429s after the first N personas);",
101
- "transient auth expiry mid-run; provider outage. Treat the affected records as missing data, not agent failures."
102
- ].join(" ");
103
- }
104
- if (r.uncostedRecords > 0) {
105
- const pct = (r.uncostedRecords / r.totalRecords * 100).toFixed(0);
106
- return [
107
- `${r.totalRecords} records with real LLM activity (in=${r.totalInputTokens}, out=${r.totalOutputTokens} tokens).`,
108
- `${r.uncostedRecords} (${pct}%) have output tokens but costUsd=0. Two distinct roots:`,
109
- "(a) cost ledger mis-wired \u2014 no usage propagation from the runtime stream into RunRecord; or",
110
- "(b) the model is unpriced at the source (sandbox/router returned $0 despite real tokens).",
111
- "For (b), price the measured tokens against the substrate table (estimateCost) instead of leaving $0."
112
- ].join(" ");
113
- }
114
- return `${r.totalRecords} records with real LLM activity (in=${r.totalInputTokens}, out=${r.totalOutputTokens} tokens, $${r.totalCostUsd.toFixed(4)}).`;
115
- }
116
- function assertRealBackend(records, opts = {}) {
117
- const report = summarizeBackendIntegrity(records);
118
- return assertBackendReport(report, opts);
119
- }
120
- function assertRealAgentReceipts(receipts, opts = {}) {
121
- const report = summarizeAgentReceiptIntegrity(receipts);
122
- return assertBackendReport(report, opts);
123
- }
124
- function assertBackendReport(report, opts) {
125
- const allowMixed = opts.allowMixed ?? true;
126
- if (report.verdict === "stub") {
127
- throw new BackendIntegrityError(
128
- `backend-integrity: ran against a stub or unconfigured backend \u2014 ${report.diagnosis}`,
129
- report
130
- );
131
- }
132
- if (!allowMixed && report.verdict === "mixed") {
133
- throw new BackendIntegrityError(
134
- `backend-integrity: partial backend failure rejected \u2014 ${report.diagnosis}`,
135
- report
136
- );
137
- }
138
- return report;
139
- }
140
-
141
- // src/verdict-cache.ts
142
- import { createHash } from "crypto";
143
- import { appendFileSync, existsSync, readFileSync } from "fs";
144
- function canonicalizeAt(value, path) {
145
- if (value === null) return "null";
146
- switch (typeof value) {
147
- case "boolean":
148
- return value ? "true" : "false";
149
- case "number":
150
- if (!Number.isFinite(value)) {
151
- throw new Error(
152
- `canonicalJson: non-finite number (${value}) at ${path} \u2014 ambiguity is an error, not a coercion`
153
- );
154
- }
155
- return JSON.stringify(value);
156
- case "string":
157
- return JSON.stringify(value);
158
- case "undefined":
159
- case "function":
160
- case "symbol":
161
- throw new Error(
162
- `canonicalJson: ${typeof value} at ${path} \u2014 ambiguity is an error, not a coercion`
163
- );
164
- case "bigint":
165
- throw new Error(`canonicalJson: bigint at ${path} \u2014 not representable in JSON`);
166
- case "object":
167
- break;
168
- }
169
- const obj = value;
170
- if (typeof obj.toJSON === "function") {
171
- return canonicalizeAt(obj.toJSON(), path);
172
- }
173
- if (Array.isArray(obj)) {
174
- return `[${obj.map((item, i) => canonicalizeAt(item, `${path}[${i}]`)).join(",")}]`;
175
- }
176
- if (obj instanceof Map || obj instanceof Set) {
177
- throw new Error(
178
- `canonicalJson: ${obj instanceof Map ? "Map" : "Set"} at ${path} \u2014 would serialize as '{}'; convert to a plain object/array first`
179
- );
180
- }
181
- const keys = Object.keys(obj).sort();
182
- const parts = keys.map((k) => `${JSON.stringify(k)}:${canonicalizeAt(obj[k], `${path}.${k}`)}`);
183
- return `{${parts.join(",")}}`;
184
- }
185
- function canonicalJson(value) {
186
- return canonicalizeAt(value, "$");
187
- }
188
- function contentHash(value) {
189
- return createHash("sha256").update(canonicalJson(value)).digest("hex");
190
- }
191
- function inMemoryVerdictCache() {
192
- const entries = /* @__PURE__ */ new Map();
193
- return {
194
- get: (key) => entries.get(key),
195
- set: (key, score) => {
196
- entries.set(key, score);
197
- }
198
- };
199
- }
200
- function parseCacheLine(line, path, lineNo) {
201
- let parsed;
202
- try {
203
- parsed = JSON.parse(line);
204
- } catch (err) {
205
- throw new Error(
206
- `fileVerdictCache: corrupt JSONL at ${path}:${lineNo} \u2014 ${err instanceof Error ? err.message : String(err)}`
207
- );
208
- }
209
- const rec = parsed;
210
- if (typeof rec !== "object" || rec === null || typeof rec.key !== "string" || typeof rec.score !== "object" || rec.score === null || typeof rec.score.composite !== "number" || typeof rec.score.dimensions !== "object") {
211
- throw new Error(
212
- `fileVerdictCache: invalid record shape at ${path}:${lineNo} \u2014 expected {key, score:{dimensions, composite, notes}}`
213
- );
214
- }
215
- return rec;
216
- }
217
- function fileVerdictCache(path) {
218
- const entries = /* @__PURE__ */ new Map();
219
- if (existsSync(path)) {
220
- const lines = readFileSync(path, "utf8").split("\n");
221
- for (let i = 0; i < lines.length; i++) {
222
- const line = lines[i];
223
- if (line === void 0 || line.trim() === "") continue;
224
- const rec = parseCacheLine(line, path, i + 1);
225
- entries.set(rec.key, rec.score);
226
- }
227
- }
228
- return {
229
- get: (key) => entries.get(key),
230
- set: (key, score) => {
231
- appendFileSync(path, `${JSON.stringify({ key, score })}
232
- `, "utf8");
233
- entries.set(key, score);
234
- }
235
- };
236
- }
237
- function cachedJudge(judge, store, options) {
238
- if (typeof options.judgeVersion !== "string" || options.judgeVersion.trim() === "") {
239
- throw new Error("cachedJudge: judgeVersion is required and must be a non-empty string");
240
- }
241
- const stats = { hits: 0, misses: 0 };
242
- const wrapped = {
243
- name: judge.name,
244
- dimensions: judge.dimensions,
245
- judgeVersion: options.judgeVersion,
246
- async score(input) {
247
- const key = contentHash({
248
- artifact: canonicalJson(input.artifact),
249
- scenarioId: input.scenario.id,
250
- judgeName: judge.name,
251
- dimensions: judge.dimensions,
252
- judgeVersion: options.judgeVersion
253
- });
254
- const cached = await store.get(key);
255
- if (cached !== void 0) {
256
- stats.hits += 1;
257
- return cached;
258
- }
259
- const score = await judge.score(input);
260
- await store.set(key, score);
261
- stats.misses += 1;
262
- return score;
263
- },
264
- stats: () => ({ ...stats })
265
- };
266
- if (judge.appliesTo) wrapped.appliesTo = judge.appliesTo;
267
- return wrapped;
268
- }
269
-
270
- // src/campaign/coverage.ts
271
- function assertCampaignDesign(scenarios, reps) {
272
- if (!Number.isSafeInteger(reps) || reps < 1) {
273
- throw new Error("campaign design requires reps to be a positive safe integer");
274
- }
275
- const scenarioIds = /* @__PURE__ */ new Set();
276
- for (const scenario of scenarios) {
277
- if (typeof scenario.id !== "string" || scenario.id.trim().length === 0) {
278
- throw new Error("campaign design requires every scenario to have a non-empty id");
279
- }
280
- if (scenarioIds.has(scenario.id)) {
281
- throw new Error(`campaign design contains duplicate scenario id '${scenario.id}'`);
282
- }
283
- if (typeof scenario.kind !== "string" || scenario.kind.trim().length === 0) {
284
- throw new Error("campaign design requires every scenario to have a non-empty kind");
285
- }
286
- if (scenario.seedGroup !== void 0 && (typeof scenario.seedGroup !== "string" || scenario.seedGroup.trim().length === 0)) {
287
- throw new Error("campaign design requires seedGroup to be a non-empty string when set");
288
- }
289
- scenarioIds.add(scenario.id);
290
- }
291
- }
292
- function campaignScenarioIdentity(scenario) {
293
- assertCampaignDesign([scenario], 1);
294
- return {
295
- id: scenario.id,
296
- kind: scenario.kind,
297
- scenarioDigest: `sha256:${contentHash(scenario)}`
298
- };
299
- }
300
- function campaignSplitDigestFromIdentities(scenarios, reps) {
301
- assertCampaignDesign(scenarios, reps);
302
- for (const scenario of scenarios) {
303
- if (!/^sha256:[a-f0-9]{64}$/.test(scenario.scenarioDigest)) {
304
- throw new Error(`campaign scenario '${scenario.id}' has an invalid digest`);
305
- }
306
- }
307
- return `sha256:${contentHash({
308
- schema: "tangle.campaign-split",
309
- scenarios: scenarios.map(({ id, kind, scenarioDigest }) => ({ id, kind, scenarioDigest })),
310
- reps
311
- })}`;
312
- }
313
- function campaignSplitDigest(scenarios, reps) {
314
- assertCampaignDesign(scenarios, reps);
315
- return campaignSplitDigestFromIdentities(scenarios.map(campaignScenarioIdentity), reps);
316
- }
317
- function assertCampaignSplitIdentity(scenarios, reps, splitDigest) {
318
- if (campaignSplitDigestFromIdentities(scenarios, reps) !== splitDigest) {
319
- throw new Error("campaign split digest does not match its retained scenario identities");
320
- }
321
- }
322
- function campaignCoverage(cells, scenarios, reps, requireJudgeScore) {
323
- assertCampaignDesign(scenarios, reps);
324
- const expectedCellIds = designedCellIds(scenarios, reps);
325
- const cellsById = /* @__PURE__ */ new Map();
326
- for (const cell of cells) {
327
- const matches = cellsById.get(cell.cellId) ?? [];
328
- matches.push(cell);
329
- cellsById.set(cell.cellId, matches);
330
- }
331
- const scorableCellIds = [];
332
- const unscorableCells = [];
333
- for (const cellId of expectedCellIds) {
334
- const matches = cellsById.get(cellId) ?? [];
335
- if (matches.length === 0) {
336
- unscorableCells.push({ cellId, reason: "missing campaign cell" });
337
- continue;
338
- }
339
- if (matches.length > 1) {
340
- unscorableCells.push({ cellId, reason: `duplicate campaign cell (${matches.length})` });
341
- continue;
342
- }
343
- const cell = matches[0];
344
- const scoreEntries = Object.entries(cell.judgeScores);
345
- const successfulScores = scoreEntries.map(([, score]) => score).filter((score) => score.failed !== true && Number.isFinite(score.composite));
346
- const nonFiniteScores = scoreEntries.filter(
347
- ([, score]) => score.failed !== true && (!Number.isFinite(score.composite) || Object.values(score.dimensions).some((value) => !Number.isFinite(value)))
348
- );
349
- const reasons = [];
350
- if (cell.error) reasons.push(cell.error);
351
- if (cell.artifact === null || cell.artifact === void 0) reasons.push("missing artifact");
352
- if (!cell.error && requireJudgeScore && successfulScores.length === 0) {
353
- reasons.push("no successful finite judge score");
354
- }
355
- if (scoreEntries.some(([, score]) => score.failed === true)) {
356
- reasons.push("judge score marked failed");
357
- }
358
- const failedPanelJudges = [
359
- ...new Set(scoreEntries.flatMap(([, score]) => score.failedJudges ?? []))
360
- ].sort();
361
- if (failedPanelJudges.length > 0) {
362
- reasons.push(`judge panel incomplete: ${failedPanelJudges.join(", ")}`);
363
- }
364
- if (nonFiniteScores.length > 0) {
365
- reasons.push(
366
- `non-finite judge score: ${nonFiniteScores.map(([name]) => name).sort().join(", ")}`
367
- );
368
- }
369
- if (reasons.length > 0) {
370
- unscorableCells.push({ cellId, reason: reasons.join("; ") });
371
- } else {
372
- scorableCellIds.push(cellId);
373
- }
374
- }
375
- const expected = new Set(expectedCellIds);
376
- for (const cell of cells) {
377
- if (cell.cellId !== `${cell.scenarioId}:${cell.rep}`) {
378
- unscorableCells.push({
379
- cellId: cell.cellId,
380
- reason: "campaign cell id does not match scenario id and rep"
381
- });
382
- continue;
383
- }
384
- if (!expected.has(cell.cellId)) {
385
- unscorableCells.push({ cellId: cell.cellId, reason: "unexpected campaign cell" });
386
- }
387
- }
388
- return {
389
- complete: unscorableCells.length === 0 && scorableCellIds.length === expectedCellIds.length,
390
- expectedCellIds,
391
- scorableCellIds,
392
- unscorableCells
393
- };
394
- }
395
- function formatCoverageFailures(coverage) {
396
- const shown = coverage.unscorableCells.slice(0, 3).map((cell) => `${cell.cellId}: ${cell.reason}`).join("; ");
397
- const remainder = coverage.unscorableCells.length - Math.min(3, coverage.unscorableCells.length);
398
- return remainder > 0 ? `${shown}; +${remainder} more` : shown || "unknown coverage failure";
399
- }
400
- function designedCellIds(scenarios, reps) {
401
- const ids = [];
402
- for (const scenario of scenarios) {
403
- for (let rep = 0; rep < reps; rep++) ids.push(`${scenario.id}:${rep}`);
404
- }
405
- return ids;
406
- }
407
-
408
- // src/campaign/run-dir.ts
409
- import { homedir } from "os";
410
- import { basename, isAbsolute, join } from "path";
411
- function tangleTracesRoot() {
412
- return join(homedir(), ".tangle", "traces");
413
- }
414
- function resolveRunDir(runDir, repo) {
415
- if (isAbsolute(runDir) || runDir.startsWith("mem://")) return runDir;
416
- const r = repo && repo.trim().length > 0 ? repo : basename(process.cwd());
417
- return join(tangleTracesRoot(), r, "runs", runDir);
418
- }
419
-
420
- // src/campaign/storage.ts
421
- import { createRequire } from "module";
422
- import { join as join2 } from "path";
423
-
424
- // src/campaign/search-ledger-file.ts
425
- import { closeSync as closeSync2, constants as constants2, fsyncSync as fsyncSync2, mkdirSync, openSync as openSync2, writeSync } from "fs";
426
- import { dirname } from "path";
427
-
428
- // src/campaign/atomic-file-lock.ts
429
- import { randomUUID } from "crypto";
430
- import {
431
- closeSync,
432
- constants,
433
- existsSync as existsSync2,
434
- fsyncSync,
435
- linkSync,
436
- openSync,
437
- readFileSync as readFileSync2,
438
- renameSync,
439
- unlinkSync,
440
- writeFileSync
441
- } from "fs";
442
- import { hostname } from "os";
443
- var AtomicFileLockError = class extends Error {
444
- name = "AtomicFileLockError";
445
- };
446
- function probeAtomicFileLock(options) {
447
- if (existsSync2(recoveryPath(options.lockPath))) return { acquired: false, reason: "recovery" };
448
- const state = ownerState(options.lockPath, options.acceptLegacyPid ?? false);
449
- if (state.state === "held") {
450
- return { acquired: false, reason: "held", holder: state.owner };
451
- }
452
- return null;
453
- }
454
- function tryAcquireAtomicFileLock(options) {
455
- const pid = options.pid ?? process.pid;
456
- if (!Number.isSafeInteger(pid) || pid <= 0) {
457
- throw new AtomicFileLockError("atomic file lock pid must be a positive integer");
458
- }
459
- const owner = { pid, host: hostname(), nonce: randomUUID() };
460
- const acceptLegacyPid = options.acceptLegacyPid ?? false;
461
- for (let attempt = 0; attempt < 8; attempt += 1) {
462
- if (existsSync2(recoveryPath(options.lockPath))) {
463
- return { acquired: false, reason: "recovery" };
464
- }
465
- if (tryLinkOwner(options.lockPath, owner, `acquire.${attempt}`)) {
466
- return acquired(options.lockPath, owner);
467
- }
468
- const holder = ownerState(options.lockPath, acceptLegacyPid);
469
- if (holder.state === "missing") continue;
470
- if (holder.state === "held") {
471
- return { acquired: false, reason: "held", holder: holder.owner };
472
- }
473
- const reclaimPath = recoveryPath(options.lockPath);
474
- if (!tryLinkOwner(reclaimPath, owner, `reclaim.${attempt}`)) {
475
- return { acquired: false, reason: "recovery" };
476
- }
477
- try {
478
- const current = ownerState(options.lockPath, acceptLegacyPid);
479
- if (current.state === "held") {
480
- return { acquired: false, reason: "held", holder: current.owner };
481
- }
482
- if (current.state === "stale") {
483
- const tombstone = `${options.lockPath}.stale.${owner.nonce}.${attempt}`;
484
- try {
485
- renameSync(options.lockPath, tombstone);
486
- unlinkSync(tombstone);
487
- } catch (error) {
488
- if (!isMissing(error)) throw error;
489
- }
490
- }
491
- if (tryLinkOwner(options.lockPath, owner, `recovered.${attempt}`)) {
492
- return acquired(options.lockPath, owner);
493
- }
494
- } finally {
495
- releaseOwnedPath(reclaimPath, owner);
496
- }
497
- }
498
- throw new AtomicFileLockError(`could not acquire atomic file lock ${options.lockPath}`);
499
- }
500
- function acquired(lockPath, owner) {
501
- return {
502
- acquired: true,
503
- lock: {
504
- owner,
505
- release: () => releaseOwnedPath(lockPath, owner)
506
- }
507
- };
508
- }
509
- function tryLinkOwner(lockPath, owner, suffix) {
510
- const ownerPath = `${lockPath}.${owner.pid}.${owner.nonce}.${suffix}.owner`;
511
- const descriptor = openSync(
512
- ownerPath,
513
- constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY,
514
- 384
515
- );
516
- try {
517
- writeFileSync(descriptor, `${canonicalOwner(owner)}
518
- `, "utf8");
519
- fsyncSync(descriptor);
520
- } catch (error) {
521
- closeSync(descriptor);
522
- unlinkIfExists(ownerPath);
523
- throw error;
524
- }
525
- closeSync(descriptor);
526
- try {
527
- linkSync(ownerPath, lockPath);
528
- return true;
529
- } catch (error) {
530
- if (!isAlreadyExists(error)) throw error;
531
- return false;
532
- } finally {
533
- unlinkIfExists(ownerPath);
534
- }
535
- }
536
- function ownerState(lockPath, acceptLegacyPid) {
537
- const owner = readOwner(lockPath, acceptLegacyPid);
538
- if (!owner) return { state: "missing" };
539
- if (owner.host !== hostname()) return { state: "held", owner };
540
- try {
541
- process.kill(owner.pid, 0);
542
- return { state: "held", owner };
543
- } catch (error) {
544
- if (isNoSuchProcess(error)) return { state: "stale", owner };
545
- return { state: "held", owner };
546
- }
547
- }
548
- function readOwner(lockPath, acceptLegacyPid) {
549
- let contents;
550
- try {
551
- contents = readFileSync2(lockPath, "utf8").trim();
552
- } catch (error) {
553
- if (isMissing(error)) return void 0;
554
- throw error;
555
- }
556
- if (acceptLegacyPid && /^[1-9]\d*$/.test(contents)) {
557
- const pid = Number(contents);
558
- if (Number.isSafeInteger(pid)) return { pid, host: hostname(), nonce: "legacy-pid" };
559
- }
560
- let value;
561
- try {
562
- value = JSON.parse(contents);
563
- } catch (error) {
564
- throw invalidOwner(lockPath, error);
565
- }
566
- if (!isOwner(value)) throw invalidOwner(lockPath);
567
- return value;
568
- }
569
- function isOwner(value) {
570
- if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
571
- const record = value;
572
- return Object.keys(record).length === 3 && Number.isSafeInteger(record.pid) && Number(record.pid) > 0 && typeof record.host === "string" && record.host.trim().length > 0 && typeof record.nonce === "string" && record.nonce.trim().length > 0;
573
- }
574
- function invalidOwner(lockPath, cause) {
575
- const error = new AtomicFileLockError(
576
- `atomic file lock has an invalid owner (${lockPath}); refusing unsafe recovery`
577
- );
578
- if (cause !== void 0) Object.defineProperty(error, "cause", { value: cause });
579
- return error;
580
- }
581
- function releaseOwnedPath(lockPath, owner) {
582
- const current = readOwner(lockPath, true);
583
- if (!current) return;
584
- if (canonicalOwner(current) !== canonicalOwner(owner)) {
585
- throw new AtomicFileLockError(`atomic file lock owner changed before release (${lockPath})`);
586
- }
587
- try {
588
- unlinkSync(lockPath);
589
- } catch (error) {
590
- if (!isMissing(error)) throw error;
591
- }
592
- }
593
- function canonicalOwner(owner) {
594
- return JSON.stringify({ host: owner.host, nonce: owner.nonce, pid: owner.pid });
595
- }
596
- function recoveryPath(lockPath) {
597
- return `${lockPath}.reclaim`;
598
- }
599
- function isAlreadyExists(error) {
600
- return error instanceof Error && "code" in error && error.code === "EEXIST";
601
- }
602
- function isMissing(error) {
603
- return error instanceof Error && "code" in error && error.code === "ENOENT";
604
- }
605
- function isNoSuchProcess(error) {
606
- return error instanceof Error && "code" in error && error.code === "ESRCH";
607
- }
608
- function unlinkIfExists(path) {
609
- try {
610
- unlinkSync(path);
611
- } catch (error) {
612
- if (!isMissing(error)) throw error;
613
- }
614
- }
615
-
616
- // src/campaign/search-ledger-errors.ts
617
- var SearchLedgerError = class extends ValidationError {
618
- };
619
- var SearchLedgerIntegrityError = class extends SearchLedgerError {
620
- };
621
- var SearchLedgerConflictError = class extends SearchLedgerError {
622
- };
623
-
624
- // src/campaign/search-ledger-file.ts
625
- function appendSearchLedgerLine(path, line) {
626
- mkdirSync(dirname(path), { recursive: true });
627
- const fd = openSync2(path, constants2.O_CREAT | constants2.O_WRONLY | constants2.O_APPEND, 384);
628
- try {
629
- writeAll(fd, Buffer.from(line, "utf8"));
630
- fsyncSync2(fd);
631
- } finally {
632
- closeSync2(fd);
633
- }
634
- fsyncDirectory(dirname(path));
635
- }
636
- function withSearchLedgerFileLock(ledgerPath, run) {
637
- const result = tryWithSearchLedgerFileLock(ledgerPath, run);
638
- if (!result.acquired) {
639
- throw new SearchLedgerIntegrityError(`search ledger lock is held (${ledgerPath})`);
640
- }
641
- return result.value;
642
- }
643
- function tryWithSearchLedgerFileLock(ledgerPath, run) {
644
- mkdirSync(dirname(ledgerPath), { recursive: true });
645
- const lockPath = `${ledgerPath}.lock`;
646
- try {
647
- const acquisition = tryAcquireAtomicFileLock({ lockPath });
648
- if (!acquisition.acquired) return { acquired: false };
649
- try {
650
- return { acquired: true, value: run() };
651
- } finally {
652
- acquisition.lock.release();
653
- }
654
- } catch (error) {
655
- if (error instanceof AtomicFileLockError) {
656
- throw new SearchLedgerIntegrityError(error.message, { cause: error });
657
- }
658
- throw error;
659
- }
660
- }
661
- function writeAll(fd, bytes) {
662
- let offset = 0;
663
- while (offset < bytes.byteLength) {
664
- const written = writeSync(fd, bytes, offset, bytes.byteLength - offset);
665
- if (written <= 0) throw new SearchLedgerIntegrityError("filesystem wrote zero bytes");
666
- offset += written;
667
- }
668
- }
669
- function fsyncDirectory(path) {
670
- const fd = openSync2(path, constants2.O_RDONLY);
671
- try {
672
- fsyncSync2(fd);
673
- } finally {
674
- closeSync2(fd);
675
- }
676
- }
677
-
678
- // src/campaign/storage.ts
679
- function fsCampaignStorage() {
680
- const nodeRequire = createRequire(import.meta.url);
681
- const { existsSync: existsSync3, mkdirSync: mkdirSync2, readFileSync: readFileSync3, statSync, writeFileSync: writeFileSync2 } = nodeRequire(
682
- "node:fs"
683
- );
684
- return {
685
- ensureDir(dir) {
686
- if (!existsSync3(dir)) mkdirSync2(dir, { recursive: true });
687
- },
688
- exists(path) {
689
- return existsSync3(path);
690
- },
691
- read(path) {
692
- try {
693
- return readFileSync3(path, "utf8");
694
- } catch {
695
- return void 0;
696
- }
697
- },
698
- write(path, content) {
699
- writeFileSync2(path, content);
700
- },
701
- append(path, content, expectedBytes) {
702
- const result = tryWithSearchLedgerFileLock(path, () => {
703
- let actualBytes = 0;
704
- try {
705
- actualBytes = statSync(path).size;
706
- } catch (error) {
707
- if (error.code !== "ENOENT") throw error;
708
- }
709
- if (actualBytes !== expectedBytes) return void 0;
710
- appendSearchLedgerLine(path, content);
711
- return expectedBytes + Buffer.byteLength(content);
712
- });
713
- return result.acquired ? result.value : void 0;
714
- }
715
- };
716
- }
717
- function inMemoryCampaignStorage() {
718
- const files = /* @__PURE__ */ new Map();
719
- const dirs = /* @__PURE__ */ new Set();
720
- return {
721
- ensureDir(dir) {
722
- dirs.add(dir);
723
- },
724
- exists(path) {
725
- return files.has(path) || dirs.has(path);
726
- },
727
- read(path) {
728
- const value = files.get(path);
729
- if (value === void 0) return void 0;
730
- return typeof value === "string" ? value : new TextDecoder().decode(value);
731
- },
732
- write(path, content) {
733
- files.set(path, content);
734
- },
735
- append(path, content, expectedBytes) {
736
- const current = files.get(path);
737
- const currentText = current === void 0 ? "" : typeof current === "string" ? current : new TextDecoder().decode(current);
738
- const currentBytes = new TextEncoder().encode(currentText).byteLength;
739
- if (currentBytes !== expectedBytes) return void 0;
740
- files.set(path, `${currentText}${content}`);
741
- return currentBytes + new TextEncoder().encode(content).byteLength;
742
- }
743
- };
744
- }
745
- function createRunCostLedger(input) {
746
- const path = join2(input.runDir, "cost-ledger.jsonl");
747
- input.storage.ensureDir(input.runDir);
748
- return new CostLedger({
749
- costCeilingUsd: input.costCeilingUsd,
750
- persistence: {
751
- read: () => {
752
- const stored = input.storage.read(path);
753
- if (stored === void 0 && input.storage.exists(path)) {
754
- throw new Error(`CostLedger: cannot read existing event log '${path}'`);
755
- }
756
- const events = stored ?? "";
757
- return {
758
- revision: String(new TextEncoder().encode(events).byteLength),
759
- events
760
- };
761
- },
762
- append: (expectedRevision, event) => {
763
- const expectedBytes = Number(expectedRevision);
764
- if (!Number.isSafeInteger(expectedBytes) || expectedBytes < 0) {
765
- throw new Error(`CostLedger: invalid storage revision '${expectedRevision}'`);
766
- }
767
- if (!input.storage.append) {
768
- throw new Error("CostLedger: CampaignStorage.append is required for paid calls");
769
- }
770
- const next = input.storage.append(path, event, expectedBytes);
771
- return next === void 0 ? void 0 : String(next);
772
- }
773
- }
774
- });
775
- }
776
-
777
- // src/campaign/run-campaign.ts
778
- async function runCampaign(opts) {
779
- const seed = opts.seed ?? 42;
780
- const reps = opts.reps ?? 1;
781
- const resumable = opts.resumable ?? true;
782
- const now = opts.now ?? (() => /* @__PURE__ */ new Date());
783
- const judges = opts.judges ?? [];
784
- const storage = opts.storage ?? fsCampaignStorage();
785
- const costPhase = opts.costPhase ?? "campaign";
786
- const dispatchShutdownTimeoutMs = opts.dispatchShutdownTimeoutMs ?? 5e3;
787
- assertCampaignDesign(opts.scenarios, reps);
788
- if (!Number.isSafeInteger(dispatchShutdownTimeoutMs) || dispatchShutdownTimeoutMs <= 0) {
789
- throw new Error("runCampaign: dispatchShutdownTimeoutMs must be a positive safe integer");
790
- }
791
- if (typeof opts.runDir !== "string" || opts.runDir.trim().length === 0) {
792
- throw new Error("runCampaign: runDir is required and must be a non-empty string");
793
- }
794
- opts.runDir = resolveRunDir(opts.runDir, opts.repo);
795
- storage.ensureDir(opts.runDir);
796
- const costLedger = opts.costLedger ?? createRunCostLedger({
797
- storage,
798
- runDir: opts.runDir,
799
- costCeilingUsd: opts.costCeiling
800
- });
801
- if (opts.costCeiling !== void 0 && costLedger.costCeilingUsd !== opts.costCeiling) {
802
- throw new Error("runCampaign: costCeiling must match the shared CostLedger ceiling");
803
- }
804
- const maxConcurrency = opts.maxConcurrency ?? 2;
805
- const manifestHash = computeManifestHash({
806
- scenarios: opts.scenarios,
807
- judges,
808
- dispatchRef: dispatchRefFor(opts.dispatch, opts.dispatchRef),
809
- seed,
810
- reps
811
- });
812
- const splitDigest = campaignSplitDigest(opts.scenarios, reps);
813
- const startedAt = now();
814
- const runAttemptId = globalThis.crypto.randomUUID();
815
- const cells = [];
816
- const artifactsByPath = {};
817
- const schedule = buildCellSchedule(opts.scenarios, seed, reps);
818
- const campaignAbort = new AbortController();
819
- const onOwnerAbort = () => campaignAbort.abort(opts.signal?.reason);
820
- if (opts.signal?.aborted) campaignAbort.abort(opts.signal.reason);
821
- else opts.signal?.addEventListener("abort", onOwnerAbort, { once: true });
822
- const campaignSignal = campaignAbort.signal;
823
- const lanes = [];
824
- let nextIdx = 0;
825
- const cellsRef = cells;
826
- let firstLaneError;
827
- let firstCellFailure;
828
- for (let i = 0; i < maxConcurrency; i++) {
829
- lanes.push(
830
- (async () => {
831
- try {
832
- while (true) {
833
- if (campaignSignal.aborted) return;
834
- const myIdx = nextIdx++;
835
- if (myIdx >= schedule.length) return;
836
- const slot = schedule[myIdx];
837
- const result = await executeCell({
838
- slot,
839
- opts,
840
- manifestHash,
841
- resumable,
842
- now,
843
- storage,
844
- buildTraceWriter: opts.buildTraceWriter ?? defaultBuildTraceWriter(storage),
845
- signal: campaignSignal,
846
- dispatchTimeoutMs: opts.dispatchTimeoutMs,
847
- dispatchShutdownTimeoutMs,
848
- costLedger,
849
- costPhase,
850
- runAttemptId,
851
- onFailure: opts.abortOnCellError ? (failure) => {
852
- if (firstCellFailure === void 0) {
853
- firstCellFailure = failure;
854
- campaignAbort.abort(failure.cause);
855
- }
856
- } : void 0
857
- });
858
- cellsRef.push(result.cell);
859
- Object.assign(artifactsByPath, result.artifactsByPath);
860
- if (opts.labeledStore && opts.labeledStore !== "off" && !result.cell.error) {
861
- await captureToStore({
862
- store: opts.labeledStore,
863
- cell: result.cell,
864
- scenario: slot.scenario,
865
- opts,
866
- now
867
- }).catch((err) => {
868
- console.warn(
869
- `[runCampaign] capture failed for ${result.cell.cellId}: ${err instanceof Error ? err.message : String(err)}`
870
- );
871
- });
872
- }
873
- if (opts.abortOnCellError && result.failure) {
874
- return;
875
- }
876
- }
877
- } catch (error) {
878
- if (firstLaneError === void 0) {
879
- firstLaneError = error;
880
- campaignAbort.abort(error);
881
- }
882
- throw error;
883
- }
884
- })()
885
- );
886
- }
887
- const laneResults = await Promise.allSettled(lanes);
888
- opts.signal?.removeEventListener("abort", onOwnerAbort);
889
- const failedLane = laneResults.find(
890
- (result) => result.status === "rejected"
891
- );
892
- if (firstCellFailure) throw firstCellFailure.cause;
893
- if (failedLane) throw firstLaneError ?? failedLane.reason;
894
- const endedAt = now();
895
- cellsRef.sort((a, b) => a.cellId.localeCompare(b.cellId));
896
- const campaignCost = costLedger.summary({ tags: { runDir: opts.runDir } });
897
- const aggregates = computeAggregates(
898
- cellsRef,
899
- judges,
900
- seed,
901
- campaignCost
902
- );
903
- return {
904
- manifestHash,
905
- splitDigest,
906
- seed,
907
- reps,
908
- startedAt: startedAt.toISOString(),
909
- endedAt: endedAt.toISOString(),
910
- durationMs: endedAt.getTime() - startedAt.getTime(),
911
- cells: cellsRef,
912
- aggregates,
913
- runDir: opts.runDir,
914
- artifactsByPath,
915
- scenarios: opts.scenarios.map(campaignScenarioIdentity)
916
- };
917
- }
918
- async function executeCell(args) {
919
- const storage = args.storage;
920
- const cellDir = join3(args.opts.runDir, args.slot.cellId.replace(/[^a-zA-Z0-9_-]/g, "_"));
921
- storage.ensureDir(cellDir);
922
- const stableCostTags = {
923
- ...args.opts.costTags ?? {},
924
- runDir: args.opts.runDir,
925
- cellId: args.slot.cellId,
926
- scenarioId: args.slot.scenario.id,
927
- rep: String(args.slot.rep)
928
- };
929
- const costTags = { ...stableCostTags, runAttemptId: args.runAttemptId };
930
- const cachePath = join3(cellDir, "cached-result.json");
931
- if (args.resumable) {
932
- const cached = readCachedCell({
933
- storage,
934
- cachePath,
935
- cellId: args.slot.cellId,
936
- manifestHash: args.manifestHash
937
- });
938
- if (cached.status === "hit") {
939
- enforceDispatchUsage(cached.cell, args.opts.expectUsage ?? "warn");
940
- const cachedHasUsage = cached.cell.costUsd > 0 || cached.cell.tokenUsage.input > 0 || cached.cell.tokenUsage.output > 0;
941
- if (cached.cell.costCallIds === void 0) {
942
- if (cachedHasUsage || Object.keys(cached.cell.judgeScores).length > 0) {
943
- throw new CostAccountingIncompleteError(
944
- `runCampaign: cached cell '${args.slot.cellId}' does not identify its ledger receipts`
945
- );
946
- }
947
- } else if (!Array.isArray(cached.cell.costCallIds) || cached.cell.costCallIds.some(
948
- (callId) => typeof callId !== "string" || callId.trim().length === 0
949
- ) || new Set(cached.cell.costCallIds).size !== cached.cell.costCallIds.length) {
950
- throw new CostAccountingIncompleteError(
951
- `runCampaign: cached cell '${args.slot.cellId}' has invalid ledger receipt IDs`
952
- );
953
- } else {
954
- const restoredCallIds = new Set(
955
- args.costLedger.list({ tags: stableCostTags }).map((receipt) => receipt.callId)
956
- );
957
- const missingCallIds = cached.cell.costCallIds.filter(
958
- (callId) => !restoredCallIds.has(callId)
959
- );
960
- if (missingCallIds.length > 0) {
961
- throw new CostAccountingIncompleteError(
962
- `runCampaign: cached cell '${args.slot.cellId}' is missing ledger receipt(s): ${missingCallIds.join(", ")}`
963
- );
964
- }
965
- }
966
- return { cell: { ...cached.cell, cached: true }, artifactsByPath: {} };
967
- }
968
- }
969
- const startMs = Date.now();
970
- const trace = args.buildTraceWriter(args.slot.cellId, cellDir);
971
- const artifactsByPath = {};
972
- let paidCallStarted = false;
973
- const artifacts = {
974
- async write(path, content) {
975
- const fullPath = join3(cellDir, path);
976
- storage.ensureDir(join3(fullPath, ".."));
977
- storage.write(fullPath, content);
978
- artifactsByPath[`${args.slot.cellId}/${path}`] = fullPath;
979
- return fullPath;
980
- },
981
- async writeJson(path, value) {
982
- return artifacts.write(path, JSON.stringify(value, null, 2));
983
- }
984
- };
985
- const cost = {
986
- async runPaidCall(input) {
987
- paidCallStarted = true;
988
- const result = await args.costLedger.runPaidCall({
989
- ...input,
990
- channel: input.channel ?? "agent",
991
- phase: args.costPhase,
992
- actor: input.actor,
993
- tags: costTags,
994
- signal: cellAbort.signal
995
- });
996
- if (result.receipt) {
997
- trace.span(`cost.${result.receipt.actor}`, { amountUsd: result.receipt.costUsd }).end();
998
- }
999
- return result;
1000
- }
1001
- };
1002
- const placement = args.opts.cellPlacement?.({
1003
- scenario: args.slot.scenario,
1004
- rep: args.slot.rep
1005
- });
1006
- const cellAbort = new AbortController();
1007
- const onCampaignAbort = () => cellAbort.abort(args.signal.reason);
1008
- if (args.signal.aborted) cellAbort.abort(args.signal.reason);
1009
- else args.signal.addEventListener("abort", onCampaignAbort, { once: true });
1010
- const ctx = {
1011
- cellId: args.slot.cellId,
1012
- rep: args.slot.rep,
1013
- seed: args.slot.cellSeed,
1014
- signal: cellAbort.signal,
1015
- trace,
1016
- artifacts,
1017
- cost,
1018
- placement
1019
- };
1020
- let artifact;
1021
- let errorMessage;
1022
- let failure;
1023
- let fatalCellError;
1024
- let dispatched;
1025
- const timeoutMs = args.dispatchTimeoutMs;
1026
- let timeoutTimer;
1027
- let removeAbortListener = () => void 0;
1028
- try {
1029
- dispatched = Promise.resolve(args.opts.dispatch(args.slot.scenario, ctx));
1030
- const aborted = new Promise((_resolve, reject) => {
1031
- const rejectAbort = () => {
1032
- const reason = cellAbort.signal.reason;
1033
- reject(reason instanceof Error ? reason : new Error(String(reason ?? "dispatch aborted")));
1034
- };
1035
- if (cellAbort.signal.aborted) {
1036
- rejectAbort();
1037
- return;
1038
- }
1039
- cellAbort.signal.addEventListener("abort", rejectAbort, { once: true });
1040
- removeAbortListener = () => cellAbort.signal.removeEventListener("abort", rejectAbort);
1041
- });
1042
- if (timeoutMs !== void 0 && timeoutMs > 0) {
1043
- artifact = await Promise.race([
1044
- dispatched,
1045
- aborted,
1046
- new Promise((_, reject) => {
1047
- timeoutTimer = setTimeout(() => {
1048
- const timeoutError = new Error(
1049
- `dispatch exceeded ${timeoutMs}ms for cell '${args.slot.cellId}' \u2014 aborted and failed loud (no silent hang)`
1050
- );
1051
- reject(timeoutError);
1052
- cellAbort.abort(timeoutError);
1053
- }, timeoutMs);
1054
- if (typeof timeoutTimer.unref === "function")
1055
- timeoutTimer.unref();
1056
- })
1057
- ]);
1058
- } else {
1059
- artifact = await Promise.race([dispatched, aborted]);
1060
- }
1061
- } catch (err) {
1062
- errorMessage = err instanceof Error ? err.message : String(err);
1063
- failure = { stage: "dispatch", cause: err };
1064
- } finally {
1065
- if (timeoutTimer) clearTimeout(timeoutTimer);
1066
- removeAbortListener();
1067
- args.signal.removeEventListener("abort", onCampaignAbort);
1068
- }
1069
- if (dispatched) {
1070
- const dispatchSettled = await settlesWithin(dispatched, args.dispatchShutdownTimeoutMs);
1071
- if (!dispatchSettled) {
1072
- await trace.flush();
1073
- throw new CostAccountingIncompleteError(
1074
- `dispatch for cell '${args.slot.cellId}' ignored cancellation and did not stop within ${args.dispatchShutdownTimeoutMs}ms; no campaign result was produced`
1075
- );
1076
- }
1077
- }
1078
- if (paidCallStarted) {
1079
- if (typeof args.costLedger.waitForIdle !== "function") {
1080
- await trace.flush();
1081
- throw new CostAccountingIncompleteError(
1082
- `cost ledger for cell '${args.slot.cellId}' cannot prove that paid calls stopped`
1083
- );
1084
- }
1085
- const paidCallsSettled = await args.costLedger.waitForIdle({
1086
- timeoutMs: args.dispatchShutdownTimeoutMs,
1087
- filter: { channel: "agent", phase: args.costPhase, tags: costTags }
1088
- });
1089
- if (!paidCallsSettled) {
1090
- await trace.flush();
1091
- throw new CostAccountingIncompleteError(
1092
- `paid calls for cell '${args.slot.cellId}' did not settle within ${args.dispatchShutdownTimeoutMs}ms; no campaign result was produced`
1093
- );
1094
- }
1095
- }
1096
- const agentReceipts = args.costLedger.list({ channel: "agent", tags: costTags });
1097
- const agentCost = args.costLedger.summary({ channel: "agent", tags: costTags });
1098
- const tokenUsage = {
1099
- input: agentCost.inputTokens,
1100
- output: agentCost.outputTokens,
1101
- ...agentCost.cachedTokens > 0 ? { cached: agentCost.cachedTokens } : {}
1102
- };
1103
- const resolvedModel = agentReceipts.at(-1)?.model;
1104
- const dispatchResult = {
1105
- cellId: args.slot.cellId,
1106
- artifact,
1107
- error: errorMessage,
1108
- costUsd: agentCost.totalCostUsd,
1109
- tokenUsage
1110
- };
1111
- try {
1112
- enforceDispatchUsage(dispatchResult, args.opts.expectUsage ?? "warn");
1113
- } catch (error) {
1114
- await trace.flush();
1115
- throw error;
1116
- }
1117
- const judgeScores = {};
1118
- if (artifact !== void 0) {
1119
- for (const judge of args.opts.judges ?? []) {
1120
- if (judge.appliesTo && !judge.appliesTo(args.slot.scenario)) continue;
1121
- try {
1122
- const score = await runJudgeCell(judge, {
1123
- artifact,
1124
- scenario: args.slot.scenario,
1125
- signal: args.signal,
1126
- costLedger: args.costLedger,
1127
- costPhase: args.costPhase,
1128
- costTags
1129
- });
1130
- judgeScores[judge.name] = score;
1131
- } catch (err) {
1132
- errorMessage = `judge '${judge.name}' failed: ${err instanceof Error ? err.message : String(err)}`;
1133
- failure = { stage: "judge", judge: judge.name, cause: err };
1134
- if (err instanceof CostAccountingIncompleteError) fatalCellError = err;
1135
- break;
1136
- }
1137
- }
1138
- }
1139
- if (failure) {
1140
- await waitForFailedCellCostSettlement({
1141
- costLedger: args.costLedger,
1142
- costPhase: args.costPhase,
1143
- costTags,
1144
- cellId: args.slot.cellId,
1145
- timeoutMs: args.dispatchShutdownTimeoutMs
1146
- });
1147
- }
1148
- const costCallIds = args.costLedger.list({ tags: costTags }).map((receipt) => receipt.callId).sort();
1149
- const cell = {
1150
- manifestHash: args.manifestHash,
1151
- cellId: args.slot.cellId,
1152
- scenarioId: args.slot.scenario.id,
1153
- rep: args.slot.rep,
1154
- artifact: artifact ?? null,
1155
- judgeScores,
1156
- costUsd: agentCost.totalCostUsd,
1157
- costEstimated: agentReceipts.some(
1158
- (receipt) => receipt.actualCostUsd === void 0 && !receipt.costUnknown
1159
- ),
1160
- costCallIds,
1161
- tokenUsage,
1162
- ...resolvedModel ? { resolvedModel } : {},
1163
- durationMs: Date.now() - startMs,
1164
- seed: args.slot.cellSeed,
1165
- cached: false,
1166
- ...failure ? { errorStage: failure.stage } : {},
1167
- ...failure?.judge ? { errorJudge: failure.judge } : {},
1168
- error: errorMessage
1169
- };
1170
- if (failure) {
1171
- const failurePath = join3(cellDir, "failure-receipt.json");
1172
- const receipt = {
1173
- schemaVersion: 1,
1174
- runAttemptId: args.runAttemptId,
1175
- recordedAt: args.now().toISOString(),
1176
- failure: {
1177
- stage: failure.stage,
1178
- ...failure.judge ? { judge: failure.judge } : {},
1179
- error: serializeCellError(failure.cause)
1180
- },
1181
- cell,
1182
- cost: args.costLedger.summary({ phase: args.costPhase, tags: costTags })
1183
- };
1184
- storage.write(failurePath, JSON.stringify(receipt, null, 2));
1185
- artifactsByPath[`${args.slot.cellId}/failure-receipt.json`] = failurePath;
1186
- args.onFailure?.(failure);
1187
- }
1188
- await trace.flush();
1189
- if (!errorMessage && args.resumable) {
1190
- storage.write(cachePath, JSON.stringify(cell));
1191
- }
1192
- if (fatalCellError !== void 0) throw fatalCellError;
1193
- return { cell, artifactsByPath, ...failure ? { failure } : {} };
1194
- }
1195
- async function waitForFailedCellCostSettlement(input) {
1196
- const filter = { phase: input.costPhase, tags: input.costTags };
1197
- if (input.costLedger.summary(filter).pendingCalls === 0) return;
1198
- if (typeof input.costLedger.waitForIdle !== "function") {
1199
- throw new CostAccountingIncompleteError(
1200
- `cost ledger for failed cell '${input.cellId}' cannot prove that paid calls stopped`
1201
- );
1202
- }
1203
- const settled = await input.costLedger.waitForIdle({
1204
- timeoutMs: input.timeoutMs,
1205
- filter
1206
- });
1207
- if (!settled || input.costLedger.summary(filter).pendingCalls > 0) {
1208
- throw new CostAccountingIncompleteError(
1209
- `paid calls for failed cell '${input.cellId}' did not settle within ${input.timeoutMs}ms; no complete failure receipt was produced`
1210
- );
1211
- }
1212
- }
1213
- function serializeCellError(error) {
1214
- if (error instanceof Error) {
1215
- return {
1216
- name: error.name,
1217
- message: error.message,
1218
- ...error.stack ? { stack: error.stack } : {}
1219
- };
1220
- }
1221
- return { name: "NonErrorThrown", message: String(error) };
1222
- }
1223
- function settlesWithin(promise, timeoutMs) {
1224
- return new Promise((resolve) => {
1225
- let completed = false;
1226
- let timer;
1227
- const finish = (value) => {
1228
- if (completed) return;
1229
- completed = true;
1230
- if (timer) clearTimeout(timer);
1231
- resolve(value);
1232
- };
1233
- timer = setTimeout(() => finish(false), timeoutMs);
1234
- promise.then(
1235
- () => finish(true),
1236
- () => finish(true)
1237
- );
1238
- });
1239
- }
1240
- function planCampaignRun(opts) {
1241
- const seed = opts.seed ?? 42;
1242
- const reps = opts.reps ?? 1;
1243
- const resumable = opts.resumable ?? true;
1244
- const storage = opts.storage ?? fsCampaignStorage();
1245
- assertCampaignDesign(opts.scenarios, reps);
1246
- if (typeof opts.runDir !== "string" || opts.runDir.trim().length === 0) {
1247
- throw new Error("planCampaignRun: runDir is required and must be a non-empty string");
1248
- }
1249
- opts.runDir = resolveRunDir(opts.runDir, opts.repo);
1250
- const manifestHash = computeManifestHash({
1251
- scenarios: opts.scenarios,
1252
- judges: opts.judges ?? [],
1253
- dispatchRef: dispatchRefFor(opts.dispatch, opts.dispatchRef),
1254
- seed,
1255
- reps
1256
- });
1257
- const splitDigest = campaignSplitDigest(opts.scenarios, reps);
1258
- const cells = buildCellSchedule(opts.scenarios, seed, reps).map((slot) => {
1259
- const cachePath = join3(
1260
- opts.runDir,
1261
- slot.cellId.replace(/[^a-zA-Z0-9_-]/g, "_"),
1262
- "cached-result.json"
1263
- );
1264
- if (!resumable) {
1265
- return {
1266
- cellId: slot.cellId,
1267
- scenarioId: slot.scenario.id,
1268
- rep: slot.rep,
1269
- seed: slot.cellSeed,
1270
- cachePath,
1271
- status: "run",
1272
- reason: "resumable-off"
1273
- };
1274
- }
1275
- const cached = readCachedCell({
1276
- storage,
1277
- cachePath,
1278
- cellId: slot.cellId,
1279
- manifestHash
1280
- });
1281
- if (cached.status === "hit") {
1282
- return {
1283
- cellId: slot.cellId,
1284
- scenarioId: slot.scenario.id,
1285
- rep: slot.rep,
1286
- seed: slot.cellSeed,
1287
- cachePath,
1288
- status: "cached"
1289
- };
1290
- }
1291
- return {
1292
- cellId: slot.cellId,
1293
- scenarioId: slot.scenario.id,
1294
- rep: slot.rep,
1295
- seed: slot.cellSeed,
1296
- cachePath,
1297
- status: "run",
1298
- reason: cached.reason
1299
- };
1300
- });
1301
- const cellsCached = cells.filter((cell) => cell.status === "cached").length;
1302
- return {
1303
- manifestHash,
1304
- splitDigest,
1305
- totalCells: cells.length,
1306
- cellsCached,
1307
- cellsToRun: cells.length - cellsCached,
1308
- cells
1309
- };
1310
- }
1311
- function enforceDispatchUsage(cell, mode) {
1312
- if (mode === "off") return;
1313
- if (cell.artifact === null || cell.artifact === void 0) return;
1314
- const zeroTokens = cell.tokenUsage.input === 0 && cell.tokenUsage.output === 0;
1315
- if (cell.costUsd !== 0 || !zeroTokens) return;
1316
- const msg = `cell '${cell.cellId}' produced an artifact but reported zero cost and zero tokens \u2014 the dispatch made no paid call through ctx.cost.runPaidCall (a stub cell)`;
1317
- if (mode === "assert") {
1318
- const report = {
1319
- totalRecords: 1,
1320
- stubRecords: 1,
1321
- realRecords: 0,
1322
- uncostedRecords: 0,
1323
- totalInputTokens: 0,
1324
- totalOutputTokens: 0,
1325
- totalCostUsd: 0,
1326
- verdict: "stub",
1327
- diagnosis: msg
1328
- };
1329
- throw new BackendIntegrityError(`expectUsage: ${msg}`, report);
1330
- }
1331
- console.warn(`[runCampaign] expectUsage: ${msg}`);
1332
- }
1333
- async function runJudgeCell(judge, input) {
1334
- const previousJudgeCalls = new Set(
1335
- input.costLedger?.list({ channel: "judge", tags: input.costTags }).map((receipt) => receipt.callId) ?? []
1336
- );
1337
- try {
1338
- const score = await judge.score(input);
1339
- assertReportedJudgeCallRecorded(judge.name, score, input, previousJudgeCalls);
1340
- return score;
1341
- } catch (error) {
1342
- assertReportedJudgeCallRecorded(judge.name, error, input, previousJudgeCalls, error);
1343
- throw error;
1344
- }
1345
- }
1346
- function assertReportedJudgeCallRecorded(judgeName, value, input, previousCallIds, cause) {
1347
- if (!hasLlmCall(value)) return;
1348
- const recorded = input.costLedger?.list({ channel: "judge", tags: input.costTags }).some((receipt) => !previousCallIds.has(receipt.callId));
1349
- if (recorded) return;
1350
- throw new CostAccountingIncompleteError(
1351
- `runCampaign: judge '${judgeName}' reported a paid LLM call without a CostLedger receipt`,
1352
- cause === void 0 ? void 0 : { cause }
1353
- );
1354
- }
1355
- function hasLlmCall(value) {
1356
- return typeof value === "object" && value !== null && "llmCall" in value && value.llmCall !== void 0;
1357
- }
1358
- function defaultBuildTraceWriter(storage) {
1359
- return (cellId, dir) => {
1360
- const spans = [];
1361
- return {
1362
- span(name, attributes) {
1363
- const startMs = Date.now();
1364
- const record = { name, cellId, startMs, ...attributes ?? {} };
1365
- const finish = {
1366
- end(endAttrs) {
1367
- record.durationMs = Date.now() - startMs;
1368
- if (endAttrs) Object.assign(record, endAttrs);
1369
- spans.push(record);
1370
- },
1371
- setAttribute(key, value) {
1372
- record[key] = value;
1373
- }
1374
- };
1375
- return finish;
1376
- },
1377
- async flush() {
1378
- storage.write(join3(dir, "spans.jsonl"), spans.map((s) => JSON.stringify(s)).join("\n"));
1379
- }
1380
- };
1381
- };
1382
- }
1383
- function buildCellSchedule(scenarios, seed, reps) {
1384
- const schedule = [];
1385
- const groupIndexes = /* @__PURE__ */ new Map();
1386
- let nextGroupIndex = 0;
1387
- for (const scenario of scenarios) {
1388
- let groupIndex;
1389
- if (scenario.seedGroup === void 0) {
1390
- groupIndex = nextGroupIndex;
1391
- nextGroupIndex += 1;
1392
- } else {
1393
- const existing = groupIndexes.get(scenario.seedGroup);
1394
- if (existing !== void 0) {
1395
- groupIndex = existing;
1396
- } else {
1397
- groupIndex = nextGroupIndex;
1398
- nextGroupIndex += 1;
1399
- groupIndexes.set(scenario.seedGroup, groupIndex);
1400
- }
1401
- }
1402
- for (let rep = 0; rep < reps; rep++) {
1403
- const cellId = `${scenario.id}:${rep}`;
1404
- const cellSeed = seed + groupIndex * reps + rep;
1405
- schedule.push({ scenario, rep, cellId, cellSeed });
1406
- }
1407
- }
1408
- return schedule;
1409
- }
1410
- function dispatchRefFor(dispatch, override) {
1411
- const ref = override ?? dispatch?.name ?? "anonymous";
1412
- if (typeof ref !== "string" || ref.trim().length === 0) {
1413
- throw new Error("runCampaign: dispatchRef must be a non-empty string when provided");
1414
- }
1415
- return ref;
1416
- }
1417
- function readCachedCell(args) {
1418
- const raw = args.storage.read(args.cachePath);
1419
- if (raw === void 0) return { status: "miss", reason: "missing" };
1420
- try {
1421
- const cached = JSON.parse(raw);
1422
- if (cached.cellId !== args.cellId) return { status: "miss", reason: "cell-mismatch" };
1423
- if (cached.manifestHash !== args.manifestHash) {
1424
- return { status: "miss", reason: "manifest-mismatch" };
1425
- }
1426
- return { status: "hit", cell: cached };
1427
- } catch {
1428
- return { status: "miss", reason: "corrupt" };
1429
- }
1430
- }
1431
- async function captureToStore(args) {
1432
- await args.store.observe({
1433
- scenario: args.scenario,
1434
- artifact: args.cell.artifact,
1435
- judgeScores: args.cell.judgeScores,
1436
- source: args.opts.captureSource ?? "eval-run",
1437
- sourceVersionHash: args.opts.captureSourceVersionHash ?? "unknown",
1438
- capturedAt: args.now().toISOString(),
1439
- redactionStatus: "raw"
1440
- });
1441
- }
1442
- function computeManifestHash(input) {
1443
- return contentHash({
1444
- scenarios: input.scenarios,
1445
- judges: input.judges.map((judge) => ({
1446
- name: judge.name,
1447
- dims: judge.dimensions,
1448
- version: judgeVersionFor(judge)
1449
- })),
1450
- dispatch: input.dispatchRef,
1451
- seed: input.seed,
1452
- reps: input.reps
1453
- });
1454
- }
1455
- function judgeVersionFor(judge) {
1456
- if (judge.judgeVersion !== void 0) {
1457
- const version = judge.judgeVersion.trim();
1458
- if (version.length === 0) {
1459
- throw new Error(`runCampaign: judge '${judge.name}' has an empty judgeVersion`);
1460
- }
1461
- return version;
1462
- }
1463
- return contentHash({
1464
- score: judge.score.toString(),
1465
- appliesTo: judge.appliesTo?.toString() ?? null
1466
- });
1467
- }
1468
- function computeAggregates(cells, judges, seed, cost) {
1469
- const cellQuality = cells.map((cell) => ({
1470
- cell,
1471
- quality: projectCampaignCellQuality(cell)
1472
- }));
1473
- const byJudge = {};
1474
- for (const judge of judges) {
1475
- const scores = [];
1476
- for (const { quality } of cellQuality) {
1477
- const s = quality.successfulJudgeScores[judge.name];
1478
- if (s !== void 0) scores.push(s.composite);
1479
- }
1480
- if (scores.length > 0) byJudge[judge.name] = aggregate(scores, seed);
1481
- }
1482
- const byScenario = {};
1483
- const scenarioGroups = /* @__PURE__ */ new Map();
1484
- for (const { cell, quality } of cellQuality) {
1485
- const score = quality.score;
1486
- if (score === void 0) continue;
1487
- const arr = scenarioGroups.get(cell.scenarioId) ?? [];
1488
- arr.push(score);
1489
- scenarioGroups.set(cell.scenarioId, arr);
1490
- }
1491
- for (const [scenarioId, samples] of scenarioGroups) {
1492
- const ag = aggregate(samples, seed);
1493
- byScenario[scenarioId] = { meanComposite: ag.mean, ci95: ag.ci95, n: ag.n };
1494
- }
1495
- const dispatchFailures = cells.filter((cell) => cell.errorStage === "dispatch");
1496
- const judgeFailures = cellQuality.filter(({ cell, quality }) => cell.errorStage === "judge" || quality.failedJudges.length > 0).map(({ cell }) => cell);
1497
- const unclassifiedFailures = cells.filter(
1498
- (cell) => Boolean(cell.error) && !cell.error?.startsWith("skipped:") && cell.errorStage === void 0
1499
- );
1500
- return {
1501
- byJudge,
1502
- byScenario,
1503
- cost,
1504
- totalCostUsd: cost.totalCostUsd,
1505
- cellsExecuted: cells.filter(
1506
- (cell) => !cell.error?.startsWith("skipped:") && cell.errorStage !== "dispatch" && !(cell.error && cell.errorStage === void 0)
1507
- ).length,
1508
- cellsSkipped: cells.filter((c) => c.error?.startsWith("skipped:")).length,
1509
- cellsCached: cells.filter((c) => c.cached).length,
1510
- cellsFailed: (/* @__PURE__ */ new Set([...dispatchFailures, ...judgeFailures, ...unclassifiedFailures])).size,
1511
- cellsDispatchFailed: dispatchFailures.length,
1512
- cellsJudgeFailed: judgeFailures.length,
1513
- cellsUnclassifiedFailed: unclassifiedFailures.length
1514
- };
1515
- }
1516
- function aggregate(samples, seed) {
1517
- const n = samples.length;
1518
- if (n === 0) throw new Error("aggregate requires at least one finite score");
1519
- const mean = samples.reduce((a, b) => a + b, 0) / n;
1520
- const variance = samples.reduce((a, b) => a + (b - mean) ** 2, 0) / Math.max(1, n - 1);
1521
- const stdev = Math.sqrt(variance);
1522
- const ci = confidenceInterval(samples, 0.95, { seed, resamples: 1e3 });
1523
- return { mean, stdev, ci95: [ci.lower, ci.upper], n };
1524
- }
1525
-
1526
- export {
1527
- BackendIntegrityError,
1528
- summarizeBackendIntegrity,
1529
- summarizeAgentReceiptIntegrity,
1530
- assertRealBackend,
1531
- assertRealAgentReceipts,
1532
- canonicalJson,
1533
- contentHash,
1534
- inMemoryVerdictCache,
1535
- fileVerdictCache,
1536
- cachedJudge,
1537
- assertCampaignDesign,
1538
- campaignScenarioIdentity,
1539
- campaignSplitDigestFromIdentities,
1540
- campaignSplitDigest,
1541
- assertCampaignSplitIdentity,
1542
- campaignCoverage,
1543
- formatCoverageFailures,
1544
- tangleTracesRoot,
1545
- resolveRunDir,
1546
- probeAtomicFileLock,
1547
- tryAcquireAtomicFileLock,
1548
- SearchLedgerError,
1549
- SearchLedgerIntegrityError,
1550
- SearchLedgerConflictError,
1551
- appendSearchLedgerLine,
1552
- withSearchLedgerFileLock,
1553
- fsCampaignStorage,
1554
- inMemoryCampaignStorage,
1555
- createRunCostLedger,
1556
- runCampaign,
1557
- planCampaignRun
1558
- };
1559
- //# sourceMappingURL=chunk-EZJEIH2R.js.map