@tangle-network/agent-eval 0.129.0 → 0.130.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (427) hide show
  1. package/CHANGELOG.md +14 -0
  2. package/README.md +1 -0
  3. package/dist/active-curriculum-C4mk67HP.js +214 -0
  4. package/dist/active-curriculum-C4mk67HP.js.map +1 -0
  5. package/dist/adversarial-smnADNFS.d.ts +21 -0
  6. package/dist/adversarial-smnADNFS.d.ts.map +1 -0
  7. package/dist/analyst/index.d.ts +81 -2872
  8. package/dist/analyst/index.d.ts.map +1 -0
  9. package/dist/analyst/index.js +319 -360
  10. package/dist/analyst/index.js.map +1 -1
  11. package/dist/analyst-BkTS3C58.d.ts +89 -0
  12. package/dist/analyst-BkTS3C58.d.ts.map +1 -0
  13. package/dist/analyst-LsnNpSkm.js +152 -0
  14. package/dist/analyst-LsnNpSkm.js.map +1 -0
  15. package/dist/analyze-runs-C1CavBMk.js +1067 -0
  16. package/dist/analyze-runs-C1CavBMk.js.map +1 -0
  17. package/dist/analyze-runs-FsgYCinh.d.ts +72 -0
  18. package/dist/analyze-runs-FsgYCinh.d.ts.map +1 -0
  19. package/dist/attribute-vocabulary-DLJ6303h.d.ts +54 -0
  20. package/dist/attribute-vocabulary-DLJ6303h.d.ts.map +1 -0
  21. package/dist/authenticity/index.d.ts +81 -79
  22. package/dist/authenticity/index.d.ts.map +1 -0
  23. package/dist/authenticity/index.js +209 -193
  24. package/dist/authenticity/index.js.map +1 -1
  25. package/dist/baseline-HsBvw_dk.js +550 -0
  26. package/dist/baseline-HsBvw_dk.js.map +1 -0
  27. package/dist/baseline-hG3K85h4.d.ts +125 -0
  28. package/dist/baseline-hG3K85h4.d.ts.map +1 -0
  29. package/dist/belief-state/index.d.ts +448 -1188
  30. package/dist/belief-state/index.d.ts.map +1 -0
  31. package/dist/belief-state/index.js +1617 -1709
  32. package/dist/belief-state/index.js.map +1 -1
  33. package/dist/benchmarks/index.d.ts +2 -891
  34. package/dist/benchmarks/index.js +2 -60
  35. package/dist/benchmarks-DviOvUNr.js +754 -0
  36. package/dist/benchmarks-DviOvUNr.js.map +1 -0
  37. package/dist/builder-eval/index.d.ts +150 -662
  38. package/dist/builder-eval/index.d.ts.map +1 -0
  39. package/dist/builder-eval/index.js +356 -345
  40. package/dist/builder-eval/index.js.map +1 -1
  41. package/dist/calibration-CNWWA6K8.js +94 -0
  42. package/dist/calibration-CNWWA6K8.js.map +1 -0
  43. package/dist/campaign/index.d.ts +5 -6381
  44. package/dist/campaign/index.js +3 -213
  45. package/dist/campaign-CBKZvQ1H.js +3885 -0
  46. package/dist/campaign-CBKZvQ1H.js.map +1 -0
  47. package/dist/cli.d.ts +1 -1
  48. package/dist/cli.js +164 -175
  49. package/dist/cli.js.map +1 -1
  50. package/dist/client-C97NMzqi.d.ts +581 -0
  51. package/dist/client-C97NMzqi.d.ts.map +1 -0
  52. package/dist/client-CYzbdJOZ.js +637 -0
  53. package/dist/client-CYzbdJOZ.js.map +1 -0
  54. package/dist/code-agent-session-BjkMTQ7H.js +1390 -0
  55. package/dist/code-agent-session-BjkMTQ7H.js.map +1 -0
  56. package/dist/code-agent-session-DqqgOJaz.d.ts +143 -0
  57. package/dist/code-agent-session-DqqgOJaz.d.ts.map +1 -0
  58. package/dist/concurrency-DIxRZF_J.js +85 -0
  59. package/dist/concurrency-DIxRZF_J.js.map +1 -0
  60. package/dist/contract/index.d.ts +645 -5565
  61. package/dist/contract/index.d.ts.map +1 -0
  62. package/dist/contract/index.js +1730 -1938
  63. package/dist/contract/index.js.map +1 -1
  64. package/dist/control.d.ts +3 -1030
  65. package/dist/control.js +2 -33
  66. package/dist/cost-ledger-DIgQUFZZ.js +801 -0
  67. package/dist/cost-ledger-DIgQUFZZ.js.map +1 -0
  68. package/dist/cost-ledger-Dye6jCgg.d.ts +236 -0
  69. package/dist/cost-ledger-Dye6jCgg.d.ts.map +1 -0
  70. package/dist/dataset-BvtnC8Dc.d.ts +115 -0
  71. package/dist/dataset-BvtnC8Dc.d.ts.map +1 -0
  72. package/dist/default-registry-C-vFCSEc.js +2579 -0
  73. package/dist/default-registry-C-vFCSEc.js.map +1 -0
  74. package/dist/default-registry-CNPo-Vsb.d.ts +540 -0
  75. package/dist/default-registry-CNPo-Vsb.d.ts.map +1 -0
  76. package/dist/emitter-CPBAhxum.js +266 -0
  77. package/dist/emitter-CPBAhxum.js.map +1 -0
  78. package/dist/emitter-DGQGoLyj.d.ts +113 -0
  79. package/dist/emitter-DGQGoLyj.d.ts.map +1 -0
  80. package/dist/errors-8YnH8WlF.js +64 -0
  81. package/dist/errors-8YnH8WlF.js.map +1 -0
  82. package/dist/errors-CEk209JS.d.ts +76 -0
  83. package/dist/errors-CEk209JS.d.ts.map +1 -0
  84. package/dist/eval-campaign-DEm6c8ru.js +349 -0
  85. package/dist/eval-campaign-DEm6c8ru.js.map +1 -0
  86. package/dist/execution-tracks-CpgFPpS5.js +93 -0
  87. package/dist/execution-tracks-CpgFPpS5.js.map +1 -0
  88. package/dist/exporters-q9iL-2Jf.js +148 -0
  89. package/dist/exporters-q9iL-2Jf.js.map +1 -0
  90. package/dist/extract-usage-BrQ8mCLX.js +155 -0
  91. package/dist/extract-usage-BrQ8mCLX.js.map +1 -0
  92. package/dist/failure-cluster-CqcvCcdR.d.ts +59 -0
  93. package/dist/failure-cluster-CqcvCcdR.d.ts.map +1 -0
  94. package/dist/feedback-trajectory-CVaeREXV.d.ts +340 -0
  95. package/dist/feedback-trajectory-CVaeREXV.d.ts.map +1 -0
  96. package/dist/fuzz.d.ts +320 -646
  97. package/dist/fuzz.d.ts.map +1 -0
  98. package/dist/fuzz.js +670 -618
  99. package/dist/fuzz.js.map +1 -1
  100. package/dist/hf-dataset-DBJXXoY1.js +763 -0
  101. package/dist/hf-dataset-DBJXXoY1.js.map +1 -0
  102. package/dist/hosted/index.d.ts +11 -831
  103. package/dist/hosted/index.d.ts.map +1 -0
  104. package/dist/hosted/index.js +2 -37
  105. package/dist/index-2JJSA6-r2.d.ts +926 -0
  106. package/dist/index-2JJSA6-r2.d.ts.map +1 -0
  107. package/dist/index-6N0aYmpW.d.ts +217 -0
  108. package/dist/index-6N0aYmpW.d.ts.map +1 -0
  109. package/dist/index-BAvgST_9.d.ts +131 -0
  110. package/dist/index-BAvgST_9.d.ts.map +1 -0
  111. package/dist/index-BvnJuTGD.d.ts +68 -0
  112. package/dist/index-BvnJuTGD.d.ts.map +1 -0
  113. package/dist/index-C61Wi7yg.d.ts +547 -0
  114. package/dist/index-C61Wi7yg.d.ts.map +1 -0
  115. package/dist/index-CAPUUKaM.d.ts +335 -0
  116. package/dist/index-CAPUUKaM.d.ts.map +1 -0
  117. package/dist/index-DE5fb3EC.d.ts +2244 -0
  118. package/dist/index-DE5fb3EC.d.ts.map +1 -0
  119. package/dist/index-DSC51roc.d.ts +102 -0
  120. package/dist/index-DSC51roc.d.ts.map +1 -0
  121. package/dist/index.d.ts +3755 -15555
  122. package/dist/index.d.ts.map +1 -0
  123. package/dist/index.js +11182 -11216
  124. package/dist/index.js.map +1 -1
  125. package/dist/integrity-BzRbCHzi.js +424 -0
  126. package/dist/integrity-BzRbCHzi.js.map +1 -0
  127. package/dist/integrity-rmVhXWA7.d.ts +61 -0
  128. package/dist/integrity-rmVhXWA7.d.ts.map +1 -0
  129. package/dist/judge-calibration-DFtEMlde.d.ts +146 -0
  130. package/dist/judge-calibration-DFtEMlde.d.ts.map +1 -0
  131. package/dist/ledger-core/index.d.ts +2 -0
  132. package/dist/ledger-core/index.js +2 -0
  133. package/dist/ledger-core-DtZz1RG0.js +388 -0
  134. package/dist/ledger-core-DtZz1RG0.js.map +1 -0
  135. package/dist/llm-client--GR4JbZE.js +687 -0
  136. package/dist/llm-client--GR4JbZE.js.map +1 -0
  137. package/dist/llm-client-B_nIBlYo.d.ts +290 -0
  138. package/dist/llm-client-B_nIBlYo.d.ts.map +1 -0
  139. package/dist/matrix/index.d.ts +3 -155
  140. package/dist/matrix/index.js +2 -8
  141. package/dist/matrix-BzQnu2S6.js +270 -0
  142. package/dist/matrix-BzQnu2S6.js.map +1 -0
  143. package/dist/meta-eval/index.d.ts +4 -1027
  144. package/dist/meta-eval/index.js +393 -390
  145. package/dist/meta-eval/index.js.map +1 -1
  146. package/dist/metrics-C9YY1OcL.js +239 -0
  147. package/dist/metrics-C9YY1OcL.js.map +1 -0
  148. package/dist/mint-yN2M2eh0.js +201 -0
  149. package/dist/mint-yN2M2eh0.js.map +1 -0
  150. package/dist/multi-layer-verifier-BHY1gWAc.d.ts +138 -0
  151. package/dist/multi-layer-verifier-BHY1gWAc.d.ts.map +1 -0
  152. package/dist/multishot/index.d.ts +273 -480
  153. package/dist/multishot/index.d.ts.map +1 -0
  154. package/dist/multishot/index.js +595 -548
  155. package/dist/multishot/index.js.map +1 -1
  156. package/dist/off-policy-DvgzvtIx.js +220 -0
  157. package/dist/off-policy-DvgzvtIx.js.map +1 -0
  158. package/dist/off-policy-mskQw8Mb.d.ts +153 -0
  159. package/dist/off-policy-mskQw8Mb.d.ts.map +1 -0
  160. package/dist/openapi.json +1 -1
  161. package/dist/opencode-sqlite-BGrHeDu3.js +318 -0
  162. package/dist/opencode-sqlite-BGrHeDu3.js.map +1 -0
  163. package/dist/outcome-store-BYHIuO0e.d.ts +65 -0
  164. package/dist/outcome-store-BYHIuO0e.d.ts.map +1 -0
  165. package/dist/outcome-store-ChBKlTd_.js +75 -0
  166. package/dist/outcome-store-ChBKlTd_.js.map +1 -0
  167. package/dist/paired-arms-D9D0wXj2.js +260 -0
  168. package/dist/paired-arms-D9D0wXj2.js.map +1 -0
  169. package/dist/pipelines/index.d.ts +95 -532
  170. package/dist/pipelines/index.d.ts.map +1 -0
  171. package/dist/pipelines/index.js +497 -478
  172. package/dist/pipelines/index.js.map +1 -1
  173. package/dist/pre-registration-DakwTRXk.js +96 -0
  174. package/dist/pre-registration-DakwTRXk.js.map +1 -0
  175. package/dist/propose-review-control-LhIWGzHl.js +1458 -0
  176. package/dist/propose-review-control-LhIWGzHl.js.map +1 -0
  177. package/dist/query-CJ_DX8vl.d.ts +28 -0
  178. package/dist/query-CJ_DX8vl.d.ts.map +1 -0
  179. package/dist/query-Di7eEQ79.js +83 -0
  180. package/dist/query-Di7eEQ79.js.map +1 -0
  181. package/dist/raw-provider-sink-BQd7mzyT.js +201 -0
  182. package/dist/raw-provider-sink-BQd7mzyT.js.map +1 -0
  183. package/dist/raw-provider-sink-BU29Sh8h.d.ts +134 -0
  184. package/dist/raw-provider-sink-BU29Sh8h.d.ts.map +1 -0
  185. package/dist/redact-7Aq1ukl-.js +107 -0
  186. package/dist/redact-7Aq1ukl-.js.map +1 -0
  187. package/dist/release-report-Cz9NKH39.js +603 -0
  188. package/dist/release-report-Cz9NKH39.js.map +1 -0
  189. package/dist/release-report-mvB2G4_J.d.ts +244 -0
  190. package/dist/release-report-mvB2G4_J.d.ts.map +1 -0
  191. package/dist/replay-GnyotH0J.js +1741 -0
  192. package/dist/replay-GnyotH0J.js.map +1 -0
  193. package/dist/replay-OoidtG1E.d.ts +749 -0
  194. package/dist/replay-OoidtG1E.d.ts.map +1 -0
  195. package/dist/reporting.d.ts +6 -1312
  196. package/dist/reporting.js +6 -51
  197. package/dist/researcher-CwTdwXG1.d.ts +314 -0
  198. package/dist/researcher-CwTdwXG1.d.ts.map +1 -0
  199. package/dist/reward-hacking-eAnOsynk.d.ts +224 -0
  200. package/dist/reward-hacking-eAnOsynk.d.ts.map +1 -0
  201. package/dist/reward-hacking-qipEpKvY.js +596 -0
  202. package/dist/reward-hacking-qipEpKvY.js.map +1 -0
  203. package/dist/reward-nw2xZGZG.js +137 -0
  204. package/dist/reward-nw2xZGZG.js.map +1 -0
  205. package/dist/rl.d.ts +760 -4010
  206. package/dist/rl.d.ts.map +1 -0
  207. package/dist/rl.js +2325 -1958
  208. package/dist/rl.js.map +1 -1
  209. package/dist/rolldown-runtime-8H4AJuhK.js +14 -0
  210. package/dist/rollout/index.d.ts +3 -2087
  211. package/dist/rollout/index.js +8 -168
  212. package/dist/rollout-BOYjemfR.js +624 -0
  213. package/dist/rollout-BOYjemfR.js.map +1 -0
  214. package/dist/rubric-predictive-validity-B3xmbmS1.js +141 -0
  215. package/dist/rubric-predictive-validity-B3xmbmS1.js.map +1 -0
  216. package/dist/rubric-predictive-validity-Ku_clp_1.d.ts +74 -0
  217. package/dist/rubric-predictive-validity-Ku_clp_1.d.ts.map +1 -0
  218. package/dist/run-evidence-ClRX_8A9.d.ts +225 -0
  219. package/dist/run-evidence-ClRX_8A9.d.ts.map +1 -0
  220. package/dist/run-record-BuoE80Dq.js +467 -0
  221. package/dist/{chunk-56TAVBOK.js.map → run-record-BuoE80Dq.js.map} +1 -1
  222. package/dist/run-record-CnZu_gjl.d.ts +357 -0
  223. package/dist/run-record-CnZu_gjl.d.ts.map +1 -0
  224. package/dist/run-score-iEEAWiBY.js +41 -0
  225. package/dist/run-score-iEEAWiBY.js.map +1 -0
  226. package/dist/runtime-trajectory-1gyaTOoC.js +93 -0
  227. package/dist/runtime-trajectory-1gyaTOoC.js.map +1 -0
  228. package/dist/runtime-trajectory-BvSZcCHD.d.ts +50 -0
  229. package/dist/runtime-trajectory-BvSZcCHD.d.ts.map +1 -0
  230. package/dist/schema-BtVldJ3T.d.ts +206 -0
  231. package/dist/schema-BtVldJ3T.d.ts.map +1 -0
  232. package/dist/schema-C6DW4ZHR.js +821 -0
  233. package/dist/schema-C6DW4ZHR.js.map +1 -0
  234. package/dist/schema-CRhEY1SO.js +69 -0
  235. package/dist/schema-CRhEY1SO.js.map +1 -0
  236. package/dist/schema-Cef2cFmb.d.ts +408 -0
  237. package/dist/schema-Cef2cFmb.d.ts.map +1 -0
  238. package/dist/semantic-concept-judge-B6cWNJ2K.js +725 -0
  239. package/dist/semantic-concept-judge-B6cWNJ2K.js.map +1 -0
  240. package/dist/sequential-Br0mAPHA.js +148 -0
  241. package/dist/sequential-Br0mAPHA.js.map +1 -0
  242. package/dist/sequential-CYwq6Ff_.d.ts +141 -0
  243. package/dist/sequential-CYwq6Ff_.d.ts.map +1 -0
  244. package/dist/series-convergence-CjO2QdRW.js +43 -0
  245. package/dist/series-convergence-CjO2QdRW.js.map +1 -0
  246. package/dist/series-convergence-ofsqPWhs.d.ts +35 -0
  247. package/dist/series-convergence-ofsqPWhs.d.ts.map +1 -0
  248. package/dist/server-m5D9cvnG.js +1040 -0
  249. package/dist/server-m5D9cvnG.js.map +1 -0
  250. package/dist/skill-usage-C_pXm7lP.d.ts +534 -0
  251. package/dist/skill-usage-C_pXm7lP.d.ts.map +1 -0
  252. package/dist/skillopt-optimization-method-B7wX7XkF.d.ts +1739 -0
  253. package/dist/skillopt-optimization-method-B7wX7XkF.d.ts.map +1 -0
  254. package/dist/skillopt-optimization-method-D4ODwFVV.js +7712 -0
  255. package/dist/skillopt-optimization-method-D4ODwFVV.js.map +1 -0
  256. package/dist/statistics-Cmj6nynr.d.ts +514 -0
  257. package/dist/statistics-Cmj6nynr.d.ts.map +1 -0
  258. package/dist/statistics-CnnxdpOg.js +1437 -0
  259. package/dist/statistics-CnnxdpOg.js.map +1 -0
  260. package/dist/store-CT9YIIve.d.ts +117 -0
  261. package/dist/store-CT9YIIve.d.ts.map +1 -0
  262. package/dist/store-CxJry_cs.d.ts +229 -0
  263. package/dist/store-CxJry_cs.d.ts.map +1 -0
  264. package/dist/storyboard/index.d.ts +84 -203
  265. package/dist/storyboard/index.d.ts.map +1 -0
  266. package/dist/storyboard/index.js +609 -542
  267. package/dist/storyboard/index.js.map +1 -1
  268. package/dist/summary-report-BNs5nmXI.js +862 -0
  269. package/dist/summary-report-BNs5nmXI.js.map +1 -0
  270. package/dist/summary-report-Cj9gdw4i.d.ts +407 -0
  271. package/dist/summary-report-Cj9gdw4i.d.ts.map +1 -0
  272. package/dist/supervisor-run/index.d.ts +2 -959
  273. package/dist/supervisor-run/index.js +2 -65
  274. package/dist/supervisor-run-_lnTLM3z.js +1679 -0
  275. package/dist/supervisor-run-_lnTLM3z.js.map +1 -0
  276. package/dist/task-failure-attributes-CQZlB3et.js +311 -0
  277. package/dist/task-failure-attributes-CQZlB3et.js.map +1 -0
  278. package/dist/test-graded-scenario-BsqWLmPt.js +318 -0
  279. package/dist/test-graded-scenario-BsqWLmPt.js.map +1 -0
  280. package/dist/test-graded-scenario-D1TaI2va.d.ts +141 -0
  281. package/dist/test-graded-scenario-D1TaI2va.d.ts.map +1 -0
  282. package/dist/tools-BmuN627J.js +1085 -0
  283. package/dist/tools-BmuN627J.js.map +1 -0
  284. package/dist/trace-attributes.d.ts +2 -52
  285. package/dist/trace-attributes.js +131 -61
  286. package/dist/trace-attributes.js.map +1 -1
  287. package/dist/traces.d.ts +12 -2365
  288. package/dist/traces.js +12 -252
  289. package/dist/trajectory-D_7rLrvE.js +56 -0
  290. package/dist/trajectory-D_7rLrvE.js.map +1 -0
  291. package/dist/types-DGsxbAEd.d.ts +387 -0
  292. package/dist/types-DGsxbAEd.d.ts.map +1 -0
  293. package/dist/types-k9tZGKUg.d.ts +640 -0
  294. package/dist/types-k9tZGKUg.d.ts.map +1 -0
  295. package/dist/verdict-Dps8_okt.d.ts +37 -0
  296. package/dist/verdict-Dps8_okt.d.ts.map +1 -0
  297. package/dist/wire/index.d.ts +702 -1173
  298. package/dist/wire/index.d.ts.map +1 -0
  299. package/dist/wire/index.js +2 -81
  300. package/package.json +17 -9
  301. package/dist/benchmarks/index.js.map +0 -1
  302. package/dist/campaign/index.js.map +0 -1
  303. package/dist/chunk-2QU3YOPR.js +0 -7374
  304. package/dist/chunk-2QU3YOPR.js.map +0 -1
  305. package/dist/chunk-3OCR4R5I.js +0 -728
  306. package/dist/chunk-3OCR4R5I.js.map +0 -1
  307. package/dist/chunk-3RF76KTD.js +0 -84
  308. package/dist/chunk-3RF76KTD.js.map +0 -1
  309. package/dist/chunk-56TAVBOK.js +0 -698
  310. package/dist/chunk-5DTSBUL2.js +0 -159
  311. package/dist/chunk-5DTSBUL2.js.map +0 -1
  312. package/dist/chunk-7FO3TNPI.js +0 -232
  313. package/dist/chunk-7FO3TNPI.js.map +0 -1
  314. package/dist/chunk-7ZZMD7UK.js +0 -386
  315. package/dist/chunk-7ZZMD7UK.js.map +0 -1
  316. package/dist/chunk-BOD4O7OF.js +0 -40
  317. package/dist/chunk-BOD4O7OF.js.map +0 -1
  318. package/dist/chunk-BSO5JDQH.js +0 -2335
  319. package/dist/chunk-BSO5JDQH.js.map +0 -1
  320. package/dist/chunk-C6LXANRU.js +0 -1550
  321. package/dist/chunk-C6LXANRU.js.map +0 -1
  322. package/dist/chunk-DODXQREJ.js +0 -752
  323. package/dist/chunk-DODXQREJ.js.map +0 -1
  324. package/dist/chunk-DRYIUNWY.js +0 -622
  325. package/dist/chunk-DRYIUNWY.js.map +0 -1
  326. package/dist/chunk-E7QXT7SX.js +0 -183
  327. package/dist/chunk-E7QXT7SX.js.map +0 -1
  328. package/dist/chunk-EG66UGL4.js +0 -341
  329. package/dist/chunk-EG66UGL4.js.map +0 -1
  330. package/dist/chunk-FXTVJPYD.js +0 -576
  331. package/dist/chunk-FXTVJPYD.js.map +0 -1
  332. package/dist/chunk-G7MGMCZD.js +0 -153
  333. package/dist/chunk-G7MGMCZD.js.map +0 -1
  334. package/dist/chunk-GGE4NNQT.js +0 -65
  335. package/dist/chunk-GGE4NNQT.js.map +0 -1
  336. package/dist/chunk-H23X7XKK.js +0 -181
  337. package/dist/chunk-H23X7XKK.js.map +0 -1
  338. package/dist/chunk-HHWE3POT.js +0 -94
  339. package/dist/chunk-HHWE3POT.js.map +0 -1
  340. package/dist/chunk-HPWUNB47.js +0 -289
  341. package/dist/chunk-HPWUNB47.js.map +0 -1
  342. package/dist/chunk-IYCLP2N2.js +0 -766
  343. package/dist/chunk-IYCLP2N2.js.map +0 -1
  344. package/dist/chunk-JHCHEVET.js +0 -274
  345. package/dist/chunk-JHCHEVET.js.map +0 -1
  346. package/dist/chunk-JQSF5DQT.js +0 -701
  347. package/dist/chunk-JQSF5DQT.js.map +0 -1
  348. package/dist/chunk-K4DBDHLK.js +0 -158
  349. package/dist/chunk-K4DBDHLK.js.map +0 -1
  350. package/dist/chunk-K6N6XJJX.js +0 -306
  351. package/dist/chunk-K6N6XJJX.js.map +0 -1
  352. package/dist/chunk-M4YBQKIJ.js +0 -1040
  353. package/dist/chunk-M4YBQKIJ.js.map +0 -1
  354. package/dist/chunk-MA6HLL3S.js +0 -65
  355. package/dist/chunk-MA6HLL3S.js.map +0 -1
  356. package/dist/chunk-MAZ26DC7.js +0 -99
  357. package/dist/chunk-MAZ26DC7.js.map +0 -1
  358. package/dist/chunk-NPCTHQIO.js +0 -91
  359. package/dist/chunk-NPCTHQIO.js.map +0 -1
  360. package/dist/chunk-NY44NC4A.js +0 -1056
  361. package/dist/chunk-NY44NC4A.js.map +0 -1
  362. package/dist/chunk-OIUOT4QD.js +0 -44
  363. package/dist/chunk-OIUOT4QD.js.map +0 -1
  364. package/dist/chunk-ONWEPEDO.js +0 -57
  365. package/dist/chunk-ONWEPEDO.js.map +0 -1
  366. package/dist/chunk-OWN5NPMC.js +0 -152
  367. package/dist/chunk-OWN5NPMC.js.map +0 -1
  368. package/dist/chunk-P6FYH6K4.js +0 -1161
  369. package/dist/chunk-P6FYH6K4.js.map +0 -1
  370. package/dist/chunk-PC4UYEBM.js +0 -166
  371. package/dist/chunk-PC4UYEBM.js.map +0 -1
  372. package/dist/chunk-PC5DOSM7.js +0 -579
  373. package/dist/chunk-PC5DOSM7.js.map +0 -1
  374. package/dist/chunk-PXE2VKMX.js +0 -140
  375. package/dist/chunk-PXE2VKMX.js.map +0 -1
  376. package/dist/chunk-PZ5AY32C.js +0 -10
  377. package/dist/chunk-PZ5AY32C.js.map +0 -1
  378. package/dist/chunk-QB6BDBP2.js +0 -4464
  379. package/dist/chunk-QB6BDBP2.js.map +0 -1
  380. package/dist/chunk-RXHCETDZ.js +0 -536
  381. package/dist/chunk-RXHCETDZ.js.map +0 -1
  382. package/dist/chunk-RZTMDUO7.js +0 -49
  383. package/dist/chunk-RZTMDUO7.js.map +0 -1
  384. package/dist/chunk-SFLLL76A.js +0 -669
  385. package/dist/chunk-SFLLL76A.js.map +0 -1
  386. package/dist/chunk-SZLVEKMJ.js +0 -1446
  387. package/dist/chunk-SZLVEKMJ.js.map +0 -1
  388. package/dist/chunk-T4SQEITX.js +0 -95
  389. package/dist/chunk-T4SQEITX.js.map +0 -1
  390. package/dist/chunk-T6RLYGAD.js +0 -158
  391. package/dist/chunk-T6RLYGAD.js.map +0 -1
  392. package/dist/chunk-TJVT4QFF.js +0 -911
  393. package/dist/chunk-TJVT4QFF.js.map +0 -1
  394. package/dist/chunk-TQ7LNKZ3.js +0 -136
  395. package/dist/chunk-TQ7LNKZ3.js.map +0 -1
  396. package/dist/chunk-U4L7JRPZ.js +0 -1706
  397. package/dist/chunk-U4L7JRPZ.js.map +0 -1
  398. package/dist/chunk-U4PHLT2N.js +0 -419
  399. package/dist/chunk-U4PHLT2N.js.map +0 -1
  400. package/dist/chunk-VCZ5FQYW.js +0 -928
  401. package/dist/chunk-VCZ5FQYW.js.map +0 -1
  402. package/dist/chunk-VI2UW6B6.js +0 -162
  403. package/dist/chunk-VI2UW6B6.js.map +0 -1
  404. package/dist/chunk-VQMK5FMP.js +0 -247
  405. package/dist/chunk-VQMK5FMP.js.map +0 -1
  406. package/dist/chunk-WGXIEX7P.js +0 -116
  407. package/dist/chunk-WGXIEX7P.js.map +0 -1
  408. package/dist/chunk-WVATSFCP.js +0 -1553
  409. package/dist/chunk-WVATSFCP.js.map +0 -1
  410. package/dist/chunk-X4YIBDER.js +0 -1662
  411. package/dist/chunk-X4YIBDER.js.map +0 -1
  412. package/dist/chunk-YQN4ICPP.js +0 -355
  413. package/dist/chunk-YQN4ICPP.js.map +0 -1
  414. package/dist/chunk-ZET2UAYW.js +0 -89
  415. package/dist/chunk-ZET2UAYW.js.map +0 -1
  416. package/dist/chunk-ZHTZ4EYI.js +0 -1212
  417. package/dist/chunk-ZHTZ4EYI.js.map +0 -1
  418. package/dist/control.js.map +0 -1
  419. package/dist/hosted/index.js.map +0 -1
  420. package/dist/matrix/index.js.map +0 -1
  421. package/dist/reporting.js.map +0 -1
  422. package/dist/rollout/index.js.map +0 -1
  423. package/dist/run-campaign-OJJ7CZF4.js +0 -18
  424. package/dist/run-campaign-OJJ7CZF4.js.map +0 -1
  425. package/dist/supervisor-run/index.js.map +0 -1
  426. package/dist/traces.js.map +0 -1
  427. package/dist/wire/index.js.map +0 -1
@@ -1,1553 +0,0 @@
1
- import {
2
- TraceEmitter
3
- } from "./chunk-VQMK5FMP.js";
4
- import {
5
- validateRunRecord
6
- } from "./chunk-56TAVBOK.js";
7
-
8
- // src/action-policy.ts
9
- function evaluateActionPolicy(action, policy = {}, options = {}) {
10
- const reasons = [];
11
- let blocked = false;
12
- let requiresApproval = Boolean(action.requiresApproval);
13
- if (policy.allowedTypes?.length && !policy.allowedTypes.includes(action.type)) {
14
- blocked = true;
15
- reasons.push(`action type "${action.type}" is not allowed`);
16
- }
17
- if (policy.blockedTypes?.includes(action.type)) {
18
- blocked = true;
19
- reasons.push(`action type "${action.type}" is blocked`);
20
- }
21
- if (policy.alwaysRequireApprovalTypes?.includes(action.type)) {
22
- requiresApproval = true;
23
- reasons.push(`action type "${action.type}" requires approval`);
24
- }
25
- if (policy.requireApprovalForExternalSideEffects && action.externalSideEffect) {
26
- requiresApproval = true;
27
- reasons.push("external side effect requires approval");
28
- }
29
- if (policy.requireApprovalAboveCostUsd !== void 0 && (action.costUsd ?? 0) > policy.requireApprovalAboveCostUsd) {
30
- requiresApproval = true;
31
- reasons.push(
32
- `cost ${action.costUsd} exceeds approval threshold ${policy.requireApprovalAboveCostUsd}`
33
- );
34
- }
35
- if (policy.maxActionCostUsd !== void 0 && (action.costUsd ?? 0) > policy.maxActionCostUsd) {
36
- blocked = true;
37
- reasons.push(`cost ${action.costUsd} exceeds max action cost ${policy.maxActionCostUsd}`);
38
- }
39
- if (policy.remainingBudgetUsd !== void 0 && (action.costUsd ?? 0) > policy.remainingBudgetUsd) {
40
- blocked = true;
41
- reasons.push(`cost ${action.costUsd} exceeds remaining budget ${policy.remainingBudgetUsd}`);
42
- }
43
- if (policy.expectedOutcomeRequired && !action.metadata?.expectedOutcome) {
44
- blocked = true;
45
- reasons.push("expected outcome is required");
46
- }
47
- if (policy.killCriteriaRequired && !action.metadata?.killCriteria) {
48
- blocked = true;
49
- reasons.push("kill criteria are required");
50
- }
51
- if (policy.autoApproveTypes?.includes(action.type) && requiresApproval) {
52
- reasons.push(
53
- `action type "${action.type}" is auto-approved only when no approval policy applies`
54
- );
55
- }
56
- if (!reasons.length) reasons.push(requiresApproval ? "approval required" : "action allowed");
57
- const label = blocked || requiresApproval ? {
58
- source: "policy",
59
- kind: blocked ? "policy_block" : "comment",
60
- value: { actionType: action.type, blocked, requiresApproval },
61
- reason: reasons.join("; "),
62
- severity: blocked ? "critical" : "warning",
63
- createdAt: options.createdAt ?? (/* @__PURE__ */ new Date()).toISOString(),
64
- metadata: { action, policy }
65
- } : void 0;
66
- return {
67
- allowed: !blocked,
68
- blocked,
69
- requiresApproval: !blocked && requiresApproval,
70
- reasons,
71
- label
72
- };
73
- }
74
-
75
- // src/detectors/index.ts
76
- function repeatedActionDetector(opts = {}) {
77
- const max = opts.maxRepeated ?? 3;
78
- const severity = opts.severity ?? "warn";
79
- const failureClass = opts.failureClass ?? "tool_recovery_failure";
80
- let last;
81
- let streak = 0;
82
- return {
83
- id: "repeated-action",
84
- get streak() {
85
- return streak;
86
- },
87
- observe(event) {
88
- const fp = event.actionFingerprint;
89
- if (fp === void 0) return null;
90
- streak = fp === last ? streak + 1 : 1;
91
- last = fp;
92
- if (max <= 0 || streak < max) return null;
93
- return {
94
- detector: "repeated-action",
95
- severity,
96
- failureClass,
97
- reason: `stuck: repeated same action for ${streak} step(s)`,
98
- streak,
99
- ...event.label ? { evidence: { action: event.label } } : {}
100
- };
101
- },
102
- reset() {
103
- last = void 0;
104
- streak = 0;
105
- }
106
- };
107
- }
108
- function noProgressDetector(opts = {}) {
109
- const max = opts.maxNoProgress ?? 3;
110
- const minScoreDelta = opts.minScoreDelta ?? 1e-3;
111
- const severity = opts.severity ?? "warn";
112
- const failureClass = opts.failureClass ?? "tool_recovery_failure";
113
- let lastState;
114
- let lastScore;
115
- let streak = 0;
116
- return {
117
- id: "no-progress",
118
- get streak() {
119
- return streak;
120
- },
121
- observe(event) {
122
- const stateUnchanged = lastState !== void 0 && lastState === event.stateFingerprint;
123
- const scoreFlat = Math.abs((event.score ?? 0) - (lastScore ?? 0)) < minScoreDelta;
124
- streak = stateUnchanged && scoreFlat ? streak + 1 : 0;
125
- lastState = event.stateFingerprint;
126
- lastScore = event.score;
127
- if (max <= 0 || streak < max) return null;
128
- return {
129
- detector: "no-progress",
130
- severity,
131
- failureClass,
132
- reason: `stuck: no state/score progress for ${streak} step(s)`,
133
- streak,
134
- ...event.label ? { evidence: { state: event.label } } : {}
135
- };
136
- },
137
- reset() {
138
- lastState = void 0;
139
- lastScore = void 0;
140
- streak = 0;
141
- }
142
- };
143
- }
144
- function errorStreakDetector(opts = {}) {
145
- const max = opts.maxErrors ?? 3;
146
- const severity = opts.severity ?? "warn";
147
- const failureClass = opts.failureClass ?? "tool_recovery_failure";
148
- let streak = 0;
149
- return {
150
- id: "error-streak",
151
- get streak() {
152
- return streak;
153
- },
154
- observe(event) {
155
- if (event.status === void 0) return null;
156
- streak = event.status === "error" ? streak + 1 : 0;
157
- if (max <= 0 || streak < max) return null;
158
- return {
159
- detector: "error-streak",
160
- severity,
161
- failureClass,
162
- reason: `stuck: ${streak} consecutive errored step(s)`,
163
- streak,
164
- ...event.label ? { evidence: { lastError: event.label } } : {}
165
- };
166
- },
167
- reset() {
168
- streak = 0;
169
- }
170
- };
171
- }
172
- function observeAll(detectors, event) {
173
- const signals = [];
174
- for (const d of detectors) {
175
- const s = d.observe(event);
176
- if (s) signals.push(s);
177
- }
178
- return signals;
179
- }
180
-
181
- // src/control-runtime.ts
182
- var DEFAULT_BUDGET = {
183
- maxSteps: 8,
184
- maxWallMs: 5 * 60 * 1e3
185
- };
186
- async function runAgentControlLoop(config) {
187
- const budget = normalizeBudget(config.budget);
188
- const actionFailure = config.actionFailure ?? "continue";
189
- const controller = new AbortController();
190
- const upstreamAbort = () => controller.abort(config.signal?.reason);
191
- if (config.signal) {
192
- if (config.signal.aborted) controller.abort(config.signal.reason);
193
- else config.signal.addEventListener("abort", upstreamAbort, { once: true });
194
- }
195
- const started = Date.now();
196
- const wallTimer = budget.maxWallMs ? setTimeout(
197
- () => controller.abort(new Error("control runtime wall timeout")),
198
- budget.maxWallMs
199
- ) : void 0;
200
- const history = [];
201
- const emitter = config.store ? new TraceEmitter(config.store) : void 0;
202
- let spentCostUsd = 0;
203
- const runtimeErrors = [];
204
- const repeatedDetector = repeatedActionDetector({
205
- maxRepeated: config.stopPolicies?.maxRepeatedActions ?? 0
206
- });
207
- const progressDetector = noProgressDetector({
208
- maxNoProgress: config.stopPolicies?.maxNoProgressSteps ?? 0,
209
- minScoreDelta: config.stopPolicies?.minScoreDelta ?? 1e-3
210
- });
211
- try {
212
- if (emitter) {
213
- await runTrace(
214
- runtimeErrors,
215
- 0,
216
- () => emitter.startRun({
217
- scenarioId: config.scenarioId ?? "agent-control-loop",
218
- projectId: config.projectId,
219
- variantId: config.variantId,
220
- layer: "meta",
221
- tags: {
222
- intent: config.intent.slice(0, 120),
223
- maxSteps: String(budget.maxSteps),
224
- ...budget.maxCostUsd !== void 0 ? { maxCostUsd: String(budget.maxCostUsd) } : {}
225
- }
226
- })
227
- );
228
- }
229
- let state;
230
- let evals;
231
- try {
232
- state = await config.observe({ history, abortSignal: controller.signal });
233
- } catch (err) {
234
- const error = runtimeError("observe", 0, err);
235
- runtimeErrors.push(error);
236
- return finish(emitter, {
237
- intent: config.intent,
238
- pass: false,
239
- completed: false,
240
- reason: error.message,
241
- steps: history,
242
- finalState: void 0,
243
- finalEvals: [],
244
- wallMs: Date.now() - started,
245
- spentCostUsd,
246
- runId: emitter?.runId ?? null,
247
- failureClass: "unknown",
248
- runtimeErrors,
249
- stoppedBy: "runtime-error"
250
- });
251
- }
252
- try {
253
- evals = await config.validate({
254
- intent: config.intent,
255
- state,
256
- history,
257
- abortSignal: controller.signal
258
- });
259
- await recordEvalSpans(emitter, evals, "initial", runtimeErrors, 0);
260
- } catch (err) {
261
- const error = runtimeError("validate", 0, err);
262
- runtimeErrors.push(error);
263
- return finish(emitter, {
264
- intent: config.intent,
265
- pass: false,
266
- completed: false,
267
- reason: error.message,
268
- steps: history,
269
- finalState: state,
270
- finalEvals: [],
271
- wallMs: Date.now() - started,
272
- spentCostUsd,
273
- runId: emitter?.runId ?? null,
274
- failureClass: "unknown",
275
- runtimeErrors,
276
- stoppedBy: "runtime-error"
277
- });
278
- }
279
- progressDetector.observe({
280
- stateFingerprint: fingerprintState(state, config.stopPolicies),
281
- score: averageScore(evals)
282
- });
283
- for (let stepIndex = 0; stepIndex < budget.maxSteps; stepIndex++) {
284
- if (controller.signal.aborted) {
285
- return finish(emitter, {
286
- intent: config.intent,
287
- pass: false,
288
- completed: false,
289
- reason: abortReason(controller.signal),
290
- score: void 0,
291
- steps: history,
292
- finalState: state,
293
- finalEvals: evals,
294
- wallMs: Date.now() - started,
295
- spentCostUsd,
296
- runId: emitter?.runId ?? null,
297
- failureClass: "timeout",
298
- runtimeErrors,
299
- stoppedBy: "abort"
300
- });
301
- }
302
- const budgetStop = budgetStopDecision(budget, spentCostUsd);
303
- if (budgetStop.stop) {
304
- return finish(emitter, {
305
- intent: config.intent,
306
- pass: false,
307
- completed: false,
308
- reason: budgetStop.reason,
309
- score: averageScore(evals),
310
- steps: history,
311
- finalState: state,
312
- finalEvals: evals,
313
- wallMs: Date.now() - started,
314
- spentCostUsd,
315
- runId: emitter?.runId ?? null,
316
- failureClass: "budget_exceeded",
317
- runtimeErrors,
318
- stoppedBy: "budget"
319
- });
320
- }
321
- const ctx = makeContext(
322
- config.intent,
323
- state,
324
- evals,
325
- history,
326
- budget,
327
- stepIndex,
328
- started,
329
- spentCostUsd,
330
- controller.signal,
331
- emitter
332
- );
333
- let stop;
334
- try {
335
- stop = config.shouldStop ? await config.shouldStop(ctx) : defaultStopDecision(evals);
336
- } catch (err) {
337
- runtimeErrors.push(runtimeError("stop-policy", stepIndex, err));
338
- return finish(emitter, {
339
- intent: config.intent,
340
- pass: false,
341
- completed: false,
342
- reason: runtimeErrors[runtimeErrors.length - 1].message,
343
- score: averageScore(evals),
344
- steps: history,
345
- finalState: state,
346
- finalEvals: evals,
347
- wallMs: Date.now() - started,
348
- spentCostUsd,
349
- runId: emitter?.runId ?? null,
350
- failureClass: "unknown",
351
- runtimeErrors,
352
- stoppedBy: "runtime-error"
353
- });
354
- }
355
- if (stop.stop) {
356
- return finish(emitter, {
357
- intent: config.intent,
358
- pass: stop.pass,
359
- completed: true,
360
- reason: stop.reason,
361
- score: stop.score,
362
- steps: history,
363
- finalState: state,
364
- finalEvals: evals,
365
- wallMs: Date.now() - started,
366
- spentCostUsd,
367
- runId: emitter?.runId ?? null,
368
- failureClass: stop.failureClass,
369
- runtimeErrors,
370
- stoppedBy: "stop-policy"
371
- });
372
- }
373
- let decision;
374
- try {
375
- decision = await config.decide(ctx);
376
- } catch (err) {
377
- runtimeErrors.push(runtimeError("decide", stepIndex, err));
378
- return finish(emitter, {
379
- intent: config.intent,
380
- pass: false,
381
- completed: false,
382
- reason: runtimeErrors[runtimeErrors.length - 1].message,
383
- score: averageScore(evals),
384
- steps: history,
385
- finalState: state,
386
- finalEvals: evals,
387
- wallMs: Date.now() - started,
388
- spentCostUsd,
389
- runId: emitter?.runId ?? null,
390
- failureClass: "unknown",
391
- runtimeErrors,
392
- stoppedBy: "runtime-error"
393
- });
394
- }
395
- if (decision.type === "stop") {
396
- const pass = decision.pass ?? false;
397
- return finish(emitter, {
398
- intent: config.intent,
399
- pass,
400
- completed: true,
401
- reason: decision.reason,
402
- score: decision.score,
403
- steps: history,
404
- finalState: state,
405
- finalEvals: evals,
406
- wallMs: Date.now() - started,
407
- spentCostUsd,
408
- runId: emitter?.runId ?? null,
409
- failureClass: pass ? void 0 : decision.failureClass ?? "unknown",
410
- runtimeErrors,
411
- stoppedBy: "policy"
412
- });
413
- }
414
- const actionFingerprint = fingerprintAction(decision.action, config.stopPolicies);
415
- const repeatedActionSignal = repeatedDetector.observe({ actionFingerprint });
416
- const repeatedActionStreak = repeatedDetector.streak;
417
- if (repeatedActionSignal) {
418
- return finish(emitter, {
419
- intent: config.intent,
420
- pass: false,
421
- completed: true,
422
- reason: repeatedActionSignal.reason,
423
- score: averageScore(evals),
424
- steps: history,
425
- finalState: state,
426
- finalEvals: evals,
427
- wallMs: Date.now() - started,
428
- spentCostUsd,
429
- runId: emitter?.runId ?? null,
430
- failureClass: "tool_recovery_failure",
431
- runtimeErrors,
432
- stoppedBy: "stop-policy"
433
- });
434
- }
435
- const beforeState = state;
436
- const evalsBefore = evals;
437
- const scoreBefore = averageScore(evals);
438
- const actionStarted = Date.now();
439
- const stepHandle = emitter ? await runTrace(
440
- runtimeErrors,
441
- stepIndex,
442
- () => emitter.tool({
443
- name: `control-step-${stepIndex}`,
444
- toolName: "agent-control-action",
445
- args: decision.action,
446
- attributes: {
447
- decision: decision.reason ?? "continue",
448
- repeatedActionStreak
449
- }
450
- })
451
- ) : void 0;
452
- let actionOutcome;
453
- try {
454
- const result = await config.act(decision.action, ctx);
455
- const rawCostUsd = config.getActionCostUsd?.({
456
- action: decision.action,
457
- result,
458
- state,
459
- evals,
460
- history
461
- });
462
- const costUsd = normalizeActionCostUsd(rawCostUsd, runtimeErrors, stepIndex);
463
- if (costUsd !== void 0 && Number.isFinite(costUsd) && costUsd > 0) {
464
- spentCostUsd += costUsd;
465
- await recordCostBudget(
466
- emitter,
467
- budget,
468
- spentCostUsd,
469
- stepHandle,
470
- runtimeErrors,
471
- stepIndex
472
- );
473
- }
474
- actionOutcome = {
475
- ok: true,
476
- result,
477
- ...costUsd !== void 0 ? { costUsd } : {},
478
- durationMs: Date.now() - actionStarted
479
- };
480
- } catch (err) {
481
- runtimeErrors.push(runtimeError("act", stepIndex, err));
482
- actionOutcome = {
483
- ok: false,
484
- error: runtimeErrors[runtimeErrors.length - 1].message,
485
- durationMs: Date.now() - actionStarted
486
- };
487
- if (actionFailure === "stop") {
488
- await runTrace(
489
- runtimeErrors,
490
- stepIndex,
491
- () => stepHandle?.fail(actionOutcome.error ?? "action failed")
492
- );
493
- const step2 = {
494
- index: stepIndex,
495
- decision,
496
- beforeState,
497
- afterState: state,
498
- evalsBefore,
499
- evalsAfter: evals,
500
- actionOutcome,
501
- startedAt: new Date(actionStarted).toISOString(),
502
- endedAt: (/* @__PURE__ */ new Date()).toISOString()
503
- };
504
- history.push(step2);
505
- await runOnStep(config.onStep, step2, runtimeErrors);
506
- return finish(emitter, {
507
- intent: config.intent,
508
- pass: false,
509
- completed: false,
510
- reason: actionOutcome.error ?? "action failed",
511
- score: averageScore(evals),
512
- steps: history,
513
- finalState: state,
514
- finalEvals: evals,
515
- wallMs: Date.now() - started,
516
- spentCostUsd,
517
- runId: emitter?.runId ?? null,
518
- failureClass: "unknown",
519
- runtimeErrors,
520
- stoppedBy: "runtime-error"
521
- });
522
- }
523
- }
524
- try {
525
- state = await config.observe({ history, abortSignal: controller.signal });
526
- } catch (err) {
527
- runtimeErrors.push(runtimeError("observe", stepIndex, err));
528
- const step2 = {
529
- index: stepIndex,
530
- decision,
531
- beforeState,
532
- afterState: beforeState,
533
- evalsBefore,
534
- evalsAfter: evals,
535
- actionOutcome,
536
- startedAt: new Date(actionStarted).toISOString(),
537
- endedAt: (/* @__PURE__ */ new Date()).toISOString()
538
- };
539
- history.push(step2);
540
- await runTrace(
541
- runtimeErrors,
542
- stepIndex,
543
- () => stepHandle?.fail(runtimeErrors[runtimeErrors.length - 1].message)
544
- );
545
- await runOnStep(config.onStep, step2, runtimeErrors);
546
- return finish(emitter, {
547
- intent: config.intent,
548
- pass: false,
549
- completed: false,
550
- reason: runtimeErrors[runtimeErrors.length - 1].message,
551
- score: averageScore(evals),
552
- steps: history,
553
- finalState: beforeState,
554
- finalEvals: evals,
555
- wallMs: Date.now() - started,
556
- spentCostUsd,
557
- runId: emitter?.runId ?? null,
558
- failureClass: "unknown",
559
- runtimeErrors,
560
- stoppedBy: "runtime-error"
561
- });
562
- }
563
- try {
564
- evals = await config.validate({
565
- intent: config.intent,
566
- state,
567
- history,
568
- abortSignal: controller.signal
569
- });
570
- await recordEvalSpans(
571
- emitter,
572
- evals,
573
- `step-${stepIndex}`,
574
- runtimeErrors,
575
- stepIndex,
576
- stepHandle?.span.spanId
577
- );
578
- } catch (err) {
579
- runtimeErrors.push(runtimeError("validate", stepIndex, err));
580
- const step2 = {
581
- index: stepIndex,
582
- decision,
583
- beforeState,
584
- afterState: state,
585
- evalsBefore,
586
- evalsAfter: evals,
587
- actionOutcome,
588
- startedAt: new Date(actionStarted).toISOString(),
589
- endedAt: (/* @__PURE__ */ new Date()).toISOString()
590
- };
591
- history.push(step2);
592
- await runTrace(
593
- runtimeErrors,
594
- stepIndex,
595
- () => stepHandle?.fail(runtimeErrors[runtimeErrors.length - 1].message)
596
- );
597
- await runOnStep(config.onStep, step2, runtimeErrors);
598
- return finish(emitter, {
599
- intent: config.intent,
600
- pass: false,
601
- completed: false,
602
- reason: runtimeErrors[runtimeErrors.length - 1].message,
603
- score: averageScore(evals),
604
- steps: history,
605
- finalState: state,
606
- finalEvals: evals,
607
- wallMs: Date.now() - started,
608
- spentCostUsd,
609
- runId: emitter?.runId ?? null,
610
- failureClass: "unknown",
611
- runtimeErrors,
612
- stoppedBy: "runtime-error"
613
- });
614
- }
615
- const scoreAfter = averageScore(evals);
616
- const stateFingerprint = fingerprintState(state, config.stopPolicies);
617
- const noProgressSignal = progressDetector.observe({ stateFingerprint, score: scoreAfter });
618
- const noProgressStreak = progressDetector.streak;
619
- const step = {
620
- index: stepIndex,
621
- decision,
622
- beforeState,
623
- afterState: state,
624
- evalsBefore,
625
- evalsAfter: evals,
626
- actionOutcome,
627
- startedAt: new Date(actionStarted).toISOString(),
628
- endedAt: (/* @__PURE__ */ new Date()).toISOString()
629
- };
630
- history.push(step);
631
- if (actionOutcome.ok) {
632
- await runTrace(
633
- runtimeErrors,
634
- stepIndex,
635
- () => stepHandle?.end({
636
- attributes: {
637
- actionCostUsd: actionOutcome.costUsd ?? null,
638
- spentCostUsd,
639
- scoreBefore: scoreBefore ?? null,
640
- scoreAfter: scoreAfter ?? null,
641
- noProgressStreak
642
- }
643
- })
644
- );
645
- } else {
646
- await runTrace(
647
- runtimeErrors,
648
- stepIndex,
649
- () => stepHandle?.fail(actionOutcome.error ?? "action failed", {
650
- attributes: {
651
- spentCostUsd,
652
- noProgressStreak
653
- }
654
- })
655
- );
656
- }
657
- await runOnStep(config.onStep, step, runtimeErrors);
658
- if (noProgressSignal) {
659
- return finish(emitter, {
660
- intent: config.intent,
661
- pass: false,
662
- completed: true,
663
- reason: noProgressSignal.reason,
664
- score: scoreAfter,
665
- steps: history,
666
- finalState: state,
667
- finalEvals: evals,
668
- wallMs: Date.now() - started,
669
- spentCostUsd,
670
- runId: emitter?.runId ?? null,
671
- failureClass: "tool_recovery_failure",
672
- runtimeErrors,
673
- stoppedBy: "stop-policy"
674
- });
675
- }
676
- const postStepBudgetStop = budgetStopDecision(budget, spentCostUsd);
677
- if (postStepBudgetStop.stop) {
678
- return finish(emitter, {
679
- intent: config.intent,
680
- pass: false,
681
- completed: false,
682
- reason: postStepBudgetStop.reason,
683
- score: scoreAfter,
684
- steps: history,
685
- finalState: state,
686
- finalEvals: evals,
687
- wallMs: Date.now() - started,
688
- spentCostUsd,
689
- runId: emitter?.runId ?? null,
690
- failureClass: "budget_exceeded",
691
- runtimeErrors,
692
- stoppedBy: "budget"
693
- });
694
- }
695
- const postStepCtx = makeContext(
696
- config.intent,
697
- state,
698
- evals,
699
- history,
700
- budget,
701
- stepIndex + 1,
702
- started,
703
- spentCostUsd,
704
- controller.signal,
705
- emitter
706
- );
707
- let postStepStop;
708
- try {
709
- postStepStop = config.shouldStop ? await config.shouldStop(postStepCtx) : defaultStopDecision(evals);
710
- } catch (err) {
711
- runtimeErrors.push(runtimeError("stop-policy", stepIndex + 1, err));
712
- return finish(emitter, {
713
- intent: config.intent,
714
- pass: false,
715
- completed: false,
716
- reason: runtimeErrors[runtimeErrors.length - 1].message,
717
- score: averageScore(evals),
718
- steps: history,
719
- finalState: state,
720
- finalEvals: evals,
721
- wallMs: Date.now() - started,
722
- spentCostUsd,
723
- runId: emitter?.runId ?? null,
724
- failureClass: "unknown",
725
- runtimeErrors,
726
- stoppedBy: "runtime-error"
727
- });
728
- }
729
- if (postStepStop.stop) {
730
- return finish(emitter, {
731
- intent: config.intent,
732
- pass: postStepStop.pass,
733
- completed: true,
734
- reason: postStepStop.reason,
735
- score: postStepStop.score,
736
- steps: history,
737
- finalState: state,
738
- finalEvals: evals,
739
- wallMs: Date.now() - started,
740
- spentCostUsd,
741
- runId: emitter?.runId ?? null,
742
- failureClass: postStepStop.failureClass,
743
- runtimeErrors,
744
- stoppedBy: "stop-policy"
745
- });
746
- }
747
- }
748
- return finish(emitter, {
749
- intent: config.intent,
750
- pass: false,
751
- completed: false,
752
- reason: `budget exhausted: maxSteps=${budget.maxSteps}`,
753
- steps: history,
754
- finalState: state,
755
- finalEvals: evals,
756
- wallMs: Date.now() - started,
757
- spentCostUsd,
758
- runId: emitter?.runId ?? null,
759
- failureClass: "budget_exceeded",
760
- runtimeErrors,
761
- stoppedBy: "budget"
762
- });
763
- } catch (err) {
764
- runtimeErrors.push(runtimeError("act", history.length, err));
765
- return finish(emitter, {
766
- intent: config.intent,
767
- pass: false,
768
- completed: false,
769
- reason: runtimeErrors[runtimeErrors.length - 1].message,
770
- steps: history,
771
- finalState: void 0,
772
- finalEvals: [],
773
- wallMs: Date.now() - started,
774
- spentCostUsd,
775
- runId: emitter?.runId ?? null,
776
- failureClass: "unknown",
777
- runtimeErrors,
778
- stoppedBy: "runtime-error"
779
- });
780
- } finally {
781
- if (wallTimer) clearTimeout(wallTimer);
782
- if (config.signal) config.signal.removeEventListener("abort", upstreamAbort);
783
- }
784
- }
785
- function stopOnNoProgress(maxNoProgressSteps, options = {}) {
786
- return { ...options, maxNoProgressSteps };
787
- }
788
- function stopOnRepeatedAction(maxRepeatedActions, options = {}) {
789
- return { ...options, maxRepeatedActions };
790
- }
791
- function objectiveEval(input) {
792
- return { ...input, objective: true };
793
- }
794
- function subjectiveEval(input) {
795
- return { ...input, objective: false };
796
- }
797
- function normalizeBudget(input) {
798
- const raw = { ...DEFAULT_BUDGET, ...input };
799
- if (!Number.isInteger(raw.maxSteps) || raw.maxSteps < 1) {
800
- throw new RangeError(
801
- `ControlRuntime budget.maxSteps must be an integer >= 1, got ${String(raw.maxSteps)}`
802
- );
803
- }
804
- const budget = { maxSteps: raw.maxSteps };
805
- if (raw.maxWallMs !== void 0) {
806
- if (typeof raw.maxWallMs !== "number" || !Number.isFinite(raw.maxWallMs) || raw.maxWallMs <= 0) {
807
- throw new RangeError(
808
- `ControlRuntime budget.maxWallMs must be a positive finite number, got ${String(raw.maxWallMs)}`
809
- );
810
- }
811
- budget.maxWallMs = raw.maxWallMs;
812
- }
813
- if (raw.maxCostUsd !== void 0) {
814
- if (typeof raw.maxCostUsd !== "number" || !Number.isFinite(raw.maxCostUsd) || raw.maxCostUsd < 0) {
815
- throw new RangeError(
816
- `ControlRuntime budget.maxCostUsd must be a nonnegative finite number, got ${String(raw.maxCostUsd)}`
817
- );
818
- }
819
- budget.maxCostUsd = raw.maxCostUsd;
820
- }
821
- return budget;
822
- }
823
- function normalizeActionCostUsd(costUsd, runtimeErrors, stepIndex) {
824
- if (costUsd === void 0) return void 0;
825
- if (!Number.isFinite(costUsd) || costUsd < 0) {
826
- runtimeErrors.push(
827
- runtimeError("act", stepIndex, new Error(`invalid action costUsd: ${String(costUsd)}`))
828
- );
829
- return void 0;
830
- }
831
- return costUsd;
832
- }
833
- function allCriticalPassed(evals) {
834
- return evals.every(
835
- (result) => result.passed || result.severity !== "critical" && result.severity !== "error"
836
- );
837
- }
838
- function makeContext(intent, state, evals, history, budget, stepIndex, started, spentCostUsd, abortSignal, emitter) {
839
- return {
840
- intent,
841
- state,
842
- evals,
843
- history,
844
- budget,
845
- stepIndex,
846
- wallMs: Date.now() - started,
847
- spentCostUsd,
848
- remainingCostUsd: budget.maxCostUsd === void 0 ? void 0 : Math.max(0, budget.maxCostUsd - spentCostUsd),
849
- abortSignal,
850
- emitter
851
- };
852
- }
853
- function defaultStopDecision(evals) {
854
- if (!evals.length) return { stop: false, pass: false, reason: "no evals yet" };
855
- const pass = allCriticalPassed(evals);
856
- return pass ? { stop: true, pass: true, reason: "all critical evals passed", score: averageScore(evals) } : {
857
- stop: false,
858
- pass: false,
859
- reason: "critical evals still failing",
860
- score: averageScore(evals)
861
- };
862
- }
863
- function averageScore(evals) {
864
- const scored = evals.map((result) => result.score).filter((score) => typeof score === "number");
865
- if (!scored.length) return void 0;
866
- return Math.round(scored.reduce((sum, score) => sum + score, 0) / scored.length * 1e3) / 1e3;
867
- }
868
- function budgetStopDecision(budget, spentCostUsd) {
869
- if (budget.maxCostUsd !== void 0 && spentCostUsd >= budget.maxCostUsd) {
870
- return {
871
- stop: true,
872
- reason: `budget exhausted: maxCostUsd=${budget.maxCostUsd}`
873
- };
874
- }
875
- return { stop: false, reason: "" };
876
- }
877
- async function recordCostBudget(emitter, budget, spentCostUsd, handle, runtimeErrors, stepIndex) {
878
- if (!emitter || budget.maxCostUsd === void 0) return;
879
- const maxCostUsd = budget.maxCostUsd;
880
- await runTrace(
881
- runtimeErrors,
882
- stepIndex,
883
- () => emitter.recordBudget({
884
- dimension: "usd",
885
- limit: maxCostUsd,
886
- consumed: spentCostUsd,
887
- remaining: Math.max(0, maxCostUsd - spentCostUsd),
888
- breached: spentCostUsd >= maxCostUsd,
889
- spanId: handle?.span.spanId
890
- })
891
- );
892
- }
893
- async function recordEvalSpans(emitter, evals, phase, runtimeErrors, stepIndex, targetSpanId) {
894
- if (!emitter) return;
895
- for (const result of evals) {
896
- await runTrace(
897
- runtimeErrors,
898
- stepIndex,
899
- () => emitter.recordJudge({
900
- judgeId: result.objective ? "objective-validator" : "subjective-judge",
901
- targetSpanId: targetSpanId ?? emitter.runId,
902
- name: `control-eval/${result.id}`,
903
- dimension: result.id,
904
- score: typeof result.score === "number" ? result.score : result.passed ? 1 : 0,
905
- rationale: result.detail,
906
- evidence: result.evidence,
907
- attributes: {
908
- phase,
909
- passed: result.passed,
910
- severity: result.severity,
911
- objective: result.objective
912
- }
913
- })
914
- );
915
- }
916
- }
917
- async function runOnStep(onStep, step, runtimeErrors) {
918
- if (!onStep) return;
919
- try {
920
- await onStep(step);
921
- } catch (err) {
922
- runtimeErrors.push(runtimeError("on-step", step.index, err));
923
- }
924
- }
925
- async function runTrace(runtimeErrors, stepIndex, write) {
926
- try {
927
- return await write();
928
- } catch (err) {
929
- runtimeErrors.push(runtimeError("trace", stepIndex, err));
930
- return void 0;
931
- }
932
- }
933
- function fingerprintState(state, policies) {
934
- if (policies?.stateFingerprint) return policies.stateFingerprint(state);
935
- return stableFingerprint(state);
936
- }
937
- function fingerprintAction(action, policies) {
938
- if (policies?.actionFingerprint) return policies.actionFingerprint(action);
939
- return stableFingerprint(action);
940
- }
941
- function stableFingerprint(value) {
942
- if (typeof value === "string") return value;
943
- if (typeof value === "number" || typeof value === "boolean" || value == null) return String(value);
944
- try {
945
- return JSON.stringify(sortForFingerprint(value));
946
- } catch {
947
- return String(value);
948
- }
949
- }
950
- function sortForFingerprint(value) {
951
- if (Array.isArray(value)) return value.map(sortForFingerprint);
952
- if (!value || typeof value !== "object") return value;
953
- const record = value;
954
- const sorted = {};
955
- for (const key of Object.keys(record).sort()) {
956
- sorted[key] = sortForFingerprint(record[key]);
957
- }
958
- return sorted;
959
- }
960
- function abortReason(signal) {
961
- const reason = signal.reason;
962
- if (reason instanceof Error) return reason.message;
963
- return reason ? String(reason) : "aborted";
964
- }
965
- function runtimeError(phase, stepIndex, err) {
966
- const message = err instanceof Error ? err.message : String(err);
967
- return { phase, stepIndex, message };
968
- }
969
- async function finish(emitter, result) {
970
- await runTrace(
971
- result.runtimeErrors,
972
- result.steps.length,
973
- () => emitter?.endRun({
974
- pass: result.pass,
975
- score: result.score ?? averageScore(result.finalEvals),
976
- failureClass: result.failureClass,
977
- notes: result.reason
978
- })
979
- );
980
- return result;
981
- }
982
-
983
- // src/run-evidence.ts
984
- function controlRunToRunRecord(run, options) {
985
- const score = finiteScore(options.score) ?? finiteScore(run.score) ?? scoreFromEvals(run.finalEvals);
986
- const outcome = options.splitTag === "holdout" ? {
987
- ...score !== void 0 ? { holdoutScore: score } : {},
988
- raw: normalizeRawMetrics(options.raw, run, score)
989
- } : {
990
- ...score !== void 0 ? { searchScore: score } : {},
991
- raw: normalizeRawMetrics(options.raw, run, score)
992
- };
993
- const terminalOutcome = run.stoppedBy === "abort" ? "cancelled" : run.stoppedBy === "runtime-error" ? "failed" : run.completed ? "succeeded" : "incomplete";
994
- const costUsd = options.costProvenance.kind === "uncaptured" ? null : options.costProvenance.usd;
995
- if (costUsd !== null && costUsd !== run.spentCostUsd) {
996
- throw new Error(
997
- `cost provenance amount ${costUsd} does not match control run spend ${run.spentCostUsd}`
998
- );
999
- }
1000
- return validateRunRecord({
1001
- runId: options.runId ?? run.runId ?? `control:${options.experimentId}:${options.candidateId}:${options.seed}:${options.splitTag}`,
1002
- experimentId: options.experimentId,
1003
- candidateId: options.candidateId,
1004
- seed: options.seed,
1005
- model: options.model,
1006
- promptHash: options.promptHash,
1007
- configHash: options.configHash,
1008
- commitSha: options.commitSha,
1009
- wallMs: run.wallMs,
1010
- ...options.queueMs !== void 0 ? { queueMs: options.queueMs } : {},
1011
- costUsd,
1012
- costProvenance: options.costProvenance,
1013
- tokenUsage: options.tokenUsage,
1014
- terminalOutcome,
1015
- ...terminalOutcome !== "succeeded" ? { terminalFailureReason: run.reason } : {},
1016
- ...options.judgeMetadata ? { judgeMetadata: options.judgeMetadata } : {},
1017
- outcome,
1018
- ...options.failureClass !== void 0 ? { failureClass: options.failureClass } : {},
1019
- ...options.failureMode !== void 0 ? { failureMode: options.failureMode } : {},
1020
- splitTag: options.splitTag,
1021
- scenarioId: options.scenarioId
1022
- });
1023
- }
1024
- function scoreFromEvals(evals) {
1025
- const scores = evals.map((e) => e.score).filter((score) => typeof score === "number" && Number.isFinite(score));
1026
- if (scores.length === 0) return void 0;
1027
- return clampScore(scores.reduce((sum, score) => sum + score, 0) / scores.length);
1028
- }
1029
- function normalizeRawMetrics(raw, run, score) {
1030
- const normalizedRaw = finiteOnly(raw ?? {});
1031
- delete normalizedRaw.score;
1032
- return {
1033
- ...normalizedRaw,
1034
- ...score !== void 0 ? { score } : {},
1035
- pass: run.pass ? 1 : 0,
1036
- completed: run.completed ? 1 : 0,
1037
- steps: run.steps.length,
1038
- runtimeErrors: run.runtimeErrors.length,
1039
- execution_error_count: executionErrorCount(run)
1040
- };
1041
- }
1042
- function executionErrorCount(run) {
1043
- const thrownActionSteps = new Set(
1044
- run.runtimeErrors.filter((error) => error.phase === "act").map((error) => error.stepIndex)
1045
- );
1046
- const failedActionsWithoutRuntimeError = run.steps.filter(
1047
- (step) => step.actionOutcome?.ok === false && !thrownActionSteps.has(step.index)
1048
- ).length;
1049
- return run.runtimeErrors.length + failedActionsWithoutRuntimeError;
1050
- }
1051
- function finiteOnly(values) {
1052
- const out = {};
1053
- for (const [key, value] of Object.entries(values)) {
1054
- if (Number.isFinite(value)) out[key] = value;
1055
- }
1056
- return out;
1057
- }
1058
- function finiteScore(value) {
1059
- return value !== void 0 && Number.isFinite(value) ? clampScore(value) : void 0;
1060
- }
1061
- function clampScore(value) {
1062
- return Math.max(0, Math.min(1, value));
1063
- }
1064
-
1065
- // src/propose-review.ts
1066
- import { appendFileSync, existsSync, mkdirSync, readFileSync } from "fs";
1067
- import { dirname } from "path";
1068
- function inMemoryReviewStore(initial = []) {
1069
- const entries = [...initial];
1070
- return {
1071
- async load() {
1072
- return [...entries];
1073
- },
1074
- async append(entry) {
1075
- entries.push(entry);
1076
- }
1077
- };
1078
- }
1079
- function jsonlReviewStore(path) {
1080
- return {
1081
- async load() {
1082
- if (!existsSync(path)) return [];
1083
- const raw = readFileSync(path, "utf8");
1084
- const out = [];
1085
- for (const line of raw.split("\n")) {
1086
- const trimmed = line.trim();
1087
- if (!trimmed) continue;
1088
- try {
1089
- out.push(JSON.parse(trimmed));
1090
- } catch {
1091
- }
1092
- }
1093
- return out;
1094
- },
1095
- async append(entry) {
1096
- mkdirSync(dirname(path), { recursive: true });
1097
- appendFileSync(path, `${JSON.stringify(entry)}
1098
- `);
1099
- }
1100
- };
1101
- }
1102
- var DEFAULT_FALLBACK_INSTRUCTION = "Inspect the verification failures above. Fix the critical issues first, then the major ones. Do not restate the failures \u2014 act on them.";
1103
- async function runProposeReview(config) {
1104
- const maxShots = config.maxShots ?? 10;
1105
- const maxWallMs = config.maxWallMs ?? 10 * 60 * 1e3;
1106
- const confidenceFloor = config.confidenceFloor ?? 0.3;
1107
- const confidenceFloorWindow = config.confidenceFloorWindow ?? 2;
1108
- const memory = config.memory ?? inMemoryReviewStore();
1109
- const fallbackInstruction = config.fallbackInstruction ?? DEFAULT_FALLBACK_INSTRUCTION;
1110
- const emitter = config.store ? new TraceEmitter(config.store) : null;
1111
- if (emitter) {
1112
- await emitter.startRun({
1113
- scenarioId: config.scenarioId ?? "propose-review",
1114
- projectId: config.projectId,
1115
- variantId: config.variantId,
1116
- layer: "meta",
1117
- tags: {
1118
- goal: config.goal.slice(0, 120),
1119
- maxShots: String(maxShots)
1120
- }
1121
- });
1122
- }
1123
- const abort = new AbortController();
1124
- const wallStart = Date.now();
1125
- const wallTimer = setTimeout(
1126
- () => abort.abort(new Error("propose-review wall timeout")),
1127
- maxWallMs
1128
- );
1129
- const shots = [];
1130
- let state = config.initialState;
1131
- let priorReview = null;
1132
- let lastVerification = { pass: false };
1133
- let failureClass;
1134
- let completed = false;
1135
- let lowConfidenceStreak = 0;
1136
- try {
1137
- for (let shot = 1; shot <= maxShots; shot++) {
1138
- if (abort.signal.aborted) {
1139
- failureClass = "timeout";
1140
- break;
1141
- }
1142
- const shotStart = Date.now();
1143
- const shotHandle = emitter ? await emitter.span({ kind: "tool", name: `shot-${shot}` }) : null;
1144
- let proposeOut;
1145
- try {
1146
- proposeOut = await config.propose({
1147
- shot,
1148
- goal: config.goal,
1149
- state,
1150
- priorReview,
1151
- abortSignal: abort.signal,
1152
- emitter: emitter ?? void 0
1153
- });
1154
- } catch (err) {
1155
- await shotHandle?.fail(err instanceof Error ? err : String(err));
1156
- failureClass = "unknown";
1157
- throw err;
1158
- }
1159
- state = proposeOut.state;
1160
- const traceSummary = proposeOut.traceSummary;
1161
- let verification;
1162
- try {
1163
- verification = await config.verify(state);
1164
- } catch (err) {
1165
- await shotHandle?.fail(err instanceof Error ? err : String(err));
1166
- failureClass = "unknown";
1167
- throw err;
1168
- }
1169
- lastVerification = verification;
1170
- const memorySnapshot = await memory.load();
1171
- const verificationDigest = {
1172
- pass: verification.pass,
1173
- score: verification.score,
1174
- failingLayers: verification.failingLayers ?? []
1175
- };
1176
- let review;
1177
- let reviewAvailable = true;
1178
- let reviewError;
1179
- if (verification.pass) {
1180
- review = {
1181
- observations: "verification passed \u2014 skipping reviewer LLM call",
1182
- diagnosis: "no failures to diagnose",
1183
- nextShotInstruction: "(done)",
1184
- shouldContinue: false,
1185
- confidence: 1
1186
- };
1187
- } else {
1188
- try {
1189
- review = await config.review({
1190
- shot,
1191
- goal: config.goal,
1192
- state,
1193
- verification,
1194
- traceSummary,
1195
- memory: memorySnapshot
1196
- });
1197
- review = coerceReview(review);
1198
- } catch (err) {
1199
- reviewAvailable = false;
1200
- reviewError = err instanceof Error ? err.message : String(err);
1201
- const lastInstruction = memorySnapshot.length > 0 ? memorySnapshot[memorySnapshot.length - 1].nextShotInstruction : fallbackInstruction;
1202
- review = {
1203
- observations: "(reviewer unavailable \u2014 using last-known instruction)",
1204
- diagnosis: reviewError,
1205
- nextShotInstruction: lastInstruction,
1206
- shouldContinue: true,
1207
- confidence: 0.3
1208
- };
1209
- }
1210
- }
1211
- const entry = {
1212
- shot,
1213
- timestamp: Date.now(),
1214
- ...review,
1215
- verification: verificationDigest
1216
- };
1217
- await memory.append(entry);
1218
- const shotRecord = {
1219
- shot,
1220
- state,
1221
- verification,
1222
- traceSummary,
1223
- review,
1224
- reviewAvailable,
1225
- reviewError,
1226
- durationMs: Date.now() - shotStart
1227
- };
1228
- shots.push(shotRecord);
1229
- await shotHandle?.end({
1230
- attributes: {
1231
- verificationPass: verification.pass,
1232
- verificationScore: verification.score ?? null,
1233
- reviewShouldContinue: review.shouldContinue,
1234
- reviewConfidence: review.confidence,
1235
- reviewAvailable
1236
- }
1237
- });
1238
- if (verification.pass) {
1239
- completed = true;
1240
- break;
1241
- }
1242
- if (!review.shouldContinue) {
1243
- break;
1244
- }
1245
- if (confidenceFloorWindow > 0 && review.confidence <= confidenceFloor) {
1246
- lowConfidenceStreak += 1;
1247
- if (lowConfidenceStreak >= confidenceFloorWindow) break;
1248
- } else {
1249
- lowConfidenceStreak = 0;
1250
- }
1251
- priorReview = review;
1252
- }
1253
- if (!completed && !failureClass) {
1254
- failureClass = shots.length >= maxShots ? "budget_exceeded" : "unknown";
1255
- }
1256
- } finally {
1257
- clearTimeout(wallTimer);
1258
- }
1259
- const score = lastVerification.pass ? 1 : typeof lastVerification.score === "number" ? lastVerification.score : 0;
1260
- if (emitter) {
1261
- await emitter.endRun({
1262
- pass: completed,
1263
- score,
1264
- failureClass,
1265
- notes: `${shots.length} shot(s); final pass=${lastVerification.pass}`
1266
- });
1267
- }
1268
- return {
1269
- runId: emitter?.runId ?? null,
1270
- completed,
1271
- shots,
1272
- finalState: state,
1273
- finalVerification: lastVerification,
1274
- failureClass,
1275
- wallMs: Date.now() - wallStart,
1276
- score
1277
- };
1278
- }
1279
- var REVIEWER_SYSTEM_PROMPT = `You are a senior reviewer directing a multi-shot build loop.
1280
- You do NOT grade \u2014 the verifier already did. Your job is to direct the worker's next shot.
1281
- You are blind to the worker's inner monologue. You see what it DID, not what it thought.
1282
- Return STRICT JSON matching the schema. No prose outside the JSON.`;
1283
- function createLlmReviewer(cfg) {
1284
- const renderState = cfg.renderState ?? ((s) => safeJson(s));
1285
- const renderTraceSummary = cfg.renderTraceSummary ?? ((s) => s === void 0 ? "(none)" : safeJson(s));
1286
- const system = cfg.systemPromptAddendum ? `${REVIEWER_SYSTEM_PROMPT}
1287
-
1288
- ${cfg.systemPromptAddendum}` : REVIEWER_SYSTEM_PROMPT;
1289
- return async (input) => {
1290
- const memoryBlock = input.memory.length === 0 ? "(no prior shots \u2014 this is shot 1)" : input.memory.map(
1291
- (m) => [
1292
- `shot ${m.shot} \u2014 verification.pass=${m.verification.pass}` + (typeof m.verification.score === "number" ? ` score=${m.verification.score.toFixed(2)}` : "") + ` confidence=${m.confidence.toFixed(2)} failing=[${(m.verification.failingLayers ?? []).join(",")}]`,
1293
- ` observations: ${m.observations.slice(0, 400)}`,
1294
- ` diagnosis: ${m.diagnosis.slice(0, 400)}`,
1295
- ` instruction given: ${m.nextShotInstruction.slice(0, 400)}`
1296
- ].join("\n")
1297
- ).join("\n\n");
1298
- const user = [
1299
- `=== GOAL ===`,
1300
- input.goal,
1301
- ``,
1302
- `=== SHOT NUMBER ===`,
1303
- String(input.shot),
1304
- ``,
1305
- `=== CURRENT STATE ===`,
1306
- renderState(input.state),
1307
- ``,
1308
- `=== TRACE SUMMARY ===`,
1309
- renderTraceSummary(input.traceSummary),
1310
- ``,
1311
- `=== VERIFICATION ===`,
1312
- summarizeVerification(input.verification),
1313
- ``,
1314
- `=== REVIEWER MEMORY (prior shots) ===`,
1315
- memoryBlock,
1316
- ``,
1317
- `=== YOUR TASK ===`,
1318
- `Return STRICT JSON:`,
1319
- `{`,
1320
- ` "observations": string (20..2000 chars, first-person worker behavior \u2014 quote counts, errors, loops)`,
1321
- ` "diagnosis": string (20..1500 chars, root cause, NOT a restatement of verification)`,
1322
- ` "nextShotInstruction": string (40..3000 chars, concrete directive to the worker)`,
1323
- ` "shouldContinue": boolean (false if verification.pass, or if thrashing, or unachievable)`,
1324
- ` "confidence": number in [0,1]`,
1325
- `}`
1326
- ].join("\n");
1327
- const raw = await cfg.callJson({ system, user });
1328
- return coerceReview(raw);
1329
- };
1330
- }
1331
- function coerceReview(raw) {
1332
- if (!raw || typeof raw !== "object") {
1333
- throw new Error("reviewer returned non-object");
1334
- }
1335
- const observations = typeof raw.observations === "string" ? raw.observations : "";
1336
- const diagnosis = typeof raw.diagnosis === "string" ? raw.diagnosis : "";
1337
- const nextShotInstruction = typeof raw.nextShotInstruction === "string" ? raw.nextShotInstruction : "";
1338
- if (!observations || !diagnosis || !nextShotInstruction) {
1339
- throw new Error("reviewer missing required string fields");
1340
- }
1341
- if (typeof raw.shouldContinue !== "boolean") {
1342
- throw new Error("reviewer missing shouldContinue boolean");
1343
- }
1344
- const confidenceRaw = Number(raw.confidence);
1345
- if (!Number.isFinite(confidenceRaw)) {
1346
- throw new Error("reviewer confidence not finite");
1347
- }
1348
- return {
1349
- observations,
1350
- diagnosis,
1351
- nextShotInstruction,
1352
- shouldContinue: raw.shouldContinue,
1353
- confidence: Math.max(0, Math.min(1, confidenceRaw))
1354
- };
1355
- }
1356
- function summarizeVerification(v) {
1357
- const header = `pass=${v.pass}` + (typeof v.score === "number" ? ` score=${v.score.toFixed(3)}` : "") + (v.failingLayers && v.failingLayers.length > 0 ? ` failing=[${v.failingLayers.join(", ")}]` : "");
1358
- const details = v.details === void 0 ? "" : `
1359
- ${safeJson(v.details).slice(0, 1500)}`;
1360
- return header + details;
1361
- }
1362
- function safeJson(x) {
1363
- try {
1364
- return JSON.stringify(x, null, 2);
1365
- } catch {
1366
- return String(x);
1367
- }
1368
- }
1369
-
1370
- // src/propose-review-control.ts
1371
- var DEFAULT_FALLBACK_INSTRUCTION2 = "Inspect the verification failures above. Fix the critical issues first, then the major ones. Do not restate the failures \u2014 act on them.";
1372
- async function runProposeReviewAsControlLoop(config) {
1373
- const maxShots = config.maxShots ?? 10;
1374
- const confidenceFloor = config.confidenceFloor ?? 0.3;
1375
- const confidenceFloorWindow = config.confidenceFloorWindow ?? 2;
1376
- const memory = config.memory ?? inMemoryReviewStore();
1377
- const fallbackInstruction = config.fallbackInstruction ?? DEFAULT_FALLBACK_INSTRUCTION2;
1378
- const failureClassFromVerification = config.failureClassFromVerification ?? controlFailureClassFromVerification;
1379
- let lowConfidenceStreak = 0;
1380
- let current = {
1381
- shot: 0,
1382
- state: config.initialState,
1383
- priorReview: null,
1384
- verification: { pass: false },
1385
- memory: await memory.load(),
1386
- completed: false,
1387
- reviewAvailable: false
1388
- };
1389
- return runAgentControlLoop({
1390
- intent: config.goal,
1391
- budget: { maxSteps: maxShots, maxWallMs: config.maxWallMs },
1392
- store: config.store,
1393
- scenarioId: config.scenarioId ?? "propose-review-control",
1394
- projectId: config.projectId,
1395
- variantId: config.variantId,
1396
- actionFailure: config.actionFailure ?? "stop",
1397
- observe: () => current,
1398
- validate: ({ state }) => [
1399
- objectiveEval({
1400
- id: "verification",
1401
- passed: state.verification.pass,
1402
- score: state.verification.score,
1403
- severity: "critical",
1404
- detail: state.verification.pass ? "verification passed" : `verification failed${state.verification.failingLayers?.length ? `: ${state.verification.failingLayers.join(", ")}` : ""}`
1405
- })
1406
- ],
1407
- shouldStop: ({ state }) => {
1408
- if (state.verification.pass) {
1409
- return {
1410
- stop: true,
1411
- pass: true,
1412
- reason: "verification passed",
1413
- score: state.verification.score
1414
- };
1415
- }
1416
- if (state.completed) {
1417
- return {
1418
- stop: true,
1419
- pass: false,
1420
- reason: "reviewer stopped continuation",
1421
- score: state.verification.score,
1422
- failureClass: failureClassFromVerification(state.verification)
1423
- };
1424
- }
1425
- return {
1426
- stop: false,
1427
- pass: false,
1428
- reason: "verification still failing",
1429
- score: state.verification.score
1430
- };
1431
- },
1432
- decide: ({ state }) => ({
1433
- type: "continue",
1434
- action: { type: "propose-review-shot", shot: state.shot + 1 },
1435
- reason: state.priorReview?.nextShotInstruction ?? fallbackInstruction
1436
- }),
1437
- act: async (action, ctx) => {
1438
- const shot = action.shot;
1439
- const proposeOut = await config.propose({
1440
- shot,
1441
- goal: config.goal,
1442
- state: current.state,
1443
- priorReview: current.priorReview,
1444
- abortSignal: ctx.abortSignal,
1445
- emitter: ctx.emitter
1446
- });
1447
- const nextState = proposeOut.state;
1448
- const verification = await config.verify(nextState);
1449
- let review = null;
1450
- let reviewAvailable = false;
1451
- let reviewError;
1452
- let shouldContinue = !verification.pass;
1453
- if (!verification.pass) {
1454
- try {
1455
- review = await config.review({
1456
- shot,
1457
- goal: config.goal,
1458
- state: nextState,
1459
- verification,
1460
- traceSummary: proposeOut.traceSummary,
1461
- memory: await memory.load()
1462
- });
1463
- reviewAvailable = true;
1464
- shouldContinue = review.shouldContinue;
1465
- lowConfidenceStreak = review.confidence <= confidenceFloor ? lowConfidenceStreak + 1 : 0;
1466
- if (confidenceFloorWindow > 0 && lowConfidenceStreak >= confidenceFloorWindow)
1467
- shouldContinue = false;
1468
- } catch (err) {
1469
- reviewError = err instanceof Error ? err.message : String(err);
1470
- review = current.priorReview ?? {
1471
- observations: "Reviewer unavailable.",
1472
- diagnosis: reviewError,
1473
- nextShotInstruction: fallbackInstruction,
1474
- shouldContinue: true,
1475
- confidence: 0
1476
- };
1477
- shouldContinue = true;
1478
- }
1479
- } else {
1480
- review = {
1481
- observations: "Verification passed.",
1482
- diagnosis: "No further revision needed.",
1483
- nextShotInstruction: "",
1484
- shouldContinue: false,
1485
- confidence: 1
1486
- };
1487
- }
1488
- const entry = {
1489
- ...review ?? {
1490
- observations: "No review.",
1491
- diagnosis: "",
1492
- nextShotInstruction: fallbackInstruction,
1493
- shouldContinue,
1494
- confidence: 0
1495
- },
1496
- shot,
1497
- timestamp: Date.now(),
1498
- verification: {
1499
- pass: verification.pass,
1500
- score: verification.score,
1501
- failingLayers: verification.failingLayers
1502
- }
1503
- };
1504
- await memory.append(entry);
1505
- current = {
1506
- shot,
1507
- state: nextState,
1508
- priorReview: review,
1509
- verification,
1510
- traceSummary: proposeOut.traceSummary,
1511
- memory: await memory.load(),
1512
- completed: verification.pass || !shouldContinue,
1513
- reviewAvailable,
1514
- reviewError
1515
- };
1516
- return {
1517
- state: nextState,
1518
- verification,
1519
- traceSummary: proposeOut.traceSummary,
1520
- review,
1521
- reviewAvailable,
1522
- reviewError
1523
- };
1524
- }
1525
- });
1526
- }
1527
- function controlFailureClassFromVerification(verification) {
1528
- if (verification.pass) return void 0;
1529
- return verification.failingLayers?.length ? "instruction_following" : "unknown";
1530
- }
1531
-
1532
- export {
1533
- evaluateActionPolicy,
1534
- repeatedActionDetector,
1535
- noProgressDetector,
1536
- errorStreakDetector,
1537
- observeAll,
1538
- runAgentControlLoop,
1539
- stopOnNoProgress,
1540
- stopOnRepeatedAction,
1541
- objectiveEval,
1542
- subjectiveEval,
1543
- allCriticalPassed,
1544
- controlRunToRunRecord,
1545
- scoreFromEvals,
1546
- inMemoryReviewStore,
1547
- jsonlReviewStore,
1548
- runProposeReview,
1549
- createLlmReviewer,
1550
- runProposeReviewAsControlLoop,
1551
- controlFailureClassFromVerification
1552
- };
1553
- //# sourceMappingURL=chunk-WVATSFCP.js.map