@tangle-network/agent-eval 0.129.0 → 0.130.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (428) hide show
  1. package/CHANGELOG.md +21 -0
  2. package/README.md +2 -0
  3. package/dist/active-curriculum-C4mk67HP.js +214 -0
  4. package/dist/active-curriculum-C4mk67HP.js.map +1 -0
  5. package/dist/adversarial-smnADNFS.d.ts +21 -0
  6. package/dist/adversarial-smnADNFS.d.ts.map +1 -0
  7. package/dist/analyst/index.d.ts +81 -2872
  8. package/dist/analyst/index.d.ts.map +1 -0
  9. package/dist/analyst/index.js +319 -360
  10. package/dist/analyst/index.js.map +1 -1
  11. package/dist/analyst-BkTS3C58.d.ts +89 -0
  12. package/dist/analyst-BkTS3C58.d.ts.map +1 -0
  13. package/dist/analyst-LsnNpSkm.js +152 -0
  14. package/dist/analyst-LsnNpSkm.js.map +1 -0
  15. package/dist/analyze-runs-C1CavBMk.js +1067 -0
  16. package/dist/analyze-runs-C1CavBMk.js.map +1 -0
  17. package/dist/analyze-runs-FsgYCinh.d.ts +72 -0
  18. package/dist/analyze-runs-FsgYCinh.d.ts.map +1 -0
  19. package/dist/attribute-vocabulary-DLJ6303h.d.ts +54 -0
  20. package/dist/attribute-vocabulary-DLJ6303h.d.ts.map +1 -0
  21. package/dist/authenticity/index.d.ts +81 -79
  22. package/dist/authenticity/index.d.ts.map +1 -0
  23. package/dist/authenticity/index.js +209 -193
  24. package/dist/authenticity/index.js.map +1 -1
  25. package/dist/baseline-HsBvw_dk.js +550 -0
  26. package/dist/baseline-HsBvw_dk.js.map +1 -0
  27. package/dist/baseline-hG3K85h4.d.ts +125 -0
  28. package/dist/baseline-hG3K85h4.d.ts.map +1 -0
  29. package/dist/belief-state/index.d.ts +448 -1188
  30. package/dist/belief-state/index.d.ts.map +1 -0
  31. package/dist/belief-state/index.js +1617 -1709
  32. package/dist/belief-state/index.js.map +1 -1
  33. package/dist/benchmarks/index.d.ts +2 -891
  34. package/dist/benchmarks/index.js +2 -60
  35. package/dist/benchmarks-BJgDGkAD.js +754 -0
  36. package/dist/benchmarks-BJgDGkAD.js.map +1 -0
  37. package/dist/builder-eval/index.d.ts +150 -662
  38. package/dist/builder-eval/index.d.ts.map +1 -0
  39. package/dist/builder-eval/index.js +356 -345
  40. package/dist/builder-eval/index.js.map +1 -1
  41. package/dist/calibration-CNWWA6K8.js +94 -0
  42. package/dist/calibration-CNWWA6K8.js.map +1 -0
  43. package/dist/campaign/index.d.ts +5 -6381
  44. package/dist/campaign/index.js +3 -213
  45. package/dist/campaign-aKJt6emI.js +3886 -0
  46. package/dist/campaign-aKJt6emI.js.map +1 -0
  47. package/dist/cli.d.ts +1 -1
  48. package/dist/cli.js +164 -175
  49. package/dist/cli.js.map +1 -1
  50. package/dist/client-C97NMzqi.d.ts +581 -0
  51. package/dist/client-C97NMzqi.d.ts.map +1 -0
  52. package/dist/client-CYzbdJOZ.js +637 -0
  53. package/dist/client-CYzbdJOZ.js.map +1 -0
  54. package/dist/code-agent-session-BjkMTQ7H.js +1390 -0
  55. package/dist/code-agent-session-BjkMTQ7H.js.map +1 -0
  56. package/dist/code-agent-session-DqqgOJaz.d.ts +143 -0
  57. package/dist/code-agent-session-DqqgOJaz.d.ts.map +1 -0
  58. package/dist/concurrency-DIxRZF_J.js +85 -0
  59. package/dist/concurrency-DIxRZF_J.js.map +1 -0
  60. package/dist/contract/index.d.ts +645 -5565
  61. package/dist/contract/index.d.ts.map +1 -0
  62. package/dist/contract/index.js +1730 -1938
  63. package/dist/contract/index.js.map +1 -1
  64. package/dist/control.d.ts +3 -1030
  65. package/dist/control.js +2 -33
  66. package/dist/cost-ledger-DIgQUFZZ.js +801 -0
  67. package/dist/cost-ledger-DIgQUFZZ.js.map +1 -0
  68. package/dist/cost-ledger-Dye6jCgg.d.ts +236 -0
  69. package/dist/cost-ledger-Dye6jCgg.d.ts.map +1 -0
  70. package/dist/dataset-BvtnC8Dc.d.ts +115 -0
  71. package/dist/dataset-BvtnC8Dc.d.ts.map +1 -0
  72. package/dist/default-registry-C-vFCSEc.js +2579 -0
  73. package/dist/default-registry-C-vFCSEc.js.map +1 -0
  74. package/dist/default-registry-CNPo-Vsb.d.ts +540 -0
  75. package/dist/default-registry-CNPo-Vsb.d.ts.map +1 -0
  76. package/dist/emitter-CPBAhxum.js +266 -0
  77. package/dist/emitter-CPBAhxum.js.map +1 -0
  78. package/dist/emitter-DGQGoLyj.d.ts +113 -0
  79. package/dist/emitter-DGQGoLyj.d.ts.map +1 -0
  80. package/dist/errors-8YnH8WlF.js +64 -0
  81. package/dist/errors-8YnH8WlF.js.map +1 -0
  82. package/dist/errors-CEk209JS.d.ts +76 -0
  83. package/dist/errors-CEk209JS.d.ts.map +1 -0
  84. package/dist/eval-campaign-DEm6c8ru.js +349 -0
  85. package/dist/eval-campaign-DEm6c8ru.js.map +1 -0
  86. package/dist/execution-tracks-CpgFPpS5.js +93 -0
  87. package/dist/execution-tracks-CpgFPpS5.js.map +1 -0
  88. package/dist/exporters-q9iL-2Jf.js +148 -0
  89. package/dist/exporters-q9iL-2Jf.js.map +1 -0
  90. package/dist/extract-usage-BrQ8mCLX.js +155 -0
  91. package/dist/extract-usage-BrQ8mCLX.js.map +1 -0
  92. package/dist/failure-cluster-CqcvCcdR.d.ts +59 -0
  93. package/dist/failure-cluster-CqcvCcdR.d.ts.map +1 -0
  94. package/dist/feedback-trajectory-CVaeREXV.d.ts +340 -0
  95. package/dist/feedback-trajectory-CVaeREXV.d.ts.map +1 -0
  96. package/dist/fuzz.d.ts +320 -646
  97. package/dist/fuzz.d.ts.map +1 -0
  98. package/dist/fuzz.js +670 -618
  99. package/dist/fuzz.js.map +1 -1
  100. package/dist/hf-dataset-DBJXXoY1.js +763 -0
  101. package/dist/hf-dataset-DBJXXoY1.js.map +1 -0
  102. package/dist/hosted/index.d.ts +11 -831
  103. package/dist/hosted/index.d.ts.map +1 -0
  104. package/dist/hosted/index.js +2 -37
  105. package/dist/index-2JJSA6-r2.d.ts +926 -0
  106. package/dist/index-2JJSA6-r2.d.ts.map +1 -0
  107. package/dist/index-6N0aYmpW.d.ts +217 -0
  108. package/dist/index-6N0aYmpW.d.ts.map +1 -0
  109. package/dist/index-BAvgST_9.d.ts +131 -0
  110. package/dist/index-BAvgST_9.d.ts.map +1 -0
  111. package/dist/index-BvnJuTGD.d.ts +68 -0
  112. package/dist/index-BvnJuTGD.d.ts.map +1 -0
  113. package/dist/index-C61Wi7yg.d.ts +547 -0
  114. package/dist/index-C61Wi7yg.d.ts.map +1 -0
  115. package/dist/index-CD_WZ_Xr.d.ts +2250 -0
  116. package/dist/index-CD_WZ_Xr.d.ts.map +1 -0
  117. package/dist/index-DSC51roc.d.ts +102 -0
  118. package/dist/index-DSC51roc.d.ts.map +1 -0
  119. package/dist/index-Em67JBjs.d.ts +335 -0
  120. package/dist/index-Em67JBjs.d.ts.map +1 -0
  121. package/dist/index.d.ts +3755 -15555
  122. package/dist/index.d.ts.map +1 -0
  123. package/dist/index.js +11182 -11216
  124. package/dist/index.js.map +1 -1
  125. package/dist/integrity-BzRbCHzi.js +424 -0
  126. package/dist/integrity-BzRbCHzi.js.map +1 -0
  127. package/dist/integrity-rmVhXWA7.d.ts +61 -0
  128. package/dist/integrity-rmVhXWA7.d.ts.map +1 -0
  129. package/dist/judge-calibration-DFtEMlde.d.ts +146 -0
  130. package/dist/judge-calibration-DFtEMlde.d.ts.map +1 -0
  131. package/dist/ledger-core/index.d.ts +2 -0
  132. package/dist/ledger-core/index.js +2 -0
  133. package/dist/ledger-core-DtZz1RG0.js +388 -0
  134. package/dist/ledger-core-DtZz1RG0.js.map +1 -0
  135. package/dist/llm-client--GR4JbZE.js +687 -0
  136. package/dist/llm-client--GR4JbZE.js.map +1 -0
  137. package/dist/llm-client-B_nIBlYo.d.ts +290 -0
  138. package/dist/llm-client-B_nIBlYo.d.ts.map +1 -0
  139. package/dist/matrix/index.d.ts +3 -155
  140. package/dist/matrix/index.js +2 -8
  141. package/dist/matrix-BzQnu2S6.js +270 -0
  142. package/dist/matrix-BzQnu2S6.js.map +1 -0
  143. package/dist/meta-eval/index.d.ts +4 -1027
  144. package/dist/meta-eval/index.js +393 -390
  145. package/dist/meta-eval/index.js.map +1 -1
  146. package/dist/metrics-C9YY1OcL.js +239 -0
  147. package/dist/metrics-C9YY1OcL.js.map +1 -0
  148. package/dist/mint-yN2M2eh0.js +201 -0
  149. package/dist/mint-yN2M2eh0.js.map +1 -0
  150. package/dist/multi-layer-verifier-BHY1gWAc.d.ts +138 -0
  151. package/dist/multi-layer-verifier-BHY1gWAc.d.ts.map +1 -0
  152. package/dist/multishot/index.d.ts +273 -480
  153. package/dist/multishot/index.d.ts.map +1 -0
  154. package/dist/multishot/index.js +595 -548
  155. package/dist/multishot/index.js.map +1 -1
  156. package/dist/off-policy-DvgzvtIx.js +220 -0
  157. package/dist/off-policy-DvgzvtIx.js.map +1 -0
  158. package/dist/off-policy-mskQw8Mb.d.ts +153 -0
  159. package/dist/off-policy-mskQw8Mb.d.ts.map +1 -0
  160. package/dist/openapi.json +1 -1
  161. package/dist/opencode-sqlite-BGrHeDu3.js +318 -0
  162. package/dist/opencode-sqlite-BGrHeDu3.js.map +1 -0
  163. package/dist/outcome-store-BYHIuO0e.d.ts +65 -0
  164. package/dist/outcome-store-BYHIuO0e.d.ts.map +1 -0
  165. package/dist/outcome-store-ChBKlTd_.js +75 -0
  166. package/dist/outcome-store-ChBKlTd_.js.map +1 -0
  167. package/dist/paired-arms-D9D0wXj2.js +260 -0
  168. package/dist/paired-arms-D9D0wXj2.js.map +1 -0
  169. package/dist/pipelines/index.d.ts +95 -532
  170. package/dist/pipelines/index.d.ts.map +1 -0
  171. package/dist/pipelines/index.js +497 -478
  172. package/dist/pipelines/index.js.map +1 -1
  173. package/dist/pre-registration-DakwTRXk.js +96 -0
  174. package/dist/pre-registration-DakwTRXk.js.map +1 -0
  175. package/dist/propose-review-control-LhIWGzHl.js +1458 -0
  176. package/dist/propose-review-control-LhIWGzHl.js.map +1 -0
  177. package/dist/query-CJ_DX8vl.d.ts +28 -0
  178. package/dist/query-CJ_DX8vl.d.ts.map +1 -0
  179. package/dist/query-Di7eEQ79.js +83 -0
  180. package/dist/query-Di7eEQ79.js.map +1 -0
  181. package/dist/raw-provider-sink-BQd7mzyT.js +201 -0
  182. package/dist/raw-provider-sink-BQd7mzyT.js.map +1 -0
  183. package/dist/raw-provider-sink-BU29Sh8h.d.ts +134 -0
  184. package/dist/raw-provider-sink-BU29Sh8h.d.ts.map +1 -0
  185. package/dist/redact-7Aq1ukl-.js +107 -0
  186. package/dist/redact-7Aq1ukl-.js.map +1 -0
  187. package/dist/release-report-Cz9NKH39.js +603 -0
  188. package/dist/release-report-Cz9NKH39.js.map +1 -0
  189. package/dist/release-report-mvB2G4_J.d.ts +244 -0
  190. package/dist/release-report-mvB2G4_J.d.ts.map +1 -0
  191. package/dist/replay-GnyotH0J.js +1741 -0
  192. package/dist/replay-GnyotH0J.js.map +1 -0
  193. package/dist/replay-OoidtG1E.d.ts +749 -0
  194. package/dist/replay-OoidtG1E.d.ts.map +1 -0
  195. package/dist/reporting.d.ts +6 -1312
  196. package/dist/reporting.js +6 -51
  197. package/dist/researcher-CwTdwXG1.d.ts +314 -0
  198. package/dist/researcher-CwTdwXG1.d.ts.map +1 -0
  199. package/dist/reward-hacking-eAnOsynk.d.ts +224 -0
  200. package/dist/reward-hacking-eAnOsynk.d.ts.map +1 -0
  201. package/dist/reward-hacking-qipEpKvY.js +596 -0
  202. package/dist/reward-hacking-qipEpKvY.js.map +1 -0
  203. package/dist/reward-nw2xZGZG.js +137 -0
  204. package/dist/reward-nw2xZGZG.js.map +1 -0
  205. package/dist/rl.d.ts +760 -4010
  206. package/dist/rl.d.ts.map +1 -0
  207. package/dist/rl.js +2325 -1958
  208. package/dist/rl.js.map +1 -1
  209. package/dist/rolldown-runtime-8H4AJuhK.js +14 -0
  210. package/dist/rollout/index.d.ts +3 -2087
  211. package/dist/rollout/index.js +8 -168
  212. package/dist/rollout-BOYjemfR.js +624 -0
  213. package/dist/rollout-BOYjemfR.js.map +1 -0
  214. package/dist/rubric-predictive-validity-B3xmbmS1.js +141 -0
  215. package/dist/rubric-predictive-validity-B3xmbmS1.js.map +1 -0
  216. package/dist/rubric-predictive-validity-Ku_clp_1.d.ts +74 -0
  217. package/dist/rubric-predictive-validity-Ku_clp_1.d.ts.map +1 -0
  218. package/dist/run-evidence-ClRX_8A9.d.ts +225 -0
  219. package/dist/run-evidence-ClRX_8A9.d.ts.map +1 -0
  220. package/dist/run-record-BuoE80Dq.js +467 -0
  221. package/dist/{chunk-56TAVBOK.js.map → run-record-BuoE80Dq.js.map} +1 -1
  222. package/dist/run-record-CnZu_gjl.d.ts +357 -0
  223. package/dist/run-record-CnZu_gjl.d.ts.map +1 -0
  224. package/dist/run-score-iEEAWiBY.js +41 -0
  225. package/dist/run-score-iEEAWiBY.js.map +1 -0
  226. package/dist/runtime-trajectory-1gyaTOoC.js +93 -0
  227. package/dist/runtime-trajectory-1gyaTOoC.js.map +1 -0
  228. package/dist/runtime-trajectory-BvSZcCHD.d.ts +50 -0
  229. package/dist/runtime-trajectory-BvSZcCHD.d.ts.map +1 -0
  230. package/dist/schema-BtVldJ3T.d.ts +206 -0
  231. package/dist/schema-BtVldJ3T.d.ts.map +1 -0
  232. package/dist/schema-C6DW4ZHR.js +821 -0
  233. package/dist/schema-C6DW4ZHR.js.map +1 -0
  234. package/dist/schema-CRhEY1SO.js +69 -0
  235. package/dist/schema-CRhEY1SO.js.map +1 -0
  236. package/dist/schema-Cef2cFmb.d.ts +408 -0
  237. package/dist/schema-Cef2cFmb.d.ts.map +1 -0
  238. package/dist/semantic-concept-judge-B6cWNJ2K.js +725 -0
  239. package/dist/semantic-concept-judge-B6cWNJ2K.js.map +1 -0
  240. package/dist/sequential-Br0mAPHA.js +148 -0
  241. package/dist/sequential-Br0mAPHA.js.map +1 -0
  242. package/dist/sequential-CYwq6Ff_.d.ts +141 -0
  243. package/dist/sequential-CYwq6Ff_.d.ts.map +1 -0
  244. package/dist/series-convergence-CjO2QdRW.js +43 -0
  245. package/dist/series-convergence-CjO2QdRW.js.map +1 -0
  246. package/dist/series-convergence-ofsqPWhs.d.ts +35 -0
  247. package/dist/series-convergence-ofsqPWhs.d.ts.map +1 -0
  248. package/dist/server-m5D9cvnG.js +1040 -0
  249. package/dist/server-m5D9cvnG.js.map +1 -0
  250. package/dist/skill-usage-C_pXm7lP.d.ts +534 -0
  251. package/dist/skill-usage-C_pXm7lP.d.ts.map +1 -0
  252. package/dist/skillopt-optimization-method-CUmHkGbI.js +7718 -0
  253. package/dist/skillopt-optimization-method-CUmHkGbI.js.map +1 -0
  254. package/dist/skillopt-optimization-method-CWKVTnks.d.ts +1740 -0
  255. package/dist/skillopt-optimization-method-CWKVTnks.d.ts.map +1 -0
  256. package/dist/statistics-Cmj6nynr.d.ts +514 -0
  257. package/dist/statistics-Cmj6nynr.d.ts.map +1 -0
  258. package/dist/statistics-CnnxdpOg.js +1437 -0
  259. package/dist/statistics-CnnxdpOg.js.map +1 -0
  260. package/dist/store-CT9YIIve.d.ts +117 -0
  261. package/dist/store-CT9YIIve.d.ts.map +1 -0
  262. package/dist/store-CxJry_cs.d.ts +229 -0
  263. package/dist/store-CxJry_cs.d.ts.map +1 -0
  264. package/dist/storyboard/index.d.ts +84 -203
  265. package/dist/storyboard/index.d.ts.map +1 -0
  266. package/dist/storyboard/index.js +609 -542
  267. package/dist/storyboard/index.js.map +1 -1
  268. package/dist/summary-report-BNs5nmXI.js +862 -0
  269. package/dist/summary-report-BNs5nmXI.js.map +1 -0
  270. package/dist/summary-report-Cj9gdw4i.d.ts +407 -0
  271. package/dist/summary-report-Cj9gdw4i.d.ts.map +1 -0
  272. package/dist/supervisor-run/index.d.ts +2 -959
  273. package/dist/supervisor-run/index.js +2 -65
  274. package/dist/supervisor-run-_lnTLM3z.js +1679 -0
  275. package/dist/supervisor-run-_lnTLM3z.js.map +1 -0
  276. package/dist/task-failure-attributes-CQZlB3et.js +311 -0
  277. package/dist/task-failure-attributes-CQZlB3et.js.map +1 -0
  278. package/dist/test-graded-scenario-BsqWLmPt.js +318 -0
  279. package/dist/test-graded-scenario-BsqWLmPt.js.map +1 -0
  280. package/dist/test-graded-scenario-D1TaI2va.d.ts +141 -0
  281. package/dist/test-graded-scenario-D1TaI2va.d.ts.map +1 -0
  282. package/dist/tools-BmuN627J.js +1085 -0
  283. package/dist/tools-BmuN627J.js.map +1 -0
  284. package/dist/trace-attributes.d.ts +2 -52
  285. package/dist/trace-attributes.js +131 -61
  286. package/dist/trace-attributes.js.map +1 -1
  287. package/dist/traces.d.ts +12 -2365
  288. package/dist/traces.js +12 -252
  289. package/dist/trajectory-D_7rLrvE.js +56 -0
  290. package/dist/trajectory-D_7rLrvE.js.map +1 -0
  291. package/dist/types-DGsxbAEd.d.ts +387 -0
  292. package/dist/types-DGsxbAEd.d.ts.map +1 -0
  293. package/dist/types-k9tZGKUg.d.ts +640 -0
  294. package/dist/types-k9tZGKUg.d.ts.map +1 -0
  295. package/dist/verdict-Dps8_okt.d.ts +37 -0
  296. package/dist/verdict-Dps8_okt.d.ts.map +1 -0
  297. package/dist/wire/index.d.ts +702 -1173
  298. package/dist/wire/index.d.ts.map +1 -0
  299. package/dist/wire/index.js +2 -81
  300. package/docs/campaign-proposers.md +1 -0
  301. package/package.json +17 -9
  302. package/dist/benchmarks/index.js.map +0 -1
  303. package/dist/campaign/index.js.map +0 -1
  304. package/dist/chunk-2QU3YOPR.js +0 -7374
  305. package/dist/chunk-2QU3YOPR.js.map +0 -1
  306. package/dist/chunk-3OCR4R5I.js +0 -728
  307. package/dist/chunk-3OCR4R5I.js.map +0 -1
  308. package/dist/chunk-3RF76KTD.js +0 -84
  309. package/dist/chunk-3RF76KTD.js.map +0 -1
  310. package/dist/chunk-56TAVBOK.js +0 -698
  311. package/dist/chunk-5DTSBUL2.js +0 -159
  312. package/dist/chunk-5DTSBUL2.js.map +0 -1
  313. package/dist/chunk-7FO3TNPI.js +0 -232
  314. package/dist/chunk-7FO3TNPI.js.map +0 -1
  315. package/dist/chunk-7ZZMD7UK.js +0 -386
  316. package/dist/chunk-7ZZMD7UK.js.map +0 -1
  317. package/dist/chunk-BOD4O7OF.js +0 -40
  318. package/dist/chunk-BOD4O7OF.js.map +0 -1
  319. package/dist/chunk-BSO5JDQH.js +0 -2335
  320. package/dist/chunk-BSO5JDQH.js.map +0 -1
  321. package/dist/chunk-C6LXANRU.js +0 -1550
  322. package/dist/chunk-C6LXANRU.js.map +0 -1
  323. package/dist/chunk-DODXQREJ.js +0 -752
  324. package/dist/chunk-DODXQREJ.js.map +0 -1
  325. package/dist/chunk-DRYIUNWY.js +0 -622
  326. package/dist/chunk-DRYIUNWY.js.map +0 -1
  327. package/dist/chunk-E7QXT7SX.js +0 -183
  328. package/dist/chunk-E7QXT7SX.js.map +0 -1
  329. package/dist/chunk-EG66UGL4.js +0 -341
  330. package/dist/chunk-EG66UGL4.js.map +0 -1
  331. package/dist/chunk-FXTVJPYD.js +0 -576
  332. package/dist/chunk-FXTVJPYD.js.map +0 -1
  333. package/dist/chunk-G7MGMCZD.js +0 -153
  334. package/dist/chunk-G7MGMCZD.js.map +0 -1
  335. package/dist/chunk-GGE4NNQT.js +0 -65
  336. package/dist/chunk-GGE4NNQT.js.map +0 -1
  337. package/dist/chunk-H23X7XKK.js +0 -181
  338. package/dist/chunk-H23X7XKK.js.map +0 -1
  339. package/dist/chunk-HHWE3POT.js +0 -94
  340. package/dist/chunk-HHWE3POT.js.map +0 -1
  341. package/dist/chunk-HPWUNB47.js +0 -289
  342. package/dist/chunk-HPWUNB47.js.map +0 -1
  343. package/dist/chunk-IYCLP2N2.js +0 -766
  344. package/dist/chunk-IYCLP2N2.js.map +0 -1
  345. package/dist/chunk-JHCHEVET.js +0 -274
  346. package/dist/chunk-JHCHEVET.js.map +0 -1
  347. package/dist/chunk-JQSF5DQT.js +0 -701
  348. package/dist/chunk-JQSF5DQT.js.map +0 -1
  349. package/dist/chunk-K4DBDHLK.js +0 -158
  350. package/dist/chunk-K4DBDHLK.js.map +0 -1
  351. package/dist/chunk-K6N6XJJX.js +0 -306
  352. package/dist/chunk-K6N6XJJX.js.map +0 -1
  353. package/dist/chunk-M4YBQKIJ.js +0 -1040
  354. package/dist/chunk-M4YBQKIJ.js.map +0 -1
  355. package/dist/chunk-MA6HLL3S.js +0 -65
  356. package/dist/chunk-MA6HLL3S.js.map +0 -1
  357. package/dist/chunk-MAZ26DC7.js +0 -99
  358. package/dist/chunk-MAZ26DC7.js.map +0 -1
  359. package/dist/chunk-NPCTHQIO.js +0 -91
  360. package/dist/chunk-NPCTHQIO.js.map +0 -1
  361. package/dist/chunk-NY44NC4A.js +0 -1056
  362. package/dist/chunk-NY44NC4A.js.map +0 -1
  363. package/dist/chunk-OIUOT4QD.js +0 -44
  364. package/dist/chunk-OIUOT4QD.js.map +0 -1
  365. package/dist/chunk-ONWEPEDO.js +0 -57
  366. package/dist/chunk-ONWEPEDO.js.map +0 -1
  367. package/dist/chunk-OWN5NPMC.js +0 -152
  368. package/dist/chunk-OWN5NPMC.js.map +0 -1
  369. package/dist/chunk-P6FYH6K4.js +0 -1161
  370. package/dist/chunk-P6FYH6K4.js.map +0 -1
  371. package/dist/chunk-PC4UYEBM.js +0 -166
  372. package/dist/chunk-PC4UYEBM.js.map +0 -1
  373. package/dist/chunk-PC5DOSM7.js +0 -579
  374. package/dist/chunk-PC5DOSM7.js.map +0 -1
  375. package/dist/chunk-PXE2VKMX.js +0 -140
  376. package/dist/chunk-PXE2VKMX.js.map +0 -1
  377. package/dist/chunk-PZ5AY32C.js +0 -10
  378. package/dist/chunk-PZ5AY32C.js.map +0 -1
  379. package/dist/chunk-QB6BDBP2.js +0 -4464
  380. package/dist/chunk-QB6BDBP2.js.map +0 -1
  381. package/dist/chunk-RXHCETDZ.js +0 -536
  382. package/dist/chunk-RXHCETDZ.js.map +0 -1
  383. package/dist/chunk-RZTMDUO7.js +0 -49
  384. package/dist/chunk-RZTMDUO7.js.map +0 -1
  385. package/dist/chunk-SFLLL76A.js +0 -669
  386. package/dist/chunk-SFLLL76A.js.map +0 -1
  387. package/dist/chunk-SZLVEKMJ.js +0 -1446
  388. package/dist/chunk-SZLVEKMJ.js.map +0 -1
  389. package/dist/chunk-T4SQEITX.js +0 -95
  390. package/dist/chunk-T4SQEITX.js.map +0 -1
  391. package/dist/chunk-T6RLYGAD.js +0 -158
  392. package/dist/chunk-T6RLYGAD.js.map +0 -1
  393. package/dist/chunk-TJVT4QFF.js +0 -911
  394. package/dist/chunk-TJVT4QFF.js.map +0 -1
  395. package/dist/chunk-TQ7LNKZ3.js +0 -136
  396. package/dist/chunk-TQ7LNKZ3.js.map +0 -1
  397. package/dist/chunk-U4L7JRPZ.js +0 -1706
  398. package/dist/chunk-U4L7JRPZ.js.map +0 -1
  399. package/dist/chunk-U4PHLT2N.js +0 -419
  400. package/dist/chunk-U4PHLT2N.js.map +0 -1
  401. package/dist/chunk-VCZ5FQYW.js +0 -928
  402. package/dist/chunk-VCZ5FQYW.js.map +0 -1
  403. package/dist/chunk-VI2UW6B6.js +0 -162
  404. package/dist/chunk-VI2UW6B6.js.map +0 -1
  405. package/dist/chunk-VQMK5FMP.js +0 -247
  406. package/dist/chunk-VQMK5FMP.js.map +0 -1
  407. package/dist/chunk-WGXIEX7P.js +0 -116
  408. package/dist/chunk-WGXIEX7P.js.map +0 -1
  409. package/dist/chunk-WVATSFCP.js +0 -1553
  410. package/dist/chunk-WVATSFCP.js.map +0 -1
  411. package/dist/chunk-X4YIBDER.js +0 -1662
  412. package/dist/chunk-X4YIBDER.js.map +0 -1
  413. package/dist/chunk-YQN4ICPP.js +0 -355
  414. package/dist/chunk-YQN4ICPP.js.map +0 -1
  415. package/dist/chunk-ZET2UAYW.js +0 -89
  416. package/dist/chunk-ZET2UAYW.js.map +0 -1
  417. package/dist/chunk-ZHTZ4EYI.js +0 -1212
  418. package/dist/chunk-ZHTZ4EYI.js.map +0 -1
  419. package/dist/control.js.map +0 -1
  420. package/dist/hosted/index.js.map +0 -1
  421. package/dist/matrix/index.js.map +0 -1
  422. package/dist/reporting.js.map +0 -1
  423. package/dist/rollout/index.js.map +0 -1
  424. package/dist/run-campaign-OJJ7CZF4.js +0 -18
  425. package/dist/run-campaign-OJJ7CZF4.js.map +0 -1
  426. package/dist/supervisor-run/index.js.map +0 -1
  427. package/dist/traces.js.map +0 -1
  428. package/dist/wire/index.js.map +0 -1
@@ -1,589 +1,636 @@
1
- import {
2
- runAgentMatrix
3
- } from "../chunk-JHCHEVET.js";
4
- import "../chunk-PZ5AY32C.js";
5
-
6
- // src/multishot/router.ts
1
+ import { t as runAgentMatrix } from "../matrix-BzQnu2S6.js";
2
+ import { mkdirSync, writeFileSync } from "node:fs";
3
+ import { join } from "node:path";
4
+ //#region src/multishot/router.ts
7
5
  async function routerCompletion(req) {
8
- const body = {
9
- model: req.model,
10
- messages: req.messages,
11
- temperature: req.temperature ?? 0.7,
12
- max_tokens: req.maxTokens ?? 2e3
13
- };
14
- if (req.tools?.length) body.tools = req.tools;
15
- const url = `${req.baseUrl.replace(/\/+$/, "")}/chat/completions`;
16
- const res = await fetch(url, {
17
- method: "POST",
18
- headers: { Authorization: `Bearer ${req.apiKey}`, "Content-Type": "application/json" },
19
- body: JSON.stringify(body),
20
- signal: req.signal
21
- });
22
- if (!res.ok) {
23
- const text = await res.text();
24
- throw new Error(`router ${res.status}: ${text.slice(0, 300)}`);
25
- }
26
- const json = await res.json();
27
- const choice = json.choices[0];
28
- if (!choice) throw new Error(`router returned no choices: ${JSON.stringify(json).slice(0, 200)}`);
29
- return { message: choice.message, usage: json.usage };
6
+ const body = {
7
+ model: req.model,
8
+ messages: req.messages,
9
+ temperature: req.temperature ?? .7,
10
+ max_tokens: req.maxTokens ?? 2e3
11
+ };
12
+ if (req.tools?.length) body.tools = req.tools;
13
+ const url = `${req.baseUrl.replace(/\/+$/, "")}/chat/completions`;
14
+ const res = await fetch(url, {
15
+ method: "POST",
16
+ headers: {
17
+ Authorization: `Bearer ${req.apiKey}`,
18
+ "Content-Type": "application/json"
19
+ },
20
+ body: JSON.stringify(body),
21
+ signal: req.signal
22
+ });
23
+ if (!res.ok) {
24
+ const text = await res.text();
25
+ throw new Error(`router ${res.status}: ${text.slice(0, 300)}`);
26
+ }
27
+ const json = await res.json();
28
+ const choice = json.choices[0];
29
+ if (!choice) throw new Error(`router returned no choices: ${JSON.stringify(json).slice(0, 200)}`);
30
+ return {
31
+ message: choice.message,
32
+ usage: json.usage
33
+ };
30
34
  }
31
35
  function estimateRouterCost(model, usage) {
32
- if (!usage) return 0;
33
- const inputTok = usage.prompt_tokens ?? 0;
34
- const outputTok = usage.completion_tokens ?? 0;
35
- let inPer1k = 3e-3;
36
- let outPer1k = 0.015;
37
- if (model.includes("gpt-4o-mini")) {
38
- inPer1k = 15e-5;
39
- outPer1k = 6e-4;
40
- } else if (model.includes("gpt-5.4") || model.includes("claude-sonnet")) {
41
- inPer1k = 3e-3;
42
- outPer1k = 0.015;
43
- } else if (model.includes("kimi") || model.includes("glm") || model.includes("deepseek")) {
44
- inPer1k = 5e-4;
45
- outPer1k = 2e-3;
46
- }
47
- return (inputTok * inPer1k + outputTok * outPer1k) / 1e3;
36
+ if (!usage) return 0;
37
+ const inputTok = usage.prompt_tokens ?? 0;
38
+ const outputTok = usage.completion_tokens ?? 0;
39
+ let inPer1k = .003;
40
+ let outPer1k = .015;
41
+ if (model.includes("gpt-4o-mini")) {
42
+ inPer1k = 15e-5;
43
+ outPer1k = 6e-4;
44
+ } else if (model.includes("gpt-5.4") || model.includes("claude-sonnet")) {
45
+ inPer1k = .003;
46
+ outPer1k = .015;
47
+ } else if (model.includes("kimi") || model.includes("glm") || model.includes("deepseek")) {
48
+ inPer1k = 5e-4;
49
+ outPer1k = .002;
50
+ }
51
+ return (inputTok * inPer1k + outputTok * outPer1k) / 1e3;
48
52
  }
49
53
  function defaultRouterBaseUrl() {
50
- return (process.env.TANGLE_ROUTER_BASE_URL ?? "https://router.tangle.tools/v1").replace(
51
- /\/+$/,
52
- ""
53
- );
54
+ return (process.env.TANGLE_ROUTER_BASE_URL ?? "https://router.tangle.tools/v1").replace(/\/+$/, "");
54
55
  }
55
56
  function requireRouterApiKey() {
56
- const key = process.env.TANGLE_API_KEY;
57
- if (!key) throw new Error("multishot requires TANGLE_API_KEY (router-scoped sk-tan-* key)");
58
- return key;
57
+ const key = process.env.TANGLE_API_KEY;
58
+ if (!key) throw new Error("multishot requires TANGLE_API_KEY (router-scoped sk-tan-* key)");
59
+ return key;
59
60
  }
60
-
61
- // src/multishot/default-tools.ts
62
- var DEFAULT_RESEARCHER_MODEL = "openai/gpt-4o-mini";
63
- var DEFAULT_CODER_MODEL = "openai/gpt-4o-mini";
64
- var GENERIC_RESEARCHER_SYSTEM = "You are a research specialist. Return a markdown brief with 3-5 findings. Each finding cites a specific source by name. Add a confidence level (high/medium/low) per finding. No fluff, no preamble.";
65
- var GENERIC_CODER_SYSTEM = "You are an expert engineer. Output ONE fenced code block containing the complete solution. Inline-comment non-obvious decisions. No explanation outside the block.";
66
- var DEFAULT_DELEGATE_RESEARCH_TOOL = {
67
- type: "function",
68
- function: {
69
- name: "delegate_research",
70
- description: "Research a topic deeply via specialist. Returns evidence-bearing items with citations. Use for audience research, competitive intel, regulatory landscape, market data, citation-grounded analysis.",
71
- parameters: {
72
- type: "object",
73
- properties: {
74
- question: { type: "string", description: "Specific question to research" },
75
- scope: {
76
- type: "string",
77
- description: "Optional scope: time window, geography, jurisdiction, segment"
78
- }
79
- },
80
- required: ["question"]
81
- }
82
- }
61
+ //#endregion
62
+ //#region src/multishot/default-tools.ts
63
+ const DEFAULT_RESEARCHER_MODEL = "openai/gpt-4o-mini";
64
+ const DEFAULT_CODER_MODEL = "openai/gpt-4o-mini";
65
+ const GENERIC_RESEARCHER_SYSTEM = "You are a research specialist. Return a markdown brief with 3-5 findings. Each finding cites a specific source by name. Add a confidence level (high/medium/low) per finding. No fluff, no preamble.";
66
+ const GENERIC_CODER_SYSTEM = "You are an expert engineer. Output ONE fenced code block containing the complete solution. Inline-comment non-obvious decisions. No explanation outside the block.";
67
+ const DEFAULT_DELEGATE_RESEARCH_TOOL = {
68
+ type: "function",
69
+ function: {
70
+ name: "delegate_research",
71
+ description: "Research a topic deeply via specialist. Returns evidence-bearing items with citations. Use for audience research, competitive intel, regulatory landscape, market data, citation-grounded analysis.",
72
+ parameters: {
73
+ type: "object",
74
+ properties: {
75
+ question: {
76
+ type: "string",
77
+ description: "Specific question to research"
78
+ },
79
+ scope: {
80
+ type: "string",
81
+ description: "Optional scope: time window, geography, jurisdiction, segment"
82
+ }
83
+ },
84
+ required: ["question"]
85
+ }
86
+ }
83
87
  };
84
- var DEFAULT_DELEGATE_CODE_TOOL = {
85
- type: "function",
86
- function: {
87
- name: "delegate_code",
88
- description: "Generate a runnable script, template, pipeline, or tool via specialist. Returns complete working code or structured markdown. Use for content pipelines, calc snippets, dashboards, compliance checklists, deadline trackers.",
89
- parameters: {
90
- type: "object",
91
- properties: {
92
- goal: { type: "string", description: "What the code must accomplish" },
93
- language: {
94
- type: "string",
95
- description: "Optional language preference (default: TypeScript)"
96
- }
97
- },
98
- required: ["goal"]
99
- }
100
- }
88
+ const DEFAULT_DELEGATE_CODE_TOOL = {
89
+ type: "function",
90
+ function: {
91
+ name: "delegate_code",
92
+ description: "Generate a runnable script, template, pipeline, or tool via specialist. Returns complete working code or structured markdown. Use for content pipelines, calc snippets, dashboards, compliance checklists, deadline trackers.",
93
+ parameters: {
94
+ type: "object",
95
+ properties: {
96
+ goal: {
97
+ type: "string",
98
+ description: "What the code must accomplish"
99
+ },
100
+ language: {
101
+ type: "string",
102
+ description: "Optional language preference (default: TypeScript)"
103
+ }
104
+ },
105
+ required: ["goal"]
106
+ }
107
+ }
101
108
  };
102
109
  function createResearchExecutor(config = {}) {
103
- const systemPrompt = config.systemPrompt ?? GENERIC_RESEARCHER_SYSTEM;
104
- const model = config.model ?? DEFAULT_RESEARCHER_MODEL;
105
- return async (args, ctx) => {
106
- const question = String(args.question ?? "");
107
- const scope = args.scope ? String(args.scope) : void 0;
108
- const { message, usage } = await routerCompletion({
109
- apiKey: ctx.apiKey,
110
- baseUrl: ctx.baseUrl,
111
- model,
112
- temperature: 0.3,
113
- maxTokens: 1800,
114
- messages: [
115
- { role: "system", content: systemPrompt },
116
- { role: "user", content: `Research: ${question}${scope ? `
117
- Scope: ${scope}` : ""}` }
118
- ],
119
- signal: ctx.signal
120
- });
121
- return { content: message.content ?? "", costUsd: estimateRouterCost(model, usage) };
122
- };
110
+ const systemPrompt = config.systemPrompt ?? GENERIC_RESEARCHER_SYSTEM;
111
+ const model = config.model ?? "openai/gpt-4o-mini";
112
+ return async (args, ctx) => {
113
+ const question = String(args.question ?? "");
114
+ const scope = args.scope ? String(args.scope) : void 0;
115
+ const { message, usage } = await routerCompletion({
116
+ apiKey: ctx.apiKey,
117
+ baseUrl: ctx.baseUrl,
118
+ model,
119
+ temperature: .3,
120
+ maxTokens: 1800,
121
+ messages: [{
122
+ role: "system",
123
+ content: systemPrompt
124
+ }, {
125
+ role: "user",
126
+ content: `Research: ${question}${scope ? `\nScope: ${scope}` : ""}`
127
+ }],
128
+ signal: ctx.signal
129
+ });
130
+ return {
131
+ content: message.content ?? "",
132
+ costUsd: estimateRouterCost(model, usage)
133
+ };
134
+ };
123
135
  }
124
136
  function createCodeExecutor(config = {}) {
125
- const systemPrompt = config.systemPrompt ?? GENERIC_CODER_SYSTEM;
126
- const model = config.model ?? DEFAULT_CODER_MODEL;
127
- return async (args, ctx) => {
128
- const goal = String(args.goal ?? "");
129
- const language = args.language ? String(args.language) : "TypeScript";
130
- const { message, usage } = await routerCompletion({
131
- apiKey: ctx.apiKey,
132
- baseUrl: ctx.baseUrl,
133
- model,
134
- temperature: 0.2,
135
- maxTokens: 2e3,
136
- messages: [
137
- { role: "system", content: `${systemPrompt}
138
-
139
- Language: ${language}` },
140
- { role: "user", content: `Produce: ${goal}` }
141
- ],
142
- signal: ctx.signal
143
- });
144
- return { content: message.content ?? "", costUsd: estimateRouterCost(model, usage) };
145
- };
137
+ const systemPrompt = config.systemPrompt ?? GENERIC_CODER_SYSTEM;
138
+ const model = config.model ?? "openai/gpt-4o-mini";
139
+ return async (args, ctx) => {
140
+ const goal = String(args.goal ?? "");
141
+ const language = args.language ? String(args.language) : "TypeScript";
142
+ const { message, usage } = await routerCompletion({
143
+ apiKey: ctx.apiKey,
144
+ baseUrl: ctx.baseUrl,
145
+ model,
146
+ temperature: .2,
147
+ maxTokens: 2e3,
148
+ messages: [{
149
+ role: "system",
150
+ content: `${systemPrompt}\n\nLanguage: ${language}`
151
+ }, {
152
+ role: "user",
153
+ content: `Produce: ${goal}`
154
+ }],
155
+ signal: ctx.signal
156
+ });
157
+ return {
158
+ content: message.content ?? "",
159
+ costUsd: estimateRouterCost(model, usage)
160
+ };
161
+ };
146
162
  }
147
163
  function defaultDelegationTools(config = {}) {
148
- return {
149
- tools: [DEFAULT_DELEGATE_RESEARCH_TOOL, DEFAULT_DELEGATE_CODE_TOOL],
150
- executors: {
151
- delegate_research: createResearchExecutor(config.research),
152
- delegate_code: createCodeExecutor(config.code)
153
- },
154
- artifactTypeFor: (name) => name === "delegate_research" ? "research" : name === "delegate_code" ? "code" : void 0
155
- };
164
+ return {
165
+ tools: [DEFAULT_DELEGATE_RESEARCH_TOOL, DEFAULT_DELEGATE_CODE_TOOL],
166
+ executors: {
167
+ delegate_research: createResearchExecutor(config.research),
168
+ delegate_code: createCodeExecutor(config.code)
169
+ },
170
+ artifactTypeFor: (name) => name === "delegate_research" ? "research" : name === "delegate_code" ? "code" : void 0
171
+ };
156
172
  }
157
-
158
- // src/multishot/judges.ts
159
- var DEFAULT_JUDGE_MODEL = "openai/gpt-4o-mini";
173
+ //#endregion
174
+ //#region src/multishot/judges.ts
175
+ const DEFAULT_JUDGE_MODEL = "openai/gpt-4o-mini";
160
176
  async function runJudge(judge, input) {
161
- const apiKey = judge.apiKey ?? requireRouterApiKey();
162
- const baseUrl = judge.baseUrl ?? defaultRouterBaseUrl();
163
- const model = judge.model ?? process.env.JUDGE_MODEL ?? DEFAULT_JUDGE_MODEL;
164
- const prompt = judge.buildPrompt(input);
165
- let raw = "";
166
- try {
167
- const { message } = await routerCompletion({
168
- apiKey,
169
- baseUrl,
170
- model,
171
- temperature: 0,
172
- maxTokens: judge.maxTokens ?? 1500,
173
- messages: [
174
- { role: "system", content: judge.systemPrompt },
175
- { role: "user", content: prompt }
176
- ]
177
- });
178
- raw = (message.content ?? "").trim();
179
- } catch (err) {
180
- return {
181
- dimensions: {},
182
- composite: 0,
183
- failed: true,
184
- notes: `judge ${judge.name} call failed: ${err instanceof Error ? err.message : String(err)}`
185
- };
186
- }
187
- let parsed = null;
188
- try {
189
- const cleaned = raw.replace(/^```json\s*/i, "").replace(/```\s*$/, "").trim();
190
- parsed = JSON.parse(cleaned);
191
- } catch {
192
- return {
193
- dimensions: {},
194
- composite: 0,
195
- failed: true,
196
- notes: `judge ${judge.name} returned non-JSON: ${raw.slice(0, 200)}`
197
- };
198
- }
199
- const dimensions = {};
200
- let sum = 0;
201
- for (const dim of judge.dimensions) {
202
- const v = Number(parsed[dim.key] ?? 0);
203
- const clamped = Number.isFinite(v) ? Math.max(0, Math.min(10, v)) : 0;
204
- dimensions[dim.key] = clamped;
205
- sum += clamped;
206
- }
207
- return {
208
- dimensions,
209
- composite: judge.dimensions.length === 0 ? 0 : sum / judge.dimensions.length,
210
- notes: typeof parsed.notes === "string" ? parsed.notes : ""
211
- };
177
+ const apiKey = judge.apiKey ?? requireRouterApiKey();
178
+ const baseUrl = judge.baseUrl ?? defaultRouterBaseUrl();
179
+ const model = judge.model ?? process.env.JUDGE_MODEL ?? "openai/gpt-4o-mini";
180
+ const prompt = judge.buildPrompt(input);
181
+ let raw = "";
182
+ try {
183
+ const { message } = await routerCompletion({
184
+ apiKey,
185
+ baseUrl,
186
+ model,
187
+ temperature: 0,
188
+ maxTokens: judge.maxTokens ?? 1500,
189
+ messages: [{
190
+ role: "system",
191
+ content: judge.systemPrompt
192
+ }, {
193
+ role: "user",
194
+ content: prompt
195
+ }]
196
+ });
197
+ raw = (message.content ?? "").trim();
198
+ } catch (err) {
199
+ return {
200
+ dimensions: {},
201
+ composite: 0,
202
+ failed: true,
203
+ notes: `judge ${judge.name} call failed: ${err instanceof Error ? err.message : String(err)}`
204
+ };
205
+ }
206
+ let parsed = null;
207
+ try {
208
+ const cleaned = raw.replace(/^```json\s*/i, "").replace(/```\s*$/, "").trim();
209
+ parsed = JSON.parse(cleaned);
210
+ } catch {
211
+ return {
212
+ dimensions: {},
213
+ composite: 0,
214
+ failed: true,
215
+ notes: `judge ${judge.name} returned non-JSON: ${raw.slice(0, 200)}`
216
+ };
217
+ }
218
+ const dimensions = {};
219
+ let sum = 0;
220
+ for (const dim of judge.dimensions) {
221
+ const v = Number(parsed[dim.key] ?? 0);
222
+ const clamped = Number.isFinite(v) ? Math.max(0, Math.min(10, v)) : 0;
223
+ dimensions[dim.key] = clamped;
224
+ sum += clamped;
225
+ }
226
+ return {
227
+ dimensions,
228
+ composite: judge.dimensions.length === 0 ? 0 : sum / judge.dimensions.length,
229
+ notes: typeof parsed.notes === "string" ? parsed.notes : ""
230
+ };
212
231
  }
232
+ /** Convenience: stringified dimension list for inclusion in a judge prompt.
233
+ * Returns lines like `- audience_fit: Does this match what the audience cares about? (0-10)`. */
213
234
  function renderDimensions(dims) {
214
- return dims.map((d) => `- ${d.key}: ${d.description}`).join("\n");
235
+ return dims.map((d) => `- ${d.key}: ${d.description}`).join("\n");
215
236
  }
237
+ /** Convenience: build the "Respond with ONLY this JSON" footer for a judge prompt. */
216
238
  function renderJsonFooter(dims) {
217
- const fields = dims.map((d) => `"${d.key}":N`).join(",");
218
- return `Respond with ONLY this JSON (no markdown, no preamble):
219
- {${fields},"notes":"1-2 sentence critique"}`;
239
+ return `Respond with ONLY this JSON (no markdown, no preamble):\n{${dims.map((d) => `"${d.key}":N`).join(",")},"notes":"1-2 sentence critique"}`;
220
240
  }
221
-
222
- // src/multishot/matrix.ts
223
- import { mkdirSync, writeFileSync } from "fs";
224
- import { join } from "path";
225
-
226
- // src/multishot/types.ts
241
+ //#endregion
242
+ //#region src/multishot/types.ts
227
243
  var MultishotDriverEmptyError = class extends Error {
228
- constructor(turn) {
229
- super(`multishot: driver returned empty content twice at turn ${turn} \u2014 failing loud`);
230
- this.turn = turn;
231
- this.name = "MultishotDriverEmptyError";
232
- }
233
- turn;
244
+ turn;
245
+ constructor(turn) {
246
+ super(`multishot: driver returned empty content twice at turn ${turn} — failing loud`);
247
+ this.turn = turn;
248
+ this.name = "MultishotDriverEmptyError";
249
+ }
234
250
  };
235
251
  var MultishotFatalToolError = class extends Error {
236
- constructor(message) {
237
- super(message);
238
- this.name = "MultishotFatalToolError";
239
- }
252
+ constructor(message) {
253
+ super(message);
254
+ this.name = "MultishotFatalToolError";
255
+ }
240
256
  };
241
-
242
- // src/multishot/multishot.ts
257
+ //#endregion
258
+ //#region src/multishot/multishot.ts
243
259
  async function runMultishot(opts) {
244
- const apiKey = opts.apiKey ?? requireRouterApiKey();
245
- const baseUrl = opts.baseUrl ?? defaultRouterBaseUrl();
246
- const maxTurns = opts.maxTurns ?? 10;
247
- const maxToolDispatches = opts.maxToolDispatches ?? 4;
248
- const agentModel = opts.agentModel ?? "openai/gpt-5.4";
249
- const driverModel = opts.driverModel ?? "openai/gpt-4o-mini";
250
- const driverModels = [driverModel, ...opts.driverFallbackModels ?? []];
251
- const agentMaxTokens = opts.agentMaxTokens ?? 2500;
252
- const toolFollowupMaxTokens = opts.toolFollowupMaxTokens ?? 2e3;
253
- const driverMaxTokens = opts.driverMaxTokens ?? 600;
254
- const bundle = opts.tools && opts.toolExecutors ? {
255
- tools: opts.tools,
256
- executors: opts.toolExecutors,
257
- artifactTypeFor: opts.artifactTypeFor ?? (() => void 0)
258
- } : defaultDelegationTools();
259
- const tools = opts.tools ?? bundle.tools;
260
- const executors = opts.toolExecutors ?? bundle.executors;
261
- const artifactTypeFor = opts.artifactTypeFor ?? bundle.artifactTypeFor;
262
- const routerTransport = (req) => routerCompletion({ apiKey, baseUrl, ...req });
263
- const agentTransport = opts.agentTransport ?? routerTransport;
264
- const driverTransport = opts.driverTransport ?? routerTransport;
265
- const start = Date.now();
266
- const transcript = [];
267
- const artifacts = [];
268
- let toolCalls = 0;
269
- let totalCostUsd = 0;
270
- const opener = opts.shape.buildOpener(opts.persona);
271
- transcript.push({ role: "user", content: opener });
272
- const systemPrompt = opts.profile.prompt?.systemPrompt ?? "";
273
- const agentMessages = [
274
- { role: "system", content: systemPrompt },
275
- { role: "user", content: opener }
276
- ];
277
- for (let turn = 0; turn < maxTurns; turn++) {
278
- if (opts.signal?.aborted) throw new Error("multishot aborted");
279
- let dispatchesThisTurn = 0;
280
- while (true) {
281
- const {
282
- message: agentMsg,
283
- usage: agentUsage,
284
- costUsd: agentCostUsd
285
- } = await agentTransport({
286
- model: agentModel,
287
- messages: agentMessages,
288
- tools,
289
- temperature: 0.7,
290
- maxTokens: dispatchesThisTurn === 0 ? agentMaxTokens : toolFollowupMaxTokens,
291
- signal: opts.signal
292
- });
293
- totalCostUsd += agentCostUsd ?? estimateRouterCost(agentModel, agentUsage);
294
- const agentText = (agentMsg.content ?? "").trim();
295
- const agentToolCalls = (agentMsg.tool_calls ?? []).map((tc) => ({
296
- id: tc.id,
297
- name: tc.function.name,
298
- args: (() => {
299
- try {
300
- return JSON.parse(tc.function.arguments);
301
- } catch {
302
- return {};
303
- }
304
- })()
305
- }));
306
- agentMessages.push({
307
- role: "assistant",
308
- content: agentText || null,
309
- ...agentMsg.tool_calls?.length ? { tool_calls: agentMsg.tool_calls } : {}
310
- });
311
- transcript.push({
312
- role: "assistant",
313
- content: agentText,
314
- toolCalls: agentToolCalls.length > 0 ? agentToolCalls : void 0
315
- });
316
- if (agentToolCalls.length === 0) break;
317
- dispatchesThisTurn += agentToolCalls.length;
318
- if (dispatchesThisTurn > maxToolDispatches) {
319
- throw new Error(
320
- `multishot: tool dispatch cap exceeded (${dispatchesThisTurn}/${maxToolDispatches}) on turn ${turn}`
321
- );
322
- }
323
- for (const tc of agentToolCalls) {
324
- toolCalls++;
325
- let toolResult = "";
326
- try {
327
- const executor = executors[tc.name];
328
- if (!executor) {
329
- toolResult = JSON.stringify({ error: `unknown tool ${tc.name}` });
330
- } else {
331
- const r = await executor(tc.args, { apiKey, baseUrl, signal: opts.signal });
332
- toolResult = r.content;
333
- totalCostUsd += r.costUsd;
334
- const artifactType = artifactTypeFor(tc.name);
335
- if (artifactType) {
336
- artifacts.push({
337
- type: artifactType,
338
- turn,
339
- invocation: { name: tc.name, args: tc.args },
340
- content: toolResult
341
- });
342
- }
343
- }
344
- } catch (err) {
345
- if (err instanceof MultishotFatalToolError) throw err;
346
- toolResult = JSON.stringify({ error: err instanceof Error ? err.message : String(err) });
347
- }
348
- agentMessages.push({ role: "tool", tool_call_id: tc.id, content: toolResult || "done" });
349
- transcript.push({ role: "tool", content: toolResult || "done", toolCallId: tc.id });
350
- }
351
- }
352
- if (turn < maxTurns - 1) {
353
- const driver = await driverTurn({
354
- transport: driverTransport,
355
- persona: opts.persona,
356
- shape: opts.shape,
357
- transcript,
358
- turn,
359
- models: driverModels,
360
- maxTokens: driverMaxTokens,
361
- signal: opts.signal
362
- });
363
- totalCostUsd += driver.costUsd;
364
- agentMessages.push({ role: "user", content: driver.content });
365
- transcript.push({ role: "user", content: driver.content });
366
- }
367
- }
368
- return { transcript, artifacts, toolCalls, durationMs: Date.now() - start, costUsd: totalCostUsd };
260
+ const apiKey = opts.apiKey ?? requireRouterApiKey();
261
+ const baseUrl = opts.baseUrl ?? defaultRouterBaseUrl();
262
+ const maxTurns = opts.maxTurns ?? 10;
263
+ const maxToolDispatches = opts.maxToolDispatches ?? 4;
264
+ const agentModel = opts.agentModel ?? "openai/gpt-5.4";
265
+ const driverModels = [opts.driverModel ?? "openai/gpt-4o-mini", ...opts.driverFallbackModels ?? []];
266
+ const agentMaxTokens = opts.agentMaxTokens ?? 2500;
267
+ const toolFollowupMaxTokens = opts.toolFollowupMaxTokens ?? 2e3;
268
+ const driverMaxTokens = opts.driverMaxTokens ?? 600;
269
+ const bundle = opts.tools && opts.toolExecutors ? {
270
+ tools: opts.tools,
271
+ executors: opts.toolExecutors,
272
+ artifactTypeFor: opts.artifactTypeFor ?? (() => void 0)
273
+ } : defaultDelegationTools();
274
+ const tools = opts.tools ?? bundle.tools;
275
+ const executors = opts.toolExecutors ?? bundle.executors;
276
+ const artifactTypeFor = opts.artifactTypeFor ?? bundle.artifactTypeFor;
277
+ const routerTransport = (req) => routerCompletion({
278
+ apiKey,
279
+ baseUrl,
280
+ ...req
281
+ });
282
+ const agentTransport = opts.agentTransport ?? routerTransport;
283
+ const driverTransport = opts.driverTransport ?? routerTransport;
284
+ const start = Date.now();
285
+ const transcript = [];
286
+ const artifacts = [];
287
+ let toolCalls = 0;
288
+ let totalCostUsd = 0;
289
+ const opener = opts.shape.buildOpener(opts.persona);
290
+ transcript.push({
291
+ role: "user",
292
+ content: opener
293
+ });
294
+ const agentMessages = [{
295
+ role: "system",
296
+ content: opts.profile.prompt?.systemPrompt ?? ""
297
+ }, {
298
+ role: "user",
299
+ content: opener
300
+ }];
301
+ for (let turn = 0; turn < maxTurns; turn++) {
302
+ if (opts.signal?.aborted) throw new Error("multishot aborted");
303
+ let dispatchesThisTurn = 0;
304
+ while (true) {
305
+ const { message: agentMsg, usage: agentUsage, costUsd: agentCostUsd } = await agentTransport({
306
+ model: agentModel,
307
+ messages: agentMessages,
308
+ tools,
309
+ temperature: .7,
310
+ maxTokens: dispatchesThisTurn === 0 ? agentMaxTokens : toolFollowupMaxTokens,
311
+ signal: opts.signal
312
+ });
313
+ totalCostUsd += agentCostUsd ?? estimateRouterCost(agentModel, agentUsage);
314
+ const agentText = (agentMsg.content ?? "").trim();
315
+ const agentToolCalls = (agentMsg.tool_calls ?? []).map((tc) => ({
316
+ id: tc.id,
317
+ name: tc.function.name,
318
+ args: (() => {
319
+ try {
320
+ return JSON.parse(tc.function.arguments);
321
+ } catch {
322
+ return {};
323
+ }
324
+ })()
325
+ }));
326
+ agentMessages.push({
327
+ role: "assistant",
328
+ content: agentText || null,
329
+ ...agentMsg.tool_calls?.length ? { tool_calls: agentMsg.tool_calls } : {}
330
+ });
331
+ transcript.push({
332
+ role: "assistant",
333
+ content: agentText,
334
+ toolCalls: agentToolCalls.length > 0 ? agentToolCalls : void 0
335
+ });
336
+ if (agentToolCalls.length === 0) break;
337
+ dispatchesThisTurn += agentToolCalls.length;
338
+ if (dispatchesThisTurn > maxToolDispatches) throw new Error(`multishot: tool dispatch cap exceeded (${dispatchesThisTurn}/${maxToolDispatches}) on turn ${turn}`);
339
+ for (const tc of agentToolCalls) {
340
+ toolCalls++;
341
+ let toolResult = "";
342
+ try {
343
+ const executor = executors[tc.name];
344
+ if (!executor) toolResult = JSON.stringify({ error: `unknown tool ${tc.name}` });
345
+ else {
346
+ const r = await executor(tc.args, {
347
+ apiKey,
348
+ baseUrl,
349
+ signal: opts.signal
350
+ });
351
+ toolResult = r.content;
352
+ totalCostUsd += r.costUsd;
353
+ const artifactType = artifactTypeFor(tc.name);
354
+ if (artifactType) artifacts.push({
355
+ type: artifactType,
356
+ turn,
357
+ invocation: {
358
+ name: tc.name,
359
+ args: tc.args
360
+ },
361
+ content: toolResult
362
+ });
363
+ }
364
+ } catch (err) {
365
+ if (err instanceof MultishotFatalToolError) throw err;
366
+ toolResult = JSON.stringify({ error: err instanceof Error ? err.message : String(err) });
367
+ }
368
+ agentMessages.push({
369
+ role: "tool",
370
+ tool_call_id: tc.id,
371
+ content: toolResult || "done"
372
+ });
373
+ transcript.push({
374
+ role: "tool",
375
+ content: toolResult || "done",
376
+ toolCallId: tc.id
377
+ });
378
+ }
379
+ }
380
+ if (turn < maxTurns - 1) {
381
+ const driver = await driverTurn({
382
+ transport: driverTransport,
383
+ persona: opts.persona,
384
+ shape: opts.shape,
385
+ transcript,
386
+ turn,
387
+ models: driverModels,
388
+ maxTokens: driverMaxTokens,
389
+ signal: opts.signal
390
+ });
391
+ totalCostUsd += driver.costUsd;
392
+ agentMessages.push({
393
+ role: "user",
394
+ content: driver.content
395
+ });
396
+ transcript.push({
397
+ role: "user",
398
+ content: driver.content
399
+ });
400
+ }
401
+ }
402
+ return {
403
+ transcript,
404
+ artifacts,
405
+ toolCalls,
406
+ durationMs: Date.now() - start,
407
+ costUsd: totalCostUsd
408
+ };
369
409
  }
370
410
  async function driverTurn(opts) {
371
- const driverSystem = opts.shape.buildDriverSystemPrompt(opts.persona);
372
- const driverMessages = [{ role: "system", content: driverSystem }];
373
- for (const msg of opts.transcript) {
374
- if (msg.role === "tool") continue;
375
- const content = driverVisibleContent(msg);
376
- if (!content) continue;
377
- if (msg.role === "assistant") driverMessages.push({ role: "user", content });
378
- else if (msg.role === "user") driverMessages.push({ role: "assistant", content });
379
- }
380
- for (const model of opts.models) {
381
- for (let attempt = 0; attempt < 2; attempt++) {
382
- const { message, usage, costUsd } = await opts.transport({
383
- model,
384
- messages: driverMessages,
385
- temperature: 0.9,
386
- maxTokens: opts.maxTokens,
387
- signal: opts.signal
388
- });
389
- const content = (message.content ?? "").trim();
390
- if (content.length > 0)
391
- return { content, costUsd: costUsd ?? estimateRouterCost(model, usage) };
392
- }
393
- }
394
- throw new MultishotDriverEmptyError(opts.turn);
411
+ const driverMessages = [{
412
+ role: "system",
413
+ content: opts.shape.buildDriverSystemPrompt(opts.persona)
414
+ }];
415
+ for (const msg of opts.transcript) {
416
+ if (msg.role === "tool") continue;
417
+ const content = driverVisibleContent(msg);
418
+ if (!content) continue;
419
+ if (msg.role === "assistant") driverMessages.push({
420
+ role: "user",
421
+ content
422
+ });
423
+ else if (msg.role === "user") driverMessages.push({
424
+ role: "assistant",
425
+ content
426
+ });
427
+ }
428
+ for (const model of opts.models) for (let attempt = 0; attempt < 2; attempt++) {
429
+ const { message, usage, costUsd } = await opts.transport({
430
+ model,
431
+ messages: driverMessages,
432
+ temperature: .9,
433
+ maxTokens: opts.maxTokens,
434
+ signal: opts.signal
435
+ });
436
+ const content = (message.content ?? "").trim();
437
+ if (content.length > 0) return {
438
+ content,
439
+ costUsd: costUsd ?? estimateRouterCost(model, usage)
440
+ };
441
+ }
442
+ throw new MultishotDriverEmptyError(opts.turn);
395
443
  }
396
444
  function driverVisibleContent(msg) {
397
- const text = msg.content.trim();
398
- if (text.length > 0) return text;
399
- if (msg.role !== "assistant" || !msg.toolCalls?.length) return null;
400
- const toolNames = msg.toolCalls.map((call) => call.name.trim()).filter(Boolean);
401
- if (toolNames.length === 0) return "Agent called tools.";
402
- return `Agent called ${toolNames.length === 1 ? "tool" : "tools"}: ${toolNames.join(", ")}.`;
445
+ const text = msg.content.trim();
446
+ if (text.length > 0) return text;
447
+ if (msg.role !== "assistant" || !msg.toolCalls?.length) return null;
448
+ const toolNames = msg.toolCalls.map((call) => call.name.trim()).filter(Boolean);
449
+ if (toolNames.length === 0) return "Agent called tools.";
450
+ return `Agent called ${toolNames.length === 1 ? "tool" : "tools"}: ${toolNames.join(", ")}.`;
403
451
  }
404
-
405
- // src/multishot/matrix.ts
452
+ //#endregion
453
+ //#region src/multishot/matrix.ts
454
+ /** Mean composite over non-failed scores. `0` when the list is empty (a
455
+ * configured judge with nothing to score contributes 0, matching the cell
456
+ * composite's long-standing semantics); `null` when scores exist but every
457
+ * one failed — no signal, so the slot must be EXCLUDED from the cell mean
458
+ * rather than dragging it to zero. */
406
459
  function meanCompositeExcludingFailed(scores) {
407
- if (scores.length === 0) return 0;
408
- const live = scores.filter((s) => !s.failed);
409
- if (live.length === 0) return null;
410
- return live.reduce((sum, s) => sum + s.composite, 0) / live.length;
460
+ if (scores.length === 0) return 0;
461
+ const live = scores.filter((s) => !s.failed);
462
+ if (live.length === 0) return null;
463
+ return live.reduce((sum, s) => sum + s.composite, 0) / live.length;
411
464
  }
465
+ /** Cell composite = mean over configured judge slots, excluding failed
466
+ * scores: a failed conversation judge or an all-failed artifact slot carries
467
+ * no signal and is dropped from the mean. `composite` is 0 only when EVERY
468
+ * configured slot failed (`allJudgesFailed` distinguishes that from a real
469
+ * zero). Pure — exported for deterministic testing. */
412
470
  function computeCellComposite(input) {
413
- const contributions = [];
414
- if (!input.conversation.failed) contributions.push(input.conversation.composite);
415
- const codeMean = input.codeReviews ? meanCompositeExcludingFailed(input.codeReviews) : void 0;
416
- if (typeof codeMean === "number") contributions.push(codeMean);
417
- const contentMean = input.contentReviews ? meanCompositeExcludingFailed(input.contentReviews) : void 0;
418
- if (typeof contentMean === "number") contributions.push(contentMean);
419
- return {
420
- composite: contributions.length === 0 ? 0 : contributions.reduce((s, v) => s + v, 0) / contributions.length,
421
- codeComposite: codeMean ?? 0,
422
- contentComposite: contentMean ?? 0,
423
- allJudgesFailed: contributions.length === 0
424
- };
471
+ const contributions = [];
472
+ if (!input.conversation.failed) contributions.push(input.conversation.composite);
473
+ const codeMean = input.codeReviews ? meanCompositeExcludingFailed(input.codeReviews) : void 0;
474
+ if (typeof codeMean === "number") contributions.push(codeMean);
475
+ const contentMean = input.contentReviews ? meanCompositeExcludingFailed(input.contentReviews) : void 0;
476
+ if (typeof contentMean === "number") contributions.push(contentMean);
477
+ return {
478
+ composite: contributions.length === 0 ? 0 : contributions.reduce((s, v) => s + v, 0) / contributions.length,
479
+ codeComposite: codeMean ?? 0,
480
+ contentComposite: contentMean ?? 0,
481
+ allJudgesFailed: contributions.length === 0
482
+ };
425
483
  }
426
484
  async function runMultishotMatrix(opts) {
427
- const codeTypes = new Set(opts.judges.codeArtifactTypes ?? ["code"]);
428
- const contentTypes = new Set(opts.judges.contentArtifactTypes ?? ["research"]);
429
- mkdirSync(opts.runDir, { recursive: true });
430
- const matrix = await runAgentMatrix({
431
- axes: [
432
- { name: "profile", values: opts.profiles },
433
- { name: "persona", values: opts.personas.map((p) => ({ id: p.id, value: p })) }
434
- ],
435
- reps: opts.reps ?? 1,
436
- maxConcurrency: opts.maxConcurrency ?? 2,
437
- costCeiling: opts.costCeiling,
438
- async runCell(cell) {
439
- const profile = cell.axes.profile?.value;
440
- const persona = cell.axes.persona?.value;
441
- const profileId = String(cell.axes.profile?.id ?? "unknown");
442
- const personaId = String(cell.axes.persona?.id ?? "unknown");
443
- const sim = await runMultishot({
444
- profile,
445
- persona,
446
- shape: opts.shape,
447
- tools: opts.tools,
448
- toolExecutors: opts.toolExecutors,
449
- artifactTypeFor: opts.artifactTypeFor,
450
- maxTurns: opts.maxTurns,
451
- maxToolDispatches: opts.maxToolDispatches,
452
- agentModel: opts.agentModel,
453
- driverModel: opts.driverModel,
454
- driverFallbackModels: opts.driverFallbackModels,
455
- agentMaxTokens: opts.agentMaxTokens,
456
- toolFollowupMaxTokens: opts.toolFollowupMaxTokens,
457
- driverMaxTokens: opts.driverMaxTokens,
458
- agentTransport: opts.agentTransport,
459
- driverTransport: opts.driverTransport,
460
- apiKey: opts.apiKey,
461
- baseUrl: opts.baseUrl
462
- });
463
- const codeArtifacts = sim.artifacts.filter((a) => codeTypes.has(a.type));
464
- const contentArtifacts = sim.artifacts.filter((a) => contentTypes.has(a.type));
465
- const [conversation, codeReviews, contentReviews] = await Promise.all([
466
- runJudge(withJudgeMaxTokens(opts.judges.conversation, opts.judgeMaxTokens), {
467
- transcript: sim.transcript,
468
- persona
469
- }),
470
- opts.judges.codeReview ? Promise.all(
471
- codeArtifacts.map(
472
- (artifact) => runJudge(withJudgeMaxTokens(opts.judges.codeReview, opts.judgeMaxTokens), {
473
- artifact,
474
- persona
475
- }).then((s) => ({
476
- ...s,
477
- turn: artifact.turn,
478
- type: artifact.type
479
- }))
480
- )
481
- ) : Promise.resolve([]),
482
- opts.judges.contentQuality ? Promise.all(
483
- contentArtifacts.map(
484
- (artifact) => runJudge(withJudgeMaxTokens(opts.judges.contentQuality, opts.judgeMaxTokens), {
485
- artifact,
486
- persona
487
- }).then((s) => ({
488
- ...s,
489
- turn: artifact.turn,
490
- type: artifact.type
491
- }))
492
- )
493
- ) : Promise.resolve([])
494
- ]);
495
- const { composite, codeComposite, contentComposite, allJudgesFailed } = computeCellComposite({
496
- conversation,
497
- codeReviews: opts.judges.codeReview ? codeReviews : void 0,
498
- contentReviews: opts.judges.contentQuality ? contentReviews : void 0
499
- });
500
- const cellScore = { composite, conversation };
501
- if (opts.judges.codeReview)
502
- cellScore.codeReview = { perArtifact: codeReviews, composite: codeComposite };
503
- if (opts.judges.contentQuality)
504
- cellScore.contentQuality = { perArtifact: contentReviews, composite: contentComposite };
505
- const cellDir = join(opts.runDir, profileId, personaId, `rep-${cell.rep}`);
506
- mkdirSync(cellDir, { recursive: true });
507
- writeFileSync(join(cellDir, "transcript.json"), JSON.stringify(sim.transcript, null, 2));
508
- writeFileSync(join(cellDir, "artifacts.json"), JSON.stringify(sim.artifacts, null, 2));
509
- writeFileSync(join(cellDir, "scores.json"), JSON.stringify(cellScore, null, 2));
510
- const notes = [`convo=${conversation.composite.toFixed(1)}`];
511
- if (opts.judges.codeReview) notes.push(`code=${codeComposite.toFixed(1)}`);
512
- if (opts.judges.contentQuality) notes.push(`content=${contentComposite.toFixed(1)}`);
513
- if (allJudgesFailed) notes.push("all-judges-failed");
514
- return {
515
- output: {
516
- turns: sim.transcript.length,
517
- toolCalls: sim.toolCalls,
518
- artifactCount: sim.artifacts.length
519
- },
520
- verdict: { valid: composite >= 5, score: composite, notes: notes.join(" ") },
521
- costUsd: sim.costUsd,
522
- durationMs: sim.durationMs
523
- };
524
- }
525
- });
526
- const summary = {
527
- cells: matrix.summary.totalCells,
528
- passRate: matrix.summary.overallPassRate,
529
- meanScore: matrix.summary.overallMeanScore,
530
- totalCostUsd: matrix.summary.totalCostUsd,
531
- durationMs: matrix.summary.durationMs,
532
- runsExecuted: matrix.summary.runsExecuted,
533
- cellsSkipped: matrix.summary.cellsSkipped,
534
- byProfile: matrix.byAxis.profile,
535
- byPersona: matrix.byAxis.persona
536
- };
537
- writeFileSync(join(opts.runDir, "summary.json"), JSON.stringify(summary, null, 2));
538
- const md = [
539
- `# Multishot matrix`,
540
- ``,
541
- `**Cells**: ${matrix.summary.totalCells} | **Pass rate**: ${(matrix.summary.overallPassRate * 100).toFixed(0)}% | **Mean**: ${matrix.summary.overallMeanScore.toFixed(2)} | **Cost**: $${matrix.summary.totalCostUsd.toFixed(2)} | **Duration**: ${(matrix.summary.durationMs / 1e3).toFixed(0)}s`,
542
- ``,
543
- `## By profile`,
544
- ``,
545
- "| profile | pass | mean | cost |",
546
- "|---|---|---|---|",
547
- ...Object.entries(matrix.byAxis.profile ?? {}).map(
548
- ([id, s]) => `| ${id} | ${(s.passRate * 100).toFixed(0)}% | ${s.meanScore.toFixed(2)} | $${s.totalCostUsd.toFixed(2)} |`
549
- ),
550
- ``,
551
- `## By persona`,
552
- ``,
553
- "| persona | pass | mean | cost |",
554
- "|---|---|---|---|",
555
- ...Object.entries(matrix.byAxis.persona ?? {}).map(
556
- ([id, s]) => `| ${id} | ${(s.passRate * 100).toFixed(0)}% | ${s.meanScore.toFixed(2)} | $${s.totalCostUsd.toFixed(2)} |`
557
- ),
558
- ``
559
- ];
560
- writeFileSync(join(opts.runDir, "summary.md"), md.join("\n"));
561
- return { matrix };
485
+ const codeTypes = new Set(opts.judges.codeArtifactTypes ?? ["code"]);
486
+ const contentTypes = new Set(opts.judges.contentArtifactTypes ?? ["research"]);
487
+ mkdirSync(opts.runDir, { recursive: true });
488
+ const matrix = await runAgentMatrix({
489
+ axes: [{
490
+ name: "profile",
491
+ values: opts.profiles
492
+ }, {
493
+ name: "persona",
494
+ values: opts.personas.map((p) => ({
495
+ id: p.id,
496
+ value: p
497
+ }))
498
+ }],
499
+ reps: opts.reps ?? 1,
500
+ maxConcurrency: opts.maxConcurrency ?? 2,
501
+ costCeiling: opts.costCeiling,
502
+ async runCell(cell) {
503
+ const profile = cell.axes.profile?.value;
504
+ const persona = cell.axes.persona?.value;
505
+ const profileId = String(cell.axes.profile?.id ?? "unknown");
506
+ const personaId = String(cell.axes.persona?.id ?? "unknown");
507
+ const sim = await runMultishot({
508
+ profile,
509
+ persona,
510
+ shape: opts.shape,
511
+ tools: opts.tools,
512
+ toolExecutors: opts.toolExecutors,
513
+ artifactTypeFor: opts.artifactTypeFor,
514
+ maxTurns: opts.maxTurns,
515
+ maxToolDispatches: opts.maxToolDispatches,
516
+ agentModel: opts.agentModel,
517
+ driverModel: opts.driverModel,
518
+ driverFallbackModels: opts.driverFallbackModels,
519
+ agentMaxTokens: opts.agentMaxTokens,
520
+ toolFollowupMaxTokens: opts.toolFollowupMaxTokens,
521
+ driverMaxTokens: opts.driverMaxTokens,
522
+ agentTransport: opts.agentTransport,
523
+ driverTransport: opts.driverTransport,
524
+ apiKey: opts.apiKey,
525
+ baseUrl: opts.baseUrl
526
+ });
527
+ const codeArtifacts = sim.artifacts.filter((a) => codeTypes.has(a.type));
528
+ const contentArtifacts = sim.artifacts.filter((a) => contentTypes.has(a.type));
529
+ const [conversation, codeReviews, contentReviews] = await Promise.all([
530
+ runJudge(withJudgeMaxTokens(opts.judges.conversation, opts.judgeMaxTokens), {
531
+ transcript: sim.transcript,
532
+ persona
533
+ }),
534
+ opts.judges.codeReview ? Promise.all(codeArtifacts.map((artifact) => runJudge(withJudgeMaxTokens(opts.judges.codeReview, opts.judgeMaxTokens), {
535
+ artifact,
536
+ persona
537
+ }).then((s) => ({
538
+ ...s,
539
+ turn: artifact.turn,
540
+ type: artifact.type
541
+ })))) : Promise.resolve([]),
542
+ opts.judges.contentQuality ? Promise.all(contentArtifacts.map((artifact) => runJudge(withJudgeMaxTokens(opts.judges.contentQuality, opts.judgeMaxTokens), {
543
+ artifact,
544
+ persona
545
+ }).then((s) => ({
546
+ ...s,
547
+ turn: artifact.turn,
548
+ type: artifact.type
549
+ })))) : Promise.resolve([])
550
+ ]);
551
+ const { composite, codeComposite, contentComposite, allJudgesFailed } = computeCellComposite({
552
+ conversation,
553
+ codeReviews: opts.judges.codeReview ? codeReviews : void 0,
554
+ contentReviews: opts.judges.contentQuality ? contentReviews : void 0
555
+ });
556
+ const cellScore = {
557
+ composite,
558
+ conversation
559
+ };
560
+ if (opts.judges.codeReview) cellScore.codeReview = {
561
+ perArtifact: codeReviews,
562
+ composite: codeComposite
563
+ };
564
+ if (opts.judges.contentQuality) cellScore.contentQuality = {
565
+ perArtifact: contentReviews,
566
+ composite: contentComposite
567
+ };
568
+ const cellDir = join(opts.runDir, profileId, personaId, `rep-${cell.rep}`);
569
+ mkdirSync(cellDir, { recursive: true });
570
+ writeFileSync(join(cellDir, "transcript.json"), JSON.stringify(sim.transcript, null, 2));
571
+ writeFileSync(join(cellDir, "artifacts.json"), JSON.stringify(sim.artifacts, null, 2));
572
+ writeFileSync(join(cellDir, "scores.json"), JSON.stringify(cellScore, null, 2));
573
+ const notes = [`convo=${conversation.composite.toFixed(1)}`];
574
+ if (opts.judges.codeReview) notes.push(`code=${codeComposite.toFixed(1)}`);
575
+ if (opts.judges.contentQuality) notes.push(`content=${contentComposite.toFixed(1)}`);
576
+ if (allJudgesFailed) notes.push("all-judges-failed");
577
+ return {
578
+ output: {
579
+ turns: sim.transcript.length,
580
+ toolCalls: sim.toolCalls,
581
+ artifactCount: sim.artifacts.length
582
+ },
583
+ verdict: {
584
+ valid: composite >= 5,
585
+ score: composite,
586
+ notes: notes.join(" ")
587
+ },
588
+ costUsd: sim.costUsd,
589
+ durationMs: sim.durationMs
590
+ };
591
+ }
592
+ });
593
+ const summary = {
594
+ cells: matrix.summary.totalCells,
595
+ passRate: matrix.summary.overallPassRate,
596
+ meanScore: matrix.summary.overallMeanScore,
597
+ totalCostUsd: matrix.summary.totalCostUsd,
598
+ durationMs: matrix.summary.durationMs,
599
+ runsExecuted: matrix.summary.runsExecuted,
600
+ cellsSkipped: matrix.summary.cellsSkipped,
601
+ byProfile: matrix.byAxis.profile,
602
+ byPersona: matrix.byAxis.persona
603
+ };
604
+ writeFileSync(join(opts.runDir, "summary.json"), JSON.stringify(summary, null, 2));
605
+ const md = [
606
+ `# Multishot matrix`,
607
+ ``,
608
+ `**Cells**: ${matrix.summary.totalCells} | **Pass rate**: ${(matrix.summary.overallPassRate * 100).toFixed(0)}% | **Mean**: ${matrix.summary.overallMeanScore.toFixed(2)} | **Cost**: $${matrix.summary.totalCostUsd.toFixed(2)} | **Duration**: ${(matrix.summary.durationMs / 1e3).toFixed(0)}s`,
609
+ ``,
610
+ `## By profile`,
611
+ ``,
612
+ "| profile | pass | mean | cost |",
613
+ "|---|---|---|---|",
614
+ ...Object.entries(matrix.byAxis.profile ?? {}).map(([id, s]) => `| ${id} | ${(s.passRate * 100).toFixed(0)}% | ${s.meanScore.toFixed(2)} | $${s.totalCostUsd.toFixed(2)} |`),
615
+ ``,
616
+ `## By persona`,
617
+ ``,
618
+ "| persona | pass | mean | cost |",
619
+ "|---|---|---|---|",
620
+ ...Object.entries(matrix.byAxis.persona ?? {}).map(([id, s]) => `| ${id} | ${(s.passRate * 100).toFixed(0)}% | ${s.meanScore.toFixed(2)} | $${s.totalCostUsd.toFixed(2)} |`),
621
+ ``
622
+ ];
623
+ writeFileSync(join(opts.runDir, "summary.md"), md.join("\n"));
624
+ return { matrix };
562
625
  }
563
626
  function withJudgeMaxTokens(judge, maxTokens) {
564
- if (maxTokens === void 0 || judge.maxTokens !== void 0) return judge;
565
- return { ...judge, maxTokens };
627
+ if (maxTokens === void 0 || judge.maxTokens !== void 0) return judge;
628
+ return {
629
+ ...judge,
630
+ maxTokens
631
+ };
566
632
  }
567
- export {
568
- DEFAULT_CODER_MODEL,
569
- DEFAULT_DELEGATE_CODE_TOOL,
570
- DEFAULT_DELEGATE_RESEARCH_TOOL,
571
- DEFAULT_JUDGE_MODEL,
572
- DEFAULT_RESEARCHER_MODEL,
573
- MultishotDriverEmptyError,
574
- MultishotFatalToolError,
575
- computeCellComposite,
576
- createCodeExecutor,
577
- createResearchExecutor,
578
- defaultDelegationTools,
579
- defaultRouterBaseUrl,
580
- estimateRouterCost,
581
- renderDimensions,
582
- renderJsonFooter,
583
- requireRouterApiKey,
584
- routerCompletion,
585
- runJudge,
586
- runMultishot,
587
- runMultishotMatrix
588
- };
633
+ //#endregion
634
+ export { DEFAULT_CODER_MODEL, DEFAULT_DELEGATE_CODE_TOOL, DEFAULT_DELEGATE_RESEARCH_TOOL, DEFAULT_JUDGE_MODEL, DEFAULT_RESEARCHER_MODEL, MultishotDriverEmptyError, MultishotFatalToolError, computeCellComposite, createCodeExecutor, createResearchExecutor, defaultDelegationTools, defaultRouterBaseUrl, estimateRouterCost, renderDimensions, renderJsonFooter, requireRouterApiKey, routerCompletion, runJudge, runMultishot, runMultishotMatrix };
635
+
589
636
  //# sourceMappingURL=index.js.map