@tangle-network/agent-eval 0.144.11 → 0.144.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (378) hide show
  1. package/CHANGELOG.md +11 -0
  2. package/dist/agent-profile-B9_GGsG8.d.ts +84 -0
  3. package/dist/agent-profile-B9_GGsG8.d.ts.map +1 -0
  4. package/dist/{agent-profile-cell-BOP-iA9Q.d.ts → agent-profile-cell-BkcRDikH.d.ts} +2 -2
  5. package/dist/{agent-profile-cell-BOP-iA9Q.d.ts.map → agent-profile-cell-BkcRDikH.d.ts.map} +1 -1
  6. package/dist/analyst/index.d.ts +134 -16
  7. package/dist/analyst/index.d.ts.map +1 -1
  8. package/dist/analyst/index.js +364 -10
  9. package/dist/analyst/index.js.map +1 -1
  10. package/dist/backend-integrity-CsVin_Wb.d.ts +280 -0
  11. package/dist/backend-integrity-CsVin_Wb.d.ts.map +1 -0
  12. package/dist/{baseline-C-GocmIW.js → baseline-BhPRQBVn.js} +3 -2
  13. package/dist/{baseline-C-GocmIW.js.map → baseline-BhPRQBVn.js.map} +1 -1
  14. package/dist/{benchmark-CWeqGl7x.js → benchmark-BhT16ep9.js} +2 -2
  15. package/dist/{benchmark-CWeqGl7x.js.map → benchmark-BhT16ep9.js.map} +1 -1
  16. package/dist/{agent-profile-DPi7IZg7.d.ts → benchmark-Ceoan7vk.d.ts} +4 -91
  17. package/dist/benchmark-Ceoan7vk.d.ts.map +1 -0
  18. package/dist/{benchmark-command-BteMFN62.js → benchmark-command-CY6cbhOB.js} +36 -79
  19. package/dist/benchmark-command-CY6cbhOB.js.map +1 -0
  20. package/dist/benchmarks/index.d.ts +244 -2
  21. package/dist/benchmarks/index.d.ts.map +1 -0
  22. package/dist/benchmarks/index.js +733 -1
  23. package/dist/benchmarks/index.js.map +1 -0
  24. package/dist/builder-eval/index.d.ts +23 -2
  25. package/dist/builder-eval/index.d.ts.map +1 -1
  26. package/dist/builder-eval/index.js +227 -3
  27. package/dist/builder-eval/index.js.map +1 -1
  28. package/dist/campaign/index.d.ts +10 -8
  29. package/dist/campaign/index.js +9 -6
  30. package/dist/{campaign-C2TTzQII.js → campaign-BYjBAypg.js} +21 -634
  31. package/dist/campaign-BYjBAypg.js.map +1 -0
  32. package/dist/{canonical-D011XM8r.js → canonical-D-XsTQ6_.js} +2 -2
  33. package/dist/{canonical-D011XM8r.js.map → canonical-D-XsTQ6_.js.map} +1 -1
  34. package/dist/{index-CQTZ-4XN.d.ts → capture-fetch-DDvpjVRU.d.ts} +3 -3
  35. package/dist/capture-fetch-DDvpjVRU.d.ts.map +1 -0
  36. package/dist/{default-registry-BmktKy8r.js → chat-client-Bp4Ebuuc.js} +1276 -1276
  37. package/dist/chat-client-Bp4Ebuuc.js.map +1 -0
  38. package/dist/cli.js +2 -2
  39. package/dist/{client-C9gzZE59.d.ts → client-KUbGulm_.d.ts} +4 -4
  40. package/dist/{client-C9gzZE59.d.ts.map → client-KUbGulm_.d.ts.map} +1 -1
  41. package/dist/contract/index.d.ts +13 -390
  42. package/dist/contract/index.d.ts.map +1 -1
  43. package/dist/contract/index.js +18 -542
  44. package/dist/contract/index.js.map +1 -1
  45. package/dist/{cost-ledger-DMFxsLKr.js → cost-ledger-BSe92yAV.js} +3 -3
  46. package/dist/{cost-ledger-DMFxsLKr.js.map → cost-ledger-BSe92yAV.js.map} +1 -1
  47. package/dist/{cost-ledger-Bv_e8XHY.d.ts → cost-ledger-DbQdN3nO.d.ts} +2 -2
  48. package/dist/{cost-ledger-Bv_e8XHY.d.ts.map → cost-ledger-DbQdN3nO.d.ts.map} +1 -1
  49. package/dist/{counterfactual-CxmxAONP.d.ts → counterfactual--bpysZF0.d.ts} +2 -14
  50. package/dist/{counterfactual-CxmxAONP.d.ts.map → counterfactual--bpysZF0.d.ts.map} +1 -1
  51. package/dist/{counterfactual-CWPTrMH7.js → counterfactual-lDfCx0Uz.js} +3 -31
  52. package/dist/{counterfactual-CWPTrMH7.js.map → counterfactual-lDfCx0Uz.js.map} +1 -1
  53. package/dist/{dataset-C8xaLXdY.d.ts → dataset-CJjKqQfA.d.ts} +2 -8
  54. package/dist/dataset-CJjKqQfA.d.ts.map +1 -0
  55. package/dist/{default-registry-Bf8Woqmq.d.ts → default-registry-Di6HP6pG.d.ts} +7 -12
  56. package/dist/default-registry-Di6HP6pG.d.ts.map +1 -0
  57. package/dist/{analyze-runs-C30yljDJ.js → define-agent-eval-iqjT--ZZ.js} +542 -130
  58. package/dist/define-agent-eval-iqjT--ZZ.js.map +1 -0
  59. package/dist/define-agent-eval-sH24zBfM.d.ts +388 -0
  60. package/dist/define-agent-eval-sH24zBfM.d.ts.map +1 -0
  61. package/dist/descriptive-B5MwKfbf.js +144 -0
  62. package/dist/descriptive-B5MwKfbf.js.map +1 -0
  63. package/dist/{dspy-rlm-engine-DbTk4JdR.js → dspy-rlm-engine-B_qhSc21.js} +4 -4
  64. package/dist/{dspy-rlm-engine-DbTk4JdR.js.map → dspy-rlm-engine-B_qhSc21.js.map} +1 -1
  65. package/dist/effect-sizes-DiH8MGOH.js +82 -0
  66. package/dist/effect-sizes-DiH8MGOH.js.map +1 -0
  67. package/dist/{engine-3hL-XqwJ.d.ts → engine-otFpE2gF.d.ts} +10 -38
  68. package/dist/engine-otFpE2gF.d.ts.map +1 -0
  69. package/dist/{errors-CKPfb2aH.d.ts → errors-DEE6u6ot.d.ts} +2 -14
  70. package/dist/{errors-CKPfb2aH.d.ts.map → errors-DEE6u6ot.d.ts.map} +1 -1
  71. package/dist/{errors-D-LKuDhb.js → errors-Dngq5h35.js} +2 -8
  72. package/dist/{errors-D-LKuDhb.js.map → errors-Dngq5h35.js.map} +1 -1
  73. package/dist/{eval-campaign-B_7wcnav.js → eval-campaign-UB-usSQ2.js} +6 -6
  74. package/dist/{eval-campaign-B_7wcnav.js.map → eval-campaign-UB-usSQ2.js.map} +1 -1
  75. package/dist/{exact-types-DSFFpLLI.d.ts → exact-types-BH1twmAJ.d.ts} +2 -2
  76. package/dist/{exact-types-DSFFpLLI.d.ts.map → exact-types-BH1twmAJ.d.ts.map} +1 -1
  77. package/dist/experiment/index.d.ts +9 -6
  78. package/dist/experiment/index.d.ts.map +1 -1
  79. package/dist/experiment/index.js +11 -7
  80. package/dist/experiment/index.js.map +1 -1
  81. package/dist/experiment-tracker-C29gXM4B.js +269 -0
  82. package/dist/experiment-tracker-C29gXM4B.js.map +1 -0
  83. package/dist/{experiment-tracker-IMntXr6J.d.ts → experiment-tracker-DWHZBAYL.d.ts} +77 -121
  84. package/dist/experiment-tracker-DWHZBAYL.d.ts.map +1 -0
  85. package/dist/{external-optimizer-contracts-CZuJNcT5.d.ts → external-optimizer-contracts-lixrOZdX.d.ts} +3 -3
  86. package/dist/{external-optimizer-contracts-CZuJNcT5.d.ts.map → external-optimizer-contracts-lixrOZdX.d.ts.map} +1 -1
  87. package/dist/external-optimizer-process-BTiNB-RH.js +301 -0
  88. package/dist/external-optimizer-process-BTiNB-RH.js.map +1 -0
  89. package/dist/{single-run-lock-DFWHEB09.js → external-optimizer-subprocess-DrJ9hR8u.js} +144 -438
  90. package/dist/external-optimizer-subprocess-DrJ9hR8u.js.map +1 -0
  91. package/dist/extract-usage-BrQ8mCLX.js +155 -0
  92. package/dist/extract-usage-BrQ8mCLX.js.map +1 -0
  93. package/dist/{failure-cluster-CqcvCcdR.d.ts → failure-cluster-BLURuWG4.d.ts} +2 -3
  94. package/dist/failure-cluster-BLURuWG4.d.ts.map +1 -0
  95. package/dist/{feedback-trajectory-CacHpxsp.d.ts → feedback-trajectory-juOozjAc.d.ts} +5 -36
  96. package/dist/feedback-trajectory-juOozjAc.d.ts.map +1 -0
  97. package/dist/fuzz.d.ts +2 -2
  98. package/dist/fuzz.js +2 -2
  99. package/dist/hosted/index.d.ts +3 -3
  100. package/dist/{index-CvSN3IG1.d.ts → index-B8Ui1mr1.d.ts} +2 -2
  101. package/dist/{index-CvSN3IG1.d.ts.map → index-B8Ui1mr1.d.ts.map} +1 -1
  102. package/dist/{skill-usage-CO9OLRBx.d.ts → index-BWDrSVfw.d.ts} +11 -136
  103. package/dist/index-BWDrSVfw.d.ts.map +1 -0
  104. package/dist/index-Ba3YrbAL.d.ts +1 -0
  105. package/dist/{index-BnEuDAK2.d.ts → index-COQYtuRF.d.ts} +3 -3
  106. package/dist/{index-BnEuDAK2.d.ts.map → index-COQYtuRF.d.ts.map} +1 -1
  107. package/dist/{index-C3ssXVLv.d.ts → index-CvjYbU0D.d.ts} +2 -2
  108. package/dist/{index-C3ssXVLv.d.ts.map → index-CvjYbU0D.d.ts.map} +1 -1
  109. package/dist/{index-DPPGNJ_R.d.ts → index-DSmEylT9.d.ts} +17 -115
  110. package/dist/index-DSmEylT9.d.ts.map +1 -0
  111. package/dist/index.d.ts +2397 -5308
  112. package/dist/index.d.ts.map +1 -1
  113. package/dist/index.js +5914 -10496
  114. package/dist/index.js.map +1 -1
  115. package/dist/{insight-report-CgX_s0Ez.d.ts → insight-report-CRi-Ufrj.d.ts} +4 -4
  116. package/dist/{insight-report-CgX_s0Ez.d.ts.map → insight-report-CRi-Ufrj.d.ts.map} +1 -1
  117. package/dist/{integrity-MLzHOfV9.js → integrity-Cy9WHAtb.js} +2 -2
  118. package/dist/{integrity-MLzHOfV9.js.map → integrity-Cy9WHAtb.js.map} +1 -1
  119. package/dist/{integrity-DY6tIbl0.js → integrity-DysDBWDu.js} +2 -2
  120. package/dist/{integrity-DY6tIbl0.js.map → integrity-DysDBWDu.js.map} +1 -1
  121. package/dist/{integrity-BuqEKu-x.d.ts → integrity-OrcI9Nau.d.ts} +3 -3
  122. package/dist/{integrity-BuqEKu-x.d.ts.map → integrity-OrcI9Nau.d.ts.map} +1 -1
  123. package/dist/internal-BDHPCnjk.js +230 -0
  124. package/dist/internal-BDHPCnjk.js.map +1 -0
  125. package/dist/judge-calibration-C5CbMYce.d.ts +117 -0
  126. package/dist/judge-calibration-C5CbMYce.d.ts.map +1 -0
  127. package/dist/judge-calibration-DZkWrm5H.js +317 -0
  128. package/dist/judge-calibration-DZkWrm5H.js.map +1 -0
  129. package/dist/{kind-factory-B8-r8-y8.js → kind-factory-CPmSd58s.js} +208 -208
  130. package/dist/{kind-factory-B8-r8-y8.js.map → kind-factory-CPmSd58s.js.map} +1 -1
  131. package/dist/ledger-core/index.d.ts +1 -1
  132. package/dist/ledger-core/index.js +2 -2
  133. package/dist/{ledger-core-DXZIqu17.js → ledger-core-BmZt19oQ.js} +110 -110
  134. package/dist/{ledger-core-DXZIqu17.js.map → ledger-core-BmZt19oQ.js.map} +1 -1
  135. package/dist/{llm-client-DzvMUsS_.js → llm-client-d0-2TT1g.js} +4 -64
  136. package/dist/{llm-client-DzvMUsS_.js.map → llm-client-d0-2TT1g.js.map} +1 -1
  137. package/dist/{skillopt-optimization-method-CQdVeM8k.js → llm-judge-dZ8P6nGI.js} +3349 -5401
  138. package/dist/llm-judge-dZ8P6nGI.js.map +1 -0
  139. package/dist/matrix/index.d.ts +2 -2
  140. package/dist/meta-eval/index.d.ts +3 -3
  141. package/dist/meta-eval/index.js +3 -3
  142. package/dist/{metrics-C9YY1OcL.js → metrics-Cl0L1KUy.js} +2 -108
  143. package/dist/{metrics-C9YY1OcL.js.map → metrics-Cl0L1KUy.js.map} +1 -1
  144. package/dist/{mint-B2O60ACG.js → mint-Dj9Ww_3I.js} +4 -4
  145. package/dist/{mint-B2O60ACG.js.map → mint-Dj9Ww_3I.js.map} +1 -1
  146. package/dist/{multi-layer-verifier-DnAqwl0h.d.ts → multi-layer-verifier-DIguZc8Z.d.ts} +3 -3
  147. package/dist/{multi-layer-verifier-DnAqwl0h.d.ts.map → multi-layer-verifier-DIguZc8Z.d.ts.map} +1 -1
  148. package/dist/multiplicity-DIWHvysC.d.ts +43 -0
  149. package/dist/multiplicity-DIWHvysC.d.ts.map +1 -0
  150. package/dist/multishot/index.d.ts +3 -3
  151. package/dist/multishot/index.js +1 -1
  152. package/dist/openapi.json +1 -1
  153. package/dist/{opencode-sqlite-8r6WUfHc.js → opencode-sqlite-DJWAXLms.js} +2 -2
  154. package/dist/{opencode-sqlite-8r6WUfHc.js.map → opencode-sqlite-DJWAXLms.js.map} +1 -1
  155. package/dist/package-version-D7lQHt_-.js +34 -0
  156. package/dist/package-version-D7lQHt_-.js.map +1 -0
  157. package/dist/paired-arms-D-XRF_fy.js +1045 -0
  158. package/dist/paired-arms-D-XRF_fy.js.map +1 -0
  159. package/dist/paired-promotion-decision-CGzg0cI_.d.ts +251 -0
  160. package/dist/paired-promotion-decision-CGzg0cI_.d.ts.map +1 -0
  161. package/dist/paired-tests-BHIhYVdu.js +213 -0
  162. package/dist/paired-tests-BHIhYVdu.js.map +1 -0
  163. package/dist/pareto-BqNW3LJR.d.ts +117 -0
  164. package/dist/pareto-BqNW3LJR.d.ts.map +1 -0
  165. package/dist/pipelines/index.d.ts +3 -64
  166. package/dist/pipelines/index.d.ts.map +1 -1
  167. package/dist/pipelines/index.js +4 -284
  168. package/dist/pipelines/index.js.map +1 -1
  169. package/dist/power-and-mde-CHIrXJll.js +195 -0
  170. package/dist/power-and-mde-CHIrXJll.js.map +1 -0
  171. package/dist/{promotion-policy-CrLrmys8.js → power-preflight-DEw-uC7q.js} +4 -184
  172. package/dist/power-preflight-DEw-uC7q.js.map +1 -0
  173. package/dist/pre-registration-CZwSFQS4.d.ts +577 -0
  174. package/dist/pre-registration-CZwSFQS4.d.ts.map +1 -0
  175. package/dist/{prime-protocol-BfSalTfR.js → prime-protocol-6tZTVsWm.js} +72 -21
  176. package/dist/prime-protocol-6tZTVsWm.js.map +1 -0
  177. package/dist/produced-state-DU79a81m.js +586 -0
  178. package/dist/produced-state-DU79a81m.js.map +1 -0
  179. package/dist/profile-cell.d.ts +1 -1
  180. package/dist/profile-cell.js +1 -1
  181. package/dist/promotion-policy-D0nPhkSy.d.ts +134 -0
  182. package/dist/promotion-policy-D0nPhkSy.d.ts.map +1 -0
  183. package/dist/promotion-policy-xzA40Evo.js +186 -0
  184. package/dist/promotion-policy-xzA40Evo.js.map +1 -0
  185. package/dist/{skillopt-optimization-method-USDKhxSA.d.ts → provenance-CCdxgLDT.d.ts} +57 -478
  186. package/dist/provenance-CCdxgLDT.d.ts.map +1 -0
  187. package/dist/registry-oJeeI4-a.d.ts +178 -0
  188. package/dist/registry-oJeeI4-a.d.ts.map +1 -0
  189. package/dist/{release-report-DA2BCu5a.d.ts → release-confidence-BFRE5WSp.d.ts} +180 -112
  190. package/dist/release-confidence-BFRE5WSp.d.ts.map +1 -0
  191. package/dist/{release-report-BUYmoKo2.js → release-confidence-CxDuiAev.js} +125 -217
  192. package/dist/release-confidence-CxDuiAev.js.map +1 -0
  193. package/dist/reporting.d.ts +6 -5
  194. package/dist/reporting.js +7 -5
  195. package/dist/{researcher-BchpD55R.d.ts → researcher-DaN4GST-.d.ts} +7 -40
  196. package/dist/{researcher-BchpD55R.d.ts.map → researcher-DaN4GST-.d.ts.map} +1 -1
  197. package/dist/{reward-hacking-BDToousL.js → reward-hacking-DNgjilrV.js} +3 -3
  198. package/dist/reward-hacking-DNgjilrV.js.map +1 -0
  199. package/dist/{reward-hacking-BI0OMAlo.d.ts → reward-hacking-DSSTuI9r.d.ts} +5 -5
  200. package/dist/{reward-hacking-BI0OMAlo.d.ts.map → reward-hacking-DSSTuI9r.d.ts.map} +1 -1
  201. package/dist/rl.d.ts +7 -7
  202. package/dist/rl.js +11 -10
  203. package/dist/rl.js.map +1 -1
  204. package/dist/rollout/index.d.ts +2 -2
  205. package/dist/rollout/index.js +4 -4
  206. package/dist/{rollout-DRrksrcV.js → rollout-BWtw0I_6.js} +3 -3
  207. package/dist/{rollout-DRrksrcV.js.map → rollout-BWtw0I_6.js.map} +1 -1
  208. package/dist/{rubric-predictive-validity-DvkjPYCe.d.ts → rubric-predictive-validity-BgxtKe4G.d.ts} +2 -2
  209. package/dist/{rubric-predictive-validity-DvkjPYCe.d.ts.map → rubric-predictive-validity-BgxtKe4G.d.ts.map} +1 -1
  210. package/dist/{rubric-predictive-validity-BRR632r1.js → rubric-predictive-validity-Cwwyd7ah.js} +2 -2
  211. package/dist/{rubric-predictive-validity-BRR632r1.js.map → rubric-predictive-validity-Cwwyd7ah.js.map} +1 -1
  212. package/dist/{run-record-BmSPWXJR.js → run-record-BvHPVS-i.js} +3 -3
  213. package/dist/{run-record-BmSPWXJR.js.map → run-record-BvHPVS-i.js.map} +1 -1
  214. package/dist/{run-record-CF4Dwpxr.d.ts → run-record-CKiihE6f.d.ts} +4 -4
  215. package/dist/{run-record-CF4Dwpxr.d.ts.map → run-record-CKiihE6f.d.ts.map} +1 -1
  216. package/dist/{proposal-findings-2GIUo1et.js → run-score-lDzV0X8j.js} +31 -31
  217. package/dist/run-score-lDzV0X8j.js.map +1 -0
  218. package/dist/sandbox-harness-BlSOu4LX.d.ts +70 -0
  219. package/dist/sandbox-harness-BlSOu4LX.d.ts.map +1 -0
  220. package/dist/{schema-Cef2cFmb2.d.ts → schema-Cef2cFmb.d.ts} +1 -1
  221. package/dist/schema-Cef2cFmb.d.ts.map +1 -0
  222. package/dist/semantic-concept-judge-BiJxScqe.js +406 -0
  223. package/dist/semantic-concept-judge-BiJxScqe.js.map +1 -0
  224. package/dist/{sequential-D-BLJBKU.js → sequential-C458DXNf.js} +4 -3
  225. package/dist/{sequential-D-BLJBKU.js.map → sequential-C458DXNf.js.map} +1 -1
  226. package/dist/sequential-eprocess-CbUt2htw.js +83 -0
  227. package/dist/sequential-eprocess-CbUt2htw.js.map +1 -0
  228. package/dist/series-convergence-D1cL1f-4.d.ts +129 -0
  229. package/dist/series-convergence-D1cL1f-4.d.ts.map +1 -0
  230. package/dist/{server-iu0ede49.js → server-ulsOdrTI.js} +5 -21
  231. package/dist/server-ulsOdrTI.js.map +1 -0
  232. package/dist/skillopt-optimization-method-C5cotF4E.d.ts +427 -0
  233. package/dist/skillopt-optimization-method-C5cotF4E.d.ts.map +1 -0
  234. package/dist/skillopt-optimization-method-jjdnc3YK.js +1999 -0
  235. package/dist/skillopt-optimization-method-jjdnc3YK.js.map +1 -0
  236. package/dist/{statistical-heldout-TQ-4CYiN.d.ts → statistical-heldout-DoFF5KJX.d.ts} +5 -278
  237. package/dist/statistical-heldout-DoFF5KJX.d.ts.map +1 -0
  238. package/dist/{store-otlp-Dw8PPIlL.js → store-otlp-CsptLYpN.js} +3 -3
  239. package/dist/{store-otlp-Dw8PPIlL.js.map → store-otlp-CsptLYpN.js.map} +1 -1
  240. package/dist/store-tool-spans-Cq9mFd-q.js +667 -0
  241. package/dist/store-tool-spans-Cq9mFd-q.js.map +1 -0
  242. package/dist/store-tool-spans-DbDLOBOb.d.ts +342 -0
  243. package/dist/store-tool-spans-DbDLOBOb.d.ts.map +1 -0
  244. package/dist/student-t-CvBq2mve.js +38 -0
  245. package/dist/student-t-CvBq2mve.js.map +1 -0
  246. package/dist/{summary-report-Lf-5I7xh.js → summary-report-Blysd6Z2.js} +6 -3
  247. package/dist/{summary-report-Lf-5I7xh.js.map → summary-report-Blysd6Z2.js.map} +1 -1
  248. package/dist/{summary-report-BNR7DWTj.d.ts → summary-report-CaL-Hnxt.d.ts} +5 -5
  249. package/dist/{summary-report-BNR7DWTj.d.ts.map → summary-report-CaL-Hnxt.d.ts.map} +1 -1
  250. package/dist/supervisor-run/index.d.ts +391 -3
  251. package/dist/supervisor-run/index.d.ts.map +1 -0
  252. package/dist/supervisor-run/index.js +1689 -2
  253. package/dist/{supervisor-run-D_sokXcO.js.map → supervisor-run/index.js.map} +1 -1
  254. package/dist/{extract-usage-CdZdoj1s.js → task-failure-attributes-CpQ4y5RD.js} +5 -157
  255. package/dist/task-failure-attributes-CpQ4y5RD.js.map +1 -0
  256. package/dist/{tool-groups-CZPGGlHf.d.ts → tool-groups-Cteb03Ps.d.ts} +3 -3
  257. package/dist/tool-groups-Cteb03Ps.d.ts.map +1 -0
  258. package/dist/tool-waste-BDdBZG1F.js +803 -0
  259. package/dist/tool-waste-BDdBZG1F.js.map +1 -0
  260. package/dist/tool-waste-DjRDEsuI.d.ts +128 -0
  261. package/dist/tool-waste-DjRDEsuI.d.ts.map +1 -0
  262. package/dist/trace-repair/index.d.ts +14 -5
  263. package/dist/trace-repair/index.d.ts.map +1 -1
  264. package/dist/trace-repair/index.js +35 -7
  265. package/dist/trace-repair/index.js.map +1 -1
  266. package/dist/traces.d.ts +406 -7
  267. package/dist/traces.d.ts.map +1 -0
  268. package/dist/traces.js +1011 -10
  269. package/dist/traces.js.map +1 -0
  270. package/dist/trajectory-replay/index.d.ts +16 -3
  271. package/dist/trajectory-replay/index.d.ts.map +1 -1
  272. package/dist/trajectory-replay/index.js +52 -5
  273. package/dist/trajectory-replay/index.js.map +1 -1
  274. package/dist/types-BEPZc6eo.d.ts +93 -0
  275. package/dist/types-BEPZc6eo.d.ts.map +1 -0
  276. package/dist/{usage-receipt-t7vAzCRQ.js → types-BI4fT3HN.js} +67 -67
  277. package/dist/types-BI4fT3HN.js.map +1 -0
  278. package/dist/{types-BnjdJ70P.d.ts → types-BZ59Ahr8.d.ts} +5 -5
  279. package/dist/{types-BnjdJ70P.d.ts.map → types-BZ59Ahr8.d.ts.map} +1 -1
  280. package/dist/{types-D216SgwM.d.ts → types-Cx3YUh2r.d.ts} +4 -241
  281. package/dist/types-Cx3YUh2r.d.ts.map +1 -0
  282. package/dist/{types-BvDKaULh.d.ts → types-DLQx4mKU.d.ts} +4 -4
  283. package/dist/{types-BvDKaULh.d.ts.map → types-DLQx4mKU.d.ts.map} +1 -1
  284. package/dist/{types-D4mog56g.d.ts → types-yLK8gXE9.d.ts} +2 -2
  285. package/dist/{types-D4mog56g.d.ts.map → types-yLK8gXE9.d.ts.map} +1 -1
  286. package/dist/verdict-BndeTAh_.js +61 -0
  287. package/dist/verdict-BndeTAh_.js.map +1 -0
  288. package/dist/verdict-E4eRNf7-.d.ts +392 -0
  289. package/dist/verdict-E4eRNf7-.d.ts.map +1 -0
  290. package/dist/{verdict-cache-BCcOh0kF.js → verdict-cache-mZf5FEiY.js} +3 -55
  291. package/dist/{verdict-cache-BCcOh0kF.js.map → verdict-cache-mZf5FEiY.js.map} +1 -1
  292. package/dist/wire/index.d.ts +3 -3
  293. package/dist/wire/index.d.ts.map +1 -1
  294. package/dist/wire/index.js +1 -1
  295. package/docs/charter.md +3 -3
  296. package/docs/control-runtime.md +3 -42
  297. package/docs/experiment.md +0 -1
  298. package/docs/feature-guide.md +2 -2
  299. package/docs/trace-repair-grader.md +1 -0
  300. package/docs/trajectory-replay.md +1 -0
  301. package/docs/verdicts.md +43 -0
  302. package/docs/verification-strategies.md +3 -2
  303. package/package.json +6 -11
  304. package/dist/agent-profile-DPi7IZg7.d.ts.map +0 -1
  305. package/dist/analyze-runs-C30yljDJ.js.map +0 -1
  306. package/dist/baseline-CavEbRyH.d.ts +0 -136
  307. package/dist/baseline-CavEbRyH.d.ts.map +0 -1
  308. package/dist/benchmark-command-BteMFN62.js.map +0 -1
  309. package/dist/benchmarks-Dzs8CKb1.js +0 -755
  310. package/dist/benchmarks-Dzs8CKb1.js.map +0 -1
  311. package/dist/campaign-C2TTzQII.js.map +0 -1
  312. package/dist/completion-verifier-DJA5BhPb.d.ts +0 -414
  313. package/dist/completion-verifier-DJA5BhPb.d.ts.map +0 -1
  314. package/dist/control.d.ts +0 -3
  315. package/dist/control.js +0 -2
  316. package/dist/dataset-C8xaLXdY.d.ts.map +0 -1
  317. package/dist/default-registry-Bf8Woqmq.d.ts.map +0 -1
  318. package/dist/default-registry-BmktKy8r.js.map +0 -1
  319. package/dist/engine-3hL-XqwJ.d.ts.map +0 -1
  320. package/dist/experiment-tracker-CnRICnMl.js +0 -500
  321. package/dist/experiment-tracker-CnRICnMl.js.map +0 -1
  322. package/dist/experiment-tracker-IMntXr6J.d.ts.map +0 -1
  323. package/dist/extract-usage-CdZdoj1s.js.map +0 -1
  324. package/dist/failure-cluster-CqcvCcdR.d.ts.map +0 -1
  325. package/dist/feedback-trajectory-CacHpxsp.d.ts.map +0 -1
  326. package/dist/index-BZ3-y4YL.d.ts +0 -391
  327. package/dist/index-BZ3-y4YL.d.ts.map +0 -1
  328. package/dist/index-CQTZ-4XN.d.ts.map +0 -1
  329. package/dist/index-DPPGNJ_R.d.ts.map +0 -1
  330. package/dist/index-YE4KdKbO2.d.ts +0 -335
  331. package/dist/index-YE4KdKbO2.d.ts.map +0 -1
  332. package/dist/paired-arms-iZ08VFMN.js +0 -260
  333. package/dist/paired-arms-iZ08VFMN.js.map +0 -1
  334. package/dist/paired-promotion-decision-B6zJ3gYM.d.ts +0 -114
  335. package/dist/paired-promotion-decision-B6zJ3gYM.d.ts.map +0 -1
  336. package/dist/prime-protocol-BfSalTfR.js.map +0 -1
  337. package/dist/promotion-policy-Ckjhzg_4.d.ts +0 -289
  338. package/dist/promotion-policy-Ckjhzg_4.d.ts.map +0 -1
  339. package/dist/promotion-policy-CrLrmys8.js.map +0 -1
  340. package/dist/proposal-findings-2GIUo1et.js.map +0 -1
  341. package/dist/propose-review-control-dSNPjFUH.js +0 -1458
  342. package/dist/propose-review-control-dSNPjFUH.js.map +0 -1
  343. package/dist/release-report-BUYmoKo2.js.map +0 -1
  344. package/dist/release-report-DA2BCu5a.d.ts.map +0 -1
  345. package/dist/replay-CohS93nE.js +0 -1859
  346. package/dist/replay-CohS93nE.js.map +0 -1
  347. package/dist/replay-DbhZ4Ked.d.ts +0 -834
  348. package/dist/replay-DbhZ4Ked.d.ts.map +0 -1
  349. package/dist/reward-hacking-BDToousL.js.map +0 -1
  350. package/dist/run-evidence-Dhi3C81V.d.ts +0 -225
  351. package/dist/run-evidence-Dhi3C81V.d.ts.map +0 -1
  352. package/dist/schema-Cef2cFmb2.d.ts.map +0 -1
  353. package/dist/semantic-concept-judge-D1z-KepS.js +0 -767
  354. package/dist/semantic-concept-judge-D1z-KepS.js.map +0 -1
  355. package/dist/series-convergence-ofsqPWhs.d.ts +0 -35
  356. package/dist/series-convergence-ofsqPWhs.d.ts.map +0 -1
  357. package/dist/server-iu0ede49.js.map +0 -1
  358. package/dist/single-run-lock-DFWHEB09.js.map +0 -1
  359. package/dist/skill-usage-CO9OLRBx.d.ts.map +0 -1
  360. package/dist/skillopt-optimization-method-CQdVeM8k.js.map +0 -1
  361. package/dist/skillopt-optimization-method-USDKhxSA.d.ts.map +0 -1
  362. package/dist/statistical-heldout-TQ-4CYiN.d.ts.map +0 -1
  363. package/dist/statistics-ByxzSiOM.js +0 -2212
  364. package/dist/statistics-ByxzSiOM.js.map +0 -1
  365. package/dist/statistics-D6Uebe_4.d.ts +0 -968
  366. package/dist/statistics-D6Uebe_4.d.ts.map +0 -1
  367. package/dist/supervisor-run-D_sokXcO.js +0 -1690
  368. package/dist/test-graded-scenario-D1TaI2va.d.ts +0 -141
  369. package/dist/test-graded-scenario-D1TaI2va.d.ts.map +0 -1
  370. package/dist/test-graded-scenario-JHcKQNpq.js +0 -318
  371. package/dist/test-graded-scenario-JHcKQNpq.js.map +0 -1
  372. package/dist/tool-groups-CZPGGlHf.d.ts.map +0 -1
  373. package/dist/tool-use-metrics-DEGMKycK.js +0 -370
  374. package/dist/tool-use-metrics-DEGMKycK.js.map +0 -1
  375. package/dist/types-D216SgwM.d.ts.map +0 -1
  376. package/dist/usage-receipt-t7vAzCRQ.js.map +0 -1
  377. package/dist/verdict-DExhxfgR.d.ts +0 -201
  378. package/dist/verdict-DExhxfgR.d.ts.map +0 -1
@@ -1,1690 +0,0 @@
1
- import { a as parseJsonl, i as parseJson, o as parseSupervisorTree, r as asRecord, s as workerSourceKey } from "./integrity-DY6tIbl0.js";
2
- import { d as transcriptFromEntries, i as openOpencodeDb, l as parseClaudeEntries, r as findOpencodeSessionsByDirectory, t as DEFAULT_OPENCODE_DB } from "./opencode-sqlite-8r6WUfHc.js";
3
- import { basename, dirname, join } from "node:path";
4
- import { appendFile, mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises";
5
- //#region src/supervisor-run/types.ts
6
- function unavailable(reason) {
7
- return { unavailable: reason };
8
- }
9
- function isUnavailable(v) {
10
- return typeof v === "object" && v !== null && typeof v.unavailable === "string";
11
- }
12
- /** Render a measured scalar for the markdown/headline: `0` and `unavailable` stay distinct. */
13
- function showMeasured(v) {
14
- if (isUnavailable(v)) return `unavailable — ${v.unavailable}`;
15
- if (v === null) return "null";
16
- return String(v);
17
- }
18
- /** A source that carries every fact the analyzer can use. */
19
- const NO_SOURCE_LIMITS = {
20
- managerTokens: null,
21
- workerTokens: null,
22
- spendUsd: null,
23
- workerVerdicts: null,
24
- deliverables: null
25
- };
26
- const SUPERVISOR_RUN_SCHEMA = "tangle.supervisor-run@1";
27
- const SUPERVISOR_RUN_ROLLUP_SCHEMA = "tangle.supervisor-run-rollup@1";
28
- //#endregion
29
- //#region src/supervisor-run/analyze.ts
30
- /**
31
- * The pure analyzer. Takes already-read bytes (`SupervisorRunSources`) and
32
- * returns the report — every metric derivable from a synthetic journal string
33
- * with no filesystem, no process, and no network. All I/O lives in a reader
34
- * (`loops-reader.ts` is one).
35
- */
36
- const NO_CACHE_COUNTERS = "the journal carries no cache-token counters for this role";
37
- /**
38
- * Analyze already-read supervisor-run bytes. Pure and synchronous: same bytes
39
- * in, same report out (modulo `generatedAt`, which `now` pins in tests).
40
- */
41
- function analyzeSupervisorRunSources(src, now = Date.now) {
42
- const gaps = [];
43
- const gap = (what, reason) => {
44
- gaps.push(`${what}: ${reason}`);
45
- return unavailable(reason);
46
- };
47
- const journalMissing = src.supRunDir === null ? "no supervisor run dir under <ws>/.agent/supervisor (or legacy <ws>/.loops/supervisor)" : "journal.jsonl absent";
48
- const haveJournal = src.journal !== null;
49
- const tree = parseSupervisorTree(src);
50
- const state = tree.state;
51
- const result = parseJson(src.result);
52
- const judge = parseJson(src.judge);
53
- const { rootId, workerSpawns, workerCloses, startedAt, completedAt } = tree;
54
- const rootSpawn = rootId === null ? null : tree.spawns.find((spawn) => spawn.id === rootId) ?? null;
55
- const spawnById = new Map(workerSpawns.map((spawn) => [spawn.id, spawn]));
56
- const spawnsByLabel = /* @__PURE__ */ new Map();
57
- for (const spawn of workerSpawns) {
58
- const matches = spawnsByLabel.get(spawn.label) ?? [];
59
- matches.push(spawn);
60
- spawnsByLabel.set(spawn.label, matches);
61
- }
62
- const spawnsForSource = (worker) => {
63
- if (worker.workerId === void 0) return spawnsByLabel.get(worker.label) ?? [];
64
- const spawn = spawnById.get(worker.workerId);
65
- return spawn === void 0 ? [] : [spawn];
66
- };
67
- const spawnForSource = (worker) => {
68
- const matches = spawnsForSource(worker);
69
- return matches.length === 1 ? matches[0] ?? null : null;
70
- };
71
- const supervisorWallMs = startedAt !== null && completedAt !== null && completedAt >= startedAt ? completedAt - startedAt : !haveJournal ? gap("supervisorWallMs", journalMissing) : gap("supervisorWallMs", "no parseable start/complete timestamps in state.json or journal");
72
- const steerRows = [];
73
- let steerQueuedTotal = 0;
74
- let steerDeliveredTotal = 0;
75
- let upLegMessages = 0;
76
- if (src.workers !== null) for (const w of src.workers) {
77
- const facts = tree.workerLogs.get(workerSourceKey(w));
78
- const queued = facts?.steersQueued ?? null;
79
- const delivered = facts?.steersDelivered ?? null;
80
- upLegMessages += facts?.questions ?? 0;
81
- if (queued !== null && delivered !== null) steerRows.push({
82
- workerId: w.workerId ?? null,
83
- worker: w.label,
84
- queued,
85
- delivered
86
- });
87
- if (queued !== null) steerQueuedTotal += queued;
88
- if (delivered !== null) steerDeliveredTotal += delivered;
89
- }
90
- const workersGapReason = src.workersMissingReason ?? "workers/ directory absent";
91
- const unavailableReasons = (pick) => {
92
- const reasons = tree.workerLogRows.map((facts) => pick(facts)).filter((reason) => reason !== null);
93
- return reasons.length === 0 ? null : `exact steer accounting unavailable for ${reasons.length} worker row(s): ${[...new Set(reasons)].join(" | ")}`;
94
- };
95
- const queuedGapReason = unavailableReasons((facts) => facts.steersQueuedUnavailable);
96
- const deliveredGapReason = unavailableReasons((facts) => facts.steersDeliveredUnavailable);
97
- const workerEventsGapReason = unavailableReasons((facts) => !facts.eventsCaptured ? "events absent" : facts.eventsInvalidRows > 0 ? "events contain malformed rows" : null);
98
- const steers = src.workers === null ? gap("steers", workersGapReason) : queuedGapReason === null ? steerQueuedTotal : gap("steers", queuedGapReason);
99
- const steersDelivered = src.workers === null ? unavailable(workersGapReason) : deliveredGapReason === null ? steerDeliveredTotal : unavailable(deliveredGapReason);
100
- const steersByWorker = src.workers === null ? unavailable(workersGapReason) : queuedGapReason === null && deliveredGapReason === null ? steerRows : unavailable(queuedGapReason ?? deliveredGapReason ?? workersGapReason);
101
- const driverSteerCalls = src.driverLog === null ? gap("driverSteerCalls", "driver.log absent") : Math.max(0, (src.driverLog.match(/supervisor_steer/g) ?? []).length - registrationMentions(src.driverLog));
102
- const timeline = [];
103
- for (const s of workerSpawns) if (s.at !== null) timeline.push({
104
- at: s.at,
105
- delta: 1
106
- });
107
- for (const c of workerCloses) if (c.at !== null) timeline.push({
108
- at: c.at,
109
- delta: -1
110
- });
111
- timeline.sort((a, b) => a.at - b.at || a.delta - b.delta);
112
- let waves = 0;
113
- const waveSizes = [];
114
- let closedSinceWaveStart = true;
115
- for (const step of timeline) if (step.delta === 1) {
116
- if (closedSinceWaveStart) {
117
- waves += 1;
118
- waveSizes.push(0);
119
- closedSinceWaveStart = false;
120
- }
121
- waveSizes[waveSizes.length - 1] = (waveSizes[waveSizes.length - 1] ?? 0) + 1;
122
- } else closedSinceWaveStart = true;
123
- let live = 0;
124
- let maxConcurrency = 0;
125
- let idleMs = 0;
126
- let sumWorkerWallMs = 0;
127
- let prev = startedAt;
128
- for (const step of timeline) {
129
- if (prev !== null && step.at >= prev) {
130
- const span = step.at - prev;
131
- if (live === 0) idleMs += span;
132
- sumWorkerWallMs += span * live;
133
- }
134
- live += step.delta;
135
- if (live > maxConcurrency) maxConcurrency = live;
136
- prev = step.at;
137
- }
138
- if (prev !== null && completedAt !== null && completedAt >= prev) {
139
- const span = completedAt - prev;
140
- if (live === 0) idleMs += span;
141
- sumWorkerWallMs += span * live;
142
- }
143
- const firstWorkerSpawnAt = workerSpawns.reduce((acc, s) => s.at === null ? acc : acc === null ? s.at : Math.min(acc, s.at), null);
144
- const closeById = new Map(workerCloses.map((close) => [close.id, close]));
145
- const childSpawnsByParent = /* @__PURE__ */ new Map();
146
- for (const spawn of workerSpawns) {
147
- if (spawn.parent === null) continue;
148
- const siblings = childSpawnsByParent.get(spawn.parent) ?? [];
149
- siblings.push(spawn);
150
- childSpawnsByParent.set(spawn.parent, siblings);
151
- }
152
- let respawns = 0;
153
- let observeThenRespawn = 0;
154
- let respawnWithoutEvidence = 0;
155
- const repeatedLabelSet = /* @__PURE__ */ new Set();
156
- for (const siblings of childSpawnsByParent.values()) {
157
- const labelCounts = /* @__PURE__ */ new Map();
158
- for (const spawn of siblings) labelCounts.set(spawn.label, (labelCounts.get(spawn.label) ?? 0) + 1);
159
- for (const [label, count] of labelCounts) if (count > 1) repeatedLabelSet.add(label);
160
- const orderedSpawns = siblings.map((spawn, index) => ({
161
- spawn,
162
- index
163
- })).filter((row) => row.spawn.at !== null).sort((a, b) => a.spawn.at - b.spawn.at || a.index - b.index);
164
- const directCloseTimes = siblings.map((spawn) => closeById.get(spawn.id)?.at ?? null).filter((at) => at !== null).sort((a, b) => a - b);
165
- const firstDirectClose = directCloseTimes[0] ?? null;
166
- for (let i = 1; i < orderedSpawns.length; i += 1) {
167
- const previous = orderedSpawns[i - 1]?.spawn.at;
168
- const current = orderedSpawns[i]?.spawn.at;
169
- if (previous === void 0 || current === void 0) continue;
170
- if (firstDirectClose === null || current <= firstDirectClose) continue;
171
- respawns += 1;
172
- if (hasNumberBetween(directCloseTimes, previous, current)) observeThenRespawn += 1;
173
- else respawnWithoutEvidence += 1;
174
- }
175
- }
176
- const repeatedLabels = [...repeatedLabelSet];
177
- const parentOf = new Map(tree.spawns.map((s) => [s.id, s.parent]));
178
- let delegationDepth = 0;
179
- for (const s of workerSpawns) {
180
- let d = 0;
181
- let cur = s.id;
182
- const seen = /* @__PURE__ */ new Set();
183
- while (cur !== null && cur !== rootId && !seen.has(cur)) {
184
- seen.add(cur);
185
- d += 1;
186
- cur = parentOf.get(cur) ?? null;
187
- }
188
- if (d > delegationDepth) delegationDepth = d;
189
- }
190
- const orchestration = {
191
- workersSpawned: haveJournal ? workerSpawns.length : gap("workersSpawned", journalMissing),
192
- workersSettled: haveJournal ? workerCloses.filter((c) => c.kind === "settled").length : unavailable(journalMissing),
193
- workersCancelled: haveJournal ? workerCloses.filter((c) => c.kind === "cancelled").length : unavailable(journalMissing),
194
- steers,
195
- steersDelivered,
196
- steersByWorker,
197
- driverSteerCalls,
198
- waves: haveJournal ? waves : unavailable(journalMissing),
199
- waveSizes: haveJournal ? waveSizes : unavailable(journalMissing),
200
- maxConcurrency: haveJournal ? maxConcurrency : unavailable(journalMissing),
201
- respawns: haveJournal ? respawns : unavailable(journalMissing),
202
- repeatedLabels: haveJournal ? repeatedLabels : unavailable(journalMissing),
203
- delegationDepth: haveJournal ? delegationDepth : unavailable(journalMissing),
204
- timeToFirstSpawnMs: startedAt !== null && firstWorkerSpawnAt !== null ? firstWorkerSpawnAt - startedAt : haveJournal ? unavailable("no worker spawn timestamps") : unavailable(journalMissing),
205
- supervisorWallMs,
206
- idleMs: isUnavailable(supervisorWallMs) ? unavailable(supervisorWallMs.unavailable) : idleMs,
207
- idlePct: isUnavailable(supervisorWallMs) || supervisorWallMs === 0 ? isUnavailable(supervisorWallMs) ? unavailable(supervisorWallMs.unavailable) : unavailable("supervisor wall is 0ms") : round(idleMs / supervisorWallMs * 100, 1),
208
- workerUtilization: isUnavailable(supervisorWallMs) || supervisorWallMs === 0 ? isUnavailable(supervisorWallMs) ? unavailable(supervisorWallMs.unavailable) : unavailable("supervisor wall is 0ms") : round(sumWorkerWallMs / supervisorWallMs, 3)
209
- };
210
- const settledByStatus = {};
211
- const settledVerdicts = {};
212
- for (const c of workerCloses) {
213
- const key = c.status ?? "unknown";
214
- settledByStatus[key] = (settledByStatus[key] ?? 0) + 1;
215
- if (c.verdict !== null) settledVerdicts[c.verdict] = (settledVerdicts[c.verdict] ?? 0) + 1;
216
- }
217
- const verdictLimit = src.limits.workerVerdicts;
218
- const acceptedLimit = verdictLimit ?? src.limits.deliverables;
219
- let accepted = 0;
220
- let emptyPass = 0;
221
- let evidenceBytes = 0;
222
- const sourceVerdicts = [];
223
- for (const w of src.workers ?? []) {
224
- const f = tree.workerLogs.get(workerSourceKey(w));
225
- if (f?.finished) evidenceBytes += f.evidenceBytes;
226
- const spawn = spawnForSource(w);
227
- const passed = (spawn === null ? null : closeById.get(spawn.id) ?? null)?.valid ?? f?.passed ?? null;
228
- if (passed !== null) sourceVerdicts.push(passed);
229
- if (passed === true) if ((w.patchBytes ?? f?.finishedPatchBytes ?? 0) > 0) accepted += 1;
230
- else emptyPass += 1;
231
- }
232
- const settledCloses = workerCloses.filter((close) => close.kind === "settled");
233
- const structuredVerdicts = settledCloses.map((close) => close.valid).filter((valid) => valid !== null);
234
- const journalVerdictsComplete = settledCloses.length > 0 && structuredVerdicts.length === settledCloses.length;
235
- const sourceVerdictsComplete = sourceVerdicts.length > 0 && sourceVerdicts.length >= settledCloses.length;
236
- const rejected = journalVerdictsComplete ? structuredVerdicts.filter((valid) => !valid).length : sourceVerdictsComplete ? sourceVerdicts.filter((valid) => !valid).length : 0;
237
- const rejectedLimit = journalVerdictsComplete || sourceVerdictsComplete ? null : settledCloses.length > 0 ? verdictLimit ?? "a settled journal verdict has no validity and no matched worker log" : verdictLimit !== null ? verdictLimit : !haveJournal && src.workers === null ? workersGapReason : null;
238
- const decision = {
239
- settledByStatus: haveJournal ? settledByStatus : gap("settledByStatus", journalMissing),
240
- settledVerdicts: verdictLimit !== null ? unavailable(verdictLimit) : haveJournal ? settledVerdicts : unavailable(journalMissing),
241
- accepted: acceptedLimit !== null ? gap("accepted", acceptedLimit) : src.workers === null ? unavailable(workersGapReason) : accepted,
242
- rejected: rejectedLimit === null ? rejected : unavailable(rejectedLimit),
243
- emptyPass: acceptedLimit !== null ? gap("emptyPass", acceptedLimit) : src.workers === null ? unavailable(workersGapReason) : emptyPass,
244
- observeThenRespawn: haveJournal ? observeThenRespawn : unavailable(journalMissing),
245
- respawnWithoutEvidence: haveJournal ? respawnWithoutEvidence : unavailable(journalMissing),
246
- reviewActions: src.workers === null ? unavailable(workersGapReason) : queuedGapReason === null ? steerQueuedTotal + upLegMessages : unavailable(queuedGapReason),
247
- workerEvidenceBytes: src.workers === null ? unavailable(workersGapReason) : workerEventsGapReason === null ? evidenceBytes : unavailable(workerEventsGapReason)
248
- };
249
- const rootChildIds = new Set(workerSpawns.filter((spawn) => spawn.parent === rootId).map((spawn) => spawn.id));
250
- const rootChildCloses = workerCloses.filter((close) => rootChildIds.has(close.id));
251
- const journalWorkerIn = rootChildCloses.reduce((a, c) => a + c.spend.tokens.input, 0);
252
- const journalWorkerOut = rootChildCloses.reduce((a, c) => a + c.spend.tokens.output, 0);
253
- const journalWorkerUsd = rootChildCloses.reduce((a, c) => a + c.spend.usd, 0);
254
- const workerUsdById = /* @__PURE__ */ new Map();
255
- for (const c of workerCloses) workerUsdById.set(c.id, (workerUsdById.get(c.id) ?? 0) + c.spend.usd);
256
- const labelById = new Map(workerSpawns.map((spawn) => [spawn.id, spawn.label]));
257
- const workerUsdByLabel = /* @__PURE__ */ new Map();
258
- for (const close of workerCloses) {
259
- const label = labelById.get(close.id);
260
- if (label === void 0) continue;
261
- workerUsdByLabel.set(label, (workerUsdByLabel.get(label) ?? 0) + close.spend.usd);
262
- }
263
- const sq = src.harnessWorkerTokens;
264
- const harnessGapReason = src.harnessMissingReason ?? "harness session store unavailable and journal settled spend is 0";
265
- const workerTokenLimit = src.limits.workerTokens;
266
- const workerIn = workerTokenLimit !== null ? gap("workers.tokensIn", workerTokenLimit) : sq !== null ? journalWorkerIn + sq.input : haveJournal ? journalWorkerIn : gap("workers.tokensIn", harnessGapReason);
267
- const workerOut = workerTokenLimit !== null ? unavailable(workerTokenLimit) : sq !== null ? journalWorkerOut + sq.output : haveJournal ? journalWorkerOut : unavailable(harnessGapReason);
268
- const stateResult = asRecord(state?.result);
269
- const stateUsd = typeof stateResult.spentUsd === "number" ? stateResult.spentUsd : null;
270
- const usdLimit = src.limits.spendUsd;
271
- const totalUsd = usdLimit !== null ? gap("totalUsd", usdLimit) : stateUsd !== null ? round(stateUsd, 6) : haveJournal ? round(tree.brain.usd + journalWorkerUsd, 6) : gap("totalUsd", journalMissing);
272
- const perWorker = (src.workers ?? []).map((w) => {
273
- const f = tree.workerLogs.get(workerSourceKey(w));
274
- const matchingSpawns = spawnsForSource(w);
275
- const spawn = spawnForSource(w);
276
- const close = spawn === null ? null : closeById.get(spawn.id) ?? null;
277
- const passed = close?.valid ?? f?.passed ?? null;
278
- const matchingRoles = new Set(matchingSpawns.map((candidate) => candidate.role));
279
- const matchingRuntimes = new Set(matchingSpawns.map((candidate) => candidate.runtime));
280
- const matchingProfiles = new Set(matchingSpawns.map((candidate) => candidate.profileDigest));
281
- const journalWallMs = spawn?.at !== null && spawn?.at !== void 0 && close?.at !== null && close?.at !== void 0 && close.at >= spawn.at ? close.at - spawn.at : null;
282
- return {
283
- workerId: w.workerId ?? null,
284
- worker: w.label,
285
- role: matchingRoles.size === 1 ? matchingSpawns[0]?.role ?? null : null,
286
- runtime: matchingRuntimes.size === 1 ? matchingSpawns[0]?.runtime ?? null : null,
287
- profileDigest: matchingProfiles.size === 1 ? matchingSpawns[0]?.profileDigest ?? null : null,
288
- status: close?.status ?? null,
289
- failure: close?.reason ?? null,
290
- infra: close?.infra ?? null,
291
- wallMs: f?.started != null && f.finishedAt != null ? f.finishedAt - f.started : journalWallMs,
292
- tokensIn: w.tokensIn ?? (close?.hasSpend ? close.spend.tokens.input : null),
293
- tokensOut: w.tokensOut ?? (close?.hasSpend ? close.spend.tokens.output : null),
294
- usd: usdLimit !== null ? null : w.workerId === void 0 ? workerUsdByLabel.get(w.label) ?? null : workerUsdById.get(w.workerId) ?? null,
295
- patchBytes: w.patchBytes ?? f?.finishedPatchBytes ?? null,
296
- passed,
297
- score: close?.score ?? f?.score ?? null
298
- };
299
- });
300
- const walls = perWorker.map((w) => w.wallMs).filter((w) => w !== null).sort((a, b) => a - b);
301
- const brainCalls = parseJsonl(src.brainLog);
302
- const managerTokenLimit = src.limits.managerTokens;
303
- const economics = {
304
- brain: {
305
- tokensIn: managerTokenLimit !== null ? gap("brain.tokensIn", managerTokenLimit) : haveJournal ? tree.brain.tokensIn : gap("brain.tokensIn", journalMissing),
306
- tokensOut: managerTokenLimit !== null ? unavailable(managerTokenLimit) : haveJournal ? tree.brain.tokensOut : unavailable(journalMissing),
307
- usd: usdLimit !== null ? unavailable(usdLimit) : haveJournal ? round(tree.brain.usd, 6) : unavailable(journalMissing),
308
- cacheRead: managerTokenLimit !== null ? unavailable(managerTokenLimit) : !haveJournal ? unavailable(journalMissing) : tree.brain.hasCache ? tree.brain.cacheRead : unavailable(NO_CACHE_COUNTERS),
309
- cacheWrite: managerTokenLimit !== null ? unavailable(managerTokenLimit) : !haveJournal ? unavailable(journalMissing) : tree.brain.hasCache ? tree.brain.cacheWrite : unavailable(NO_CACHE_COUNTERS),
310
- source: managerTokenLimit ?? (haveJournal ? `journal metered events (n=${tree.brain.meteredCount})` : journalMissing)
311
- },
312
- brainTruncations: src.brainLog === null ? gap("brain.brainTruncations", src.brainLogMissingReason ?? (src.supRunDir === null ? "no supervisor run dir under <ws>/.agent/supervisor (or legacy <ws>/.loops/supervisor)" : "brain.jsonl absent — loops predates the brain-call tap, so truncation cannot be ruled out")) : brainCalls.filter((c) => c.finish_reason === "length").length,
313
- workers: {
314
- tokensIn: workerIn,
315
- tokensOut: workerOut,
316
- cacheRead: workerTokenLimit !== null ? unavailable(workerTokenLimit) : sq?.cacheRead !== void 0 ? sq.cacheRead : unavailable(NO_CACHE_COUNTERS),
317
- cacheWrite: workerTokenLimit !== null ? unavailable(workerTokenLimit) : sq?.cacheWrite !== void 0 ? sq.cacheWrite : unavailable(NO_CACHE_COUNTERS),
318
- usd: usdLimit !== null ? unavailable(usdLimit) : haveJournal ? round(journalWorkerUsd, 6) : unavailable(journalMissing),
319
- source: workerTokenLimit !== null ? workerTokenLimit : sq !== null ? `journal settled spend + ${sq.store} sessions (n=${sq.sessions})` : `journal settled spend only — ${src.harnessMissingReason ?? "harness session store unavailable"}`
320
- },
321
- totalUsd,
322
- totalUsdSource: usdLimit !== null ? usdLimit : stateUsd !== null ? `state.json result.spentUsd${rootChildCloses.length > 0 && journalWorkerUsd === 0 ? " — brain-priced only; worker CLI inference is unpriced (see worker token counts)" : ""}` : haveJournal ? "journal metered + settled usd" : journalMissing,
323
- costPerAcceptedPatchUsd: isUnavailable(totalUsd) ? unavailable(totalUsd.unavailable) : isUnavailable(decision.accepted) ? unavailable(decision.accepted.unavailable) : decision.accepted === 0 ? unavailable("no accepted worker patch (cost has no denominator)") : round(totalUsd / decision.accepted, 6),
324
- workerWallMsDistribution: walls.length === 0 ? unavailable(src.workers === null ? workersGapReason : "no worker start/finish pairs captured") : {
325
- n: walls.length,
326
- min: walls[0],
327
- p50: quantile(walls, .5),
328
- p90: quantile(walls, .9),
329
- max: walls[walls.length - 1],
330
- sum: walls.reduce((a, b) => a + b, 0)
331
- },
332
- perWorker: src.workers === null ? unavailable(workersGapReason) : perWorker
333
- };
334
- const patchStats = src.patch === null ? gap("patch", src.limits.deliverables ?? "delivered patch file absent") : parsePatch(src.patch);
335
- const outcome = {
336
- supStatus: pickString(state, "status") ?? pickString(result, "sup_status") ?? gap("supStatus", "no state.json / result.json status"),
337
- supVerdict: pickString(state, "verdict") ?? pickString(result, "sup_verdict") ?? unavailable("no state.json / result.json verdict"),
338
- delivered: typeof stateResult.delivered === "boolean" ? stateResult.delivered : typeof result?.delivered === "boolean" ? result.delivered : unavailable("no delivered flag in state.json or result.json"),
339
- judgeResolved: judge === null ? gap("judge", "judge.json absent") : typeof judge.resolved === "boolean" ? judge.resolved : null,
340
- judgeScore: judge === null ? unavailable("judge.json absent") : typeof judge.score === "number" ? judge.score : null,
341
- judgePassed: judge === null ? unavailable("judge.json absent") : typeof judge.passed === "number" ? judge.passed : null,
342
- judgeTotal: judge === null ? unavailable("judge.json absent") : typeof judge.total === "number" ? judge.total : null,
343
- verifyPass: typeof result?.verify_pass === "boolean" ? result.verify_pass : gap("verifyPass", "result.json absent or has no verify_pass"),
344
- verifyRc: typeof result?.verify_rc === "number" ? result.verify_rc : unavailable("result.json absent or has no verify_rc"),
345
- patch: patchStats,
346
- judgeSource: src.judgeSource
347
- };
348
- return {
349
- schema: SUPERVISOR_RUN_SCHEMA,
350
- runRef: src.runRef,
351
- instanceId: src.instanceId,
352
- arm: src.arm,
353
- supervisorId: rootId !== null ? rootId : unavailable(journalMissing),
354
- supervisorProfileDigest: rootSpawn?.profileDigest !== null && rootSpawn?.profileDigest !== void 0 ? rootSpawn.profileDigest : gap("supervisorProfileDigest", "root spawned event has no profile digest"),
355
- generatedAt: new Date(now()).toISOString(),
356
- orchestration,
357
- decision,
358
- economics,
359
- outcome,
360
- gaps,
361
- traceCommand: src.traceCommand ?? "npx --yes @tangle-network/traces@latest analyze --harness opencode --cwd <worker-clone-cwd>"
362
- };
363
- }
364
- /** Whether sorted values contain one value in the inclusive interval. */
365
- function hasNumberBetween(sorted, low, high) {
366
- let left = 0;
367
- let right = sorted.length;
368
- while (left < right) {
369
- const middle = left + Math.floor((right - left) / 2);
370
- if (sorted[middle] < low) left = middle + 1;
371
- else right = middle;
372
- }
373
- return left < sorted.length && sorted[left] <= high;
374
- }
375
- /** `[driver] registered tools: …supervisor_steer…` is a banner, not an invocation. */
376
- function registrationMentions(driverLog) {
377
- let n = 0;
378
- for (const line of driverLog.split("\n")) if (line.includes("registered tools:") && line.includes("supervisor_steer")) n += 1;
379
- return n;
380
- }
381
- function pickString(rec, key) {
382
- const v = rec?.[key];
383
- return typeof v === "string" ? v : null;
384
- }
385
- function round(v, digits) {
386
- const f = 10 ** digits;
387
- return Math.round(v * f) / f;
388
- }
389
- function quantile(sorted, q) {
390
- if (sorted.length === 0) return 0;
391
- return sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil(q * sorted.length) - 1))];
392
- }
393
- /** Unified-diff stats. Counts `+++ b/<path>` targets, body +/- lines, and test-file touches. */
394
- function parsePatch(text) {
395
- const files = /* @__PURE__ */ new Set();
396
- const testFiles = /* @__PURE__ */ new Set();
397
- let added = 0;
398
- let removed = 0;
399
- for (const line of text.split("\n")) {
400
- if (line.startsWith("+++ ")) {
401
- const p = line.slice(4).trim().replace(/^b\//, "");
402
- if (p !== "/dev/null") {
403
- files.add(p);
404
- if (isTestPath(p)) testFiles.add(p);
405
- }
406
- continue;
407
- }
408
- if (line.startsWith("--- ") || line.startsWith("diff --git") || line.startsWith("index ")) continue;
409
- if (line.startsWith("+")) added += 1;
410
- else if (line.startsWith("-")) removed += 1;
411
- }
412
- return {
413
- files: files.size,
414
- linesAdded: added,
415
- linesRemoved: removed,
416
- testFilesTouched: [...testFiles].sort()
417
- };
418
- }
419
- function isTestPath(p) {
420
- const base = p.split("/").pop() ?? p;
421
- return /(^|\/)(tests?|__tests__|testing|spec)(\/|$)/.test(p) || /\.(test|spec)\.[cm]?[jt]sx?$/.test(base) || /^test_.*\.py$/.test(base) || /_test\.py$/.test(base);
422
- }
423
- /**
424
- * Aggregate many supervisor-run reports. A metric no run could measure stays
425
- * `unavailable` rather than becoming a 0-valued mean, and cells whose steer
426
- * count was unavailable are counted separately from cells that measured zero.
427
- */
428
- function rollupSupervisorRuns(reports) {
429
- const known = (vals) => vals.filter((v) => !isUnavailable(v));
430
- const steerVals = known(reports.map((r) => r.orchestration.steers));
431
- const waveVals = known(reports.map((r) => r.orchestration.waves));
432
- const concVals = known(reports.map((r) => r.orchestration.maxConcurrency));
433
- const utilVals = known(reports.map((r) => r.orchestration.workerUtilization));
434
- const idleVals = known(reports.map((r) => r.orchestration.idlePct));
435
- const spawnVals = known(reports.map((r) => r.orchestration.workersSpawned));
436
- const acceptVals = known(reports.map((r) => r.decision.accepted));
437
- const usdVals = known(reports.map((r) => r.economics.totalUsd));
438
- const resolvedVals = known(reports.map((r) => r.outcome.judgeResolved));
439
- const sum = (xs) => xs.reduce((a, b) => a + b, 0);
440
- const mean = (xs) => xs.length === 0 ? unavailable("no cell reported this metric") : round(sum(xs) / xs.length, 3);
441
- const perCell = reports.map((r) => ({
442
- instanceId: r.instanceId,
443
- arm: r.arm,
444
- steers: r.orchestration.steers,
445
- waves: r.orchestration.waves,
446
- utilization: r.orchestration.workerUtilization,
447
- idlePct: r.orchestration.idlePct,
448
- resolved: r.outcome.judgeResolved,
449
- usd: r.economics.totalUsd
450
- }));
451
- return {
452
- schema: SUPERVISOR_RUN_ROLLUP_SCHEMA,
453
- cells: reports.length,
454
- steersTotal: steerVals.length === 0 ? unavailable("no cell reported a steer count") : sum(steerVals),
455
- cellsWithSteers: steerVals.length === 0 ? unavailable("no cell reported a steer count") : steerVals.filter((n) => n > 0).length,
456
- cellsWithUnavailableSteers: reports.filter((r) => isUnavailable(r.orchestration.steers)).length,
457
- wavesMean: mean(waveVals),
458
- maxConcurrencyMax: concVals.length === 0 ? unavailable("no cell reported concurrency") : Math.max(...concVals),
459
- utilizationMean: mean(utilVals),
460
- idlePctMean: mean(idleVals),
461
- workersSpawnedTotal: spawnVals.length === 0 ? unavailable("no cell reported spawns") : sum(spawnVals),
462
- acceptedTotal: acceptVals.length === 0 ? unavailable("no cell reported acceptance") : sum(acceptVals),
463
- usdTotal: usdVals.length === 0 ? unavailable("no cell reported spend") : round(sum(usdVals), 6),
464
- resolvedCount: resolvedVals.length === 0 ? unavailable("no cell reported a judge verdict") : resolvedVals.filter((v) => v === true).length,
465
- perCell
466
- };
467
- }
468
- //#endregion
469
- //#region src/supervisor-run/claude-code-reader.ts
470
- /**
471
- * Supervision-tree reader over a THIRD-PARTY harness: Claude Code.
472
- *
473
- * `loops-reader.ts` reads a supervisor we wrote, whose journal was designed
474
- * for this analysis. This reader reads a harness we do not control, whose
475
- * transcript was designed for replaying a chat — and recovers the same tree
476
- * from it. If both produce a `SupervisorRunSources`, the tree model is a
477
- * property of multi-agent runs, not of our journal format.
478
- *
479
- * ## Where the tree hides in a Claude Code transcript
480
- *
481
- * | Tree fact | Claude Code evidence |
482
- * |---|---|
483
- * | spawn | assistant `tool_use` (`Agent` / `Task`), answered by a `tool_result` whose `toolUseResult.agentId` names the child |
484
- * | settle | a `<task-notification>` block in a later user line: `<task-id>` = agentId, `<status>` |
485
- * | steer | assistant `tool_use` (`SendMessage`) with `input.to` = agentId — mid-task, to a LIVE child |
486
- * | delivered | that steer's `tool_result` carrying `success` / `resumedAgentId` |
487
- * | cancel | assistant `tool_use` (`TaskStop`) targeting an agentId |
488
- * | brain spend| `message.usage` on the main thread's assistant lines |
489
- * | worker spend| `message.usage` inside `<session>/subagents/agent-<id>.jsonl` |
490
- * | depth | a child transcript that itself contains `Agent` tool_use lines |
491
- *
492
- * Every one of those is read through `parseClaudeEntries` — the SAME line
493
- * parser `src/rollout/readers/claude-jsonl.ts` uses for solo rollouts. There
494
- * is no second transcript parser.
495
- *
496
- * ## What Claude Code cannot say
497
- *
498
- * It records tokens but never a price, runs no per-worker verify, and keeps no
499
- * per-worker patch. Those are declared once in `limits`, so the analyzer
500
- * reports `unavailable — <reason>` instead of the $0 / 0-accepted that summing
501
- * an empty field would produce. See `SourceLimits`.
502
- *
503
- * ## Metric coverage vs the loops journal
504
- *
505
- * Measured on a real 52-agent session (fixture:
506
- * `tests/fixtures/supervisor-run/claude-code-session-*`).
507
- *
508
- * | Metric | loops | Claude Code | Why |
509
- * |---|---|---|---|
510
- * | workersSpawned / Settled / Cancelled | full | full | spawn tool_use + task-notification + TaskStop |
511
- * | steers / steersDelivered / steersByWorker | full | full | `SendMessage`; delivery from its tool_result |
512
- * | waves / waveSizes / maxConcurrency | full | full | derived from spawn/settle instants |
513
- * | respawns / repeatedLabels | full | full | same derivation |
514
- * | delegationDepth | full | full | a child transcript's own spawn calls |
515
- * | timeToFirstSpawn / supervisorWall | full | full | transcript instants |
516
- * | idleMs / idlePct / workerUtilization | full | PARTIAL | an agent that never notifies is counted live to the end of the transcript |
517
- * | observeThenRespawn / respawnWithoutEvidence | full | full | ordering of spawn vs settle instants |
518
- * | workerEvidenceBytes | full | PARTIAL | the child's closing message; 0 for pruned transcripts |
519
- * | brain tokens in/out + cache | full | full | main-thread `message.usage` |
520
- * | worker tokens in/out + cache | via harness join | full or unavailable | totals are refused if any spawned transcript was pruned; retained per-worker rows remain available |
521
- * | perWorker wall | full | full | spawn → settle instants |
522
- * | accepted / rejected / emptyPass / settledVerdicts | full | NONE | no per-worker verify step exists |
523
- * | brain/worker/total usd, costPerAcceptedPatch | full | NONE | transcripts carry no price |
524
- * | patch stats, delivered, verifyPass/Rc | full | NONE | no diff is handed back |
525
- * | judgeResolved / Score / Passed / Total | full | NONE | no judge in the loop |
526
- * | driverSteerCalls, brainTruncations | full | NONE | no outer driver log, no per-call finish_reason tap |
527
- */
528
- /** Tool names that spawn a child agent. `Task` is the older name for `Agent`. */
529
- const DEFAULT_SPAWN_TOOLS = ["Agent", "Task"];
530
- /** Tool names that deliver a message to an ALREADY-RUNNING child agent. */
531
- const DEFAULT_STEER_TOOLS = ["SendMessage"];
532
- /** Tool names that stop a running child agent. */
533
- const DEFAULT_CANCEL_TOOLS = ["TaskStop", "KillAgent"];
534
- const SPEND_UNPRICED = "Claude Code transcripts record token usage but never a price — usd is not in the store";
535
- const NO_VERDICTS = "Claude Code runs no per-worker verify step — a subagent reports prose, not pass/fail";
536
- const NO_DELIVERABLES = "Claude Code retains no per-worker patch — subagents commit to git, they do not hand back a diff";
537
- const isRecord = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
538
- const str = (v) => typeof v === "string" ? v : null;
539
- function blockText(content) {
540
- if (typeof content === "string") return content;
541
- if (!Array.isArray(content)) return "";
542
- const parts = [];
543
- for (const b of content) if (isRecord(b) && b.type === "text" && typeof b.text === "string") parts.push(b.text);
544
- return parts.join("\n");
545
- }
546
- const tag = (xml, name) => {
547
- const m = xml.match(new RegExp(`<${name}>([\\s\\S]*?)</${name}>`));
548
- return m === null ? null : m[1];
549
- };
550
- /**
551
- * Task notifications are the settle instants. A notification fires each time an
552
- * agent stops, so a resumed agent produces several — they are kept in order and
553
- * the LAST one is the settle the analyzer sees, with the earlier ones acting as
554
- * the intermediate stops they actually were.
555
- */
556
- function parseNotifications(text, at) {
557
- const out = [];
558
- for (const m of text.matchAll(/<task-notification>[\s\S]*?<\/task-notification>/g)) {
559
- const xml = m[0];
560
- const taskId = tag(xml, "task-id");
561
- if (taskId === null) continue;
562
- out.push({
563
- taskId: taskId.trim(),
564
- toolUseId: tag(xml, "tool-use-id")?.trim() ?? null,
565
- status: tag(xml, "status")?.trim() ?? "unknown",
566
- summary: tag(xml, "summary")?.trim() ?? null,
567
- at
568
- });
569
- }
570
- return out;
571
- }
572
- /** Project entries of ONE thread (main or a single sidechain) into tool traffic. */
573
- function threadCalls(entries) {
574
- const uses = [];
575
- const results = /* @__PURE__ */ new Map();
576
- const notifications = [];
577
- let firstAt = null;
578
- let lastAt = null;
579
- for (const entry of entries) {
580
- if (entry.timestamp !== null) {
581
- if (firstAt === null) firstAt = entry.timestamp;
582
- lastAt = entry.timestamp;
583
- }
584
- const content = entry.message.content;
585
- if (entry.type === "assistant") {
586
- if (!Array.isArray(content)) continue;
587
- for (const block of content) {
588
- if (!isRecord(block) || block.type !== "tool_use") continue;
589
- const id = str(block.id);
590
- const name = str(block.name);
591
- if (id === null || name === null) continue;
592
- uses.push({
593
- id,
594
- name,
595
- input: isRecord(block.input) ? block.input : {},
596
- at: entry.timestamp
597
- });
598
- }
599
- continue;
600
- }
601
- if (typeof content === "string") {
602
- notifications.push(...parseNotifications(content, entry.timestamp));
603
- continue;
604
- }
605
- if (!Array.isArray(content)) continue;
606
- for (const block of content) {
607
- if (!isRecord(block)) continue;
608
- if (block.type === "tool_result" && typeof block.tool_use_id === "string") results.set(block.tool_use_id, {
609
- id: block.tool_use_id,
610
- at: entry.timestamp,
611
- structured: entry.toolUseResult,
612
- text: blockText(block.content)
613
- });
614
- else if (block.type === "text" && typeof block.text === "string") notifications.push(...parseNotifications(block.text, entry.timestamp));
615
- }
616
- }
617
- return {
618
- uses,
619
- results,
620
- notifications,
621
- firstAt,
622
- lastAt
623
- };
624
- }
625
- /**
626
- * The agent id a spawn produced. Claude Code puts it in the structured
627
- * `toolUseResult`; the same id is echoed in the result text for transcripts
628
- * written before that field existed, so both are tried before giving up.
629
- */
630
- function spawnedAgentId(result) {
631
- if (result === void 0) return null;
632
- if (isRecord(result.structured)) {
633
- const id = str(result.structured.agentId);
634
- if (id !== null) return id;
635
- }
636
- return result.text.match(/agentId:\s*([A-Za-z0-9_-]+)/)?.[1] ?? null;
637
- }
638
- async function readChildren(dir) {
639
- const names = await readdir(dir).catch(() => null);
640
- if (names === null) return [];
641
- const out = [];
642
- for (const name of names.filter((n) => n.endsWith(".jsonl")).sort()) {
643
- const path = join(dir, name);
644
- const raw = await readFile(path, "utf8").catch(() => null);
645
- if (raw === null) continue;
646
- const entries = parseClaudeEntries(raw);
647
- const projected = transcriptFromEntries(entries, { includeSidechain: true });
648
- const metaRaw = await readFile(path.replace(/\.jsonl$/, ".meta.json"), "utf8").catch(() => null);
649
- let meta = {};
650
- if (metaRaw !== null) try {
651
- const parsed = JSON.parse(metaRaw);
652
- if (isRecord(parsed)) meta = parsed;
653
- } catch {
654
- meta = {};
655
- }
656
- const agentId = entries.find((e) => e.agentId !== null)?.agentId ?? name.replace(/^agent-/, "").replace(/\.jsonl$/, "");
657
- out.push({
658
- agentId,
659
- path,
660
- description: str(meta.description),
661
- spawnToolUseId: str(meta.toolUseId),
662
- spawnDepth: typeof meta.spawnDepth === "number" ? meta.spawnDepth : null,
663
- entries,
664
- tokensIn: projected.usage.tokensIn,
665
- tokensOut: projected.usage.tokensOut,
666
- cacheRead: projected.usage.cacheRead,
667
- cacheWrite: projected.usage.cacheWrite,
668
- firstAt: projected.startedAt,
669
- lastAt: projected.endedAt,
670
- model: projected.model,
671
- finalReport: [...projected.messages].reverse().find((m) => m.role === "assistant" && typeof m.content === "string")?.content ?? null
672
- });
673
- }
674
- return out;
675
- }
676
- /** A journal line in the dialect `parseSupervisorTree` reads. */
677
- const line = (obj) => JSON.stringify(obj);
678
- /**
679
- * Read a Claude Code session (plus its subagent transcripts) as supervision-tree
680
- * source bytes. Never throws on a missing artifact.
681
- */
682
- async function readClaudeCodeSupervisorRun(opts) {
683
- const spawnTools = new Set(opts.spawnTools ?? DEFAULT_SPAWN_TOOLS);
684
- const steerTools = new Set(opts.steerTools ?? DEFAULT_STEER_TOOLS);
685
- const cancelTools = new Set(opts.cancelTools ?? DEFAULT_CANCEL_TOOLS);
686
- const raw = await readFile(opts.transcriptPath, "utf8").catch(() => null);
687
- const sessionId = basename(opts.transcriptPath).replace(/\.jsonl$/, "");
688
- const runRef = opts.runRef ?? opts.transcriptPath;
689
- const limits = {
690
- managerTokens: null,
691
- workerTokens: null,
692
- spendUsd: SPEND_UNPRICED,
693
- workerVerdicts: NO_VERDICTS,
694
- deliverables: null
695
- };
696
- const traceCommand = `npx --yes @tangle-network/traces@latest analyze --harness claude-code --session ${sessionId}`;
697
- if (raw === null) return {
698
- runRef,
699
- instanceId: opts.instanceId ?? sessionId,
700
- arm: opts.arm ?? null,
701
- supRunDir: null,
702
- journal: null,
703
- brainLog: null,
704
- state: null,
705
- progress: null,
706
- workers: null,
707
- workersMissingReason: `session transcript unreadable at ${opts.transcriptPath}`,
708
- result: null,
709
- judge: null,
710
- judgeSource: null,
711
- patch: null,
712
- driverLog: null,
713
- harnessWorkerTokens: null,
714
- harnessMissingReason: `session transcript unreadable at ${opts.transcriptPath}`,
715
- limits: {
716
- ...limits,
717
- managerTokens: `session transcript unreadable at ${opts.transcriptPath}`,
718
- workerTokens: `session transcript unreadable at ${opts.transcriptPath}`,
719
- deliverables: NO_DELIVERABLES
720
- },
721
- traceCommand
722
- };
723
- const allEntries = parseClaudeEntries(raw);
724
- const main = threadCalls(allEntries.filter((e) => !e.isSidechain));
725
- const mainTranscript = transcriptFromEntries(allEntries);
726
- const subagentsDir = opts.subagentsDir === void 0 ? join(dirname(opts.transcriptPath), sessionId, "subagents") : opts.subagentsDir;
727
- const children = subagentsDir === null ? [] : await readChildren(subagentsDir);
728
- const childByAgentId = new Map(children.map((c) => [c.agentId, c]));
729
- const childBySpawnToolUseId = new Map(children.filter((c) => c.spawnToolUseId !== null).map((c) => [c.spawnToolUseId, c]));
730
- const threads = [{
731
- id: sessionId,
732
- calls: main
733
- }];
734
- for (const child of children) threads.push({
735
- id: child.agentId,
736
- calls: threadCalls(child.entries)
737
- });
738
- const spawns = [];
739
- const steersByTarget = /* @__PURE__ */ new Map();
740
- const cancels = [];
741
- const settles = [];
742
- for (const thread of threads) {
743
- for (const use of thread.calls.uses) {
744
- if (spawnTools.has(use.name)) {
745
- const result = thread.calls.results.get(use.id);
746
- const agentId = spawnedAgentId(result) ?? childBySpawnToolUseId.get(use.id)?.agentId ?? null;
747
- if (agentId === null) continue;
748
- const label = str(use.input.description) ?? childByAgentId.get(agentId)?.description ?? agentId;
749
- spawns.push({
750
- agentId,
751
- parentId: thread.id,
752
- label,
753
- at: result?.at ?? use.at,
754
- model: isRecord(result?.structured) ? str(result.structured.resolvedModel) : null
755
- });
756
- continue;
757
- }
758
- if (steerTools.has(use.name)) {
759
- const target = str(use.input.to) ?? str(use.input.recipient);
760
- if (target === null) continue;
761
- const result = thread.calls.results.get(use.id);
762
- const structured = isRecord(result?.structured) ? result.structured : null;
763
- const delivered = structured?.success === true || str(structured?.resumedAgentId) === target;
764
- const rows = steersByTarget.get(target) ?? [];
765
- rows.push({
766
- requestId: use.id,
767
- at: use.at,
768
- delivered
769
- });
770
- steersByTarget.set(target, rows);
771
- continue;
772
- }
773
- if (cancelTools.has(use.name)) {
774
- const target = str(use.input.agentId) ?? str(use.input.to) ?? str(use.input.taskId);
775
- if (target !== null) cancels.push({
776
- agentId: target,
777
- at: use.at
778
- });
779
- }
780
- }
781
- settles.push(...thread.calls.notifications);
782
- }
783
- const spawnedIds = new Set(spawns.map((s) => s.agentId));
784
- const cancelledIds = new Set(cancels.filter((c) => spawnedIds.has(c.agentId)).map((c) => c.agentId));
785
- const startedAt = main.firstAt;
786
- const completedAt = main.lastAt;
787
- const journalLines = [line({
788
- kind: "spawned",
789
- id: sessionId,
790
- parent: null,
791
- label: `session:${sessionId}`,
792
- role: "supervisor",
793
- at: startedAt
794
- })];
795
- for (const s of spawns) journalLines.push(line({
796
- kind: "spawned",
797
- id: s.agentId,
798
- parent: s.parentId,
799
- label: s.label,
800
- role: "worker",
801
- at: s.at
802
- }));
803
- const lastNotification = /* @__PURE__ */ new Map();
804
- for (const n of settles) {
805
- if (!spawnedIds.has(n.taskId)) continue;
806
- lastNotification.set(n.taskId, n);
807
- }
808
- for (const [agentId, n] of lastNotification) {
809
- if (cancelledIds.has(agentId)) continue;
810
- journalLines.push(line({
811
- kind: "settled",
812
- id: agentId,
813
- status: n.status,
814
- verdict: n.summary,
815
- at: n.at
816
- }));
817
- }
818
- const lastCancel = /* @__PURE__ */ new Map();
819
- for (const c of cancels) {
820
- if (!spawnedIds.has(c.agentId)) continue;
821
- lastCancel.set(c.agentId, c);
822
- }
823
- for (const c of lastCancel.values()) journalLines.push(line({
824
- kind: "cancelled",
825
- id: c.agentId,
826
- reason: "stopped by supervisor",
827
- at: c.at
828
- }));
829
- journalLines.push(line({
830
- kind: "metered",
831
- id: sessionId,
832
- spend: { tokens: {
833
- input: mainTranscript.usage.tokensIn,
834
- output: mainTranscript.usage.tokensOut,
835
- cacheRead: mainTranscript.usage.cacheRead,
836
- cacheWrite: mainTranscript.usage.cacheWrite
837
- } },
838
- at: completedAt
839
- }));
840
- const settleAtByAgent = new Map([...lastNotification.entries()].map(([id, n]) => [id, n.at]));
841
- const spawnAtByAgent = new Map(spawns.map((s) => [s.agentId, s.at]));
842
- const workers = [];
843
- for (const spawn of spawns) {
844
- const { agentId, label } = spawn;
845
- const events = [];
846
- const inbox = [];
847
- const child = childByAgentId.get(agentId) ?? null;
848
- const startAt = child?.firstAt ?? spawnAtByAgent.get(agentId) ?? null;
849
- const endAt = settleAtByAgent.get(agentId) ?? child?.lastAt ?? null;
850
- if (startAt !== null) events.push(line({
851
- kind: "started",
852
- label,
853
- at: startAt,
854
- agentId
855
- }));
856
- for (const steer of steersByTarget.get(agentId) ?? []) {
857
- inbox.push(line({
858
- id: steer.requestId,
859
- at: steer.at,
860
- worker: label,
861
- message: "steer"
862
- }));
863
- events.push(line({
864
- kind: "message",
865
- label,
866
- direction: "down",
867
- at: steer.at,
868
- requestId: steer.requestId,
869
- delivered: steer.delivered
870
- }));
871
- }
872
- if (endAt !== null) events.push(line({
873
- kind: "finished",
874
- label,
875
- at: endAt,
876
- agentId,
877
- ...child?.finalReport === null || child?.finalReport === void 0 ? {} : { evidence: child.finalReport }
878
- }));
879
- workers.push({
880
- workerId: agentId,
881
- label,
882
- events: events.length === 0 ? "" : `${events.join("\n")}\n`,
883
- inbox: inbox.length === 0 ? "" : `${inbox.join("\n")}\n`,
884
- patchBytes: null,
885
- transcriptRef: child?.path ?? null,
886
- patchPath: null,
887
- tokensIn: child?.tokensIn ?? null,
888
- tokensOut: child?.tokensOut ?? null,
889
- cacheRead: child?.cacheRead ?? null,
890
- cacheWrite: child?.cacheWrite ?? null
891
- });
892
- }
893
- const joined = children.length;
894
- const harnessWorkerTokens = joined === 0 ? null : {
895
- store: "claude-code subagent transcripts",
896
- sessions: joined,
897
- input: children.reduce((a, c) => a + c.tokensIn, 0),
898
- output: children.reduce((a, c) => a + c.tokensOut, 0),
899
- cacheRead: children.reduce((a, c) => a + c.cacheRead, 0),
900
- cacheWrite: children.reduce((a, c) => a + c.cacheWrite, 0)
901
- };
902
- const missingChildren = spawns.filter((s) => !childByAgentId.has(s.agentId)).length;
903
- const harnessMissingReason = subagentsDir === null ? "subagent transcript join disabled" : joined === 0 ? `no subagent transcripts under ${subagentsDir}` : missingChildren === 0 ? null : `${missingChildren}/${spawns.length} spawned agents have no retained transcript under ${subagentsDir} (Claude Code prunes them; their tokens are unrecoverable)`;
904
- const liveWorkers = spawns.filter((s) => !lastNotification.has(s.agentId) && !cancelledIds.has(s.agentId)).length;
905
- const state = JSON.stringify({
906
- id: sessionId,
907
- status: liveWorkers > 0 ? "running" : "idle",
908
- startedAt,
909
- completedAt,
910
- result: { delivered: null }
911
- });
912
- return {
913
- runRef,
914
- instanceId: opts.instanceId ?? sessionId,
915
- arm: opts.arm ?? null,
916
- supRunDir: subagentsDir,
917
- journal: `${journalLines.join("\n")}\n`,
918
- brainLog: null,
919
- state,
920
- progress: null,
921
- workers,
922
- workersMissingReason: null,
923
- result: null,
924
- judge: null,
925
- judgeSource: null,
926
- patch: null,
927
- driverLog: null,
928
- harnessWorkerTokens,
929
- harnessMissingReason,
930
- limits: {
931
- ...limits,
932
- workerTokens: spawns.length === 0 ? null : harnessMissingReason,
933
- deliverables: NO_DELIVERABLES
934
- },
935
- rootTranscriptRef: opts.transcriptPath,
936
- traceCommand
937
- };
938
- }
939
- /** A `SupervisorRunReader` over a Claude Code session — the same contract loops implements. */
940
- function claudeCodeSupervisorRunReader(opts) {
941
- return {
942
- runRef: opts.runRef ?? opts.transcriptPath,
943
- read: () => readClaudeCodeSupervisorRun(opts)
944
- };
945
- }
946
- //#endregion
947
- //#region src/supervisor-run/render.ts
948
- /**
949
- * Human-readable renderings of a supervisor-run report. Zero and unavailable
950
- * render differently on purpose (`0` vs `unavailable — <reason>`), because the
951
- * two have driven opposite conclusions about the same architecture.
952
- */
953
- /**
954
- * The block appended to a run log after every run — the answers an operator asks
955
- * for, in the log tail, with no extra command.
956
- */
957
- function renderSupervisorRunHeadline(r) {
958
- const o = r.orchestration;
959
- const steerNote = isUnavailable(o.steers) ? `unavailable — ${o.steers.unavailable}` : o.steers === 0 ? "0 (spawn→wait→respawn only; no mid-task steering)" : `${o.steers} queued / ${showMeasured(o.steersDelivered)} delivered`;
960
- return [
961
- `RUN-REPORT ${r.instanceId ?? "?"} [${r.arm ?? "?"}]`,
962
- ` steers=${steerNote}`,
963
- ` waves=${showMeasured(o.waves)} sizes=${isUnavailable(o.waveSizes) ? `unavailable — ${o.waveSizes.unavailable}` : `[${o.waveSizes.join(",")}]`} workers=${showMeasured(o.workersSpawned)} settled=${showMeasured(o.workersSettled)} cancelled=${showMeasured(o.workersCancelled)}`,
964
- ` concurrency max=${showMeasured(o.maxConcurrency)} utilization=${showMeasured(o.workerUtilization)} idle=${fmtMs(o.idleMs)} (${showMeasured(o.idlePct)}%) wall=${fmtMs(o.supervisorWallMs)}`,
965
- ` respawns=${showMeasured(o.respawns)} evidence→respawn=${showMeasured(r.decision.observeThenRespawn)} blind-respawn=${showMeasured(r.decision.respawnWithoutEvidence)} depth=${showMeasured(o.delegationDepth)}`,
966
- ` accepted=${showMeasured(r.decision.accepted)} rejected=${showMeasured(r.decision.rejected)} empty-pass=${showMeasured(r.decision.emptyPass)}`,
967
- ` brain=$${showMeasured(r.economics.brain.usd)} total=$${showMeasured(r.economics.totalUsd)} judge.resolved=${showMeasured(r.outcome.judgeResolved)} score=${showMeasured(r.outcome.judgeScore)} verify=${showMeasured(r.outcome.verifyPass)}`,
968
- r.gaps.length > 0 ? ` gaps(${r.gaps.length}): ${r.gaps.join("; ")}` : " gaps: none"
969
- ].join("\n");
970
- }
971
- function fmtMs(v) {
972
- if (isUnavailable(v)) return `unavailable — ${v.unavailable}`;
973
- if (v < 1e3) return `${v}ms`;
974
- const s = v / 1e3;
975
- if (s < 120) return `${round(s, 1)}s`;
976
- return `${round(s / 60, 1)}min`;
977
- }
978
- function renderSupervisorRunMarkdown(r) {
979
- const o = r.orchestration;
980
- const d = r.decision;
981
- const e = r.economics;
982
- const out = [];
983
- out.push(`# Run report — ${r.instanceId ?? "unknown instance"} [${r.arm ?? "unknown arm"}]`);
984
- out.push("");
985
- out.push("```");
986
- out.push(renderSupervisorRunHeadline(r));
987
- out.push("```");
988
- out.push("");
989
- out.push(`- Run: \`${r.runRef}\``);
990
- out.push(`- Supervisor: \`${showMeasured(r.supervisorId)}\``);
991
- out.push(`- Supervisor profile: \`${showMeasured(r.supervisorProfileDigest)}\``);
992
- out.push(`- Generated: ${r.generatedAt}`);
993
- out.push("");
994
- out.push("## Orchestration");
995
- out.push("");
996
- out.push("| Metric | Value |");
997
- out.push("|---|---|");
998
- out.push(`| Workers spawned | ${showMeasured(o.workersSpawned)} |`);
999
- out.push(`| Workers settled | ${showMeasured(o.workersSettled)} |`);
1000
- out.push(`| Workers cancelled | ${showMeasured(o.workersCancelled)} |`);
1001
- out.push(`| **Steers (mid-task messages to live workers)** | **${showMeasured(o.steers)}** |`);
1002
- out.push(`| Steers delivered | ${showMeasured(o.steersDelivered)} |`);
1003
- out.push(`| Outer-driver \`supervisor_steer\` calls | ${showMeasured(o.driverSteerCalls)} |`);
1004
- out.push(`| Spawn waves | ${showMeasured(o.waves)} |`);
1005
- out.push(`| Wave sizes | ${isUnavailable(o.waveSizes) ? showMeasured(o.waveSizes) : `[${o.waveSizes.join(", ")}]`} |`);
1006
- out.push(`| Max concurrency | ${showMeasured(o.maxConcurrency)} |`);
1007
- out.push(`| Respawns (after same parent's first settle) | ${showMeasured(o.respawns)} |`);
1008
- out.push(`| Repeated labels | ${isUnavailable(o.repeatedLabels) ? showMeasured(o.repeatedLabels) : o.repeatedLabels.length === 0 ? "none" : o.repeatedLabels.join(", ")} |`);
1009
- out.push(`| Delegation depth | ${showMeasured(o.delegationDepth)} |`);
1010
- out.push(`| Time to first spawn | ${fmtMs(o.timeToFirstSpawnMs)} |`);
1011
- out.push(`| Supervisor wall | ${fmtMs(o.supervisorWallMs)} |`);
1012
- out.push(`| Idle (zero live workers) | ${fmtMs(o.idleMs)} (${showMeasured(o.idlePct)}%) |`);
1013
- out.push(`| Worker utilization (Σ worker wall ÷ supervisor wall) | ${showMeasured(o.workerUtilization)} |`);
1014
- out.push("");
1015
- if (!isUnavailable(o.steersByWorker) && o.steersByWorker.length > 0) {
1016
- out.push("### Steers per worker");
1017
- out.push("");
1018
- out.push("| Worker id | Label | Queued | Delivered |");
1019
- out.push("|---|---|---:|---:|");
1020
- for (const s of o.steersByWorker) out.push(`| ${s.workerId === null ? "unavailable — legacy label join" : `\`${s.workerId}\``} | \`${s.worker}\` | ${s.queued} | ${s.delivered} |`);
1021
- out.push("");
1022
- } else if (isUnavailable(o.steersByWorker)) out.push(`### Steers per worker\n\nunavailable — ${o.steersByWorker.unavailable}\n`);
1023
- out.push("## Decision quality");
1024
- out.push("");
1025
- out.push("| Metric | Value |");
1026
- out.push("|---|---|");
1027
- out.push(`| Settled by status | ${fmtCounts(d.settledByStatus)} |`);
1028
- out.push(`| Settled verdicts | ${fmtCounts(d.settledVerdicts)} |`);
1029
- out.push(`| Accepted (verify green + patch bytes) | ${showMeasured(d.accepted)} |`);
1030
- out.push(`| Rejected (verify red) | ${showMeasured(d.rejected)} |`);
1031
- out.push(`| Empty pass (green, no patch) | ${showMeasured(d.emptyPass)} |`);
1032
- out.push(`| Evidence → respawn sequences | ${showMeasured(d.observeThenRespawn)} |`);
1033
- out.push(`| Respawn with no same-parent settled evidence in front | ${showMeasured(d.respawnWithoutEvidence)} |`);
1034
- out.push(`| Review actions (steers + worker questions) | ${showMeasured(d.reviewActions)} |`);
1035
- out.push(`| Worker evidence returned | ${isUnavailable(d.workerEvidenceBytes) ? showMeasured(d.workerEvidenceBytes) : `${d.workerEvidenceBytes} bytes`} |`);
1036
- out.push("");
1037
- out.push("## Economics");
1038
- out.push("");
1039
- out.push("| Role | Tokens in | Tokens out | Cache read | Cache write | USD | Source |");
1040
- out.push("|---|---:|---:|---:|---:|---:|---|");
1041
- out.push(`| brain | ${showMeasured(e.brain.tokensIn)} | ${showMeasured(e.brain.tokensOut)} | ${showMeasured(e.brain.cacheRead)} | ${showMeasured(e.brain.cacheWrite)} | ${showMeasured(e.brain.usd)} | ${e.brain.source} |`);
1042
- out.push(`| workers | ${showMeasured(e.workers.tokensIn)} | ${showMeasured(e.workers.tokensOut)} | ${showMeasured(e.workers.cacheRead)} | ${showMeasured(e.workers.cacheWrite)} | ${showMeasured(e.workers.usd)} | ${e.workers.source} |`);
1043
- out.push("");
1044
- if (!isUnavailable(e.brainTruncations) && e.brainTruncations > 0) out.push(`- **BRAIN OUTPUT TRUNCATED: ${e.brainTruncations} completion(s) hit \`finish_reason: "length"\`** — the supervisor acted on a half-written plan. Its output ceiling is too low; see \`brain.jsonl\` for the per-call \`req_max_tokens\`.`);
1045
- else out.push(`- Brain completions truncated (finish_reason=length): ${showMeasured(e.brainTruncations)}`);
1046
- out.push(`- Total USD: ${showMeasured(e.totalUsd)} (source: ${e.totalUsdSource})`);
1047
- out.push(`- Cost per accepted patch: ${showMeasured(e.costPerAcceptedPatchUsd)}`);
1048
- if (isUnavailable(e.workerWallMsDistribution)) out.push(`- Worker wall distribution: unavailable — ${e.workerWallMsDistribution.unavailable}`);
1049
- else {
1050
- const w = e.workerWallMsDistribution;
1051
- out.push(`- Worker wall (n=${w.n}): min ${fmtMs(w.min)} / p50 ${fmtMs(w.p50)} / p90 ${fmtMs(w.p90)} / max ${fmtMs(w.max)} / Σ ${fmtMs(w.sum)}`);
1052
- }
1053
- out.push("");
1054
- if (!isUnavailable(e.perWorker) && e.perWorker.length > 0) {
1055
- out.push("| Worker id | Label | Role | Runtime | Profile digest | Status | Failure | Infra | Wall | Tokens in | Tokens out | Patch bytes | Verify passed | Score |");
1056
- out.push("|---|---|---|---|---|---|---|---|---:|---:|---:|---:|---|---:|");
1057
- for (const w of e.perWorker) out.push(`| ${w.workerId === null ? "unavailable — legacy label join" : `\`${w.workerId}\``} | \`${w.worker}\` | ${w.role ?? "unavailable — source recorded no role"} | ${w.runtime ?? "unavailable — source recorded no runtime"} | ${w.profileDigest === null ? "unavailable — source recorded no profile digest" : `\`${w.profileDigest}\``} | ${w.status ?? "unavailable — no terminal event"} | ${w.failure ?? "none recorded"} | ${w.infra ?? "unavailable"} | ${w.wallMs === null ? "unavailable — no spawn/finish pair" : fmtMs(w.wallMs)} | ${w.tokensIn ?? "unavailable — store does not attribute tokens per worker"} | ${w.tokensOut ?? "unavailable — store does not attribute tokens per worker"} | ${w.patchBytes ?? "unavailable — no worker patch file"} | ${w.passed === null ? "unavailable — no verdict" : String(w.passed)} | ${w.score ?? "unavailable — no numeric score"} |`);
1058
- out.push("");
1059
- }
1060
- out.push("## Outcome");
1061
- out.push("");
1062
- out.push("| Metric | Value |");
1063
- out.push("|---|---|");
1064
- out.push(`| Supervisor status | ${showMeasured(r.outcome.supStatus)} |`);
1065
- out.push(`| Supervisor verdict | ${showMeasured(r.outcome.supVerdict)} |`);
1066
- out.push(`| Delivered | ${showMeasured(r.outcome.delivered)} |`);
1067
- out.push(`| Judge resolved | ${showMeasured(r.outcome.judgeResolved)} |`);
1068
- out.push(`| Judge score | ${showMeasured(r.outcome.judgeScore)} |`);
1069
- out.push(`| Judge passed / total | ${showMeasured(r.outcome.judgePassed)} / ${showMeasured(r.outcome.judgeTotal)} |`);
1070
- out.push(`| Judge source | ${r.outcome.judgeSource ?? "unavailable — no judge.json and no ledger row"} |`);
1071
- out.push(`| Verify gate | pass=${showMeasured(r.outcome.verifyPass)} rc=${showMeasured(r.outcome.verifyRc)} |`);
1072
- if (isUnavailable(r.outcome.patch)) out.push(`| Patch | unavailable — ${r.outcome.patch.unavailable} |`);
1073
- else {
1074
- const p = r.outcome.patch;
1075
- out.push(`| Patch | ${p.files} file(s), +${p.linesAdded}/-${p.linesRemoved}, test files touched: ${p.testFilesTouched.length === 0 ? "none" : p.testFilesTouched.join(", ")} |`);
1076
- }
1077
- out.push("");
1078
- out.push("## Gaps");
1079
- out.push("");
1080
- if (r.gaps.length === 0) out.push("None — every metric above is backed by a present artifact.");
1081
- else for (const g of r.gaps) out.push(`- ${g}`);
1082
- out.push("");
1083
- out.push(`> Harness-session view of the same run (model calls, stuck loops, tool errors): \`${r.traceCommand}\``);
1084
- out.push("");
1085
- return out.join("\n");
1086
- }
1087
- function fmtCounts(v) {
1088
- if (isUnavailable(v)) return showMeasured(v);
1089
- const entries = Object.entries(v);
1090
- return entries.length === 0 ? "none" : entries.map(([k, n]) => `${k}=${n}`).join(", ");
1091
- }
1092
- function renderSupervisorRollupMarkdown(rollup, title = "Round rollup") {
1093
- const out = [];
1094
- out.push(`# ${title}`);
1095
- out.push("");
1096
- out.push(`- Cells: ${rollup.cells}`);
1097
- out.push(`- **Steers across all cells: ${showMeasured(rollup.steersTotal)}** (cells with ≥1 steer: ${showMeasured(rollup.cellsWithSteers)}; cells where the steer count is unavailable: ${rollup.cellsWithUnavailableSteers})`);
1098
- out.push(`- Waves per cell (mean): ${showMeasured(rollup.wavesMean)}`);
1099
- out.push(`- Max concurrency observed: ${showMeasured(rollup.maxConcurrencyMax)}`);
1100
- out.push(`- Worker utilization (mean): ${showMeasured(rollup.utilizationMean)}`);
1101
- out.push(`- Idle share (mean): ${showMeasured(rollup.idlePctMean)}%`);
1102
- out.push(`- Workers spawned: ${showMeasured(rollup.workersSpawnedTotal)} · accepted: ${showMeasured(rollup.acceptedTotal)}`);
1103
- out.push(`- Spend: $${showMeasured(rollup.usdTotal)} · judged resolved: ${showMeasured(rollup.resolvedCount)}/${rollup.cells}`);
1104
- out.push("");
1105
- out.push("| Instance | Arm | Steers | Waves | Utilization | Idle % | Resolved | USD |");
1106
- out.push("|---|---|---:|---:|---:|---:|---|---:|");
1107
- for (const c of rollup.perCell) out.push(`| ${c.instanceId ?? "?"} | ${c.arm ?? "?"} | ${showMeasured(c.steers)} | ${showMeasured(c.waves)} | ${showMeasured(c.utilization)} | ${showMeasured(c.idlePct)} | ${showMeasured(c.resolved)} | ${showMeasured(c.usd)} |`);
1108
- out.push("");
1109
- return out.join("\n");
1110
- }
1111
- //#endregion
1112
- //#region src/supervisor-run/runtime-reader.ts
1113
- /**
1114
- * Reader for agent-runtime's file-backed supervision context.
1115
- *
1116
- * Runtime stores multiple recursive trees in one `spawn-journal.jsonl`.
1117
- * Each line is an envelope whose `root` identifies the local tree. A nested
1118
- * driver is represented twice: once as a child spawn in its parent's tree and
1119
- * once as the parentless root marker of its own tree. This reader removes only
1120
- * that duplicate root marker and preserves the remaining envelopes for the
1121
- * supervisor-run analyzer. Runtime stores profile identity below `identity`
1122
- * and does not emit Eval's role field, so this boundary projects those fields
1123
- * without changing Runtime's journal dialect.
1124
- */
1125
- const JOURNAL_FILE = "spawn-journal.jsonl";
1126
- const RESULT_FILE = "result.json";
1127
- const TRAJECTORY_FILE = "trajectory.json";
1128
- async function readMaybe$1(path) {
1129
- return readFile(path, "utf8").catch((error) => {
1130
- if (typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT") return null;
1131
- throw error;
1132
- });
1133
- }
1134
- function record(value) {
1135
- return typeof value === "object" && value !== null && !Array.isArray(value) ? value : null;
1136
- }
1137
- function nonEmptyString(value) {
1138
- return typeof value === "string" && value.length > 0 ? value : null;
1139
- }
1140
- function profileDigest(event) {
1141
- const direct = nonEmptyString(event.profileDigest);
1142
- if (direct !== null) return direct;
1143
- const identity = record(event.identity);
1144
- return identity === null ? null : nonEmptyString(identity.profileDigest);
1145
- }
1146
- function formatError(path, line, detail) {
1147
- return /* @__PURE__ */ new Error(`${path}:${line}: invalid Runtime spawn journal: ${detail}`);
1148
- }
1149
- function parseEnvelopeJournal(text, path) {
1150
- const begins = [];
1151
- const events = [];
1152
- const begun = /* @__PURE__ */ new Map();
1153
- for (const [index, sourceLine] of text.split("\n").entries()) {
1154
- const line = index + 1;
1155
- const trimmed = sourceLine.trim();
1156
- if (trimmed.length === 0) continue;
1157
- let parsed;
1158
- try {
1159
- parsed = JSON.parse(trimmed);
1160
- } catch {
1161
- throw formatError(path, line, "line is not JSON");
1162
- }
1163
- const envelope = record(parsed);
1164
- if (envelope === null) throw formatError(path, line, "line is not an object");
1165
- const kind = nonEmptyString(envelope.kind);
1166
- const root = nonEmptyString(envelope.root);
1167
- if (root === null) throw formatError(path, line, "root must be a non-empty string");
1168
- if (kind === "begin") {
1169
- const at = nonEmptyString(envelope.at);
1170
- if (at === null || !Number.isFinite(Date.parse(at))) throw formatError(path, line, "begin.at must be an ISO timestamp");
1171
- if (begun.has(root)) throw formatError(path, line, `tree ${JSON.stringify(root)} began twice`);
1172
- const begin = {
1173
- root,
1174
- at,
1175
- line
1176
- };
1177
- begun.set(root, begin);
1178
- begins.push(begin);
1179
- continue;
1180
- }
1181
- if (kind !== "event") throw formatError(path, line, "kind must be 'begin' or 'event'");
1182
- if (!begun.has(root)) throw formatError(path, line, `event for tree ${JSON.stringify(root)} precedes begin`);
1183
- const event = record(envelope.event);
1184
- if (event === null) throw formatError(path, line, "event must be an object");
1185
- if (nonEmptyString(event.kind) === null) throw formatError(path, line, "event.kind must be a non-empty string");
1186
- events.push({
1187
- root,
1188
- event: { ...event },
1189
- line
1190
- });
1191
- }
1192
- if (begins.length === 0) throw formatError(path, 1, "no begin record");
1193
- const parentSpawnsById = /* @__PURE__ */ new Map();
1194
- const rootMarkersByTree = /* @__PURE__ */ new Map();
1195
- for (const entry of events) {
1196
- if (entry.event.kind !== "spawned") continue;
1197
- const id = nonEmptyString(entry.event.id);
1198
- if (id === null) continue;
1199
- if (nonEmptyString(entry.event.parent) !== null) {
1200
- const matches = parentSpawnsById.get(id) ?? [];
1201
- matches.push(entry);
1202
- parentSpawnsById.set(id, matches);
1203
- }
1204
- if (entry.root === id && entry.event.parent === void 0) {
1205
- const markers = rootMarkersByTree.get(entry.root) ?? [];
1206
- markers.push(entry);
1207
- rootMarkersByTree.set(entry.root, markers);
1208
- }
1209
- }
1210
- const nestedRoots = /* @__PURE__ */ new Set();
1211
- const nestedParentSpawns = /* @__PURE__ */ new Map();
1212
- for (const begin of begins) {
1213
- const parentSpawns = (parentSpawnsById.get(begin.root) ?? []).filter((entry) => entry.root !== begin.root);
1214
- if (parentSpawns.length > 1) throw formatError(path, begin.line, `tree ${JSON.stringify(begin.root)} has ${parentSpawns.length} parent spawns`);
1215
- if (parentSpawns.length === 1) {
1216
- nestedRoots.add(begin.root);
1217
- nestedParentSpawns.set(begin.root, parentSpawns[0]);
1218
- }
1219
- }
1220
- const topRoots = begins.filter((begin) => !nestedRoots.has(begin.root));
1221
- if (topRoots.length !== 1) throw formatError(path, topRoots[0]?.line ?? 1, `expected one top-level tree, found ${topRoots.length}`);
1222
- const top = topRoots[0];
1223
- for (const nestedRoot of nestedRoots) {
1224
- const marker = rootMarkersByTree.get(nestedRoot) ?? [];
1225
- if (marker.length !== 1) throw formatError(path, begun.get(nestedRoot)?.line ?? 1, `nested tree ${JSON.stringify(nestedRoot)} must contain one root marker`);
1226
- const parentSpawn = nestedParentSpawns.get(nestedRoot);
1227
- if (parentSpawn === void 0) throw formatError(path, begun.get(nestedRoot)?.line ?? 1, `nested tree ${JSON.stringify(nestedRoot)} has no parent spawn`);
1228
- const markerDigest = profileDigest(marker[0]?.event ?? {});
1229
- const parentDigest = profileDigest(parentSpawn.event);
1230
- if (markerDigest !== null && parentDigest !== null && markerDigest !== parentDigest) throw formatError(path, marker[0]?.line ?? 1, `nested tree ${JSON.stringify(nestedRoot)} disagrees with its parent profile digest`);
1231
- if (parentDigest === null && markerDigest !== null) parentSpawn.event.profileDigest = markerDigest;
1232
- }
1233
- const supervisorIds = /* @__PURE__ */ new Set([top.root, ...nestedRoots]);
1234
- const normalized = events.filter((entry) => !(nestedRoots.has(entry.root) && entry.event.kind === "spawned" && entry.event.id === entry.root && entry.event.parent === void 0)).map((entry) => {
1235
- const event = { ...entry.event };
1236
- if (event.kind === "spawned") {
1237
- const digest = profileDigest(event);
1238
- if (event.profileDigest === void 0 && digest !== null) event.profileDigest = digest;
1239
- if (event.role === void 0) event.role = supervisorIds.has(nonEmptyString(event.id) ?? "") ? "supervisor" : "worker";
1240
- }
1241
- return {
1242
- root: entry.root,
1243
- event
1244
- };
1245
- });
1246
- if ((rootMarkersByTree.get(top.root) ?? []).length !== 1) throw formatError(path, top.line, `top-level tree ${JSON.stringify(top.root)} must contain one root marker`);
1247
- return {
1248
- root: top.root,
1249
- startedAt: top.at,
1250
- journal: `${normalized.map((entry) => JSON.stringify({
1251
- kind: "event",
1252
- root: entry.root,
1253
- event: entry.event
1254
- })).join("\n")}\n`,
1255
- events: normalized.map((entry) => entry.event)
1256
- };
1257
- }
1258
- function parseOptionalRecord(text, path) {
1259
- if (text === null) return null;
1260
- let parsed;
1261
- try {
1262
- parsed = JSON.parse(text);
1263
- } catch {
1264
- throw new Error(`${path}: invalid JSON`);
1265
- }
1266
- const value = record(parsed);
1267
- if (value === null) throw new Error(`${path}: expected a JSON object`);
1268
- return value;
1269
- }
1270
- function spendRecord(value) {
1271
- const spend = record(value);
1272
- if (spend === null) return null;
1273
- const tokens = record(spend.tokens);
1274
- if (tokens === null || typeof tokens.input !== "number" || !Number.isFinite(tokens.input) || tokens.input < 0 || typeof tokens.output !== "number" || !Number.isFinite(tokens.output) || tokens.output < 0 || typeof spend.usd !== "number" || !Number.isFinite(spend.usd) || spend.usd < 0 || spend.usdKnown !== void 0 && typeof spend.usdKnown !== "boolean") return null;
1275
- return spend;
1276
- }
1277
- function sourceLimits(root, events, workerIds) {
1278
- const rootMeters = events.filter((event) => event.kind === "metered" && event.id === root);
1279
- const invalidRootMeters = rootMeters.filter((event) => spendRecord(event.spend) === null);
1280
- const rootMeterReason = rootMeters.length === 0 ? "Runtime journal has no root metered event" : invalidRootMeters.length > 0 ? `${invalidRootMeters.length} root metered event(s) lack complete spend` : null;
1281
- const closes = events.filter((event) => workerIds.has(nonEmptyString(event.id) ?? "") && (event.kind === "settled" || event.kind === "cancelled"));
1282
- const settledById = /* @__PURE__ */ new Map();
1283
- for (const event of closes) {
1284
- const id = nonEmptyString(event.id);
1285
- if (id === null) continue;
1286
- const matches = settledById.get(id) ?? [];
1287
- matches.push(event);
1288
- settledById.set(id, matches);
1289
- }
1290
- const incompleteWorkers = [...workerIds].filter((id) => {
1291
- const terminal = settledById.get(id);
1292
- return terminal?.length !== 1 || terminal[0]?.kind !== "settled" || spendRecord(terminal[0]?.spent) === null;
1293
- });
1294
- const unknownUsd = [...rootMeters.map((event) => spendRecord(event.spend)), ...closes.filter((event) => event.kind === "settled").map((event) => spendRecord(event.spent))].filter((spend) => spend !== null).some((spend) => spend.usdKnown === false);
1295
- const missingVerdicts = [...workerIds].filter((id) => {
1296
- const terminal = settledById.get(id)?.[0];
1297
- if (terminal?.kind !== "settled") return true;
1298
- return typeof record(terminal.verdict)?.valid !== "boolean";
1299
- });
1300
- return {
1301
- managerTokens: rootMeterReason,
1302
- workerTokens: incompleteWorkers.length === 0 ? null : `${incompleteWorkers.length}/${workerIds.size} child invocation(s) lack one settled spend record`,
1303
- spendUsd: rootMeterReason !== null ? rootMeterReason : unknownUsd ? "Runtime recorded usdKnown:false for at least one spend" : incompleteWorkers.length > 0 ? "at least one child invocation lacks a settled spend record" : null,
1304
- workerVerdicts: missingVerdicts.length === 0 ? null : `${missingVerdicts.length}/${workerIds.size} child invocation(s) lack a structured validity verdict`,
1305
- deliverables: workerIds.size === 0 ? null : "Runtime FileRunContext does not retain per-child delivered patches"
1306
- };
1307
- }
1308
- function runtimeState(root, startedAt, result, trajectory, resultPath, trajectoryPath) {
1309
- const resultKind = result?.kind === "completed" || result?.kind === "interrupted" ? result.kind : null;
1310
- if (resultKind !== null && result !== null) {
1311
- const resultRoot = nonEmptyString(record(result.tree)?.root);
1312
- if (resultRoot === null) throw new Error(`${resultPath}: Runtime ${resultKind} result has no tree.root`);
1313
- if (resultRoot !== root) throw new Error(`${resultPath}: root ${JSON.stringify(resultRoot)} does not match journal root ${JSON.stringify(root)}`);
1314
- }
1315
- if (trajectory !== null && nonEmptyString(trajectory.root) === null) throw new Error(`${trajectoryPath}: Runtime trajectory has no root`);
1316
- if (typeof trajectory?.root === "string" && trajectory.root !== root) throw new Error(`${trajectoryPath}: root ${JSON.stringify(trajectory.root)} does not match journal root ${JSON.stringify(root)}`);
1317
- if (trajectory !== null && !Array.isArray(trajectory.nodes)) throw new Error(`${trajectoryPath}: Runtime trajectory nodes must be an array`);
1318
- let status = null;
1319
- if (resultKind === "completed") status = "completed";
1320
- else if (resultKind === "interrupted") status = "interrupted";
1321
- else if (Array.isArray(trajectory?.nodes)) {
1322
- const rootNodes = trajectory.nodes.map((node) => record(node)).filter((node) => node?.id === root);
1323
- if (rootNodes.length > 1) throw new Error(`${trajectoryPath}: Runtime trajectory contains duplicate root nodes`);
1324
- status = nonEmptyString(rootNodes[0]?.status);
1325
- }
1326
- return JSON.stringify({
1327
- id: root,
1328
- startedAt,
1329
- ...status === null ? {} : { status }
1330
- });
1331
- }
1332
- /**
1333
- * Read one agent-runtime `createFileRunContext(dir)` directory.
1334
- *
1335
- * The reader translates storage envelopes only. It does not assign research
1336
- * roles, interpret artifacts, or turn process completion into a quality
1337
- * verdict.
1338
- */
1339
- async function readRuntimeSupervisorRun(runDir) {
1340
- const journalPath = join(runDir, JOURNAL_FILE);
1341
- const normalized = parseEnvelopeJournal(await readFile(journalPath, "utf8"), journalPath);
1342
- const resultText = await readMaybe$1(join(runDir, RESULT_FILE));
1343
- const trajectoryText = await readMaybe$1(join(runDir, TRAJECTORY_FILE));
1344
- const result = parseOptionalRecord(resultText, join(runDir, RESULT_FILE));
1345
- const trajectory = parseOptionalRecord(trajectoryText, join(runDir, TRAJECTORY_FILE));
1346
- const resultPath = join(runDir, RESULT_FILE);
1347
- const trajectoryPath = join(runDir, TRAJECTORY_FILE);
1348
- const childSpawns = normalized.events.filter((event) => event.kind === "spawned" && nonEmptyString(event.id) !== null).filter((event) => event.id !== normalized.root);
1349
- const workerIds = new Set(childSpawns.map((event) => nonEmptyString(event.id)).filter((id) => id !== null));
1350
- const workers = childSpawns.map((event) => ({
1351
- workerId: nonEmptyString(event.id),
1352
- label: nonEmptyString(event.label) ?? String(event.id),
1353
- events: null,
1354
- inbox: null,
1355
- patchBytes: null,
1356
- transcriptRef: null,
1357
- patchPath: null
1358
- }));
1359
- return {
1360
- runRef: runDir,
1361
- instanceId: normalized.root,
1362
- arm: null,
1363
- supRunDir: runDir,
1364
- journal: normalized.journal,
1365
- brainLog: null,
1366
- brainLogMissingReason: "Runtime FileRunContext records spend but not model completion finish reasons",
1367
- state: runtimeState(normalized.root, normalized.startedAt, result, trajectory, resultPath, trajectoryPath),
1368
- progress: null,
1369
- workers,
1370
- workersMissingReason: null,
1371
- result: resultText,
1372
- judge: null,
1373
- judgeSource: null,
1374
- patch: null,
1375
- driverLog: null,
1376
- harnessWorkerTokens: null,
1377
- harnessMissingReason: "Runtime FileRunContext has no external worker-token join",
1378
- limits: sourceLimits(normalized.root, normalized.events, workerIds),
1379
- rootTranscriptRef: null,
1380
- traceCommand: "unavailable — Runtime FileRunContext records no provider-session trace identity"
1381
- };
1382
- }
1383
- /** The agent-runtime file-backed layout as a `SupervisorRunReader`. */
1384
- function runtimeSupervisorRunReader(runDir) {
1385
- return {
1386
- runRef: runDir,
1387
- read: () => readRuntimeSupervisorRun(runDir)
1388
- };
1389
- }
1390
- /** True when a directory contains Runtime's canonical file-backed journal. */
1391
- async function isRuntimeSupervisorRunDir(runDir) {
1392
- return stat(join(runDir, JOURNAL_FILE)).then((entry) => entry.isFile()).catch((error) => {
1393
- if (typeof error === "object" && error !== null && "code" in error && (error.code === "ENOENT" || error.code === "ENOTDIR")) return false;
1394
- throw error;
1395
- });
1396
- }
1397
- //#endregion
1398
- //#region src/supervisor-run/loops-reader.ts
1399
- /**
1400
- * ONE implementation of `SupervisorRunReader`: the on-disk layout the loops
1401
- * supervisor writes — `<runDir>/ws/.agent/supervisor/<id>/{journal.jsonl,
1402
- * state.json, progress.ndjson, workers/*.ndjson}` alongside the run's
1403
- * `result.json` / `judge.json` / `driver.log` / delivered patch. Runs written
1404
- * before the `.agent` rename live under `<ws>/.loops/supervisor/<id>` and are
1405
- * still found via fallback.
1406
- *
1407
- * Nothing in `analyze.ts` knows this layout exists. A different store (an
1408
- * archive, an object bucket, a database) implements the same interface and
1409
- * gets the same report.
1410
- *
1411
- * Worker token recovery reuses the rollout module's opencode reader rather
1412
- * than opening a second sqlite path — one store client, one corruption policy.
1413
- */
1414
- async function readMaybe(path) {
1415
- return readFile(path, "utf8").catch(() => null);
1416
- }
1417
- /**
1418
- * Locate the (single) supervisor run dir under `<ws>/.agent/supervisor`, falling back to the
1419
- * pre-rename `<ws>/.loops/supervisor` so runs written by older supervisors stay analyzable.
1420
- */
1421
- async function findSupervisorRunDirIn(ws) {
1422
- for (const stateDir of [".agent", ".loops"]) {
1423
- const root = join(ws, stateDir, "supervisor");
1424
- const dirs = (await readdir(root, { withFileTypes: true }).catch(() => [])).filter((e) => e.isDirectory()).map((e) => join(root, e.name));
1425
- if (dirs[0] !== void 0) return dirs[0];
1426
- }
1427
- return null;
1428
- }
1429
- /**
1430
- * Read a loops supervisor run directory into source bytes. Never throws on a
1431
- * missing artifact — an absent file becomes a `null` field, which is what makes
1432
- * the dependent metric `unavailable` instead of 0.
1433
- */
1434
- async function readLoopsSupervisorRun(runDir, opts = {}) {
1435
- const ws = opts.ws ?? join(runDir, "ws");
1436
- const supRunDir = await findSupervisorRunDirIn(ws);
1437
- const result = await readMaybe(join(runDir, "result.json"));
1438
- const resultObj = parseJson(result);
1439
- const journal = supRunDir === null ? null : await readMaybe(join(supRunDir, "journal.jsonl"));
1440
- const journalWorkerSpawns = parseJsonl(journal).filter((event) => event.kind === "spawned" && typeof event.parent === "string" && event.role !== "supervisor").length;
1441
- let workers = null;
1442
- let workersMissingReason = null;
1443
- const workerCwds = [];
1444
- let workerStarts = 0;
1445
- if (supRunDir === null) workersMissingReason = `no supervisor run dir under ${join(ws, ".agent", "supervisor")} (or legacy ${join(ws, ".loops", "supervisor")})`;
1446
- else {
1447
- const workersDir = join(supRunDir, "workers");
1448
- const entries = await readdir(workersDir).catch(() => null);
1449
- if (entries === null) workersMissingReason = `workers/ directory absent under ${supRunDir}`;
1450
- else {
1451
- const labels = [...new Set(entries.filter((f) => f.endsWith(".ndjson")).map((f) => f.replace(/\.inbox\.ndjson$/, "").replace(/\.ndjson$/, "")))].sort();
1452
- workers = [];
1453
- for (const label of labels) {
1454
- const events = await readMaybe(join(workersDir, `${label}.ndjson`));
1455
- const inbox = await readMaybe(join(workersDir, `${label}.inbox.ndjson`));
1456
- const patch = await readMaybe(join(workersDir, `${label}.patch`));
1457
- const startedRows = parseJsonl(events).filter((event) => event.kind === "started");
1458
- workerStarts += startedRows.length;
1459
- const startedIds = startedRows.map((event) => typeof event.workerId === "string" ? event.workerId : typeof event.agentId === "string" ? event.agentId : null).filter((id) => id !== null);
1460
- const distinctStartedIds = new Set(startedIds);
1461
- const workerId = startedRows.length > 0 && startedIds.length === startedRows.length && distinctStartedIds.size === 1 ? startedIds[0] : void 0;
1462
- workers.push({
1463
- ...workerId === void 0 ? {} : { workerId },
1464
- label,
1465
- events,
1466
- inbox,
1467
- patchBytes: patch === null ? null : Buffer.byteLength(patch)
1468
- });
1469
- for (const ev of startedRows) if (ev.kind === "started" && typeof ev.cwd === "string") workerCwds.push(ev.cwd);
1470
- }
1471
- }
1472
- }
1473
- let harnessWorkerTokens = null;
1474
- let harnessMissingReason = null;
1475
- let workerCwdsWithoutSessions = 0;
1476
- if (opts.opencodeDb === null) harnessMissingReason = "opencode join disabled";
1477
- else if (workerCwds.length === 0) harnessMissingReason = "no worker clone cwds in workers/*.ndjson (nothing to join)";
1478
- else {
1479
- const db = await openOpencodeDb(opts.opencodeDb ?? DEFAULT_OPENCODE_DB);
1480
- if (db === null) harnessMissingReason = `opencode session store unreadable at ${opts.opencodeDb ?? DEFAULT_OPENCODE_DB}`;
1481
- else try {
1482
- const seen = /* @__PURE__ */ new Set();
1483
- let sessions = 0;
1484
- let input = 0;
1485
- let output = 0;
1486
- const distinctWorkerCwds = new Set(workerCwds);
1487
- for (const cwd of distinctWorkerCwds) {
1488
- const rows = findOpencodeSessionsByDirectory(db, cwd);
1489
- if (rows.length === 0) workerCwdsWithoutSessions += 1;
1490
- for (const row of rows) {
1491
- if (seen.has(row.id)) continue;
1492
- seen.add(row.id);
1493
- sessions += 1;
1494
- input += row.tokensInput;
1495
- output += row.tokensOutput + row.tokensReasoning;
1496
- }
1497
- }
1498
- harnessWorkerTokens = {
1499
- store: "opencode",
1500
- sessions,
1501
- input,
1502
- output
1503
- };
1504
- if (workerCwdsWithoutSessions > 0) harnessMissingReason = `${workerCwdsWithoutSessions}/${distinctWorkerCwds.size} worker clone cwds have no opencode session`;
1505
- } finally {
1506
- db.close();
1507
- }
1508
- }
1509
- const workerInvocations = Math.max(journalWorkerSpawns, workerStarts);
1510
- const workerTokenGaps = [];
1511
- if (workerCwds.length < workerInvocations) workerTokenGaps.push(`${workerInvocations - workerCwds.length}/${workerInvocations} worker invocations have no clone cwd for the opencode token join`);
1512
- if (workerInvocations > 0 && harnessWorkerTokens === null) workerTokenGaps.push(harnessMissingReason ?? "worker harness token join unavailable");
1513
- else if (workerCwdsWithoutSessions > 0 && harnessMissingReason !== null) workerTokenGaps.push(harnessMissingReason);
1514
- const workerTokenLimit = workerTokenGaps.length === 0 ? null : workerTokenGaps.join("; ");
1515
- const patchPath = opts.patchPath ?? (typeof resultObj?.patchPath === "string" ? resultObj.patchPath : null);
1516
- let judge = await readMaybe(join(runDir, "judge.json"));
1517
- let judgeSource = judge === null ? null : join(runDir, "judge.json");
1518
- if (judge === null && opts.ledgerPath !== void 0) {
1519
- const row = await findLedgerRow(opts.ledgerPath, runDir);
1520
- if (row !== null) {
1521
- judge = JSON.stringify(row);
1522
- judgeSource = `${opts.ledgerPath} (ledger row)`;
1523
- }
1524
- }
1525
- return {
1526
- runRef: runDir,
1527
- instanceId: typeof resultObj?.iid === "string" ? resultObj.iid : instanceIdFromPath(runDir),
1528
- arm: typeof resultObj?.arm === "string" ? resultObj.arm : basename(runDir),
1529
- supRunDir,
1530
- journal,
1531
- brainLog: supRunDir === null ? null : await readMaybe(join(supRunDir, "brain.jsonl")),
1532
- state: supRunDir === null ? null : await readMaybe(join(supRunDir, "state.json")),
1533
- progress: supRunDir === null ? null : await readMaybe(join(supRunDir, "progress.ndjson")),
1534
- workers,
1535
- workersMissingReason,
1536
- result,
1537
- judge,
1538
- judgeSource,
1539
- patch: patchPath === null ? null : await readMaybe(patchPath),
1540
- driverLog: await readMaybe(join(runDir, "driver.log")),
1541
- harnessWorkerTokens,
1542
- harnessMissingReason,
1543
- limits: {
1544
- ...NO_SOURCE_LIMITS,
1545
- workerTokens: workerTokenLimit
1546
- },
1547
- traceCommand: null
1548
- };
1549
- }
1550
- /** The loops on-disk layout, as a `SupervisorRunReader`. */
1551
- function loopsSupervisorRunReader(runDir, opts = {}) {
1552
- return {
1553
- runRef: runDir,
1554
- read: () => readLoopsSupervisorRun(runDir, opts)
1555
- };
1556
- }
1557
- /** The ledger row whose `runDir` is this run (falling back to iid + arm match). */
1558
- async function findLedgerRow(ledgerPath, runDir) {
1559
- const rows = parseJsonl(await readMaybe(ledgerPath));
1560
- const exact = rows.find((r) => r.runDir === runDir);
1561
- if (exact !== void 0) return exact;
1562
- const iid = instanceIdFromPath(runDir);
1563
- const arm = basename(runDir);
1564
- return rows.find((r) => r.iid === iid && r.arm === arm) ?? null;
1565
- }
1566
- /** `<outDir>/runs/<iid>/<arm>` → `<iid>`. */
1567
- function instanceIdFromPath(runDir) {
1568
- const parts = runDir.split("/").filter(Boolean);
1569
- const armIdx = parts.length - 1;
1570
- const iid = parts[armIdx - 1];
1571
- return parts[armIdx - 2] === "runs" && iid !== void 0 ? iid : null;
1572
- }
1573
- /**
1574
- * Analyze a supervisor run. Accepts a run directory (read through the loops
1575
- * reader), any `SupervisorRunReader`, or already-read source bytes — so a
1576
- * caller with its own store never has to touch the filesystem layout.
1577
- */
1578
- async function analyzeSupervisorRun(input, opts = {}) {
1579
- if (typeof input === "string") return analyzeSupervisorRunSources(await isRuntimeSupervisorRunDir(input) ? await readRuntimeSupervisorRun(input) : await readLoopsSupervisorRun(input, opts));
1580
- if (isReader(input)) return analyzeSupervisorRunSources(await input.read());
1581
- return analyzeSupervisorRunSources(input);
1582
- }
1583
- function isReader(input) {
1584
- return typeof input.read === "function";
1585
- }
1586
- /**
1587
- * Read a completed run, write `run-report.json` + `run-report.md` beside its
1588
- * artifacts, and append the headline block to the run log. Never throws on a
1589
- * missing artifact — a run that produced nothing still yields a report whose
1590
- * every metric says why.
1591
- */
1592
- async function writeSupervisorRunReport(runDir, opts = {}) {
1593
- const report = analyzeSupervisorRunSources(await isRuntimeSupervisorRunDir(runDir) ? await readRuntimeSupervisorRun(runDir) : await readLoopsSupervisorRun(runDir, opts));
1594
- const md = renderSupervisorRunMarkdown(report);
1595
- const dest = opts.reportDir ?? runDir;
1596
- const stem = opts.reportDir === void 0 ? "run-report" : supervisorReportStem(runDir);
1597
- if (opts.reportDir !== void 0) await mkdir(opts.reportDir, { recursive: true }).catch(() => {});
1598
- await writeFile(join(dest, `${stem}.json`), JSON.stringify(report, null, 1)).catch(() => {});
1599
- await writeFile(join(dest, `${stem}.md`), md).catch(() => {});
1600
- const headline = renderSupervisorRunHeadline(report);
1601
- if (opts.appendHeadlineTo !== void 0) await appendFile(opts.appendHeadlineTo, `${headline}\n`).catch(() => {});
1602
- if (opts.echo !== false) console.log(headline);
1603
- return report;
1604
- }
1605
- /**
1606
- * File stem for out-of-tree reports. Built from the run path's identifying
1607
- * segments — candidate tag (the segment under `arm-runs/`), rep, instance, arm
1608
- * — so two runs of the same instance from different candidates/reps never
1609
- * overwrite each other.
1610
- */
1611
- function supervisorReportStem(runDir) {
1612
- const parts = runDir.split("/").filter(Boolean);
1613
- const arm = parts[parts.length - 1] ?? "cell";
1614
- const iid = parts[parts.length - 2] ?? "instance";
1615
- const rep = parts.find((p) => /^rep-\d+$/.test(p));
1616
- const armRunsIdx = parts.indexOf("arm-runs");
1617
- return [
1618
- armRunsIdx >= 0 ? parts[armRunsIdx + 1] : void 0,
1619
- rep,
1620
- iid,
1621
- arm
1622
- ].filter((s) => s !== void 0 && s !== "runs").join(".").replace(/[^A-Za-z0-9._-]/g, "_");
1623
- }
1624
- /**
1625
- * Best-effort wrapper for a hot path: a reporting failure must never kill a run
1626
- * that already produced real work. Returns null and logs the reason instead.
1627
- */
1628
- async function writeSupervisorRunReportSafe(runDir, opts = {}) {
1629
- try {
1630
- return await writeSupervisorRunReport(runDir, opts);
1631
- } catch (err) {
1632
- console.log(`RUN-REPORT failed for ${runDir}: ${err instanceof Error ? err.message : String(err)}`);
1633
- return null;
1634
- }
1635
- }
1636
- /**
1637
- * Report every run under an experiment `outDir` (any depth of
1638
- * `runs/<iid>/<arm>`), write each run's report, and write the rollup at
1639
- * `<outDir>/run-report-round.{json,md}`.
1640
- */
1641
- async function reportSupervisorRound(outDir, opts = {}) {
1642
- const runDirs = await findSupervisorRunDirs(outDir);
1643
- const reports = [];
1644
- for (const runDir of runDirs) {
1645
- const r = await writeSupervisorRunReportSafe(runDir, {
1646
- ...opts,
1647
- echo: opts.echo ?? false
1648
- });
1649
- if (r !== null) reports.push(r);
1650
- }
1651
- const rollup = rollupSupervisorRuns(reports);
1652
- const md = renderSupervisorRollupMarkdown(rollup, opts.title ?? `Round rollup — ${basename(outDir)}`);
1653
- const dest = opts.reportDir ?? outDir;
1654
- if (opts.reportDir !== void 0) await mkdir(opts.reportDir, { recursive: true }).catch(() => {});
1655
- await writeFile(join(dest, "run-report-round.json"), JSON.stringify(rollup, null, 1)).catch(() => {});
1656
- await writeFile(join(dest, "run-report-round.md"), md).catch(() => {});
1657
- if (opts.appendHeadlineTo !== void 0) await appendFile(opts.appendHeadlineTo, `${md}\n`).catch(() => {});
1658
- if (opts.echo !== false) console.log(md);
1659
- return rollup;
1660
- }
1661
- /**
1662
- * Every loops or Runtime supervisor run below `root`.
1663
- *
1664
- * When `root` itself is one run, return no children so callers can distinguish
1665
- * a single report from a parent-directory rollup.
1666
- */
1667
- async function findSupervisorRunDirs(root) {
1668
- if (await isRuntimeSupervisorRunDir(root) || await findSupervisorRunDirIn(join(root, "ws")) !== null) return [];
1669
- const found = [];
1670
- const walk = async (dir, depth) => {
1671
- if (depth > 8) return;
1672
- const entries = await readdir(dir, { withFileTypes: true }).catch(() => []);
1673
- for (const e of entries) {
1674
- if (!e.isDirectory()) continue;
1675
- if (e.name === "node_modules" || e.name === ".git") continue;
1676
- const full = join(dir, e.name);
1677
- if (await isRuntimeSupervisorRunDir(full) || await findSupervisorRunDirIn(join(full, "ws")) !== null) {
1678
- found.push(full);
1679
- continue;
1680
- }
1681
- await walk(full, depth + 1);
1682
- }
1683
- };
1684
- await walk(root, 0);
1685
- return found.sort();
1686
- }
1687
- //#endregion
1688
- export { rollupSupervisorRuns as C, isUnavailable as D, SUPERVISOR_RUN_SCHEMA as E, showMeasured as O, parsePatch as S, SUPERVISOR_RUN_ROLLUP_SCHEMA as T, DEFAULT_SPAWN_TOOLS as _, readLoopsSupervisorRun as a, readClaudeCodeSupervisorRun as b, writeSupervisorRunReport as c, readRuntimeSupervisorRun as d, runtimeSupervisorRunReader as f, DEFAULT_CANCEL_TOOLS as g, renderSupervisorRunMarkdown as h, loopsSupervisorRunReader as i, unavailable as k, writeSupervisorRunReportSafe as l, renderSupervisorRunHeadline as m, findSupervisorRunDirIn as n, reportSupervisorRound as o, renderSupervisorRollupMarkdown as p, findSupervisorRunDirs as r, supervisorReportStem as s, analyzeSupervisorRun as t, isRuntimeSupervisorRunDir as u, DEFAULT_STEER_TOOLS as v, NO_SOURCE_LIMITS as w, analyzeSupervisorRunSources as x, claudeCodeSupervisorRunReader as y };
1689
-
1690
- //# sourceMappingURL=supervisor-run-D_sokXcO.js.map