@tangle-network/agent-eval 0.144.11 → 0.144.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (378) hide show
  1. package/CHANGELOG.md +11 -0
  2. package/dist/agent-profile-B9_GGsG8.d.ts +84 -0
  3. package/dist/agent-profile-B9_GGsG8.d.ts.map +1 -0
  4. package/dist/{agent-profile-cell-BOP-iA9Q.d.ts → agent-profile-cell-BkcRDikH.d.ts} +2 -2
  5. package/dist/{agent-profile-cell-BOP-iA9Q.d.ts.map → agent-profile-cell-BkcRDikH.d.ts.map} +1 -1
  6. package/dist/analyst/index.d.ts +134 -16
  7. package/dist/analyst/index.d.ts.map +1 -1
  8. package/dist/analyst/index.js +364 -10
  9. package/dist/analyst/index.js.map +1 -1
  10. package/dist/backend-integrity-CsVin_Wb.d.ts +280 -0
  11. package/dist/backend-integrity-CsVin_Wb.d.ts.map +1 -0
  12. package/dist/{baseline-C-GocmIW.js → baseline-BhPRQBVn.js} +3 -2
  13. package/dist/{baseline-C-GocmIW.js.map → baseline-BhPRQBVn.js.map} +1 -1
  14. package/dist/{benchmark-CWeqGl7x.js → benchmark-BhT16ep9.js} +2 -2
  15. package/dist/{benchmark-CWeqGl7x.js.map → benchmark-BhT16ep9.js.map} +1 -1
  16. package/dist/{agent-profile-DPi7IZg7.d.ts → benchmark-Ceoan7vk.d.ts} +4 -91
  17. package/dist/benchmark-Ceoan7vk.d.ts.map +1 -0
  18. package/dist/{benchmark-command-BteMFN62.js → benchmark-command-CY6cbhOB.js} +36 -79
  19. package/dist/benchmark-command-CY6cbhOB.js.map +1 -0
  20. package/dist/benchmarks/index.d.ts +244 -2
  21. package/dist/benchmarks/index.d.ts.map +1 -0
  22. package/dist/benchmarks/index.js +733 -1
  23. package/dist/benchmarks/index.js.map +1 -0
  24. package/dist/builder-eval/index.d.ts +23 -2
  25. package/dist/builder-eval/index.d.ts.map +1 -1
  26. package/dist/builder-eval/index.js +227 -3
  27. package/dist/builder-eval/index.js.map +1 -1
  28. package/dist/campaign/index.d.ts +10 -8
  29. package/dist/campaign/index.js +9 -6
  30. package/dist/{campaign-C2TTzQII.js → campaign-BYjBAypg.js} +21 -634
  31. package/dist/campaign-BYjBAypg.js.map +1 -0
  32. package/dist/{canonical-D011XM8r.js → canonical-D-XsTQ6_.js} +2 -2
  33. package/dist/{canonical-D011XM8r.js.map → canonical-D-XsTQ6_.js.map} +1 -1
  34. package/dist/{index-CQTZ-4XN.d.ts → capture-fetch-DDvpjVRU.d.ts} +3 -3
  35. package/dist/capture-fetch-DDvpjVRU.d.ts.map +1 -0
  36. package/dist/{default-registry-BmktKy8r.js → chat-client-Bp4Ebuuc.js} +1276 -1276
  37. package/dist/chat-client-Bp4Ebuuc.js.map +1 -0
  38. package/dist/cli.js +2 -2
  39. package/dist/{client-C9gzZE59.d.ts → client-KUbGulm_.d.ts} +4 -4
  40. package/dist/{client-C9gzZE59.d.ts.map → client-KUbGulm_.d.ts.map} +1 -1
  41. package/dist/contract/index.d.ts +13 -390
  42. package/dist/contract/index.d.ts.map +1 -1
  43. package/dist/contract/index.js +18 -542
  44. package/dist/contract/index.js.map +1 -1
  45. package/dist/{cost-ledger-DMFxsLKr.js → cost-ledger-BSe92yAV.js} +3 -3
  46. package/dist/{cost-ledger-DMFxsLKr.js.map → cost-ledger-BSe92yAV.js.map} +1 -1
  47. package/dist/{cost-ledger-Bv_e8XHY.d.ts → cost-ledger-DbQdN3nO.d.ts} +2 -2
  48. package/dist/{cost-ledger-Bv_e8XHY.d.ts.map → cost-ledger-DbQdN3nO.d.ts.map} +1 -1
  49. package/dist/{counterfactual-CxmxAONP.d.ts → counterfactual--bpysZF0.d.ts} +2 -14
  50. package/dist/{counterfactual-CxmxAONP.d.ts.map → counterfactual--bpysZF0.d.ts.map} +1 -1
  51. package/dist/{counterfactual-CWPTrMH7.js → counterfactual-lDfCx0Uz.js} +3 -31
  52. package/dist/{counterfactual-CWPTrMH7.js.map → counterfactual-lDfCx0Uz.js.map} +1 -1
  53. package/dist/{dataset-C8xaLXdY.d.ts → dataset-CJjKqQfA.d.ts} +2 -8
  54. package/dist/dataset-CJjKqQfA.d.ts.map +1 -0
  55. package/dist/{default-registry-Bf8Woqmq.d.ts → default-registry-Di6HP6pG.d.ts} +7 -12
  56. package/dist/default-registry-Di6HP6pG.d.ts.map +1 -0
  57. package/dist/{analyze-runs-C30yljDJ.js → define-agent-eval-iqjT--ZZ.js} +542 -130
  58. package/dist/define-agent-eval-iqjT--ZZ.js.map +1 -0
  59. package/dist/define-agent-eval-sH24zBfM.d.ts +388 -0
  60. package/dist/define-agent-eval-sH24zBfM.d.ts.map +1 -0
  61. package/dist/descriptive-B5MwKfbf.js +144 -0
  62. package/dist/descriptive-B5MwKfbf.js.map +1 -0
  63. package/dist/{dspy-rlm-engine-DbTk4JdR.js → dspy-rlm-engine-B_qhSc21.js} +4 -4
  64. package/dist/{dspy-rlm-engine-DbTk4JdR.js.map → dspy-rlm-engine-B_qhSc21.js.map} +1 -1
  65. package/dist/effect-sizes-DiH8MGOH.js +82 -0
  66. package/dist/effect-sizes-DiH8MGOH.js.map +1 -0
  67. package/dist/{engine-3hL-XqwJ.d.ts → engine-otFpE2gF.d.ts} +10 -38
  68. package/dist/engine-otFpE2gF.d.ts.map +1 -0
  69. package/dist/{errors-CKPfb2aH.d.ts → errors-DEE6u6ot.d.ts} +2 -14
  70. package/dist/{errors-CKPfb2aH.d.ts.map → errors-DEE6u6ot.d.ts.map} +1 -1
  71. package/dist/{errors-D-LKuDhb.js → errors-Dngq5h35.js} +2 -8
  72. package/dist/{errors-D-LKuDhb.js.map → errors-Dngq5h35.js.map} +1 -1
  73. package/dist/{eval-campaign-B_7wcnav.js → eval-campaign-UB-usSQ2.js} +6 -6
  74. package/dist/{eval-campaign-B_7wcnav.js.map → eval-campaign-UB-usSQ2.js.map} +1 -1
  75. package/dist/{exact-types-DSFFpLLI.d.ts → exact-types-BH1twmAJ.d.ts} +2 -2
  76. package/dist/{exact-types-DSFFpLLI.d.ts.map → exact-types-BH1twmAJ.d.ts.map} +1 -1
  77. package/dist/experiment/index.d.ts +9 -6
  78. package/dist/experiment/index.d.ts.map +1 -1
  79. package/dist/experiment/index.js +11 -7
  80. package/dist/experiment/index.js.map +1 -1
  81. package/dist/experiment-tracker-C29gXM4B.js +269 -0
  82. package/dist/experiment-tracker-C29gXM4B.js.map +1 -0
  83. package/dist/{experiment-tracker-IMntXr6J.d.ts → experiment-tracker-DWHZBAYL.d.ts} +77 -121
  84. package/dist/experiment-tracker-DWHZBAYL.d.ts.map +1 -0
  85. package/dist/{external-optimizer-contracts-CZuJNcT5.d.ts → external-optimizer-contracts-lixrOZdX.d.ts} +3 -3
  86. package/dist/{external-optimizer-contracts-CZuJNcT5.d.ts.map → external-optimizer-contracts-lixrOZdX.d.ts.map} +1 -1
  87. package/dist/external-optimizer-process-BTiNB-RH.js +301 -0
  88. package/dist/external-optimizer-process-BTiNB-RH.js.map +1 -0
  89. package/dist/{single-run-lock-DFWHEB09.js → external-optimizer-subprocess-DrJ9hR8u.js} +144 -438
  90. package/dist/external-optimizer-subprocess-DrJ9hR8u.js.map +1 -0
  91. package/dist/extract-usage-BrQ8mCLX.js +155 -0
  92. package/dist/extract-usage-BrQ8mCLX.js.map +1 -0
  93. package/dist/{failure-cluster-CqcvCcdR.d.ts → failure-cluster-BLURuWG4.d.ts} +2 -3
  94. package/dist/failure-cluster-BLURuWG4.d.ts.map +1 -0
  95. package/dist/{feedback-trajectory-CacHpxsp.d.ts → feedback-trajectory-juOozjAc.d.ts} +5 -36
  96. package/dist/feedback-trajectory-juOozjAc.d.ts.map +1 -0
  97. package/dist/fuzz.d.ts +2 -2
  98. package/dist/fuzz.js +2 -2
  99. package/dist/hosted/index.d.ts +3 -3
  100. package/dist/{index-CvSN3IG1.d.ts → index-B8Ui1mr1.d.ts} +2 -2
  101. package/dist/{index-CvSN3IG1.d.ts.map → index-B8Ui1mr1.d.ts.map} +1 -1
  102. package/dist/{skill-usage-CO9OLRBx.d.ts → index-BWDrSVfw.d.ts} +11 -136
  103. package/dist/index-BWDrSVfw.d.ts.map +1 -0
  104. package/dist/index-Ba3YrbAL.d.ts +1 -0
  105. package/dist/{index-BnEuDAK2.d.ts → index-COQYtuRF.d.ts} +3 -3
  106. package/dist/{index-BnEuDAK2.d.ts.map → index-COQYtuRF.d.ts.map} +1 -1
  107. package/dist/{index-C3ssXVLv.d.ts → index-CvjYbU0D.d.ts} +2 -2
  108. package/dist/{index-C3ssXVLv.d.ts.map → index-CvjYbU0D.d.ts.map} +1 -1
  109. package/dist/{index-DPPGNJ_R.d.ts → index-DSmEylT9.d.ts} +17 -115
  110. package/dist/index-DSmEylT9.d.ts.map +1 -0
  111. package/dist/index.d.ts +2397 -5308
  112. package/dist/index.d.ts.map +1 -1
  113. package/dist/index.js +5914 -10496
  114. package/dist/index.js.map +1 -1
  115. package/dist/{insight-report-CgX_s0Ez.d.ts → insight-report-CRi-Ufrj.d.ts} +4 -4
  116. package/dist/{insight-report-CgX_s0Ez.d.ts.map → insight-report-CRi-Ufrj.d.ts.map} +1 -1
  117. package/dist/{integrity-MLzHOfV9.js → integrity-Cy9WHAtb.js} +2 -2
  118. package/dist/{integrity-MLzHOfV9.js.map → integrity-Cy9WHAtb.js.map} +1 -1
  119. package/dist/{integrity-DY6tIbl0.js → integrity-DysDBWDu.js} +2 -2
  120. package/dist/{integrity-DY6tIbl0.js.map → integrity-DysDBWDu.js.map} +1 -1
  121. package/dist/{integrity-BuqEKu-x.d.ts → integrity-OrcI9Nau.d.ts} +3 -3
  122. package/dist/{integrity-BuqEKu-x.d.ts.map → integrity-OrcI9Nau.d.ts.map} +1 -1
  123. package/dist/internal-BDHPCnjk.js +230 -0
  124. package/dist/internal-BDHPCnjk.js.map +1 -0
  125. package/dist/judge-calibration-C5CbMYce.d.ts +117 -0
  126. package/dist/judge-calibration-C5CbMYce.d.ts.map +1 -0
  127. package/dist/judge-calibration-DZkWrm5H.js +317 -0
  128. package/dist/judge-calibration-DZkWrm5H.js.map +1 -0
  129. package/dist/{kind-factory-B8-r8-y8.js → kind-factory-CPmSd58s.js} +208 -208
  130. package/dist/{kind-factory-B8-r8-y8.js.map → kind-factory-CPmSd58s.js.map} +1 -1
  131. package/dist/ledger-core/index.d.ts +1 -1
  132. package/dist/ledger-core/index.js +2 -2
  133. package/dist/{ledger-core-DXZIqu17.js → ledger-core-BmZt19oQ.js} +110 -110
  134. package/dist/{ledger-core-DXZIqu17.js.map → ledger-core-BmZt19oQ.js.map} +1 -1
  135. package/dist/{llm-client-DzvMUsS_.js → llm-client-d0-2TT1g.js} +4 -64
  136. package/dist/{llm-client-DzvMUsS_.js.map → llm-client-d0-2TT1g.js.map} +1 -1
  137. package/dist/{skillopt-optimization-method-CQdVeM8k.js → llm-judge-dZ8P6nGI.js} +3349 -5401
  138. package/dist/llm-judge-dZ8P6nGI.js.map +1 -0
  139. package/dist/matrix/index.d.ts +2 -2
  140. package/dist/meta-eval/index.d.ts +3 -3
  141. package/dist/meta-eval/index.js +3 -3
  142. package/dist/{metrics-C9YY1OcL.js → metrics-Cl0L1KUy.js} +2 -108
  143. package/dist/{metrics-C9YY1OcL.js.map → metrics-Cl0L1KUy.js.map} +1 -1
  144. package/dist/{mint-B2O60ACG.js → mint-Dj9Ww_3I.js} +4 -4
  145. package/dist/{mint-B2O60ACG.js.map → mint-Dj9Ww_3I.js.map} +1 -1
  146. package/dist/{multi-layer-verifier-DnAqwl0h.d.ts → multi-layer-verifier-DIguZc8Z.d.ts} +3 -3
  147. package/dist/{multi-layer-verifier-DnAqwl0h.d.ts.map → multi-layer-verifier-DIguZc8Z.d.ts.map} +1 -1
  148. package/dist/multiplicity-DIWHvysC.d.ts +43 -0
  149. package/dist/multiplicity-DIWHvysC.d.ts.map +1 -0
  150. package/dist/multishot/index.d.ts +3 -3
  151. package/dist/multishot/index.js +1 -1
  152. package/dist/openapi.json +1 -1
  153. package/dist/{opencode-sqlite-8r6WUfHc.js → opencode-sqlite-DJWAXLms.js} +2 -2
  154. package/dist/{opencode-sqlite-8r6WUfHc.js.map → opencode-sqlite-DJWAXLms.js.map} +1 -1
  155. package/dist/package-version-D7lQHt_-.js +34 -0
  156. package/dist/package-version-D7lQHt_-.js.map +1 -0
  157. package/dist/paired-arms-D-XRF_fy.js +1045 -0
  158. package/dist/paired-arms-D-XRF_fy.js.map +1 -0
  159. package/dist/paired-promotion-decision-CGzg0cI_.d.ts +251 -0
  160. package/dist/paired-promotion-decision-CGzg0cI_.d.ts.map +1 -0
  161. package/dist/paired-tests-BHIhYVdu.js +213 -0
  162. package/dist/paired-tests-BHIhYVdu.js.map +1 -0
  163. package/dist/pareto-BqNW3LJR.d.ts +117 -0
  164. package/dist/pareto-BqNW3LJR.d.ts.map +1 -0
  165. package/dist/pipelines/index.d.ts +3 -64
  166. package/dist/pipelines/index.d.ts.map +1 -1
  167. package/dist/pipelines/index.js +4 -284
  168. package/dist/pipelines/index.js.map +1 -1
  169. package/dist/power-and-mde-CHIrXJll.js +195 -0
  170. package/dist/power-and-mde-CHIrXJll.js.map +1 -0
  171. package/dist/{promotion-policy-CrLrmys8.js → power-preflight-DEw-uC7q.js} +4 -184
  172. package/dist/power-preflight-DEw-uC7q.js.map +1 -0
  173. package/dist/pre-registration-CZwSFQS4.d.ts +577 -0
  174. package/dist/pre-registration-CZwSFQS4.d.ts.map +1 -0
  175. package/dist/{prime-protocol-BfSalTfR.js → prime-protocol-6tZTVsWm.js} +72 -21
  176. package/dist/prime-protocol-6tZTVsWm.js.map +1 -0
  177. package/dist/produced-state-DU79a81m.js +586 -0
  178. package/dist/produced-state-DU79a81m.js.map +1 -0
  179. package/dist/profile-cell.d.ts +1 -1
  180. package/dist/profile-cell.js +1 -1
  181. package/dist/promotion-policy-D0nPhkSy.d.ts +134 -0
  182. package/dist/promotion-policy-D0nPhkSy.d.ts.map +1 -0
  183. package/dist/promotion-policy-xzA40Evo.js +186 -0
  184. package/dist/promotion-policy-xzA40Evo.js.map +1 -0
  185. package/dist/{skillopt-optimization-method-USDKhxSA.d.ts → provenance-CCdxgLDT.d.ts} +57 -478
  186. package/dist/provenance-CCdxgLDT.d.ts.map +1 -0
  187. package/dist/registry-oJeeI4-a.d.ts +178 -0
  188. package/dist/registry-oJeeI4-a.d.ts.map +1 -0
  189. package/dist/{release-report-DA2BCu5a.d.ts → release-confidence-BFRE5WSp.d.ts} +180 -112
  190. package/dist/release-confidence-BFRE5WSp.d.ts.map +1 -0
  191. package/dist/{release-report-BUYmoKo2.js → release-confidence-CxDuiAev.js} +125 -217
  192. package/dist/release-confidence-CxDuiAev.js.map +1 -0
  193. package/dist/reporting.d.ts +6 -5
  194. package/dist/reporting.js +7 -5
  195. package/dist/{researcher-BchpD55R.d.ts → researcher-DaN4GST-.d.ts} +7 -40
  196. package/dist/{researcher-BchpD55R.d.ts.map → researcher-DaN4GST-.d.ts.map} +1 -1
  197. package/dist/{reward-hacking-BDToousL.js → reward-hacking-DNgjilrV.js} +3 -3
  198. package/dist/reward-hacking-DNgjilrV.js.map +1 -0
  199. package/dist/{reward-hacking-BI0OMAlo.d.ts → reward-hacking-DSSTuI9r.d.ts} +5 -5
  200. package/dist/{reward-hacking-BI0OMAlo.d.ts.map → reward-hacking-DSSTuI9r.d.ts.map} +1 -1
  201. package/dist/rl.d.ts +7 -7
  202. package/dist/rl.js +11 -10
  203. package/dist/rl.js.map +1 -1
  204. package/dist/rollout/index.d.ts +2 -2
  205. package/dist/rollout/index.js +4 -4
  206. package/dist/{rollout-DRrksrcV.js → rollout-BWtw0I_6.js} +3 -3
  207. package/dist/{rollout-DRrksrcV.js.map → rollout-BWtw0I_6.js.map} +1 -1
  208. package/dist/{rubric-predictive-validity-DvkjPYCe.d.ts → rubric-predictive-validity-BgxtKe4G.d.ts} +2 -2
  209. package/dist/{rubric-predictive-validity-DvkjPYCe.d.ts.map → rubric-predictive-validity-BgxtKe4G.d.ts.map} +1 -1
  210. package/dist/{rubric-predictive-validity-BRR632r1.js → rubric-predictive-validity-Cwwyd7ah.js} +2 -2
  211. package/dist/{rubric-predictive-validity-BRR632r1.js.map → rubric-predictive-validity-Cwwyd7ah.js.map} +1 -1
  212. package/dist/{run-record-BmSPWXJR.js → run-record-BvHPVS-i.js} +3 -3
  213. package/dist/{run-record-BmSPWXJR.js.map → run-record-BvHPVS-i.js.map} +1 -1
  214. package/dist/{run-record-CF4Dwpxr.d.ts → run-record-CKiihE6f.d.ts} +4 -4
  215. package/dist/{run-record-CF4Dwpxr.d.ts.map → run-record-CKiihE6f.d.ts.map} +1 -1
  216. package/dist/{proposal-findings-2GIUo1et.js → run-score-lDzV0X8j.js} +31 -31
  217. package/dist/run-score-lDzV0X8j.js.map +1 -0
  218. package/dist/sandbox-harness-BlSOu4LX.d.ts +70 -0
  219. package/dist/sandbox-harness-BlSOu4LX.d.ts.map +1 -0
  220. package/dist/{schema-Cef2cFmb2.d.ts → schema-Cef2cFmb.d.ts} +1 -1
  221. package/dist/schema-Cef2cFmb.d.ts.map +1 -0
  222. package/dist/semantic-concept-judge-BiJxScqe.js +406 -0
  223. package/dist/semantic-concept-judge-BiJxScqe.js.map +1 -0
  224. package/dist/{sequential-D-BLJBKU.js → sequential-C458DXNf.js} +4 -3
  225. package/dist/{sequential-D-BLJBKU.js.map → sequential-C458DXNf.js.map} +1 -1
  226. package/dist/sequential-eprocess-CbUt2htw.js +83 -0
  227. package/dist/sequential-eprocess-CbUt2htw.js.map +1 -0
  228. package/dist/series-convergence-D1cL1f-4.d.ts +129 -0
  229. package/dist/series-convergence-D1cL1f-4.d.ts.map +1 -0
  230. package/dist/{server-iu0ede49.js → server-ulsOdrTI.js} +5 -21
  231. package/dist/server-ulsOdrTI.js.map +1 -0
  232. package/dist/skillopt-optimization-method-C5cotF4E.d.ts +427 -0
  233. package/dist/skillopt-optimization-method-C5cotF4E.d.ts.map +1 -0
  234. package/dist/skillopt-optimization-method-jjdnc3YK.js +1999 -0
  235. package/dist/skillopt-optimization-method-jjdnc3YK.js.map +1 -0
  236. package/dist/{statistical-heldout-TQ-4CYiN.d.ts → statistical-heldout-DoFF5KJX.d.ts} +5 -278
  237. package/dist/statistical-heldout-DoFF5KJX.d.ts.map +1 -0
  238. package/dist/{store-otlp-Dw8PPIlL.js → store-otlp-CsptLYpN.js} +3 -3
  239. package/dist/{store-otlp-Dw8PPIlL.js.map → store-otlp-CsptLYpN.js.map} +1 -1
  240. package/dist/store-tool-spans-Cq9mFd-q.js +667 -0
  241. package/dist/store-tool-spans-Cq9mFd-q.js.map +1 -0
  242. package/dist/store-tool-spans-DbDLOBOb.d.ts +342 -0
  243. package/dist/store-tool-spans-DbDLOBOb.d.ts.map +1 -0
  244. package/dist/student-t-CvBq2mve.js +38 -0
  245. package/dist/student-t-CvBq2mve.js.map +1 -0
  246. package/dist/{summary-report-Lf-5I7xh.js → summary-report-Blysd6Z2.js} +6 -3
  247. package/dist/{summary-report-Lf-5I7xh.js.map → summary-report-Blysd6Z2.js.map} +1 -1
  248. package/dist/{summary-report-BNR7DWTj.d.ts → summary-report-CaL-Hnxt.d.ts} +5 -5
  249. package/dist/{summary-report-BNR7DWTj.d.ts.map → summary-report-CaL-Hnxt.d.ts.map} +1 -1
  250. package/dist/supervisor-run/index.d.ts +391 -3
  251. package/dist/supervisor-run/index.d.ts.map +1 -0
  252. package/dist/supervisor-run/index.js +1689 -2
  253. package/dist/{supervisor-run-D_sokXcO.js.map → supervisor-run/index.js.map} +1 -1
  254. package/dist/{extract-usage-CdZdoj1s.js → task-failure-attributes-CpQ4y5RD.js} +5 -157
  255. package/dist/task-failure-attributes-CpQ4y5RD.js.map +1 -0
  256. package/dist/{tool-groups-CZPGGlHf.d.ts → tool-groups-Cteb03Ps.d.ts} +3 -3
  257. package/dist/tool-groups-Cteb03Ps.d.ts.map +1 -0
  258. package/dist/tool-waste-BDdBZG1F.js +803 -0
  259. package/dist/tool-waste-BDdBZG1F.js.map +1 -0
  260. package/dist/tool-waste-DjRDEsuI.d.ts +128 -0
  261. package/dist/tool-waste-DjRDEsuI.d.ts.map +1 -0
  262. package/dist/trace-repair/index.d.ts +14 -5
  263. package/dist/trace-repair/index.d.ts.map +1 -1
  264. package/dist/trace-repair/index.js +35 -7
  265. package/dist/trace-repair/index.js.map +1 -1
  266. package/dist/traces.d.ts +406 -7
  267. package/dist/traces.d.ts.map +1 -0
  268. package/dist/traces.js +1011 -10
  269. package/dist/traces.js.map +1 -0
  270. package/dist/trajectory-replay/index.d.ts +16 -3
  271. package/dist/trajectory-replay/index.d.ts.map +1 -1
  272. package/dist/trajectory-replay/index.js +52 -5
  273. package/dist/trajectory-replay/index.js.map +1 -1
  274. package/dist/types-BEPZc6eo.d.ts +93 -0
  275. package/dist/types-BEPZc6eo.d.ts.map +1 -0
  276. package/dist/{usage-receipt-t7vAzCRQ.js → types-BI4fT3HN.js} +67 -67
  277. package/dist/types-BI4fT3HN.js.map +1 -0
  278. package/dist/{types-BnjdJ70P.d.ts → types-BZ59Ahr8.d.ts} +5 -5
  279. package/dist/{types-BnjdJ70P.d.ts.map → types-BZ59Ahr8.d.ts.map} +1 -1
  280. package/dist/{types-D216SgwM.d.ts → types-Cx3YUh2r.d.ts} +4 -241
  281. package/dist/types-Cx3YUh2r.d.ts.map +1 -0
  282. package/dist/{types-BvDKaULh.d.ts → types-DLQx4mKU.d.ts} +4 -4
  283. package/dist/{types-BvDKaULh.d.ts.map → types-DLQx4mKU.d.ts.map} +1 -1
  284. package/dist/{types-D4mog56g.d.ts → types-yLK8gXE9.d.ts} +2 -2
  285. package/dist/{types-D4mog56g.d.ts.map → types-yLK8gXE9.d.ts.map} +1 -1
  286. package/dist/verdict-BndeTAh_.js +61 -0
  287. package/dist/verdict-BndeTAh_.js.map +1 -0
  288. package/dist/verdict-E4eRNf7-.d.ts +392 -0
  289. package/dist/verdict-E4eRNf7-.d.ts.map +1 -0
  290. package/dist/{verdict-cache-BCcOh0kF.js → verdict-cache-mZf5FEiY.js} +3 -55
  291. package/dist/{verdict-cache-BCcOh0kF.js.map → verdict-cache-mZf5FEiY.js.map} +1 -1
  292. package/dist/wire/index.d.ts +3 -3
  293. package/dist/wire/index.d.ts.map +1 -1
  294. package/dist/wire/index.js +1 -1
  295. package/docs/charter.md +3 -3
  296. package/docs/control-runtime.md +3 -42
  297. package/docs/experiment.md +0 -1
  298. package/docs/feature-guide.md +2 -2
  299. package/docs/trace-repair-grader.md +1 -0
  300. package/docs/trajectory-replay.md +1 -0
  301. package/docs/verdicts.md +43 -0
  302. package/docs/verification-strategies.md +3 -2
  303. package/package.json +6 -11
  304. package/dist/agent-profile-DPi7IZg7.d.ts.map +0 -1
  305. package/dist/analyze-runs-C30yljDJ.js.map +0 -1
  306. package/dist/baseline-CavEbRyH.d.ts +0 -136
  307. package/dist/baseline-CavEbRyH.d.ts.map +0 -1
  308. package/dist/benchmark-command-BteMFN62.js.map +0 -1
  309. package/dist/benchmarks-Dzs8CKb1.js +0 -755
  310. package/dist/benchmarks-Dzs8CKb1.js.map +0 -1
  311. package/dist/campaign-C2TTzQII.js.map +0 -1
  312. package/dist/completion-verifier-DJA5BhPb.d.ts +0 -414
  313. package/dist/completion-verifier-DJA5BhPb.d.ts.map +0 -1
  314. package/dist/control.d.ts +0 -3
  315. package/dist/control.js +0 -2
  316. package/dist/dataset-C8xaLXdY.d.ts.map +0 -1
  317. package/dist/default-registry-Bf8Woqmq.d.ts.map +0 -1
  318. package/dist/default-registry-BmktKy8r.js.map +0 -1
  319. package/dist/engine-3hL-XqwJ.d.ts.map +0 -1
  320. package/dist/experiment-tracker-CnRICnMl.js +0 -500
  321. package/dist/experiment-tracker-CnRICnMl.js.map +0 -1
  322. package/dist/experiment-tracker-IMntXr6J.d.ts.map +0 -1
  323. package/dist/extract-usage-CdZdoj1s.js.map +0 -1
  324. package/dist/failure-cluster-CqcvCcdR.d.ts.map +0 -1
  325. package/dist/feedback-trajectory-CacHpxsp.d.ts.map +0 -1
  326. package/dist/index-BZ3-y4YL.d.ts +0 -391
  327. package/dist/index-BZ3-y4YL.d.ts.map +0 -1
  328. package/dist/index-CQTZ-4XN.d.ts.map +0 -1
  329. package/dist/index-DPPGNJ_R.d.ts.map +0 -1
  330. package/dist/index-YE4KdKbO2.d.ts +0 -335
  331. package/dist/index-YE4KdKbO2.d.ts.map +0 -1
  332. package/dist/paired-arms-iZ08VFMN.js +0 -260
  333. package/dist/paired-arms-iZ08VFMN.js.map +0 -1
  334. package/dist/paired-promotion-decision-B6zJ3gYM.d.ts +0 -114
  335. package/dist/paired-promotion-decision-B6zJ3gYM.d.ts.map +0 -1
  336. package/dist/prime-protocol-BfSalTfR.js.map +0 -1
  337. package/dist/promotion-policy-Ckjhzg_4.d.ts +0 -289
  338. package/dist/promotion-policy-Ckjhzg_4.d.ts.map +0 -1
  339. package/dist/promotion-policy-CrLrmys8.js.map +0 -1
  340. package/dist/proposal-findings-2GIUo1et.js.map +0 -1
  341. package/dist/propose-review-control-dSNPjFUH.js +0 -1458
  342. package/dist/propose-review-control-dSNPjFUH.js.map +0 -1
  343. package/dist/release-report-BUYmoKo2.js.map +0 -1
  344. package/dist/release-report-DA2BCu5a.d.ts.map +0 -1
  345. package/dist/replay-CohS93nE.js +0 -1859
  346. package/dist/replay-CohS93nE.js.map +0 -1
  347. package/dist/replay-DbhZ4Ked.d.ts +0 -834
  348. package/dist/replay-DbhZ4Ked.d.ts.map +0 -1
  349. package/dist/reward-hacking-BDToousL.js.map +0 -1
  350. package/dist/run-evidence-Dhi3C81V.d.ts +0 -225
  351. package/dist/run-evidence-Dhi3C81V.d.ts.map +0 -1
  352. package/dist/schema-Cef2cFmb2.d.ts.map +0 -1
  353. package/dist/semantic-concept-judge-D1z-KepS.js +0 -767
  354. package/dist/semantic-concept-judge-D1z-KepS.js.map +0 -1
  355. package/dist/series-convergence-ofsqPWhs.d.ts +0 -35
  356. package/dist/series-convergence-ofsqPWhs.d.ts.map +0 -1
  357. package/dist/server-iu0ede49.js.map +0 -1
  358. package/dist/single-run-lock-DFWHEB09.js.map +0 -1
  359. package/dist/skill-usage-CO9OLRBx.d.ts.map +0 -1
  360. package/dist/skillopt-optimization-method-CQdVeM8k.js.map +0 -1
  361. package/dist/skillopt-optimization-method-USDKhxSA.d.ts.map +0 -1
  362. package/dist/statistical-heldout-TQ-4CYiN.d.ts.map +0 -1
  363. package/dist/statistics-ByxzSiOM.js +0 -2212
  364. package/dist/statistics-ByxzSiOM.js.map +0 -1
  365. package/dist/statistics-D6Uebe_4.d.ts +0 -968
  366. package/dist/statistics-D6Uebe_4.d.ts.map +0 -1
  367. package/dist/supervisor-run-D_sokXcO.js +0 -1690
  368. package/dist/test-graded-scenario-D1TaI2va.d.ts +0 -141
  369. package/dist/test-graded-scenario-D1TaI2va.d.ts.map +0 -1
  370. package/dist/test-graded-scenario-JHcKQNpq.js +0 -318
  371. package/dist/test-graded-scenario-JHcKQNpq.js.map +0 -1
  372. package/dist/tool-groups-CZPGGlHf.d.ts.map +0 -1
  373. package/dist/tool-use-metrics-DEGMKycK.js +0 -370
  374. package/dist/tool-use-metrics-DEGMKycK.js.map +0 -1
  375. package/dist/types-D216SgwM.d.ts.map +0 -1
  376. package/dist/usage-receipt-t7vAzCRQ.js.map +0 -1
  377. package/dist/verdict-DExhxfgR.d.ts +0 -201
  378. package/dist/verdict-DExhxfgR.d.ts.map +0 -1
@@ -1,141 +0,0 @@
1
- import { f as Run, o as FailureClass } from "./schema-BtVldJ3T.js";
2
- import { s as TraceStore } from "./store-CT9YIIve.js";
3
- import { i as TraceEmitter } from "./emitter-DGQGoLyj.js";
4
- //#region src/sandbox-harness.d.ts
5
- interface HarnessConfig {
6
- /** Setup command (e.g. "pnpm install"). Non-zero exit fails the run. */
7
- setupCommand?: string;
8
- /** Run command (e.g. "pnpm build"). */
9
- runCommand?: string;
10
- /** Test command (e.g. "pnpm test --run"). Drives the test count + pass count. */
11
- testCommand?: string;
12
- /** Absolute cwd for the subprocess driver. Ignored by docker driver. */
13
- cwd?: string;
14
- /** Max wall-clock per phase in ms. Default 10 minutes. */
15
- timeoutMs?: number;
16
- /**
17
- * Cap on captured stdout+stderr bytes per phase. A runaway process can
18
- * otherwise grow the in-memory buffer without bound. Once hit, further
19
- * output is dropped and `outputTruncated` is set. Default 16 MiB.
20
- */
21
- maxOutputBytes?: number;
22
- /** Image for the docker driver. */
23
- image?: string;
24
- /** Extra env vars (validated; shell-escaped). */
25
- env?: Record<string, string>;
26
- /** Parser for the test output — maps stdout/stderr/exit code → pass count. */
27
- testParser?: TestOutputParser;
28
- }
29
- interface TestOutputParser {
30
- id: string;
31
- parse(stdout: string, stderr: string, exitCode: number): {
32
- testsTotal: number;
33
- testsPassed: number;
34
- } | undefined;
35
- }
36
- interface SandboxResult {
37
- phase: 'setup' | 'run' | 'test';
38
- exitCode: number;
39
- stdout: string;
40
- stderr: string;
41
- wallMs: number;
42
- testsTotal?: number;
43
- testsPassed?: number;
44
- /**
45
- * True when the process was killed because it exceeded `timeoutMs`. A
46
- * SIGKILLed child can still close with exit code 0; callers MUST treat
47
- * a timed-out phase as a hard failure regardless of `exitCode`, never
48
- * as a pass. `undefined`/`false` means the process completed on its own.
49
- */
50
- killedByTimeout?: boolean;
51
- /**
52
- * True when captured stdout/stderr hit `maxOutputBytes` and further
53
- * output was dropped. The result is still returned (the process was
54
- * not killed for this), but downstream parsers see truncated text.
55
- */
56
- outputTruncated?: boolean;
57
- }
58
- interface SandboxDriver {
59
- id: string;
60
- exec(phase: SandboxResult['phase'], command: string, config: HarnessConfig): Promise<SandboxResult>;
61
- }
62
- /** Vitest default summary line: "Tests X passed | Y failed". */
63
- declare const vitestTestParser: TestOutputParser;
64
- /** Pytest default: "collected N items" + " X passed, Y failed". */
65
- declare const pytestTestParser: TestOutputParser;
66
- /** Jest: "Tests: X passed, Y total" (and optional failed). */
67
- declare const jestTestParser: TestOutputParser;
68
- /** Composite parser — tries a list of parsers in order. */
69
- declare function composeParsers(...parsers: TestOutputParser[]): TestOutputParser;
70
- interface SubprocessSandboxDriverOptions {
71
- /**
72
- * Default cwd for all `exec` calls. Used when the per-call `HarnessConfig`
73
- * does not set its own `cwd`. Lets callers bind the driver to a working
74
- * directory once instead of spreading cwd into every harness config —
75
- * useful when the harness config is constructed far from the call site
76
- * (e.g. starter-foundry's promoter passes a static HarnessConfig per
77
- * family taxonomy but needs a per-run composed-scaffold cwd).
78
- */
79
- cwd?: string;
80
- /**
81
- * Default env merged into every `exec` call's env (per-call `HarnessConfig.env`
82
- * still wins on key collision). Same ergonomic rationale as `cwd` above.
83
- */
84
- env?: Record<string, string>;
85
- }
86
- declare class SubprocessSandboxDriver implements SandboxDriver {
87
- id: string;
88
- private defaultCwd?;
89
- private defaultEnv?;
90
- constructor(options?: SubprocessSandboxDriverOptions);
91
- exec(phase: SandboxResult['phase'], command: string, config: HarnessConfig): Promise<SandboxResult>;
92
- }
93
- declare class DockerSandboxDriver implements SandboxDriver {
94
- id: string;
95
- exec(phase: SandboxResult['phase'], command: string, config: HarnessConfig): Promise<SandboxResult>;
96
- }
97
- interface SandboxHarnessResult {
98
- passed: boolean;
99
- setup?: SandboxResult;
100
- run?: SandboxResult;
101
- test?: SandboxResult;
102
- totalWallMs: number;
103
- /** Final score — 0 when no tests; otherwise testsPassed/testsTotal. */
104
- score: number;
105
- }
106
- declare class SandboxHarness {
107
- private driver;
108
- constructor(driver?: SandboxDriver);
109
- run(config: HarnessConfig, emitter: TraceEmitter): Promise<SandboxHarnessResult>;
110
- }
111
- //#endregion
112
- //#region src/test-graded-scenario.d.ts
113
- interface TestGradedScenario {
114
- id: string;
115
- description?: string;
116
- harness: HarnessConfig;
117
- /** Optional pass threshold in 0..1 (default 1.0 = all tests must pass). */
118
- passThreshold?: number;
119
- /** Provenance for dataset tracking. */
120
- datasetVersion?: string;
121
- /** Free-form tags (difficulty, category, etc.). */
122
- tags?: Record<string, string>;
123
- }
124
- interface TestGradedRunOptions {
125
- variantId?: string;
126
- driver?: SandboxDriver;
127
- /** Metadata recorded on the Run (codeSha, promptSha, modelFingerprint, seed). */
128
- provenance?: Pick<Run, 'codeSha' | 'promptSha' | 'modelFingerprint' | 'seed' | 'envFingerprint'>;
129
- }
130
- interface TestGradedRunResult {
131
- runId: string;
132
- scenario: TestGradedScenario;
133
- harness: SandboxHarnessResult;
134
- pass: boolean;
135
- score: number;
136
- failureClass?: FailureClass;
137
- }
138
- declare function runTestGradedScenario(scenario: TestGradedScenario, store: TraceStore, options?: TestGradedRunOptions): Promise<TestGradedRunResult>;
139
- //#endregion
140
- export { vitestTestParser as _, DockerSandboxDriver as a, SandboxHarness as c, SubprocessSandboxDriver as d, SubprocessSandboxDriverOptions as f, pytestTestParser as g, jestTestParser as h, runTestGradedScenario as i, SandboxHarnessResult as l, composeParsers as m, TestGradedRunResult as n, HarnessConfig as o, TestOutputParser as p, TestGradedScenario as r, SandboxDriver as s, TestGradedRunOptions as t, SandboxResult as u };
141
- //# sourceMappingURL=test-graded-scenario-D1TaI2va.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"test-graded-scenario-D1TaI2va.d.ts","names":[],"sources":["../src/sandbox-harness.ts","../src/test-graded-scenario.ts"],"mappings":";;;;UAkBiB;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;EAMA;;EAEA;;EAEA,MAAM;;EAEN,aAAa;;UAGE;EACf;EACA,MACE,gBACA,gBACA;IACG;IAAoB;;;UAGV;EACf;EACA;EACA;EACA;EACA;EACA;EACA;;;;;;;EAOA;;;;;;EAMA;;UAGe;EACf;EACA,KACE,OAAO,wBACP,iBACA,QAAQ,gBACP,QAAQ;;;cAMA,kBAAkB;;cAqBlB,kBAAkB;;cAWlB,gBAAgB;;iBAUb,kBAAkB,SAAS,qBAAqB;UAe/C;;;;;;;;;EASf;;;;;EAKA,MAAM;;cAGK,mCAAmC;EAC9C;UACQ;UACA;EAER,YAAY,UAAS;EAKf,KACJ,OAAO,wBACP,iBACA,QAAQ,gBACP,QAAQ;;cAqIA,+BAA+B;EAC1C;EAEM,KACJ,OAAO,wBACP,iBACA,QAAQ,gBACP,QAAQ;;UAkBI;EACf;EACA,QAAQ;EACR,MAAM;EACN,OAAO;EACP;;EAEA;;cAGW;UACH;EACR,YAAY,SAAQ;EAId,IAAI,QAAQ,eAAe,SAAS,eAAe,QAAQ;;;;UC1UlD;EACf;EACA;EACA,SAAS;;EAET;;EAEA;;EAEA,OAAO;;UAGQ;EACf;EACA,SAAS;;EAET,aAAa,KAAK;;UAGH;EACf;EACA,UAAU;EACV,SAAS;EACT;EACA;EACA,eAAe;;iBAGK,sBACpB,UAAU,oBACV,OAAO,YACP,UAAS,uBACR,QAAQ"}
@@ -1,318 +0,0 @@
1
- import { r as ConfigError } from "./errors-D-LKuDhb.js";
2
- import { t as TraceEmitter } from "./emitter-CPBAhxum.js";
3
- //#region src/sandbox-harness.ts
4
- /**
5
- * SandboxHarness — executes a scenario in an isolated environment and
6
- * emits a rich SandboxSpan into the trace.
7
- *
8
- * Two built-in drivers:
9
- * - `SubprocessSandboxDriver` — spawn in a local cwd with env vars.
10
- * Fast, no dependencies, fine for unit tests and most CI gates.
11
- * - `DockerSandboxDriver` — lifted from tangle-router's sandbox path;
12
- * shells out to `docker run`. Stronger isolation, slower startup.
13
- *
14
- * Consumers implement `SandboxDriver` for custom backends (Firecracker,
15
- * Cloudflare sandbox product, etc.). The harness doesn't care which.
16
- */
17
- /** Vitest default summary line: "Tests X passed | Y failed". */
18
- const vitestTestParser = {
19
- id: "vitest",
20
- parse(stdout) {
21
- const m = stdout.match(/Tests\s+(\d+)\s+(passed|failed)(?:\s*\|\s*(\d+)\s+(passed|failed))?/i);
22
- if (!m) return void 0;
23
- let passed = 0;
24
- let failed = 0;
25
- const a = parseInt(m[1], 10);
26
- if (m[2].toLowerCase() === "passed") passed += a;
27
- else failed += a;
28
- if (m[3] && m[4]) {
29
- const b = parseInt(m[3], 10);
30
- if (m[4].toLowerCase() === "passed") passed += b;
31
- else failed += b;
32
- }
33
- return {
34
- testsTotal: passed + failed,
35
- testsPassed: passed
36
- };
37
- }
38
- };
39
- /** Pytest default: "collected N items" + " X passed, Y failed". */
40
- const pytestTestParser = {
41
- id: "pytest",
42
- parse(stdout) {
43
- const total = stdout.match(/collected\s+(\d+)\s+items?/i);
44
- const passed = stdout.match(/(\d+)\s+passed/);
45
- if (!total || !passed) return void 0;
46
- return {
47
- testsTotal: parseInt(total[1], 10),
48
- testsPassed: parseInt(passed[1], 10)
49
- };
50
- }
51
- };
52
- /** Jest: "Tests: X passed, Y total" (and optional failed). */
53
- const jestTestParser = {
54
- id: "jest",
55
- parse(stdout) {
56
- const m = stdout.match(/Tests:\s+(?:(\d+)\s+failed[^,]*,\s*)?(\d+)\s+passed,\s+(\d+)\s+total/i);
57
- if (!m) return void 0;
58
- return {
59
- testsTotal: parseInt(m[3], 10),
60
- testsPassed: parseInt(m[2], 10)
61
- };
62
- }
63
- };
64
- /** Composite parser — tries a list of parsers in order. */
65
- function composeParsers(...parsers) {
66
- return {
67
- id: parsers.map((p) => p.id).join("|"),
68
- parse(stdout, stderr, exitCode) {
69
- for (const p of parsers) {
70
- const res = p.parse(stdout, stderr, exitCode);
71
- if (res) return res;
72
- }
73
- }
74
- };
75
- }
76
- var SubprocessSandboxDriver = class {
77
- id = "subprocess";
78
- defaultCwd;
79
- defaultEnv;
80
- constructor(options = {}) {
81
- this.defaultCwd = options.cwd;
82
- this.defaultEnv = options.env;
83
- }
84
- async exec(phase, command, config) {
85
- const { spawn } = await import("node:child_process");
86
- const start = Date.now();
87
- const effectiveCwd = config.cwd ?? this.defaultCwd;
88
- const effectiveEnv = {
89
- ...process.env,
90
- ...this.defaultEnv ?? {},
91
- ...config.env ?? {}
92
- };
93
- const maxOutputBytes = config.maxOutputBytes ?? 16 * 1024 * 1024;
94
- return await new Promise((resolve) => {
95
- const child = spawn(command, {
96
- shell: true,
97
- cwd: effectiveCwd,
98
- env: effectiveEnv,
99
- detached: process.platform !== "win32"
100
- });
101
- let stdout = "";
102
- let stderr = "";
103
- let outputBytes = 0;
104
- let outputTruncated = false;
105
- let killedByTimeout = false;
106
- let settled = false;
107
- const onStdout = (d) => {
108
- const chunk = capture(String(d));
109
- if (chunk) stdout += chunk;
110
- };
111
- const onStderr = (d) => {
112
- const chunk = capture(String(d));
113
- if (chunk) stderr += chunk;
114
- };
115
- function capture(s) {
116
- if (outputBytes >= maxOutputBytes) {
117
- outputTruncated = true;
118
- return "";
119
- }
120
- const room = maxOutputBytes - outputBytes;
121
- if (s.length > room) {
122
- outputBytes = maxOutputBytes;
123
- outputTruncated = true;
124
- return s.slice(0, room);
125
- }
126
- outputBytes += s.length;
127
- return s;
128
- }
129
- child.stdout?.on("data", onStdout);
130
- child.stderr?.on("data", onStderr);
131
- const cleanup = () => {
132
- clearTimeout(timeout);
133
- if (forceResolve) clearTimeout(forceResolve);
134
- child.stdout?.off("data", onStdout);
135
- child.stderr?.off("data", onStderr);
136
- child.removeAllListeners("close");
137
- child.removeAllListeners("error");
138
- };
139
- const finish = (outcome) => {
140
- if (settled) return;
141
- settled = true;
142
- cleanup();
143
- const wallMs = Date.now() - start;
144
- const exitCode = killedByTimeout ? outcome.code && outcome.code !== 0 ? outcome.code : 124 : outcome.code ?? 1;
145
- const parsed = !killedByTimeout && phase === "test" && config.testParser ? config.testParser.parse(stdout, stderr, exitCode) : void 0;
146
- resolve({
147
- phase,
148
- exitCode,
149
- stdout,
150
- stderr: outcome.runnerError ? stderr + outcome.runnerError : stderr,
151
- wallMs,
152
- testsTotal: parsed?.testsTotal,
153
- testsPassed: parsed?.testsPassed,
154
- killedByTimeout: killedByTimeout || void 0,
155
- outputTruncated: outputTruncated || void 0
156
- });
157
- };
158
- const killTree = () => {
159
- try {
160
- if (process.platform !== "win32" && typeof child.pid === "number") process.kill(-child.pid, "SIGKILL");
161
- else child.kill("SIGKILL");
162
- } catch (err) {
163
- console.warn("[sandbox-harness] SIGKILL on timeout failed:", err);
164
- }
165
- };
166
- let forceResolve;
167
- const timeout = setTimeout(() => {
168
- killedByTimeout = true;
169
- killTree();
170
- forceResolve = setTimeout(() => finish({ code: null }), 2e3);
171
- }, config.timeoutMs ?? 10 * 6e4);
172
- child.on("close", (code) => {
173
- if (forceResolve) clearTimeout(forceResolve);
174
- finish({ code });
175
- });
176
- child.on("error", (err) => {
177
- if (forceResolve) clearTimeout(forceResolve);
178
- finish({
179
- code: 127,
180
- runnerError: String(err)
181
- });
182
- });
183
- });
184
- }
185
- };
186
- var DockerSandboxDriver = class {
187
- id = "docker";
188
- async exec(phase, command, config) {
189
- if (!config.image) throw new ConfigError("DockerSandboxDriver requires config.image");
190
- const sub = new SubprocessSandboxDriver();
191
- const wrapped = `docker run --rm ${Object.entries(config.env ?? {}).map(([k, v]) => `-e ${shellQuote(k)}=${shellQuote(v)}`).join(" ")} ${shellQuote(config.image)} sh -c ${shellQuote(command)}`;
192
- return sub.exec(phase, wrapped, {
193
- ...config,
194
- env: void 0
195
- });
196
- }
197
- };
198
- function shellQuote(v) {
199
- if (/^[A-Za-z0-9_\-/.@:=]+$/.test(v)) return v;
200
- return `'${v.replace(/'/g, `'\\''`)}'`;
201
- }
202
- var SandboxHarness = class {
203
- driver;
204
- constructor(driver = new SubprocessSandboxDriver()) {
205
- this.driver = driver;
206
- }
207
- async run(config, emitter) {
208
- const handle = await emitter.sandbox({
209
- name: `sandbox(${this.driver.id})`,
210
- image: config.image,
211
- command: [
212
- config.setupCommand,
213
- config.runCommand,
214
- config.testCommand
215
- ].filter(Boolean).join(" && ")
216
- });
217
- const result = {
218
- passed: false,
219
- totalWallMs: 0,
220
- score: 0
221
- };
222
- try {
223
- if (config.setupCommand) {
224
- result.setup = await this.driver.exec("setup", config.setupCommand, config);
225
- result.totalWallMs += result.setup.wallMs;
226
- if (result.setup.killedByTimeout || result.setup.exitCode !== 0) {
227
- const reason = result.setup.killedByTimeout ? `setup timed out after ${result.setup.wallMs}ms` : `setup failed (exit ${result.setup.exitCode})`;
228
- await handle.fail(reason, {
229
- exitCode: result.setup.exitCode,
230
- wallMs: result.totalWallMs
231
- });
232
- return result;
233
- }
234
- }
235
- if (config.runCommand) {
236
- result.run = await this.driver.exec("run", config.runCommand, config);
237
- result.totalWallMs += result.run.wallMs;
238
- if (result.run.killedByTimeout || result.run.exitCode !== 0) {
239
- const reason = result.run.killedByTimeout ? `run timed out after ${result.run.wallMs}ms` : `run failed (exit ${result.run.exitCode})`;
240
- await handle.fail(reason, {
241
- exitCode: result.run.exitCode,
242
- wallMs: result.totalWallMs
243
- });
244
- return result;
245
- }
246
- }
247
- if (config.testCommand) {
248
- result.test = await this.driver.exec("test", config.testCommand, config);
249
- result.totalWallMs += result.test.wallMs;
250
- const passed = !result.test.killedByTimeout && result.test.exitCode === 0;
251
- result.passed = passed;
252
- if (result.test.testsTotal !== void 0 && result.test.testsTotal > 0) result.score = (result.test.testsPassed ?? 0) / result.test.testsTotal;
253
- else result.score = passed ? 1 : 0;
254
- await handle.end({
255
- exitCode: result.test.exitCode,
256
- testsTotal: result.test.testsTotal,
257
- testsPassed: result.test.testsPassed,
258
- wallMs: result.totalWallMs,
259
- status: passed ? "ok" : "error"
260
- });
261
- } else {
262
- result.passed = true;
263
- result.score = 1;
264
- await handle.end({ wallMs: result.totalWallMs });
265
- }
266
- } catch (err) {
267
- await handle.fail(err instanceof Error ? err : String(err));
268
- throw err;
269
- }
270
- return result;
271
- }
272
- };
273
- //#endregion
274
- //#region src/test-graded-scenario.ts
275
- async function runTestGradedScenario(scenario, store, options = {}) {
276
- const emitter = new TraceEmitter(store);
277
- await emitter.startRun({
278
- scenarioId: scenario.id,
279
- variantId: options.variantId,
280
- datasetVersion: scenario.datasetVersion,
281
- tags: scenario.tags,
282
- ...options.provenance
283
- });
284
- const result = await new SandboxHarness(options.driver).run(scenario.harness, emitter);
285
- const threshold = scenario.passThreshold ?? 1;
286
- const pass = result.passed && result.score >= threshold;
287
- const setupFailed = result.setup !== void 0 && result.setup.exitCode !== 0;
288
- const runFailed = result.run !== void 0 && result.run.exitCode !== 0;
289
- const testFailed = result.test !== void 0 && result.test.exitCode !== 0;
290
- const failureClass = pass ? "success" : setupFailed || runFailed ? "sandbox_failure" : testFailed ? "format_drift" : "unknown";
291
- await emitter.endRun({
292
- pass,
293
- score: result.score,
294
- failureClass,
295
- notes: pass ? void 0 : reasonForFailure(result)
296
- });
297
- return {
298
- runId: emitter.runId,
299
- scenario,
300
- harness: result,
301
- pass,
302
- score: result.score,
303
- failureClass
304
- };
305
- }
306
- function reasonForFailure(result) {
307
- if (result.setup && result.setup.exitCode !== 0) return `setup failed: exit ${result.setup.exitCode}`;
308
- if (result.run && result.run.exitCode !== 0) return `run failed: exit ${result.run.exitCode}`;
309
- if (result.test) {
310
- if (result.test.testsTotal !== void 0) return `tests: ${result.test.testsPassed ?? 0}/${result.test.testsTotal}`;
311
- return `test exit ${result.test.exitCode}`;
312
- }
313
- return "no test command";
314
- }
315
- //#endregion
316
- export { composeParsers as a, vitestTestParser as c, SubprocessSandboxDriver as i, DockerSandboxDriver as n, jestTestParser as o, SandboxHarness as r, pytestTestParser as s, runTestGradedScenario as t };
317
-
318
- //# sourceMappingURL=test-graded-scenario-JHcKQNpq.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"test-graded-scenario-JHcKQNpq.js","names":[],"sources":["../src/sandbox-harness.ts","../src/test-graded-scenario.ts"],"sourcesContent":["/**\n * SandboxHarness — executes a scenario in an isolated environment and\n * emits a rich SandboxSpan into the trace.\n *\n * Two built-in drivers:\n * - `SubprocessSandboxDriver` — spawn in a local cwd with env vars.\n * Fast, no dependencies, fine for unit tests and most CI gates.\n * - `DockerSandboxDriver` — lifted from tangle-router's sandbox path;\n * shells out to `docker run`. Stronger isolation, slower startup.\n *\n * Consumers implement `SandboxDriver` for custom backends (Firecracker,\n * Cloudflare sandbox product, etc.). The harness doesn't care which.\n */\n\nimport { ConfigError } from './errors'\nimport type { TraceEmitter } from './trace/emitter'\nimport type { SandboxSpan } from './trace/schema'\n\nexport interface HarnessConfig {\n /** Setup command (e.g. \"pnpm install\"). Non-zero exit fails the run. */\n setupCommand?: string\n /** Run command (e.g. \"pnpm build\"). */\n runCommand?: string\n /** Test command (e.g. \"pnpm test --run\"). Drives the test count + pass count. */\n testCommand?: string\n /** Absolute cwd for the subprocess driver. Ignored by docker driver. */\n cwd?: string\n /** Max wall-clock per phase in ms. Default 10 minutes. */\n timeoutMs?: number\n /**\n * Cap on captured stdout+stderr bytes per phase. A runaway process can\n * otherwise grow the in-memory buffer without bound. Once hit, further\n * output is dropped and `outputTruncated` is set. Default 16 MiB.\n */\n maxOutputBytes?: number\n /** Image for the docker driver. */\n image?: string\n /** Extra env vars (validated; shell-escaped). */\n env?: Record<string, string>\n /** Parser for the test output — maps stdout/stderr/exit code → pass count. */\n testParser?: TestOutputParser\n}\n\nexport interface TestOutputParser {\n id: string\n parse(\n stdout: string,\n stderr: string,\n exitCode: number,\n ): { testsTotal: number; testsPassed: number } | undefined\n}\n\nexport interface SandboxResult {\n phase: 'setup' | 'run' | 'test'\n exitCode: number\n stdout: string\n stderr: string\n wallMs: number\n testsTotal?: number\n testsPassed?: number\n /**\n * True when the process was killed because it exceeded `timeoutMs`. A\n * SIGKILLed child can still close with exit code 0; callers MUST treat\n * a timed-out phase as a hard failure regardless of `exitCode`, never\n * as a pass. `undefined`/`false` means the process completed on its own.\n */\n killedByTimeout?: boolean\n /**\n * True when captured stdout/stderr hit `maxOutputBytes` and further\n * output was dropped. The result is still returned (the process was\n * not killed for this), but downstream parsers see truncated text.\n */\n outputTruncated?: boolean\n}\n\nexport interface SandboxDriver {\n id: string\n exec(\n phase: SandboxResult['phase'],\n command: string,\n config: HarnessConfig,\n ): Promise<SandboxResult>\n}\n\n// ── Parsers ──────────────────────────────────────────────────────────\n\n/** Vitest default summary line: \"Tests X passed | Y failed\". */\nexport const vitestTestParser: TestOutputParser = {\n id: 'vitest',\n parse(stdout) {\n const m = stdout.match(/Tests\\s+(\\d+)\\s+(passed|failed)(?:\\s*\\|\\s*(\\d+)\\s+(passed|failed))?/i)\n if (!m) return undefined\n let passed = 0\n let failed = 0\n const a = parseInt(m[1]!, 10)\n const aLabel = m[2]!.toLowerCase()\n if (aLabel === 'passed') passed += a\n else failed += a\n if (m[3] && m[4]) {\n const b = parseInt(m[3], 10)\n if (m[4].toLowerCase() === 'passed') passed += b\n else failed += b\n }\n return { testsTotal: passed + failed, testsPassed: passed }\n },\n}\n\n/** Pytest default: \"collected N items\" + \" X passed, Y failed\". */\nexport const pytestTestParser: TestOutputParser = {\n id: 'pytest',\n parse(stdout) {\n const total = stdout.match(/collected\\s+(\\d+)\\s+items?/i)\n const passed = stdout.match(/(\\d+)\\s+passed/)\n if (!total || !passed) return undefined\n return { testsTotal: parseInt(total[1]!, 10), testsPassed: parseInt(passed[1]!, 10) }\n },\n}\n\n/** Jest: \"Tests: X passed, Y total\" (and optional failed). */\nexport const jestTestParser: TestOutputParser = {\n id: 'jest',\n parse(stdout) {\n const m = stdout.match(/Tests:\\s+(?:(\\d+)\\s+failed[^,]*,\\s*)?(\\d+)\\s+passed,\\s+(\\d+)\\s+total/i)\n if (!m) return undefined\n return { testsTotal: parseInt(m[3]!, 10), testsPassed: parseInt(m[2]!, 10) }\n },\n}\n\n/** Composite parser — tries a list of parsers in order. */\nexport function composeParsers(...parsers: TestOutputParser[]): TestOutputParser {\n return {\n id: parsers.map((p) => p.id).join('|'),\n parse(stdout, stderr, exitCode) {\n for (const p of parsers) {\n const res = p.parse(stdout, stderr, exitCode)\n if (res) return res\n }\n return undefined\n },\n }\n}\n\n// ── Drivers ──────────────────────────────────────────────────────────\n\nexport interface SubprocessSandboxDriverOptions {\n /**\n * Default cwd for all `exec` calls. Used when the per-call `HarnessConfig`\n * does not set its own `cwd`. Lets callers bind the driver to a working\n * directory once instead of spreading cwd into every harness config —\n * useful when the harness config is constructed far from the call site\n * (e.g. starter-foundry's promoter passes a static HarnessConfig per\n * family taxonomy but needs a per-run composed-scaffold cwd).\n */\n cwd?: string\n /**\n * Default env merged into every `exec` call's env (per-call `HarnessConfig.env`\n * still wins on key collision). Same ergonomic rationale as `cwd` above.\n */\n env?: Record<string, string>\n}\n\nexport class SubprocessSandboxDriver implements SandboxDriver {\n id = 'subprocess'\n private defaultCwd?: string\n private defaultEnv?: Record<string, string>\n\n constructor(options: SubprocessSandboxDriverOptions = {}) {\n this.defaultCwd = options.cwd\n this.defaultEnv = options.env\n }\n\n async exec(\n phase: SandboxResult['phase'],\n command: string,\n config: HarnessConfig,\n ): Promise<SandboxResult> {\n const { spawn } = await import('node:child_process')\n const start = Date.now()\n // Per-call config wins; fall back to constructor defaults. Honoring\n // the constructor `cwd` keeps the subprocess from inheriting Node's\n // cwd when only the constructor arg is supplied.\n const effectiveCwd = config.cwd ?? this.defaultCwd\n const effectiveEnv = { ...process.env, ...(this.defaultEnv ?? {}), ...(config.env ?? {}) }\n const maxOutputBytes = config.maxOutputBytes ?? 16 * 1024 * 1024\n return await new Promise<SandboxResult>((resolve) => {\n const child = spawn(command, {\n shell: true,\n cwd: effectiveCwd,\n env: effectiveEnv,\n // Own process group so a timeout can SIGKILL the whole tree, not\n // just the shell — otherwise a runaway grandchild keeps the pipes\n // open and `close` never fires (the timeout silently does nothing).\n detached: process.platform !== 'win32',\n })\n let stdout = ''\n let stderr = ''\n let outputBytes = 0\n let outputTruncated = false\n let killedByTimeout = false\n let settled = false\n\n const onStdout = (d: unknown) => {\n const chunk = capture(String(d))\n if (chunk) stdout += chunk\n }\n const onStderr = (d: unknown) => {\n const chunk = capture(String(d))\n if (chunk) stderr += chunk\n }\n // Bound the in-memory buffer so a runaway process can't OOM the\n // harness. Once the cap is hit we keep draining the streams (to let\n // the child reach `close`) but discard the bytes.\n function capture(s: string): string {\n if (outputBytes >= maxOutputBytes) {\n outputTruncated = true\n return ''\n }\n const room = maxOutputBytes - outputBytes\n if (s.length > room) {\n outputBytes = maxOutputBytes\n outputTruncated = true\n return s.slice(0, room)\n }\n outputBytes += s.length\n return s\n }\n\n child.stdout?.on('data', onStdout)\n child.stderr?.on('data', onStderr)\n\n const cleanup = () => {\n clearTimeout(timeout)\n if (forceResolve) clearTimeout(forceResolve)\n child.stdout?.off('data', onStdout)\n child.stderr?.off('data', onStderr)\n child.removeAllListeners('close')\n child.removeAllListeners('error')\n }\n\n // Resolve exactly once. `code`/`signal` come from `close`; on the\n // timeout path we force a non-zero exit so a SIGKILLed child that\n // reports exit 0 can never read as a clean pass downstream.\n const finish = (outcome: { code: number | null; runnerError?: string }) => {\n if (settled) return\n settled = true\n cleanup()\n const wallMs = Date.now() - start\n const exitCode = killedByTimeout\n ? outcome.code && outcome.code !== 0\n ? outcome.code\n : 124\n : (outcome.code ?? 1)\n const parsed =\n !killedByTimeout && phase === 'test' && config.testParser\n ? config.testParser.parse(stdout, stderr, exitCode)\n : undefined\n resolve({\n phase,\n exitCode,\n stdout,\n stderr: outcome.runnerError ? stderr + outcome.runnerError : stderr,\n wallMs,\n testsTotal: parsed?.testsTotal,\n testsPassed: parsed?.testsPassed,\n killedByTimeout: killedByTimeout || undefined,\n outputTruncated: outputTruncated || undefined,\n })\n }\n\n // Kill the whole process group on win32-less platforms (we spawned\n // detached). On a clean close this is a no-op.\n const killTree = () => {\n try {\n if (process.platform !== 'win32' && typeof child.pid === 'number') {\n process.kill(-child.pid, 'SIGKILL')\n } else {\n child.kill('SIGKILL')\n }\n } catch (err) {\n console.warn('[sandbox-harness] SIGKILL on timeout failed:', err)\n }\n }\n\n let forceResolve: ReturnType<typeof setTimeout> | undefined\n const timeout = setTimeout(\n () => {\n killedByTimeout = true\n killTree()\n // Give `close` a brief window to fire (flushing any final output)\n // after the group dies; if an orphaned grandchild keeps the pipes\n // open, force-resolve so the harness can't hang on a runaway.\n forceResolve = setTimeout(() => finish({ code: null }), 2000)\n },\n config.timeoutMs ?? 10 * 60_000,\n )\n\n child.on('close', (code) => {\n if (forceResolve) clearTimeout(forceResolve)\n finish({ code })\n })\n child.on('error', (err) => {\n if (forceResolve) clearTimeout(forceResolve)\n finish({ code: 127, runnerError: String(err) })\n })\n })\n }\n}\n\nexport class DockerSandboxDriver implements SandboxDriver {\n id = 'docker'\n\n async exec(\n phase: SandboxResult['phase'],\n command: string,\n config: HarnessConfig,\n ): Promise<SandboxResult> {\n if (!config.image) throw new ConfigError('DockerSandboxDriver requires config.image')\n const sub = new SubprocessSandboxDriver()\n const envArgs = Object.entries(config.env ?? {})\n .map(([k, v]) => `-e ${shellQuote(k)}=${shellQuote(v)}`)\n .join(' ')\n const wrapped = `docker run --rm ${envArgs} ${shellQuote(config.image)} sh -c ${shellQuote(command)}`\n return sub.exec(phase, wrapped, { ...config, env: undefined })\n }\n}\n\nfunction shellQuote(v: string): string {\n if (/^[A-Za-z0-9_\\-/.@:=]+$/.test(v)) return v\n return `'${v.replace(/'/g, `'\\\\''`)}'`\n}\n\n// ── Harness orchestration ────────────────────────────────────────────\n\nexport interface SandboxHarnessResult {\n passed: boolean\n setup?: SandboxResult\n run?: SandboxResult\n test?: SandboxResult\n totalWallMs: number\n /** Final score — 0 when no tests; otherwise testsPassed/testsTotal. */\n score: number\n}\n\nexport class SandboxHarness {\n private driver: SandboxDriver\n constructor(driver: SandboxDriver = new SubprocessSandboxDriver()) {\n this.driver = driver\n }\n\n async run(config: HarnessConfig, emitter: TraceEmitter): Promise<SandboxHarnessResult> {\n const handle = await emitter.sandbox({\n name: `sandbox(${this.driver.id})`,\n image: config.image,\n command: [config.setupCommand, config.runCommand, config.testCommand]\n .filter(Boolean)\n .join(' && '),\n })\n const result: SandboxHarnessResult = { passed: false, totalWallMs: 0, score: 0 }\n try {\n if (config.setupCommand) {\n result.setup = await this.driver.exec('setup', config.setupCommand, config)\n result.totalWallMs += result.setup.wallMs\n if (result.setup.killedByTimeout || result.setup.exitCode !== 0) {\n const reason = result.setup.killedByTimeout\n ? `setup timed out after ${result.setup.wallMs}ms`\n : `setup failed (exit ${result.setup.exitCode})`\n await handle.fail(reason, {\n exitCode: result.setup.exitCode,\n wallMs: result.totalWallMs,\n } as Partial<SandboxSpan>)\n return result\n }\n }\n if (config.runCommand) {\n result.run = await this.driver.exec('run', config.runCommand, config)\n result.totalWallMs += result.run.wallMs\n if (result.run.killedByTimeout || result.run.exitCode !== 0) {\n const reason = result.run.killedByTimeout\n ? `run timed out after ${result.run.wallMs}ms`\n : `run failed (exit ${result.run.exitCode})`\n await handle.fail(reason, {\n exitCode: result.run.exitCode,\n wallMs: result.totalWallMs,\n } as Partial<SandboxSpan>)\n return result\n }\n }\n if (config.testCommand) {\n result.test = await this.driver.exec('test', config.testCommand, config)\n result.totalWallMs += result.test.wallMs\n // A timed-out test phase is a hard fail regardless of the exit code\n // a SIGKILLed child happens to report — never let it score as a pass.\n const passed = !result.test.killedByTimeout && result.test.exitCode === 0\n result.passed = passed\n if (result.test.testsTotal !== undefined && result.test.testsTotal > 0) {\n result.score = (result.test.testsPassed ?? 0) / result.test.testsTotal\n } else {\n result.score = passed ? 1 : 0\n }\n await handle.end({\n exitCode: result.test.exitCode,\n testsTotal: result.test.testsTotal,\n testsPassed: result.test.testsPassed,\n wallMs: result.totalWallMs,\n status: passed ? 'ok' : 'error',\n } as Partial<SandboxSpan>)\n } else {\n result.passed = true\n result.score = 1\n await handle.end({ wallMs: result.totalWallMs } as Partial<SandboxSpan>)\n }\n } catch (err) {\n await handle.fail(err instanceof Error ? err : String(err))\n throw err\n }\n return result\n }\n}\n","/**\n * TestGradedScenario — a scenario whose score comes from a test suite.\n *\n * This is the SWE-bench pattern generalized. The scenario ships:\n * - fixture data (setup instructions)\n * - a test command the harness runs\n * - optional assertion overrides\n *\n * The runner emits a run, delegates to SandboxHarness, records the\n * outcome, and returns a structured verdict. Consumers bind their own\n * agent execution to this contract.\n */\n\nimport type { HarnessConfig, SandboxDriver, SandboxHarnessResult } from './sandbox-harness'\nimport { SandboxHarness } from './sandbox-harness'\nimport { TraceEmitter } from './trace/emitter'\nimport type { FailureClass, Run } from './trace/schema'\nimport type { TraceStore } from './trace/store'\n\nexport interface TestGradedScenario {\n id: string\n description?: string\n harness: HarnessConfig\n /** Optional pass threshold in 0..1 (default 1.0 = all tests must pass). */\n passThreshold?: number\n /** Provenance for dataset tracking. */\n datasetVersion?: string\n /** Free-form tags (difficulty, category, etc.). */\n tags?: Record<string, string>\n}\n\nexport interface TestGradedRunOptions {\n variantId?: string\n driver?: SandboxDriver\n /** Metadata recorded on the Run (codeSha, promptSha, modelFingerprint, seed). */\n provenance?: Pick<Run, 'codeSha' | 'promptSha' | 'modelFingerprint' | 'seed' | 'envFingerprint'>\n}\n\nexport interface TestGradedRunResult {\n runId: string\n scenario: TestGradedScenario\n harness: SandboxHarnessResult\n pass: boolean\n score: number\n failureClass?: FailureClass\n}\n\nexport async function runTestGradedScenario(\n scenario: TestGradedScenario,\n store: TraceStore,\n options: TestGradedRunOptions = {},\n): Promise<TestGradedRunResult> {\n const emitter = new TraceEmitter(store)\n await emitter.startRun({\n scenarioId: scenario.id,\n variantId: options.variantId,\n datasetVersion: scenario.datasetVersion,\n tags: scenario.tags,\n ...options.provenance,\n })\n const harness = new SandboxHarness(options.driver)\n const result = await harness.run(scenario.harness, emitter)\n const threshold = scenario.passThreshold ?? 1.0\n const pass = result.passed && result.score >= threshold\n const setupFailed = result.setup !== undefined && result.setup.exitCode !== 0\n const runFailed = result.run !== undefined && result.run.exitCode !== 0\n const testFailed = result.test !== undefined && result.test.exitCode !== 0\n const failureClass: FailureClass | undefined = pass\n ? 'success'\n : setupFailed || runFailed\n ? 'sandbox_failure'\n : testFailed\n ? 'format_drift'\n : 'unknown'\n await emitter.endRun({\n pass,\n score: result.score,\n failureClass,\n notes: pass ? undefined : reasonForFailure(result),\n })\n return {\n runId: emitter.runId,\n scenario,\n harness: result,\n pass,\n score: result.score,\n failureClass,\n }\n}\n\nfunction reasonForFailure(result: SandboxHarnessResult): string {\n if (result.setup && result.setup.exitCode !== 0)\n return `setup failed: exit ${result.setup.exitCode}`\n if (result.run && result.run.exitCode !== 0) return `run failed: exit ${result.run.exitCode}`\n if (result.test) {\n if (result.test.testsTotal !== undefined) {\n return `tests: ${result.test.testsPassed ?? 0}/${result.test.testsTotal}`\n }\n return `test exit ${result.test.exitCode}`\n }\n return 'no test command'\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAuFA,MAAa,mBAAqC;CAChD,IAAI;CACJ,MAAM,QAAQ;EACZ,MAAM,IAAI,OAAO,MAAM,sEAAsE;EAC7F,IAAI,CAAC,GAAG,OAAO,KAAA;EACf,IAAI,SAAS;EACb,IAAI,SAAS;EACb,MAAM,IAAI,SAAS,EAAE,IAAK,EAAE;EAE5B,IADe,EAAE,EAAE,CAAE,YACZ,MAAM,UAAU,UAAU;OAC9B,UAAU;EACf,IAAI,EAAE,MAAM,EAAE,IAAI;GAChB,MAAM,IAAI,SAAS,EAAE,IAAI,EAAE;GAC3B,IAAI,EAAE,EAAE,CAAC,YAAY,MAAM,UAAU,UAAU;QAC1C,UAAU;EACjB;EACA,OAAO;GAAE,YAAY,SAAS;GAAQ,aAAa;EAAO;CAC5D;AACF;;AAGA,MAAa,mBAAqC;CAChD,IAAI;CACJ,MAAM,QAAQ;EACZ,MAAM,QAAQ,OAAO,MAAM,6BAA6B;EACxD,MAAM,SAAS,OAAO,MAAM,gBAAgB;EAC5C,IAAI,CAAC,SAAS,CAAC,QAAQ,OAAO,KAAA;EAC9B,OAAO;GAAE,YAAY,SAAS,MAAM,IAAK,EAAE;GAAG,aAAa,SAAS,OAAO,IAAK,EAAE;EAAE;CACtF;AACF;;AAGA,MAAa,iBAAmC;CAC9C,IAAI;CACJ,MAAM,QAAQ;EACZ,MAAM,IAAI,OAAO,MAAM,uEAAuE;EAC9F,IAAI,CAAC,GAAG,OAAO,KAAA;EACf,OAAO;GAAE,YAAY,SAAS,EAAE,IAAK,EAAE;GAAG,aAAa,SAAS,EAAE,IAAK,EAAE;EAAE;CAC7E;AACF;;AAGA,SAAgB,eAAe,GAAG,SAA+C;CAC/E,OAAO;EACL,IAAI,QAAQ,KAAK,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,GAAG;EACrC,MAAM,QAAQ,QAAQ,UAAU;GAC9B,KAAK,MAAM,KAAK,SAAS;IACvB,MAAM,MAAM,EAAE,MAAM,QAAQ,QAAQ,QAAQ;IAC5C,IAAI,KAAK,OAAO;GAClB;EAEF;CACF;AACF;AAqBA,IAAa,0BAAb,MAA8D;CAC5D,KAAK;CACL;CACA;CAEA,YAAY,UAA0C,CAAC,GAAG;EACxD,KAAK,aAAa,QAAQ;EAC1B,KAAK,aAAa,QAAQ;CAC5B;CAEA,MAAM,KACJ,OACA,SACA,QACwB;EACxB,MAAM,EAAE,UAAU,MAAM,OAAO;EAC/B,MAAM,QAAQ,KAAK,IAAI;EAIvB,MAAM,eAAe,OAAO,OAAO,KAAK;EACxC,MAAM,eAAe;GAAE,GAAG,QAAQ;GAAK,GAAI,KAAK,cAAc,CAAC;GAAI,GAAI,OAAO,OAAO,CAAC;EAAG;EACzF,MAAM,iBAAiB,OAAO,kBAAkB,KAAK,OAAO;EAC5D,OAAO,MAAM,IAAI,SAAwB,YAAY;GACnD,MAAM,QAAQ,MAAM,SAAS;IAC3B,OAAO;IACP,KAAK;IACL,KAAK;IAIL,UAAU,QAAQ,aAAa;GACjC,CAAC;GACD,IAAI,SAAS;GACb,IAAI,SAAS;GACb,IAAI,cAAc;GAClB,IAAI,kBAAkB;GACtB,IAAI,kBAAkB;GACtB,IAAI,UAAU;GAEd,MAAM,YAAY,MAAe;IAC/B,MAAM,QAAQ,QAAQ,OAAO,CAAC,CAAC;IAC/B,IAAI,OAAO,UAAU;GACvB;GACA,MAAM,YAAY,MAAe;IAC/B,MAAM,QAAQ,QAAQ,OAAO,CAAC,CAAC;IAC/B,IAAI,OAAO,UAAU;GACvB;GAIA,SAAS,QAAQ,GAAmB;IAClC,IAAI,eAAe,gBAAgB;KACjC,kBAAkB;KAClB,OAAO;IACT;IACA,MAAM,OAAO,iBAAiB;IAC9B,IAAI,EAAE,SAAS,MAAM;KACnB,cAAc;KACd,kBAAkB;KAClB,OAAO,EAAE,MAAM,GAAG,IAAI;IACxB;IACA,eAAe,EAAE;IACjB,OAAO;GACT;GAEA,MAAM,QAAQ,GAAG,QAAQ,QAAQ;GACjC,MAAM,QAAQ,GAAG,QAAQ,QAAQ;GAEjC,MAAM,gBAAgB;IACpB,aAAa,OAAO;IACpB,IAAI,cAAc,aAAa,YAAY;IAC3C,MAAM,QAAQ,IAAI,QAAQ,QAAQ;IAClC,MAAM,QAAQ,IAAI,QAAQ,QAAQ;IAClC,MAAM,mBAAmB,OAAO;IAChC,MAAM,mBAAmB,OAAO;GAClC;GAKA,MAAM,UAAU,YAA2D;IACzE,IAAI,SAAS;IACb,UAAU;IACV,QAAQ;IACR,MAAM,SAAS,KAAK,IAAI,IAAI;IAC5B,MAAM,WAAW,kBACb,QAAQ,QAAQ,QAAQ,SAAS,IAC/B,QAAQ,OACR,MACD,QAAQ,QAAQ;IACrB,MAAM,SACJ,CAAC,mBAAmB,UAAU,UAAU,OAAO,aAC3C,OAAO,WAAW,MAAM,QAAQ,QAAQ,QAAQ,IAChD,KAAA;IACN,QAAQ;KACN;KACA;KACA;KACA,QAAQ,QAAQ,cAAc,SAAS,QAAQ,cAAc;KAC7D;KACA,YAAY,QAAQ;KACpB,aAAa,QAAQ;KACrB,iBAAiB,mBAAmB,KAAA;KACpC,iBAAiB,mBAAmB,KAAA;IACtC,CAAC;GACH;GAIA,MAAM,iBAAiB;IACrB,IAAI;KACF,IAAI,QAAQ,aAAa,WAAW,OAAO,MAAM,QAAQ,UACvD,QAAQ,KAAK,CAAC,MAAM,KAAK,SAAS;UAElC,MAAM,KAAK,SAAS;IAExB,SAAS,KAAK;KACZ,QAAQ,KAAK,gDAAgD,GAAG;IAClE;GACF;GAEA,IAAI;GACJ,MAAM,UAAU,iBACR;IACJ,kBAAkB;IAClB,SAAS;IAIT,eAAe,iBAAiB,OAAO,EAAE,MAAM,KAAK,CAAC,GAAG,GAAI;GAC9D,GACA,OAAO,aAAa,KAAK,GAC3B;GAEA,MAAM,GAAG,UAAU,SAAS;IAC1B,IAAI,cAAc,aAAa,YAAY;IAC3C,OAAO,EAAE,KAAK,CAAC;GACjB,CAAC;GACD,MAAM,GAAG,UAAU,QAAQ;IACzB,IAAI,cAAc,aAAa,YAAY;IAC3C,OAAO;KAAE,MAAM;KAAK,aAAa,OAAO,GAAG;IAAE,CAAC;GAChD,CAAC;EACH,CAAC;CACH;AACF;AAEA,IAAa,sBAAb,MAA0D;CACxD,KAAK;CAEL,MAAM,KACJ,OACA,SACA,QACwB;EACxB,IAAI,CAAC,OAAO,OAAO,MAAM,IAAI,YAAY,2CAA2C;EACpF,MAAM,MAAM,IAAI,wBAAwB;EAIxC,MAAM,UAAU,mBAHA,OAAO,QAAQ,OAAO,OAAO,CAAC,CAAC,CAAC,CAC7C,KAAK,CAAC,GAAG,OAAO,MAAM,WAAW,CAAC,EAAE,GAAG,WAAW,CAAC,GAAG,CAAC,CACvD,KAAK,GACiC,EAAE,GAAG,WAAW,OAAO,KAAK,EAAE,SAAS,WAAW,OAAO;EAClG,OAAO,IAAI,KAAK,OAAO,SAAS;GAAE,GAAG;GAAQ,KAAK,KAAA;EAAU,CAAC;CAC/D;AACF;AAEA,SAAS,WAAW,GAAmB;CACrC,IAAI,yBAAyB,KAAK,CAAC,GAAG,OAAO;CAC7C,OAAO,IAAI,EAAE,QAAQ,MAAM,OAAO,EAAE;AACtC;AAcA,IAAa,iBAAb,MAA4B;CAC1B;CACA,YAAY,SAAwB,IAAI,wBAAwB,GAAG;EACjE,KAAK,SAAS;CAChB;CAEA,MAAM,IAAI,QAAuB,SAAsD;EACrF,MAAM,SAAS,MAAM,QAAQ,QAAQ;GACnC,MAAM,WAAW,KAAK,OAAO,GAAG;GAChC,OAAO,OAAO;GACd,SAAS;IAAC,OAAO;IAAc,OAAO;IAAY,OAAO;GAAW,CAAC,CAClE,OAAO,OAAO,CAAC,CACf,KAAK,MAAM;EAChB,CAAC;EACD,MAAM,SAA+B;GAAE,QAAQ;GAAO,aAAa;GAAG,OAAO;EAAE;EAC/E,IAAI;GACF,IAAI,OAAO,cAAc;IACvB,OAAO,QAAQ,MAAM,KAAK,OAAO,KAAK,SAAS,OAAO,cAAc,MAAM;IAC1E,OAAO,eAAe,OAAO,MAAM;IACnC,IAAI,OAAO,MAAM,mBAAmB,OAAO,MAAM,aAAa,GAAG;KAC/D,MAAM,SAAS,OAAO,MAAM,kBACxB,yBAAyB,OAAO,MAAM,OAAO,MAC7C,sBAAsB,OAAO,MAAM,SAAS;KAChD,MAAM,OAAO,KAAK,QAAQ;MACxB,UAAU,OAAO,MAAM;MACvB,QAAQ,OAAO;KACjB,CAAyB;KACzB,OAAO;IACT;GACF;GACA,IAAI,OAAO,YAAY;IACrB,OAAO,MAAM,MAAM,KAAK,OAAO,KAAK,OAAO,OAAO,YAAY,MAAM;IACpE,OAAO,eAAe,OAAO,IAAI;IACjC,IAAI,OAAO,IAAI,mBAAmB,OAAO,IAAI,aAAa,GAAG;KAC3D,MAAM,SAAS,OAAO,IAAI,kBACtB,uBAAuB,OAAO,IAAI,OAAO,MACzC,oBAAoB,OAAO,IAAI,SAAS;KAC5C,MAAM,OAAO,KAAK,QAAQ;MACxB,UAAU,OAAO,IAAI;MACrB,QAAQ,OAAO;KACjB,CAAyB;KACzB,OAAO;IACT;GACF;GACA,IAAI,OAAO,aAAa;IACtB,OAAO,OAAO,MAAM,KAAK,OAAO,KAAK,QAAQ,OAAO,aAAa,MAAM;IACvE,OAAO,eAAe,OAAO,KAAK;IAGlC,MAAM,SAAS,CAAC,OAAO,KAAK,mBAAmB,OAAO,KAAK,aAAa;IACxE,OAAO,SAAS;IAChB,IAAI,OAAO,KAAK,eAAe,KAAA,KAAa,OAAO,KAAK,aAAa,GACnE,OAAO,SAAS,OAAO,KAAK,eAAe,KAAK,OAAO,KAAK;SAE5D,OAAO,QAAQ,SAAS,IAAI;IAE9B,MAAM,OAAO,IAAI;KACf,UAAU,OAAO,KAAK;KACtB,YAAY,OAAO,KAAK;KACxB,aAAa,OAAO,KAAK;KACzB,QAAQ,OAAO;KACf,QAAQ,SAAS,OAAO;IAC1B,CAAyB;GAC3B,OAAO;IACL,OAAO,SAAS;IAChB,OAAO,QAAQ;IACf,MAAM,OAAO,IAAI,EAAE,QAAQ,OAAO,YAAY,CAAyB;GACzE;EACF,SAAS,KAAK;GACZ,MAAM,OAAO,KAAK,eAAe,QAAQ,MAAM,OAAO,GAAG,CAAC;GAC1D,MAAM;EACR;EACA,OAAO;CACT;AACF;;;AClXA,eAAsB,sBACpB,UACA,OACA,UAAgC,CAAC,GACH;CAC9B,MAAM,UAAU,IAAI,aAAa,KAAK;CACtC,MAAM,QAAQ,SAAS;EACrB,YAAY,SAAS;EACrB,WAAW,QAAQ;EACnB,gBAAgB,SAAS;EACzB,MAAM,SAAS;EACf,GAAG,QAAQ;CACb,CAAC;CAED,MAAM,SAAS,MAAM,IADD,eAAe,QAAQ,MAChB,CAAC,CAAC,IAAI,SAAS,SAAS,OAAO;CAC1D,MAAM,YAAY,SAAS,iBAAiB;CAC5C,MAAM,OAAO,OAAO,UAAU,OAAO,SAAS;CAC9C,MAAM,cAAc,OAAO,UAAU,KAAA,KAAa,OAAO,MAAM,aAAa;CAC5E,MAAM,YAAY,OAAO,QAAQ,KAAA,KAAa,OAAO,IAAI,aAAa;CACtE,MAAM,aAAa,OAAO,SAAS,KAAA,KAAa,OAAO,KAAK,aAAa;CACzE,MAAM,eAAyC,OAC3C,YACA,eAAe,YACb,oBACA,aACE,iBACA;CACR,MAAM,QAAQ,OAAO;EACnB;EACA,OAAO,OAAO;EACd;EACA,OAAO,OAAO,KAAA,IAAY,iBAAiB,MAAM;CACnD,CAAC;CACD,OAAO;EACL,OAAO,QAAQ;EACf;EACA,SAAS;EACT;EACA,OAAO,OAAO;EACd;CACF;AACF;AAEA,SAAS,iBAAiB,QAAsC;CAC9D,IAAI,OAAO,SAAS,OAAO,MAAM,aAAa,GAC5C,OAAO,sBAAsB,OAAO,MAAM;CAC5C,IAAI,OAAO,OAAO,OAAO,IAAI,aAAa,GAAG,OAAO,oBAAoB,OAAO,IAAI;CACnF,IAAI,OAAO,MAAM;EACf,IAAI,OAAO,KAAK,eAAe,KAAA,GAC7B,OAAO,UAAU,OAAO,KAAK,eAAe,EAAE,GAAG,OAAO,KAAK;EAE/D,OAAO,aAAa,OAAO,KAAK;CAClC;CACA,OAAO;AACT"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"tool-groups-CZPGGlHf.d.ts","names":[],"sources":["../src/analyst/tool-groups.ts"],"mappings":";;;;KAmBY;;;;;;;;;;;;;;;;;;;;iBAgDI,wBACd,OAAO,oBACP,OAAO,qBACN"}