@tangle-network/agent-eval 0.144.11 → 0.144.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (378) hide show
  1. package/CHANGELOG.md +11 -0
  2. package/dist/agent-profile-B9_GGsG8.d.ts +84 -0
  3. package/dist/agent-profile-B9_GGsG8.d.ts.map +1 -0
  4. package/dist/{agent-profile-cell-BOP-iA9Q.d.ts → agent-profile-cell-BkcRDikH.d.ts} +2 -2
  5. package/dist/{agent-profile-cell-BOP-iA9Q.d.ts.map → agent-profile-cell-BkcRDikH.d.ts.map} +1 -1
  6. package/dist/analyst/index.d.ts +134 -16
  7. package/dist/analyst/index.d.ts.map +1 -1
  8. package/dist/analyst/index.js +364 -10
  9. package/dist/analyst/index.js.map +1 -1
  10. package/dist/backend-integrity-CsVin_Wb.d.ts +280 -0
  11. package/dist/backend-integrity-CsVin_Wb.d.ts.map +1 -0
  12. package/dist/{baseline-C-GocmIW.js → baseline-BhPRQBVn.js} +3 -2
  13. package/dist/{baseline-C-GocmIW.js.map → baseline-BhPRQBVn.js.map} +1 -1
  14. package/dist/{benchmark-CWeqGl7x.js → benchmark-BhT16ep9.js} +2 -2
  15. package/dist/{benchmark-CWeqGl7x.js.map → benchmark-BhT16ep9.js.map} +1 -1
  16. package/dist/{agent-profile-DPi7IZg7.d.ts → benchmark-Ceoan7vk.d.ts} +4 -91
  17. package/dist/benchmark-Ceoan7vk.d.ts.map +1 -0
  18. package/dist/{benchmark-command-BteMFN62.js → benchmark-command-CY6cbhOB.js} +36 -79
  19. package/dist/benchmark-command-CY6cbhOB.js.map +1 -0
  20. package/dist/benchmarks/index.d.ts +244 -2
  21. package/dist/benchmarks/index.d.ts.map +1 -0
  22. package/dist/benchmarks/index.js +733 -1
  23. package/dist/benchmarks/index.js.map +1 -0
  24. package/dist/builder-eval/index.d.ts +23 -2
  25. package/dist/builder-eval/index.d.ts.map +1 -1
  26. package/dist/builder-eval/index.js +227 -3
  27. package/dist/builder-eval/index.js.map +1 -1
  28. package/dist/campaign/index.d.ts +10 -8
  29. package/dist/campaign/index.js +9 -6
  30. package/dist/{campaign-C2TTzQII.js → campaign-BYjBAypg.js} +21 -634
  31. package/dist/campaign-BYjBAypg.js.map +1 -0
  32. package/dist/{canonical-D011XM8r.js → canonical-D-XsTQ6_.js} +2 -2
  33. package/dist/{canonical-D011XM8r.js.map → canonical-D-XsTQ6_.js.map} +1 -1
  34. package/dist/{index-CQTZ-4XN.d.ts → capture-fetch-DDvpjVRU.d.ts} +3 -3
  35. package/dist/capture-fetch-DDvpjVRU.d.ts.map +1 -0
  36. package/dist/{default-registry-BmktKy8r.js → chat-client-Bp4Ebuuc.js} +1276 -1276
  37. package/dist/chat-client-Bp4Ebuuc.js.map +1 -0
  38. package/dist/cli.js +2 -2
  39. package/dist/{client-C9gzZE59.d.ts → client-KUbGulm_.d.ts} +4 -4
  40. package/dist/{client-C9gzZE59.d.ts.map → client-KUbGulm_.d.ts.map} +1 -1
  41. package/dist/contract/index.d.ts +13 -390
  42. package/dist/contract/index.d.ts.map +1 -1
  43. package/dist/contract/index.js +18 -542
  44. package/dist/contract/index.js.map +1 -1
  45. package/dist/{cost-ledger-DMFxsLKr.js → cost-ledger-BSe92yAV.js} +3 -3
  46. package/dist/{cost-ledger-DMFxsLKr.js.map → cost-ledger-BSe92yAV.js.map} +1 -1
  47. package/dist/{cost-ledger-Bv_e8XHY.d.ts → cost-ledger-DbQdN3nO.d.ts} +2 -2
  48. package/dist/{cost-ledger-Bv_e8XHY.d.ts.map → cost-ledger-DbQdN3nO.d.ts.map} +1 -1
  49. package/dist/{counterfactual-CxmxAONP.d.ts → counterfactual--bpysZF0.d.ts} +2 -14
  50. package/dist/{counterfactual-CxmxAONP.d.ts.map → counterfactual--bpysZF0.d.ts.map} +1 -1
  51. package/dist/{counterfactual-CWPTrMH7.js → counterfactual-lDfCx0Uz.js} +3 -31
  52. package/dist/{counterfactual-CWPTrMH7.js.map → counterfactual-lDfCx0Uz.js.map} +1 -1
  53. package/dist/{dataset-C8xaLXdY.d.ts → dataset-CJjKqQfA.d.ts} +2 -8
  54. package/dist/dataset-CJjKqQfA.d.ts.map +1 -0
  55. package/dist/{default-registry-Bf8Woqmq.d.ts → default-registry-Di6HP6pG.d.ts} +7 -12
  56. package/dist/default-registry-Di6HP6pG.d.ts.map +1 -0
  57. package/dist/{analyze-runs-C30yljDJ.js → define-agent-eval-iqjT--ZZ.js} +542 -130
  58. package/dist/define-agent-eval-iqjT--ZZ.js.map +1 -0
  59. package/dist/define-agent-eval-sH24zBfM.d.ts +388 -0
  60. package/dist/define-agent-eval-sH24zBfM.d.ts.map +1 -0
  61. package/dist/descriptive-B5MwKfbf.js +144 -0
  62. package/dist/descriptive-B5MwKfbf.js.map +1 -0
  63. package/dist/{dspy-rlm-engine-DbTk4JdR.js → dspy-rlm-engine-B_qhSc21.js} +4 -4
  64. package/dist/{dspy-rlm-engine-DbTk4JdR.js.map → dspy-rlm-engine-B_qhSc21.js.map} +1 -1
  65. package/dist/effect-sizes-DiH8MGOH.js +82 -0
  66. package/dist/effect-sizes-DiH8MGOH.js.map +1 -0
  67. package/dist/{engine-3hL-XqwJ.d.ts → engine-otFpE2gF.d.ts} +10 -38
  68. package/dist/engine-otFpE2gF.d.ts.map +1 -0
  69. package/dist/{errors-CKPfb2aH.d.ts → errors-DEE6u6ot.d.ts} +2 -14
  70. package/dist/{errors-CKPfb2aH.d.ts.map → errors-DEE6u6ot.d.ts.map} +1 -1
  71. package/dist/{errors-D-LKuDhb.js → errors-Dngq5h35.js} +2 -8
  72. package/dist/{errors-D-LKuDhb.js.map → errors-Dngq5h35.js.map} +1 -1
  73. package/dist/{eval-campaign-B_7wcnav.js → eval-campaign-UB-usSQ2.js} +6 -6
  74. package/dist/{eval-campaign-B_7wcnav.js.map → eval-campaign-UB-usSQ2.js.map} +1 -1
  75. package/dist/{exact-types-DSFFpLLI.d.ts → exact-types-BH1twmAJ.d.ts} +2 -2
  76. package/dist/{exact-types-DSFFpLLI.d.ts.map → exact-types-BH1twmAJ.d.ts.map} +1 -1
  77. package/dist/experiment/index.d.ts +9 -6
  78. package/dist/experiment/index.d.ts.map +1 -1
  79. package/dist/experiment/index.js +11 -7
  80. package/dist/experiment/index.js.map +1 -1
  81. package/dist/experiment-tracker-C29gXM4B.js +269 -0
  82. package/dist/experiment-tracker-C29gXM4B.js.map +1 -0
  83. package/dist/{experiment-tracker-IMntXr6J.d.ts → experiment-tracker-DWHZBAYL.d.ts} +77 -121
  84. package/dist/experiment-tracker-DWHZBAYL.d.ts.map +1 -0
  85. package/dist/{external-optimizer-contracts-CZuJNcT5.d.ts → external-optimizer-contracts-lixrOZdX.d.ts} +3 -3
  86. package/dist/{external-optimizer-contracts-CZuJNcT5.d.ts.map → external-optimizer-contracts-lixrOZdX.d.ts.map} +1 -1
  87. package/dist/external-optimizer-process-BTiNB-RH.js +301 -0
  88. package/dist/external-optimizer-process-BTiNB-RH.js.map +1 -0
  89. package/dist/{single-run-lock-DFWHEB09.js → external-optimizer-subprocess-DrJ9hR8u.js} +144 -438
  90. package/dist/external-optimizer-subprocess-DrJ9hR8u.js.map +1 -0
  91. package/dist/extract-usage-BrQ8mCLX.js +155 -0
  92. package/dist/extract-usage-BrQ8mCLX.js.map +1 -0
  93. package/dist/{failure-cluster-CqcvCcdR.d.ts → failure-cluster-BLURuWG4.d.ts} +2 -3
  94. package/dist/failure-cluster-BLURuWG4.d.ts.map +1 -0
  95. package/dist/{feedback-trajectory-CacHpxsp.d.ts → feedback-trajectory-juOozjAc.d.ts} +5 -36
  96. package/dist/feedback-trajectory-juOozjAc.d.ts.map +1 -0
  97. package/dist/fuzz.d.ts +2 -2
  98. package/dist/fuzz.js +2 -2
  99. package/dist/hosted/index.d.ts +3 -3
  100. package/dist/{index-CvSN3IG1.d.ts → index-B8Ui1mr1.d.ts} +2 -2
  101. package/dist/{index-CvSN3IG1.d.ts.map → index-B8Ui1mr1.d.ts.map} +1 -1
  102. package/dist/{skill-usage-CO9OLRBx.d.ts → index-BWDrSVfw.d.ts} +11 -136
  103. package/dist/index-BWDrSVfw.d.ts.map +1 -0
  104. package/dist/index-Ba3YrbAL.d.ts +1 -0
  105. package/dist/{index-BnEuDAK2.d.ts → index-COQYtuRF.d.ts} +3 -3
  106. package/dist/{index-BnEuDAK2.d.ts.map → index-COQYtuRF.d.ts.map} +1 -1
  107. package/dist/{index-C3ssXVLv.d.ts → index-CvjYbU0D.d.ts} +2 -2
  108. package/dist/{index-C3ssXVLv.d.ts.map → index-CvjYbU0D.d.ts.map} +1 -1
  109. package/dist/{index-DPPGNJ_R.d.ts → index-DSmEylT9.d.ts} +17 -115
  110. package/dist/index-DSmEylT9.d.ts.map +1 -0
  111. package/dist/index.d.ts +2397 -5308
  112. package/dist/index.d.ts.map +1 -1
  113. package/dist/index.js +5914 -10496
  114. package/dist/index.js.map +1 -1
  115. package/dist/{insight-report-CgX_s0Ez.d.ts → insight-report-CRi-Ufrj.d.ts} +4 -4
  116. package/dist/{insight-report-CgX_s0Ez.d.ts.map → insight-report-CRi-Ufrj.d.ts.map} +1 -1
  117. package/dist/{integrity-MLzHOfV9.js → integrity-Cy9WHAtb.js} +2 -2
  118. package/dist/{integrity-MLzHOfV9.js.map → integrity-Cy9WHAtb.js.map} +1 -1
  119. package/dist/{integrity-DY6tIbl0.js → integrity-DysDBWDu.js} +2 -2
  120. package/dist/{integrity-DY6tIbl0.js.map → integrity-DysDBWDu.js.map} +1 -1
  121. package/dist/{integrity-BuqEKu-x.d.ts → integrity-OrcI9Nau.d.ts} +3 -3
  122. package/dist/{integrity-BuqEKu-x.d.ts.map → integrity-OrcI9Nau.d.ts.map} +1 -1
  123. package/dist/internal-BDHPCnjk.js +230 -0
  124. package/dist/internal-BDHPCnjk.js.map +1 -0
  125. package/dist/judge-calibration-C5CbMYce.d.ts +117 -0
  126. package/dist/judge-calibration-C5CbMYce.d.ts.map +1 -0
  127. package/dist/judge-calibration-DZkWrm5H.js +317 -0
  128. package/dist/judge-calibration-DZkWrm5H.js.map +1 -0
  129. package/dist/{kind-factory-B8-r8-y8.js → kind-factory-CPmSd58s.js} +208 -208
  130. package/dist/{kind-factory-B8-r8-y8.js.map → kind-factory-CPmSd58s.js.map} +1 -1
  131. package/dist/ledger-core/index.d.ts +1 -1
  132. package/dist/ledger-core/index.js +2 -2
  133. package/dist/{ledger-core-DXZIqu17.js → ledger-core-BmZt19oQ.js} +110 -110
  134. package/dist/{ledger-core-DXZIqu17.js.map → ledger-core-BmZt19oQ.js.map} +1 -1
  135. package/dist/{llm-client-DzvMUsS_.js → llm-client-d0-2TT1g.js} +4 -64
  136. package/dist/{llm-client-DzvMUsS_.js.map → llm-client-d0-2TT1g.js.map} +1 -1
  137. package/dist/{skillopt-optimization-method-CQdVeM8k.js → llm-judge-dZ8P6nGI.js} +3349 -5401
  138. package/dist/llm-judge-dZ8P6nGI.js.map +1 -0
  139. package/dist/matrix/index.d.ts +2 -2
  140. package/dist/meta-eval/index.d.ts +3 -3
  141. package/dist/meta-eval/index.js +3 -3
  142. package/dist/{metrics-C9YY1OcL.js → metrics-Cl0L1KUy.js} +2 -108
  143. package/dist/{metrics-C9YY1OcL.js.map → metrics-Cl0L1KUy.js.map} +1 -1
  144. package/dist/{mint-B2O60ACG.js → mint-Dj9Ww_3I.js} +4 -4
  145. package/dist/{mint-B2O60ACG.js.map → mint-Dj9Ww_3I.js.map} +1 -1
  146. package/dist/{multi-layer-verifier-DnAqwl0h.d.ts → multi-layer-verifier-DIguZc8Z.d.ts} +3 -3
  147. package/dist/{multi-layer-verifier-DnAqwl0h.d.ts.map → multi-layer-verifier-DIguZc8Z.d.ts.map} +1 -1
  148. package/dist/multiplicity-DIWHvysC.d.ts +43 -0
  149. package/dist/multiplicity-DIWHvysC.d.ts.map +1 -0
  150. package/dist/multishot/index.d.ts +3 -3
  151. package/dist/multishot/index.js +1 -1
  152. package/dist/openapi.json +1 -1
  153. package/dist/{opencode-sqlite-8r6WUfHc.js → opencode-sqlite-DJWAXLms.js} +2 -2
  154. package/dist/{opencode-sqlite-8r6WUfHc.js.map → opencode-sqlite-DJWAXLms.js.map} +1 -1
  155. package/dist/package-version-D7lQHt_-.js +34 -0
  156. package/dist/package-version-D7lQHt_-.js.map +1 -0
  157. package/dist/paired-arms-D-XRF_fy.js +1045 -0
  158. package/dist/paired-arms-D-XRF_fy.js.map +1 -0
  159. package/dist/paired-promotion-decision-CGzg0cI_.d.ts +251 -0
  160. package/dist/paired-promotion-decision-CGzg0cI_.d.ts.map +1 -0
  161. package/dist/paired-tests-BHIhYVdu.js +213 -0
  162. package/dist/paired-tests-BHIhYVdu.js.map +1 -0
  163. package/dist/pareto-BqNW3LJR.d.ts +117 -0
  164. package/dist/pareto-BqNW3LJR.d.ts.map +1 -0
  165. package/dist/pipelines/index.d.ts +3 -64
  166. package/dist/pipelines/index.d.ts.map +1 -1
  167. package/dist/pipelines/index.js +4 -284
  168. package/dist/pipelines/index.js.map +1 -1
  169. package/dist/power-and-mde-CHIrXJll.js +195 -0
  170. package/dist/power-and-mde-CHIrXJll.js.map +1 -0
  171. package/dist/{promotion-policy-CrLrmys8.js → power-preflight-DEw-uC7q.js} +4 -184
  172. package/dist/power-preflight-DEw-uC7q.js.map +1 -0
  173. package/dist/pre-registration-CZwSFQS4.d.ts +577 -0
  174. package/dist/pre-registration-CZwSFQS4.d.ts.map +1 -0
  175. package/dist/{prime-protocol-BfSalTfR.js → prime-protocol-6tZTVsWm.js} +72 -21
  176. package/dist/prime-protocol-6tZTVsWm.js.map +1 -0
  177. package/dist/produced-state-DU79a81m.js +586 -0
  178. package/dist/produced-state-DU79a81m.js.map +1 -0
  179. package/dist/profile-cell.d.ts +1 -1
  180. package/dist/profile-cell.js +1 -1
  181. package/dist/promotion-policy-D0nPhkSy.d.ts +134 -0
  182. package/dist/promotion-policy-D0nPhkSy.d.ts.map +1 -0
  183. package/dist/promotion-policy-xzA40Evo.js +186 -0
  184. package/dist/promotion-policy-xzA40Evo.js.map +1 -0
  185. package/dist/{skillopt-optimization-method-USDKhxSA.d.ts → provenance-CCdxgLDT.d.ts} +57 -478
  186. package/dist/provenance-CCdxgLDT.d.ts.map +1 -0
  187. package/dist/registry-oJeeI4-a.d.ts +178 -0
  188. package/dist/registry-oJeeI4-a.d.ts.map +1 -0
  189. package/dist/{release-report-DA2BCu5a.d.ts → release-confidence-BFRE5WSp.d.ts} +180 -112
  190. package/dist/release-confidence-BFRE5WSp.d.ts.map +1 -0
  191. package/dist/{release-report-BUYmoKo2.js → release-confidence-CxDuiAev.js} +125 -217
  192. package/dist/release-confidence-CxDuiAev.js.map +1 -0
  193. package/dist/reporting.d.ts +6 -5
  194. package/dist/reporting.js +7 -5
  195. package/dist/{researcher-BchpD55R.d.ts → researcher-DaN4GST-.d.ts} +7 -40
  196. package/dist/{researcher-BchpD55R.d.ts.map → researcher-DaN4GST-.d.ts.map} +1 -1
  197. package/dist/{reward-hacking-BDToousL.js → reward-hacking-DNgjilrV.js} +3 -3
  198. package/dist/reward-hacking-DNgjilrV.js.map +1 -0
  199. package/dist/{reward-hacking-BI0OMAlo.d.ts → reward-hacking-DSSTuI9r.d.ts} +5 -5
  200. package/dist/{reward-hacking-BI0OMAlo.d.ts.map → reward-hacking-DSSTuI9r.d.ts.map} +1 -1
  201. package/dist/rl.d.ts +7 -7
  202. package/dist/rl.js +11 -10
  203. package/dist/rl.js.map +1 -1
  204. package/dist/rollout/index.d.ts +2 -2
  205. package/dist/rollout/index.js +4 -4
  206. package/dist/{rollout-DRrksrcV.js → rollout-BWtw0I_6.js} +3 -3
  207. package/dist/{rollout-DRrksrcV.js.map → rollout-BWtw0I_6.js.map} +1 -1
  208. package/dist/{rubric-predictive-validity-DvkjPYCe.d.ts → rubric-predictive-validity-BgxtKe4G.d.ts} +2 -2
  209. package/dist/{rubric-predictive-validity-DvkjPYCe.d.ts.map → rubric-predictive-validity-BgxtKe4G.d.ts.map} +1 -1
  210. package/dist/{rubric-predictive-validity-BRR632r1.js → rubric-predictive-validity-Cwwyd7ah.js} +2 -2
  211. package/dist/{rubric-predictive-validity-BRR632r1.js.map → rubric-predictive-validity-Cwwyd7ah.js.map} +1 -1
  212. package/dist/{run-record-BmSPWXJR.js → run-record-BvHPVS-i.js} +3 -3
  213. package/dist/{run-record-BmSPWXJR.js.map → run-record-BvHPVS-i.js.map} +1 -1
  214. package/dist/{run-record-CF4Dwpxr.d.ts → run-record-CKiihE6f.d.ts} +4 -4
  215. package/dist/{run-record-CF4Dwpxr.d.ts.map → run-record-CKiihE6f.d.ts.map} +1 -1
  216. package/dist/{proposal-findings-2GIUo1et.js → run-score-lDzV0X8j.js} +31 -31
  217. package/dist/run-score-lDzV0X8j.js.map +1 -0
  218. package/dist/sandbox-harness-BlSOu4LX.d.ts +70 -0
  219. package/dist/sandbox-harness-BlSOu4LX.d.ts.map +1 -0
  220. package/dist/{schema-Cef2cFmb2.d.ts → schema-Cef2cFmb.d.ts} +1 -1
  221. package/dist/schema-Cef2cFmb.d.ts.map +1 -0
  222. package/dist/semantic-concept-judge-BiJxScqe.js +406 -0
  223. package/dist/semantic-concept-judge-BiJxScqe.js.map +1 -0
  224. package/dist/{sequential-D-BLJBKU.js → sequential-C458DXNf.js} +4 -3
  225. package/dist/{sequential-D-BLJBKU.js.map → sequential-C458DXNf.js.map} +1 -1
  226. package/dist/sequential-eprocess-CbUt2htw.js +83 -0
  227. package/dist/sequential-eprocess-CbUt2htw.js.map +1 -0
  228. package/dist/series-convergence-D1cL1f-4.d.ts +129 -0
  229. package/dist/series-convergence-D1cL1f-4.d.ts.map +1 -0
  230. package/dist/{server-iu0ede49.js → server-ulsOdrTI.js} +5 -21
  231. package/dist/server-ulsOdrTI.js.map +1 -0
  232. package/dist/skillopt-optimization-method-C5cotF4E.d.ts +427 -0
  233. package/dist/skillopt-optimization-method-C5cotF4E.d.ts.map +1 -0
  234. package/dist/skillopt-optimization-method-jjdnc3YK.js +1999 -0
  235. package/dist/skillopt-optimization-method-jjdnc3YK.js.map +1 -0
  236. package/dist/{statistical-heldout-TQ-4CYiN.d.ts → statistical-heldout-DoFF5KJX.d.ts} +5 -278
  237. package/dist/statistical-heldout-DoFF5KJX.d.ts.map +1 -0
  238. package/dist/{store-otlp-Dw8PPIlL.js → store-otlp-CsptLYpN.js} +3 -3
  239. package/dist/{store-otlp-Dw8PPIlL.js.map → store-otlp-CsptLYpN.js.map} +1 -1
  240. package/dist/store-tool-spans-Cq9mFd-q.js +667 -0
  241. package/dist/store-tool-spans-Cq9mFd-q.js.map +1 -0
  242. package/dist/store-tool-spans-DbDLOBOb.d.ts +342 -0
  243. package/dist/store-tool-spans-DbDLOBOb.d.ts.map +1 -0
  244. package/dist/student-t-CvBq2mve.js +38 -0
  245. package/dist/student-t-CvBq2mve.js.map +1 -0
  246. package/dist/{summary-report-Lf-5I7xh.js → summary-report-Blysd6Z2.js} +6 -3
  247. package/dist/{summary-report-Lf-5I7xh.js.map → summary-report-Blysd6Z2.js.map} +1 -1
  248. package/dist/{summary-report-BNR7DWTj.d.ts → summary-report-CaL-Hnxt.d.ts} +5 -5
  249. package/dist/{summary-report-BNR7DWTj.d.ts.map → summary-report-CaL-Hnxt.d.ts.map} +1 -1
  250. package/dist/supervisor-run/index.d.ts +391 -3
  251. package/dist/supervisor-run/index.d.ts.map +1 -0
  252. package/dist/supervisor-run/index.js +1689 -2
  253. package/dist/{supervisor-run-D_sokXcO.js.map → supervisor-run/index.js.map} +1 -1
  254. package/dist/{extract-usage-CdZdoj1s.js → task-failure-attributes-CpQ4y5RD.js} +5 -157
  255. package/dist/task-failure-attributes-CpQ4y5RD.js.map +1 -0
  256. package/dist/{tool-groups-CZPGGlHf.d.ts → tool-groups-Cteb03Ps.d.ts} +3 -3
  257. package/dist/tool-groups-Cteb03Ps.d.ts.map +1 -0
  258. package/dist/tool-waste-BDdBZG1F.js +803 -0
  259. package/dist/tool-waste-BDdBZG1F.js.map +1 -0
  260. package/dist/tool-waste-DjRDEsuI.d.ts +128 -0
  261. package/dist/tool-waste-DjRDEsuI.d.ts.map +1 -0
  262. package/dist/trace-repair/index.d.ts +14 -5
  263. package/dist/trace-repair/index.d.ts.map +1 -1
  264. package/dist/trace-repair/index.js +35 -7
  265. package/dist/trace-repair/index.js.map +1 -1
  266. package/dist/traces.d.ts +406 -7
  267. package/dist/traces.d.ts.map +1 -0
  268. package/dist/traces.js +1011 -10
  269. package/dist/traces.js.map +1 -0
  270. package/dist/trajectory-replay/index.d.ts +16 -3
  271. package/dist/trajectory-replay/index.d.ts.map +1 -1
  272. package/dist/trajectory-replay/index.js +52 -5
  273. package/dist/trajectory-replay/index.js.map +1 -1
  274. package/dist/types-BEPZc6eo.d.ts +93 -0
  275. package/dist/types-BEPZc6eo.d.ts.map +1 -0
  276. package/dist/{usage-receipt-t7vAzCRQ.js → types-BI4fT3HN.js} +67 -67
  277. package/dist/types-BI4fT3HN.js.map +1 -0
  278. package/dist/{types-BnjdJ70P.d.ts → types-BZ59Ahr8.d.ts} +5 -5
  279. package/dist/{types-BnjdJ70P.d.ts.map → types-BZ59Ahr8.d.ts.map} +1 -1
  280. package/dist/{types-D216SgwM.d.ts → types-Cx3YUh2r.d.ts} +4 -241
  281. package/dist/types-Cx3YUh2r.d.ts.map +1 -0
  282. package/dist/{types-BvDKaULh.d.ts → types-DLQx4mKU.d.ts} +4 -4
  283. package/dist/{types-BvDKaULh.d.ts.map → types-DLQx4mKU.d.ts.map} +1 -1
  284. package/dist/{types-D4mog56g.d.ts → types-yLK8gXE9.d.ts} +2 -2
  285. package/dist/{types-D4mog56g.d.ts.map → types-yLK8gXE9.d.ts.map} +1 -1
  286. package/dist/verdict-BndeTAh_.js +61 -0
  287. package/dist/verdict-BndeTAh_.js.map +1 -0
  288. package/dist/verdict-E4eRNf7-.d.ts +392 -0
  289. package/dist/verdict-E4eRNf7-.d.ts.map +1 -0
  290. package/dist/{verdict-cache-BCcOh0kF.js → verdict-cache-mZf5FEiY.js} +3 -55
  291. package/dist/{verdict-cache-BCcOh0kF.js.map → verdict-cache-mZf5FEiY.js.map} +1 -1
  292. package/dist/wire/index.d.ts +3 -3
  293. package/dist/wire/index.d.ts.map +1 -1
  294. package/dist/wire/index.js +1 -1
  295. package/docs/charter.md +3 -3
  296. package/docs/control-runtime.md +3 -42
  297. package/docs/experiment.md +0 -1
  298. package/docs/feature-guide.md +2 -2
  299. package/docs/trace-repair-grader.md +1 -0
  300. package/docs/trajectory-replay.md +1 -0
  301. package/docs/verdicts.md +43 -0
  302. package/docs/verification-strategies.md +3 -2
  303. package/package.json +6 -11
  304. package/dist/agent-profile-DPi7IZg7.d.ts.map +0 -1
  305. package/dist/analyze-runs-C30yljDJ.js.map +0 -1
  306. package/dist/baseline-CavEbRyH.d.ts +0 -136
  307. package/dist/baseline-CavEbRyH.d.ts.map +0 -1
  308. package/dist/benchmark-command-BteMFN62.js.map +0 -1
  309. package/dist/benchmarks-Dzs8CKb1.js +0 -755
  310. package/dist/benchmarks-Dzs8CKb1.js.map +0 -1
  311. package/dist/campaign-C2TTzQII.js.map +0 -1
  312. package/dist/completion-verifier-DJA5BhPb.d.ts +0 -414
  313. package/dist/completion-verifier-DJA5BhPb.d.ts.map +0 -1
  314. package/dist/control.d.ts +0 -3
  315. package/dist/control.js +0 -2
  316. package/dist/dataset-C8xaLXdY.d.ts.map +0 -1
  317. package/dist/default-registry-Bf8Woqmq.d.ts.map +0 -1
  318. package/dist/default-registry-BmktKy8r.js.map +0 -1
  319. package/dist/engine-3hL-XqwJ.d.ts.map +0 -1
  320. package/dist/experiment-tracker-CnRICnMl.js +0 -500
  321. package/dist/experiment-tracker-CnRICnMl.js.map +0 -1
  322. package/dist/experiment-tracker-IMntXr6J.d.ts.map +0 -1
  323. package/dist/extract-usage-CdZdoj1s.js.map +0 -1
  324. package/dist/failure-cluster-CqcvCcdR.d.ts.map +0 -1
  325. package/dist/feedback-trajectory-CacHpxsp.d.ts.map +0 -1
  326. package/dist/index-BZ3-y4YL.d.ts +0 -391
  327. package/dist/index-BZ3-y4YL.d.ts.map +0 -1
  328. package/dist/index-CQTZ-4XN.d.ts.map +0 -1
  329. package/dist/index-DPPGNJ_R.d.ts.map +0 -1
  330. package/dist/index-YE4KdKbO2.d.ts +0 -335
  331. package/dist/index-YE4KdKbO2.d.ts.map +0 -1
  332. package/dist/paired-arms-iZ08VFMN.js +0 -260
  333. package/dist/paired-arms-iZ08VFMN.js.map +0 -1
  334. package/dist/paired-promotion-decision-B6zJ3gYM.d.ts +0 -114
  335. package/dist/paired-promotion-decision-B6zJ3gYM.d.ts.map +0 -1
  336. package/dist/prime-protocol-BfSalTfR.js.map +0 -1
  337. package/dist/promotion-policy-Ckjhzg_4.d.ts +0 -289
  338. package/dist/promotion-policy-Ckjhzg_4.d.ts.map +0 -1
  339. package/dist/promotion-policy-CrLrmys8.js.map +0 -1
  340. package/dist/proposal-findings-2GIUo1et.js.map +0 -1
  341. package/dist/propose-review-control-dSNPjFUH.js +0 -1458
  342. package/dist/propose-review-control-dSNPjFUH.js.map +0 -1
  343. package/dist/release-report-BUYmoKo2.js.map +0 -1
  344. package/dist/release-report-DA2BCu5a.d.ts.map +0 -1
  345. package/dist/replay-CohS93nE.js +0 -1859
  346. package/dist/replay-CohS93nE.js.map +0 -1
  347. package/dist/replay-DbhZ4Ked.d.ts +0 -834
  348. package/dist/replay-DbhZ4Ked.d.ts.map +0 -1
  349. package/dist/reward-hacking-BDToousL.js.map +0 -1
  350. package/dist/run-evidence-Dhi3C81V.d.ts +0 -225
  351. package/dist/run-evidence-Dhi3C81V.d.ts.map +0 -1
  352. package/dist/schema-Cef2cFmb2.d.ts.map +0 -1
  353. package/dist/semantic-concept-judge-D1z-KepS.js +0 -767
  354. package/dist/semantic-concept-judge-D1z-KepS.js.map +0 -1
  355. package/dist/series-convergence-ofsqPWhs.d.ts +0 -35
  356. package/dist/series-convergence-ofsqPWhs.d.ts.map +0 -1
  357. package/dist/server-iu0ede49.js.map +0 -1
  358. package/dist/single-run-lock-DFWHEB09.js.map +0 -1
  359. package/dist/skill-usage-CO9OLRBx.d.ts.map +0 -1
  360. package/dist/skillopt-optimization-method-CQdVeM8k.js.map +0 -1
  361. package/dist/skillopt-optimization-method-USDKhxSA.d.ts.map +0 -1
  362. package/dist/statistical-heldout-TQ-4CYiN.d.ts.map +0 -1
  363. package/dist/statistics-ByxzSiOM.js +0 -2212
  364. package/dist/statistics-ByxzSiOM.js.map +0 -1
  365. package/dist/statistics-D6Uebe_4.d.ts +0 -968
  366. package/dist/statistics-D6Uebe_4.d.ts.map +0 -1
  367. package/dist/supervisor-run-D_sokXcO.js +0 -1690
  368. package/dist/test-graded-scenario-D1TaI2va.d.ts +0 -141
  369. package/dist/test-graded-scenario-D1TaI2va.d.ts.map +0 -1
  370. package/dist/test-graded-scenario-JHcKQNpq.js +0 -318
  371. package/dist/test-graded-scenario-JHcKQNpq.js.map +0 -1
  372. package/dist/tool-groups-CZPGGlHf.d.ts.map +0 -1
  373. package/dist/tool-use-metrics-DEGMKycK.js +0 -370
  374. package/dist/tool-use-metrics-DEGMKycK.js.map +0 -1
  375. package/dist/types-D216SgwM.d.ts.map +0 -1
  376. package/dist/usage-receipt-t7vAzCRQ.js.map +0 -1
  377. package/dist/verdict-DExhxfgR.d.ts +0 -201
  378. package/dist/verdict-DExhxfgR.d.ts.map +0 -1
@@ -1,414 +0,0 @@
1
- import { t as DefaultVerdict } from "./verdict-DExhxfgR.js";
2
- import { c as CostLedgerHandle } from "./cost-ledger-Bv_e8XHY.js";
3
- import { A as ChatClient, Ft as RawProviderSink } from "./types-D216SgwM.js";
4
- import { c as AnalystRunInputs, i as AnalystFinding, l as AnalystRunResult, n as AnalystContext, s as AnalystRunEvent, t as Analyst, u as AnalystRunSummary } from "./types-BvDKaULh.js";
5
- import { i as ExactAnalystRunEvent, o as ExactAnalystRunResult, u as ExactExecutionComponentIdentity } from "./exact-types-DSFFpLLI.js";
6
- //#region src/analyst/registry.d.ts
7
- interface AnalystHooks {
8
- /** Legacy runs may mutate ctx; exact runs provide a frozen observational context. */
9
- onBeforeAnalyze?(args: {
10
- analyst: Analyst;
11
- ctx: AnalystContext;
12
- runId: string;
13
- }): void | Promise<void>;
14
- /** After every analyst (ok | failed | skipped). Use for telemetry, ingestion, rotation. */
15
- onAfterAnalyze?(args: {
16
- analyst: Analyst;
17
- summary: AnalystRunSummary;
18
- findings: AnalystFinding[];
19
- runId: string;
20
- }): void | Promise<void>;
21
- /**
22
- * On analyst exception. Hook MAY return findings to convert the
23
- * error into structured findings; the summary still reports 'failed'.
24
- * Return void to keep the default empty-findings behavior.
25
- */
26
- onError?(args: {
27
- analyst: Analyst;
28
- error: Error;
29
- runId: string;
30
- }): AnalystFinding[] | undefined | Promise<AnalystFinding[] | undefined>;
31
- /** Once after registry.run() completes. Use for final aggregation, persistence. */
32
- onComplete?(args: {
33
- result: AnalystRunResult;
34
- }): void | Promise<void>;
35
- }
36
- interface BudgetPolicy {
37
- /** Overall USD cap across the registry.run(). */
38
- totalUsd?: number;
39
- /** Per-analyst weight for the default allocator. Missing ids get weight 1. */
40
- weights?: Record<string, number>;
41
- /**
42
- * Custom allocator — receives the analyst, remaining/total budget, and
43
- * the count of analysts that will run. Returns the per-analyst budget
44
- * (or undefined only when the run has no overall cap). Overrides weights
45
- * when set.
46
- */
47
- allocate?: (args: {
48
- analyst: Analyst;
49
- totalUsd: number | undefined;
50
- remainingUsd: number | undefined;
51
- runningCount: number;
52
- }) => number | undefined;
53
- }
54
- interface AnalystRegistryOptions {
55
- /** Shared chat client passed to every LLM analyst via AnalystContext. */
56
- chat?: ChatClient;
57
- /** Logger callback. Defaults to a no-op. */
58
- log?: (msg: string, fields?: Record<string, unknown>) => void;
59
- /** Hooks invoked around analyze() — observability + customization seam. */
60
- hooks?: AnalystHooks;
61
- /** Required identity/config when `runExact` applies `hooks`. */
62
- hooksIdentity?: ExactExecutionComponentIdentity;
63
- /** Default budget when run() doesn't override. */
64
- defaultBudget?: BudgetPolicy;
65
- /** Required identity/config when `runExact` provides `chat`. */
66
- chatIdentity?: ExactExecutionComponentIdentity;
67
- }
68
- interface RegistryRunOpts {
69
- /** Restrict to a subset of registered analysts by id. */
70
- only?: string[];
71
- /** Skip these analysts even if registered. Useful for cheap iteration. */
72
- skip?: string[];
73
- /** Budget policy — totalUsd + optional weights/allocator. Falls back to options.defaultBudget. */
74
- budget?: BudgetPolicy;
75
- /** Active-work cap for the complete registry run. Model receipt settlement may follow. */
76
- timeoutMs?: number;
77
- /** Abort signal — forwarded into every analyst's context. */
78
- signal?: AbortSignal;
79
- /** Shared paid-call account forwarded to every analyst. */
80
- costLedger?: CostLedgerHandle;
81
- /** Attribution phase for calls written to `costLedger`. */
82
- costPhase?: string;
83
- /** Tags echoed into AnalystContext.tags — useful for tracking environment/version in findings. */
84
- tags?: Record<string, string>;
85
- /**
86
- * Prior-run findings made available as retrieval context to every
87
- * analyst via `ctx.priorFindings`. The registry forwards the slice
88
- * whose `analyst_id` matches each registered analyst so a kind sees
89
- * only its own history. Pass `{ '*': findings }` to broadcast to
90
- * every analyst (useful when several kinds share the same historical
91
- * context). For findings from this run, use `chainFindings` instead.
92
- */
93
- priorFindings?: ReadonlyArray<AnalystFinding> | Record<string, ReadonlyArray<AnalystFinding>>;
94
- /**
95
- * Pass findings produced earlier in this registry run to each later analyst
96
- * via `ctx.upstreamFindings`. Registration order is dependency order.
97
- * Disabled by default because independent analyst suites must opt in.
98
- */
99
- chainFindings?: boolean;
100
- }
101
- /** A caller-selected allocation rule for an exact analyst run. */
102
- type ExactAnalystBudgetPolicy = {
103
- readonly kind: 'equal';
104
- readonly totalUsd: number;
105
- } | {
106
- readonly kind: 'weighted';
107
- readonly totalUsd: number;
108
- /** One explicit weight for every selected analyst id. */
109
- readonly weights: Readonly<Record<string, number>>;
110
- };
111
- /**
112
- * Complete per-run analyst policy.
113
- *
114
- * Every field is required. `null` explicitly disables an optional resource or context channel.
115
- * `analystIds` is execution order; it is not filtered through registry insertion order.
116
- * Missing-input behavior is explicit so callers cannot accidentally inherit it by omission.
117
- * Exact runs are serial; more elaborate scheduling belongs in the caller's runtime.
118
- */
119
- interface ExactRegistryRunOpts {
120
- readonly analystIds: readonly string[];
121
- readonly budget: ExactAnalystBudgetPolicy | null;
122
- readonly totalTimeoutMs: number | null;
123
- readonly signal: AbortSignal | null;
124
- readonly costLedger: CostLedgerHandle | null;
125
- readonly costLedgerIdentity: ExactExecutionComponentIdentity | null;
126
- readonly costPhase: string | null;
127
- readonly tags: Readonly<Record<string, string>> | null;
128
- readonly priorFindings: ReadonlyArray<AnalystFinding> | Readonly<Record<string, ReadonlyArray<AnalystFinding>>> | null;
129
- readonly chainFindings: boolean;
130
- readonly missingInputMode: 'skip' | 'abort';
131
- readonly applyRegistryHooks: boolean;
132
- readonly useRegistryChat: boolean;
133
- }
134
- /** A post-start exact-run failure; completed work remains attached for accounting and review. */
135
- declare class ExactAnalystRunExecutionError extends Error {
136
- readonly name: string;
137
- readonly result: ExactAnalystRunResult;
138
- constructor(message: string, result: ExactAnalystRunResult, options?: ErrorOptions);
139
- }
140
- declare class AnalystRegistry {
141
- private readonly analysts;
142
- private readonly options;
143
- constructor(options?: AnalystRegistryOptions);
144
- register(analyst: Analyst): void;
145
- list(): ReadonlyArray<{
146
- id: string;
147
- description: string;
148
- version: string;
149
- cost: Analyst['cost'];
150
- }>;
151
- run(runId: string, inputs: AnalystRunInputs, runOpts?: RegistryRunOpts): Promise<AnalystRunResult>;
152
- /** Run exactly the ordered analysts and complete policy supplied by the caller. */
153
- runExact(runId: string, inputs: AnalystRunInputs, runOpts: ExactRegistryRunOpts): Promise<ExactAnalystRunResult>;
154
- /** Streaming counterpart to {@link runExact}. */
155
- runExactStream(runId: string, inputs: AnalystRunInputs, runOpts: ExactRegistryRunOpts): AsyncGenerator<ExactAnalystRunEvent, void, void>;
156
- /**
157
- * Streaming counterpart to `run()`. Emits `AnalystRunEvent` values
158
- * in real time — `run-started`, then per-analyst `skipped` /
159
- * `started` / `completed`, then a terminal `run-completed` whose
160
- * payload is the full `AnalystRunResult`. UIs use this to render
161
- * progress; persistence consumers use `run()` and read the result.
162
- *
163
- * Hooks (`onBeforeAnalyze` / `onAfterAnalyze` / `onError` /
164
- * `onComplete`) fire as before — streaming is additive, not a hook
165
- * replacement.
166
- */
167
- runStream(runId: string, inputs: AnalystRunInputs, runOpts?: RegistryRunOpts): AsyncGenerator<AnalystRunEvent, void, void>;
168
- private normalizeLegacyPlan;
169
- private normalizeExactPlan;
170
- private executePlanStream;
171
- private selectAnalysts;
172
- private selectExactAnalysts;
173
- private routeInput;
174
- }
175
- /** Validate the canonical exact-run policy before any analyst can start. */
176
- declare function assertExactRegistryRunOpts(value: unknown): asserts value is ExactRegistryRunOpts;
177
- //#endregion
178
- //#region src/artifact-validator.d.ts
179
- /**
180
- * Artifact validators.
181
- *
182
- * Generic "score a produced artifact" primitive. Tax uses it for PDF form
183
- * correctness, research for sourced briefs, browser for task assertions, coding
184
- * for social posts. One interface, many validators; all plug into
185
- * `BenchmarkRunner` the same way.
186
- *
187
- * A validator receives an `Artifact` (file on disk, JSON blob, text, binary)
188
- * plus a `ValidationContext` (scenario id, the turns that produced it) and
189
- * returns a `ValidationResult` with pass/fail + 0..1 score + structured
190
- * issues.
191
- */
192
- interface Artifact {
193
- /** Logical kind — validators type-guard on this */
194
- kind: 'file' | 'json' | 'text' | 'binary' | string;
195
- /** Filesystem-style path, optional */
196
- path?: string;
197
- /** String content for text/json/file kinds */
198
- content?: string;
199
- /** Binary content (if kind === 'binary') */
200
- bytes?: Uint8Array;
201
- /** Caller-supplied metadata (mimeType, sha256, size, etc.) */
202
- metadata?: Record<string, unknown>;
203
- }
204
- interface ValidationContext {
205
- scenarioId: string;
206
- turnIndex?: number;
207
- /** Prior artifacts for multi-artifact scenarios */
208
- priorArtifacts?: Artifact[];
209
- /** Free-form hints the validator uses for domain-specific checks */
210
- hints?: Record<string, unknown>;
211
- }
212
- interface ValidationIssue {
213
- severity: 'error' | 'warning' | 'info';
214
- message: string;
215
- /** Optional path into the artifact (e.g. JSON path or byte offset) */
216
- locus?: string;
217
- }
218
- interface ValidationResult {
219
- pass: boolean;
220
- /** 0–1 normalized score. Validators should be monotonic in pass-ness. */
221
- score: number;
222
- issues: ValidationIssue[];
223
- /** Diagnostic payload for reporters */
224
- evidence?: Record<string, unknown>;
225
- }
226
- interface ArtifactValidator {
227
- /** Stable identifier for the validator; appears in reports. */
228
- name: string;
229
- /** Optional description for human-facing reports. */
230
- description?: string;
231
- /** Called once per artifact; validators are expected to be pure + idempotent. */
232
- validate(artifact: Artifact, context: ValidationContext): Promise<ValidationResult>;
233
- }
234
- /**
235
- * Run every validator on the same artifact; aggregate pass as AND, score as
236
- * (weighted) mean, issues concatenated. Weights default to 1 each.
237
- */
238
- declare function composeValidators(validators: ArtifactValidator[], options?: {
239
- name?: string;
240
- weights?: number[];
241
- }): ArtifactValidator;
242
- /** Pass if the artifact body matches a provided regex. */
243
- declare function regexMatch(name: string, pattern: RegExp): ArtifactValidator;
244
- /** Pass if JSON parses and every required key is present. */
245
- declare function jsonHasKeys(name: string, requiredPaths: string[]): ArtifactValidator;
246
- /** Pass if min ≤ byte length ≤ max. */
247
- declare function byteLengthRange(name: string, min: number, max: number): ArtifactValidator;
248
- /** Pass if the artifact contains every required substring (case-insensitive by default). */
249
- declare function containsAll(name: string, required: string[], options?: {
250
- caseSensitive?: boolean;
251
- }): ArtifactValidator;
252
- //#endregion
253
- //#region src/completion-verifier.d.ts
254
- /** What kind of produced state can satisfy a requirement structurally. */
255
- type SatisfiedBy = 'artifact' | 'proposal' | 'tool-call' | 'any';
256
- interface CompletionRequirement {
257
- /** Stable id from the task gold (e.g. a persona's `expected_requirements[].req_id`). */
258
- reqId: string;
259
- /** Human-readable description of the required deliverable. */
260
- title: string;
261
- /** Optional kind/category hint, matched against a produced item's kind. */
262
- category?: string;
263
- /** What produced state satisfies this requirement. Defaults to 'any'. */
264
- satisfiedBy?: SatisfiedBy;
265
- }
266
- interface TaskGold {
267
- taskId: string;
268
- requirements: CompletionRequirement[];
269
- }
270
- interface ProducedProposal {
271
- id: string;
272
- title: string;
273
- status: 'pending' | 'approved' | 'rejected';
274
- /** Optional persisted body — when present, enables a correctness check. */
275
- content?: string;
276
- }
277
- /** Everything observable about what a run actually produced. */
278
- interface ProducedState {
279
- /** Persisted vault artifacts. Reuses the shared `Artifact` shape. */
280
- artifacts: Artifact[];
281
- /** Proposals / filings the agent created. */
282
- proposals: ProducedProposal[];
283
- /** Names of tools the agent invoked. */
284
- toolCalls: string[];
285
- }
286
- interface RequirementCheck {
287
- reqId: string;
288
- title: string;
289
- /** A produced item of the right kind matched the requirement, non-empty. */
290
- structurallyPresent: boolean;
291
- /**
292
- * Whether the matched item actually fulfils the requirement. `null` when
293
- * not structurally present, when the matched item carries no content
294
- * to assess, or when the correctness check itself failed (`unmeasured`).
295
- */
296
- correct: boolean | null;
297
- /** structurallyPresent && !unmeasured && correct !== false. */
298
- satisfied: boolean;
299
- /**
300
- * Set when the correctness check itself errored (LLM call failure or an
301
- * unparseable response after retry). The requirement's fulfilment is
302
- * UNKNOWN — `correct` stays null, `satisfied` is false, and
303
- * `completionVerdict` excludes the row from `completionRate`'s
304
- * denominator. Never folded into a zero: a synthetic zero is
305
- * indistinguishable from a real failure (see `JudgeParseError`).
306
- */
307
- unmeasured?: true;
308
- /** Why the correctness check could not be measured (present iff `unmeasured`). */
309
- unmeasuredReason?: string;
310
- /** Human-readable evidence for the verdict. */
311
- evidence: string[];
312
- }
313
- /** Extends the substrate verdict spine: `valid` = `fullyComplete` and
314
- * `score` = `completionRate` — derived in `completionVerdict()`, the one
315
- * place those equalities hold by construction. */
316
- interface CompletionVerdict extends DefaultVerdict {
317
- taskId: string;
318
- requirements: RequirementCheck[];
319
- /** satisfied / MEASURABLE requirements (unmeasured rows leave the denominator). */
320
- completionRate: number;
321
- /** Every measurable requirement satisfied (false when anything is unmeasured). */
322
- fullyComplete: boolean;
323
- /** Requirements whose correctness check errored — reported, never scored as zero. */
324
- unmeasuredCount: number;
325
- }
326
- /**
327
- * Construct a `CompletionVerdict` from the per-requirement checks, deriving
328
- * `completionRate` / `fullyComplete` and the spine fields (`valid` =
329
- * `fullyComplete`, `score` = `completionRate`) in one place. Throws on zero
330
- * requirements — a verdict over nothing is a misconfiguration, mirroring
331
- * `verifyCompletion`'s gold-spec guard.
332
- */
333
- declare function completionVerdict(input: {
334
- taskId: string;
335
- requirements: RequirementCheck[];
336
- }): CompletionVerdict;
337
- /**
338
- * Decides whether a produced item's content actually fulfils a requirement.
339
- * Injected so the structural verifier stays pure and unit-testable; the
340
- * production implementation is `createLlmCorrectnessChecker`.
341
- */
342
- type CorrectnessChecker = (requirement: CompletionRequirement, content: string) => Promise<{
343
- correct: boolean;
344
- reason: string;
345
- }>;
346
- /**
347
- * Verify whether a run completed the task. `checkCorrectness` is injected —
348
- * `createLlmCorrectnessChecker` for production, a deterministic stub in tests.
349
- *
350
- * Throws on a gold spec with no requirements: an eval task that requires
351
- * nothing is a misconfiguration, not a vacuously-complete task.
352
- */
353
- declare function verifyCompletion(gold: TaskGold, state: ProducedState, checkCorrectness: CorrectnessChecker): Promise<CompletionVerdict>;
354
- interface LlmCorrectnessCheckerOpts {
355
- model?: string;
356
- /** Optional ledger for direct use. */
357
- costLedger?: CostLedgerHandle;
358
- costPhase?: string;
359
- costTags?: Record<string, string>;
360
- signal?: AbortSignal;
361
- /** Max chars of artifact content sent to the checker. */
362
- maxContentChars?: number;
363
- /**
364
- * Checker LLM calls per requirement before giving up (parse failures and
365
- * call errors both consume attempts). The failure then surfaces as an
366
- * `unmeasured` requirement, never a zero.
367
- */
368
- maxAttempts?: number;
369
- /**
370
- * Forensic capture of every checker request/response/error — without it a
371
- * checker failure is unauditable (the agent-turn raws never contain the
372
- * checker's own calls). Same sink contract as `LlmClient`.
373
- */
374
- rawSink?: RawProviderSink;
375
- }
376
- /**
377
- * Parse the correctness checker's model response. Tolerates a response
378
- * truncated mid-JSON (max_tokens cap) by auto-closing the prefix — the
379
- * verdict boolean usually lands in the first few tokens, so a recovered
380
- * prefix with a boolean `correct` is a real measurement, not a guess.
381
- * Fails loud (JudgeParseError) when no boolean verdict is recoverable.
382
- */
383
- declare function parseCorrectnessResponse(raw: string): {
384
- correct: boolean;
385
- reason: string;
386
- };
387
- /**
388
- * Production `CorrectnessChecker` — one LLM call per matched artifact,
389
- * deterministic (temperature 0), structured JSON out. Judges fulfilment
390
- * only: a plan, a gesture, or a description of what should be done does not
391
- * fulfil a requirement — the artifact must BE the deliverable.
392
- */
393
- declare function createLlmCorrectnessChecker(chat: ChatClient, opts?: LlmCorrectnessCheckerOpts): CorrectnessChecker;
394
- /**
395
- * Deterministic `CorrectnessChecker` — the no-LLM counterpart to
396
- * `createLlmCorrectnessChecker`. A produced item fulfils a requirement when its
397
- * content is substantive (≥ `minContentLength` chars) AND recalls ≥ `minRecall`
398
- * of the requirement title's significant tokens. No network.
399
- *
400
- * Polarity-blind: token recall credits a negation that contains the
401
- * requirement's tokens ("I will NOT produce the comparison" recalls every token
402
- * of "produce the comparison"). The structural match stage is ALSO lexical, so
403
- * pairing the two collapses to a single gameable gate. Use this only as an
404
- * opt-in structural pre-filter or for tasks whose requirements have no polarity
405
- * to invert; for produced-state grading the correctness checker MUST be semantic
406
- * (`createLlmCorrectnessChecker`). See the anti-game fixtures in the test suite.
407
- */
408
- declare function createTokenRecallChecker(opts?: {
409
- minRecall?: number;
410
- minContentLength?: number;
411
- }): CorrectnessChecker;
412
- //#endregion
413
- export { ExactAnalystRunExecutionError as A, jsonHasKeys as C, AnalystRegistryOptions as D, AnalystRegistry as E, RegistryRunOpts as M, assertExactRegistryRunOpts as N, BudgetPolicy as O, containsAll as S, AnalystHooks as T, ValidationContext as _, ProducedProposal as a, byteLengthRange as b, SatisfiedBy as c, createLlmCorrectnessChecker as d, createTokenRecallChecker as f, ArtifactValidator as g, Artifact as h, LlmCorrectnessCheckerOpts as i, ExactRegistryRunOpts as j, ExactAnalystBudgetPolicy as k, TaskGold as l, verifyCompletion as m, CompletionVerdict as n, ProducedState as o, parseCorrectnessResponse as p, CorrectnessChecker as r, RequirementCheck as s, CompletionRequirement as t, completionVerdict as u, ValidationIssue as v, regexMatch as w, composeValidators as x, ValidationResult as y };
414
- //# sourceMappingURL=completion-verifier-DJA5BhPb.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"completion-verifier-DJA5BhPb.d.ts","names":[],"sources":["../src/analyst/registry.ts","../src/artifact-validator.ts","../src/completion-verifier.ts"],"mappings":";;;;;;UAqDiB;;EAEf,iBAAiB;IACf,SAAS;IACT,KAAK;IACL;aACS;;EAEX,gBAAgB;IACd,SAAS;IACT,SAAS;IACT,UAAU;IACV;aACS;;;;;;EAMX,SAAS;IACP,SAAS;IACT,OAAO;IACP;MACE,+BAA+B,QAAQ;;EAE3C,YAAY;IAAQ,QAAQ;aAA4B;;UAGzC;;EAEf;;EAEA,UAAU;;;;;;;EAOV,YAAY;IACV,SAAS;IACT;IACA;IACA;;;UAIa;;EAEf,OAAO;;EAEP,OAAO,aAAa,SAAS;;EAE7B,QAAQ;;EAER,gBAAgB;;EAEhB,gBAAgB;;EAEhB,eAAe;;UAGA;;EAEf;;EAEA;;EAEA,SAAS;;EAET;;EAEA,SAAS;;EAET,aAAa;;EAEb;;EAEA,OAAO;;;;;;;;;EASP,gBAAgB,cAAc,kBAAkB,eAAe,cAAc;;;;;;EAM7E;;;KAIU;WAEG;WACA;;WAGA;WACA;;WAEA,SAAS,SAAS;;;;;;;;;;UAWhB;WACN;WACA,QAAQ;WACR;WACA,QAAQ;WACR,YAAY;WACZ,oBAAoB;WACpB;WACA,MAAM,SAAS;WACf,eACL,cAAc,kBACd,SAAS,eAAe,cAAc;WAEjC;WACA;WACA;WACA;;;cAqCE,sCAAsC;WACxC;WACA,QAAQ;EAEjB,YAAY,iBAAiB,QAAQ,uBAAuB,UAAU;;cAU3D;mBACM;mBACA;EAEjB,YAAY,UAAS;EAIrB,SAAS,SAAS;EAsBlB,QAAQ;IACN;IACA;IACA;IACA,MAAM;;EAUF,IACJ,eACA,QAAQ,kBACR,UAAS,kBACR,QAAQ;;EAUL,SACJ,eACA,QAAQ,kBACR,SAAS,uBACR,QAAQ;;EAQJ,eACL,eACA,QAAQ,kBACR,SAAS,uBACR,eAAe;;;;;;;;;;;;EAmBX,UACL,eACA,QAAQ,kBACR,UAAS,kBACR,eAAe;UAIV;UA8BA;UAwFO;UA2aP;UAaA;UAUA;;;iBAiGM,2BAA2B,yBAAyB,SAAS;;;;;;;;;;;;;;;;UC79B5D;;EAEf;;EAEA;;EAEA;;EAEA,QAAQ;;EAER,WAAW;;UAGI;EACf;EACA;;EAEA,iBAAiB;;EAEjB,QAAQ;;UAGO;EACf;EACA;;EAEA;;UAGe;EACf;;EAEA;EACA,QAAQ;;EAER,WAAW;;UAGI;;EAEf;;EAEA;;EAEA,SAAS,UAAU,UAAU,SAAS,oBAAoB,QAAQ;;;;;;iBAWpD,kBACd,YAAY,qBACZ;EAAY;EAAe;IAC1B;;iBAgCa,WAAW,cAAc,SAAS,SAAS;;iBAkB3C,YAAY,cAAc,0BAA0B;;iBAuCpD,gBAAgB,cAAc,aAAa,cAAc;;iBAoBzD,YACd,cACA,oBACA;EAAY;IACX;;;;KClJS;UAEK;;EAEf;;EAEA;;EAEA;;EAEA,cAAc;;UAGC;EACf;EACA,cAAc;;UAGC;EACf;EACA;EACA;;EAEA;;;UAIe;;EAEf,WAAW;;EAEX,WAAW;;EAEX;;UAGe;EACf;EACA;;EAEA;;;;;;EAMA;;EAEA;;;;;;;;;EASA;;EAEA;;EAEA;;;;;UAMe,0BAA0B;EACzC;EACA,cAAc;;EAEd;;EAEA;;EAEA;;;;;;;;;iBAUc,kBAAkB;EAChC;EACA,cAAc;IACZ;;;;;;KAoCQ,sBACV,aAAa,uBACb,oBACG;EAAU;EAAkB;;;;;;;;;iBAqLX,iBACpB,MAAM,UACN,OAAO,eACP,kBAAkB,qBACjB,QAAQ;UAoFM;EACf;;EAEA,aAAa;EACb;EACA,WAAW;EACX,SAAS;;EAET;;;;;;EAMA;;;;;;EAMA,UAAU;;;;;;;;;iBAUI,yBAAyB;EAAgB;EAAkB;;;;;;;;iBAiC3D,4BACd,MAAM,YACN,OAAM,4BACL;;;;;;;;;;;;;;;iBAyJa,yBACd;EAAQ;EAAoB;IAC3B"}
package/dist/control.d.ts DELETED
@@ -1,3 +0,0 @@
1
- import { $ as ControlRunResult, J as ControlActionOutcome, Q as ControlEvalResult, X as ControlContext, Y as ControlBudget, Z as ControlDecision, at as StopDecision, ct as runAgentControlLoop, dt as subjectiveEval, et as ControlRuntimeConfig, it as ControlStopPolicies, lt as stopOnNoProgress, nt as ControlSeverity, ot as allCriticalPassed, q as ControlActionFailureMode, rt as ControlStep, st as objectiveEval, tt as ControlRuntimeError, ut as stopOnRepeatedAction } from "./feedback-trajectory-CacHpxsp.js";
2
- import { A as ActionExecutionPolicy, M as evaluateActionPolicy, _ as ProposeReviewReport, a as ProposeReviewControlAction, c as ProposeReviewControlState, g as ProposeReviewConfig, i as scoreFromEvals, j as ActionPolicyDecision, k as runProposeReview, n as RunEvidenceMetadata, o as ProposeReviewControlConfig, r as controlRunToRunRecord, s as ProposeReviewControlResult, t as ControlRunToRunRecordOptions, u as runProposeReviewAsControlLoop } from "./run-evidence-Dhi3C81V.js";
3
- export { type ActionExecutionPolicy, type ActionPolicyDecision, type ControlActionFailureMode, type ControlActionOutcome, type ControlBudget, type ControlContext, type ControlDecision, type ControlEvalResult, type ControlRunResult, type ControlRunToRunRecordOptions, type ControlRuntimeConfig, type ControlRuntimeError, type ControlSeverity, type ControlStep, type ControlStopPolicies, type ProposeReviewConfig, type ProposeReviewControlAction, type ProposeReviewControlConfig, type ProposeReviewControlResult, type ProposeReviewControlState, type ProposeReviewReport, type RunEvidenceMetadata, type StopDecision, allCriticalPassed, controlRunToRunRecord, evaluateActionPolicy, objectiveEval, runAgentControlLoop, runProposeReview, runProposeReviewAsControlLoop, scoreFromEvals, stopOnNoProgress, stopOnRepeatedAction, subjectiveEval };
package/dist/control.js DELETED
@@ -1,2 +0,0 @@
1
- import { c as scoreFromEvals, d as runAgentControlLoop, f as stopOnNoProgress, l as allCriticalPassed, m as subjectiveEval, n as runProposeReviewAsControlLoop, o as runProposeReview, p as stopOnRepeatedAction, s as controlRunToRunRecord, u as objectiveEval, y as evaluateActionPolicy } from "./propose-review-control-dSNPjFUH.js";
2
- export { allCriticalPassed, controlRunToRunRecord, evaluateActionPolicy, objectiveEval, runAgentControlLoop, runProposeReview, runProposeReviewAsControlLoop, scoreFromEvals, stopOnNoProgress, stopOnRepeatedAction, subjectiveEval };
@@ -1 +0,0 @@
1
- {"version":3,"file":"dataset-C8xaLXdY.d.ts","names":[],"sources":["../src/dataset.ts"],"mappings":";;;;;;;;;;;;;;;;;;KAiBY;KACA;UAEK;EACf;;EAEA;EACA,QAAQ;EACR,aAAa;;EAEb;;;;;;;;;;EAUA;EACA,OAAO;;UAGQ;EACf;EACA;EACA;EACA;EACA;;EAEA;;UAGe;EACf;EACA,YAAY;;EAEZ;EACA;EACA,aAAa,OAAO;;UAGL;EACf,QAAQ;EACR,aAAa;;EAEb;EACA;;EAEA,UAAU,UAAU;;EAEpB;;;cAMW,2BAA2B;EACtC,YAAY;;cAOD;WACF;WACA,YAAY;UACb;UACA;EAER,YAAY;IACV;IACA,YAAY;IACZ,WAAW;IACX;;;EASF,gBAAgB;MAIZ;;;;;EAQJ,MAAM,UAAS,eAAoB;;;;;EAqB7B,YAAY,QAAQ;;EAgB1B,MAAM,YAAW;IAAU;IAAc;OAA0B;EAWnE;EAIA,IAAI,UAAU;EAQd,OAAO;;;;;EAWP;SAQO,UACL,eACA,UAAU,KAAK,oEACd;;iBAaiB,cAAc,WAAW,oBAAoB"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"default-registry-Bf8Woqmq.d.ts","names":[],"sources":["../src/trace-analyst/behavioral-metrics.ts","../src/analyst/behavioral-analyst.ts","../src/analyst/kind-factory.ts","../src/analyst/default-registry.ts"],"mappings":";;;;;;KAoBY;UAMK;EACf,MAAM;EACN;;EAEA;;EAEA,UAAU;;UAGK;;EAEf;EACA;;EAEA,gBAAgB;;EAEhB;EACA;EACA,eAAe;EACf;EACA;;EAEA;EACA;EACA,SAAS;;UAGM;EACf;EACA;EACA,sBAAsB;EACtB,uBAAuB;;;;;;iBAoFT,oBAAoB,gBAAgB,qBAAqB;;;UC9GxD;;EAEf;;EAEA;;;;;;iBA+Dc,yBACd,SAAS,mBACT;EAAQ;EAAoB;IAC3B;;iBAkCa,kBACd,UAAS,2BACR,oBAAoB;;;;UCrHN;EACf;EACA;EACA;EACA;;EAEA,sBAAsB,SAAS;;EAE/B;;EAEA,WAAW;;EAEX,kBACE,OAAO,oBACP,SAAS,wCACe;EAC1B,SAAS,QAAQ;EACjB,eAAe,KAAK,mBAAmB,SAAS,mBAAmB;EACnE;EACA;;UAGe;EACf,QAAQ;;EAER;EACA;;;iBAIoB,gBAAgB;EACpC,YAAY;EACZ,QAAQ;EACR,OAAO;EACP,SAAS;EACT;IACE,QAAQ;;iBAgFI,mBACd,YAAY,wBACZ,SAAS,4BACR,oBAAoB;iBA+QP,oBAAoB,OAAO;iBAe3B,uBAAuB,UAAU;;;UCnahC;;;;;;EAMf,SAAS;EACT,uBAAuB;;EAEvB;EACA,aAAa;EACb,WAAW;;iBAGG,4BACd,UAAS,gCACR"}