@tangle-network/agent-eval 0.172.0 → 0.173.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (219) hide show
  1. package/CHANGELOG.md +49 -0
  2. package/README.md +17 -2
  3. package/dist/adapters/http.d.ts +2 -2
  4. package/dist/analyst/index.d.ts +13 -13
  5. package/dist/analyst/index.js +7 -6
  6. package/dist/analyst/index.js.map +1 -1
  7. package/dist/{attestation-CJBGmMVh.d.ts → attestation-c1QvaBdX.d.ts} +2 -2
  8. package/dist/{attestation-CJBGmMVh.d.ts.map → attestation-c1QvaBdX.d.ts.map} +1 -1
  9. package/dist/{backend-integrity-e79K3UPD.d.ts → backend-integrity-CeuTgqsd.d.ts} +3 -4
  10. package/dist/backend-integrity-CeuTgqsd.d.ts.map +1 -0
  11. package/dist/{benchmark-h-h4bfqj.d.ts → benchmark-BjLGkfnN.d.ts} +3 -3
  12. package/dist/{benchmark-h-h4bfqj.d.ts.map → benchmark-BjLGkfnN.d.ts.map} +1 -1
  13. package/dist/{benchmark-command-DoFcisuM.js → benchmark-command-9S20PRel.js} +9 -9
  14. package/dist/{benchmark-command-DoFcisuM.js.map → benchmark-command-9S20PRel.js.map} +1 -1
  15. package/dist/benchmarks/index.d.ts +5 -5
  16. package/dist/benchmarks/index.js +3 -3
  17. package/dist/{bounded-process-CVOC_D3H.js → bounded-process-BBZob7vl.js} +64 -10
  18. package/dist/bounded-process-BBZob7vl.js.map +1 -0
  19. package/dist/builder-eval/index.d.ts +3 -3
  20. package/dist/builder-eval/index.js +2 -2
  21. package/dist/campaign/index.d.ts +9 -9
  22. package/dist/campaign/index.js +6 -6
  23. package/dist/{campaign-Dp35pBbS.js → campaign-pxS0wmo4.js} +8 -8
  24. package/dist/{campaign-Dp35pBbS.js.map → campaign-pxS0wmo4.js.map} +1 -1
  25. package/dist/chat-client-Db4bqYfA.js +115 -0
  26. package/dist/chat-client-Db4bqYfA.js.map +1 -0
  27. package/dist/{chat-json-call-5Jxna-aV.js → chat-json-call-C26igCih.js} +16 -6
  28. package/dist/chat-json-call-C26igCih.js.map +1 -0
  29. package/dist/cli.js +31 -17
  30. package/dist/cli.js.map +1 -1
  31. package/dist/{client-BvwNkIRN.js → client-BlLY6o2w.js} +2 -2
  32. package/dist/{client-BvwNkIRN.js.map → client-BlLY6o2w.js.map} +1 -1
  33. package/dist/{client-Df7wdslk.d.ts → client-DlqdbM7n.d.ts} +4 -4
  34. package/dist/{client-Df7wdslk.d.ts.map → client-DlqdbM7n.d.ts.map} +1 -1
  35. package/dist/contract/index.d.ts +13 -13
  36. package/dist/contract/index.js +202 -10
  37. package/dist/contract/index.js.map +1 -1
  38. package/dist/{counterfactual-Bee5_BIn.d.ts → counterfactual-CLgrwhkY.d.ts} +4 -4
  39. package/dist/{counterfactual-Bee5_BIn.d.ts.map → counterfactual-CLgrwhkY.d.ts.map} +1 -1
  40. package/dist/{chat-client-DI79OPye.js → default-registry-B0bKikCb.js} +3 -39
  41. package/dist/default-registry-B0bKikCb.js.map +1 -0
  42. package/dist/{default-registry-XxedTLwu.d.ts → default-registry-BKwc8bN5.d.ts} +6 -6
  43. package/dist/{default-registry-XxedTLwu.d.ts.map → default-registry-BKwc8bN5.d.ts.map} +1 -1
  44. package/dist/{define-agent-eval-0wW7gFhr.d.ts → define-agent-eval-CY6qdlGV.d.ts} +6 -6
  45. package/dist/{define-agent-eval-0wW7gFhr.d.ts.map → define-agent-eval-CY6qdlGV.d.ts.map} +1 -1
  46. package/dist/{define-agent-eval-jS8xj_Q_.js → define-agent-eval-D_i_s69h.js} +7 -7
  47. package/dist/{define-agent-eval-jS8xj_Q_.js.map → define-agent-eval-D_i_s69h.js.map} +1 -1
  48. package/dist/{dspy-rlm-engine-CS3qcCEk.js → dspy-rlm-engine-D5byiHn9.js} +5 -6
  49. package/dist/dspy-rlm-engine-D5byiHn9.js.map +1 -0
  50. package/dist/{emitter-Bvnu0VzL.d.ts → emitter-Cs0egaFd.d.ts} +3 -3
  51. package/dist/{emitter-Bvnu0VzL.d.ts.map → emitter-Cs0egaFd.d.ts.map} +1 -1
  52. package/dist/{engine-BfRay1qD.d.ts → engine-DhFir3Ys.d.ts} +23 -8
  53. package/dist/{engine-BfRay1qD.d.ts.map → engine-DhFir3Ys.d.ts.map} +1 -1
  54. package/dist/{eval-campaign-JDTeE6Pl.js → eval-campaign-BeAjdhzC.js} +2 -2
  55. package/dist/{eval-campaign-JDTeE6Pl.js.map → eval-campaign-BeAjdhzC.js.map} +1 -1
  56. package/dist/{exact-types-BEecmnWm.d.ts → exact-types-BKOEILRP.d.ts} +2 -2
  57. package/dist/{exact-types-BEecmnWm.d.ts.map → exact-types-BKOEILRP.d.ts.map} +1 -1
  58. package/dist/experiment/index.d.ts +4 -4
  59. package/dist/{external-optimizer-contracts-szBJ_1vh.d.ts → external-optimizer-contracts-CQCpyrIL.d.ts} +2 -2
  60. package/dist/{external-optimizer-contracts-szBJ_1vh.d.ts.map → external-optimizer-contracts-CQCpyrIL.d.ts.map} +1 -1
  61. package/dist/{external-optimizer-process-CQxylYeG.js → external-optimizer-process-Cq_Pg15r.js} +2 -2
  62. package/dist/{external-optimizer-process-CQxylYeG.js.map → external-optimizer-process-Cq_Pg15r.js.map} +1 -1
  63. package/dist/{external-optimizer-subprocess-Cex8Da2i.js → external-optimizer-subprocess-DgNebftP.js} +2 -2
  64. package/dist/{external-optimizer-subprocess-Cex8Da2i.js.map → external-optimizer-subprocess-DgNebftP.js.map} +1 -1
  65. package/dist/failure-cluster-OldNRoAt.d.ts +154 -0
  66. package/dist/failure-cluster-OldNRoAt.d.ts.map +1 -0
  67. package/dist/{feedback-trajectory-DIqpCyF0.d.ts → feedback-trajectory-CMnv_uYs.d.ts} +6 -6
  68. package/dist/{feedback-trajectory-DIqpCyF0.d.ts.map → feedback-trajectory-CMnv_uYs.d.ts.map} +1 -1
  69. package/dist/{heldout-gate-JgNRDZwZ.d.ts → heldout-gate-Df5hsqmm.d.ts} +7 -7
  70. package/dist/{heldout-gate-JgNRDZwZ.d.ts.map → heldout-gate-Df5hsqmm.d.ts.map} +1 -1
  71. package/dist/hosted/index.d.ts +2 -2
  72. package/dist/hosted/index.js +1 -1
  73. package/dist/{index-DnglhM0A.d.ts → index-BQqOjerE.d.ts} +94 -13
  74. package/dist/index-BQqOjerE.d.ts.map +1 -0
  75. package/dist/{index-_vPrVMRX.d.ts → index-CFDffsKz.d.ts} +11 -11
  76. package/dist/{index-_vPrVMRX.d.ts.map → index-CFDffsKz.d.ts.map} +1 -1
  77. package/dist/{index-DDAPhUJJ.d.ts → index-D0Db5X-4.d.ts} +45 -9
  78. package/dist/index-D0Db5X-4.d.ts.map +1 -0
  79. package/dist/{index-DMoxLG8P.d.ts → index-e7LXeRVa.d.ts} +3 -3
  80. package/dist/{index-DMoxLG8P.d.ts.map → index-e7LXeRVa.d.ts.map} +1 -1
  81. package/dist/index.d.ts +172 -46
  82. package/dist/index.d.ts.map +1 -1
  83. package/dist/index.js +64 -41
  84. package/dist/index.js.map +1 -1
  85. package/dist/{insight-report-08F022xN.d.ts → insight-report-DETqPc_A.d.ts} +4 -4
  86. package/dist/{insight-report-08F022xN.d.ts.map → insight-report-DETqPc_A.d.ts.map} +1 -1
  87. package/dist/{integrity-B_EDELom.d.ts → integrity-BKTcA-HP.d.ts} +3 -3
  88. package/dist/{integrity-B_EDELom.d.ts.map → integrity-BKTcA-HP.d.ts.map} +1 -1
  89. package/dist/{kind-factory-DMeEoMQZ.js → kind-factory-gP6lDySe.js} +164 -149
  90. package/dist/kind-factory-gP6lDySe.js.map +1 -0
  91. package/dist/{llm-client-BFMRpmqb.js → llm-client-CxQtdtd6.js} +12 -5
  92. package/dist/llm-client-CxQtdtd6.js.map +1 -0
  93. package/dist/{llm-judge-aQHIk5_-.js → llm-judge-B2YxbAJb.js} +81 -7
  94. package/dist/{llm-judge-aQHIk5_-.js.map → llm-judge-B2YxbAJb.js.map} +1 -1
  95. package/dist/{matrix-Ch8JO1pG.d.ts → matrix-DGu8KhSs.d.ts} +2 -2
  96. package/dist/{matrix-Ch8JO1pG.d.ts.map → matrix-DGu8KhSs.d.ts.map} +1 -1
  97. package/dist/meta-eval/index.d.ts +100 -5
  98. package/dist/meta-eval/index.d.ts.map +1 -1
  99. package/dist/meta-eval/index.js +200 -2
  100. package/dist/meta-eval/index.js.map +1 -1
  101. package/dist/{mint-DjfDUMHr.js → mint-vWOdD8Ae.js} +2 -2
  102. package/dist/{mint-DjfDUMHr.js.map → mint-vWOdD8Ae.js.map} +1 -1
  103. package/dist/multishot/golden/index.d.ts +1 -1
  104. package/dist/multishot/index.d.ts +2 -2
  105. package/dist/openapi.json +1 -1
  106. package/dist/pipelines/index.d.ts +5 -5
  107. package/dist/pipelines/index.js +3 -3
  108. package/dist/{pre-registration-DHz6P_6f.d.ts → pre-registration-BoI4ucR3.d.ts} +2 -2
  109. package/dist/{pre-registration-DHz6P_6f.d.ts.map → pre-registration-BoI4ucR3.d.ts.map} +1 -1
  110. package/dist/{produced-state-CxmbFxFd.js → produced-state-7VYDwtkk.js} +3 -3
  111. package/dist/{produced-state-CxmbFxFd.js.map → produced-state-7VYDwtkk.js.map} +1 -1
  112. package/dist/{promotion-policy-BBBcz5_3.d.ts → promotion-policy-CvMda3kU.d.ts} +2 -2
  113. package/dist/{promotion-policy-BBBcz5_3.d.ts.map → promotion-policy-CvMda3kU.d.ts.map} +1 -1
  114. package/dist/{provenance-Dp-vvyrU.d.ts → provenance-CRY67X50.d.ts} +39 -165
  115. package/dist/provenance-CRY67X50.d.ts.map +1 -0
  116. package/dist/{query-BPGMVlbM.js → query-D1nLIKt7.js} +2 -2
  117. package/dist/{query-BPGMVlbM.js.map → query-D1nLIKt7.js.map} +1 -1
  118. package/dist/{query-Na5gEIGd.d.ts → query-D6W6MaGx.d.ts} +3 -3
  119. package/dist/{query-Na5gEIGd.d.ts.map → query-D6W6MaGx.d.ts.map} +1 -1
  120. package/dist/{registry-xEb_xfns.d.ts → registry-7pOUBrtX.d.ts} +4 -4
  121. package/dist/{registry-xEb_xfns.d.ts.map → registry-7pOUBrtX.d.ts.map} +1 -1
  122. package/dist/{release-confidence-D6lQw_o7.d.ts → release-confidence-BAcNYOf1.d.ts} +4 -4
  123. package/dist/{release-confidence-D6lQw_o7.d.ts.map → release-confidence-BAcNYOf1.d.ts.map} +1 -1
  124. package/dist/{release-confidence-CzUHc4z4.js → release-confidence-BsGEg_xg.js} +3 -3
  125. package/dist/{release-confidence-CzUHc4z4.js.map → release-confidence-BsGEg_xg.js.map} +1 -1
  126. package/dist/reporting.d.ts +3 -3
  127. package/dist/reporting.js +1 -1
  128. package/dist/{researcher-CMUTQXD7.d.ts → researcher-jsW1X94L.d.ts} +7 -8
  129. package/dist/researcher-jsW1X94L.d.ts.map +1 -0
  130. package/dist/{reward-hacking-SkxYgT0x.js → reward-hacking-CKW4teig.js} +2 -2
  131. package/dist/{reward-hacking-SkxYgT0x.js.map → reward-hacking-CKW4teig.js.map} +1 -1
  132. package/dist/{reward-hacking-CgPRUesA.d.ts → reward-hacking-ZXEi9VCq.d.ts} +2 -2
  133. package/dist/{reward-hacking-CgPRUesA.d.ts.map → reward-hacking-ZXEi9VCq.d.ts.map} +1 -1
  134. package/dist/rl.d.ts +7 -7
  135. package/dist/rl.js +4 -4
  136. package/dist/rollout/index.d.ts +1 -1
  137. package/dist/rollout/index.js +2 -2
  138. package/dist/{rollout-Crypdx8s.js → rollout-C-znbbYg.js} +2 -2
  139. package/dist/{rollout-Crypdx8s.js.map → rollout-C-znbbYg.js.map} +1 -1
  140. package/dist/{rubric-predictive-validity-DluJLCKQ.d.ts → rubric-predictive-validity-Dl1dvKCv.d.ts} +2 -2
  141. package/dist/{rubric-predictive-validity-DluJLCKQ.d.ts.map → rubric-predictive-validity-Dl1dvKCv.d.ts.map} +1 -1
  142. package/dist/{run-record-DQjRcYwA.d.ts → run-record-DTv1MdjK.d.ts} +2 -2
  143. package/dist/{run-record-DQjRcYwA.d.ts.map → run-record-DTv1MdjK.d.ts.map} +1 -1
  144. package/dist/{run-record-DLORoL7t.js → run-record-ZIsR9Fif.js} +2 -2
  145. package/dist/{run-record-DLORoL7t.js.map → run-record-ZIsR9Fif.js.map} +1 -1
  146. package/dist/{schema-DID1Cqct.d.ts → schema-CR5cpjQ3.d.ts} +57 -2
  147. package/dist/{schema-DID1Cqct.d.ts.map → schema-CR5cpjQ3.d.ts.map} +1 -1
  148. package/dist/{schema-CdIX2aHu.js → schema-CSf6qWgZ.js} +35 -1
  149. package/dist/{schema-CdIX2aHu.js.map → schema-CSf6qWgZ.js.map} +1 -1
  150. package/dist/semantic-concept-judge-Ct3QU7t5.js +780 -0
  151. package/dist/semantic-concept-judge-Ct3QU7t5.js.map +1 -0
  152. package/dist/{series-convergence-D9WgpXGi.d.ts → series-convergence-DeG33RpC.d.ts} +2 -2
  153. package/dist/{series-convergence-D9WgpXGi.d.ts.map → series-convergence-DeG33RpC.d.ts.map} +1 -1
  154. package/dist/{server-CCEnywOR.js → server-BR6onwZB.js} +2 -2
  155. package/dist/{server-CCEnywOR.js.map → server-BR6onwZB.js.map} +1 -1
  156. package/dist/{skillopt-optimization-method-LHi02MzH.js → skillopt-optimization-method-BzdphODy.js} +6 -6
  157. package/dist/{skillopt-optimization-method-LHi02MzH.js.map → skillopt-optimization-method-BzdphODy.js.map} +1 -1
  158. package/dist/{statistical-heldout-Yldkntvy.d.ts → statistical-heldout-DTyB_6-1.d.ts} +3 -3
  159. package/dist/{statistical-heldout-Yldkntvy.d.ts.map → statistical-heldout-DTyB_6-1.d.ts.map} +1 -1
  160. package/dist/{store-Cq9oOrI1.d.ts → store-BErPvYBr.d.ts} +2 -2
  161. package/dist/{store-Cq9oOrI1.d.ts.map → store-BErPvYBr.d.ts.map} +1 -1
  162. package/dist/{store-otlp-CHjBvWQY.js → store-otlp-Dow0pk_5.js} +2 -2
  163. package/dist/{store-otlp-CHjBvWQY.js.map → store-otlp-Dow0pk_5.js.map} +1 -1
  164. package/dist/{store-tool-spans-BvdUbeOB.d.ts → store-tool-spans-CCZNsihA.d.ts} +8 -8
  165. package/dist/{store-tool-spans-BvdUbeOB.d.ts.map → store-tool-spans-CCZNsihA.d.ts.map} +1 -1
  166. package/dist/{store-tool-spans-B9o6tU8f.js → store-tool-spans-CeNj_m2L.js} +3 -3
  167. package/dist/{store-tool-spans-B9o6tU8f.js.map → store-tool-spans-CeNj_m2L.js.map} +1 -1
  168. package/dist/storyboard/index.d.ts +1 -1
  169. package/dist/{summary-report-DRstQNBX.d.ts → summary-report-gMrbYawB.d.ts} +3 -3
  170. package/dist/{summary-report-DRstQNBX.d.ts.map → summary-report-gMrbYawB.d.ts.map} +1 -1
  171. package/dist/{task-failure-attributes-CBGtLS_H.js → task-failure-attributes-CZjZeBsY.js} +3 -3
  172. package/dist/{task-failure-attributes-CBGtLS_H.js.map → task-failure-attributes-CZjZeBsY.js.map} +1 -1
  173. package/dist/{tool-groups-DjwlMBvW.d.ts → tool-groups-Cp4Xdzrp.d.ts} +3 -3
  174. package/dist/tool-groups-Cp4Xdzrp.d.ts.map +1 -0
  175. package/dist/{tool-waste-CwGHzBzX.js → tool-waste-B9tdWV6g.js} +253 -6
  176. package/dist/tool-waste-B9tdWV6g.js.map +1 -0
  177. package/dist/{tool-waste-BrmLKxMw.d.ts → tool-waste-D23I0zWm.d.ts} +4 -4
  178. package/dist/{tool-waste-BrmLKxMw.d.ts.map → tool-waste-D23I0zWm.d.ts.map} +1 -1
  179. package/dist/trace-repair/index.d.ts +2 -2
  180. package/dist/traces.d.ts +10 -10
  181. package/dist/traces.js +7 -7
  182. package/dist/{trajectory-r1bQqvBQ.d.ts → trajectory-D7qrNvaN.d.ts} +3 -3
  183. package/dist/{trajectory-r1bQqvBQ.d.ts.map → trajectory-D7qrNvaN.d.ts.map} +1 -1
  184. package/dist/trajectory-replay/index.d.ts +3 -3
  185. package/dist/{types-CCZ34qmV.d.ts → types-BDV4PiMR.d.ts} +3 -3
  186. package/dist/{types-CCZ34qmV.d.ts.map → types-BDV4PiMR.d.ts.map} +1 -1
  187. package/dist/{types-nokrtr7M.d.ts → types-Ba5UQyVD.d.ts} +4 -4
  188. package/dist/{types-nokrtr7M.d.ts.map → types-Ba5UQyVD.d.ts.map} +1 -1
  189. package/dist/{types-DMoNFDWi.d.ts → types-DN2WdT5S.d.ts} +3 -3
  190. package/dist/{types-DMoNFDWi.d.ts.map → types-DN2WdT5S.d.ts.map} +1 -1
  191. package/dist/types-gvRsyJLh.d.ts +831 -0
  192. package/dist/types-gvRsyJLh.d.ts.map +1 -0
  193. package/dist/wire/index.d.ts +3 -3
  194. package/dist/wire/index.js +1 -1
  195. package/docs/plants.md +69 -1
  196. package/docs/public-api.md +6 -4
  197. package/docs/trace-analysis.md +55 -8
  198. package/package.json +1 -1
  199. package/dist/backend-integrity-e79K3UPD.d.ts.map +0 -1
  200. package/dist/bounded-process-CVOC_D3H.js.map +0 -1
  201. package/dist/chat-client-DI79OPye.js.map +0 -1
  202. package/dist/chat-json-call-5Jxna-aV.js.map +0 -1
  203. package/dist/dspy-rlm-engine-CS3qcCEk.js.map +0 -1
  204. package/dist/failure-cluster-6YSvsKlp.d.ts +0 -58
  205. package/dist/failure-cluster-6YSvsKlp.d.ts.map +0 -1
  206. package/dist/index-DDAPhUJJ.d.ts.map +0 -1
  207. package/dist/index-DnglhM0A.d.ts.map +0 -1
  208. package/dist/kind-factory-DMeEoMQZ.js.map +0 -1
  209. package/dist/llm-client-BFMRpmqb.js.map +0 -1
  210. package/dist/provenance-Dp-vvyrU.d.ts.map +0 -1
  211. package/dist/raw-provider-sink-BU29Sh8h.d.ts +0 -134
  212. package/dist/raw-provider-sink-BU29Sh8h.d.ts.map +0 -1
  213. package/dist/researcher-CMUTQXD7.d.ts.map +0 -1
  214. package/dist/semantic-concept-judge-I36eejJx.js +0 -382
  215. package/dist/semantic-concept-judge-I36eejJx.js.map +0 -1
  216. package/dist/tool-groups-DjwlMBvW.d.ts.map +0 -1
  217. package/dist/tool-waste-CwGHzBzX.js.map +0 -1
  218. package/dist/types-Bfk0uxRj.d.ts +0 -443
  219. package/dist/types-Bfk0uxRj.d.ts.map +0 -1
@@ -14,6 +14,22 @@ const TRACE_SCHEMA_VERSION = "1.0.0";
14
14
  /**
15
15
  * The failure taxonomy. `FailureClass` derives from this array, so the type
16
16
  * and the runtime list cannot name different sets.
17
+ *
18
+ * Two layers, and the split is what `FAILURE_BLAME` reads
19
+ * (`src/failure-taxonomy.ts`):
20
+ * - behaviour classes name what the agent did wrong, and a run that ends in
21
+ * one is evidence about the agent;
22
+ * - transport classes name a death before or around the model turn — the
23
+ * bridge refused, the box never provisioned, the stream tore — and a run
24
+ * that ends in one of the machine-blamed members is not evidence about the
25
+ * agent at all.
26
+ *
27
+ * The transport layer exists because a taxonomy with no name for "the run
28
+ * never reached a model" charges an infrastructure outage to the agent under
29
+ * test. Measured: six director cells settled with the runtime journal's own
30
+ * `infra` flag reading FALSE while the real reason was
31
+ * `host-executor: acquire timeout after 900000ms (in_flight=10/10)` — the
32
+ * bridge's host lane had saturated.
17
33
  */
18
34
  const FAILURE_CLASSES = [
19
35
  "success",
@@ -50,6 +66,24 @@ const FAILURE_CLASSES = [
50
66
  "contradictory_evidence",
51
67
  "ambiguous_user_intent",
52
68
  "knowledge_readiness_blocked",
69
+ "bridge_unreachable",
70
+ "bridge_lane_saturated",
71
+ "router_usage_mismatch",
72
+ "session_profile_conflict",
73
+ "runtime_restart",
74
+ "root_driver_failed",
75
+ "box_provision_failure",
76
+ "box_teardown_failure",
77
+ "profile_refused",
78
+ "empty_completion",
79
+ "stream_protocol_error",
80
+ "stream_incomplete",
81
+ "message_sealed",
82
+ "process_killed",
83
+ "cancelled",
84
+ "terminated",
85
+ "traversal_cap_exhausted",
86
+ "unreported",
53
87
  "unknown"
54
88
  ];
55
89
  function isLlmSpan(s) {
@@ -64,4 +98,4 @@ function isJudgeSpan(s) {
64
98
  //#endregion
65
99
  export { isToolSpan as a, isLlmSpan as i, TRACE_SCHEMA_VERSION as n, isJudgeSpan as r, FAILURE_CLASSES as t };
66
100
 
67
- //# sourceMappingURL=schema-CdIX2aHu.js.map
101
+ //# sourceMappingURL=schema-CSf6qWgZ.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"schema-CdIX2aHu.js","names":[],"sources":["../src/trace/schema.ts"],"sourcesContent":["/**\n * TraceSchema v1 — the canonical data model for agent-eval.\n *\n * Every score, every failure class, every pipeline in the framework is\n * a view over this data. Shape it once, live with it.\n *\n * Wire-compatible with OpenTelemetry span semantics (see trace/otel.ts)\n * but extended with agent-specific span kinds (llm, tool, retrieval,\n * judge, sandbox) and first-class BudgetLedger / Artifact / JudgeVerdict\n * entities that OTEL leaves as free-form attributes.\n */\n\nexport const TRACE_SCHEMA_VERSION = '1.0.0'\n\n// ── Run ──────────────────────────────────────────────────────────────\n\nexport type RunStatus = 'running' | 'completed' | 'failed' | 'aborted'\n\nexport interface BudgetSpec {\n tokens?: number\n wallMs?: number\n calls?: number\n usd?: number\n}\n\nexport interface RunOutcome {\n score?: number\n pass?: boolean\n failureClass?: FailureClass\n notes?: string\n}\n\n/**\n * Layer — optional classification in a nested build workflow.\n * `builder`: the meta-agent editing a project (e.g. agent-builder Forge chat).\n * `app-build`: sandbox harness that compiled + tested the generated scaffold.\n * `app-runtime`: a run of the generated agent against a domain scenario.\n * `meta`: any meta-eval (judge replay, correlation analysis).\n */\nexport type RunLayer = 'builder' | 'app-build' | 'app-runtime' | 'meta' | 'custom'\n\nexport interface Run {\n runId: string\n /**\n * Stable identifier of the scenario being executed.\n *\n * Always populated on the persisted Run — but `TraceEmitter.startRun` accepts\n * input WITHOUT this field, substituting a sensible default\n * (`run.layer ?? run.tags?.['kind'] ?? 'runtime'`) when the caller has no\n * curated scenario to anchor to (runtime / operator / meta-eval runs). This\n * keeps the persisted shape unambiguous for downstream filters + aggregations\n * while removing the boilerplate of inventing placeholder ids at the call site.\n */\n scenarioId: string\n variantId?: string\n datasetVersion?: string\n /** Git SHA of agent code at run time. */\n codeSha?: string\n /** Hash of the prompt template + any system prompt. */\n promptSha?: string\n /** Model id + date + system-prompt hash, concatenated. */\n modelFingerprint?: string\n seed?: number\n /** Arbitrary environment markers (shell, docker version, tz). */\n envFingerprint?: Record<string, string>\n /** Version of the redaction rules applied to this run. */\n redactionVersion?: string\n /** Parent run in a nested build workflow. A builder run's children are\n * app-build runs; those children are app-runtime runs. */\n parentRunId?: string\n /** Stable project identifier — groups runs across chats + sessions. */\n projectId?: string\n /** Chat/conversation identifier within a project. */\n chatId?: string\n /** Layer classification — hint for aggregation; not enforced. */\n layer?: RunLayer\n startedAt: number\n endedAt?: number\n status: RunStatus\n outcome?: RunOutcome\n budget?: BudgetSpec\n /** Free-form labels for downstream grouping. */\n tags?: Record<string, string>\n}\n\n// ── Spans (hierarchical work units) ──────────────────────────────────\n\nexport type SpanKind = 'agent' | 'llm' | 'tool' | 'retrieval' | 'judge' | 'sandbox' | 'custom'\n\nexport type SpanStatus = 'ok' | 'error'\n\nexport interface SpanBase {\n spanId: string\n parentSpanId?: string\n runId: string\n kind: SpanKind\n name: string\n startedAt: number\n endedAt?: number\n status?: SpanStatus\n error?: string\n /** Anything not covered by typed fields. Kept deliberately free-form. */\n attributes?: Record<string, unknown>\n}\n\nexport interface Message {\n role: 'system' | 'user' | 'assistant' | 'tool'\n content: string\n tokens?: number\n /** Multi-modal content descriptors; blobs themselves live in Artifacts. */\n images?: Array<{ artifactId?: string; url?: string; mime?: string }>\n}\n\nexport interface LlmSpan extends SpanBase {\n kind: 'llm'\n model: string\n messages: Message[]\n output?: string\n inputTokens?: number\n /** All generated tokens, including the reasoning subset when present. */\n outputTokens?: number\n cachedTokens?: number\n cacheWriteTokens?: number\n /** Reasoning-token subset of `outputTokens`. */\n reasoningTokens?: number\n costUsd?: number\n finishReason?: string\n}\n\nexport interface ToolSpan extends SpanBase {\n kind: 'tool'\n toolName: string\n args: unknown\n /** False when the source observed the call but did not capture its arguments. */\n argsCaptured?: boolean\n result?: unknown\n latencyMs?: number\n}\n\nexport interface RetrievalSpan extends SpanBase {\n kind: 'retrieval'\n query: string\n hits: Array<{ docId: string; score: number; content?: string }>\n}\n\nexport interface JudgeSpan extends SpanBase {\n kind: 'judge'\n judgeId: string\n /** Span this judgment applies to. */\n targetSpanId: string\n dimension: string\n /** Numeric score (free-range; interpretation up to the judge). */\n score: number\n rationale?: string\n evidence?: string\n}\n\nexport interface SandboxSpan extends SpanBase {\n kind: 'sandbox'\n image?: string\n command?: string\n exitCode?: number\n testsTotal?: number\n testsPassed?: number\n stdoutHash?: string\n stderrHash?: string\n /** Duration in ms; the harness fills this explicitly (endedAt - startedAt may miss setup). */\n wallMs?: number\n}\n\nexport interface GenericSpan extends SpanBase {\n kind: 'agent' | 'custom'\n}\n\nexport type Span = LlmSpan | ToolSpan | RetrievalSpan | JudgeSpan | SandboxSpan | GenericSpan\n\n// ── Events (point-in-time occurrences within a span) ─────────────────\n\nexport type EventKind =\n | 'log'\n | 'error'\n | 'budget_decrement'\n | 'budget_breach'\n | 'state_mutation'\n | 'policy_violation'\n | 'redaction_applied'\n | 'custom'\n\nexport interface TraceEvent {\n eventId: string\n runId: string\n spanId?: string\n kind: EventKind\n timestamp: number\n payload: Record<string, unknown>\n}\n\n// ── Budget ledger (running token/wall/call/$ accounting) ─────────────\n\nexport interface BudgetLedgerEntry {\n runId: string\n dimension: keyof BudgetSpec\n limit: number\n consumed: number\n remaining: number\n timestamp: number\n breached: boolean\n /** Span that triggered this entry, if any. */\n spanId?: string\n}\n\n// ── Artifacts (blobs addressed by hash) ──────────────────────────────\n\nexport interface Artifact {\n artifactId: string\n runId: string\n spanId?: string\n contentType: string\n sizeBytes: number\n /** sha256 in hex. */\n hash: string\n /** External storage URL (R2, S3, filesystem path). */\n storageUrl?: string\n /** Inline content for small blobs — keep under ~64KB. */\n inlineContent?: string\n}\n\n// ── Failure taxonomy ─────────────────────────────────────────────────\n\n/**\n * The failure taxonomy. `FailureClass` derives from this array, so the type\n * and the runtime list cannot name different sets.\n */\nexport const FAILURE_CLASSES = [\n 'success',\n 'reasoning_error',\n 'tool_selection_error',\n 'tool_argument_error',\n 'tool_recovery_failure',\n 'hallucination',\n 'instruction_following',\n 'safety_refusal_miss',\n 'policy_violation',\n 'budget_exceeded',\n 'format_drift',\n 'permission_escalation',\n 'pii_leak',\n 'cost_overrun',\n 'timeout',\n 'sandbox_failure',\n 'missing_user_data',\n 'missing_domain_data',\n 'missing_codebase_context',\n 'missing_runtime_context',\n 'missing_credentials',\n 'missing_integration_connection',\n 'missing_integration_scope',\n 'integration_approval_required',\n 'integration_auth_expired',\n 'integration_provider_failure',\n 'bad_integration_manifest',\n 'unsafe_integration_write_denied',\n 'stale_external_data',\n 'bad_retrieval',\n 'insufficient_evidence',\n 'contradictory_evidence',\n 'ambiguous_user_intent',\n 'knowledge_readiness_blocked',\n 'unknown',\n] as const\n\nexport type FailureClass = (typeof FAILURE_CLASSES)[number]\n\n// ── Helpers ──────────────────────────────────────────────────────────\n\nexport function isLlmSpan(s: Span): s is LlmSpan {\n return s.kind === 'llm'\n}\nexport function isToolSpan(s: Span): s is ToolSpan {\n return s.kind === 'tool'\n}\nexport function isJudgeSpan(s: Span): s is JudgeSpan {\n return s.kind === 'judge'\n}\n"],"mappings":";;;;;;;;;;;;AAYA,MAAa,uBAAuB;;;;;AA6NpC,MAAa,kBAAkB;CAC7B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF;AAMA,SAAgB,UAAU,GAAuB;CAC/C,OAAO,EAAE,SAAS;AACpB;AACA,SAAgB,WAAW,GAAwB;CACjD,OAAO,EAAE,SAAS;AACpB;AACA,SAAgB,YAAY,GAAyB;CACnD,OAAO,EAAE,SAAS;AACpB"}
1
+ {"version":3,"file":"schema-CSf6qWgZ.js","names":[],"sources":["../src/trace/schema.ts"],"sourcesContent":["/**\n * TraceSchema v1 — the canonical data model for agent-eval.\n *\n * Every score, every failure class, every pipeline in the framework is\n * a view over this data. Shape it once, live with it.\n *\n * Wire-compatible with OpenTelemetry span semantics (see trace/otel.ts)\n * but extended with agent-specific span kinds (llm, tool, retrieval,\n * judge, sandbox) and first-class BudgetLedger / Artifact / JudgeVerdict\n * entities that OTEL leaves as free-form attributes.\n */\n\nexport const TRACE_SCHEMA_VERSION = '1.0.0'\n\n// ── Run ──────────────────────────────────────────────────────────────\n\nexport type RunStatus = 'running' | 'completed' | 'failed' | 'aborted'\n\nexport interface BudgetSpec {\n tokens?: number\n wallMs?: number\n calls?: number\n usd?: number\n}\n\nexport interface RunOutcome {\n score?: number\n pass?: boolean\n failureClass?: FailureClass\n notes?: string\n}\n\n/**\n * Layer — optional classification in a nested build workflow.\n * `builder`: the meta-agent editing a project (e.g. agent-builder Forge chat).\n * `app-build`: sandbox harness that compiled + tested the generated scaffold.\n * `app-runtime`: a run of the generated agent against a domain scenario.\n * `meta`: any meta-eval (judge replay, correlation analysis).\n */\nexport type RunLayer = 'builder' | 'app-build' | 'app-runtime' | 'meta' | 'custom'\n\nexport interface Run {\n runId: string\n /**\n * Stable identifier of the scenario being executed.\n *\n * Always populated on the persisted Run — but `TraceEmitter.startRun` accepts\n * input WITHOUT this field, substituting a sensible default\n * (`run.layer ?? run.tags?.['kind'] ?? 'runtime'`) when the caller has no\n * curated scenario to anchor to (runtime / operator / meta-eval runs). This\n * keeps the persisted shape unambiguous for downstream filters + aggregations\n * while removing the boilerplate of inventing placeholder ids at the call site.\n */\n scenarioId: string\n variantId?: string\n datasetVersion?: string\n /** Git SHA of agent code at run time. */\n codeSha?: string\n /** Hash of the prompt template + any system prompt. */\n promptSha?: string\n /** Model id + date + system-prompt hash, concatenated. */\n modelFingerprint?: string\n seed?: number\n /** Arbitrary environment markers (shell, docker version, tz). */\n envFingerprint?: Record<string, string>\n /** Version of the redaction rules applied to this run. */\n redactionVersion?: string\n /** Parent run in a nested build workflow. A builder run's children are\n * app-build runs; those children are app-runtime runs. */\n parentRunId?: string\n /** Stable project identifier — groups runs across chats + sessions. */\n projectId?: string\n /** Chat/conversation identifier within a project. */\n chatId?: string\n /** Layer classification — hint for aggregation; not enforced. */\n layer?: RunLayer\n startedAt: number\n endedAt?: number\n status: RunStatus\n outcome?: RunOutcome\n budget?: BudgetSpec\n /** Free-form labels for downstream grouping. */\n tags?: Record<string, string>\n}\n\n// ── Spans (hierarchical work units) ──────────────────────────────────\n\nexport type SpanKind = 'agent' | 'llm' | 'tool' | 'retrieval' | 'judge' | 'sandbox' | 'custom'\n\nexport type SpanStatus = 'ok' | 'error'\n\nexport interface SpanBase {\n spanId: string\n parentSpanId?: string\n runId: string\n kind: SpanKind\n name: string\n startedAt: number\n endedAt?: number\n status?: SpanStatus\n error?: string\n /** Anything not covered by typed fields. Kept deliberately free-form. */\n attributes?: Record<string, unknown>\n}\n\nexport interface Message {\n role: 'system' | 'user' | 'assistant' | 'tool'\n content: string\n tokens?: number\n /** Multi-modal content descriptors; blobs themselves live in Artifacts. */\n images?: Array<{ artifactId?: string; url?: string; mime?: string }>\n}\n\nexport interface LlmSpan extends SpanBase {\n kind: 'llm'\n model: string\n messages: Message[]\n output?: string\n inputTokens?: number\n /** All generated tokens, including the reasoning subset when present. */\n outputTokens?: number\n cachedTokens?: number\n cacheWriteTokens?: number\n /** Reasoning-token subset of `outputTokens`. */\n reasoningTokens?: number\n costUsd?: number\n finishReason?: string\n}\n\nexport interface ToolSpan extends SpanBase {\n kind: 'tool'\n toolName: string\n args: unknown\n /** False when the source observed the call but did not capture its arguments. */\n argsCaptured?: boolean\n result?: unknown\n latencyMs?: number\n}\n\nexport interface RetrievalSpan extends SpanBase {\n kind: 'retrieval'\n query: string\n hits: Array<{ docId: string; score: number; content?: string }>\n}\n\nexport interface JudgeSpan extends SpanBase {\n kind: 'judge'\n judgeId: string\n /** Span this judgment applies to. */\n targetSpanId: string\n dimension: string\n /** Numeric score (free-range; interpretation up to the judge). */\n score: number\n rationale?: string\n evidence?: string\n}\n\nexport interface SandboxSpan extends SpanBase {\n kind: 'sandbox'\n image?: string\n command?: string\n exitCode?: number\n testsTotal?: number\n testsPassed?: number\n stdoutHash?: string\n stderrHash?: string\n /** Duration in ms; the harness fills this explicitly (endedAt - startedAt may miss setup). */\n wallMs?: number\n}\n\nexport interface GenericSpan extends SpanBase {\n kind: 'agent' | 'custom'\n}\n\nexport type Span = LlmSpan | ToolSpan | RetrievalSpan | JudgeSpan | SandboxSpan | GenericSpan\n\n// ── Events (point-in-time occurrences within a span) ─────────────────\n\nexport type EventKind =\n | 'log'\n | 'error'\n | 'budget_decrement'\n | 'budget_breach'\n | 'state_mutation'\n | 'policy_violation'\n | 'redaction_applied'\n | 'custom'\n\nexport interface TraceEvent {\n eventId: string\n runId: string\n spanId?: string\n kind: EventKind\n timestamp: number\n payload: Record<string, unknown>\n}\n\n// ── Budget ledger (running token/wall/call/$ accounting) ─────────────\n\nexport interface BudgetLedgerEntry {\n runId: string\n dimension: keyof BudgetSpec\n limit: number\n consumed: number\n remaining: number\n timestamp: number\n breached: boolean\n /** Span that triggered this entry, if any. */\n spanId?: string\n}\n\n// ── Artifacts (blobs addressed by hash) ──────────────────────────────\n\nexport interface Artifact {\n artifactId: string\n runId: string\n spanId?: string\n contentType: string\n sizeBytes: number\n /** sha256 in hex. */\n hash: string\n /** External storage URL (R2, S3, filesystem path). */\n storageUrl?: string\n /** Inline content for small blobs — keep under ~64KB. */\n inlineContent?: string\n}\n\n// ── Failure taxonomy ─────────────────────────────────────────────────\n\n/**\n * The failure taxonomy. `FailureClass` derives from this array, so the type\n * and the runtime list cannot name different sets.\n *\n * Two layers, and the split is what `FAILURE_BLAME` reads\n * (`src/failure-taxonomy.ts`):\n * - behaviour classes name what the agent did wrong, and a run that ends in\n * one is evidence about the agent;\n * - transport classes name a death before or around the model turn — the\n * bridge refused, the box never provisioned, the stream tore — and a run\n * that ends in one of the machine-blamed members is not evidence about the\n * agent at all.\n *\n * The transport layer exists because a taxonomy with no name for \"the run\n * never reached a model\" charges an infrastructure outage to the agent under\n * test. Measured: six director cells settled with the runtime journal's own\n * `infra` flag reading FALSE while the real reason was\n * `host-executor: acquire timeout after 900000ms (in_flight=10/10)` — the\n * bridge's host lane had saturated.\n */\nexport const FAILURE_CLASSES = [\n 'success',\n 'reasoning_error',\n 'tool_selection_error',\n 'tool_argument_error',\n 'tool_recovery_failure',\n 'hallucination',\n 'instruction_following',\n 'safety_refusal_miss',\n 'policy_violation',\n 'budget_exceeded',\n 'format_drift',\n 'permission_escalation',\n 'pii_leak',\n 'cost_overrun',\n 'timeout',\n 'sandbox_failure',\n 'missing_user_data',\n 'missing_domain_data',\n 'missing_codebase_context',\n 'missing_runtime_context',\n 'missing_credentials',\n 'missing_integration_connection',\n 'missing_integration_scope',\n 'integration_approval_required',\n 'integration_auth_expired',\n 'integration_provider_failure',\n 'bad_integration_manifest',\n 'unsafe_integration_write_denied',\n 'stale_external_data',\n 'bad_retrieval',\n 'insufficient_evidence',\n 'contradictory_evidence',\n 'ambiguous_user_intent',\n 'knowledge_readiness_blocked',\n\n // ── Transport: the run never reached a model, or the turn never completed ──\n /** The bridge refused the connection or dropped it before a terminal event. */\n 'bridge_unreachable',\n /** The bridge accepted the request and its execution lane was already full. */\n 'bridge_lane_saturated',\n /** The router could not reconcile a completion with its own recorded usage. */\n 'router_usage_mismatch',\n /** A session was bound to one AgentProfile and asked to serve another. */\n 'session_profile_conflict',\n /** The agent runtime restarted underneath a live run. */\n 'runtime_restart',\n /** The root driver of a recursive run failed before it could dispatch work. */\n 'root_driver_failed',\n /** A sandbox box never finished provisioning. Distinct from `sandbox_failure`,\n * which is a command the agent ran exiting non-zero inside a live box. */\n 'box_provision_failure',\n /** A sandbox box did not confirm destruction after the run. */\n 'box_teardown_failure',\n /** The harness refused an agent-authored profile: an unknown model, a\n * workspace file it may not overwrite, a system prompt it may not replace. */\n 'profile_refused',\n /** The model was reached and returned no visible output. */\n 'empty_completion',\n /** The response stream carried malformed protocol content. */\n 'stream_protocol_error',\n /** The response stream ended before a terminal event. */\n 'stream_incomplete',\n /** A message was appended to a conversation the provider had already sealed. */\n 'message_sealed',\n /** The model process was killed by a signal or exited with no code. */\n 'process_killed',\n /** The run was cancelled by an operator. */\n 'cancelled',\n /** The run was terminated without a more specific reason. */\n 'terminated',\n /** The agent exhausted its traversal or iteration cap. */\n 'traversal_cap_exhausted',\n\n /** No reason was recorded at all. Distinct from `unknown`, which means a\n * reason exists and no rule matched it. */\n 'unreported',\n 'unknown',\n] as const\n\nexport type FailureClass = (typeof FAILURE_CLASSES)[number]\n\n// ── Helpers ──────────────────────────────────────────────────────────\n\nexport function isLlmSpan(s: Span): s is LlmSpan {\n return s.kind === 'llm'\n}\nexport function isToolSpan(s: Span): s is ToolSpan {\n return s.kind === 'tool'\n}\nexport function isJudgeSpan(s: Span): s is JudgeSpan {\n return s.kind === 'judge'\n}\n"],"mappings":";;;;;;;;;;;;AAYA,MAAa,uBAAuB;;;;;;;;;;;;;;;;;;;;;AA6OpC,MAAa,kBAAkB;CAC7B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAIA;CAEA;CAEA;CAEA;CAEA;CAEA;CAGA;CAEA;CAGA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAIA;CACA;AACF;AAMA,SAAgB,UAAU,GAAuB;CAC/C,OAAO,EAAE,SAAS;AACpB;AACA,SAAgB,WAAW,GAAwB;CACjD,OAAO,EAAE,SAAS;AACpB;AACA,SAAgB,YAAY,GAAyB;CACnD,OAAO,EAAE,SAAS;AACpB"}