@tangle-network/agent-eval 0.187.2 → 0.188.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (219) hide show
  1. package/CHANGELOG.md +55 -0
  2. package/dist/adapters/http.d.ts +2 -2
  3. package/dist/{agent-profile-uAIQK3Nr.d.ts → agent-profile-CfbuZk8b.d.ts} +4 -4
  4. package/dist/{agent-profile-uAIQK3Nr.d.ts.map → agent-profile-CfbuZk8b.d.ts.map} +1 -1
  5. package/dist/analyst/index.d.ts +12 -12
  6. package/dist/analyst/index.d.ts.map +1 -1
  7. package/dist/analyst/index.js +6 -6
  8. package/dist/{analyze-runs-DapYZz02.d.ts → analyze-runs-CLz5O01x.d.ts} +5 -5
  9. package/dist/{analyze-runs-DapYZz02.d.ts.map → analyze-runs-CLz5O01x.d.ts.map} +1 -1
  10. package/dist/{benchmark-command-DGokT8zd.js → benchmark-command-CdUGku3W.js} +12 -12
  11. package/dist/benchmark-command-CdUGku3W.js.map +1 -0
  12. package/dist/benchmarks/index.d.ts +4 -4
  13. package/dist/benchmarks/index.js +3 -3
  14. package/dist/builder-eval/index.d.ts +3 -3
  15. package/dist/builder-eval/index.js +1 -1
  16. package/dist/campaign/index.d.ts +5 -5
  17. package/dist/campaign/index.js +5 -5
  18. package/dist/{campaign-DlxdTOvw.js → campaign-DZuysBbS.js} +6 -6
  19. package/dist/{campaign-DlxdTOvw.js.map → campaign-DZuysBbS.js.map} +1 -1
  20. package/dist/{chat-client-DqPqWzVV.js → chat-client-C4dg7Ng_.js} +2 -2
  21. package/dist/{chat-client-DqPqWzVV.js.map → chat-client-C4dg7Ng_.js.map} +1 -1
  22. package/dist/{chat-json-call-C-DnDxEy.js → chat-json-call-JVGbfx95.js} +2 -2
  23. package/dist/{chat-json-call-C-DnDxEy.js.map → chat-json-call-JVGbfx95.js.map} +1 -1
  24. package/dist/cli.js +4 -4
  25. package/dist/{client-DCc7a8ZC.d.ts → client-CFj9dfcn.d.ts} +4 -4
  26. package/dist/{client-DCc7a8ZC.d.ts.map → client-CFj9dfcn.d.ts.map} +1 -1
  27. package/dist/contract/index.d.ts +11 -11
  28. package/dist/contract/index.js +6 -6
  29. package/dist/{counterfactual-CLgrwhkY.d.ts → counterfactual-CPaWN99-.d.ts} +4 -4
  30. package/dist/{counterfactual-CLgrwhkY.d.ts.map → counterfactual-CPaWN99-.d.ts.map} +1 -1
  31. package/dist/{counterfactual-Bjq1mlUu.js → counterfactual-DtMWvn7D.js} +2 -2
  32. package/dist/{counterfactual-Bjq1mlUu.js.map → counterfactual-DtMWvn7D.js.map} +1 -1
  33. package/dist/{default-registry-DMxeb4GD.d.ts → default-registry-BLsGfnoO.d.ts} +6 -6
  34. package/dist/{default-registry-DMxeb4GD.d.ts.map → default-registry-BLsGfnoO.d.ts.map} +1 -1
  35. package/dist/{default-registry-Ca2IWH1X.js → default-registry-BtZ_t-9b.js} +2 -2
  36. package/dist/{default-registry-Ca2IWH1X.js.map → default-registry-BtZ_t-9b.js.map} +1 -1
  37. package/dist/{define-agent-eval-C0EqJ9Gz.js → define-agent-eval-ChtddasD.js} +3 -3
  38. package/dist/{define-agent-eval-C0EqJ9Gz.js.map → define-agent-eval-ChtddasD.js.map} +1 -1
  39. package/dist/{define-agent-eval-CGBwPPaq.d.ts → define-agent-eval-CsicmDxt.d.ts} +7 -7
  40. package/dist/{define-agent-eval-CGBwPPaq.d.ts.map → define-agent-eval-CsicmDxt.d.ts.map} +1 -1
  41. package/dist/diagnosis/index.d.ts +38 -96
  42. package/dist/diagnosis/index.d.ts.map +1 -1
  43. package/dist/diagnosis/index.js +83 -134
  44. package/dist/diagnosis/index.js.map +1 -1
  45. package/dist/{dspy-rlm-engine-BE9lCzzL.js → dspy-rlm-engine-sPX7B5ac.js} +3 -3
  46. package/dist/{dspy-rlm-engine-BE9lCzzL.js.map → dspy-rlm-engine-sPX7B5ac.js.map} +1 -1
  47. package/dist/emitter-DbMXUIVP.js +334 -0
  48. package/dist/emitter-DbMXUIVP.js.map +1 -0
  49. package/dist/{emitter-Cs0egaFd.d.ts → emitter-HcCmFQl_.d.ts} +37 -6
  50. package/dist/emitter-HcCmFQl_.d.ts.map +1 -0
  51. package/dist/{engine-DNlKhIm0.d.ts → engine-Gw2n1irt.d.ts} +15 -5
  52. package/dist/engine-Gw2n1irt.d.ts.map +1 -0
  53. package/dist/{eval-campaign-XFHEpN86.js → eval-campaign-CeotQ1ZR.js} +7 -4
  54. package/dist/{eval-campaign-XFHEpN86.js.map → eval-campaign-CeotQ1ZR.js.map} +1 -1
  55. package/dist/evaluation.d.ts +2 -2
  56. package/dist/{exact-types-BEHN7tnq.d.ts → exact-types-CrhPoDp8.d.ts} +2 -2
  57. package/dist/{exact-types-BEHN7tnq.d.ts.map → exact-types-CrhPoDp8.d.ts.map} +1 -1
  58. package/dist/experiment/index.d.ts +3 -3
  59. package/dist/experiment/index.js +1 -2
  60. package/dist/experiment/index.js.map +1 -1
  61. package/dist/{external-optimizer-process-BbhvXTFx.js → external-optimizer-process-D3BLpZyR.js} +2 -2
  62. package/dist/{external-optimizer-process-BbhvXTFx.js.map → external-optimizer-process-D3BLpZyR.js.map} +1 -1
  63. package/dist/{external-optimizer-subprocess-DMgmqTB6.js → external-optimizer-subprocess-BrfsgL92.js} +2 -2
  64. package/dist/{external-optimizer-subprocess-DMgmqTB6.js.map → external-optimizer-subprocess-BrfsgL92.js.map} +1 -1
  65. package/dist/{failure-cluster-OldNRoAt.d.ts → failure-cluster-CbgEDcFJ.d.ts} +3 -3
  66. package/dist/{failure-cluster-OldNRoAt.d.ts.map → failure-cluster-CbgEDcFJ.d.ts.map} +1 -1
  67. package/dist/failure-taxonomy-CjabTATj.js +538 -0
  68. package/dist/failure-taxonomy-CjabTATj.js.map +1 -0
  69. package/dist/{feedback-trajectory-CytTR7u4.d.ts → feedback-trajectory-BJi4t3Vf.d.ts} +6 -6
  70. package/dist/{feedback-trajectory-CytTR7u4.d.ts.map → feedback-trajectory-BJi4t3Vf.d.ts.map} +1 -1
  71. package/dist/hosted/index.d.ts +2 -2
  72. package/dist/{index-CKZa_rnt.d.ts → index-B-s8qDRJ.d.ts} +2 -2
  73. package/dist/{index-CKZa_rnt.d.ts.map → index-B-s8qDRJ.d.ts.map} +1 -1
  74. package/dist/{index-YZrbnBSV.d.ts → index-BD1oSbHr.d.ts} +11 -11
  75. package/dist/{index-YZrbnBSV.d.ts.map → index-BD1oSbHr.d.ts.map} +1 -1
  76. package/dist/{index-vYNHCV2G.d.ts → index-Ca0_IjFu.d.ts} +7 -7
  77. package/dist/{index-vYNHCV2G.d.ts.map → index-Ca0_IjFu.d.ts.map} +1 -1
  78. package/dist/{index-BRsrfSHT.d.ts → index-eTTnPe3r.d.ts} +9 -9
  79. package/dist/{index-BRsrfSHT.d.ts.map → index-eTTnPe3r.d.ts.map} +1 -1
  80. package/dist/index.d.ts +383 -68
  81. package/dist/index.d.ts.map +1 -1
  82. package/dist/index.js +1482 -207
  83. package/dist/index.js.map +1 -1
  84. package/dist/{insight-report-DpOo-3iL.d.ts → insight-report-DeE1solA.d.ts} +4 -4
  85. package/dist/{insight-report-DpOo-3iL.d.ts.map → insight-report-DeE1solA.d.ts.map} +1 -1
  86. package/dist/{integrity-Cy9WHAtb.js → integrity-BFP4kjGj.js} +6 -1
  87. package/dist/integrity-BFP4kjGj.js.map +1 -0
  88. package/dist/{integrity-rGOfSUle.d.ts → integrity-CaeNeusS.d.ts} +4 -4
  89. package/dist/integrity-CaeNeusS.d.ts.map +1 -0
  90. package/dist/jev.d.ts +2 -2
  91. package/dist/{kind-factory-V79Hw0AU.js → kind-factory-g1ROw67A.js} +15 -4
  92. package/dist/{kind-factory-V79Hw0AU.js.map → kind-factory-g1ROw67A.js.map} +1 -1
  93. package/dist/{llm-client-DDhZ29Vm.js → llm-client-DF9bPNYA.js} +2 -2
  94. package/dist/{llm-client-DDhZ29Vm.js.map → llm-client-DF9bPNYA.js.map} +1 -1
  95. package/dist/{llm-judge-BVso5x-u.js → llm-judge-DF18J3CR.js} +9 -9
  96. package/dist/llm-judge-DF18J3CR.js.map +1 -0
  97. package/dist/{matrix-CPKsRY8Y.d.ts → matrix-BvA1WUpj.d.ts} +2 -2
  98. package/dist/{matrix-CPKsRY8Y.d.ts.map → matrix-BvA1WUpj.d.ts.map} +1 -1
  99. package/dist/meta-eval/index.d.ts +4 -4
  100. package/dist/{mint-BI14xehJ.d.ts → mint-CJFSYBIU.d.ts} +3 -3
  101. package/dist/{mint-BI14xehJ.d.ts.map → mint-CJFSYBIU.d.ts.map} +1 -1
  102. package/dist/multishot/golden/index.d.ts +1 -1
  103. package/dist/multishot/index.d.ts +2 -2
  104. package/dist/openapi.json +1 -1
  105. package/dist/pipelines/index.d.ts +67 -12
  106. package/dist/pipelines/index.d.ts.map +1 -1
  107. package/dist/pipelines/index.js +178 -41
  108. package/dist/pipelines/index.js.map +1 -1
  109. package/dist/{prime-protocol-CDBcL9NM.d.ts → prime-protocol-CF8Z43Fj.d.ts} +2 -2
  110. package/dist/{prime-protocol-CDBcL9NM.d.ts.map → prime-protocol-CF8Z43Fj.d.ts.map} +1 -1
  111. package/dist/{produced-state-Cn9ECr4v.js → produced-state-xsMMjYVc.js} +3 -3
  112. package/dist/{produced-state-Cn9ECr4v.js.map → produced-state-xsMMjYVc.js.map} +1 -1
  113. package/dist/{query-D6W6MaGx.d.ts → query-DBUQhehH.d.ts} +3 -3
  114. package/dist/{query-D6W6MaGx.d.ts.map → query-DBUQhehH.d.ts.map} +1 -1
  115. package/dist/random-DGEyr3zG.d.ts +83 -0
  116. package/dist/random-DGEyr3zG.d.ts.map +1 -0
  117. package/dist/{raw-provider-sink-BQd7mzyT.js → raw-provider-sink-C96CcqeG.js} +19 -56
  118. package/dist/raw-provider-sink-C96CcqeG.js.map +1 -0
  119. package/dist/redact-DPoH1-OW.d.ts +158 -0
  120. package/dist/redact-DPoH1-OW.d.ts.map +1 -0
  121. package/dist/redact-DxwlUAvh.js +824 -0
  122. package/dist/redact-DxwlUAvh.js.map +1 -0
  123. package/dist/{registry-D5Ly3GRV.d.ts → registry-BM2Kdk8T.d.ts} +4 -4
  124. package/dist/{registry-D5Ly3GRV.d.ts.map → registry-BM2Kdk8T.d.ts.map} +1 -1
  125. package/dist/{release-confidence-CSp3MsU4.d.ts → release-confidence-CG4Z9QLB.d.ts} +4 -4
  126. package/dist/{release-confidence-CSp3MsU4.d.ts.map → release-confidence-CG4Z9QLB.d.ts.map} +1 -1
  127. package/dist/{report-command-CSTjzFGY.js → report-command-DKJ_yAIj.js} +21 -12
  128. package/dist/{report-command-CSTjzFGY.js.map → report-command-DKJ_yAIj.js.map} +1 -1
  129. package/dist/reporting.d.ts +3 -3
  130. package/dist/{researcher-D7E92STT.d.ts → researcher-Bjcd_8T_.d.ts} +7 -7
  131. package/dist/{researcher-D7E92STT.d.ts.map → researcher-Bjcd_8T_.d.ts.map} +1 -1
  132. package/dist/{reward-hacking-BIQ2BWLG.d.ts → reward-hacking-DrKXJtgH.d.ts} +2 -2
  133. package/dist/{reward-hacking-BIQ2BWLG.d.ts.map → reward-hacking-DrKXJtgH.d.ts.map} +1 -1
  134. package/dist/rl.d.ts +8 -8
  135. package/dist/rl.js +1 -1
  136. package/dist/rollout/index.d.ts +2 -2
  137. package/dist/{rubric-predictive-validity-CTU8y8Cv.d.ts → rubric-predictive-validity-B14yBAmM.d.ts} +2 -2
  138. package/dist/{rubric-predictive-validity-CTU8y8Cv.d.ts.map → rubric-predictive-validity-B14yBAmM.d.ts.map} +1 -1
  139. package/dist/{run-record-D9wS35fK.d.ts → run-record-xEULpgML.d.ts} +2 -2
  140. package/dist/{run-record-D9wS35fK.d.ts.map → run-record-xEULpgML.d.ts.map} +1 -1
  141. package/dist/{schema-CR5cpjQ3.d.ts → schema-2g7trUwH.d.ts} +17 -2
  142. package/dist/schema-2g7trUwH.d.ts.map +1 -0
  143. package/dist/schema-CSf6qWgZ.js.map +1 -1
  144. package/dist/{semantic-concept-judge-Cnt0TWp0.js → semantic-concept-judge-70LxrCSL.js} +5 -5
  145. package/dist/{semantic-concept-judge-Cnt0TWp0.js.map → semantic-concept-judge-70LxrCSL.js.map} +1 -1
  146. package/dist/{series-convergence-BnMs_uAr.d.ts → series-convergence-DY7ha0kx.d.ts} +2 -2
  147. package/dist/{series-convergence-BnMs_uAr.d.ts.map → series-convergence-DY7ha0kx.d.ts.map} +1 -1
  148. package/dist/{server-DmpcFiVK.js → server-C8XvqCg8.js} +2 -2
  149. package/dist/{server-DmpcFiVK.js.map → server-C8XvqCg8.js.map} +1 -1
  150. package/dist/{skillopt-optimization-method-Cc9SVLqi.js → skillopt-optimization-method-C60h0Got.js} +5 -5
  151. package/dist/{skillopt-optimization-method-Cc9SVLqi.js.map → skillopt-optimization-method-C60h0Got.js.map} +1 -1
  152. package/dist/spans-W0o8pbEg.d.ts +60 -0
  153. package/dist/spans-W0o8pbEg.d.ts.map +1 -0
  154. package/dist/{statistical-heldout-DTmhTW8P.d.ts → statistical-heldout-CPv-y6sK.d.ts} +3 -3
  155. package/dist/{statistical-heldout-DTmhTW8P.d.ts.map → statistical-heldout-CPv-y6sK.d.ts.map} +1 -1
  156. package/dist/{store-BErPvYBr.d.ts → store-C4avFjIi.d.ts} +2 -2
  157. package/dist/{store-BErPvYBr.d.ts.map → store-C4avFjIi.d.ts.map} +1 -1
  158. package/dist/{store-otlp-2gYCy-v0.js → store-otlp-DPgzSQKS.js} +2 -2
  159. package/dist/{store-otlp-2gYCy-v0.js.map → store-otlp-DPgzSQKS.js.map} +1 -1
  160. package/dist/{store-tool-spans-B3bX9yqn.d.ts → store-tool-spans-C2lLwDzg.d.ts} +9 -54
  161. package/dist/store-tool-spans-C2lLwDzg.d.ts.map +1 -0
  162. package/dist/{store-tool-spans-B1e6Auc2.js → store-tool-spans-ClPxcvh2.js} +4 -4
  163. package/dist/{store-tool-spans-B1e6Auc2.js.map → store-tool-spans-ClPxcvh2.js.map} +1 -1
  164. package/dist/storyboard/index.d.ts +1 -1
  165. package/dist/{summary-report-Iy6kXo8x.d.ts → summary-report-SNvJoAvD.d.ts} +3 -3
  166. package/dist/{summary-report-Iy6kXo8x.d.ts.map → summary-report-SNvJoAvD.d.ts.map} +1 -1
  167. package/dist/supervisor-run/index.d.ts.map +1 -1
  168. package/dist/supervisor-run/index.js +1 -1
  169. package/dist/{tool-groups-veQYPiTy.d.ts → tool-groups-CCQqAa16.d.ts} +3 -3
  170. package/dist/tool-groups-CCQqAa16.d.ts.map +1 -0
  171. package/dist/{tool-waste-nbHPAxei.js → tool-waste-CJDfxH45.js} +3 -537
  172. package/dist/tool-waste-CJDfxH45.js.map +1 -0
  173. package/dist/{tool-waste-D23I0zWm.d.ts → tool-waste-CP5DO383.d.ts} +4 -4
  174. package/dist/{tool-waste-D23I0zWm.d.ts.map → tool-waste-CP5DO383.d.ts.map} +1 -1
  175. package/dist/trace-repair/index.d.ts +2 -2
  176. package/dist/traces.d.ts +12 -11
  177. package/dist/traces.d.ts.map +1 -1
  178. package/dist/traces.js +8 -8
  179. package/dist/{trajectory-D7qrNvaN.d.ts → trajectory-Y1fmdlw7.d.ts} +3 -3
  180. package/dist/{trajectory-D7qrNvaN.d.ts.map → trajectory-Y1fmdlw7.d.ts.map} +1 -1
  181. package/dist/trajectory-replay/index.d.ts +3 -3
  182. package/dist/trajectory-replay/index.js +2 -2
  183. package/dist/{types--18oHwVF.d.ts → types-5ak2PqnJ.d.ts} +3 -3
  184. package/dist/{types--18oHwVF.d.ts.map → types-5ak2PqnJ.d.ts.map} +1 -1
  185. package/dist/{types-BTRcIqFr.d.ts → types-B1vDZ694.d.ts} +4 -4
  186. package/dist/{types-BTRcIqFr.d.ts.map → types-B1vDZ694.d.ts.map} +1 -1
  187. package/dist/{types-CBbLtr2J.d.ts → types-C0bWoSKC.d.ts} +10 -11
  188. package/dist/types-C0bWoSKC.d.ts.map +1 -0
  189. package/dist/{types-BjB2wZM9.d.ts → types-DDp2Mb8Y.d.ts} +3 -3
  190. package/dist/{types-BjB2wZM9.d.ts.map → types-DDp2Mb8Y.d.ts.map} +1 -1
  191. package/dist/wire/index.d.ts +3 -3
  192. package/dist/wire/index.js +1 -1
  193. package/docs/diagnosis.md +2 -3
  194. package/docs/eval-surface-map.md +1 -1
  195. package/docs/experiment.md +0 -1
  196. package/docs/redaction.md +59 -0
  197. package/docs/trace-contracts.md +102 -0
  198. package/docs/verdicts.md +1 -1
  199. package/package.json +3 -2
  200. package/dist/benchmark-command-DGokT8zd.js.map +0 -1
  201. package/dist/emitter-Cs0egaFd.d.ts.map +0 -1
  202. package/dist/emitter-DeQHiDMm.js +0 -257
  203. package/dist/emitter-DeQHiDMm.js.map +0 -1
  204. package/dist/engine-DNlKhIm0.d.ts.map +0 -1
  205. package/dist/experiment-tracker-BKEumQug.js +0 -280
  206. package/dist/experiment-tracker-BKEumQug.js.map +0 -1
  207. package/dist/experiment-tracker-URRGP9ek.d.ts +0 -237
  208. package/dist/experiment-tracker-URRGP9ek.d.ts.map +0 -1
  209. package/dist/integrity-Cy9WHAtb.js.map +0 -1
  210. package/dist/integrity-rGOfSUle.d.ts.map +0 -1
  211. package/dist/llm-judge-BVso5x-u.js.map +0 -1
  212. package/dist/raw-provider-sink-BQd7mzyT.js.map +0 -1
  213. package/dist/redact-7Aq1ukl-.js +0 -107
  214. package/dist/redact-7Aq1ukl-.js.map +0 -1
  215. package/dist/schema-CR5cpjQ3.d.ts.map +0 -1
  216. package/dist/store-tool-spans-B3bX9yqn.d.ts.map +0 -1
  217. package/dist/tool-groups-veQYPiTy.d.ts.map +0 -1
  218. package/dist/tool-waste-nbHPAxei.js.map +0 -1
  219. package/dist/types-CBbLtr2J.d.ts.map +0 -1
package/docs/verdicts.md CHANGED
@@ -30,7 +30,7 @@ Their additional fields distinguish failed checks from incomplete or unmeasured
30
30
  | --- | --- | --- | --- | --- |
31
31
  | `MultiLayerVerifier.run()` | `VerificationReport` | `composite` | Ordered verification layers | `layers`, `allPass`, and optional `taskScore` |
32
32
  | `verifyCompletion()` | `CompletionVerdict` | The supplied checker's strategy | Completion requirements matched against produced state | Requirement evidence, `correct`, and `unmeasured` |
33
- | `evaluateTraceContract()` | `ContractVerdict` | `invariant` | Temporal rules over recorded spans | Rule results and assumptions about ordering and predicates |
33
+ | `evaluateTraceContract()` | `ContractVerdict` | `invariant` | Ordering, count, token, run, and argument rules over recorded spans | `status` (`pass`, `fail`, or `error`), `ruleExecutions`, `violations`, and custom-predicate assumptions |
34
34
  | `evaluateOracles()` | `OracleReport` | `test` | Declared expected-outcome assertions | `results`, `passCount`, and `failCount` |
35
35
  | `replayVerify()` | `ReplayVerdict` | `replication` | Failure reproduction and an optional fix under re-execution | Prefix fidelity, signature matches, and both execution arms |
36
36
  | `verifyFindings()` | `VerifyFindingsRun` | `replication` | Analyst findings checked through replay | `executions`, `counts`, and individual verifications |
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@tangle-network/agent-eval",
3
- "version": "0.187.2",
3
+ "version": "0.188.0",
4
4
  "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
5
5
  "homepage": "https://github.com/tangle-network/agent-eval#readme",
6
6
  "repository": {
@@ -213,7 +213,8 @@
213
213
  "contract:finding": "tsx scripts/emit-finding-contract.ts",
214
214
  "contract:finding:check": "tsx scripts/emit-finding-contract.ts --check",
215
215
  "evidence:render": "tsx scripts/render-evidence-index.ts",
216
- "evidence:check": "tsx scripts/render-evidence-index.ts --check"
216
+ "evidence:check": "tsx scripts/render-evidence-index.ts --check",
217
+ "redaction:corpus": "tsx scripts/redaction-corpus.ts"
217
218
  },
218
219
  "dependencies": {
219
220
  "@asteasolutions/zod-to-openapi": "^9.1.0",