@tangle-network/agent-eval 0.187.2 → 0.188.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +55 -0
- package/dist/adapters/http.d.ts +2 -2
- package/dist/{agent-profile-uAIQK3Nr.d.ts → agent-profile-CfbuZk8b.d.ts} +4 -4
- package/dist/{agent-profile-uAIQK3Nr.d.ts.map → agent-profile-CfbuZk8b.d.ts.map} +1 -1
- package/dist/analyst/index.d.ts +12 -12
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +6 -6
- package/dist/{analyze-runs-DapYZz02.d.ts → analyze-runs-CLz5O01x.d.ts} +5 -5
- package/dist/{analyze-runs-DapYZz02.d.ts.map → analyze-runs-CLz5O01x.d.ts.map} +1 -1
- package/dist/{benchmark-command-DGokT8zd.js → benchmark-command-CdUGku3W.js} +12 -12
- package/dist/benchmark-command-CdUGku3W.js.map +1 -0
- package/dist/benchmarks/index.d.ts +4 -4
- package/dist/benchmarks/index.js +3 -3
- package/dist/builder-eval/index.d.ts +3 -3
- package/dist/builder-eval/index.js +1 -1
- package/dist/campaign/index.d.ts +5 -5
- package/dist/campaign/index.js +5 -5
- package/dist/{campaign-DlxdTOvw.js → campaign-DZuysBbS.js} +6 -6
- package/dist/{campaign-DlxdTOvw.js.map → campaign-DZuysBbS.js.map} +1 -1
- package/dist/{chat-client-DqPqWzVV.js → chat-client-C4dg7Ng_.js} +2 -2
- package/dist/{chat-client-DqPqWzVV.js.map → chat-client-C4dg7Ng_.js.map} +1 -1
- package/dist/{chat-json-call-C-DnDxEy.js → chat-json-call-JVGbfx95.js} +2 -2
- package/dist/{chat-json-call-C-DnDxEy.js.map → chat-json-call-JVGbfx95.js.map} +1 -1
- package/dist/cli.js +4 -4
- package/dist/{client-DCc7a8ZC.d.ts → client-CFj9dfcn.d.ts} +4 -4
- package/dist/{client-DCc7a8ZC.d.ts.map → client-CFj9dfcn.d.ts.map} +1 -1
- package/dist/contract/index.d.ts +11 -11
- package/dist/contract/index.js +6 -6
- package/dist/{counterfactual-CLgrwhkY.d.ts → counterfactual-CPaWN99-.d.ts} +4 -4
- package/dist/{counterfactual-CLgrwhkY.d.ts.map → counterfactual-CPaWN99-.d.ts.map} +1 -1
- package/dist/{counterfactual-Bjq1mlUu.js → counterfactual-DtMWvn7D.js} +2 -2
- package/dist/{counterfactual-Bjq1mlUu.js.map → counterfactual-DtMWvn7D.js.map} +1 -1
- package/dist/{default-registry-DMxeb4GD.d.ts → default-registry-BLsGfnoO.d.ts} +6 -6
- package/dist/{default-registry-DMxeb4GD.d.ts.map → default-registry-BLsGfnoO.d.ts.map} +1 -1
- package/dist/{default-registry-Ca2IWH1X.js → default-registry-BtZ_t-9b.js} +2 -2
- package/dist/{default-registry-Ca2IWH1X.js.map → default-registry-BtZ_t-9b.js.map} +1 -1
- package/dist/{define-agent-eval-C0EqJ9Gz.js → define-agent-eval-ChtddasD.js} +3 -3
- package/dist/{define-agent-eval-C0EqJ9Gz.js.map → define-agent-eval-ChtddasD.js.map} +1 -1
- package/dist/{define-agent-eval-CGBwPPaq.d.ts → define-agent-eval-CsicmDxt.d.ts} +7 -7
- package/dist/{define-agent-eval-CGBwPPaq.d.ts.map → define-agent-eval-CsicmDxt.d.ts.map} +1 -1
- package/dist/diagnosis/index.d.ts +38 -96
- package/dist/diagnosis/index.d.ts.map +1 -1
- package/dist/diagnosis/index.js +83 -134
- package/dist/diagnosis/index.js.map +1 -1
- package/dist/{dspy-rlm-engine-BE9lCzzL.js → dspy-rlm-engine-sPX7B5ac.js} +3 -3
- package/dist/{dspy-rlm-engine-BE9lCzzL.js.map → dspy-rlm-engine-sPX7B5ac.js.map} +1 -1
- package/dist/emitter-DbMXUIVP.js +334 -0
- package/dist/emitter-DbMXUIVP.js.map +1 -0
- package/dist/{emitter-Cs0egaFd.d.ts → emitter-HcCmFQl_.d.ts} +37 -6
- package/dist/emitter-HcCmFQl_.d.ts.map +1 -0
- package/dist/{engine-DNlKhIm0.d.ts → engine-Gw2n1irt.d.ts} +15 -5
- package/dist/engine-Gw2n1irt.d.ts.map +1 -0
- package/dist/{eval-campaign-XFHEpN86.js → eval-campaign-CeotQ1ZR.js} +7 -4
- package/dist/{eval-campaign-XFHEpN86.js.map → eval-campaign-CeotQ1ZR.js.map} +1 -1
- package/dist/evaluation.d.ts +2 -2
- package/dist/{exact-types-BEHN7tnq.d.ts → exact-types-CrhPoDp8.d.ts} +2 -2
- package/dist/{exact-types-BEHN7tnq.d.ts.map → exact-types-CrhPoDp8.d.ts.map} +1 -1
- package/dist/experiment/index.d.ts +3 -3
- package/dist/experiment/index.js +1 -2
- package/dist/experiment/index.js.map +1 -1
- package/dist/{external-optimizer-process-BbhvXTFx.js → external-optimizer-process-D3BLpZyR.js} +2 -2
- package/dist/{external-optimizer-process-BbhvXTFx.js.map → external-optimizer-process-D3BLpZyR.js.map} +1 -1
- package/dist/{external-optimizer-subprocess-DMgmqTB6.js → external-optimizer-subprocess-BrfsgL92.js} +2 -2
- package/dist/{external-optimizer-subprocess-DMgmqTB6.js.map → external-optimizer-subprocess-BrfsgL92.js.map} +1 -1
- package/dist/{failure-cluster-OldNRoAt.d.ts → failure-cluster-CbgEDcFJ.d.ts} +3 -3
- package/dist/{failure-cluster-OldNRoAt.d.ts.map → failure-cluster-CbgEDcFJ.d.ts.map} +1 -1
- package/dist/failure-taxonomy-CjabTATj.js +538 -0
- package/dist/failure-taxonomy-CjabTATj.js.map +1 -0
- package/dist/{feedback-trajectory-CytTR7u4.d.ts → feedback-trajectory-BJi4t3Vf.d.ts} +6 -6
- package/dist/{feedback-trajectory-CytTR7u4.d.ts.map → feedback-trajectory-BJi4t3Vf.d.ts.map} +1 -1
- package/dist/hosted/index.d.ts +2 -2
- package/dist/{index-CKZa_rnt.d.ts → index-B-s8qDRJ.d.ts} +2 -2
- package/dist/{index-CKZa_rnt.d.ts.map → index-B-s8qDRJ.d.ts.map} +1 -1
- package/dist/{index-YZrbnBSV.d.ts → index-BD1oSbHr.d.ts} +11 -11
- package/dist/{index-YZrbnBSV.d.ts.map → index-BD1oSbHr.d.ts.map} +1 -1
- package/dist/{index-vYNHCV2G.d.ts → index-Ca0_IjFu.d.ts} +7 -7
- package/dist/{index-vYNHCV2G.d.ts.map → index-Ca0_IjFu.d.ts.map} +1 -1
- package/dist/{index-BRsrfSHT.d.ts → index-eTTnPe3r.d.ts} +9 -9
- package/dist/{index-BRsrfSHT.d.ts.map → index-eTTnPe3r.d.ts.map} +1 -1
- package/dist/index.d.ts +383 -68
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1482 -207
- package/dist/index.js.map +1 -1
- package/dist/{insight-report-DpOo-3iL.d.ts → insight-report-DeE1solA.d.ts} +4 -4
- package/dist/{insight-report-DpOo-3iL.d.ts.map → insight-report-DeE1solA.d.ts.map} +1 -1
- package/dist/{integrity-Cy9WHAtb.js → integrity-BFP4kjGj.js} +6 -1
- package/dist/integrity-BFP4kjGj.js.map +1 -0
- package/dist/{integrity-rGOfSUle.d.ts → integrity-CaeNeusS.d.ts} +4 -4
- package/dist/integrity-CaeNeusS.d.ts.map +1 -0
- package/dist/jev.d.ts +2 -2
- package/dist/{kind-factory-V79Hw0AU.js → kind-factory-g1ROw67A.js} +15 -4
- package/dist/{kind-factory-V79Hw0AU.js.map → kind-factory-g1ROw67A.js.map} +1 -1
- package/dist/{llm-client-DDhZ29Vm.js → llm-client-DF9bPNYA.js} +2 -2
- package/dist/{llm-client-DDhZ29Vm.js.map → llm-client-DF9bPNYA.js.map} +1 -1
- package/dist/{llm-judge-BVso5x-u.js → llm-judge-DF18J3CR.js} +9 -9
- package/dist/llm-judge-DF18J3CR.js.map +1 -0
- package/dist/{matrix-CPKsRY8Y.d.ts → matrix-BvA1WUpj.d.ts} +2 -2
- package/dist/{matrix-CPKsRY8Y.d.ts.map → matrix-BvA1WUpj.d.ts.map} +1 -1
- package/dist/meta-eval/index.d.ts +4 -4
- package/dist/{mint-BI14xehJ.d.ts → mint-CJFSYBIU.d.ts} +3 -3
- package/dist/{mint-BI14xehJ.d.ts.map → mint-CJFSYBIU.d.ts.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/pipelines/index.d.ts +67 -12
- package/dist/pipelines/index.d.ts.map +1 -1
- package/dist/pipelines/index.js +178 -41
- package/dist/pipelines/index.js.map +1 -1
- package/dist/{prime-protocol-CDBcL9NM.d.ts → prime-protocol-CF8Z43Fj.d.ts} +2 -2
- package/dist/{prime-protocol-CDBcL9NM.d.ts.map → prime-protocol-CF8Z43Fj.d.ts.map} +1 -1
- package/dist/{produced-state-Cn9ECr4v.js → produced-state-xsMMjYVc.js} +3 -3
- package/dist/{produced-state-Cn9ECr4v.js.map → produced-state-xsMMjYVc.js.map} +1 -1
- package/dist/{query-D6W6MaGx.d.ts → query-DBUQhehH.d.ts} +3 -3
- package/dist/{query-D6W6MaGx.d.ts.map → query-DBUQhehH.d.ts.map} +1 -1
- package/dist/random-DGEyr3zG.d.ts +83 -0
- package/dist/random-DGEyr3zG.d.ts.map +1 -0
- package/dist/{raw-provider-sink-BQd7mzyT.js → raw-provider-sink-C96CcqeG.js} +19 -56
- package/dist/raw-provider-sink-C96CcqeG.js.map +1 -0
- package/dist/redact-DPoH1-OW.d.ts +158 -0
- package/dist/redact-DPoH1-OW.d.ts.map +1 -0
- package/dist/redact-DxwlUAvh.js +824 -0
- package/dist/redact-DxwlUAvh.js.map +1 -0
- package/dist/{registry-D5Ly3GRV.d.ts → registry-BM2Kdk8T.d.ts} +4 -4
- package/dist/{registry-D5Ly3GRV.d.ts.map → registry-BM2Kdk8T.d.ts.map} +1 -1
- package/dist/{release-confidence-CSp3MsU4.d.ts → release-confidence-CG4Z9QLB.d.ts} +4 -4
- package/dist/{release-confidence-CSp3MsU4.d.ts.map → release-confidence-CG4Z9QLB.d.ts.map} +1 -1
- package/dist/{report-command-CSTjzFGY.js → report-command-DKJ_yAIj.js} +21 -12
- package/dist/{report-command-CSTjzFGY.js.map → report-command-DKJ_yAIj.js.map} +1 -1
- package/dist/reporting.d.ts +3 -3
- package/dist/{researcher-D7E92STT.d.ts → researcher-Bjcd_8T_.d.ts} +7 -7
- package/dist/{researcher-D7E92STT.d.ts.map → researcher-Bjcd_8T_.d.ts.map} +1 -1
- package/dist/{reward-hacking-BIQ2BWLG.d.ts → reward-hacking-DrKXJtgH.d.ts} +2 -2
- package/dist/{reward-hacking-BIQ2BWLG.d.ts.map → reward-hacking-DrKXJtgH.d.ts.map} +1 -1
- package/dist/rl.d.ts +8 -8
- package/dist/rl.js +1 -1
- package/dist/rollout/index.d.ts +2 -2
- package/dist/{rubric-predictive-validity-CTU8y8Cv.d.ts → rubric-predictive-validity-B14yBAmM.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-CTU8y8Cv.d.ts.map → rubric-predictive-validity-B14yBAmM.d.ts.map} +1 -1
- package/dist/{run-record-D9wS35fK.d.ts → run-record-xEULpgML.d.ts} +2 -2
- package/dist/{run-record-D9wS35fK.d.ts.map → run-record-xEULpgML.d.ts.map} +1 -1
- package/dist/{schema-CR5cpjQ3.d.ts → schema-2g7trUwH.d.ts} +17 -2
- package/dist/schema-2g7trUwH.d.ts.map +1 -0
- package/dist/schema-CSf6qWgZ.js.map +1 -1
- package/dist/{semantic-concept-judge-Cnt0TWp0.js → semantic-concept-judge-70LxrCSL.js} +5 -5
- package/dist/{semantic-concept-judge-Cnt0TWp0.js.map → semantic-concept-judge-70LxrCSL.js.map} +1 -1
- package/dist/{series-convergence-BnMs_uAr.d.ts → series-convergence-DY7ha0kx.d.ts} +2 -2
- package/dist/{series-convergence-BnMs_uAr.d.ts.map → series-convergence-DY7ha0kx.d.ts.map} +1 -1
- package/dist/{server-DmpcFiVK.js → server-C8XvqCg8.js} +2 -2
- package/dist/{server-DmpcFiVK.js.map → server-C8XvqCg8.js.map} +1 -1
- package/dist/{skillopt-optimization-method-Cc9SVLqi.js → skillopt-optimization-method-C60h0Got.js} +5 -5
- package/dist/{skillopt-optimization-method-Cc9SVLqi.js.map → skillopt-optimization-method-C60h0Got.js.map} +1 -1
- package/dist/spans-W0o8pbEg.d.ts +60 -0
- package/dist/spans-W0o8pbEg.d.ts.map +1 -0
- package/dist/{statistical-heldout-DTmhTW8P.d.ts → statistical-heldout-CPv-y6sK.d.ts} +3 -3
- package/dist/{statistical-heldout-DTmhTW8P.d.ts.map → statistical-heldout-CPv-y6sK.d.ts.map} +1 -1
- package/dist/{store-BErPvYBr.d.ts → store-C4avFjIi.d.ts} +2 -2
- package/dist/{store-BErPvYBr.d.ts.map → store-C4avFjIi.d.ts.map} +1 -1
- package/dist/{store-otlp-2gYCy-v0.js → store-otlp-DPgzSQKS.js} +2 -2
- package/dist/{store-otlp-2gYCy-v0.js.map → store-otlp-DPgzSQKS.js.map} +1 -1
- package/dist/{store-tool-spans-B3bX9yqn.d.ts → store-tool-spans-C2lLwDzg.d.ts} +9 -54
- package/dist/store-tool-spans-C2lLwDzg.d.ts.map +1 -0
- package/dist/{store-tool-spans-B1e6Auc2.js → store-tool-spans-ClPxcvh2.js} +4 -4
- package/dist/{store-tool-spans-B1e6Auc2.js.map → store-tool-spans-ClPxcvh2.js.map} +1 -1
- package/dist/storyboard/index.d.ts +1 -1
- package/dist/{summary-report-Iy6kXo8x.d.ts → summary-report-SNvJoAvD.d.ts} +3 -3
- package/dist/{summary-report-Iy6kXo8x.d.ts.map → summary-report-SNvJoAvD.d.ts.map} +1 -1
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +1 -1
- package/dist/{tool-groups-veQYPiTy.d.ts → tool-groups-CCQqAa16.d.ts} +3 -3
- package/dist/tool-groups-CCQqAa16.d.ts.map +1 -0
- package/dist/{tool-waste-nbHPAxei.js → tool-waste-CJDfxH45.js} +3 -537
- package/dist/tool-waste-CJDfxH45.js.map +1 -0
- package/dist/{tool-waste-D23I0zWm.d.ts → tool-waste-CP5DO383.d.ts} +4 -4
- package/dist/{tool-waste-D23I0zWm.d.ts.map → tool-waste-CP5DO383.d.ts.map} +1 -1
- package/dist/trace-repair/index.d.ts +2 -2
- package/dist/traces.d.ts +12 -11
- package/dist/traces.d.ts.map +1 -1
- package/dist/traces.js +8 -8
- package/dist/{trajectory-D7qrNvaN.d.ts → trajectory-Y1fmdlw7.d.ts} +3 -3
- package/dist/{trajectory-D7qrNvaN.d.ts.map → trajectory-Y1fmdlw7.d.ts.map} +1 -1
- package/dist/trajectory-replay/index.d.ts +3 -3
- package/dist/trajectory-replay/index.js +2 -2
- package/dist/{types--18oHwVF.d.ts → types-5ak2PqnJ.d.ts} +3 -3
- package/dist/{types--18oHwVF.d.ts.map → types-5ak2PqnJ.d.ts.map} +1 -1
- package/dist/{types-BTRcIqFr.d.ts → types-B1vDZ694.d.ts} +4 -4
- package/dist/{types-BTRcIqFr.d.ts.map → types-B1vDZ694.d.ts.map} +1 -1
- package/dist/{types-CBbLtr2J.d.ts → types-C0bWoSKC.d.ts} +10 -11
- package/dist/types-C0bWoSKC.d.ts.map +1 -0
- package/dist/{types-BjB2wZM9.d.ts → types-DDp2Mb8Y.d.ts} +3 -3
- package/dist/{types-BjB2wZM9.d.ts.map → types-DDp2Mb8Y.d.ts.map} +1 -1
- package/dist/wire/index.d.ts +3 -3
- package/dist/wire/index.js +1 -1
- package/docs/diagnosis.md +2 -3
- package/docs/eval-surface-map.md +1 -1
- package/docs/experiment.md +0 -1
- package/docs/redaction.md +59 -0
- package/docs/trace-contracts.md +102 -0
- package/docs/verdicts.md +1 -1
- package/package.json +3 -2
- package/dist/benchmark-command-DGokT8zd.js.map +0 -1
- package/dist/emitter-Cs0egaFd.d.ts.map +0 -1
- package/dist/emitter-DeQHiDMm.js +0 -257
- package/dist/emitter-DeQHiDMm.js.map +0 -1
- package/dist/engine-DNlKhIm0.d.ts.map +0 -1
- package/dist/experiment-tracker-BKEumQug.js +0 -280
- package/dist/experiment-tracker-BKEumQug.js.map +0 -1
- package/dist/experiment-tracker-URRGP9ek.d.ts +0 -237
- package/dist/experiment-tracker-URRGP9ek.d.ts.map +0 -1
- package/dist/integrity-Cy9WHAtb.js.map +0 -1
- package/dist/integrity-rGOfSUle.d.ts.map +0 -1
- package/dist/llm-judge-BVso5x-u.js.map +0 -1
- package/dist/raw-provider-sink-BQd7mzyT.js.map +0 -1
- package/dist/redact-7Aq1ukl-.js +0 -107
- package/dist/redact-7Aq1ukl-.js.map +0 -1
- package/dist/schema-CR5cpjQ3.d.ts.map +0 -1
- package/dist/store-tool-spans-B3bX9yqn.d.ts.map +0 -1
- package/dist/tool-groups-veQYPiTy.d.ts.map +0 -1
- package/dist/tool-waste-nbHPAxei.js.map +0 -1
- package/dist/types-CBbLtr2J.d.ts.map +0 -1
package/docs/verdicts.md
CHANGED
|
@@ -30,7 +30,7 @@ Their additional fields distinguish failed checks from incomplete or unmeasured
|
|
|
30
30
|
| --- | --- | --- | --- | --- |
|
|
31
31
|
| `MultiLayerVerifier.run()` | `VerificationReport` | `composite` | Ordered verification layers | `layers`, `allPass`, and optional `taskScore` |
|
|
32
32
|
| `verifyCompletion()` | `CompletionVerdict` | The supplied checker's strategy | Completion requirements matched against produced state | Requirement evidence, `correct`, and `unmeasured` |
|
|
33
|
-
| `evaluateTraceContract()` | `ContractVerdict` | `invariant` |
|
|
33
|
+
| `evaluateTraceContract()` | `ContractVerdict` | `invariant` | Ordering, count, token, run, and argument rules over recorded spans | `status` (`pass`, `fail`, or `error`), `ruleExecutions`, `violations`, and custom-predicate assumptions |
|
|
34
34
|
| `evaluateOracles()` | `OracleReport` | `test` | Declared expected-outcome assertions | `results`, `passCount`, and `failCount` |
|
|
35
35
|
| `replayVerify()` | `ReplayVerdict` | `replication` | Failure reproduction and an optional fix under re-execution | Prefix fidelity, signature matches, and both execution arms |
|
|
36
36
|
| `verifyFindings()` | `VerifyFindingsRun` | `replication` | Analyst findings checked through replay | `executions`, `counts`, and individual verifications |
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tangle-network/agent-eval",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.188.0",
|
|
4
4
|
"description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
|
|
5
5
|
"homepage": "https://github.com/tangle-network/agent-eval#readme",
|
|
6
6
|
"repository": {
|
|
@@ -213,7 +213,8 @@
|
|
|
213
213
|
"contract:finding": "tsx scripts/emit-finding-contract.ts",
|
|
214
214
|
"contract:finding:check": "tsx scripts/emit-finding-contract.ts --check",
|
|
215
215
|
"evidence:render": "tsx scripts/render-evidence-index.ts",
|
|
216
|
-
"evidence:check": "tsx scripts/render-evidence-index.ts --check"
|
|
216
|
+
"evidence:check": "tsx scripts/render-evidence-index.ts --check",
|
|
217
|
+
"redaction:corpus": "tsx scripts/redaction-corpus.ts"
|
|
217
218
|
},
|
|
218
219
|
"dependencies": {
|
|
219
220
|
"@asteasolutions/zod-to-openapi": "^9.1.0",
|