@tangle-network/agent-eval 0.216.1 → 0.217.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +102 -57
- package/README.md +2 -2
- package/dist/adapters/http.d.ts +2 -2
- package/dist/{agent-profile-VFcQncbW.d.ts → agent-profile-DTt8uGP9.d.ts} +4 -17
- package/dist/agent-profile-DTt8uGP9.d.ts.map +1 -0
- package/dist/{agent-profile-cell-BK8mlqlq.js → agent-profile-cell-B1V1gUZs.js} +4 -53
- package/dist/agent-profile-cell-B1V1gUZs.js.map +1 -0
- package/dist/{agent-profile-cell-BNp-Js9F.d.ts → agent-profile-cell-BcpFbNd5.d.ts} +2 -22
- package/dist/agent-profile-cell-BcpFbNd5.d.ts.map +1 -0
- package/dist/{agreement-irr-BEWLIF3_.js → agreement-irr-BhAqjluA.js} +5 -5
- package/dist/agreement-irr-BhAqjluA.js.map +1 -0
- package/dist/analyst/index.d.ts +36 -169
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +17 -192
- package/dist/analyst/index.js.map +1 -1
- package/dist/{analyze-runs-C84irm4i.js → analyze-runs-DqPJaJ4X.js} +225 -21
- package/dist/analyze-runs-DqPJaJ4X.js.map +1 -0
- package/dist/{analyze-runs-D68fd-su.d.ts → analyze-runs-DyeNcenW.d.ts} +5 -5
- package/dist/{analyze-runs-D68fd-su.d.ts.map → analyze-runs-DyeNcenW.d.ts.map} +1 -1
- package/dist/authenticity/index.d.ts +0 -23
- package/dist/authenticity/index.d.ts.map +1 -1
- package/dist/authenticity/index.js +1 -34
- package/dist/authenticity/index.js.map +1 -1
- package/dist/{benchmark-command-By0vh8q7.js → benchmark-command-xQK8nOI6.js} +554 -89
- package/dist/benchmark-command-xQK8nOI6.js.map +1 -0
- package/dist/benchmarks/index.d.ts +5 -6
- package/dist/benchmarks/index.d.ts.map +1 -1
- package/dist/benchmarks/index.js +4 -4
- package/dist/benchmarks/index.js.map +1 -1
- package/dist/builder-eval/index.d.ts +3 -3
- package/dist/builder-eval/index.js +2 -2
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/campaign/index.d.ts +7 -7
- package/dist/campaign/index.js +15 -15
- package/dist/{campaign-BudZ70zo.js → campaign-CJGkxAQ6.js} +23 -530
- package/dist/campaign-CJGkxAQ6.js.map +1 -0
- package/dist/{campaign-evidence-Clrn4hBV.js → campaign-evidence-CQYw8_0H.js} +10 -18
- package/dist/{campaign-evidence-Clrn4hBV.js.map → campaign-evidence-CQYw8_0H.js.map} +1 -1
- package/dist/{canonical-B2RqLofv.js → canonical-Dq2hOQ8L.js} +2 -2
- package/dist/{canonical-B2RqLofv.js.map → canonical-Dq2hOQ8L.js.map} +1 -1
- package/dist/cell-spend-DEWR3nj-.js +90 -0
- package/dist/cell-spend-DEWR3nj-.js.map +1 -0
- package/dist/{chat-client-l-MVDlDk.js → chat-client-ldqxYYTN.js} +2 -2
- package/dist/{chat-client-l-MVDlDk.js.map → chat-client-ldqxYYTN.js.map} +1 -1
- package/dist/{chat-json-call-puMArmtW.js → chat-json-call-DavzexSp.js} +3 -2
- package/dist/chat-json-call-DavzexSp.js.map +1 -0
- package/dist/{claim-q4HQhK1G.js → claim-BPVuyHuy.js} +3 -3
- package/dist/{claim-q4HQhK1G.js.map → claim-BPVuyHuy.js.map} +1 -1
- package/dist/cli.js +113 -12
- package/dist/cli.js.map +1 -1
- package/dist/{client-BzL9niqi.d.ts → client-hwo3cJjR.d.ts} +3 -16
- package/dist/client-hwo3cJjR.d.ts.map +1 -0
- package/dist/{cluster-bootstrap-qcUWYFl-.js → cluster-bootstrap-DEwwhl0C.js} +3 -3
- package/dist/{cluster-bootstrap-qcUWYFl-.js.map → cluster-bootstrap-DEwwhl0C.js.map} +1 -1
- package/dist/{produced-state-BsDmnwxc.js → completion-verifier-B_jw6C9p.js} +9 -64
- package/dist/completion-verifier-B_jw6C9p.js.map +1 -0
- package/dist/contract/index.d.ts +12 -12
- package/dist/contract/index.js +25 -401
- package/dist/contract/index.js.map +1 -1
- package/dist/{cost-ledger-BP8V4kUQ.js → cost-ledger-D0352Z4d.js} +4 -8
- package/dist/cost-ledger-D0352Z4d.js.map +1 -0
- package/dist/dataset-DQqhOCPt.d.ts.map +1 -1
- package/dist/{default-registry-COhk9g36.js → default-registry-ClYjq3QE.js} +8 -11
- package/dist/default-registry-ClYjq3QE.js.map +1 -0
- package/dist/{default-registry-DzHiGruW.d.ts → default-registry-DXgLmoDI.d.ts} +7 -50
- package/dist/default-registry-DXgLmoDI.d.ts.map +1 -0
- package/dist/{define-agent-eval-CNTPzIkp.js → define-agent-eval-BDxNlP3D.js} +10 -10
- package/dist/{define-agent-eval-CNTPzIkp.js.map → define-agent-eval-BDxNlP3D.js.map} +1 -1
- package/dist/{define-agent-eval-WSck0CZJ.d.ts → define-agent-eval-BIkH_gAR.d.ts} +8 -8
- package/dist/define-agent-eval-BIkH_gAR.d.ts.map +1 -0
- package/dist/{descriptive-DboxPZrG.js → descriptive-CVNy48rR.js} +2 -2
- package/dist/{descriptive-DboxPZrG.js.map → descriptive-CVNy48rR.js.map} +1 -1
- package/dist/diagnosis/index.d.ts +4 -14
- package/dist/diagnosis/index.d.ts.map +1 -1
- package/dist/diagnosis/index.js +67 -28
- package/dist/diagnosis/index.js.map +1 -1
- package/dist/{dspy-rlm-engine-LRCu61m5.js → dspy-rlm-engine-DhdCr1Dc.js} +5 -10
- package/dist/dspy-rlm-engine-DhdCr1Dc.js.map +1 -0
- package/dist/{emitter-HcCmFQl_.d.ts → emitter-DABYYTcH.d.ts} +5 -5
- package/dist/emitter-DABYYTcH.d.ts.map +1 -0
- package/dist/emitter-DbMXUIVP.js.map +1 -1
- package/dist/{engine-3A_4jOlI.d.ts → engine-BvZ0R-36.d.ts} +17 -21
- package/dist/engine-BvZ0R-36.d.ts.map +1 -0
- package/dist/{equal-terms-BtdXayCJ.js → equal-terms-DbPGcVtI.js} +2 -2
- package/dist/equal-terms-DbPGcVtI.js.map +1 -0
- package/dist/{errors-DVZiqS7K.js → errors-BcOs5MKW.js} +2 -8
- package/dist/{errors-DVZiqS7K.js.map → errors-BcOs5MKW.js.map} +1 -1
- package/dist/{eval-campaign-90jeaCAI.js → eval-campaign-BLEfBGbA.js} +6 -6
- package/dist/{eval-campaign-90jeaCAI.js.map → eval-campaign-BLEfBGbA.js.map} +1 -1
- package/dist/{evaluation-6Yoi74gM.d.ts → evaluation-jTjC_QQD.d.ts} +7 -7
- package/dist/evaluation-jTjC_QQD.d.ts.map +1 -0
- package/dist/evaluation.d.ts +2 -2
- package/dist/evaluation.js +1 -1
- package/dist/evaluation.js.map +1 -1
- package/dist/{exact-types-ClugMJfI.d.ts → exact-types-CflbJ9f3.d.ts} +2 -2
- package/dist/{exact-types-ClugMJfI.d.ts.map → exact-types-CflbJ9f3.d.ts.map} +1 -1
- package/dist/{exec-ChJ6d5if.js → exec-CpbtOXVt.js} +2 -24
- package/dist/{exec-ChJ6d5if.js.map → exec-CpbtOXVt.js.map} +1 -1
- package/dist/execution-tracks-CpgFPpS5.js.map +1 -1
- package/dist/experiment/index.d.ts +9 -112
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +149 -241
- package/dist/experiment/index.js.map +1 -1
- package/dist/{exporters-X4xwla8A.js → exporters-Cl6IeNAu.js} +3 -3
- package/dist/exporters-Cl6IeNAu.js.map +1 -0
- package/dist/{exporters-ChTnR90l.d.ts → exporters-Dk3oEbK_.d.ts} +2 -4
- package/dist/{exporters-ChTnR90l.d.ts.map → exporters-Dk3oEbK_.d.ts.map} +1 -1
- package/dist/{external-optimizer-process-BjMGUuds.js → external-optimizer-process-KVFGvYX1.js} +4 -3
- package/dist/external-optimizer-process-KVFGvYX1.js.map +1 -0
- package/dist/{external-optimizer-subprocess-BqW0hHcI.js → external-optimizer-subprocess-XIHBOSKt.js} +9 -11
- package/dist/external-optimizer-subprocess-XIHBOSKt.js.map +1 -0
- package/dist/{extract-usage-BrnzRJ6H.js → extract-usage-BZLU01kR.js} +2 -23
- package/dist/{extract-usage-BrnzRJ6H.js.map → extract-usage-BZLU01kR.js.map} +1 -1
- package/dist/{failure-cluster-CbgEDcFJ.d.ts → failure-cluster-DN0t63Z0.d.ts} +4 -43
- package/dist/failure-cluster-DN0t63Z0.d.ts.map +1 -0
- package/dist/{failure-taxonomy-CjabTATj.js → failure-taxonomy-Bx5oWqaW.js} +2 -19
- package/dist/{failure-taxonomy-CjabTATj.js.map → failure-taxonomy-Bx5oWqaW.js.map} +1 -1
- package/dist/{feedback-trajectory-C4EG0nNH.d.ts → feedback-trajectory-BWmhRBGu.d.ts} +7 -9
- package/dist/feedback-trajectory-BWmhRBGu.d.ts.map +1 -0
- package/dist/{semantic-concept-judge-BQF8te0T.js → findings-store-BMVQtEsm.js} +9 -249
- package/dist/findings-store-BMVQtEsm.js.map +1 -0
- package/dist/fuzz.d.ts +19 -62
- package/dist/fuzz.d.ts.map +1 -1
- package/dist/fuzz.js +62 -108
- package/dist/fuzz.js.map +1 -1
- package/dist/{hf-dataset-BR2yhuNn.js → hf-dataset-D9f7-oX7.js} +4 -24
- package/dist/hf-dataset-D9f7-oX7.js.map +1 -0
- package/dist/hosted/index.d.ts +2 -2
- package/dist/hosted/index.js +2 -2
- package/dist/{index-DHKEAv3s.d.ts → index-9jJ2fKSX.d.ts} +2 -64
- package/dist/index-9jJ2fKSX.d.ts.map +1 -0
- package/dist/{index-DV-jSzxF.d.ts → index-BLGqZ81N.d.ts} +10 -15
- package/dist/{index-DV-jSzxF.d.ts.map → index-BLGqZ81N.d.ts.map} +1 -1
- package/dist/{index-DPcngtwt.d.ts → index-BiT6RtNH.d.ts} +11 -36
- package/dist/index-BiT6RtNH.d.ts.map +1 -0
- package/dist/{index-BO41eGj5.d.ts → index-CX-GAUpw.d.ts} +3 -7
- package/dist/index-CX-GAUpw.d.ts.map +1 -0
- package/dist/{index-BFRO6chs.d.ts → index-CiRwLv2R.d.ts} +5 -69
- package/dist/index-CiRwLv2R.d.ts.map +1 -0
- package/dist/{index-CyszSg4O.d.ts → index-ZuWx52WG.d.ts} +115 -347
- package/dist/index-ZuWx52WG.d.ts.map +1 -0
- package/dist/index.d.ts +211 -218
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +402 -1040
- package/dist/index.js.map +1 -1
- package/dist/{integrity-dLiDe2kD.js → integrity-DOd9Q86O.js} +7 -9
- package/dist/integrity-DOd9Q86O.js.map +1 -0
- package/dist/{integrity-BXOsAb3i.js → integrity-W4CADcFc.js} +3 -7
- package/dist/{integrity-BXOsAb3i.js.map → integrity-W4CADcFc.js.map} +1 -1
- package/dist/{integrity-DAMEDFaC.d.ts → integrity-_qgZLA5p.d.ts} +4 -6
- package/dist/{integrity-DAMEDFaC.d.ts.map → integrity-_qgZLA5p.d.ts.map} +1 -1
- package/dist/{internal-Epl-YBZW.js → internal-BTK91Bqt.js} +3 -3
- package/dist/{internal-Epl-YBZW.js.map → internal-BTK91Bqt.js.map} +1 -1
- package/dist/{interval-CELpye1n.js → interval-Dq9gt7jx.js} +3 -3
- package/dist/{interval-CELpye1n.js.map → interval-Dq9gt7jx.js.map} +1 -1
- package/dist/{journal-DfRz9kps.js → journal-CR4QikYp.js} +110 -110
- package/dist/journal-CR4QikYp.js.map +1 -0
- package/dist/json-BhdsJaOQ.js +111 -0
- package/dist/json-BhdsJaOQ.js.map +1 -0
- package/dist/{judge-calibration-BpCy5XQl.js → judge-calibration-BkXMIWA4.js} +3 -3
- package/dist/{judge-calibration-BpCy5XQl.js.map → judge-calibration-BkXMIWA4.js.map} +1 -1
- package/dist/{judge-gate-iY9s4L44.js → judge-gate-Co5tuh57.js} +4 -27
- package/dist/{judge-gate-iY9s4L44.js.map → judge-gate-Co5tuh57.js.map} +1 -1
- package/dist/{judge-gate-CE7CQAgt.d.ts → judge-gate-DsL0yFcb.d.ts} +8 -9
- package/dist/judge-gate-DsL0yFcb.d.ts.map +1 -0
- package/dist/{kind-factory-BobI5Jy3.js → kind-factory-Gy4fXixd.js} +88 -17
- package/dist/kind-factory-Gy4fXixd.js.map +1 -0
- package/dist/ledger-core/index.d.ts +2 -2
- package/dist/ledger-core/index.js +4 -4
- package/dist/{ledger-core-Gq7yp8lP.js → ledger-core-DrJhbms9.js} +5 -84
- package/dist/ledger-core-DrJhbms9.js.map +1 -0
- package/dist/{llm-client-CyBAoYiX.js → llm-client-BshCREn6.js} +6 -108
- package/dist/llm-client-BshCREn6.js.map +1 -0
- package/dist/{llm-judge-CG12SQJq.js → llm-judge-C8jMTNNG.js} +24 -32
- package/dist/llm-judge-C8jMTNNG.js.map +1 -0
- package/dist/matrix/index.d.ts +3 -3
- package/dist/matrix/index.js +3 -2
- package/dist/{matrix-Dm0BQW7U.js → matrix-B8l-Nsxd.js} +3 -88
- package/dist/matrix-B8l-Nsxd.js.map +1 -0
- package/dist/{matrix-DSrSxjjy.d.ts → matrix-DHe-dwVJ.d.ts} +9 -18
- package/dist/matrix-DHe-dwVJ.d.ts.map +1 -0
- package/dist/{mean-DHoMiudD.js → mean-BbA_JCzW.js} +39 -5
- package/dist/mean-BbA_JCzW.js.map +1 -0
- package/dist/meta-eval/index.d.ts +5 -5
- package/dist/meta-eval/index.js +10 -10
- package/dist/metrics-BKaPBtIG.js.map +1 -1
- package/dist/{mint-DNU8QPG4.d.ts → mint-BRZXuwRi.d.ts} +4 -4
- package/dist/{mint-DNU8QPG4.d.ts.map → mint-BRZXuwRi.d.ts.map} +1 -1
- package/dist/{mint-QaD5cA0u.js → mint-zszyYlpA.js} +4 -4
- package/dist/{mint-QaD5cA0u.js.map → mint-zszyYlpA.js.map} +1 -1
- package/dist/{multi-layer-verifier-BUaQ4C17.d.ts → multi-layer-verifier-DEgMUk6v.d.ts} +2 -2
- package/dist/{multi-layer-verifier-BUaQ4C17.d.ts.map → multi-layer-verifier-DEgMUk6v.d.ts.map} +1 -1
- package/dist/{multiplicity-DQUFcbW4.js → multiplicity-Sg0U7h8j.js} +2 -2
- package/dist/{multiplicity-DQUFcbW4.js.map → multiplicity-Sg0U7h8j.js.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -23
- package/dist/multishot/golden/index.d.ts.map +1 -1
- package/dist/multishot/golden/index.js +3 -3
- package/dist/multishot/golden/index.js.map +1 -1
- package/dist/multishot/index.d.ts +3 -12
- package/dist/multishot/index.d.ts.map +1 -1
- package/dist/multishot/index.js +4 -3
- package/dist/multishot/index.js.map +1 -1
- package/dist/openapi.json +114 -10
- package/dist/{otel-spans-B9tUXRHF.js → otel-spans-CdN909k1.js} +3 -3
- package/dist/{otel-spans-B9tUXRHF.js.map → otel-spans-CdN909k1.js.map} +1 -1
- package/dist/{paired-arms-CwMGJsrS.js → paired-arms-Ct4LTKc1.js} +32 -9
- package/dist/paired-arms-Ct4LTKc1.js.map +1 -0
- package/dist/{paired-binary-CvpFHfKZ.js → paired-binary-B_nejA44.js} +3 -36
- package/dist/paired-binary-B_nejA44.js.map +1 -0
- package/dist/{paired-promotion-decision-DPKz8ZnF.d.ts → paired-promotion-decision-CQMJ5xGN.d.ts} +2 -34
- package/dist/paired-promotion-decision-CQMJ5xGN.d.ts.map +1 -0
- package/dist/{paired-promotion-decision-DsAPYGCm.js → paired-promotion-decision-D0SHb-ZL.js} +3 -3
- package/dist/{paired-promotion-decision-DsAPYGCm.js.map → paired-promotion-decision-D0SHb-ZL.js.map} +1 -1
- package/dist/{pareto-Cm6ndrWN.js → pareto-BzsxidFD.js} +11 -11
- package/dist/pareto-BzsxidFD.js.map +1 -0
- package/dist/pipelines/index.d.ts +16 -19
- package/dist/pipelines/index.d.ts.map +1 -1
- package/dist/pipelines/index.js +4 -68
- package/dist/pipelines/index.js.map +1 -1
- package/dist/{power-and-mde-Hj2ubb4B.js → power-and-mde-N5nU4_Ly.js} +3 -3
- package/dist/{power-and-mde-Hj2ubb4B.js.map → power-and-mde-N5nU4_Ly.js.map} +1 -1
- package/dist/{prime-protocol-CHKgA3a6.js → prime-protocol-BFgGrEMP.js} +3 -3
- package/dist/prime-protocol-BFgGrEMP.js.map +1 -0
- package/dist/{prime-protocol-HiB9HX7E.d.ts → prime-protocol-BWG2DQIk.d.ts} +3 -21
- package/dist/prime-protocol-BWG2DQIk.d.ts.map +1 -0
- package/dist/profile-cell.d.ts +2 -2
- package/dist/profile-cell.js +2 -2
- package/dist/{promotion-gate-BawrdPYr.d.ts → promotion-gate-DaY95yU6.d.ts} +7 -8
- package/dist/{promotion-gate-BawrdPYr.d.ts.map → promotion-gate-DaY95yU6.d.ts.map} +1 -1
- package/dist/{promotion-policy-KO57gbyV.js → promotion-policy-6A8osGr_.js} +4 -4
- package/dist/{promotion-policy-KO57gbyV.js.map → promotion-policy-6A8osGr_.js.map} +1 -1
- package/dist/{proposal-findings-CY1xKn_a.js → proposal-findings-BbWidvUk.js} +3 -3
- package/dist/proposal-findings-BbWidvUk.js.map +1 -0
- package/dist/{query-BUCZCK7F.js → query-B_RZ36an.js} +5 -5
- package/dist/query-B_RZ36an.js.map +1 -0
- package/dist/{query-DK1nWcmp.d.ts → query-Cmv_U8vf.d.ts} +4 -24
- package/dist/{query-DK1nWcmp.d.ts.map → query-Cmv_U8vf.d.ts.map} +1 -1
- package/dist/{random-ByUSnV9w.js → random-DDDcC7YU.js} +2 -2
- package/dist/{random-ByUSnV9w.js.map → random-DDDcC7YU.js.map} +1 -1
- package/dist/{raw-provider-sink-DoHz4JNG.js → raw-provider-sink-75w4FWBj.js} +2 -2
- package/dist/{raw-provider-sink-DoHz4JNG.js.map → raw-provider-sink-75w4FWBj.js.map} +1 -1
- package/dist/{redact-mtCCcT8z.d.ts → redact-DXdlFb_U.d.ts} +2 -3
- package/dist/redact-DXdlFb_U.d.ts.map +1 -0
- package/dist/{redact-B5xXKqQ3.js → redact-mtkTiJVk.js} +2 -7
- package/dist/{redact-B5xXKqQ3.js.map → redact-mtkTiJVk.js.map} +1 -1
- package/dist/{registry-D7_hm2fK.d.ts → registry-ssNoHzwD.d.ts} +4 -4
- package/dist/{registry-D7_hm2fK.d.ts.map → registry-ssNoHzwD.d.ts.map} +1 -1
- package/dist/{release-confidence-CpuP65S5.js → release-confidence-CYIFH-fp.js} +6 -11
- package/dist/release-confidence-CYIFH-fp.js.map +1 -0
- package/dist/{report-command-SC9nqb4M.js → render-C-w2EZ6p.js} +205 -363
- package/dist/render-C-w2EZ6p.js.map +1 -0
- package/dist/reporting.d.ts +6 -6
- package/dist/reporting.js +8 -8
- package/dist/{researcher-CEcfku-T.d.ts → researcher-0ABVIdGa.d.ts} +9 -9
- package/dist/{researcher-CEcfku-T.d.ts.map → researcher-0ABVIdGa.d.ts.map} +1 -1
- package/dist/{reward-CpCmAM8L.d.ts → reward-DrZ71vnf.d.ts} +3 -26
- package/dist/reward-DrZ71vnf.d.ts.map +1 -0
- package/dist/{reward-hacking-E0d-WVcP.js → reward-hacking-ByuETaIb.js} +3 -3
- package/dist/{reward-hacking-E0d-WVcP.js.map → reward-hacking-ByuETaIb.js.map} +1 -1
- package/dist/{reward-hacking-RVdZ6Kkt.d.ts → reward-hacking-E52wq86n.d.ts} +5 -18
- package/dist/reward-hacking-E52wq86n.d.ts.map +1 -0
- package/dist/reward-nw2xZGZG.js.map +1 -1
- package/dist/rl.d.ts +117 -295
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +27 -747
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +5 -5
- package/dist/rollout/index.js +7 -8
- package/dist/{rollout-DZV1t2Wg.js → rollout-D6C50mfA.js} +88 -13
- package/dist/rollout-D6C50mfA.js.map +1 -0
- package/dist/{rubric-predictive-validity-DaFW70y9.js → rubric-predictive-validity-D3_oI72N.js} +3 -3
- package/dist/{rubric-predictive-validity-DaFW70y9.js.map → rubric-predictive-validity-D3_oI72N.js.map} +1 -1
- package/dist/{rubric-predictive-validity-C03EC7ZE.d.ts → rubric-predictive-validity-DiwT8PbL.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-C03EC7ZE.d.ts.map → rubric-predictive-validity-DiwT8PbL.d.ts.map} +1 -1
- package/dist/{run-record-C1E24G2A.d.ts → run-record-CaxbLq6Y.d.ts} +4 -10
- package/dist/run-record-CaxbLq6Y.d.ts.map +1 -0
- package/dist/{run-record-C5e7FjhJ.js → run-record-CysKLB93.js} +3 -3
- package/dist/run-record-CysKLB93.js.map +1 -0
- package/dist/{run-record-Boc5IUTp.js → run-record-DhEJgoPz.js} +4 -4
- package/dist/{run-record-Boc5IUTp.js.map → run-record-DhEJgoPz.js.map} +1 -1
- package/dist/sandbox-harness-BlSOu4LX.d.ts.map +1 -1
- package/dist/{schema-CSf6qWgZ.js → schema-Bt2vLFm3.js} +2 -14
- package/dist/schema-Bt2vLFm3.js.map +1 -0
- package/dist/schema-C-ziyJD7.d.ts.map +1 -1
- package/dist/{schema-2g7trUwH.d.ts → schema-D6EVKzTB.d.ts} +4 -5
- package/dist/schema-D6EVKzTB.d.ts.map +1 -0
- package/dist/{schema-BuiujZ-g.js → schema-DxcKTzbi.js} +2 -2
- package/dist/schema-DxcKTzbi.js.map +1 -0
- package/dist/search/index.d.ts +3 -31
- package/dist/search/index.d.ts.map +1 -1
- package/dist/search/index.js +4 -4
- package/dist/{search-kernel-DyNEa6IU.d.ts → search-kernel-DxqjBxA-.d.ts} +10 -27
- package/dist/search-kernel-DxqjBxA-.d.ts.map +1 -0
- package/dist/{search-ledger-CVUGLDZp.js → search-ledger-TUvB5gWH.js} +7 -7
- package/dist/{search-ledger-CVUGLDZp.js.map → search-ledger-TUvB5gWH.js.map} +1 -1
- package/dist/{search-ledger-file-DDQUgvCg.js → search-ledger-file-UKCO2r8a.js} +4 -4
- package/dist/search-ledger-file-UKCO2r8a.js.map +1 -0
- package/dist/{search-shipper-BOSD46qv.js → search-shipper-Cw2z3-td.js} +4 -4
- package/dist/search-shipper-Cw2z3-td.js.map +1 -0
- package/dist/{search-summary-BQCSWf1g.js → search-summary-DQIvJsSB.js} +5 -5
- package/dist/{search-summary-BQCSWf1g.js.map → search-summary-DQIvJsSB.js.map} +1 -1
- package/dist/{sequential-CzK5DarL.js → sequential-BVv1sjsw.js} +2 -32
- package/dist/{sequential-CzK5DarL.js.map → sequential-BVv1sjsw.js.map} +1 -1
- package/dist/{sequential-Bf_pbE50.js → sequential-CimQrbjv.js} +5 -5
- package/dist/sequential-CimQrbjv.js.map +1 -0
- package/dist/{sequential-BhsrMupG.d.ts → sequential-KwGspfeR.d.ts} +2 -9
- package/dist/{sequential-BhsrMupG.d.ts.map → sequential-KwGspfeR.d.ts.map} +1 -1
- package/dist/{sequential-eprocess-DM_qENCl.js → sequential-eprocess-DodI6RoF.js} +2 -2
- package/dist/{sequential-eprocess-DM_qENCl.js.map → sequential-eprocess-DodI6RoF.js.map} +1 -1
- package/dist/{server-e9oWO_bN.js → server-DoVbqqtJ.js} +101 -101
- package/dist/server-DoVbqqtJ.js.map +1 -0
- package/dist/{single-run-lock-CV9Eyd_q.js → single-run-lock-U58yf8N6.js} +2 -2
- package/dist/{single-run-lock-CV9Eyd_q.js.map → single-run-lock-U58yf8N6.js.map} +1 -1
- package/dist/{skillopt-optimization-method-BiwA8nfX.js → skillopt-optimization-method-kyL94-ri.js} +17 -95
- package/dist/skillopt-optimization-method-kyL94-ri.js.map +1 -0
- package/dist/{spans-BHAUXMGN.d.ts → spans-CVngGij3.d.ts} +2 -2
- package/dist/{spans-BHAUXMGN.d.ts.map → spans-CVngGij3.d.ts.map} +1 -1
- package/dist/{statistical-heldout-42NlCNDe.js → statistical-heldout-Bksv92Th.js} +3 -3
- package/dist/statistical-heldout-Bksv92Th.js.map +1 -0
- package/dist/{statistical-heldout-CYpGaoDH.d.ts → statistical-heldout-C9AbgREN.d.ts} +8 -67
- package/dist/statistical-heldout-C9AbgREN.d.ts.map +1 -0
- package/dist/{steps-Bg5-KVx1.d.ts → steps-DlepJPwD.d.ts} +2 -56
- package/dist/steps-DlepJPwD.d.ts.map +1 -0
- package/dist/{store-C4avFjIi.d.ts → store-B3nWRy3B.d.ts} +2 -2
- package/dist/{store-C4avFjIi.d.ts.map → store-B3nWRy3B.d.ts.map} +1 -1
- package/dist/{store-otlp-DJ9dtA5x.js → store-otlp-BteDAbHK.js} +3 -3
- package/dist/{store-otlp-DJ9dtA5x.js.map → store-otlp-BteDAbHK.js.map} +1 -1
- package/dist/{store-tool-spans-CIX5arAG.js → store-tool-spans-D5l_OfTJ.js} +8 -8
- package/dist/store-tool-spans-D5l_OfTJ.js.map +1 -0
- package/dist/{store-tool-spans-DdCSPa7C.d.ts → store-tool-spans-DZS0HWVf.d.ts} +9 -18
- package/dist/store-tool-spans-DZS0HWVf.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts +4 -11
- package/dist/storyboard/index.d.ts.map +1 -1
- package/dist/storyboard/index.js +1 -16
- package/dist/storyboard/index.js.map +1 -1
- package/dist/{summary-report-V_JACR4Y.js → summary-report-BsXJoYX5.js} +88 -10
- package/dist/summary-report-BsXJoYX5.js.map +1 -0
- package/dist/{summary-report-bdFDXq0b.d.ts → summary-report-DM7tlml1.d.ts} +10 -19
- package/dist/summary-report-DM7tlml1.d.ts.map +1 -0
- package/dist/supervisor-run/index.d.ts +60 -28
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +4 -458
- package/dist/supervisor-run/index.js.map +1 -1
- package/dist/{systemone-BomhHr2c.d.ts → systemone-Bp_Ac0p1.d.ts} +5 -5
- package/dist/{systemone-BomhHr2c.d.ts.map → systemone-Bp_Ac0p1.d.ts.map} +1 -1
- package/dist/systemone-protocol.js +1 -1
- package/dist/{systemone-review-CUVcQJu6.d.ts → systemone-review-CwE6JGIp.d.ts} +2 -2
- package/dist/{systemone-review-CUVcQJu6.d.ts.map → systemone-review-CwE6JGIp.d.ts.map} +1 -1
- package/dist/{systemone-review-D7sXwsqq.js → systemone-review-DzdOJ8I6.js} +3 -3
- package/dist/{systemone-review-D7sXwsqq.js.map → systemone-review-DzdOJ8I6.js.map} +1 -1
- package/dist/systemone.d.ts +5 -5
- package/dist/systemone.js +5 -5
- package/dist/systemone.js.map +1 -1
- package/dist/{task-failure-attributes-hEmASjix.js → task-failure-attributes-BMsblaDh.js} +3 -3
- package/dist/task-failure-attributes-BMsblaDh.js.map +1 -0
- package/dist/{terminal-record-DZyQZ3pz.js → terminal-record-78g6E4vh.js} +5 -5
- package/dist/terminal-record-78g6E4vh.js.map +1 -0
- package/dist/{tool-groups-C3NEcdCM.d.ts → tool-groups-DyNz2JZ_.d.ts} +3 -3
- package/dist/tool-groups-DyNz2JZ_.d.ts.map +1 -0
- package/dist/{tool-waste-BKgvd-TP.d.ts → tool-waste-CkjzWmeY.d.ts} +25 -28
- package/dist/tool-waste-CkjzWmeY.d.ts.map +1 -0
- package/dist/{tool-waste-BlesqTqW.js → tool-waste-DLsjS_p-.js} +5 -5
- package/dist/{tool-waste-BlesqTqW.js.map → tool-waste-DLsjS_p-.js.map} +1 -1
- package/dist/{trace-questions-CM2CAocU.d.ts → trace-questions-xxwgsuMi.d.ts} +4 -9
- package/dist/{trace-questions-CM2CAocU.d.ts.map → trace-questions-xxwgsuMi.d.ts.map} +1 -1
- package/dist/trace-repair/index.d.ts +14 -211
- package/dist/trace-repair/index.d.ts.map +1 -1
- package/dist/trace-repair/index.js +865 -981
- package/dist/trace-repair/index.js.map +1 -1
- package/dist/traces.d.ts +18 -46
- package/dist/traces.d.ts.map +1 -1
- package/dist/traces.js +13 -13
- package/dist/traces.js.map +1 -1
- package/dist/trajectory-replay/index.d.ts +3 -117
- package/dist/trajectory-replay/index.d.ts.map +1 -1
- package/dist/trajectory-replay/index.js +98 -56
- package/dist/trajectory-replay/index.js.map +1 -1
- package/dist/{tree-C1XYeXvy.js → tree-BGRglkKd.js} +13 -10
- package/dist/tree-BGRglkKd.js.map +1 -0
- package/dist/{types-B3jzCp0p.js → types-BXgASBCL.js} +2 -2
- package/dist/{types-B3jzCp0p.js.map → types-BXgASBCL.js.map} +1 -1
- package/dist/{types-C1tnkEgM.d.ts → types-CW8ewATZ.d.ts} +4 -113
- package/dist/types-CW8ewATZ.d.ts.map +1 -0
- package/dist/{types-BqkVmHfv.d.ts → types-CmXY0PQf.d.ts} +2 -4
- package/dist/types-CmXY0PQf.d.ts.map +1 -0
- package/dist/{types-C02qtnG8.d.ts → types-D-mgRzlh.d.ts} +7 -28
- package/dist/types-D-mgRzlh.d.ts.map +1 -0
- package/dist/{types-Cq8mSbq8.d.ts → types-D8GhJoHk.d.ts} +7 -9
- package/dist/{types-Cq8mSbq8.d.ts.map → types-D8GhJoHk.d.ts.map} +1 -1
- package/dist/{types-IXD2HJzu.js → types-DqFMNTgK.js} +2 -4
- package/dist/types-DqFMNTgK.js.map +1 -0
- package/dist/{types-Cbe2fWBa.d.ts → types-DtMbFw6U.d.ts} +3 -3
- package/dist/{types-Cbe2fWBa.d.ts.map → types-DtMbFw6U.d.ts.map} +1 -1
- package/dist/{verdict-Dl0tiT1X.js → verdict-BGQmxQ8A.js} +3 -3
- package/dist/verdict-BGQmxQ8A.js.map +1 -0
- package/dist/{verdict-E4eRNf7-.d.ts → verdict-C24NELtN.d.ts} +2 -15
- package/dist/{verdict-E4eRNf7-.d.ts.map → verdict-C24NELtN.d.ts.map} +1 -1
- package/dist/{verdict-cache-Bm8EKL2s.js → verdict-cache-CL_7tePD.js} +2 -2
- package/dist/verdict-cache-CL_7tePD.js.map +1 -0
- package/dist/wire/index.d.ts +31 -410
- package/dist/wire/index.d.ts.map +1 -1
- package/dist/wire/index.js +2 -2
- package/docs/changes/jev-evidence-review.md +1 -1
- package/docs/changes/jev-product-adoption.md +1 -1
- package/docs/changes/jev-recorded-evidence.md +1 -1
- package/docs/design/statistics-decisions.md +17 -17
- package/docs/design.md +1 -1
- package/docs/evaluation-integrity.md +1 -1
- package/docs/proofs/campaign-attempt-retention-20261001/README.md +2 -2
- package/docs/public-api.md +138 -529
- package/docs/rollout.md +1 -1
- package/docs/trace-repair-admission.md +1 -1
- package/docs/wire-protocol.md +1 -1
- package/package.json +1 -1
- package/dist/active-curriculum-B_Lc9kfO.js +0 -61
- package/dist/active-curriculum-B_Lc9kfO.js.map +0 -1
- package/dist/agent-profile-VFcQncbW.d.ts.map +0 -1
- package/dist/agent-profile-cell-BK8mlqlq.js.map +0 -1
- package/dist/agent-profile-cell-BNp-Js9F.d.ts.map +0 -1
- package/dist/agreement-irr-BEWLIF3_.js.map +0 -1
- package/dist/analyze-runs-C84irm4i.js.map +0 -1
- package/dist/baseline-Udi_PLuT.js +0 -259
- package/dist/baseline-Udi_PLuT.js.map +0 -1
- package/dist/benchmark-CXmVVHH2.js +0 -562
- package/dist/benchmark-CXmVVHH2.js.map +0 -1
- package/dist/benchmark-command-By0vh8q7.js.map +0 -1
- package/dist/campaign-BudZ70zo.js.map +0 -1
- package/dist/chat-json-call-puMArmtW.js.map +0 -1
- package/dist/claude-jsonl-dZir12m6.js +0 -86
- package/dist/claude-jsonl-dZir12m6.js.map +0 -1
- package/dist/client-BzL9niqi.d.ts.map +0 -1
- package/dist/cost-ledger-BP8V4kUQ.js.map +0 -1
- package/dist/counterfactual-CPaWN99-.d.ts +0 -60
- package/dist/counterfactual-CPaWN99-.d.ts.map +0 -1
- package/dist/counterfactual-Dn3veueY.js +0 -98
- package/dist/counterfactual-Dn3veueY.js.map +0 -1
- package/dist/default-registry-COhk9g36.js.map +0 -1
- package/dist/default-registry-DzHiGruW.d.ts.map +0 -1
- package/dist/define-agent-eval-WSck0CZJ.d.ts.map +0 -1
- package/dist/dspy-rlm-engine-LRCu61m5.js.map +0 -1
- package/dist/effect-sizes-BhzEpXMc.js +0 -82
- package/dist/effect-sizes-BhzEpXMc.js.map +0 -1
- package/dist/emitter-HcCmFQl_.d.ts.map +0 -1
- package/dist/engine-3A_4jOlI.d.ts.map +0 -1
- package/dist/equal-terms-BtdXayCJ.js.map +0 -1
- package/dist/evaluation-6Yoi74gM.d.ts.map +0 -1
- package/dist/exporters-X4xwla8A.js.map +0 -1
- package/dist/external-optimizer-process-BjMGUuds.js.map +0 -1
- package/dist/external-optimizer-subprocess-BqW0hHcI.js.map +0 -1
- package/dist/failure-cluster-CbgEDcFJ.d.ts.map +0 -1
- package/dist/feedback-trajectory-C4EG0nNH.d.ts.map +0 -1
- package/dist/hf-dataset-BR2yhuNn.js.map +0 -1
- package/dist/index-BFRO6chs.d.ts.map +0 -1
- package/dist/index-BO41eGj5.d.ts.map +0 -1
- package/dist/index-CyszSg4O.d.ts.map +0 -1
- package/dist/index-DHKEAv3s.d.ts.map +0 -1
- package/dist/index-DPcngtwt.d.ts.map +0 -1
- package/dist/integrity-dLiDe2kD.js.map +0 -1
- package/dist/journal-DfRz9kps.js.map +0 -1
- package/dist/judge-gate-CE7CQAgt.d.ts.map +0 -1
- package/dist/kind-factory-BobI5Jy3.js.map +0 -1
- package/dist/ledger-core-Gq7yp8lP.js.map +0 -1
- package/dist/llm-client-CyBAoYiX.js.map +0 -1
- package/dist/llm-judge-CG12SQJq.js.map +0 -1
- package/dist/matrix-DSrSxjjy.d.ts.map +0 -1
- package/dist/matrix-Dm0BQW7U.js.map +0 -1
- package/dist/mean-DHoMiudD.js.map +0 -1
- package/dist/normal-DsJZtrJc.js +0 -25
- package/dist/normal-DsJZtrJc.js.map +0 -1
- package/dist/paired-arms-CwMGJsrS.js.map +0 -1
- package/dist/paired-binary-CvpFHfKZ.js.map +0 -1
- package/dist/paired-promotion-decision-DPKz8ZnF.d.ts.map +0 -1
- package/dist/pareto-Cm6ndrWN.js.map +0 -1
- package/dist/prime-protocol-CHKgA3a6.js.map +0 -1
- package/dist/prime-protocol-HiB9HX7E.d.ts.map +0 -1
- package/dist/produced-state-BsDmnwxc.js.map +0 -1
- package/dist/proposal-findings-CY1xKn_a.js.map +0 -1
- package/dist/query-BUCZCK7F.js.map +0 -1
- package/dist/redact-mtCCcT8z.d.ts.map +0 -1
- package/dist/release-confidence-CpuP65S5.js.map +0 -1
- package/dist/report-command-SC9nqb4M.js.map +0 -1
- package/dist/reward-CpCmAM8L.d.ts.map +0 -1
- package/dist/reward-hacking-RVdZ6Kkt.d.ts.map +0 -1
- package/dist/rollout-DZV1t2Wg.js.map +0 -1
- package/dist/run-record-C1E24G2A.d.ts.map +0 -1
- package/dist/run-record-C5e7FjhJ.js.map +0 -1
- package/dist/schema-2g7trUwH.d.ts.map +0 -1
- package/dist/schema-BuiujZ-g.js.map +0 -1
- package/dist/schema-CSf6qWgZ.js.map +0 -1
- package/dist/search-kernel-DyNEa6IU.d.ts.map +0 -1
- package/dist/search-ledger-file-DDQUgvCg.js.map +0 -1
- package/dist/search-shipper-BOSD46qv.js.map +0 -1
- package/dist/semantic-concept-judge-BQF8te0T.js.map +0 -1
- package/dist/sequential-Bf_pbE50.js.map +0 -1
- package/dist/server-e9oWO_bN.js.map +0 -1
- package/dist/skillopt-optimization-method-BiwA8nfX.js.map +0 -1
- package/dist/statistical-heldout-42NlCNDe.js.map +0 -1
- package/dist/statistical-heldout-CYpGaoDH.d.ts.map +0 -1
- package/dist/steps-Bg5-KVx1.d.ts.map +0 -1
- package/dist/store-tool-spans-CIX5arAG.js.map +0 -1
- package/dist/store-tool-spans-DdCSPa7C.d.ts.map +0 -1
- package/dist/student-t-DVPW7CdL.js +0 -39
- package/dist/student-t-DVPW7CdL.js.map +0 -1
- package/dist/summary-report-V_JACR4Y.js.map +0 -1
- package/dist/summary-report-bdFDXq0b.d.ts.map +0 -1
- package/dist/task-failure-attributes-hEmASjix.js.map +0 -1
- package/dist/terminal-record-DZyQZ3pz.js.map +0 -1
- package/dist/tool-groups-C3NEcdCM.d.ts.map +0 -1
- package/dist/tool-waste-BKgvd-TP.d.ts.map +0 -1
- package/dist/trajectory-Y1fmdlw7.d.ts +0 -24
- package/dist/trajectory-Y1fmdlw7.d.ts.map +0 -1
- package/dist/tree-C1XYeXvy.js.map +0 -1
- package/dist/types-BqkVmHfv.d.ts.map +0 -1
- package/dist/types-C02qtnG8.d.ts.map +0 -1
- package/dist/types-C1tnkEgM.d.ts.map +0 -1
- package/dist/types-IXD2HJzu.js.map +0 -1
- package/dist/usage-receipt-Cx3xNeyB.js +0 -76
- package/dist/usage-receipt-Cx3xNeyB.js.map +0 -1
- package/dist/verdict-Dl0tiT1X.js.map +0 -1
- package/dist/verdict-cache-Bm8EKL2s.js.map +0 -1
- package/docs/design/self-improvement-evidence-audit.md +0 -263
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,51 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval-
|
|
|
4
4
|
|
|
5
5
|
---
|
|
6
6
|
|
|
7
|
+
## [0.217.1] — 2026-10-21
|
|
8
|
+
|
|
9
|
+
0.217.0 was tagged but never published: its package fixture check failed. 0.217.1 is the first published release of the 0.217 changes; the breaking changes listed under 0.217.0 below apply to it.
|
|
10
|
+
|
|
11
|
+
- fix(release): follow the rl function rename in the package fixtures
|
|
12
|
+
|
|
13
|
+
## [0.217.0] — 2026-10-21
|
|
14
|
+
|
|
15
|
+
**Breaking (0.x minor).**
|
|
16
|
+
|
|
17
|
+
- 442 value exports no `tangle-network` repository imports are no longer published (1,663 → 1,221 across subpaths; the list is the diff of `docs/public-api.md` in #1039). Consumer sweep: all 105 non-archived org repos on 2026-10-21.
|
|
18
|
+
- Published names renamed so one name means one concept (#1040): `/multishot` `JudgeConfig` → `MultishotJudgeConfig`; `/wire` `FeedbackTrajectory`, `TraceEvent`, `FailureMode`, `RubricDimension` → `WireFeedbackTrajectory`, `WireTraceEvent`, `WireFailureMode`, `WireRubricDimension`; `/systemone` `Evaluator` → `CostedEvaluator`; `/rl` `toSftRows` → `toSftRowsWithLookups`, `paretoFrontier` → `paretoCostScoreFrontier`; plus type-only renames in `/fuzz`, `/experiment`, `/traces`, `/search`, `/trace-repair`, `/campaign`.
|
|
19
|
+
- Consumer repos that import a renamed name: gtm-agent, tax-agent, super-agent, legal-agent, creative-agent. Their rename PRs are linked from the GitHub release notes.
|
|
20
|
+
|
|
21
|
+
- refactor!: follow the rl function rename in the example, contract test and census
|
|
22
|
+
- test: follow the renamed types and functions
|
|
23
|
+
- chore: remove the one-off prune tooling
|
|
24
|
+
- chore: regenerate the API census and re-pin the analyst digest
|
|
25
|
+
- chore(hooks): chain the global ratchets entry point, not loc-ratchet alone (#1038)
|
|
26
|
+
- test: delete the tests that only covered the removed names
|
|
27
|
+
- refactor!: one name per concept across published types and functions
|
|
28
|
+
- refactor!: keep the names other modules import, delete the pruned tests
|
|
29
|
+
- refactor!: stop publishing names no tangle-network repo imports, and delete what that orphaned
|
|
30
|
+
- wip: prune tooling v2
|
|
31
|
+
- wip: prune tooling
|
|
32
|
+
- chore: regenerate the API census and re-pin the analyst digest
|
|
33
|
+
- refactor: import ContinuousAgreement from its owner, not a forwarding file
|
|
34
|
+
- test: type the search matcher inline
|
|
35
|
+
- refactor: delete the run-record store class, eval tools, cached judge and tool-span gate nothing runs
|
|
36
|
+
- refactor: delete four scripts nothing runs or cites
|
|
37
|
+
- chore: drop the waiver for the deleted TokenCounter
|
|
38
|
+
- refactor: delete exports and declarations nothing imports
|
|
39
|
+
- chore(analyst): re-pin the implementation digest after the internal renames
|
|
40
|
+
- refactor: rename the internal side of three more name collisions
|
|
41
|
+
- docs(search-policy): name AIDE's search_policy without a file path
|
|
42
|
+
- chore(analyst): re-pin the current benchmark implementation digest
|
|
43
|
+
- docs: regenerate the public API census
|
|
44
|
+
- fix: drop unused isRecord imports; correct stale line references in two docs
|
|
45
|
+
- refactor: one owner per duplicated name; rename what differs
|
|
46
|
+
- adopt-stack merges after the required approval lands, instead of refusing in the first second (#1035)
|
|
47
|
+
- docs: one root AGENTS.md map; CLAUDE.md imports it
|
|
48
|
+
- docs: point stale doc references at current code; archive superseded records
|
|
49
|
+
- adopt-stack runs the caller's own sync step after pinning (#1033)
|
|
50
|
+
- One reusable workflow adopts each new Tangle stack release, peer ranges included (#1032)
|
|
51
|
+
|
|
7
52
|
## [0.216.1] — 2026-10-21
|
|
8
53
|
|
|
9
54
|
- fix(experiment): declare the diagnostic target outcome direction
|
|
@@ -261,7 +306,7 @@ Prepared but never published; 0.210.0 is the first release that carries these ch
|
|
|
261
306
|
- Search lenses (`/search`, search-tree-design §12): `tree`, `operatorYield`, `front` and `taskMatrix`, each a pure function of `SearchState` returning JSON for a view plus one named signal a `SearchPolicy` can read. `agent-eval search show` gains `--tree`/`--operator-yield`/`--front`/`--task-matrix` to print each lens's text form.
|
|
262
307
|
`operatorYield` groups edges by operator and reports outcome counts and improvement-per-known-dollar yield (via `searchPosterior`, excluding any node with an unknown-cost cell rather than flooring it). `front` is the Pareto frontier over score and known cost, reusing the existing `paretoFrontier` primitive, with room for caller-declared extra axes. `taskMatrix` clusters nodes and units by single-linkage clustering with a data-derived cutoff and reports each unit cluster's specialist gain (best node's mean minus the cluster mean).
|
|
263
308
|
`incumbentWithOperatorBandit({ seed, fixedWeights? })` (`/campaign`) is a hill-climb `SearchPolicy` that draws its expansion operator from `operatorYield`'s signal once an operator has 6 or more measured outcomes (falling back to a fixed prior per operator until then, and adding a measured yield to that same prior rather than replacing it, so the two stay on comparable scales); `incumbent` itself is unchanged.
|
|
264
|
-
`scripts/generate-synthetic-search-ledger.ts` and `scripts/import-vb-climb.ts` (the latter through the GEPA importer) produce real ledgers these lenses are proved against;
|
|
309
|
+
`scripts/generate-synthetic-search-ledger.ts` and `scripts/import-vb-climb.ts` (the latter through the GEPA importer) produce real ledgers these lenses are proved against; the operator-bandit proof script (since removed) is a real-run (no-mock) proof of the bandit's draw distribution against the lens's own signal.
|
|
265
310
|
- `metaSearch(searches)` (`/search`) treats each search as a node whose genome is its policy configuration and whose score is its claim's held-out lift per known dollar; an unscored search carries its reason, never 0. Searches form a forest by derivation and containment.
|
|
266
311
|
`runNestedSearch` runs an outer search whose cells are inner searches on the same kernel; `scripts/search-meta-sim.ts` runs one on the simulator, and `agent-eval search show --meta [--json]` prints the lens.
|
|
267
312
|
- `editCredit(state, { readArtifact })` (`/search`) cuts each lineage edge's parent-to-child diff into paragraph edits with stable ids and credits an edit on the clean lineage steps where it alone changed, pairing the carrying side against the lacking side on shared units through `pairedDeltaTest`. It reports linked edits, interacting pairs and inline skill candidates, with the signal `reusableHunks`; `agent-eval search show --edit-credit [--json]` prints it.
|
|
@@ -491,7 +536,7 @@ Prepared but never published; 0.210.0 is the first release that carries these ch
|
|
|
491
536
|
Each question is one `runTraceAnalyst` call, so a finding still needs its minimum distinct citations (default 2), each resolvable in the store.
|
|
492
537
|
The verifier reads the claim through `citedSpansOnly`, a store that exposes exactly the cited spans and refuses every wider read; a finding it does not support is returned with `verified: false`.
|
|
493
538
|
One failed question never stops the others; its failure is its outcome.
|
|
494
|
-
It is the question runner of the agent-runtime continuation panel (discovery
|
|
539
|
+
It is the question runner of the agent-runtime continuation panel (discovery discovery note 38, section 5), and `traces ask` can call the same code.
|
|
495
540
|
|
|
496
541
|
### Changed
|
|
497
542
|
|
|
@@ -552,7 +597,7 @@ Prepared but never published; 0.210.0 is the first release that carries these ch
|
|
|
552
597
|
- `/ledger-core` `FileLedgerJournal` can cache a projector snapshot at its trusted head, so opening a large journal only does its expensive work — the codec's own row parsing and the domain projector's `apply` — on the tail after the cache, not the whole file.
|
|
553
598
|
A codec opts in with `snapshotProjection: { serialize, restore }`; `projectorSnapshot: { everyEntries }` on `FileLedgerJournalOptions` refreshes the cache every that many entries past the last one, on a pinning append or an explicit `pinTrustedHead()`.
|
|
554
599
|
Opening still verifies the generic hash chain of every row up to the cached checkpoint — self-consistency and chain linkage, the two checks every row already gets — so tampering anywhere before the checkpoint is refused exactly as a full replay would refuse it; only the checkpoint row and the true tail go through the codec.
|
|
555
|
-
Measured with
|
|
600
|
+
Measured with a snapshot proof script (since removed) against real hash-chained journals: a snapshot-seeded open always matched a full replay's projection exactly, and used 6.00x less CPU at 300 entries (11.1ms vs 66.7ms, checkpoint at 250), 3.26x less at 1,200 (68.4ms vs 222.8ms, checkpoint at 1,000) and 3.49x less at 4,000 (143.3ms vs 499.5ms, checkpoint at 3,500); the ratio tracks how much of the file the checkpoint covers, since both opens still read every byte — the chain check does not skip any of them (`~/webb/_wt/_mq-notes/E12-proof/`).
|
|
556
601
|
Same-length tampering of a row before the checkpoint is refused, identically to a full replay of the same bytes; tampering after it, in the true tail, is still caught too.
|
|
557
602
|
No existing codec (`search-ledger`, `final-evidence`) opts in yet; this is the ledger-core mechanism only.
|
|
558
603
|
|
|
@@ -588,7 +633,7 @@ Prepared but never published; 0.210.0 is the first release that carries these ch
|
|
|
588
633
|
- `/ledger-core` `FileLedgerJournal` verifies incrementally.
|
|
589
634
|
An instance verifies its whole file on its first call, then each append or replay checks that the head row it verified is still in place and reads, verifies, and projects only the rows past it.
|
|
590
635
|
An append reads one entry's bytes instead of the whole file, so writing a ledger costs O(n) instead of O(n²); `openSearchLedger` and `openFinalEvidenceLedger` gain the same speed.
|
|
591
|
-
At 20,000 entries an append read 571 bytes instead of 10,852,776 (
|
|
636
|
+
At 20,000 entries an append read 571 bytes instead of 10,852,776 (a journal-load script (since removed)).
|
|
592
637
|
A replaced, shrunk, or rewritten head is verified whole again; a same-length rewrite of earlier rows is refused by the next new instance, not by one already open.
|
|
593
638
|
A new entry passes through the codec's own row parse before it is written, so a journal never writes a row it would refuse to read.
|
|
594
639
|
Entries are deep-frozen; code that mutated an entry or event from a replay now throws.
|
|
@@ -663,7 +708,7 @@ Prepared but never published; 0.210.0 is the first release that carries these ch
|
|
|
663
708
|
- **The rule-list redactor is replaced by the redaction core.** Removed `DEFAULT_REDACTION_RULES`, `RedactionRule`, `redactString` and `redactValue` (root and `/traces`), and `/diagnosis` `DIAGNOSIS_SECRET_RULES`, `holdsSecret`, `redactSecrets`, `redactSecretsDeep`, `SECRET_ASSIGNMENT_PATTERN` and their types. `RedactionReport` now has `byDetector` and `findings` instead of `byRule`, and `REDACTION_VERSION` is `2.0.0`. **Migration:** `redactValue(v)` becomes `redact(v).value`, `redactString(s).output` becomes `redactText(s)`, and a custom rule list becomes `knownSecrets` or a case in the corpus. `defaultProviderRedactor` now runs the core, so a credential header keeps its name with a `[REDACTED:credential-key]` value, and `redactedFields` holds JSON Pointers.
|
|
664
709
|
- **`ExperimentTracker` and its git-provenance/persistence machinery are gone.** Removed `ExperimentTracker`, `fileExperimentStore`, `inMemoryExperimentStore`, `Experiment`, and `ExperimentProvenance` (root and `/experiment`). The class had no in-repo, agent-runtime, blueprint-agent, or agent-dev-container caller — a manual experiment becomes a search with `proposer.kind: 'human'` in the upcoming search-tree system. **Migration:** if you called `new ExperimentTracker({ store, provenanceReader })`, replace it with your own store (`create`/`addRep`/`list` become plain reads and writes of whatever you persist) plus `computeExperimentStats` and `improvementVerdict` directly — those two pure functions, `ExperimentRep`, `ExperimentStats`, `ImprovementThresholds`, `ImprovementVerdictResult`, and `ExperimentVerdict` are unchanged and still exported from the package root (blueprint-agent's held-out gate uses them as-is). Provenance capture (`git rev-parse HEAD`, etc.) is no longer built in; shell out yourself if you need it.
|
|
665
710
|
- `contractJudge`, `matchSpan`, and `assertTraceContract`, which the package root never exported.
|
|
666
|
-
- The unit tests in
|
|
711
|
+
- The unit tests in the trace-contracts test; `traces check` over recorded sessions is the proof.
|
|
667
712
|
|
|
668
713
|
## [0.187.2] — 2026-09-24
|
|
669
714
|
|
|
@@ -1219,7 +1264,7 @@ Prepared but never published; 0.210.0 is the first release that carries these ch
|
|
|
1219
1264
|
|
|
1220
1265
|
- Every byte-identical private copy of a shared primitive now routes through one owner. No public export changes and no behavior changes.
|
|
1221
1266
|
- `mulberry32` had five implementations: the exported owner in `src/statistics/random.ts`, named private copies in `src/trajectory-replay/batch.ts` and `src/judge-calibration.ts`, and unnamed inline copies in `src/rl/contamination.ts` (`shuffleOrder`) and `src/fuzz/explorer.ts` (`FuzzExplorer.rng`). The copies used `seed >>> 0` where the owner uses `seed | 0` — the same 32 bits, so the streams are identical (verified over 150,000 draws across 15 seeds). All four are deleted. The replay fix-case sample, the judge-calibration bootstrap, the contamination probe's order-shuffle perturbation, and the fuzz explorer now inherit the owner's refusal of a non-finite seed instead of silently seeding from `NaN >>> 0 === 0`. `src/statistics/random.test.ts` pins the stream with known-answer vectors, so a future edit to the PRNG cannot silently move an already-published statistic.
|
|
1222
|
-
- `toAttributes` and `msToNs` were byte-identical in `src/trace/otel.ts` and
|
|
1267
|
+
- `toAttributes` and `msToNs` were byte-identical in `src/trace/otel.ts` and the removed OTel exporter module. Both exporters emit the same `OtlpSpan`, so two copies of the value-tagging rule meant the same attribute could be typed differently depending on which exporter ran. They are now `toOtlpAttributes` and `msToUnixNano` in the private `src/trace/otlp-encoding.ts`.
|
|
1223
1268
|
- `cryptoRandomId` / `cryptoEventId` / `cryptoId` were three byte-identical id mints in `src/trace/emitter.ts`, `src/llm-client.ts`, and `src/builder-eval/builder-session.ts`. Each carried a `Date.now()`-plus-`Math.random()` fallback that is unreachable on every runtime this package supports (`engines.node >= 20`, where `globalThis.crypto.randomUUID` is always present). One private `newRecordId` in `src/record-id.ts` replaces all three, and the weaker id path is gone.
|
|
1224
1269
|
|
|
1225
1270
|
---
|
|
@@ -1281,7 +1326,7 @@ Prepared but never published; 0.210.0 is the first release that carries these ch
|
|
|
1281
1326
|
- `createChatClient` loses its `router`, `direct-provider`, and `cli-bridge` variants. `custom`, `sandbox-sdk`, and `mock` remain, and `ChatTransport` narrows to those three.
|
|
1282
1327
|
- The root barrel no longer exports `callLlm`, `callLlmJson`, `LlmClient`, `LlmClientOptions`, `assertLlmRoute`, `LlmRouteRequirements`, or `probeLlm`. `assertLlmRoute` and `probeLlm` are deleted outright: the caller holds the endpoint, so the caller owns both the route check and the reachability probe. The canonical contract stays public — `LlmCallRequest`, `LlmCallResult` (including `logprobs`, `toolCalls`, `servedModel`), `LlmMessage`, `LlmUsage`, `costReceiptFromLlm`, `costReceiptFromLlmError`, `maximumChargeForLlmRequest`, `isTransientLlmError`, `stripFencedJson`.
|
|
1283
1328
|
- `createOpenAiCompatibleExecutionOwner` is gone from `/campaign`. Agent Runtime already owns that role with `profileOptimizerModelCall`, which executes one exact `AgentProfile` and reports profile-digest evidence; two owners for one role was the defect. `examples/_shared/openai-compatible-owner.ts` is the caller-side reference implementation, and it is example code, not a published export.
|
|
1284
|
-
-
|
|
1329
|
+
- the multishot router module is deleted with `routerCompletion`, `requireRouterApiKey`, and `defaultRouterBaseUrl`. `runMultishot`, `runMultishotMatrix`, and `runJudge` now require a caller-supplied `MultishotTransport`; `JudgeConfig.transport` is required and `JUDGE_MODEL` is no longer read from the environment. `MultishotToolExecutor` receives `{ transport, signal }` instead of `{ apiKey, baseUrl, signal }`, and the optional `toolTransport` names the leg the built-in delegate tools run on. `estimateRouterCost` is now `estimateMultishotCost` in `src/multishot/cost.ts`.
|
|
1285
1330
|
- `preflightModels` and `assertModelsServed` take `request: ModelEndpointRequest` instead of `baseUrl` and `apiKey`. Agent Eval asks for a `list-models` or a `probe` check and reads the `Response`, so status, the provider's own `error.message`, `budgetExhausted`, and served-model substitution stay exactly as detectable as before.
|
|
1286
1331
|
- `runIntentMatchJudge`, `runSemanticConceptJudge`, `handleJudge`, `dispatchRpc`, and `createApp` take `chat: ChatClient` (plus optional `pricing`) instead of `llm: LlmClientOptions`. `/v1/judge` refuses with `llm_not_configured` (503) when no transport is configured, which replaces the old route assertion.
|
|
1287
1332
|
- The internal OpenAI-compatible client has no default endpoint. `DEFAULT_BASE_URL = 'https://router.tangle.tools/v1'` is deleted and `baseUrl` is required, so a misconfigured caller fails loudly instead of silently billing the public router — the failure `assertLlmRoute` existed to catch, now unrepresentable.
|
|
@@ -1323,7 +1368,7 @@ Prepared but never published; 0.210.0 is the first release that carries these ch
|
|
|
1323
1368
|
|
|
1324
1369
|
### Fixed
|
|
1325
1370
|
|
|
1326
|
-
- Every result-bearing random draw is seeded (#411, criterion 2). Five byte-identical private copies of `mulberry32` lived beside the canonical one, in `meta-eval/rubric-predictive-validity.ts`, `rl/active-curriculum.ts`, `rl/adaptation-eval.ts`, `summary-report.ts`, and `promotion-gate.ts`; four fell back to `Math.random` when the caller passed no seed, so the rubric verdict, the Thompson allocation, the adaptation-curve intervals, and the reported posterior were silently non-reproducible. `meta-eval/correlation-study.ts` called `Math.random` directly inside its bootstrap and offered no seed at all. All six now route through `makeRng`, which derives the seed from the observations when the caller supplies none, so the same input reproduces the same interval; `correlationStudy` gains the `seed` option its sibling already had. `Math.random` references in `src/` fall from 19 to 10, and every remaining one produces an identifier or a retry delay that no reported number reads. The seat-by-seat audit table is in `docs/design/statistics-decisions.md`.
|
|
1371
|
+
- Every result-bearing random draw is seeded (#411, criterion 2). Five byte-identical private copies of `mulberry32` lived beside the canonical one, in `src/meta-eval/rubric-predictive-validity.ts`, `src/rl/active-curriculum.ts`, `src/rl/adaptation-eval.ts`, `summary-report.ts`, and `promotion-gate.ts`; four fell back to `Math.random` when the caller passed no seed, so the rubric verdict, the Thompson allocation, the adaptation-curve intervals, and the reported posterior were silently non-reproducible. `src/meta-eval/correlation-study.ts` called `Math.random` directly inside its bootstrap and offered no seed at all. All six now route through `makeRng`, which derives the seed from the observations when the caller supplies none, so the same input reproduces the same interval; `correlationStudy` gains the `seed` option its sibling already had. `Math.random` references in `src/` fall from 19 to 10, and every remaining one produces an identifier or a retry delay that no reported number reads. The seat-by-seat audit table is in `docs/design/statistics-decisions.md`.
|
|
1327
1372
|
- Two documentation fences imported `@tangle-network/agent-eval/../src/trace-repair`, which cannot resolve; they now import `@tangle-network/agent-eval/trace-repair`. Three fences that are not TypeScript — an object fragment, a call elided as `{ ... }`, and a method sketch — are marked `text`.
|
|
1328
1373
|
|
|
1329
1374
|
### Changed
|
|
@@ -1357,7 +1402,7 @@ Prepared but never published; 0.210.0 is the first release that carries these ch
|
|
|
1357
1402
|
|
|
1358
1403
|
- `runOptimization({ searchLedger })` and `selfImprove({ searchLedger })` record the candidate search into the canonical `SearchLedger` and return a bounded `searchHistory` receipt (#633). The loop emits the plan (slots = generations x populationSize, one candidate-generation operation per generation, one selection operation, one task per designed scenario-replicate cell), one registration per candidate carrying the exact parent surface it mutated, one attempt per scored cell with the cell's own outcome and accounting, one decision per candidate, and the terminal event. `FileSearchLedger` now has a first-party caller in its own package. The terminal event is appended only when canonical replay accounts for the whole planned denominator, so an interrupted or partly unscored search reports the gap instead of claiming a closed search.
|
|
1359
1404
|
- `search-plan-extended`: a rolling search appends candidate slots and operations to an existing plan instead of opening a second ledger. Replay merges the first plan with every extension, the generation invariant continues across rounds, and the planless refusal is unchanged. The planned task denominator stays frozen.
|
|
1360
|
-
- `gepaOptimizationMethod({ searchLedger: { identity } })` records GEPA's own candidate population — its parent graph and per-scenario selection scores — into the same ledger through `recordCandidatePopulationSearch()`. `compareOptimizationMethods({ searchHistoryPolicy: 'require-complete' })` now accepts a first-party method, which is what
|
|
1405
|
+
- `gepaOptimizationMethod({ searchLedger: { identity } })` records GEPA's own candidate population — its parent graph and per-scenario selection scores — into the same ledger through `recordCandidatePopulationSearch()`. `compareOptimizationMethods({ searchHistoryPolicy: 'require-complete' })` now accepts a first-party method, which is what the search-history receipts note promised.
|
|
1361
1406
|
|
|
1362
1407
|
---
|
|
1363
1408
|
|
|
@@ -1421,7 +1466,7 @@ Prepared but never published; 0.210.0 is the first release that carries these ch
|
|
|
1421
1466
|
- `foreign-agent-quickstart` moved to the standard `LLM_BASE_URL`/`LLM_API_KEY`/`LLM_MODEL` variables and declares `expectUsage: 'off'` for its unmetered demo transport.
|
|
1422
1467
|
- Optimizer examples (`self-improve-optimizer`, `compare-optimization-methods`, GSM8K, AppWorld) encode the measured GEPA lessons: `LLM_BASE_URL` is required (no silent `api.openai.com` default), every GEPA engine run pins `reflection_lm_kwargs: { num_retries: 0 }` via the shared `examples/_shared/gepa-reflection.ts`, worker `maxTokens` is env-tunable with a reasoning-model headroom note, and `GEPA_MAX_EVALUATIONS` documents the >= train-partition-size floor.
|
|
1423
1468
|
- Model ids refreshed across examples: retired `gpt-4o`/`gpt-4.1-mini`/`claude-sonnet-4-6` literals replaced with served ids (`deepseek-v4-flash`, `glm-5.3`; RunRecord literals carry the required snapshot suffix).
|
|
1424
|
-
- Removed
|
|
1469
|
+
- Removed the AppWorld halo chat launcher and `halo-chat.sh`: launchers for an external analysis engine that no documented example references. `run-bench.ts` requires `APPWORLD_DIR` explicitly instead of defaulting to a machine-local path.
|
|
1425
1470
|
|
|
1426
1471
|
---
|
|
1427
1472
|
|
|
@@ -1515,14 +1560,14 @@ One repository still names three of them. `starter-foundry` imports `runAssertio
|
|
|
1515
1560
|
|
|
1516
1561
|
### Removed
|
|
1517
1562
|
|
|
1518
|
-
- `decideNextUserTurn`, `DecideNextUserTurnOpts` and `buildDriverSystemPrompt`, with
|
|
1519
|
-
- `ConvergenceTracker` and
|
|
1563
|
+
- `decideNextUserTurn`, `DecideNextUserTurnOpts` and `buildDriverSystemPrompt`, with the driver module and the `user-simulation-driver` example. A role written as a code function cannot be optimized, and a persona driver is an `AgentProfile` on a graph edge, not a packaged function. All five first-party callers now own the role in their own repositories, where the prompt is product data they can change and measure: gtm-agent, legal-agent, insurance-agent, workcomp-agent and creative-agent.
|
|
1564
|
+
- `ConvergenceTracker` and the convergence tracker module. Its only caller was `AgentDriver`, which 0.145.22 deleted, and it never reached the barrel. `analyzeSeries` in `src/series-convergence.ts` is unaffected — it reads drift across runs, not progress within one.
|
|
1520
1565
|
- `PersonaConfig.feedbackPatterns`, the `FeedbackPattern` type, and `PersonaConfig.driverModel`. `feedbackPatterns` told `AgentDriver` which product approvals to reject, and `driverModel` picked its driver model; with the class gone nothing reads either, in this package or in any repository that depends on it. A field that advertises behaviour the package no longer has is worse than no field.
|
|
1521
1566
|
- Nothing else on the public surface changes. `PersonaConfig` and `DriverState` stay: a harness that writes its own driver still describes a persona and a produced state with them.
|
|
1522
1567
|
|
|
1523
1568
|
### Migration
|
|
1524
1569
|
|
|
1525
|
-
A caller that drove a conversation with `decideNextUserTurn` owns two things now: the system prompt that states what its simulated professional demands, and one priced call that turns the transcript into the next message. Both stay reachable from this package — `CostLedger.runPaidCall` prices and attributes the call, `maximumChargeForLlmRequest` caps it, `costReceiptFromLlm` records it, and `assertServedModel` (published in 0.145.22) holds the transport to the model id it was asked for. gtm-agent's
|
|
1570
|
+
A caller that drove a conversation with `decideNextUserTurn` owns two things now: the system prompt that states what its simulated professional demands, and one priced call that turns the transcript into the next message. Both stay reachable from this package — `CostLedger.runPaidCall` prices and attributes the call, `maximumChargeForLlmRequest` caps it, `costReceiptFromLlm` records it, and `assertServedModel` (published in 0.145.22) holds the transport to the model id it was asked for. gtm-agent's the eval persona driver is the worked example.
|
|
1526
1571
|
|
|
1527
1572
|
A caller pinned below 0.145.22 cannot import `assertServedModel`. Check what the pinned version did before porting: `decideNextUserTurn` did not call the guard before the 0.145.x line, so a port that omits it there matches the pinned behaviour exactly.
|
|
1528
1573
|
## [0.146.0] — 2026-08-16
|
|
@@ -1798,7 +1843,7 @@ Breaking release: the deep clean. The measured consumer-import census is the com
|
|
|
1798
1843
|
|
|
1799
1844
|
### Changed
|
|
1800
1845
|
|
|
1801
|
-
-
|
|
1846
|
+
- the statistics module is now the `src/statistics/` directory (12 focused modules); every existing import specifier still compiles.
|
|
1802
1847
|
- `runCampaign` internals split into phase modules; its import path and options are unchanged.
|
|
1803
1848
|
- Every verifier lands in `DefaultVerdict`, with `certification` naming what certified the verdict (checker identity, strategy, assumptions, evidence digest).
|
|
1804
1849
|
|
|
@@ -2656,10 +2701,10 @@ threshold and should be re-run on this release.
|
|
|
2656
2701
|
(`outcome.realness.gated`) existed in exactly one function, `rolloutReward`, called from exactly one
|
|
2657
2702
|
place — `mintRolloutRows`. The same derivation, `outcome.holdoutScore ?? outcome.searchScore`, was
|
|
2658
2703
|
hand-rolled at 20 other sites with no gate. Six of those sites feed exported training data: the GRPO
|
|
2659
|
-
default reward and the SFT row metadata (`rl/exporters.ts`), the DPO preference ordering
|
|
2660
|
-
(`rl/preferences.ts`), the probabilistic verifiable-reward fallback (`rl/verifiable-reward.ts`), the
|
|
2661
|
-
corpus `minScore` filter (`rl/corpus.ts`), and the published datasheet's reward statistics
|
|
2662
|
-
(`rl/dataset.ts`). A gamed run therefore trained at its claimed score, and in DPO it became the
|
|
2704
|
+
default reward and the SFT row metadata (`src/rl/exporters.ts`), the DPO preference ordering
|
|
2705
|
+
(`src/rl/preferences.ts`), the probabilistic verifiable-reward fallback (`src/rl/verifiable-reward.ts`), the
|
|
2706
|
+
corpus `minScore` filter (`src/rl/corpus.ts`), and the published datasheet's reward statistics
|
|
2707
|
+
(`src/rl/dataset.ts`). A gamed run therefore trained at its claimed score, and in DPO it became the
|
|
2663
2708
|
*chosen* side of a pair against its honest sibling. Every one of those six now derives its reward
|
|
2664
2709
|
through the gate.
|
|
2665
2710
|
|
|
@@ -2673,7 +2718,7 @@ threshold and should be re-run on this release.
|
|
|
2673
2718
|
allocation need the ungated number, and gating a detector's proxy would make it report "clean" on
|
|
2674
2719
|
precisely the population being gamed — so the fix names the choice rather than removing it.
|
|
2675
2720
|
- A regression test (`src/rollout/reward-invariant.test.ts`) with two halves: a source-level check that
|
|
2676
|
-
the bare derivation appears nowhere outside `rollout/reward.ts`, and a behavioural check that pushes
|
|
2721
|
+
the bare derivation appears nowhere outside `src/rollout/reward.ts`, and a behavioural check that pushes
|
|
2677
2722
|
one gated record with a 0.95 score through mint, GRPO, SFT, DPO, verifiable reward, the dataset
|
|
2678
2723
|
bundle, and the corpus filter, asserting 0 in each. Against the pre-fix tree the source check reports
|
|
2679
2724
|
21 offending lines and 7 of the 10 tests fail.
|
|
@@ -2684,7 +2729,7 @@ threshold and should be re-run on this release.
|
|
|
2684
2729
|
Use `trainingReward` for score derivation or `rolloutRewardFields` when producing a rollout outcome.
|
|
2685
2730
|
- The 14 analysis, reporting, and detection sites that legitimately want the raw number now call
|
|
2686
2731
|
`observedScore` explicitly. Behaviour is unchanged at all 14, including the two sites that
|
|
2687
|
-
deliberately prefer the search split (`rl/active-curriculum.ts`, via the new `ScorePreference`
|
|
2732
|
+
deliberately prefer the search split (`src/rl/active-curriculum.ts`, via the new `ScorePreference`
|
|
2688
2733
|
argument). (`description-length-gate.ts` reads through `runTaskScore`, which 0.127.0 stripped of
|
|
2689
2734
|
its obsolete `raw.score` fallback.)
|
|
2690
2735
|
|
|
@@ -2731,10 +2776,10 @@ unchanged.
|
|
|
2731
2776
|
`HeldOutGate` gets the same treatment: gated runs are dropped from both sides before pairing, with
|
|
2732
2777
|
`evidence.realnessGatedRuns` surfacing how many. Never a silent 0 — a shrunken denominator has to say
|
|
2733
2778
|
by how much.
|
|
2734
|
-
- **Ten remaining hand-rolled derivations routed by classification**: `rl/sim-fidelity.ts` (×2, RAW —
|
|
2779
|
+
- **Ten remaining hand-rolled derivations routed by classification**: `src/rl/sim-fidelity.ts` (×2, RAW —
|
|
2735
2780
|
gating a sim-vs-production divergence measure would report the simulator as more faithful precisely
|
|
2736
|
-
where it is gamed),
|
|
2737
|
-
`eval-trace-store.ts` (GATED, above), `contract/analyze-runs.ts` (×3, RAW), `summary-report.ts` (×4,
|
|
2781
|
+
where it is gamed), the belief-state code-agent corpus module (GATED — its output becomes corpus labels),
|
|
2782
|
+
`eval-trace-store.ts` (GATED, above), `src/contract/analyze-runs.ts` (×3, RAW), `summary-report.ts` (×4,
|
|
2738
2783
|
RAW), `release-confidence.ts` (RAW), `held-out-gate.ts` (RAW, over an already-degated set).
|
|
2739
2784
|
- **`trainingReward` no longer collapses an unscored record to 0.** It returns `reward: null`, matching
|
|
2740
2785
|
the schema's own "a labeled gap, never 0" rule; a gated run still returns 0, because that IS a
|
|
@@ -2814,7 +2859,7 @@ unchanged.
|
|
|
2814
2859
|
|
|
2815
2860
|
### Fixed — unrelated flake encountered on the way
|
|
2816
2861
|
|
|
2817
|
-
- `node:sqlite` is loaded through `createRequire` in `rollout/readers/opencode-sqlite.ts` and its test.
|
|
2862
|
+
- `node:sqlite` is loaded through `createRequire` in `src/rollout/readers/opencode-sqlite.ts` and its test.
|
|
2818
2863
|
esbuild and Vite both rewrite an `import()` of a builtin and strip the `node:` prefix, producing a bogus
|
|
2819
2864
|
`sqlite` package lookup; composing the specifier at runtime did not reliably defeat it, so the failure
|
|
2820
2865
|
moved between workers whenever a test file was added. A require obtained from `createRequire` is not an
|
|
@@ -2872,7 +2917,7 @@ Artifacts that carry run ids without embedded reward state still require explici
|
|
|
2872
2917
|
|
|
2873
2918
|
#### Deliberately NOT gated
|
|
2874
2919
|
|
|
2875
|
-
`rl/reward-hacking.ts` reads the deterministic reward through the new
|
|
2920
|
+
`src/rl/reward-hacking.ts` reads the deterministic reward through the new
|
|
2876
2921
|
`VerifiableRewardExtractionOptions.applyRealnessGate: false`, which preserves its previous behaviour
|
|
2877
2922
|
exactly. Its `judge_drift` and `reward_disagreement` signals measure the GAP between the judge reward
|
|
2878
2923
|
and the deterministic one; a deterministic reward another gate already forced to 0 opens that gap by
|
|
@@ -2883,7 +2928,7 @@ empty options object gates — the opt-out is explicit and greppable.
|
|
|
2883
2928
|
### Known, not fixed here
|
|
2884
2929
|
|
|
2885
2930
|
- `description-length-gate.ts` gives a gated run claiming `score: 1.0` the largest possible improvement
|
|
2886
|
-
to its objective, and `product-benchmark/export.ts` publishes a gated run with `pass: true`. Each
|
|
2931
|
+
to its objective, and `src/product-benchmark/export.ts` publishes a gated run with `pass: true`. Each
|
|
2887
2932
|
site carries a comment naming the hole.
|
|
2888
2933
|
- `extractVerifiableReward(report)` — the `VerificationReport` signature — cannot gate and does not
|
|
2889
2934
|
claim to: `realness` lives on the `RunRecord`, not on the report. Documented on the function.
|
|
@@ -3473,7 +3518,7 @@ Additive (new exports + optional fields only); no consumer bump required.
|
|
|
3473
3518
|
|
|
3474
3519
|
### Added
|
|
3475
3520
|
|
|
3476
|
-
- **FAPO proposer policy** in the campaign proposer family — feeds `runImprovementLoop` / `runCampaign` proposer-driven self-improvement (`gepaProposer` at
|
|
3521
|
+
- **FAPO proposer policy** in the campaign proposer family — feeds `runImprovementLoop` / `runCampaign` proposer-driven self-improvement (`gepaProposer` at the GEPA proposer module).
|
|
3477
3522
|
|
|
3478
3523
|
### Changed
|
|
3479
3524
|
|
|
@@ -3515,8 +3560,8 @@ One clean, canonical version of five generic patterns the fleet kept hand-rollin
|
|
|
3515
3560
|
|
|
3516
3561
|
### Added
|
|
3517
3562
|
|
|
3518
|
-
- **`ExperimentTracker` + `improvementVerdict` + `computeExperimentStats` (root).** Git-provenanced experiment log with N-rep stats (`median` / `mean` / `min` / `max` / `iqr` / `stddev` / `passRate` / `n` + a `stable` flag) and a `KEEP` / `REGRESSION` / `NOISE` / `ITERATE` verdict against a parent. Provenance (`ProvenanceReader`, default `gitProvenanceReader`) and persistence (`ExperimentStore`, with `inMemoryExperimentStore` + `fileExperimentStore`) are injected, so the stats + verdict are pure and unit-testable without a repo or disk. Thresholds (`keepThreshold` / `regressionThreshold` / `iqrUnstableAbove` / `stddevUnstableAbove` / `minRepsForVerdict`) are configurable. Replaces the per-agent
|
|
3519
|
-
- **`EvalTraceStore` + `runScore` (root).** JSONL save / query / compare over the analysis-time `RunRecord` row: `query(filter)`, `getBest(scenarioId)`, and `compareRuns(a, b)` (paired on matched scenarios, best-score-per-scenario). Persistence is injected via `RunRecordBackend` (`inMemoryRunRecordBackend` / `jsonlRunRecordBackend`, which fail loud on a malformed line). Does NOT fork `FileSystemTraceStore` (the rich TraceSchema-v1 span store) — it is the analysis projection beside it. Replaces the hand-rolled
|
|
3563
|
+
- **`ExperimentTracker` + `improvementVerdict` + `computeExperimentStats` (root).** Git-provenanced experiment log with N-rep stats (`median` / `mean` / `min` / `max` / `iqr` / `stddev` / `passRate` / `n` + a `stable` flag) and a `KEEP` / `REGRESSION` / `NOISE` / `ITERATE` verdict against a parent. Provenance (`ProvenanceReader`, default `gitProvenanceReader`) and persistence (`ExperimentStore`, with `inMemoryExperimentStore` + `fileExperimentStore`) are injected, so the stats + verdict are pure and unit-testable without a repo or disk. Thresholds (`keepThreshold` / `regressionThreshold` / `iqrUnstableAbove` / `stddevUnstableAbove` / `minRepsForVerdict`) are configurable. Replaces the per-agent the eval experiment tracker copies (tax / insurance / legal).
|
|
3564
|
+
- **`EvalTraceStore` + `runScore` (root).** JSONL save / query / compare over the analysis-time `RunRecord` row: `query(filter)`, `getBest(scenarioId)`, and `compareRuns(a, b)` (paired on matched scenarios, best-score-per-scenario). Persistence is injected via `RunRecordBackend` (`inMemoryRunRecordBackend` / `jsonlRunRecordBackend`, which fail loud on a malformed line). Does NOT fork `FileSystemTraceStore` (the rich TraceSchema-v1 span store) — it is the analysis projection beside it. Replaces the hand-rolled the eval trace store copies.
|
|
3520
3565
|
- **`CostLedger` + `costForUsage` + `modelPriceKey` (root).** Per-run token + USD accounting folded over the substrate's `resolveModelPricing` / `isModelPriced`, with an explicit `costUnknown` axis so a $0 from an unpriced model is never mistaken for a measured free run. Classifies spend by channel (`agent` / `judge` / `verifier` / …), surfaces `unpricedModels` + `fullyPriced`, and computes `costPerCompletedTask`. Generalizes physim's `costForUsage` / `modelPriceKey` and the tax / gtm / agent-builder copies.
|
|
3521
3566
|
- **`extractUsage` + `extractUsageFromSse` + `extractUsageFromResponse` (`/traces` + root).** Token-usage extraction from a chat-completions response or an SSE stream — OpenAI / Anthropic / camelCase shapes, cache-read tokens, and per-chunk SSE accumulation — returning `null` (not a silent zero) when no usage is present. `captureFetchToRawSink` gains an optional `onUsage` callback that emits the parsed usage off each response (reusing the body it already reads — no extra clone), so a caller folds usage → cost without re-cloning. Replaces the insurance / legal / gtm `raw-capture.ts` copies.
|
|
3522
3567
|
- **`partitionHeldOut` + `assignHeldOutTag` + `hashToUnit` + `fnv1a32` (root).** Deterministic FNV-1a id+seed held-out splitter. `assignHeldOutTag` stamps a single id; `partitionHeldOut` splits a whole id list into disjoint search / holdout sets and fails loud on duplicate ids, empty input, an under-floor holdout (`minHoldout` / `minSearch` significance floor), or an out-of-range `holdoutFraction`. Generalizes agent-builder's `deterministicSplit` and the frontier persona-splitter; complements the existing 3-way benchmark `deterministicSplit` in `/benchmarks`.
|
|
@@ -3667,7 +3712,7 @@ An adversarial review of a real "ship +4.0 lift" decision found it was a **tripl
|
|
|
3667
3712
|
### Fixed / Added
|
|
3668
3713
|
|
|
3669
3714
|
- **No-op guard** (`runImprovementLoop`) — when the winner is byte-identical to the baseline (no candidate beat the training baseline, empty diff), the loop now forces `hold` and skips the meaningless baseline-vs-itself holdout pass, instead of shipping the noise delta.
|
|
3670
|
-
- **Statistical held-out gate** — `defaultProductionGate`'s held-out check is now a **paired bootstrap CI**, not a point estimate. It pairs candidate vs baseline holdout cells by **full `cellId` (`scenario:rep`)** — never averaging reps away — and ships only when the CI lower bound clears `deltaThreshold` (default 0 ⇒ confidently positive). Below `minProductiveRuns` (default 3) paired observations it HOLDS with `few_runs` rather than reading a degenerate interval. (New module `src/campaign/gates/statistical-heldout.ts`; reuses `pairedBootstrap` from `src/statistics
|
|
3715
|
+
- **Statistical held-out gate** — `defaultProductionGate`'s held-out check is now a **paired bootstrap CI**, not a point estimate. It pairs candidate vs baseline holdout cells by **full `cellId` (`scenario:rep`)** — never averaging reps away — and ships only when the CI lower bound clears `deltaThreshold` (default 0 ⇒ confidently positive). Below `minProductiveRuns` (default 3) paired observations it HOLDS with `few_runs` rather than reading a degenerate interval. (New module `src/campaign/gates/statistical-heldout.ts`; reuses `pairedBootstrap` from `src/statistics/`.)
|
|
3671
3716
|
- **Per-dimension regression guard (anti-Goodhart)** — `criticalDimensions` + `regressionTolerance` on `DefaultProductionGateOptions`. The gate HOLDS if any guarded dimension's paired-delta CI lower bound falls below −tolerance, even when the net composite rose. Tolerance auto-scales (0.05 on [0,1], 5 on 0-100) so a default expressed for one scale isn't a silent no-op on the other.
|
|
3672
3717
|
- **Exports** `pairHoldout`, `heldoutSignificance`, `dimensionRegressions`, `detectScale` from `/campaign`.
|
|
3673
3718
|
|
|
@@ -3705,7 +3750,7 @@ Closes the optimizer-completeness gap (#101/#100). `gepaDriver` was reflection-o
|
|
|
3705
3750
|
|
|
3706
3751
|
### Added
|
|
3707
3752
|
|
|
3708
|
-
- **GEPA Pareto frontier + combine-complementary-lessons (#101).** `runOptimization` now accumulates every scored surface as a per-scenario objective vector and recomputes the non-dominated set before each generation, handing it to the driver as `ctx.paretoParents` (new `ParetoParent` type). A surface uniquely best on one hard scenario survives even when its mean composite is lower. `gepaDriver` spends one population slot merging the frontier parents' complementary strengths (toggle via `combineParents`, default on; fires only when the frontier has >1 member). `RunOptimizationResult.paretoFrontier` exposes the final frontier. Dominance is computed by the package-canonical `paretoFrontier` (`src/pareto.ts`) — the parallel
|
|
3753
|
+
- **GEPA Pareto frontier + combine-complementary-lessons (#101).** `runOptimization` now accumulates every scored surface as a per-scenario objective vector and recomputes the non-dominated set before each generation, handing it to the driver as `ctx.paretoParents` (new `ParetoParent` type). A surface uniquely best on one hard scenario survives even when its mean composite is lower. `gepaDriver` spends one population slot merging the frontier parents' complementary strengths (toggle via `combineParents`, default on; fires only when the frontier has >1 member). `RunOptimizationResult.paretoFrontier` exposes the final frontier. Dominance is computed by the package-canonical `paretoFrontier` (`src/pareto.ts`) — the parallel the campaign Pareto module fork has been deleted (one dominance implementation).
|
|
3709
3754
|
- **SkillOpt patch-mode driver + `runSkillOpt` preset (#100)** (Microsoft, arXiv:2605.23904). `skillOptDriver` proposes BOUNDED add/delete/replace patches to one skill document (`applySkillPatch`, `SkillPatch`); `runSkillOpt` is the held-out-gated epoch hill-climb: reflect on TRAIN weaknesses → propose ≤ `editBudget` ops → score on the held-out split → ACCEPT only on STRICT held-out improvement, else buffer the rejected edit; with edit-budget annealing (the "textual learning rate") and a slow-update meta note. The held-out composite is monotonically non-decreasing by construction — a regression can never ship. Proposals reflect on train evidence only (no held-out leakage).
|
|
3710
3755
|
- **`compareDrivers` head-to-head lift benchmark (the forcing function).** Runs N optimizer entries on ONE corpus, scores the baseline + every promoted surface UNIFORMLY on the same held-out scenarios, and reports per-driver lift + paired-bootstrap CI + pairwise "which driver wins" CIs, ranked (cost breaks a lift tie). Ships `gepaReflectionEntry` / `gepaParetoEntry` / `skillOptEntry` to wire the real optimizers. Optimizer quality is now a number with a confidence interval — a driver regression turns a build red instead of going invisible.
|
|
3711
3756
|
- **`campaignMeanComposite` / `campaignBreakdown`** (`score-utils`) — the one definition of "composite of a campaign" + per-scenario/dimension breakdown, now shared by `runOptimization`, `runSkillOpt`, and `compareDrivers` (extracted from `runOptimization`'s private copies).
|
|
@@ -3775,7 +3820,7 @@ Pure additive/doc surface (`expectUsage` defaults to non-breaking `'warn'`). Ful
|
|
|
3775
3820
|
|
|
3776
3821
|
### Architectural context
|
|
3777
3822
|
|
|
3778
|
-
Part of the self-improvement-protocol design (
|
|
3823
|
+
Part of the self-improvement-protocol design (the self-improvement protocol design). This is 0.53.0 of the roadmap that ends at 1.0.0 (profile-versioning + composite driver) and 1.1.0 (empirical-proof publication).
|
|
3779
3824
|
|
|
3780
3825
|
### Notes
|
|
3781
3826
|
|
|
@@ -3797,9 +3842,9 @@ After cloning and reading the actual SkillOpt source (microsoft/SkillOpt) and th
|
|
|
3797
3842
|
|
|
3798
3843
|
### Added
|
|
3799
3844
|
|
|
3800
|
-
-
|
|
3801
|
-
-
|
|
3802
|
-
-
|
|
3845
|
+
- **the driver honesty spec** — primary-source comparison vs GEPA and SkillOpt. Quotes the actual source. Names 13 deviations between 0.51.0's `skillOptDriver` and the real SkillOpt pipeline.
|
|
3846
|
+
- **the Hermes self-improvement audit** — corrected audit after cloning NousResearch/hermes-agent. Hermes has two loops, not one: the 7-day curator (housekeeping) AND a per-turn `background_review` fork that uses **user corrective feedback as a first-class skill-update signal** ("stop doing X", "you always do Y"). Signal source we don't capture today.
|
|
3847
|
+
- **the profile-versioning spec** — architecture for the offline/online drift problem. Symmetric-fork framing (both writers are peers, neither is the authority). `AgentProfileVersion` content-hashing, `ProfileDiff` patch/replace types, 4-way `DriftGateDecision` (ship-substrate / ship-harness / merge / inconclusive), opt-in `driftPolicy` (ignore / reject-on-drift / benchmark-branches), four conflict-resolution cases including semantic-duplication detection. Phase 0 forcing-function experiment specified.
|
|
3803
3848
|
|
|
3804
3849
|
### Where we beat the prior art (now named explicitly)
|
|
3805
3850
|
|
|
@@ -3807,7 +3852,7 @@ Our `defaultProductionGate` uses paired bootstrap CI + Cohen's d + MDE + p-value
|
|
|
3807
3852
|
|
|
3808
3853
|
### Notes
|
|
3809
3854
|
|
|
3810
|
-
`gepaDriver({ constraints })` covers every use case the deleted `skillOptDriver` covered. The single `skillOptDriver` test file was removed; 13 new tests under
|
|
3855
|
+
`gepaDriver({ constraints })` covers every use case the deleted `skillOptDriver` covered. The single `skillOptDriver` test file was removed; 13 new tests under the GEPA driver constraints test cover the absorbed behavior + the unconstrained baseline behavior. Full suite 1444 / 1444 green.
|
|
3811
3856
|
|
|
3812
3857
|
---
|
|
3813
3858
|
|
|
@@ -3898,8 +3943,8 @@ Docs-only patch. No code changes, no behavior changes, no API surface changes vs
|
|
|
3898
3943
|
|
|
3899
3944
|
### Added
|
|
3900
3945
|
|
|
3901
|
-
-
|
|
3902
|
-
-
|
|
3946
|
+
- the OTel-to-hosted bridge module — generic OTel→hosted bridge (`createOtelBridge` / `OtelBridge` / `OtelBridgeOptions`). Stringifies array-valued attributes instead of dropping them.
|
|
3947
|
+
- the contract diff module — `keyForCell` uses `JSON.stringify([scenarioId, rep])` (no separator collisions); `Number.isFinite` coercion on dimension deltas (no NaN propagating to dashboards).
|
|
3903
3948
|
- `examples/hosted-ingest-server/server.ts` — `REFERENCE_RECEIVER_START=1|0` env var as the primary start signal; idempotency cache prunes on read with the wire-spec 24h TTL.
|
|
3904
3949
|
|
|
3905
3950
|
### Changed
|
|
@@ -3909,7 +3954,7 @@ Docs-only patch. No code changes, no behavior changes, no API surface changes vs
|
|
|
3909
3954
|
|
|
3910
3955
|
### Removed
|
|
3911
3956
|
|
|
3912
|
-
-
|
|
3957
|
+
- the traceai adapter — replaced by the OTel-to-hosted bridge module. No back-compat shim.
|
|
3913
3958
|
|
|
3914
3959
|
---
|
|
3915
3960
|
|
|
@@ -3918,8 +3963,8 @@ Docs-only patch. No code changes, no behavior changes, no API surface changes vs
|
|
|
3918
3963
|
### Added
|
|
3919
3964
|
|
|
3920
3965
|
- `src/verdict.ts` — `DefaultVerdict` substrate primitive (moved DOWN from agent-runtime).
|
|
3921
|
-
-
|
|
3922
|
-
-
|
|
3966
|
+
- the contract diff module — `diffRuns` / `diffGenerations` / `diffRunBaselineToWinner` for v3-vs-v4 dashboard rendering, CI reporting, and any consumer comparing improvement-loop output.
|
|
3967
|
+
- the traceai adapter — OTel→hosted bridge (renamed to `otel.ts` in 0.49.0).
|
|
3923
3968
|
- `tests/hosted-roundtrip.test.ts` — proves wire-format binary compat between client and reference receiver.
|
|
3924
3969
|
- Python `HostedClient` (`clients/python/src/agent_eval_rpc/hosted.py`) — TS↔Python wire-format parity with bearer auth, idempotency, and exponential backoff on 5xx/408/429.
|
|
3925
3970
|
- `CLAUDE.md` repo-layering rule: agent-eval is the substrate; agent-runtime + agent-knowledge depend on it; the reverse is forbidden.
|
|
@@ -4006,7 +4051,7 @@ aggregate improved?*
|
|
|
4006
4051
|
contract into `RunRecord` rows and `runEvalCampaign` so every campaign row
|
|
4007
4052
|
is keyed by `(profile, scenario, seed)` end-to-end. (#79)
|
|
4008
4053
|
- **Stats consolidation** — `pairedBootstrap`, power analysis, and the
|
|
4009
|
-
paired/Welch primitives now all live in `src/statistics
|
|
4054
|
+
paired/Welch primitives now all live in `src/statistics/`. (#73)
|
|
4010
4055
|
- **LLM retry classifier unified** across `llm-client` and `judge-retry`
|
|
4011
4056
|
via `isTransientLlmError`. (#74)
|
|
4012
4057
|
- **`pr-review-benchmark` source committed** — the module was exported from
|
|
@@ -4051,8 +4096,8 @@ tag/npm drift.
|
|
|
4051
4096
|
### Republish of 0.31.0 — dist drift fix
|
|
4052
4097
|
|
|
4053
4098
|
The `v0.31.0` tag's npm tarball shipped a stale `dist/` — `JudgeScoresRecord`
|
|
4054
|
-
was missing from `
|
|
4055
|
-
propagation never made it into `
|
|
4099
|
+
was missing from the built `index.d.ts` and the `recordOutcome.judgeScores`
|
|
4100
|
+
propagation never made it into the built `index.js`, even though the source on
|
|
4056
4101
|
the tagged commit had both. Consumers that bumped to `^0.31.0` got a
|
|
4057
4102
|
typecheck failure on `RunOutcome.judgeScores` (since the type wasn't
|
|
4058
4103
|
re-exported) and a silent drop on the wire (since the campaign runner
|
|
@@ -4062,8 +4107,8 @@ Cause: a build artifact picked up by the publish workflow predated the
|
|
|
4062
4107
|
source merge. The retag forces a clean `pnpm build` and republish; this
|
|
4063
4108
|
patch carries no source change beyond the version bump.
|
|
4064
4109
|
|
|
4065
|
-
Verified after this tag: `
|
|
4066
|
-
`
|
|
4110
|
+
Verified after this tag: the built `index.d.ts` contains `JudgeScoresRecord`,
|
|
4111
|
+
the built `index.js` propagates `outcome.judgeScores` end-to-end via
|
|
4067
4112
|
`recordOutcome.judgeScores`, and a downstream `pnpm install
|
|
4068
4113
|
@tangle-network/agent-eval@0.31.1` types-clean against the shape
|
|
4069
4114
|
documented in 0.31.0.
|
|
@@ -4107,7 +4152,7 @@ Every `perJudge[judge][dim]` and every `perDimMean[dim]` and the
|
|
|
4107
4152
|
`composite` must be finite numbers — the NaN-as-silent-zero bug class
|
|
4108
4153
|
banned by `CLAUDE.md` cannot pass the boundary. `failedJudges` must be
|
|
4109
4154
|
an array of non-empty strings; `notes` must be a string. Round-trip
|
|
4110
|
-
tested in
|
|
4155
|
+
tested in the run-record test.
|
|
4111
4156
|
|
|
4112
4157
|
### Fail-loud contract
|
|
4113
4158
|
|
|
@@ -4229,7 +4274,7 @@ different formulas with different domains of validity.
|
|
|
4229
4274
|
|
|
4230
4275
|
### Added
|
|
4231
4276
|
|
|
4232
|
-
- **`corpusInterRaterAgreement(records, opts?)`** (`src/statistics
|
|
4277
|
+
- **`corpusInterRaterAgreement(records, opts?)`** (`src/statistics/`) —
|
|
4233
4278
|
takes a flat list of `{itemId, judgeName, dimension, score}` records.
|
|
4234
4279
|
For each dimension, pivots to the [n_items × n_judges] matrix of items
|
|
4235
4280
|
every judge rated and delegates to `continuousAgreement` (ICC(2,1) +
|
|
@@ -4369,7 +4414,7 @@ clean primitive that wires them together end-to-end.
|
|
|
4369
4414
|
|
|
4370
4415
|
### Added
|
|
4371
4416
|
|
|
4372
|
-
- **`runProductionLoop({ ... })`** (
|
|
4417
|
+
- **`runProductionLoop({ ... })`** (the production loop module,
|
|
4373
4418
|
`@experimental`) — one call = one cycle. Ingests production traces
|
|
4374
4419
|
and feedback, clusters failures, runs evolve against the worst
|
|
4375
4420
|
cluster, gates with `HeldOutGate` + `evaluateReleaseConfidence`
|
|
@@ -4379,7 +4424,7 @@ clean primitive that wires them together end-to-end.
|
|
|
4379
4424
|
the primitive doesn't own scheduling.
|
|
4380
4425
|
|
|
4381
4426
|
- **`proposeAutomatedPullRequest(client, input)`** + two transports
|
|
4382
|
-
(`src/auto-pr.ts`, `@experimental`):
|
|
4427
|
+
(`src/campaign/auto-pr.ts`, `@experimental`):
|
|
4383
4428
|
- `httpGithubClient({ token, ... })` — direct REST against
|
|
4384
4429
|
`api.github.com`, no extra deps. Idempotent on branch name:
|
|
4385
4430
|
existing open PRs are returned, not duplicated.
|
|
@@ -4473,7 +4518,7 @@ first-touch usability and contract clarity. The visible deltas:
|
|
|
4473
4518
|
`contamination`, `compute-curves`. Experimental: `process-reward`,
|
|
4474
4519
|
`adversarial`, `active-curriculum`, `reward-hacking`, `adaptation-eval`,
|
|
4475
4520
|
`exporters`, `rl-campaign`, `predictive-validity-researcher`, `auto-research`.
|
|
4476
|
-
Tags are visible in IDE hover and emitted into `
|
|
4521
|
+
Tags are visible in IDE hover and emitted into the built `rl.d.ts` so consumers
|
|
4477
4522
|
can see the contract at the call site.
|
|
4478
4523
|
|
|
4479
4524
|
### Added
|
|
@@ -4551,12 +4596,12 @@ is now demonstrably real, not aspirational.
|
|
|
4551
4596
|
because it's the cleanest fit between agent-eval's exporters and
|
|
4552
4597
|
prime-rl's entrypoints (DPO/PRM go to TRL; offline GRPO requires a
|
|
4553
4598
|
custom verifiers env — both called out in the README).
|
|
4554
|
-
-
|
|
4599
|
+
- **the three-package architecture note** — the contracts between
|
|
4555
4600
|
agent-eval, agent-knowledge, agent-runtime. Dependency direction (both
|
|
4556
4601
|
consume agent-eval; agent-eval imports neither), shared data
|
|
4557
4602
|
interchange (RunRecord, Scenario, KnowledgeBundle), and known
|
|
4558
4603
|
contract gaps tracked as follow-ups.
|
|
4559
|
-
-
|
|
4604
|
+
- **the auto-research loop end-to-end note** — the runnable composition
|
|
4560
4605
|
pattern with the explicit invariants every iteration must preserve
|
|
4561
4606
|
(canonical RunRecord with scenarioId, capture wired by construction,
|
|
4562
4607
|
stable comparator, deterministic mutator).
|
|
@@ -4868,7 +4913,7 @@ Locked at `agent-eval-rpc==0.21.0` to match the npm package.
|
|
|
4868
4913
|
- Knowledge readiness recommended actions now honor non-blocking gap
|
|
4869
4914
|
acquisition modes such as `ask_user`, `search_web`, `query_connector`, and
|
|
4870
4915
|
`inspect_repo`.
|
|
4871
|
-
- Npm builds now generate `
|
|
4916
|
+
- Npm builds now generate the built `openapi.json`, and the package exports
|
|
4872
4917
|
`@tangle-network/agent-eval/openapi.json`.
|
|
4873
4918
|
- Npm and Python client versions are locked at `0.20.9`.
|
|
4874
4919
|
|
|
@@ -5034,7 +5079,7 @@ The v0.15 primitives were framed as "paper-grade" but most are production-rigor
|
|
|
5034
5079
|
- `paretoFigure` → `paretoChart`
|
|
5035
5080
|
- `gainDistributionFigure` → `gainHistogram`
|
|
5036
5081
|
- `PaperTable` / `PaperTableOptions` / `PaperTableRow` types → `SummaryTable` / `SummaryTableOptions` / `SummaryTableRow`
|
|
5037
|
-
- File:
|
|
5082
|
+
- File: the paper-report module → `src/summary-report.ts`
|
|
5038
5083
|
|
|
5039
5084
|
### Migration
|
|
5040
5085
|
|
|
@@ -5086,7 +5131,7 @@ optimization with held-out promotion gates.
|
|
|
5086
5131
|
- `deterministicSplit(itemId, seed?)`: stable 60/20/20 split via
|
|
5087
5132
|
FNV-1a hash. Default seed `agent-eval-v1`.
|
|
5088
5133
|
- **`summaryTable`, `paretoChart`, `gainHistogram`**
|
|
5089
|
-
(`
|
|
5134
|
+
(`src/summary-report.ts`) — Table 1 + Pareto + gain-distribution specs.
|
|
5090
5135
|
Returns data structures (markdown table, point lists, histogram bins);
|
|
5091
5136
|
caller picks the plotting library.
|
|
5092
5137
|
- **`runCanaries`** (`src/canary.ts`) — three liveness canaries:
|
|
@@ -5094,7 +5139,7 @@ optimization with held-out promotion gates.
|
|
|
5094
5139
|
judge calibration drift (KS test on confidence distribution), eval-set
|
|
5095
5140
|
distribution shift (chi-square on category bucket counts).
|
|
5096
5141
|
- **`pairedBootstrap`, `pairedWilcoxon`, `bhAdjust`**
|
|
5097
|
-
(
|
|
5142
|
+
(the paired-stats module) — paper-style aliases + the missing paired
|
|
5098
5143
|
bootstrap CI primitive. Deterministic with optional seed.
|
|
5099
5144
|
|
|
5100
5145
|
### Notes
|
package/README.md
CHANGED
|
@@ -149,7 +149,7 @@ See [evaluation integrity](./docs/evaluation-integrity.md) for the complete API
|
|
|
149
149
|
|
|
150
150
|
These controls check the evidence behind a result.
|
|
151
151
|
They do not establish that an optimizer beats a direct edit or simple search.
|
|
152
|
-
The [historical evidence audit](./
|
|
152
|
+
The [historical evidence audit](./archive/2026-09-13-self-improvement-evidence-audit.md) records prior gains, failed transfer, and missing comparisons.
|
|
153
153
|
|
|
154
154
|
Set `searchHistoryPolicy: 'require-complete'` when a method's search must be closed, with every cell, operation and node accounted for, before final evidence is exposed.
|
|
155
155
|
The [search ledger](./docs/search-ledger.md) records every search as nodes, edges and cells, and its receipt is the bounded proof.
|
|
@@ -201,7 +201,7 @@ The [benchmark-book review](./docs/design/mlbenchmarks-book-review.md) records t
|
|
|
201
201
|
|
|
202
202
|
## Development
|
|
203
203
|
|
|
204
|
-
Read [
|
|
204
|
+
Read [AGENTS.md](./AGENTS.md) first: dependency boundaries, ledger canonicalization, and local conventions apply to every change.
|
|
205
205
|
|
|
206
206
|
```sh
|
|
207
207
|
pnpm install
|
package/dist/adapters/http.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { u as CostReceipt } from "../cost-ledger-Jk62OBfF.js";
|
|
2
|
-
import { R as Scenario, d as DispatchContext, f as DispatchFn } from "../types-
|
|
3
|
-
import "../index-
|
|
2
|
+
import { R as Scenario, d as DispatchContext, f as DispatchFn } from "../types-D8GhJoHk.js";
|
|
3
|
+
import "../index-BiT6RtNH.js";
|
|
4
4
|
//#region src/adapters/http.d.ts
|
|
5
5
|
export interface HttpDispatchOptions<TScenario extends Scenario, _TArtifact> {
|
|
6
6
|
/** Static endpoint URL. Mutually exclusive with `resolveUrl`. */
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { d as CostReceiptInput, p as CustomTokenPricing } from "./cost-ledger-Jk62OBfF.js";
|
|
2
|
-
import {
|
|
3
|
-
import { C as ChatResponse, S as ChatRequest } from "./types-
|
|
4
|
-
import { c as RegistryRunOpts, t as AnalystRegistry } from "./registry-D7_hm2fK.js";
|
|
2
|
+
import { W as TraceAnalysisStore, i as AnalystFinding, m as EvidenceRef, p as AnalystUsageReceipt } from "./types-CW8ewATZ.js";
|
|
3
|
+
import { C as ChatResponse, S as ChatRequest } from "./types-D-mgRzlh.js";
|
|
5
4
|
import { AgentProfile, AgentProfile as AgentProfile$1, HarnessType, HarnessType as HarnessType$1 } from "@tangle-network/agent-interface";
|
|
6
5
|
//#region src/campaign/external-optimizer-contracts.d.ts
|
|
7
6
|
interface ExternalOptimizerProcessLimits {
|
|
@@ -33,8 +32,6 @@ interface ExternalTextEvaluationRequest {
|
|
|
33
32
|
candidate: ExternalTextCandidate;
|
|
34
33
|
exampleId: string;
|
|
35
34
|
}
|
|
36
|
-
declare function resolveExternalOptimizerProcessLimits(value: Partial<ExternalOptimizerProcessLimits> | undefined, label?: string): ExternalOptimizerProcessLimits;
|
|
37
|
-
declare function resolveExternalOptimizerCallbackLimits(value: Partial<ExternalOptimizerCallbackLimits> | undefined, label?: string): ExternalOptimizerCallbackLimits;
|
|
38
35
|
type DeepReadonly<T> = T extends ((...args: never[]) => unknown) ? T : T extends readonly (infer U)[] ? readonly DeepReadonly<U>[] : T extends object ? { readonly [K in keyof T]: DeepReadonly<T[K]>; } : T;
|
|
39
36
|
/** Provider-neutral request parsed once from the optimizer's loopback protocol. */
|
|
40
37
|
type ExternalOptimizerChatRequest = DeepReadonly<Omit<ChatRequest, 'model'> & {
|
|
@@ -171,9 +168,6 @@ interface ExternalOptimizerWireCounts {
|
|
|
171
168
|
successfulCompletions: number;
|
|
172
169
|
}
|
|
173
170
|
//#endregion
|
|
174
|
-
//#region src/analyst/benchmark-scoring.d.ts
|
|
175
|
-
declare function scoreAnalystFindings(testCase: Pick<AnalystBenchmarkCase, 'id' | 'expectedIssues' | 'labeledEvidence'>, findings: readonly AnalystFinding[]): AnalystFindingScore;
|
|
176
|
-
//#endregion
|
|
177
171
|
//#region src/analyst/benchmark.d.ts
|
|
178
172
|
interface AnalystEvidenceExpectation {
|
|
179
173
|
uri: string;
|
|
@@ -395,13 +389,6 @@ interface RunAnalystBenchmarkOptions<TInput> {
|
|
|
395
389
|
signal?: AbortSignal;
|
|
396
390
|
}
|
|
397
391
|
declare function runAnalystBenchmark<TInput>(options: RunAnalystBenchmarkOptions<TInput>): Promise<AnalystBenchmarkResult>;
|
|
398
|
-
declare function registryBenchmarkRunner(options: {
|
|
399
|
-
id: string;
|
|
400
|
-
registry: AnalystRegistry;
|
|
401
|
-
runOptions?: Omit<RegistryRunOpts, 'signal'>;
|
|
402
|
-
/** Count any selected analyst failure as a failed benchmark run. */
|
|
403
|
-
failOnAnalystFailure?: boolean;
|
|
404
|
-
}): AnalystBenchmarkRunner<AnalystRunInputs>;
|
|
405
392
|
//#endregion
|
|
406
393
|
//#region src/agent-profile.d.ts
|
|
407
394
|
/**
|
|
@@ -484,5 +471,5 @@ declare function agentProfileId(profile: AgentProfile): string;
|
|
|
484
471
|
*/
|
|
485
472
|
declare function agentProfileHash(profile: AgentProfile): string;
|
|
486
473
|
//#endregion
|
|
487
|
-
export {
|
|
488
|
-
//# sourceMappingURL=agent-profile-
|
|
474
|
+
export { traceStoreEvidenceResolver as A, ExternalOptimizerModelCallRequest as B, AnalystEvidenceResolutionError as C, AnalystLatencyDistribution as D, AnalystIssueExpectation as E, ExternalOptimizerEndpointFormat as F, ExternalOptimizerRunnerCommand as G, ExternalOptimizerModelExecutionObservation as H, ExternalOptimizerEvaluationObservation as I, ExternalTextEvaluationRequest as J, ExternalOptimizerWireCounts as K, ExternalOptimizerEvaluationRefusalReason as L, DEFAULT_EXTERNAL_OPTIMIZER_PROCESS_LIMITS as M, ExternalOptimizerCallbackLimits as N, RunAnalystBenchmarkOptions as O, ExternalOptimizerChatRequest as P, ExternalOptimizerModelBudget as R, AnalystEvidenceResolution as S, AnalystFindingScore as T, ExternalOptimizerProcessLimits as U, ExternalOptimizerModelCallResult as V, ExternalOptimizerResumeMode as W, AnalystBenchmarkProvenance as _, ProfileAxisSpec as a, AnalystBenchmarkSummary as b, expandProfileAxes as c, AnalystBenchmarkDatasetRef as d, AnalystBenchmarkDescriptor as f, AnalystBenchmarkOutput as g, AnalystBenchmarkObservation as h, HarnessType$1 as i, DEFAULT_EXTERNAL_OPTIMIZER_CALLBACK_LIMITS as j, runAnalystBenchmark as k, harnessAxisOf as l, AnalystBenchmarkLabelState as m, CODING_HARNESSES as n, agentProfileHash as o, AnalystBenchmarkError as p, ExternalTextCandidate as q, HARNESS_NATIVE_MODEL as r, agentProfileId as s, AgentProfile$1 as t, AnalystBenchmarkCase as u, AnalystBenchmarkResult as v, AnalystEvidenceResolver as w, AnalystEvidenceExpectation as x, AnalystBenchmarkRunner as y, ExternalOptimizerModelCall as z };
|
|
475
|
+
//# sourceMappingURL=agent-profile-DTt8uGP9.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"agent-profile-DTt8uGP9.d.ts","names":["_$___cost_ledger0.CostReceiptInput"],"sources":["../src/campaign/external-optimizer-contracts.ts","../src/analyst/benchmark.ts","../src/agent-profile.ts"],"mappings":";;;;;UAciB;;EAEf;;EAEA;;EAEA;;cAGW,2CAA2C,SAAS;UAOhD;;EAEf;;EAEA;;cAGW,4CAA4C,SAAS;UAMjD;EACf;EACA;EACA,MAAM,OAAO;;EAEb,SAAS,QAAQ;;KAGP;KAEA,iCAAiC;UAE5B;EACf,WAAW;EACX;;KAqDG,aAAa,KAAK,eAAc,6BACjC,IACA,0BAA0B,gBACf,aAAa,OACtB,+BACc,WAAW,IAAI,aAAa,EAAE,SAC1C;;KAGI,+BAA+B,aACzC,KAAK;EAA0B;;KAGrB;;UAMK;;WAEN;;WAEA,SAAS;;WAET,iBAAiB;WACjB,QAAQ;;;KAIP;WAEG;;WAEA,UAAU;;;;;;;WAOV,SAPUA;;WASV;;WAGA;;WAEA;;WAEA,SATkCA;;WAWlC;;;;;;;;;;;;KAaH,8BACV,SAAS,sCACN,QAAQ;;KAGD;WAEG;WACA;WACA;WACA;WACA;WACA;WACA;WACA;;WAGA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;;KAGH;;;;;;KAUA;WAEG;WACA;WACA,WAAW;WACX;;WAGA;WACA;WACA,WAAW;WACX;WACA;;WAEA;WACA;;WAGA;WACA;WACA,QAAQ;WACR,YAAY;WACZ;WACA;;UAGE;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;;;;;EAUA;;;;;;EAMA,UAAU;;EAEV;;;UAIe;;EAEf;;EAEA;;;;UCpQe;EACf;EACA,OAAO;;UAGQ;EACf;EACA;EACA;EACA;EACA,oBAAoB;EACpB;;EAEA,4BAA4B;;KAGlB;UAEK,qBAAqB;EACpC;;EAEA;;EAEA,YAAY;EACZ,OAAO;EACP,yBAAyB;;EAEzB,2BAA2B;EAC3B;EACA,WAAW;;UAGI;EACf;EACA;EACA;EACA;EACA;EACA,mBAAmB;EACnB;EACA;EACA;EACA;;EAEA;;EAEA;;EAEA;EACA;;UAGe;EACf,UAAU;EACV;EACA;;UAGe;EACf;EACA;EACA,oBAAoB;EACpB,QAAQ;;EAER;;KAGU,wBAAwB,qBAAqB;EACvD;EACA,WAAW;EACX,UAAU;EACV,SAAS;gBACK;;;;;iBAMA,2BAA2B,QACzC,WAAW,OAAO,WAAW,qBAC5B,wBAAwB;UAoBV;EACf,mBAAmB;EACnB,QAAQ;EACR,WAAW;;;;;EAKX;;EAEA,QAAQ;;UAGO;EACf;EACA;EACA;EACA;;UAGe,uBAAuB;EACtC;EACA,QACE,OAAO,QACP;IAAW;IAAgB;IAAoB,SAAS;MACvD,yBAAyB,QAAQ;;UAGrB;EACf;EACA;EACA;EACA,YAAY;EACZ;EACA;EACA;EACA;EACA,mBAAmB;EACnB,OAAO;EACP,qBAAqB;EACrB;EACA,eAAe;EACf,QAAQ;EACR,iBAAiB;EACjB,QAAQ;;UAGO;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;EAEA;;EAEA;;EAEA;;EAEA;EACA,WAAW;EACX;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;;UAGe;EACf;EACA,UAAU;EACV;EACA,cAAc;EACd,WAAW;;UAGI,mCAAmC;EAClD;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;EACf,YAAY;EACZ,cAAc;EACd,WAAW;;UAGI,2BAA2B;EAC1C,gBAAgB,qBAAqB;EACrC,kBAAkB,uBAAuB;EACzC;EACA;EACA;EACA,kBAAkB,wBAAwB;EAC1C,YAAY;;EAEZ,+BAA+B;EAC/B,iBAAiB,aAAa,uCAAuC;EACrE,SAAS;;iBAGW,oBAAoB,QACxC,SAAS,2BAA2B,UACnC,QAAQ;;;;;;;;;;cCnQE,2BAA2B;UAOvB;;;EAGf,MAAM;;EAEN,qBAAqB;;;EAGrB;;;;;;EAMA;;;;;;;cAQW;;;;;;;;;;;;;;;;;;;iBAoBG,kBAAkB,MAAM,kBAAkB;;;;;;;;iBAwD1C,cACd,SAAS,KAAK;EACX,SAAS;EAAa;;;;;;;;;iBAiBX,eAAe,SAAS;;;;;;;;;;;iBAmDxB,iBAAiB,SAAS"}
|