@tangle-network/agent-eval 0.216.0 → 0.217.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +141 -57
- package/README.md +2 -2
- package/dist/adapters/http.d.ts +3 -3
- package/dist/{agent-profile-CIA2Atnt.d.ts → agent-profile-DTt8uGP9.d.ts} +5 -18
- package/dist/agent-profile-DTt8uGP9.d.ts.map +1 -0
- package/dist/{agent-profile-cell-BYc0bxsL.js → agent-profile-cell-B1V1gUZs.js} +4 -53
- package/dist/agent-profile-cell-B1V1gUZs.js.map +1 -0
- package/dist/{agent-profile-cell-BNp-Js9F.d.ts → agent-profile-cell-BcpFbNd5.d.ts} +2 -22
- package/dist/agent-profile-cell-BcpFbNd5.d.ts.map +1 -0
- package/dist/{agreement-irr-DsTURxdr.js → agreement-irr-BhAqjluA.js} +5 -5
- package/dist/agreement-irr-BhAqjluA.js.map +1 -0
- package/dist/analyst/index.d.ts +37 -170
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +17 -192
- package/dist/analyst/index.js.map +1 -1
- package/dist/{analyze-runs-3xpby3p2.js → analyze-runs-DqPJaJ4X.js} +763 -555
- package/dist/analyze-runs-DqPJaJ4X.js.map +1 -0
- package/dist/{analyze-runs-CSDAh8A7.d.ts → analyze-runs-DyeNcenW.d.ts} +18 -16
- package/dist/analyze-runs-DyeNcenW.d.ts.map +1 -0
- package/dist/authenticity/index.d.ts +0 -23
- package/dist/authenticity/index.d.ts.map +1 -1
- package/dist/authenticity/index.js +1 -34
- package/dist/authenticity/index.js.map +1 -1
- package/dist/{benchmark-command-CByl9dxG.js → benchmark-command-xQK8nOI6.js} +591 -109
- package/dist/benchmark-command-xQK8nOI6.js.map +1 -0
- package/dist/benchmarks/index.d.ts +5 -6
- package/dist/benchmarks/index.d.ts.map +1 -1
- package/dist/benchmarks/index.js +4 -4
- package/dist/benchmarks/index.js.map +1 -1
- package/dist/builder-eval/index.d.ts +3 -3
- package/dist/builder-eval/index.js +2 -2
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/campaign/index.d.ts +8 -8
- package/dist/campaign/index.js +15 -15
- package/dist/{campaign-BQ-Jujuj.js → campaign-CJGkxAQ6.js} +256 -765
- package/dist/campaign-CJGkxAQ6.js.map +1 -0
- package/dist/{campaign-evidence-OcUciD-N.js → campaign-evidence-CQYw8_0H.js} +10 -18
- package/dist/{campaign-evidence-OcUciD-N.js.map → campaign-evidence-CQYw8_0H.js.map} +1 -1
- package/dist/{canonical-BsZVgBhW.js → canonical-Dq2hOQ8L.js} +2 -2
- package/dist/{canonical-BsZVgBhW.js.map → canonical-Dq2hOQ8L.js.map} +1 -1
- package/dist/cell-spend-DEWR3nj-.js +90 -0
- package/dist/cell-spend-DEWR3nj-.js.map +1 -0
- package/dist/{chat-client-DthPOYGd.js → chat-client-ldqxYYTN.js} +2 -2
- package/dist/{chat-client-DthPOYGd.js.map → chat-client-ldqxYYTN.js.map} +1 -1
- package/dist/{chat-json-call-BrKsyspg.js → chat-json-call-DavzexSp.js} +3 -2
- package/dist/chat-json-call-DavzexSp.js.map +1 -0
- package/dist/{claim-BXcZw-jo.js → claim-BPVuyHuy.js} +4 -4
- package/dist/claim-BPVuyHuy.js.map +1 -0
- package/dist/cli.js +113 -12
- package/dist/cli.js.map +1 -1
- package/dist/{client-BzL9niqi.d.ts → client-hwo3cJjR.d.ts} +3 -16
- package/dist/client-hwo3cJjR.d.ts.map +1 -0
- package/dist/{cluster-bootstrap-B3JywQQn.js → cluster-bootstrap-DEwwhl0C.js} +3 -3
- package/dist/{cluster-bootstrap-B3JywQQn.js.map → cluster-bootstrap-DEwwhl0C.js.map} +1 -1
- package/dist/{produced-state-ZCtDBxAE.js → completion-verifier-B_jw6C9p.js} +9 -64
- package/dist/completion-verifier-B_jw6C9p.js.map +1 -0
- package/dist/contract/index.d.ts +13 -13
- package/dist/contract/index.js +1041 -1473
- package/dist/contract/index.js.map +1 -1
- package/dist/{cost-ledger-D2FnJPBF.js → cost-ledger-D0352Z4d.js} +832 -817
- package/dist/cost-ledger-D0352Z4d.js.map +1 -0
- package/dist/{cost-ledger-CO7UyEcQ.d.ts → cost-ledger-Jk62OBfF.d.ts} +12 -47
- package/dist/cost-ledger-Jk62OBfF.d.ts.map +1 -0
- package/dist/dataset-DQqhOCPt.d.ts.map +1 -1
- package/dist/{default-registry-Cn1LQ2Ve.js → default-registry-ClYjq3QE.js} +8 -11
- package/dist/default-registry-ClYjq3QE.js.map +1 -0
- package/dist/{default-registry-BrWh62zf.d.ts → default-registry-DXgLmoDI.d.ts} +7 -50
- package/dist/default-registry-DXgLmoDI.d.ts.map +1 -0
- package/dist/{define-agent-eval-Ch3MWX6k.js → define-agent-eval-BDxNlP3D.js} +10 -10
- package/dist/define-agent-eval-BDxNlP3D.js.map +1 -0
- package/dist/{define-agent-eval-CEWkX5wZ.d.ts → define-agent-eval-BIkH_gAR.d.ts} +9 -9
- package/dist/define-agent-eval-BIkH_gAR.d.ts.map +1 -0
- package/dist/{descriptive-CnU8Yq29.js → descriptive-CVNy48rR.js} +2 -2
- package/dist/{descriptive-CnU8Yq29.js.map → descriptive-CVNy48rR.js.map} +1 -1
- package/dist/diagnosis/index.d.ts +4 -14
- package/dist/diagnosis/index.d.ts.map +1 -1
- package/dist/diagnosis/index.js +67 -28
- package/dist/diagnosis/index.js.map +1 -1
- package/dist/{dspy-rlm-engine-D-KX_e64.js → dspy-rlm-engine-DhdCr1Dc.js} +5 -10
- package/dist/dspy-rlm-engine-DhdCr1Dc.js.map +1 -0
- package/dist/{emitter-HcCmFQl_.d.ts → emitter-DABYYTcH.d.ts} +5 -5
- package/dist/emitter-DABYYTcH.d.ts.map +1 -0
- package/dist/emitter-DbMXUIVP.js.map +1 -1
- package/dist/{engine-C_zNWfM8.d.ts → engine-BvZ0R-36.d.ts} +18 -22
- package/dist/engine-BvZ0R-36.d.ts.map +1 -0
- package/dist/{equal-terms-Blx9Zk5I.js → equal-terms-DbPGcVtI.js} +2 -2
- package/dist/equal-terms-DbPGcVtI.js.map +1 -0
- package/dist/{errors-Dngq5h35.js → errors-BcOs5MKW.js} +6 -8
- package/dist/{errors-Dngq5h35.js.map → errors-BcOs5MKW.js.map} +1 -1
- package/dist/{eval-campaign-4byTtqrf.js → eval-campaign-BLEfBGbA.js} +6 -6
- package/dist/{eval-campaign-4byTtqrf.js.map → eval-campaign-BLEfBGbA.js.map} +1 -1
- package/dist/{evaluation-Cn2dRZIz.d.ts → evaluation-jTjC_QQD.d.ts} +8 -8
- package/dist/evaluation-jTjC_QQD.d.ts.map +1 -0
- package/dist/evaluation.d.ts +2 -2
- package/dist/evaluation.js +1 -1
- package/dist/evaluation.js.map +1 -1
- package/dist/{exact-types-CD0lf5Qd.d.ts → exact-types-CflbJ9f3.d.ts} +2 -2
- package/dist/{exact-types-CD0lf5Qd.d.ts.map → exact-types-CflbJ9f3.d.ts.map} +1 -1
- package/dist/{exec-ChJ6d5if.js → exec-CpbtOXVt.js} +2 -24
- package/dist/{exec-ChJ6d5if.js.map → exec-CpbtOXVt.js.map} +1 -1
- package/dist/execution-tracks-CpgFPpS5.js.map +1 -1
- package/dist/experiment/index.d.ts +331 -33
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +303 -101
- package/dist/experiment/index.js.map +1 -1
- package/dist/{exporters-X4xwla8A.js → exporters-Cl6IeNAu.js} +3 -3
- package/dist/exporters-Cl6IeNAu.js.map +1 -0
- package/dist/{exporters-ChTnR90l.d.ts → exporters-Dk3oEbK_.d.ts} +2 -4
- package/dist/{exporters-ChTnR90l.d.ts.map → exporters-Dk3oEbK_.d.ts.map} +1 -1
- package/dist/{external-optimizer-process-nnsY5D_4.js → external-optimizer-process-KVFGvYX1.js} +4 -3
- package/dist/external-optimizer-process-KVFGvYX1.js.map +1 -0
- package/dist/{external-optimizer-subprocess-DcerlZSG.js → external-optimizer-subprocess-XIHBOSKt.js} +551 -553
- package/dist/external-optimizer-subprocess-XIHBOSKt.js.map +1 -0
- package/dist/{extract-usage-BrnzRJ6H.js → extract-usage-BZLU01kR.js} +2 -23
- package/dist/{extract-usage-BrnzRJ6H.js.map → extract-usage-BZLU01kR.js.map} +1 -1
- package/dist/{failure-cluster-CbgEDcFJ.d.ts → failure-cluster-DN0t63Z0.d.ts} +4 -43
- package/dist/failure-cluster-DN0t63Z0.d.ts.map +1 -0
- package/dist/{failure-taxonomy-CjabTATj.js → failure-taxonomy-Bx5oWqaW.js} +2 -19
- package/dist/{failure-taxonomy-CjabTATj.js.map → failure-taxonomy-Bx5oWqaW.js.map} +1 -1
- package/dist/{feedback-trajectory-DkaDj0dj.d.ts → feedback-trajectory-BWmhRBGu.d.ts} +8 -13
- package/dist/feedback-trajectory-BWmhRBGu.d.ts.map +1 -0
- package/dist/{semantic-concept-judge-zQBG9hHU.js → findings-store-BMVQtEsm.js} +13 -255
- package/dist/findings-store-BMVQtEsm.js.map +1 -0
- package/dist/fuzz.d.ts +20 -63
- package/dist/fuzz.d.ts.map +1 -1
- package/dist/fuzz.js +62 -108
- package/dist/fuzz.js.map +1 -1
- package/dist/{hf-dataset-BR2yhuNn.js → hf-dataset-D9f7-oX7.js} +4 -24
- package/dist/hf-dataset-D9f7-oX7.js.map +1 -0
- package/dist/hosted/index.d.ts +2 -2
- package/dist/hosted/index.js +2 -2
- package/dist/{index-DHKEAv3s.d.ts → index-9jJ2fKSX.d.ts} +2 -64
- package/dist/index-9jJ2fKSX.d.ts.map +1 -0
- package/dist/{index-oQ4aHziK.d.ts → index-BLGqZ81N.d.ts} +11 -16
- package/dist/{index-oQ4aHziK.d.ts.map → index-BLGqZ81N.d.ts.map} +1 -1
- package/dist/{index-DlH0Y9gI.d.ts → index-BiT6RtNH.d.ts} +31 -43
- package/dist/index-BiT6RtNH.d.ts.map +1 -0
- package/dist/{index-Ch9VxCpK.d.ts → index-CX-GAUpw.d.ts} +4 -8
- package/dist/index-CX-GAUpw.d.ts.map +1 -0
- package/dist/{index-CPJJr98o.d.ts → index-CiRwLv2R.d.ts} +5 -68
- package/dist/index-CiRwLv2R.d.ts.map +1 -0
- package/dist/{index-NvGBiO8H.d.ts → index-ZuWx52WG.d.ts} +266 -495
- package/dist/index-ZuWx52WG.d.ts.map +1 -0
- package/dist/index.d.ts +2256 -2165
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +514 -1192
- package/dist/index.js.map +1 -1
- package/dist/{integrity-BV1ZDf42.js → integrity-DOd9Q86O.js} +7 -12
- package/dist/integrity-DOd9Q86O.js.map +1 -0
- package/dist/{integrity-BvnVARVl.js → integrity-W4CADcFc.js} +3 -7
- package/dist/{integrity-BvnVARVl.js.map → integrity-W4CADcFc.js.map} +1 -1
- package/dist/{integrity-CCAxGVK6.d.ts → integrity-_qgZLA5p.d.ts} +4 -6
- package/dist/{integrity-CCAxGVK6.d.ts.map → integrity-_qgZLA5p.d.ts.map} +1 -1
- package/dist/{internal-CRHwS9iz.js → internal-BTK91Bqt.js} +3 -3
- package/dist/{internal-CRHwS9iz.js.map → internal-BTK91Bqt.js.map} +1 -1
- package/dist/interval-Dq9gt7jx.js +219 -0
- package/dist/interval-Dq9gt7jx.js.map +1 -0
- package/dist/interval-ohuOJurX.d.ts +47 -0
- package/dist/interval-ohuOJurX.d.ts.map +1 -0
- package/dist/{journal-DvKCvjty.js → journal-CR4QikYp.js} +110 -110
- package/dist/journal-CR4QikYp.js.map +1 -0
- package/dist/json-BhdsJaOQ.js +111 -0
- package/dist/json-BhdsJaOQ.js.map +1 -0
- package/dist/{judge-calibration-N44V04Se.js → judge-calibration-BkXMIWA4.js} +3 -3
- package/dist/{judge-calibration-N44V04Se.js.map → judge-calibration-BkXMIWA4.js.map} +1 -1
- package/dist/{judge-gate-B1qYnkpi.js → judge-gate-Co5tuh57.js} +4 -27
- package/dist/judge-gate-Co5tuh57.js.map +1 -0
- package/dist/{judge-gate-DNcUrRFA.d.ts → judge-gate-DsL0yFcb.d.ts} +9 -10
- package/dist/judge-gate-DsL0yFcb.d.ts.map +1 -0
- package/dist/{kind-factory-CbZ-6tvu.js → kind-factory-Gy4fXixd.js} +88 -17
- package/dist/kind-factory-Gy4fXixd.js.map +1 -0
- package/dist/ledger-core/index.d.ts +2 -2
- package/dist/ledger-core/index.js +4 -4
- package/dist/{ledger-core-B2hahzrt.js → ledger-core-DrJhbms9.js} +5 -84
- package/dist/ledger-core-DrJhbms9.js.map +1 -0
- package/dist/{llm-client-DQveoxSM.js → llm-client-BshCREn6.js} +90 -219
- package/dist/llm-client-BshCREn6.js.map +1 -0
- package/dist/{llm-judge-PXV7Xljl.js → llm-judge-C8jMTNNG.js} +501 -931
- package/dist/llm-judge-C8jMTNNG.js.map +1 -0
- package/dist/matrix/index.d.ts +4 -4
- package/dist/matrix/index.js +3 -2
- package/dist/{matrix-_BV9-Lk5.js → matrix-B8l-Nsxd.js} +7 -96
- package/dist/matrix-B8l-Nsxd.js.map +1 -0
- package/dist/{matrix-CHMiebFx.d.ts → matrix-DHe-dwVJ.d.ts} +10 -19
- package/dist/matrix-DHe-dwVJ.d.ts.map +1 -0
- package/dist/{paired-tests-jbwPdSz5.js → mean-BbA_JCzW.js} +45 -5
- package/dist/mean-BbA_JCzW.js.map +1 -0
- package/dist/meta-eval/index.d.ts +6 -6
- package/dist/meta-eval/index.js +10 -10
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/metrics-BKaPBtIG.js.map +1 -1
- package/dist/mint-BRZXuwRi.d.ts +81 -0
- package/dist/mint-BRZXuwRi.d.ts.map +1 -0
- package/dist/{mint-Csor1BLJ.js → mint-zszyYlpA.js} +4 -4
- package/dist/{mint-Csor1BLJ.js.map → mint-zszyYlpA.js.map} +1 -1
- package/dist/{multi-layer-verifier-BUaQ4C17.d.ts → multi-layer-verifier-DEgMUk6v.d.ts} +2 -2
- package/dist/{multi-layer-verifier-BUaQ4C17.d.ts.map → multi-layer-verifier-DEgMUk6v.d.ts.map} +1 -1
- package/dist/{multiplicity-D9cjmStq.js → multiplicity-Sg0U7h8j.js} +2 -2
- package/dist/{multiplicity-D9cjmStq.js.map → multiplicity-Sg0U7h8j.js.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -23
- package/dist/multishot/golden/index.d.ts.map +1 -1
- package/dist/multishot/golden/index.js +5 -8
- package/dist/multishot/golden/index.js.map +1 -1
- package/dist/multishot/index.d.ts +3 -12
- package/dist/multishot/index.d.ts.map +1 -1
- package/dist/multishot/index.js +4 -3
- package/dist/multishot/index.js.map +1 -1
- package/dist/nonnegative-number-DvdMxg_f.js +9 -0
- package/dist/nonnegative-number-DvdMxg_f.js.map +1 -0
- package/dist/openapi.json +114 -10
- package/dist/{otel-spans-C8YIJWy1.js → otel-spans-CdN909k1.js} +3 -3
- package/dist/{otel-spans-C8YIJWy1.js.map → otel-spans-CdN909k1.js.map} +1 -1
- package/dist/{paired-arms-DzKn0TfY.js → paired-arms-Ct4LTKc1.js} +32 -9
- package/dist/paired-arms-Ct4LTKc1.js.map +1 -0
- package/dist/{paired-binary-Q36bhn2R.js → paired-binary-B_nejA44.js} +3 -36
- package/dist/paired-binary-B_nejA44.js.map +1 -0
- package/dist/{paired-promotion-decision-DPKz8ZnF.d.ts → paired-promotion-decision-CQMJ5xGN.d.ts} +2 -34
- package/dist/paired-promotion-decision-CQMJ5xGN.d.ts.map +1 -0
- package/dist/{paired-promotion-decision-BN6IQTiu.js → paired-promotion-decision-D0SHb-ZL.js} +3 -3
- package/dist/{paired-promotion-decision-BN6IQTiu.js.map → paired-promotion-decision-D0SHb-ZL.js.map} +1 -1
- package/dist/{search-kernel-BztlERLv.js → pareto-BzsxidFD.js} +883 -861
- package/dist/pareto-BzsxidFD.js.map +1 -0
- package/dist/pipelines/index.d.ts +16 -19
- package/dist/pipelines/index.d.ts.map +1 -1
- package/dist/pipelines/index.js +4 -68
- package/dist/pipelines/index.js.map +1 -1
- package/dist/{power-and-mde-fK5Qocwo.js → power-and-mde-N5nU4_Ly.js} +3 -3
- package/dist/{power-and-mde-fK5Qocwo.js.map → power-and-mde-N5nU4_Ly.js.map} +1 -1
- package/dist/predicate-FdKmElA_.js +38 -0
- package/dist/predicate-FdKmElA_.js.map +1 -0
- package/dist/{prime-protocol-DcAK9pC8.js → prime-protocol-BFgGrEMP.js} +3 -3
- package/dist/prime-protocol-BFgGrEMP.js.map +1 -0
- package/dist/{prime-protocol-CEE4BR9N.d.ts → prime-protocol-BWG2DQIk.d.ts} +4 -22
- package/dist/prime-protocol-BWG2DQIk.d.ts.map +1 -0
- package/dist/profile-cell.d.ts +2 -2
- package/dist/profile-cell.js +2 -2
- package/dist/{release-confidence-BK2QCB5M.d.ts → promotion-gate-DaY95yU6.d.ts} +105 -106
- package/dist/promotion-gate-DaY95yU6.d.ts.map +1 -0
- package/dist/{promotion-policy-CWW35UWq.js → promotion-policy-6A8osGr_.js} +4 -4
- package/dist/{promotion-policy-CWW35UWq.js.map → promotion-policy-6A8osGr_.js.map} +1 -1
- package/dist/{proposal-findings-BZiMTzAF.js → proposal-findings-BbWidvUk.js} +3 -3
- package/dist/proposal-findings-BbWidvUk.js.map +1 -0
- package/dist/{query-B1D1qp2Z.js → query-B_RZ36an.js} +5 -5
- package/dist/query-B_RZ36an.js.map +1 -0
- package/dist/{query-DK1nWcmp.d.ts → query-Cmv_U8vf.d.ts} +4 -24
- package/dist/{query-DK1nWcmp.d.ts.map → query-Cmv_U8vf.d.ts.map} +1 -1
- package/dist/{random-Dn5fPWkt.js → random-DDDcC7YU.js} +2 -2
- package/dist/{random-Dn5fPWkt.js.map → random-DDDcC7YU.js.map} +1 -1
- package/dist/{raw-provider-sink-rB2DYxr6.js → raw-provider-sink-75w4FWBj.js} +2 -2
- package/dist/{raw-provider-sink-rB2DYxr6.js.map → raw-provider-sink-75w4FWBj.js.map} +1 -1
- package/dist/{redact-Cveoq6fJ.d.ts → redact-DXdlFb_U.d.ts} +4 -21
- package/dist/redact-DXdlFb_U.d.ts.map +1 -0
- package/dist/{redact-Dk1ZdJEm.js → redact-mtkTiJVk.js} +92 -98
- package/dist/redact-mtkTiJVk.js.map +1 -0
- package/dist/{registry-JYfPxcVz.d.ts → registry-ssNoHzwD.d.ts} +5 -5
- package/dist/{registry-JYfPxcVz.d.ts.map → registry-ssNoHzwD.d.ts.map} +1 -1
- package/dist/{release-confidence-Dbe9AS_X.js → release-confidence-CYIFH-fp.js} +6 -11
- package/dist/release-confidence-CYIFH-fp.js.map +1 -0
- package/dist/{report-command-y8H6RSX8.js → render-C-w2EZ6p.js} +331 -554
- package/dist/render-C-w2EZ6p.js.map +1 -0
- package/dist/reporting-html-Kvlc8oMt.js +100 -0
- package/dist/reporting-html-Kvlc8oMt.js.map +1 -0
- package/dist/reporting.d.ts +6 -6
- package/dist/reporting.js +8 -8
- package/dist/{researcher-f6mZhh58.d.ts → researcher-0ABVIdGa.d.ts} +9 -9
- package/dist/{researcher-f6mZhh58.d.ts.map → researcher-0ABVIdGa.d.ts.map} +1 -1
- package/dist/reward-DrZ71vnf.d.ts +55 -0
- package/dist/reward-DrZ71vnf.d.ts.map +1 -0
- package/dist/{reward-hacking-BFcQgkux.js → reward-hacking-ByuETaIb.js} +3 -3
- package/dist/{reward-hacking-BFcQgkux.js.map → reward-hacking-ByuETaIb.js.map} +1 -1
- package/dist/{reward-hacking-DHA7c1l8.d.ts → reward-hacking-E52wq86n.d.ts} +5 -18
- package/dist/reward-hacking-E52wq86n.d.ts.map +1 -0
- package/dist/reward-nw2xZGZG.js.map +1 -1
- package/dist/rl.d.ts +117 -295
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +27 -747
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +5 -4
- package/dist/rollout/index.js +7 -8
- package/dist/{rollout-D9bKKBnZ.js → rollout-D6C50mfA.js} +88 -13
- package/dist/rollout-D6C50mfA.js.map +1 -0
- package/dist/round-decimal-COrKglMh.js +10 -0
- package/dist/round-decimal-COrKglMh.js.map +1 -0
- package/dist/{rubric-predictive-validity-BsgkBfT0.js → rubric-predictive-validity-D3_oI72N.js} +3 -3
- package/dist/{rubric-predictive-validity-BsgkBfT0.js.map → rubric-predictive-validity-D3_oI72N.js.map} +1 -1
- package/dist/{rubric-predictive-validity-2SgHxoa5.d.ts → rubric-predictive-validity-DiwT8PbL.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-2SgHxoa5.d.ts.map → rubric-predictive-validity-DiwT8PbL.d.ts.map} +1 -1
- package/dist/{run-record-D1Hw-Jml.d.ts → run-record-CaxbLq6Y.d.ts} +5 -11
- package/dist/run-record-CaxbLq6Y.d.ts.map +1 -0
- package/dist/{run-record-ES07wfz9.js → run-record-CysKLB93.js} +3 -5
- package/dist/run-record-CysKLB93.js.map +1 -0
- package/dist/{run-record-DlTZ_lqe.js → run-record-DhEJgoPz.js} +4 -4
- package/dist/{run-record-DlTZ_lqe.js.map → run-record-DhEJgoPz.js.map} +1 -1
- package/dist/sandbox-harness-BlSOu4LX.d.ts.map +1 -1
- package/dist/{schema-CSf6qWgZ.js → schema-Bt2vLFm3.js} +2 -14
- package/dist/schema-Bt2vLFm3.js.map +1 -0
- package/dist/schema-C-ziyJD7.d.ts.map +1 -1
- package/dist/{schema-2g7trUwH.d.ts → schema-D6EVKzTB.d.ts} +4 -5
- package/dist/schema-D6EVKzTB.d.ts.map +1 -0
- package/dist/{schema-BuiujZ-g.js → schema-DxcKTzbi.js} +2 -2
- package/dist/schema-DxcKTzbi.js.map +1 -0
- package/dist/search/index.d.ts +39 -53
- package/dist/search/index.d.ts.map +1 -1
- package/dist/search/index.js +4 -4
- package/dist/{search-kernel-CpcKH5Ih.d.ts → search-kernel-DxqjBxA-.d.ts} +18 -31
- package/dist/search-kernel-DxqjBxA-.d.ts.map +1 -0
- package/dist/{search-ledger-d7V8o9Lk.js → search-ledger-TUvB5gWH.js} +189 -199
- package/dist/search-ledger-TUvB5gWH.js.map +1 -0
- package/dist/{search-ledger-file-BqOo-bVo.js → search-ledger-file-UKCO2r8a.js} +4 -4
- package/dist/search-ledger-file-UKCO2r8a.js.map +1 -0
- package/dist/search-ledger-types-p2xKfiDJ.d.ts.map +1 -1
- package/dist/{search-shipper-er4pK5_Y.js → search-shipper-Cw2z3-td.js} +4 -4
- package/dist/search-shipper-Cw2z3-td.js.map +1 -0
- package/dist/{search-summary-CBvz3u_S.js → search-summary-DQIvJsSB.js} +5 -5
- package/dist/{search-summary-CBvz3u_S.js.map → search-summary-DQIvJsSB.js.map} +1 -1
- package/dist/{sequential-CzK5DarL.js → sequential-BVv1sjsw.js} +2 -32
- package/dist/{sequential-CzK5DarL.js.map → sequential-BVv1sjsw.js.map} +1 -1
- package/dist/{sequential-D-1PPCmM.js → sequential-CimQrbjv.js} +5 -5
- package/dist/sequential-CimQrbjv.js.map +1 -0
- package/dist/{sequential-BhsrMupG.d.ts → sequential-KwGspfeR.d.ts} +2 -9
- package/dist/{sequential-BhsrMupG.d.ts.map → sequential-KwGspfeR.d.ts.map} +1 -1
- package/dist/{sequential-eprocess-D1jKoihe.js → sequential-eprocess-DodI6RoF.js} +2 -2
- package/dist/{sequential-eprocess-D1jKoihe.js.map → sequential-eprocess-DodI6RoF.js.map} +1 -1
- package/dist/{server-DbcC7xMl.js → server-DoVbqqtJ.js} +101 -101
- package/dist/server-DoVbqqtJ.js.map +1 -0
- package/dist/{single-run-lock-fVu3MZWH.js → single-run-lock-U58yf8N6.js} +2 -2
- package/dist/{single-run-lock-fVu3MZWH.js.map → single-run-lock-U58yf8N6.js.map} +1 -1
- package/dist/{skillopt-optimization-method-CCtzralq.js → skillopt-optimization-method-kyL94-ri.js} +428 -91
- package/dist/skillopt-optimization-method-kyL94-ri.js.map +1 -0
- package/dist/{spans-D9_ehFuk.d.ts → spans-CVngGij3.d.ts} +2 -2
- package/dist/{spans-D9_ehFuk.d.ts.map → spans-CVngGij3.d.ts.map} +1 -1
- package/dist/{statistical-heldout-1Gbmex7v.js → statistical-heldout-Bksv92Th.js} +3 -3
- package/dist/statistical-heldout-Bksv92Th.js.map +1 -0
- package/dist/{statistical-heldout-BeR4bOdg.d.ts → statistical-heldout-C9AbgREN.d.ts} +8 -67
- package/dist/statistical-heldout-C9AbgREN.d.ts.map +1 -0
- package/dist/{steps-Bg5-KVx1.d.ts → steps-DlepJPwD.d.ts} +2 -56
- package/dist/steps-DlepJPwD.d.ts.map +1 -0
- package/dist/{store-C4avFjIi.d.ts → store-B3nWRy3B.d.ts} +2 -2
- package/dist/{store-C4avFjIi.d.ts.map → store-B3nWRy3B.d.ts.map} +1 -1
- package/dist/{store-otlp-DtB4Slsr.js → store-otlp-BteDAbHK.js} +135 -132
- package/dist/store-otlp-BteDAbHK.js.map +1 -0
- package/dist/{store-tool-spans-fZIigrpP.js → store-tool-spans-D5l_OfTJ.js} +9 -9
- package/dist/store-tool-spans-D5l_OfTJ.js.map +1 -0
- package/dist/{store-tool-spans-DR1SrCLK.d.ts → store-tool-spans-DZS0HWVf.d.ts} +157 -165
- package/dist/store-tool-spans-DZS0HWVf.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts +4 -11
- package/dist/storyboard/index.d.ts.map +1 -1
- package/dist/storyboard/index.js +21 -19
- package/dist/storyboard/index.js.map +1 -1
- package/dist/{summary-report-DtdRucM0.js → summary-report-BsXJoYX5.js} +588 -598
- package/dist/summary-report-BsXJoYX5.js.map +1 -0
- package/dist/{summary-report-C-CtItCN.d.ts → summary-report-DM7tlml1.d.ts} +16 -20
- package/dist/summary-report-DM7tlml1.d.ts.map +1 -0
- package/dist/supervisor-run/index.d.ts +189 -153
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +58 -450
- package/dist/supervisor-run/index.js.map +1 -1
- package/dist/{systemone-CpTLAlqb.d.ts → systemone-Bp_Ac0p1.d.ts} +6 -6
- package/dist/{systemone-CpTLAlqb.d.ts.map → systemone-Bp_Ac0p1.d.ts.map} +1 -1
- package/dist/systemone-protocol.js +1 -1
- package/dist/{systemone-review-B_yo1xsd.d.ts → systemone-review-CwE6JGIp.d.ts} +2 -2
- package/dist/{systemone-review-B_yo1xsd.d.ts.map → systemone-review-CwE6JGIp.d.ts.map} +1 -1
- package/dist/{systemone-review-BZWfrtq1.js → systemone-review-DzdOJ8I6.js} +3 -3
- package/dist/{systemone-review-BZWfrtq1.js.map → systemone-review-DzdOJ8I6.js.map} +1 -1
- package/dist/systemone.d.ts +5 -5
- package/dist/systemone.js +5 -5
- package/dist/systemone.js.map +1 -1
- package/dist/{task-failure-attributes-DngaLdE4.js → task-failure-attributes-BMsblaDh.js} +3 -3
- package/dist/task-failure-attributes-BMsblaDh.js.map +1 -0
- package/dist/{terminal-record-BHocwFw1.js → terminal-record-78g6E4vh.js} +17 -13
- package/dist/terminal-record-78g6E4vh.js.map +1 -0
- package/dist/{tool-groups-mcxOXVlb.d.ts → tool-groups-DyNz2JZ_.d.ts} +3 -3
- package/dist/tool-groups-DyNz2JZ_.d.ts.map +1 -0
- package/dist/{tool-waste-Wi4g6Ekb.d.ts → tool-waste-CkjzWmeY.d.ts} +53 -56
- package/dist/tool-waste-CkjzWmeY.d.ts.map +1 -0
- package/dist/{tool-waste-D52PqU9Q.js → tool-waste-DLsjS_p-.js} +5 -5
- package/dist/{tool-waste-D52PqU9Q.js.map → tool-waste-DLsjS_p-.js.map} +1 -1
- package/dist/{trace-questions-DG7nqLQ7.d.ts → trace-questions-xxwgsuMi.d.ts} +5 -10
- package/dist/{trace-questions-DG7nqLQ7.d.ts.map → trace-questions-xxwgsuMi.d.ts.map} +1 -1
- package/dist/trace-repair/index.d.ts +15 -212
- package/dist/trace-repair/index.d.ts.map +1 -1
- package/dist/trace-repair/index.js +865 -987
- package/dist/trace-repair/index.js.map +1 -1
- package/dist/traces.d.ts +76 -86
- package/dist/traces.d.ts.map +1 -1
- package/dist/traces.js +13 -13
- package/dist/traces.js.map +1 -1
- package/dist/trajectory-replay/index.d.ts +3 -117
- package/dist/trajectory-replay/index.d.ts.map +1 -1
- package/dist/trajectory-replay/index.js +98 -56
- package/dist/trajectory-replay/index.js.map +1 -1
- package/dist/{tree-rXLW3M_4.js → tree-BGRglkKd.js} +2355 -2414
- package/dist/tree-BGRglkKd.js.map +1 -0
- package/dist/{types-B3jzCp0p.js → types-BXgASBCL.js} +2 -2
- package/dist/{types-B3jzCp0p.js.map → types-BXgASBCL.js.map} +1 -1
- package/dist/{types-oKRhQAde.d.ts → types-CW8ewATZ.d.ts} +5 -114
- package/dist/types-CW8ewATZ.d.ts.map +1 -0
- package/dist/{types-BqkVmHfv.d.ts → types-CmXY0PQf.d.ts} +2 -4
- package/dist/types-CmXY0PQf.d.ts.map +1 -0
- package/dist/{types-DBZ7keha.d.ts → types-D-mgRzlh.d.ts} +9 -72
- package/dist/types-D-mgRzlh.d.ts.map +1 -0
- package/dist/{types-B4C5nFw6.d.ts → types-D8GhJoHk.d.ts} +79 -75
- package/dist/types-D8GhJoHk.d.ts.map +1 -0
- package/dist/{types-IXD2HJzu.js → types-DqFMNTgK.js} +2 -4
- package/dist/types-DqFMNTgK.js.map +1 -0
- package/dist/{types-CfZMIzB_.d.ts → types-DtMbFw6U.d.ts} +3 -3
- package/dist/{types-CfZMIzB_.d.ts.map → types-DtMbFw6U.d.ts.map} +1 -1
- package/dist/{verdict-DhPoXApk.js → verdict-BGQmxQ8A.js} +3 -3
- package/dist/verdict-BGQmxQ8A.js.map +1 -0
- package/dist/{verdict-E4eRNf7-.d.ts → verdict-C24NELtN.d.ts} +2 -15
- package/dist/{verdict-E4eRNf7-.d.ts.map → verdict-C24NELtN.d.ts.map} +1 -1
- package/dist/{verdict-cache-BQEsVy9n.js → verdict-cache-CL_7tePD.js} +2 -2
- package/dist/verdict-cache-CL_7tePD.js.map +1 -0
- package/dist/wire/index.d.ts +32 -411
- package/dist/wire/index.d.ts.map +1 -1
- package/dist/wire/index.js +2 -2
- package/docs/changes/jev-evidence-review.md +1 -1
- package/docs/changes/jev-product-adoption.md +1 -1
- package/docs/changes/jev-recorded-evidence.md +1 -1
- package/docs/design/mlbenchmarks-book-review.md +1 -1
- package/docs/design/mlbenchmarks-review/probes.mts +23 -31
- package/docs/design/statistics-decisions.md +17 -17
- package/docs/design.md +1 -1
- package/docs/evaluation-integrity.md +1 -1
- package/docs/experiment.md +1 -1
- package/docs/proofs/campaign-attempt-retention-20261001/README.md +2 -2
- package/docs/public-api.md +138 -529
- package/docs/rollout.md +1 -1
- package/docs/trace-repair-admission.md +1 -1
- package/docs/wire-protocol.md +1 -1
- package/package.json +1 -1
- package/dist/active-curriculum-B_Lc9kfO.js +0 -61
- package/dist/active-curriculum-B_Lc9kfO.js.map +0 -1
- package/dist/agent-profile-CIA2Atnt.d.ts.map +0 -1
- package/dist/agent-profile-cell-BNp-Js9F.d.ts.map +0 -1
- package/dist/agent-profile-cell-BYc0bxsL.js.map +0 -1
- package/dist/agreement-irr-DsTURxdr.js.map +0 -1
- package/dist/analyze-runs-3xpby3p2.js.map +0 -1
- package/dist/analyze-runs-CSDAh8A7.d.ts.map +0 -1
- package/dist/ast-DJ6Bc7ab.js +0 -558
- package/dist/ast-DJ6Bc7ab.js.map +0 -1
- package/dist/ast-hI-vjW6J.d.ts +0 -457
- package/dist/ast-hI-vjW6J.d.ts.map +0 -1
- package/dist/baseline-Dm8966du.js +0 -259
- package/dist/baseline-Dm8966du.js.map +0 -1
- package/dist/benchmark-BywPeAUo.js +0 -562
- package/dist/benchmark-BywPeAUo.js.map +0 -1
- package/dist/benchmark-command-CByl9dxG.js.map +0 -1
- package/dist/campaign-BQ-Jujuj.js.map +0 -1
- package/dist/chat-json-call-BrKsyspg.js.map +0 -1
- package/dist/claim-BXcZw-jo.js.map +0 -1
- package/dist/claude-jsonl-dZir12m6.js +0 -86
- package/dist/claude-jsonl-dZir12m6.js.map +0 -1
- package/dist/client-BzL9niqi.d.ts.map +0 -1
- package/dist/cost-ledger-CO7UyEcQ.d.ts.map +0 -1
- package/dist/cost-ledger-D2FnJPBF.js.map +0 -1
- package/dist/counterfactual-CPaWN99-.d.ts +0 -60
- package/dist/counterfactual-CPaWN99-.d.ts.map +0 -1
- package/dist/counterfactual-DtMWvn7D.js +0 -98
- package/dist/counterfactual-DtMWvn7D.js.map +0 -1
- package/dist/default-registry-BrWh62zf.d.ts.map +0 -1
- package/dist/default-registry-Cn1LQ2Ve.js.map +0 -1
- package/dist/define-agent-eval-CEWkX5wZ.d.ts.map +0 -1
- package/dist/define-agent-eval-Ch3MWX6k.js.map +0 -1
- package/dist/dspy-rlm-engine-D-KX_e64.js.map +0 -1
- package/dist/effect-sizes-DiH8MGOH.js +0 -82
- package/dist/effect-sizes-DiH8MGOH.js.map +0 -1
- package/dist/emitter-HcCmFQl_.d.ts.map +0 -1
- package/dist/engine-C_zNWfM8.d.ts.map +0 -1
- package/dist/equal-terms-Blx9Zk5I.js.map +0 -1
- package/dist/evaluation-Cn2dRZIz.d.ts.map +0 -1
- package/dist/exporters-X4xwla8A.js.map +0 -1
- package/dist/external-optimizer-process-nnsY5D_4.js.map +0 -1
- package/dist/external-optimizer-subprocess-DcerlZSG.js.map +0 -1
- package/dist/failure-cluster-CbgEDcFJ.d.ts.map +0 -1
- package/dist/feedback-trajectory-DkaDj0dj.d.ts.map +0 -1
- package/dist/hf-dataset-BR2yhuNn.js.map +0 -1
- package/dist/index-CPJJr98o.d.ts.map +0 -1
- package/dist/index-Ch9VxCpK.d.ts.map +0 -1
- package/dist/index-DHKEAv3s.d.ts.map +0 -1
- package/dist/index-DlH0Y9gI.d.ts.map +0 -1
- package/dist/index-NvGBiO8H.d.ts.map +0 -1
- package/dist/integrity-BV1ZDf42.js.map +0 -1
- package/dist/journal-DvKCvjty.js.map +0 -1
- package/dist/judge-gate-B1qYnkpi.js.map +0 -1
- package/dist/judge-gate-DNcUrRFA.d.ts.map +0 -1
- package/dist/kind-factory-CbZ-6tvu.js.map +0 -1
- package/dist/ledger-core-B2hahzrt.js.map +0 -1
- package/dist/llm-client-DQveoxSM.js.map +0 -1
- package/dist/llm-judge-PXV7Xljl.js.map +0 -1
- package/dist/matrix-CHMiebFx.d.ts.map +0 -1
- package/dist/matrix-_BV9-Lk5.js.map +0 -1
- package/dist/mint-DCGoBYhB.d.ts +0 -155
- package/dist/mint-DCGoBYhB.d.ts.map +0 -1
- package/dist/normal-DsJZtrJc.js +0 -25
- package/dist/normal-DsJZtrJc.js.map +0 -1
- package/dist/paired-arms-DzKn0TfY.js.map +0 -1
- package/dist/paired-binary-Q36bhn2R.js.map +0 -1
- package/dist/paired-promotion-decision-DPKz8ZnF.d.ts.map +0 -1
- package/dist/paired-tests-jbwPdSz5.js.map +0 -1
- package/dist/prime-protocol-CEE4BR9N.d.ts.map +0 -1
- package/dist/prime-protocol-DcAK9pC8.js.map +0 -1
- package/dist/produced-state-ZCtDBxAE.js.map +0 -1
- package/dist/proposal-findings-BZiMTzAF.js.map +0 -1
- package/dist/query-B1D1qp2Z.js.map +0 -1
- package/dist/redact-Cveoq6fJ.d.ts.map +0 -1
- package/dist/redact-Dk1ZdJEm.js.map +0 -1
- package/dist/release-confidence-BK2QCB5M.d.ts.map +0 -1
- package/dist/release-confidence-Dbe9AS_X.js.map +0 -1
- package/dist/report-command-y8H6RSX8.js.map +0 -1
- package/dist/reward-hacking-DHA7c1l8.d.ts.map +0 -1
- package/dist/rollout-D9bKKBnZ.js.map +0 -1
- package/dist/run-record-D1Hw-Jml.d.ts.map +0 -1
- package/dist/run-record-ES07wfz9.js.map +0 -1
- package/dist/schema-2g7trUwH.d.ts.map +0 -1
- package/dist/schema-BuiujZ-g.js.map +0 -1
- package/dist/schema-CSf6qWgZ.js.map +0 -1
- package/dist/search-kernel-BztlERLv.js.map +0 -1
- package/dist/search-kernel-CpcKH5Ih.d.ts.map +0 -1
- package/dist/search-ledger-d7V8o9Lk.js.map +0 -1
- package/dist/search-ledger-file-BqOo-bVo.js.map +0 -1
- package/dist/search-shipper-er4pK5_Y.js.map +0 -1
- package/dist/semantic-concept-judge-zQBG9hHU.js.map +0 -1
- package/dist/sequential-D-1PPCmM.js.map +0 -1
- package/dist/server-DbcC7xMl.js.map +0 -1
- package/dist/skillopt-optimization-method-CCtzralq.js.map +0 -1
- package/dist/statistical-heldout-1Gbmex7v.js.map +0 -1
- package/dist/statistical-heldout-BeR4bOdg.d.ts.map +0 -1
- package/dist/steps-Bg5-KVx1.d.ts.map +0 -1
- package/dist/store-otlp-DtB4Slsr.js.map +0 -1
- package/dist/store-tool-spans-DR1SrCLK.d.ts.map +0 -1
- package/dist/store-tool-spans-fZIigrpP.js.map +0 -1
- package/dist/student-t-BQeKJx2N.js +0 -39
- package/dist/student-t-BQeKJx2N.js.map +0 -1
- package/dist/summary-report-C-CtItCN.d.ts.map +0 -1
- package/dist/summary-report-DtdRucM0.js.map +0 -1
- package/dist/task-failure-attributes-DngaLdE4.js.map +0 -1
- package/dist/terminal-record-BHocwFw1.js.map +0 -1
- package/dist/tool-groups-mcxOXVlb.d.ts.map +0 -1
- package/dist/tool-waste-Wi4g6Ekb.d.ts.map +0 -1
- package/dist/trajectory-Y1fmdlw7.d.ts +0 -24
- package/dist/trajectory-Y1fmdlw7.d.ts.map +0 -1
- package/dist/tree-rXLW3M_4.js.map +0 -1
- package/dist/types-B4C5nFw6.d.ts.map +0 -1
- package/dist/types-BqkVmHfv.d.ts.map +0 -1
- package/dist/types-DBZ7keha.d.ts.map +0 -1
- package/dist/types-IXD2HJzu.js.map +0 -1
- package/dist/types-oKRhQAde.d.ts.map +0 -1
- package/dist/usage-receipt-Cx3xNeyB.js +0 -76
- package/dist/usage-receipt-Cx3xNeyB.js.map +0 -1
- package/dist/verdict-DhPoXApk.js.map +0 -1
- package/dist/verdict-cache-BQEsVy9n.js.map +0 -1
- package/docs/design/self-improvement-evidence-audit.md +0 -263
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,90 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval-
|
|
|
4
4
|
|
|
5
5
|
---
|
|
6
6
|
|
|
7
|
+
## [0.217.1] — 2026-10-21
|
|
8
|
+
|
|
9
|
+
0.217.0 was tagged but never published: its package fixture check failed. 0.217.1 is the first published release of the 0.217 changes; the breaking changes listed under 0.217.0 below apply to it.
|
|
10
|
+
|
|
11
|
+
- fix(release): follow the rl function rename in the package fixtures
|
|
12
|
+
|
|
13
|
+
## [0.217.0] — 2026-10-21
|
|
14
|
+
|
|
15
|
+
**Breaking (0.x minor).**
|
|
16
|
+
|
|
17
|
+
- 442 value exports no `tangle-network` repository imports are no longer published (1,663 → 1,221 across subpaths; the list is the diff of `docs/public-api.md` in #1039). Consumer sweep: all 105 non-archived org repos on 2026-10-21.
|
|
18
|
+
- Published names renamed so one name means one concept (#1040): `/multishot` `JudgeConfig` → `MultishotJudgeConfig`; `/wire` `FeedbackTrajectory`, `TraceEvent`, `FailureMode`, `RubricDimension` → `WireFeedbackTrajectory`, `WireTraceEvent`, `WireFailureMode`, `WireRubricDimension`; `/systemone` `Evaluator` → `CostedEvaluator`; `/rl` `toSftRows` → `toSftRowsWithLookups`, `paretoFrontier` → `paretoCostScoreFrontier`; plus type-only renames in `/fuzz`, `/experiment`, `/traces`, `/search`, `/trace-repair`, `/campaign`.
|
|
19
|
+
- Consumer repos that import a renamed name: gtm-agent, tax-agent, super-agent, legal-agent, creative-agent. Their rename PRs are linked from the GitHub release notes.
|
|
20
|
+
|
|
21
|
+
- refactor!: follow the rl function rename in the example, contract test and census
|
|
22
|
+
- test: follow the renamed types and functions
|
|
23
|
+
- chore: remove the one-off prune tooling
|
|
24
|
+
- chore: regenerate the API census and re-pin the analyst digest
|
|
25
|
+
- chore(hooks): chain the global ratchets entry point, not loc-ratchet alone (#1038)
|
|
26
|
+
- test: delete the tests that only covered the removed names
|
|
27
|
+
- refactor!: one name per concept across published types and functions
|
|
28
|
+
- refactor!: keep the names other modules import, delete the pruned tests
|
|
29
|
+
- refactor!: stop publishing names no tangle-network repo imports, and delete what that orphaned
|
|
30
|
+
- wip: prune tooling v2
|
|
31
|
+
- wip: prune tooling
|
|
32
|
+
- chore: regenerate the API census and re-pin the analyst digest
|
|
33
|
+
- refactor: import ContinuousAgreement from its owner, not a forwarding file
|
|
34
|
+
- test: type the search matcher inline
|
|
35
|
+
- refactor: delete the run-record store class, eval tools, cached judge and tool-span gate nothing runs
|
|
36
|
+
- refactor: delete four scripts nothing runs or cites
|
|
37
|
+
- chore: drop the waiver for the deleted TokenCounter
|
|
38
|
+
- refactor: delete exports and declarations nothing imports
|
|
39
|
+
- chore(analyst): re-pin the implementation digest after the internal renames
|
|
40
|
+
- refactor: rename the internal side of three more name collisions
|
|
41
|
+
- docs(search-policy): name AIDE's search_policy without a file path
|
|
42
|
+
- chore(analyst): re-pin the current benchmark implementation digest
|
|
43
|
+
- docs: regenerate the public API census
|
|
44
|
+
- fix: drop unused isRecord imports; correct stale line references in two docs
|
|
45
|
+
- refactor: one owner per duplicated name; rename what differs
|
|
46
|
+
- adopt-stack merges after the required approval lands, instead of refusing in the first second (#1035)
|
|
47
|
+
- docs: one root AGENTS.md map; CLAUDE.md imports it
|
|
48
|
+
- docs: point stale doc references at current code; archive superseded records
|
|
49
|
+
- adopt-stack runs the caller's own sync step after pinning (#1033)
|
|
50
|
+
- One reusable workflow adopts each new Tangle stack release, peer ranges included (#1032)
|
|
51
|
+
|
|
52
|
+
## [0.216.1] — 2026-10-21
|
|
53
|
+
|
|
54
|
+
- fix(experiment): declare the diagnostic target outcome direction
|
|
55
|
+
- fix(experiment): record current probe refusals and reuse historical receipts
|
|
56
|
+
- fix(experiment): migrate the documented AST probe to its owner
|
|
57
|
+
- refactor(campaign): own optimization method contracts beside execution
|
|
58
|
+
- refactor(search): separate meta-search rendering and share label formatting
|
|
59
|
+
- refactor(control): separate execution contracts and evaluation policy
|
|
60
|
+
- refactor(campaign): share cross-surface evidence projections
|
|
61
|
+
- refactor(supervisor): isolate runtime source coverage and spending
|
|
62
|
+
- refactor(experiment): put validation rules in their domain owners
|
|
63
|
+
- refactor(supervisor): separate patch statistics and run rollups
|
|
64
|
+
- refactor(campaign): isolate optimization surface execution state
|
|
65
|
+
- refactor(traces): separate buffered OTLP decoding and error clusters
|
|
66
|
+
- refactor: share redaction detectors and remove forwarding constructor
|
|
67
|
+
- refactor: separate public API types and preserve runtime order
|
|
68
|
+
- refactor(gepa): use the canonical seeded random generator
|
|
69
|
+
- refactor(analysis): share arithmetic mean with domain empty fallback
|
|
70
|
+
- refactor(tb): call the shared test-suite reader directly
|
|
71
|
+
- refactor(reporting): reuse the canonical arithmetic mean
|
|
72
|
+
- docs(search): preserve adaptive calibration declines in revision checks
|
|
73
|
+
- docs(search): declare minimum effect for adaptive held-out proof
|
|
74
|
+
- fix(tb-gated-stop): remove unused facade import after extraction
|
|
75
|
+
- refactor: share measured comparison record assembly
|
|
76
|
+
- refactor: share TerminalBench test suite traversal
|
|
77
|
+
- refactor: separate measured comparison responsibilities
|
|
78
|
+
- refactor: separate code agent intake responsibilities
|
|
79
|
+
- refactor: separate llm client responsibilities
|
|
80
|
+
- refactor: separate gepa analyst campaign responsibilities
|
|
81
|
+
- refactor: separate landscape responsibilities
|
|
82
|
+
- refactor: separate edit credit responsibilities
|
|
83
|
+
- refactor: separate analyze runs responsibilities
|
|
84
|
+
- refactor: separate tb gated stop responsibilities
|
|
85
|
+
- refactor: separate summary report responsibilities
|
|
86
|
+
- refactor: separate optimizer model proxy responsibilities
|
|
87
|
+
- refactor: separate cost ledger responsibilities
|
|
88
|
+
- refactor: separate skill manifold responsibilities
|
|
89
|
+
- refactor: separate search runtime responsibilities
|
|
90
|
+
|
|
7
91
|
## [0.216.0] — 2026-10-20
|
|
8
92
|
|
|
9
93
|
- factory: The pre-commit hook runs where no global hooks path is set (#1003)
|
|
@@ -222,7 +306,7 @@ Prepared but never published; 0.210.0 is the first release that carries these ch
|
|
|
222
306
|
- Search lenses (`/search`, search-tree-design §12): `tree`, `operatorYield`, `front` and `taskMatrix`, each a pure function of `SearchState` returning JSON for a view plus one named signal a `SearchPolicy` can read. `agent-eval search show` gains `--tree`/`--operator-yield`/`--front`/`--task-matrix` to print each lens's text form.
|
|
223
307
|
`operatorYield` groups edges by operator and reports outcome counts and improvement-per-known-dollar yield (via `searchPosterior`, excluding any node with an unknown-cost cell rather than flooring it). `front` is the Pareto frontier over score and known cost, reusing the existing `paretoFrontier` primitive, with room for caller-declared extra axes. `taskMatrix` clusters nodes and units by single-linkage clustering with a data-derived cutoff and reports each unit cluster's specialist gain (best node's mean minus the cluster mean).
|
|
224
308
|
`incumbentWithOperatorBandit({ seed, fixedWeights? })` (`/campaign`) is a hill-climb `SearchPolicy` that draws its expansion operator from `operatorYield`'s signal once an operator has 6 or more measured outcomes (falling back to a fixed prior per operator until then, and adding a measured yield to that same prior rather than replacing it, so the two stay on comparable scales); `incumbent` itself is unchanged.
|
|
225
|
-
`scripts/generate-synthetic-search-ledger.ts` and `scripts/import-vb-climb.ts` (the latter through the GEPA importer) produce real ledgers these lenses are proved against;
|
|
309
|
+
`scripts/generate-synthetic-search-ledger.ts` and `scripts/import-vb-climb.ts` (the latter through the GEPA importer) produce real ledgers these lenses are proved against; the operator-bandit proof script (since removed) is a real-run (no-mock) proof of the bandit's draw distribution against the lens's own signal.
|
|
226
310
|
- `metaSearch(searches)` (`/search`) treats each search as a node whose genome is its policy configuration and whose score is its claim's held-out lift per known dollar; an unscored search carries its reason, never 0. Searches form a forest by derivation and containment.
|
|
227
311
|
`runNestedSearch` runs an outer search whose cells are inner searches on the same kernel; `scripts/search-meta-sim.ts` runs one on the simulator, and `agent-eval search show --meta [--json]` prints the lens.
|
|
228
312
|
- `editCredit(state, { readArtifact })` (`/search`) cuts each lineage edge's parent-to-child diff into paragraph edits with stable ids and credits an edit on the clean lineage steps where it alone changed, pairing the carrying side against the lacking side on shared units through `pairedDeltaTest`. It reports linked edits, interacting pairs and inline skill candidates, with the signal `reusableHunks`; `agent-eval search show --edit-credit [--json]` prints it.
|
|
@@ -452,7 +536,7 @@ Prepared but never published; 0.210.0 is the first release that carries these ch
|
|
|
452
536
|
Each question is one `runTraceAnalyst` call, so a finding still needs its minimum distinct citations (default 2), each resolvable in the store.
|
|
453
537
|
The verifier reads the claim through `citedSpansOnly`, a store that exposes exactly the cited spans and refuses every wider read; a finding it does not support is returned with `verified: false`.
|
|
454
538
|
One failed question never stops the others; its failure is its outcome.
|
|
455
|
-
It is the question runner of the agent-runtime continuation panel (discovery
|
|
539
|
+
It is the question runner of the agent-runtime continuation panel (discovery discovery note 38, section 5), and `traces ask` can call the same code.
|
|
456
540
|
|
|
457
541
|
### Changed
|
|
458
542
|
|
|
@@ -513,7 +597,7 @@ Prepared but never published; 0.210.0 is the first release that carries these ch
|
|
|
513
597
|
- `/ledger-core` `FileLedgerJournal` can cache a projector snapshot at its trusted head, so opening a large journal only does its expensive work — the codec's own row parsing and the domain projector's `apply` — on the tail after the cache, not the whole file.
|
|
514
598
|
A codec opts in with `snapshotProjection: { serialize, restore }`; `projectorSnapshot: { everyEntries }` on `FileLedgerJournalOptions` refreshes the cache every that many entries past the last one, on a pinning append or an explicit `pinTrustedHead()`.
|
|
515
599
|
Opening still verifies the generic hash chain of every row up to the cached checkpoint — self-consistency and chain linkage, the two checks every row already gets — so tampering anywhere before the checkpoint is refused exactly as a full replay would refuse it; only the checkpoint row and the true tail go through the codec.
|
|
516
|
-
Measured with
|
|
600
|
+
Measured with a snapshot proof script (since removed) against real hash-chained journals: a snapshot-seeded open always matched a full replay's projection exactly, and used 6.00x less CPU at 300 entries (11.1ms vs 66.7ms, checkpoint at 250), 3.26x less at 1,200 (68.4ms vs 222.8ms, checkpoint at 1,000) and 3.49x less at 4,000 (143.3ms vs 499.5ms, checkpoint at 3,500); the ratio tracks how much of the file the checkpoint covers, since both opens still read every byte — the chain check does not skip any of them (`~/webb/_wt/_mq-notes/E12-proof/`).
|
|
517
601
|
Same-length tampering of a row before the checkpoint is refused, identically to a full replay of the same bytes; tampering after it, in the true tail, is still caught too.
|
|
518
602
|
No existing codec (`search-ledger`, `final-evidence`) opts in yet; this is the ledger-core mechanism only.
|
|
519
603
|
|
|
@@ -549,7 +633,7 @@ Prepared but never published; 0.210.0 is the first release that carries these ch
|
|
|
549
633
|
- `/ledger-core` `FileLedgerJournal` verifies incrementally.
|
|
550
634
|
An instance verifies its whole file on its first call, then each append or replay checks that the head row it verified is still in place and reads, verifies, and projects only the rows past it.
|
|
551
635
|
An append reads one entry's bytes instead of the whole file, so writing a ledger costs O(n) instead of O(n²); `openSearchLedger` and `openFinalEvidenceLedger` gain the same speed.
|
|
552
|
-
At 20,000 entries an append read 571 bytes instead of 10,852,776 (
|
|
636
|
+
At 20,000 entries an append read 571 bytes instead of 10,852,776 (a journal-load script (since removed)).
|
|
553
637
|
A replaced, shrunk, or rewritten head is verified whole again; a same-length rewrite of earlier rows is refused by the next new instance, not by one already open.
|
|
554
638
|
A new entry passes through the codec's own row parse before it is written, so a journal never writes a row it would refuse to read.
|
|
555
639
|
Entries are deep-frozen; code that mutated an entry or event from a replay now throws.
|
|
@@ -624,7 +708,7 @@ Prepared but never published; 0.210.0 is the first release that carries these ch
|
|
|
624
708
|
- **The rule-list redactor is replaced by the redaction core.** Removed `DEFAULT_REDACTION_RULES`, `RedactionRule`, `redactString` and `redactValue` (root and `/traces`), and `/diagnosis` `DIAGNOSIS_SECRET_RULES`, `holdsSecret`, `redactSecrets`, `redactSecretsDeep`, `SECRET_ASSIGNMENT_PATTERN` and their types. `RedactionReport` now has `byDetector` and `findings` instead of `byRule`, and `REDACTION_VERSION` is `2.0.0`. **Migration:** `redactValue(v)` becomes `redact(v).value`, `redactString(s).output` becomes `redactText(s)`, and a custom rule list becomes `knownSecrets` or a case in the corpus. `defaultProviderRedactor` now runs the core, so a credential header keeps its name with a `[REDACTED:credential-key]` value, and `redactedFields` holds JSON Pointers.
|
|
625
709
|
- **`ExperimentTracker` and its git-provenance/persistence machinery are gone.** Removed `ExperimentTracker`, `fileExperimentStore`, `inMemoryExperimentStore`, `Experiment`, and `ExperimentProvenance` (root and `/experiment`). The class had no in-repo, agent-runtime, blueprint-agent, or agent-dev-container caller — a manual experiment becomes a search with `proposer.kind: 'human'` in the upcoming search-tree system. **Migration:** if you called `new ExperimentTracker({ store, provenanceReader })`, replace it with your own store (`create`/`addRep`/`list` become plain reads and writes of whatever you persist) plus `computeExperimentStats` and `improvementVerdict` directly — those two pure functions, `ExperimentRep`, `ExperimentStats`, `ImprovementThresholds`, `ImprovementVerdictResult`, and `ExperimentVerdict` are unchanged and still exported from the package root (blueprint-agent's held-out gate uses them as-is). Provenance capture (`git rev-parse HEAD`, etc.) is no longer built in; shell out yourself if you need it.
|
|
626
710
|
- `contractJudge`, `matchSpan`, and `assertTraceContract`, which the package root never exported.
|
|
627
|
-
- The unit tests in
|
|
711
|
+
- The unit tests in the trace-contracts test; `traces check` over recorded sessions is the proof.
|
|
628
712
|
|
|
629
713
|
## [0.187.2] — 2026-09-24
|
|
630
714
|
|
|
@@ -1180,7 +1264,7 @@ Prepared but never published; 0.210.0 is the first release that carries these ch
|
|
|
1180
1264
|
|
|
1181
1265
|
- Every byte-identical private copy of a shared primitive now routes through one owner. No public export changes and no behavior changes.
|
|
1182
1266
|
- `mulberry32` had five implementations: the exported owner in `src/statistics/random.ts`, named private copies in `src/trajectory-replay/batch.ts` and `src/judge-calibration.ts`, and unnamed inline copies in `src/rl/contamination.ts` (`shuffleOrder`) and `src/fuzz/explorer.ts` (`FuzzExplorer.rng`). The copies used `seed >>> 0` where the owner uses `seed | 0` — the same 32 bits, so the streams are identical (verified over 150,000 draws across 15 seeds). All four are deleted. The replay fix-case sample, the judge-calibration bootstrap, the contamination probe's order-shuffle perturbation, and the fuzz explorer now inherit the owner's refusal of a non-finite seed instead of silently seeding from `NaN >>> 0 === 0`. `src/statistics/random.test.ts` pins the stream with known-answer vectors, so a future edit to the PRNG cannot silently move an already-published statistic.
|
|
1183
|
-
- `toAttributes` and `msToNs` were byte-identical in `src/trace/otel.ts` and
|
|
1267
|
+
- `toAttributes` and `msToNs` were byte-identical in `src/trace/otel.ts` and the removed OTel exporter module. Both exporters emit the same `OtlpSpan`, so two copies of the value-tagging rule meant the same attribute could be typed differently depending on which exporter ran. They are now `toOtlpAttributes` and `msToUnixNano` in the private `src/trace/otlp-encoding.ts`.
|
|
1184
1268
|
- `cryptoRandomId` / `cryptoEventId` / `cryptoId` were three byte-identical id mints in `src/trace/emitter.ts`, `src/llm-client.ts`, and `src/builder-eval/builder-session.ts`. Each carried a `Date.now()`-plus-`Math.random()` fallback that is unreachable on every runtime this package supports (`engines.node >= 20`, where `globalThis.crypto.randomUUID` is always present). One private `newRecordId` in `src/record-id.ts` replaces all three, and the weaker id path is gone.
|
|
1185
1269
|
|
|
1186
1270
|
---
|
|
@@ -1242,7 +1326,7 @@ Prepared but never published; 0.210.0 is the first release that carries these ch
|
|
|
1242
1326
|
- `createChatClient` loses its `router`, `direct-provider`, and `cli-bridge` variants. `custom`, `sandbox-sdk`, and `mock` remain, and `ChatTransport` narrows to those three.
|
|
1243
1327
|
- The root barrel no longer exports `callLlm`, `callLlmJson`, `LlmClient`, `LlmClientOptions`, `assertLlmRoute`, `LlmRouteRequirements`, or `probeLlm`. `assertLlmRoute` and `probeLlm` are deleted outright: the caller holds the endpoint, so the caller owns both the route check and the reachability probe. The canonical contract stays public — `LlmCallRequest`, `LlmCallResult` (including `logprobs`, `toolCalls`, `servedModel`), `LlmMessage`, `LlmUsage`, `costReceiptFromLlm`, `costReceiptFromLlmError`, `maximumChargeForLlmRequest`, `isTransientLlmError`, `stripFencedJson`.
|
|
1244
1328
|
- `createOpenAiCompatibleExecutionOwner` is gone from `/campaign`. Agent Runtime already owns that role with `profileOptimizerModelCall`, which executes one exact `AgentProfile` and reports profile-digest evidence; two owners for one role was the defect. `examples/_shared/openai-compatible-owner.ts` is the caller-side reference implementation, and it is example code, not a published export.
|
|
1245
|
-
-
|
|
1329
|
+
- the multishot router module is deleted with `routerCompletion`, `requireRouterApiKey`, and `defaultRouterBaseUrl`. `runMultishot`, `runMultishotMatrix`, and `runJudge` now require a caller-supplied `MultishotTransport`; `JudgeConfig.transport` is required and `JUDGE_MODEL` is no longer read from the environment. `MultishotToolExecutor` receives `{ transport, signal }` instead of `{ apiKey, baseUrl, signal }`, and the optional `toolTransport` names the leg the built-in delegate tools run on. `estimateRouterCost` is now `estimateMultishotCost` in `src/multishot/cost.ts`.
|
|
1246
1330
|
- `preflightModels` and `assertModelsServed` take `request: ModelEndpointRequest` instead of `baseUrl` and `apiKey`. Agent Eval asks for a `list-models` or a `probe` check and reads the `Response`, so status, the provider's own `error.message`, `budgetExhausted`, and served-model substitution stay exactly as detectable as before.
|
|
1247
1331
|
- `runIntentMatchJudge`, `runSemanticConceptJudge`, `handleJudge`, `dispatchRpc`, and `createApp` take `chat: ChatClient` (plus optional `pricing`) instead of `llm: LlmClientOptions`. `/v1/judge` refuses with `llm_not_configured` (503) when no transport is configured, which replaces the old route assertion.
|
|
1248
1332
|
- The internal OpenAI-compatible client has no default endpoint. `DEFAULT_BASE_URL = 'https://router.tangle.tools/v1'` is deleted and `baseUrl` is required, so a misconfigured caller fails loudly instead of silently billing the public router — the failure `assertLlmRoute` existed to catch, now unrepresentable.
|
|
@@ -1284,7 +1368,7 @@ Prepared but never published; 0.210.0 is the first release that carries these ch
|
|
|
1284
1368
|
|
|
1285
1369
|
### Fixed
|
|
1286
1370
|
|
|
1287
|
-
- Every result-bearing random draw is seeded (#411, criterion 2). Five byte-identical private copies of `mulberry32` lived beside the canonical one, in `meta-eval/rubric-predictive-validity.ts`, `rl/active-curriculum.ts`, `rl/adaptation-eval.ts`, `summary-report.ts`, and `promotion-gate.ts`; four fell back to `Math.random` when the caller passed no seed, so the rubric verdict, the Thompson allocation, the adaptation-curve intervals, and the reported posterior were silently non-reproducible. `meta-eval/correlation-study.ts` called `Math.random` directly inside its bootstrap and offered no seed at all. All six now route through `makeRng`, which derives the seed from the observations when the caller supplies none, so the same input reproduces the same interval; `correlationStudy` gains the `seed` option its sibling already had. `Math.random` references in `src/` fall from 19 to 10, and every remaining one produces an identifier or a retry delay that no reported number reads. The seat-by-seat audit table is in `docs/design/statistics-decisions.md`.
|
|
1371
|
+
- Every result-bearing random draw is seeded (#411, criterion 2). Five byte-identical private copies of `mulberry32` lived beside the canonical one, in `src/meta-eval/rubric-predictive-validity.ts`, `src/rl/active-curriculum.ts`, `src/rl/adaptation-eval.ts`, `summary-report.ts`, and `promotion-gate.ts`; four fell back to `Math.random` when the caller passed no seed, so the rubric verdict, the Thompson allocation, the adaptation-curve intervals, and the reported posterior were silently non-reproducible. `src/meta-eval/correlation-study.ts` called `Math.random` directly inside its bootstrap and offered no seed at all. All six now route through `makeRng`, which derives the seed from the observations when the caller supplies none, so the same input reproduces the same interval; `correlationStudy` gains the `seed` option its sibling already had. `Math.random` references in `src/` fall from 19 to 10, and every remaining one produces an identifier or a retry delay that no reported number reads. The seat-by-seat audit table is in `docs/design/statistics-decisions.md`.
|
|
1288
1372
|
- Two documentation fences imported `@tangle-network/agent-eval/../src/trace-repair`, which cannot resolve; they now import `@tangle-network/agent-eval/trace-repair`. Three fences that are not TypeScript — an object fragment, a call elided as `{ ... }`, and a method sketch — are marked `text`.
|
|
1289
1373
|
|
|
1290
1374
|
### Changed
|
|
@@ -1318,7 +1402,7 @@ Prepared but never published; 0.210.0 is the first release that carries these ch
|
|
|
1318
1402
|
|
|
1319
1403
|
- `runOptimization({ searchLedger })` and `selfImprove({ searchLedger })` record the candidate search into the canonical `SearchLedger` and return a bounded `searchHistory` receipt (#633). The loop emits the plan (slots = generations x populationSize, one candidate-generation operation per generation, one selection operation, one task per designed scenario-replicate cell), one registration per candidate carrying the exact parent surface it mutated, one attempt per scored cell with the cell's own outcome and accounting, one decision per candidate, and the terminal event. `FileSearchLedger` now has a first-party caller in its own package. The terminal event is appended only when canonical replay accounts for the whole planned denominator, so an interrupted or partly unscored search reports the gap instead of claiming a closed search.
|
|
1320
1404
|
- `search-plan-extended`: a rolling search appends candidate slots and operations to an existing plan instead of opening a second ledger. Replay merges the first plan with every extension, the generation invariant continues across rounds, and the planless refusal is unchanged. The planned task denominator stays frozen.
|
|
1321
|
-
- `gepaOptimizationMethod({ searchLedger: { identity } })` records GEPA's own candidate population — its parent graph and per-scenario selection scores — into the same ledger through `recordCandidatePopulationSearch()`. `compareOptimizationMethods({ searchHistoryPolicy: 'require-complete' })` now accepts a first-party method, which is what
|
|
1405
|
+
- `gepaOptimizationMethod({ searchLedger: { identity } })` records GEPA's own candidate population — its parent graph and per-scenario selection scores — into the same ledger through `recordCandidatePopulationSearch()`. `compareOptimizationMethods({ searchHistoryPolicy: 'require-complete' })` now accepts a first-party method, which is what the search-history receipts note promised.
|
|
1322
1406
|
|
|
1323
1407
|
---
|
|
1324
1408
|
|
|
@@ -1382,7 +1466,7 @@ Prepared but never published; 0.210.0 is the first release that carries these ch
|
|
|
1382
1466
|
- `foreign-agent-quickstart` moved to the standard `LLM_BASE_URL`/`LLM_API_KEY`/`LLM_MODEL` variables and declares `expectUsage: 'off'` for its unmetered demo transport.
|
|
1383
1467
|
- Optimizer examples (`self-improve-optimizer`, `compare-optimization-methods`, GSM8K, AppWorld) encode the measured GEPA lessons: `LLM_BASE_URL` is required (no silent `api.openai.com` default), every GEPA engine run pins `reflection_lm_kwargs: { num_retries: 0 }` via the shared `examples/_shared/gepa-reflection.ts`, worker `maxTokens` is env-tunable with a reasoning-model headroom note, and `GEPA_MAX_EVALUATIONS` documents the >= train-partition-size floor.
|
|
1384
1468
|
- Model ids refreshed across examples: retired `gpt-4o`/`gpt-4.1-mini`/`claude-sonnet-4-6` literals replaced with served ids (`deepseek-v4-flash`, `glm-5.3`; RunRecord literals carry the required snapshot suffix).
|
|
1385
|
-
- Removed
|
|
1469
|
+
- Removed the AppWorld halo chat launcher and `halo-chat.sh`: launchers for an external analysis engine that no documented example references. `run-bench.ts` requires `APPWORLD_DIR` explicitly instead of defaulting to a machine-local path.
|
|
1386
1470
|
|
|
1387
1471
|
---
|
|
1388
1472
|
|
|
@@ -1476,14 +1560,14 @@ One repository still names three of them. `starter-foundry` imports `runAssertio
|
|
|
1476
1560
|
|
|
1477
1561
|
### Removed
|
|
1478
1562
|
|
|
1479
|
-
- `decideNextUserTurn`, `DecideNextUserTurnOpts` and `buildDriverSystemPrompt`, with
|
|
1480
|
-
- `ConvergenceTracker` and
|
|
1563
|
+
- `decideNextUserTurn`, `DecideNextUserTurnOpts` and `buildDriverSystemPrompt`, with the driver module and the `user-simulation-driver` example. A role written as a code function cannot be optimized, and a persona driver is an `AgentProfile` on a graph edge, not a packaged function. All five first-party callers now own the role in their own repositories, where the prompt is product data they can change and measure: gtm-agent, legal-agent, insurance-agent, workcomp-agent and creative-agent.
|
|
1564
|
+
- `ConvergenceTracker` and the convergence tracker module. Its only caller was `AgentDriver`, which 0.145.22 deleted, and it never reached the barrel. `analyzeSeries` in `src/series-convergence.ts` is unaffected — it reads drift across runs, not progress within one.
|
|
1481
1565
|
- `PersonaConfig.feedbackPatterns`, the `FeedbackPattern` type, and `PersonaConfig.driverModel`. `feedbackPatterns` told `AgentDriver` which product approvals to reject, and `driverModel` picked its driver model; with the class gone nothing reads either, in this package or in any repository that depends on it. A field that advertises behaviour the package no longer has is worse than no field.
|
|
1482
1566
|
- Nothing else on the public surface changes. `PersonaConfig` and `DriverState` stay: a harness that writes its own driver still describes a persona and a produced state with them.
|
|
1483
1567
|
|
|
1484
1568
|
### Migration
|
|
1485
1569
|
|
|
1486
|
-
A caller that drove a conversation with `decideNextUserTurn` owns two things now: the system prompt that states what its simulated professional demands, and one priced call that turns the transcript into the next message. Both stay reachable from this package — `CostLedger.runPaidCall` prices and attributes the call, `maximumChargeForLlmRequest` caps it, `costReceiptFromLlm` records it, and `assertServedModel` (published in 0.145.22) holds the transport to the model id it was asked for. gtm-agent's
|
|
1570
|
+
A caller that drove a conversation with `decideNextUserTurn` owns two things now: the system prompt that states what its simulated professional demands, and one priced call that turns the transcript into the next message. Both stay reachable from this package — `CostLedger.runPaidCall` prices and attributes the call, `maximumChargeForLlmRequest` caps it, `costReceiptFromLlm` records it, and `assertServedModel` (published in 0.145.22) holds the transport to the model id it was asked for. gtm-agent's the eval persona driver is the worked example.
|
|
1487
1571
|
|
|
1488
1572
|
A caller pinned below 0.145.22 cannot import `assertServedModel`. Check what the pinned version did before porting: `decideNextUserTurn` did not call the guard before the 0.145.x line, so a port that omits it there matches the pinned behaviour exactly.
|
|
1489
1573
|
## [0.146.0] — 2026-08-16
|
|
@@ -1759,7 +1843,7 @@ Breaking release: the deep clean. The measured consumer-import census is the com
|
|
|
1759
1843
|
|
|
1760
1844
|
### Changed
|
|
1761
1845
|
|
|
1762
|
-
-
|
|
1846
|
+
- the statistics module is now the `src/statistics/` directory (12 focused modules); every existing import specifier still compiles.
|
|
1763
1847
|
- `runCampaign` internals split into phase modules; its import path and options are unchanged.
|
|
1764
1848
|
- Every verifier lands in `DefaultVerdict`, with `certification` naming what certified the verdict (checker identity, strategy, assumptions, evidence digest).
|
|
1765
1849
|
|
|
@@ -2617,10 +2701,10 @@ threshold and should be re-run on this release.
|
|
|
2617
2701
|
(`outcome.realness.gated`) existed in exactly one function, `rolloutReward`, called from exactly one
|
|
2618
2702
|
place — `mintRolloutRows`. The same derivation, `outcome.holdoutScore ?? outcome.searchScore`, was
|
|
2619
2703
|
hand-rolled at 20 other sites with no gate. Six of those sites feed exported training data: the GRPO
|
|
2620
|
-
default reward and the SFT row metadata (`rl/exporters.ts`), the DPO preference ordering
|
|
2621
|
-
(`rl/preferences.ts`), the probabilistic verifiable-reward fallback (`rl/verifiable-reward.ts`), the
|
|
2622
|
-
corpus `minScore` filter (`rl/corpus.ts`), and the published datasheet's reward statistics
|
|
2623
|
-
(`rl/dataset.ts`). A gamed run therefore trained at its claimed score, and in DPO it became the
|
|
2704
|
+
default reward and the SFT row metadata (`src/rl/exporters.ts`), the DPO preference ordering
|
|
2705
|
+
(`src/rl/preferences.ts`), the probabilistic verifiable-reward fallback (`src/rl/verifiable-reward.ts`), the
|
|
2706
|
+
corpus `minScore` filter (`src/rl/corpus.ts`), and the published datasheet's reward statistics
|
|
2707
|
+
(`src/rl/dataset.ts`). A gamed run therefore trained at its claimed score, and in DPO it became the
|
|
2624
2708
|
*chosen* side of a pair against its honest sibling. Every one of those six now derives its reward
|
|
2625
2709
|
through the gate.
|
|
2626
2710
|
|
|
@@ -2634,7 +2718,7 @@ threshold and should be re-run on this release.
|
|
|
2634
2718
|
allocation need the ungated number, and gating a detector's proxy would make it report "clean" on
|
|
2635
2719
|
precisely the population being gamed — so the fix names the choice rather than removing it.
|
|
2636
2720
|
- A regression test (`src/rollout/reward-invariant.test.ts`) with two halves: a source-level check that
|
|
2637
|
-
the bare derivation appears nowhere outside `rollout/reward.ts`, and a behavioural check that pushes
|
|
2721
|
+
the bare derivation appears nowhere outside `src/rollout/reward.ts`, and a behavioural check that pushes
|
|
2638
2722
|
one gated record with a 0.95 score through mint, GRPO, SFT, DPO, verifiable reward, the dataset
|
|
2639
2723
|
bundle, and the corpus filter, asserting 0 in each. Against the pre-fix tree the source check reports
|
|
2640
2724
|
21 offending lines and 7 of the 10 tests fail.
|
|
@@ -2645,7 +2729,7 @@ threshold and should be re-run on this release.
|
|
|
2645
2729
|
Use `trainingReward` for score derivation or `rolloutRewardFields` when producing a rollout outcome.
|
|
2646
2730
|
- The 14 analysis, reporting, and detection sites that legitimately want the raw number now call
|
|
2647
2731
|
`observedScore` explicitly. Behaviour is unchanged at all 14, including the two sites that
|
|
2648
|
-
deliberately prefer the search split (`rl/active-curriculum.ts`, via the new `ScorePreference`
|
|
2732
|
+
deliberately prefer the search split (`src/rl/active-curriculum.ts`, via the new `ScorePreference`
|
|
2649
2733
|
argument). (`description-length-gate.ts` reads through `runTaskScore`, which 0.127.0 stripped of
|
|
2650
2734
|
its obsolete `raw.score` fallback.)
|
|
2651
2735
|
|
|
@@ -2692,10 +2776,10 @@ unchanged.
|
|
|
2692
2776
|
`HeldOutGate` gets the same treatment: gated runs are dropped from both sides before pairing, with
|
|
2693
2777
|
`evidence.realnessGatedRuns` surfacing how many. Never a silent 0 — a shrunken denominator has to say
|
|
2694
2778
|
by how much.
|
|
2695
|
-
- **Ten remaining hand-rolled derivations routed by classification**: `rl/sim-fidelity.ts` (×2, RAW —
|
|
2779
|
+
- **Ten remaining hand-rolled derivations routed by classification**: `src/rl/sim-fidelity.ts` (×2, RAW —
|
|
2696
2780
|
gating a sim-vs-production divergence measure would report the simulator as more faithful precisely
|
|
2697
|
-
where it is gamed),
|
|
2698
|
-
`eval-trace-store.ts` (GATED, above), `contract/analyze-runs.ts` (×3, RAW), `summary-report.ts` (×4,
|
|
2781
|
+
where it is gamed), the belief-state code-agent corpus module (GATED — its output becomes corpus labels),
|
|
2782
|
+
`eval-trace-store.ts` (GATED, above), `src/contract/analyze-runs.ts` (×3, RAW), `summary-report.ts` (×4,
|
|
2699
2783
|
RAW), `release-confidence.ts` (RAW), `held-out-gate.ts` (RAW, over an already-degated set).
|
|
2700
2784
|
- **`trainingReward` no longer collapses an unscored record to 0.** It returns `reward: null`, matching
|
|
2701
2785
|
the schema's own "a labeled gap, never 0" rule; a gated run still returns 0, because that IS a
|
|
@@ -2775,7 +2859,7 @@ unchanged.
|
|
|
2775
2859
|
|
|
2776
2860
|
### Fixed — unrelated flake encountered on the way
|
|
2777
2861
|
|
|
2778
|
-
- `node:sqlite` is loaded through `createRequire` in `rollout/readers/opencode-sqlite.ts` and its test.
|
|
2862
|
+
- `node:sqlite` is loaded through `createRequire` in `src/rollout/readers/opencode-sqlite.ts` and its test.
|
|
2779
2863
|
esbuild and Vite both rewrite an `import()` of a builtin and strip the `node:` prefix, producing a bogus
|
|
2780
2864
|
`sqlite` package lookup; composing the specifier at runtime did not reliably defeat it, so the failure
|
|
2781
2865
|
moved between workers whenever a test file was added. A require obtained from `createRequire` is not an
|
|
@@ -2833,7 +2917,7 @@ Artifacts that carry run ids without embedded reward state still require explici
|
|
|
2833
2917
|
|
|
2834
2918
|
#### Deliberately NOT gated
|
|
2835
2919
|
|
|
2836
|
-
`rl/reward-hacking.ts` reads the deterministic reward through the new
|
|
2920
|
+
`src/rl/reward-hacking.ts` reads the deterministic reward through the new
|
|
2837
2921
|
`VerifiableRewardExtractionOptions.applyRealnessGate: false`, which preserves its previous behaviour
|
|
2838
2922
|
exactly. Its `judge_drift` and `reward_disagreement` signals measure the GAP between the judge reward
|
|
2839
2923
|
and the deterministic one; a deterministic reward another gate already forced to 0 opens that gap by
|
|
@@ -2844,7 +2928,7 @@ empty options object gates — the opt-out is explicit and greppable.
|
|
|
2844
2928
|
### Known, not fixed here
|
|
2845
2929
|
|
|
2846
2930
|
- `description-length-gate.ts` gives a gated run claiming `score: 1.0` the largest possible improvement
|
|
2847
|
-
to its objective, and `product-benchmark/export.ts` publishes a gated run with `pass: true`. Each
|
|
2931
|
+
to its objective, and `src/product-benchmark/export.ts` publishes a gated run with `pass: true`. Each
|
|
2848
2932
|
site carries a comment naming the hole.
|
|
2849
2933
|
- `extractVerifiableReward(report)` — the `VerificationReport` signature — cannot gate and does not
|
|
2850
2934
|
claim to: `realness` lives on the `RunRecord`, not on the report. Documented on the function.
|
|
@@ -3434,7 +3518,7 @@ Additive (new exports + optional fields only); no consumer bump required.
|
|
|
3434
3518
|
|
|
3435
3519
|
### Added
|
|
3436
3520
|
|
|
3437
|
-
- **FAPO proposer policy** in the campaign proposer family — feeds `runImprovementLoop` / `runCampaign` proposer-driven self-improvement (`gepaProposer` at
|
|
3521
|
+
- **FAPO proposer policy** in the campaign proposer family — feeds `runImprovementLoop` / `runCampaign` proposer-driven self-improvement (`gepaProposer` at the GEPA proposer module).
|
|
3438
3522
|
|
|
3439
3523
|
### Changed
|
|
3440
3524
|
|
|
@@ -3476,8 +3560,8 @@ One clean, canonical version of five generic patterns the fleet kept hand-rollin
|
|
|
3476
3560
|
|
|
3477
3561
|
### Added
|
|
3478
3562
|
|
|
3479
|
-
- **`ExperimentTracker` + `improvementVerdict` + `computeExperimentStats` (root).** Git-provenanced experiment log with N-rep stats (`median` / `mean` / `min` / `max` / `iqr` / `stddev` / `passRate` / `n` + a `stable` flag) and a `KEEP` / `REGRESSION` / `NOISE` / `ITERATE` verdict against a parent. Provenance (`ProvenanceReader`, default `gitProvenanceReader`) and persistence (`ExperimentStore`, with `inMemoryExperimentStore` + `fileExperimentStore`) are injected, so the stats + verdict are pure and unit-testable without a repo or disk. Thresholds (`keepThreshold` / `regressionThreshold` / `iqrUnstableAbove` / `stddevUnstableAbove` / `minRepsForVerdict`) are configurable. Replaces the per-agent
|
|
3480
|
-
- **`EvalTraceStore` + `runScore` (root).** JSONL save / query / compare over the analysis-time `RunRecord` row: `query(filter)`, `getBest(scenarioId)`, and `compareRuns(a, b)` (paired on matched scenarios, best-score-per-scenario). Persistence is injected via `RunRecordBackend` (`inMemoryRunRecordBackend` / `jsonlRunRecordBackend`, which fail loud on a malformed line). Does NOT fork `FileSystemTraceStore` (the rich TraceSchema-v1 span store) — it is the analysis projection beside it. Replaces the hand-rolled
|
|
3563
|
+
- **`ExperimentTracker` + `improvementVerdict` + `computeExperimentStats` (root).** Git-provenanced experiment log with N-rep stats (`median` / `mean` / `min` / `max` / `iqr` / `stddev` / `passRate` / `n` + a `stable` flag) and a `KEEP` / `REGRESSION` / `NOISE` / `ITERATE` verdict against a parent. Provenance (`ProvenanceReader`, default `gitProvenanceReader`) and persistence (`ExperimentStore`, with `inMemoryExperimentStore` + `fileExperimentStore`) are injected, so the stats + verdict are pure and unit-testable without a repo or disk. Thresholds (`keepThreshold` / `regressionThreshold` / `iqrUnstableAbove` / `stddevUnstableAbove` / `minRepsForVerdict`) are configurable. Replaces the per-agent the eval experiment tracker copies (tax / insurance / legal).
|
|
3564
|
+
- **`EvalTraceStore` + `runScore` (root).** JSONL save / query / compare over the analysis-time `RunRecord` row: `query(filter)`, `getBest(scenarioId)`, and `compareRuns(a, b)` (paired on matched scenarios, best-score-per-scenario). Persistence is injected via `RunRecordBackend` (`inMemoryRunRecordBackend` / `jsonlRunRecordBackend`, which fail loud on a malformed line). Does NOT fork `FileSystemTraceStore` (the rich TraceSchema-v1 span store) — it is the analysis projection beside it. Replaces the hand-rolled the eval trace store copies.
|
|
3481
3565
|
- **`CostLedger` + `costForUsage` + `modelPriceKey` (root).** Per-run token + USD accounting folded over the substrate's `resolveModelPricing` / `isModelPriced`, with an explicit `costUnknown` axis so a $0 from an unpriced model is never mistaken for a measured free run. Classifies spend by channel (`agent` / `judge` / `verifier` / …), surfaces `unpricedModels` + `fullyPriced`, and computes `costPerCompletedTask`. Generalizes physim's `costForUsage` / `modelPriceKey` and the tax / gtm / agent-builder copies.
|
|
3482
3566
|
- **`extractUsage` + `extractUsageFromSse` + `extractUsageFromResponse` (`/traces` + root).** Token-usage extraction from a chat-completions response or an SSE stream — OpenAI / Anthropic / camelCase shapes, cache-read tokens, and per-chunk SSE accumulation — returning `null` (not a silent zero) when no usage is present. `captureFetchToRawSink` gains an optional `onUsage` callback that emits the parsed usage off each response (reusing the body it already reads — no extra clone), so a caller folds usage → cost without re-cloning. Replaces the insurance / legal / gtm `raw-capture.ts` copies.
|
|
3483
3567
|
- **`partitionHeldOut` + `assignHeldOutTag` + `hashToUnit` + `fnv1a32` (root).** Deterministic FNV-1a id+seed held-out splitter. `assignHeldOutTag` stamps a single id; `partitionHeldOut` splits a whole id list into disjoint search / holdout sets and fails loud on duplicate ids, empty input, an under-floor holdout (`minHoldout` / `minSearch` significance floor), or an out-of-range `holdoutFraction`. Generalizes agent-builder's `deterministicSplit` and the frontier persona-splitter; complements the existing 3-way benchmark `deterministicSplit` in `/benchmarks`.
|
|
@@ -3628,7 +3712,7 @@ An adversarial review of a real "ship +4.0 lift" decision found it was a **tripl
|
|
|
3628
3712
|
### Fixed / Added
|
|
3629
3713
|
|
|
3630
3714
|
- **No-op guard** (`runImprovementLoop`) — when the winner is byte-identical to the baseline (no candidate beat the training baseline, empty diff), the loop now forces `hold` and skips the meaningless baseline-vs-itself holdout pass, instead of shipping the noise delta.
|
|
3631
|
-
- **Statistical held-out gate** — `defaultProductionGate`'s held-out check is now a **paired bootstrap CI**, not a point estimate. It pairs candidate vs baseline holdout cells by **full `cellId` (`scenario:rep`)** — never averaging reps away — and ships only when the CI lower bound clears `deltaThreshold` (default 0 ⇒ confidently positive). Below `minProductiveRuns` (default 3) paired observations it HOLDS with `few_runs` rather than reading a degenerate interval. (New module `src/campaign/gates/statistical-heldout.ts`; reuses `pairedBootstrap` from `src/statistics
|
|
3715
|
+
- **Statistical held-out gate** — `defaultProductionGate`'s held-out check is now a **paired bootstrap CI**, not a point estimate. It pairs candidate vs baseline holdout cells by **full `cellId` (`scenario:rep`)** — never averaging reps away — and ships only when the CI lower bound clears `deltaThreshold` (default 0 ⇒ confidently positive). Below `minProductiveRuns` (default 3) paired observations it HOLDS with `few_runs` rather than reading a degenerate interval. (New module `src/campaign/gates/statistical-heldout.ts`; reuses `pairedBootstrap` from `src/statistics/`.)
|
|
3632
3716
|
- **Per-dimension regression guard (anti-Goodhart)** — `criticalDimensions` + `regressionTolerance` on `DefaultProductionGateOptions`. The gate HOLDS if any guarded dimension's paired-delta CI lower bound falls below −tolerance, even when the net composite rose. Tolerance auto-scales (0.05 on [0,1], 5 on 0-100) so a default expressed for one scale isn't a silent no-op on the other.
|
|
3633
3717
|
- **Exports** `pairHoldout`, `heldoutSignificance`, `dimensionRegressions`, `detectScale` from `/campaign`.
|
|
3634
3718
|
|
|
@@ -3666,7 +3750,7 @@ Closes the optimizer-completeness gap (#101/#100). `gepaDriver` was reflection-o
|
|
|
3666
3750
|
|
|
3667
3751
|
### Added
|
|
3668
3752
|
|
|
3669
|
-
- **GEPA Pareto frontier + combine-complementary-lessons (#101).** `runOptimization` now accumulates every scored surface as a per-scenario objective vector and recomputes the non-dominated set before each generation, handing it to the driver as `ctx.paretoParents` (new `ParetoParent` type). A surface uniquely best on one hard scenario survives even when its mean composite is lower. `gepaDriver` spends one population slot merging the frontier parents' complementary strengths (toggle via `combineParents`, default on; fires only when the frontier has >1 member). `RunOptimizationResult.paretoFrontier` exposes the final frontier. Dominance is computed by the package-canonical `paretoFrontier` (`src/pareto.ts`) — the parallel
|
|
3753
|
+
- **GEPA Pareto frontier + combine-complementary-lessons (#101).** `runOptimization` now accumulates every scored surface as a per-scenario objective vector and recomputes the non-dominated set before each generation, handing it to the driver as `ctx.paretoParents` (new `ParetoParent` type). A surface uniquely best on one hard scenario survives even when its mean composite is lower. `gepaDriver` spends one population slot merging the frontier parents' complementary strengths (toggle via `combineParents`, default on; fires only when the frontier has >1 member). `RunOptimizationResult.paretoFrontier` exposes the final frontier. Dominance is computed by the package-canonical `paretoFrontier` (`src/pareto.ts`) — the parallel the campaign Pareto module fork has been deleted (one dominance implementation).
|
|
3670
3754
|
- **SkillOpt patch-mode driver + `runSkillOpt` preset (#100)** (Microsoft, arXiv:2605.23904). `skillOptDriver` proposes BOUNDED add/delete/replace patches to one skill document (`applySkillPatch`, `SkillPatch`); `runSkillOpt` is the held-out-gated epoch hill-climb: reflect on TRAIN weaknesses → propose ≤ `editBudget` ops → score on the held-out split → ACCEPT only on STRICT held-out improvement, else buffer the rejected edit; with edit-budget annealing (the "textual learning rate") and a slow-update meta note. The held-out composite is monotonically non-decreasing by construction — a regression can never ship. Proposals reflect on train evidence only (no held-out leakage).
|
|
3671
3755
|
- **`compareDrivers` head-to-head lift benchmark (the forcing function).** Runs N optimizer entries on ONE corpus, scores the baseline + every promoted surface UNIFORMLY on the same held-out scenarios, and reports per-driver lift + paired-bootstrap CI + pairwise "which driver wins" CIs, ranked (cost breaks a lift tie). Ships `gepaReflectionEntry` / `gepaParetoEntry` / `skillOptEntry` to wire the real optimizers. Optimizer quality is now a number with a confidence interval — a driver regression turns a build red instead of going invisible.
|
|
3672
3756
|
- **`campaignMeanComposite` / `campaignBreakdown`** (`score-utils`) — the one definition of "composite of a campaign" + per-scenario/dimension breakdown, now shared by `runOptimization`, `runSkillOpt`, and `compareDrivers` (extracted from `runOptimization`'s private copies).
|
|
@@ -3736,7 +3820,7 @@ Pure additive/doc surface (`expectUsage` defaults to non-breaking `'warn'`). Ful
|
|
|
3736
3820
|
|
|
3737
3821
|
### Architectural context
|
|
3738
3822
|
|
|
3739
|
-
Part of the self-improvement-protocol design (
|
|
3823
|
+
Part of the self-improvement-protocol design (the self-improvement protocol design). This is 0.53.0 of the roadmap that ends at 1.0.0 (profile-versioning + composite driver) and 1.1.0 (empirical-proof publication).
|
|
3740
3824
|
|
|
3741
3825
|
### Notes
|
|
3742
3826
|
|
|
@@ -3758,9 +3842,9 @@ After cloning and reading the actual SkillOpt source (microsoft/SkillOpt) and th
|
|
|
3758
3842
|
|
|
3759
3843
|
### Added
|
|
3760
3844
|
|
|
3761
|
-
-
|
|
3762
|
-
-
|
|
3763
|
-
-
|
|
3845
|
+
- **the driver honesty spec** — primary-source comparison vs GEPA and SkillOpt. Quotes the actual source. Names 13 deviations between 0.51.0's `skillOptDriver` and the real SkillOpt pipeline.
|
|
3846
|
+
- **the Hermes self-improvement audit** — corrected audit after cloning NousResearch/hermes-agent. Hermes has two loops, not one: the 7-day curator (housekeeping) AND a per-turn `background_review` fork that uses **user corrective feedback as a first-class skill-update signal** ("stop doing X", "you always do Y"). Signal source we don't capture today.
|
|
3847
|
+
- **the profile-versioning spec** — architecture for the offline/online drift problem. Symmetric-fork framing (both writers are peers, neither is the authority). `AgentProfileVersion` content-hashing, `ProfileDiff` patch/replace types, 4-way `DriftGateDecision` (ship-substrate / ship-harness / merge / inconclusive), opt-in `driftPolicy` (ignore / reject-on-drift / benchmark-branches), four conflict-resolution cases including semantic-duplication detection. Phase 0 forcing-function experiment specified.
|
|
3764
3848
|
|
|
3765
3849
|
### Where we beat the prior art (now named explicitly)
|
|
3766
3850
|
|
|
@@ -3768,7 +3852,7 @@ Our `defaultProductionGate` uses paired bootstrap CI + Cohen's d + MDE + p-value
|
|
|
3768
3852
|
|
|
3769
3853
|
### Notes
|
|
3770
3854
|
|
|
3771
|
-
`gepaDriver({ constraints })` covers every use case the deleted `skillOptDriver` covered. The single `skillOptDriver` test file was removed; 13 new tests under
|
|
3855
|
+
`gepaDriver({ constraints })` covers every use case the deleted `skillOptDriver` covered. The single `skillOptDriver` test file was removed; 13 new tests under the GEPA driver constraints test cover the absorbed behavior + the unconstrained baseline behavior. Full suite 1444 / 1444 green.
|
|
3772
3856
|
|
|
3773
3857
|
---
|
|
3774
3858
|
|
|
@@ -3859,8 +3943,8 @@ Docs-only patch. No code changes, no behavior changes, no API surface changes vs
|
|
|
3859
3943
|
|
|
3860
3944
|
### Added
|
|
3861
3945
|
|
|
3862
|
-
-
|
|
3863
|
-
-
|
|
3946
|
+
- the OTel-to-hosted bridge module — generic OTel→hosted bridge (`createOtelBridge` / `OtelBridge` / `OtelBridgeOptions`). Stringifies array-valued attributes instead of dropping them.
|
|
3947
|
+
- the contract diff module — `keyForCell` uses `JSON.stringify([scenarioId, rep])` (no separator collisions); `Number.isFinite` coercion on dimension deltas (no NaN propagating to dashboards).
|
|
3864
3948
|
- `examples/hosted-ingest-server/server.ts` — `REFERENCE_RECEIVER_START=1|0` env var as the primary start signal; idempotency cache prunes on read with the wire-spec 24h TTL.
|
|
3865
3949
|
|
|
3866
3950
|
### Changed
|
|
@@ -3870,7 +3954,7 @@ Docs-only patch. No code changes, no behavior changes, no API surface changes vs
|
|
|
3870
3954
|
|
|
3871
3955
|
### Removed
|
|
3872
3956
|
|
|
3873
|
-
-
|
|
3957
|
+
- the traceai adapter — replaced by the OTel-to-hosted bridge module. No back-compat shim.
|
|
3874
3958
|
|
|
3875
3959
|
---
|
|
3876
3960
|
|
|
@@ -3879,8 +3963,8 @@ Docs-only patch. No code changes, no behavior changes, no API surface changes vs
|
|
|
3879
3963
|
### Added
|
|
3880
3964
|
|
|
3881
3965
|
- `src/verdict.ts` — `DefaultVerdict` substrate primitive (moved DOWN from agent-runtime).
|
|
3882
|
-
-
|
|
3883
|
-
-
|
|
3966
|
+
- the contract diff module — `diffRuns` / `diffGenerations` / `diffRunBaselineToWinner` for v3-vs-v4 dashboard rendering, CI reporting, and any consumer comparing improvement-loop output.
|
|
3967
|
+
- the traceai adapter — OTel→hosted bridge (renamed to `otel.ts` in 0.49.0).
|
|
3884
3968
|
- `tests/hosted-roundtrip.test.ts` — proves wire-format binary compat between client and reference receiver.
|
|
3885
3969
|
- Python `HostedClient` (`clients/python/src/agent_eval_rpc/hosted.py`) — TS↔Python wire-format parity with bearer auth, idempotency, and exponential backoff on 5xx/408/429.
|
|
3886
3970
|
- `CLAUDE.md` repo-layering rule: agent-eval is the substrate; agent-runtime + agent-knowledge depend on it; the reverse is forbidden.
|
|
@@ -3967,7 +4051,7 @@ aggregate improved?*
|
|
|
3967
4051
|
contract into `RunRecord` rows and `runEvalCampaign` so every campaign row
|
|
3968
4052
|
is keyed by `(profile, scenario, seed)` end-to-end. (#79)
|
|
3969
4053
|
- **Stats consolidation** — `pairedBootstrap`, power analysis, and the
|
|
3970
|
-
paired/Welch primitives now all live in `src/statistics
|
|
4054
|
+
paired/Welch primitives now all live in `src/statistics/`. (#73)
|
|
3971
4055
|
- **LLM retry classifier unified** across `llm-client` and `judge-retry`
|
|
3972
4056
|
via `isTransientLlmError`. (#74)
|
|
3973
4057
|
- **`pr-review-benchmark` source committed** — the module was exported from
|
|
@@ -4012,8 +4096,8 @@ tag/npm drift.
|
|
|
4012
4096
|
### Republish of 0.31.0 — dist drift fix
|
|
4013
4097
|
|
|
4014
4098
|
The `v0.31.0` tag's npm tarball shipped a stale `dist/` — `JudgeScoresRecord`
|
|
4015
|
-
was missing from `
|
|
4016
|
-
propagation never made it into `
|
|
4099
|
+
was missing from the built `index.d.ts` and the `recordOutcome.judgeScores`
|
|
4100
|
+
propagation never made it into the built `index.js`, even though the source on
|
|
4017
4101
|
the tagged commit had both. Consumers that bumped to `^0.31.0` got a
|
|
4018
4102
|
typecheck failure on `RunOutcome.judgeScores` (since the type wasn't
|
|
4019
4103
|
re-exported) and a silent drop on the wire (since the campaign runner
|
|
@@ -4023,8 +4107,8 @@ Cause: a build artifact picked up by the publish workflow predated the
|
|
|
4023
4107
|
source merge. The retag forces a clean `pnpm build` and republish; this
|
|
4024
4108
|
patch carries no source change beyond the version bump.
|
|
4025
4109
|
|
|
4026
|
-
Verified after this tag: `
|
|
4027
|
-
`
|
|
4110
|
+
Verified after this tag: the built `index.d.ts` contains `JudgeScoresRecord`,
|
|
4111
|
+
the built `index.js` propagates `outcome.judgeScores` end-to-end via
|
|
4028
4112
|
`recordOutcome.judgeScores`, and a downstream `pnpm install
|
|
4029
4113
|
@tangle-network/agent-eval@0.31.1` types-clean against the shape
|
|
4030
4114
|
documented in 0.31.0.
|
|
@@ -4068,7 +4152,7 @@ Every `perJudge[judge][dim]` and every `perDimMean[dim]` and the
|
|
|
4068
4152
|
`composite` must be finite numbers — the NaN-as-silent-zero bug class
|
|
4069
4153
|
banned by `CLAUDE.md` cannot pass the boundary. `failedJudges` must be
|
|
4070
4154
|
an array of non-empty strings; `notes` must be a string. Round-trip
|
|
4071
|
-
tested in
|
|
4155
|
+
tested in the run-record test.
|
|
4072
4156
|
|
|
4073
4157
|
### Fail-loud contract
|
|
4074
4158
|
|
|
@@ -4190,7 +4274,7 @@ different formulas with different domains of validity.
|
|
|
4190
4274
|
|
|
4191
4275
|
### Added
|
|
4192
4276
|
|
|
4193
|
-
- **`corpusInterRaterAgreement(records, opts?)`** (`src/statistics
|
|
4277
|
+
- **`corpusInterRaterAgreement(records, opts?)`** (`src/statistics/`) —
|
|
4194
4278
|
takes a flat list of `{itemId, judgeName, dimension, score}` records.
|
|
4195
4279
|
For each dimension, pivots to the [n_items × n_judges] matrix of items
|
|
4196
4280
|
every judge rated and delegates to `continuousAgreement` (ICC(2,1) +
|
|
@@ -4330,7 +4414,7 @@ clean primitive that wires them together end-to-end.
|
|
|
4330
4414
|
|
|
4331
4415
|
### Added
|
|
4332
4416
|
|
|
4333
|
-
- **`runProductionLoop({ ... })`** (
|
|
4417
|
+
- **`runProductionLoop({ ... })`** (the production loop module,
|
|
4334
4418
|
`@experimental`) — one call = one cycle. Ingests production traces
|
|
4335
4419
|
and feedback, clusters failures, runs evolve against the worst
|
|
4336
4420
|
cluster, gates with `HeldOutGate` + `evaluateReleaseConfidence`
|
|
@@ -4340,7 +4424,7 @@ clean primitive that wires them together end-to-end.
|
|
|
4340
4424
|
the primitive doesn't own scheduling.
|
|
4341
4425
|
|
|
4342
4426
|
- **`proposeAutomatedPullRequest(client, input)`** + two transports
|
|
4343
|
-
(`src/auto-pr.ts`, `@experimental`):
|
|
4427
|
+
(`src/campaign/auto-pr.ts`, `@experimental`):
|
|
4344
4428
|
- `httpGithubClient({ token, ... })` — direct REST against
|
|
4345
4429
|
`api.github.com`, no extra deps. Idempotent on branch name:
|
|
4346
4430
|
existing open PRs are returned, not duplicated.
|
|
@@ -4434,7 +4518,7 @@ first-touch usability and contract clarity. The visible deltas:
|
|
|
4434
4518
|
`contamination`, `compute-curves`. Experimental: `process-reward`,
|
|
4435
4519
|
`adversarial`, `active-curriculum`, `reward-hacking`, `adaptation-eval`,
|
|
4436
4520
|
`exporters`, `rl-campaign`, `predictive-validity-researcher`, `auto-research`.
|
|
4437
|
-
Tags are visible in IDE hover and emitted into `
|
|
4521
|
+
Tags are visible in IDE hover and emitted into the built `rl.d.ts` so consumers
|
|
4438
4522
|
can see the contract at the call site.
|
|
4439
4523
|
|
|
4440
4524
|
### Added
|
|
@@ -4512,12 +4596,12 @@ is now demonstrably real, not aspirational.
|
|
|
4512
4596
|
because it's the cleanest fit between agent-eval's exporters and
|
|
4513
4597
|
prime-rl's entrypoints (DPO/PRM go to TRL; offline GRPO requires a
|
|
4514
4598
|
custom verifiers env — both called out in the README).
|
|
4515
|
-
-
|
|
4599
|
+
- **the three-package architecture note** — the contracts between
|
|
4516
4600
|
agent-eval, agent-knowledge, agent-runtime. Dependency direction (both
|
|
4517
4601
|
consume agent-eval; agent-eval imports neither), shared data
|
|
4518
4602
|
interchange (RunRecord, Scenario, KnowledgeBundle), and known
|
|
4519
4603
|
contract gaps tracked as follow-ups.
|
|
4520
|
-
-
|
|
4604
|
+
- **the auto-research loop end-to-end note** — the runnable composition
|
|
4521
4605
|
pattern with the explicit invariants every iteration must preserve
|
|
4522
4606
|
(canonical RunRecord with scenarioId, capture wired by construction,
|
|
4523
4607
|
stable comparator, deterministic mutator).
|
|
@@ -4829,7 +4913,7 @@ Locked at `agent-eval-rpc==0.21.0` to match the npm package.
|
|
|
4829
4913
|
- Knowledge readiness recommended actions now honor non-blocking gap
|
|
4830
4914
|
acquisition modes such as `ask_user`, `search_web`, `query_connector`, and
|
|
4831
4915
|
`inspect_repo`.
|
|
4832
|
-
- Npm builds now generate `
|
|
4916
|
+
- Npm builds now generate the built `openapi.json`, and the package exports
|
|
4833
4917
|
`@tangle-network/agent-eval/openapi.json`.
|
|
4834
4918
|
- Npm and Python client versions are locked at `0.20.9`.
|
|
4835
4919
|
|
|
@@ -4995,7 +5079,7 @@ The v0.15 primitives were framed as "paper-grade" but most are production-rigor
|
|
|
4995
5079
|
- `paretoFigure` → `paretoChart`
|
|
4996
5080
|
- `gainDistributionFigure` → `gainHistogram`
|
|
4997
5081
|
- `PaperTable` / `PaperTableOptions` / `PaperTableRow` types → `SummaryTable` / `SummaryTableOptions` / `SummaryTableRow`
|
|
4998
|
-
- File:
|
|
5082
|
+
- File: the paper-report module → `src/summary-report.ts`
|
|
4999
5083
|
|
|
5000
5084
|
### Migration
|
|
5001
5085
|
|
|
@@ -5047,7 +5131,7 @@ optimization with held-out promotion gates.
|
|
|
5047
5131
|
- `deterministicSplit(itemId, seed?)`: stable 60/20/20 split via
|
|
5048
5132
|
FNV-1a hash. Default seed `agent-eval-v1`.
|
|
5049
5133
|
- **`summaryTable`, `paretoChart`, `gainHistogram`**
|
|
5050
|
-
(`
|
|
5134
|
+
(`src/summary-report.ts`) — Table 1 + Pareto + gain-distribution specs.
|
|
5051
5135
|
Returns data structures (markdown table, point lists, histogram bins);
|
|
5052
5136
|
caller picks the plotting library.
|
|
5053
5137
|
- **`runCanaries`** (`src/canary.ts`) — three liveness canaries:
|
|
@@ -5055,7 +5139,7 @@ optimization with held-out promotion gates.
|
|
|
5055
5139
|
judge calibration drift (KS test on confidence distribution), eval-set
|
|
5056
5140
|
distribution shift (chi-square on category bucket counts).
|
|
5057
5141
|
- **`pairedBootstrap`, `pairedWilcoxon`, `bhAdjust`**
|
|
5058
|
-
(
|
|
5142
|
+
(the paired-stats module) — paper-style aliases + the missing paired
|
|
5059
5143
|
bootstrap CI primitive. Deterministic with optional seed.
|
|
5060
5144
|
|
|
5061
5145
|
### Notes
|
package/README.md
CHANGED
|
@@ -149,7 +149,7 @@ See [evaluation integrity](./docs/evaluation-integrity.md) for the complete API
|
|
|
149
149
|
|
|
150
150
|
These controls check the evidence behind a result.
|
|
151
151
|
They do not establish that an optimizer beats a direct edit or simple search.
|
|
152
|
-
The [historical evidence audit](./
|
|
152
|
+
The [historical evidence audit](./archive/2026-09-13-self-improvement-evidence-audit.md) records prior gains, failed transfer, and missing comparisons.
|
|
153
153
|
|
|
154
154
|
Set `searchHistoryPolicy: 'require-complete'` when a method's search must be closed, with every cell, operation and node accounted for, before final evidence is exposed.
|
|
155
155
|
The [search ledger](./docs/search-ledger.md) records every search as nodes, edges and cells, and its receipt is the bounded proof.
|
|
@@ -201,7 +201,7 @@ The [benchmark-book review](./docs/design/mlbenchmarks-book-review.md) records t
|
|
|
201
201
|
|
|
202
202
|
## Development
|
|
203
203
|
|
|
204
|
-
Read [
|
|
204
|
+
Read [AGENTS.md](./AGENTS.md) first: dependency boundaries, ledger canonicalization, and local conventions apply to every change.
|
|
205
205
|
|
|
206
206
|
```sh
|
|
207
207
|
pnpm install
|
package/dist/adapters/http.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { R as Scenario, d as DispatchContext, f as DispatchFn } from "../types-
|
|
3
|
-
import "../index-
|
|
1
|
+
import { u as CostReceipt } from "../cost-ledger-Jk62OBfF.js";
|
|
2
|
+
import { R as Scenario, d as DispatchContext, f as DispatchFn } from "../types-D8GhJoHk.js";
|
|
3
|
+
import "../index-BiT6RtNH.js";
|
|
4
4
|
//#region src/adapters/http.d.ts
|
|
5
5
|
export interface HttpDispatchOptions<TScenario extends Scenario, _TArtifact> {
|
|
6
6
|
/** Static endpoint URL. Mutually exclusive with `resolveUrl`. */
|