@tangle-network/agent-eval 0.216.1 → 0.218.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +137 -57
- package/README.md +2 -2
- package/dist/adapters/http.d.ts +2 -2
- package/dist/{agent-profile-VFcQncbW.d.ts → agent-profile-DTt8uGP9.d.ts} +4 -17
- package/dist/agent-profile-DTt8uGP9.d.ts.map +1 -0
- package/dist/{agent-profile-cell-BK8mlqlq.js → agent-profile-cell-B1V1gUZs.js} +4 -53
- package/dist/agent-profile-cell-B1V1gUZs.js.map +1 -0
- package/dist/{agent-profile-cell-BNp-Js9F.d.ts → agent-profile-cell-BcpFbNd5.d.ts} +2 -22
- package/dist/agent-profile-cell-BcpFbNd5.d.ts.map +1 -0
- package/dist/{agreement-irr-BEWLIF3_.js → agreement-irr-BhAqjluA.js} +5 -5
- package/dist/agreement-irr-BhAqjluA.js.map +1 -0
- package/dist/analyst/index.d.ts +36 -169
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +17 -192
- package/dist/analyst/index.js.map +1 -1
- package/dist/{analyze-runs-D68fd-su.d.ts → analyze-runs-DjlZOVXx.d.ts} +5 -5
- package/dist/{analyze-runs-D68fd-su.d.ts.map → analyze-runs-DjlZOVXx.d.ts.map} +1 -1
- package/dist/{analyze-runs-C84irm4i.js → analyze-runs-DqPJaJ4X.js} +225 -21
- package/dist/analyze-runs-DqPJaJ4X.js.map +1 -0
- package/dist/authenticity/index.d.ts +0 -23
- package/dist/authenticity/index.d.ts.map +1 -1
- package/dist/authenticity/index.js +1 -34
- package/dist/authenticity/index.js.map +1 -1
- package/dist/{benchmark-command-By0vh8q7.js → benchmark-command-BupdHRyf.js} +1225 -661
- package/dist/benchmark-command-BupdHRyf.js.map +1 -0
- package/dist/benchmarks/index.d.ts +5 -6
- package/dist/benchmarks/index.d.ts.map +1 -1
- package/dist/benchmarks/index.js +4 -4
- package/dist/benchmarks/index.js.map +1 -1
- package/dist/builder-eval/index.d.ts +3 -3
- package/dist/builder-eval/index.js +2 -2
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/campaign/index.d.ts +7 -7
- package/dist/campaign/index.js +15 -15
- package/dist/{campaign-BudZ70zo.js → campaign-CGvUTnmc.js} +324 -800
- package/dist/campaign-CGvUTnmc.js.map +1 -0
- package/dist/{campaign-evidence-Clrn4hBV.js → campaign-evidence-DunzcJGv.js} +10 -18
- package/dist/{campaign-evidence-Clrn4hBV.js.map → campaign-evidence-DunzcJGv.js.map} +1 -1
- package/dist/{canonical-B2RqLofv.js → canonical-Dq2hOQ8L.js} +2 -2
- package/dist/{canonical-B2RqLofv.js.map → canonical-Dq2hOQ8L.js.map} +1 -1
- package/dist/cell-spend-DEWR3nj-.js +90 -0
- package/dist/cell-spend-DEWR3nj-.js.map +1 -0
- package/dist/{chat-client-l-MVDlDk.js → chat-client-B2pXlPdx.js} +2 -2
- package/dist/{chat-client-l-MVDlDk.js.map → chat-client-B2pXlPdx.js.map} +1 -1
- package/dist/{chat-json-call-puMArmtW.js → chat-json-call-Dm9c-b41.js} +3 -2
- package/dist/chat-json-call-Dm9c-b41.js.map +1 -0
- package/dist/{claim-q4HQhK1G.js → claim-BPVuyHuy.js} +3 -3
- package/dist/{claim-q4HQhK1G.js.map → claim-BPVuyHuy.js.map} +1 -1
- package/dist/cli.js +113 -12
- package/dist/cli.js.map +1 -1
- package/dist/{client-BzL9niqi.d.ts → client-hwo3cJjR.d.ts} +3 -16
- package/dist/client-hwo3cJjR.d.ts.map +1 -0
- package/dist/{cluster-bootstrap-qcUWYFl-.js → cluster-bootstrap-DEwwhl0C.js} +3 -3
- package/dist/{cluster-bootstrap-qcUWYFl-.js.map → cluster-bootstrap-DEwwhl0C.js.map} +1 -1
- package/dist/{produced-state-BsDmnwxc.js → completion-verifier-DbLVf3i8.js} +9 -64
- package/dist/completion-verifier-DbLVf3i8.js.map +1 -0
- package/dist/contract/index.d.ts +12 -12
- package/dist/contract/index.js +107 -403
- package/dist/contract/index.js.map +1 -1
- package/dist/{cost-ledger-BP8V4kUQ.js → cost-ledger-D0352Z4d.js} +4 -8
- package/dist/cost-ledger-D0352Z4d.js.map +1 -0
- package/dist/dataset-DQqhOCPt.d.ts.map +1 -1
- package/dist/{default-registry-COhk9g36.js → default-registry-BJ-Pq6yL.js} +314 -285
- package/dist/default-registry-BJ-Pq6yL.js.map +1 -0
- package/dist/{default-registry-DzHiGruW.d.ts → default-registry-DXgLmoDI.d.ts} +7 -50
- package/dist/default-registry-DXgLmoDI.d.ts.map +1 -0
- package/dist/{define-agent-eval-CNTPzIkp.js → define-agent-eval-BGdFKlrc.js} +170 -110
- package/dist/define-agent-eval-BGdFKlrc.js.map +1 -0
- package/dist/{define-agent-eval-WSck0CZJ.d.ts → define-agent-eval-BlH10zEw.d.ts} +8 -8
- package/dist/define-agent-eval-BlH10zEw.d.ts.map +1 -0
- package/dist/{descriptive-DboxPZrG.js → descriptive-CVNy48rR.js} +2 -2
- package/dist/{descriptive-DboxPZrG.js.map → descriptive-CVNy48rR.js.map} +1 -1
- package/dist/diagnosis/index.d.ts +4 -14
- package/dist/diagnosis/index.d.ts.map +1 -1
- package/dist/diagnosis/index.js +67 -28
- package/dist/diagnosis/index.js.map +1 -1
- package/dist/{dspy-rlm-engine-LRCu61m5.js → dspy-rlm-engine-CSMwudLO.js} +5 -10
- package/dist/dspy-rlm-engine-CSMwudLO.js.map +1 -0
- package/dist/{emitter-HcCmFQl_.d.ts → emitter-DABYYTcH.d.ts} +5 -5
- package/dist/emitter-DABYYTcH.d.ts.map +1 -0
- package/dist/emitter-DbMXUIVP.js.map +1 -1
- package/dist/{engine-3A_4jOlI.d.ts → engine-BvZ0R-36.d.ts} +17 -21
- package/dist/engine-BvZ0R-36.d.ts.map +1 -0
- package/dist/{equal-terms-BtdXayCJ.js → equal-terms-DbPGcVtI.js} +2 -2
- package/dist/equal-terms-DbPGcVtI.js.map +1 -0
- package/dist/{errors-DVZiqS7K.js → errors-BcOs5MKW.js} +2 -8
- package/dist/{errors-DVZiqS7K.js.map → errors-BcOs5MKW.js.map} +1 -1
- package/dist/eval-campaign-BUMIxKTE.js +393 -0
- package/dist/eval-campaign-BUMIxKTE.js.map +1 -0
- package/dist/{evaluation-6Yoi74gM.d.ts → evaluation-BOZ38vOH.d.ts} +7 -7
- package/dist/evaluation-BOZ38vOH.d.ts.map +1 -0
- package/dist/evaluation.d.ts +2 -2
- package/dist/evaluation.js +1 -1
- package/dist/evaluation.js.map +1 -1
- package/dist/{exact-types-ClugMJfI.d.ts → exact-types-CflbJ9f3.d.ts} +2 -2
- package/dist/{exact-types-ClugMJfI.d.ts.map → exact-types-CflbJ9f3.d.ts.map} +1 -1
- package/dist/{exec-ChJ6d5if.js → exec-CpbtOXVt.js} +2 -24
- package/dist/{exec-ChJ6d5if.js.map → exec-CpbtOXVt.js.map} +1 -1
- package/dist/execution-tracks-CpgFPpS5.js.map +1 -1
- package/dist/experiment/index.d.ts +9 -112
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +258 -311
- package/dist/experiment/index.js.map +1 -1
- package/dist/{exporters-X4xwla8A.js → exporters-DWvM16LE.js} +3 -3
- package/dist/exporters-DWvM16LE.js.map +1 -0
- package/dist/{exporters-ChTnR90l.d.ts → exporters-Dk3oEbK_.d.ts} +2 -4
- package/dist/{exporters-ChTnR90l.d.ts.map → exporters-Dk3oEbK_.d.ts.map} +1 -1
- package/dist/{external-optimizer-process-BjMGUuds.js → external-optimizer-process-C3Wm7Vhn.js} +4 -3
- package/dist/external-optimizer-process-C3Wm7Vhn.js.map +1 -0
- package/dist/{external-optimizer-subprocess-BqW0hHcI.js → external-optimizer-subprocess-B5gdLRZv.js} +9 -11
- package/dist/external-optimizer-subprocess-B5gdLRZv.js.map +1 -0
- package/dist/{extract-usage-BrnzRJ6H.js → extract-usage-BZLU01kR.js} +2 -23
- package/dist/{extract-usage-BrnzRJ6H.js.map → extract-usage-BZLU01kR.js.map} +1 -1
- package/dist/{failure-cluster-CbgEDcFJ.d.ts → failure-cluster-DN0t63Z0.d.ts} +4 -43
- package/dist/failure-cluster-DN0t63Z0.d.ts.map +1 -0
- package/dist/{failure-taxonomy-CjabTATj.js → failure-taxonomy-Bx5oWqaW.js} +2 -19
- package/dist/{failure-taxonomy-CjabTATj.js.map → failure-taxonomy-Bx5oWqaW.js.map} +1 -1
- package/dist/{feedback-trajectory-C4EG0nNH.d.ts → feedback-trajectory-BWmhRBGu.d.ts} +7 -9
- package/dist/feedback-trajectory-BWmhRBGu.d.ts.map +1 -0
- package/dist/{semantic-concept-judge-BQF8te0T.js → findings-store-BtTzPL0d.js} +9 -249
- package/dist/findings-store-BtTzPL0d.js.map +1 -0
- package/dist/fuzz.d.ts +19 -62
- package/dist/fuzz.d.ts.map +1 -1
- package/dist/fuzz.js +62 -108
- package/dist/fuzz.js.map +1 -1
- package/dist/{hf-dataset-BR2yhuNn.js → hf-dataset-CKxIJQba.js} +4 -24
- package/dist/hf-dataset-CKxIJQba.js.map +1 -0
- package/dist/hosted/index.d.ts +2 -2
- package/dist/hosted/index.js +2 -2
- package/dist/{index-CyszSg4O.d.ts → index-B16FNdW5.d.ts} +145 -360
- package/dist/index-B16FNdW5.d.ts.map +1 -0
- package/dist/{index-DV-jSzxF.d.ts → index-BLGqZ81N.d.ts} +10 -15
- package/dist/{index-DV-jSzxF.d.ts.map → index-BLGqZ81N.d.ts.map} +1 -1
- package/dist/{index-BO41eGj5.d.ts → index-CX-GAUpw.d.ts} +3 -7
- package/dist/index-CX-GAUpw.d.ts.map +1 -0
- package/dist/{index-DHKEAv3s.d.ts → index-CalAsXMR.d.ts} +8 -68
- package/dist/index-CalAsXMR.d.ts.map +1 -0
- package/dist/{index-BFRO6chs.d.ts → index-CiRwLv2R.d.ts} +5 -69
- package/dist/index-CiRwLv2R.d.ts.map +1 -0
- package/dist/{index-DPcngtwt.d.ts → index-DbMOXYnP.d.ts} +11 -36
- package/dist/index-DbMOXYnP.d.ts.map +1 -0
- package/dist/index.d.ts +211 -218
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +999 -1714
- package/dist/index.js.map +1 -1
- package/dist/{integrity-dLiDe2kD.js → integrity-C8oUeB0g.js} +326 -203
- package/dist/integrity-C8oUeB0g.js.map +1 -0
- package/dist/{integrity-BXOsAb3i.js → integrity-W4CADcFc.js} +3 -7
- package/dist/{integrity-BXOsAb3i.js.map → integrity-W4CADcFc.js.map} +1 -1
- package/dist/{integrity-DAMEDFaC.d.ts → integrity-_qgZLA5p.d.ts} +4 -6
- package/dist/{integrity-DAMEDFaC.d.ts.map → integrity-_qgZLA5p.d.ts.map} +1 -1
- package/dist/{internal-Epl-YBZW.js → internal-BTK91Bqt.js} +3 -3
- package/dist/{internal-Epl-YBZW.js.map → internal-BTK91Bqt.js.map} +1 -1
- package/dist/{interval-CELpye1n.js → interval-Dq9gt7jx.js} +3 -3
- package/dist/{interval-CELpye1n.js.map → interval-Dq9gt7jx.js.map} +1 -1
- package/dist/{journal-DfRz9kps.js → journal-C7om8Gum.js} +157 -147
- package/dist/journal-C7om8Gum.js.map +1 -0
- package/dist/json-BhdsJaOQ.js +111 -0
- package/dist/json-BhdsJaOQ.js.map +1 -0
- package/dist/{judge-calibration-BpCy5XQl.js → judge-calibration-BkXMIWA4.js} +3 -3
- package/dist/{judge-calibration-BpCy5XQl.js.map → judge-calibration-BkXMIWA4.js.map} +1 -1
- package/dist/{judge-gate-iY9s4L44.js → judge-gate-Co5tuh57.js} +4 -27
- package/dist/{judge-gate-iY9s4L44.js.map → judge-gate-Co5tuh57.js.map} +1 -1
- package/dist/{judge-gate-CE7CQAgt.d.ts → judge-gate-DsL0yFcb.d.ts} +8 -9
- package/dist/judge-gate-DsL0yFcb.d.ts.map +1 -0
- package/dist/{kind-factory-BobI5Jy3.js → kind-factory-Gy4fXixd.js} +88 -17
- package/dist/kind-factory-Gy4fXixd.js.map +1 -0
- package/dist/ledger-core/index.d.ts +2 -2
- package/dist/ledger-core/index.js +4 -4
- package/dist/ledger-core/lock-holder.d.ts +2 -0
- package/dist/ledger-core/lock-holder.js +181 -0
- package/dist/ledger-core/lock-holder.js.map +1 -0
- package/dist/{ledger-core-Gq7yp8lP.js → ledger-core-CAxUdDk0.js} +5 -84
- package/dist/ledger-core-CAxUdDk0.js.map +1 -0
- package/dist/{llm-client-CyBAoYiX.js → llm-client-_Y6ic-tf.js} +243 -355
- package/dist/llm-client-_Y6ic-tf.js.map +1 -0
- package/dist/{llm-judge-CG12SQJq.js → llm-judge-Djxh-Uhs.js} +662 -482
- package/dist/llm-judge-Djxh-Uhs.js.map +1 -0
- package/dist/lock-holder-DaJbfpci.d.ts +86 -0
- package/dist/lock-holder-DaJbfpci.d.ts.map +1 -0
- package/dist/matrix/index.d.ts +3 -3
- package/dist/matrix/index.js +3 -2
- package/dist/{matrix-Dm0BQW7U.js → matrix-B8l-Nsxd.js} +3 -88
- package/dist/matrix-B8l-Nsxd.js.map +1 -0
- package/dist/{matrix-DSrSxjjy.d.ts → matrix-CCSBna4q.d.ts} +9 -18
- package/dist/matrix-CCSBna4q.d.ts.map +1 -0
- package/dist/{mean-DHoMiudD.js → mean-BbA_JCzW.js} +39 -5
- package/dist/mean-BbA_JCzW.js.map +1 -0
- package/dist/meta-eval/index.d.ts +5 -5
- package/dist/meta-eval/index.js +10 -10
- package/dist/metrics-BKaPBtIG.js.map +1 -1
- package/dist/{mint-DNU8QPG4.d.ts → mint-BRZXuwRi.d.ts} +4 -4
- package/dist/{mint-DNU8QPG4.d.ts.map → mint-BRZXuwRi.d.ts.map} +1 -1
- package/dist/{mint-QaD5cA0u.js → mint-CbdwjTPW.js} +4 -4
- package/dist/{mint-QaD5cA0u.js.map → mint-CbdwjTPW.js.map} +1 -1
- package/dist/{multi-layer-verifier-BUaQ4C17.d.ts → multi-layer-verifier-DEgMUk6v.d.ts} +2 -2
- package/dist/{multi-layer-verifier-BUaQ4C17.d.ts.map → multi-layer-verifier-DEgMUk6v.d.ts.map} +1 -1
- package/dist/{multiplicity-DQUFcbW4.js → multiplicity-Sg0U7h8j.js} +2 -2
- package/dist/{multiplicity-DQUFcbW4.js.map → multiplicity-Sg0U7h8j.js.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -23
- package/dist/multishot/golden/index.d.ts.map +1 -1
- package/dist/multishot/golden/index.js +3 -3
- package/dist/multishot/golden/index.js.map +1 -1
- package/dist/multishot/index.d.ts +3 -12
- package/dist/multishot/index.d.ts.map +1 -1
- package/dist/multishot/index.js +4 -3
- package/dist/multishot/index.js.map +1 -1
- package/dist/openapi.json +114 -10
- package/dist/{otel-spans-B9tUXRHF.js → otel-spans-CdN909k1.js} +3 -3
- package/dist/{otel-spans-B9tUXRHF.js.map → otel-spans-CdN909k1.js.map} +1 -1
- package/dist/{paired-arms-CwMGJsrS.js → paired-arms-Ct4LTKc1.js} +32 -9
- package/dist/paired-arms-Ct4LTKc1.js.map +1 -0
- package/dist/{paired-binary-CvpFHfKZ.js → paired-binary-B_nejA44.js} +3 -36
- package/dist/paired-binary-B_nejA44.js.map +1 -0
- package/dist/{paired-promotion-decision-DPKz8ZnF.d.ts → paired-promotion-decision-CQMJ5xGN.d.ts} +2 -34
- package/dist/paired-promotion-decision-CQMJ5xGN.d.ts.map +1 -0
- package/dist/{paired-promotion-decision-DsAPYGCm.js → paired-promotion-decision-D0SHb-ZL.js} +3 -3
- package/dist/{paired-promotion-decision-DsAPYGCm.js.map → paired-promotion-decision-D0SHb-ZL.js.map} +1 -1
- package/dist/{pareto-Cm6ndrWN.js → pareto-CTv8vozZ.js} +11 -11
- package/dist/pareto-CTv8vozZ.js.map +1 -0
- package/dist/pipelines/index.d.ts +16 -19
- package/dist/pipelines/index.d.ts.map +1 -1
- package/dist/pipelines/index.js +4 -68
- package/dist/pipelines/index.js.map +1 -1
- package/dist/{power-and-mde-Hj2ubb4B.js → power-and-mde-N5nU4_Ly.js} +3 -3
- package/dist/{power-and-mde-Hj2ubb4B.js.map → power-and-mde-N5nU4_Ly.js.map} +1 -1
- package/dist/{prime-protocol-CHKgA3a6.js → prime-protocol-BFgGrEMP.js} +3 -3
- package/dist/prime-protocol-BFgGrEMP.js.map +1 -0
- package/dist/{prime-protocol-HiB9HX7E.d.ts → prime-protocol-BWG2DQIk.d.ts} +3 -21
- package/dist/prime-protocol-BWG2DQIk.d.ts.map +1 -0
- package/dist/profile-cell.d.ts +2 -2
- package/dist/profile-cell.js +2 -2
- package/dist/{promotion-gate-BawrdPYr.d.ts → promotion-gate-CxEHRsZW.d.ts} +7 -8
- package/dist/{promotion-gate-BawrdPYr.d.ts.map → promotion-gate-CxEHRsZW.d.ts.map} +1 -1
- package/dist/{promotion-policy-KO57gbyV.js → promotion-policy-6A8osGr_.js} +4 -4
- package/dist/{promotion-policy-KO57gbyV.js.map → promotion-policy-6A8osGr_.js.map} +1 -1
- package/dist/{proposal-findings-CY1xKn_a.js → proposal-findings-n-5WBbEx.js} +3 -3
- package/dist/proposal-findings-n-5WBbEx.js.map +1 -0
- package/dist/{query-BUCZCK7F.js → query-B_RZ36an.js} +5 -5
- package/dist/query-B_RZ36an.js.map +1 -0
- package/dist/{query-DK1nWcmp.d.ts → query-Cmv_U8vf.d.ts} +4 -24
- package/dist/{query-DK1nWcmp.d.ts.map → query-Cmv_U8vf.d.ts.map} +1 -1
- package/dist/{random-ByUSnV9w.js → random-DDDcC7YU.js} +2 -2
- package/dist/{random-ByUSnV9w.js.map → random-DDDcC7YU.js.map} +1 -1
- package/dist/{raw-provider-sink-DoHz4JNG.js → raw-provider-sink-75w4FWBj.js} +2 -2
- package/dist/{raw-provider-sink-DoHz4JNG.js.map → raw-provider-sink-75w4FWBj.js.map} +1 -1
- package/dist/{redact-mtCCcT8z.d.ts → redact-DXdlFb_U.d.ts} +2 -3
- package/dist/redact-DXdlFb_U.d.ts.map +1 -0
- package/dist/{redact-B5xXKqQ3.js → redact-mtkTiJVk.js} +2 -7
- package/dist/{redact-B5xXKqQ3.js.map → redact-mtkTiJVk.js.map} +1 -1
- package/dist/{registry-D7_hm2fK.d.ts → registry-ssNoHzwD.d.ts} +4 -4
- package/dist/{registry-D7_hm2fK.d.ts.map → registry-ssNoHzwD.d.ts.map} +1 -1
- package/dist/{release-confidence-CpuP65S5.js → release-confidence-CYIFH-fp.js} +6 -11
- package/dist/release-confidence-CYIFH-fp.js.map +1 -0
- package/dist/{report-command-SC9nqb4M.js → render-Cs34y0iX.js} +625 -587
- package/dist/render-Cs34y0iX.js.map +1 -0
- package/dist/reporting.d.ts +6 -6
- package/dist/reporting.js +8 -8
- package/dist/{researcher-CEcfku-T.d.ts → researcher-GU1tV4dn.d.ts} +9 -9
- package/dist/{researcher-CEcfku-T.d.ts.map → researcher-GU1tV4dn.d.ts.map} +1 -1
- package/dist/{reward-CpCmAM8L.d.ts → reward-DrZ71vnf.d.ts} +3 -26
- package/dist/reward-DrZ71vnf.d.ts.map +1 -0
- package/dist/{reward-hacking-E0d-WVcP.js → reward-hacking-ByuETaIb.js} +3 -3
- package/dist/{reward-hacking-E0d-WVcP.js.map → reward-hacking-ByuETaIb.js.map} +1 -1
- package/dist/{reward-hacking-RVdZ6Kkt.d.ts → reward-hacking-E52wq86n.d.ts} +5 -18
- package/dist/reward-hacking-E52wq86n.d.ts.map +1 -0
- package/dist/reward-nw2xZGZG.js.map +1 -1
- package/dist/rl.d.ts +117 -295
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +27 -747
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +5 -5
- package/dist/rollout/index.js +7 -8
- package/dist/{rollout-DZV1t2Wg.js → rollout-Dd1ucT_M.js} +88 -13
- package/dist/rollout-Dd1ucT_M.js.map +1 -0
- package/dist/{rubric-predictive-validity-DaFW70y9.js → rubric-predictive-validity-D3_oI72N.js} +3 -3
- package/dist/{rubric-predictive-validity-DaFW70y9.js.map → rubric-predictive-validity-D3_oI72N.js.map} +1 -1
- package/dist/{rubric-predictive-validity-C03EC7ZE.d.ts → rubric-predictive-validity-DiwT8PbL.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-C03EC7ZE.d.ts.map → rubric-predictive-validity-DiwT8PbL.d.ts.map} +1 -1
- package/dist/{run-record-C1E24G2A.d.ts → run-record-CaxbLq6Y.d.ts} +4 -10
- package/dist/run-record-CaxbLq6Y.d.ts.map +1 -0
- package/dist/{run-record-C5e7FjhJ.js → run-record-CysKLB93.js} +3 -3
- package/dist/run-record-CysKLB93.js.map +1 -0
- package/dist/{run-record-Boc5IUTp.js → run-record-DhEJgoPz.js} +4 -4
- package/dist/{run-record-Boc5IUTp.js.map → run-record-DhEJgoPz.js.map} +1 -1
- package/dist/sandbox-harness-BlSOu4LX.d.ts.map +1 -1
- package/dist/{schema-CSf6qWgZ.js → schema-Bt2vLFm3.js} +2 -14
- package/dist/schema-Bt2vLFm3.js.map +1 -0
- package/dist/schema-C-ziyJD7.d.ts.map +1 -1
- package/dist/{schema-BuiujZ-g.js → schema-CvhfLjtZ.js} +78 -48
- package/dist/schema-CvhfLjtZ.js.map +1 -0
- package/dist/{schema-2g7trUwH.d.ts → schema-D6EVKzTB.d.ts} +4 -5
- package/dist/schema-D6EVKzTB.d.ts.map +1 -0
- package/dist/search/index.d.ts +3 -31
- package/dist/search/index.d.ts.map +1 -1
- package/dist/search/index.js +4 -4
- package/dist/{search-kernel-DyNEa6IU.d.ts → search-kernel-p9JB3SmL.d.ts} +10 -27
- package/dist/search-kernel-p9JB3SmL.d.ts.map +1 -0
- package/dist/{search-ledger-CVUGLDZp.js → search-ledger-ClpN4I3e.js} +7 -7
- package/dist/{search-ledger-CVUGLDZp.js.map → search-ledger-ClpN4I3e.js.map} +1 -1
- package/dist/{search-ledger-file-DDQUgvCg.js → search-ledger-file-Bd5Tioq6.js} +4 -4
- package/dist/search-ledger-file-Bd5Tioq6.js.map +1 -0
- package/dist/{search-shipper-BOSD46qv.js → search-shipper-Bg2HW3Uv.js} +4 -4
- package/dist/search-shipper-Bg2HW3Uv.js.map +1 -0
- package/dist/{search-summary-BQCSWf1g.js → search-summary-CrBuvVnf.js} +5 -5
- package/dist/{search-summary-BQCSWf1g.js.map → search-summary-CrBuvVnf.js.map} +1 -1
- package/dist/{sequential-CzK5DarL.js → sequential-BVv1sjsw.js} +2 -32
- package/dist/{sequential-CzK5DarL.js.map → sequential-BVv1sjsw.js.map} +1 -1
- package/dist/{sequential-Bf_pbE50.js → sequential-CimQrbjv.js} +5 -5
- package/dist/sequential-CimQrbjv.js.map +1 -0
- package/dist/{sequential-BhsrMupG.d.ts → sequential-KwGspfeR.d.ts} +2 -9
- package/dist/{sequential-BhsrMupG.d.ts.map → sequential-KwGspfeR.d.ts.map} +1 -1
- package/dist/{sequential-eprocess-DM_qENCl.js → sequential-eprocess-DodI6RoF.js} +2 -2
- package/dist/{sequential-eprocess-DM_qENCl.js.map → sequential-eprocess-DodI6RoF.js.map} +1 -1
- package/dist/{server-e9oWO_bN.js → server-C6M0qR8h.js} +101 -101
- package/dist/server-C6M0qR8h.js.map +1 -0
- package/dist/{single-run-lock-CV9Eyd_q.js → single-run-lock-Dekd8_Wv.js} +7 -7
- package/dist/single-run-lock-Dekd8_Wv.js.map +1 -0
- package/dist/{skillopt-optimization-method-BiwA8nfX.js → skillopt-optimization-method-5dlex7MJ.js} +1386 -1184
- package/dist/skillopt-optimization-method-5dlex7MJ.js.map +1 -0
- package/dist/{spans-BHAUXMGN.d.ts → spans-CVngGij3.d.ts} +2 -2
- package/dist/{spans-BHAUXMGN.d.ts.map → spans-CVngGij3.d.ts.map} +1 -1
- package/dist/{statistical-heldout-CYpGaoDH.d.ts → statistical-heldout-BHfLZHrm.d.ts} +8 -67
- package/dist/statistical-heldout-BHfLZHrm.d.ts.map +1 -0
- package/dist/{statistical-heldout-42NlCNDe.js → statistical-heldout-Bksv92Th.js} +3 -3
- package/dist/statistical-heldout-Bksv92Th.js.map +1 -0
- package/dist/{steps-Bg5-KVx1.d.ts → steps-DlepJPwD.d.ts} +2 -56
- package/dist/steps-DlepJPwD.d.ts.map +1 -0
- package/dist/{store-C4avFjIi.d.ts → store-B3nWRy3B.d.ts} +2 -2
- package/dist/{store-C4avFjIi.d.ts.map → store-B3nWRy3B.d.ts.map} +1 -1
- package/dist/{store-otlp-DJ9dtA5x.js → store-otlp-BteDAbHK.js} +3 -3
- package/dist/{store-otlp-DJ9dtA5x.js.map → store-otlp-BteDAbHK.js.map} +1 -1
- package/dist/{store-tool-spans-CIX5arAG.js → store-tool-spans-D5l_OfTJ.js} +8 -8
- package/dist/store-tool-spans-D5l_OfTJ.js.map +1 -0
- package/dist/{store-tool-spans-DdCSPa7C.d.ts → store-tool-spans-DZS0HWVf.d.ts} +9 -18
- package/dist/store-tool-spans-DZS0HWVf.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts +4 -11
- package/dist/storyboard/index.d.ts.map +1 -1
- package/dist/storyboard/index.js +1 -16
- package/dist/storyboard/index.js.map +1 -1
- package/dist/{summary-report-V_JACR4Y.js → summary-report-BsXJoYX5.js} +88 -10
- package/dist/summary-report-BsXJoYX5.js.map +1 -0
- package/dist/{summary-report-bdFDXq0b.d.ts → summary-report-CkbRcSuu.d.ts} +25 -19
- package/dist/summary-report-CkbRcSuu.d.ts.map +1 -0
- package/dist/supervisor-run/index.d.ts +60 -28
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +4 -458
- package/dist/supervisor-run/index.js.map +1 -1
- package/dist/{systemone-BomhHr2c.d.ts → systemone-BMzDJb1i.d.ts} +5 -5
- package/dist/{systemone-BomhHr2c.d.ts.map → systemone-BMzDJb1i.d.ts.map} +1 -1
- package/dist/systemone-protocol.js +1 -1
- package/dist/{systemone-review-CUVcQJu6.d.ts → systemone-review-CwE6JGIp.d.ts} +2 -2
- package/dist/{systemone-review-CUVcQJu6.d.ts.map → systemone-review-CwE6JGIp.d.ts.map} +1 -1
- package/dist/{systemone-review-D7sXwsqq.js → systemone-review-DzdOJ8I6.js} +3 -3
- package/dist/{systemone-review-D7sXwsqq.js.map → systemone-review-DzdOJ8I6.js.map} +1 -1
- package/dist/systemone.d.ts +5 -5
- package/dist/systemone.js +5 -5
- package/dist/systemone.js.map +1 -1
- package/dist/{task-failure-attributes-hEmASjix.js → task-failure-attributes-BMsblaDh.js} +3 -3
- package/dist/task-failure-attributes-BMsblaDh.js.map +1 -0
- package/dist/{terminal-record-DZyQZ3pz.js → terminal-record-D4Rv-j1W.js} +244 -196
- package/dist/terminal-record-D4Rv-j1W.js.map +1 -0
- package/dist/{tool-groups-C3NEcdCM.d.ts → tool-groups-DyNz2JZ_.d.ts} +3 -3
- package/dist/tool-groups-DyNz2JZ_.d.ts.map +1 -0
- package/dist/{tool-waste-BKgvd-TP.d.ts → tool-waste-CkjzWmeY.d.ts} +25 -28
- package/dist/tool-waste-CkjzWmeY.d.ts.map +1 -0
- package/dist/{tool-waste-BlesqTqW.js → tool-waste-DLsjS_p-.js} +5 -5
- package/dist/{tool-waste-BlesqTqW.js.map → tool-waste-DLsjS_p-.js.map} +1 -1
- package/dist/{trace-questions-CM2CAocU.d.ts → trace-questions-xxwgsuMi.d.ts} +4 -9
- package/dist/{trace-questions-CM2CAocU.d.ts.map → trace-questions-xxwgsuMi.d.ts.map} +1 -1
- package/dist/trace-repair/index.d.ts +14 -211
- package/dist/trace-repair/index.d.ts.map +1 -1
- package/dist/trace-repair/index.js +865 -981
- package/dist/trace-repair/index.js.map +1 -1
- package/dist/traces.d.ts +18 -46
- package/dist/traces.d.ts.map +1 -1
- package/dist/traces.js +13 -13
- package/dist/traces.js.map +1 -1
- package/dist/trajectory-replay/index.d.ts +7 -118
- package/dist/trajectory-replay/index.d.ts.map +1 -1
- package/dist/trajectory-replay/index.js +921 -869
- package/dist/trajectory-replay/index.js.map +1 -1
- package/dist/{tree-C1XYeXvy.js → tree-B5e1uAeL.js} +132 -21
- package/dist/tree-B5e1uAeL.js.map +1 -0
- package/dist/{types-Cbe2fWBa.d.ts → types-BKCzzmit.d.ts} +3 -3
- package/dist/{types-Cbe2fWBa.d.ts.map → types-BKCzzmit.d.ts.map} +1 -1
- package/dist/{types-B3jzCp0p.js → types-BXgASBCL.js} +2 -2
- package/dist/{types-B3jzCp0p.js.map → types-BXgASBCL.js.map} +1 -1
- package/dist/{types-C1tnkEgM.d.ts → types-CW8ewATZ.d.ts} +4 -113
- package/dist/types-CW8ewATZ.d.ts.map +1 -0
- package/dist/{types-BqkVmHfv.d.ts → types-CmXY0PQf.d.ts} +2 -4
- package/dist/types-CmXY0PQf.d.ts.map +1 -0
- package/dist/{types-C02qtnG8.d.ts → types-D-mgRzlh.d.ts} +7 -28
- package/dist/types-D-mgRzlh.d.ts.map +1 -0
- package/dist/{types-IXD2HJzu.js → types-DqFMNTgK.js} +2 -4
- package/dist/types-DqFMNTgK.js.map +1 -0
- package/dist/{types-Cq8mSbq8.d.ts → types-DtkO1G9L.d.ts} +7 -9
- package/dist/{types-Cq8mSbq8.d.ts.map → types-DtkO1G9L.d.ts.map} +1 -1
- package/dist/{verdict-Dl0tiT1X.js → verdict-BGQmxQ8A.js} +3 -3
- package/dist/verdict-BGQmxQ8A.js.map +1 -0
- package/dist/{verdict-E4eRNf7-.d.ts → verdict-C24NELtN.d.ts} +2 -15
- package/dist/{verdict-E4eRNf7-.d.ts.map → verdict-C24NELtN.d.ts.map} +1 -1
- package/dist/{verdict-cache-Bm8EKL2s.js → verdict-cache-CL_7tePD.js} +2 -2
- package/dist/verdict-cache-CL_7tePD.js.map +1 -0
- package/dist/wire/index.d.ts +31 -410
- package/dist/wire/index.d.ts.map +1 -1
- package/dist/wire/index.js +2 -2
- package/docs/changes/jev-evidence-review.md +1 -1
- package/docs/changes/jev-product-adoption.md +1 -1
- package/docs/changes/jev-recorded-evidence.md +1 -1
- package/docs/design/statistics-decisions.md +18 -18
- package/docs/design.md +1 -1
- package/docs/eval-surface-map.md +1 -0
- package/docs/evaluation-integrity.md +1 -1
- package/docs/profile-json-identity.md +1 -2
- package/docs/proofs/campaign-attempt-retention-20261001/README.md +2 -2
- package/docs/public-api.md +158 -538
- package/docs/rollout.md +1 -1
- package/docs/trace-repair-admission.md +1 -1
- package/docs/wire-protocol.md +1 -1
- package/package.json +7 -2
- package/dist/active-curriculum-B_Lc9kfO.js +0 -61
- package/dist/active-curriculum-B_Lc9kfO.js.map +0 -1
- package/dist/agent-profile-VFcQncbW.d.ts.map +0 -1
- package/dist/agent-profile-cell-BK8mlqlq.js.map +0 -1
- package/dist/agent-profile-cell-BNp-Js9F.d.ts.map +0 -1
- package/dist/agreement-irr-BEWLIF3_.js.map +0 -1
- package/dist/analyze-runs-C84irm4i.js.map +0 -1
- package/dist/baseline-Udi_PLuT.js +0 -259
- package/dist/baseline-Udi_PLuT.js.map +0 -1
- package/dist/benchmark-CXmVVHH2.js +0 -562
- package/dist/benchmark-CXmVVHH2.js.map +0 -1
- package/dist/benchmark-command-By0vh8q7.js.map +0 -1
- package/dist/campaign-BudZ70zo.js.map +0 -1
- package/dist/chat-json-call-puMArmtW.js.map +0 -1
- package/dist/claude-jsonl-dZir12m6.js +0 -86
- package/dist/claude-jsonl-dZir12m6.js.map +0 -1
- package/dist/client-BzL9niqi.d.ts.map +0 -1
- package/dist/cost-ledger-BP8V4kUQ.js.map +0 -1
- package/dist/counterfactual-CPaWN99-.d.ts +0 -60
- package/dist/counterfactual-CPaWN99-.d.ts.map +0 -1
- package/dist/counterfactual-Dn3veueY.js +0 -98
- package/dist/counterfactual-Dn3veueY.js.map +0 -1
- package/dist/default-registry-COhk9g36.js.map +0 -1
- package/dist/default-registry-DzHiGruW.d.ts.map +0 -1
- package/dist/define-agent-eval-CNTPzIkp.js.map +0 -1
- package/dist/define-agent-eval-WSck0CZJ.d.ts.map +0 -1
- package/dist/dspy-rlm-engine-LRCu61m5.js.map +0 -1
- package/dist/effect-sizes-BhzEpXMc.js +0 -82
- package/dist/effect-sizes-BhzEpXMc.js.map +0 -1
- package/dist/emitter-HcCmFQl_.d.ts.map +0 -1
- package/dist/engine-3A_4jOlI.d.ts.map +0 -1
- package/dist/equal-terms-BtdXayCJ.js.map +0 -1
- package/dist/eval-campaign-90jeaCAI.js +0 -343
- package/dist/eval-campaign-90jeaCAI.js.map +0 -1
- package/dist/evaluation-6Yoi74gM.d.ts.map +0 -1
- package/dist/exporters-X4xwla8A.js.map +0 -1
- package/dist/external-optimizer-process-BjMGUuds.js.map +0 -1
- package/dist/external-optimizer-subprocess-BqW0hHcI.js.map +0 -1
- package/dist/failure-cluster-CbgEDcFJ.d.ts.map +0 -1
- package/dist/feedback-trajectory-C4EG0nNH.d.ts.map +0 -1
- package/dist/hf-dataset-BR2yhuNn.js.map +0 -1
- package/dist/index-BFRO6chs.d.ts.map +0 -1
- package/dist/index-BO41eGj5.d.ts.map +0 -1
- package/dist/index-CyszSg4O.d.ts.map +0 -1
- package/dist/index-DHKEAv3s.d.ts.map +0 -1
- package/dist/index-DPcngtwt.d.ts.map +0 -1
- package/dist/integrity-dLiDe2kD.js.map +0 -1
- package/dist/journal-DfRz9kps.js.map +0 -1
- package/dist/judge-gate-CE7CQAgt.d.ts.map +0 -1
- package/dist/kind-factory-BobI5Jy3.js.map +0 -1
- package/dist/ledger-core-Gq7yp8lP.js.map +0 -1
- package/dist/llm-client-CyBAoYiX.js.map +0 -1
- package/dist/llm-judge-CG12SQJq.js.map +0 -1
- package/dist/matrix-DSrSxjjy.d.ts.map +0 -1
- package/dist/matrix-Dm0BQW7U.js.map +0 -1
- package/dist/mean-DHoMiudD.js.map +0 -1
- package/dist/normal-DsJZtrJc.js +0 -25
- package/dist/normal-DsJZtrJc.js.map +0 -1
- package/dist/paired-arms-CwMGJsrS.js.map +0 -1
- package/dist/paired-binary-CvpFHfKZ.js.map +0 -1
- package/dist/paired-promotion-decision-DPKz8ZnF.d.ts.map +0 -1
- package/dist/pareto-Cm6ndrWN.js.map +0 -1
- package/dist/prime-protocol-CHKgA3a6.js.map +0 -1
- package/dist/prime-protocol-HiB9HX7E.d.ts.map +0 -1
- package/dist/produced-state-BsDmnwxc.js.map +0 -1
- package/dist/proposal-findings-CY1xKn_a.js.map +0 -1
- package/dist/query-BUCZCK7F.js.map +0 -1
- package/dist/redact-mtCCcT8z.d.ts.map +0 -1
- package/dist/release-confidence-CpuP65S5.js.map +0 -1
- package/dist/report-command-SC9nqb4M.js.map +0 -1
- package/dist/reward-CpCmAM8L.d.ts.map +0 -1
- package/dist/reward-hacking-RVdZ6Kkt.d.ts.map +0 -1
- package/dist/rollout-DZV1t2Wg.js.map +0 -1
- package/dist/run-record-C1E24G2A.d.ts.map +0 -1
- package/dist/run-record-C5e7FjhJ.js.map +0 -1
- package/dist/schema-2g7trUwH.d.ts.map +0 -1
- package/dist/schema-BuiujZ-g.js.map +0 -1
- package/dist/schema-CSf6qWgZ.js.map +0 -1
- package/dist/search-kernel-DyNEa6IU.d.ts.map +0 -1
- package/dist/search-ledger-file-DDQUgvCg.js.map +0 -1
- package/dist/search-shipper-BOSD46qv.js.map +0 -1
- package/dist/semantic-concept-judge-BQF8te0T.js.map +0 -1
- package/dist/sequential-Bf_pbE50.js.map +0 -1
- package/dist/server-e9oWO_bN.js.map +0 -1
- package/dist/single-run-lock-CV9Eyd_q.js.map +0 -1
- package/dist/skillopt-optimization-method-BiwA8nfX.js.map +0 -1
- package/dist/statistical-heldout-42NlCNDe.js.map +0 -1
- package/dist/statistical-heldout-CYpGaoDH.d.ts.map +0 -1
- package/dist/steps-Bg5-KVx1.d.ts.map +0 -1
- package/dist/store-tool-spans-CIX5arAG.js.map +0 -1
- package/dist/store-tool-spans-DdCSPa7C.d.ts.map +0 -1
- package/dist/student-t-DVPW7CdL.js +0 -39
- package/dist/student-t-DVPW7CdL.js.map +0 -1
- package/dist/summary-report-V_JACR4Y.js.map +0 -1
- package/dist/summary-report-bdFDXq0b.d.ts.map +0 -1
- package/dist/task-failure-attributes-hEmASjix.js.map +0 -1
- package/dist/terminal-record-DZyQZ3pz.js.map +0 -1
- package/dist/tool-groups-C3NEcdCM.d.ts.map +0 -1
- package/dist/tool-waste-BKgvd-TP.d.ts.map +0 -1
- package/dist/trajectory-Y1fmdlw7.d.ts +0 -24
- package/dist/trajectory-Y1fmdlw7.d.ts.map +0 -1
- package/dist/tree-C1XYeXvy.js.map +0 -1
- package/dist/types-BqkVmHfv.d.ts.map +0 -1
- package/dist/types-C02qtnG8.d.ts.map +0 -1
- package/dist/types-C1tnkEgM.d.ts.map +0 -1
- package/dist/types-IXD2HJzu.js.map +0 -1
- package/dist/usage-receipt-Cx3xNeyB.js +0 -76
- package/dist/usage-receipt-Cx3xNeyB.js.map +0 -1
- package/dist/verdict-Dl0tiT1X.js.map +0 -1
- package/dist/verdict-cache-Bm8EKL2s.js.map +0 -1
- package/docs/design/self-improvement-evidence-audit.md +0 -263
package/dist/rl.d.ts
CHANGED
|
@@ -1,20 +1,18 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { a as RunRecord, c as RunSplitTag } from "./run-record-
|
|
3
|
-
import { a as PairedPromotionDecision,
|
|
4
|
-
import { a as CampaignResult } from "./types-
|
|
5
|
-
import { s as TraceStore } from "./store-C4avFjIi.js";
|
|
1
|
+
import { _ as Span } from "./schema-D6EVKzTB.js";
|
|
2
|
+
import { a as RunRecord, c as RunSplitTag } from "./run-record-CaxbLq6Y.js";
|
|
3
|
+
import { a as PairedPromotionDecision, u as PairedBootstrapResult } from "./paired-promotion-decision-CQMJ5xGN.js";
|
|
4
|
+
import { a as CampaignResult } from "./types-DtkO1G9L.js";
|
|
6
5
|
import { a as MintedRolloutLine, b as RolloutSplit, t as ChatMessage } from "./schema-C-ziyJD7.js";
|
|
7
|
-
import { t as MintRolloutOptions } from "./mint-
|
|
8
|
-
import { s as
|
|
9
|
-
import { a as detectRewardHacking, c as VerifiableRewardSource, d as filterDeterministicallyRewarded, i as RewardHackingSignal, l as extractVerifiableReward, n as RewardHackingFinding, o as VerifiableReward, r as RewardHackingReport, s as VerifiableRewardExtractionOptions, t as DetectRewardHackingInput, u as extractVerifiableRewardsFromRecords } from "./reward-hacking-RVdZ6Kkt.js";
|
|
6
|
+
import { t as MintRolloutOptions } from "./mint-BRZXuwRi.js";
|
|
7
|
+
import { a as detectRewardHacking, c as VerifiableRewardSource, i as RewardHackingSignal, l as extractVerifiableReward, n as RewardHackingFinding, o as VerifiableReward, r as RewardHackingReport, s as VerifiableRewardExtractionOptions, t as DetectRewardHackingInput, u as extractVerifiableRewardsFromRecords } from "./reward-hacking-E52wq86n.js";
|
|
10
8
|
import { i as InMemoryOutcomeStore, n as FileSystemOutcomeStore, o as OutcomeStore, r as FileSystemOutcomeStoreOptions, t as DeploymentOutcome } from "./outcome-store-CNt4iZ67.js";
|
|
11
|
-
import { a as SystemOneReviewReport, t as PreparedSystemOneReview } from "./systemone-review-
|
|
12
|
-
import { n as TraceQuestionOutcome, r as TraceQuestionSpec } from "./trace-questions-
|
|
13
|
-
import { t as InterimReleaseConfidence } from "./sequential-
|
|
9
|
+
import { a as SystemOneReviewReport, t as PreparedSystemOneReview } from "./systemone-review-CwE6JGIp.js";
|
|
10
|
+
import { n as TraceQuestionOutcome, r as TraceQuestionSpec } from "./trace-questions-xxwgsuMi.js";
|
|
11
|
+
import { t as InterimReleaseConfidence } from "./sequential-KwGspfeR.js";
|
|
14
12
|
import { t as AdversarialMutation } from "./adversarial-smnADNFS.js";
|
|
15
|
-
import { a as SteeringChange, f as EvalCampaignOptions, h as runEvalCampaign, i as Researcher, n as ExperimentResult, p as EvalCampaignResult, r as FailureMode, t as ExperimentPlan } from "./researcher-
|
|
16
|
-
import { o as TrainingExportOptions } from "./exporters-
|
|
17
|
-
import { a as RubricPredictiveValidityReport, t as OutcomeMetricSpec } from "./rubric-predictive-validity-
|
|
13
|
+
import { a as SteeringChange, f as EvalCampaignOptions, h as runEvalCampaign, i as Researcher, n as ExperimentResult, p as EvalCampaignResult, r as FailureMode, t as ExperimentPlan } from "./researcher-GU1tV4dn.js";
|
|
14
|
+
import { o as TrainingExportOptions } from "./exporters-Dk3oEbK_.js";
|
|
15
|
+
import { a as RubricPredictiveValidityReport, t as OutcomeMetricSpec } from "./rubric-predictive-validity-DiwT8PbL.js";
|
|
18
16
|
//#region src/rl/active-curriculum.d.ts
|
|
19
17
|
/**
|
|
20
18
|
* Adaptive curriculum: active scenario selection by variance.
|
|
@@ -31,13 +29,13 @@ import { a as RubricPredictiveValidityReport, t as OutcomeMetricSpec } from "./r
|
|
|
31
29
|
* posterior, in `campaign/search-policy.ts`). Scenario generation belongs to
|
|
32
30
|
* the adversarial primitive.
|
|
33
31
|
*/
|
|
34
|
-
|
|
32
|
+
interface CellObservation {
|
|
35
33
|
variantId: string;
|
|
36
34
|
scenarioId: string;
|
|
37
35
|
/** Observed score in [0, 1]. */
|
|
38
36
|
score: number;
|
|
39
37
|
}
|
|
40
|
-
|
|
38
|
+
interface CurriculumAllocation {
|
|
41
39
|
variantId: string;
|
|
42
40
|
scenarioId: string;
|
|
43
41
|
/** How many additional reps to run on this cell. */
|
|
@@ -45,7 +43,7 @@ export interface CurriculumAllocation {
|
|
|
45
43
|
/** Strategy-specific reason for the allocation. */
|
|
46
44
|
reason: string;
|
|
47
45
|
}
|
|
48
|
-
|
|
46
|
+
interface VarianceCurriculumOptions {
|
|
49
47
|
/** Total reps to allocate across all cells. */
|
|
50
48
|
budget: number;
|
|
51
49
|
/**
|
|
@@ -60,17 +58,6 @@ export interface VarianceCurriculumOptions {
|
|
|
60
58
|
*/
|
|
61
59
|
floorPerCell?: number;
|
|
62
60
|
}
|
|
63
|
-
/**
|
|
64
|
-
* Variance-proportional allocation. For each cell, estimate variance from
|
|
65
|
-
* past observations + a prior, then allocate the budget proportional to
|
|
66
|
-
* (sqrt(variance) + 1/sqrt(n)) — a classical optimal-allocation rule
|
|
67
|
-
* (Neyman 1934) that balances "explore noisy cells" with "explore
|
|
68
|
-
* under-sampled cells."
|
|
69
|
-
*/
|
|
70
|
-
export declare function varianceBasedCurriculum(observations: CellObservation[], candidateCells: Array<{
|
|
71
|
-
variantId: string;
|
|
72
|
-
scenarioId: string;
|
|
73
|
-
}>, opts: VarianceCurriculumOptions): CurriculumAllocation[];
|
|
74
61
|
//#endregion
|
|
75
62
|
//#region src/rl/adaptation-eval.d.ts
|
|
76
63
|
export interface AdaptationRunner<S> {
|
|
@@ -161,15 +148,15 @@ export declare function compareAdaptationCurves(a: AdaptationCurve, b: Adaptatio
|
|
|
161
148
|
export declare function firstPassK(curve: AdaptationCurve, threshold?: number): number | null;
|
|
162
149
|
//#endregion
|
|
163
150
|
//#region src/rl/claim-integrity.d.ts
|
|
164
|
-
|
|
165
|
-
|
|
151
|
+
type ScopeValue = string | number | boolean | null;
|
|
152
|
+
type ScopeOperator = 'eq' | 'ne' | 'lt' | 'le' | 'gt' | 'ge' | 'in' | 'notIn';
|
|
166
153
|
/** One comparison of a claim parameter. A rule holds when all of its conditions hold. */
|
|
167
|
-
|
|
154
|
+
interface ScopeCondition {
|
|
168
155
|
readonly parameter: string;
|
|
169
156
|
readonly op: ScopeOperator;
|
|
170
157
|
readonly value: ScopeValue | readonly ScopeValue[];
|
|
171
158
|
}
|
|
172
|
-
|
|
159
|
+
interface ScopeRule {
|
|
173
160
|
/** Stable id, cited in evidence. */
|
|
174
161
|
readonly id: string;
|
|
175
162
|
readonly when: readonly ScopeCondition[];
|
|
@@ -180,7 +167,7 @@ export interface ScopeRule {
|
|
|
180
167
|
/** Where the source says so: page and line. */
|
|
181
168
|
readonly cite: string;
|
|
182
169
|
}
|
|
183
|
-
|
|
170
|
+
interface SourceExcerpt {
|
|
184
171
|
readonly cite: string;
|
|
185
172
|
readonly text: string;
|
|
186
173
|
}
|
|
@@ -189,7 +176,7 @@ export interface SourceExcerpt {
|
|
|
189
176
|
* the checker's author, and kept out of the agent's reach: it is a held-out check, and an agent that
|
|
190
177
|
* could read it would learn which cases to hide.
|
|
191
178
|
*/
|
|
192
|
-
|
|
179
|
+
interface SourceScope {
|
|
193
180
|
/** The statement or program id a claim names. */
|
|
194
181
|
readonly statement: string;
|
|
195
182
|
/** Citation of the source. */
|
|
@@ -215,7 +202,7 @@ export interface SourceScope {
|
|
|
215
202
|
readonly checked?: readonly ScopeRule[];
|
|
216
203
|
}
|
|
217
204
|
/** One distinct claim an exact checker judged, with what the detector reads. */
|
|
218
|
-
|
|
205
|
+
interface IntegrityClaim {
|
|
219
206
|
/** Stable id of the distinct claim. */
|
|
220
207
|
readonly id: string;
|
|
221
208
|
/** The lane (run lineage) that filed it. */
|
|
@@ -232,14 +219,14 @@ export interface IntegrityClaim {
|
|
|
232
219
|
/** The claim page. Read by the text signal only; never sent to a referee or to a System One model. */
|
|
233
220
|
readonly text?: string | null;
|
|
234
221
|
}
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
222
|
+
type IntegritySignalKind = 'exact' | 'anomaly' | 'text' | 'model' | 'context';
|
|
223
|
+
type IntegritySignalId = 'scope-declared' | 'excluded-case' | 'listed-in-source' | 'inside-checked-range' | 'parameter-farming' | 'grader-reference' | 'referee' | 'systemone' | 'trajectory-monitor' | 'checker-read-first';
|
|
224
|
+
interface IntegrityEvidence {
|
|
238
225
|
readonly kind: 'source' | 'claim' | 'lane' | 'trace' | 'model';
|
|
239
226
|
readonly ref: string;
|
|
240
227
|
readonly quote?: string;
|
|
241
228
|
}
|
|
242
|
-
|
|
229
|
+
interface IntegritySignal {
|
|
243
230
|
readonly id: IntegritySignalId;
|
|
244
231
|
readonly kind: IntegritySignalKind;
|
|
245
232
|
/** `true` against the claim; `false` evaluated with nothing against it; `null` not evaluated. */
|
|
@@ -257,7 +244,7 @@ export declare function ruleHolds(rule: ScopeRule, claim: Pick<IntegrityClaim, '
|
|
|
257
244
|
* source reports checked (claims of new only; an anomaly, since sources do err).
|
|
258
245
|
*/
|
|
259
246
|
export declare function scopeSignals(claim: IntegrityClaim, scope: SourceScope | undefined): IntegritySignal[];
|
|
260
|
-
|
|
247
|
+
interface FarmingOptions {
|
|
261
248
|
/** Distinct claims that must share one parameter value while another parameter varies. Default 3. */
|
|
262
249
|
readonly minimumRepeats?: number;
|
|
263
250
|
}
|
|
@@ -295,7 +282,7 @@ export declare const GRADER_REFERENCE_PHRASES: readonly [{
|
|
|
295
282
|
/** Whether a claim page names the checker or the printed letter as its justification. */
|
|
296
283
|
export declare function graderReferenceSignal(text: string | null | undefined): IntegritySignal;
|
|
297
284
|
/** One recorded action of the lane's agents, oldest first. */
|
|
298
|
-
|
|
285
|
+
interface TrajectoryEvent {
|
|
299
286
|
readonly at: string | number;
|
|
300
287
|
readonly ref: string;
|
|
301
288
|
/** The action as text: a tool name and its arguments. */
|
|
@@ -311,7 +298,7 @@ export declare function checkerReadSignal(events: readonly TrajectoryEvent[], op
|
|
|
311
298
|
firstClaimAt: string | number | null;
|
|
312
299
|
}): IntegritySignal;
|
|
313
300
|
declare const REFEREE_VERDICTS: readonly ['holds', 'excluded', 'trivial', 'known', 'unclear'];
|
|
314
|
-
|
|
301
|
+
type RefereeVerdict = (typeof REFEREE_VERDICTS)[number];
|
|
315
302
|
/** The referee's messages for one claim, for any chat model. */
|
|
316
303
|
export declare function sourceScopeRefereeMessages(claim: IntegrityClaim, scope: SourceScope): ({
|
|
317
304
|
role: 'system';
|
|
@@ -320,7 +307,7 @@ export declare function sourceScopeRefereeMessages(claim: IntegrityClaim, scope:
|
|
|
320
307
|
role: 'user';
|
|
321
308
|
content: string;
|
|
322
309
|
})[];
|
|
323
|
-
|
|
310
|
+
interface SourceScopeRefereeAnswer {
|
|
324
311
|
readonly verdict: RefereeVerdict;
|
|
325
312
|
readonly cite: string | null;
|
|
326
313
|
readonly quote: string | null;
|
|
@@ -360,8 +347,8 @@ declare const claimIntegritySystemOneQuestions: {
|
|
|
360
347
|
readonly instructions: "Is the claim's case inside the statement the source means, so that a verified result there would count for the source's authors?";
|
|
361
348
|
};
|
|
362
349
|
};
|
|
363
|
-
|
|
364
|
-
|
|
350
|
+
type ClaimIntegritySystemOneQuestions = typeof claimIntegritySystemOneQuestions;
|
|
351
|
+
interface ClaimIntegritySystemOneOptions {
|
|
365
352
|
readonly model: string;
|
|
366
353
|
/**
|
|
367
354
|
* Support mass at or above which a question counts against the claim. The caller calibrates it:
|
|
@@ -392,12 +379,11 @@ export declare function monitorSignal(outcomes: readonly TraceQuestionOutcome[]
|
|
|
392
379
|
} | null, options?: {
|
|
393
380
|
costUsd?: number | null;
|
|
394
381
|
}): IntegritySignal;
|
|
395
|
-
|
|
396
|
-
export interface ClaimIntegrityPolicy {
|
|
382
|
+
interface ClaimIntegrityPolicy {
|
|
397
383
|
/** Signals whose firing is recorded but moves no verdict, such as a judge not yet calibrated. */
|
|
398
384
|
readonly recordOnly?: readonly IntegritySignalId[];
|
|
399
385
|
}
|
|
400
|
-
|
|
386
|
+
interface ClaimIntegrityVerdict {
|
|
401
387
|
readonly verdict: 'pass' | 'hack' | 'unknown';
|
|
402
388
|
readonly reasons: readonly string[];
|
|
403
389
|
/** Whether a trajectory monitor should read the lane before a person decides. */
|
|
@@ -444,7 +430,7 @@ export declare function claimIntegrityVerdict(signals: readonly IntegritySignal[
|
|
|
444
430
|
* or LLM-call count. We accept whatever the caller provides; the curve
|
|
445
431
|
* is on whatever axis they pick.
|
|
446
432
|
*/
|
|
447
|
-
|
|
433
|
+
interface ComputeCurveBudget {
|
|
448
434
|
/** Identifier — for the report. Common: '1x', '4x', '16x'. */
|
|
449
435
|
id: string;
|
|
450
436
|
/** Numeric value on the chosen axis (tokens, calls, USD, ms — caller picks). */
|
|
@@ -452,7 +438,7 @@ export interface ComputeCurveBudget {
|
|
|
452
438
|
/** Free-form metadata (the caller can carry per-budget config). */
|
|
453
439
|
meta?: Record<string, unknown>;
|
|
454
440
|
}
|
|
455
|
-
|
|
441
|
+
interface ComputeCurvePoint {
|
|
456
442
|
budgetId: string;
|
|
457
443
|
cost: number;
|
|
458
444
|
score: number;
|
|
@@ -463,7 +449,7 @@ export interface ComputeCurvePoint {
|
|
|
463
449
|
/** Any extra metrics the runner returned. */
|
|
464
450
|
metrics?: Record<string, number>;
|
|
465
451
|
}
|
|
466
|
-
|
|
452
|
+
interface ComputeCurve {
|
|
467
453
|
candidateId: string;
|
|
468
454
|
points: ComputeCurvePoint[];
|
|
469
455
|
/** Rough exponent fit: score ≈ a + b * log(cost). Useful for "how steep is the curve?" */
|
|
@@ -471,7 +457,7 @@ export interface ComputeCurve {
|
|
|
471
457
|
/** Best (highest-score) point on the curve. */
|
|
472
458
|
best: ComputeCurvePoint;
|
|
473
459
|
}
|
|
474
|
-
|
|
460
|
+
interface RunComputeCurveOptions {
|
|
475
461
|
candidateId: string;
|
|
476
462
|
budgets: ComputeCurveBudget[];
|
|
477
463
|
/**
|
|
@@ -485,8 +471,7 @@ export interface RunComputeCurveOptions {
|
|
|
485
471
|
metrics?: Record<string, number>;
|
|
486
472
|
}>;
|
|
487
473
|
}
|
|
488
|
-
|
|
489
|
-
export interface ComputeBestOfNOptions<O> {
|
|
474
|
+
interface ComputeBestOfNOptions<O> {
|
|
490
475
|
/** Number of independent samples to draw. */
|
|
491
476
|
n: number;
|
|
492
477
|
/** Sampler — produces one rollout. */
|
|
@@ -494,7 +479,7 @@ export interface ComputeBestOfNOptions<O> {
|
|
|
494
479
|
/** Score one rollout. */
|
|
495
480
|
scoreFn: (rollout: O) => Promise<number> | number;
|
|
496
481
|
}
|
|
497
|
-
|
|
482
|
+
interface ComputeBestOfNResult<O> {
|
|
498
483
|
best: O;
|
|
499
484
|
bestScore: number;
|
|
500
485
|
scores: number[];
|
|
@@ -504,13 +489,13 @@ export interface ComputeBestOfNResult<O> {
|
|
|
504
489
|
}
|
|
505
490
|
/** The simplest test-time scaling primitive. */
|
|
506
491
|
export declare function bestOfN<O>(opts: ComputeBestOfNOptions<O>): Promise<ComputeBestOfNResult<O>>;
|
|
507
|
-
|
|
492
|
+
interface SelfConsistencyOptions<O> {
|
|
508
493
|
n: number;
|
|
509
494
|
sample: (sampleIdx: number) => Promise<O>;
|
|
510
495
|
/** Extract the canonical answer key (string) from a rollout. */
|
|
511
496
|
answerKey: (rollout: O) => string;
|
|
512
497
|
}
|
|
513
|
-
|
|
498
|
+
interface SelfConsistencyResult<O> {
|
|
514
499
|
/** Modal answer (the majority vote). */
|
|
515
500
|
answer: string;
|
|
516
501
|
/** Fraction of samples voting for the modal answer in [0, 1]. */
|
|
@@ -522,24 +507,19 @@ export interface SelfConsistencyResult<O> {
|
|
|
522
507
|
/** All rollouts. */
|
|
523
508
|
rollouts: O[];
|
|
524
509
|
}
|
|
525
|
-
/**
|
|
526
|
-
* Self-consistency / majority-vote test-time scaling. For tasks with a
|
|
527
|
-
* small categorical answer space (math problems, multiple choice).
|
|
528
|
-
*/
|
|
529
|
-
export declare function selfConsistency<O>(opts: SelfConsistencyOptions<O>): Promise<SelfConsistencyResult<O>>;
|
|
530
510
|
/**
|
|
531
511
|
* Pareto frontier over (candidate, compute, score) tuples. A point is on
|
|
532
512
|
* the frontier iff no other point dominates it in both score (higher
|
|
533
513
|
* better) and cost (lower better). Returns the frontier sorted ascending
|
|
534
514
|
* by cost.
|
|
535
515
|
*/
|
|
536
|
-
|
|
516
|
+
interface ParetoPointInput {
|
|
537
517
|
candidateId: string;
|
|
538
518
|
budgetId: string;
|
|
539
519
|
cost: number;
|
|
540
520
|
score: number;
|
|
541
521
|
}
|
|
542
|
-
export declare function
|
|
522
|
+
export declare function paretoCostScoreFrontier(points: ParetoPointInput[]): ParetoPointInput[];
|
|
543
523
|
//#endregion
|
|
544
524
|
//#region src/rl/contamination.d.ts
|
|
545
525
|
/**
|
|
@@ -551,15 +531,15 @@ export declare function paretoFrontier(points: ParetoPointInput[]): ParetoPointI
|
|
|
551
531
|
* does not identify contamination as the cause. Per-item differences have no
|
|
552
532
|
* calibrated sampling null and carry no p-values or q-values.
|
|
553
533
|
*/
|
|
554
|
-
|
|
555
|
-
|
|
534
|
+
type ScenarioPerturbationKind = 'rename_variables' | 'shuffle_order' | 'paraphrase' | 'inject_irrelevant_clause' | 'custom';
|
|
535
|
+
interface ScenarioPerturbation<S> {
|
|
556
536
|
kind: ScenarioPerturbationKind;
|
|
557
537
|
/** Apply to one scenario, return its perturbed sibling. */
|
|
558
538
|
apply: (scenario: S) => Promise<S> | S;
|
|
559
539
|
/** Optional id — for the report. */
|
|
560
540
|
id?: string;
|
|
561
541
|
}
|
|
562
|
-
|
|
542
|
+
interface ContaminationProbeInput<S> {
|
|
563
543
|
/** Stable, unique identity of every original scenario. */
|
|
564
544
|
scenarioId: (s: S) => string;
|
|
565
545
|
/** Original scenarios. */
|
|
@@ -577,7 +557,7 @@ export interface ContaminationProbeInput<S> {
|
|
|
577
557
|
*/
|
|
578
558
|
scoreFn: (s: S) => Promise<number>;
|
|
579
559
|
}
|
|
580
|
-
|
|
560
|
+
interface ContaminationProbeOptions {
|
|
581
561
|
/** Drop scores below this from the probe; treats partial failures separately. Default 0. */
|
|
582
562
|
scoreFloor?: number;
|
|
583
563
|
/** Significance threshold for the single global paired test. Default 0.05. */
|
|
@@ -588,7 +568,7 @@ export interface ContaminationProbeOptions {
|
|
|
588
568
|
*/
|
|
589
569
|
minMedianDrop?: number;
|
|
590
570
|
}
|
|
591
|
-
|
|
571
|
+
interface ContaminationProbeReport {
|
|
592
572
|
perScenario: Array<{
|
|
593
573
|
scenarioId: string;
|
|
594
574
|
originalScore: number;
|
|
@@ -611,31 +591,6 @@ export interface ContaminationProbeReport {
|
|
|
611
591
|
excludedScenarioIds: string[];
|
|
612
592
|
}
|
|
613
593
|
export declare function runContaminationProbe<S>(input: ContaminationProbeInput<S>, opts?: ContaminationProbeOptions): Promise<ContaminationProbeReport>;
|
|
614
|
-
/**
|
|
615
|
-
* Identifier-rename perturbation for code/text scenarios. Replaces every
|
|
616
|
-
* occurrence of the listed identifiers with synthesized aliases. Use when
|
|
617
|
-
* the scenario's structural difficulty is independent of variable names
|
|
618
|
-
* (e.g. SWE-Bench-style coding tasks).
|
|
619
|
-
*/
|
|
620
|
-
export declare function renameVariables<S extends {
|
|
621
|
-
prompt: string;
|
|
622
|
-
}>(identifiers: string[], rename?: (name: string, idx: number) => string): ScenarioPerturbation<S>;
|
|
623
|
-
/**
|
|
624
|
-
* Order-shuffle perturbation. Reshuffles a list-shaped section of the
|
|
625
|
-
* prompt (for QA scenarios that present options A/B/C/D — answer depends
|
|
626
|
-
* on the option labels, not order). Caller provides the section extractor.
|
|
627
|
-
*/
|
|
628
|
-
export declare function shuffleOrder<S extends {
|
|
629
|
-
prompt: string;
|
|
630
|
-
}>(shuffleSection: (prompt: string, rng: () => number) => string, seed: number): ScenarioPerturbation<S>;
|
|
631
|
-
/**
|
|
632
|
-
* Inject-irrelevant-clause perturbation. Adds a benign sentence that
|
|
633
|
-
* shouldn't change the answer. Tests for "did the model just memorize
|
|
634
|
-
* the input string."
|
|
635
|
-
*/
|
|
636
|
-
export declare function injectIrrelevantClause<S extends {
|
|
637
|
-
prompt: string;
|
|
638
|
-
}>(clause: string, position?: 'prefix' | 'suffix'): ScenarioPerturbation<S>;
|
|
639
594
|
//#endregion
|
|
640
595
|
//#region src/rl/rollout-input.d.ts
|
|
641
596
|
/**
|
|
@@ -790,7 +745,7 @@ export declare function toAnthropicFormat(triples: PreferenceTriple[], context:
|
|
|
790
745
|
}>;
|
|
791
746
|
//#endregion
|
|
792
747
|
//#region src/rl/process-reward.d.ts
|
|
793
|
-
|
|
748
|
+
interface StepReward {
|
|
794
749
|
/** Trace span this reward attaches to. */
|
|
795
750
|
spanId: string;
|
|
796
751
|
runId: string;
|
|
@@ -812,13 +767,13 @@ export interface StepReward {
|
|
|
812
767
|
/** Optional weight — how much this step contributes to credit assignment. */
|
|
813
768
|
weight?: number;
|
|
814
769
|
}
|
|
815
|
-
|
|
770
|
+
interface StepScorer {
|
|
816
771
|
/** Span kinds this scorer applies to. */
|
|
817
772
|
appliesTo: Span['kind'][];
|
|
818
773
|
/** Returns null to skip the span; returns a `StepReward` shape (without index/runId/spanId, which are filled in). */
|
|
819
774
|
score(span: Span): Promise<Omit<StepReward, 'spanId' | 'runId' | 'stepIndex'>> | null | undefined;
|
|
820
775
|
}
|
|
821
|
-
|
|
776
|
+
interface ExtractStepRewardsOptions {
|
|
822
777
|
/**
|
|
823
778
|
* Ordered list of scorers. Each span runs through scorers in order;
|
|
824
779
|
* the first non-null result wins. If no scorer applies, the span is
|
|
@@ -828,8 +783,7 @@ export interface ExtractStepRewardsOptions {
|
|
|
828
783
|
/** Optional filter — return null to drop the span entirely before scoring. */
|
|
829
784
|
preFilter?: (span: Span) => boolean;
|
|
830
785
|
}
|
|
831
|
-
|
|
832
|
-
export interface RunwiseStepSummary {
|
|
786
|
+
interface RunwiseStepSummary {
|
|
833
787
|
runId: string;
|
|
834
788
|
totalSteps: number;
|
|
835
789
|
meanReward: number;
|
|
@@ -841,8 +795,7 @@ export interface RunwiseStepSummary {
|
|
|
841
795
|
worstStepDelta: number;
|
|
842
796
|
worstStepIndex: number | null;
|
|
843
797
|
}
|
|
844
|
-
|
|
845
|
-
export interface PrmTrainingTriple {
|
|
798
|
+
interface PrmTrainingTriple {
|
|
846
799
|
/** Prefix run-id (or composite key) — the trajectory up to step k-1. */
|
|
847
800
|
prefixRunId: string;
|
|
848
801
|
prefixStepIndex: number;
|
|
@@ -877,13 +830,13 @@ export declare function prmTrainingPairs(stepRewardsByRun: Map<string, StepRewar
|
|
|
877
830
|
}): PrmTrainingTriple[];
|
|
878
831
|
//#endregion
|
|
879
832
|
//#region src/rl/exporters.d.ts
|
|
880
|
-
|
|
833
|
+
interface DpoLookups {
|
|
881
834
|
/** Resolve the prompt text for a run (typically from a trace store / raw event sink). */
|
|
882
835
|
promptOf: (runId: string) => string | Promise<string>;
|
|
883
836
|
/** Resolve the assistant completion text for a run. */
|
|
884
837
|
completionOf: (runId: string) => string | Promise<string>;
|
|
885
838
|
}
|
|
886
|
-
|
|
839
|
+
interface DpoExportRow {
|
|
887
840
|
prompt: string;
|
|
888
841
|
chosen: string;
|
|
889
842
|
rejected: string;
|
|
@@ -893,7 +846,7 @@ export interface DpoExportRow {
|
|
|
893
846
|
meta?: Record<string, unknown>;
|
|
894
847
|
}
|
|
895
848
|
/** The minted lines for the runs a `PreferenceTriple` names on each side. */
|
|
896
|
-
|
|
849
|
+
type DpoLineContext = RolloutLineContext;
|
|
897
850
|
/**
|
|
898
851
|
* Convert preference triples to TRL-compatible DPO rows. The shape
|
|
899
852
|
* `{prompt, chosen, rejected}` is the canonical HuggingFace DPODataset
|
|
@@ -910,17 +863,15 @@ export type DpoLineContext = RolloutLineContext;
|
|
|
910
863
|
* why dropping, not zeroing, is the right disposition for a preference pair.
|
|
911
864
|
*/
|
|
912
865
|
export declare function toDpoRows(triples: PreferenceTriple[], lookups: DpoLookups, context: DpoLineContext): Promise<DpoExportRow[]>;
|
|
913
|
-
/** Serialize DPO rows as JSONL. One line per row. */
|
|
914
|
-
export declare function toDpoJsonl(rows: DpoExportRow[]): string;
|
|
915
866
|
/** The one training policy, owned by `rollout/exporters`; the `/rl` name for it. */
|
|
916
|
-
|
|
917
|
-
|
|
867
|
+
type TrainingLineSelectionOptions = TrainingExportOptions;
|
|
868
|
+
interface GrpoLookups extends Pick<TrainingLineSelectionOptions, 'allowHeldOutTrainingData' | 'splitFilter'> {
|
|
918
869
|
/** Resolve the prompt text for a rollout, keyed by `line.run_id`. */
|
|
919
870
|
promptOf: (runId: string) => string | Promise<string>;
|
|
920
871
|
/** Resolve the assistant completion text for a rollout. */
|
|
921
872
|
completionOf: (runId: string) => string | Promise<string>;
|
|
922
873
|
}
|
|
923
|
-
|
|
874
|
+
interface GrpoExportRow {
|
|
924
875
|
prompt: string;
|
|
925
876
|
completions: string[];
|
|
926
877
|
rewards: number[];
|
|
@@ -943,11 +894,10 @@ export interface GrpoExportRow {
|
|
|
943
894
|
* On the line path a realness-gated line stays in its group at reward 0 rather
|
|
944
895
|
* than being dropped. 0 is the honest label for a faked success and is usable
|
|
945
896
|
* signal; removing the line would also move the group's baseline, just in the
|
|
946
|
-
* other direction. (SFT differs — see `
|
|
897
|
+
* other direction. (SFT differs — see `toSftRowsWithLookups`.)
|
|
947
898
|
*/
|
|
948
899
|
export declare function toGrpoRows(lines: MintedRolloutLine[], lookups: GrpoLookups): Promise<GrpoExportRow[]>;
|
|
949
|
-
|
|
950
|
-
export interface SftLookups extends TrainingLineSelectionOptions {
|
|
900
|
+
interface SftLookups extends TrainingLineSelectionOptions {
|
|
951
901
|
/**
|
|
952
902
|
* Resolve the prompt text for a GAP line (one minted without a transcript),
|
|
953
903
|
* keyed by `line.run_id`. A line with a captured transcript never consults it.
|
|
@@ -960,7 +910,7 @@ export interface SftLookups extends TrainingLineSelectionOptions {
|
|
|
960
910
|
/** Extra filter on top of the realness gate (e.g., low score, failed cases). */
|
|
961
911
|
include?: (line: MintedRolloutLine) => boolean;
|
|
962
912
|
}
|
|
963
|
-
|
|
913
|
+
interface SftExportRow {
|
|
964
914
|
/** The canonical chat transcript, tool turns and nullable content included. */
|
|
965
915
|
messages: ChatMessage[];
|
|
966
916
|
meta?: Record<string, unknown>;
|
|
@@ -982,9 +932,9 @@ export interface SftExportRow {
|
|
|
982
932
|
* says which path a row took. A captured transcript whose every turn is copied
|
|
983
933
|
* context yields no row rather than a lookup-built substitute.
|
|
984
934
|
*/
|
|
985
|
-
export declare function
|
|
935
|
+
export declare function toSftRowsWithLookups(lines: MintedRolloutLine[], lookups: SftLookups): Promise<SftExportRow[]>;
|
|
986
936
|
export declare function toSftJsonl(rows: SftExportRow[]): string;
|
|
987
|
-
|
|
937
|
+
interface PrmLookups {
|
|
988
938
|
/** Resolve the prompt text for a run. */
|
|
989
939
|
promptOf: (runId: string) => string | Promise<string>;
|
|
990
940
|
/** Resolve the trajectory step text for a (runId, spanId) pair. */
|
|
@@ -992,7 +942,7 @@ export interface PrmLookups {
|
|
|
992
942
|
/** Optional: sequence of prefix span ids leading up to the divergence. */
|
|
993
943
|
prefixOf?: (runId: string, prefixStepIndex: number) => string[] | Promise<string[]>;
|
|
994
944
|
}
|
|
995
|
-
|
|
945
|
+
interface PrmExportRow {
|
|
996
946
|
prompt: string;
|
|
997
947
|
/** Span ids for the steps before divergence — caller resolves text via `stepTextOf`. */
|
|
998
948
|
prefixSpanIds: string[];
|
|
@@ -1004,7 +954,7 @@ export interface PrmExportRow {
|
|
|
1004
954
|
marginScore: number;
|
|
1005
955
|
meta?: Record<string, unknown>;
|
|
1006
956
|
}
|
|
1007
|
-
|
|
957
|
+
interface PrmLineContext extends RolloutLineContext {
|
|
1008
958
|
/**
|
|
1009
959
|
* The `maxSteps` cap the lines were minted with, if any.
|
|
1010
960
|
*
|
|
@@ -1036,8 +986,7 @@ export interface PrmLineContext extends RolloutLineContext {
|
|
|
1036
986
|
* the exact step the gaming happened.
|
|
1037
987
|
*/
|
|
1038
988
|
export declare function toPrmRows(triples: PrmTrainingTriple[], lookups: PrmLookups, context: PrmLineContext): Promise<PrmExportRow[]>;
|
|
1039
|
-
|
|
1040
|
-
export interface StepRewardJsonlRow {
|
|
989
|
+
interface StepRewardJsonlRow {
|
|
1041
990
|
runId: string;
|
|
1042
991
|
spanId: string;
|
|
1043
992
|
stepIndex: number;
|
|
@@ -1045,26 +994,15 @@ export interface StepRewardJsonlRow {
|
|
|
1045
994
|
determinism: 'deterministic' | 'probabilistic';
|
|
1046
995
|
weight: number;
|
|
1047
996
|
}
|
|
1048
|
-
/**
|
|
1049
|
-
* Step-level reward rows as JSONL.
|
|
1050
|
-
*
|
|
1051
|
-
* `context` is REQUIRED for the same reason it is on `toDpoRows` and
|
|
1052
|
-
* `toPrmRows`: this is a line-less input carrying a reward number. Steps
|
|
1053
|
-
* belonging to a realness-gated run are dropped rather than zeroed — a
|
|
1054
|
-
* per-step reward of 0 across a whole trajectory is a claim that every step was
|
|
1055
|
-
* bad, which is a different (and false) statement from "this run's success was
|
|
1056
|
-
* fabricated, so its step-level credit assignment is meaningless".
|
|
1057
|
-
*/
|
|
1058
|
-
export declare function stepRewardsToJsonl(stepRewards: StepReward[], context: RolloutLineContext): string;
|
|
1059
997
|
//#endregion
|
|
1060
998
|
//#region src/rl/dataset.d.ts
|
|
1061
|
-
|
|
999
|
+
type RewardKind = 'deterministic' | 'probabilistic' | 'mixed';
|
|
1062
1000
|
declare const DATASET_FORMATS: readonly ['grpo', 'sft', 'dpo'];
|
|
1063
|
-
|
|
1001
|
+
type DatasetFormat = (typeof DATASET_FORMATS)[number];
|
|
1064
1002
|
export declare function validateDatasetFormats(value: unknown): DatasetFormat[];
|
|
1065
1003
|
/** Caller-declared context — the qualitative half of the datasheet that can't
|
|
1066
1004
|
* be computed from records. */
|
|
1067
|
-
|
|
1005
|
+
interface RlDatasetConfig {
|
|
1068
1006
|
name: string;
|
|
1069
1007
|
version: string;
|
|
1070
1008
|
/** Product/task domain, e.g. 'legal-m&a', 'tax-1040'. */
|
|
@@ -1093,7 +1031,7 @@ export interface RlDatasetConfig {
|
|
|
1093
1031
|
verifiableRewardFilter?: boolean;
|
|
1094
1032
|
};
|
|
1095
1033
|
}
|
|
1096
|
-
|
|
1034
|
+
interface RewardStats {
|
|
1097
1035
|
n: number;
|
|
1098
1036
|
mean: number | null;
|
|
1099
1037
|
median: number | null;
|
|
@@ -1101,7 +1039,7 @@ export interface RewardStats {
|
|
|
1101
1039
|
max: number | null;
|
|
1102
1040
|
std: number | null;
|
|
1103
1041
|
}
|
|
1104
|
-
|
|
1042
|
+
interface RlDatasetStats {
|
|
1105
1043
|
records: number;
|
|
1106
1044
|
/** Rollouts carrying an explicit task-quality score. */
|
|
1107
1045
|
scoredRecords: number;
|
|
@@ -1125,12 +1063,12 @@ export interface RlDatasetStats {
|
|
|
1125
1063
|
*/
|
|
1126
1064
|
rolloutsWithoutCost: number;
|
|
1127
1065
|
}
|
|
1128
|
-
|
|
1066
|
+
interface RlDatasetManifest extends RlDatasetConfig {
|
|
1129
1067
|
formats: DatasetFormat[];
|
|
1130
1068
|
rowCounts: Partial<Record<DatasetFormat, number>>;
|
|
1131
1069
|
stats: RlDatasetStats;
|
|
1132
1070
|
}
|
|
1133
|
-
|
|
1071
|
+
interface RlDatasetBundle {
|
|
1134
1072
|
manifest: RlDatasetManifest;
|
|
1135
1073
|
/** Relative filename -> contents. Write these to a directory to publish. */
|
|
1136
1074
|
files: Record<string, string>;
|
|
@@ -1147,33 +1085,24 @@ export declare function buildRlDataset(lines: MintedRolloutLine[], lookups: Grpo
|
|
|
1147
1085
|
triples: PreferenceTriple[];
|
|
1148
1086
|
lookups: DpoLookups;
|
|
1149
1087
|
}): Promise<RlDatasetBundle>;
|
|
1150
|
-
/** Render the "Datasheet for Datasets" card that a buyer reads. */
|
|
1151
|
-
export declare function datasheetToMarkdown(m: RlDatasetManifest): string;
|
|
1152
1088
|
//#endregion
|
|
1153
1089
|
//#region src/rl/corpus.d.ts
|
|
1154
1090
|
/** A corpus record is a RunRecord carrying the trajectory text the harness
|
|
1155
1091
|
* captured. `prompt`/`completion` are top-level (the validator ignores extras). */
|
|
1156
|
-
|
|
1092
|
+
type CorpusRecord = RunRecord & {
|
|
1157
1093
|
prompt?: string;
|
|
1158
1094
|
completion?: string;
|
|
1159
1095
|
};
|
|
1160
|
-
|
|
1096
|
+
interface CorpusAppendResult {
|
|
1161
1097
|
appended: number;
|
|
1162
1098
|
/** Skipped because a record with the same runId was already in the corpus
|
|
1163
1099
|
* (idempotent appends — NOT re-run collapsing; re-runs get fresh runIds). */
|
|
1164
1100
|
skipped: number;
|
|
1165
1101
|
total: number;
|
|
1166
1102
|
}
|
|
1167
|
-
/**
|
|
1168
|
-
* Append graded records to the corpus (append-only JSONL). Deduplicates by
|
|
1169
|
-
* `runId` against what's already on disk so re-running the same harness is
|
|
1170
|
-
* idempotent. Creates the file and parent dir. This is the call every eval
|
|
1171
|
-
* harness makes by default after producing its records.
|
|
1172
|
-
*/
|
|
1173
|
-
export declare function appendToCorpus(records: CorpusRecord[], corpusPath: string): CorpusAppendResult;
|
|
1174
1103
|
/** Read the full corpus. Returns [] if the corpus does not exist yet. */
|
|
1175
1104
|
export declare function readCorpus(corpusPath: string): CorpusRecord[];
|
|
1176
|
-
|
|
1105
|
+
interface HarvestOptions {
|
|
1177
1106
|
/** Keep only records scoring >= this (rejection-sampling for SFT). */
|
|
1178
1107
|
minScore?: number;
|
|
1179
1108
|
/** Keep only these source splits. Held-out rows still require the explicit override below. */
|
|
@@ -1181,20 +1110,6 @@ export interface HarvestOptions {
|
|
|
1181
1110
|
/** Permit held-out rows in training files. Default false. */
|
|
1182
1111
|
allowHeldOutTrainingData?: boolean;
|
|
1183
1112
|
}
|
|
1184
|
-
/**
|
|
1185
|
-
* Harvest the accumulated corpus into a publishable RL dataset bundle. Reads
|
|
1186
|
-
* trajectory text from each record's top-level `prompt`/`completion`; records
|
|
1187
|
-
* missing either are excluded (a graded score with no trajectory can't train).
|
|
1188
|
-
* Optionally filters by score / split. Throws (via buildRlDataset) if nothing
|
|
1189
|
-
* survives — an empty dataset must never be published.
|
|
1190
|
-
*
|
|
1191
|
-
* `minScore` is applied to the GATED reward (`trainingScore`), so a gamed run
|
|
1192
|
-
* cannot buy its way into the published bundle with its claimed score —
|
|
1193
|
-
* `minScore` is exactly the door a reward-hacked run would otherwise clear for
|
|
1194
|
-
* SFT. Unscored records are dropped before packaging: a missing label is not a
|
|
1195
|
-
* zero, and it is not publishable either.
|
|
1196
|
-
*/
|
|
1197
|
-
export declare function buildDatasetFromCorpus(corpusPath: string, config: RlDatasetConfig, opts?: HarvestOptions): Promise<RlDatasetBundle>;
|
|
1198
1113
|
//#endregion
|
|
1199
1114
|
//#region src/rl/off-policy.d.ts
|
|
1200
1115
|
/**
|
|
@@ -1236,7 +1151,7 @@ export declare function buildDatasetFromCorpus(corpusPath: string, config: RlDat
|
|
|
1236
1151
|
* complement with `replayCampaign` (exact replay where the request hashes
|
|
1237
1152
|
* match) for high-confidence answers and OPE for the gap.
|
|
1238
1153
|
*/
|
|
1239
|
-
|
|
1154
|
+
interface OffPolicyTrajectory {
|
|
1240
1155
|
/** Stable id, for traceability through the dataset. */
|
|
1241
1156
|
runId: string;
|
|
1242
1157
|
/** Reward observed under the behavior policy (the realized outcome). */
|
|
@@ -1267,13 +1182,13 @@ export interface OffPolicyTrajectory {
|
|
|
1267
1182
|
*/
|
|
1268
1183
|
vHatTarget?: number | null;
|
|
1269
1184
|
}
|
|
1270
|
-
|
|
1185
|
+
interface OffPolicyContributionCounts {
|
|
1271
1186
|
/** Contributions using the contextual-bandit doubly-robust formula. */
|
|
1272
1187
|
dr: number;
|
|
1273
1188
|
/** Contributions using exact IPS because no reward-model estimate was supplied. */
|
|
1274
1189
|
ipsFallback: number;
|
|
1275
1190
|
}
|
|
1276
|
-
|
|
1191
|
+
interface OffPolicyEstimate {
|
|
1277
1192
|
/** Estimated value of the target policy. */
|
|
1278
1193
|
value: number;
|
|
1279
1194
|
/** Standard error of the estimate. */
|
|
@@ -1290,7 +1205,7 @@ export interface OffPolicyEstimate {
|
|
|
1290
1205
|
/** Populated by `doublyRobust` to expose which formula each row used. */
|
|
1291
1206
|
contributionCounts?: OffPolicyContributionCounts;
|
|
1292
1207
|
}
|
|
1293
|
-
|
|
1208
|
+
interface OffPolicyOptions {
|
|
1294
1209
|
/**
|
|
1295
1210
|
* Cap importance weights at this value (Ionides 2008 truncated IS) to
|
|
1296
1211
|
* trade unbiasedness for variance reduction. Default `Infinity` (no cap).
|
|
@@ -1336,16 +1251,6 @@ export declare function selfNormalizedImportanceWeighting(trajectories: OffPolic
|
|
|
1336
1251
|
* fitting and evaluating Q on the same outcomes leaks the answer.
|
|
1337
1252
|
*/
|
|
1338
1253
|
export declare function doublyRobust(trajectories: OffPolicyTrajectory[], opts?: OffPolicyOptions): OffPolicyEstimate;
|
|
1339
|
-
/**
|
|
1340
|
-
* Convenience: run all three estimators and return them side-by-side.
|
|
1341
|
-
* The recommended diagnostic — agreement across estimators is a much
|
|
1342
|
-
* stronger signal than any single one.
|
|
1343
|
-
*/
|
|
1344
|
-
export declare function offPolicyEstimateAll(trajectories: OffPolicyTrajectory[], opts?: OffPolicyOptions): {
|
|
1345
|
-
ips: OffPolicyEstimate;
|
|
1346
|
-
snips: OffPolicyEstimate;
|
|
1347
|
-
dr: OffPolicyEstimate;
|
|
1348
|
-
};
|
|
1349
1254
|
//#endregion
|
|
1350
1255
|
//#region src/rl/predictive-validity-researcher.d.ts
|
|
1351
1256
|
export interface PredictiveValidityResearcherOptions {
|
|
@@ -1494,7 +1399,7 @@ export interface RLCampaignResult {
|
|
|
1494
1399
|
export declare function runRLCampaign<V>(opts: RunRLCampaignOptions<V>): Promise<RLCampaignResult>;
|
|
1495
1400
|
//#endregion
|
|
1496
1401
|
//#region src/rl/run-record-adapters.d.ts
|
|
1497
|
-
|
|
1402
|
+
interface AdapterContext {
|
|
1498
1403
|
/** Logical experiment id — typically the campaign or sweep identifier. */
|
|
1499
1404
|
experimentId: string;
|
|
1500
1405
|
/** Snapshot model id (e.g. `claude-sonnet-4-6@2025-04-15`). */
|
|
@@ -1522,73 +1427,17 @@ export interface AdapterContext {
|
|
|
1522
1427
|
export declare function campaignToRunRecords(campaign: CampaignResult, ctx: AdapterContext & {
|
|
1523
1428
|
candidateId?: string;
|
|
1524
1429
|
}): RunRecord[];
|
|
1525
|
-
/**
|
|
1526
|
-
* Convert a `MultiLayerVerifier` `VerificationReport` into a `RunRecord`.
|
|
1527
|
-
* A split score is emitted only when `report.taskScore` proves the configured
|
|
1528
|
-
* scoring panel completed. Partial scores remain in `outcome.raw` for
|
|
1529
|
-
* diagnosis. Layer errors and timeouts become judge or execution telemetry;
|
|
1530
|
-
* only a scored `fail` layer may produce task-failure detail.
|
|
1531
|
-
*/
|
|
1532
|
-
export declare function verificationReportToRunRecord(report: VerificationReport, ctx: AdapterContext & {
|
|
1533
|
-
candidateId: string;
|
|
1534
|
-
scenarioId: string;
|
|
1535
|
-
}, opts?: {
|
|
1536
|
-
runId?: string;
|
|
1537
|
-
}): RunRecord;
|
|
1538
1430
|
//#endregion
|
|
1539
1431
|
//#region src/rl/sim-fidelity.d.ts
|
|
1540
1432
|
/** Extracts a flat behavioral feature map from one record. `string` values
|
|
1541
1433
|
* are categorical, `number` values are quantile-bucketed over the union of
|
|
1542
1434
|
* both sides, `null` means the feature is absent on this record and is
|
|
1543
1435
|
* counted explicitly as its own category (never silently dropped). */
|
|
1544
|
-
|
|
1545
|
-
/** Reserved histogram category for `null` feature values. A capture-rate
|
|
1546
|
-
* difference (one side instruments a signal, the other does not) registers
|
|
1547
|
-
* as divergence by design: a simulator that produces no tool traces is not
|
|
1548
|
-
* representative of production that does. */
|
|
1549
|
-
export declare const ABSENT_CATEGORY = "(absent)";
|
|
1550
|
-
/**
|
|
1551
|
-
* Default feature set — ONLY fields verified present on both simulated and
|
|
1552
|
-
* production records:
|
|
1553
|
-
*
|
|
1554
|
-
* - `score`, `wall_ms`, `output_tokens` — mandatory per the `RunRecord`
|
|
1555
|
-
* validator (non-finite values read as absent rather than poisoning a
|
|
1556
|
-
* bucket).
|
|
1557
|
-
* - `failure_class` — optional taxonomy field; absent counted explicitly.
|
|
1558
|
-
* - `turn_count`, `tool_errors`, `tool_error_recovery` — derived from the
|
|
1559
|
-
* `outcome.raw` counters the intake adapters and eval harnesses write
|
|
1560
|
-
* (`turns_completed`, `assistant_messages`, `tool_errors`,
|
|
1561
|
-
* `turns_aborted`); absent on records whose producer did not capture
|
|
1562
|
-
* them, counted explicitly.
|
|
1563
|
-
* - `completion_length` — from the optional `CorpusRecord` trajectory
|
|
1564
|
-
* text; the message-length proxy when records come from the corpus.
|
|
1565
|
-
*
|
|
1566
|
-
* `RunRecord` carries event COUNTS, not event ordering, so
|
|
1567
|
-
* `tool_error_recovery` is a counts-only derivation: errors occurred and the
|
|
1568
|
-
* run still completed cleanly ('recovered') vs aborted or classified as a
|
|
1569
|
-
* failure ('unrecovered') — not a literal error→retry sequence check.
|
|
1570
|
-
*/
|
|
1571
|
-
export declare const defaultBehaviorFeatures: BehaviorFeatures;
|
|
1572
|
-
/**
|
|
1573
|
-
* Jensen-Shannon divergence between two categorical histograms (raw counts;
|
|
1574
|
-
* normalized internally). Log base 2 → bounded [0,1]: 0 = identical
|
|
1575
|
-
* distributions, 1 = disjoint support. Symmetric, defined even where the
|
|
1576
|
-
* supports differ — exactly the regime sim-vs-production comparison lives in.
|
|
1577
|
-
* Throws on zero-mass or negative/non-finite counts: an empty histogram has
|
|
1578
|
-
* no distribution and a silent 0 would read as "perfectly representative".
|
|
1579
|
-
*/
|
|
1580
|
-
export declare function jsDivergence(p: Record<string, number>, q: Record<string, number>): number;
|
|
1581
|
-
/**
|
|
1582
|
-
* Deterministic quantile edges over a value set (the UNION of both sides, so
|
|
1583
|
-
* sim and production land in the same buckets). Linear interpolation between
|
|
1584
|
-
* order statistics; duplicate edges from heavy ties collapse into fewer,
|
|
1585
|
-
* wider buckets. Returns `bucketCount - 1` edges before deduplication.
|
|
1586
|
-
*/
|
|
1587
|
-
export declare function quantileEdges(values: number[], bucketCount?: number): number[];
|
|
1436
|
+
type BehaviorFeatures = (record: RunRecord) => Record<string, string | number | null>;
|
|
1588
1437
|
/** Stable half-open bucket label for a value against quantile edges:
|
|
1589
1438
|
* `[-inf,e0)`, `[e0,e1)`, …, `[eLast,+inf)`. */
|
|
1590
1439
|
export declare function bucketLabel(value: number, edges: number[]): string;
|
|
1591
|
-
|
|
1440
|
+
interface FeatureShift {
|
|
1592
1441
|
/** Category label (a string value, a numeric bucket, or `ABSENT_CATEGORY`). */
|
|
1593
1442
|
value: string;
|
|
1594
1443
|
/** Probability of this category among ALL simulated records (nulls included
|
|
@@ -1597,7 +1446,7 @@ export interface FeatureShift {
|
|
|
1597
1446
|
/** Probability among ALL production records. */
|
|
1598
1447
|
pProd: number;
|
|
1599
1448
|
}
|
|
1600
|
-
|
|
1449
|
+
interface FeatureDivergence {
|
|
1601
1450
|
feature: string;
|
|
1602
1451
|
/** Jensen-Shannon divergence in [0,1] for this feature. */
|
|
1603
1452
|
divergence: number;
|
|
@@ -1608,8 +1457,8 @@ export interface FeatureDivergence {
|
|
|
1608
1457
|
/** Non-null observations on the production side. */
|
|
1609
1458
|
nProd: number;
|
|
1610
1459
|
}
|
|
1611
|
-
|
|
1612
|
-
|
|
1460
|
+
type FidelityVerdict = 'representative' | 'skewed' | 'insufficient-data';
|
|
1461
|
+
interface FidelityReport {
|
|
1613
1462
|
perDimension: FeatureDivergence[];
|
|
1614
1463
|
/** 1 − mean divergence over features with sufficient data. NaN when the
|
|
1615
1464
|
* verdict is 'insufficient-data' — a 0 would read as "maximally skewed"
|
|
@@ -1622,20 +1471,13 @@ export interface FidelityReport {
|
|
|
1622
1471
|
* 'skewed' below, 'insufficient-data' when no feature met minN. */
|
|
1623
1472
|
verdict: FidelityVerdict;
|
|
1624
1473
|
}
|
|
1625
|
-
|
|
1474
|
+
interface SimFidelityOptions {
|
|
1626
1475
|
/** Feature extractor. Defaults to `defaultBehaviorFeatures`. */
|
|
1627
1476
|
features?: BehaviorFeatures;
|
|
1628
1477
|
/** Minimum non-null observations per side per feature. Default 20. */
|
|
1629
1478
|
minNPerFeature?: number;
|
|
1630
1479
|
}
|
|
1631
|
-
|
|
1632
|
-
* Compare a simulator's RunRecords against production RunRecords, feature by
|
|
1633
|
-
* feature. Numeric features are bucketed by deterministic quantiles of the
|
|
1634
|
-
* union; nulls count as an explicit `ABSENT_CATEGORY`. Throws on empty
|
|
1635
|
-
* inputs — "no records" is a wiring error, not a distribution.
|
|
1636
|
-
*/
|
|
1637
|
-
export declare function simFidelityReport(simulated: RunRecord[], production: RunRecord[], opts?: SimFidelityOptions): FidelityReport;
|
|
1638
|
-
export interface EasyModeOptions {
|
|
1480
|
+
interface EasyModeOptions {
|
|
1639
1481
|
/** A run passes when its score (holdout, else search) >= this. Default 0.5
|
|
1640
1482
|
* — matches the pass-threshold convention across the rl/ primitives. */
|
|
1641
1483
|
passThreshold?: number;
|
|
@@ -1643,7 +1485,7 @@ export interface EasyModeOptions {
|
|
|
1643
1485
|
* a 10-point inflation is enough to flip most promotion gates. */
|
|
1644
1486
|
inflationTolerance?: number;
|
|
1645
1487
|
}
|
|
1646
|
-
|
|
1488
|
+
interface EasyModeReport {
|
|
1647
1489
|
simPassRate: number;
|
|
1648
1490
|
prodPassRate: number;
|
|
1649
1491
|
/** simPassRate − prodPassRate. Positive = the simulator is easier than reality. */
|
|
@@ -1652,14 +1494,6 @@ export interface EasyModeReport {
|
|
|
1652
1494
|
* simulator overstate production performance. */
|
|
1653
1495
|
inflated: boolean;
|
|
1654
1496
|
}
|
|
1655
|
-
/**
|
|
1656
|
-
* The headline simulator failure mode as its own named artifact: a simulator
|
|
1657
|
-
* that creates "easy mode" inflates pass-rate relative to production, and
|
|
1658
|
-
* every score measured against it overstates reality. Throws on empty inputs
|
|
1659
|
-
* and on records carrying neither score — a silently-skipped record would
|
|
1660
|
-
* bias the very rate this check exists to keep honest.
|
|
1661
|
-
*/
|
|
1662
|
-
export declare function easyModeCheck(simulated: RunRecord[], production: RunRecord[], opts?: EasyModeOptions): EasyModeReport;
|
|
1663
1497
|
//#endregion
|
|
1664
1498
|
//#region src/rl/tournament.d.ts
|
|
1665
1499
|
/**
|
|
@@ -1688,7 +1522,7 @@ export declare function easyModeCheck(simulated: RunRecord[], production: RunRec
|
|
|
1688
1522
|
* tournament eval as the most sample-efficient and most rank-stable
|
|
1689
1523
|
* method when you have many candidates.
|
|
1690
1524
|
*/
|
|
1691
|
-
|
|
1525
|
+
interface PairwiseOutcome {
|
|
1692
1526
|
/** Winner candidate id. */
|
|
1693
1527
|
winner: string;
|
|
1694
1528
|
/** Loser candidate id. */
|
|
@@ -1705,7 +1539,7 @@ export interface PairwiseOutcome {
|
|
|
1705
1539
|
*/
|
|
1706
1540
|
weight?: number;
|
|
1707
1541
|
}
|
|
1708
|
-
|
|
1542
|
+
interface BradleyTerryRating {
|
|
1709
1543
|
candidateId: string;
|
|
1710
1544
|
/** Latent strength θ ≥ 0 from the BT MLE. */
|
|
1711
1545
|
strength: number;
|
|
@@ -1716,7 +1550,7 @@ export interface BradleyTerryRating {
|
|
|
1716
1550
|
/** Win count (+ 0.5 per draw). */
|
|
1717
1551
|
wins: number;
|
|
1718
1552
|
}
|
|
1719
|
-
|
|
1553
|
+
interface BradleyTerryFit {
|
|
1720
1554
|
ratings: BradleyTerryRating[];
|
|
1721
1555
|
/** Iterations of the MM algorithm before convergence. */
|
|
1722
1556
|
iterations: number;
|
|
@@ -1746,23 +1580,19 @@ export declare function fitBradleyTerry(outcomes: PairwiseOutcome[], opts?: {
|
|
|
1746
1580
|
* `applyEloUpdate` mutates the map in place and returns the deltas so
|
|
1747
1581
|
* the caller can log per-comparison rating changes.
|
|
1748
1582
|
*/
|
|
1749
|
-
|
|
1583
|
+
interface EloOptions {
|
|
1750
1584
|
/** Default rating for unseen candidates. Default 1500. */
|
|
1751
1585
|
defaultRating?: number;
|
|
1752
1586
|
/** K-factor controls the step size. Default 32 (FIDE-ish). */
|
|
1753
1587
|
kFactor?: number;
|
|
1754
1588
|
}
|
|
1755
|
-
export declare function applyEloUpdate(ratings: Map<string, number>, outcome: PairwiseOutcome, opts?: EloOptions): {
|
|
1756
|
-
winnerDelta: number;
|
|
1757
|
-
loserDelta: number;
|
|
1758
|
-
};
|
|
1759
1589
|
/**
|
|
1760
1590
|
* Build pairwise outcomes from the campaign artifact: for every scenario
|
|
1761
1591
|
* shared by two candidates, the higher-scoring run wins. Useful when you
|
|
1762
1592
|
* want a tournament view of an existing campaign without an additional
|
|
1763
1593
|
* pairwise judge call.
|
|
1764
1594
|
*/
|
|
1765
|
-
|
|
1595
|
+
interface BuildPairwiseFromCampaignInput {
|
|
1766
1596
|
runs: Array<{
|
|
1767
1597
|
candidateId: string;
|
|
1768
1598
|
/** Stable identifier for the matching unit (typically scenarioId). */
|
|
@@ -1775,7 +1605,6 @@ export interface BuildPairwiseFromCampaignInput {
|
|
|
1775
1605
|
*/
|
|
1776
1606
|
drawMargin?: number;
|
|
1777
1607
|
}
|
|
1778
|
-
export declare function buildPairwiseFromCampaign(input: BuildPairwiseFromCampaignInput): PairwiseOutcome[];
|
|
1779
1608
|
//#endregion
|
|
1780
1609
|
//#region src/rl/verified-findings-dataset.d.ts
|
|
1781
1610
|
/**
|
|
@@ -1808,9 +1637,9 @@ export declare function buildPairwiseFromCampaign(input: BuildPairwiseFromCampai
|
|
|
1808
1637
|
* per-case files contribute prefix-divergence detail and run ids only, and
|
|
1809
1638
|
* are cross-checked against the report where they overlap.
|
|
1810
1639
|
*/
|
|
1811
|
-
|
|
1640
|
+
declare const VERIFIED_FINDING_SCHEMA = "agent-eval/verified-finding@0";
|
|
1812
1641
|
/** One case row from a replay-verify `batch-report.json`. */
|
|
1813
|
-
|
|
1642
|
+
interface ReplayBatchCase {
|
|
1814
1643
|
corpus: string;
|
|
1815
1644
|
trajId: string;
|
|
1816
1645
|
image: string;
|
|
@@ -1837,7 +1666,7 @@ export interface ReplayBatchCase {
|
|
|
1837
1666
|
fix: ReplayBatchFix | null;
|
|
1838
1667
|
wallMs: number;
|
|
1839
1668
|
}
|
|
1840
|
-
|
|
1669
|
+
interface ReplayBatchFix {
|
|
1841
1670
|
attempted: boolean;
|
|
1842
1671
|
sampledOut: boolean;
|
|
1843
1672
|
command: string | null;
|
|
@@ -1845,12 +1674,12 @@ export interface ReplayBatchFix {
|
|
|
1845
1674
|
armBExit: number | null;
|
|
1846
1675
|
failureVanished: boolean | null;
|
|
1847
1676
|
}
|
|
1848
|
-
|
|
1677
|
+
interface ReplayBatchReportFile {
|
|
1849
1678
|
generatedAt: string;
|
|
1850
1679
|
cases: ReplayBatchCase[];
|
|
1851
1680
|
}
|
|
1852
1681
|
/** Gold label entry for one trajectory (CodeTraceBench annotation format). */
|
|
1853
|
-
|
|
1682
|
+
interface GoldLabelEntry {
|
|
1854
1683
|
traj_id: string;
|
|
1855
1684
|
solved: boolean;
|
|
1856
1685
|
step_count: number;
|
|
@@ -1864,12 +1693,12 @@ export interface GoldLabelEntry {
|
|
|
1864
1693
|
}>;
|
|
1865
1694
|
}
|
|
1866
1695
|
/** One step from a normalized trajectory `steps.json`. */
|
|
1867
|
-
|
|
1696
|
+
interface NormalizedStep {
|
|
1868
1697
|
step_id: number;
|
|
1869
1698
|
action: string;
|
|
1870
1699
|
observation?: string | null;
|
|
1871
1700
|
}
|
|
1872
|
-
|
|
1701
|
+
interface RecordedPrefixDivergence {
|
|
1873
1702
|
step: number;
|
|
1874
1703
|
/** `unknown-expectation` marks a step the recording carries no returncode
|
|
1875
1704
|
* for: it could not be confirmed, so it is not agreement. */
|
|
@@ -1879,19 +1708,19 @@ export interface PrefixDivergence {
|
|
|
1879
1708
|
actualExit: number;
|
|
1880
1709
|
}
|
|
1881
1710
|
/** Optional extract from a per-case `replay-verdict.json` (arm A detail only). */
|
|
1882
|
-
|
|
1711
|
+
interface CaseVerdictDetail {
|
|
1883
1712
|
k: number;
|
|
1884
1713
|
prefixExecuted: number;
|
|
1885
1714
|
recordedReturncode: number;
|
|
1886
1715
|
signatureBasis: string | null;
|
|
1887
|
-
prefixDivergences:
|
|
1716
|
+
prefixDivergences: RecordedPrefixDivergence[];
|
|
1888
1717
|
armACommand: string | null;
|
|
1889
1718
|
runIds: {
|
|
1890
1719
|
original: string | null;
|
|
1891
1720
|
armA: string | null;
|
|
1892
1721
|
};
|
|
1893
1722
|
}
|
|
1894
|
-
|
|
1723
|
+
interface DatasetTrajectoryStep {
|
|
1895
1724
|
stepId: number;
|
|
1896
1725
|
action: string;
|
|
1897
1726
|
observation: string | null;
|
|
@@ -1899,15 +1728,15 @@ export interface TrajectoryStep {
|
|
|
1899
1728
|
observationTruncated: boolean;
|
|
1900
1729
|
observationChars: number;
|
|
1901
1730
|
}
|
|
1902
|
-
|
|
1903
|
-
|
|
1731
|
+
type FixOutcome = 'flipped' | 'not-flipped' | 'generation-failed' | 'not-attempted';
|
|
1732
|
+
interface VerifiedFindingFix {
|
|
1904
1733
|
outcome: FixOutcome;
|
|
1905
1734
|
command: string | null;
|
|
1906
1735
|
llmError: string | null;
|
|
1907
1736
|
armBExit: number | null;
|
|
1908
1737
|
failureVanished: boolean | null;
|
|
1909
1738
|
}
|
|
1910
|
-
|
|
1739
|
+
interface VerifiedFindingRow {
|
|
1911
1740
|
schema: typeof VERIFIED_FINDING_SCHEMA;
|
|
1912
1741
|
/** `<runId>/<corpus>/<trajId>` — unique across batches. */
|
|
1913
1742
|
caseId: string;
|
|
@@ -1938,7 +1767,7 @@ export interface VerifiedFindingRow {
|
|
|
1938
1767
|
start: number;
|
|
1939
1768
|
end: number;
|
|
1940
1769
|
};
|
|
1941
|
-
steps:
|
|
1770
|
+
steps: DatasetTrajectoryStep[];
|
|
1942
1771
|
};
|
|
1943
1772
|
verification: {
|
|
1944
1773
|
reproduced: boolean;
|
|
@@ -1953,7 +1782,7 @@ export interface VerifiedFindingRow {
|
|
|
1953
1782
|
/** Steps the recording carries no returncode for. Nonzero here means part
|
|
1954
1783
|
* of the prefix replay was never confirmed against the recording. */
|
|
1955
1784
|
prefixUnknownExpectations: number;
|
|
1956
|
-
prefixDivergenceDetail:
|
|
1785
|
+
prefixDivergenceDetail: RecordedPrefixDivergence[] | null;
|
|
1957
1786
|
armAExit: number | null;
|
|
1958
1787
|
armAReturncodeMatch: boolean;
|
|
1959
1788
|
armACommand: string | null;
|
|
@@ -1976,7 +1805,7 @@ export interface VerifiedFindingRow {
|
|
|
1976
1805
|
armARunId: string | null;
|
|
1977
1806
|
};
|
|
1978
1807
|
}
|
|
1979
|
-
|
|
1808
|
+
interface VerifiedFindingsSummary {
|
|
1980
1809
|
rows: number;
|
|
1981
1810
|
reproduced: number;
|
|
1982
1811
|
/** Reproduced AND arm A matched the failure signature — the batch report's headline strict rate.
|
|
@@ -1990,7 +1819,7 @@ export interface VerifiedFindingsSummary {
|
|
|
1990
1819
|
fixFlipped: number;
|
|
1991
1820
|
}>;
|
|
1992
1821
|
}
|
|
1993
|
-
|
|
1822
|
+
interface BuildVerifiedFindingRowArgs {
|
|
1994
1823
|
batchCase: ReplayBatchCase;
|
|
1995
1824
|
label: GoldLabelEntry;
|
|
1996
1825
|
steps: NormalizedStep[];
|
|
@@ -2004,19 +1833,12 @@ export interface BuildVerifiedFindingRowArgs {
|
|
|
2004
1833
|
detail?: CaseVerdictDetail;
|
|
2005
1834
|
maxObservationChars?: number;
|
|
2006
1835
|
}
|
|
2007
|
-
|
|
2008
|
-
* Join one batch case with its gold label and trajectory into a row.
|
|
2009
|
-
* Throws on any join inconsistency — never emits a partially joined row.
|
|
2010
|
-
*/
|
|
2011
|
-
export declare function buildVerifiedFindingRow(args: BuildVerifiedFindingRowArgs): VerifiedFindingRow;
|
|
2012
|
-
export declare function summarizeVerifiedFindings(rows: VerifiedFindingRow[]): VerifiedFindingsSummary;
|
|
2013
|
-
export declare function verifiedFindingsToJsonl(rows: VerifiedFindingRow[]): string;
|
|
2014
|
-
export interface VerifiedFindingsCorpusSource {
|
|
1836
|
+
interface VerifiedFindingsCorpusSource {
|
|
2015
1837
|
labelsPath: string;
|
|
2016
1838
|
/** Directory containing `normalized/<trajId>/steps.json`. */
|
|
2017
1839
|
preparedDir: string;
|
|
2018
1840
|
}
|
|
2019
|
-
|
|
1841
|
+
interface VerifiedFindingsSource {
|
|
2020
1842
|
batchReportPath: string;
|
|
2021
1843
|
/** Batch run identifier embedded in every caseId, e.g. 'run2-20260802'. */
|
|
2022
1844
|
runId: string;
|
|
@@ -2026,7 +1848,7 @@ export interface VerifiedFindingsSource {
|
|
|
2026
1848
|
runDir?: string;
|
|
2027
1849
|
maxObservationChars?: number;
|
|
2028
1850
|
}
|
|
2029
|
-
|
|
1851
|
+
interface VerifiedFindingsDataset {
|
|
2030
1852
|
rows: VerifiedFindingRow[];
|
|
2031
1853
|
summary: VerifiedFindingsSummary;
|
|
2032
1854
|
provenance: {
|
|
@@ -2048,5 +1870,5 @@ export interface VerifiedFindingsDataset {
|
|
|
2048
1870
|
*/
|
|
2049
1871
|
export declare function loadVerifiedFindingsDataset(source: VerifiedFindingsSource): VerifiedFindingsDataset;
|
|
2050
1872
|
//#endregion
|
|
2051
|
-
export { AdversarialMutation, type DeploymentOutcome, DetectRewardHackingInput, FileSystemOutcomeStore, type FileSystemOutcomeStoreOptions, InMemoryOutcomeStore, type OutcomeStore, RewardHackingFinding, RewardHackingReport, RewardHackingSignal, type RolloutLineContext, VerifiableReward, VerifiableRewardExtractionOptions, VerifiableRewardSource, detectRewardHacking, extractVerifiableReward, extractVerifiableRewardsFromRecords,
|
|
1873
|
+
export { type AdapterContext, AdversarialMutation, type BehaviorFeatures, type BradleyTerryFit, type BradleyTerryRating, type BuildPairwiseFromCampaignInput, type BuildVerifiedFindingRowArgs, type CaseVerdictDetail, type CellObservation, type ClaimIntegrityPolicy, type ClaimIntegritySystemOneOptions, type ClaimIntegritySystemOneQuestions, type ClaimIntegrityVerdict, type ComputeBestOfNOptions, type ComputeBestOfNResult, type ComputeCurve, type ComputeCurveBudget, type ComputeCurvePoint, type ContaminationProbeInput, type ContaminationProbeOptions, type ContaminationProbeReport, type CorpusAppendResult, type CorpusRecord, type CurriculumAllocation, type DatasetFormat, type DatasetTrajectoryStep, type DeploymentOutcome, DetectRewardHackingInput, type DpoExportRow, type DpoLineContext, type DpoLookups, type EasyModeOptions, type EasyModeReport, type EloOptions, type ExtractStepRewardsOptions, type FarmingOptions, type FeatureDivergence, type FeatureShift, type FidelityReport, type FidelityVerdict, FileSystemOutcomeStore, type FileSystemOutcomeStoreOptions, type FixOutcome, type GoldLabelEntry, type GrpoExportRow, type GrpoLookups, type HarvestOptions, InMemoryOutcomeStore, type IntegrityClaim, type IntegrityEvidence, type IntegritySignal, type IntegritySignalId, type IntegritySignalKind, type NormalizedStep, type OffPolicyContributionCounts, type OffPolicyEstimate, type OffPolicyOptions, type OffPolicyTrajectory, type OutcomeStore, type PairwiseOutcome, type ParetoPointInput, type PrmExportRow, type PrmLineContext, type PrmLookups, type PrmTrainingTriple, type RecordedPrefixDivergence, type RefereeVerdict, type ReplayBatchCase, type ReplayBatchFix, type ReplayBatchReportFile, RewardHackingFinding, RewardHackingReport, RewardHackingSignal, type RewardKind, type RewardStats, type RlDatasetBundle, type RlDatasetConfig, type RlDatasetManifest, type RlDatasetStats, type RolloutLineContext, type RunComputeCurveOptions, type RunwiseStepSummary, type ScenarioPerturbation, type ScenarioPerturbationKind, type ScopeCondition, type ScopeOperator, type ScopeRule, type ScopeValue, type SelfConsistencyOptions, type SelfConsistencyResult, type SftExportRow, type SftLookups, type SimFidelityOptions, type SourceExcerpt, type SourceScope, type SourceScopeRefereeAnswer, type StepReward, type StepRewardJsonlRow, type StepScorer, type TrainingLineSelectionOptions, type TrajectoryEvent, type VarianceCurriculumOptions, type VerifiableReward, type VerifiableRewardExtractionOptions, type VerifiableRewardSource, type VerifiedFindingFix, type VerifiedFindingRow, type VerifiedFindingsCorpusSource, type VerifiedFindingsDataset, type VerifiedFindingsSource, type VerifiedFindingsSummary, detectRewardHacking, extractVerifiableReward, extractVerifiableRewardsFromRecords, runEvalCampaign };
|
|
2052
1874
|
//# sourceMappingURL=rl.d.ts.map
|