@tangle-network/agent-eval 0.216.1 → 0.218.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +137 -57
- package/README.md +2 -2
- package/dist/adapters/http.d.ts +2 -2
- package/dist/{agent-profile-VFcQncbW.d.ts → agent-profile-DTt8uGP9.d.ts} +4 -17
- package/dist/agent-profile-DTt8uGP9.d.ts.map +1 -0
- package/dist/{agent-profile-cell-BK8mlqlq.js → agent-profile-cell-B1V1gUZs.js} +4 -53
- package/dist/agent-profile-cell-B1V1gUZs.js.map +1 -0
- package/dist/{agent-profile-cell-BNp-Js9F.d.ts → agent-profile-cell-BcpFbNd5.d.ts} +2 -22
- package/dist/agent-profile-cell-BcpFbNd5.d.ts.map +1 -0
- package/dist/{agreement-irr-BEWLIF3_.js → agreement-irr-BhAqjluA.js} +5 -5
- package/dist/agreement-irr-BhAqjluA.js.map +1 -0
- package/dist/analyst/index.d.ts +36 -169
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +17 -192
- package/dist/analyst/index.js.map +1 -1
- package/dist/{analyze-runs-D68fd-su.d.ts → analyze-runs-DjlZOVXx.d.ts} +5 -5
- package/dist/{analyze-runs-D68fd-su.d.ts.map → analyze-runs-DjlZOVXx.d.ts.map} +1 -1
- package/dist/{analyze-runs-C84irm4i.js → analyze-runs-DqPJaJ4X.js} +225 -21
- package/dist/analyze-runs-DqPJaJ4X.js.map +1 -0
- package/dist/authenticity/index.d.ts +0 -23
- package/dist/authenticity/index.d.ts.map +1 -1
- package/dist/authenticity/index.js +1 -34
- package/dist/authenticity/index.js.map +1 -1
- package/dist/{benchmark-command-By0vh8q7.js → benchmark-command-BupdHRyf.js} +1225 -661
- package/dist/benchmark-command-BupdHRyf.js.map +1 -0
- package/dist/benchmarks/index.d.ts +5 -6
- package/dist/benchmarks/index.d.ts.map +1 -1
- package/dist/benchmarks/index.js +4 -4
- package/dist/benchmarks/index.js.map +1 -1
- package/dist/builder-eval/index.d.ts +3 -3
- package/dist/builder-eval/index.js +2 -2
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/campaign/index.d.ts +7 -7
- package/dist/campaign/index.js +15 -15
- package/dist/{campaign-BudZ70zo.js → campaign-CGvUTnmc.js} +324 -800
- package/dist/campaign-CGvUTnmc.js.map +1 -0
- package/dist/{campaign-evidence-Clrn4hBV.js → campaign-evidence-DunzcJGv.js} +10 -18
- package/dist/{campaign-evidence-Clrn4hBV.js.map → campaign-evidence-DunzcJGv.js.map} +1 -1
- package/dist/{canonical-B2RqLofv.js → canonical-Dq2hOQ8L.js} +2 -2
- package/dist/{canonical-B2RqLofv.js.map → canonical-Dq2hOQ8L.js.map} +1 -1
- package/dist/cell-spend-DEWR3nj-.js +90 -0
- package/dist/cell-spend-DEWR3nj-.js.map +1 -0
- package/dist/{chat-client-l-MVDlDk.js → chat-client-B2pXlPdx.js} +2 -2
- package/dist/{chat-client-l-MVDlDk.js.map → chat-client-B2pXlPdx.js.map} +1 -1
- package/dist/{chat-json-call-puMArmtW.js → chat-json-call-Dm9c-b41.js} +3 -2
- package/dist/chat-json-call-Dm9c-b41.js.map +1 -0
- package/dist/{claim-q4HQhK1G.js → claim-BPVuyHuy.js} +3 -3
- package/dist/{claim-q4HQhK1G.js.map → claim-BPVuyHuy.js.map} +1 -1
- package/dist/cli.js +113 -12
- package/dist/cli.js.map +1 -1
- package/dist/{client-BzL9niqi.d.ts → client-hwo3cJjR.d.ts} +3 -16
- package/dist/client-hwo3cJjR.d.ts.map +1 -0
- package/dist/{cluster-bootstrap-qcUWYFl-.js → cluster-bootstrap-DEwwhl0C.js} +3 -3
- package/dist/{cluster-bootstrap-qcUWYFl-.js.map → cluster-bootstrap-DEwwhl0C.js.map} +1 -1
- package/dist/{produced-state-BsDmnwxc.js → completion-verifier-DbLVf3i8.js} +9 -64
- package/dist/completion-verifier-DbLVf3i8.js.map +1 -0
- package/dist/contract/index.d.ts +12 -12
- package/dist/contract/index.js +107 -403
- package/dist/contract/index.js.map +1 -1
- package/dist/{cost-ledger-BP8V4kUQ.js → cost-ledger-D0352Z4d.js} +4 -8
- package/dist/cost-ledger-D0352Z4d.js.map +1 -0
- package/dist/dataset-DQqhOCPt.d.ts.map +1 -1
- package/dist/{default-registry-COhk9g36.js → default-registry-BJ-Pq6yL.js} +314 -285
- package/dist/default-registry-BJ-Pq6yL.js.map +1 -0
- package/dist/{default-registry-DzHiGruW.d.ts → default-registry-DXgLmoDI.d.ts} +7 -50
- package/dist/default-registry-DXgLmoDI.d.ts.map +1 -0
- package/dist/{define-agent-eval-CNTPzIkp.js → define-agent-eval-BGdFKlrc.js} +170 -110
- package/dist/define-agent-eval-BGdFKlrc.js.map +1 -0
- package/dist/{define-agent-eval-WSck0CZJ.d.ts → define-agent-eval-BlH10zEw.d.ts} +8 -8
- package/dist/define-agent-eval-BlH10zEw.d.ts.map +1 -0
- package/dist/{descriptive-DboxPZrG.js → descriptive-CVNy48rR.js} +2 -2
- package/dist/{descriptive-DboxPZrG.js.map → descriptive-CVNy48rR.js.map} +1 -1
- package/dist/diagnosis/index.d.ts +4 -14
- package/dist/diagnosis/index.d.ts.map +1 -1
- package/dist/diagnosis/index.js +67 -28
- package/dist/diagnosis/index.js.map +1 -1
- package/dist/{dspy-rlm-engine-LRCu61m5.js → dspy-rlm-engine-CSMwudLO.js} +5 -10
- package/dist/dspy-rlm-engine-CSMwudLO.js.map +1 -0
- package/dist/{emitter-HcCmFQl_.d.ts → emitter-DABYYTcH.d.ts} +5 -5
- package/dist/emitter-DABYYTcH.d.ts.map +1 -0
- package/dist/emitter-DbMXUIVP.js.map +1 -1
- package/dist/{engine-3A_4jOlI.d.ts → engine-BvZ0R-36.d.ts} +17 -21
- package/dist/engine-BvZ0R-36.d.ts.map +1 -0
- package/dist/{equal-terms-BtdXayCJ.js → equal-terms-DbPGcVtI.js} +2 -2
- package/dist/equal-terms-DbPGcVtI.js.map +1 -0
- package/dist/{errors-DVZiqS7K.js → errors-BcOs5MKW.js} +2 -8
- package/dist/{errors-DVZiqS7K.js.map → errors-BcOs5MKW.js.map} +1 -1
- package/dist/eval-campaign-BUMIxKTE.js +393 -0
- package/dist/eval-campaign-BUMIxKTE.js.map +1 -0
- package/dist/{evaluation-6Yoi74gM.d.ts → evaluation-BOZ38vOH.d.ts} +7 -7
- package/dist/evaluation-BOZ38vOH.d.ts.map +1 -0
- package/dist/evaluation.d.ts +2 -2
- package/dist/evaluation.js +1 -1
- package/dist/evaluation.js.map +1 -1
- package/dist/{exact-types-ClugMJfI.d.ts → exact-types-CflbJ9f3.d.ts} +2 -2
- package/dist/{exact-types-ClugMJfI.d.ts.map → exact-types-CflbJ9f3.d.ts.map} +1 -1
- package/dist/{exec-ChJ6d5if.js → exec-CpbtOXVt.js} +2 -24
- package/dist/{exec-ChJ6d5if.js.map → exec-CpbtOXVt.js.map} +1 -1
- package/dist/execution-tracks-CpgFPpS5.js.map +1 -1
- package/dist/experiment/index.d.ts +9 -112
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +258 -311
- package/dist/experiment/index.js.map +1 -1
- package/dist/{exporters-X4xwla8A.js → exporters-DWvM16LE.js} +3 -3
- package/dist/exporters-DWvM16LE.js.map +1 -0
- package/dist/{exporters-ChTnR90l.d.ts → exporters-Dk3oEbK_.d.ts} +2 -4
- package/dist/{exporters-ChTnR90l.d.ts.map → exporters-Dk3oEbK_.d.ts.map} +1 -1
- package/dist/{external-optimizer-process-BjMGUuds.js → external-optimizer-process-C3Wm7Vhn.js} +4 -3
- package/dist/external-optimizer-process-C3Wm7Vhn.js.map +1 -0
- package/dist/{external-optimizer-subprocess-BqW0hHcI.js → external-optimizer-subprocess-B5gdLRZv.js} +9 -11
- package/dist/external-optimizer-subprocess-B5gdLRZv.js.map +1 -0
- package/dist/{extract-usage-BrnzRJ6H.js → extract-usage-BZLU01kR.js} +2 -23
- package/dist/{extract-usage-BrnzRJ6H.js.map → extract-usage-BZLU01kR.js.map} +1 -1
- package/dist/{failure-cluster-CbgEDcFJ.d.ts → failure-cluster-DN0t63Z0.d.ts} +4 -43
- package/dist/failure-cluster-DN0t63Z0.d.ts.map +1 -0
- package/dist/{failure-taxonomy-CjabTATj.js → failure-taxonomy-Bx5oWqaW.js} +2 -19
- package/dist/{failure-taxonomy-CjabTATj.js.map → failure-taxonomy-Bx5oWqaW.js.map} +1 -1
- package/dist/{feedback-trajectory-C4EG0nNH.d.ts → feedback-trajectory-BWmhRBGu.d.ts} +7 -9
- package/dist/feedback-trajectory-BWmhRBGu.d.ts.map +1 -0
- package/dist/{semantic-concept-judge-BQF8te0T.js → findings-store-BtTzPL0d.js} +9 -249
- package/dist/findings-store-BtTzPL0d.js.map +1 -0
- package/dist/fuzz.d.ts +19 -62
- package/dist/fuzz.d.ts.map +1 -1
- package/dist/fuzz.js +62 -108
- package/dist/fuzz.js.map +1 -1
- package/dist/{hf-dataset-BR2yhuNn.js → hf-dataset-CKxIJQba.js} +4 -24
- package/dist/hf-dataset-CKxIJQba.js.map +1 -0
- package/dist/hosted/index.d.ts +2 -2
- package/dist/hosted/index.js +2 -2
- package/dist/{index-CyszSg4O.d.ts → index-B16FNdW5.d.ts} +145 -360
- package/dist/index-B16FNdW5.d.ts.map +1 -0
- package/dist/{index-DV-jSzxF.d.ts → index-BLGqZ81N.d.ts} +10 -15
- package/dist/{index-DV-jSzxF.d.ts.map → index-BLGqZ81N.d.ts.map} +1 -1
- package/dist/{index-BO41eGj5.d.ts → index-CX-GAUpw.d.ts} +3 -7
- package/dist/index-CX-GAUpw.d.ts.map +1 -0
- package/dist/{index-DHKEAv3s.d.ts → index-CalAsXMR.d.ts} +8 -68
- package/dist/index-CalAsXMR.d.ts.map +1 -0
- package/dist/{index-BFRO6chs.d.ts → index-CiRwLv2R.d.ts} +5 -69
- package/dist/index-CiRwLv2R.d.ts.map +1 -0
- package/dist/{index-DPcngtwt.d.ts → index-DbMOXYnP.d.ts} +11 -36
- package/dist/index-DbMOXYnP.d.ts.map +1 -0
- package/dist/index.d.ts +211 -218
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +999 -1714
- package/dist/index.js.map +1 -1
- package/dist/{integrity-dLiDe2kD.js → integrity-C8oUeB0g.js} +326 -203
- package/dist/integrity-C8oUeB0g.js.map +1 -0
- package/dist/{integrity-BXOsAb3i.js → integrity-W4CADcFc.js} +3 -7
- package/dist/{integrity-BXOsAb3i.js.map → integrity-W4CADcFc.js.map} +1 -1
- package/dist/{integrity-DAMEDFaC.d.ts → integrity-_qgZLA5p.d.ts} +4 -6
- package/dist/{integrity-DAMEDFaC.d.ts.map → integrity-_qgZLA5p.d.ts.map} +1 -1
- package/dist/{internal-Epl-YBZW.js → internal-BTK91Bqt.js} +3 -3
- package/dist/{internal-Epl-YBZW.js.map → internal-BTK91Bqt.js.map} +1 -1
- package/dist/{interval-CELpye1n.js → interval-Dq9gt7jx.js} +3 -3
- package/dist/{interval-CELpye1n.js.map → interval-Dq9gt7jx.js.map} +1 -1
- package/dist/{journal-DfRz9kps.js → journal-C7om8Gum.js} +157 -147
- package/dist/journal-C7om8Gum.js.map +1 -0
- package/dist/json-BhdsJaOQ.js +111 -0
- package/dist/json-BhdsJaOQ.js.map +1 -0
- package/dist/{judge-calibration-BpCy5XQl.js → judge-calibration-BkXMIWA4.js} +3 -3
- package/dist/{judge-calibration-BpCy5XQl.js.map → judge-calibration-BkXMIWA4.js.map} +1 -1
- package/dist/{judge-gate-iY9s4L44.js → judge-gate-Co5tuh57.js} +4 -27
- package/dist/{judge-gate-iY9s4L44.js.map → judge-gate-Co5tuh57.js.map} +1 -1
- package/dist/{judge-gate-CE7CQAgt.d.ts → judge-gate-DsL0yFcb.d.ts} +8 -9
- package/dist/judge-gate-DsL0yFcb.d.ts.map +1 -0
- package/dist/{kind-factory-BobI5Jy3.js → kind-factory-Gy4fXixd.js} +88 -17
- package/dist/kind-factory-Gy4fXixd.js.map +1 -0
- package/dist/ledger-core/index.d.ts +2 -2
- package/dist/ledger-core/index.js +4 -4
- package/dist/ledger-core/lock-holder.d.ts +2 -0
- package/dist/ledger-core/lock-holder.js +181 -0
- package/dist/ledger-core/lock-holder.js.map +1 -0
- package/dist/{ledger-core-Gq7yp8lP.js → ledger-core-CAxUdDk0.js} +5 -84
- package/dist/ledger-core-CAxUdDk0.js.map +1 -0
- package/dist/{llm-client-CyBAoYiX.js → llm-client-_Y6ic-tf.js} +243 -355
- package/dist/llm-client-_Y6ic-tf.js.map +1 -0
- package/dist/{llm-judge-CG12SQJq.js → llm-judge-Djxh-Uhs.js} +662 -482
- package/dist/llm-judge-Djxh-Uhs.js.map +1 -0
- package/dist/lock-holder-DaJbfpci.d.ts +86 -0
- package/dist/lock-holder-DaJbfpci.d.ts.map +1 -0
- package/dist/matrix/index.d.ts +3 -3
- package/dist/matrix/index.js +3 -2
- package/dist/{matrix-Dm0BQW7U.js → matrix-B8l-Nsxd.js} +3 -88
- package/dist/matrix-B8l-Nsxd.js.map +1 -0
- package/dist/{matrix-DSrSxjjy.d.ts → matrix-CCSBna4q.d.ts} +9 -18
- package/dist/matrix-CCSBna4q.d.ts.map +1 -0
- package/dist/{mean-DHoMiudD.js → mean-BbA_JCzW.js} +39 -5
- package/dist/mean-BbA_JCzW.js.map +1 -0
- package/dist/meta-eval/index.d.ts +5 -5
- package/dist/meta-eval/index.js +10 -10
- package/dist/metrics-BKaPBtIG.js.map +1 -1
- package/dist/{mint-DNU8QPG4.d.ts → mint-BRZXuwRi.d.ts} +4 -4
- package/dist/{mint-DNU8QPG4.d.ts.map → mint-BRZXuwRi.d.ts.map} +1 -1
- package/dist/{mint-QaD5cA0u.js → mint-CbdwjTPW.js} +4 -4
- package/dist/{mint-QaD5cA0u.js.map → mint-CbdwjTPW.js.map} +1 -1
- package/dist/{multi-layer-verifier-BUaQ4C17.d.ts → multi-layer-verifier-DEgMUk6v.d.ts} +2 -2
- package/dist/{multi-layer-verifier-BUaQ4C17.d.ts.map → multi-layer-verifier-DEgMUk6v.d.ts.map} +1 -1
- package/dist/{multiplicity-DQUFcbW4.js → multiplicity-Sg0U7h8j.js} +2 -2
- package/dist/{multiplicity-DQUFcbW4.js.map → multiplicity-Sg0U7h8j.js.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -23
- package/dist/multishot/golden/index.d.ts.map +1 -1
- package/dist/multishot/golden/index.js +3 -3
- package/dist/multishot/golden/index.js.map +1 -1
- package/dist/multishot/index.d.ts +3 -12
- package/dist/multishot/index.d.ts.map +1 -1
- package/dist/multishot/index.js +4 -3
- package/dist/multishot/index.js.map +1 -1
- package/dist/openapi.json +114 -10
- package/dist/{otel-spans-B9tUXRHF.js → otel-spans-CdN909k1.js} +3 -3
- package/dist/{otel-spans-B9tUXRHF.js.map → otel-spans-CdN909k1.js.map} +1 -1
- package/dist/{paired-arms-CwMGJsrS.js → paired-arms-Ct4LTKc1.js} +32 -9
- package/dist/paired-arms-Ct4LTKc1.js.map +1 -0
- package/dist/{paired-binary-CvpFHfKZ.js → paired-binary-B_nejA44.js} +3 -36
- package/dist/paired-binary-B_nejA44.js.map +1 -0
- package/dist/{paired-promotion-decision-DPKz8ZnF.d.ts → paired-promotion-decision-CQMJ5xGN.d.ts} +2 -34
- package/dist/paired-promotion-decision-CQMJ5xGN.d.ts.map +1 -0
- package/dist/{paired-promotion-decision-DsAPYGCm.js → paired-promotion-decision-D0SHb-ZL.js} +3 -3
- package/dist/{paired-promotion-decision-DsAPYGCm.js.map → paired-promotion-decision-D0SHb-ZL.js.map} +1 -1
- package/dist/{pareto-Cm6ndrWN.js → pareto-CTv8vozZ.js} +11 -11
- package/dist/pareto-CTv8vozZ.js.map +1 -0
- package/dist/pipelines/index.d.ts +16 -19
- package/dist/pipelines/index.d.ts.map +1 -1
- package/dist/pipelines/index.js +4 -68
- package/dist/pipelines/index.js.map +1 -1
- package/dist/{power-and-mde-Hj2ubb4B.js → power-and-mde-N5nU4_Ly.js} +3 -3
- package/dist/{power-and-mde-Hj2ubb4B.js.map → power-and-mde-N5nU4_Ly.js.map} +1 -1
- package/dist/{prime-protocol-CHKgA3a6.js → prime-protocol-BFgGrEMP.js} +3 -3
- package/dist/prime-protocol-BFgGrEMP.js.map +1 -0
- package/dist/{prime-protocol-HiB9HX7E.d.ts → prime-protocol-BWG2DQIk.d.ts} +3 -21
- package/dist/prime-protocol-BWG2DQIk.d.ts.map +1 -0
- package/dist/profile-cell.d.ts +2 -2
- package/dist/profile-cell.js +2 -2
- package/dist/{promotion-gate-BawrdPYr.d.ts → promotion-gate-CxEHRsZW.d.ts} +7 -8
- package/dist/{promotion-gate-BawrdPYr.d.ts.map → promotion-gate-CxEHRsZW.d.ts.map} +1 -1
- package/dist/{promotion-policy-KO57gbyV.js → promotion-policy-6A8osGr_.js} +4 -4
- package/dist/{promotion-policy-KO57gbyV.js.map → promotion-policy-6A8osGr_.js.map} +1 -1
- package/dist/{proposal-findings-CY1xKn_a.js → proposal-findings-n-5WBbEx.js} +3 -3
- package/dist/proposal-findings-n-5WBbEx.js.map +1 -0
- package/dist/{query-BUCZCK7F.js → query-B_RZ36an.js} +5 -5
- package/dist/query-B_RZ36an.js.map +1 -0
- package/dist/{query-DK1nWcmp.d.ts → query-Cmv_U8vf.d.ts} +4 -24
- package/dist/{query-DK1nWcmp.d.ts.map → query-Cmv_U8vf.d.ts.map} +1 -1
- package/dist/{random-ByUSnV9w.js → random-DDDcC7YU.js} +2 -2
- package/dist/{random-ByUSnV9w.js.map → random-DDDcC7YU.js.map} +1 -1
- package/dist/{raw-provider-sink-DoHz4JNG.js → raw-provider-sink-75w4FWBj.js} +2 -2
- package/dist/{raw-provider-sink-DoHz4JNG.js.map → raw-provider-sink-75w4FWBj.js.map} +1 -1
- package/dist/{redact-mtCCcT8z.d.ts → redact-DXdlFb_U.d.ts} +2 -3
- package/dist/redact-DXdlFb_U.d.ts.map +1 -0
- package/dist/{redact-B5xXKqQ3.js → redact-mtkTiJVk.js} +2 -7
- package/dist/{redact-B5xXKqQ3.js.map → redact-mtkTiJVk.js.map} +1 -1
- package/dist/{registry-D7_hm2fK.d.ts → registry-ssNoHzwD.d.ts} +4 -4
- package/dist/{registry-D7_hm2fK.d.ts.map → registry-ssNoHzwD.d.ts.map} +1 -1
- package/dist/{release-confidence-CpuP65S5.js → release-confidence-CYIFH-fp.js} +6 -11
- package/dist/release-confidence-CYIFH-fp.js.map +1 -0
- package/dist/{report-command-SC9nqb4M.js → render-Cs34y0iX.js} +625 -587
- package/dist/render-Cs34y0iX.js.map +1 -0
- package/dist/reporting.d.ts +6 -6
- package/dist/reporting.js +8 -8
- package/dist/{researcher-CEcfku-T.d.ts → researcher-GU1tV4dn.d.ts} +9 -9
- package/dist/{researcher-CEcfku-T.d.ts.map → researcher-GU1tV4dn.d.ts.map} +1 -1
- package/dist/{reward-CpCmAM8L.d.ts → reward-DrZ71vnf.d.ts} +3 -26
- package/dist/reward-DrZ71vnf.d.ts.map +1 -0
- package/dist/{reward-hacking-E0d-WVcP.js → reward-hacking-ByuETaIb.js} +3 -3
- package/dist/{reward-hacking-E0d-WVcP.js.map → reward-hacking-ByuETaIb.js.map} +1 -1
- package/dist/{reward-hacking-RVdZ6Kkt.d.ts → reward-hacking-E52wq86n.d.ts} +5 -18
- package/dist/reward-hacking-E52wq86n.d.ts.map +1 -0
- package/dist/reward-nw2xZGZG.js.map +1 -1
- package/dist/rl.d.ts +117 -295
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +27 -747
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +5 -5
- package/dist/rollout/index.js +7 -8
- package/dist/{rollout-DZV1t2Wg.js → rollout-Dd1ucT_M.js} +88 -13
- package/dist/rollout-Dd1ucT_M.js.map +1 -0
- package/dist/{rubric-predictive-validity-DaFW70y9.js → rubric-predictive-validity-D3_oI72N.js} +3 -3
- package/dist/{rubric-predictive-validity-DaFW70y9.js.map → rubric-predictive-validity-D3_oI72N.js.map} +1 -1
- package/dist/{rubric-predictive-validity-C03EC7ZE.d.ts → rubric-predictive-validity-DiwT8PbL.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-C03EC7ZE.d.ts.map → rubric-predictive-validity-DiwT8PbL.d.ts.map} +1 -1
- package/dist/{run-record-C1E24G2A.d.ts → run-record-CaxbLq6Y.d.ts} +4 -10
- package/dist/run-record-CaxbLq6Y.d.ts.map +1 -0
- package/dist/{run-record-C5e7FjhJ.js → run-record-CysKLB93.js} +3 -3
- package/dist/run-record-CysKLB93.js.map +1 -0
- package/dist/{run-record-Boc5IUTp.js → run-record-DhEJgoPz.js} +4 -4
- package/dist/{run-record-Boc5IUTp.js.map → run-record-DhEJgoPz.js.map} +1 -1
- package/dist/sandbox-harness-BlSOu4LX.d.ts.map +1 -1
- package/dist/{schema-CSf6qWgZ.js → schema-Bt2vLFm3.js} +2 -14
- package/dist/schema-Bt2vLFm3.js.map +1 -0
- package/dist/schema-C-ziyJD7.d.ts.map +1 -1
- package/dist/{schema-BuiujZ-g.js → schema-CvhfLjtZ.js} +78 -48
- package/dist/schema-CvhfLjtZ.js.map +1 -0
- package/dist/{schema-2g7trUwH.d.ts → schema-D6EVKzTB.d.ts} +4 -5
- package/dist/schema-D6EVKzTB.d.ts.map +1 -0
- package/dist/search/index.d.ts +3 -31
- package/dist/search/index.d.ts.map +1 -1
- package/dist/search/index.js +4 -4
- package/dist/{search-kernel-DyNEa6IU.d.ts → search-kernel-p9JB3SmL.d.ts} +10 -27
- package/dist/search-kernel-p9JB3SmL.d.ts.map +1 -0
- package/dist/{search-ledger-CVUGLDZp.js → search-ledger-ClpN4I3e.js} +7 -7
- package/dist/{search-ledger-CVUGLDZp.js.map → search-ledger-ClpN4I3e.js.map} +1 -1
- package/dist/{search-ledger-file-DDQUgvCg.js → search-ledger-file-Bd5Tioq6.js} +4 -4
- package/dist/search-ledger-file-Bd5Tioq6.js.map +1 -0
- package/dist/{search-shipper-BOSD46qv.js → search-shipper-Bg2HW3Uv.js} +4 -4
- package/dist/search-shipper-Bg2HW3Uv.js.map +1 -0
- package/dist/{search-summary-BQCSWf1g.js → search-summary-CrBuvVnf.js} +5 -5
- package/dist/{search-summary-BQCSWf1g.js.map → search-summary-CrBuvVnf.js.map} +1 -1
- package/dist/{sequential-CzK5DarL.js → sequential-BVv1sjsw.js} +2 -32
- package/dist/{sequential-CzK5DarL.js.map → sequential-BVv1sjsw.js.map} +1 -1
- package/dist/{sequential-Bf_pbE50.js → sequential-CimQrbjv.js} +5 -5
- package/dist/sequential-CimQrbjv.js.map +1 -0
- package/dist/{sequential-BhsrMupG.d.ts → sequential-KwGspfeR.d.ts} +2 -9
- package/dist/{sequential-BhsrMupG.d.ts.map → sequential-KwGspfeR.d.ts.map} +1 -1
- package/dist/{sequential-eprocess-DM_qENCl.js → sequential-eprocess-DodI6RoF.js} +2 -2
- package/dist/{sequential-eprocess-DM_qENCl.js.map → sequential-eprocess-DodI6RoF.js.map} +1 -1
- package/dist/{server-e9oWO_bN.js → server-C6M0qR8h.js} +101 -101
- package/dist/server-C6M0qR8h.js.map +1 -0
- package/dist/{single-run-lock-CV9Eyd_q.js → single-run-lock-Dekd8_Wv.js} +7 -7
- package/dist/single-run-lock-Dekd8_Wv.js.map +1 -0
- package/dist/{skillopt-optimization-method-BiwA8nfX.js → skillopt-optimization-method-5dlex7MJ.js} +1386 -1184
- package/dist/skillopt-optimization-method-5dlex7MJ.js.map +1 -0
- package/dist/{spans-BHAUXMGN.d.ts → spans-CVngGij3.d.ts} +2 -2
- package/dist/{spans-BHAUXMGN.d.ts.map → spans-CVngGij3.d.ts.map} +1 -1
- package/dist/{statistical-heldout-CYpGaoDH.d.ts → statistical-heldout-BHfLZHrm.d.ts} +8 -67
- package/dist/statistical-heldout-BHfLZHrm.d.ts.map +1 -0
- package/dist/{statistical-heldout-42NlCNDe.js → statistical-heldout-Bksv92Th.js} +3 -3
- package/dist/statistical-heldout-Bksv92Th.js.map +1 -0
- package/dist/{steps-Bg5-KVx1.d.ts → steps-DlepJPwD.d.ts} +2 -56
- package/dist/steps-DlepJPwD.d.ts.map +1 -0
- package/dist/{store-C4avFjIi.d.ts → store-B3nWRy3B.d.ts} +2 -2
- package/dist/{store-C4avFjIi.d.ts.map → store-B3nWRy3B.d.ts.map} +1 -1
- package/dist/{store-otlp-DJ9dtA5x.js → store-otlp-BteDAbHK.js} +3 -3
- package/dist/{store-otlp-DJ9dtA5x.js.map → store-otlp-BteDAbHK.js.map} +1 -1
- package/dist/{store-tool-spans-CIX5arAG.js → store-tool-spans-D5l_OfTJ.js} +8 -8
- package/dist/store-tool-spans-D5l_OfTJ.js.map +1 -0
- package/dist/{store-tool-spans-DdCSPa7C.d.ts → store-tool-spans-DZS0HWVf.d.ts} +9 -18
- package/dist/store-tool-spans-DZS0HWVf.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts +4 -11
- package/dist/storyboard/index.d.ts.map +1 -1
- package/dist/storyboard/index.js +1 -16
- package/dist/storyboard/index.js.map +1 -1
- package/dist/{summary-report-V_JACR4Y.js → summary-report-BsXJoYX5.js} +88 -10
- package/dist/summary-report-BsXJoYX5.js.map +1 -0
- package/dist/{summary-report-bdFDXq0b.d.ts → summary-report-CkbRcSuu.d.ts} +25 -19
- package/dist/summary-report-CkbRcSuu.d.ts.map +1 -0
- package/dist/supervisor-run/index.d.ts +60 -28
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +4 -458
- package/dist/supervisor-run/index.js.map +1 -1
- package/dist/{systemone-BomhHr2c.d.ts → systemone-BMzDJb1i.d.ts} +5 -5
- package/dist/{systemone-BomhHr2c.d.ts.map → systemone-BMzDJb1i.d.ts.map} +1 -1
- package/dist/systemone-protocol.js +1 -1
- package/dist/{systemone-review-CUVcQJu6.d.ts → systemone-review-CwE6JGIp.d.ts} +2 -2
- package/dist/{systemone-review-CUVcQJu6.d.ts.map → systemone-review-CwE6JGIp.d.ts.map} +1 -1
- package/dist/{systemone-review-D7sXwsqq.js → systemone-review-DzdOJ8I6.js} +3 -3
- package/dist/{systemone-review-D7sXwsqq.js.map → systemone-review-DzdOJ8I6.js.map} +1 -1
- package/dist/systemone.d.ts +5 -5
- package/dist/systemone.js +5 -5
- package/dist/systemone.js.map +1 -1
- package/dist/{task-failure-attributes-hEmASjix.js → task-failure-attributes-BMsblaDh.js} +3 -3
- package/dist/task-failure-attributes-BMsblaDh.js.map +1 -0
- package/dist/{terminal-record-DZyQZ3pz.js → terminal-record-D4Rv-j1W.js} +244 -196
- package/dist/terminal-record-D4Rv-j1W.js.map +1 -0
- package/dist/{tool-groups-C3NEcdCM.d.ts → tool-groups-DyNz2JZ_.d.ts} +3 -3
- package/dist/tool-groups-DyNz2JZ_.d.ts.map +1 -0
- package/dist/{tool-waste-BKgvd-TP.d.ts → tool-waste-CkjzWmeY.d.ts} +25 -28
- package/dist/tool-waste-CkjzWmeY.d.ts.map +1 -0
- package/dist/{tool-waste-BlesqTqW.js → tool-waste-DLsjS_p-.js} +5 -5
- package/dist/{tool-waste-BlesqTqW.js.map → tool-waste-DLsjS_p-.js.map} +1 -1
- package/dist/{trace-questions-CM2CAocU.d.ts → trace-questions-xxwgsuMi.d.ts} +4 -9
- package/dist/{trace-questions-CM2CAocU.d.ts.map → trace-questions-xxwgsuMi.d.ts.map} +1 -1
- package/dist/trace-repair/index.d.ts +14 -211
- package/dist/trace-repair/index.d.ts.map +1 -1
- package/dist/trace-repair/index.js +865 -981
- package/dist/trace-repair/index.js.map +1 -1
- package/dist/traces.d.ts +18 -46
- package/dist/traces.d.ts.map +1 -1
- package/dist/traces.js +13 -13
- package/dist/traces.js.map +1 -1
- package/dist/trajectory-replay/index.d.ts +7 -118
- package/dist/trajectory-replay/index.d.ts.map +1 -1
- package/dist/trajectory-replay/index.js +921 -869
- package/dist/trajectory-replay/index.js.map +1 -1
- package/dist/{tree-C1XYeXvy.js → tree-B5e1uAeL.js} +132 -21
- package/dist/tree-B5e1uAeL.js.map +1 -0
- package/dist/{types-Cbe2fWBa.d.ts → types-BKCzzmit.d.ts} +3 -3
- package/dist/{types-Cbe2fWBa.d.ts.map → types-BKCzzmit.d.ts.map} +1 -1
- package/dist/{types-B3jzCp0p.js → types-BXgASBCL.js} +2 -2
- package/dist/{types-B3jzCp0p.js.map → types-BXgASBCL.js.map} +1 -1
- package/dist/{types-C1tnkEgM.d.ts → types-CW8ewATZ.d.ts} +4 -113
- package/dist/types-CW8ewATZ.d.ts.map +1 -0
- package/dist/{types-BqkVmHfv.d.ts → types-CmXY0PQf.d.ts} +2 -4
- package/dist/types-CmXY0PQf.d.ts.map +1 -0
- package/dist/{types-C02qtnG8.d.ts → types-D-mgRzlh.d.ts} +7 -28
- package/dist/types-D-mgRzlh.d.ts.map +1 -0
- package/dist/{types-IXD2HJzu.js → types-DqFMNTgK.js} +2 -4
- package/dist/types-DqFMNTgK.js.map +1 -0
- package/dist/{types-Cq8mSbq8.d.ts → types-DtkO1G9L.d.ts} +7 -9
- package/dist/{types-Cq8mSbq8.d.ts.map → types-DtkO1G9L.d.ts.map} +1 -1
- package/dist/{verdict-Dl0tiT1X.js → verdict-BGQmxQ8A.js} +3 -3
- package/dist/verdict-BGQmxQ8A.js.map +1 -0
- package/dist/{verdict-E4eRNf7-.d.ts → verdict-C24NELtN.d.ts} +2 -15
- package/dist/{verdict-E4eRNf7-.d.ts.map → verdict-C24NELtN.d.ts.map} +1 -1
- package/dist/{verdict-cache-Bm8EKL2s.js → verdict-cache-CL_7tePD.js} +2 -2
- package/dist/verdict-cache-CL_7tePD.js.map +1 -0
- package/dist/wire/index.d.ts +31 -410
- package/dist/wire/index.d.ts.map +1 -1
- package/dist/wire/index.js +2 -2
- package/docs/changes/jev-evidence-review.md +1 -1
- package/docs/changes/jev-product-adoption.md +1 -1
- package/docs/changes/jev-recorded-evidence.md +1 -1
- package/docs/design/statistics-decisions.md +18 -18
- package/docs/design.md +1 -1
- package/docs/eval-surface-map.md +1 -0
- package/docs/evaluation-integrity.md +1 -1
- package/docs/profile-json-identity.md +1 -2
- package/docs/proofs/campaign-attempt-retention-20261001/README.md +2 -2
- package/docs/public-api.md +158 -538
- package/docs/rollout.md +1 -1
- package/docs/trace-repair-admission.md +1 -1
- package/docs/wire-protocol.md +1 -1
- package/package.json +7 -2
- package/dist/active-curriculum-B_Lc9kfO.js +0 -61
- package/dist/active-curriculum-B_Lc9kfO.js.map +0 -1
- package/dist/agent-profile-VFcQncbW.d.ts.map +0 -1
- package/dist/agent-profile-cell-BK8mlqlq.js.map +0 -1
- package/dist/agent-profile-cell-BNp-Js9F.d.ts.map +0 -1
- package/dist/agreement-irr-BEWLIF3_.js.map +0 -1
- package/dist/analyze-runs-C84irm4i.js.map +0 -1
- package/dist/baseline-Udi_PLuT.js +0 -259
- package/dist/baseline-Udi_PLuT.js.map +0 -1
- package/dist/benchmark-CXmVVHH2.js +0 -562
- package/dist/benchmark-CXmVVHH2.js.map +0 -1
- package/dist/benchmark-command-By0vh8q7.js.map +0 -1
- package/dist/campaign-BudZ70zo.js.map +0 -1
- package/dist/chat-json-call-puMArmtW.js.map +0 -1
- package/dist/claude-jsonl-dZir12m6.js +0 -86
- package/dist/claude-jsonl-dZir12m6.js.map +0 -1
- package/dist/client-BzL9niqi.d.ts.map +0 -1
- package/dist/cost-ledger-BP8V4kUQ.js.map +0 -1
- package/dist/counterfactual-CPaWN99-.d.ts +0 -60
- package/dist/counterfactual-CPaWN99-.d.ts.map +0 -1
- package/dist/counterfactual-Dn3veueY.js +0 -98
- package/dist/counterfactual-Dn3veueY.js.map +0 -1
- package/dist/default-registry-COhk9g36.js.map +0 -1
- package/dist/default-registry-DzHiGruW.d.ts.map +0 -1
- package/dist/define-agent-eval-CNTPzIkp.js.map +0 -1
- package/dist/define-agent-eval-WSck0CZJ.d.ts.map +0 -1
- package/dist/dspy-rlm-engine-LRCu61m5.js.map +0 -1
- package/dist/effect-sizes-BhzEpXMc.js +0 -82
- package/dist/effect-sizes-BhzEpXMc.js.map +0 -1
- package/dist/emitter-HcCmFQl_.d.ts.map +0 -1
- package/dist/engine-3A_4jOlI.d.ts.map +0 -1
- package/dist/equal-terms-BtdXayCJ.js.map +0 -1
- package/dist/eval-campaign-90jeaCAI.js +0 -343
- package/dist/eval-campaign-90jeaCAI.js.map +0 -1
- package/dist/evaluation-6Yoi74gM.d.ts.map +0 -1
- package/dist/exporters-X4xwla8A.js.map +0 -1
- package/dist/external-optimizer-process-BjMGUuds.js.map +0 -1
- package/dist/external-optimizer-subprocess-BqW0hHcI.js.map +0 -1
- package/dist/failure-cluster-CbgEDcFJ.d.ts.map +0 -1
- package/dist/feedback-trajectory-C4EG0nNH.d.ts.map +0 -1
- package/dist/hf-dataset-BR2yhuNn.js.map +0 -1
- package/dist/index-BFRO6chs.d.ts.map +0 -1
- package/dist/index-BO41eGj5.d.ts.map +0 -1
- package/dist/index-CyszSg4O.d.ts.map +0 -1
- package/dist/index-DHKEAv3s.d.ts.map +0 -1
- package/dist/index-DPcngtwt.d.ts.map +0 -1
- package/dist/integrity-dLiDe2kD.js.map +0 -1
- package/dist/journal-DfRz9kps.js.map +0 -1
- package/dist/judge-gate-CE7CQAgt.d.ts.map +0 -1
- package/dist/kind-factory-BobI5Jy3.js.map +0 -1
- package/dist/ledger-core-Gq7yp8lP.js.map +0 -1
- package/dist/llm-client-CyBAoYiX.js.map +0 -1
- package/dist/llm-judge-CG12SQJq.js.map +0 -1
- package/dist/matrix-DSrSxjjy.d.ts.map +0 -1
- package/dist/matrix-Dm0BQW7U.js.map +0 -1
- package/dist/mean-DHoMiudD.js.map +0 -1
- package/dist/normal-DsJZtrJc.js +0 -25
- package/dist/normal-DsJZtrJc.js.map +0 -1
- package/dist/paired-arms-CwMGJsrS.js.map +0 -1
- package/dist/paired-binary-CvpFHfKZ.js.map +0 -1
- package/dist/paired-promotion-decision-DPKz8ZnF.d.ts.map +0 -1
- package/dist/pareto-Cm6ndrWN.js.map +0 -1
- package/dist/prime-protocol-CHKgA3a6.js.map +0 -1
- package/dist/prime-protocol-HiB9HX7E.d.ts.map +0 -1
- package/dist/produced-state-BsDmnwxc.js.map +0 -1
- package/dist/proposal-findings-CY1xKn_a.js.map +0 -1
- package/dist/query-BUCZCK7F.js.map +0 -1
- package/dist/redact-mtCCcT8z.d.ts.map +0 -1
- package/dist/release-confidence-CpuP65S5.js.map +0 -1
- package/dist/report-command-SC9nqb4M.js.map +0 -1
- package/dist/reward-CpCmAM8L.d.ts.map +0 -1
- package/dist/reward-hacking-RVdZ6Kkt.d.ts.map +0 -1
- package/dist/rollout-DZV1t2Wg.js.map +0 -1
- package/dist/run-record-C1E24G2A.d.ts.map +0 -1
- package/dist/run-record-C5e7FjhJ.js.map +0 -1
- package/dist/schema-2g7trUwH.d.ts.map +0 -1
- package/dist/schema-BuiujZ-g.js.map +0 -1
- package/dist/schema-CSf6qWgZ.js.map +0 -1
- package/dist/search-kernel-DyNEa6IU.d.ts.map +0 -1
- package/dist/search-ledger-file-DDQUgvCg.js.map +0 -1
- package/dist/search-shipper-BOSD46qv.js.map +0 -1
- package/dist/semantic-concept-judge-BQF8te0T.js.map +0 -1
- package/dist/sequential-Bf_pbE50.js.map +0 -1
- package/dist/server-e9oWO_bN.js.map +0 -1
- package/dist/single-run-lock-CV9Eyd_q.js.map +0 -1
- package/dist/skillopt-optimization-method-BiwA8nfX.js.map +0 -1
- package/dist/statistical-heldout-42NlCNDe.js.map +0 -1
- package/dist/statistical-heldout-CYpGaoDH.d.ts.map +0 -1
- package/dist/steps-Bg5-KVx1.d.ts.map +0 -1
- package/dist/store-tool-spans-CIX5arAG.js.map +0 -1
- package/dist/store-tool-spans-DdCSPa7C.d.ts.map +0 -1
- package/dist/student-t-DVPW7CdL.js +0 -39
- package/dist/student-t-DVPW7CdL.js.map +0 -1
- package/dist/summary-report-V_JACR4Y.js.map +0 -1
- package/dist/summary-report-bdFDXq0b.d.ts.map +0 -1
- package/dist/task-failure-attributes-hEmASjix.js.map +0 -1
- package/dist/terminal-record-DZyQZ3pz.js.map +0 -1
- package/dist/tool-groups-C3NEcdCM.d.ts.map +0 -1
- package/dist/tool-waste-BKgvd-TP.d.ts.map +0 -1
- package/dist/trajectory-Y1fmdlw7.d.ts +0 -24
- package/dist/trajectory-Y1fmdlw7.d.ts.map +0 -1
- package/dist/tree-C1XYeXvy.js.map +0 -1
- package/dist/types-BqkVmHfv.d.ts.map +0 -1
- package/dist/types-C02qtnG8.d.ts.map +0 -1
- package/dist/types-C1tnkEgM.d.ts.map +0 -1
- package/dist/types-IXD2HJzu.js.map +0 -1
- package/dist/usage-receipt-Cx3xNeyB.js +0 -76
- package/dist/usage-receipt-Cx3xNeyB.js.map +0 -1
- package/dist/verdict-Dl0tiT1X.js.map +0 -1
- package/dist/verdict-cache-Bm8EKL2s.js.map +0 -1
- package/docs/design/self-improvement-evidence-audit.md +0 -263
|
@@ -1,24 +1,25 @@
|
|
|
1
|
-
import { i as JudgeError, s as ValidationError } from "./errors-
|
|
2
|
-
import { a as hashCanonical, i as compareCodeUnits, r as canonicalString, t as LEDGER_HASH_PATTERN } from "./canonical-
|
|
3
|
-
import { i as modelHasSnapshot, s as searchCellRunId } from "./run-record-
|
|
4
|
-
import { t as mulberry32 } from "./random-
|
|
5
|
-
import { o as summarizeNumberSeries, s as weightedComposite, t as confidenceInterval } from "./descriptive-
|
|
6
|
-
import { r as minimumPairsForPairedDeltaTest } from "./paired-promotion-decision-
|
|
7
|
-
import { a as heldoutSignificance, i as dimensionRegressions, o as pairHoldout } from "./statistical-heldout-
|
|
8
|
-
import { n as contentHash } from "./verdict-cache-
|
|
9
|
-
import { r as projectCampaignCellQuality, t as campaignCellCostProvenance } from "./run-record-
|
|
10
|
-
import { n as CostAccountingIncompleteError, t as CostLedger } from "./cost-ledger-
|
|
11
|
-
import {
|
|
1
|
+
import { i as JudgeError, s as ValidationError } from "./errors-BcOs5MKW.js";
|
|
2
|
+
import { a as hashCanonical, i as compareCodeUnits, r as canonicalString, t as LEDGER_HASH_PATTERN } from "./canonical-Dq2hOQ8L.js";
|
|
3
|
+
import { i as modelHasSnapshot, s as searchCellRunId } from "./run-record-DhEJgoPz.js";
|
|
4
|
+
import { t as mulberry32 } from "./random-DDDcC7YU.js";
|
|
5
|
+
import { o as summarizeNumberSeries, s as weightedComposite, t as confidenceInterval } from "./descriptive-CVNy48rR.js";
|
|
6
|
+
import { r as minimumPairsForPairedDeltaTest } from "./paired-promotion-decision-D0SHb-ZL.js";
|
|
7
|
+
import { a as heldoutSignificance, i as dimensionRegressions, o as pairHoldout } from "./statistical-heldout-Bksv92Th.js";
|
|
8
|
+
import { n as contentHash } from "./verdict-cache-CL_7tePD.js";
|
|
9
|
+
import { r as projectCampaignCellQuality, t as campaignCellCostProvenance } from "./run-record-CysKLB93.js";
|
|
10
|
+
import { n as CostAccountingIncompleteError, t as CostLedger } from "./cost-ledger-D0352Z4d.js";
|
|
11
|
+
import { A as surfaceHashMatches, D as surfaceContentHash, E as renderSurfaceDiff, F as assertCampaignSplitIdentity, I as assertCompleteCampaign, L as campaignCoverage, O as surfaceDispatchRef, P as assertCampaignDesign, R as campaignScenarioIdentity, V as formatCoverageFailures, b as FinalEvidenceConflictError, f as campaignBreakdown, j as BackendIntegrityError, k as surfaceHash, m as campaignMeanCompositeOrNull, p as campaignMeanComposite, x as FinalEvidenceError, z as campaignSplitDigest } from "./campaign-evidence-DunzcJGv.js";
|
|
12
12
|
import { t as deepFreezeCanonicalJson } from "./deep-freeze-CurcOQg3.js";
|
|
13
|
-
import {
|
|
14
|
-
import { r as summarizeEvaluationUnits, t as defineEvaluationClaim } from "./claim-
|
|
15
|
-
import {
|
|
16
|
-
import { t as detectRewardHacking } from "./reward-hacking-
|
|
17
|
-
import { t as assertProposalFindings } from "./proposal-findings-
|
|
18
|
-
import { i as runSearch, m as searchExpansionIndex, n as paretoFrontier, o as searchPolicyView, p as SEARCH_KERNEL_SOURCE, r as paretoFrontierWithCrowding, s as EXPANSION_OPERATORS, w as estimateNode } from "./pareto-
|
|
19
|
-
import { a as validateSearchLedgerEvent, d as searchNodeId, f as searchTaskSetDigest, i as replaySearchLedgerText, n as openSearchLedger, u as searchCellId } from "./search-ledger-
|
|
20
|
-
import { n as searchProposerView, t as renderSearchSummary } from "./search-summary-
|
|
21
|
-
import {
|
|
13
|
+
import { M as fsCampaignStorage, h as isExternalTextCandidate, j as createRunCostLedger } from "./external-optimizer-subprocess-B5gdLRZv.js";
|
|
14
|
+
import { r as summarizeEvaluationUnits, t as defineEvaluationClaim } from "./claim-BPVuyHuy.js";
|
|
15
|
+
import { d as detectPersonalData, s as redactText, t as REDACTION_VERSION, u as detectCredential } from "./redact-mtkTiJVk.js";
|
|
16
|
+
import { t as detectRewardHacking } from "./reward-hacking-ByuETaIb.js";
|
|
17
|
+
import { t as assertProposalFindings } from "./proposal-findings-n-5WBbEx.js";
|
|
18
|
+
import { i as runSearch, m as searchExpansionIndex, n as paretoFrontier, o as searchPolicyView, p as SEARCH_KERNEL_SOURCE, r as paretoFrontierWithCrowding, s as EXPANSION_OPERATORS, w as estimateNode } from "./pareto-CTv8vozZ.js";
|
|
19
|
+
import { a as validateSearchLedgerEvent, d as searchNodeId, f as searchTaskSetDigest, i as replaySearchLedgerText, n as openSearchLedger, u as searchCellId } from "./search-ledger-ClpN4I3e.js";
|
|
20
|
+
import { n as searchProposerView, t as renderSearchSummary } from "./search-summary-CrBuvVnf.js";
|
|
21
|
+
import { i as stripFencedJson, n as isRecord } from "./json-BhdsJaOQ.js";
|
|
22
|
+
import { a as maximumChargeForLlmRequest, i as costReceiptFromLlmError, r as costReceiptFromLlm } from "./llm-client-_Y6ic-tf.js";
|
|
22
23
|
import { createHash } from "node:crypto";
|
|
23
24
|
import { writeFileSync } from "node:fs";
|
|
24
25
|
import { z } from "zod";
|
|
@@ -336,52 +337,41 @@ function hasLlmCall(value) {
|
|
|
336
337
|
* persisted cell result. `runCampaign` owns the schedule; this module owns
|
|
337
338
|
* everything that happens inside a single cell.
|
|
338
339
|
*/
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
const
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
340
|
+
/** A cached cell is reused only when its receipts still match the ledger; invalid cache entries need an explicit rerun. */
|
|
341
|
+
function reuseCachedCell(args, cachePath, stableCostTags) {
|
|
342
|
+
const cached = readCachedCell({
|
|
343
|
+
storage: args.storage,
|
|
344
|
+
cachePath,
|
|
345
|
+
cellId: args.slot.cellId,
|
|
346
|
+
manifestHash: args.manifestHash
|
|
347
|
+
});
|
|
348
|
+
if (cached.status === "miss" && cacheIssueRequiresExplicitRerun(cached.reason) && !args.opts.rerunInvalidCachedCells) throw invalidCachedCellsError([{
|
|
349
|
+
cellId: args.slot.cellId,
|
|
350
|
+
reason: cached.reason
|
|
351
|
+
}]);
|
|
352
|
+
if (cached.status === "hit" && (!cached.cell.error || args.opts.reuseFailedCells)) {
|
|
353
|
+
const receiptProblem = cachedCellReceiptProblem(cached.cell, args.costLedger, stableCostTags);
|
|
354
|
+
if (receiptProblem === void 0) {
|
|
355
|
+
const cell = withCurrentAgentModelEvidence(cached.cell, args.costLedger, stableCostTags);
|
|
356
|
+
enforceDispatchUsage(cell, args.opts.expectUsage ?? "warn");
|
|
357
|
+
return {
|
|
358
|
+
cell: {
|
|
359
|
+
...cell,
|
|
360
|
+
cached: true
|
|
361
|
+
},
|
|
362
|
+
artifactsByPath: {}
|
|
363
|
+
};
|
|
364
|
+
}
|
|
365
|
+
if (!args.opts.rerunInvalidCachedCells) throw invalidCachedCellsError([{
|
|
361
366
|
cellId: args.slot.cellId,
|
|
362
|
-
reason:
|
|
367
|
+
reason: "invalid-cost-receipts",
|
|
368
|
+
detail: receiptProblem
|
|
363
369
|
}]);
|
|
364
|
-
if (cached.status === "hit" && (!cached.cell.error || args.opts.reuseFailedCells)) {
|
|
365
|
-
const receiptProblem = cachedCellReceiptProblem(cached.cell, args.costLedger, stableCostTags);
|
|
366
|
-
if (receiptProblem === void 0) {
|
|
367
|
-
const cell = withCurrentAgentModelEvidence(cached.cell, args.costLedger, stableCostTags);
|
|
368
|
-
enforceDispatchUsage(cell, args.opts.expectUsage ?? "warn");
|
|
369
|
-
return {
|
|
370
|
-
cell: {
|
|
371
|
-
...cell,
|
|
372
|
-
cached: true
|
|
373
|
-
},
|
|
374
|
-
artifactsByPath: {}
|
|
375
|
-
};
|
|
376
|
-
}
|
|
377
|
-
if (!args.opts.rerunInvalidCachedCells) throw invalidCachedCellsError([{
|
|
378
|
-
cellId: args.slot.cellId,
|
|
379
|
-
reason: "invalid-cost-receipts",
|
|
380
|
-
detail: receiptProblem
|
|
381
|
-
}]);
|
|
382
|
-
}
|
|
383
370
|
}
|
|
384
|
-
|
|
371
|
+
}
|
|
372
|
+
/** The directories, writers, abort chain and dispatch context of one execution attempt. */
|
|
373
|
+
function openAttempt(args, cellDir, attemptCostTags) {
|
|
374
|
+
const { storage } = args;
|
|
385
375
|
const attempt = {
|
|
386
376
|
runAttemptId: args.runAttemptId,
|
|
387
377
|
number: args.attempt
|
|
@@ -406,7 +396,7 @@ async function executeCell(args) {
|
|
|
406
396
|
artifactsByPath[`${args.slot.cellId}/${attemptPrefix}/identity.json`] = identityPath;
|
|
407
397
|
storage.write(join(cellDir, "latest-attempt.json"), JSON.stringify(attempt));
|
|
408
398
|
const trace = args.buildTraceWriter(args.slot.cellId, traceDir);
|
|
409
|
-
|
|
399
|
+
const paid = { started: false };
|
|
410
400
|
const artifacts = {
|
|
411
401
|
async write(path, content) {
|
|
412
402
|
const fullPath = resolve(artifactDir, path);
|
|
@@ -421,8 +411,10 @@ async function executeCell(args) {
|
|
|
421
411
|
return artifacts.write(path, JSON.stringify(value, null, 2));
|
|
422
412
|
}
|
|
423
413
|
};
|
|
414
|
+
const cellAbort = new AbortController();
|
|
415
|
+
const onCampaignAbort = () => cellAbort.abort(args.signal.reason);
|
|
424
416
|
const cost = { async runPaidCall(input) {
|
|
425
|
-
|
|
417
|
+
paid.started = true;
|
|
426
418
|
const result = await args.costLedger.runPaidCall({
|
|
427
419
|
...input,
|
|
428
420
|
channel: input.channel ?? "agent",
|
|
@@ -438,25 +430,36 @@ async function executeCell(args) {
|
|
|
438
430
|
scenario: args.slot.scenario,
|
|
439
431
|
rep: args.slot.rep
|
|
440
432
|
});
|
|
441
|
-
const cellAbort = new AbortController();
|
|
442
|
-
const onCampaignAbort = () => cellAbort.abort(args.signal.reason);
|
|
443
433
|
if (args.signal.aborted) cellAbort.abort(args.signal.reason);
|
|
444
434
|
else args.signal.addEventListener("abort", onCampaignAbort, { once: true });
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
signal: cellAbort.signal,
|
|
435
|
+
return {
|
|
436
|
+
attempt,
|
|
437
|
+
attemptDir,
|
|
438
|
+
attemptPrefix,
|
|
439
|
+
artifactsByPath,
|
|
451
440
|
trace,
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
441
|
+
paid,
|
|
442
|
+
cellAbort,
|
|
443
|
+
onCampaignAbort,
|
|
444
|
+
ctx: {
|
|
445
|
+
cellId: args.slot.cellId,
|
|
446
|
+
runAttemptId: args.runAttemptId,
|
|
447
|
+
rep: args.slot.rep,
|
|
448
|
+
seed: args.slot.cellSeed,
|
|
449
|
+
signal: cellAbort.signal,
|
|
450
|
+
trace,
|
|
451
|
+
artifacts,
|
|
452
|
+
cost,
|
|
453
|
+
placement
|
|
454
|
+
}
|
|
455
455
|
};
|
|
456
|
+
}
|
|
457
|
+
/** Dispatch the scenario under the per-cell abort and the dispatch deadline. A failure is recorded, not thrown. */
|
|
458
|
+
async function dispatchCell(args, run) {
|
|
459
|
+
const { cellAbort, onCampaignAbort, ctx } = run;
|
|
456
460
|
let artifact;
|
|
457
461
|
let errorMessage;
|
|
458
462
|
let failure;
|
|
459
|
-
let fatalCellError;
|
|
460
463
|
let dispatched;
|
|
461
464
|
const timeoutMs = args.dispatchTimeoutMs;
|
|
462
465
|
let timeoutTimer;
|
|
@@ -499,29 +502,41 @@ async function executeCell(args) {
|
|
|
499
502
|
removeAbortListener();
|
|
500
503
|
args.signal.removeEventListener("abort", onCampaignAbort);
|
|
501
504
|
}
|
|
505
|
+
return {
|
|
506
|
+
artifact,
|
|
507
|
+
errorMessage,
|
|
508
|
+
failure,
|
|
509
|
+
dispatched
|
|
510
|
+
};
|
|
511
|
+
}
|
|
512
|
+
/** A dispatch that ignored cancellation, or paid calls that never settled, mean no trustworthy cell result exists. */
|
|
513
|
+
async function awaitCellShutdown(args, run, dispatched, costTags) {
|
|
514
|
+
const { trace, paid } = run;
|
|
502
515
|
if (dispatched) {
|
|
503
516
|
if (!await settlesWithin(dispatched, args.dispatchShutdownTimeoutMs)) {
|
|
504
517
|
await trace.flush();
|
|
505
518
|
throw new CostAccountingIncompleteError(`dispatch for cell '${args.slot.cellId}' ignored cancellation and did not stop within ${args.dispatchShutdownTimeoutMs}ms; no campaign result was produced`);
|
|
506
519
|
}
|
|
507
520
|
}
|
|
508
|
-
if (
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
}
|
|
520
|
-
})) {
|
|
521
|
-
await trace.flush();
|
|
522
|
-
throw new CostAccountingIncompleteError(`paid calls for cell '${args.slot.cellId}' did not settle within ${args.dispatchShutdownTimeoutMs}ms; no campaign result was produced`);
|
|
521
|
+
if (!paid.started) return;
|
|
522
|
+
if (typeof args.costLedger.waitForIdle !== "function") {
|
|
523
|
+
await trace.flush();
|
|
524
|
+
throw new CostAccountingIncompleteError(`cost ledger for cell '${args.slot.cellId}' cannot prove that paid calls stopped`);
|
|
525
|
+
}
|
|
526
|
+
if (!await args.costLedger.waitForIdle({
|
|
527
|
+
timeoutMs: args.dispatchShutdownTimeoutMs,
|
|
528
|
+
filter: {
|
|
529
|
+
channel: "agent",
|
|
530
|
+
phase: args.costPhase,
|
|
531
|
+
tags: costTags
|
|
523
532
|
}
|
|
533
|
+
})) {
|
|
534
|
+
await trace.flush();
|
|
535
|
+
throw new CostAccountingIncompleteError(`paid calls for cell '${args.slot.cellId}' did not settle within ${args.dispatchShutdownTimeoutMs}ms; no campaign result was produced`);
|
|
524
536
|
}
|
|
537
|
+
}
|
|
538
|
+
/** The agent spend of this attempt: cost, token usage and the models that served it. */
|
|
539
|
+
function measureAgentUsage(args, costTags) {
|
|
525
540
|
const agentReceipts = args.costLedger.list({
|
|
526
541
|
channel: "agent",
|
|
527
542
|
tags: costTags
|
|
@@ -530,29 +545,29 @@ async function executeCell(args) {
|
|
|
530
545
|
channel: "agent",
|
|
531
546
|
tags: costTags
|
|
532
547
|
});
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
artifact,
|
|
545
|
-
error: errorMessage,
|
|
546
|
-
costUsd: agentCost.totalCostUsd,
|
|
547
|
-
tokenUsage
|
|
548
|
+
return {
|
|
549
|
+
agentCost,
|
|
550
|
+
tokenUsage: {
|
|
551
|
+
input: agentCost.inputTokens,
|
|
552
|
+
output: agentCost.outputTokens,
|
|
553
|
+
...agentCost.usageComplete ? {} : { tokensKnown: false },
|
|
554
|
+
...agentCost.reasoningTokens !== void 0 && agentCost.reasoningTokens > 0 ? { reasoning: agentCost.reasoningTokens } : {},
|
|
555
|
+
...agentCost.cachedTokens > 0 ? { cached: agentCost.cachedTokens } : {},
|
|
556
|
+
...agentCost.cacheWriteTokens !== void 0 && agentCost.cacheWriteTokens > 0 ? { cacheWrite: agentCost.cacheWriteTokens } : {}
|
|
557
|
+
},
|
|
558
|
+
agentModelEvidence: modelEvidenceFromReceipts(agentReceipts)
|
|
548
559
|
};
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
560
|
+
}
|
|
561
|
+
/**
|
|
562
|
+
* Run judges (only if we have an artifact). A judge that throws invalidates
|
|
563
|
+
* the cell — recorded as `error`, NOT folded into a fake composite:0 (a fake
|
|
564
|
+
* zero is indistinguishable from a real zero and poisons every aggregate).
|
|
565
|
+
*/
|
|
566
|
+
async function judgeArtifact(args, artifact, attemptCostTags) {
|
|
555
567
|
const judgeScores = {};
|
|
568
|
+
let errorMessage;
|
|
569
|
+
let failure;
|
|
570
|
+
let fatalCellError;
|
|
556
571
|
if (artifact !== void 0) for (const judge of args.opts.judges ?? []) {
|
|
557
572
|
if (judge.appliesTo && !judge.appliesTo(args.slot.scenario)) continue;
|
|
558
573
|
try {
|
|
@@ -576,6 +591,79 @@ async function executeCell(args) {
|
|
|
576
591
|
break;
|
|
577
592
|
}
|
|
578
593
|
}
|
|
594
|
+
return {
|
|
595
|
+
judgeScores,
|
|
596
|
+
errorMessage,
|
|
597
|
+
failure,
|
|
598
|
+
fatalCellError
|
|
599
|
+
};
|
|
600
|
+
}
|
|
601
|
+
/** Write the failure receipt and decide whether `cellRetry` dispatches this slot again. */
|
|
602
|
+
function recordCellFailure(args, run, failure, cell, costTags, fatalCellError) {
|
|
603
|
+
const { attemptDir, attemptPrefix, artifactsByPath } = run;
|
|
604
|
+
const receipt = {
|
|
605
|
+
schemaVersion: 1,
|
|
606
|
+
runAttemptId: args.runAttemptId,
|
|
607
|
+
recordedAt: args.now().toISOString(),
|
|
608
|
+
failure: {
|
|
609
|
+
stage: failure.stage,
|
|
610
|
+
...failure.judge ? { judge: failure.judge } : {},
|
|
611
|
+
error: serializeCellError(failure.cause)
|
|
612
|
+
},
|
|
613
|
+
cell,
|
|
614
|
+
cost: args.costLedger.summary({
|
|
615
|
+
phase: args.costPhase,
|
|
616
|
+
tags: costTags
|
|
617
|
+
})
|
|
618
|
+
};
|
|
619
|
+
const retry = args.cellRetry !== void 0 && args.attempt < args.cellRetry.attempts && !args.signal.aborted && fatalCellError === void 0 && args.cellRetry.retryable(receipt.failure);
|
|
620
|
+
const failurePath = join(attemptDir, "failure-receipt.json");
|
|
621
|
+
args.storage.write(failurePath, JSON.stringify(receipt, null, 2));
|
|
622
|
+
artifactsByPath[`${args.slot.cellId}/${attemptPrefix}/failure-receipt.json`] = failurePath;
|
|
623
|
+
if (!retry) args.onFailure?.(failure);
|
|
624
|
+
return retry;
|
|
625
|
+
}
|
|
626
|
+
async function executeCell(args) {
|
|
627
|
+
const storage = args.storage;
|
|
628
|
+
const cellDir = cellDirectory(args.opts.runDir, args.slot.cellId);
|
|
629
|
+
storage.ensureDir(cellDir);
|
|
630
|
+
const stableCostTags = stableCostTagsFor(args.opts, args.slot);
|
|
631
|
+
const costTags = {
|
|
632
|
+
...stableCostTags,
|
|
633
|
+
runAttemptId: args.runAttemptId
|
|
634
|
+
};
|
|
635
|
+
const attemptCostTags = {
|
|
636
|
+
...costTags,
|
|
637
|
+
attemptNumber: String(args.attempt)
|
|
638
|
+
};
|
|
639
|
+
const cachePath = join(cellDir, "cached-result.json");
|
|
640
|
+
if (args.resumable) {
|
|
641
|
+
const reused = reuseCachedCell(args, cachePath, stableCostTags);
|
|
642
|
+
if (reused) return reused;
|
|
643
|
+
}
|
|
644
|
+
const startMs = Date.now();
|
|
645
|
+
const run = openAttempt(args, cellDir, attemptCostTags);
|
|
646
|
+
const { attempt, attemptDir, attemptPrefix, artifactsByPath, trace } = run;
|
|
647
|
+
const dispatch = await dispatchCell(args, run);
|
|
648
|
+
await awaitCellShutdown(args, run, dispatch.dispatched, costTags);
|
|
649
|
+
const { artifact } = dispatch;
|
|
650
|
+
const { agentCost, tokenUsage, agentModelEvidence } = measureAgentUsage(args, costTags);
|
|
651
|
+
try {
|
|
652
|
+
enforceDispatchUsage({
|
|
653
|
+
cellId: args.slot.cellId,
|
|
654
|
+
artifact,
|
|
655
|
+
error: dispatch.errorMessage,
|
|
656
|
+
costUsd: agentCost.totalCostUsd,
|
|
657
|
+
tokenUsage
|
|
658
|
+
}, args.opts.expectUsage ?? "warn");
|
|
659
|
+
} catch (error) {
|
|
660
|
+
await trace.flush();
|
|
661
|
+
throw error;
|
|
662
|
+
}
|
|
663
|
+
const judged = await judgeArtifact(args, artifact, attemptCostTags);
|
|
664
|
+
const failure = judged.failure ?? dispatch.failure;
|
|
665
|
+
const errorMessage = judged.failure ? judged.errorMessage : dispatch.errorMessage;
|
|
666
|
+
const { judgeScores, fatalCellError } = judged;
|
|
579
667
|
if (failure) await waitForFailedCellCostSettlement({
|
|
580
668
|
costLedger: args.costLedger,
|
|
581
669
|
costPhase: args.costPhase,
|
|
@@ -605,29 +693,7 @@ async function executeCell(args) {
|
|
|
605
693
|
...failure?.judge ? { errorJudge: failure.judge } : {},
|
|
606
694
|
error: errorMessage
|
|
607
695
|
};
|
|
608
|
-
|
|
609
|
-
if (failure) {
|
|
610
|
-
const receipt = {
|
|
611
|
-
schemaVersion: 1,
|
|
612
|
-
runAttemptId: args.runAttemptId,
|
|
613
|
-
recordedAt: args.now().toISOString(),
|
|
614
|
-
failure: {
|
|
615
|
-
stage: failure.stage,
|
|
616
|
-
...failure.judge ? { judge: failure.judge } : {},
|
|
617
|
-
error: serializeCellError(failure.cause)
|
|
618
|
-
},
|
|
619
|
-
cell,
|
|
620
|
-
cost: args.costLedger.summary({
|
|
621
|
-
phase: args.costPhase,
|
|
622
|
-
tags: costTags
|
|
623
|
-
})
|
|
624
|
-
};
|
|
625
|
-
retry = args.cellRetry !== void 0 && args.attempt < args.cellRetry.attempts && !args.signal.aborted && fatalCellError === void 0 && args.cellRetry.retryable(receipt.failure);
|
|
626
|
-
const failurePath = join(attemptDir, "failure-receipt.json");
|
|
627
|
-
storage.write(failurePath, JSON.stringify(receipt, null, 2));
|
|
628
|
-
artifactsByPath[`${args.slot.cellId}/${attemptPrefix}/failure-receipt.json`] = failurePath;
|
|
629
|
-
if (!retry) args.onFailure?.(failure);
|
|
630
|
-
}
|
|
696
|
+
const retry = failure ? recordCellFailure(args, run, failure, cell, costTags, fatalCellError) : false;
|
|
631
697
|
await trace.flush();
|
|
632
698
|
const resultPath = join(attemptDir, "result.json");
|
|
633
699
|
storage.write(resultPath, JSON.stringify(cell));
|
|
@@ -1889,261 +1955,269 @@ function excerptAt(source, at, needleLength) {
|
|
|
1889
1955
|
return (start > 0 ? "…" : "") + source.slice(start, end) + (end < source.length ? "…" : "");
|
|
1890
1956
|
}
|
|
1891
1957
|
//#endregion
|
|
1892
|
-
//#region src/campaign/gates/default-production-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
const
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
|
|
1958
|
+
//#region src/campaign/gates/default-production-checks.ts
|
|
1959
|
+
function newFindings() {
|
|
1960
|
+
const reasons = [];
|
|
1961
|
+
const contributing = [];
|
|
1962
|
+
const requiredUnavailable = /* @__PURE__ */ new Set();
|
|
1963
|
+
const unavailable = (name, required, reason, detail = {}) => {
|
|
1964
|
+
contributing.push({
|
|
1965
|
+
name,
|
|
1966
|
+
status: "not_evaluated",
|
|
1967
|
+
detail: {
|
|
1968
|
+
reason,
|
|
1969
|
+
required,
|
|
1970
|
+
...detail
|
|
1971
|
+
}
|
|
1972
|
+
});
|
|
1973
|
+
if (required) {
|
|
1974
|
+
requiredUnavailable.add(name);
|
|
1975
|
+
reasons.push(`${name}: ${reason}`);
|
|
1976
|
+
}
|
|
1977
|
+
};
|
|
1910
1978
|
return {
|
|
1911
|
-
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
|
|
1938
|
-
|
|
1939
|
-
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
|
|
1943
|
-
|
|
1944
|
-
delta
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
deltaMedianDiagnostic: sig.medianBootstrap.median,
|
|
1962
|
-
deltaMedian: sig.medianBootstrap.median,
|
|
1963
|
-
tieFraction: sig.tieFraction,
|
|
1964
|
-
ciLow: dec.low,
|
|
1965
|
-
ciHigh: dec.high,
|
|
1966
|
-
bootstrapCiLow: sig.bootstrap.low,
|
|
1967
|
-
bootstrapCiHigh: sig.bootstrap.high,
|
|
1968
|
-
confidence: dec.confidence,
|
|
1969
|
-
deltaThreshold,
|
|
1970
|
-
fewRuns: sig.fewRuns
|
|
1971
|
-
}
|
|
1972
|
-
});
|
|
1973
|
-
if (sig.fewRuns) requiredUnavailable.add("heldout-significance");
|
|
1974
|
-
if (!heldoutPass) {
|
|
1975
|
-
const tieNote = sig.tieFraction >= .4 ? `; ${(sig.tieFraction * 100).toFixed(0)}% tied observation units` : "";
|
|
1976
|
-
const ci = `${(dec.confidence * 100).toFixed(0)}% CI [${dec.low.toFixed(3)}, ${dec.high.toFixed(3)}]`;
|
|
1977
|
-
reasons.push(sig.fewRuns ? `held-out: only ${sig.n} paired observation units (< ${sig.minimumRequired}) — too few to claim significance` : dec.indeterminate ? `held-out: ${dec.indeterminateCause}, so the paired CI is ${ci} and carries no direction — it cannot clear threshold ${deltaThreshold} on evidence${tieNote}` : dec.exactTestVetoes ? `held-out: McNemar exact p=${dec.mcnemar?.pValue.toExponential(2)} does not reject at α=${(1 - dec.confidence).toFixed(4)} (${dec.label} Δ ${delta.toFixed(3)}, ${ci}${tieNote})` : dec.method === "exact-sign" ? `held-out: exact one-sided sign test p=${dec.pValue} does not reject at α=${((1 - dec.confidence) / 2).toFixed(4)} (${dec.label} Δ ${delta.toFixed(3)}, diagnostic ${ci}${tieNote})` : `held-out CI.low ${dec.low.toFixed(3)} ≤ threshold ${deltaThreshold} (${dec.label} Δ ${delta.toFixed(3)}, ${ci}${tieNote})`);
|
|
1978
|
-
}
|
|
1979
|
+
reasons,
|
|
1980
|
+
contributing,
|
|
1981
|
+
requiredUnavailable,
|
|
1982
|
+
unavailable
|
|
1983
|
+
};
|
|
1984
|
+
}
|
|
1985
|
+
/** Held-out significance of the candidate over the baseline, paired by cell before any averaging. */
|
|
1986
|
+
function checkHeldout(g, cfg, ctx) {
|
|
1987
|
+
const { reasons, contributing, requiredUnavailable, unavailable } = g;
|
|
1988
|
+
const { deltaThreshold, confidence, resamples, seed, minProductiveRuns, heldoutStatistic, scenarioIds, independentUnitByScenarioId } = cfg;
|
|
1989
|
+
let delta;
|
|
1990
|
+
if (!ctx.baselineJudgeScores) unavailable("heldout-significance", true, "baselineJudgeScores is required for a held-out comparison");
|
|
1991
|
+
else {
|
|
1992
|
+
const sig = heldoutSignificance(pairHoldout(ctx.judgeScores, ctx.baselineJudgeScores, scenarioIds, (s) => s.composite), {
|
|
1993
|
+
deltaThreshold,
|
|
1994
|
+
minProductiveRuns,
|
|
1995
|
+
confidence,
|
|
1996
|
+
resamples,
|
|
1997
|
+
seed,
|
|
1998
|
+
statistic: heldoutStatistic,
|
|
1999
|
+
independentUnitByScenarioId
|
|
2000
|
+
});
|
|
2001
|
+
const dec = sig.decision;
|
|
2002
|
+
delta = dec.delta;
|
|
2003
|
+
const heldoutPass = sig.significant;
|
|
2004
|
+
contributing.push({
|
|
2005
|
+
name: "heldout-significance",
|
|
2006
|
+
status: sig.fewRuns ? "not_evaluated" : heldoutPass ? "pass" : "fail",
|
|
2007
|
+
detail: {
|
|
2008
|
+
n: sig.n,
|
|
2009
|
+
pairedCellN: sig.pairedCellN,
|
|
2010
|
+
observationUnit: sig.observationUnit,
|
|
2011
|
+
unitIds: sig.unitIds,
|
|
2012
|
+
delta,
|
|
2013
|
+
decisionStatistic: sig.decisionStatistic,
|
|
2014
|
+
decisionMethod: sig.decisionMethod,
|
|
2015
|
+
binaryScale: dec.binaryScale,
|
|
2016
|
+
mcnemar: dec.mcnemar,
|
|
2017
|
+
indeterminate: dec.indeterminate,
|
|
2018
|
+
deltaMean: sig.bootstrap.mean,
|
|
2019
|
+
deltaMedianDiagnostic: sig.medianBootstrap.median,
|
|
2020
|
+
deltaMedian: sig.medianBootstrap.median,
|
|
2021
|
+
tieFraction: sig.tieFraction,
|
|
2022
|
+
ciLow: dec.low,
|
|
2023
|
+
ciHigh: dec.high,
|
|
2024
|
+
bootstrapCiLow: sig.bootstrap.low,
|
|
2025
|
+
bootstrapCiHigh: sig.bootstrap.high,
|
|
2026
|
+
confidence: dec.confidence,
|
|
2027
|
+
deltaThreshold,
|
|
2028
|
+
fewRuns: sig.fewRuns
|
|
1979
2029
|
}
|
|
1980
|
-
|
|
1981
|
-
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
|
|
1990
|
-
|
|
1991
|
-
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
|
|
1995
|
-
|
|
1996
|
-
|
|
1997
|
-
|
|
1998
|
-
|
|
1999
|
-
|
|
2000
|
-
|
|
2001
|
-
|
|
2002
|
-
|
|
2003
|
-
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
|
|
2031
|
-
|
|
2032
|
-
|
|
2033
|
-
|
|
2034
|
-
|
|
2035
|
-
|
|
2030
|
+
});
|
|
2031
|
+
if (sig.fewRuns) requiredUnavailable.add("heldout-significance");
|
|
2032
|
+
if (!heldoutPass) {
|
|
2033
|
+
const tieNote = sig.tieFraction >= .4 ? `; ${(sig.tieFraction * 100).toFixed(0)}% tied observation units` : "";
|
|
2034
|
+
const ci = `${(dec.confidence * 100).toFixed(0)}% CI [${dec.low.toFixed(3)}, ${dec.high.toFixed(3)}]`;
|
|
2035
|
+
reasons.push(sig.fewRuns ? `held-out: only ${sig.n} paired observation units (< ${sig.minimumRequired}) — too few to claim significance` : dec.indeterminate ? `held-out: ${dec.indeterminateCause}, so the paired CI is ${ci} and carries no direction — it cannot clear threshold ${deltaThreshold} on evidence${tieNote}` : dec.exactTestVetoes ? `held-out: McNemar exact p=${dec.mcnemar?.pValue.toExponential(2)} does not reject at α=${(1 - dec.confidence).toFixed(4)} (${dec.label} Δ ${delta.toFixed(3)}, ${ci}${tieNote})` : dec.method === "exact-sign" ? `held-out: exact one-sided sign test p=${dec.pValue} does not reject at α=${((1 - dec.confidence) / 2).toFixed(4)} (${dec.label} Δ ${delta.toFixed(3)}, diagnostic ${ci}${tieNote})` : `held-out CI.low ${dec.low.toFixed(3)} ≤ threshold ${deltaThreshold} (${dec.label} Δ ${delta.toFixed(3)}, ${ci}${tieNote})`);
|
|
2036
|
+
}
|
|
2037
|
+
}
|
|
2038
|
+
return delta;
|
|
2039
|
+
}
|
|
2040
|
+
/** Anti-Goodhart guard: a net composite gain must not hide a regression on a critical dimension. */
|
|
2041
|
+
function checkDimensions(g, cfg, ctx) {
|
|
2042
|
+
const { reasons, contributing, requiredUnavailable, unavailable } = g;
|
|
2043
|
+
const { options, confidence, resamples, seed, minProductiveRuns, explicitlyRequired, scenarioIds, independentUnitByScenarioId } = cfg;
|
|
2044
|
+
const dimensionsProvided = options.criticalDimensions !== void 0;
|
|
2045
|
+
const criticalDimensions = [...new Set(options.criticalDimensions ?? [])];
|
|
2046
|
+
const dimensionsRequired = dimensionsProvided || explicitlyRequired.has("dimension-regression");
|
|
2047
|
+
if (!dimensionsProvided) unavailable("dimension-regression", dimensionsRequired, "criticalDimensions is not configured");
|
|
2048
|
+
else if (criticalDimensions.length === 0) unavailable("dimension-regression", true, "criticalDimensions must contain at least one dimension");
|
|
2049
|
+
else if (!ctx.baselineJudgeScores) unavailable("dimension-regression", true, "baselineJudgeScores is required to measure dimension regressions", { guarded: criticalDimensions });
|
|
2050
|
+
else {
|
|
2051
|
+
const dimRegs = dimensionRegressions(ctx.judgeScores, ctx.baselineJudgeScores, scenarioIds, criticalDimensions, {
|
|
2052
|
+
tolerance: options.regressionTolerance,
|
|
2053
|
+
confidence,
|
|
2054
|
+
resamples,
|
|
2055
|
+
seed,
|
|
2056
|
+
independentUnitByScenarioId,
|
|
2057
|
+
minProductiveRuns
|
|
2058
|
+
});
|
|
2059
|
+
const measured = new Set(dimRegs.map((result) => result.dimension));
|
|
2060
|
+
const missingDimensions = criticalDimensions.filter((dimension) => !measured.has(dimension));
|
|
2061
|
+
const regressed = dimRegs.filter((result) => result.regressed);
|
|
2062
|
+
const incompleteDimensions = dimRegs.filter((result) => result.fewRuns || result.missingCellIds.length > 0 || result.missingScenarioIds.length > 0);
|
|
2063
|
+
const dimensionStatus = regressed.length > 0 ? "fail" : missingDimensions.length > 0 || incompleteDimensions.length > 0 ? "not_evaluated" : "pass";
|
|
2064
|
+
contributing.push({
|
|
2065
|
+
name: "dimension-regression",
|
|
2066
|
+
status: dimensionStatus,
|
|
2067
|
+
detail: {
|
|
2068
|
+
guarded: criticalDimensions,
|
|
2069
|
+
missingDimensions,
|
|
2070
|
+
incompleteDimensions: incompleteDimensions.map((result) => result.dimension),
|
|
2071
|
+
regressions: dimRegs.map((result) => ({
|
|
2072
|
+
dimension: result.dimension,
|
|
2073
|
+
ciLow: result.ci.low,
|
|
2074
|
+
ciHigh: result.ci.high,
|
|
2075
|
+
decisionStatistic: result.decisionStatistic,
|
|
2076
|
+
indeterminate: result.indeterminate,
|
|
2077
|
+
bootstrapCiLow: result.bootstrap.low,
|
|
2078
|
+
median: result.bootstrap.median,
|
|
2079
|
+
tolerance: result.tolerance,
|
|
2080
|
+
n: result.n,
|
|
2081
|
+
pairedCellN: result.pairedCellN,
|
|
2082
|
+
observationUnit: result.observationUnit,
|
|
2083
|
+
minimumRequired: result.minimumRequired,
|
|
2084
|
+
fewRuns: result.fewRuns,
|
|
2085
|
+
missingCellIds: result.missingCellIds,
|
|
2086
|
+
missingScenarioIds: result.missingScenarioIds,
|
|
2087
|
+
regressed: result.regressed
|
|
2088
|
+
}))
|
|
2036
2089
|
}
|
|
2037
|
-
|
|
2038
|
-
|
|
2039
|
-
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
|
|
2053
|
-
|
|
2054
|
-
|
|
2055
|
-
|
|
2056
|
-
|
|
2057
|
-
|
|
2058
|
-
|
|
2059
|
-
|
|
2060
|
-
|
|
2090
|
+
});
|
|
2091
|
+
if (missingDimensions.length > 0) {
|
|
2092
|
+
requiredUnavailable.add("dimension-regression");
|
|
2093
|
+
reasons.push(`critical dimension(s) were not scored: ${missingDimensions.join(", ")}`);
|
|
2094
|
+
}
|
|
2095
|
+
if (incompleteDimensions.length > 0) {
|
|
2096
|
+
requiredUnavailable.add("dimension-regression");
|
|
2097
|
+
reasons.push(`critical dimension evidence is incomplete: ${incompleteDimensions.map((result) => `${result.dimension} has ${result.n} paired observation units (minimum ${result.minimumRequired}), ${result.missingCellIds.length} unscored cells, ${result.missingScenarioIds.length} unscored scenarios`).join("; ")}`);
|
|
2098
|
+
}
|
|
2099
|
+
if (regressed.length > 0) reasons.push(`critical dimension(s) regressed: ${regressed.map((result) => result.bootstrap.low < -result.tolerance ? `${result.dimension} bootstrap CI.low ${result.bootstrap.low.toFixed(3)} < -${result.tolerance}` : `${result.dimension} paired test supports a drop greater than ${result.tolerance}`).join("; ")}`);
|
|
2100
|
+
}
|
|
2101
|
+
}
|
|
2102
|
+
/** Total spend of the improvement run against `budgetUsd`, from complete cost accounting only. */
|
|
2103
|
+
function checkBudget(g, cfg, ctx) {
|
|
2104
|
+
const { reasons, contributing, requiredUnavailable, unavailable } = g;
|
|
2105
|
+
const { options, explicitlyRequired } = cfg;
|
|
2106
|
+
const budgetUsd = options.budgetUsd;
|
|
2107
|
+
const budgetConfigured = budgetUsd !== void 0;
|
|
2108
|
+
const budgetRequired = budgetConfigured || explicitlyRequired.has("budget");
|
|
2109
|
+
if (!budgetConfigured) unavailable("budget", budgetRequired, "budgetUsd is not configured");
|
|
2110
|
+
else if (!ctx.costLedger) unavailable("budget", true, "costLedger is required to measure complete run spend", { budgetUsd });
|
|
2111
|
+
else {
|
|
2112
|
+
const cost = ctx.costLedger.summary();
|
|
2113
|
+
const accountingComplete = cost.accountingComplete && cost.pendingCalls === 0 && cost.unresolvedCalls === 0;
|
|
2114
|
+
const budgetPass = cost.totalCostUsd <= budgetUsd;
|
|
2115
|
+
contributing.push({
|
|
2116
|
+
name: "budget",
|
|
2117
|
+
status: !accountingComplete ? "not_evaluated" : budgetPass ? "pass" : "fail",
|
|
2118
|
+
detail: {
|
|
2119
|
+
totalCostUsd: cost.totalCostUsd,
|
|
2120
|
+
budgetUsd,
|
|
2121
|
+
accountingComplete,
|
|
2122
|
+
pendingCalls: cost.pendingCalls,
|
|
2123
|
+
unresolvedCalls: cost.unresolvedCalls
|
|
2061
2124
|
}
|
|
2062
|
-
|
|
2063
|
-
|
|
2064
|
-
|
|
2065
|
-
|
|
2066
|
-
|
|
2067
|
-
|
|
2068
|
-
|
|
2069
|
-
|
|
2070
|
-
|
|
2071
|
-
|
|
2072
|
-
|
|
2073
|
-
|
|
2074
|
-
|
|
2075
|
-
|
|
2076
|
-
|
|
2077
|
-
|
|
2078
|
-
|
|
2079
|
-
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
2125
|
+
});
|
|
2126
|
+
if (!accountingComplete) {
|
|
2127
|
+
requiredUnavailable.add("budget");
|
|
2128
|
+
reasons.push("budget: cost accounting is incomplete");
|
|
2129
|
+
} else if (!budgetPass) reasons.push(`spend ${cost.totalCostUsd.toFixed(2)} > budget ${budgetUsd}`);
|
|
2130
|
+
}
|
|
2131
|
+
}
|
|
2132
|
+
/** Static red-team screening of the candidate artifacts. */
|
|
2133
|
+
function checkRedTeam(g, cfg, ctx) {
|
|
2134
|
+
const { reasons, contributing, requiredUnavailable, unavailable } = g;
|
|
2135
|
+
const { options, explicitlyRequired } = cfg;
|
|
2136
|
+
const redTeamRequired = options.redTeamBattery !== void 0 || explicitlyRequired.has("red-team");
|
|
2137
|
+
if (!options.redTeamBattery) unavailable("red-team", redTeamRequired, "redTeamBattery is not configured");
|
|
2138
|
+
else if (options.redTeamBattery.length === 0) unavailable("red-team", true, "redTeamBattery must contain at least one case");
|
|
2139
|
+
else {
|
|
2140
|
+
const redTeam = probeRedTeam(ctx.candidateArtifacts, options.redTeamBattery);
|
|
2141
|
+
const incomplete = redTeam.unsupportedCases.length > 0 || redTeam.unscoredArtifacts.length > 0 || redTeam.evaluatedCases === 0;
|
|
2142
|
+
const redTeamStatus = redTeam.findings.length > 0 ? "fail" : incomplete ? "not_evaluated" : "pass";
|
|
2143
|
+
contributing.push({
|
|
2144
|
+
name: "red-team",
|
|
2145
|
+
status: redTeamStatus,
|
|
2146
|
+
detail: {
|
|
2147
|
+
evaluatedCases: redTeam.evaluatedCases,
|
|
2148
|
+
unsupportedCases: redTeam.unsupportedCases,
|
|
2149
|
+
unscoredArtifacts: redTeam.unscoredArtifacts,
|
|
2150
|
+
failures: redTeam.findings.length,
|
|
2151
|
+
sample: redTeam.findings.slice(0, 3)
|
|
2085
2152
|
}
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
|
|
2095
|
-
|
|
2096
|
-
|
|
2097
|
-
|
|
2098
|
-
|
|
2099
|
-
|
|
2100
|
-
|
|
2101
|
-
|
|
2102
|
-
|
|
2103
|
-
|
|
2104
|
-
|
|
2105
|
-
|
|
2106
|
-
|
|
2107
|
-
|
|
2108
|
-
|
|
2109
|
-
|
|
2110
|
-
|
|
2153
|
+
});
|
|
2154
|
+
if (incomplete) requiredUnavailable.add("red-team");
|
|
2155
|
+
if (redTeam.unsupportedCases.length > 0) reasons.push(`red-team: static artifact screening cannot evaluate case(s): ${redTeam.unsupportedCases.join(", ")}`);
|
|
2156
|
+
if (redTeam.unscoredArtifacts.length > 0) reasons.push(`red-team: no text could be extracted from artifact(s): ${redTeam.unscoredArtifacts.join(", ")}`);
|
|
2157
|
+
if (redTeam.evaluatedCases === 0) reasons.push("red-team: no artifact/case pair was evaluated");
|
|
2158
|
+
if (redTeam.findings.length > 0) reasons.push(`red-team probe failed (${redTeam.findings.length} findings)`);
|
|
2159
|
+
}
|
|
2160
|
+
}
|
|
2161
|
+
/** Reward-hacking detector over the run-history window. */
|
|
2162
|
+
function checkRewardHacking(g, cfg) {
|
|
2163
|
+
const { reasons, contributing, unavailable } = g;
|
|
2164
|
+
const { options, explicitlyRequired } = cfg;
|
|
2165
|
+
const rewardHacking = options.rewardHacking;
|
|
2166
|
+
const rewardHackingConfigured = rewardHacking !== void 0;
|
|
2167
|
+
const rewardHackingRequired = rewardHackingConfigured || explicitlyRequired.has("reward-hacking");
|
|
2168
|
+
if (!rewardHackingConfigured) unavailable("reward-hacking", rewardHackingRequired, "rewardHacking is not configured");
|
|
2169
|
+
else if (!options.recentRuns) unavailable("reward-hacking", rewardHackingRequired, "recentRuns is not configured");
|
|
2170
|
+
else if (options.recentRuns.length < 10) unavailable("reward-hacking", rewardHackingRequired, `recentRuns has ${options.recentRuns.length} record(s); at least 10 are required`);
|
|
2171
|
+
else {
|
|
2172
|
+
const rewardHackingReport = detectRewardHacking({
|
|
2173
|
+
runs: options.recentRuns,
|
|
2174
|
+
...rewardHacking
|
|
2175
|
+
});
|
|
2176
|
+
if (!rewardHackingReport.evaluatedSignals.includes("reward_divergence")) unavailable("reward-hacking", true, "independent proxy/truth evidence is insufficient to evaluate reward divergence", { report: rewardHackingReport });
|
|
2177
|
+
else {
|
|
2178
|
+
const gamingThreshold = rewardHacking.thresholds?.gaming ?? .6;
|
|
2179
|
+
const gamingFindings = rewardHackingReport.findings.filter((finding) => finding.severity >= gamingThreshold);
|
|
2180
|
+
const rewardHackingPass = gamingFindings.length === 0 && rewardHackingReport.verdict !== "gaming";
|
|
2181
|
+
contributing.push({
|
|
2182
|
+
name: "reward-hacking",
|
|
2183
|
+
status: rewardHackingPass ? "pass" : "fail",
|
|
2184
|
+
detail: {
|
|
2185
|
+
report: rewardHackingReport,
|
|
2186
|
+
gamingFindingCount: gamingFindings.length
|
|
2111
2187
|
}
|
|
2112
|
-
}
|
|
2113
|
-
|
|
2114
|
-
|
|
2115
|
-
|
|
2116
|
-
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
|
|
2121
|
-
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
|
|
2132
|
-
|
|
2133
|
-
|
|
2134
|
-
|
|
2188
|
+
});
|
|
2189
|
+
if (!rewardHackingPass) reasons.push(`reward-hacking detector flagged ${gamingFindings.length} gaming-severity findings (verdict=${rewardHackingReport.verdict})`);
|
|
2190
|
+
}
|
|
2191
|
+
}
|
|
2192
|
+
}
|
|
2193
|
+
/** Canary checks over the run history. */
|
|
2194
|
+
function checkCanary(g, cfg) {
|
|
2195
|
+
const { reasons, contributing, unavailable } = g;
|
|
2196
|
+
const { options, explicitlyRequired } = cfg;
|
|
2197
|
+
const canary = options.canary;
|
|
2198
|
+
const canaryConfigured = canary !== void 0;
|
|
2199
|
+
const canaryRequired = canaryConfigured || explicitlyRequired.has("canary");
|
|
2200
|
+
if (!canaryConfigured && !canaryRequired) unavailable("canary", false, "canary is not configured");
|
|
2201
|
+
else if (!options.recentRuns) unavailable("canary", canaryRequired, "recentRuns is not configured");
|
|
2202
|
+
else {
|
|
2203
|
+
const canaryReport = runCanaries(options.recentRuns, canary ?? {});
|
|
2204
|
+
const incomplete = canaryReport.evaluations.filter((evaluation) => evaluation.status === "not_evaluated");
|
|
2205
|
+
if (incomplete.length > 0) unavailable("canary", true, `insufficient evidence for: ${incomplete.map((evaluation) => evaluation.kind).join(", ")}`, { report: canaryReport });
|
|
2206
|
+
else {
|
|
2207
|
+
const errorAlerts = canaryReport.alerts.filter((alert) => alert.severity === "error");
|
|
2208
|
+
const canaryPass = errorAlerts.length === 0;
|
|
2209
|
+
contributing.push({
|
|
2210
|
+
name: "canary",
|
|
2211
|
+
status: canaryPass ? "pass" : "fail",
|
|
2212
|
+
detail: {
|
|
2213
|
+
totalAlerts: canaryReport.alerts.length,
|
|
2214
|
+
errorAlerts: errorAlerts.length,
|
|
2215
|
+
report: canaryReport
|
|
2135
2216
|
}
|
|
2136
|
-
}
|
|
2137
|
-
|
|
2138
|
-
const notEvaluated = contributing.filter((check) => check.status === "not_evaluated").map((check) => check.name);
|
|
2139
|
-
return {
|
|
2140
|
-
decision,
|
|
2141
|
-
reasons: reasons.length > 0 ? reasons : notEvaluated.length > 0 ? [`evaluated checks passed; not evaluated: ${notEvaluated.join(", ")}`] : ["all checks passed"],
|
|
2142
|
-
contributingGates: contributing,
|
|
2143
|
-
delta
|
|
2144
|
-
};
|
|
2217
|
+
});
|
|
2218
|
+
if (!canaryPass) reasons.push(`canary error alerts: ${errorAlerts.length}`);
|
|
2145
2219
|
}
|
|
2146
|
-
}
|
|
2220
|
+
}
|
|
2147
2221
|
}
|
|
2148
2222
|
function probeRedTeam(artifacts, battery) {
|
|
2149
2223
|
const supportedCases = battery.filter((testCase) => testCase.payload.expected === "ignore" && !testCase.payload.forbiddenTools?.length);
|
|
@@ -2184,6 +2258,62 @@ function extractText(artifact) {
|
|
|
2184
2258
|
}
|
|
2185
2259
|
}
|
|
2186
2260
|
//#endregion
|
|
2261
|
+
//#region src/campaign/gates/default-production-gate.ts
|
|
2262
|
+
/**
|
|
2263
|
+
* Opinionated production gate composing held-out significance, red-team, reward-hacking, and canary checks into a single `Gate.decide` decision.
|
|
2264
|
+
*/
|
|
2265
|
+
function defaultProductionGate(options) {
|
|
2266
|
+
const deltaThreshold = options.deltaThreshold ?? 0;
|
|
2267
|
+
const confidence = options.confidence ?? .95;
|
|
2268
|
+
const resamples = options.bootstrapResamples ?? 2e3;
|
|
2269
|
+
const seed = options.bootstrapSeed ?? 1337;
|
|
2270
|
+
const minProductiveRuns = options.minProductiveRuns ?? 3;
|
|
2271
|
+
const heldoutStatistic = options.heldoutStatistic ?? "mean";
|
|
2272
|
+
const explicitlyRequired = new Set(options.requiredChecks ?? []);
|
|
2273
|
+
const scenarioIds = new Set(options.holdoutScenarios.map((scenario) => scenario.id));
|
|
2274
|
+
const independentUnitByScenarioId = options.independentUnitByScenarioId === void 0 ? void 0 : new Map(options.independentUnitByScenarioId);
|
|
2275
|
+
if (independentUnitByScenarioId !== void 0) for (const id of scenarioIds) {
|
|
2276
|
+
const unit = independentUnitByScenarioId.get(id);
|
|
2277
|
+
if (typeof unit !== "string" || unit.length === 0 || unit.trim() !== unit) throw new Error(`defaultProductionGate: missing independent unit for holdout scenario '${id}'`);
|
|
2278
|
+
}
|
|
2279
|
+
const cfg = {
|
|
2280
|
+
options,
|
|
2281
|
+
deltaThreshold,
|
|
2282
|
+
confidence,
|
|
2283
|
+
resamples,
|
|
2284
|
+
seed,
|
|
2285
|
+
minProductiveRuns,
|
|
2286
|
+
heldoutStatistic,
|
|
2287
|
+
explicitlyRequired,
|
|
2288
|
+
scenarioIds,
|
|
2289
|
+
independentUnitByScenarioId
|
|
2290
|
+
};
|
|
2291
|
+
return {
|
|
2292
|
+
name: "defaultProductionGate",
|
|
2293
|
+
async decide(ctx) {
|
|
2294
|
+
const findings = newFindings();
|
|
2295
|
+
const delta = checkHeldout(findings, cfg, ctx);
|
|
2296
|
+
checkDimensions(findings, cfg, ctx);
|
|
2297
|
+
checkBudget(findings, cfg, ctx);
|
|
2298
|
+
checkRedTeam(findings, cfg, ctx);
|
|
2299
|
+
checkRewardHacking(findings, cfg);
|
|
2300
|
+
checkCanary(findings, cfg);
|
|
2301
|
+
return verdictOf(findings, delta);
|
|
2302
|
+
}
|
|
2303
|
+
};
|
|
2304
|
+
}
|
|
2305
|
+
/** Hold when any check failed or a required check could not be evaluated; otherwise ship. */
|
|
2306
|
+
function verdictOf({ reasons, contributing, requiredUnavailable }, delta) {
|
|
2307
|
+
const decision = contributing.some((check) => check.status === "fail") || requiredUnavailable.size > 0 ? "hold" : "ship";
|
|
2308
|
+
const notEvaluated = contributing.filter((check) => check.status === "not_evaluated").map((check) => check.name);
|
|
2309
|
+
return {
|
|
2310
|
+
decision,
|
|
2311
|
+
reasons: reasons.length > 0 ? reasons : notEvaluated.length > 0 ? [`evaluated checks passed; not evaluated: ${notEvaluated.join(", ")}`] : ["all checks passed"],
|
|
2312
|
+
contributingGates: contributing,
|
|
2313
|
+
delta
|
|
2314
|
+
};
|
|
2315
|
+
}
|
|
2316
|
+
//#endregion
|
|
2187
2317
|
//#region src/campaign/judge-snapshot.ts
|
|
2188
2318
|
/** Capture configuration and callbacks; receiver and closed-over state remain caller-owned. */
|
|
2189
2319
|
function captureJudge(judge) {
|
|
@@ -3707,7 +3837,7 @@ function crowdedFrontierParent(options) {
|
|
|
3707
3837
|
});
|
|
3708
3838
|
}
|
|
3709
3839
|
/**
|
|
3710
|
-
* AIDE's
|
|
3840
|
+
* AIDE's search_policy with three changes
|
|
3711
3841
|
* for noisy scores.
|
|
3712
3842
|
*
|
|
3713
3843
|
* 1. **Draft** while fewer than `drafts` nodes were drafted: the root is the
|
|
@@ -4056,15 +4186,6 @@ function mean(values) {
|
|
|
4056
4186
|
function isProposedCandidate(value) {
|
|
4057
4187
|
return typeof value === "object" && value !== null && "surface" in value && "label" in value && "rationale" in value;
|
|
4058
4188
|
}
|
|
4059
|
-
const LABEL_TRUST_RANK = {
|
|
4060
|
-
unverified: 0,
|
|
4061
|
-
"verified-signal": 1,
|
|
4062
|
-
"human-rated": 2
|
|
4063
|
-
};
|
|
4064
|
-
/** Ordinal rank for a label-trust tier; absent ⇒ `unverified` (rank 0). */
|
|
4065
|
-
function labelTrustRank(trust) {
|
|
4066
|
-
return LABEL_TRUST_RANK[trust ?? "unverified"];
|
|
4067
|
-
}
|
|
4068
4189
|
//#endregion
|
|
4069
4190
|
//#region src/campaign/presets/optimization-surface-nodes.ts
|
|
4070
4191
|
/** Surface-node campaigns, cached cell adoption, and proposer history from the search ledger. */
|
|
@@ -4431,13 +4552,9 @@ function toScoredSurfaceOutcome(hash, campaign, coverage, generation) {
|
|
|
4431
4552
|
* again on the same run directory and options continues an interrupted run
|
|
4432
4553
|
* instead of starting over, and reruns nothing that settled.
|
|
4433
4554
|
*/
|
|
4434
|
-
/** Compose the optimization proposer and cell executor on the shared search kernel. */
|
|
4435
4555
|
/** Improve a surface as a search on the kernel; see the module comment. */
|
|
4436
4556
|
async function runOptimization(opts) {
|
|
4437
|
-
const candidateConcurrency = opts
|
|
4438
|
-
if (typeof opts.runDir !== "string" || opts.runDir.trim().length === 0) throw new Error("runOptimization: runDir is required and must be a non-empty string");
|
|
4439
|
-
if (!Number.isInteger(candidateConcurrency) || candidateConcurrency < 1) throw new Error("runOptimization: candidateConcurrency must be a positive integer");
|
|
4440
|
-
for (const [name, value] of [["populationSize", opts.populationSize], ["maxGenerations", opts.maxGenerations]]) if (!Number.isSafeInteger(value) || value < 0) throw new Error(`runOptimization: ${name} must be a non-negative integer`);
|
|
4557
|
+
const candidateConcurrency = assertOptimizationOptions(opts);
|
|
4441
4558
|
const initialFindings = immutableProposalSnapshot(assertProposalFindings(opts.findings ?? [], "runOptimization initial proposal findings"), "initial findings");
|
|
4442
4559
|
const baselineSurface = immutableProposalSnapshot(opts.baselineSurface, "baseline surface");
|
|
4443
4560
|
const runDir = resolveRunDir(opts.runDir, opts.repo);
|
|
@@ -4476,7 +4593,53 @@ async function runOptimization(opts) {
|
|
|
4476
4593
|
const recorder = await SearchRecorder.open({
|
|
4477
4594
|
ledger: binding.ledger,
|
|
4478
4595
|
storage
|
|
4479
|
-
}, {
|
|
4596
|
+
}, searchOpening({
|
|
4597
|
+
opts,
|
|
4598
|
+
identity,
|
|
4599
|
+
execution,
|
|
4600
|
+
baselineSurface,
|
|
4601
|
+
splitDigest,
|
|
4602
|
+
policy,
|
|
4603
|
+
allocationName: allocation.name,
|
|
4604
|
+
seed
|
|
4605
|
+
}));
|
|
4606
|
+
const run = {
|
|
4607
|
+
opts,
|
|
4608
|
+
runDir,
|
|
4609
|
+
costLedger,
|
|
4610
|
+
reps,
|
|
4611
|
+
requireJudgeScore,
|
|
4612
|
+
premeasured,
|
|
4613
|
+
identity,
|
|
4614
|
+
policy,
|
|
4615
|
+
recorder,
|
|
4616
|
+
nodes: new SurfaceNodes(opts, recorder, runDir, costLedger, execution, premeasured),
|
|
4617
|
+
findings: initialFindings
|
|
4618
|
+
};
|
|
4619
|
+
return optimizationResult(run, baselineSurface, await runSearch({
|
|
4620
|
+
recorder,
|
|
4621
|
+
root: baselineSurface,
|
|
4622
|
+
codec: surfaceCodec,
|
|
4623
|
+
policy,
|
|
4624
|
+
allocation,
|
|
4625
|
+
proposer: optimizationProposerPort(run),
|
|
4626
|
+
executor: optimizationExecutor(run, candidateConcurrency),
|
|
4627
|
+
maxExpansions: opts.maxGenerations,
|
|
4628
|
+
signal: opts.signal
|
|
4629
|
+
}));
|
|
4630
|
+
}
|
|
4631
|
+
/** Reject malformed run settings before any state is touched; returns the candidate concurrency. */
|
|
4632
|
+
function assertOptimizationOptions(opts) {
|
|
4633
|
+
const candidateConcurrency = opts.candidateConcurrency ?? 1;
|
|
4634
|
+
if (typeof opts.runDir !== "string" || opts.runDir.trim().length === 0) throw new Error("runOptimization: runDir is required and must be a non-empty string");
|
|
4635
|
+
if (!Number.isInteger(candidateConcurrency) || candidateConcurrency < 1) throw new Error("runOptimization: candidateConcurrency must be a positive integer");
|
|
4636
|
+
for (const [name, value] of [["populationSize", opts.populationSize], ["maxGenerations", opts.maxGenerations]]) if (!Number.isSafeInteger(value) || value < 0) throw new Error(`runOptimization: ${name} must be a non-negative integer`);
|
|
4637
|
+
return candidateConcurrency;
|
|
4638
|
+
}
|
|
4639
|
+
/** The search ledger's opening record: subject, objective, train split, policy and budget. */
|
|
4640
|
+
function searchOpening(args) {
|
|
4641
|
+
const { opts, identity, baselineSurface, splitDigest, policy } = args;
|
|
4642
|
+
return {
|
|
4480
4643
|
subject: identity.subject ?? opts.proposer.kind,
|
|
4481
4644
|
process: {
|
|
4482
4645
|
name: "runOptimization",
|
|
@@ -4504,8 +4667,8 @@ async function runOptimization(opts) {
|
|
|
4504
4667
|
},
|
|
4505
4668
|
policy: {
|
|
4506
4669
|
expansion: policy.name,
|
|
4507
|
-
allocation:
|
|
4508
|
-
seed
|
|
4670
|
+
allocation: args.allocationName,
|
|
4671
|
+
seed: args.seed
|
|
4509
4672
|
},
|
|
4510
4673
|
budget: {
|
|
4511
4674
|
maxUsd: opts.costCeiling ?? null,
|
|
@@ -4517,12 +4680,14 @@ async function runOptimization(opts) {
|
|
|
4517
4680
|
},
|
|
4518
4681
|
containment: null,
|
|
4519
4682
|
derivedFrom: null,
|
|
4520
|
-
identity: execution
|
|
4521
|
-
}
|
|
4522
|
-
|
|
4683
|
+
identity: args.execution
|
|
4684
|
+
};
|
|
4685
|
+
}
|
|
4686
|
+
/** One lane that measures each candidate cell through the surface nodes. */
|
|
4687
|
+
function optimizationExecutor(run, candidateConcurrency) {
|
|
4688
|
+
const { opts, nodes } = run;
|
|
4523
4689
|
const scenarioById = new Map(opts.scenarios.map((scenario) => [scenario.id, scenario]));
|
|
4524
|
-
|
|
4525
|
-
const executor = {
|
|
4690
|
+
return {
|
|
4526
4691
|
lanes: () => [{
|
|
4527
4692
|
name: "campaign",
|
|
4528
4693
|
capacity: candidateConcurrency * (opts.maxConcurrency ?? 2),
|
|
@@ -4533,83 +4698,29 @@ async function runOptimization(opts) {
|
|
|
4533
4698
|
run: (work) => nodes.runCell(work, scenarioById),
|
|
4534
4699
|
adopt: (work) => nodes.adoptCell(work)
|
|
4535
4700
|
};
|
|
4701
|
+
}
|
|
4702
|
+
/** The proposer port: each proposal analyses the last generation, then asks the caller's proposer. */
|
|
4703
|
+
function optimizationProposerPort(run) {
|
|
4704
|
+
const { opts, identity, recorder, nodes, policy } = run;
|
|
4536
4705
|
const proposer = opts.proposer;
|
|
4537
4706
|
const proposerSource = identity.proposer.source;
|
|
4538
|
-
|
|
4707
|
+
return {
|
|
4539
4708
|
name: proposer.kind,
|
|
4540
4709
|
kind: "optimizer",
|
|
4541
4710
|
source: proposerSource,
|
|
4542
4711
|
execution: identity.proposer,
|
|
4543
4712
|
childrenPerProposal: opts.populationSize,
|
|
4544
4713
|
async propose(request) {
|
|
4545
|
-
const
|
|
4546
|
-
"search.proposal",
|
|
4547
|
-
"analysis.baseline",
|
|
4548
|
-
"analysis.generation"
|
|
4549
|
-
];
|
|
4550
|
-
const before = new Set(phases.flatMap((phase) => costLedger.list({ phase })).map((receipt) => receipt.callId));
|
|
4551
|
-
const accounting = () => {
|
|
4552
|
-
const fresh = phases.flatMap((phase) => costLedger.list({ phase })).filter((receipt) => !before.has(receipt.callId));
|
|
4553
|
-
return {
|
|
4554
|
-
accounting: searchReceiptAccounting(fresh),
|
|
4555
|
-
execution: opts.searchLedger ? identity.proposer : searchProposalExecution(fresh, identity.model.provider, proposerSource)
|
|
4556
|
-
};
|
|
4557
|
-
};
|
|
4714
|
+
const accounting = proposalAccounting(run);
|
|
4558
4715
|
const earlier = await recorder.state();
|
|
4559
4716
|
const root = earlier.rootNodeId;
|
|
4560
4717
|
const rootCampaign = nodes.campaign(earlier, root);
|
|
4561
|
-
|
|
4562
|
-
if (!rootCoverage.complete) throw new Error(`runOptimization: ${premeasured ? "premeasured baseline" : "baseline"} is incomplete (${rootCoverage.scorableCellIds.length}/${rootCoverage.expectedCellIds.length} designed cells scorable) — ${formatCoverageFailures(rootCoverage)}. Refusing to optimize against an incomplete incumbent.`);
|
|
4718
|
+
assertBaselineComplete(run, rootCampaign);
|
|
4563
4719
|
const generation = nodes.generations(earlier).length;
|
|
4564
|
-
if (opts.analyzeGeneration)
|
|
4565
|
-
const previous = generation - 1;
|
|
4566
|
-
const candidates = (previous < 0 ? [root] : nodes.expansionNodes(earlier, nodes.generations(earlier)[previous])).map((nodeId) => {
|
|
4567
|
-
const campaign = nodes.campaign(earlier, nodeId);
|
|
4568
|
-
return {
|
|
4569
|
-
surfaceHash: surfaceHash(nodes.surface(earlier, nodeId)),
|
|
4570
|
-
campaign,
|
|
4571
|
-
composite: campaignMeanCompositeOrNull(campaign)
|
|
4572
|
-
};
|
|
4573
|
-
});
|
|
4574
|
-
const history = nodes.history(earlier, policy);
|
|
4575
|
-
if (candidates.some(({ campaign }) => campaign.cells.length > 0)) {
|
|
4576
|
-
const fresh = await opts.analyzeGeneration({
|
|
4577
|
-
generation: previous,
|
|
4578
|
-
runDir: previous < 0 ? rootCampaign.runDir : `${runDir}/candidates`,
|
|
4579
|
-
candidates,
|
|
4580
|
-
history,
|
|
4581
|
-
costLedger,
|
|
4582
|
-
costPhase: previous < 0 ? "analysis.baseline" : "analysis.generation"
|
|
4583
|
-
});
|
|
4584
|
-
if (!Array.isArray(fresh)) throw new TypeError("runOptimization: analyzeGeneration must return an array");
|
|
4585
|
-
findings = immutableProposalSnapshot(assertProposalFindings(fresh, "runOptimization analysis findings"), "analysis findings");
|
|
4586
|
-
}
|
|
4587
|
-
}
|
|
4720
|
+
if (opts.analyzeGeneration) await analyzePreviousGeneration(run, earlier, generation, root, rootCampaign.runDir);
|
|
4588
4721
|
const state = await recorder.state();
|
|
4589
4722
|
const proposalHistory = immutableProposalSnapshot(nodes.history(state, policy), "history");
|
|
4590
|
-
const
|
|
4591
|
-
const context = Object.freeze({
|
|
4592
|
-
currentSurface: immutableProposalSnapshot(parent.artifact, "current surface"),
|
|
4593
|
-
operator: request.operator,
|
|
4594
|
-
parents: immutableProposalSnapshot(request.parents.map(({ nodeId, artifact }) => ({
|
|
4595
|
-
nodeId,
|
|
4596
|
-
artifact: immutableProposalSnapshot(artifact, "parent surface")
|
|
4597
|
-
})), "parents"),
|
|
4598
|
-
train: searchProposerView(state),
|
|
4599
|
-
summary: renderSearchSummary(state, { split: "train" }),
|
|
4600
|
-
history: proposalHistory,
|
|
4601
|
-
findings: immutableProposalSnapshot(assertProposalFindings(findings, "runOptimization proposal findings"), "findings"),
|
|
4602
|
-
populationSize: opts.populationSize,
|
|
4603
|
-
generation,
|
|
4604
|
-
signal: request.signal,
|
|
4605
|
-
baselineOutcome: immutableProposalSnapshot(nodes.outcome(state, root), "baseline"),
|
|
4606
|
-
incumbentOutcome: immutableProposalSnapshot(nodes.outcome(state, request.leader), "incumbent outcome"),
|
|
4607
|
-
parentOutcome: immutableProposalSnapshot(nodes.outcome(state, parent.nodeId), "parent outcome"),
|
|
4608
|
-
maxImprovementShots: opts.maxImprovementShots,
|
|
4609
|
-
paretoParents: immutableProposalSnapshot(nodes.frontier(state), "Pareto parents"),
|
|
4610
|
-
costLedger,
|
|
4611
|
-
costPhase: "search.proposal"
|
|
4612
|
-
});
|
|
4723
|
+
const context = proposeContext(run, request, state, generation, root, proposalHistory);
|
|
4613
4724
|
if (proposer.decide?.({ history: proposalHistory }).stop) return {
|
|
4614
4725
|
children: [],
|
|
4615
4726
|
stop: "the proposer decided to stop",
|
|
@@ -4632,17 +4743,86 @@ async function runOptimization(opts) {
|
|
|
4632
4743
|
};
|
|
4633
4744
|
}
|
|
4634
4745
|
};
|
|
4635
|
-
|
|
4636
|
-
|
|
4637
|
-
|
|
4638
|
-
|
|
4639
|
-
|
|
4640
|
-
|
|
4641
|
-
|
|
4642
|
-
|
|
4643
|
-
|
|
4644
|
-
|
|
4746
|
+
}
|
|
4747
|
+
/** Snapshot the proposal-phase receipts now; the returned reader totals only receipts added since. */
|
|
4748
|
+
function proposalAccounting(run) {
|
|
4749
|
+
const { opts, identity, costLedger } = run;
|
|
4750
|
+
const phases = [
|
|
4751
|
+
"search.proposal",
|
|
4752
|
+
"analysis.baseline",
|
|
4753
|
+
"analysis.generation"
|
|
4754
|
+
];
|
|
4755
|
+
const before = new Set(phases.flatMap((phase) => costLedger.list({ phase })).map((receipt) => receipt.callId));
|
|
4756
|
+
return () => {
|
|
4757
|
+
const fresh = phases.flatMap((phase) => costLedger.list({ phase })).filter((receipt) => !before.has(receipt.callId));
|
|
4758
|
+
return {
|
|
4759
|
+
accounting: searchReceiptAccounting(fresh),
|
|
4760
|
+
execution: opts.searchLedger ? identity.proposer : searchProposalExecution(fresh, identity.model.provider, identity.proposer.source)
|
|
4761
|
+
};
|
|
4762
|
+
};
|
|
4763
|
+
}
|
|
4764
|
+
/** Refuse to optimize against a baseline whose designed cells are not all scorable. */
|
|
4765
|
+
function assertBaselineComplete(run, rootCampaign) {
|
|
4766
|
+
const { opts, reps, requireJudgeScore, premeasured } = run;
|
|
4767
|
+
const rootCoverage = campaignCoverage(rootCampaign.cells, opts.scenarios, reps, requireJudgeScore);
|
|
4768
|
+
if (!rootCoverage.complete) throw new Error(`runOptimization: ${premeasured ? "premeasured baseline" : "baseline"} is incomplete (${rootCoverage.scorableCellIds.length}/${rootCoverage.expectedCellIds.length} designed cells scorable) — ${formatCoverageFailures(rootCoverage)}. Refusing to optimize against an incomplete incumbent.`);
|
|
4769
|
+
}
|
|
4770
|
+
/** Run the caller's analysis over the previous generation (or the baseline) and keep its findings. */
|
|
4771
|
+
async function analyzePreviousGeneration(run, earlier, generation, root, rootRunDir) {
|
|
4772
|
+
const { opts, nodes, policy, costLedger, runDir } = run;
|
|
4773
|
+
const previous = generation - 1;
|
|
4774
|
+
const candidates = (previous < 0 ? [root] : nodes.expansionNodes(earlier, nodes.generations(earlier)[previous])).map((nodeId) => {
|
|
4775
|
+
const campaign = nodes.campaign(earlier, nodeId);
|
|
4776
|
+
return {
|
|
4777
|
+
surfaceHash: surfaceHash(nodes.surface(earlier, nodeId)),
|
|
4778
|
+
campaign,
|
|
4779
|
+
composite: campaignMeanCompositeOrNull(campaign)
|
|
4780
|
+
};
|
|
4645
4781
|
});
|
|
4782
|
+
const history = nodes.history(earlier, policy);
|
|
4783
|
+
if (candidates.some(({ campaign }) => campaign.cells.length > 0)) {
|
|
4784
|
+
const fresh = await opts.analyzeGeneration({
|
|
4785
|
+
generation: previous,
|
|
4786
|
+
runDir: previous < 0 ? rootRunDir : `${runDir}/candidates`,
|
|
4787
|
+
candidates,
|
|
4788
|
+
history,
|
|
4789
|
+
costLedger,
|
|
4790
|
+
costPhase: previous < 0 ? "analysis.baseline" : "analysis.generation"
|
|
4791
|
+
});
|
|
4792
|
+
if (!Array.isArray(fresh)) throw new TypeError("runOptimization: analyzeGeneration must return an array");
|
|
4793
|
+
run.findings = immutableProposalSnapshot(assertProposalFindings(fresh, "runOptimization analysis findings"), "analysis findings");
|
|
4794
|
+
}
|
|
4795
|
+
}
|
|
4796
|
+
/** The frozen view of the search the caller's proposer reads. */
|
|
4797
|
+
function proposeContext(run, request, state, generation, root, proposalHistory) {
|
|
4798
|
+
const { opts, nodes, costLedger } = run;
|
|
4799
|
+
const parent = request.parents[0];
|
|
4800
|
+
return Object.freeze({
|
|
4801
|
+
currentSurface: immutableProposalSnapshot(parent.artifact, "current surface"),
|
|
4802
|
+
operator: request.operator,
|
|
4803
|
+
parents: immutableProposalSnapshot(request.parents.map(({ nodeId, artifact }) => ({
|
|
4804
|
+
nodeId,
|
|
4805
|
+
artifact: immutableProposalSnapshot(artifact, "parent surface")
|
|
4806
|
+
})), "parents"),
|
|
4807
|
+
train: searchProposerView(state),
|
|
4808
|
+
summary: renderSearchSummary(state, { split: "train" }),
|
|
4809
|
+
history: proposalHistory,
|
|
4810
|
+
findings: immutableProposalSnapshot(assertProposalFindings(run.findings, "runOptimization proposal findings"), "findings"),
|
|
4811
|
+
populationSize: opts.populationSize,
|
|
4812
|
+
generation,
|
|
4813
|
+
signal: request.signal,
|
|
4814
|
+
baselineOutcome: immutableProposalSnapshot(nodes.outcome(state, root), "baseline"),
|
|
4815
|
+
incumbentOutcome: immutableProposalSnapshot(nodes.outcome(state, request.leader), "incumbent outcome"),
|
|
4816
|
+
parentOutcome: immutableProposalSnapshot(nodes.outcome(state, parent.nodeId), "parent outcome"),
|
|
4817
|
+
maxImprovementShots: opts.maxImprovementShots,
|
|
4818
|
+
paretoParents: immutableProposalSnapshot(nodes.frontier(state), "Pareto parents"),
|
|
4819
|
+
costLedger,
|
|
4820
|
+
costPhase: "search.proposal"
|
|
4821
|
+
});
|
|
4822
|
+
}
|
|
4823
|
+
/** Read the finished search into the leader, its provenance and every generation's measured surfaces. */
|
|
4824
|
+
function optimizationResult(run, baselineSurface, result) {
|
|
4825
|
+
const { opts, nodes, policy, recorder, costLedger, runDir } = run;
|
|
4646
4826
|
const state = result.state;
|
|
4647
4827
|
const winnerSurface = nodes.surface(state, result.leader);
|
|
4648
4828
|
const firstEdge = state.edge(state.node(result.leader).edgeIds[0]);
|
|
@@ -4668,7 +4848,7 @@ async function runOptimization(opts) {
|
|
|
4668
4848
|
baselineCampaign: nodes.campaign(state, state.rootNodeId),
|
|
4669
4849
|
cost: costLedger.summary(),
|
|
4670
4850
|
searchHistory: recorder.receipt({
|
|
4671
|
-
producerId: proposer.kind,
|
|
4851
|
+
producerId: opts.proposer.kind,
|
|
4672
4852
|
runId: runDir
|
|
4673
4853
|
}),
|
|
4674
4854
|
paretoFrontier: nodes.frontier(state)
|
|
@@ -5750,6 +5930,6 @@ function firstString(value) {
|
|
|
5750
5930
|
return typeof value === "string" && value.trim() ? value : void 0;
|
|
5751
5931
|
}
|
|
5752
5932
|
//#endregion
|
|
5753
|
-
export {
|
|
5933
|
+
export { planCampaignRun as $, verifySearchHistoryReceipt as A, defaultProductionGate as B, searchEdgeId as C, assertSearchHistoryMatchesState as D, assertSearchHistoryAdmissionOptions as E, asha as F, runCanaries as G, redTeamDataset as H, uniform as I, evaluationUnitMap as J, assertIndependentEvaluationSplit as K, runFinalComparison as L, searchModelIdentity as M, searchProposalExecution as N, createSearchHistoryReceipt as O, searchReceiptAccounting as P, runCampaign as Q, openAutoPr as R, developmentClaim as S, surfaceNode as T, redTeamReport as U, DEFAULT_RED_TEAM_CORPUS as V, scoreRedTeamOutput as W, reserveFinalEvidence as X, exposeFinalEvidence as Y, runEval as Z, beam as _, aggregateRunScore as a, incumbentWithOperatorBandit as b, assertGepaCandidatePopulationSummary as c, costFromLedgerSummary as d, resolveRunDir as et, optimizationTokenUsageFromSummary as f, aide as g, isProposedCandidate as h, transientDispatchFailure as i, campaignCellSearchResult as j, verifySearchHistoryArtifact as k, readGepaCandidatePopulationArtifact as l, runOptimization as m, JudgeParseError as n, cellCachePath as nt, clamp01 as o, runImprovementLoop as p, captureFinalEvidencePolicy as q, isTransientTransportFailure as r, computeManifestHash as rt, executeOptimizationMethod as s, llmJudge as t, buildCellSchedule as tt, combineComparisonCosts as u, crowdedFrontierParent as v, surfaceDiff as w, SearchRecorder as x, incumbent as y, captureJudge as z };
|
|
5754
5934
|
|
|
5755
|
-
//# sourceMappingURL=llm-judge-
|
|
5935
|
+
//# sourceMappingURL=llm-judge-Djxh-Uhs.js.map
|