@tangle-network/agent-eval 0.216.1 → 0.218.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +137 -57
- package/README.md +2 -2
- package/dist/adapters/http.d.ts +2 -2
- package/dist/{agent-profile-VFcQncbW.d.ts → agent-profile-DTt8uGP9.d.ts} +4 -17
- package/dist/agent-profile-DTt8uGP9.d.ts.map +1 -0
- package/dist/{agent-profile-cell-BK8mlqlq.js → agent-profile-cell-B1V1gUZs.js} +4 -53
- package/dist/agent-profile-cell-B1V1gUZs.js.map +1 -0
- package/dist/{agent-profile-cell-BNp-Js9F.d.ts → agent-profile-cell-BcpFbNd5.d.ts} +2 -22
- package/dist/agent-profile-cell-BcpFbNd5.d.ts.map +1 -0
- package/dist/{agreement-irr-BEWLIF3_.js → agreement-irr-BhAqjluA.js} +5 -5
- package/dist/agreement-irr-BhAqjluA.js.map +1 -0
- package/dist/analyst/index.d.ts +36 -169
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +17 -192
- package/dist/analyst/index.js.map +1 -1
- package/dist/{analyze-runs-D68fd-su.d.ts → analyze-runs-DjlZOVXx.d.ts} +5 -5
- package/dist/{analyze-runs-D68fd-su.d.ts.map → analyze-runs-DjlZOVXx.d.ts.map} +1 -1
- package/dist/{analyze-runs-C84irm4i.js → analyze-runs-DqPJaJ4X.js} +225 -21
- package/dist/analyze-runs-DqPJaJ4X.js.map +1 -0
- package/dist/authenticity/index.d.ts +0 -23
- package/dist/authenticity/index.d.ts.map +1 -1
- package/dist/authenticity/index.js +1 -34
- package/dist/authenticity/index.js.map +1 -1
- package/dist/{benchmark-command-By0vh8q7.js → benchmark-command-BupdHRyf.js} +1225 -661
- package/dist/benchmark-command-BupdHRyf.js.map +1 -0
- package/dist/benchmarks/index.d.ts +5 -6
- package/dist/benchmarks/index.d.ts.map +1 -1
- package/dist/benchmarks/index.js +4 -4
- package/dist/benchmarks/index.js.map +1 -1
- package/dist/builder-eval/index.d.ts +3 -3
- package/dist/builder-eval/index.js +2 -2
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/campaign/index.d.ts +7 -7
- package/dist/campaign/index.js +15 -15
- package/dist/{campaign-BudZ70zo.js → campaign-CGvUTnmc.js} +324 -800
- package/dist/campaign-CGvUTnmc.js.map +1 -0
- package/dist/{campaign-evidence-Clrn4hBV.js → campaign-evidence-DunzcJGv.js} +10 -18
- package/dist/{campaign-evidence-Clrn4hBV.js.map → campaign-evidence-DunzcJGv.js.map} +1 -1
- package/dist/{canonical-B2RqLofv.js → canonical-Dq2hOQ8L.js} +2 -2
- package/dist/{canonical-B2RqLofv.js.map → canonical-Dq2hOQ8L.js.map} +1 -1
- package/dist/cell-spend-DEWR3nj-.js +90 -0
- package/dist/cell-spend-DEWR3nj-.js.map +1 -0
- package/dist/{chat-client-l-MVDlDk.js → chat-client-B2pXlPdx.js} +2 -2
- package/dist/{chat-client-l-MVDlDk.js.map → chat-client-B2pXlPdx.js.map} +1 -1
- package/dist/{chat-json-call-puMArmtW.js → chat-json-call-Dm9c-b41.js} +3 -2
- package/dist/chat-json-call-Dm9c-b41.js.map +1 -0
- package/dist/{claim-q4HQhK1G.js → claim-BPVuyHuy.js} +3 -3
- package/dist/{claim-q4HQhK1G.js.map → claim-BPVuyHuy.js.map} +1 -1
- package/dist/cli.js +113 -12
- package/dist/cli.js.map +1 -1
- package/dist/{client-BzL9niqi.d.ts → client-hwo3cJjR.d.ts} +3 -16
- package/dist/client-hwo3cJjR.d.ts.map +1 -0
- package/dist/{cluster-bootstrap-qcUWYFl-.js → cluster-bootstrap-DEwwhl0C.js} +3 -3
- package/dist/{cluster-bootstrap-qcUWYFl-.js.map → cluster-bootstrap-DEwwhl0C.js.map} +1 -1
- package/dist/{produced-state-BsDmnwxc.js → completion-verifier-DbLVf3i8.js} +9 -64
- package/dist/completion-verifier-DbLVf3i8.js.map +1 -0
- package/dist/contract/index.d.ts +12 -12
- package/dist/contract/index.js +107 -403
- package/dist/contract/index.js.map +1 -1
- package/dist/{cost-ledger-BP8V4kUQ.js → cost-ledger-D0352Z4d.js} +4 -8
- package/dist/cost-ledger-D0352Z4d.js.map +1 -0
- package/dist/dataset-DQqhOCPt.d.ts.map +1 -1
- package/dist/{default-registry-COhk9g36.js → default-registry-BJ-Pq6yL.js} +314 -285
- package/dist/default-registry-BJ-Pq6yL.js.map +1 -0
- package/dist/{default-registry-DzHiGruW.d.ts → default-registry-DXgLmoDI.d.ts} +7 -50
- package/dist/default-registry-DXgLmoDI.d.ts.map +1 -0
- package/dist/{define-agent-eval-CNTPzIkp.js → define-agent-eval-BGdFKlrc.js} +170 -110
- package/dist/define-agent-eval-BGdFKlrc.js.map +1 -0
- package/dist/{define-agent-eval-WSck0CZJ.d.ts → define-agent-eval-BlH10zEw.d.ts} +8 -8
- package/dist/define-agent-eval-BlH10zEw.d.ts.map +1 -0
- package/dist/{descriptive-DboxPZrG.js → descriptive-CVNy48rR.js} +2 -2
- package/dist/{descriptive-DboxPZrG.js.map → descriptive-CVNy48rR.js.map} +1 -1
- package/dist/diagnosis/index.d.ts +4 -14
- package/dist/diagnosis/index.d.ts.map +1 -1
- package/dist/diagnosis/index.js +67 -28
- package/dist/diagnosis/index.js.map +1 -1
- package/dist/{dspy-rlm-engine-LRCu61m5.js → dspy-rlm-engine-CSMwudLO.js} +5 -10
- package/dist/dspy-rlm-engine-CSMwudLO.js.map +1 -0
- package/dist/{emitter-HcCmFQl_.d.ts → emitter-DABYYTcH.d.ts} +5 -5
- package/dist/emitter-DABYYTcH.d.ts.map +1 -0
- package/dist/emitter-DbMXUIVP.js.map +1 -1
- package/dist/{engine-3A_4jOlI.d.ts → engine-BvZ0R-36.d.ts} +17 -21
- package/dist/engine-BvZ0R-36.d.ts.map +1 -0
- package/dist/{equal-terms-BtdXayCJ.js → equal-terms-DbPGcVtI.js} +2 -2
- package/dist/equal-terms-DbPGcVtI.js.map +1 -0
- package/dist/{errors-DVZiqS7K.js → errors-BcOs5MKW.js} +2 -8
- package/dist/{errors-DVZiqS7K.js.map → errors-BcOs5MKW.js.map} +1 -1
- package/dist/eval-campaign-BUMIxKTE.js +393 -0
- package/dist/eval-campaign-BUMIxKTE.js.map +1 -0
- package/dist/{evaluation-6Yoi74gM.d.ts → evaluation-BOZ38vOH.d.ts} +7 -7
- package/dist/evaluation-BOZ38vOH.d.ts.map +1 -0
- package/dist/evaluation.d.ts +2 -2
- package/dist/evaluation.js +1 -1
- package/dist/evaluation.js.map +1 -1
- package/dist/{exact-types-ClugMJfI.d.ts → exact-types-CflbJ9f3.d.ts} +2 -2
- package/dist/{exact-types-ClugMJfI.d.ts.map → exact-types-CflbJ9f3.d.ts.map} +1 -1
- package/dist/{exec-ChJ6d5if.js → exec-CpbtOXVt.js} +2 -24
- package/dist/{exec-ChJ6d5if.js.map → exec-CpbtOXVt.js.map} +1 -1
- package/dist/execution-tracks-CpgFPpS5.js.map +1 -1
- package/dist/experiment/index.d.ts +9 -112
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +258 -311
- package/dist/experiment/index.js.map +1 -1
- package/dist/{exporters-X4xwla8A.js → exporters-DWvM16LE.js} +3 -3
- package/dist/exporters-DWvM16LE.js.map +1 -0
- package/dist/{exporters-ChTnR90l.d.ts → exporters-Dk3oEbK_.d.ts} +2 -4
- package/dist/{exporters-ChTnR90l.d.ts.map → exporters-Dk3oEbK_.d.ts.map} +1 -1
- package/dist/{external-optimizer-process-BjMGUuds.js → external-optimizer-process-C3Wm7Vhn.js} +4 -3
- package/dist/external-optimizer-process-C3Wm7Vhn.js.map +1 -0
- package/dist/{external-optimizer-subprocess-BqW0hHcI.js → external-optimizer-subprocess-B5gdLRZv.js} +9 -11
- package/dist/external-optimizer-subprocess-B5gdLRZv.js.map +1 -0
- package/dist/{extract-usage-BrnzRJ6H.js → extract-usage-BZLU01kR.js} +2 -23
- package/dist/{extract-usage-BrnzRJ6H.js.map → extract-usage-BZLU01kR.js.map} +1 -1
- package/dist/{failure-cluster-CbgEDcFJ.d.ts → failure-cluster-DN0t63Z0.d.ts} +4 -43
- package/dist/failure-cluster-DN0t63Z0.d.ts.map +1 -0
- package/dist/{failure-taxonomy-CjabTATj.js → failure-taxonomy-Bx5oWqaW.js} +2 -19
- package/dist/{failure-taxonomy-CjabTATj.js.map → failure-taxonomy-Bx5oWqaW.js.map} +1 -1
- package/dist/{feedback-trajectory-C4EG0nNH.d.ts → feedback-trajectory-BWmhRBGu.d.ts} +7 -9
- package/dist/feedback-trajectory-BWmhRBGu.d.ts.map +1 -0
- package/dist/{semantic-concept-judge-BQF8te0T.js → findings-store-BtTzPL0d.js} +9 -249
- package/dist/findings-store-BtTzPL0d.js.map +1 -0
- package/dist/fuzz.d.ts +19 -62
- package/dist/fuzz.d.ts.map +1 -1
- package/dist/fuzz.js +62 -108
- package/dist/fuzz.js.map +1 -1
- package/dist/{hf-dataset-BR2yhuNn.js → hf-dataset-CKxIJQba.js} +4 -24
- package/dist/hf-dataset-CKxIJQba.js.map +1 -0
- package/dist/hosted/index.d.ts +2 -2
- package/dist/hosted/index.js +2 -2
- package/dist/{index-CyszSg4O.d.ts → index-B16FNdW5.d.ts} +145 -360
- package/dist/index-B16FNdW5.d.ts.map +1 -0
- package/dist/{index-DV-jSzxF.d.ts → index-BLGqZ81N.d.ts} +10 -15
- package/dist/{index-DV-jSzxF.d.ts.map → index-BLGqZ81N.d.ts.map} +1 -1
- package/dist/{index-BO41eGj5.d.ts → index-CX-GAUpw.d.ts} +3 -7
- package/dist/index-CX-GAUpw.d.ts.map +1 -0
- package/dist/{index-DHKEAv3s.d.ts → index-CalAsXMR.d.ts} +8 -68
- package/dist/index-CalAsXMR.d.ts.map +1 -0
- package/dist/{index-BFRO6chs.d.ts → index-CiRwLv2R.d.ts} +5 -69
- package/dist/index-CiRwLv2R.d.ts.map +1 -0
- package/dist/{index-DPcngtwt.d.ts → index-DbMOXYnP.d.ts} +11 -36
- package/dist/index-DbMOXYnP.d.ts.map +1 -0
- package/dist/index.d.ts +211 -218
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +999 -1714
- package/dist/index.js.map +1 -1
- package/dist/{integrity-dLiDe2kD.js → integrity-C8oUeB0g.js} +326 -203
- package/dist/integrity-C8oUeB0g.js.map +1 -0
- package/dist/{integrity-BXOsAb3i.js → integrity-W4CADcFc.js} +3 -7
- package/dist/{integrity-BXOsAb3i.js.map → integrity-W4CADcFc.js.map} +1 -1
- package/dist/{integrity-DAMEDFaC.d.ts → integrity-_qgZLA5p.d.ts} +4 -6
- package/dist/{integrity-DAMEDFaC.d.ts.map → integrity-_qgZLA5p.d.ts.map} +1 -1
- package/dist/{internal-Epl-YBZW.js → internal-BTK91Bqt.js} +3 -3
- package/dist/{internal-Epl-YBZW.js.map → internal-BTK91Bqt.js.map} +1 -1
- package/dist/{interval-CELpye1n.js → interval-Dq9gt7jx.js} +3 -3
- package/dist/{interval-CELpye1n.js.map → interval-Dq9gt7jx.js.map} +1 -1
- package/dist/{journal-DfRz9kps.js → journal-C7om8Gum.js} +157 -147
- package/dist/journal-C7om8Gum.js.map +1 -0
- package/dist/json-BhdsJaOQ.js +111 -0
- package/dist/json-BhdsJaOQ.js.map +1 -0
- package/dist/{judge-calibration-BpCy5XQl.js → judge-calibration-BkXMIWA4.js} +3 -3
- package/dist/{judge-calibration-BpCy5XQl.js.map → judge-calibration-BkXMIWA4.js.map} +1 -1
- package/dist/{judge-gate-iY9s4L44.js → judge-gate-Co5tuh57.js} +4 -27
- package/dist/{judge-gate-iY9s4L44.js.map → judge-gate-Co5tuh57.js.map} +1 -1
- package/dist/{judge-gate-CE7CQAgt.d.ts → judge-gate-DsL0yFcb.d.ts} +8 -9
- package/dist/judge-gate-DsL0yFcb.d.ts.map +1 -0
- package/dist/{kind-factory-BobI5Jy3.js → kind-factory-Gy4fXixd.js} +88 -17
- package/dist/kind-factory-Gy4fXixd.js.map +1 -0
- package/dist/ledger-core/index.d.ts +2 -2
- package/dist/ledger-core/index.js +4 -4
- package/dist/ledger-core/lock-holder.d.ts +2 -0
- package/dist/ledger-core/lock-holder.js +181 -0
- package/dist/ledger-core/lock-holder.js.map +1 -0
- package/dist/{ledger-core-Gq7yp8lP.js → ledger-core-CAxUdDk0.js} +5 -84
- package/dist/ledger-core-CAxUdDk0.js.map +1 -0
- package/dist/{llm-client-CyBAoYiX.js → llm-client-_Y6ic-tf.js} +243 -355
- package/dist/llm-client-_Y6ic-tf.js.map +1 -0
- package/dist/{llm-judge-CG12SQJq.js → llm-judge-Djxh-Uhs.js} +662 -482
- package/dist/llm-judge-Djxh-Uhs.js.map +1 -0
- package/dist/lock-holder-DaJbfpci.d.ts +86 -0
- package/dist/lock-holder-DaJbfpci.d.ts.map +1 -0
- package/dist/matrix/index.d.ts +3 -3
- package/dist/matrix/index.js +3 -2
- package/dist/{matrix-Dm0BQW7U.js → matrix-B8l-Nsxd.js} +3 -88
- package/dist/matrix-B8l-Nsxd.js.map +1 -0
- package/dist/{matrix-DSrSxjjy.d.ts → matrix-CCSBna4q.d.ts} +9 -18
- package/dist/matrix-CCSBna4q.d.ts.map +1 -0
- package/dist/{mean-DHoMiudD.js → mean-BbA_JCzW.js} +39 -5
- package/dist/mean-BbA_JCzW.js.map +1 -0
- package/dist/meta-eval/index.d.ts +5 -5
- package/dist/meta-eval/index.js +10 -10
- package/dist/metrics-BKaPBtIG.js.map +1 -1
- package/dist/{mint-DNU8QPG4.d.ts → mint-BRZXuwRi.d.ts} +4 -4
- package/dist/{mint-DNU8QPG4.d.ts.map → mint-BRZXuwRi.d.ts.map} +1 -1
- package/dist/{mint-QaD5cA0u.js → mint-CbdwjTPW.js} +4 -4
- package/dist/{mint-QaD5cA0u.js.map → mint-CbdwjTPW.js.map} +1 -1
- package/dist/{multi-layer-verifier-BUaQ4C17.d.ts → multi-layer-verifier-DEgMUk6v.d.ts} +2 -2
- package/dist/{multi-layer-verifier-BUaQ4C17.d.ts.map → multi-layer-verifier-DEgMUk6v.d.ts.map} +1 -1
- package/dist/{multiplicity-DQUFcbW4.js → multiplicity-Sg0U7h8j.js} +2 -2
- package/dist/{multiplicity-DQUFcbW4.js.map → multiplicity-Sg0U7h8j.js.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -23
- package/dist/multishot/golden/index.d.ts.map +1 -1
- package/dist/multishot/golden/index.js +3 -3
- package/dist/multishot/golden/index.js.map +1 -1
- package/dist/multishot/index.d.ts +3 -12
- package/dist/multishot/index.d.ts.map +1 -1
- package/dist/multishot/index.js +4 -3
- package/dist/multishot/index.js.map +1 -1
- package/dist/openapi.json +114 -10
- package/dist/{otel-spans-B9tUXRHF.js → otel-spans-CdN909k1.js} +3 -3
- package/dist/{otel-spans-B9tUXRHF.js.map → otel-spans-CdN909k1.js.map} +1 -1
- package/dist/{paired-arms-CwMGJsrS.js → paired-arms-Ct4LTKc1.js} +32 -9
- package/dist/paired-arms-Ct4LTKc1.js.map +1 -0
- package/dist/{paired-binary-CvpFHfKZ.js → paired-binary-B_nejA44.js} +3 -36
- package/dist/paired-binary-B_nejA44.js.map +1 -0
- package/dist/{paired-promotion-decision-DPKz8ZnF.d.ts → paired-promotion-decision-CQMJ5xGN.d.ts} +2 -34
- package/dist/paired-promotion-decision-CQMJ5xGN.d.ts.map +1 -0
- package/dist/{paired-promotion-decision-DsAPYGCm.js → paired-promotion-decision-D0SHb-ZL.js} +3 -3
- package/dist/{paired-promotion-decision-DsAPYGCm.js.map → paired-promotion-decision-D0SHb-ZL.js.map} +1 -1
- package/dist/{pareto-Cm6ndrWN.js → pareto-CTv8vozZ.js} +11 -11
- package/dist/pareto-CTv8vozZ.js.map +1 -0
- package/dist/pipelines/index.d.ts +16 -19
- package/dist/pipelines/index.d.ts.map +1 -1
- package/dist/pipelines/index.js +4 -68
- package/dist/pipelines/index.js.map +1 -1
- package/dist/{power-and-mde-Hj2ubb4B.js → power-and-mde-N5nU4_Ly.js} +3 -3
- package/dist/{power-and-mde-Hj2ubb4B.js.map → power-and-mde-N5nU4_Ly.js.map} +1 -1
- package/dist/{prime-protocol-CHKgA3a6.js → prime-protocol-BFgGrEMP.js} +3 -3
- package/dist/prime-protocol-BFgGrEMP.js.map +1 -0
- package/dist/{prime-protocol-HiB9HX7E.d.ts → prime-protocol-BWG2DQIk.d.ts} +3 -21
- package/dist/prime-protocol-BWG2DQIk.d.ts.map +1 -0
- package/dist/profile-cell.d.ts +2 -2
- package/dist/profile-cell.js +2 -2
- package/dist/{promotion-gate-BawrdPYr.d.ts → promotion-gate-CxEHRsZW.d.ts} +7 -8
- package/dist/{promotion-gate-BawrdPYr.d.ts.map → promotion-gate-CxEHRsZW.d.ts.map} +1 -1
- package/dist/{promotion-policy-KO57gbyV.js → promotion-policy-6A8osGr_.js} +4 -4
- package/dist/{promotion-policy-KO57gbyV.js.map → promotion-policy-6A8osGr_.js.map} +1 -1
- package/dist/{proposal-findings-CY1xKn_a.js → proposal-findings-n-5WBbEx.js} +3 -3
- package/dist/proposal-findings-n-5WBbEx.js.map +1 -0
- package/dist/{query-BUCZCK7F.js → query-B_RZ36an.js} +5 -5
- package/dist/query-B_RZ36an.js.map +1 -0
- package/dist/{query-DK1nWcmp.d.ts → query-Cmv_U8vf.d.ts} +4 -24
- package/dist/{query-DK1nWcmp.d.ts.map → query-Cmv_U8vf.d.ts.map} +1 -1
- package/dist/{random-ByUSnV9w.js → random-DDDcC7YU.js} +2 -2
- package/dist/{random-ByUSnV9w.js.map → random-DDDcC7YU.js.map} +1 -1
- package/dist/{raw-provider-sink-DoHz4JNG.js → raw-provider-sink-75w4FWBj.js} +2 -2
- package/dist/{raw-provider-sink-DoHz4JNG.js.map → raw-provider-sink-75w4FWBj.js.map} +1 -1
- package/dist/{redact-mtCCcT8z.d.ts → redact-DXdlFb_U.d.ts} +2 -3
- package/dist/redact-DXdlFb_U.d.ts.map +1 -0
- package/dist/{redact-B5xXKqQ3.js → redact-mtkTiJVk.js} +2 -7
- package/dist/{redact-B5xXKqQ3.js.map → redact-mtkTiJVk.js.map} +1 -1
- package/dist/{registry-D7_hm2fK.d.ts → registry-ssNoHzwD.d.ts} +4 -4
- package/dist/{registry-D7_hm2fK.d.ts.map → registry-ssNoHzwD.d.ts.map} +1 -1
- package/dist/{release-confidence-CpuP65S5.js → release-confidence-CYIFH-fp.js} +6 -11
- package/dist/release-confidence-CYIFH-fp.js.map +1 -0
- package/dist/{report-command-SC9nqb4M.js → render-Cs34y0iX.js} +625 -587
- package/dist/render-Cs34y0iX.js.map +1 -0
- package/dist/reporting.d.ts +6 -6
- package/dist/reporting.js +8 -8
- package/dist/{researcher-CEcfku-T.d.ts → researcher-GU1tV4dn.d.ts} +9 -9
- package/dist/{researcher-CEcfku-T.d.ts.map → researcher-GU1tV4dn.d.ts.map} +1 -1
- package/dist/{reward-CpCmAM8L.d.ts → reward-DrZ71vnf.d.ts} +3 -26
- package/dist/reward-DrZ71vnf.d.ts.map +1 -0
- package/dist/{reward-hacking-E0d-WVcP.js → reward-hacking-ByuETaIb.js} +3 -3
- package/dist/{reward-hacking-E0d-WVcP.js.map → reward-hacking-ByuETaIb.js.map} +1 -1
- package/dist/{reward-hacking-RVdZ6Kkt.d.ts → reward-hacking-E52wq86n.d.ts} +5 -18
- package/dist/reward-hacking-E52wq86n.d.ts.map +1 -0
- package/dist/reward-nw2xZGZG.js.map +1 -1
- package/dist/rl.d.ts +117 -295
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +27 -747
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +5 -5
- package/dist/rollout/index.js +7 -8
- package/dist/{rollout-DZV1t2Wg.js → rollout-Dd1ucT_M.js} +88 -13
- package/dist/rollout-Dd1ucT_M.js.map +1 -0
- package/dist/{rubric-predictive-validity-DaFW70y9.js → rubric-predictive-validity-D3_oI72N.js} +3 -3
- package/dist/{rubric-predictive-validity-DaFW70y9.js.map → rubric-predictive-validity-D3_oI72N.js.map} +1 -1
- package/dist/{rubric-predictive-validity-C03EC7ZE.d.ts → rubric-predictive-validity-DiwT8PbL.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-C03EC7ZE.d.ts.map → rubric-predictive-validity-DiwT8PbL.d.ts.map} +1 -1
- package/dist/{run-record-C1E24G2A.d.ts → run-record-CaxbLq6Y.d.ts} +4 -10
- package/dist/run-record-CaxbLq6Y.d.ts.map +1 -0
- package/dist/{run-record-C5e7FjhJ.js → run-record-CysKLB93.js} +3 -3
- package/dist/run-record-CysKLB93.js.map +1 -0
- package/dist/{run-record-Boc5IUTp.js → run-record-DhEJgoPz.js} +4 -4
- package/dist/{run-record-Boc5IUTp.js.map → run-record-DhEJgoPz.js.map} +1 -1
- package/dist/sandbox-harness-BlSOu4LX.d.ts.map +1 -1
- package/dist/{schema-CSf6qWgZ.js → schema-Bt2vLFm3.js} +2 -14
- package/dist/schema-Bt2vLFm3.js.map +1 -0
- package/dist/schema-C-ziyJD7.d.ts.map +1 -1
- package/dist/{schema-BuiujZ-g.js → schema-CvhfLjtZ.js} +78 -48
- package/dist/schema-CvhfLjtZ.js.map +1 -0
- package/dist/{schema-2g7trUwH.d.ts → schema-D6EVKzTB.d.ts} +4 -5
- package/dist/schema-D6EVKzTB.d.ts.map +1 -0
- package/dist/search/index.d.ts +3 -31
- package/dist/search/index.d.ts.map +1 -1
- package/dist/search/index.js +4 -4
- package/dist/{search-kernel-DyNEa6IU.d.ts → search-kernel-p9JB3SmL.d.ts} +10 -27
- package/dist/search-kernel-p9JB3SmL.d.ts.map +1 -0
- package/dist/{search-ledger-CVUGLDZp.js → search-ledger-ClpN4I3e.js} +7 -7
- package/dist/{search-ledger-CVUGLDZp.js.map → search-ledger-ClpN4I3e.js.map} +1 -1
- package/dist/{search-ledger-file-DDQUgvCg.js → search-ledger-file-Bd5Tioq6.js} +4 -4
- package/dist/search-ledger-file-Bd5Tioq6.js.map +1 -0
- package/dist/{search-shipper-BOSD46qv.js → search-shipper-Bg2HW3Uv.js} +4 -4
- package/dist/search-shipper-Bg2HW3Uv.js.map +1 -0
- package/dist/{search-summary-BQCSWf1g.js → search-summary-CrBuvVnf.js} +5 -5
- package/dist/{search-summary-BQCSWf1g.js.map → search-summary-CrBuvVnf.js.map} +1 -1
- package/dist/{sequential-CzK5DarL.js → sequential-BVv1sjsw.js} +2 -32
- package/dist/{sequential-CzK5DarL.js.map → sequential-BVv1sjsw.js.map} +1 -1
- package/dist/{sequential-Bf_pbE50.js → sequential-CimQrbjv.js} +5 -5
- package/dist/sequential-CimQrbjv.js.map +1 -0
- package/dist/{sequential-BhsrMupG.d.ts → sequential-KwGspfeR.d.ts} +2 -9
- package/dist/{sequential-BhsrMupG.d.ts.map → sequential-KwGspfeR.d.ts.map} +1 -1
- package/dist/{sequential-eprocess-DM_qENCl.js → sequential-eprocess-DodI6RoF.js} +2 -2
- package/dist/{sequential-eprocess-DM_qENCl.js.map → sequential-eprocess-DodI6RoF.js.map} +1 -1
- package/dist/{server-e9oWO_bN.js → server-C6M0qR8h.js} +101 -101
- package/dist/server-C6M0qR8h.js.map +1 -0
- package/dist/{single-run-lock-CV9Eyd_q.js → single-run-lock-Dekd8_Wv.js} +7 -7
- package/dist/single-run-lock-Dekd8_Wv.js.map +1 -0
- package/dist/{skillopt-optimization-method-BiwA8nfX.js → skillopt-optimization-method-5dlex7MJ.js} +1386 -1184
- package/dist/skillopt-optimization-method-5dlex7MJ.js.map +1 -0
- package/dist/{spans-BHAUXMGN.d.ts → spans-CVngGij3.d.ts} +2 -2
- package/dist/{spans-BHAUXMGN.d.ts.map → spans-CVngGij3.d.ts.map} +1 -1
- package/dist/{statistical-heldout-CYpGaoDH.d.ts → statistical-heldout-BHfLZHrm.d.ts} +8 -67
- package/dist/statistical-heldout-BHfLZHrm.d.ts.map +1 -0
- package/dist/{statistical-heldout-42NlCNDe.js → statistical-heldout-Bksv92Th.js} +3 -3
- package/dist/statistical-heldout-Bksv92Th.js.map +1 -0
- package/dist/{steps-Bg5-KVx1.d.ts → steps-DlepJPwD.d.ts} +2 -56
- package/dist/steps-DlepJPwD.d.ts.map +1 -0
- package/dist/{store-C4avFjIi.d.ts → store-B3nWRy3B.d.ts} +2 -2
- package/dist/{store-C4avFjIi.d.ts.map → store-B3nWRy3B.d.ts.map} +1 -1
- package/dist/{store-otlp-DJ9dtA5x.js → store-otlp-BteDAbHK.js} +3 -3
- package/dist/{store-otlp-DJ9dtA5x.js.map → store-otlp-BteDAbHK.js.map} +1 -1
- package/dist/{store-tool-spans-CIX5arAG.js → store-tool-spans-D5l_OfTJ.js} +8 -8
- package/dist/store-tool-spans-D5l_OfTJ.js.map +1 -0
- package/dist/{store-tool-spans-DdCSPa7C.d.ts → store-tool-spans-DZS0HWVf.d.ts} +9 -18
- package/dist/store-tool-spans-DZS0HWVf.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts +4 -11
- package/dist/storyboard/index.d.ts.map +1 -1
- package/dist/storyboard/index.js +1 -16
- package/dist/storyboard/index.js.map +1 -1
- package/dist/{summary-report-V_JACR4Y.js → summary-report-BsXJoYX5.js} +88 -10
- package/dist/summary-report-BsXJoYX5.js.map +1 -0
- package/dist/{summary-report-bdFDXq0b.d.ts → summary-report-CkbRcSuu.d.ts} +25 -19
- package/dist/summary-report-CkbRcSuu.d.ts.map +1 -0
- package/dist/supervisor-run/index.d.ts +60 -28
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +4 -458
- package/dist/supervisor-run/index.js.map +1 -1
- package/dist/{systemone-BomhHr2c.d.ts → systemone-BMzDJb1i.d.ts} +5 -5
- package/dist/{systemone-BomhHr2c.d.ts.map → systemone-BMzDJb1i.d.ts.map} +1 -1
- package/dist/systemone-protocol.js +1 -1
- package/dist/{systemone-review-CUVcQJu6.d.ts → systemone-review-CwE6JGIp.d.ts} +2 -2
- package/dist/{systemone-review-CUVcQJu6.d.ts.map → systemone-review-CwE6JGIp.d.ts.map} +1 -1
- package/dist/{systemone-review-D7sXwsqq.js → systemone-review-DzdOJ8I6.js} +3 -3
- package/dist/{systemone-review-D7sXwsqq.js.map → systemone-review-DzdOJ8I6.js.map} +1 -1
- package/dist/systemone.d.ts +5 -5
- package/dist/systemone.js +5 -5
- package/dist/systemone.js.map +1 -1
- package/dist/{task-failure-attributes-hEmASjix.js → task-failure-attributes-BMsblaDh.js} +3 -3
- package/dist/task-failure-attributes-BMsblaDh.js.map +1 -0
- package/dist/{terminal-record-DZyQZ3pz.js → terminal-record-D4Rv-j1W.js} +244 -196
- package/dist/terminal-record-D4Rv-j1W.js.map +1 -0
- package/dist/{tool-groups-C3NEcdCM.d.ts → tool-groups-DyNz2JZ_.d.ts} +3 -3
- package/dist/tool-groups-DyNz2JZ_.d.ts.map +1 -0
- package/dist/{tool-waste-BKgvd-TP.d.ts → tool-waste-CkjzWmeY.d.ts} +25 -28
- package/dist/tool-waste-CkjzWmeY.d.ts.map +1 -0
- package/dist/{tool-waste-BlesqTqW.js → tool-waste-DLsjS_p-.js} +5 -5
- package/dist/{tool-waste-BlesqTqW.js.map → tool-waste-DLsjS_p-.js.map} +1 -1
- package/dist/{trace-questions-CM2CAocU.d.ts → trace-questions-xxwgsuMi.d.ts} +4 -9
- package/dist/{trace-questions-CM2CAocU.d.ts.map → trace-questions-xxwgsuMi.d.ts.map} +1 -1
- package/dist/trace-repair/index.d.ts +14 -211
- package/dist/trace-repair/index.d.ts.map +1 -1
- package/dist/trace-repair/index.js +865 -981
- package/dist/trace-repair/index.js.map +1 -1
- package/dist/traces.d.ts +18 -46
- package/dist/traces.d.ts.map +1 -1
- package/dist/traces.js +13 -13
- package/dist/traces.js.map +1 -1
- package/dist/trajectory-replay/index.d.ts +7 -118
- package/dist/trajectory-replay/index.d.ts.map +1 -1
- package/dist/trajectory-replay/index.js +921 -869
- package/dist/trajectory-replay/index.js.map +1 -1
- package/dist/{tree-C1XYeXvy.js → tree-B5e1uAeL.js} +132 -21
- package/dist/tree-B5e1uAeL.js.map +1 -0
- package/dist/{types-Cbe2fWBa.d.ts → types-BKCzzmit.d.ts} +3 -3
- package/dist/{types-Cbe2fWBa.d.ts.map → types-BKCzzmit.d.ts.map} +1 -1
- package/dist/{types-B3jzCp0p.js → types-BXgASBCL.js} +2 -2
- package/dist/{types-B3jzCp0p.js.map → types-BXgASBCL.js.map} +1 -1
- package/dist/{types-C1tnkEgM.d.ts → types-CW8ewATZ.d.ts} +4 -113
- package/dist/types-CW8ewATZ.d.ts.map +1 -0
- package/dist/{types-BqkVmHfv.d.ts → types-CmXY0PQf.d.ts} +2 -4
- package/dist/types-CmXY0PQf.d.ts.map +1 -0
- package/dist/{types-C02qtnG8.d.ts → types-D-mgRzlh.d.ts} +7 -28
- package/dist/types-D-mgRzlh.d.ts.map +1 -0
- package/dist/{types-IXD2HJzu.js → types-DqFMNTgK.js} +2 -4
- package/dist/types-DqFMNTgK.js.map +1 -0
- package/dist/{types-Cq8mSbq8.d.ts → types-DtkO1G9L.d.ts} +7 -9
- package/dist/{types-Cq8mSbq8.d.ts.map → types-DtkO1G9L.d.ts.map} +1 -1
- package/dist/{verdict-Dl0tiT1X.js → verdict-BGQmxQ8A.js} +3 -3
- package/dist/verdict-BGQmxQ8A.js.map +1 -0
- package/dist/{verdict-E4eRNf7-.d.ts → verdict-C24NELtN.d.ts} +2 -15
- package/dist/{verdict-E4eRNf7-.d.ts.map → verdict-C24NELtN.d.ts.map} +1 -1
- package/dist/{verdict-cache-Bm8EKL2s.js → verdict-cache-CL_7tePD.js} +2 -2
- package/dist/verdict-cache-CL_7tePD.js.map +1 -0
- package/dist/wire/index.d.ts +31 -410
- package/dist/wire/index.d.ts.map +1 -1
- package/dist/wire/index.js +2 -2
- package/docs/changes/jev-evidence-review.md +1 -1
- package/docs/changes/jev-product-adoption.md +1 -1
- package/docs/changes/jev-recorded-evidence.md +1 -1
- package/docs/design/statistics-decisions.md +18 -18
- package/docs/design.md +1 -1
- package/docs/eval-surface-map.md +1 -0
- package/docs/evaluation-integrity.md +1 -1
- package/docs/profile-json-identity.md +1 -2
- package/docs/proofs/campaign-attempt-retention-20261001/README.md +2 -2
- package/docs/public-api.md +158 -538
- package/docs/rollout.md +1 -1
- package/docs/trace-repair-admission.md +1 -1
- package/docs/wire-protocol.md +1 -1
- package/package.json +7 -2
- package/dist/active-curriculum-B_Lc9kfO.js +0 -61
- package/dist/active-curriculum-B_Lc9kfO.js.map +0 -1
- package/dist/agent-profile-VFcQncbW.d.ts.map +0 -1
- package/dist/agent-profile-cell-BK8mlqlq.js.map +0 -1
- package/dist/agent-profile-cell-BNp-Js9F.d.ts.map +0 -1
- package/dist/agreement-irr-BEWLIF3_.js.map +0 -1
- package/dist/analyze-runs-C84irm4i.js.map +0 -1
- package/dist/baseline-Udi_PLuT.js +0 -259
- package/dist/baseline-Udi_PLuT.js.map +0 -1
- package/dist/benchmark-CXmVVHH2.js +0 -562
- package/dist/benchmark-CXmVVHH2.js.map +0 -1
- package/dist/benchmark-command-By0vh8q7.js.map +0 -1
- package/dist/campaign-BudZ70zo.js.map +0 -1
- package/dist/chat-json-call-puMArmtW.js.map +0 -1
- package/dist/claude-jsonl-dZir12m6.js +0 -86
- package/dist/claude-jsonl-dZir12m6.js.map +0 -1
- package/dist/client-BzL9niqi.d.ts.map +0 -1
- package/dist/cost-ledger-BP8V4kUQ.js.map +0 -1
- package/dist/counterfactual-CPaWN99-.d.ts +0 -60
- package/dist/counterfactual-CPaWN99-.d.ts.map +0 -1
- package/dist/counterfactual-Dn3veueY.js +0 -98
- package/dist/counterfactual-Dn3veueY.js.map +0 -1
- package/dist/default-registry-COhk9g36.js.map +0 -1
- package/dist/default-registry-DzHiGruW.d.ts.map +0 -1
- package/dist/define-agent-eval-CNTPzIkp.js.map +0 -1
- package/dist/define-agent-eval-WSck0CZJ.d.ts.map +0 -1
- package/dist/dspy-rlm-engine-LRCu61m5.js.map +0 -1
- package/dist/effect-sizes-BhzEpXMc.js +0 -82
- package/dist/effect-sizes-BhzEpXMc.js.map +0 -1
- package/dist/emitter-HcCmFQl_.d.ts.map +0 -1
- package/dist/engine-3A_4jOlI.d.ts.map +0 -1
- package/dist/equal-terms-BtdXayCJ.js.map +0 -1
- package/dist/eval-campaign-90jeaCAI.js +0 -343
- package/dist/eval-campaign-90jeaCAI.js.map +0 -1
- package/dist/evaluation-6Yoi74gM.d.ts.map +0 -1
- package/dist/exporters-X4xwla8A.js.map +0 -1
- package/dist/external-optimizer-process-BjMGUuds.js.map +0 -1
- package/dist/external-optimizer-subprocess-BqW0hHcI.js.map +0 -1
- package/dist/failure-cluster-CbgEDcFJ.d.ts.map +0 -1
- package/dist/feedback-trajectory-C4EG0nNH.d.ts.map +0 -1
- package/dist/hf-dataset-BR2yhuNn.js.map +0 -1
- package/dist/index-BFRO6chs.d.ts.map +0 -1
- package/dist/index-BO41eGj5.d.ts.map +0 -1
- package/dist/index-CyszSg4O.d.ts.map +0 -1
- package/dist/index-DHKEAv3s.d.ts.map +0 -1
- package/dist/index-DPcngtwt.d.ts.map +0 -1
- package/dist/integrity-dLiDe2kD.js.map +0 -1
- package/dist/journal-DfRz9kps.js.map +0 -1
- package/dist/judge-gate-CE7CQAgt.d.ts.map +0 -1
- package/dist/kind-factory-BobI5Jy3.js.map +0 -1
- package/dist/ledger-core-Gq7yp8lP.js.map +0 -1
- package/dist/llm-client-CyBAoYiX.js.map +0 -1
- package/dist/llm-judge-CG12SQJq.js.map +0 -1
- package/dist/matrix-DSrSxjjy.d.ts.map +0 -1
- package/dist/matrix-Dm0BQW7U.js.map +0 -1
- package/dist/mean-DHoMiudD.js.map +0 -1
- package/dist/normal-DsJZtrJc.js +0 -25
- package/dist/normal-DsJZtrJc.js.map +0 -1
- package/dist/paired-arms-CwMGJsrS.js.map +0 -1
- package/dist/paired-binary-CvpFHfKZ.js.map +0 -1
- package/dist/paired-promotion-decision-DPKz8ZnF.d.ts.map +0 -1
- package/dist/pareto-Cm6ndrWN.js.map +0 -1
- package/dist/prime-protocol-CHKgA3a6.js.map +0 -1
- package/dist/prime-protocol-HiB9HX7E.d.ts.map +0 -1
- package/dist/produced-state-BsDmnwxc.js.map +0 -1
- package/dist/proposal-findings-CY1xKn_a.js.map +0 -1
- package/dist/query-BUCZCK7F.js.map +0 -1
- package/dist/redact-mtCCcT8z.d.ts.map +0 -1
- package/dist/release-confidence-CpuP65S5.js.map +0 -1
- package/dist/report-command-SC9nqb4M.js.map +0 -1
- package/dist/reward-CpCmAM8L.d.ts.map +0 -1
- package/dist/reward-hacking-RVdZ6Kkt.d.ts.map +0 -1
- package/dist/rollout-DZV1t2Wg.js.map +0 -1
- package/dist/run-record-C1E24G2A.d.ts.map +0 -1
- package/dist/run-record-C5e7FjhJ.js.map +0 -1
- package/dist/schema-2g7trUwH.d.ts.map +0 -1
- package/dist/schema-BuiujZ-g.js.map +0 -1
- package/dist/schema-CSf6qWgZ.js.map +0 -1
- package/dist/search-kernel-DyNEa6IU.d.ts.map +0 -1
- package/dist/search-ledger-file-DDQUgvCg.js.map +0 -1
- package/dist/search-shipper-BOSD46qv.js.map +0 -1
- package/dist/semantic-concept-judge-BQF8te0T.js.map +0 -1
- package/dist/sequential-Bf_pbE50.js.map +0 -1
- package/dist/server-e9oWO_bN.js.map +0 -1
- package/dist/single-run-lock-CV9Eyd_q.js.map +0 -1
- package/dist/skillopt-optimization-method-BiwA8nfX.js.map +0 -1
- package/dist/statistical-heldout-42NlCNDe.js.map +0 -1
- package/dist/statistical-heldout-CYpGaoDH.d.ts.map +0 -1
- package/dist/steps-Bg5-KVx1.d.ts.map +0 -1
- package/dist/store-tool-spans-CIX5arAG.js.map +0 -1
- package/dist/store-tool-spans-DdCSPa7C.d.ts.map +0 -1
- package/dist/student-t-DVPW7CdL.js +0 -39
- package/dist/student-t-DVPW7CdL.js.map +0 -1
- package/dist/summary-report-V_JACR4Y.js.map +0 -1
- package/dist/summary-report-bdFDXq0b.d.ts.map +0 -1
- package/dist/task-failure-attributes-hEmASjix.js.map +0 -1
- package/dist/terminal-record-DZyQZ3pz.js.map +0 -1
- package/dist/tool-groups-C3NEcdCM.d.ts.map +0 -1
- package/dist/tool-waste-BKgvd-TP.d.ts.map +0 -1
- package/dist/trajectory-Y1fmdlw7.d.ts +0 -24
- package/dist/trajectory-Y1fmdlw7.d.ts.map +0 -1
- package/dist/tree-C1XYeXvy.js.map +0 -1
- package/dist/types-BqkVmHfv.d.ts.map +0 -1
- package/dist/types-C02qtnG8.d.ts.map +0 -1
- package/dist/types-C1tnkEgM.d.ts.map +0 -1
- package/dist/types-IXD2HJzu.js.map +0 -1
- package/dist/usage-receipt-Cx3xNeyB.js +0 -76
- package/dist/usage-receipt-Cx3xNeyB.js.map +0 -1
- package/dist/verdict-Dl0tiT1X.js.map +0 -1
- package/dist/verdict-cache-Bm8EKL2s.js.map +0 -1
- package/docs/design/self-improvement-evidence-audit.md +0 -263
package/dist/index.js
CHANGED
|
@@ -1,66 +1,64 @@
|
|
|
1
1
|
import { t as __exportAll } from "./rolldown-runtime-8H4AJuhK.js";
|
|
2
|
-
import { i as JudgeError, n as CaptureIntegrityError, o as NotFoundError, r as ConfigError, s as ValidationError, t as AgentEvalError } from "./errors-
|
|
3
|
-
import { a as hashCanonical, r as canonicalString } from "./canonical-
|
|
4
|
-
import { a as
|
|
5
|
-
import {
|
|
2
|
+
import { i as JudgeError, n as CaptureIntegrityError, o as NotFoundError, r as ConfigError, s as ValidationError, t as AgentEvalError } from "./errors-BcOs5MKW.js";
|
|
3
|
+
import { a as hashCanonical, r as canonicalString } from "./canonical-Dq2hOQ8L.js";
|
|
4
|
+
import { a as HARNESS_NATIVE_MODEL, c as agentProfileModelId, i as CODING_HARNESSES, l as expandProfileAxes, n as createLlmCorrectnessChecker, o as agentProfileHash, r as verifyCompletion, s as agentProfileId, t as completionVerdict, u as harnessAxisOf } from "./completion-verifier-DbLVf3i8.js";
|
|
5
|
+
import { a as groupRunsByAgentProfileCell, c as verifyAgentProfileCell, i as buildAgentProfileCell, l as hashJson, o as toAgentProfileJson, r as agentProfileCellKey, s as validateAgentProfileCell, t as AGENT_PROFILE_KINDS, u as manifestContentDigest } from "./agent-profile-cell-B1V1gUZs.js";
|
|
6
6
|
import { r as observedSplitScore, t as isRealnessGated } from "./reward-nw2xZGZG.js";
|
|
7
|
-
import {
|
|
8
|
-
import { a as parseRunRecordSafe, c as validateRunRecord, i as modelHasSnapshot, n as UNKNOWN_MODEL, o as runTaskScore, r as isRunRecord,
|
|
9
|
-
import { i as summaryTable, n as gainHistogram,
|
|
7
|
+
import { i as isToolSpan, n as isJudgeSpan, r as isLlmSpan, t as FAILURE_CLASSES } from "./schema-Bt2vLFm3.js";
|
|
8
|
+
import { a as parseRunRecordSafe, c as validateRunRecord, i as modelHasSnapshot, n as UNKNOWN_MODEL, o as runTaskScore, r as isRunRecord, t as RunRecordValidationError } from "./run-record-DhEJgoPz.js";
|
|
9
|
+
import { a as cliffsDelta, c as pairedCohensDz, i as summaryTable, n as gainHistogram, o as cohensD, s as interpretCliffs } from "./summary-report-BsXJoYX5.js";
|
|
10
10
|
import { t as interpolatedQuantile } from "./quantile--0yb07pD.js";
|
|
11
|
-
import { a as medianInPlace, i as makeRng, t as assertFiniteSample } from "./internal-
|
|
12
|
-
import { t as mulberry32 } from "./random-
|
|
13
|
-
import { a as spearmanR, c as weightedMean, i as ranks, n as partialCredit, o as summarizeNumberSeries, r as pearsonR, s as weightedComposite, t as confidenceInterval } from "./descriptive-
|
|
14
|
-
import { n as calibrateJudgeContinuous, o as verbosityBias, r as continuousAgreement, t as calibrateJudge } from "./judge-calibration-
|
|
15
|
-
import { n as corpusInterRaterAgreementFromJudgeScores, r as interRaterReliability, t as corpusInterRaterAgreement } from "./agreement-irr-
|
|
16
|
-
import { i as
|
|
17
|
-
import { a as
|
|
18
|
-
import { n as bonferroni, r as holm, t as benjaminiHochberg } from "./multiplicity-
|
|
19
|
-
import { a as
|
|
20
|
-
import {
|
|
21
|
-
import { a as requiredSampleSize, i as requiredPairedSampleSize, n as mcnemarRequiredN, r as pairedMde, t as mcnemarPower } from "./power-and-mde-
|
|
22
|
-
import {
|
|
23
|
-
import { t as
|
|
24
|
-
import {
|
|
25
|
-
import { n as
|
|
26
|
-
import {
|
|
27
|
-
import { n as contentHash, r as fileVerdictCache, t as canonicalJson } from "./verdict-cache-Bm8EKL2s.js";
|
|
28
|
-
import { J as redTeamReport, K as DEFAULT_RED_TEAM_CORPUS, X as runCanaries, Y as scoreRedTeamOutput, a as transientDispatchFailure, i as quotaExhaustedUntil, o as aggregateRunScore, q as redTeamDataset, rt as runCampaign, s as clamp01, t as llmJudge } from "./llm-judge-CG12SQJq.js";
|
|
11
|
+
import { a as medianInPlace, i as makeRng, t as assertFiniteSample } from "./internal-BTK91Bqt.js";
|
|
12
|
+
import { t as mulberry32 } from "./random-DDDcC7YU.js";
|
|
13
|
+
import { a as spearmanR, c as weightedMean, i as ranks, n as partialCredit, o as summarizeNumberSeries, r as pearsonR, s as weightedComposite, t as confidenceInterval } from "./descriptive-CVNy48rR.js";
|
|
14
|
+
import { n as calibrateJudgeContinuous, o as verbosityBias, r as continuousAgreement, t as calibrateJudge } from "./judge-calibration-BkXMIWA4.js";
|
|
15
|
+
import { n as corpusInterRaterAgreementFromJudgeScores, r as interRaterReliability, t as corpusInterRaterAgreement } from "./agreement-irr-BhAqjluA.js";
|
|
16
|
+
import { a as wilcoxonSignedRank, i as mannWhitneyU, n as pairArms, r as pairRunRecords, t as comparePairedArms } from "./paired-arms-Ct4LTKc1.js";
|
|
17
|
+
import { a as welchsTTest, c as fisherExactTwoSided, d as studentTTwoSidedPValue, f as twoByTwoPValue, i as iqr, l as linearRegressionSlope, n as checkCanaries, o as chiSquared, s as chiSquaredPValue, t as analyzeRuns, u as normalTwoSidedPValue } from "./analyze-runs-DqPJaJ4X.js";
|
|
18
|
+
import { n as bonferroni, r as holm, t as benjaminiHochberg } from "./multiplicity-Sg0U7h8j.js";
|
|
19
|
+
import { a as pairedRiskDifferenceScore, i as pairedRiskDifferenceExact, o as passAtK, r as pairedRiskDifference, s as wilson, t as mcnemar } from "./paired-binary-B_nejA44.js";
|
|
20
|
+
import { i as pairedBootstrap, n as BOOTSTRAP_GATE_MIN_N, o as pairedSignTest, s as pairedTTest, t as mean$1 } from "./mean-BbA_JCzW.js";
|
|
21
|
+
import { a as requiredSampleSize, i as requiredPairedSampleSize, n as mcnemarRequiredN, r as pairedMde, t as mcnemarPower } from "./power-and-mde-N5nU4_Ly.js";
|
|
22
|
+
import { t as eProcess } from "./sequential-eprocess-DodI6RoF.js";
|
|
23
|
+
import { i as pairedDeltaTest, n as pairedDecisionShape, r as minimumPairsForPairedDeltaTest, t as decidePairedPromotion } from "./paired-promotion-decision-D0SHb-ZL.js";
|
|
24
|
+
import { c as runsForScenario, i as judgeSpans, t as argHash } from "./query-B_RZ36an.js";
|
|
25
|
+
import { n as contentHash, r as fileVerdictCache, t as canonicalJson } from "./verdict-cache-CL_7tePD.js";
|
|
26
|
+
import { G as runCanaries, H as redTeamDataset, Q as runCampaign, U as redTeamReport, V as DEFAULT_RED_TEAM_CORPUS, W as scoreRedTeamOutput, a as aggregateRunScore, i as transientDispatchFailure, o as clamp01, t as llmJudge } from "./llm-judge-Djxh-Uhs.js";
|
|
29
27
|
import { a as resolveModelPricing, i as isModelPriced, n as estimateCost, r as estimateTokens, t as MODEL_PRICING } from "./metrics-BKaPBtIG.js";
|
|
30
|
-
import {
|
|
31
|
-
import {
|
|
32
|
-
import {
|
|
33
|
-
import { a as
|
|
34
|
-
import { S as requiredPairsForPairedPromotion, n as paretoFrontier, t as dominates, x as pairedPromotionPower } from "./pareto-
|
|
35
|
-
import {
|
|
36
|
-
import {
|
|
37
|
-
import { t as
|
|
38
|
-
import { t as
|
|
39
|
-
import { t as
|
|
40
|
-
import { G as DEFAULT_TRACE_ANALYST_OUTPUT_TOKENS, U as parseFindingSubject, h as TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, l as createBoundedTraceAnalysisStore, m as DEFAULT_TRACE_ANALYST_BUDGETS, t as createTraceAnalyst } from "./kind-factory-BobI5Jy3.js";
|
|
41
|
-
import { _ as completedAnalystReviewQuality, b as validateAnalystReviewDecisions, c as FAILURE_MODE_KIND_SPEC, g as assertUniqueFindingIds, h as analystRunDigest, i as DEFAULT_TRACE_ANALYST_KINDS, m as analystFindingDigest, n as AnalystRegistry, t as buildDefaultAnalystRegistry, v as readAnalystReview, y as snapshotAnalystRun } from "./default-registry-COhk9g36.js";
|
|
28
|
+
import { c as costForTokenPricing, i as CostCeilingReachedError, l as costForUsage, n as CostAccountingIncompleteError, t as CostLedger } from "./cost-ledger-D0352Z4d.js";
|
|
29
|
+
import { D as surfaceContentHash, M as assertRealBackend, N as summarizeBackendIntegrity, _ as parseReflectionResponse, a as ATTESTATION_ALGORITHM, g as buildReflectionPrompt, j as BackendIntegrityError, o as attest, s as verifyAttestation } from "./campaign-evidence-DunzcJGv.js";
|
|
30
|
+
import { a as redact, c as shareAllowed, n as assessShareSafety, o as redactForShare, s as redactText, t as REDACTION_VERSION } from "./redact-mtkTiJVk.js";
|
|
31
|
+
import { a as makeProposalFinding, i as makeFinding, r as computeFindingId } from "./types-DqFMNTgK.js";
|
|
32
|
+
import { S as requiredPairsForPairedPromotion, n as paretoFrontier, t as dominates, x as pairedPromotionPower } from "./pareto-CTv8vozZ.js";
|
|
33
|
+
import { a as LlmCallError, i as stripFencedJson, n as isRecord, o as LlmResponseError } from "./json-BhdsJaOQ.js";
|
|
34
|
+
import { r as selfImprove, t as defineAgentEval } from "./define-agent-eval-BGdFKlrc.js";
|
|
35
|
+
import { n as equivalenceVerdict, t as certificationEvidenceDigest } from "./verdict-BGQmxQ8A.js";
|
|
36
|
+
import { c as createBoundedTraceAnalysisStore, p as DEFAULT_TRACE_ANALYST_BUDGETS, t as createTraceAnalyst } from "./kind-factory-Gy4fXixd.js";
|
|
37
|
+
import { _ as readAnalystReview, c as FAILURE_MODE_KIND_SPEC, g as completedAnalystReviewQuality, h as assertUniqueFindingIds, i as DEFAULT_TRACE_ANALYST_KINDS, m as analystRunDigest, n as AnalystRegistry, p as analystFindingDigest, t as buildDefaultAnalystRegistry, v as snapshotAnalystRun, y as validateAnalystReviewDecisions } from "./default-registry-BJ-Pq6yL.js";
|
|
42
38
|
import { OUTPUT_VALUE } from "./trace-attributes.js";
|
|
43
|
-
import {
|
|
44
|
-
import { n as InMemoryRawProviderSink, r as NoopRawProviderSink, t as FileSystemRawProviderSink } from "./raw-provider-sink-
|
|
45
|
-
import {
|
|
46
|
-
import {
|
|
39
|
+
import { n as extractUsageFromSse, t as extractUsage } from "./extract-usage-BZLU01kR.js";
|
|
40
|
+
import { n as InMemoryRawProviderSink, r as NoopRawProviderSink, t as FileSystemRawProviderSink } from "./raw-provider-sink-75w4FWBj.js";
|
|
41
|
+
import { _ as exportRunAsOtlp, b as captureFetchToRawSink, c as describeTraceInsightScope, g as analyzeTraces, h as tokenizeDomainWords, l as domainEvidencePattern, m as scoreTraceInsightReadiness, n as toolSpansToTraceAnalysisStore, o as buildTraceInsightContext, s as buildTraceInsightPrompt, u as inferDomainKeywords } from "./store-tool-spans-D5l_OfTJ.js";
|
|
42
|
+
import { t as TraceEmitter } from "./emitter-DbMXUIVP.js";
|
|
43
|
+
import { n as assertRunCaptured, t as RunIntegrityError } from "./integrity-W4CADcFc.js";
|
|
47
44
|
import { n as InMemoryTraceStore, t as FileSystemTraceStore } from "./store-BhNuLdJw.js";
|
|
48
45
|
import { t as packageVersion$1 } from "./package-version-D7lQHt_-.js";
|
|
49
|
-
import {
|
|
46
|
+
import { t as buildTrajectory } from "./trajectory-D_7rLrvE.js";
|
|
47
|
+
import { i as failureBlame, n as classifyFailure, r as classifyFailureReason } from "./failure-taxonomy-Bx5oWqaW.js";
|
|
50
48
|
import { n as runBoundedProcess, t as DEFAULT_BOUNDED_PROCESS_TIMEOUT_MS } from "./bounded-process-Cj9YjXbE.js";
|
|
51
|
-
import { t as runEvalCampaign } from "./eval-campaign-
|
|
52
|
-
import "./rollout-
|
|
53
|
-
import { t as mintRolloutRows } from "./mint-
|
|
54
|
-
import { n as pairedEvalueSequence, t as evaluateInterimReleaseConfidence } from "./sequential-
|
|
55
|
-
import {
|
|
56
|
-
import { n as paidJsonChat } from "./chat-json-call-
|
|
57
|
-
import { t as createDspyRlmTraceEngine } from "./dspy-rlm-engine-
|
|
58
|
-
import {
|
|
59
|
-
import { a as budgetBreachView, i as failureClusterView, n as computeToolUseMetrics, r as judgeAgreementView, t as toolWasteView } from "./tool-waste-
|
|
60
|
-
import {
|
|
61
|
-
import { a as spanRecordsToTraceAnalysisStore, i as otlpTextToTraceAnalysisStore, r as OtlpFileTraceStore } from "./store-otlp-
|
|
62
|
-
import { n as
|
|
63
|
-
import { t as createChatClient } from "./chat-client-
|
|
49
|
+
import { t as runEvalCampaign } from "./eval-campaign-BUMIxKTE.js";
|
|
50
|
+
import "./rollout-Dd1ucT_M.js";
|
|
51
|
+
import { t as mintRolloutRows } from "./mint-CbdwjTPW.js";
|
|
52
|
+
import { n as pairedEvalueSequence, t as evaluateInterimReleaseConfidence } from "./sequential-BVv1sjsw.js";
|
|
53
|
+
import { a as maximumChargeForLlmRequest, c as ModelSubstitutionError, f as assertServedModels, g as judgeFamily, h as assertCrossFamily, i as costReceiptFromLlmError, l as assertCrossFamilyServed, m as CrossFamilyError, o as backoffMs, p as checkServedModel, r as costReceiptFromLlm, s as isTransientLlmError, u as assertServedModel } from "./llm-client-_Y6ic-tf.js";
|
|
54
|
+
import { n as paidJsonChat } from "./chat-json-call-Dm9c-b41.js";
|
|
55
|
+
import { t as createDspyRlmTraceEngine } from "./dspy-rlm-engine-CSMwudLO.js";
|
|
56
|
+
import { n as diffFindings, r as createChatTraceEngine, t as FindingsStore } from "./findings-store-BtTzPL0d.js";
|
|
57
|
+
import { a as budgetBreachView, i as failureClusterView, n as computeToolUseMetrics, r as judgeAgreementView, t as toolWasteView } from "./tool-waste-DLsjS_p-.js";
|
|
58
|
+
import { a as judgeGateDecision, c as snapshotFromJudgeAgreement, g as analyzeSeries, i as assertJudgeMayGate, n as JudgeGateRefusedError, o as measureJudgeAgreement, r as addCalibrationVerdicts, s as registerCalibrationSet, t as DEFAULT_JUDGE_GATE_POLICY } from "./judge-gate-Co5tuh57.js";
|
|
59
|
+
import { a as spanRecordsToTraceAnalysisStore, i as otlpTextToTraceAnalysisStore, r as OtlpFileTraceStore } from "./store-otlp-BteDAbHK.js";
|
|
60
|
+
import { n as bootstrapCi, t as evaluateReleaseConfidence } from "./release-confidence-CYIFH-fp.js";
|
|
61
|
+
import { t as createChatClient } from "./chat-client-B2pXlPdx.js";
|
|
64
62
|
import { createHash } from "node:crypto";
|
|
65
63
|
import { accessSync, appendFileSync, constants, cpSync, existsSync, mkdirSync, promises, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
|
|
66
64
|
import { basename, delimiter, dirname, extname, isAbsolute, join, relative, resolve } from "node:path";
|
|
@@ -3460,37 +3458,6 @@ var ProductClient = class {
|
|
|
3460
3458
|
* implementations live with their consumer because they depend on
|
|
3461
3459
|
* SDK-specific Box / Sandbox types that don't belong in this package.
|
|
3462
3460
|
*/
|
|
3463
|
-
/**
|
|
3464
|
-
* Host-process runner, mapped onto {@link runBoundedProcess} so the package
|
|
3465
|
-
* has one spawn site rather than two.
|
|
3466
|
-
*
|
|
3467
|
-
* It used `spawnSync` and therefore had none of the bounds that runner exists
|
|
3468
|
-
* to provide: a `capMs` timeout killed the direct child and left any
|
|
3469
|
-
* descendant it had started running, a killed run could still report exit
|
|
3470
|
-
* code 0, captured output was unbounded, and the whole run blocked the event
|
|
3471
|
-
* loop. This runner spawns build, test and lint commands, which are exactly
|
|
3472
|
-
* the commands most likely to start descendants.
|
|
3473
|
-
*
|
|
3474
|
-
* It is not the package's last private spawn: `nodeProcessRunner`
|
|
3475
|
-
* (`src/trace-repair/docker-environment.ts`) and `runProcess`
|
|
3476
|
-
* (`src/campaign/external-optimizer-subprocess.ts`) still spawn their own
|
|
3477
|
-
* detached children, and several one-shot `spawnSync` git and CLI calls
|
|
3478
|
-
* remain. What this removes is the second spawn behind the `CommandRunner`
|
|
3479
|
-
* contract, which is the one that graded build, test and lint commands.
|
|
3480
|
-
*
|
|
3481
|
-
* Two behaviours the mapping changes on purpose:
|
|
3482
|
-
* - a run with no `capMs` is now bounded by
|
|
3483
|
-
* {@link LOCAL_COMMAND_RUNNER_DEFAULT_CAP_MS} instead of running forever;
|
|
3484
|
-
* - the run no longer blocks the event loop, so a minutes-long `capMs` no
|
|
3485
|
-
* longer stalls everything else in the process.
|
|
3486
|
-
*/
|
|
3487
|
-
/**
|
|
3488
|
-
* Wall clock {@link localCommandRunner} applies when the caller names no
|
|
3489
|
-
* `capMs`. It is `runBoundedProcess`'s own default, stated here because this
|
|
3490
|
-
* runner used to be unbounded: a consumer whose command runs longer names its
|
|
3491
|
-
* own `capMs`, and a run the bound stops says so with `timedOut: true`.
|
|
3492
|
-
*/
|
|
3493
|
-
const LOCAL_COMMAND_RUNNER_DEFAULT_CAP_MS = DEFAULT_BOUNDED_PROCESS_TIMEOUT_MS;
|
|
3494
3461
|
const localCommandRunner = {
|
|
3495
3462
|
name: "local",
|
|
3496
3463
|
async run(input) {
|
|
@@ -3771,7 +3738,6 @@ function observeAll(detectors, event) {
|
|
|
3771
3738
|
//#region src/control-runtime.ts
|
|
3772
3739
|
async function runAgentControlLoop(config) {
|
|
3773
3740
|
const budget = normalizeBudget(config.budget);
|
|
3774
|
-
const actionFailure = config.actionFailure ?? "continue";
|
|
3775
3741
|
const controller = new AbortController();
|
|
3776
3742
|
const upstreamAbort = () => controller.abort(config.signal?.reason);
|
|
3777
3743
|
if (config.signal) {
|
|
@@ -3780,473 +3746,51 @@ async function runAgentControlLoop(config) {
|
|
|
3780
3746
|
}
|
|
3781
3747
|
const started = Date.now();
|
|
3782
3748
|
const wallTimer = budget.maxWallMs ? setTimeout(() => controller.abort(/* @__PURE__ */ new Error("control runtime wall timeout")), budget.maxWallMs) : void 0;
|
|
3783
|
-
const
|
|
3784
|
-
|
|
3785
|
-
|
|
3786
|
-
|
|
3787
|
-
|
|
3788
|
-
|
|
3789
|
-
|
|
3790
|
-
|
|
3791
|
-
|
|
3749
|
+
const run = {
|
|
3750
|
+
config,
|
|
3751
|
+
budget,
|
|
3752
|
+
signal: controller.signal,
|
|
3753
|
+
started,
|
|
3754
|
+
history: [],
|
|
3755
|
+
emitter: config.store ? new TraceEmitter(config.store, { onCaptureError: () => void 0 }) : void 0,
|
|
3756
|
+
runtimeErrors: [],
|
|
3757
|
+
repeatedDetector: repeatedActionDetector({ maxRepeated: config.stopPolicies?.maxRepeatedActions ?? 0 }),
|
|
3758
|
+
progressDetector: noProgressDetector({
|
|
3759
|
+
maxNoProgress: config.stopPolicies?.maxNoProgressSteps ?? 0,
|
|
3760
|
+
minScoreDelta: config.stopPolicies?.minScoreDelta ?? .001
|
|
3761
|
+
}),
|
|
3762
|
+
spentCostUsd: 0,
|
|
3763
|
+
state: void 0,
|
|
3764
|
+
evals: []
|
|
3765
|
+
};
|
|
3792
3766
|
try {
|
|
3793
|
-
|
|
3794
|
-
|
|
3795
|
-
|
|
3796
|
-
|
|
3797
|
-
|
|
3798
|
-
|
|
3799
|
-
intent: config.intent.slice(0, 120),
|
|
3800
|
-
maxSteps: String(budget.maxSteps),
|
|
3801
|
-
...budget.maxCostUsd !== void 0 ? { maxCostUsd: String(budget.maxCostUsd) } : {}
|
|
3802
|
-
}
|
|
3803
|
-
}));
|
|
3804
|
-
let state;
|
|
3805
|
-
let evals;
|
|
3806
|
-
try {
|
|
3807
|
-
state = await config.observe({
|
|
3808
|
-
history,
|
|
3809
|
-
abortSignal: controller.signal
|
|
3810
|
-
});
|
|
3811
|
-
} catch (err) {
|
|
3812
|
-
const error = runtimeError("observe", 0, err);
|
|
3813
|
-
runtimeErrors.push(error);
|
|
3814
|
-
return finish(emitter, {
|
|
3815
|
-
intent: config.intent,
|
|
3816
|
-
pass: false,
|
|
3817
|
-
completed: false,
|
|
3818
|
-
reason: error.message,
|
|
3819
|
-
steps: history,
|
|
3820
|
-
finalState: void 0,
|
|
3821
|
-
finalEvals: [],
|
|
3822
|
-
wallMs: Date.now() - started,
|
|
3823
|
-
spentCostUsd,
|
|
3824
|
-
runId: emitter?.runId ?? null,
|
|
3825
|
-
failureClass: "unknown",
|
|
3826
|
-
runtimeErrors,
|
|
3827
|
-
stoppedBy: "runtime-error"
|
|
3828
|
-
});
|
|
3829
|
-
}
|
|
3830
|
-
try {
|
|
3831
|
-
evals = await config.validate({
|
|
3832
|
-
intent: config.intent,
|
|
3833
|
-
state,
|
|
3834
|
-
history,
|
|
3835
|
-
abortSignal: controller.signal
|
|
3836
|
-
});
|
|
3837
|
-
await recordEvalSpans(emitter, evals, "initial", runtimeErrors, 0);
|
|
3838
|
-
} catch (err) {
|
|
3839
|
-
const error = runtimeError("validate", 0, err);
|
|
3840
|
-
runtimeErrors.push(error);
|
|
3841
|
-
return finish(emitter, {
|
|
3842
|
-
intent: config.intent,
|
|
3843
|
-
pass: false,
|
|
3844
|
-
completed: false,
|
|
3845
|
-
reason: error.message,
|
|
3846
|
-
steps: history,
|
|
3847
|
-
finalState: state,
|
|
3848
|
-
finalEvals: [],
|
|
3849
|
-
wallMs: Date.now() - started,
|
|
3850
|
-
spentCostUsd,
|
|
3851
|
-
runId: emitter?.runId ?? null,
|
|
3852
|
-
failureClass: "unknown",
|
|
3853
|
-
runtimeErrors,
|
|
3854
|
-
stoppedBy: "runtime-error"
|
|
3855
|
-
});
|
|
3856
|
-
}
|
|
3857
|
-
progressDetector.observe({
|
|
3858
|
-
stateFingerprint: fingerprintState(state, config.stopPolicies),
|
|
3859
|
-
score: averageScore(evals)
|
|
3767
|
+
await startTrace(run);
|
|
3768
|
+
const failedStart = await observeInitial(run);
|
|
3769
|
+
if (failedStart) return failedStart;
|
|
3770
|
+
run.progressDetector.observe({
|
|
3771
|
+
stateFingerprint: fingerprintState(run.state, config.stopPolicies),
|
|
3772
|
+
score: averageScore(run.evals)
|
|
3860
3773
|
});
|
|
3861
3774
|
for (let stepIndex = 0; stepIndex < budget.maxSteps; stepIndex++) {
|
|
3862
|
-
|
|
3863
|
-
|
|
3864
|
-
pass: false,
|
|
3865
|
-
completed: false,
|
|
3866
|
-
reason: abortReason(controller.signal),
|
|
3867
|
-
score: void 0,
|
|
3868
|
-
steps: history,
|
|
3869
|
-
finalState: state,
|
|
3870
|
-
finalEvals: evals,
|
|
3871
|
-
wallMs: Date.now() - started,
|
|
3872
|
-
spentCostUsd,
|
|
3873
|
-
runId: emitter?.runId ?? null,
|
|
3874
|
-
failureClass: "timeout",
|
|
3875
|
-
runtimeErrors,
|
|
3876
|
-
stoppedBy: "abort"
|
|
3877
|
-
});
|
|
3878
|
-
const budgetStop = budgetStopDecision(budget, spentCostUsd);
|
|
3879
|
-
if (budgetStop.stop) return finish(emitter, {
|
|
3880
|
-
intent: config.intent,
|
|
3881
|
-
pass: false,
|
|
3882
|
-
completed: false,
|
|
3883
|
-
reason: budgetStop.reason,
|
|
3884
|
-
score: averageScore(evals),
|
|
3885
|
-
steps: history,
|
|
3886
|
-
finalState: state,
|
|
3887
|
-
finalEvals: evals,
|
|
3888
|
-
wallMs: Date.now() - started,
|
|
3889
|
-
spentCostUsd,
|
|
3890
|
-
runId: emitter?.runId ?? null,
|
|
3891
|
-
failureClass: "budget_exceeded",
|
|
3892
|
-
runtimeErrors,
|
|
3893
|
-
stoppedBy: "budget"
|
|
3894
|
-
});
|
|
3895
|
-
const ctx = makeContext(config.intent, state, evals, history, budget, stepIndex, started, spentCostUsd, controller.signal, emitter);
|
|
3896
|
-
let stop;
|
|
3897
|
-
try {
|
|
3898
|
-
stop = config.shouldStop ? await config.shouldStop(ctx) : defaultStopDecision(evals);
|
|
3899
|
-
} catch (err) {
|
|
3900
|
-
runtimeErrors.push(runtimeError("stop-policy", stepIndex, err));
|
|
3901
|
-
return finish(emitter, {
|
|
3902
|
-
intent: config.intent,
|
|
3903
|
-
pass: false,
|
|
3904
|
-
completed: false,
|
|
3905
|
-
reason: runtimeErrors[runtimeErrors.length - 1].message,
|
|
3906
|
-
score: averageScore(evals),
|
|
3907
|
-
steps: history,
|
|
3908
|
-
finalState: state,
|
|
3909
|
-
finalEvals: evals,
|
|
3910
|
-
wallMs: Date.now() - started,
|
|
3911
|
-
spentCostUsd,
|
|
3912
|
-
runId: emitter?.runId ?? null,
|
|
3913
|
-
failureClass: "unknown",
|
|
3914
|
-
runtimeErrors,
|
|
3915
|
-
stoppedBy: "runtime-error"
|
|
3916
|
-
});
|
|
3917
|
-
}
|
|
3918
|
-
if (stop.stop) return finish(emitter, {
|
|
3919
|
-
intent: config.intent,
|
|
3920
|
-
pass: stop.pass,
|
|
3921
|
-
completed: true,
|
|
3922
|
-
reason: stop.reason,
|
|
3923
|
-
score: stop.score,
|
|
3924
|
-
steps: history,
|
|
3925
|
-
finalState: state,
|
|
3926
|
-
finalEvals: evals,
|
|
3927
|
-
wallMs: Date.now() - started,
|
|
3928
|
-
spentCostUsd,
|
|
3929
|
-
runId: emitter?.runId ?? null,
|
|
3930
|
-
failureClass: stop.failureClass,
|
|
3931
|
-
runtimeErrors,
|
|
3932
|
-
stoppedBy: "stop-policy"
|
|
3933
|
-
});
|
|
3934
|
-
let decision;
|
|
3935
|
-
try {
|
|
3936
|
-
decision = await config.decide(ctx);
|
|
3937
|
-
} catch (err) {
|
|
3938
|
-
runtimeErrors.push(runtimeError("decide", stepIndex, err));
|
|
3939
|
-
return finish(emitter, {
|
|
3940
|
-
intent: config.intent,
|
|
3941
|
-
pass: false,
|
|
3942
|
-
completed: false,
|
|
3943
|
-
reason: runtimeErrors[runtimeErrors.length - 1].message,
|
|
3944
|
-
score: averageScore(evals),
|
|
3945
|
-
steps: history,
|
|
3946
|
-
finalState: state,
|
|
3947
|
-
finalEvals: evals,
|
|
3948
|
-
wallMs: Date.now() - started,
|
|
3949
|
-
spentCostUsd,
|
|
3950
|
-
runId: emitter?.runId ?? null,
|
|
3951
|
-
failureClass: "unknown",
|
|
3952
|
-
runtimeErrors,
|
|
3953
|
-
stoppedBy: "runtime-error"
|
|
3954
|
-
});
|
|
3955
|
-
}
|
|
3956
|
-
if (decision.type === "stop") {
|
|
3957
|
-
const pass = decision.pass ?? false;
|
|
3958
|
-
return finish(emitter, {
|
|
3959
|
-
intent: config.intent,
|
|
3960
|
-
pass,
|
|
3961
|
-
completed: true,
|
|
3962
|
-
reason: decision.reason,
|
|
3963
|
-
score: decision.score,
|
|
3964
|
-
steps: history,
|
|
3965
|
-
finalState: state,
|
|
3966
|
-
finalEvals: evals,
|
|
3967
|
-
wallMs: Date.now() - started,
|
|
3968
|
-
spentCostUsd,
|
|
3969
|
-
runId: emitter?.runId ?? null,
|
|
3970
|
-
failureClass: pass ? void 0 : decision.failureClass ?? "unknown",
|
|
3971
|
-
runtimeErrors,
|
|
3972
|
-
stoppedBy: "policy"
|
|
3973
|
-
});
|
|
3974
|
-
}
|
|
3975
|
-
const actionFingerprint = fingerprintAction(decision.action, config.stopPolicies);
|
|
3976
|
-
const repeatedActionSignal = repeatedDetector.observe({ actionFingerprint });
|
|
3977
|
-
const repeatedActionStreak = repeatedDetector.streak;
|
|
3978
|
-
if (repeatedActionSignal) return finish(emitter, {
|
|
3979
|
-
intent: config.intent,
|
|
3980
|
-
pass: false,
|
|
3981
|
-
completed: true,
|
|
3982
|
-
reason: repeatedActionSignal.reason,
|
|
3983
|
-
score: averageScore(evals),
|
|
3984
|
-
steps: history,
|
|
3985
|
-
finalState: state,
|
|
3986
|
-
finalEvals: evals,
|
|
3987
|
-
wallMs: Date.now() - started,
|
|
3988
|
-
spentCostUsd,
|
|
3989
|
-
runId: emitter?.runId ?? null,
|
|
3990
|
-
failureClass: "tool_recovery_failure",
|
|
3991
|
-
runtimeErrors,
|
|
3992
|
-
stoppedBy: "stop-policy"
|
|
3993
|
-
});
|
|
3994
|
-
const beforeState = state;
|
|
3995
|
-
const evalsBefore = evals;
|
|
3996
|
-
const scoreBefore = averageScore(evals);
|
|
3997
|
-
const actionStarted = Date.now();
|
|
3998
|
-
const stepHandle = emitter ? await runTrace(emitter, runtimeErrors, stepIndex, () => emitter.tool({
|
|
3999
|
-
name: `control-step-${stepIndex}`,
|
|
4000
|
-
toolName: "agent-control-action",
|
|
4001
|
-
args: decision.action,
|
|
4002
|
-
attributes: {
|
|
4003
|
-
decision: decision.reason ?? "continue",
|
|
4004
|
-
repeatedActionStreak
|
|
4005
|
-
}
|
|
4006
|
-
})) : void 0;
|
|
4007
|
-
let actionOutcome;
|
|
4008
|
-
try {
|
|
4009
|
-
const result = await config.act(decision.action, ctx);
|
|
4010
|
-
const rawCostUsd = config.getActionCostUsd?.({
|
|
4011
|
-
action: decision.action,
|
|
4012
|
-
result,
|
|
4013
|
-
state,
|
|
4014
|
-
evals,
|
|
4015
|
-
history
|
|
4016
|
-
});
|
|
4017
|
-
const costUsd = normalizeActionCostUsd(rawCostUsd, runtimeErrors, stepIndex);
|
|
4018
|
-
if (costUsd !== void 0 && Number.isFinite(costUsd) && costUsd > 0) {
|
|
4019
|
-
spentCostUsd += costUsd;
|
|
4020
|
-
await recordCostBudget(emitter, budget, spentCostUsd, stepHandle, runtimeErrors, stepIndex);
|
|
4021
|
-
}
|
|
4022
|
-
actionOutcome = {
|
|
4023
|
-
ok: true,
|
|
4024
|
-
result,
|
|
4025
|
-
...costUsd !== void 0 ? { costUsd } : {},
|
|
4026
|
-
durationMs: Date.now() - actionStarted
|
|
4027
|
-
};
|
|
4028
|
-
} catch (err) {
|
|
4029
|
-
runtimeErrors.push(runtimeError("act", stepIndex, err));
|
|
4030
|
-
actionOutcome = {
|
|
4031
|
-
ok: false,
|
|
4032
|
-
error: runtimeErrors[runtimeErrors.length - 1].message,
|
|
4033
|
-
durationMs: Date.now() - actionStarted
|
|
4034
|
-
};
|
|
4035
|
-
if (actionFailure === "stop") {
|
|
4036
|
-
await runTrace(emitter, runtimeErrors, stepIndex, () => stepHandle?.fail(actionOutcome.error ?? "action failed"));
|
|
4037
|
-
const step = {
|
|
4038
|
-
index: stepIndex,
|
|
4039
|
-
decision,
|
|
4040
|
-
beforeState,
|
|
4041
|
-
afterState: state,
|
|
4042
|
-
evalsBefore,
|
|
4043
|
-
evalsAfter: evals,
|
|
4044
|
-
actionOutcome,
|
|
4045
|
-
startedAt: new Date(actionStarted).toISOString(),
|
|
4046
|
-
endedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
4047
|
-
};
|
|
4048
|
-
history.push(step);
|
|
4049
|
-
await runOnStep(config.onStep, step, runtimeErrors);
|
|
4050
|
-
return finish(emitter, {
|
|
4051
|
-
intent: config.intent,
|
|
4052
|
-
pass: false,
|
|
4053
|
-
completed: false,
|
|
4054
|
-
reason: actionOutcome.error ?? "action failed",
|
|
4055
|
-
score: averageScore(evals),
|
|
4056
|
-
steps: history,
|
|
4057
|
-
finalState: state,
|
|
4058
|
-
finalEvals: evals,
|
|
4059
|
-
wallMs: Date.now() - started,
|
|
4060
|
-
spentCostUsd,
|
|
4061
|
-
runId: emitter?.runId ?? null,
|
|
4062
|
-
failureClass: "unknown",
|
|
4063
|
-
runtimeErrors,
|
|
4064
|
-
stoppedBy: "runtime-error"
|
|
4065
|
-
});
|
|
4066
|
-
}
|
|
4067
|
-
}
|
|
4068
|
-
let observationPhase = "observe";
|
|
4069
|
-
try {
|
|
4070
|
-
state = await config.observe({
|
|
4071
|
-
history,
|
|
4072
|
-
abortSignal: controller.signal
|
|
4073
|
-
});
|
|
4074
|
-
observationPhase = "validate";
|
|
4075
|
-
evals = await config.validate({
|
|
4076
|
-
intent: config.intent,
|
|
4077
|
-
state,
|
|
4078
|
-
history,
|
|
4079
|
-
abortSignal: controller.signal
|
|
4080
|
-
});
|
|
4081
|
-
await recordEvalSpans(emitter, evals, `step-${stepIndex}`, runtimeErrors, stepIndex, stepHandle?.span.spanId);
|
|
4082
|
-
} catch (err) {
|
|
4083
|
-
runtimeErrors.push(runtimeError(observationPhase, stepIndex, err));
|
|
4084
|
-
const afterState = observationPhase === "observe" ? beforeState : state;
|
|
4085
|
-
const step = {
|
|
4086
|
-
index: stepIndex,
|
|
4087
|
-
decision,
|
|
4088
|
-
beforeState,
|
|
4089
|
-
afterState,
|
|
4090
|
-
evalsBefore,
|
|
4091
|
-
evalsAfter: evals,
|
|
4092
|
-
actionOutcome,
|
|
4093
|
-
startedAt: new Date(actionStarted).toISOString(),
|
|
4094
|
-
endedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
4095
|
-
};
|
|
4096
|
-
history.push(step);
|
|
4097
|
-
await runTrace(emitter, runtimeErrors, stepIndex, () => stepHandle?.fail(runtimeErrors[runtimeErrors.length - 1].message));
|
|
4098
|
-
await runOnStep(config.onStep, step, runtimeErrors);
|
|
4099
|
-
return finish(emitter, {
|
|
4100
|
-
intent: config.intent,
|
|
4101
|
-
pass: false,
|
|
4102
|
-
completed: false,
|
|
4103
|
-
reason: runtimeErrors[runtimeErrors.length - 1].message,
|
|
4104
|
-
score: averageScore(evals),
|
|
4105
|
-
steps: history,
|
|
4106
|
-
finalState: afterState,
|
|
4107
|
-
finalEvals: evals,
|
|
4108
|
-
wallMs: Date.now() - started,
|
|
4109
|
-
spentCostUsd,
|
|
4110
|
-
runId: emitter?.runId ?? null,
|
|
4111
|
-
failureClass: "unknown",
|
|
4112
|
-
runtimeErrors,
|
|
4113
|
-
stoppedBy: "runtime-error"
|
|
4114
|
-
});
|
|
4115
|
-
}
|
|
4116
|
-
const scoreAfter = averageScore(evals);
|
|
4117
|
-
const stateFingerprint = fingerprintState(state, config.stopPolicies);
|
|
4118
|
-
const noProgressSignal = progressDetector.observe({
|
|
4119
|
-
stateFingerprint,
|
|
4120
|
-
score: scoreAfter
|
|
4121
|
-
});
|
|
4122
|
-
const noProgressStreak = progressDetector.streak;
|
|
4123
|
-
const step = {
|
|
4124
|
-
index: stepIndex,
|
|
4125
|
-
decision,
|
|
4126
|
-
beforeState,
|
|
4127
|
-
afterState: state,
|
|
4128
|
-
evalsBefore,
|
|
4129
|
-
evalsAfter: evals,
|
|
4130
|
-
actionOutcome,
|
|
4131
|
-
startedAt: new Date(actionStarted).toISOString(),
|
|
4132
|
-
endedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
4133
|
-
};
|
|
4134
|
-
history.push(step);
|
|
4135
|
-
if (actionOutcome.ok) await runTrace(emitter, runtimeErrors, stepIndex, () => stepHandle?.end({ attributes: {
|
|
4136
|
-
actionCostUsd: actionOutcome.costUsd ?? null,
|
|
4137
|
-
spentCostUsd,
|
|
4138
|
-
scoreBefore: scoreBefore ?? null,
|
|
4139
|
-
scoreAfter: scoreAfter ?? null,
|
|
4140
|
-
noProgressStreak
|
|
4141
|
-
} }));
|
|
4142
|
-
else await runTrace(emitter, runtimeErrors, stepIndex, () => stepHandle?.fail(actionOutcome.error ?? "action failed", { attributes: {
|
|
4143
|
-
spentCostUsd,
|
|
4144
|
-
noProgressStreak
|
|
4145
|
-
} }));
|
|
4146
|
-
await runOnStep(config.onStep, step, runtimeErrors);
|
|
4147
|
-
if (noProgressSignal) return finish(emitter, {
|
|
4148
|
-
intent: config.intent,
|
|
4149
|
-
pass: false,
|
|
4150
|
-
completed: true,
|
|
4151
|
-
reason: noProgressSignal.reason,
|
|
4152
|
-
score: scoreAfter,
|
|
4153
|
-
steps: history,
|
|
4154
|
-
finalState: state,
|
|
4155
|
-
finalEvals: evals,
|
|
4156
|
-
wallMs: Date.now() - started,
|
|
4157
|
-
spentCostUsd,
|
|
4158
|
-
runId: emitter?.runId ?? null,
|
|
4159
|
-
failureClass: "tool_recovery_failure",
|
|
4160
|
-
runtimeErrors,
|
|
4161
|
-
stoppedBy: "stop-policy"
|
|
4162
|
-
});
|
|
4163
|
-
const postStepBudgetStop = budgetStopDecision(budget, spentCostUsd);
|
|
4164
|
-
if (postStepBudgetStop.stop) return finish(emitter, {
|
|
4165
|
-
intent: config.intent,
|
|
4166
|
-
pass: false,
|
|
4167
|
-
completed: false,
|
|
4168
|
-
reason: postStepBudgetStop.reason,
|
|
4169
|
-
score: scoreAfter,
|
|
4170
|
-
steps: history,
|
|
4171
|
-
finalState: state,
|
|
4172
|
-
finalEvals: evals,
|
|
4173
|
-
wallMs: Date.now() - started,
|
|
4174
|
-
spentCostUsd,
|
|
4175
|
-
runId: emitter?.runId ?? null,
|
|
4176
|
-
failureClass: "budget_exceeded",
|
|
4177
|
-
runtimeErrors,
|
|
4178
|
-
stoppedBy: "budget"
|
|
4179
|
-
});
|
|
4180
|
-
const postStepCtx = makeContext(config.intent, state, evals, history, budget, stepIndex + 1, started, spentCostUsd, controller.signal, emitter);
|
|
4181
|
-
let postStepStop;
|
|
4182
|
-
try {
|
|
4183
|
-
postStepStop = config.shouldStop ? await config.shouldStop(postStepCtx) : defaultStopDecision(evals);
|
|
4184
|
-
} catch (err) {
|
|
4185
|
-
runtimeErrors.push(runtimeError("stop-policy", stepIndex + 1, err));
|
|
4186
|
-
return finish(emitter, {
|
|
4187
|
-
intent: config.intent,
|
|
4188
|
-
pass: false,
|
|
4189
|
-
completed: false,
|
|
4190
|
-
reason: runtimeErrors[runtimeErrors.length - 1].message,
|
|
4191
|
-
score: averageScore(evals),
|
|
4192
|
-
steps: history,
|
|
4193
|
-
finalState: state,
|
|
4194
|
-
finalEvals: evals,
|
|
4195
|
-
wallMs: Date.now() - started,
|
|
4196
|
-
spentCostUsd,
|
|
4197
|
-
runId: emitter?.runId ?? null,
|
|
4198
|
-
failureClass: "unknown",
|
|
4199
|
-
runtimeErrors,
|
|
4200
|
-
stoppedBy: "runtime-error"
|
|
4201
|
-
});
|
|
4202
|
-
}
|
|
4203
|
-
if (postStepStop.stop) return finish(emitter, {
|
|
4204
|
-
intent: config.intent,
|
|
4205
|
-
pass: postStepStop.pass,
|
|
4206
|
-
completed: true,
|
|
4207
|
-
reason: postStepStop.reason,
|
|
4208
|
-
score: postStepStop.score,
|
|
4209
|
-
steps: history,
|
|
4210
|
-
finalState: state,
|
|
4211
|
-
finalEvals: evals,
|
|
4212
|
-
wallMs: Date.now() - started,
|
|
4213
|
-
spentCostUsd,
|
|
4214
|
-
runId: emitter?.runId ?? null,
|
|
4215
|
-
failureClass: postStepStop.failureClass,
|
|
4216
|
-
runtimeErrors,
|
|
4217
|
-
stoppedBy: "stop-policy"
|
|
4218
|
-
});
|
|
3775
|
+
const ended = await runStep(run, stepIndex);
|
|
3776
|
+
if (ended) return ended;
|
|
4219
3777
|
}
|
|
4220
|
-
return
|
|
4221
|
-
intent: config.intent,
|
|
3778
|
+
return settle(run, {
|
|
4222
3779
|
pass: false,
|
|
4223
3780
|
completed: false,
|
|
4224
3781
|
reason: `budget exhausted: maxSteps=${budget.maxSteps}`,
|
|
4225
|
-
steps: history,
|
|
4226
|
-
finalState: state,
|
|
4227
|
-
finalEvals: evals,
|
|
4228
|
-
wallMs: Date.now() - started,
|
|
4229
|
-
spentCostUsd,
|
|
4230
|
-
runId: emitter?.runId ?? null,
|
|
4231
3782
|
failureClass: "budget_exceeded",
|
|
4232
|
-
runtimeErrors,
|
|
4233
3783
|
stoppedBy: "budget"
|
|
4234
3784
|
});
|
|
4235
3785
|
} catch (err) {
|
|
4236
|
-
runtimeErrors.push(runtimeError("act", history.length, err));
|
|
4237
|
-
return
|
|
4238
|
-
intent: config.intent,
|
|
3786
|
+
run.runtimeErrors.push(runtimeError("act", run.history.length, err));
|
|
3787
|
+
return settle(run, {
|
|
4239
3788
|
pass: false,
|
|
4240
3789
|
completed: false,
|
|
4241
|
-
reason:
|
|
4242
|
-
steps: history,
|
|
3790
|
+
reason: lastErrorMessage(run),
|
|
4243
3791
|
finalState: void 0,
|
|
4244
3792
|
finalEvals: [],
|
|
4245
|
-
wallMs: Date.now() - started,
|
|
4246
|
-
spentCostUsd,
|
|
4247
|
-
runId: emitter?.runId ?? null,
|
|
4248
3793
|
failureClass: "unknown",
|
|
4249
|
-
runtimeErrors,
|
|
4250
3794
|
stoppedBy: "runtime-error"
|
|
4251
3795
|
});
|
|
4252
3796
|
} finally {
|
|
@@ -4254,6 +3798,349 @@ async function runAgentControlLoop(config) {
|
|
|
4254
3798
|
if (config.signal) config.signal.removeEventListener("abort", upstreamAbort);
|
|
4255
3799
|
}
|
|
4256
3800
|
}
|
|
3801
|
+
/** End the run with `how`; everything not named there is read from the run as it stands now. */
|
|
3802
|
+
function settle(run, how) {
|
|
3803
|
+
return finish(run.emitter, {
|
|
3804
|
+
intent: run.config.intent,
|
|
3805
|
+
pass: how.pass,
|
|
3806
|
+
completed: how.completed,
|
|
3807
|
+
reason: how.reason,
|
|
3808
|
+
..."score" in how ? { score: how.score } : {},
|
|
3809
|
+
steps: run.history,
|
|
3810
|
+
finalState: "finalState" in how ? how.finalState : run.state,
|
|
3811
|
+
finalEvals: how.finalEvals ?? run.evals,
|
|
3812
|
+
wallMs: Date.now() - run.started,
|
|
3813
|
+
spentCostUsd: run.spentCostUsd,
|
|
3814
|
+
runId: run.emitter?.runId ?? null,
|
|
3815
|
+
failureClass: how.failureClass,
|
|
3816
|
+
runtimeErrors: run.runtimeErrors,
|
|
3817
|
+
stoppedBy: how.stoppedBy
|
|
3818
|
+
});
|
|
3819
|
+
}
|
|
3820
|
+
function lastErrorMessage(run) {
|
|
3821
|
+
return run.runtimeErrors[run.runtimeErrors.length - 1].message;
|
|
3822
|
+
}
|
|
3823
|
+
/** End the run on the runtime error just recorded. */
|
|
3824
|
+
function settleOnRuntimeError(run, finalState) {
|
|
3825
|
+
return settle(run, {
|
|
3826
|
+
pass: false,
|
|
3827
|
+
completed: false,
|
|
3828
|
+
reason: lastErrorMessage(run),
|
|
3829
|
+
score: averageScore(run.evals),
|
|
3830
|
+
...finalState ? { finalState: finalState.value } : {},
|
|
3831
|
+
failureClass: "unknown",
|
|
3832
|
+
stoppedBy: "runtime-error"
|
|
3833
|
+
});
|
|
3834
|
+
}
|
|
3835
|
+
async function startTrace(run) {
|
|
3836
|
+
const { emitter, config, budget } = run;
|
|
3837
|
+
if (!emitter) return;
|
|
3838
|
+
await runTrace(emitter, run.runtimeErrors, 0, () => emitter.startRun({
|
|
3839
|
+
scenarioId: config.scenarioId ?? "agent-control-loop",
|
|
3840
|
+
projectId: config.projectId,
|
|
3841
|
+
variantId: config.variantId,
|
|
3842
|
+
layer: "meta",
|
|
3843
|
+
tags: {
|
|
3844
|
+
intent: config.intent.slice(0, 120),
|
|
3845
|
+
maxSteps: String(budget.maxSteps),
|
|
3846
|
+
...budget.maxCostUsd !== void 0 ? { maxCostUsd: String(budget.maxCostUsd) } : {}
|
|
3847
|
+
}
|
|
3848
|
+
}));
|
|
3849
|
+
}
|
|
3850
|
+
/** The first observation and validation. Returns the result that ends the run when either fails. */
|
|
3851
|
+
async function observeInitial(run) {
|
|
3852
|
+
try {
|
|
3853
|
+
run.state = await run.config.observe({
|
|
3854
|
+
history: run.history,
|
|
3855
|
+
abortSignal: run.signal
|
|
3856
|
+
});
|
|
3857
|
+
} catch (err) {
|
|
3858
|
+
const error = runtimeError("observe", 0, err);
|
|
3859
|
+
run.runtimeErrors.push(error);
|
|
3860
|
+
return settle(run, {
|
|
3861
|
+
pass: false,
|
|
3862
|
+
completed: false,
|
|
3863
|
+
reason: error.message,
|
|
3864
|
+
failureClass: "unknown",
|
|
3865
|
+
stoppedBy: "runtime-error"
|
|
3866
|
+
});
|
|
3867
|
+
}
|
|
3868
|
+
try {
|
|
3869
|
+
run.evals = await run.config.validate({
|
|
3870
|
+
intent: run.config.intent,
|
|
3871
|
+
state: run.state,
|
|
3872
|
+
history: run.history,
|
|
3873
|
+
abortSignal: run.signal
|
|
3874
|
+
});
|
|
3875
|
+
await recordEvalSpans(run.emitter, run.evals, "initial", run.runtimeErrors, 0);
|
|
3876
|
+
} catch (err) {
|
|
3877
|
+
const error = runtimeError("validate", 0, err);
|
|
3878
|
+
run.runtimeErrors.push(error);
|
|
3879
|
+
return settle(run, {
|
|
3880
|
+
pass: false,
|
|
3881
|
+
completed: false,
|
|
3882
|
+
reason: error.message,
|
|
3883
|
+
finalEvals: [],
|
|
3884
|
+
failureClass: "unknown",
|
|
3885
|
+
stoppedBy: "runtime-error"
|
|
3886
|
+
});
|
|
3887
|
+
}
|
|
3888
|
+
}
|
|
3889
|
+
/** One decide-act-observe step. Returns the result that ends the run, or nothing to go on. */
|
|
3890
|
+
async function runStep(run, stepIndex) {
|
|
3891
|
+
const halted = await checkAbortAndBudget(run);
|
|
3892
|
+
if (halted) return halted;
|
|
3893
|
+
const ctx = makeRunContext(run, stepIndex);
|
|
3894
|
+
const stopped = await checkStopPolicy(run, ctx, stepIndex);
|
|
3895
|
+
if (stopped) return stopped;
|
|
3896
|
+
const chosen = await decideAction(run, ctx, stepIndex);
|
|
3897
|
+
if ("ended" in chosen) return chosen.ended;
|
|
3898
|
+
const { decision } = chosen.next;
|
|
3899
|
+
const repeated = await detectRepeatedAction(run, decision);
|
|
3900
|
+
if ("ended" in repeated) return repeated.ended;
|
|
3901
|
+
const attempt = await attemptAction(run, ctx, decision, stepIndex, repeated.next);
|
|
3902
|
+
if ("ended" in attempt) return attempt.ended;
|
|
3903
|
+
const observed = await observeAfterAction(run, attempt.next, stepIndex);
|
|
3904
|
+
if (observed) return observed;
|
|
3905
|
+
return recordStepAndCheck(run, attempt.next, stepIndex);
|
|
3906
|
+
}
|
|
3907
|
+
async function checkAbortAndBudget(run) {
|
|
3908
|
+
if (run.signal.aborted) return settle(run, {
|
|
3909
|
+
pass: false,
|
|
3910
|
+
completed: false,
|
|
3911
|
+
reason: abortReason(run.signal),
|
|
3912
|
+
score: void 0,
|
|
3913
|
+
failureClass: "timeout",
|
|
3914
|
+
stoppedBy: "abort"
|
|
3915
|
+
});
|
|
3916
|
+
const budgetStop = budgetStopDecision(run.budget, run.spentCostUsd);
|
|
3917
|
+
if (!budgetStop.stop) return void 0;
|
|
3918
|
+
return settle(run, {
|
|
3919
|
+
pass: false,
|
|
3920
|
+
completed: false,
|
|
3921
|
+
reason: budgetStop.reason,
|
|
3922
|
+
score: averageScore(run.evals),
|
|
3923
|
+
failureClass: "budget_exceeded",
|
|
3924
|
+
stoppedBy: "budget"
|
|
3925
|
+
});
|
|
3926
|
+
}
|
|
3927
|
+
function makeRunContext(run, stepIndex) {
|
|
3928
|
+
return makeContext(run.config.intent, run.state, run.evals, run.history, run.budget, stepIndex, run.started, run.spentCostUsd, run.signal, run.emitter);
|
|
3929
|
+
}
|
|
3930
|
+
/** Ask the stop policy about the current state. `errorStep` is the step a failing policy is recorded against. */
|
|
3931
|
+
async function checkStopPolicy(run, ctx, errorStep) {
|
|
3932
|
+
let stop;
|
|
3933
|
+
try {
|
|
3934
|
+
stop = run.config.shouldStop ? await run.config.shouldStop(ctx) : defaultStopDecision(run.evals);
|
|
3935
|
+
} catch (err) {
|
|
3936
|
+
run.runtimeErrors.push(runtimeError("stop-policy", errorStep, err));
|
|
3937
|
+
return settleOnRuntimeError(run);
|
|
3938
|
+
}
|
|
3939
|
+
if (!stop.stop) return void 0;
|
|
3940
|
+
return settle(run, {
|
|
3941
|
+
pass: stop.pass,
|
|
3942
|
+
completed: true,
|
|
3943
|
+
reason: stop.reason,
|
|
3944
|
+
score: stop.score,
|
|
3945
|
+
failureClass: stop.failureClass,
|
|
3946
|
+
stoppedBy: "stop-policy"
|
|
3947
|
+
});
|
|
3948
|
+
}
|
|
3949
|
+
async function decideAction(run, ctx, stepIndex) {
|
|
3950
|
+
let decision;
|
|
3951
|
+
try {
|
|
3952
|
+
decision = await run.config.decide(ctx);
|
|
3953
|
+
} catch (err) {
|
|
3954
|
+
run.runtimeErrors.push(runtimeError("decide", stepIndex, err));
|
|
3955
|
+
return { ended: await settleOnRuntimeError(run) };
|
|
3956
|
+
}
|
|
3957
|
+
if (decision.type !== "stop") return { next: { decision } };
|
|
3958
|
+
const pass = decision.pass ?? false;
|
|
3959
|
+
return { ended: await settle(run, {
|
|
3960
|
+
pass,
|
|
3961
|
+
completed: true,
|
|
3962
|
+
reason: decision.reason,
|
|
3963
|
+
score: decision.score,
|
|
3964
|
+
failureClass: pass ? void 0 : decision.failureClass ?? "unknown",
|
|
3965
|
+
stoppedBy: "policy"
|
|
3966
|
+
}) };
|
|
3967
|
+
}
|
|
3968
|
+
/** Feed the chosen action to the repeated-action detector; its streak is returned for the step's trace. */
|
|
3969
|
+
async function detectRepeatedAction(run, decision) {
|
|
3970
|
+
const actionFingerprint = fingerprintAction(decision.action, run.config.stopPolicies);
|
|
3971
|
+
const signal = run.repeatedDetector.observe({ actionFingerprint });
|
|
3972
|
+
if (!signal) return { next: run.repeatedDetector.streak };
|
|
3973
|
+
return { ended: await settle(run, {
|
|
3974
|
+
pass: false,
|
|
3975
|
+
completed: true,
|
|
3976
|
+
reason: signal.reason,
|
|
3977
|
+
score: averageScore(run.evals),
|
|
3978
|
+
failureClass: "tool_recovery_failure",
|
|
3979
|
+
stoppedBy: "stop-policy"
|
|
3980
|
+
}) };
|
|
3981
|
+
}
|
|
3982
|
+
/** Run the chosen action under its trace span and price it. A failed action ends the run only when `actionFailure` is 'stop'. */
|
|
3983
|
+
async function attemptAction(run, ctx, decision, stepIndex, repeatedActionStreak) {
|
|
3984
|
+
const { config, emitter, runtimeErrors } = run;
|
|
3985
|
+
const beforeState = run.state;
|
|
3986
|
+
const evalsBefore = run.evals;
|
|
3987
|
+
const scoreBefore = averageScore(run.evals);
|
|
3988
|
+
const actionStarted = Date.now();
|
|
3989
|
+
const stepHandle = emitter ? await runTrace(emitter, runtimeErrors, stepIndex, () => emitter.tool({
|
|
3990
|
+
name: `control-step-${stepIndex}`,
|
|
3991
|
+
toolName: "agent-control-action",
|
|
3992
|
+
args: decision.action,
|
|
3993
|
+
attributes: {
|
|
3994
|
+
decision: decision.reason ?? "continue",
|
|
3995
|
+
repeatedActionStreak
|
|
3996
|
+
}
|
|
3997
|
+
})) : void 0;
|
|
3998
|
+
let outcome;
|
|
3999
|
+
try {
|
|
4000
|
+
const result = await config.act(decision.action, ctx);
|
|
4001
|
+
const rawCostUsd = config.getActionCostUsd?.({
|
|
4002
|
+
action: decision.action,
|
|
4003
|
+
result,
|
|
4004
|
+
state: run.state,
|
|
4005
|
+
evals: run.evals,
|
|
4006
|
+
history: run.history
|
|
4007
|
+
});
|
|
4008
|
+
const costUsd = normalizeActionCostUsd(rawCostUsd, runtimeErrors, stepIndex);
|
|
4009
|
+
if (costUsd !== void 0 && Number.isFinite(costUsd) && costUsd > 0) {
|
|
4010
|
+
run.spentCostUsd += costUsd;
|
|
4011
|
+
await recordCostBudget(emitter, run.budget, run.spentCostUsd, stepHandle, runtimeErrors, stepIndex);
|
|
4012
|
+
}
|
|
4013
|
+
outcome = {
|
|
4014
|
+
ok: true,
|
|
4015
|
+
result,
|
|
4016
|
+
...costUsd !== void 0 ? { costUsd } : {},
|
|
4017
|
+
durationMs: Date.now() - actionStarted
|
|
4018
|
+
};
|
|
4019
|
+
} catch (err) {
|
|
4020
|
+
runtimeErrors.push(runtimeError("act", stepIndex, err));
|
|
4021
|
+
outcome = {
|
|
4022
|
+
ok: false,
|
|
4023
|
+
error: lastErrorMessage(run),
|
|
4024
|
+
durationMs: Date.now() - actionStarted
|
|
4025
|
+
};
|
|
4026
|
+
if ((config.actionFailure ?? "continue") === "stop") {
|
|
4027
|
+
await runTrace(emitter, runtimeErrors, stepIndex, () => stepHandle?.fail(outcome.error ?? "action failed"));
|
|
4028
|
+
const step = buildStep(run, {
|
|
4029
|
+
decision,
|
|
4030
|
+
beforeState,
|
|
4031
|
+
evalsBefore,
|
|
4032
|
+
scoreBefore,
|
|
4033
|
+
actionStarted,
|
|
4034
|
+
stepHandle,
|
|
4035
|
+
outcome
|
|
4036
|
+
}, stepIndex, run.state);
|
|
4037
|
+
run.history.push(step);
|
|
4038
|
+
await runOnStep(config.onStep, step, runtimeErrors);
|
|
4039
|
+
return { ended: await settle(run, {
|
|
4040
|
+
pass: false,
|
|
4041
|
+
completed: false,
|
|
4042
|
+
reason: outcome.error ?? "action failed",
|
|
4043
|
+
score: averageScore(run.evals),
|
|
4044
|
+
failureClass: "unknown",
|
|
4045
|
+
stoppedBy: "runtime-error"
|
|
4046
|
+
}) };
|
|
4047
|
+
}
|
|
4048
|
+
}
|
|
4049
|
+
return { next: {
|
|
4050
|
+
decision,
|
|
4051
|
+
beforeState,
|
|
4052
|
+
evalsBefore,
|
|
4053
|
+
scoreBefore,
|
|
4054
|
+
actionStarted,
|
|
4055
|
+
stepHandle,
|
|
4056
|
+
outcome
|
|
4057
|
+
} };
|
|
4058
|
+
}
|
|
4059
|
+
function buildStep(run, attempt, stepIndex, afterState) {
|
|
4060
|
+
return {
|
|
4061
|
+
index: stepIndex,
|
|
4062
|
+
decision: attempt.decision,
|
|
4063
|
+
beforeState: attempt.beforeState,
|
|
4064
|
+
afterState,
|
|
4065
|
+
evalsBefore: attempt.evalsBefore,
|
|
4066
|
+
evalsAfter: run.evals,
|
|
4067
|
+
actionOutcome: attempt.outcome,
|
|
4068
|
+
startedAt: new Date(attempt.actionStarted).toISOString(),
|
|
4069
|
+
endedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
4070
|
+
};
|
|
4071
|
+
}
|
|
4072
|
+
/** Observe and validate after the action. A failure records the step and ends the run. */
|
|
4073
|
+
async function observeAfterAction(run, attempt, stepIndex) {
|
|
4074
|
+
const { config, emitter, runtimeErrors } = run;
|
|
4075
|
+
let observationPhase = "observe";
|
|
4076
|
+
try {
|
|
4077
|
+
run.state = await config.observe({
|
|
4078
|
+
history: run.history,
|
|
4079
|
+
abortSignal: run.signal
|
|
4080
|
+
});
|
|
4081
|
+
observationPhase = "validate";
|
|
4082
|
+
run.evals = await config.validate({
|
|
4083
|
+
intent: config.intent,
|
|
4084
|
+
state: run.state,
|
|
4085
|
+
history: run.history,
|
|
4086
|
+
abortSignal: run.signal
|
|
4087
|
+
});
|
|
4088
|
+
await recordEvalSpans(emitter, run.evals, `step-${stepIndex}`, runtimeErrors, stepIndex, attempt.stepHandle?.span.spanId);
|
|
4089
|
+
return;
|
|
4090
|
+
} catch (err) {
|
|
4091
|
+
runtimeErrors.push(runtimeError(observationPhase, stepIndex, err));
|
|
4092
|
+
const afterState = observationPhase === "observe" ? attempt.beforeState : run.state;
|
|
4093
|
+
const step = buildStep(run, attempt, stepIndex, afterState);
|
|
4094
|
+
run.history.push(step);
|
|
4095
|
+
await runTrace(emitter, runtimeErrors, stepIndex, () => attempt.stepHandle?.fail(lastErrorMessage(run)));
|
|
4096
|
+
await runOnStep(config.onStep, step, runtimeErrors);
|
|
4097
|
+
return settleOnRuntimeError(run, { value: afterState });
|
|
4098
|
+
}
|
|
4099
|
+
}
|
|
4100
|
+
/** Record the finished step, then apply the no-progress, budget and stop-policy checks that follow it. */
|
|
4101
|
+
async function recordStepAndCheck(run, attempt, stepIndex) {
|
|
4102
|
+
const { config, emitter, runtimeErrors } = run;
|
|
4103
|
+
const scoreAfter = averageScore(run.evals);
|
|
4104
|
+
const stateFingerprint = fingerprintState(run.state, config.stopPolicies);
|
|
4105
|
+
const noProgressSignal = run.progressDetector.observe({
|
|
4106
|
+
stateFingerprint,
|
|
4107
|
+
score: scoreAfter
|
|
4108
|
+
});
|
|
4109
|
+
const noProgressStreak = run.progressDetector.streak;
|
|
4110
|
+
const step = buildStep(run, attempt, stepIndex, run.state);
|
|
4111
|
+
run.history.push(step);
|
|
4112
|
+
const { outcome, stepHandle } = attempt;
|
|
4113
|
+
if (outcome.ok) await runTrace(emitter, runtimeErrors, stepIndex, () => stepHandle?.end({ attributes: {
|
|
4114
|
+
actionCostUsd: outcome.costUsd ?? null,
|
|
4115
|
+
spentCostUsd: run.spentCostUsd,
|
|
4116
|
+
scoreBefore: attempt.scoreBefore ?? null,
|
|
4117
|
+
scoreAfter: scoreAfter ?? null,
|
|
4118
|
+
noProgressStreak
|
|
4119
|
+
} }));
|
|
4120
|
+
else await runTrace(emitter, runtimeErrors, stepIndex, () => stepHandle?.fail(outcome.error ?? "action failed", { attributes: {
|
|
4121
|
+
spentCostUsd: run.spentCostUsd,
|
|
4122
|
+
noProgressStreak
|
|
4123
|
+
} }));
|
|
4124
|
+
await runOnStep(config.onStep, step, runtimeErrors);
|
|
4125
|
+
if (noProgressSignal) return settle(run, {
|
|
4126
|
+
pass: false,
|
|
4127
|
+
completed: true,
|
|
4128
|
+
reason: noProgressSignal.reason,
|
|
4129
|
+
score: scoreAfter,
|
|
4130
|
+
failureClass: "tool_recovery_failure",
|
|
4131
|
+
stoppedBy: "stop-policy"
|
|
4132
|
+
});
|
|
4133
|
+
const postStepBudgetStop = budgetStopDecision(run.budget, run.spentCostUsd);
|
|
4134
|
+
if (postStepBudgetStop.stop) return settle(run, {
|
|
4135
|
+
pass: false,
|
|
4136
|
+
completed: false,
|
|
4137
|
+
reason: postStepBudgetStop.reason,
|
|
4138
|
+
score: scoreAfter,
|
|
4139
|
+
failureClass: "budget_exceeded",
|
|
4140
|
+
stoppedBy: "budget"
|
|
4141
|
+
});
|
|
4142
|
+
return checkStopPolicy(run, makeRunContext(run, stepIndex + 1), stepIndex + 1);
|
|
4143
|
+
}
|
|
4257
4144
|
function makeContext(intent, state, evals, history, budget, stepIndex, started, spentCostUsd, abortSignal, emitter) {
|
|
4258
4145
|
return {
|
|
4259
4146
|
intent,
|
|
@@ -4424,65 +4311,6 @@ async function discoverPersonas(dir, opts = {}) {
|
|
|
4424
4311
|
return results;
|
|
4425
4312
|
}
|
|
4426
4313
|
//#endregion
|
|
4427
|
-
//#region src/improvement-verdict.ts
|
|
4428
|
-
const DEFAULTS = {
|
|
4429
|
-
keepThreshold: 5,
|
|
4430
|
-
regressionThreshold: 5,
|
|
4431
|
-
iqrUnstableAbove: 10,
|
|
4432
|
-
stddevUnstableAbove: Number.POSITIVE_INFINITY,
|
|
4433
|
-
minRepsForVerdict: 3
|
|
4434
|
-
};
|
|
4435
|
-
function resolveThresholds(t) {
|
|
4436
|
-
const r = {
|
|
4437
|
-
...DEFAULTS,
|
|
4438
|
-
...t ?? {}
|
|
4439
|
-
};
|
|
4440
|
-
if (r.keepThreshold < 0) throw new ValidationError(`improvement-verdict: keepThreshold must be >= 0, got ${r.keepThreshold}`);
|
|
4441
|
-
if (r.regressionThreshold < 0) throw new ValidationError(`improvement-verdict: regressionThreshold must be >= 0, got ${r.regressionThreshold}`);
|
|
4442
|
-
if (r.minRepsForVerdict < 1) throw new ValidationError(`improvement-verdict: minRepsForVerdict must be >= 1, got ${r.minRepsForVerdict}`);
|
|
4443
|
-
return r;
|
|
4444
|
-
}
|
|
4445
|
-
/**
|
|
4446
|
-
* Verdict for a candidate against its parent. Pure — operates on already-computed
|
|
4447
|
-
* stats. KEEP/REGRESSION require both sides to have `>= minRepsForVerdict` reps
|
|
4448
|
-
* AND the candidate to be `stable`; otherwise the result is NOISE (unstable) or
|
|
4449
|
-
* ITERATE (not enough reps / no parent).
|
|
4450
|
-
*/
|
|
4451
|
-
function improvementVerdict(candidate, parent, thresholds) {
|
|
4452
|
-
const t = resolveThresholds(thresholds);
|
|
4453
|
-
if (!parent) return {
|
|
4454
|
-
verdict: "ITERATE",
|
|
4455
|
-
medianDelta: null,
|
|
4456
|
-
reason: "no parent experiment to compare against"
|
|
4457
|
-
};
|
|
4458
|
-
if (candidate.n < t.minRepsForVerdict || parent.n < t.minRepsForVerdict) return {
|
|
4459
|
-
verdict: "ITERATE",
|
|
4460
|
-
medianDelta: null,
|
|
4461
|
-
reason: `need >= ${t.minRepsForVerdict} reps on both sides (candidate n=${candidate.n}, parent n=${parent.n})`
|
|
4462
|
-
};
|
|
4463
|
-
if (!candidate.stable) return {
|
|
4464
|
-
verdict: "NOISE",
|
|
4465
|
-
medianDelta: candidate.median - parent.median,
|
|
4466
|
-
reason: `candidate unstable (iqr=${candidate.iqr}, stddev=${candidate.stddev.toFixed(2)})`
|
|
4467
|
-
};
|
|
4468
|
-
const medianDelta = candidate.median - parent.median;
|
|
4469
|
-
if (medianDelta > t.keepThreshold) return {
|
|
4470
|
-
verdict: "KEEP",
|
|
4471
|
-
medianDelta,
|
|
4472
|
-
reason: `median +${medianDelta} > +${t.keepThreshold}`
|
|
4473
|
-
};
|
|
4474
|
-
if (medianDelta < -t.regressionThreshold) return {
|
|
4475
|
-
verdict: "REGRESSION",
|
|
4476
|
-
medianDelta,
|
|
4477
|
-
reason: `median ${medianDelta} < -${t.regressionThreshold}`
|
|
4478
|
-
};
|
|
4479
|
-
return {
|
|
4480
|
-
verdict: "NOISE",
|
|
4481
|
-
medianDelta,
|
|
4482
|
-
reason: `median delta ${medianDelta} inside noise band [-${t.regressionThreshold}, +${t.keepThreshold}]`
|
|
4483
|
-
};
|
|
4484
|
-
}
|
|
4485
|
-
//#endregion
|
|
4486
4314
|
//#region src/propose-review.ts
|
|
4487
4315
|
/**
|
|
4488
4316
|
* Propose / Verify / Review — the core multi-shot primitive.
|
|
@@ -4553,14 +4381,14 @@ function jsonlReviewStore(path) {
|
|
|
4553
4381
|
}
|
|
4554
4382
|
};
|
|
4555
4383
|
}
|
|
4556
|
-
const DEFAULT_FALLBACK_INSTRUCTION
|
|
4384
|
+
const DEFAULT_FALLBACK_INSTRUCTION = "Inspect the verification failures above. Fix the critical issues first, then the major ones. Do not restate the failures — act on them.";
|
|
4557
4385
|
async function runProposeReview(config) {
|
|
4558
4386
|
const maxShots = config.maxShots ?? 10;
|
|
4559
4387
|
const maxWallMs = config.maxWallMs ?? 6e5;
|
|
4560
4388
|
const confidenceFloor = config.confidenceFloor ?? .3;
|
|
4561
4389
|
const confidenceFloorWindow = config.confidenceFloorWindow ?? 2;
|
|
4562
4390
|
const memory = config.memory ?? inMemoryReviewStore();
|
|
4563
|
-
const fallbackInstruction = config.fallbackInstruction ?? DEFAULT_FALLBACK_INSTRUCTION
|
|
4391
|
+
const fallbackInstruction = config.fallbackInstruction ?? DEFAULT_FALLBACK_INSTRUCTION;
|
|
4564
4392
|
const emitter = config.store ? new TraceEmitter(config.store) : null;
|
|
4565
4393
|
if (emitter) await emitter.startRun({
|
|
4566
4394
|
scenarioId: config.scenarioId ?? "propose-review",
|
|
@@ -4793,196 +4621,6 @@ function safeJson(x) {
|
|
|
4793
4621
|
}
|
|
4794
4622
|
}
|
|
4795
4623
|
//#endregion
|
|
4796
|
-
//#region src/propose-review-control.ts
|
|
4797
|
-
const DEFAULT_FALLBACK_INSTRUCTION = "Inspect the verification failures above. Fix the critical issues first, then the major ones. Do not restate the failures — act on them.";
|
|
4798
|
-
async function runProposeReviewAsControlLoop(config) {
|
|
4799
|
-
const maxShots = config.maxShots ?? 10;
|
|
4800
|
-
const confidenceFloor = config.confidenceFloor ?? .3;
|
|
4801
|
-
const confidenceFloorWindow = config.confidenceFloorWindow ?? 2;
|
|
4802
|
-
const memory = config.memory ?? inMemoryReviewStore();
|
|
4803
|
-
const fallbackInstruction = config.fallbackInstruction ?? DEFAULT_FALLBACK_INSTRUCTION;
|
|
4804
|
-
const failureClassFromVerification = config.failureClassFromVerification ?? controlFailureClassFromVerification;
|
|
4805
|
-
let lowConfidenceStreak = 0;
|
|
4806
|
-
let current = {
|
|
4807
|
-
shot: 0,
|
|
4808
|
-
state: config.initialState,
|
|
4809
|
-
priorReview: null,
|
|
4810
|
-
verification: { pass: false },
|
|
4811
|
-
memory: await memory.load(),
|
|
4812
|
-
completed: false,
|
|
4813
|
-
reviewAvailable: false
|
|
4814
|
-
};
|
|
4815
|
-
return runAgentControlLoop({
|
|
4816
|
-
intent: config.goal,
|
|
4817
|
-
budget: {
|
|
4818
|
-
maxSteps: maxShots,
|
|
4819
|
-
maxWallMs: config.maxWallMs
|
|
4820
|
-
},
|
|
4821
|
-
store: config.store,
|
|
4822
|
-
scenarioId: config.scenarioId ?? "propose-review-control",
|
|
4823
|
-
projectId: config.projectId,
|
|
4824
|
-
variantId: config.variantId,
|
|
4825
|
-
actionFailure: config.actionFailure ?? "stop",
|
|
4826
|
-
observe: () => current,
|
|
4827
|
-
validate: ({ state }) => [objectiveEval({
|
|
4828
|
-
id: "verification",
|
|
4829
|
-
passed: state.verification.pass,
|
|
4830
|
-
score: state.verification.score,
|
|
4831
|
-
severity: "critical",
|
|
4832
|
-
detail: state.verification.pass ? "verification passed" : `verification failed${state.verification.failingLayers?.length ? `: ${state.verification.failingLayers.join(", ")}` : ""}`
|
|
4833
|
-
})],
|
|
4834
|
-
shouldStop: ({ state }) => {
|
|
4835
|
-
if (state.verification.pass) return {
|
|
4836
|
-
stop: true,
|
|
4837
|
-
pass: true,
|
|
4838
|
-
reason: "verification passed",
|
|
4839
|
-
score: state.verification.score
|
|
4840
|
-
};
|
|
4841
|
-
if (state.completed) return {
|
|
4842
|
-
stop: true,
|
|
4843
|
-
pass: false,
|
|
4844
|
-
reason: "reviewer stopped continuation",
|
|
4845
|
-
score: state.verification.score,
|
|
4846
|
-
failureClass: failureClassFromVerification(state.verification)
|
|
4847
|
-
};
|
|
4848
|
-
return {
|
|
4849
|
-
stop: false,
|
|
4850
|
-
pass: false,
|
|
4851
|
-
reason: "verification still failing",
|
|
4852
|
-
score: state.verification.score
|
|
4853
|
-
};
|
|
4854
|
-
},
|
|
4855
|
-
decide: ({ state }) => ({
|
|
4856
|
-
type: "continue",
|
|
4857
|
-
action: {
|
|
4858
|
-
type: "propose-review-shot",
|
|
4859
|
-
shot: state.shot + 1
|
|
4860
|
-
},
|
|
4861
|
-
reason: state.priorReview?.nextShotInstruction ?? fallbackInstruction
|
|
4862
|
-
}),
|
|
4863
|
-
act: async (action, ctx) => {
|
|
4864
|
-
const shot = action.shot;
|
|
4865
|
-
const proposeOut = await config.propose({
|
|
4866
|
-
shot,
|
|
4867
|
-
goal: config.goal,
|
|
4868
|
-
state: current.state,
|
|
4869
|
-
priorReview: current.priorReview,
|
|
4870
|
-
abortSignal: ctx.abortSignal,
|
|
4871
|
-
emitter: ctx.emitter
|
|
4872
|
-
});
|
|
4873
|
-
const nextState = proposeOut.state;
|
|
4874
|
-
const verification = await config.verify(nextState);
|
|
4875
|
-
let review = null;
|
|
4876
|
-
let reviewAvailable = false;
|
|
4877
|
-
let reviewError;
|
|
4878
|
-
let shouldContinue = !verification.pass;
|
|
4879
|
-
if (!verification.pass) try {
|
|
4880
|
-
review = await config.review({
|
|
4881
|
-
shot,
|
|
4882
|
-
goal: config.goal,
|
|
4883
|
-
state: nextState,
|
|
4884
|
-
verification,
|
|
4885
|
-
traceSummary: proposeOut.traceSummary,
|
|
4886
|
-
memory: await memory.load()
|
|
4887
|
-
});
|
|
4888
|
-
reviewAvailable = true;
|
|
4889
|
-
shouldContinue = review.shouldContinue;
|
|
4890
|
-
lowConfidenceStreak = review.confidence <= confidenceFloor ? lowConfidenceStreak + 1 : 0;
|
|
4891
|
-
if (confidenceFloorWindow > 0 && lowConfidenceStreak >= confidenceFloorWindow) shouldContinue = false;
|
|
4892
|
-
} catch (err) {
|
|
4893
|
-
reviewError = err instanceof Error ? err.message : String(err);
|
|
4894
|
-
review = current.priorReview ?? {
|
|
4895
|
-
observations: "Reviewer unavailable.",
|
|
4896
|
-
diagnosis: reviewError,
|
|
4897
|
-
nextShotInstruction: fallbackInstruction,
|
|
4898
|
-
shouldContinue: true,
|
|
4899
|
-
confidence: 0
|
|
4900
|
-
};
|
|
4901
|
-
shouldContinue = true;
|
|
4902
|
-
}
|
|
4903
|
-
else review = {
|
|
4904
|
-
observations: "Verification passed.",
|
|
4905
|
-
diagnosis: "No further revision needed.",
|
|
4906
|
-
nextShotInstruction: "",
|
|
4907
|
-
shouldContinue: false,
|
|
4908
|
-
confidence: 1
|
|
4909
|
-
};
|
|
4910
|
-
const entry = {
|
|
4911
|
-
...review ?? {
|
|
4912
|
-
observations: "No review.",
|
|
4913
|
-
diagnosis: "",
|
|
4914
|
-
nextShotInstruction: fallbackInstruction,
|
|
4915
|
-
shouldContinue,
|
|
4916
|
-
confidence: 0
|
|
4917
|
-
},
|
|
4918
|
-
shot,
|
|
4919
|
-
timestamp: Date.now(),
|
|
4920
|
-
verification: {
|
|
4921
|
-
pass: verification.pass,
|
|
4922
|
-
score: verification.score,
|
|
4923
|
-
failingLayers: verification.failingLayers
|
|
4924
|
-
}
|
|
4925
|
-
};
|
|
4926
|
-
await memory.append(entry);
|
|
4927
|
-
current = {
|
|
4928
|
-
shot,
|
|
4929
|
-
state: nextState,
|
|
4930
|
-
priorReview: review,
|
|
4931
|
-
verification,
|
|
4932
|
-
traceSummary: proposeOut.traceSummary,
|
|
4933
|
-
memory: await memory.load(),
|
|
4934
|
-
completed: verification.pass || !shouldContinue,
|
|
4935
|
-
reviewAvailable,
|
|
4936
|
-
reviewError
|
|
4937
|
-
};
|
|
4938
|
-
return {
|
|
4939
|
-
state: nextState,
|
|
4940
|
-
verification,
|
|
4941
|
-
traceSummary: proposeOut.traceSummary,
|
|
4942
|
-
review,
|
|
4943
|
-
reviewAvailable,
|
|
4944
|
-
reviewError
|
|
4945
|
-
};
|
|
4946
|
-
}
|
|
4947
|
-
});
|
|
4948
|
-
}
|
|
4949
|
-
function controlFailureClassFromVerification(verification) {
|
|
4950
|
-
if (verification.pass) return void 0;
|
|
4951
|
-
return verification.failingLayers?.length ? "instruction_following" : "unknown";
|
|
4952
|
-
}
|
|
4953
|
-
//#endregion
|
|
4954
|
-
//#region src/analyst/steer-firewall.ts
|
|
4955
|
-
/** True iff the finding is a JUDGE VERDICT (an acceptance score lifted into a
|
|
4956
|
-
* finding), identified by provenance set at the lift site — independent of
|
|
4957
|
-
* whatever evidence it cites. */
|
|
4958
|
-
function isJudgeVerdict(finding) {
|
|
4959
|
-
return finding.derived_from_judge === true;
|
|
4960
|
-
}
|
|
4961
|
-
/**
|
|
4962
|
-
* THE steer firewall. Fail-loud guard for any path that admits analyst findings
|
|
4963
|
-
* as STEERING input (the `f(trace)` role): rejects — naming the offenders — any
|
|
4964
|
-
* finding whose provenance is a judge verdict, rather than let `J` leak into the
|
|
4965
|
-
* loop. Returns the findings unchanged for chaining.
|
|
4966
|
-
*
|
|
4967
|
-
* Call this at the chokepoint where a detector that ALSO scores/gates has its
|
|
4968
|
-
* findings turned into a steer (the judge-and-steer dual-role case). It keys on
|
|
4969
|
-
* provenance, so it correctly admits evidence-less trace-analyst observations and
|
|
4970
|
-
* correctly rejects an artifact-citing judge verdict — the cases an evidence
|
|
4971
|
-
* check gets backwards.
|
|
4972
|
-
*
|
|
4973
|
-
* It is necessary, not sufficient: it stops PROVENANCE-tagged verdicts. A judge
|
|
4974
|
-
* whose output is laundered through a hand-built finding with no provenance flag
|
|
4975
|
-
* is out of its reach — provenance must be honestly set at every judge→finding
|
|
4976
|
-
* lift (today: createJudgeAdapter). That is why the integrity rule lives at the
|
|
4977
|
-
* lift site, and why ProposeContext.judgeScores?: never is the complementary
|
|
4978
|
-
* compile-time tripwire on the obvious direct channel.
|
|
4979
|
-
*/
|
|
4980
|
-
function assertNoJudgeVerdict(findings, context = "steer") {
|
|
4981
|
-
const leaks = findings.filter(isJudgeVerdict);
|
|
4982
|
-
if (leaks.length > 0) throw new Error(`${context}: a judge verdict cannot be admitted as steering input — that is the held-out judge leaking into the loop. Offending judge-derived findings: [${leaks.map((f) => f.finding_id).join(", ")}]. Steering consumes observations of behavior, never acceptance verdicts.`);
|
|
4983
|
-
return findings;
|
|
4984
|
-
}
|
|
4985
|
-
//#endregion
|
|
4986
4624
|
//#region src/analyst/policy-edit.ts
|
|
4987
4625
|
const POLICY_EDIT_AXES = [
|
|
4988
4626
|
"carrier",
|
|
@@ -5013,20 +4651,7 @@ var PolicyEditValidationError = class extends ValidationError {
|
|
|
5013
4651
|
this.path = path;
|
|
5014
4652
|
}
|
|
5015
4653
|
};
|
|
5016
|
-
const DEFAULT_MIN_SCORE = .7;
|
|
5017
|
-
const DEFAULT_MIN_EXPECTED_GAIN = .01;
|
|
5018
4654
|
const POLICY_EDIT_ID = /^policy-edit:sha256:[0-9a-f]{64}$/;
|
|
5019
|
-
function makePolicyEdit(init) {
|
|
5020
|
-
const normalized = normalizePolicyEdit({
|
|
5021
|
-
schemaVersion: "policy-edit/v1",
|
|
5022
|
-
...init,
|
|
5023
|
-
source: normalizeSource(init.source)
|
|
5024
|
-
});
|
|
5025
|
-
return validatePolicyEdit({
|
|
5026
|
-
...normalized,
|
|
5027
|
-
editId: init.editId ?? computePolicyEditId(normalized)
|
|
5028
|
-
});
|
|
5029
|
-
}
|
|
5030
4655
|
function computePolicyEditId(edit) {
|
|
5031
4656
|
const { editId: _editId, schemaVersion, ...material } = edit;
|
|
5032
4657
|
return `policy-edit:${hashCanonical({
|
|
@@ -5080,363 +4705,6 @@ function validatePolicyEditCandidateRecord(input) {
|
|
|
5080
4705
|
policyEdit: validatePolicyEdit(snapshot)
|
|
5081
4706
|
};
|
|
5082
4707
|
}
|
|
5083
|
-
function policyEditsFromFindings(findings, opts = {}) {
|
|
5084
|
-
assertNoJudgeVerdict(findings, "policyEditsFromFindings");
|
|
5085
|
-
const edits = [];
|
|
5086
|
-
for (const finding of findings) {
|
|
5087
|
-
const edit = policyEditFromFinding(finding, opts);
|
|
5088
|
-
if (edit) edits.push(edit);
|
|
5089
|
-
}
|
|
5090
|
-
return edits;
|
|
5091
|
-
}
|
|
5092
|
-
function policyEditFromFinding(finding, opts = {}) {
|
|
5093
|
-
assertNoJudgeVerdict([finding], "policyEditFromFinding");
|
|
5094
|
-
if (!finding.recommended_action?.trim()) return null;
|
|
5095
|
-
const expectedGain = resolveExpectedGain(finding, opts);
|
|
5096
|
-
if (!expectedGain) return null;
|
|
5097
|
-
if (typeof finding.confidence !== "number" || !Number.isFinite(finding.confidence) || finding.confidence < 0 || finding.confidence > 1) return null;
|
|
5098
|
-
const routed = routeFindingSubject(finding.subject, opts);
|
|
5099
|
-
const risk = resolveRisk(finding, opts);
|
|
5100
|
-
return makePolicyEdit({
|
|
5101
|
-
axis: routed.axis,
|
|
5102
|
-
target: routed.target,
|
|
5103
|
-
change: {
|
|
5104
|
-
kind: "text",
|
|
5105
|
-
mode: "append",
|
|
5106
|
-
value: finding.recommended_action.trim()
|
|
5107
|
-
},
|
|
5108
|
-
claim: finding.claim,
|
|
5109
|
-
rationale: finding.rationale,
|
|
5110
|
-
expectedGain,
|
|
5111
|
-
confidence: finding.confidence,
|
|
5112
|
-
risk,
|
|
5113
|
-
validationPlan: finding.validation_plan,
|
|
5114
|
-
source: {
|
|
5115
|
-
findingIds: [finding.finding_id],
|
|
5116
|
-
analystIds: [finding.analyst_id],
|
|
5117
|
-
evidenceRefs: finding.evidence_refs,
|
|
5118
|
-
derivedFromJudge: finding.derived_from_judge
|
|
5119
|
-
}
|
|
5120
|
-
});
|
|
5121
|
-
}
|
|
5122
|
-
function scorePolicyEditReadiness(edit, opts = {}) {
|
|
5123
|
-
validatePolicyEdit(edit);
|
|
5124
|
-
const minExpectedGain = opts.minExpectedGain ?? DEFAULT_MIN_EXPECTED_GAIN;
|
|
5125
|
-
const requireEvidence = opts.requireEvidence ?? false;
|
|
5126
|
-
const evidenceScore = Math.min(1, edit.source.evidenceRefs.length / 2);
|
|
5127
|
-
const confidenceScore = clamp01$3(edit.confidence);
|
|
5128
|
-
const gainScore = clamp01$3(Math.abs(edit.expectedGain.amount) / Math.max(minExpectedGain * 5, .001));
|
|
5129
|
-
const targetScore = targetSpecificityScore(edit);
|
|
5130
|
-
const riskPenalty = edit.risk === "high" && opts.allowHighRisk !== true ? .35 : edit.risk === "unknown" ? .2 : 0;
|
|
5131
|
-
return clamp01$3((requireEvidence ? .3 * evidenceScore + .25 * confidenceScore + .25 * gainScore + .2 * targetScore : .35 * confidenceScore + .35 * gainScore + .3 * targetScore + .1 * evidenceScore) - riskPenalty);
|
|
5132
|
-
}
|
|
5133
|
-
function admitPolicyEdit(edit, opts = {}) {
|
|
5134
|
-
const validated = validatePolicyEdit(edit);
|
|
5135
|
-
const score = scorePolicyEditReadiness(validated, opts);
|
|
5136
|
-
const reasons = [];
|
|
5137
|
-
const minExpectedGain = opts.minExpectedGain ?? DEFAULT_MIN_EXPECTED_GAIN;
|
|
5138
|
-
const requireEvidence = opts.requireEvidence ?? false;
|
|
5139
|
-
if (validated.source.derivedFromJudge) reasons.push("source is judge-derived; judge verdicts cannot steer policy edits");
|
|
5140
|
-
if (requireEvidence && validated.source.evidenceRefs.length === 0) reasons.push("missing evidence refs");
|
|
5141
|
-
if (Math.abs(validated.expectedGain.amount) < minExpectedGain) reasons.push(`expected gain below ${minExpectedGain}`);
|
|
5142
|
-
if (validated.risk === "high" && opts.allowHighRisk !== true) reasons.push("high-risk edit requires explicit allowHighRisk");
|
|
5143
|
-
if (score < (opts.minScore ?? DEFAULT_MIN_SCORE)) reasons.push(`readiness score ${score.toFixed(3)} below ${(opts.minScore ?? DEFAULT_MIN_SCORE).toFixed(3)}`);
|
|
5144
|
-
return {
|
|
5145
|
-
edit: validated,
|
|
5146
|
-
decision: reasons.length === 0 ? "admit" : "reject",
|
|
5147
|
-
score,
|
|
5148
|
-
reasons
|
|
5149
|
-
};
|
|
5150
|
-
}
|
|
5151
|
-
function applyPolicyEditToSurface(surface, edit) {
|
|
5152
|
-
const validated = validatePolicyEdit(edit);
|
|
5153
|
-
if (validated.change.kind === "text") return applyTextChange(surface, validated.change);
|
|
5154
|
-
return applyJsonChange(surface, validated.change);
|
|
5155
|
-
}
|
|
5156
|
-
function routeFindingSubject(subject, opts) {
|
|
5157
|
-
const parsed = parseFindingSubject(subject);
|
|
5158
|
-
if (!parsed) return {
|
|
5159
|
-
axis: opts.defaultAxis ?? "representation",
|
|
5160
|
-
target: { surface: opts.defaultTargetSurface ?? "prompt" }
|
|
5161
|
-
};
|
|
5162
|
-
return routeParsedSubject(parsed);
|
|
5163
|
-
}
|
|
5164
|
-
function routeParsedSubject(subject) {
|
|
5165
|
-
switch (subject.kind) {
|
|
5166
|
-
case "system-prompt": return {
|
|
5167
|
-
axis: "representation",
|
|
5168
|
-
target: {
|
|
5169
|
-
surface: "prompt",
|
|
5170
|
-
path: `system-prompt:${subject.section}`
|
|
5171
|
-
}
|
|
5172
|
-
};
|
|
5173
|
-
case "skill": return {
|
|
5174
|
-
axis: "agent_profile",
|
|
5175
|
-
target: {
|
|
5176
|
-
surface: "agent-profile",
|
|
5177
|
-
path: `skill:${subject.name}`
|
|
5178
|
-
}
|
|
5179
|
-
};
|
|
5180
|
-
case "tool-doc": return {
|
|
5181
|
-
axis: "tool_contract",
|
|
5182
|
-
target: {
|
|
5183
|
-
surface: "tool-contract",
|
|
5184
|
-
path: subject.aspect ? `tool-doc:${subject.tool}:${subject.aspect}` : `tool-doc:${subject.tool}`
|
|
5185
|
-
}
|
|
5186
|
-
};
|
|
5187
|
-
case "new-tool": return {
|
|
5188
|
-
axis: "tool_contract",
|
|
5189
|
-
target: {
|
|
5190
|
-
surface: "tool-contract",
|
|
5191
|
-
path: `new-tool:${subject.name}`
|
|
5192
|
-
}
|
|
5193
|
-
};
|
|
5194
|
-
case "mcp": return {
|
|
5195
|
-
axis: "tool_contract",
|
|
5196
|
-
target: {
|
|
5197
|
-
surface: "agent-profile",
|
|
5198
|
-
path: subject.tool ? `mcp:${subject.server}:${subject.tool}` : `mcp:${subject.server}`
|
|
5199
|
-
}
|
|
5200
|
-
};
|
|
5201
|
-
case "hook": return {
|
|
5202
|
-
axis: "agent_profile",
|
|
5203
|
-
target: {
|
|
5204
|
-
surface: "agent-profile",
|
|
5205
|
-
path: `hook:${subject.name}`
|
|
5206
|
-
}
|
|
5207
|
-
};
|
|
5208
|
-
case "subagent": return {
|
|
5209
|
-
axis: "routing",
|
|
5210
|
-
target: {
|
|
5211
|
-
surface: "agent-profile",
|
|
5212
|
-
path: `subagent:${subject.name}`
|
|
5213
|
-
}
|
|
5214
|
-
};
|
|
5215
|
-
case "workflow": return {
|
|
5216
|
-
axis: "routing",
|
|
5217
|
-
target: {
|
|
5218
|
-
surface: "runtime-config",
|
|
5219
|
-
path: `workflow:${subject.name}`
|
|
5220
|
-
}
|
|
5221
|
-
};
|
|
5222
|
-
case "rollout-policy": return {
|
|
5223
|
-
axis: rolloutPolicyAxis(subject.field),
|
|
5224
|
-
target: {
|
|
5225
|
-
surface: "runtime-config",
|
|
5226
|
-
path: `rollout-policy:${subject.field}`
|
|
5227
|
-
}
|
|
5228
|
-
};
|
|
5229
|
-
case "agent-profile": return {
|
|
5230
|
-
axis: "agent_profile",
|
|
5231
|
-
target: {
|
|
5232
|
-
surface: "agent-profile",
|
|
5233
|
-
path: `agent-profile:${subject.field}`
|
|
5234
|
-
}
|
|
5235
|
-
};
|
|
5236
|
-
case "code": return {
|
|
5237
|
-
axis: "representation",
|
|
5238
|
-
target: {
|
|
5239
|
-
surface: "code",
|
|
5240
|
-
path: `code:${subject.path}`
|
|
5241
|
-
}
|
|
5242
|
-
};
|
|
5243
|
-
case "rag": return {
|
|
5244
|
-
axis: "memory",
|
|
5245
|
-
target: {
|
|
5246
|
-
surface: "memory",
|
|
5247
|
-
path: `rag:${subject.corpus}:${subject.docId}`
|
|
5248
|
-
}
|
|
5249
|
-
};
|
|
5250
|
-
case "memory": return {
|
|
5251
|
-
axis: "memory",
|
|
5252
|
-
target: {
|
|
5253
|
-
surface: "memory",
|
|
5254
|
-
path: `memory:${subject.key}`
|
|
5255
|
-
}
|
|
5256
|
-
};
|
|
5257
|
-
case "scaffolding": return {
|
|
5258
|
-
axis: "routing",
|
|
5259
|
-
target: {
|
|
5260
|
-
surface: "runtime-config",
|
|
5261
|
-
path: `scaffolding:${subject.concern}`
|
|
5262
|
-
}
|
|
5263
|
-
};
|
|
5264
|
-
case "output-schema": return {
|
|
5265
|
-
axis: "output_contract",
|
|
5266
|
-
target: {
|
|
5267
|
-
surface: "runtime-config",
|
|
5268
|
-
path: `output-schema:${subject.field}`
|
|
5269
|
-
}
|
|
5270
|
-
};
|
|
5271
|
-
case "knowledge.wiki": return {
|
|
5272
|
-
axis: "memory",
|
|
5273
|
-
target: {
|
|
5274
|
-
surface: "memory",
|
|
5275
|
-
path: `agent-knowledge:wiki:${subject.slug}${subject.heading ? `#${subject.heading}` : ""}`
|
|
5276
|
-
}
|
|
5277
|
-
};
|
|
5278
|
-
case "knowledge.claim": return {
|
|
5279
|
-
axis: "memory",
|
|
5280
|
-
target: {
|
|
5281
|
-
surface: "memory",
|
|
5282
|
-
path: `agent-knowledge:claim:${subject.topic}`
|
|
5283
|
-
}
|
|
5284
|
-
};
|
|
5285
|
-
case "knowledge.raw": return {
|
|
5286
|
-
axis: "memory",
|
|
5287
|
-
target: {
|
|
5288
|
-
surface: "memory",
|
|
5289
|
-
path: `agent-knowledge:raw:${subject.sourceId}`
|
|
5290
|
-
}
|
|
5291
|
-
};
|
|
5292
|
-
case "knowledge.stale": return {
|
|
5293
|
-
axis: "memory",
|
|
5294
|
-
target: {
|
|
5295
|
-
surface: "memory",
|
|
5296
|
-
path: `agent-knowledge:stale:${subject.slug}`
|
|
5297
|
-
}
|
|
5298
|
-
};
|
|
5299
|
-
case "websearch.outdated": return {
|
|
5300
|
-
axis: "memory",
|
|
5301
|
-
target: {
|
|
5302
|
-
surface: "memory",
|
|
5303
|
-
path: `websearch:outdated:${subject.topic}`
|
|
5304
|
-
}
|
|
5305
|
-
};
|
|
5306
|
-
case "prior-run-summary": return {
|
|
5307
|
-
axis: "memory",
|
|
5308
|
-
target: {
|
|
5309
|
-
surface: "memory",
|
|
5310
|
-
path: `prior-run-summary:${subject.topic}`
|
|
5311
|
-
}
|
|
5312
|
-
};
|
|
5313
|
-
case "cluster": return {
|
|
5314
|
-
axis: "representation",
|
|
5315
|
-
target: {
|
|
5316
|
-
surface: "prompt",
|
|
5317
|
-
path: subject.label
|
|
5318
|
-
}
|
|
5319
|
-
};
|
|
5320
|
-
}
|
|
5321
|
-
}
|
|
5322
|
-
function rolloutPolicyAxis(field) {
|
|
5323
|
-
const normalized = field.toLowerCase();
|
|
5324
|
-
if (/budget|max(?:imum)?[-_. ]?(?:turns?|tokens?|cost)|timeout|deadline/.test(normalized)) return "budget";
|
|
5325
|
-
if (/temperature|top[-_. ]?p|sampling|seed|shots?|parallel|concurrency/.test(normalized)) return "sampling";
|
|
5326
|
-
if (/output|schema|format/.test(normalized)) return "output_contract";
|
|
5327
|
-
return "routing";
|
|
5328
|
-
}
|
|
5329
|
-
function resolveExpectedGain(finding, opts) {
|
|
5330
|
-
if (typeof opts.expectedGain === "function") return opts.expectedGain(finding) ?? null;
|
|
5331
|
-
if (opts.expectedGain) return opts.expectedGain;
|
|
5332
|
-
return readExpectedGainFromMetadata(finding.metadata);
|
|
5333
|
-
}
|
|
5334
|
-
function readExpectedGainFromMetadata(metadata) {
|
|
5335
|
-
const raw = readPolicyEditMetadata(metadata)?.expectedGain ?? readPolicyEditMetadata(metadata)?.expected_gain;
|
|
5336
|
-
if (!raw || typeof raw !== "object") return null;
|
|
5337
|
-
const obj = raw;
|
|
5338
|
-
if (typeof obj.metric !== "string" || obj.direction !== "increase" && obj.direction !== "decrease" || typeof obj.amount !== "number" || !Number.isFinite(obj.amount) || obj.amount <= 0) return null;
|
|
5339
|
-
const out = {
|
|
5340
|
-
metric: obj.metric,
|
|
5341
|
-
direction: obj.direction,
|
|
5342
|
-
amount: obj.amount
|
|
5343
|
-
};
|
|
5344
|
-
if (obj.unit === "absolute" || obj.unit === "relative" || obj.unit === "percent" || obj.unit === "score") out.unit = obj.unit;
|
|
5345
|
-
if (typeof obj.rationale === "string") out.rationale = obj.rationale;
|
|
5346
|
-
return out;
|
|
5347
|
-
}
|
|
5348
|
-
function readPolicyEditMetadata(metadata) {
|
|
5349
|
-
const raw = metadata?.policyEdit ?? metadata?.policy_edit;
|
|
5350
|
-
return raw && typeof raw === "object" ? raw : null;
|
|
5351
|
-
}
|
|
5352
|
-
function resolveRisk(finding, opts) {
|
|
5353
|
-
if (typeof opts.risk === "function") return opts.risk(finding);
|
|
5354
|
-
if (opts.risk) return opts.risk;
|
|
5355
|
-
const raw = readPolicyEditMetadata(finding.metadata)?.risk;
|
|
5356
|
-
if (raw === "low" || raw === "medium" || raw === "high" || raw === "unknown") return raw;
|
|
5357
|
-
if (finding.severity === "critical" || finding.severity === "high") return "medium";
|
|
5358
|
-
return "low";
|
|
5359
|
-
}
|
|
5360
|
-
function applyTextChange(surface, change) {
|
|
5361
|
-
if (typeof surface !== "string") throw new PolicyEditValidationError("text policy edits require a string surface", "change");
|
|
5362
|
-
if (change.mode === "append") {
|
|
5363
|
-
if (hasExactTextBlock(surface, change.value)) return surface;
|
|
5364
|
-
return `${surface.trimEnd()}\n\n${change.value}`.trimStart();
|
|
5365
|
-
}
|
|
5366
|
-
if (change.mode === "prepend") {
|
|
5367
|
-
if (hasExactTextBlock(surface, change.value)) return surface;
|
|
5368
|
-
return `${change.value}\n\n${surface.trimStart()}`.trimEnd();
|
|
5369
|
-
}
|
|
5370
|
-
const find = expectNonEmpty(change.find, "change.find");
|
|
5371
|
-
if (!surface.includes(find)) throw new PolicyEditValidationError("replace target not found in surface", "change.find");
|
|
5372
|
-
return surface.replace(find, change.value);
|
|
5373
|
-
}
|
|
5374
|
-
function applyJsonChange(surface, change) {
|
|
5375
|
-
const root = parseJsonSurface(surface);
|
|
5376
|
-
const path = splitPath(change.path);
|
|
5377
|
-
if (change.mode === "remove") return setJsonAtPath(root, path, void 0, "remove");
|
|
5378
|
-
if (change.mode === "set") return setJsonAtPath(root, path, change.value ?? null, "set");
|
|
5379
|
-
const prior = readJsonAtPath(root, path);
|
|
5380
|
-
return setJsonAtPath(root, path, prior && typeof prior === "object" && !Array.isArray(prior) && change.value && typeof change.value === "object" && !Array.isArray(change.value) ? {
|
|
5381
|
-
...prior,
|
|
5382
|
-
...change.value
|
|
5383
|
-
} : change.value ?? null, "set");
|
|
5384
|
-
}
|
|
5385
|
-
function parseJsonSurface(surface) {
|
|
5386
|
-
if (typeof surface === "string") try {
|
|
5387
|
-
return JSON.parse(surface);
|
|
5388
|
-
} catch {
|
|
5389
|
-
throw new PolicyEditValidationError("json policy edits require a JSON string surface", "change");
|
|
5390
|
-
}
|
|
5391
|
-
assertJson(surface, "surface");
|
|
5392
|
-
return surface;
|
|
5393
|
-
}
|
|
5394
|
-
function readJsonAtPath(root, path) {
|
|
5395
|
-
let cursor = root;
|
|
5396
|
-
for (const part of path) {
|
|
5397
|
-
if (!cursor || typeof cursor !== "object" || Array.isArray(cursor)) return void 0;
|
|
5398
|
-
cursor = cursor[part];
|
|
5399
|
-
}
|
|
5400
|
-
return cursor;
|
|
5401
|
-
}
|
|
5402
|
-
function setJsonAtPath(root, path, value, mode) {
|
|
5403
|
-
if (path.length === 0) {
|
|
5404
|
-
if (mode === "remove") return null;
|
|
5405
|
-
return value ?? null;
|
|
5406
|
-
}
|
|
5407
|
-
if (root === null || typeof root !== "object" || Array.isArray(root)) throw new PolicyEditValidationError("json edit root must be an object", "change.path");
|
|
5408
|
-
const out = { ...root };
|
|
5409
|
-
let cursor = out;
|
|
5410
|
-
for (let i = 0; i < path.length - 1; i++) {
|
|
5411
|
-
const key = path[i];
|
|
5412
|
-
const existing = cursor[key];
|
|
5413
|
-
if (mode === "remove" && (!existing || typeof existing !== "object" || Array.isArray(existing))) return out;
|
|
5414
|
-
const next = existing && typeof existing === "object" && !Array.isArray(existing) ? { ...existing } : {};
|
|
5415
|
-
cursor[key] = next;
|
|
5416
|
-
cursor = next;
|
|
5417
|
-
}
|
|
5418
|
-
const leaf = path[path.length - 1];
|
|
5419
|
-
if (mode === "remove") delete cursor[leaf];
|
|
5420
|
-
else cursor[leaf] = value ?? null;
|
|
5421
|
-
return out;
|
|
5422
|
-
}
|
|
5423
|
-
function normalizePolicyEdit(input) {
|
|
5424
|
-
const out = {
|
|
5425
|
-
schemaVersion: "policy-edit/v1",
|
|
5426
|
-
axis: input.axis,
|
|
5427
|
-
target: normalizeTarget(input.target),
|
|
5428
|
-
change: normalizeChange(input.change),
|
|
5429
|
-
claim: input.claim.trim(),
|
|
5430
|
-
expectedGain: normalizeExpectedGain(input.expectedGain),
|
|
5431
|
-
confidence: input.confidence,
|
|
5432
|
-
risk: input.risk,
|
|
5433
|
-
source: normalizeSource(input.source)
|
|
5434
|
-
};
|
|
5435
|
-
if (input.rationale?.trim()) out.rationale = input.rationale.trim();
|
|
5436
|
-
if (input.validationPlan?.trim()) out.validationPlan = input.validationPlan.trim();
|
|
5437
|
-
if (input.metadata) out.metadata = input.metadata;
|
|
5438
|
-
return out;
|
|
5439
|
-
}
|
|
5440
4708
|
function assertJsonSafe(value, path, ancestors = /* @__PURE__ */ new WeakSet()) {
|
|
5441
4709
|
if (value === null || typeof value === "string" || typeof value === "boolean") return;
|
|
5442
4710
|
if (typeof value === "number") {
|
|
@@ -5458,50 +4726,6 @@ function assertJsonSafe(value, path, ancestors = /* @__PURE__ */ new WeakSet())
|
|
|
5458
4726
|
}
|
|
5459
4727
|
ancestors.delete(value);
|
|
5460
4728
|
}
|
|
5461
|
-
function normalizeTarget(target) {
|
|
5462
|
-
const out = { surface: target.surface };
|
|
5463
|
-
if (target.path?.trim()) out.path = target.path.trim();
|
|
5464
|
-
if (target.agentProfileCell) out.agentProfileCell = validateAgentProfileCell(target.agentProfileCell);
|
|
5465
|
-
if (target.label?.trim()) out.label = target.label.trim();
|
|
5466
|
-
return out;
|
|
5467
|
-
}
|
|
5468
|
-
function normalizeChange(change) {
|
|
5469
|
-
if (change.kind === "text") {
|
|
5470
|
-
const out = {
|
|
5471
|
-
kind: "text",
|
|
5472
|
-
mode: change.mode,
|
|
5473
|
-
value: change.value.trim()
|
|
5474
|
-
};
|
|
5475
|
-
if (change.find?.trim()) out.find = change.find.trim();
|
|
5476
|
-
return out;
|
|
5477
|
-
}
|
|
5478
|
-
const out = {
|
|
5479
|
-
kind: "json",
|
|
5480
|
-
mode: change.mode,
|
|
5481
|
-
path: change.path.trim()
|
|
5482
|
-
};
|
|
5483
|
-
if (change.value !== void 0) out.value = change.value;
|
|
5484
|
-
return out;
|
|
5485
|
-
}
|
|
5486
|
-
function normalizeExpectedGain(gain) {
|
|
5487
|
-
const out = {
|
|
5488
|
-
metric: gain.metric.trim(),
|
|
5489
|
-
direction: gain.direction,
|
|
5490
|
-
amount: gain.amount
|
|
5491
|
-
};
|
|
5492
|
-
if (gain.unit) out.unit = gain.unit;
|
|
5493
|
-
if (gain.rationale?.trim()) out.rationale = gain.rationale.trim();
|
|
5494
|
-
return out;
|
|
5495
|
-
}
|
|
5496
|
-
function normalizeSource(source) {
|
|
5497
|
-
const out = {
|
|
5498
|
-
findingIds: uniqueSorted(source.findingIds.map((s) => s.trim()).filter(Boolean)),
|
|
5499
|
-
analystIds: uniqueSorted(source.analystIds.map((s) => s.trim()).filter(Boolean)),
|
|
5500
|
-
evidenceRefs: source.evidenceRefs
|
|
5501
|
-
};
|
|
5502
|
-
if (source.derivedFromJudge) out.derivedFromJudge = true;
|
|
5503
|
-
return out;
|
|
5504
|
-
}
|
|
5505
4729
|
function validateTarget(target) {
|
|
5506
4730
|
if (!target || typeof target !== "object") throw new PolicyEditValidationError("expected object", "target");
|
|
5507
4731
|
const obj = target;
|
|
@@ -5584,14 +4808,6 @@ function assertJson(value, path) {
|
|
|
5584
4808
|
}
|
|
5585
4809
|
throw new PolicyEditValidationError("expected JSON-compatible value", path);
|
|
5586
4810
|
}
|
|
5587
|
-
function targetSpecificityScore(edit) {
|
|
5588
|
-
let score = .4;
|
|
5589
|
-
if (edit.target.path) score += .25;
|
|
5590
|
-
if (edit.target.agentProfileCell) score += .15;
|
|
5591
|
-
if (edit.change.kind === "json" || edit.change.mode === "replace") score += .2;
|
|
5592
|
-
else if (edit.change.value.length > 0) score += .1;
|
|
5593
|
-
return clamp01$3(score);
|
|
5594
|
-
}
|
|
5595
4811
|
const FORBIDDEN_PATH_KEYS = /* @__PURE__ */ new Set([
|
|
5596
4812
|
"__proto__",
|
|
5597
4813
|
"constructor",
|
|
@@ -5609,21 +4825,9 @@ function expectLiteral(value, expected, path) {
|
|
|
5609
4825
|
function expectString$1(value, path) {
|
|
5610
4826
|
if (typeof value !== "string" || value.trim().length === 0) throw new PolicyEditValidationError("expected non-empty string", path);
|
|
5611
4827
|
}
|
|
5612
|
-
function expectNonEmpty(value, path) {
|
|
5613
|
-
expectString$1(value, path);
|
|
5614
|
-
return value;
|
|
5615
|
-
}
|
|
5616
4828
|
function expectConfidence(value, path) {
|
|
5617
4829
|
if (typeof value !== "number" || !Number.isFinite(value) || value < 0 || value > 1) throw new PolicyEditValidationError("expected finite number in [0,1]", path);
|
|
5618
4830
|
}
|
|
5619
|
-
function hasExactTextBlock(surface, value) {
|
|
5620
|
-
const needle = normalizeTextBlock(value);
|
|
5621
|
-
const normalizedSurface = surface.replace(/\r\n/g, "\n");
|
|
5622
|
-
return [...normalizedSurface.split(/\n{2,}/), ...normalizedSurface.split("\n")].some((block) => normalizeTextBlock(block) === needle);
|
|
5623
|
-
}
|
|
5624
|
-
function normalizeTextBlock(value) {
|
|
5625
|
-
return value.replace(/\r\n/g, "\n").trim();
|
|
5626
|
-
}
|
|
5627
4831
|
function expectOneOf(value, allowed, path) {
|
|
5628
4832
|
if (typeof value !== "string" || !allowed.includes(value)) throw new PolicyEditValidationError(`expected one of ${allowed.join(", ")}`, path);
|
|
5629
4833
|
}
|
|
@@ -5635,15 +4839,6 @@ function expectNonEmptyStringArray(value, path) {
|
|
|
5635
4839
|
expectStringArray$1(value, path);
|
|
5636
4840
|
if (value.length === 0) throw new PolicyEditValidationError("expected non-empty array", path);
|
|
5637
4841
|
}
|
|
5638
|
-
function uniqueSorted(values) {
|
|
5639
|
-
return [...new Set(values)].sort();
|
|
5640
|
-
}
|
|
5641
|
-
function clamp01$3(n) {
|
|
5642
|
-
if (!Number.isFinite(n)) return 0;
|
|
5643
|
-
if (n < 0) return 0;
|
|
5644
|
-
if (n > 1) return 1;
|
|
5645
|
-
return n;
|
|
5646
|
-
}
|
|
5647
4842
|
//#endregion
|
|
5648
4843
|
//#region src/capability-headroom.ts
|
|
5649
4844
|
/**
|
|
@@ -6208,58 +5403,6 @@ function partitionHeldOut(ids, options = {}) {
|
|
|
6208
5403
|
function assertFraction(f) {
|
|
6209
5404
|
if (!Number.isFinite(f) || f <= 0 || f >= 1) throw new ValidationError(`partitionHeldOut: holdoutFraction must be in (0, 1), got ${f}`);
|
|
6210
5405
|
}
|
|
6211
|
-
/**
|
|
6212
|
-
* Partition items into disjoint train, selection and test sets, the three
|
|
6213
|
-
* partitions `compareOptimizationMethods` and Runtime `improve()` take.
|
|
6214
|
-
*
|
|
6215
|
-
* Assignment is per unit: `hashToUnit(unit, seed)` below `testFraction` is
|
|
6216
|
-
* test, below `testFraction + selectionFraction` is selection, and the rest is
|
|
6217
|
-
* train. A unit therefore keeps its partition as the corpus grows, and items of
|
|
6218
|
-
* one unit never straddle two partitions. Fail-loud like `partitionHeldOut`:
|
|
6219
|
-
* empty input, a duplicate or empty id, an empty unit, fractions outside
|
|
6220
|
-
* (0, 1) or summing to 1 or more, and a partition below its floor all throw.
|
|
6221
|
-
* Order within each partition follows the input order.
|
|
6222
|
-
*/
|
|
6223
|
-
function partitionTrainSelectionTest(items, options = {}) {
|
|
6224
|
-
const seed = options.seed ?? "train-selection-test-v1";
|
|
6225
|
-
const selectionFraction = options.selectionFraction ?? .25;
|
|
6226
|
-
const testFraction = options.testFraction ?? .25;
|
|
6227
|
-
assertSplitFraction("selectionFraction", selectionFraction);
|
|
6228
|
-
assertSplitFraction("testFraction", testFraction);
|
|
6229
|
-
if (selectionFraction + testFraction >= 1) throw new ValidationError(`partitionTrainSelectionTest: selectionFraction + testFraction must be below 1, got ${selectionFraction + testFraction}`);
|
|
6230
|
-
if (items.length === 0) throw new ValidationError("partitionTrainSelectionTest: no items supplied");
|
|
6231
|
-
const unitOf = options.unitOf ?? ((item) => item.id);
|
|
6232
|
-
const seen = /* @__PURE__ */ new Set();
|
|
6233
|
-
const partition = {
|
|
6234
|
-
train: [],
|
|
6235
|
-
selection: [],
|
|
6236
|
-
test: [],
|
|
6237
|
-
seed,
|
|
6238
|
-
selectionFraction,
|
|
6239
|
-
testFraction
|
|
6240
|
-
};
|
|
6241
|
-
for (const item of items) {
|
|
6242
|
-
if (typeof item.id !== "string" || item.id.length === 0) throw new ValidationError(`partitionTrainSelectionTest: ids must be non-empty strings, got ${JSON.stringify(item.id)}`);
|
|
6243
|
-
if (seen.has(item.id)) throw new ValidationError(`partitionTrainSelectionTest: duplicate id "${item.id}" — one item cannot be scored in two partitions`);
|
|
6244
|
-
seen.add(item.id);
|
|
6245
|
-
const unit = unitOf(item);
|
|
6246
|
-
if (typeof unit !== "string" || unit.length === 0) throw new ValidationError(`partitionTrainSelectionTest: item "${item.id}" has no unit`);
|
|
6247
|
-
const position = hashToUnit(unit, seed);
|
|
6248
|
-
if (position < testFraction) partition.test.push(item);
|
|
6249
|
-
else if (position < testFraction + selectionFraction) partition.selection.push(item);
|
|
6250
|
-
else partition.train.push(item);
|
|
6251
|
-
}
|
|
6252
|
-
const floors = [
|
|
6253
|
-
["train", options.minTrain ?? 1],
|
|
6254
|
-
["selection", options.minSelection ?? 1],
|
|
6255
|
-
["test", options.minTest ?? 1]
|
|
6256
|
-
];
|
|
6257
|
-
for (const [name, floor] of floors) if (partition[name].length < floor) throw new ValidationError(`partitionTrainSelectionTest: ${name} has ${partition[name].length} item(s), below the floor of ${floor} (n=${items.length}, selectionFraction=${selectionFraction}, testFraction=${testFraction}, seed=${seed})`);
|
|
6258
|
-
return partition;
|
|
6259
|
-
}
|
|
6260
|
-
function assertSplitFraction(name, value) {
|
|
6261
|
-
if (!Number.isFinite(value) || value <= 0 || value >= 1) throw new ValidationError(`partitionTrainSelectionTest: ${name} must be in (0, 1), got ${value}`);
|
|
6262
|
-
}
|
|
6263
5406
|
//#endregion
|
|
6264
5407
|
//#region src/product-benchmark/export.ts
|
|
6265
5408
|
/**
|
|
@@ -7224,9 +6367,6 @@ function parseRuntimeTrajectoryHookEvent(input) {
|
|
|
7224
6367
|
function runtimeTrajectoryRecordKey(record, index, recordIdOf) {
|
|
7225
6368
|
return stringOrUndefined(recordIdOf?.(record, index)) ?? stringOrUndefined(record.id) ?? `record[${index}]`;
|
|
7226
6369
|
}
|
|
7227
|
-
function isRecord(value) {
|
|
7228
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
7229
|
-
}
|
|
7230
6370
|
function stringOrUndefined(value) {
|
|
7231
6371
|
return typeof value === "string" && value.length > 0 ? value : void 0;
|
|
7232
6372
|
}
|
|
@@ -7475,6 +6615,152 @@ function wranglerDeployRunner(input) {
|
|
|
7475
6615
|
} };
|
|
7476
6616
|
}
|
|
7477
6617
|
//#endregion
|
|
6618
|
+
//#region src/held-out-gate-inputs.ts
|
|
6619
|
+
/** Drop gamed runs, pair both splits and measure coverage, scores, gaps and cost. */
|
|
6620
|
+
function collectGateInputs(candidate, baseline, baselineKey) {
|
|
6621
|
+
const candidateId = inferCandidateId(candidate, baselineKey);
|
|
6622
|
+
const baselineId = baselineKey;
|
|
6623
|
+
assertScenarioIdentities([...candidate, ...baseline]);
|
|
6624
|
+
const realnessGatedRuns = [...candidate, ...baseline].filter(isRealnessGated).length;
|
|
6625
|
+
const honestCandidate = candidate.filter((run) => !isRealnessGated(run));
|
|
6626
|
+
const honestBaseline = baseline.filter((run) => !isRealnessGated(run));
|
|
6627
|
+
const candidateSearch = scoredRuns(honestCandidate, "search");
|
|
6628
|
+
const baselineSearch = scoredRuns(honestBaseline, "search");
|
|
6629
|
+
const candidateHoldout = scoredRuns(honestCandidate, "holdout");
|
|
6630
|
+
const baselineHoldout = scoredRuns(honestBaseline, "holdout");
|
|
6631
|
+
const searchPairing = pairRunRecords(baselineSearch, candidateSearch);
|
|
6632
|
+
const holdoutPairing = pairRunRecords(baselineHoldout, candidateHoldout);
|
|
6633
|
+
const searchCoverage = splitCoverage(honestBaseline, honestCandidate, "search");
|
|
6634
|
+
const holdoutCoverage = splitCoverage(honestBaseline, honestCandidate, "holdout");
|
|
6635
|
+
const splitScoreOf = (run, split) => observedSplitScore(run, split);
|
|
6636
|
+
const beforeSearch = searchPairing.pairs.map((pair) => splitScoreOf(pair.baseline, "search"));
|
|
6637
|
+
const afterSearch = searchPairing.pairs.map((pair) => splitScoreOf(pair.treatment, "search"));
|
|
6638
|
+
const beforeHoldout = holdoutPairing.pairs.map((pair) => splitScoreOf(pair.baseline, "holdout"));
|
|
6639
|
+
const afterHoldout = holdoutPairing.pairs.map((pair) => splitScoreOf(pair.treatment, "holdout"));
|
|
6640
|
+
const productiveRuns = beforeHoldout.length;
|
|
6641
|
+
const candidateSearchMean = meanOrNull(afterSearch);
|
|
6642
|
+
const candidateHoldoutMean = meanOrNull(afterHoldout);
|
|
6643
|
+
const baselineSearchMean = meanOrNull(beforeSearch);
|
|
6644
|
+
const baselineHoldoutMean = meanOrNull(beforeHoldout);
|
|
6645
|
+
const overfitGap = diffOrNull(candidateSearchMean, candidateHoldoutMean);
|
|
6646
|
+
const baselineOverfitGap = diffOrNull(baselineSearchMean, baselineHoldoutMean);
|
|
6647
|
+
const medianCandidateCost = completeCostMedian([...searchPairing.pairs.map((pair) => pair.treatment), ...holdoutPairing.pairs.map((pair) => pair.treatment)]);
|
|
6648
|
+
const medianBaselineCost = completeCostMedian([...searchPairing.pairs.map((pair) => pair.baseline), ...holdoutPairing.pairs.map((pair) => pair.baseline)]);
|
|
6649
|
+
return {
|
|
6650
|
+
candidateId,
|
|
6651
|
+
baselineId,
|
|
6652
|
+
beforeHoldout,
|
|
6653
|
+
afterHoldout,
|
|
6654
|
+
productiveRuns,
|
|
6655
|
+
overfitGap,
|
|
6656
|
+
baselineOverfitGap,
|
|
6657
|
+
medianCandidateCost,
|
|
6658
|
+
medianBaselineCost,
|
|
6659
|
+
searchCoverage,
|
|
6660
|
+
holdoutCoverage,
|
|
6661
|
+
missingSplitScores: [
|
|
6662
|
+
candidateSearch.length === 0 ? "candidate search" : null,
|
|
6663
|
+
candidateHoldout.length === 0 ? "candidate holdout" : null,
|
|
6664
|
+
baselineSearch.length === 0 ? "baseline search" : null,
|
|
6665
|
+
baselineHoldout.length === 0 ? "baseline holdout" : null
|
|
6666
|
+
].filter((label) => label !== null),
|
|
6667
|
+
commonEvidence: {
|
|
6668
|
+
productiveRuns,
|
|
6669
|
+
unpairedCandidateRuns: holdoutPairing.unpairedTreatment.length,
|
|
6670
|
+
unpairedBaselineRuns: holdoutPairing.unpairedBaseline.length,
|
|
6671
|
+
searchScore: candidateSearchMean,
|
|
6672
|
+
holdoutScore: candidateHoldoutMean,
|
|
6673
|
+
overfitGap,
|
|
6674
|
+
baselineOverfitGap,
|
|
6675
|
+
medianCandidateCost,
|
|
6676
|
+
medianBaselineCost,
|
|
6677
|
+
realnessGatedRuns,
|
|
6678
|
+
searchCoverage,
|
|
6679
|
+
holdoutCoverage
|
|
6680
|
+
}
|
|
6681
|
+
};
|
|
6682
|
+
}
|
|
6683
|
+
function inferCandidateId(candidate, baselineKey) {
|
|
6684
|
+
for (const run of candidate) if (run.candidateId && run.candidateId !== baselineKey) return run.candidateId;
|
|
6685
|
+
return candidate[0]?.candidateId ?? "(unknown candidate)";
|
|
6686
|
+
}
|
|
6687
|
+
function assertScenarioIdentities(runs) {
|
|
6688
|
+
for (const run of runs) if (typeof run.scenarioId !== "string" || run.scenarioId.trim() === "") throw new Error(`HeldOutGate: run ${run.runId} is missing scenarioId`);
|
|
6689
|
+
}
|
|
6690
|
+
/**
|
|
6691
|
+
* Rows carrying a finite score on one split. RAW (`observedSplitScore`) by
|
|
6692
|
+
* choice: the gated runs are already removed by the caller, so applying the
|
|
6693
|
+
* gate again here would only zero runs that are no longer in the set, and the
|
|
6694
|
+
* reported means must describe the runs that actually decided the promotion.
|
|
6695
|
+
*/
|
|
6696
|
+
function scoredRuns(runs, split) {
|
|
6697
|
+
return runs.filter((run) => {
|
|
6698
|
+
if (run.splitTag !== split) return false;
|
|
6699
|
+
const v = observedSplitScore(run, split);
|
|
6700
|
+
return typeof v === "number" && Number.isFinite(v);
|
|
6701
|
+
});
|
|
6702
|
+
}
|
|
6703
|
+
/** Every row tagged to one split, scored or not — the work the split was
|
|
6704
|
+
* DEALT, as opposed to the work it managed to measure. */
|
|
6705
|
+
function dealtRuns(runs, split) {
|
|
6706
|
+
return runs.filter((run) => run.splitTag === split);
|
|
6707
|
+
}
|
|
6708
|
+
function hasFiniteScore(run, split) {
|
|
6709
|
+
const v = observedSplitScore(run, split);
|
|
6710
|
+
return typeof v === "number" && Number.isFinite(v);
|
|
6711
|
+
}
|
|
6712
|
+
/**
|
|
6713
|
+
* Partition one split's dealt work into answered / unscored / one-armed.
|
|
6714
|
+
*
|
|
6715
|
+
* Pairs the UNFILTERED rows with the same `pairRunRecords` the decision uses,
|
|
6716
|
+
* so the denominator comes out of the same identity rules as the numerator and
|
|
6717
|
+
* cannot drift from it. Deliberately a SECOND pairing rather than a widening of
|
|
6718
|
+
* the decision's own: the decision keeps pairing exactly the rows it always
|
|
6719
|
+
* paired, so this check can only ever add a refusal, never move a verdict that
|
|
6720
|
+
* was already being made on a complete set.
|
|
6721
|
+
*/
|
|
6722
|
+
function splitCoverage(baseline, candidate, split) {
|
|
6723
|
+
const pairing = pairRunRecords(dealtRuns(baseline, split), dealtRuns(candidate, split));
|
|
6724
|
+
let answered = 0;
|
|
6725
|
+
for (const pair of pairing.pairs) if (hasFiniteScore(pair.baseline, split) && hasFiniteScore(pair.treatment, split)) answered += 1;
|
|
6726
|
+
const unscoredPairs = pairing.pairs.length - answered;
|
|
6727
|
+
const candidateOnly = pairing.unpairedTreatment.length;
|
|
6728
|
+
const baselineOnly = pairing.unpairedBaseline.length;
|
|
6729
|
+
const dealt = pairing.pairs.length + candidateOnly + baselineOnly;
|
|
6730
|
+
return {
|
|
6731
|
+
dealt,
|
|
6732
|
+
answered,
|
|
6733
|
+
unscoredPairs,
|
|
6734
|
+
candidateOnly,
|
|
6735
|
+
baselineOnly,
|
|
6736
|
+
coverage: dealt === 0 ? 0 : answered / dealt
|
|
6737
|
+
};
|
|
6738
|
+
}
|
|
6739
|
+
function meanOrNull(xs) {
|
|
6740
|
+
if (xs.length === 0) return null;
|
|
6741
|
+
return xs.reduce((s, x) => s + x, 0) / xs.length;
|
|
6742
|
+
}
|
|
6743
|
+
function diffOrNull(a, b) {
|
|
6744
|
+
if (a === null || b === null) return null;
|
|
6745
|
+
return a - b;
|
|
6746
|
+
}
|
|
6747
|
+
function medianFinite(xs) {
|
|
6748
|
+
const ys = xs.filter((x) => Number.isFinite(x)).sort((x, y) => x - y);
|
|
6749
|
+
if (ys.length === 0) return null;
|
|
6750
|
+
const mid = Math.floor(ys.length / 2);
|
|
6751
|
+
return ys.length % 2 === 0 ? (ys[mid - 1] + ys[mid]) / 2 : ys[mid];
|
|
6752
|
+
}
|
|
6753
|
+
function completeCostMedian(runs) {
|
|
6754
|
+
if (runs.length === 0) return null;
|
|
6755
|
+
const costs = [];
|
|
6756
|
+
for (const run of runs) {
|
|
6757
|
+
const usd = run.costProvenance.usd;
|
|
6758
|
+
if (usd === null) return null;
|
|
6759
|
+
costs.push(usd);
|
|
6760
|
+
}
|
|
6761
|
+
return medianFinite(costs);
|
|
6762
|
+
}
|
|
6763
|
+
//#endregion
|
|
7478
6764
|
//#region src/held-out-gate.ts
|
|
7479
6765
|
/**
|
|
7480
6766
|
* HeldOutGate — first-class held-out paired-delta promotion gate.
|
|
@@ -7634,55 +6920,35 @@ var HeldOutGate = class {
|
|
|
7634
6920
|
* Pairing is by `(experimentId, scenarioId, seed)`.
|
|
7635
6921
|
* Missing or duplicate identities throw instead of comparing by position. */
|
|
7636
6922
|
evaluate(candidate, baseline) {
|
|
7637
|
-
const
|
|
7638
|
-
const
|
|
7639
|
-
|
|
7640
|
-
const
|
|
7641
|
-
|
|
7642
|
-
const
|
|
7643
|
-
const
|
|
7644
|
-
|
|
7645
|
-
const
|
|
7646
|
-
|
|
7647
|
-
|
|
7648
|
-
|
|
7649
|
-
|
|
7650
|
-
|
|
7651
|
-
|
|
7652
|
-
|
|
7653
|
-
const afterSearch = searchPairing.pairs.map((pair) => splitScoreOf(pair.treatment, "search"));
|
|
7654
|
-
const beforeHoldout = holdoutPairing.pairs.map((pair) => splitScoreOf(pair.baseline, "holdout"));
|
|
7655
|
-
const afterHoldout = holdoutPairing.pairs.map((pair) => splitScoreOf(pair.treatment, "holdout"));
|
|
7656
|
-
const productiveRuns = beforeHoldout.length;
|
|
7657
|
-
const candidateSearchMean = meanOrNull(afterSearch);
|
|
7658
|
-
const candidateHoldoutMean = meanOrNull(afterHoldout);
|
|
7659
|
-
const baselineSearchMean = meanOrNull(beforeSearch);
|
|
7660
|
-
const baselineHoldoutMean = meanOrNull(beforeHoldout);
|
|
7661
|
-
const overfitGap = diffOrNull(candidateSearchMean, candidateHoldoutMean);
|
|
7662
|
-
const baselineOverfitGap = diffOrNull(baselineSearchMean, baselineHoldoutMean);
|
|
7663
|
-
const medianCandidateCost = completeCostMedian([...searchPairing.pairs.map((pair) => pair.treatment), ...holdoutPairing.pairs.map((pair) => pair.treatment)]);
|
|
7664
|
-
const medianBaselineCost = completeCostMedian([...searchPairing.pairs.map((pair) => pair.baseline), ...holdoutPairing.pairs.map((pair) => pair.baseline)]);
|
|
7665
|
-
const commonEvidence = {
|
|
7666
|
-
productiveRuns,
|
|
7667
|
-
unpairedCandidateRuns: holdoutPairing.unpairedTreatment.length,
|
|
7668
|
-
unpairedBaselineRuns: holdoutPairing.unpairedBaseline.length,
|
|
7669
|
-
searchScore: candidateSearchMean,
|
|
7670
|
-
holdoutScore: candidateHoldoutMean,
|
|
7671
|
-
overfitGap,
|
|
7672
|
-
baselineOverfitGap,
|
|
7673
|
-
medianCandidateCost,
|
|
7674
|
-
medianBaselineCost,
|
|
7675
|
-
realnessGatedRuns,
|
|
7676
|
-
searchCoverage,
|
|
7677
|
-
holdoutCoverage
|
|
6923
|
+
const inputs = collectGateInputs(candidate, baseline, this.baselineKey);
|
|
6924
|
+
const missingEvidence = this.refuseOnMissingEvidence(inputs);
|
|
6925
|
+
if (missingEvidence) return missingEvidence;
|
|
6926
|
+
const { overfitGap, baselineOverfitGap } = inputs;
|
|
6927
|
+
if (overfitGap === null || baselineOverfitGap === null) throw new Error("HeldOutGate: complete split scores did not produce overfit gaps");
|
|
6928
|
+
const { decision, evidence } = this.decidePairedDelta(inputs);
|
|
6929
|
+
const refusal = this.refuseOnDelta(inputs, decision, evidence) ?? this.refuseOnOverfitOrCost(inputs, evidence, overfitGap, baselineOverfitGap);
|
|
6930
|
+
if (refusal) return refusal;
|
|
6931
|
+
const { candidateId, baselineId, productiveRuns, medianCandidateCost, medianBaselineCost } = inputs;
|
|
6932
|
+
return {
|
|
6933
|
+
promote: true,
|
|
6934
|
+
candidateId,
|
|
6935
|
+
baselineId,
|
|
6936
|
+
evidence,
|
|
6937
|
+
reason: `promote: paired holdout ${decision.label} Δ=${fmt(decision.delta)} CI=[${fmt(decision.low)}, ${fmt(decision.high)}] over ${productiveRuns} pairs; overfit gap candidate=${fmt(overfitGap)} vs baseline=${fmt(baselineOverfitGap)}; median cost candidate=$${fmt(medianCandidateCost)} vs baseline=$${fmt(medianBaselineCost)}`,
|
|
6938
|
+
rejectionCode: null
|
|
7678
6939
|
};
|
|
6940
|
+
}
|
|
6941
|
+
/** Refusals decided before any interval is computed: absent split scores,
|
|
6942
|
+
* incomplete coverage, too few paired runs. */
|
|
6943
|
+
refuseOnMissingEvidence(inputs) {
|
|
6944
|
+
const { productiveRuns, searchCoverage, holdoutCoverage } = inputs;
|
|
7679
6945
|
/** Evidence for the early exits, where no CI was computed at all. The
|
|
7680
6946
|
* statistic that WOULD have decided is still reported so a caller can see
|
|
7681
6947
|
* which branch their data lands in without producing a verdict. */
|
|
7682
|
-
const earlyShape = this.chooseShape(beforeHoldout, afterHoldout);
|
|
6948
|
+
const earlyShape = this.chooseShape(inputs.beforeHoldout, inputs.afterHoldout);
|
|
7683
6949
|
const noCiEvidence = {
|
|
7684
|
-
...commonEvidence,
|
|
7685
|
-
medianPairedDelta: productiveRuns > 0 ? medianDelta(beforeHoldout, afterHoldout) : null,
|
|
6950
|
+
...inputs.commonEvidence,
|
|
6951
|
+
medianPairedDelta: productiveRuns > 0 ? medianDelta(inputs.beforeHoldout, inputs.afterHoldout) : null,
|
|
7686
6952
|
deltaStatistic: earlyShape.statistic,
|
|
7687
6953
|
decidingDelta: null,
|
|
7688
6954
|
pairedCI: null,
|
|
@@ -7691,38 +6957,19 @@ var HeldOutGate = class {
|
|
|
7691
6957
|
binaryScale: earlyShape.binaryScale,
|
|
7692
6958
|
tieFraction: earlyShape.tieFraction
|
|
7693
6959
|
};
|
|
7694
|
-
|
|
7695
|
-
candidateSearch.length === 0 ? "candidate search" : null,
|
|
7696
|
-
candidateHoldout.length === 0 ? "candidate holdout" : null,
|
|
7697
|
-
baselineSearch.length === 0 ? "baseline search" : null,
|
|
7698
|
-
baselineHoldout.length === 0 ? "baseline holdout" : null
|
|
7699
|
-
].filter((label) => label !== null);
|
|
7700
|
-
if (missingSplitScores.length > 0) return {
|
|
7701
|
-
promote: false,
|
|
7702
|
-
candidateId,
|
|
7703
|
-
baselineId,
|
|
7704
|
-
evidence: noCiEvidence,
|
|
7705
|
-
reason: `missing_split_scores: ${missingSplitScores.join(", ")} score evidence is absent`,
|
|
7706
|
-
rejectionCode: "missing_split_scores"
|
|
7707
|
-
};
|
|
6960
|
+
if (inputs.missingSplitScores.length > 0) return rejectGate(inputs, noCiEvidence, "missing_split_scores", `missing_split_scores: ${inputs.missingSplitScores.join(", ")} score evidence is absent`);
|
|
7708
6961
|
const shortfalls = [["holdout", holdoutCoverage], ["search", searchCoverage]].filter(([, cov]) => cov.coverage < this.minCoverage);
|
|
7709
|
-
if (shortfalls.length > 0) return {
|
|
7710
|
-
|
|
7711
|
-
|
|
7712
|
-
|
|
7713
|
-
|
|
7714
|
-
|
|
7715
|
-
|
|
7716
|
-
|
|
7717
|
-
|
|
7718
|
-
|
|
7719
|
-
|
|
7720
|
-
baselineId,
|
|
7721
|
-
evidence: noCiEvidence,
|
|
7722
|
-
reason: `few_runs: ${productiveRuns} paired holdout observation(s) < min ${this.minProductiveRuns}`,
|
|
7723
|
-
rejectionCode: "few_runs"
|
|
7724
|
-
};
|
|
7725
|
-
if (overfitGap === null || baselineOverfitGap === null) throw new Error("HeldOutGate: complete split scores did not produce overfit gaps");
|
|
6962
|
+
if (shortfalls.length > 0) return rejectGate(inputs, noCiEvidence, "incomplete_coverage", `incomplete_coverage: ${shortfalls.map(([split, cov]) => describeCoverage(split, cov)).join("; ")} — required ${fmt(this.minCoverage)} of every dealt item scored on both arms. A missing score is missing data, not an absent item: the gate will not decide on the subset that answered`);
|
|
6963
|
+
if (productiveRuns < this.minProductiveRuns) return rejectGate(inputs, noCiEvidence, "few_runs", `few_runs: ${productiveRuns} paired holdout observation(s) < min ${this.minProductiveRuns}`);
|
|
6964
|
+
return null;
|
|
6965
|
+
}
|
|
6966
|
+
/** The paired holdout delta, decided by the one shared rule (see
|
|
6967
|
+
* `paired-promotion-decision.ts`): the estimator comes from the outcome's
|
|
6968
|
+
* shape, McNemar vetoes at a non-negative threshold, and a zero-width
|
|
6969
|
+
* interval is refused. `medianPairedDelta` stays the literal median on
|
|
6970
|
+
* EVERY path so the number that made the old gate blind remains visible. */
|
|
6971
|
+
decidePairedDelta(inputs) {
|
|
6972
|
+
const { beforeHoldout, afterHoldout } = inputs;
|
|
7726
6973
|
const decision = decidePairedPromotion(beforeHoldout, afterHoldout, {
|
|
7727
6974
|
threshold: this.pairedDeltaThreshold,
|
|
7728
6975
|
confidence: this.confidence,
|
|
@@ -7733,81 +6980,46 @@ var HeldOutGate = class {
|
|
|
7733
6980
|
});
|
|
7734
6981
|
const literalMedian = medianDelta(beforeHoldout, afterHoldout);
|
|
7735
6982
|
const wilcoxon = wilcoxonSignedRank(beforeHoldout, afterHoldout);
|
|
7736
|
-
const { delta: decidingDelta, low, high
|
|
7737
|
-
|
|
7738
|
-
|
|
7739
|
-
|
|
7740
|
-
|
|
7741
|
-
|
|
7742
|
-
|
|
7743
|
-
|
|
7744
|
-
|
|
7745
|
-
|
|
7746
|
-
|
|
7747
|
-
|
|
7748
|
-
|
|
7749
|
-
|
|
7750
|
-
|
|
7751
|
-
|
|
7752
|
-
|
|
7753
|
-
candidateId,
|
|
7754
|
-
baselineId,
|
|
7755
|
-
evidence,
|
|
7756
|
-
reason: `indeterminate_delta: ${decision.indeterminateCause}, so the paired holdout CI is [${fmt(low)}, ${fmt(high)}] and carries no direction — it cannot clear threshold ${fmt(this.pairedDeltaThreshold)} on evidence`,
|
|
7757
|
-
rejectionCode: "indeterminate_delta"
|
|
7758
|
-
};
|
|
7759
|
-
if (!decision.sufficient) return {
|
|
7760
|
-
promote: false,
|
|
7761
|
-
candidateId,
|
|
7762
|
-
baselineId,
|
|
7763
|
-
evidence,
|
|
7764
|
-
reason: `few_runs: ${productiveRuns} paired holdout observations < ${decision.minimumPairs} required for ${deltaLabel} estimation`,
|
|
7765
|
-
rejectionCode: "few_runs"
|
|
6983
|
+
const { delta: decidingDelta, low, high } = decision;
|
|
6984
|
+
return {
|
|
6985
|
+
decision,
|
|
6986
|
+
evidence: {
|
|
6987
|
+
...inputs.commonEvidence,
|
|
6988
|
+
medianPairedDelta: literalMedian,
|
|
6989
|
+
deltaStatistic: decision.statistic,
|
|
6990
|
+
decidingDelta,
|
|
6991
|
+
pairedCI: {
|
|
6992
|
+
low,
|
|
6993
|
+
high
|
|
6994
|
+
},
|
|
6995
|
+
pairedPValue: wilcoxon.p,
|
|
6996
|
+
mcnemar: decision.mcnemar,
|
|
6997
|
+
binaryScale: decision.binaryScale,
|
|
6998
|
+
tieFraction: decision.tieFraction
|
|
6999
|
+
}
|
|
7766
7000
|
};
|
|
7001
|
+
}
|
|
7002
|
+
/** Refusals read off the paired delta: a zero-width interval, too few pairs
|
|
7003
|
+
* for the chosen estimator, or an interval that does not clear the threshold. */
|
|
7004
|
+
refuseOnDelta(inputs, decision, evidence) {
|
|
7005
|
+
const { delta: decidingDelta, low, high, label: deltaLabel } = decision;
|
|
7006
|
+
if (decision.indeterminate) return rejectGate(inputs, evidence, "indeterminate_delta", `indeterminate_delta: ${decision.indeterminateCause}, so the paired holdout CI is [${fmt(low)}, ${fmt(high)}] and carries no direction — it cannot clear threshold ${fmt(this.pairedDeltaThreshold)} on evidence`);
|
|
7007
|
+
if (!decision.sufficient) return rejectGate(inputs, evidence, "few_runs", `few_runs: ${inputs.productiveRuns} paired holdout observations < ${decision.minimumPairs} required for ${deltaLabel} estimation`);
|
|
7767
7008
|
const exactTestVetoes = decision.exactTestVetoes;
|
|
7768
7009
|
if (!decision.clearsThreshold || exactTestVetoes) {
|
|
7769
7010
|
const detail = exactTestVetoes ? ` McNemar exact p=${decision.mcnemar?.pValue.toExponential(2)} does not reject at α=${fmt(1 - this.confidence)}.` : decision.methodDetail;
|
|
7770
|
-
return {
|
|
7771
|
-
promote: false,
|
|
7772
|
-
candidateId,
|
|
7773
|
-
baselineId,
|
|
7774
|
-
evidence,
|
|
7775
|
-
reason: `negative_delta: paired holdout ${deltaLabel} Δ=${fmt(decidingDelta)} CI=[${fmt(low)}, ${fmt(high)}] does not clear threshold ${fmt(this.pairedDeltaThreshold)}.${detail}`,
|
|
7776
|
-
rejectionCode: "negative_delta"
|
|
7777
|
-
};
|
|
7011
|
+
return rejectGate(inputs, evidence, "negative_delta", `negative_delta: paired holdout ${deltaLabel} Δ=${fmt(decidingDelta)} CI=[${fmt(low)}, ${fmt(high)}] does not clear threshold ${fmt(this.pairedDeltaThreshold)}.${detail}`);
|
|
7778
7012
|
}
|
|
7779
|
-
|
|
7780
|
-
|
|
7781
|
-
|
|
7782
|
-
|
|
7783
|
-
|
|
7784
|
-
|
|
7785
|
-
|
|
7786
|
-
|
|
7787
|
-
if (this.costPerTaskCeiling !== void 0 && medianCandidateCost
|
|
7788
|
-
|
|
7789
|
-
candidateId,
|
|
7790
|
-
baselineId,
|
|
7791
|
-
evidence,
|
|
7792
|
-
reason: "missing_cost: candidate cost evidence is incomplete",
|
|
7793
|
-
rejectionCode: "missing_cost"
|
|
7794
|
-
};
|
|
7795
|
-
if (this.costPerTaskCeiling !== void 0 && medianCandidateCost !== null && medianCandidateCost > this.costPerTaskCeiling) return {
|
|
7796
|
-
promote: false,
|
|
7797
|
-
candidateId,
|
|
7798
|
-
baselineId,
|
|
7799
|
-
evidence,
|
|
7800
|
-
reason: `cost_ceiling: candidate median cost $${fmt(medianCandidateCost)} exceeds ceiling $${fmt(this.costPerTaskCeiling)} (baseline $${fmt(medianBaselineCost)})`,
|
|
7801
|
-
rejectionCode: "cost_ceiling"
|
|
7802
|
-
};
|
|
7803
|
-
return {
|
|
7804
|
-
promote: true,
|
|
7805
|
-
candidateId,
|
|
7806
|
-
baselineId,
|
|
7807
|
-
evidence,
|
|
7808
|
-
reason: `promote: paired holdout ${deltaLabel} Δ=${fmt(decidingDelta)} CI=[${fmt(low)}, ${fmt(high)}] over ${productiveRuns} pairs; overfit gap candidate=${fmt(overfitGap)} vs baseline=${fmt(baselineOverfitGap)}; median cost candidate=$${fmt(medianCandidateCost)} vs baseline=$${fmt(medianBaselineCost)}`,
|
|
7809
|
-
rejectionCode: null
|
|
7810
|
-
};
|
|
7013
|
+
return null;
|
|
7014
|
+
}
|
|
7015
|
+
/** The overfit-gap and cost-ceiling refusals, run after the quality gates so
|
|
7016
|
+
* a cost-driven rejection never masks a quality failure. */
|
|
7017
|
+
refuseOnOverfitOrCost(inputs, evidence, overfitGap, baselineOverfitGap) {
|
|
7018
|
+
const { medianCandidateCost, medianBaselineCost } = inputs;
|
|
7019
|
+
if (overfitGap > baselineOverfitGap + this.overfitGapThreshold) return rejectGate(inputs, evidence, "overfit_gap", `overfit_gap: candidate gap=${fmt(overfitGap)} exceeds baseline gap=${fmt(baselineOverfitGap)} by more than ${fmt(this.overfitGapThreshold)}`);
|
|
7020
|
+
if (this.costPerTaskCeiling !== void 0 && medianCandidateCost === null) return rejectGate(inputs, evidence, "missing_cost", "missing_cost: candidate cost evidence is incomplete");
|
|
7021
|
+
if (this.costPerTaskCeiling !== void 0 && medianCandidateCost !== null && medianCandidateCost > this.costPerTaskCeiling) return rejectGate(inputs, evidence, "cost_ceiling", `cost_ceiling: candidate median cost $${fmt(medianCandidateCost)} exceeds ceiling $${fmt(this.costPerTaskCeiling)} (baseline $${fmt(medianBaselineCost)})`);
|
|
7022
|
+
return null;
|
|
7811
7023
|
}
|
|
7812
7024
|
/**
|
|
7813
7025
|
* Which paired statistic decides this data, and the shape facts behind it —
|
|
@@ -7819,60 +7031,15 @@ var HeldOutGate = class {
|
|
|
7819
7031
|
return pairedDecisionShape(before, after, this.deltaStatistic === "median" ? "median" : "mean");
|
|
7820
7032
|
}
|
|
7821
7033
|
};
|
|
7822
|
-
|
|
7823
|
-
|
|
7824
|
-
return candidate[0]?.candidateId ?? "(unknown candidate)";
|
|
7825
|
-
}
|
|
7826
|
-
function assertScenarioIdentities(runs) {
|
|
7827
|
-
for (const run of runs) if (typeof run.scenarioId !== "string" || run.scenarioId.trim() === "") throw new Error(`HeldOutGate: run ${run.runId} is missing scenarioId`);
|
|
7828
|
-
}
|
|
7829
|
-
/**
|
|
7830
|
-
* Rows carrying a finite score on one split. RAW (`observedSplitScore`) by
|
|
7831
|
-
* choice: the gated runs are already removed by the caller, so applying the
|
|
7832
|
-
* gate again here would only zero runs that are no longer in the set, and the
|
|
7833
|
-
* reported means must describe the runs that actually decided the promotion.
|
|
7834
|
-
*/
|
|
7835
|
-
function scoredRuns(runs, split) {
|
|
7836
|
-
return runs.filter((run) => {
|
|
7837
|
-
if (run.splitTag !== split) return false;
|
|
7838
|
-
const v = observedSplitScore(run, split);
|
|
7839
|
-
return typeof v === "number" && Number.isFinite(v);
|
|
7840
|
-
});
|
|
7841
|
-
}
|
|
7842
|
-
/** Every row tagged to one split, scored or not — the work the split was
|
|
7843
|
-
* DEALT, as opposed to the work it managed to measure. */
|
|
7844
|
-
function dealtRuns(runs, split) {
|
|
7845
|
-
return runs.filter((run) => run.splitTag === split);
|
|
7846
|
-
}
|
|
7847
|
-
function hasFiniteScore(run, split) {
|
|
7848
|
-
const v = observedSplitScore(run, split);
|
|
7849
|
-
return typeof v === "number" && Number.isFinite(v);
|
|
7850
|
-
}
|
|
7851
|
-
/**
|
|
7852
|
-
* Partition one split's dealt work into answered / unscored / one-armed.
|
|
7853
|
-
*
|
|
7854
|
-
* Pairs the UNFILTERED rows with the same `pairRunRecords` the decision uses,
|
|
7855
|
-
* so the denominator comes out of the same identity rules as the numerator and
|
|
7856
|
-
* cannot drift from it. Deliberately a SECOND pairing rather than a widening of
|
|
7857
|
-
* the decision's own: the decision keeps pairing exactly the rows it always
|
|
7858
|
-
* paired, so this check can only ever add a refusal, never move a verdict that
|
|
7859
|
-
* was already being made on a complete set.
|
|
7860
|
-
*/
|
|
7861
|
-
function splitCoverage(baseline, candidate, split) {
|
|
7862
|
-
const pairing = pairRunRecords(dealtRuns(baseline, split), dealtRuns(candidate, split));
|
|
7863
|
-
let answered = 0;
|
|
7864
|
-
for (const pair of pairing.pairs) if (hasFiniteScore(pair.baseline, split) && hasFiniteScore(pair.treatment, split)) answered += 1;
|
|
7865
|
-
const unscoredPairs = pairing.pairs.length - answered;
|
|
7866
|
-
const candidateOnly = pairing.unpairedTreatment.length;
|
|
7867
|
-
const baselineOnly = pairing.unpairedBaseline.length;
|
|
7868
|
-
const dealt = pairing.pairs.length + candidateOnly + baselineOnly;
|
|
7034
|
+
/** A refusal verdict carrying the evidence gathered so far. */
|
|
7035
|
+
function rejectGate(inputs, evidence, rejectionCode, reason) {
|
|
7869
7036
|
return {
|
|
7870
|
-
|
|
7871
|
-
|
|
7872
|
-
|
|
7873
|
-
|
|
7874
|
-
|
|
7875
|
-
|
|
7037
|
+
promote: false,
|
|
7038
|
+
candidateId: inputs.candidateId,
|
|
7039
|
+
baselineId: inputs.baselineId,
|
|
7040
|
+
evidence,
|
|
7041
|
+
reason,
|
|
7042
|
+
rejectionCode
|
|
7876
7043
|
};
|
|
7877
7044
|
}
|
|
7878
7045
|
function describeCoverage(split, cov) {
|
|
@@ -7883,42 +7050,52 @@ function describeCoverage(split, cov) {
|
|
|
7883
7050
|
].filter((part) => part !== null);
|
|
7884
7051
|
return `${split} scored ${cov.answered} of ${cov.dealt} dealt items (coverage ${fmt(cov.coverage)})` + (dark.length > 0 ? ` — ${dark.join(", ")}` : "");
|
|
7885
7052
|
}
|
|
7886
|
-
function meanOrNull(xs) {
|
|
7887
|
-
if (xs.length === 0) return null;
|
|
7888
|
-
return xs.reduce((s, x) => s + x, 0) / xs.length;
|
|
7889
|
-
}
|
|
7890
|
-
function diffOrNull(a, b) {
|
|
7891
|
-
if (a === null || b === null) return null;
|
|
7892
|
-
return a - b;
|
|
7893
|
-
}
|
|
7894
7053
|
function medianDelta(before, after) {
|
|
7895
7054
|
const ds = before.map((b, i) => after[i] - b).sort((x, y) => x - y);
|
|
7896
7055
|
if (ds.length === 0) throw new Error("HeldOutGate: median delta requires at least one pair");
|
|
7897
7056
|
const mid = Math.floor(ds.length / 2);
|
|
7898
7057
|
return ds.length % 2 === 0 ? (ds[mid - 1] + ds[mid]) / 2 : ds[mid];
|
|
7899
7058
|
}
|
|
7900
|
-
function medianFinite(xs) {
|
|
7901
|
-
const ys = xs.filter((x) => Number.isFinite(x)).sort((x, y) => x - y);
|
|
7902
|
-
if (ys.length === 0) return null;
|
|
7903
|
-
const mid = Math.floor(ys.length / 2);
|
|
7904
|
-
return ys.length % 2 === 0 ? (ys[mid - 1] + ys[mid]) / 2 : ys[mid];
|
|
7905
|
-
}
|
|
7906
|
-
function completeCostMedian(runs) {
|
|
7907
|
-
if (runs.length === 0) return null;
|
|
7908
|
-
const costs = [];
|
|
7909
|
-
for (const run of runs) {
|
|
7910
|
-
const usd = run.costProvenance.usd;
|
|
7911
|
-
if (usd === null) return null;
|
|
7912
|
-
costs.push(usd);
|
|
7913
|
-
}
|
|
7914
|
-
return medianFinite(costs);
|
|
7915
|
-
}
|
|
7916
7059
|
function fmt(x) {
|
|
7917
7060
|
if (x === null) return "n/a";
|
|
7918
7061
|
return x.toFixed(4);
|
|
7919
7062
|
}
|
|
7920
7063
|
//#endregion
|
|
7921
7064
|
//#region src/hidden-criteria-grading.ts
|
|
7065
|
+
/**
|
|
7066
|
+
* Hidden-criteria grading firewall — grade an agent on criteria it never saw.
|
|
7067
|
+
*
|
|
7068
|
+
* A trustworthy benchmark splits every scenario's data by WHERE each field is
|
|
7069
|
+
* allowed to flow, then proves the held-out / judge-only fields never reach the
|
|
7070
|
+
* agent during the run. The coding bench expresses this with four destinations
|
|
7071
|
+
* (prompt / develop-against / held-out suite / rubric); this module lifts the
|
|
7072
|
+
* domain-FREE core out of it so research, legal, tax, content — any domain —
|
|
7073
|
+
* can declare the same routing and get the same firewall enforcement and the
|
|
7074
|
+
* same held-out-weighted composite, plugging in its OWN grader.
|
|
7075
|
+
*
|
|
7076
|
+
* Two reusable pieces, both domain-agnostic:
|
|
7077
|
+
*
|
|
7078
|
+
* 1. FIELD ROUTING BY DESTINATION. A scenario declares each field's
|
|
7079
|
+
* `FieldDestination`; `assertNoHiddenLeak` is a pure checker that throws if
|
|
7080
|
+
* a grading-only or judge-only field's value appears in what reaches the
|
|
7081
|
+
* agent. The domain decides which fields exist and where they go — the
|
|
7082
|
+
* substrate only enforces "hidden stays hidden".
|
|
7083
|
+
*
|
|
7084
|
+
* 2. HIDDEN-CRITERIA GRADING. The domain supplies its own grader
|
|
7085
|
+
* `(artifact, hiddenCriteria) => { passRate, total }` — the coding
|
|
7086
|
+
* node-test executor is ONE such grader a consumer plugs in; the substrate
|
|
7087
|
+
* bakes in NO node/test/TS/exec/regex. `gradeOnHidden` runs that grader
|
|
7088
|
+
* behind the firewall and `blendHeldout` composes its pass rate with a
|
|
7089
|
+
* judge score into the final number the leaderboard ranks on.
|
|
7090
|
+
*
|
|
7091
|
+
* Shape mirrors `treatment-gate`/`authenticity`: pure predicates and pure
|
|
7092
|
+
* composition over already-computed values, fail-loud, with the
|
|
7093
|
+
* "which field / which weight / which grader" decisions left as parameters and
|
|
7094
|
+
* no domain literal anywhere in the module.
|
|
7095
|
+
*
|
|
7096
|
+
* Lives next to `test-graded-scenario` and `partition-held-out` — it is a
|
|
7097
|
+
* scorecard/grading concept that makes sense without a running agent loop.
|
|
7098
|
+
*/
|
|
7922
7099
|
/** The destinations a value must be kept OUT of the agent context for. */
|
|
7923
7100
|
const hiddenDestinations = /* @__PURE__ */ new Set(["grading-only", "judge-only"]);
|
|
7924
7101
|
/** True for the destinations whose values must never reach the agent context. */
|
|
@@ -7926,26 +7103,6 @@ function isHiddenDestination(destination) {
|
|
|
7926
7103
|
return hiddenDestinations.has(destination);
|
|
7927
7104
|
}
|
|
7928
7105
|
/**
|
|
7929
|
-
* Route a domain's named fields by destination into the firewall's input shape.
|
|
7930
|
-
* The `routing` declares each field's destination; the `values` carry each
|
|
7931
|
-
* field's renderable string. A field present in `routing` but missing from
|
|
7932
|
-
* `values` is an authoring error (fail loud) — every routed field must have a
|
|
7933
|
-
* value the firewall can check.
|
|
7934
|
-
*/
|
|
7935
|
-
function routeFields(routing, values) {
|
|
7936
|
-
const out = [];
|
|
7937
|
-
for (const name of Object.keys(routing)) {
|
|
7938
|
-
const value = values[name];
|
|
7939
|
-
if (value === void 0) throw new ValidationError(`routed field "${name}" has a destination but no value — every routed field must carry its value`);
|
|
7940
|
-
out.push({
|
|
7941
|
-
name,
|
|
7942
|
-
value,
|
|
7943
|
-
destination: routing[name]
|
|
7944
|
-
});
|
|
7945
|
-
}
|
|
7946
|
-
return out;
|
|
7947
|
-
}
|
|
7948
|
-
/**
|
|
7949
7106
|
* The FIREWALL. Throws `ValidationError` if any `grading-only`/`judge-only`
|
|
7950
7107
|
* field's value is found inside `agentContext` — the exact text that reaches the
|
|
7951
7108
|
* agent during the run (its prompt, its seeded files concatenated, whatever the
|
|
@@ -7974,34 +7131,6 @@ function assertNoHiddenLeak(fields, agentContext, opts = {}) {
|
|
|
7974
7131
|
}
|
|
7975
7132
|
return fields;
|
|
7976
7133
|
}
|
|
7977
|
-
/** Normalize a grader's raw `{passed, total}` into a `HiddenGradeResult` with a
|
|
7978
|
-
* consistent, fail-loud `passRate` — the canonical "honest zero on no-run"
|
|
7979
|
-
* rule, single-sourced so every domain grader gets it. */
|
|
7980
|
-
function hiddenGrade(passed, total, notes) {
|
|
7981
|
-
const p = Number.isFinite(passed) && passed > 0 ? Math.floor(passed) : 0;
|
|
7982
|
-
const t = Number.isFinite(total) && total > 0 ? Math.floor(total) : 0;
|
|
7983
|
-
const passRate = t > 0 ? Math.min(1, p / t) : 0;
|
|
7984
|
-
return {
|
|
7985
|
-
passed: Math.min(p, t > 0 ? t : p),
|
|
7986
|
-
total: t,
|
|
7987
|
-
passRate,
|
|
7988
|
-
notes
|
|
7989
|
-
};
|
|
7990
|
-
}
|
|
7991
|
-
/**
|
|
7992
|
-
* Run a domain's hidden-criteria grader behind the firewall. Before grading, it
|
|
7993
|
-
* re-asserts the firewall against the agent context the run actually used —
|
|
7994
|
-
* proving (at grading time, on real data) that the hidden criteria never
|
|
7995
|
-
* reached the agent — then invokes the grader and returns its pass rate. A
|
|
7996
|
-
* domain that wants the firewall and the grader wired together in one call uses
|
|
7997
|
-
* this; a domain that already asserted the firewall at dispatch time can call
|
|
7998
|
-
* its grader directly and feed the result to `blendHeldout`.
|
|
7999
|
-
*/
|
|
8000
|
-
async function gradeOnHidden(args) {
|
|
8001
|
-
assertNoHiddenLeak(args.firewall.fields, args.firewall.agentContext, args.firewall.options);
|
|
8002
|
-
const result = await args.grader(args.artifact, args.hiddenCriteria, args.signal);
|
|
8003
|
-
return hiddenGrade(result.passed, result.total, result.notes);
|
|
8004
|
-
}
|
|
8005
7134
|
/** Default blend: 0.7 hidden correctness, 0.3 judge quality. The coding bench's
|
|
8006
7135
|
* long-standing split — execution truth dominates, style refines. */
|
|
8007
7136
|
const defaultBlendWeights = {
|
|
@@ -8032,33 +7161,6 @@ function blendHeldout(heldoutPassRate, judgeScore, weights = defaultBlendWeights
|
|
|
8032
7161
|
const judge = clampUnit(judgeScore);
|
|
8033
7162
|
return w.heldout * heldout + w.judge * judge;
|
|
8034
7163
|
}
|
|
8035
|
-
/**
|
|
8036
|
-
* Wrap a judge's `score` so the `composite` it REPORTS is the held-out-weighted
|
|
8037
|
-
* blend. The judge still scores its quality dimensions (recorded, secondary),
|
|
8038
|
-
* but the composite that downstream selection/scorecard reads becomes
|
|
8039
|
-
* `blendHeldout(heldoutPassRate(artifact), judgeComposite, weights)`. The held-
|
|
8040
|
-
* out pass rate is read off the artifact via `heldoutPassRate` — already
|
|
8041
|
-
* computed before the judge runs — so no second grading pass is needed.
|
|
8042
|
-
*
|
|
8043
|
-
* Generic over the artifact type, inferred from `heldoutPassRate`, so it
|
|
8044
|
-
* composes with both a `campaign` `JudgeConfig.score` and a bare scoring
|
|
8045
|
-
* function. The input is the judge's `{ artifact, ... }` — any extra fields
|
|
8046
|
-
* (`scenario`, `signal`) ride through untouched via the index signature.
|
|
8047
|
-
*/
|
|
8048
|
-
function withHeldoutBlend(score, heldoutPassRate, weights = defaultBlendWeights) {
|
|
8049
|
-
return async (input) => {
|
|
8050
|
-
const base = await score(input);
|
|
8051
|
-
if (base.failed) return base;
|
|
8052
|
-
const rate = clampUnit(heldoutPassRate(input.artifact));
|
|
8053
|
-
const composite = blendHeldout(rate, base.composite, weights);
|
|
8054
|
-
const w = normalizeWeights(weights);
|
|
8055
|
-
return {
|
|
8056
|
-
...base,
|
|
8057
|
-
composite,
|
|
8058
|
-
notes: `composite=${composite.toFixed(3)} (held-out ${(rate * 100).toFixed(0)}% × ${w.heldout.toFixed(2)} + quality ${base.composite.toFixed(3)} × ${w.judge.toFixed(2)})` + (base.notes ? ` — ${base.notes}` : "")
|
|
8059
|
-
};
|
|
8060
|
-
};
|
|
8061
|
-
}
|
|
8062
7164
|
function clampUnit(value) {
|
|
8063
7165
|
if (!Number.isFinite(value)) return 0;
|
|
8064
7166
|
return Math.max(0, Math.min(1, value));
|
|
@@ -8141,12 +7243,12 @@ function assertSingleBackend(agent, judge, opts = {}) {
|
|
|
8141
7243
|
//#endregion
|
|
8142
7244
|
//#region src/intent-match-judge.ts
|
|
8143
7245
|
const INTENT_MATCH_JUDGE_VERSION = "intent-match-judge-v1-2026-04-24";
|
|
8144
|
-
const DEFAULT_MODEL = "claude-sonnet-4-6";
|
|
8145
|
-
const DEFAULT_TIMEOUT = 3e5;
|
|
8146
|
-
const DEFAULT_MAX_TOKENS = 800;
|
|
8147
|
-
const DEFAULT_MAX_SOURCE = 25e3;
|
|
8148
|
-
const DEFAULT_MAX_PER_FILE = 12e3;
|
|
8149
|
-
const DEFAULT_MAX_HTML = 2e4;
|
|
7246
|
+
const DEFAULT_MODEL$1 = "claude-sonnet-4-6";
|
|
7247
|
+
const DEFAULT_TIMEOUT$1 = 3e5;
|
|
7248
|
+
const DEFAULT_MAX_TOKENS$1 = 800;
|
|
7249
|
+
const DEFAULT_MAX_SOURCE$1 = 25e3;
|
|
7250
|
+
const DEFAULT_MAX_PER_FILE$1 = 12e3;
|
|
7251
|
+
const DEFAULT_MAX_HTML$1 = 2e4;
|
|
8150
7252
|
const INTENT_SCHEMA = {
|
|
8151
7253
|
type: "object",
|
|
8152
7254
|
additionalProperties: false,
|
|
@@ -8164,11 +7266,11 @@ const INTENT_SCHEMA = {
|
|
|
8164
7266
|
}
|
|
8165
7267
|
}
|
|
8166
7268
|
};
|
|
8167
|
-
function truncate(body, cap, label) {
|
|
7269
|
+
function truncate$1(body, cap, label) {
|
|
8168
7270
|
if (body.length <= cap) return body;
|
|
8169
7271
|
return `${body.slice(0, cap)}\n… [truncated ${body.length - cap} chars of ${label}]`;
|
|
8170
7272
|
}
|
|
8171
|
-
function buildPrompt(input, opts) {
|
|
7273
|
+
function buildPrompt$1(input, opts) {
|
|
8172
7274
|
const sourceBlob = input.sourceFiles.filter((f) => f.content.length <= opts.maxPerFileChars).map((f) => `--- FILE: ${f.path} ---\n${f.content}`).join("\n\n");
|
|
8173
7275
|
const html = input.servedHtml ?? "";
|
|
8174
7276
|
return `You are evaluating whether an agent built THE RIGHT APP for a user request.
|
|
@@ -8181,8 +7283,8 @@ project entirely?
|
|
|
8181
7283
|
USER REQUEST:
|
|
8182
7284
|
${input.userRequest}
|
|
8183
7285
|
|
|
8184
|
-
${input.artifactLabel ? `ARTIFACT METADATA:\n name: ${input.artifactLabel}\n description: ${input.artifactDescription ?? ""}\n\n` : ""}${html ? `SERVED HTML (what the preview returns):\n${truncate(html, opts.maxHtmlChars, "HTML")}\n\n` : ""}SOURCE FILES (the agent's workdir):
|
|
8185
|
-
${truncate(sourceBlob, opts.maxSourceChars, "source")}
|
|
7286
|
+
${input.artifactLabel ? `ARTIFACT METADATA:\n name: ${input.artifactLabel}\n description: ${input.artifactDescription ?? ""}\n\n` : ""}${html ? `SERVED HTML (what the preview returns):\n${truncate$1(html, opts.maxHtmlChars, "HTML")}\n\n` : ""}SOURCE FILES (the agent's workdir):
|
|
7287
|
+
${truncate$1(sourceBlob, opts.maxSourceChars, "source")}
|
|
8186
7288
|
|
|
8187
7289
|
Score 0–1:
|
|
8188
7290
|
1.0 — unmistakably the right app. Even with bugs, gaps, or missing
|
|
@@ -8214,12 +7316,12 @@ async function runIntentMatchJudge(input, options) {
|
|
|
8214
7316
|
const start = Date.now();
|
|
8215
7317
|
const opts = {
|
|
8216
7318
|
chat: options.chat,
|
|
8217
|
-
model: options.model ?? options.chat.defaultModel ?? DEFAULT_MODEL,
|
|
8218
|
-
timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT,
|
|
8219
|
-
maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS,
|
|
8220
|
-
maxSourceChars: options.maxSourceChars ?? DEFAULT_MAX_SOURCE,
|
|
8221
|
-
maxPerFileChars: options.maxPerFileChars ?? DEFAULT_MAX_PER_FILE,
|
|
8222
|
-
maxHtmlChars: options.maxHtmlChars ?? DEFAULT_MAX_HTML,
|
|
7319
|
+
model: options.model ?? options.chat.defaultModel ?? DEFAULT_MODEL$1,
|
|
7320
|
+
timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT$1,
|
|
7321
|
+
maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS$1,
|
|
7322
|
+
maxSourceChars: options.maxSourceChars ?? DEFAULT_MAX_SOURCE$1,
|
|
7323
|
+
maxPerFileChars: options.maxPerFileChars ?? DEFAULT_MAX_PER_FILE$1,
|
|
7324
|
+
maxHtmlChars: options.maxHtmlChars ?? DEFAULT_MAX_HTML$1,
|
|
8223
7325
|
...options.pricing ? { pricing: options.pricing } : {},
|
|
8224
7326
|
costLedger: options.costLedger ?? new CostLedger(),
|
|
8225
7327
|
costPhase: options.costPhase ?? "judge.intent-match",
|
|
@@ -8245,7 +7347,7 @@ async function runIntentMatchJudge(input, options) {
|
|
|
8245
7347
|
content: "You are a holistic code reviewer answering one question: did the agent build the right app for the user. Return strict JSON. No prose outside."
|
|
8246
7348
|
}, {
|
|
8247
7349
|
role: "user",
|
|
8248
|
-
content: buildPrompt(input, opts)
|
|
7350
|
+
content: buildPrompt$1(input, opts)
|
|
8249
7351
|
}],
|
|
8250
7352
|
jsonSchema: {
|
|
8251
7353
|
name: "intent_match_judge",
|
|
@@ -8920,38 +8022,6 @@ function mergeSignals(a, b) {
|
|
|
8920
8022
|
* `'test'`-strategy certification: an oracle set IS a small answer key,
|
|
8921
8023
|
* and certifies nothing outside its own coverage.
|
|
8922
8024
|
*/
|
|
8923
|
-
function textInSnapshot(needle, opts = {}) {
|
|
8924
|
-
const id = `text-in-snapshot(${needle})`;
|
|
8925
|
-
return {
|
|
8926
|
-
id,
|
|
8927
|
-
check(obs) {
|
|
8928
|
-
const hay = obs.text ?? "";
|
|
8929
|
-
const found = opts.caseSensitive ? hay.includes(needle) : hay.toLowerCase().includes(needle.toLowerCase());
|
|
8930
|
-
return {
|
|
8931
|
-
id,
|
|
8932
|
-
pass: found,
|
|
8933
|
-
detail: found ? `"${needle}" found` : `"${needle}" not present in observation`,
|
|
8934
|
-
evidence: found ? excerpt(hay, needle, opts.caseSensitive) : void 0
|
|
8935
|
-
};
|
|
8936
|
-
}
|
|
8937
|
-
};
|
|
8938
|
-
}
|
|
8939
|
-
function urlContains(fragment) {
|
|
8940
|
-
const id = `url-contains(${fragment})`;
|
|
8941
|
-
return {
|
|
8942
|
-
id,
|
|
8943
|
-
check(obs) {
|
|
8944
|
-
const url = obs.url ?? "";
|
|
8945
|
-
const pass = url.toLowerCase().includes(fragment.toLowerCase());
|
|
8946
|
-
return {
|
|
8947
|
-
id,
|
|
8948
|
-
pass,
|
|
8949
|
-
detail: pass ? `url ok (${url})` : `url "${url}" missing "${fragment}"`,
|
|
8950
|
-
evidence: url
|
|
8951
|
-
};
|
|
8952
|
-
}
|
|
8953
|
-
};
|
|
8954
|
-
}
|
|
8955
8025
|
function jsonShape(expected) {
|
|
8956
8026
|
const id = `json-shape(${Object.keys(expected).join(",")})`;
|
|
8957
8027
|
return {
|
|
@@ -8991,77 +8061,6 @@ function jsonShape(expected) {
|
|
|
8991
8061
|
}
|
|
8992
8062
|
};
|
|
8993
8063
|
}
|
|
8994
|
-
function regexMatches(pattern) {
|
|
8995
|
-
const id = `regex(${pattern.source})`;
|
|
8996
|
-
return {
|
|
8997
|
-
id,
|
|
8998
|
-
check(obs) {
|
|
8999
|
-
const m = (obs.text ?? "").match(pattern);
|
|
9000
|
-
return {
|
|
9001
|
-
id,
|
|
9002
|
-
pass: m !== null,
|
|
9003
|
-
detail: m ? `matched "${m[0]}"` : `pattern ${pattern.source} not matched`,
|
|
9004
|
-
evidence: m?.[0]
|
|
9005
|
-
};
|
|
9006
|
-
}
|
|
9007
|
-
};
|
|
9008
|
-
}
|
|
9009
|
-
/**
|
|
9010
|
-
* Anti-bot detector — distinguishes genuine failures from blocked navigation
|
|
9011
|
-
* (cloudflare, recaptcha, etc). Returns an Oracle that PASSES when no block
|
|
9012
|
-
* marker is present; on block, detail names the blocker so runners can tag
|
|
9013
|
-
* results as "blocked" rather than "failed". Lifted from browser-agent-driver.
|
|
9014
|
-
*/
|
|
9015
|
-
function notBlocked() {
|
|
9016
|
-
const id = "not-blocked";
|
|
9017
|
-
const markers = [
|
|
9018
|
-
{
|
|
9019
|
-
name: "cloudflare",
|
|
9020
|
-
re: /just a moment|verifying you are human|cf-chl-|cloudflare/i
|
|
9021
|
-
},
|
|
9022
|
-
{
|
|
9023
|
-
name: "recaptcha",
|
|
9024
|
-
re: /recaptcha|i'?m not a robot|challenge.?form/i
|
|
9025
|
-
},
|
|
9026
|
-
{
|
|
9027
|
-
name: "hcaptcha",
|
|
9028
|
-
re: /hcaptcha/i
|
|
9029
|
-
},
|
|
9030
|
-
{
|
|
9031
|
-
name: "akamai",
|
|
9032
|
-
re: /akamai|pragma: no-cache/i
|
|
9033
|
-
},
|
|
9034
|
-
{
|
|
9035
|
-
name: "perimeterx",
|
|
9036
|
-
re: /perimeterx|px-captcha/i
|
|
9037
|
-
},
|
|
9038
|
-
{
|
|
9039
|
-
name: "rate-limit",
|
|
9040
|
-
re: /rate.?limit|429 too many requests/i
|
|
9041
|
-
},
|
|
9042
|
-
{
|
|
9043
|
-
name: "access-denied",
|
|
9044
|
-
re: /access denied|403 forbidden/i
|
|
9045
|
-
}
|
|
9046
|
-
];
|
|
9047
|
-
return {
|
|
9048
|
-
id,
|
|
9049
|
-
check(obs) {
|
|
9050
|
-
const hay = obs.text ?? "";
|
|
9051
|
-
for (const { name, re } of markers) if (re.test(hay)) return {
|
|
9052
|
-
id,
|
|
9053
|
-
pass: false,
|
|
9054
|
-
detail: `blocked by ${name}`,
|
|
9055
|
-
evidence: (hay.match(re) ?? [])[0]
|
|
9056
|
-
};
|
|
9057
|
-
return {
|
|
9058
|
-
id,
|
|
9059
|
-
pass: true,
|
|
9060
|
-
detail: "no anti-bot block detected"
|
|
9061
|
-
};
|
|
9062
|
-
}
|
|
9063
|
-
};
|
|
9064
|
-
}
|
|
9065
8064
|
/** Run all oracles against one observation and aggregate. */
|
|
9066
8065
|
function evaluateOracles(obs, oracles) {
|
|
9067
8066
|
const results = oracles.map((o) => o.check(obs));
|
|
@@ -9088,19 +8087,305 @@ function evaluateOracles(obs, oracles) {
|
|
|
9088
8087
|
}
|
|
9089
8088
|
};
|
|
9090
8089
|
}
|
|
9091
|
-
function excerpt(hay, needle, caseSensitive = false) {
|
|
9092
|
-
const haySearch = caseSensitive ? hay : hay.toLowerCase();
|
|
9093
|
-
const needleSearch = caseSensitive ? needle : needle.toLowerCase();
|
|
9094
|
-
const idx = haySearch.indexOf(needleSearch);
|
|
9095
|
-
if (idx === -1) return "";
|
|
9096
|
-
const start = Math.max(0, idx - 20);
|
|
9097
|
-
const end = Math.min(hay.length, idx + needle.length + 20);
|
|
9098
|
-
return (start > 0 ? "…" : "") + hay.slice(start, end) + (end < hay.length ? "…" : "");
|
|
9099
|
-
}
|
|
9100
8090
|
function isObject(v) {
|
|
9101
8091
|
return typeof v === "object" && v !== null && !Array.isArray(v);
|
|
9102
8092
|
}
|
|
9103
8093
|
//#endregion
|
|
8094
|
+
//#region src/produced-state.ts
|
|
8095
|
+
function artifactKind(mimeType) {
|
|
8096
|
+
if (!mimeType) return "file";
|
|
8097
|
+
if (mimeType.includes("json")) return "json";
|
|
8098
|
+
if (mimeType.startsWith("text/")) return "text";
|
|
8099
|
+
return "file";
|
|
8100
|
+
}
|
|
8101
|
+
/**
|
|
8102
|
+
* Normalize an emission-ordered stream into its latest observed produced state.
|
|
8103
|
+
* Artifacts are keyed by their exact output path (name, then URI, then id);
|
|
8104
|
+
* proposals by id. Later observations replace earlier ones, including missing
|
|
8105
|
+
* content or a rejected status. An obsolete version cannot prove completion.
|
|
8106
|
+
* Distinct paths remain distinct; no path normalization or aliasing is inferred.
|
|
8107
|
+
* Results retain first-seen identity order. Tool names describe invocation,
|
|
8108
|
+
* not success, and are deduplicated in first-seen order.
|
|
8109
|
+
*
|
|
8110
|
+
* An artifact without observed content yields empty content, which the
|
|
8111
|
+
* completion oracle rejects. This projection does not verify external effects,
|
|
8112
|
+
* recover dropped events, or establish freshness beyond the supplied stream.
|
|
8113
|
+
*/
|
|
8114
|
+
function extractProducedState(events) {
|
|
8115
|
+
const artifacts = /* @__PURE__ */ new Map();
|
|
8116
|
+
const proposals = /* @__PURE__ */ new Map();
|
|
8117
|
+
const toolCalls = [];
|
|
8118
|
+
const seenTools = /* @__PURE__ */ new Set();
|
|
8119
|
+
for (const ev of events) if (ev.type === "tool_call") {
|
|
8120
|
+
const name = ev.toolName;
|
|
8121
|
+
if (name && !seenTools.has(name)) {
|
|
8122
|
+
seenTools.add(name);
|
|
8123
|
+
toolCalls.push(name);
|
|
8124
|
+
}
|
|
8125
|
+
} else if (ev.type === "artifact") {
|
|
8126
|
+
const a = ev;
|
|
8127
|
+
const path = a.name ?? a.uri ?? a.artifactId;
|
|
8128
|
+
artifacts.set(path, {
|
|
8129
|
+
kind: artifactKind(a.mimeType),
|
|
8130
|
+
path,
|
|
8131
|
+
content: a.content ?? ""
|
|
8132
|
+
});
|
|
8133
|
+
} else if (ev.type === "proposal_created") {
|
|
8134
|
+
const p = ev;
|
|
8135
|
+
proposals.set(p.proposalId, {
|
|
8136
|
+
id: p.proposalId,
|
|
8137
|
+
title: p.title,
|
|
8138
|
+
status: p.status ?? "pending",
|
|
8139
|
+
...p.content !== void 0 ? { content: p.content } : {}
|
|
8140
|
+
});
|
|
8141
|
+
}
|
|
8142
|
+
return {
|
|
8143
|
+
artifacts: [...artifacts.values()],
|
|
8144
|
+
proposals: [...proposals.values()],
|
|
8145
|
+
toolCalls
|
|
8146
|
+
};
|
|
8147
|
+
}
|
|
8148
|
+
//#endregion
|
|
8149
|
+
//#region src/semantic-concept-judge.ts
|
|
8150
|
+
const DEFAULT_COMPLEXITY_WEIGHTS = {
|
|
8151
|
+
render: 1,
|
|
8152
|
+
integrate: 2,
|
|
8153
|
+
compute: 2.5
|
|
8154
|
+
};
|
|
8155
|
+
const SEMANTIC_CONCEPT_JUDGE_VERSION = "semantic-concept-judge-v1-2026-04-24";
|
|
8156
|
+
const DEFAULT_MAX_SOURCE = 45e3;
|
|
8157
|
+
const DEFAULT_MAX_HTML = 3e4;
|
|
8158
|
+
const DEFAULT_MAX_PER_FILE = 2e4;
|
|
8159
|
+
const DEFAULT_TIMEOUT = 3e5;
|
|
8160
|
+
const DEFAULT_MAX_TOKENS = 16e3;
|
|
8161
|
+
const DEFAULT_MODEL = "claude-sonnet-4-6";
|
|
8162
|
+
const SEMANTIC_SCHEMA = {
|
|
8163
|
+
type: "object",
|
|
8164
|
+
additionalProperties: false,
|
|
8165
|
+
required: ["summary", "concepts"],
|
|
8166
|
+
properties: {
|
|
8167
|
+
summary: {
|
|
8168
|
+
type: "string",
|
|
8169
|
+
minLength: 20,
|
|
8170
|
+
maxLength: 600
|
|
8171
|
+
},
|
|
8172
|
+
concepts: {
|
|
8173
|
+
type: "array",
|
|
8174
|
+
minItems: 1,
|
|
8175
|
+
items: {
|
|
8176
|
+
type: "object",
|
|
8177
|
+
additionalProperties: false,
|
|
8178
|
+
required: [
|
|
8179
|
+
"concept",
|
|
8180
|
+
"present",
|
|
8181
|
+
"score",
|
|
8182
|
+
"evidence",
|
|
8183
|
+
"severity"
|
|
8184
|
+
],
|
|
8185
|
+
properties: {
|
|
8186
|
+
concept: {
|
|
8187
|
+
type: "string",
|
|
8188
|
+
minLength: 1,
|
|
8189
|
+
maxLength: 120
|
|
8190
|
+
},
|
|
8191
|
+
present: { type: "boolean" },
|
|
8192
|
+
score: {
|
|
8193
|
+
type: "number",
|
|
8194
|
+
minimum: 0,
|
|
8195
|
+
maximum: 10
|
|
8196
|
+
},
|
|
8197
|
+
evidence: {
|
|
8198
|
+
type: "string",
|
|
8199
|
+
minLength: 5,
|
|
8200
|
+
maxLength: 400
|
|
8201
|
+
},
|
|
8202
|
+
severity: {
|
|
8203
|
+
type: "string",
|
|
8204
|
+
enum: [
|
|
8205
|
+
"critical",
|
|
8206
|
+
"major",
|
|
8207
|
+
"minor",
|
|
8208
|
+
"info"
|
|
8209
|
+
]
|
|
8210
|
+
}
|
|
8211
|
+
}
|
|
8212
|
+
}
|
|
8213
|
+
}
|
|
8214
|
+
}
|
|
8215
|
+
};
|
|
8216
|
+
function truncate(body, cap, label) {
|
|
8217
|
+
if (body.length <= cap) return body;
|
|
8218
|
+
return `${body.slice(0, cap)}\n… [truncated ${body.length - cap} chars of ${label}]`;
|
|
8219
|
+
}
|
|
8220
|
+
function buildPrompt(input, opts) {
|
|
8221
|
+
const sourceBlob = input.sourceFiles.filter((f) => f.content.length <= opts.maxPerFileChars).map((f) => `--- FILE: ${f.path} ---\n${f.content}`).join("\n\n");
|
|
8222
|
+
const html = input.servedHtml ?? "";
|
|
8223
|
+
return `You are a strict code-review judge evaluating whether an agent's 0-to-1 build actually implements the features the user asked for.
|
|
8224
|
+
|
|
8225
|
+
You MUST distinguish:
|
|
8226
|
+
(a) WORKING code that implements the concept (rendered UI, wired handler, real API call),
|
|
8227
|
+
(b) KEYWORD-PRESENT stub (comments mentioning the concept, variable names, TODOs),
|
|
8228
|
+
(c) ABSENT (concept nowhere).
|
|
8229
|
+
|
|
8230
|
+
A comment like "// TODO: add mint button" is NOT present — score 2-3. Only count a concept as present if there is real functional code: a rendered component, a call handler wired to state or a network call, a computed value actually used.
|
|
8231
|
+
|
|
8232
|
+
USER REQUEST (what the agent was asked to build):
|
|
8233
|
+
${input.userRequest}
|
|
8234
|
+
|
|
8235
|
+
${input.artifactLabel ? `ARTIFACT METADATA:\n name: ${input.artifactLabel}\n description: ${input.artifactDescription ?? ""}\n\n` : ""}EXPECTED CONCEPTS (each must be graded independently):
|
|
8236
|
+
${input.expectedConcepts.map((c, i) => ` ${i + 1}. "${c.name}"${c.keywords?.length ? ` — hints: [${c.keywords.slice(0, 6).join(" | ")}]` : ""}`).join("\n")}
|
|
8237
|
+
|
|
8238
|
+
${html ? `SERVED HTML (what the preview returns when hit):\n${truncate(html, opts.maxHtmlChars, "HTML")}\n\n` : ""}SOURCE FILES (the agent's workdir):
|
|
8239
|
+
${truncate(sourceBlob, opts.maxSourceChars, "source")}
|
|
8240
|
+
|
|
8241
|
+
For EACH concept, return:
|
|
8242
|
+
- concept: the concept name as given (match exactly)
|
|
8243
|
+
- present: boolean — does a working implementation exist?
|
|
8244
|
+
- score: 0-10 — 10 = production-ready; 7 = functional but thin; 4 = partial/stubbed; 2 = keyword-only comment; 0 = absent
|
|
8245
|
+
- evidence: cite "<file>:<line>" or "served-html:<selector>" pointing at the strongest supporting code. If the concept is absent or stubbed, explain what's missing.
|
|
8246
|
+
- severity:
|
|
8247
|
+
"info" when present: true AND score >= 7
|
|
8248
|
+
"minor" when present: true AND 4 <= score < 7
|
|
8249
|
+
"major" when present: false OR score < 4
|
|
8250
|
+
"critical" when the concept is not only absent but a core user flow depends on it
|
|
8251
|
+
|
|
8252
|
+
Also produce a "summary" (one sentence, 20-600 chars): overall verdict on whether this is a shippable implementation of the user request vs a keyword-dense placeholder.
|
|
8253
|
+
|
|
8254
|
+
BE SKEPTICAL. Keyword matching already passed — your job is to catch what keyword matching misses. If the agent shipped a working build, say so. If it shipped a stub, say so. Don't grade on effort.
|
|
8255
|
+
|
|
8256
|
+
Return STRICT JSON. No prose outside the JSON.`;
|
|
8257
|
+
}
|
|
8258
|
+
/**
|
|
8259
|
+
* Run the semantic concept judge. Soft-fails to available=false on
|
|
8260
|
+
* LLM/JSON errors — callers in a MultiLayerVerifier pipeline can treat
|
|
8261
|
+
* that as "skip" rather than "fail."
|
|
8262
|
+
*/
|
|
8263
|
+
async function runSemanticConceptJudge(input, options) {
|
|
8264
|
+
const start = Date.now();
|
|
8265
|
+
const totalCount = input.expectedConcepts.length;
|
|
8266
|
+
if (totalCount === 0) return {
|
|
8267
|
+
kind: "semantic-concept",
|
|
8268
|
+
version: SEMANTIC_CONCEPT_JUDGE_VERSION,
|
|
8269
|
+
score: 0,
|
|
8270
|
+
presentCount: 0,
|
|
8271
|
+
totalCount: 0,
|
|
8272
|
+
findings: [],
|
|
8273
|
+
summary: "no expected concepts declared",
|
|
8274
|
+
durationMs: 0,
|
|
8275
|
+
costUsd: null,
|
|
8276
|
+
available: false,
|
|
8277
|
+
error: "no expected concepts declared"
|
|
8278
|
+
};
|
|
8279
|
+
const opts = {
|
|
8280
|
+
chat: options.chat,
|
|
8281
|
+
model: options.model ?? options.chat.defaultModel ?? DEFAULT_MODEL,
|
|
8282
|
+
timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT,
|
|
8283
|
+
maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS,
|
|
8284
|
+
maxSourceChars: options.maxSourceChars ?? DEFAULT_MAX_SOURCE,
|
|
8285
|
+
maxPerFileChars: options.maxPerFileChars ?? DEFAULT_MAX_PER_FILE,
|
|
8286
|
+
maxHtmlChars: options.maxHtmlChars ?? DEFAULT_MAX_HTML,
|
|
8287
|
+
...options.pricing ? { pricing: options.pricing } : {},
|
|
8288
|
+
costLedger: options.costLedger ?? new CostLedger(),
|
|
8289
|
+
costPhase: options.costPhase ?? "judge.semantic-concept",
|
|
8290
|
+
costTags: options.costTags ?? {},
|
|
8291
|
+
signal: options.signal ?? new AbortController().signal,
|
|
8292
|
+
weightConcepts: options.weightConcepts ?? "mean",
|
|
8293
|
+
complexityWeights: {
|
|
8294
|
+
...DEFAULT_COMPLEXITY_WEIGHTS,
|
|
8295
|
+
...options.complexityWeights ?? {}
|
|
8296
|
+
}
|
|
8297
|
+
};
|
|
8298
|
+
const weightForConcept = (spec) => {
|
|
8299
|
+
if (opts.weightConcepts === "mean") return 1;
|
|
8300
|
+
if (spec.weight != null) return spec.weight;
|
|
8301
|
+
if (opts.weightConcepts === "complexity") return opts.complexityWeights[spec.complexity ?? "render"] ?? 1;
|
|
8302
|
+
return 1;
|
|
8303
|
+
};
|
|
8304
|
+
const weightByName = new Map(input.expectedConcepts.map((c) => [c.name, weightForConcept(c)]));
|
|
8305
|
+
let receipt;
|
|
8306
|
+
try {
|
|
8307
|
+
const request = {
|
|
8308
|
+
model: opts.model,
|
|
8309
|
+
messages: [{
|
|
8310
|
+
role: "system",
|
|
8311
|
+
content: "You are a strict code-review judge. Return strict JSON only. No prose outside the JSON. A keyword in a comment is NOT a working implementation."
|
|
8312
|
+
}, {
|
|
8313
|
+
role: "user",
|
|
8314
|
+
content: buildPrompt(input, opts)
|
|
8315
|
+
}],
|
|
8316
|
+
jsonSchema: {
|
|
8317
|
+
name: "semantic_concept_judge",
|
|
8318
|
+
schema: SEMANTIC_SCHEMA
|
|
8319
|
+
},
|
|
8320
|
+
temperature: 0,
|
|
8321
|
+
maxTokens: opts.maxTokens,
|
|
8322
|
+
timeoutMs: opts.timeoutMs
|
|
8323
|
+
};
|
|
8324
|
+
const paid = await paidJsonChat({
|
|
8325
|
+
chat: opts.chat,
|
|
8326
|
+
request,
|
|
8327
|
+
ledger: opts.costLedger,
|
|
8328
|
+
channel: "judge",
|
|
8329
|
+
phase: opts.costPhase,
|
|
8330
|
+
actor: "semantic-concept",
|
|
8331
|
+
tags: opts.costTags,
|
|
8332
|
+
signal: opts.signal,
|
|
8333
|
+
...opts.pricing ? { pricing: opts.pricing } : {}
|
|
8334
|
+
});
|
|
8335
|
+
receipt = paid.receipt;
|
|
8336
|
+
if (!paid.succeeded) throw paid.error;
|
|
8337
|
+
const { value } = paid;
|
|
8338
|
+
if (!value?.concepts || !Array.isArray(value.concepts)) throw new Error("judge returned malformed response — expected array under \"concepts\"");
|
|
8339
|
+
const findings = value.concepts.map((c) => ({
|
|
8340
|
+
concept: String(c.concept),
|
|
8341
|
+
present: Boolean(c.present),
|
|
8342
|
+
score: Math.max(0, Math.min(10, Number(c.score ?? 0))),
|
|
8343
|
+
evidence: String(c.evidence ?? ""),
|
|
8344
|
+
severity: [
|
|
8345
|
+
"critical",
|
|
8346
|
+
"major",
|
|
8347
|
+
"minor",
|
|
8348
|
+
"info"
|
|
8349
|
+
].includes(c.severity) ? c.severity : "info"
|
|
8350
|
+
}));
|
|
8351
|
+
const presentCount = findings.filter((f) => f.present && f.score >= 7).length;
|
|
8352
|
+
let weightSum = 0;
|
|
8353
|
+
let weightedScoreSum = 0;
|
|
8354
|
+
for (const f of findings) {
|
|
8355
|
+
const w = weightByName.get(f.concept) ?? 1;
|
|
8356
|
+
weightSum += w;
|
|
8357
|
+
weightedScoreSum += w * f.score;
|
|
8358
|
+
}
|
|
8359
|
+
const scoreAvg = weightSum > 0 ? weightedScoreSum / weightSum : findings.reduce((a, f) => a + f.score, 0) / Math.max(1, findings.length);
|
|
8360
|
+
return {
|
|
8361
|
+
kind: "semantic-concept",
|
|
8362
|
+
version: SEMANTIC_CONCEPT_JUDGE_VERSION,
|
|
8363
|
+
score: Number((scoreAvg / 10).toFixed(3)),
|
|
8364
|
+
presentCount,
|
|
8365
|
+
totalCount,
|
|
8366
|
+
findings,
|
|
8367
|
+
summary: String(value.summary ?? ""),
|
|
8368
|
+
durationMs: Date.now() - start,
|
|
8369
|
+
costUsd: paid.receipt.costUnknown ? null : paid.receipt.costUsd,
|
|
8370
|
+
available: true
|
|
8371
|
+
};
|
|
8372
|
+
} catch (err) {
|
|
8373
|
+
return {
|
|
8374
|
+
kind: "semantic-concept",
|
|
8375
|
+
version: SEMANTIC_CONCEPT_JUDGE_VERSION,
|
|
8376
|
+
score: 0,
|
|
8377
|
+
presentCount: 0,
|
|
8378
|
+
totalCount,
|
|
8379
|
+
findings: [],
|
|
8380
|
+
summary: "",
|
|
8381
|
+
durationMs: Date.now() - start,
|
|
8382
|
+
costUsd: receipt && !receipt.costUnknown ? receipt.costUsd : null,
|
|
8383
|
+
available: false,
|
|
8384
|
+
error: err instanceof Error ? err.message : String(err)
|
|
8385
|
+
};
|
|
8386
|
+
}
|
|
8387
|
+
}
|
|
8388
|
+
//#endregion
|
|
9104
8389
|
//#region src/treatment-gate.ts
|
|
9105
8390
|
/**
|
|
9106
8391
|
* Core predicate. Gated (treatment-not-applied) ONLY when telemetry was
|
|
@@ -9670,6 +8955,6 @@ function rankRows(rows, weights) {
|
|
|
9670
8955
|
})).sort((a, b) => b.mean - a.mean);
|
|
9671
8956
|
}
|
|
9672
8957
|
//#endregion
|
|
9673
|
-
export { AGENT_PROFILE_KINDS, ATTESTATION_ALGORITHM, AgentEvalError, AnalystRegistry, BOOTSTRAP_GATE_MIN_N, BackendIntegrityError, BudgetBreachError, BudgetGuard, CODING_HARNESSES, ConfigError, CostAccountingIncompleteError,
|
|
8958
|
+
export { AGENT_PROFILE_KINDS, ATTESTATION_ALGORITHM, AgentEvalError, AnalystRegistry, BOOTSTRAP_GATE_MIN_N, BackendIntegrityError, BudgetBreachError, BudgetGuard, CODING_HARNESSES, ConfigError, CostAccountingIncompleteError, CostCeilingReachedError, CostLedger, CostTracker, CrossFamilyError, DEFAULT_BOUNDED_PROCESS_TIMEOUT_MS, DEFAULT_JUDGE_GATE_POLICY, DEFAULT_RED_TEAM_CORPUS, DEFAULT_TRACE_ANALYST_BUDGETS, DEFAULT_TRACE_ANALYST_KINDS, ERROR_COUNT_PATTERNS, EquivalenceProtocolError, FAILURE_CLASSES, FAILURE_MODE_KIND_SPEC, FileSystemFeedbackTrajectoryStore, FileSystemRawProviderSink, FileSystemTraceStore, FindingsStore, HARNESS_NATIVE_MODEL, HeldOutGate, InMemoryFeedbackTrajectoryStore, InMemoryRawProviderSink, InMemoryTraceStore, JudgeError, JudgeGateRefusedError, LlmCallError, LlmResponseError, MODEL_PRICING, ModelSubstitutionError, MultiLayerVerifier, NoopRawProviderSink, NotFoundError, OUTPUT_VALUE, OtlpFileTraceStore, PairwiseSteeringOptimizer, ProductClient, PromptRegistry, REDACTION_VERSION, RunIntegrityError, RunRecordValidationError, SEMANTIC_CONCEPT_JUDGE_VERSION, TraceEmitter, UNKNOWN_MODEL, VERIFICATION_STRATEGIES, VERIFICATION_STRATEGY_SOURCES, ValidationError, acquisitionPlansForKnowledgeGaps, addCalibrationVerdicts, agentProfileCellKey, agentProfileHash, agentProfileId, aggregateJudgeVerdicts, aggregateRunScore, analystRunDigest, analystRunToFeedbackTrajectory, analystRunToReviewRequests, analyzeAntiSlop, analyzeRuns, analyzeSeries, analyzeTraces, argHash, assertCapabilityHeadroom, assertCrossFamily, assertCrossFamilyServed, assertJudgeMayGate, assertNoHiddenLeak, assertProductBenchmarkRun, assertRealBackend, assertRunCaptured, assertServedModel, assertServedModels, assertSingleBackend, assessShareSafety, assignFeedbackSplit, attest, benjaminiHochberg, blendHeldout, blockingKnowledgeEval, bonferroni, bootstrapCi, budgetBreachView, buildAgentProfileCell, buildDefaultAnalystRegistry, buildReflectionPrompt, buildTraceInsightContext, buildTraceInsightPrompt, buildTrajectory, calibrateJudge, calibrateJudgeContinuous, canonicalJson, capabilityHeadroom, captureFetchToRawSink, certificationEvidenceDigest, checkCanaries, checkServedModel, checkTraceContracts, chiSquared, chiSquaredPValue, clamp01, classifyFailure, classifyFailureReason, cliffsDelta, cohensD, comparePairedArms, compileTraceContractSpec, completionVerdict, computeFindingId, computeToolUseMetrics, confidenceInterval, contentHash, continuousAgreement, controlRunToFeedbackTrajectory, corpusInterRaterAgreement, corpusInterRaterAgreementFromJudgeScores, costForTokenPricing, costForUsage, costReceiptFromLlm, costReceiptFromLlmError, createAntiSlopJudge, createBoundedTraceAnalysisStore, createChatClient, createChatTraceEngine, createDspyRlmTraceEngine, createFeedbackTrajectory, createLlmCorrectnessChecker, createLlmReviewer, createTraceAnalyst, decidePairedPromotion, defineAgentEval, defineEquivalenceCheck, deployGateLayer, describeTraceInsightScope, diffFindings, diffScorecard, discoverPersonas, domainEvidencePattern, dominates, eProcess, ensembleJudge, equivalenceVerdict, errorStreakDetector, estimateCost, estimateTokens, evaluateActionPolicy, evaluateInterimReleaseConfidence, evaluateOracles, evaluateReleaseConfidence, evaluateTraceContract, expandProfileAxes, explainTraceContract, exportProductBenchmark, exportProductBenchmarkRuns, exportRunAsOtlp, extractErrorCount, extractProducedState, extractUsage, extractUsageFromSse, failureBlame, failureClusterView, feedbackTrajectoriesToDatasetScenarios, feedbackTrajectoriesToOptimizerRows, feedbackTrajectoryToOptimizerRow, fileVerdictCache, fisherExactTwoSided, formatScorecardDiff, gainHistogram, gateTreatmentApplied, gradeSemanticStatus, groupRunsByAgentProfileCell, harnessAxisOf, hashContent, hashJson, holm, inMemoryReviewStore, inferDomainKeywords, interRaterReliability, interpretCliffs, iqr, isJudgeSpan, isLlmSpan, isModelPriced, isRunRecord, isToolSpan, isTransientLlmError, jsonShape, jsonlReviewStore, jsonlRunRecordBackend, judgeAgreementView, judgeFamily, judgeGateDecision, judgeSpans, knowledgeReadinessTracePayload, leaderboard, linearRegressionSlope, lintTraceContractSpec, llmJudge, loadScorecard, localCommandRunner, makeFinding, makePolicyEditCandidateRecord, makeProposalFinding, manifestContentDigest, mannWhitneyU, maximumChargeForLlmRequest, mcnemar, mcnemarPower, mcnemarRequiredN, measureJudgeAgreement, minimumPairsForPairedDeltaTest, mintRolloutRows, modelHasSnapshot, mulberry32, normalTwoSidedPValue, objectiveEval, observeAll, otlpTextToTraceAnalysisStore, pairArms, pairRunRecords, pairedBootstrap, pairedCohensDz, pairedDeltaTest, pairedEvalueSequence, pairedMde, pairedPromotionPower, pairedRiskDifference, pairedRiskDifferenceExact, pairedRiskDifferenceScore, pairedSignTest, pairedTTest, paretoFrontier, parseReflectionResponse, parseRunRecordSafe, partialCredit, partitionHeldOut, passAtK, pearsonR, preflightModels, productBenchmarkRepoIdentity, profile_exports as profile, projectRuntimeTrajectoryEvidence, proposeSynthesisTargets, ranks, readProductBenchmarkManifest, recordRuns, recordRunsToScorecard, redTeamDataset, redTeamReport, redact, redactForShare, redactText, registerCalibrationSet, renderPreferenceMemoryMarkdown, repeatedActionDetector, requiredPairedSampleSize, requiredPairsForPairedPromotion, requiredSampleSize, resolveModelPricing, resolveSeat, runAgentControlLoop, runBoundedProcess, runCampaign, runCanaries, runEquivalenceCheck, runEvalCampaign, runIntentMatchJudge, runKeywordCoverageJudge, runKeywordCoverageJudgeUrl, runProposeReview, runSemanticConceptJudge, runTaskScore, runsForScenario, sampleMedian, scoreKnowledgeReadiness, scoreRedTeamOutput, scoreTraceInsightReadiness, seatPresets, selfImprove, shareAllowed, snapshotFromJudgeAgreement, spanRecordsToTraceAnalysisStore, spearmanR, stripFencedJson, studentTTwoSidedPValue, subjectiveEval, summarizeBackendIntegrity, summarizeNumberSeries, summarizePreferenceMemory, summarizeSample, summaryTable, toAgentProfileJson, tokenizeDomainWords, toolSpansToTraceAnalysisStore, toolWasteView, traceContract, transientDispatchFailure, twoByTwoPValue, userQuestionsForKnowledgeGaps, validateRunRecord, verbosityBias, verifyAgentProfileCell, verifyAttestation, verifyCompletion, viteDeployRunner, weightedComposite, weightedMean, welchsTTest, wilcoxonSignedRank, wilson, withAssignedFeedbackSplit, withJudgeRetry, wranglerDeployRunner };
|
|
9674
8959
|
|
|
9675
8960
|
//# sourceMappingURL=index.js.map
|