@tangle-network/agent-eval 0.216.0 → 0.217.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +141 -57
- package/README.md +2 -2
- package/dist/adapters/http.d.ts +3 -3
- package/dist/{agent-profile-CIA2Atnt.d.ts → agent-profile-DTt8uGP9.d.ts} +5 -18
- package/dist/agent-profile-DTt8uGP9.d.ts.map +1 -0
- package/dist/{agent-profile-cell-BYc0bxsL.js → agent-profile-cell-B1V1gUZs.js} +4 -53
- package/dist/agent-profile-cell-B1V1gUZs.js.map +1 -0
- package/dist/{agent-profile-cell-BNp-Js9F.d.ts → agent-profile-cell-BcpFbNd5.d.ts} +2 -22
- package/dist/agent-profile-cell-BcpFbNd5.d.ts.map +1 -0
- package/dist/{agreement-irr-DsTURxdr.js → agreement-irr-BhAqjluA.js} +5 -5
- package/dist/agreement-irr-BhAqjluA.js.map +1 -0
- package/dist/analyst/index.d.ts +37 -170
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +17 -192
- package/dist/analyst/index.js.map +1 -1
- package/dist/{analyze-runs-3xpby3p2.js → analyze-runs-DqPJaJ4X.js} +763 -555
- package/dist/analyze-runs-DqPJaJ4X.js.map +1 -0
- package/dist/{analyze-runs-CSDAh8A7.d.ts → analyze-runs-DyeNcenW.d.ts} +18 -16
- package/dist/analyze-runs-DyeNcenW.d.ts.map +1 -0
- package/dist/authenticity/index.d.ts +0 -23
- package/dist/authenticity/index.d.ts.map +1 -1
- package/dist/authenticity/index.js +1 -34
- package/dist/authenticity/index.js.map +1 -1
- package/dist/{benchmark-command-CByl9dxG.js → benchmark-command-xQK8nOI6.js} +591 -109
- package/dist/benchmark-command-xQK8nOI6.js.map +1 -0
- package/dist/benchmarks/index.d.ts +5 -6
- package/dist/benchmarks/index.d.ts.map +1 -1
- package/dist/benchmarks/index.js +4 -4
- package/dist/benchmarks/index.js.map +1 -1
- package/dist/builder-eval/index.d.ts +3 -3
- package/dist/builder-eval/index.js +2 -2
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/campaign/index.d.ts +8 -8
- package/dist/campaign/index.js +15 -15
- package/dist/{campaign-BQ-Jujuj.js → campaign-CJGkxAQ6.js} +256 -765
- package/dist/campaign-CJGkxAQ6.js.map +1 -0
- package/dist/{campaign-evidence-OcUciD-N.js → campaign-evidence-CQYw8_0H.js} +10 -18
- package/dist/{campaign-evidence-OcUciD-N.js.map → campaign-evidence-CQYw8_0H.js.map} +1 -1
- package/dist/{canonical-BsZVgBhW.js → canonical-Dq2hOQ8L.js} +2 -2
- package/dist/{canonical-BsZVgBhW.js.map → canonical-Dq2hOQ8L.js.map} +1 -1
- package/dist/cell-spend-DEWR3nj-.js +90 -0
- package/dist/cell-spend-DEWR3nj-.js.map +1 -0
- package/dist/{chat-client-DthPOYGd.js → chat-client-ldqxYYTN.js} +2 -2
- package/dist/{chat-client-DthPOYGd.js.map → chat-client-ldqxYYTN.js.map} +1 -1
- package/dist/{chat-json-call-BrKsyspg.js → chat-json-call-DavzexSp.js} +3 -2
- package/dist/chat-json-call-DavzexSp.js.map +1 -0
- package/dist/{claim-BXcZw-jo.js → claim-BPVuyHuy.js} +4 -4
- package/dist/claim-BPVuyHuy.js.map +1 -0
- package/dist/cli.js +113 -12
- package/dist/cli.js.map +1 -1
- package/dist/{client-BzL9niqi.d.ts → client-hwo3cJjR.d.ts} +3 -16
- package/dist/client-hwo3cJjR.d.ts.map +1 -0
- package/dist/{cluster-bootstrap-B3JywQQn.js → cluster-bootstrap-DEwwhl0C.js} +3 -3
- package/dist/{cluster-bootstrap-B3JywQQn.js.map → cluster-bootstrap-DEwwhl0C.js.map} +1 -1
- package/dist/{produced-state-ZCtDBxAE.js → completion-verifier-B_jw6C9p.js} +9 -64
- package/dist/completion-verifier-B_jw6C9p.js.map +1 -0
- package/dist/contract/index.d.ts +13 -13
- package/dist/contract/index.js +1041 -1473
- package/dist/contract/index.js.map +1 -1
- package/dist/{cost-ledger-D2FnJPBF.js → cost-ledger-D0352Z4d.js} +832 -817
- package/dist/cost-ledger-D0352Z4d.js.map +1 -0
- package/dist/{cost-ledger-CO7UyEcQ.d.ts → cost-ledger-Jk62OBfF.d.ts} +12 -47
- package/dist/cost-ledger-Jk62OBfF.d.ts.map +1 -0
- package/dist/dataset-DQqhOCPt.d.ts.map +1 -1
- package/dist/{default-registry-Cn1LQ2Ve.js → default-registry-ClYjq3QE.js} +8 -11
- package/dist/default-registry-ClYjq3QE.js.map +1 -0
- package/dist/{default-registry-BrWh62zf.d.ts → default-registry-DXgLmoDI.d.ts} +7 -50
- package/dist/default-registry-DXgLmoDI.d.ts.map +1 -0
- package/dist/{define-agent-eval-Ch3MWX6k.js → define-agent-eval-BDxNlP3D.js} +10 -10
- package/dist/define-agent-eval-BDxNlP3D.js.map +1 -0
- package/dist/{define-agent-eval-CEWkX5wZ.d.ts → define-agent-eval-BIkH_gAR.d.ts} +9 -9
- package/dist/define-agent-eval-BIkH_gAR.d.ts.map +1 -0
- package/dist/{descriptive-CnU8Yq29.js → descriptive-CVNy48rR.js} +2 -2
- package/dist/{descriptive-CnU8Yq29.js.map → descriptive-CVNy48rR.js.map} +1 -1
- package/dist/diagnosis/index.d.ts +4 -14
- package/dist/diagnosis/index.d.ts.map +1 -1
- package/dist/diagnosis/index.js +67 -28
- package/dist/diagnosis/index.js.map +1 -1
- package/dist/{dspy-rlm-engine-D-KX_e64.js → dspy-rlm-engine-DhdCr1Dc.js} +5 -10
- package/dist/dspy-rlm-engine-DhdCr1Dc.js.map +1 -0
- package/dist/{emitter-HcCmFQl_.d.ts → emitter-DABYYTcH.d.ts} +5 -5
- package/dist/emitter-DABYYTcH.d.ts.map +1 -0
- package/dist/emitter-DbMXUIVP.js.map +1 -1
- package/dist/{engine-C_zNWfM8.d.ts → engine-BvZ0R-36.d.ts} +18 -22
- package/dist/engine-BvZ0R-36.d.ts.map +1 -0
- package/dist/{equal-terms-Blx9Zk5I.js → equal-terms-DbPGcVtI.js} +2 -2
- package/dist/equal-terms-DbPGcVtI.js.map +1 -0
- package/dist/{errors-Dngq5h35.js → errors-BcOs5MKW.js} +6 -8
- package/dist/{errors-Dngq5h35.js.map → errors-BcOs5MKW.js.map} +1 -1
- package/dist/{eval-campaign-4byTtqrf.js → eval-campaign-BLEfBGbA.js} +6 -6
- package/dist/{eval-campaign-4byTtqrf.js.map → eval-campaign-BLEfBGbA.js.map} +1 -1
- package/dist/{evaluation-Cn2dRZIz.d.ts → evaluation-jTjC_QQD.d.ts} +8 -8
- package/dist/evaluation-jTjC_QQD.d.ts.map +1 -0
- package/dist/evaluation.d.ts +2 -2
- package/dist/evaluation.js +1 -1
- package/dist/evaluation.js.map +1 -1
- package/dist/{exact-types-CD0lf5Qd.d.ts → exact-types-CflbJ9f3.d.ts} +2 -2
- package/dist/{exact-types-CD0lf5Qd.d.ts.map → exact-types-CflbJ9f3.d.ts.map} +1 -1
- package/dist/{exec-ChJ6d5if.js → exec-CpbtOXVt.js} +2 -24
- package/dist/{exec-ChJ6d5if.js.map → exec-CpbtOXVt.js.map} +1 -1
- package/dist/execution-tracks-CpgFPpS5.js.map +1 -1
- package/dist/experiment/index.d.ts +331 -33
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +303 -101
- package/dist/experiment/index.js.map +1 -1
- package/dist/{exporters-X4xwla8A.js → exporters-Cl6IeNAu.js} +3 -3
- package/dist/exporters-Cl6IeNAu.js.map +1 -0
- package/dist/{exporters-ChTnR90l.d.ts → exporters-Dk3oEbK_.d.ts} +2 -4
- package/dist/{exporters-ChTnR90l.d.ts.map → exporters-Dk3oEbK_.d.ts.map} +1 -1
- package/dist/{external-optimizer-process-nnsY5D_4.js → external-optimizer-process-KVFGvYX1.js} +4 -3
- package/dist/external-optimizer-process-KVFGvYX1.js.map +1 -0
- package/dist/{external-optimizer-subprocess-DcerlZSG.js → external-optimizer-subprocess-XIHBOSKt.js} +551 -553
- package/dist/external-optimizer-subprocess-XIHBOSKt.js.map +1 -0
- package/dist/{extract-usage-BrnzRJ6H.js → extract-usage-BZLU01kR.js} +2 -23
- package/dist/{extract-usage-BrnzRJ6H.js.map → extract-usage-BZLU01kR.js.map} +1 -1
- package/dist/{failure-cluster-CbgEDcFJ.d.ts → failure-cluster-DN0t63Z0.d.ts} +4 -43
- package/dist/failure-cluster-DN0t63Z0.d.ts.map +1 -0
- package/dist/{failure-taxonomy-CjabTATj.js → failure-taxonomy-Bx5oWqaW.js} +2 -19
- package/dist/{failure-taxonomy-CjabTATj.js.map → failure-taxonomy-Bx5oWqaW.js.map} +1 -1
- package/dist/{feedback-trajectory-DkaDj0dj.d.ts → feedback-trajectory-BWmhRBGu.d.ts} +8 -13
- package/dist/feedback-trajectory-BWmhRBGu.d.ts.map +1 -0
- package/dist/{semantic-concept-judge-zQBG9hHU.js → findings-store-BMVQtEsm.js} +13 -255
- package/dist/findings-store-BMVQtEsm.js.map +1 -0
- package/dist/fuzz.d.ts +20 -63
- package/dist/fuzz.d.ts.map +1 -1
- package/dist/fuzz.js +62 -108
- package/dist/fuzz.js.map +1 -1
- package/dist/{hf-dataset-BR2yhuNn.js → hf-dataset-D9f7-oX7.js} +4 -24
- package/dist/hf-dataset-D9f7-oX7.js.map +1 -0
- package/dist/hosted/index.d.ts +2 -2
- package/dist/hosted/index.js +2 -2
- package/dist/{index-DHKEAv3s.d.ts → index-9jJ2fKSX.d.ts} +2 -64
- package/dist/index-9jJ2fKSX.d.ts.map +1 -0
- package/dist/{index-oQ4aHziK.d.ts → index-BLGqZ81N.d.ts} +11 -16
- package/dist/{index-oQ4aHziK.d.ts.map → index-BLGqZ81N.d.ts.map} +1 -1
- package/dist/{index-DlH0Y9gI.d.ts → index-BiT6RtNH.d.ts} +31 -43
- package/dist/index-BiT6RtNH.d.ts.map +1 -0
- package/dist/{index-Ch9VxCpK.d.ts → index-CX-GAUpw.d.ts} +4 -8
- package/dist/index-CX-GAUpw.d.ts.map +1 -0
- package/dist/{index-CPJJr98o.d.ts → index-CiRwLv2R.d.ts} +5 -68
- package/dist/index-CiRwLv2R.d.ts.map +1 -0
- package/dist/{index-NvGBiO8H.d.ts → index-ZuWx52WG.d.ts} +266 -495
- package/dist/index-ZuWx52WG.d.ts.map +1 -0
- package/dist/index.d.ts +2256 -2165
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +514 -1192
- package/dist/index.js.map +1 -1
- package/dist/{integrity-BV1ZDf42.js → integrity-DOd9Q86O.js} +7 -12
- package/dist/integrity-DOd9Q86O.js.map +1 -0
- package/dist/{integrity-BvnVARVl.js → integrity-W4CADcFc.js} +3 -7
- package/dist/{integrity-BvnVARVl.js.map → integrity-W4CADcFc.js.map} +1 -1
- package/dist/{integrity-CCAxGVK6.d.ts → integrity-_qgZLA5p.d.ts} +4 -6
- package/dist/{integrity-CCAxGVK6.d.ts.map → integrity-_qgZLA5p.d.ts.map} +1 -1
- package/dist/{internal-CRHwS9iz.js → internal-BTK91Bqt.js} +3 -3
- package/dist/{internal-CRHwS9iz.js.map → internal-BTK91Bqt.js.map} +1 -1
- package/dist/interval-Dq9gt7jx.js +219 -0
- package/dist/interval-Dq9gt7jx.js.map +1 -0
- package/dist/interval-ohuOJurX.d.ts +47 -0
- package/dist/interval-ohuOJurX.d.ts.map +1 -0
- package/dist/{journal-DvKCvjty.js → journal-CR4QikYp.js} +110 -110
- package/dist/journal-CR4QikYp.js.map +1 -0
- package/dist/json-BhdsJaOQ.js +111 -0
- package/dist/json-BhdsJaOQ.js.map +1 -0
- package/dist/{judge-calibration-N44V04Se.js → judge-calibration-BkXMIWA4.js} +3 -3
- package/dist/{judge-calibration-N44V04Se.js.map → judge-calibration-BkXMIWA4.js.map} +1 -1
- package/dist/{judge-gate-B1qYnkpi.js → judge-gate-Co5tuh57.js} +4 -27
- package/dist/judge-gate-Co5tuh57.js.map +1 -0
- package/dist/{judge-gate-DNcUrRFA.d.ts → judge-gate-DsL0yFcb.d.ts} +9 -10
- package/dist/judge-gate-DsL0yFcb.d.ts.map +1 -0
- package/dist/{kind-factory-CbZ-6tvu.js → kind-factory-Gy4fXixd.js} +88 -17
- package/dist/kind-factory-Gy4fXixd.js.map +1 -0
- package/dist/ledger-core/index.d.ts +2 -2
- package/dist/ledger-core/index.js +4 -4
- package/dist/{ledger-core-B2hahzrt.js → ledger-core-DrJhbms9.js} +5 -84
- package/dist/ledger-core-DrJhbms9.js.map +1 -0
- package/dist/{llm-client-DQveoxSM.js → llm-client-BshCREn6.js} +90 -219
- package/dist/llm-client-BshCREn6.js.map +1 -0
- package/dist/{llm-judge-PXV7Xljl.js → llm-judge-C8jMTNNG.js} +501 -931
- package/dist/llm-judge-C8jMTNNG.js.map +1 -0
- package/dist/matrix/index.d.ts +4 -4
- package/dist/matrix/index.js +3 -2
- package/dist/{matrix-_BV9-Lk5.js → matrix-B8l-Nsxd.js} +7 -96
- package/dist/matrix-B8l-Nsxd.js.map +1 -0
- package/dist/{matrix-CHMiebFx.d.ts → matrix-DHe-dwVJ.d.ts} +10 -19
- package/dist/matrix-DHe-dwVJ.d.ts.map +1 -0
- package/dist/{paired-tests-jbwPdSz5.js → mean-BbA_JCzW.js} +45 -5
- package/dist/mean-BbA_JCzW.js.map +1 -0
- package/dist/meta-eval/index.d.ts +6 -6
- package/dist/meta-eval/index.js +10 -10
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/metrics-BKaPBtIG.js.map +1 -1
- package/dist/mint-BRZXuwRi.d.ts +81 -0
- package/dist/mint-BRZXuwRi.d.ts.map +1 -0
- package/dist/{mint-Csor1BLJ.js → mint-zszyYlpA.js} +4 -4
- package/dist/{mint-Csor1BLJ.js.map → mint-zszyYlpA.js.map} +1 -1
- package/dist/{multi-layer-verifier-BUaQ4C17.d.ts → multi-layer-verifier-DEgMUk6v.d.ts} +2 -2
- package/dist/{multi-layer-verifier-BUaQ4C17.d.ts.map → multi-layer-verifier-DEgMUk6v.d.ts.map} +1 -1
- package/dist/{multiplicity-D9cjmStq.js → multiplicity-Sg0U7h8j.js} +2 -2
- package/dist/{multiplicity-D9cjmStq.js.map → multiplicity-Sg0U7h8j.js.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -23
- package/dist/multishot/golden/index.d.ts.map +1 -1
- package/dist/multishot/golden/index.js +5 -8
- package/dist/multishot/golden/index.js.map +1 -1
- package/dist/multishot/index.d.ts +3 -12
- package/dist/multishot/index.d.ts.map +1 -1
- package/dist/multishot/index.js +4 -3
- package/dist/multishot/index.js.map +1 -1
- package/dist/nonnegative-number-DvdMxg_f.js +9 -0
- package/dist/nonnegative-number-DvdMxg_f.js.map +1 -0
- package/dist/openapi.json +114 -10
- package/dist/{otel-spans-C8YIJWy1.js → otel-spans-CdN909k1.js} +3 -3
- package/dist/{otel-spans-C8YIJWy1.js.map → otel-spans-CdN909k1.js.map} +1 -1
- package/dist/{paired-arms-DzKn0TfY.js → paired-arms-Ct4LTKc1.js} +32 -9
- package/dist/paired-arms-Ct4LTKc1.js.map +1 -0
- package/dist/{paired-binary-Q36bhn2R.js → paired-binary-B_nejA44.js} +3 -36
- package/dist/paired-binary-B_nejA44.js.map +1 -0
- package/dist/{paired-promotion-decision-DPKz8ZnF.d.ts → paired-promotion-decision-CQMJ5xGN.d.ts} +2 -34
- package/dist/paired-promotion-decision-CQMJ5xGN.d.ts.map +1 -0
- package/dist/{paired-promotion-decision-BN6IQTiu.js → paired-promotion-decision-D0SHb-ZL.js} +3 -3
- package/dist/{paired-promotion-decision-BN6IQTiu.js.map → paired-promotion-decision-D0SHb-ZL.js.map} +1 -1
- package/dist/{search-kernel-BztlERLv.js → pareto-BzsxidFD.js} +883 -861
- package/dist/pareto-BzsxidFD.js.map +1 -0
- package/dist/pipelines/index.d.ts +16 -19
- package/dist/pipelines/index.d.ts.map +1 -1
- package/dist/pipelines/index.js +4 -68
- package/dist/pipelines/index.js.map +1 -1
- package/dist/{power-and-mde-fK5Qocwo.js → power-and-mde-N5nU4_Ly.js} +3 -3
- package/dist/{power-and-mde-fK5Qocwo.js.map → power-and-mde-N5nU4_Ly.js.map} +1 -1
- package/dist/predicate-FdKmElA_.js +38 -0
- package/dist/predicate-FdKmElA_.js.map +1 -0
- package/dist/{prime-protocol-DcAK9pC8.js → prime-protocol-BFgGrEMP.js} +3 -3
- package/dist/prime-protocol-BFgGrEMP.js.map +1 -0
- package/dist/{prime-protocol-CEE4BR9N.d.ts → prime-protocol-BWG2DQIk.d.ts} +4 -22
- package/dist/prime-protocol-BWG2DQIk.d.ts.map +1 -0
- package/dist/profile-cell.d.ts +2 -2
- package/dist/profile-cell.js +2 -2
- package/dist/{release-confidence-BK2QCB5M.d.ts → promotion-gate-DaY95yU6.d.ts} +105 -106
- package/dist/promotion-gate-DaY95yU6.d.ts.map +1 -0
- package/dist/{promotion-policy-CWW35UWq.js → promotion-policy-6A8osGr_.js} +4 -4
- package/dist/{promotion-policy-CWW35UWq.js.map → promotion-policy-6A8osGr_.js.map} +1 -1
- package/dist/{proposal-findings-BZiMTzAF.js → proposal-findings-BbWidvUk.js} +3 -3
- package/dist/proposal-findings-BbWidvUk.js.map +1 -0
- package/dist/{query-B1D1qp2Z.js → query-B_RZ36an.js} +5 -5
- package/dist/query-B_RZ36an.js.map +1 -0
- package/dist/{query-DK1nWcmp.d.ts → query-Cmv_U8vf.d.ts} +4 -24
- package/dist/{query-DK1nWcmp.d.ts.map → query-Cmv_U8vf.d.ts.map} +1 -1
- package/dist/{random-Dn5fPWkt.js → random-DDDcC7YU.js} +2 -2
- package/dist/{random-Dn5fPWkt.js.map → random-DDDcC7YU.js.map} +1 -1
- package/dist/{raw-provider-sink-rB2DYxr6.js → raw-provider-sink-75w4FWBj.js} +2 -2
- package/dist/{raw-provider-sink-rB2DYxr6.js.map → raw-provider-sink-75w4FWBj.js.map} +1 -1
- package/dist/{redact-Cveoq6fJ.d.ts → redact-DXdlFb_U.d.ts} +4 -21
- package/dist/redact-DXdlFb_U.d.ts.map +1 -0
- package/dist/{redact-Dk1ZdJEm.js → redact-mtkTiJVk.js} +92 -98
- package/dist/redact-mtkTiJVk.js.map +1 -0
- package/dist/{registry-JYfPxcVz.d.ts → registry-ssNoHzwD.d.ts} +5 -5
- package/dist/{registry-JYfPxcVz.d.ts.map → registry-ssNoHzwD.d.ts.map} +1 -1
- package/dist/{release-confidence-Dbe9AS_X.js → release-confidence-CYIFH-fp.js} +6 -11
- package/dist/release-confidence-CYIFH-fp.js.map +1 -0
- package/dist/{report-command-y8H6RSX8.js → render-C-w2EZ6p.js} +331 -554
- package/dist/render-C-w2EZ6p.js.map +1 -0
- package/dist/reporting-html-Kvlc8oMt.js +100 -0
- package/dist/reporting-html-Kvlc8oMt.js.map +1 -0
- package/dist/reporting.d.ts +6 -6
- package/dist/reporting.js +8 -8
- package/dist/{researcher-f6mZhh58.d.ts → researcher-0ABVIdGa.d.ts} +9 -9
- package/dist/{researcher-f6mZhh58.d.ts.map → researcher-0ABVIdGa.d.ts.map} +1 -1
- package/dist/reward-DrZ71vnf.d.ts +55 -0
- package/dist/reward-DrZ71vnf.d.ts.map +1 -0
- package/dist/{reward-hacking-BFcQgkux.js → reward-hacking-ByuETaIb.js} +3 -3
- package/dist/{reward-hacking-BFcQgkux.js.map → reward-hacking-ByuETaIb.js.map} +1 -1
- package/dist/{reward-hacking-DHA7c1l8.d.ts → reward-hacking-E52wq86n.d.ts} +5 -18
- package/dist/reward-hacking-E52wq86n.d.ts.map +1 -0
- package/dist/reward-nw2xZGZG.js.map +1 -1
- package/dist/rl.d.ts +117 -295
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +27 -747
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +5 -4
- package/dist/rollout/index.js +7 -8
- package/dist/{rollout-D9bKKBnZ.js → rollout-D6C50mfA.js} +88 -13
- package/dist/rollout-D6C50mfA.js.map +1 -0
- package/dist/round-decimal-COrKglMh.js +10 -0
- package/dist/round-decimal-COrKglMh.js.map +1 -0
- package/dist/{rubric-predictive-validity-BsgkBfT0.js → rubric-predictive-validity-D3_oI72N.js} +3 -3
- package/dist/{rubric-predictive-validity-BsgkBfT0.js.map → rubric-predictive-validity-D3_oI72N.js.map} +1 -1
- package/dist/{rubric-predictive-validity-2SgHxoa5.d.ts → rubric-predictive-validity-DiwT8PbL.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-2SgHxoa5.d.ts.map → rubric-predictive-validity-DiwT8PbL.d.ts.map} +1 -1
- package/dist/{run-record-D1Hw-Jml.d.ts → run-record-CaxbLq6Y.d.ts} +5 -11
- package/dist/run-record-CaxbLq6Y.d.ts.map +1 -0
- package/dist/{run-record-ES07wfz9.js → run-record-CysKLB93.js} +3 -5
- package/dist/run-record-CysKLB93.js.map +1 -0
- package/dist/{run-record-DlTZ_lqe.js → run-record-DhEJgoPz.js} +4 -4
- package/dist/{run-record-DlTZ_lqe.js.map → run-record-DhEJgoPz.js.map} +1 -1
- package/dist/sandbox-harness-BlSOu4LX.d.ts.map +1 -1
- package/dist/{schema-CSf6qWgZ.js → schema-Bt2vLFm3.js} +2 -14
- package/dist/schema-Bt2vLFm3.js.map +1 -0
- package/dist/schema-C-ziyJD7.d.ts.map +1 -1
- package/dist/{schema-2g7trUwH.d.ts → schema-D6EVKzTB.d.ts} +4 -5
- package/dist/schema-D6EVKzTB.d.ts.map +1 -0
- package/dist/{schema-BuiujZ-g.js → schema-DxcKTzbi.js} +2 -2
- package/dist/schema-DxcKTzbi.js.map +1 -0
- package/dist/search/index.d.ts +39 -53
- package/dist/search/index.d.ts.map +1 -1
- package/dist/search/index.js +4 -4
- package/dist/{search-kernel-CpcKH5Ih.d.ts → search-kernel-DxqjBxA-.d.ts} +18 -31
- package/dist/search-kernel-DxqjBxA-.d.ts.map +1 -0
- package/dist/{search-ledger-d7V8o9Lk.js → search-ledger-TUvB5gWH.js} +189 -199
- package/dist/search-ledger-TUvB5gWH.js.map +1 -0
- package/dist/{search-ledger-file-BqOo-bVo.js → search-ledger-file-UKCO2r8a.js} +4 -4
- package/dist/search-ledger-file-UKCO2r8a.js.map +1 -0
- package/dist/search-ledger-types-p2xKfiDJ.d.ts.map +1 -1
- package/dist/{search-shipper-er4pK5_Y.js → search-shipper-Cw2z3-td.js} +4 -4
- package/dist/search-shipper-Cw2z3-td.js.map +1 -0
- package/dist/{search-summary-CBvz3u_S.js → search-summary-DQIvJsSB.js} +5 -5
- package/dist/{search-summary-CBvz3u_S.js.map → search-summary-DQIvJsSB.js.map} +1 -1
- package/dist/{sequential-CzK5DarL.js → sequential-BVv1sjsw.js} +2 -32
- package/dist/{sequential-CzK5DarL.js.map → sequential-BVv1sjsw.js.map} +1 -1
- package/dist/{sequential-D-1PPCmM.js → sequential-CimQrbjv.js} +5 -5
- package/dist/sequential-CimQrbjv.js.map +1 -0
- package/dist/{sequential-BhsrMupG.d.ts → sequential-KwGspfeR.d.ts} +2 -9
- package/dist/{sequential-BhsrMupG.d.ts.map → sequential-KwGspfeR.d.ts.map} +1 -1
- package/dist/{sequential-eprocess-D1jKoihe.js → sequential-eprocess-DodI6RoF.js} +2 -2
- package/dist/{sequential-eprocess-D1jKoihe.js.map → sequential-eprocess-DodI6RoF.js.map} +1 -1
- package/dist/{server-DbcC7xMl.js → server-DoVbqqtJ.js} +101 -101
- package/dist/server-DoVbqqtJ.js.map +1 -0
- package/dist/{single-run-lock-fVu3MZWH.js → single-run-lock-U58yf8N6.js} +2 -2
- package/dist/{single-run-lock-fVu3MZWH.js.map → single-run-lock-U58yf8N6.js.map} +1 -1
- package/dist/{skillopt-optimization-method-CCtzralq.js → skillopt-optimization-method-kyL94-ri.js} +428 -91
- package/dist/skillopt-optimization-method-kyL94-ri.js.map +1 -0
- package/dist/{spans-D9_ehFuk.d.ts → spans-CVngGij3.d.ts} +2 -2
- package/dist/{spans-D9_ehFuk.d.ts.map → spans-CVngGij3.d.ts.map} +1 -1
- package/dist/{statistical-heldout-1Gbmex7v.js → statistical-heldout-Bksv92Th.js} +3 -3
- package/dist/statistical-heldout-Bksv92Th.js.map +1 -0
- package/dist/{statistical-heldout-BeR4bOdg.d.ts → statistical-heldout-C9AbgREN.d.ts} +8 -67
- package/dist/statistical-heldout-C9AbgREN.d.ts.map +1 -0
- package/dist/{steps-Bg5-KVx1.d.ts → steps-DlepJPwD.d.ts} +2 -56
- package/dist/steps-DlepJPwD.d.ts.map +1 -0
- package/dist/{store-C4avFjIi.d.ts → store-B3nWRy3B.d.ts} +2 -2
- package/dist/{store-C4avFjIi.d.ts.map → store-B3nWRy3B.d.ts.map} +1 -1
- package/dist/{store-otlp-DtB4Slsr.js → store-otlp-BteDAbHK.js} +135 -132
- package/dist/store-otlp-BteDAbHK.js.map +1 -0
- package/dist/{store-tool-spans-fZIigrpP.js → store-tool-spans-D5l_OfTJ.js} +9 -9
- package/dist/store-tool-spans-D5l_OfTJ.js.map +1 -0
- package/dist/{store-tool-spans-DR1SrCLK.d.ts → store-tool-spans-DZS0HWVf.d.ts} +157 -165
- package/dist/store-tool-spans-DZS0HWVf.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts +4 -11
- package/dist/storyboard/index.d.ts.map +1 -1
- package/dist/storyboard/index.js +21 -19
- package/dist/storyboard/index.js.map +1 -1
- package/dist/{summary-report-DtdRucM0.js → summary-report-BsXJoYX5.js} +588 -598
- package/dist/summary-report-BsXJoYX5.js.map +1 -0
- package/dist/{summary-report-C-CtItCN.d.ts → summary-report-DM7tlml1.d.ts} +16 -20
- package/dist/summary-report-DM7tlml1.d.ts.map +1 -0
- package/dist/supervisor-run/index.d.ts +189 -153
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +58 -450
- package/dist/supervisor-run/index.js.map +1 -1
- package/dist/{systemone-CpTLAlqb.d.ts → systemone-Bp_Ac0p1.d.ts} +6 -6
- package/dist/{systemone-CpTLAlqb.d.ts.map → systemone-Bp_Ac0p1.d.ts.map} +1 -1
- package/dist/systemone-protocol.js +1 -1
- package/dist/{systemone-review-B_yo1xsd.d.ts → systemone-review-CwE6JGIp.d.ts} +2 -2
- package/dist/{systemone-review-B_yo1xsd.d.ts.map → systemone-review-CwE6JGIp.d.ts.map} +1 -1
- package/dist/{systemone-review-BZWfrtq1.js → systemone-review-DzdOJ8I6.js} +3 -3
- package/dist/{systemone-review-BZWfrtq1.js.map → systemone-review-DzdOJ8I6.js.map} +1 -1
- package/dist/systemone.d.ts +5 -5
- package/dist/systemone.js +5 -5
- package/dist/systemone.js.map +1 -1
- package/dist/{task-failure-attributes-DngaLdE4.js → task-failure-attributes-BMsblaDh.js} +3 -3
- package/dist/task-failure-attributes-BMsblaDh.js.map +1 -0
- package/dist/{terminal-record-BHocwFw1.js → terminal-record-78g6E4vh.js} +17 -13
- package/dist/terminal-record-78g6E4vh.js.map +1 -0
- package/dist/{tool-groups-mcxOXVlb.d.ts → tool-groups-DyNz2JZ_.d.ts} +3 -3
- package/dist/tool-groups-DyNz2JZ_.d.ts.map +1 -0
- package/dist/{tool-waste-Wi4g6Ekb.d.ts → tool-waste-CkjzWmeY.d.ts} +53 -56
- package/dist/tool-waste-CkjzWmeY.d.ts.map +1 -0
- package/dist/{tool-waste-D52PqU9Q.js → tool-waste-DLsjS_p-.js} +5 -5
- package/dist/{tool-waste-D52PqU9Q.js.map → tool-waste-DLsjS_p-.js.map} +1 -1
- package/dist/{trace-questions-DG7nqLQ7.d.ts → trace-questions-xxwgsuMi.d.ts} +5 -10
- package/dist/{trace-questions-DG7nqLQ7.d.ts.map → trace-questions-xxwgsuMi.d.ts.map} +1 -1
- package/dist/trace-repair/index.d.ts +15 -212
- package/dist/trace-repair/index.d.ts.map +1 -1
- package/dist/trace-repair/index.js +865 -987
- package/dist/trace-repair/index.js.map +1 -1
- package/dist/traces.d.ts +76 -86
- package/dist/traces.d.ts.map +1 -1
- package/dist/traces.js +13 -13
- package/dist/traces.js.map +1 -1
- package/dist/trajectory-replay/index.d.ts +3 -117
- package/dist/trajectory-replay/index.d.ts.map +1 -1
- package/dist/trajectory-replay/index.js +98 -56
- package/dist/trajectory-replay/index.js.map +1 -1
- package/dist/{tree-rXLW3M_4.js → tree-BGRglkKd.js} +2355 -2414
- package/dist/tree-BGRglkKd.js.map +1 -0
- package/dist/{types-B3jzCp0p.js → types-BXgASBCL.js} +2 -2
- package/dist/{types-B3jzCp0p.js.map → types-BXgASBCL.js.map} +1 -1
- package/dist/{types-oKRhQAde.d.ts → types-CW8ewATZ.d.ts} +5 -114
- package/dist/types-CW8ewATZ.d.ts.map +1 -0
- package/dist/{types-BqkVmHfv.d.ts → types-CmXY0PQf.d.ts} +2 -4
- package/dist/types-CmXY0PQf.d.ts.map +1 -0
- package/dist/{types-DBZ7keha.d.ts → types-D-mgRzlh.d.ts} +9 -72
- package/dist/types-D-mgRzlh.d.ts.map +1 -0
- package/dist/{types-B4C5nFw6.d.ts → types-D8GhJoHk.d.ts} +79 -75
- package/dist/types-D8GhJoHk.d.ts.map +1 -0
- package/dist/{types-IXD2HJzu.js → types-DqFMNTgK.js} +2 -4
- package/dist/types-DqFMNTgK.js.map +1 -0
- package/dist/{types-CfZMIzB_.d.ts → types-DtMbFw6U.d.ts} +3 -3
- package/dist/{types-CfZMIzB_.d.ts.map → types-DtMbFw6U.d.ts.map} +1 -1
- package/dist/{verdict-DhPoXApk.js → verdict-BGQmxQ8A.js} +3 -3
- package/dist/verdict-BGQmxQ8A.js.map +1 -0
- package/dist/{verdict-E4eRNf7-.d.ts → verdict-C24NELtN.d.ts} +2 -15
- package/dist/{verdict-E4eRNf7-.d.ts.map → verdict-C24NELtN.d.ts.map} +1 -1
- package/dist/{verdict-cache-BQEsVy9n.js → verdict-cache-CL_7tePD.js} +2 -2
- package/dist/verdict-cache-CL_7tePD.js.map +1 -0
- package/dist/wire/index.d.ts +32 -411
- package/dist/wire/index.d.ts.map +1 -1
- package/dist/wire/index.js +2 -2
- package/docs/changes/jev-evidence-review.md +1 -1
- package/docs/changes/jev-product-adoption.md +1 -1
- package/docs/changes/jev-recorded-evidence.md +1 -1
- package/docs/design/mlbenchmarks-book-review.md +1 -1
- package/docs/design/mlbenchmarks-review/probes.mts +23 -31
- package/docs/design/statistics-decisions.md +17 -17
- package/docs/design.md +1 -1
- package/docs/evaluation-integrity.md +1 -1
- package/docs/experiment.md +1 -1
- package/docs/proofs/campaign-attempt-retention-20261001/README.md +2 -2
- package/docs/public-api.md +138 -529
- package/docs/rollout.md +1 -1
- package/docs/trace-repair-admission.md +1 -1
- package/docs/wire-protocol.md +1 -1
- package/package.json +1 -1
- package/dist/active-curriculum-B_Lc9kfO.js +0 -61
- package/dist/active-curriculum-B_Lc9kfO.js.map +0 -1
- package/dist/agent-profile-CIA2Atnt.d.ts.map +0 -1
- package/dist/agent-profile-cell-BNp-Js9F.d.ts.map +0 -1
- package/dist/agent-profile-cell-BYc0bxsL.js.map +0 -1
- package/dist/agreement-irr-DsTURxdr.js.map +0 -1
- package/dist/analyze-runs-3xpby3p2.js.map +0 -1
- package/dist/analyze-runs-CSDAh8A7.d.ts.map +0 -1
- package/dist/ast-DJ6Bc7ab.js +0 -558
- package/dist/ast-DJ6Bc7ab.js.map +0 -1
- package/dist/ast-hI-vjW6J.d.ts +0 -457
- package/dist/ast-hI-vjW6J.d.ts.map +0 -1
- package/dist/baseline-Dm8966du.js +0 -259
- package/dist/baseline-Dm8966du.js.map +0 -1
- package/dist/benchmark-BywPeAUo.js +0 -562
- package/dist/benchmark-BywPeAUo.js.map +0 -1
- package/dist/benchmark-command-CByl9dxG.js.map +0 -1
- package/dist/campaign-BQ-Jujuj.js.map +0 -1
- package/dist/chat-json-call-BrKsyspg.js.map +0 -1
- package/dist/claim-BXcZw-jo.js.map +0 -1
- package/dist/claude-jsonl-dZir12m6.js +0 -86
- package/dist/claude-jsonl-dZir12m6.js.map +0 -1
- package/dist/client-BzL9niqi.d.ts.map +0 -1
- package/dist/cost-ledger-CO7UyEcQ.d.ts.map +0 -1
- package/dist/cost-ledger-D2FnJPBF.js.map +0 -1
- package/dist/counterfactual-CPaWN99-.d.ts +0 -60
- package/dist/counterfactual-CPaWN99-.d.ts.map +0 -1
- package/dist/counterfactual-DtMWvn7D.js +0 -98
- package/dist/counterfactual-DtMWvn7D.js.map +0 -1
- package/dist/default-registry-BrWh62zf.d.ts.map +0 -1
- package/dist/default-registry-Cn1LQ2Ve.js.map +0 -1
- package/dist/define-agent-eval-CEWkX5wZ.d.ts.map +0 -1
- package/dist/define-agent-eval-Ch3MWX6k.js.map +0 -1
- package/dist/dspy-rlm-engine-D-KX_e64.js.map +0 -1
- package/dist/effect-sizes-DiH8MGOH.js +0 -82
- package/dist/effect-sizes-DiH8MGOH.js.map +0 -1
- package/dist/emitter-HcCmFQl_.d.ts.map +0 -1
- package/dist/engine-C_zNWfM8.d.ts.map +0 -1
- package/dist/equal-terms-Blx9Zk5I.js.map +0 -1
- package/dist/evaluation-Cn2dRZIz.d.ts.map +0 -1
- package/dist/exporters-X4xwla8A.js.map +0 -1
- package/dist/external-optimizer-process-nnsY5D_4.js.map +0 -1
- package/dist/external-optimizer-subprocess-DcerlZSG.js.map +0 -1
- package/dist/failure-cluster-CbgEDcFJ.d.ts.map +0 -1
- package/dist/feedback-trajectory-DkaDj0dj.d.ts.map +0 -1
- package/dist/hf-dataset-BR2yhuNn.js.map +0 -1
- package/dist/index-CPJJr98o.d.ts.map +0 -1
- package/dist/index-Ch9VxCpK.d.ts.map +0 -1
- package/dist/index-DHKEAv3s.d.ts.map +0 -1
- package/dist/index-DlH0Y9gI.d.ts.map +0 -1
- package/dist/index-NvGBiO8H.d.ts.map +0 -1
- package/dist/integrity-BV1ZDf42.js.map +0 -1
- package/dist/journal-DvKCvjty.js.map +0 -1
- package/dist/judge-gate-B1qYnkpi.js.map +0 -1
- package/dist/judge-gate-DNcUrRFA.d.ts.map +0 -1
- package/dist/kind-factory-CbZ-6tvu.js.map +0 -1
- package/dist/ledger-core-B2hahzrt.js.map +0 -1
- package/dist/llm-client-DQveoxSM.js.map +0 -1
- package/dist/llm-judge-PXV7Xljl.js.map +0 -1
- package/dist/matrix-CHMiebFx.d.ts.map +0 -1
- package/dist/matrix-_BV9-Lk5.js.map +0 -1
- package/dist/mint-DCGoBYhB.d.ts +0 -155
- package/dist/mint-DCGoBYhB.d.ts.map +0 -1
- package/dist/normal-DsJZtrJc.js +0 -25
- package/dist/normal-DsJZtrJc.js.map +0 -1
- package/dist/paired-arms-DzKn0TfY.js.map +0 -1
- package/dist/paired-binary-Q36bhn2R.js.map +0 -1
- package/dist/paired-promotion-decision-DPKz8ZnF.d.ts.map +0 -1
- package/dist/paired-tests-jbwPdSz5.js.map +0 -1
- package/dist/prime-protocol-CEE4BR9N.d.ts.map +0 -1
- package/dist/prime-protocol-DcAK9pC8.js.map +0 -1
- package/dist/produced-state-ZCtDBxAE.js.map +0 -1
- package/dist/proposal-findings-BZiMTzAF.js.map +0 -1
- package/dist/query-B1D1qp2Z.js.map +0 -1
- package/dist/redact-Cveoq6fJ.d.ts.map +0 -1
- package/dist/redact-Dk1ZdJEm.js.map +0 -1
- package/dist/release-confidence-BK2QCB5M.d.ts.map +0 -1
- package/dist/release-confidence-Dbe9AS_X.js.map +0 -1
- package/dist/report-command-y8H6RSX8.js.map +0 -1
- package/dist/reward-hacking-DHA7c1l8.d.ts.map +0 -1
- package/dist/rollout-D9bKKBnZ.js.map +0 -1
- package/dist/run-record-D1Hw-Jml.d.ts.map +0 -1
- package/dist/run-record-ES07wfz9.js.map +0 -1
- package/dist/schema-2g7trUwH.d.ts.map +0 -1
- package/dist/schema-BuiujZ-g.js.map +0 -1
- package/dist/schema-CSf6qWgZ.js.map +0 -1
- package/dist/search-kernel-BztlERLv.js.map +0 -1
- package/dist/search-kernel-CpcKH5Ih.d.ts.map +0 -1
- package/dist/search-ledger-d7V8o9Lk.js.map +0 -1
- package/dist/search-ledger-file-BqOo-bVo.js.map +0 -1
- package/dist/search-shipper-er4pK5_Y.js.map +0 -1
- package/dist/semantic-concept-judge-zQBG9hHU.js.map +0 -1
- package/dist/sequential-D-1PPCmM.js.map +0 -1
- package/dist/server-DbcC7xMl.js.map +0 -1
- package/dist/skillopt-optimization-method-CCtzralq.js.map +0 -1
- package/dist/statistical-heldout-1Gbmex7v.js.map +0 -1
- package/dist/statistical-heldout-BeR4bOdg.d.ts.map +0 -1
- package/dist/steps-Bg5-KVx1.d.ts.map +0 -1
- package/dist/store-otlp-DtB4Slsr.js.map +0 -1
- package/dist/store-tool-spans-DR1SrCLK.d.ts.map +0 -1
- package/dist/store-tool-spans-fZIigrpP.js.map +0 -1
- package/dist/student-t-BQeKJx2N.js +0 -39
- package/dist/student-t-BQeKJx2N.js.map +0 -1
- package/dist/summary-report-C-CtItCN.d.ts.map +0 -1
- package/dist/summary-report-DtdRucM0.js.map +0 -1
- package/dist/task-failure-attributes-DngaLdE4.js.map +0 -1
- package/dist/terminal-record-BHocwFw1.js.map +0 -1
- package/dist/tool-groups-mcxOXVlb.d.ts.map +0 -1
- package/dist/tool-waste-Wi4g6Ekb.d.ts.map +0 -1
- package/dist/trajectory-Y1fmdlw7.d.ts +0 -24
- package/dist/trajectory-Y1fmdlw7.d.ts.map +0 -1
- package/dist/tree-rXLW3M_4.js.map +0 -1
- package/dist/types-B4C5nFw6.d.ts.map +0 -1
- package/dist/types-BqkVmHfv.d.ts.map +0 -1
- package/dist/types-DBZ7keha.d.ts.map +0 -1
- package/dist/types-IXD2HJzu.js.map +0 -1
- package/dist/types-oKRhQAde.d.ts.map +0 -1
- package/dist/usage-receipt-Cx3xNeyB.js +0 -76
- package/dist/usage-receipt-Cx3xNeyB.js.map +0 -1
- package/dist/verdict-DhPoXApk.js.map +0 -1
- package/dist/verdict-cache-BQEsVy9n.js.map +0 -1
- package/docs/design/self-improvement-evidence-audit.md +0 -263
package/dist/index.js
CHANGED
|
@@ -1,66 +1,64 @@
|
|
|
1
1
|
import { t as __exportAll } from "./rolldown-runtime-8H4AJuhK.js";
|
|
2
|
-
import { i as JudgeError, n as CaptureIntegrityError, o as NotFoundError, r as ConfigError, s as ValidationError, t as AgentEvalError } from "./errors-
|
|
3
|
-
import { a as hashCanonical, r as canonicalString } from "./canonical-
|
|
4
|
-
import { a as
|
|
5
|
-
import {
|
|
6
|
-
import { t as interpolatedQuantile } from "./quantile--0yb07pD.js";
|
|
7
|
-
import { a as medianInPlace, i as makeRng, t as assertFiniteSample } from "./internal-CRHwS9iz.js";
|
|
8
|
-
import { i as pairedCohensDz, n as cohensD, r as interpretCliffs, t as cliffsDelta } from "./effect-sizes-DiH8MGOH.js";
|
|
9
|
-
import { t as mulberry32 } from "./random-Dn5fPWkt.js";
|
|
10
|
-
import { a as chiSquaredPValue, c as normalTwoSidedPValue, i as chiSquared, l as studentTTwoSidedPValue, n as iqr, o as fisherExactTwoSided, r as welchsTTest, s as linearRegressionSlope, u as twoByTwoPValue } from "./baseline-Dm8966du.js";
|
|
11
|
-
import { a as spearmanR, c as weightedMean, i as ranks, n as partialCredit, o as summarizeNumberSeries, r as pearsonR, s as weightedComposite, t as confidenceInterval } from "./descriptive-CnU8Yq29.js";
|
|
12
|
-
import { n as calibrateJudgeContinuous, o as verbosityBias, r as continuousAgreement, t as calibrateJudge } from "./judge-calibration-N44V04Se.js";
|
|
13
|
-
import { n as corpusInterRaterAgreementFromJudgeScores, r as interRaterReliability, t as corpusInterRaterAgreement } from "./agreement-irr-DsTURxdr.js";
|
|
14
|
-
import { n as bonferroni, r as holm, t as benjaminiHochberg } from "./multiplicity-D9cjmStq.js";
|
|
15
|
-
import { a as pairedRiskDifferenceExact, c as wilson, i as pairedRiskDifference, n as mcnemar, o as pairedRiskDifferenceScore, r as pairedBinaryScale, s as passAtK, t as isBinaryOutcomeVector } from "./paired-binary-Q36bhn2R.js";
|
|
16
|
-
import { a as pairedSignTest, i as pairedDeltaTieFraction, n as DECISION_PAIRED_DELTA_STATISTIC, o as pairedTTest, r as pairedBootstrap, t as BOOTSTRAP_GATE_MIN_N } from "./paired-tests-jbwPdSz5.js";
|
|
17
|
-
import { a as requiredSampleSize, i as requiredPairedSampleSize, n as mcnemarRequiredN, r as pairedMde, t as mcnemarPower } from "./power-and-mde-fK5Qocwo.js";
|
|
18
|
-
import { a as wilcoxonSignedRank, i as mannWhitneyU, n as pairArms, r as pairRunRecords, t as comparePairedArms } from "./paired-arms-DzKn0TfY.js";
|
|
19
|
-
import { t as eProcess } from "./sequential-eprocess-D1jKoihe.js";
|
|
20
|
-
import { i as pairedDeltaTest, n as pairedDecisionShape, r as minimumPairsForPairedDeltaTest, t as decidePairedPromotion } from "./paired-promotion-decision-BN6IQTiu.js";
|
|
21
|
-
import { a as isToolSpan, i as isLlmSpan, r as isJudgeSpan, t as FAILURE_CLASSES } from "./schema-CSf6qWgZ.js";
|
|
22
|
-
import { o as judgeSpans, r as argHash, u as runsForScenario } from "./query-B1D1qp2Z.js";
|
|
23
|
-
import { r as checkCanaries, t as analyzeRuns } from "./analyze-runs-3xpby3p2.js";
|
|
2
|
+
import { i as JudgeError, n as CaptureIntegrityError, o as NotFoundError, r as ConfigError, s as ValidationError, t as AgentEvalError } from "./errors-BcOs5MKW.js";
|
|
3
|
+
import { a as hashCanonical, r as canonicalString } from "./canonical-Dq2hOQ8L.js";
|
|
4
|
+
import { a as HARNESS_NATIVE_MODEL, c as agentProfileModelId, i as CODING_HARNESSES, l as expandProfileAxes, n as createLlmCorrectnessChecker, o as agentProfileHash, r as verifyCompletion, s as agentProfileId, t as completionVerdict, u as harnessAxisOf } from "./completion-verifier-B_jw6C9p.js";
|
|
5
|
+
import { a as groupRunsByAgentProfileCell, c as verifyAgentProfileCell, i as buildAgentProfileCell, l as hashJson, o as toAgentProfileJson, r as agentProfileCellKey, s as validateAgentProfileCell, t as AGENT_PROFILE_KINDS, u as manifestContentDigest } from "./agent-profile-cell-B1V1gUZs.js";
|
|
24
6
|
import { r as observedSplitScore, t as isRealnessGated } from "./reward-nw2xZGZG.js";
|
|
25
|
-
import {
|
|
26
|
-
import { a as
|
|
27
|
-
import { n as
|
|
28
|
-
import {
|
|
7
|
+
import { i as isToolSpan, n as isJudgeSpan, r as isLlmSpan, t as FAILURE_CLASSES } from "./schema-Bt2vLFm3.js";
|
|
8
|
+
import { a as parseRunRecordSafe, c as validateRunRecord, i as modelHasSnapshot, n as UNKNOWN_MODEL, o as runTaskScore, r as isRunRecord, t as RunRecordValidationError } from "./run-record-DhEJgoPz.js";
|
|
9
|
+
import { a as cliffsDelta, c as pairedCohensDz, i as summaryTable, n as gainHistogram, o as cohensD, s as interpretCliffs } from "./summary-report-BsXJoYX5.js";
|
|
10
|
+
import { t as interpolatedQuantile } from "./quantile--0yb07pD.js";
|
|
11
|
+
import { a as medianInPlace, i as makeRng, t as assertFiniteSample } from "./internal-BTK91Bqt.js";
|
|
12
|
+
import { t as mulberry32 } from "./random-DDDcC7YU.js";
|
|
13
|
+
import { a as spearmanR, c as weightedMean, i as ranks, n as partialCredit, o as summarizeNumberSeries, r as pearsonR, s as weightedComposite, t as confidenceInterval } from "./descriptive-CVNy48rR.js";
|
|
14
|
+
import { n as calibrateJudgeContinuous, o as verbosityBias, r as continuousAgreement, t as calibrateJudge } from "./judge-calibration-BkXMIWA4.js";
|
|
15
|
+
import { n as corpusInterRaterAgreementFromJudgeScores, r as interRaterReliability, t as corpusInterRaterAgreement } from "./agreement-irr-BhAqjluA.js";
|
|
16
|
+
import { a as wilcoxonSignedRank, i as mannWhitneyU, n as pairArms, r as pairRunRecords, t as comparePairedArms } from "./paired-arms-Ct4LTKc1.js";
|
|
17
|
+
import { a as welchsTTest, c as fisherExactTwoSided, d as studentTTwoSidedPValue, f as twoByTwoPValue, i as iqr, l as linearRegressionSlope, n as checkCanaries, o as chiSquared, s as chiSquaredPValue, t as analyzeRuns, u as normalTwoSidedPValue } from "./analyze-runs-DqPJaJ4X.js";
|
|
18
|
+
import { n as bonferroni, r as holm, t as benjaminiHochberg } from "./multiplicity-Sg0U7h8j.js";
|
|
19
|
+
import { a as pairedRiskDifferenceScore, i as pairedRiskDifferenceExact, o as passAtK, r as pairedRiskDifference, s as wilson, t as mcnemar } from "./paired-binary-B_nejA44.js";
|
|
20
|
+
import { i as pairedBootstrap, n as BOOTSTRAP_GATE_MIN_N, o as pairedSignTest, s as pairedTTest, t as mean$1 } from "./mean-BbA_JCzW.js";
|
|
21
|
+
import { a as requiredSampleSize, i as requiredPairedSampleSize, n as mcnemarRequiredN, r as pairedMde, t as mcnemarPower } from "./power-and-mde-N5nU4_Ly.js";
|
|
22
|
+
import { t as eProcess } from "./sequential-eprocess-DodI6RoF.js";
|
|
23
|
+
import { i as pairedDeltaTest, n as pairedDecisionShape, r as minimumPairsForPairedDeltaTest, t as decidePairedPromotion } from "./paired-promotion-decision-D0SHb-ZL.js";
|
|
24
|
+
import { c as runsForScenario, i as judgeSpans, t as argHash } from "./query-B_RZ36an.js";
|
|
25
|
+
import { n as contentHash, r as fileVerdictCache, t as canonicalJson } from "./verdict-cache-CL_7tePD.js";
|
|
26
|
+
import { G as runCanaries, H as redTeamDataset, Q as runCampaign, U as redTeamReport, V as DEFAULT_RED_TEAM_CORPUS, W as scoreRedTeamOutput, a as aggregateRunScore, i as transientDispatchFailure, o as clamp01, t as llmJudge } from "./llm-judge-C8jMTNNG.js";
|
|
29
27
|
import { a as resolveModelPricing, i as isModelPriced, n as estimateCost, r as estimateTokens, t as MODEL_PRICING } from "./metrics-BKaPBtIG.js";
|
|
30
|
-
import {
|
|
31
|
-
import {
|
|
32
|
-
import {
|
|
33
|
-
import { a as
|
|
34
|
-
import {
|
|
35
|
-
import {
|
|
36
|
-
import { r as selfImprove, t as defineAgentEval } from "./define-agent-eval-
|
|
37
|
-
import { n as equivalenceVerdict, t as certificationEvidenceDigest } from "./verdict-
|
|
38
|
-
import { t as
|
|
39
|
-
import { t as
|
|
40
|
-
import { t as runCounterfactual } from "./counterfactual-DtMWvn7D.js";
|
|
41
|
-
import { G as DEFAULT_TRACE_ANALYST_OUTPUT_TOKENS, U as parseFindingSubject, h as TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, l as createBoundedTraceAnalysisStore, m as DEFAULT_TRACE_ANALYST_BUDGETS, t as createTraceAnalyst } from "./kind-factory-CbZ-6tvu.js";
|
|
42
|
-
import { _ as completedAnalystReviewQuality, b as validateAnalystReviewDecisions, c as FAILURE_MODE_KIND_SPEC, g as assertUniqueFindingIds, h as analystRunDigest, i as DEFAULT_TRACE_ANALYST_KINDS, m as analystFindingDigest, n as AnalystRegistry, t as buildDefaultAnalystRegistry, v as readAnalystReview, y as snapshotAnalystRun } from "./default-registry-Cn1LQ2Ve.js";
|
|
28
|
+
import { c as costForTokenPricing, i as CostCeilingReachedError, l as costForUsage, n as CostAccountingIncompleteError, t as CostLedger } from "./cost-ledger-D0352Z4d.js";
|
|
29
|
+
import { D as surfaceContentHash, M as assertRealBackend, N as summarizeBackendIntegrity, _ as parseReflectionResponse, a as ATTESTATION_ALGORITHM, g as buildReflectionPrompt, j as BackendIntegrityError, o as attest, s as verifyAttestation } from "./campaign-evidence-CQYw8_0H.js";
|
|
30
|
+
import { a as redact, c as shareAllowed, n as assessShareSafety, o as redactForShare, s as redactText, t as REDACTION_VERSION } from "./redact-mtkTiJVk.js";
|
|
31
|
+
import { a as makeProposalFinding, i as makeFinding, r as computeFindingId } from "./types-DqFMNTgK.js";
|
|
32
|
+
import { S as requiredPairsForPairedPromotion, n as paretoFrontier, t as dominates, x as pairedPromotionPower } from "./pareto-BzsxidFD.js";
|
|
33
|
+
import { a as LlmCallError, i as stripFencedJson, n as isRecord, o as LlmResponseError } from "./json-BhdsJaOQ.js";
|
|
34
|
+
import { r as selfImprove, t as defineAgentEval } from "./define-agent-eval-BDxNlP3D.js";
|
|
35
|
+
import { n as equivalenceVerdict, t as certificationEvidenceDigest } from "./verdict-BGQmxQ8A.js";
|
|
36
|
+
import { c as createBoundedTraceAnalysisStore, p as DEFAULT_TRACE_ANALYST_BUDGETS, t as createTraceAnalyst } from "./kind-factory-Gy4fXixd.js";
|
|
37
|
+
import { _ as readAnalystReview, c as FAILURE_MODE_KIND_SPEC, g as completedAnalystReviewQuality, h as assertUniqueFindingIds, i as DEFAULT_TRACE_ANALYST_KINDS, m as analystRunDigest, n as AnalystRegistry, p as analystFindingDigest, t as buildDefaultAnalystRegistry, v as snapshotAnalystRun, y as validateAnalystReviewDecisions } from "./default-registry-ClYjq3QE.js";
|
|
43
38
|
import { OUTPUT_VALUE } from "./trace-attributes.js";
|
|
44
|
-
import {
|
|
45
|
-
import { n as InMemoryRawProviderSink, r as NoopRawProviderSink, t as FileSystemRawProviderSink } from "./raw-provider-sink-
|
|
46
|
-
import {
|
|
47
|
-
import {
|
|
39
|
+
import { n as extractUsageFromSse, t as extractUsage } from "./extract-usage-BZLU01kR.js";
|
|
40
|
+
import { n as InMemoryRawProviderSink, r as NoopRawProviderSink, t as FileSystemRawProviderSink } from "./raw-provider-sink-75w4FWBj.js";
|
|
41
|
+
import { _ as exportRunAsOtlp, b as captureFetchToRawSink, c as describeTraceInsightScope, g as analyzeTraces, h as tokenizeDomainWords, l as domainEvidencePattern, m as scoreTraceInsightReadiness, n as toolSpansToTraceAnalysisStore, o as buildTraceInsightContext, s as buildTraceInsightPrompt, u as inferDomainKeywords } from "./store-tool-spans-D5l_OfTJ.js";
|
|
42
|
+
import { t as TraceEmitter } from "./emitter-DbMXUIVP.js";
|
|
43
|
+
import { n as assertRunCaptured, t as RunIntegrityError } from "./integrity-W4CADcFc.js";
|
|
48
44
|
import { n as InMemoryTraceStore, t as FileSystemTraceStore } from "./store-BhNuLdJw.js";
|
|
49
45
|
import { t as packageVersion$1 } from "./package-version-D7lQHt_-.js";
|
|
50
|
-
import {
|
|
46
|
+
import { t as buildTrajectory } from "./trajectory-D_7rLrvE.js";
|
|
47
|
+
import { i as failureBlame, n as classifyFailure, r as classifyFailureReason } from "./failure-taxonomy-Bx5oWqaW.js";
|
|
51
48
|
import { n as runBoundedProcess, t as DEFAULT_BOUNDED_PROCESS_TIMEOUT_MS } from "./bounded-process-Cj9YjXbE.js";
|
|
52
|
-
import { t as runEvalCampaign } from "./eval-campaign-
|
|
53
|
-
import "./rollout-
|
|
54
|
-
import { t as mintRolloutRows } from "./mint-
|
|
55
|
-
import { n as pairedEvalueSequence, t as evaluateInterimReleaseConfidence } from "./sequential-
|
|
56
|
-
import {
|
|
57
|
-
import {
|
|
58
|
-
import {
|
|
59
|
-
import {
|
|
60
|
-
import {
|
|
61
|
-
import { a as
|
|
62
|
-
import {
|
|
63
|
-
import { t as
|
|
49
|
+
import { t as runEvalCampaign } from "./eval-campaign-BLEfBGbA.js";
|
|
50
|
+
import "./rollout-D6C50mfA.js";
|
|
51
|
+
import { t as mintRolloutRows } from "./mint-zszyYlpA.js";
|
|
52
|
+
import { n as pairedEvalueSequence, t as evaluateInterimReleaseConfidence } from "./sequential-BVv1sjsw.js";
|
|
53
|
+
import { a as maximumChargeForLlmRequest, c as ModelSubstitutionError, f as assertServedModels, g as judgeFamily, h as assertCrossFamily, i as costReceiptFromLlmError, l as assertCrossFamilyServed, m as CrossFamilyError, o as backoffMs, p as checkServedModel, r as costReceiptFromLlm, s as isTransientLlmError, u as assertServedModel } from "./llm-client-BshCREn6.js";
|
|
54
|
+
import { n as paidJsonChat } from "./chat-json-call-DavzexSp.js";
|
|
55
|
+
import { t as createDspyRlmTraceEngine } from "./dspy-rlm-engine-DhdCr1Dc.js";
|
|
56
|
+
import { n as diffFindings, r as createChatTraceEngine, t as FindingsStore } from "./findings-store-BMVQtEsm.js";
|
|
57
|
+
import { a as budgetBreachView, i as failureClusterView, n as computeToolUseMetrics, r as judgeAgreementView, t as toolWasteView } from "./tool-waste-DLsjS_p-.js";
|
|
58
|
+
import { a as judgeGateDecision, c as snapshotFromJudgeAgreement, g as analyzeSeries, i as assertJudgeMayGate, n as JudgeGateRefusedError, o as measureJudgeAgreement, r as addCalibrationVerdicts, s as registerCalibrationSet, t as DEFAULT_JUDGE_GATE_POLICY } from "./judge-gate-Co5tuh57.js";
|
|
59
|
+
import { a as spanRecordsToTraceAnalysisStore, i as otlpTextToTraceAnalysisStore, r as OtlpFileTraceStore } from "./store-otlp-BteDAbHK.js";
|
|
60
|
+
import { n as bootstrapCi, t as evaluateReleaseConfidence } from "./release-confidence-CYIFH-fp.js";
|
|
61
|
+
import { t as createChatClient } from "./chat-client-ldqxYYTN.js";
|
|
64
62
|
import { createHash } from "node:crypto";
|
|
65
63
|
import { accessSync, appendFileSync, constants, cpSync, existsSync, mkdirSync, promises, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
|
|
66
64
|
import { basename, delimiter, dirname, extname, isAbsolute, join, relative, resolve } from "node:path";
|
|
@@ -3460,37 +3458,6 @@ var ProductClient = class {
|
|
|
3460
3458
|
* implementations live with their consumer because they depend on
|
|
3461
3459
|
* SDK-specific Box / Sandbox types that don't belong in this package.
|
|
3462
3460
|
*/
|
|
3463
|
-
/**
|
|
3464
|
-
* Host-process runner, mapped onto {@link runBoundedProcess} so the package
|
|
3465
|
-
* has one spawn site rather than two.
|
|
3466
|
-
*
|
|
3467
|
-
* It used `spawnSync` and therefore had none of the bounds that runner exists
|
|
3468
|
-
* to provide: a `capMs` timeout killed the direct child and left any
|
|
3469
|
-
* descendant it had started running, a killed run could still report exit
|
|
3470
|
-
* code 0, captured output was unbounded, and the whole run blocked the event
|
|
3471
|
-
* loop. This runner spawns build, test and lint commands, which are exactly
|
|
3472
|
-
* the commands most likely to start descendants.
|
|
3473
|
-
*
|
|
3474
|
-
* It is not the package's last private spawn: `nodeProcessRunner`
|
|
3475
|
-
* (`src/trace-repair/docker-environment.ts`) and `runProcess`
|
|
3476
|
-
* (`src/campaign/external-optimizer-subprocess.ts`) still spawn their own
|
|
3477
|
-
* detached children, and several one-shot `spawnSync` git and CLI calls
|
|
3478
|
-
* remain. What this removes is the second spawn behind the `CommandRunner`
|
|
3479
|
-
* contract, which is the one that graded build, test and lint commands.
|
|
3480
|
-
*
|
|
3481
|
-
* Two behaviours the mapping changes on purpose:
|
|
3482
|
-
* - a run with no `capMs` is now bounded by
|
|
3483
|
-
* {@link LOCAL_COMMAND_RUNNER_DEFAULT_CAP_MS} instead of running forever;
|
|
3484
|
-
* - the run no longer blocks the event loop, so a minutes-long `capMs` no
|
|
3485
|
-
* longer stalls everything else in the process.
|
|
3486
|
-
*/
|
|
3487
|
-
/**
|
|
3488
|
-
* Wall clock {@link localCommandRunner} applies when the caller names no
|
|
3489
|
-
* `capMs`. It is `runBoundedProcess`'s own default, stated here because this
|
|
3490
|
-
* runner used to be unbounded: a consumer whose command runs longer names its
|
|
3491
|
-
* own `capMs`, and a run the bound stops says so with `timedOut: true`.
|
|
3492
|
-
*/
|
|
3493
|
-
const LOCAL_COMMAND_RUNNER_DEFAULT_CAP_MS = DEFAULT_BOUNDED_PROCESS_TIMEOUT_MS;
|
|
3494
3461
|
const localCommandRunner = {
|
|
3495
3462
|
name: "local",
|
|
3496
3463
|
async run(input) {
|
|
@@ -3585,6 +3552,77 @@ function isExecutable(path) {
|
|
|
3585
3552
|
}
|
|
3586
3553
|
}
|
|
3587
3554
|
//#endregion
|
|
3555
|
+
//#region src/control-runtime/policy.ts
|
|
3556
|
+
const DEFAULT_BUDGET = {
|
|
3557
|
+
maxSteps: 8,
|
|
3558
|
+
maxWallMs: 3e5
|
|
3559
|
+
};
|
|
3560
|
+
function objectiveEval(input) {
|
|
3561
|
+
return {
|
|
3562
|
+
...input,
|
|
3563
|
+
objective: true
|
|
3564
|
+
};
|
|
3565
|
+
}
|
|
3566
|
+
function subjectiveEval(input) {
|
|
3567
|
+
return {
|
|
3568
|
+
...input,
|
|
3569
|
+
objective: false
|
|
3570
|
+
};
|
|
3571
|
+
}
|
|
3572
|
+
function normalizeBudget(input) {
|
|
3573
|
+
const raw = {
|
|
3574
|
+
...DEFAULT_BUDGET,
|
|
3575
|
+
...input
|
|
3576
|
+
};
|
|
3577
|
+
if (!Number.isInteger(raw.maxSteps) || raw.maxSteps < 1) throw new RangeError(`ControlRuntime budget.maxSteps must be an integer >= 1, got ${String(raw.maxSteps)}`);
|
|
3578
|
+
const budget = { maxSteps: raw.maxSteps };
|
|
3579
|
+
if (raw.maxWallMs !== void 0) {
|
|
3580
|
+
if (typeof raw.maxWallMs !== "number" || !Number.isFinite(raw.maxWallMs) || raw.maxWallMs <= 0) throw new RangeError(`ControlRuntime budget.maxWallMs must be a positive finite number, got ${String(raw.maxWallMs)}`);
|
|
3581
|
+
budget.maxWallMs = raw.maxWallMs;
|
|
3582
|
+
}
|
|
3583
|
+
if (raw.maxCostUsd !== void 0) {
|
|
3584
|
+
if (typeof raw.maxCostUsd !== "number" || !Number.isFinite(raw.maxCostUsd) || raw.maxCostUsd < 0) throw new RangeError(`ControlRuntime budget.maxCostUsd must be a nonnegative finite number, got ${String(raw.maxCostUsd)}`);
|
|
3585
|
+
budget.maxCostUsd = raw.maxCostUsd;
|
|
3586
|
+
}
|
|
3587
|
+
return budget;
|
|
3588
|
+
}
|
|
3589
|
+
function allCriticalPassed(evals) {
|
|
3590
|
+
return evals.every((result) => result.passed || result.severity !== "critical" && result.severity !== "error");
|
|
3591
|
+
}
|
|
3592
|
+
function defaultStopDecision(evals) {
|
|
3593
|
+
if (!evals.length) return {
|
|
3594
|
+
stop: false,
|
|
3595
|
+
pass: false,
|
|
3596
|
+
reason: "no evals yet"
|
|
3597
|
+
};
|
|
3598
|
+
return allCriticalPassed(evals) ? {
|
|
3599
|
+
stop: true,
|
|
3600
|
+
pass: true,
|
|
3601
|
+
reason: "all critical evals passed",
|
|
3602
|
+
score: averageScore(evals)
|
|
3603
|
+
} : {
|
|
3604
|
+
stop: false,
|
|
3605
|
+
pass: false,
|
|
3606
|
+
reason: "critical evals still failing",
|
|
3607
|
+
score: averageScore(evals)
|
|
3608
|
+
};
|
|
3609
|
+
}
|
|
3610
|
+
function averageScore(evals) {
|
|
3611
|
+
const scored = evals.map((result) => result.score).filter((score) => typeof score === "number");
|
|
3612
|
+
if (!scored.length) return void 0;
|
|
3613
|
+
return Math.round(mean$1(scored) * 1e3) / 1e3;
|
|
3614
|
+
}
|
|
3615
|
+
function budgetStopDecision(budget, spentCostUsd) {
|
|
3616
|
+
if (budget.maxCostUsd !== void 0 && spentCostUsd >= budget.maxCostUsd) return {
|
|
3617
|
+
stop: true,
|
|
3618
|
+
reason: `budget exhausted: maxCostUsd=${budget.maxCostUsd}`
|
|
3619
|
+
};
|
|
3620
|
+
return {
|
|
3621
|
+
stop: false,
|
|
3622
|
+
reason: ""
|
|
3623
|
+
};
|
|
3624
|
+
}
|
|
3625
|
+
//#endregion
|
|
3588
3626
|
//#region src/detectors/index.ts
|
|
3589
3627
|
/** Same action fingerprint N consecutive steps = a stuck loop (the #1 long-horizon failure mode). */
|
|
3590
3628
|
function repeatedActionDetector(opts = {}) {
|
|
@@ -3698,21 +3736,6 @@ function observeAll(detectors, event) {
|
|
|
3698
3736
|
}
|
|
3699
3737
|
//#endregion
|
|
3700
3738
|
//#region src/control-runtime.ts
|
|
3701
|
-
/**
|
|
3702
|
-
* Policy-based agent control runtime.
|
|
3703
|
-
*
|
|
3704
|
-
* This is the minimal reusable loop behind driver-agent patterns:
|
|
3705
|
-
*
|
|
3706
|
-
* observe state -> validate -> decide next action -> act -> observe -> ...
|
|
3707
|
-
*
|
|
3708
|
-
* It deliberately does not model named "topologies". Direct execution,
|
|
3709
|
-
* critic/revise, driver intervention, specialist calls, and human escalation
|
|
3710
|
-
* are all just actions chosen by the control policy.
|
|
3711
|
-
*/
|
|
3712
|
-
const DEFAULT_BUDGET = {
|
|
3713
|
-
maxSteps: 8,
|
|
3714
|
-
maxWallMs: 3e5
|
|
3715
|
-
};
|
|
3716
3739
|
async function runAgentControlLoop(config) {
|
|
3717
3740
|
const budget = normalizeBudget(config.budget);
|
|
3718
3741
|
const actionFailure = config.actionFailure ?? "continue";
|
|
@@ -4009,45 +4032,13 @@ async function runAgentControlLoop(config) {
|
|
|
4009
4032
|
});
|
|
4010
4033
|
}
|
|
4011
4034
|
}
|
|
4035
|
+
let observationPhase = "observe";
|
|
4012
4036
|
try {
|
|
4013
4037
|
state = await config.observe({
|
|
4014
4038
|
history,
|
|
4015
4039
|
abortSignal: controller.signal
|
|
4016
4040
|
});
|
|
4017
|
-
|
|
4018
|
-
runtimeErrors.push(runtimeError("observe", stepIndex, err));
|
|
4019
|
-
const step = {
|
|
4020
|
-
index: stepIndex,
|
|
4021
|
-
decision,
|
|
4022
|
-
beforeState,
|
|
4023
|
-
afterState: beforeState,
|
|
4024
|
-
evalsBefore,
|
|
4025
|
-
evalsAfter: evals,
|
|
4026
|
-
actionOutcome,
|
|
4027
|
-
startedAt: new Date(actionStarted).toISOString(),
|
|
4028
|
-
endedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
4029
|
-
};
|
|
4030
|
-
history.push(step);
|
|
4031
|
-
await runTrace(emitter, runtimeErrors, stepIndex, () => stepHandle?.fail(runtimeErrors[runtimeErrors.length - 1].message));
|
|
4032
|
-
await runOnStep(config.onStep, step, runtimeErrors);
|
|
4033
|
-
return finish(emitter, {
|
|
4034
|
-
intent: config.intent,
|
|
4035
|
-
pass: false,
|
|
4036
|
-
completed: false,
|
|
4037
|
-
reason: runtimeErrors[runtimeErrors.length - 1].message,
|
|
4038
|
-
score: averageScore(evals),
|
|
4039
|
-
steps: history,
|
|
4040
|
-
finalState: beforeState,
|
|
4041
|
-
finalEvals: evals,
|
|
4042
|
-
wallMs: Date.now() - started,
|
|
4043
|
-
spentCostUsd,
|
|
4044
|
-
runId: emitter?.runId ?? null,
|
|
4045
|
-
failureClass: "unknown",
|
|
4046
|
-
runtimeErrors,
|
|
4047
|
-
stoppedBy: "runtime-error"
|
|
4048
|
-
});
|
|
4049
|
-
}
|
|
4050
|
-
try {
|
|
4041
|
+
observationPhase = "validate";
|
|
4051
4042
|
evals = await config.validate({
|
|
4052
4043
|
intent: config.intent,
|
|
4053
4044
|
state,
|
|
@@ -4056,12 +4047,13 @@ async function runAgentControlLoop(config) {
|
|
|
4056
4047
|
});
|
|
4057
4048
|
await recordEvalSpans(emitter, evals, `step-${stepIndex}`, runtimeErrors, stepIndex, stepHandle?.span.spanId);
|
|
4058
4049
|
} catch (err) {
|
|
4059
|
-
runtimeErrors.push(runtimeError(
|
|
4050
|
+
runtimeErrors.push(runtimeError(observationPhase, stepIndex, err));
|
|
4051
|
+
const afterState = observationPhase === "observe" ? beforeState : state;
|
|
4060
4052
|
const step = {
|
|
4061
4053
|
index: stepIndex,
|
|
4062
4054
|
decision,
|
|
4063
4055
|
beforeState,
|
|
4064
|
-
afterState
|
|
4056
|
+
afterState,
|
|
4065
4057
|
evalsBefore,
|
|
4066
4058
|
evalsAfter: evals,
|
|
4067
4059
|
actionOutcome,
|
|
@@ -4078,7 +4070,7 @@ async function runAgentControlLoop(config) {
|
|
|
4078
4070
|
reason: runtimeErrors[runtimeErrors.length - 1].message,
|
|
4079
4071
|
score: averageScore(evals),
|
|
4080
4072
|
steps: history,
|
|
4081
|
-
finalState:
|
|
4073
|
+
finalState: afterState,
|
|
4082
4074
|
finalEvals: evals,
|
|
4083
4075
|
wallMs: Date.now() - started,
|
|
4084
4076
|
spentCostUsd,
|
|
@@ -4229,46 +4221,6 @@ async function runAgentControlLoop(config) {
|
|
|
4229
4221
|
if (config.signal) config.signal.removeEventListener("abort", upstreamAbort);
|
|
4230
4222
|
}
|
|
4231
4223
|
}
|
|
4232
|
-
function objectiveEval(input) {
|
|
4233
|
-
return {
|
|
4234
|
-
...input,
|
|
4235
|
-
objective: true
|
|
4236
|
-
};
|
|
4237
|
-
}
|
|
4238
|
-
function subjectiveEval(input) {
|
|
4239
|
-
return {
|
|
4240
|
-
...input,
|
|
4241
|
-
objective: false
|
|
4242
|
-
};
|
|
4243
|
-
}
|
|
4244
|
-
function normalizeBudget(input) {
|
|
4245
|
-
const raw = {
|
|
4246
|
-
...DEFAULT_BUDGET,
|
|
4247
|
-
...input
|
|
4248
|
-
};
|
|
4249
|
-
if (!Number.isInteger(raw.maxSteps) || raw.maxSteps < 1) throw new RangeError(`ControlRuntime budget.maxSteps must be an integer >= 1, got ${String(raw.maxSteps)}`);
|
|
4250
|
-
const budget = { maxSteps: raw.maxSteps };
|
|
4251
|
-
if (raw.maxWallMs !== void 0) {
|
|
4252
|
-
if (typeof raw.maxWallMs !== "number" || !Number.isFinite(raw.maxWallMs) || raw.maxWallMs <= 0) throw new RangeError(`ControlRuntime budget.maxWallMs must be a positive finite number, got ${String(raw.maxWallMs)}`);
|
|
4253
|
-
budget.maxWallMs = raw.maxWallMs;
|
|
4254
|
-
}
|
|
4255
|
-
if (raw.maxCostUsd !== void 0) {
|
|
4256
|
-
if (typeof raw.maxCostUsd !== "number" || !Number.isFinite(raw.maxCostUsd) || raw.maxCostUsd < 0) throw new RangeError(`ControlRuntime budget.maxCostUsd must be a nonnegative finite number, got ${String(raw.maxCostUsd)}`);
|
|
4257
|
-
budget.maxCostUsd = raw.maxCostUsd;
|
|
4258
|
-
}
|
|
4259
|
-
return budget;
|
|
4260
|
-
}
|
|
4261
|
-
function normalizeActionCostUsd(costUsd, runtimeErrors, stepIndex) {
|
|
4262
|
-
if (costUsd === void 0) return void 0;
|
|
4263
|
-
if (!Number.isFinite(costUsd) || costUsd < 0) {
|
|
4264
|
-
runtimeErrors.push(runtimeError("act", stepIndex, /* @__PURE__ */ new Error(`invalid action costUsd: ${String(costUsd)}`)));
|
|
4265
|
-
return;
|
|
4266
|
-
}
|
|
4267
|
-
return costUsd;
|
|
4268
|
-
}
|
|
4269
|
-
function allCriticalPassed(evals) {
|
|
4270
|
-
return evals.every((result) => result.passed || result.severity !== "critical" && result.severity !== "error");
|
|
4271
|
-
}
|
|
4272
4224
|
function makeContext(intent, state, evals, history, budget, stepIndex, started, spentCostUsd, abortSignal, emitter) {
|
|
4273
4225
|
return {
|
|
4274
4226
|
intent,
|
|
@@ -4284,37 +4236,35 @@ function makeContext(intent, state, evals, history, budget, stepIndex, started,
|
|
|
4284
4236
|
emitter
|
|
4285
4237
|
};
|
|
4286
4238
|
}
|
|
4287
|
-
function
|
|
4288
|
-
if (
|
|
4289
|
-
|
|
4290
|
-
pass: false,
|
|
4291
|
-
reason: "no evals yet"
|
|
4292
|
-
};
|
|
4293
|
-
return allCriticalPassed(evals) ? {
|
|
4294
|
-
stop: true,
|
|
4295
|
-
pass: true,
|
|
4296
|
-
reason: "all critical evals passed",
|
|
4297
|
-
score: averageScore(evals)
|
|
4298
|
-
} : {
|
|
4299
|
-
stop: false,
|
|
4300
|
-
pass: false,
|
|
4301
|
-
reason: "critical evals still failing",
|
|
4302
|
-
score: averageScore(evals)
|
|
4303
|
-
};
|
|
4239
|
+
function fingerprintState(state, policies) {
|
|
4240
|
+
if (policies?.stateFingerprint) return policies.stateFingerprint(state);
|
|
4241
|
+
return stableFingerprint(state);
|
|
4304
4242
|
}
|
|
4305
|
-
function
|
|
4306
|
-
|
|
4307
|
-
|
|
4308
|
-
return Math.round(scored.reduce((sum, score) => sum + score, 0) / scored.length * 1e3) / 1e3;
|
|
4243
|
+
function fingerprintAction(action, policies) {
|
|
4244
|
+
if (policies?.actionFingerprint) return policies.actionFingerprint(action);
|
|
4245
|
+
return stableFingerprint(action);
|
|
4309
4246
|
}
|
|
4310
|
-
|
|
4311
|
-
|
|
4312
|
-
|
|
4313
|
-
|
|
4314
|
-
|
|
4247
|
+
/** Default state/action fingerprint: the value itself for a string, `String(value)`
|
|
4248
|
+
* for a number, boolean, null or undefined, and RFC 8785 canonical JSON for
|
|
4249
|
+
* everything else. A value with no canonical JSON form (an `undefined`-valued
|
|
4250
|
+
* field, a non-finite number, a class instance) throws: two states that differ
|
|
4251
|
+
* in such a field must not fingerprint alike, and a caller with such state
|
|
4252
|
+
* supplies `stopPolicies.stateFingerprint` / `actionFingerprint`. */
|
|
4253
|
+
function stableFingerprint(value) {
|
|
4254
|
+
if (typeof value === "string") return value;
|
|
4255
|
+
if (typeof value === "number" || typeof value === "boolean" || value == null) return String(value);
|
|
4256
|
+
return canonicalString(value);
|
|
4257
|
+
}
|
|
4258
|
+
function abortReason(signal) {
|
|
4259
|
+
const reason = signal.reason;
|
|
4260
|
+
if (reason instanceof Error) return reason.message;
|
|
4261
|
+
return reason ? String(reason) : "aborted";
|
|
4262
|
+
}
|
|
4263
|
+
function runtimeError(phase, stepIndex, err) {
|
|
4315
4264
|
return {
|
|
4316
|
-
|
|
4317
|
-
|
|
4265
|
+
phase,
|
|
4266
|
+
stepIndex,
|
|
4267
|
+
message: err instanceof Error ? err.message : String(err)
|
|
4318
4268
|
};
|
|
4319
4269
|
}
|
|
4320
4270
|
async function recordCostBudget(emitter, budget, spentCostUsd, handle, runtimeErrors, stepIndex) {
|
|
@@ -4374,37 +4324,6 @@ async function runTrace(emitter, runtimeErrors, stepIndex, write) {
|
|
|
4374
4324
|
if (after.dropped > droppedBefore) runtimeErrors.push(runtimeError("trace", stepIndex, after.lastError ?? "trace write dropped"));
|
|
4375
4325
|
return value;
|
|
4376
4326
|
}
|
|
4377
|
-
function fingerprintState(state, policies) {
|
|
4378
|
-
if (policies?.stateFingerprint) return policies.stateFingerprint(state);
|
|
4379
|
-
return stableFingerprint(state);
|
|
4380
|
-
}
|
|
4381
|
-
function fingerprintAction(action, policies) {
|
|
4382
|
-
if (policies?.actionFingerprint) return policies.actionFingerprint(action);
|
|
4383
|
-
return stableFingerprint(action);
|
|
4384
|
-
}
|
|
4385
|
-
/** Default state/action fingerprint: the value itself for a string, `String(value)`
|
|
4386
|
-
* for a number, boolean, null or undefined, and RFC 8785 canonical JSON for
|
|
4387
|
-
* everything else. A value with no canonical JSON form (an `undefined`-valued
|
|
4388
|
-
* field, a non-finite number, a class instance) throws: two states that differ
|
|
4389
|
-
* in such a field must not fingerprint alike, and a caller with such state
|
|
4390
|
-
* supplies `stopPolicies.stateFingerprint` / `actionFingerprint`. */
|
|
4391
|
-
function stableFingerprint(value) {
|
|
4392
|
-
if (typeof value === "string") return value;
|
|
4393
|
-
if (typeof value === "number" || typeof value === "boolean" || value == null) return String(value);
|
|
4394
|
-
return canonicalString(value);
|
|
4395
|
-
}
|
|
4396
|
-
function abortReason(signal) {
|
|
4397
|
-
const reason = signal.reason;
|
|
4398
|
-
if (reason instanceof Error) return reason.message;
|
|
4399
|
-
return reason ? String(reason) : "aborted";
|
|
4400
|
-
}
|
|
4401
|
-
function runtimeError(phase, stepIndex, err) {
|
|
4402
|
-
return {
|
|
4403
|
-
phase,
|
|
4404
|
-
stepIndex,
|
|
4405
|
-
message: err instanceof Error ? err.message : String(err)
|
|
4406
|
-
};
|
|
4407
|
-
}
|
|
4408
4327
|
async function finish(emitter, result) {
|
|
4409
4328
|
await runTrace(emitter, result.runtimeErrors, result.steps.length, () => emitter?.endRun({
|
|
4410
4329
|
pass: result.pass,
|
|
@@ -4414,6 +4333,14 @@ async function finish(emitter, result) {
|
|
|
4414
4333
|
}));
|
|
4415
4334
|
return result;
|
|
4416
4335
|
}
|
|
4336
|
+
function normalizeActionCostUsd(costUsd, runtimeErrors, stepIndex) {
|
|
4337
|
+
if (costUsd === void 0) return void 0;
|
|
4338
|
+
if (!Number.isFinite(costUsd) || costUsd < 0) {
|
|
4339
|
+
runtimeErrors.push(runtimeError("act", stepIndex, /* @__PURE__ */ new Error(`invalid action costUsd: ${String(costUsd)}`)));
|
|
4340
|
+
return;
|
|
4341
|
+
}
|
|
4342
|
+
return costUsd;
|
|
4343
|
+
}
|
|
4417
4344
|
//#endregion
|
|
4418
4345
|
//#region src/discover-personas.ts
|
|
4419
4346
|
/**
|
|
@@ -4464,65 +4391,6 @@ async function discoverPersonas(dir, opts = {}) {
|
|
|
4464
4391
|
return results;
|
|
4465
4392
|
}
|
|
4466
4393
|
//#endregion
|
|
4467
|
-
//#region src/improvement-verdict.ts
|
|
4468
|
-
const DEFAULTS = {
|
|
4469
|
-
keepThreshold: 5,
|
|
4470
|
-
regressionThreshold: 5,
|
|
4471
|
-
iqrUnstableAbove: 10,
|
|
4472
|
-
stddevUnstableAbove: Number.POSITIVE_INFINITY,
|
|
4473
|
-
minRepsForVerdict: 3
|
|
4474
|
-
};
|
|
4475
|
-
function resolveThresholds(t) {
|
|
4476
|
-
const r = {
|
|
4477
|
-
...DEFAULTS,
|
|
4478
|
-
...t ?? {}
|
|
4479
|
-
};
|
|
4480
|
-
if (r.keepThreshold < 0) throw new ValidationError(`improvement-verdict: keepThreshold must be >= 0, got ${r.keepThreshold}`);
|
|
4481
|
-
if (r.regressionThreshold < 0) throw new ValidationError(`improvement-verdict: regressionThreshold must be >= 0, got ${r.regressionThreshold}`);
|
|
4482
|
-
if (r.minRepsForVerdict < 1) throw new ValidationError(`improvement-verdict: minRepsForVerdict must be >= 1, got ${r.minRepsForVerdict}`);
|
|
4483
|
-
return r;
|
|
4484
|
-
}
|
|
4485
|
-
/**
|
|
4486
|
-
* Verdict for a candidate against its parent. Pure — operates on already-computed
|
|
4487
|
-
* stats. KEEP/REGRESSION require both sides to have `>= minRepsForVerdict` reps
|
|
4488
|
-
* AND the candidate to be `stable`; otherwise the result is NOISE (unstable) or
|
|
4489
|
-
* ITERATE (not enough reps / no parent).
|
|
4490
|
-
*/
|
|
4491
|
-
function improvementVerdict(candidate, parent, thresholds) {
|
|
4492
|
-
const t = resolveThresholds(thresholds);
|
|
4493
|
-
if (!parent) return {
|
|
4494
|
-
verdict: "ITERATE",
|
|
4495
|
-
medianDelta: null,
|
|
4496
|
-
reason: "no parent experiment to compare against"
|
|
4497
|
-
};
|
|
4498
|
-
if (candidate.n < t.minRepsForVerdict || parent.n < t.minRepsForVerdict) return {
|
|
4499
|
-
verdict: "ITERATE",
|
|
4500
|
-
medianDelta: null,
|
|
4501
|
-
reason: `need >= ${t.minRepsForVerdict} reps on both sides (candidate n=${candidate.n}, parent n=${parent.n})`
|
|
4502
|
-
};
|
|
4503
|
-
if (!candidate.stable) return {
|
|
4504
|
-
verdict: "NOISE",
|
|
4505
|
-
medianDelta: candidate.median - parent.median,
|
|
4506
|
-
reason: `candidate unstable (iqr=${candidate.iqr}, stddev=${candidate.stddev.toFixed(2)})`
|
|
4507
|
-
};
|
|
4508
|
-
const medianDelta = candidate.median - parent.median;
|
|
4509
|
-
if (medianDelta > t.keepThreshold) return {
|
|
4510
|
-
verdict: "KEEP",
|
|
4511
|
-
medianDelta,
|
|
4512
|
-
reason: `median +${medianDelta} > +${t.keepThreshold}`
|
|
4513
|
-
};
|
|
4514
|
-
if (medianDelta < -t.regressionThreshold) return {
|
|
4515
|
-
verdict: "REGRESSION",
|
|
4516
|
-
medianDelta,
|
|
4517
|
-
reason: `median ${medianDelta} < -${t.regressionThreshold}`
|
|
4518
|
-
};
|
|
4519
|
-
return {
|
|
4520
|
-
verdict: "NOISE",
|
|
4521
|
-
medianDelta,
|
|
4522
|
-
reason: `median delta ${medianDelta} inside noise band [-${t.regressionThreshold}, +${t.keepThreshold}]`
|
|
4523
|
-
};
|
|
4524
|
-
}
|
|
4525
|
-
//#endregion
|
|
4526
4394
|
//#region src/propose-review.ts
|
|
4527
4395
|
/**
|
|
4528
4396
|
* Propose / Verify / Review — the core multi-shot primitive.
|
|
@@ -4593,14 +4461,14 @@ function jsonlReviewStore(path) {
|
|
|
4593
4461
|
}
|
|
4594
4462
|
};
|
|
4595
4463
|
}
|
|
4596
|
-
const DEFAULT_FALLBACK_INSTRUCTION
|
|
4464
|
+
const DEFAULT_FALLBACK_INSTRUCTION = "Inspect the verification failures above. Fix the critical issues first, then the major ones. Do not restate the failures — act on them.";
|
|
4597
4465
|
async function runProposeReview(config) {
|
|
4598
4466
|
const maxShots = config.maxShots ?? 10;
|
|
4599
4467
|
const maxWallMs = config.maxWallMs ?? 6e5;
|
|
4600
4468
|
const confidenceFloor = config.confidenceFloor ?? .3;
|
|
4601
4469
|
const confidenceFloorWindow = config.confidenceFloorWindow ?? 2;
|
|
4602
4470
|
const memory = config.memory ?? inMemoryReviewStore();
|
|
4603
|
-
const fallbackInstruction = config.fallbackInstruction ?? DEFAULT_FALLBACK_INSTRUCTION
|
|
4471
|
+
const fallbackInstruction = config.fallbackInstruction ?? DEFAULT_FALLBACK_INSTRUCTION;
|
|
4604
4472
|
const emitter = config.store ? new TraceEmitter(config.store) : null;
|
|
4605
4473
|
if (emitter) await emitter.startRun({
|
|
4606
4474
|
scenarioId: config.scenarioId ?? "propose-review",
|
|
@@ -4833,196 +4701,6 @@ function safeJson(x) {
|
|
|
4833
4701
|
}
|
|
4834
4702
|
}
|
|
4835
4703
|
//#endregion
|
|
4836
|
-
//#region src/propose-review-control.ts
|
|
4837
|
-
const DEFAULT_FALLBACK_INSTRUCTION = "Inspect the verification failures above. Fix the critical issues first, then the major ones. Do not restate the failures — act on them.";
|
|
4838
|
-
async function runProposeReviewAsControlLoop(config) {
|
|
4839
|
-
const maxShots = config.maxShots ?? 10;
|
|
4840
|
-
const confidenceFloor = config.confidenceFloor ?? .3;
|
|
4841
|
-
const confidenceFloorWindow = config.confidenceFloorWindow ?? 2;
|
|
4842
|
-
const memory = config.memory ?? inMemoryReviewStore();
|
|
4843
|
-
const fallbackInstruction = config.fallbackInstruction ?? DEFAULT_FALLBACK_INSTRUCTION;
|
|
4844
|
-
const failureClassFromVerification = config.failureClassFromVerification ?? controlFailureClassFromVerification;
|
|
4845
|
-
let lowConfidenceStreak = 0;
|
|
4846
|
-
let current = {
|
|
4847
|
-
shot: 0,
|
|
4848
|
-
state: config.initialState,
|
|
4849
|
-
priorReview: null,
|
|
4850
|
-
verification: { pass: false },
|
|
4851
|
-
memory: await memory.load(),
|
|
4852
|
-
completed: false,
|
|
4853
|
-
reviewAvailable: false
|
|
4854
|
-
};
|
|
4855
|
-
return runAgentControlLoop({
|
|
4856
|
-
intent: config.goal,
|
|
4857
|
-
budget: {
|
|
4858
|
-
maxSteps: maxShots,
|
|
4859
|
-
maxWallMs: config.maxWallMs
|
|
4860
|
-
},
|
|
4861
|
-
store: config.store,
|
|
4862
|
-
scenarioId: config.scenarioId ?? "propose-review-control",
|
|
4863
|
-
projectId: config.projectId,
|
|
4864
|
-
variantId: config.variantId,
|
|
4865
|
-
actionFailure: config.actionFailure ?? "stop",
|
|
4866
|
-
observe: () => current,
|
|
4867
|
-
validate: ({ state }) => [objectiveEval({
|
|
4868
|
-
id: "verification",
|
|
4869
|
-
passed: state.verification.pass,
|
|
4870
|
-
score: state.verification.score,
|
|
4871
|
-
severity: "critical",
|
|
4872
|
-
detail: state.verification.pass ? "verification passed" : `verification failed${state.verification.failingLayers?.length ? `: ${state.verification.failingLayers.join(", ")}` : ""}`
|
|
4873
|
-
})],
|
|
4874
|
-
shouldStop: ({ state }) => {
|
|
4875
|
-
if (state.verification.pass) return {
|
|
4876
|
-
stop: true,
|
|
4877
|
-
pass: true,
|
|
4878
|
-
reason: "verification passed",
|
|
4879
|
-
score: state.verification.score
|
|
4880
|
-
};
|
|
4881
|
-
if (state.completed) return {
|
|
4882
|
-
stop: true,
|
|
4883
|
-
pass: false,
|
|
4884
|
-
reason: "reviewer stopped continuation",
|
|
4885
|
-
score: state.verification.score,
|
|
4886
|
-
failureClass: failureClassFromVerification(state.verification)
|
|
4887
|
-
};
|
|
4888
|
-
return {
|
|
4889
|
-
stop: false,
|
|
4890
|
-
pass: false,
|
|
4891
|
-
reason: "verification still failing",
|
|
4892
|
-
score: state.verification.score
|
|
4893
|
-
};
|
|
4894
|
-
},
|
|
4895
|
-
decide: ({ state }) => ({
|
|
4896
|
-
type: "continue",
|
|
4897
|
-
action: {
|
|
4898
|
-
type: "propose-review-shot",
|
|
4899
|
-
shot: state.shot + 1
|
|
4900
|
-
},
|
|
4901
|
-
reason: state.priorReview?.nextShotInstruction ?? fallbackInstruction
|
|
4902
|
-
}),
|
|
4903
|
-
act: async (action, ctx) => {
|
|
4904
|
-
const shot = action.shot;
|
|
4905
|
-
const proposeOut = await config.propose({
|
|
4906
|
-
shot,
|
|
4907
|
-
goal: config.goal,
|
|
4908
|
-
state: current.state,
|
|
4909
|
-
priorReview: current.priorReview,
|
|
4910
|
-
abortSignal: ctx.abortSignal,
|
|
4911
|
-
emitter: ctx.emitter
|
|
4912
|
-
});
|
|
4913
|
-
const nextState = proposeOut.state;
|
|
4914
|
-
const verification = await config.verify(nextState);
|
|
4915
|
-
let review = null;
|
|
4916
|
-
let reviewAvailable = false;
|
|
4917
|
-
let reviewError;
|
|
4918
|
-
let shouldContinue = !verification.pass;
|
|
4919
|
-
if (!verification.pass) try {
|
|
4920
|
-
review = await config.review({
|
|
4921
|
-
shot,
|
|
4922
|
-
goal: config.goal,
|
|
4923
|
-
state: nextState,
|
|
4924
|
-
verification,
|
|
4925
|
-
traceSummary: proposeOut.traceSummary,
|
|
4926
|
-
memory: await memory.load()
|
|
4927
|
-
});
|
|
4928
|
-
reviewAvailable = true;
|
|
4929
|
-
shouldContinue = review.shouldContinue;
|
|
4930
|
-
lowConfidenceStreak = review.confidence <= confidenceFloor ? lowConfidenceStreak + 1 : 0;
|
|
4931
|
-
if (confidenceFloorWindow > 0 && lowConfidenceStreak >= confidenceFloorWindow) shouldContinue = false;
|
|
4932
|
-
} catch (err) {
|
|
4933
|
-
reviewError = err instanceof Error ? err.message : String(err);
|
|
4934
|
-
review = current.priorReview ?? {
|
|
4935
|
-
observations: "Reviewer unavailable.",
|
|
4936
|
-
diagnosis: reviewError,
|
|
4937
|
-
nextShotInstruction: fallbackInstruction,
|
|
4938
|
-
shouldContinue: true,
|
|
4939
|
-
confidence: 0
|
|
4940
|
-
};
|
|
4941
|
-
shouldContinue = true;
|
|
4942
|
-
}
|
|
4943
|
-
else review = {
|
|
4944
|
-
observations: "Verification passed.",
|
|
4945
|
-
diagnosis: "No further revision needed.",
|
|
4946
|
-
nextShotInstruction: "",
|
|
4947
|
-
shouldContinue: false,
|
|
4948
|
-
confidence: 1
|
|
4949
|
-
};
|
|
4950
|
-
const entry = {
|
|
4951
|
-
...review ?? {
|
|
4952
|
-
observations: "No review.",
|
|
4953
|
-
diagnosis: "",
|
|
4954
|
-
nextShotInstruction: fallbackInstruction,
|
|
4955
|
-
shouldContinue,
|
|
4956
|
-
confidence: 0
|
|
4957
|
-
},
|
|
4958
|
-
shot,
|
|
4959
|
-
timestamp: Date.now(),
|
|
4960
|
-
verification: {
|
|
4961
|
-
pass: verification.pass,
|
|
4962
|
-
score: verification.score,
|
|
4963
|
-
failingLayers: verification.failingLayers
|
|
4964
|
-
}
|
|
4965
|
-
};
|
|
4966
|
-
await memory.append(entry);
|
|
4967
|
-
current = {
|
|
4968
|
-
shot,
|
|
4969
|
-
state: nextState,
|
|
4970
|
-
priorReview: review,
|
|
4971
|
-
verification,
|
|
4972
|
-
traceSummary: proposeOut.traceSummary,
|
|
4973
|
-
memory: await memory.load(),
|
|
4974
|
-
completed: verification.pass || !shouldContinue,
|
|
4975
|
-
reviewAvailable,
|
|
4976
|
-
reviewError
|
|
4977
|
-
};
|
|
4978
|
-
return {
|
|
4979
|
-
state: nextState,
|
|
4980
|
-
verification,
|
|
4981
|
-
traceSummary: proposeOut.traceSummary,
|
|
4982
|
-
review,
|
|
4983
|
-
reviewAvailable,
|
|
4984
|
-
reviewError
|
|
4985
|
-
};
|
|
4986
|
-
}
|
|
4987
|
-
});
|
|
4988
|
-
}
|
|
4989
|
-
function controlFailureClassFromVerification(verification) {
|
|
4990
|
-
if (verification.pass) return void 0;
|
|
4991
|
-
return verification.failingLayers?.length ? "instruction_following" : "unknown";
|
|
4992
|
-
}
|
|
4993
|
-
//#endregion
|
|
4994
|
-
//#region src/analyst/steer-firewall.ts
|
|
4995
|
-
/** True iff the finding is a JUDGE VERDICT (an acceptance score lifted into a
|
|
4996
|
-
* finding), identified by provenance set at the lift site — independent of
|
|
4997
|
-
* whatever evidence it cites. */
|
|
4998
|
-
function isJudgeVerdict(finding) {
|
|
4999
|
-
return finding.derived_from_judge === true;
|
|
5000
|
-
}
|
|
5001
|
-
/**
|
|
5002
|
-
* THE steer firewall. Fail-loud guard for any path that admits analyst findings
|
|
5003
|
-
* as STEERING input (the `f(trace)` role): rejects — naming the offenders — any
|
|
5004
|
-
* finding whose provenance is a judge verdict, rather than let `J` leak into the
|
|
5005
|
-
* loop. Returns the findings unchanged for chaining.
|
|
5006
|
-
*
|
|
5007
|
-
* Call this at the chokepoint where a detector that ALSO scores/gates has its
|
|
5008
|
-
* findings turned into a steer (the judge-and-steer dual-role case). It keys on
|
|
5009
|
-
* provenance, so it correctly admits evidence-less trace-analyst observations and
|
|
5010
|
-
* correctly rejects an artifact-citing judge verdict — the cases an evidence
|
|
5011
|
-
* check gets backwards.
|
|
5012
|
-
*
|
|
5013
|
-
* It is necessary, not sufficient: it stops PROVENANCE-tagged verdicts. A judge
|
|
5014
|
-
* whose output is laundered through a hand-built finding with no provenance flag
|
|
5015
|
-
* is out of its reach — provenance must be honestly set at every judge→finding
|
|
5016
|
-
* lift (today: createJudgeAdapter). That is why the integrity rule lives at the
|
|
5017
|
-
* lift site, and why ProposeContext.judgeScores?: never is the complementary
|
|
5018
|
-
* compile-time tripwire on the obvious direct channel.
|
|
5019
|
-
*/
|
|
5020
|
-
function assertNoJudgeVerdict(findings, context = "steer") {
|
|
5021
|
-
const leaks = findings.filter(isJudgeVerdict);
|
|
5022
|
-
if (leaks.length > 0) throw new Error(`${context}: a judge verdict cannot be admitted as steering input — that is the held-out judge leaking into the loop. Offending judge-derived findings: [${leaks.map((f) => f.finding_id).join(", ")}]. Steering consumes observations of behavior, never acceptance verdicts.`);
|
|
5023
|
-
return findings;
|
|
5024
|
-
}
|
|
5025
|
-
//#endregion
|
|
5026
4704
|
//#region src/analyst/policy-edit.ts
|
|
5027
4705
|
const POLICY_EDIT_AXES = [
|
|
5028
4706
|
"carrier",
|
|
@@ -5053,20 +4731,7 @@ var PolicyEditValidationError = class extends ValidationError {
|
|
|
5053
4731
|
this.path = path;
|
|
5054
4732
|
}
|
|
5055
4733
|
};
|
|
5056
|
-
const DEFAULT_MIN_SCORE = .7;
|
|
5057
|
-
const DEFAULT_MIN_EXPECTED_GAIN = .01;
|
|
5058
4734
|
const POLICY_EDIT_ID = /^policy-edit:sha256:[0-9a-f]{64}$/;
|
|
5059
|
-
function makePolicyEdit(init) {
|
|
5060
|
-
const normalized = normalizePolicyEdit({
|
|
5061
|
-
schemaVersion: "policy-edit/v1",
|
|
5062
|
-
...init,
|
|
5063
|
-
source: normalizeSource(init.source)
|
|
5064
|
-
});
|
|
5065
|
-
return validatePolicyEdit({
|
|
5066
|
-
...normalized,
|
|
5067
|
-
editId: init.editId ?? computePolicyEditId(normalized)
|
|
5068
|
-
});
|
|
5069
|
-
}
|
|
5070
4735
|
function computePolicyEditId(edit) {
|
|
5071
4736
|
const { editId: _editId, schemaVersion, ...material } = edit;
|
|
5072
4737
|
return `policy-edit:${hashCanonical({
|
|
@@ -5120,363 +4785,6 @@ function validatePolicyEditCandidateRecord(input) {
|
|
|
5120
4785
|
policyEdit: validatePolicyEdit(snapshot)
|
|
5121
4786
|
};
|
|
5122
4787
|
}
|
|
5123
|
-
function policyEditsFromFindings(findings, opts = {}) {
|
|
5124
|
-
assertNoJudgeVerdict(findings, "policyEditsFromFindings");
|
|
5125
|
-
const edits = [];
|
|
5126
|
-
for (const finding of findings) {
|
|
5127
|
-
const edit = policyEditFromFinding(finding, opts);
|
|
5128
|
-
if (edit) edits.push(edit);
|
|
5129
|
-
}
|
|
5130
|
-
return edits;
|
|
5131
|
-
}
|
|
5132
|
-
function policyEditFromFinding(finding, opts = {}) {
|
|
5133
|
-
assertNoJudgeVerdict([finding], "policyEditFromFinding");
|
|
5134
|
-
if (!finding.recommended_action?.trim()) return null;
|
|
5135
|
-
const expectedGain = resolveExpectedGain(finding, opts);
|
|
5136
|
-
if (!expectedGain) return null;
|
|
5137
|
-
if (typeof finding.confidence !== "number" || !Number.isFinite(finding.confidence) || finding.confidence < 0 || finding.confidence > 1) return null;
|
|
5138
|
-
const routed = routeFindingSubject(finding.subject, opts);
|
|
5139
|
-
const risk = resolveRisk(finding, opts);
|
|
5140
|
-
return makePolicyEdit({
|
|
5141
|
-
axis: routed.axis,
|
|
5142
|
-
target: routed.target,
|
|
5143
|
-
change: {
|
|
5144
|
-
kind: "text",
|
|
5145
|
-
mode: "append",
|
|
5146
|
-
value: finding.recommended_action.trim()
|
|
5147
|
-
},
|
|
5148
|
-
claim: finding.claim,
|
|
5149
|
-
rationale: finding.rationale,
|
|
5150
|
-
expectedGain,
|
|
5151
|
-
confidence: finding.confidence,
|
|
5152
|
-
risk,
|
|
5153
|
-
validationPlan: finding.validation_plan,
|
|
5154
|
-
source: {
|
|
5155
|
-
findingIds: [finding.finding_id],
|
|
5156
|
-
analystIds: [finding.analyst_id],
|
|
5157
|
-
evidenceRefs: finding.evidence_refs,
|
|
5158
|
-
derivedFromJudge: finding.derived_from_judge
|
|
5159
|
-
}
|
|
5160
|
-
});
|
|
5161
|
-
}
|
|
5162
|
-
function scorePolicyEditReadiness(edit, opts = {}) {
|
|
5163
|
-
validatePolicyEdit(edit);
|
|
5164
|
-
const minExpectedGain = opts.minExpectedGain ?? DEFAULT_MIN_EXPECTED_GAIN;
|
|
5165
|
-
const requireEvidence = opts.requireEvidence ?? false;
|
|
5166
|
-
const evidenceScore = Math.min(1, edit.source.evidenceRefs.length / 2);
|
|
5167
|
-
const confidenceScore = clamp01$3(edit.confidence);
|
|
5168
|
-
const gainScore = clamp01$3(Math.abs(edit.expectedGain.amount) / Math.max(minExpectedGain * 5, .001));
|
|
5169
|
-
const targetScore = targetSpecificityScore(edit);
|
|
5170
|
-
const riskPenalty = edit.risk === "high" && opts.allowHighRisk !== true ? .35 : edit.risk === "unknown" ? .2 : 0;
|
|
5171
|
-
return clamp01$3((requireEvidence ? .3 * evidenceScore + .25 * confidenceScore + .25 * gainScore + .2 * targetScore : .35 * confidenceScore + .35 * gainScore + .3 * targetScore + .1 * evidenceScore) - riskPenalty);
|
|
5172
|
-
}
|
|
5173
|
-
function admitPolicyEdit(edit, opts = {}) {
|
|
5174
|
-
const validated = validatePolicyEdit(edit);
|
|
5175
|
-
const score = scorePolicyEditReadiness(validated, opts);
|
|
5176
|
-
const reasons = [];
|
|
5177
|
-
const minExpectedGain = opts.minExpectedGain ?? DEFAULT_MIN_EXPECTED_GAIN;
|
|
5178
|
-
const requireEvidence = opts.requireEvidence ?? false;
|
|
5179
|
-
if (validated.source.derivedFromJudge) reasons.push("source is judge-derived; judge verdicts cannot steer policy edits");
|
|
5180
|
-
if (requireEvidence && validated.source.evidenceRefs.length === 0) reasons.push("missing evidence refs");
|
|
5181
|
-
if (Math.abs(validated.expectedGain.amount) < minExpectedGain) reasons.push(`expected gain below ${minExpectedGain}`);
|
|
5182
|
-
if (validated.risk === "high" && opts.allowHighRisk !== true) reasons.push("high-risk edit requires explicit allowHighRisk");
|
|
5183
|
-
if (score < (opts.minScore ?? DEFAULT_MIN_SCORE)) reasons.push(`readiness score ${score.toFixed(3)} below ${(opts.minScore ?? DEFAULT_MIN_SCORE).toFixed(3)}`);
|
|
5184
|
-
return {
|
|
5185
|
-
edit: validated,
|
|
5186
|
-
decision: reasons.length === 0 ? "admit" : "reject",
|
|
5187
|
-
score,
|
|
5188
|
-
reasons
|
|
5189
|
-
};
|
|
5190
|
-
}
|
|
5191
|
-
function applyPolicyEditToSurface(surface, edit) {
|
|
5192
|
-
const validated = validatePolicyEdit(edit);
|
|
5193
|
-
if (validated.change.kind === "text") return applyTextChange(surface, validated.change);
|
|
5194
|
-
return applyJsonChange(surface, validated.change);
|
|
5195
|
-
}
|
|
5196
|
-
function routeFindingSubject(subject, opts) {
|
|
5197
|
-
const parsed = parseFindingSubject(subject);
|
|
5198
|
-
if (!parsed) return {
|
|
5199
|
-
axis: opts.defaultAxis ?? "representation",
|
|
5200
|
-
target: { surface: opts.defaultTargetSurface ?? "prompt" }
|
|
5201
|
-
};
|
|
5202
|
-
return routeParsedSubject(parsed);
|
|
5203
|
-
}
|
|
5204
|
-
function routeParsedSubject(subject) {
|
|
5205
|
-
switch (subject.kind) {
|
|
5206
|
-
case "system-prompt": return {
|
|
5207
|
-
axis: "representation",
|
|
5208
|
-
target: {
|
|
5209
|
-
surface: "prompt",
|
|
5210
|
-
path: `system-prompt:${subject.section}`
|
|
5211
|
-
}
|
|
5212
|
-
};
|
|
5213
|
-
case "skill": return {
|
|
5214
|
-
axis: "agent_profile",
|
|
5215
|
-
target: {
|
|
5216
|
-
surface: "agent-profile",
|
|
5217
|
-
path: `skill:${subject.name}`
|
|
5218
|
-
}
|
|
5219
|
-
};
|
|
5220
|
-
case "tool-doc": return {
|
|
5221
|
-
axis: "tool_contract",
|
|
5222
|
-
target: {
|
|
5223
|
-
surface: "tool-contract",
|
|
5224
|
-
path: subject.aspect ? `tool-doc:${subject.tool}:${subject.aspect}` : `tool-doc:${subject.tool}`
|
|
5225
|
-
}
|
|
5226
|
-
};
|
|
5227
|
-
case "new-tool": return {
|
|
5228
|
-
axis: "tool_contract",
|
|
5229
|
-
target: {
|
|
5230
|
-
surface: "tool-contract",
|
|
5231
|
-
path: `new-tool:${subject.name}`
|
|
5232
|
-
}
|
|
5233
|
-
};
|
|
5234
|
-
case "mcp": return {
|
|
5235
|
-
axis: "tool_contract",
|
|
5236
|
-
target: {
|
|
5237
|
-
surface: "agent-profile",
|
|
5238
|
-
path: subject.tool ? `mcp:${subject.server}:${subject.tool}` : `mcp:${subject.server}`
|
|
5239
|
-
}
|
|
5240
|
-
};
|
|
5241
|
-
case "hook": return {
|
|
5242
|
-
axis: "agent_profile",
|
|
5243
|
-
target: {
|
|
5244
|
-
surface: "agent-profile",
|
|
5245
|
-
path: `hook:${subject.name}`
|
|
5246
|
-
}
|
|
5247
|
-
};
|
|
5248
|
-
case "subagent": return {
|
|
5249
|
-
axis: "routing",
|
|
5250
|
-
target: {
|
|
5251
|
-
surface: "agent-profile",
|
|
5252
|
-
path: `subagent:${subject.name}`
|
|
5253
|
-
}
|
|
5254
|
-
};
|
|
5255
|
-
case "workflow": return {
|
|
5256
|
-
axis: "routing",
|
|
5257
|
-
target: {
|
|
5258
|
-
surface: "runtime-config",
|
|
5259
|
-
path: `workflow:${subject.name}`
|
|
5260
|
-
}
|
|
5261
|
-
};
|
|
5262
|
-
case "rollout-policy": return {
|
|
5263
|
-
axis: rolloutPolicyAxis(subject.field),
|
|
5264
|
-
target: {
|
|
5265
|
-
surface: "runtime-config",
|
|
5266
|
-
path: `rollout-policy:${subject.field}`
|
|
5267
|
-
}
|
|
5268
|
-
};
|
|
5269
|
-
case "agent-profile": return {
|
|
5270
|
-
axis: "agent_profile",
|
|
5271
|
-
target: {
|
|
5272
|
-
surface: "agent-profile",
|
|
5273
|
-
path: `agent-profile:${subject.field}`
|
|
5274
|
-
}
|
|
5275
|
-
};
|
|
5276
|
-
case "code": return {
|
|
5277
|
-
axis: "representation",
|
|
5278
|
-
target: {
|
|
5279
|
-
surface: "code",
|
|
5280
|
-
path: `code:${subject.path}`
|
|
5281
|
-
}
|
|
5282
|
-
};
|
|
5283
|
-
case "rag": return {
|
|
5284
|
-
axis: "memory",
|
|
5285
|
-
target: {
|
|
5286
|
-
surface: "memory",
|
|
5287
|
-
path: `rag:${subject.corpus}:${subject.docId}`
|
|
5288
|
-
}
|
|
5289
|
-
};
|
|
5290
|
-
case "memory": return {
|
|
5291
|
-
axis: "memory",
|
|
5292
|
-
target: {
|
|
5293
|
-
surface: "memory",
|
|
5294
|
-
path: `memory:${subject.key}`
|
|
5295
|
-
}
|
|
5296
|
-
};
|
|
5297
|
-
case "scaffolding": return {
|
|
5298
|
-
axis: "routing",
|
|
5299
|
-
target: {
|
|
5300
|
-
surface: "runtime-config",
|
|
5301
|
-
path: `scaffolding:${subject.concern}`
|
|
5302
|
-
}
|
|
5303
|
-
};
|
|
5304
|
-
case "output-schema": return {
|
|
5305
|
-
axis: "output_contract",
|
|
5306
|
-
target: {
|
|
5307
|
-
surface: "runtime-config",
|
|
5308
|
-
path: `output-schema:${subject.field}`
|
|
5309
|
-
}
|
|
5310
|
-
};
|
|
5311
|
-
case "knowledge.wiki": return {
|
|
5312
|
-
axis: "memory",
|
|
5313
|
-
target: {
|
|
5314
|
-
surface: "memory",
|
|
5315
|
-
path: `agent-knowledge:wiki:${subject.slug}${subject.heading ? `#${subject.heading}` : ""}`
|
|
5316
|
-
}
|
|
5317
|
-
};
|
|
5318
|
-
case "knowledge.claim": return {
|
|
5319
|
-
axis: "memory",
|
|
5320
|
-
target: {
|
|
5321
|
-
surface: "memory",
|
|
5322
|
-
path: `agent-knowledge:claim:${subject.topic}`
|
|
5323
|
-
}
|
|
5324
|
-
};
|
|
5325
|
-
case "knowledge.raw": return {
|
|
5326
|
-
axis: "memory",
|
|
5327
|
-
target: {
|
|
5328
|
-
surface: "memory",
|
|
5329
|
-
path: `agent-knowledge:raw:${subject.sourceId}`
|
|
5330
|
-
}
|
|
5331
|
-
};
|
|
5332
|
-
case "knowledge.stale": return {
|
|
5333
|
-
axis: "memory",
|
|
5334
|
-
target: {
|
|
5335
|
-
surface: "memory",
|
|
5336
|
-
path: `agent-knowledge:stale:${subject.slug}`
|
|
5337
|
-
}
|
|
5338
|
-
};
|
|
5339
|
-
case "websearch.outdated": return {
|
|
5340
|
-
axis: "memory",
|
|
5341
|
-
target: {
|
|
5342
|
-
surface: "memory",
|
|
5343
|
-
path: `websearch:outdated:${subject.topic}`
|
|
5344
|
-
}
|
|
5345
|
-
};
|
|
5346
|
-
case "prior-run-summary": return {
|
|
5347
|
-
axis: "memory",
|
|
5348
|
-
target: {
|
|
5349
|
-
surface: "memory",
|
|
5350
|
-
path: `prior-run-summary:${subject.topic}`
|
|
5351
|
-
}
|
|
5352
|
-
};
|
|
5353
|
-
case "cluster": return {
|
|
5354
|
-
axis: "representation",
|
|
5355
|
-
target: {
|
|
5356
|
-
surface: "prompt",
|
|
5357
|
-
path: subject.label
|
|
5358
|
-
}
|
|
5359
|
-
};
|
|
5360
|
-
}
|
|
5361
|
-
}
|
|
5362
|
-
function rolloutPolicyAxis(field) {
|
|
5363
|
-
const normalized = field.toLowerCase();
|
|
5364
|
-
if (/budget|max(?:imum)?[-_. ]?(?:turns?|tokens?|cost)|timeout|deadline/.test(normalized)) return "budget";
|
|
5365
|
-
if (/temperature|top[-_. ]?p|sampling|seed|shots?|parallel|concurrency/.test(normalized)) return "sampling";
|
|
5366
|
-
if (/output|schema|format/.test(normalized)) return "output_contract";
|
|
5367
|
-
return "routing";
|
|
5368
|
-
}
|
|
5369
|
-
function resolveExpectedGain(finding, opts) {
|
|
5370
|
-
if (typeof opts.expectedGain === "function") return opts.expectedGain(finding) ?? null;
|
|
5371
|
-
if (opts.expectedGain) return opts.expectedGain;
|
|
5372
|
-
return readExpectedGainFromMetadata(finding.metadata);
|
|
5373
|
-
}
|
|
5374
|
-
function readExpectedGainFromMetadata(metadata) {
|
|
5375
|
-
const raw = readPolicyEditMetadata(metadata)?.expectedGain ?? readPolicyEditMetadata(metadata)?.expected_gain;
|
|
5376
|
-
if (!raw || typeof raw !== "object") return null;
|
|
5377
|
-
const obj = raw;
|
|
5378
|
-
if (typeof obj.metric !== "string" || obj.direction !== "increase" && obj.direction !== "decrease" || typeof obj.amount !== "number" || !Number.isFinite(obj.amount) || obj.amount <= 0) return null;
|
|
5379
|
-
const out = {
|
|
5380
|
-
metric: obj.metric,
|
|
5381
|
-
direction: obj.direction,
|
|
5382
|
-
amount: obj.amount
|
|
5383
|
-
};
|
|
5384
|
-
if (obj.unit === "absolute" || obj.unit === "relative" || obj.unit === "percent" || obj.unit === "score") out.unit = obj.unit;
|
|
5385
|
-
if (typeof obj.rationale === "string") out.rationale = obj.rationale;
|
|
5386
|
-
return out;
|
|
5387
|
-
}
|
|
5388
|
-
function readPolicyEditMetadata(metadata) {
|
|
5389
|
-
const raw = metadata?.policyEdit ?? metadata?.policy_edit;
|
|
5390
|
-
return raw && typeof raw === "object" ? raw : null;
|
|
5391
|
-
}
|
|
5392
|
-
function resolveRisk(finding, opts) {
|
|
5393
|
-
if (typeof opts.risk === "function") return opts.risk(finding);
|
|
5394
|
-
if (opts.risk) return opts.risk;
|
|
5395
|
-
const raw = readPolicyEditMetadata(finding.metadata)?.risk;
|
|
5396
|
-
if (raw === "low" || raw === "medium" || raw === "high" || raw === "unknown") return raw;
|
|
5397
|
-
if (finding.severity === "critical" || finding.severity === "high") return "medium";
|
|
5398
|
-
return "low";
|
|
5399
|
-
}
|
|
5400
|
-
function applyTextChange(surface, change) {
|
|
5401
|
-
if (typeof surface !== "string") throw new PolicyEditValidationError("text policy edits require a string surface", "change");
|
|
5402
|
-
if (change.mode === "append") {
|
|
5403
|
-
if (hasExactTextBlock(surface, change.value)) return surface;
|
|
5404
|
-
return `${surface.trimEnd()}\n\n${change.value}`.trimStart();
|
|
5405
|
-
}
|
|
5406
|
-
if (change.mode === "prepend") {
|
|
5407
|
-
if (hasExactTextBlock(surface, change.value)) return surface;
|
|
5408
|
-
return `${change.value}\n\n${surface.trimStart()}`.trimEnd();
|
|
5409
|
-
}
|
|
5410
|
-
const find = expectNonEmpty(change.find, "change.find");
|
|
5411
|
-
if (!surface.includes(find)) throw new PolicyEditValidationError("replace target not found in surface", "change.find");
|
|
5412
|
-
return surface.replace(find, change.value);
|
|
5413
|
-
}
|
|
5414
|
-
function applyJsonChange(surface, change) {
|
|
5415
|
-
const root = parseJsonSurface(surface);
|
|
5416
|
-
const path = splitPath(change.path);
|
|
5417
|
-
if (change.mode === "remove") return setJsonAtPath(root, path, void 0, "remove");
|
|
5418
|
-
if (change.mode === "set") return setJsonAtPath(root, path, change.value ?? null, "set");
|
|
5419
|
-
const prior = readJsonAtPath(root, path);
|
|
5420
|
-
return setJsonAtPath(root, path, prior && typeof prior === "object" && !Array.isArray(prior) && change.value && typeof change.value === "object" && !Array.isArray(change.value) ? {
|
|
5421
|
-
...prior,
|
|
5422
|
-
...change.value
|
|
5423
|
-
} : change.value ?? null, "set");
|
|
5424
|
-
}
|
|
5425
|
-
function parseJsonSurface(surface) {
|
|
5426
|
-
if (typeof surface === "string") try {
|
|
5427
|
-
return JSON.parse(surface);
|
|
5428
|
-
} catch {
|
|
5429
|
-
throw new PolicyEditValidationError("json policy edits require a JSON string surface", "change");
|
|
5430
|
-
}
|
|
5431
|
-
assertJson(surface, "surface");
|
|
5432
|
-
return surface;
|
|
5433
|
-
}
|
|
5434
|
-
function readJsonAtPath(root, path) {
|
|
5435
|
-
let cursor = root;
|
|
5436
|
-
for (const part of path) {
|
|
5437
|
-
if (!cursor || typeof cursor !== "object" || Array.isArray(cursor)) return void 0;
|
|
5438
|
-
cursor = cursor[part];
|
|
5439
|
-
}
|
|
5440
|
-
return cursor;
|
|
5441
|
-
}
|
|
5442
|
-
function setJsonAtPath(root, path, value, mode) {
|
|
5443
|
-
if (path.length === 0) {
|
|
5444
|
-
if (mode === "remove") return null;
|
|
5445
|
-
return value ?? null;
|
|
5446
|
-
}
|
|
5447
|
-
if (root === null || typeof root !== "object" || Array.isArray(root)) throw new PolicyEditValidationError("json edit root must be an object", "change.path");
|
|
5448
|
-
const out = { ...root };
|
|
5449
|
-
let cursor = out;
|
|
5450
|
-
for (let i = 0; i < path.length - 1; i++) {
|
|
5451
|
-
const key = path[i];
|
|
5452
|
-
const existing = cursor[key];
|
|
5453
|
-
if (mode === "remove" && (!existing || typeof existing !== "object" || Array.isArray(existing))) return out;
|
|
5454
|
-
const next = existing && typeof existing === "object" && !Array.isArray(existing) ? { ...existing } : {};
|
|
5455
|
-
cursor[key] = next;
|
|
5456
|
-
cursor = next;
|
|
5457
|
-
}
|
|
5458
|
-
const leaf = path[path.length - 1];
|
|
5459
|
-
if (mode === "remove") delete cursor[leaf];
|
|
5460
|
-
else cursor[leaf] = value ?? null;
|
|
5461
|
-
return out;
|
|
5462
|
-
}
|
|
5463
|
-
function normalizePolicyEdit(input) {
|
|
5464
|
-
const out = {
|
|
5465
|
-
schemaVersion: "policy-edit/v1",
|
|
5466
|
-
axis: input.axis,
|
|
5467
|
-
target: normalizeTarget(input.target),
|
|
5468
|
-
change: normalizeChange(input.change),
|
|
5469
|
-
claim: input.claim.trim(),
|
|
5470
|
-
expectedGain: normalizeExpectedGain(input.expectedGain),
|
|
5471
|
-
confidence: input.confidence,
|
|
5472
|
-
risk: input.risk,
|
|
5473
|
-
source: normalizeSource(input.source)
|
|
5474
|
-
};
|
|
5475
|
-
if (input.rationale?.trim()) out.rationale = input.rationale.trim();
|
|
5476
|
-
if (input.validationPlan?.trim()) out.validationPlan = input.validationPlan.trim();
|
|
5477
|
-
if (input.metadata) out.metadata = input.metadata;
|
|
5478
|
-
return out;
|
|
5479
|
-
}
|
|
5480
4788
|
function assertJsonSafe(value, path, ancestors = /* @__PURE__ */ new WeakSet()) {
|
|
5481
4789
|
if (value === null || typeof value === "string" || typeof value === "boolean") return;
|
|
5482
4790
|
if (typeof value === "number") {
|
|
@@ -5498,50 +4806,6 @@ function assertJsonSafe(value, path, ancestors = /* @__PURE__ */ new WeakSet())
|
|
|
5498
4806
|
}
|
|
5499
4807
|
ancestors.delete(value);
|
|
5500
4808
|
}
|
|
5501
|
-
function normalizeTarget(target) {
|
|
5502
|
-
const out = { surface: target.surface };
|
|
5503
|
-
if (target.path?.trim()) out.path = target.path.trim();
|
|
5504
|
-
if (target.agentProfileCell) out.agentProfileCell = validateAgentProfileCell(target.agentProfileCell);
|
|
5505
|
-
if (target.label?.trim()) out.label = target.label.trim();
|
|
5506
|
-
return out;
|
|
5507
|
-
}
|
|
5508
|
-
function normalizeChange(change) {
|
|
5509
|
-
if (change.kind === "text") {
|
|
5510
|
-
const out = {
|
|
5511
|
-
kind: "text",
|
|
5512
|
-
mode: change.mode,
|
|
5513
|
-
value: change.value.trim()
|
|
5514
|
-
};
|
|
5515
|
-
if (change.find?.trim()) out.find = change.find.trim();
|
|
5516
|
-
return out;
|
|
5517
|
-
}
|
|
5518
|
-
const out = {
|
|
5519
|
-
kind: "json",
|
|
5520
|
-
mode: change.mode,
|
|
5521
|
-
path: change.path.trim()
|
|
5522
|
-
};
|
|
5523
|
-
if (change.value !== void 0) out.value = change.value;
|
|
5524
|
-
return out;
|
|
5525
|
-
}
|
|
5526
|
-
function normalizeExpectedGain(gain) {
|
|
5527
|
-
const out = {
|
|
5528
|
-
metric: gain.metric.trim(),
|
|
5529
|
-
direction: gain.direction,
|
|
5530
|
-
amount: gain.amount
|
|
5531
|
-
};
|
|
5532
|
-
if (gain.unit) out.unit = gain.unit;
|
|
5533
|
-
if (gain.rationale?.trim()) out.rationale = gain.rationale.trim();
|
|
5534
|
-
return out;
|
|
5535
|
-
}
|
|
5536
|
-
function normalizeSource(source) {
|
|
5537
|
-
const out = {
|
|
5538
|
-
findingIds: uniqueSorted(source.findingIds.map((s) => s.trim()).filter(Boolean)),
|
|
5539
|
-
analystIds: uniqueSorted(source.analystIds.map((s) => s.trim()).filter(Boolean)),
|
|
5540
|
-
evidenceRefs: source.evidenceRefs
|
|
5541
|
-
};
|
|
5542
|
-
if (source.derivedFromJudge) out.derivedFromJudge = true;
|
|
5543
|
-
return out;
|
|
5544
|
-
}
|
|
5545
4809
|
function validateTarget(target) {
|
|
5546
4810
|
if (!target || typeof target !== "object") throw new PolicyEditValidationError("expected object", "target");
|
|
5547
4811
|
const obj = target;
|
|
@@ -5624,14 +4888,6 @@ function assertJson(value, path) {
|
|
|
5624
4888
|
}
|
|
5625
4889
|
throw new PolicyEditValidationError("expected JSON-compatible value", path);
|
|
5626
4890
|
}
|
|
5627
|
-
function targetSpecificityScore(edit) {
|
|
5628
|
-
let score = .4;
|
|
5629
|
-
if (edit.target.path) score += .25;
|
|
5630
|
-
if (edit.target.agentProfileCell) score += .15;
|
|
5631
|
-
if (edit.change.kind === "json" || edit.change.mode === "replace") score += .2;
|
|
5632
|
-
else if (edit.change.value.length > 0) score += .1;
|
|
5633
|
-
return clamp01$3(score);
|
|
5634
|
-
}
|
|
5635
4891
|
const FORBIDDEN_PATH_KEYS = /* @__PURE__ */ new Set([
|
|
5636
4892
|
"__proto__",
|
|
5637
4893
|
"constructor",
|
|
@@ -5649,21 +4905,9 @@ function expectLiteral(value, expected, path) {
|
|
|
5649
4905
|
function expectString$1(value, path) {
|
|
5650
4906
|
if (typeof value !== "string" || value.trim().length === 0) throw new PolicyEditValidationError("expected non-empty string", path);
|
|
5651
4907
|
}
|
|
5652
|
-
function expectNonEmpty(value, path) {
|
|
5653
|
-
expectString$1(value, path);
|
|
5654
|
-
return value;
|
|
5655
|
-
}
|
|
5656
4908
|
function expectConfidence(value, path) {
|
|
5657
4909
|
if (typeof value !== "number" || !Number.isFinite(value) || value < 0 || value > 1) throw new PolicyEditValidationError("expected finite number in [0,1]", path);
|
|
5658
4910
|
}
|
|
5659
|
-
function hasExactTextBlock(surface, value) {
|
|
5660
|
-
const needle = normalizeTextBlock(value);
|
|
5661
|
-
const normalizedSurface = surface.replace(/\r\n/g, "\n");
|
|
5662
|
-
return [...normalizedSurface.split(/\n{2,}/), ...normalizedSurface.split("\n")].some((block) => normalizeTextBlock(block) === needle);
|
|
5663
|
-
}
|
|
5664
|
-
function normalizeTextBlock(value) {
|
|
5665
|
-
return value.replace(/\r\n/g, "\n").trim();
|
|
5666
|
-
}
|
|
5667
4911
|
function expectOneOf(value, allowed, path) {
|
|
5668
4912
|
if (typeof value !== "string" || !allowed.includes(value)) throw new PolicyEditValidationError(`expected one of ${allowed.join(", ")}`, path);
|
|
5669
4913
|
}
|
|
@@ -5675,15 +4919,6 @@ function expectNonEmptyStringArray(value, path) {
|
|
|
5675
4919
|
expectStringArray$1(value, path);
|
|
5676
4920
|
if (value.length === 0) throw new PolicyEditValidationError("expected non-empty array", path);
|
|
5677
4921
|
}
|
|
5678
|
-
function uniqueSorted(values) {
|
|
5679
|
-
return [...new Set(values)].sort();
|
|
5680
|
-
}
|
|
5681
|
-
function clamp01$3(n) {
|
|
5682
|
-
if (!Number.isFinite(n)) return 0;
|
|
5683
|
-
if (n < 0) return 0;
|
|
5684
|
-
if (n > 1) return 1;
|
|
5685
|
-
return n;
|
|
5686
|
-
}
|
|
5687
4922
|
//#endregion
|
|
5688
4923
|
//#region src/capability-headroom.ts
|
|
5689
4924
|
/**
|
|
@@ -6248,58 +5483,6 @@ function partitionHeldOut(ids, options = {}) {
|
|
|
6248
5483
|
function assertFraction(f) {
|
|
6249
5484
|
if (!Number.isFinite(f) || f <= 0 || f >= 1) throw new ValidationError(`partitionHeldOut: holdoutFraction must be in (0, 1), got ${f}`);
|
|
6250
5485
|
}
|
|
6251
|
-
/**
|
|
6252
|
-
* Partition items into disjoint train, selection and test sets, the three
|
|
6253
|
-
* partitions `compareOptimizationMethods` and Runtime `improve()` take.
|
|
6254
|
-
*
|
|
6255
|
-
* Assignment is per unit: `hashToUnit(unit, seed)` below `testFraction` is
|
|
6256
|
-
* test, below `testFraction + selectionFraction` is selection, and the rest is
|
|
6257
|
-
* train. A unit therefore keeps its partition as the corpus grows, and items of
|
|
6258
|
-
* one unit never straddle two partitions. Fail-loud like `partitionHeldOut`:
|
|
6259
|
-
* empty input, a duplicate or empty id, an empty unit, fractions outside
|
|
6260
|
-
* (0, 1) or summing to 1 or more, and a partition below its floor all throw.
|
|
6261
|
-
* Order within each partition follows the input order.
|
|
6262
|
-
*/
|
|
6263
|
-
function partitionTrainSelectionTest(items, options = {}) {
|
|
6264
|
-
const seed = options.seed ?? "train-selection-test-v1";
|
|
6265
|
-
const selectionFraction = options.selectionFraction ?? .25;
|
|
6266
|
-
const testFraction = options.testFraction ?? .25;
|
|
6267
|
-
assertSplitFraction("selectionFraction", selectionFraction);
|
|
6268
|
-
assertSplitFraction("testFraction", testFraction);
|
|
6269
|
-
if (selectionFraction + testFraction >= 1) throw new ValidationError(`partitionTrainSelectionTest: selectionFraction + testFraction must be below 1, got ${selectionFraction + testFraction}`);
|
|
6270
|
-
if (items.length === 0) throw new ValidationError("partitionTrainSelectionTest: no items supplied");
|
|
6271
|
-
const unitOf = options.unitOf ?? ((item) => item.id);
|
|
6272
|
-
const seen = /* @__PURE__ */ new Set();
|
|
6273
|
-
const partition = {
|
|
6274
|
-
train: [],
|
|
6275
|
-
selection: [],
|
|
6276
|
-
test: [],
|
|
6277
|
-
seed,
|
|
6278
|
-
selectionFraction,
|
|
6279
|
-
testFraction
|
|
6280
|
-
};
|
|
6281
|
-
for (const item of items) {
|
|
6282
|
-
if (typeof item.id !== "string" || item.id.length === 0) throw new ValidationError(`partitionTrainSelectionTest: ids must be non-empty strings, got ${JSON.stringify(item.id)}`);
|
|
6283
|
-
if (seen.has(item.id)) throw new ValidationError(`partitionTrainSelectionTest: duplicate id "${item.id}" — one item cannot be scored in two partitions`);
|
|
6284
|
-
seen.add(item.id);
|
|
6285
|
-
const unit = unitOf(item);
|
|
6286
|
-
if (typeof unit !== "string" || unit.length === 0) throw new ValidationError(`partitionTrainSelectionTest: item "${item.id}" has no unit`);
|
|
6287
|
-
const position = hashToUnit(unit, seed);
|
|
6288
|
-
if (position < testFraction) partition.test.push(item);
|
|
6289
|
-
else if (position < testFraction + selectionFraction) partition.selection.push(item);
|
|
6290
|
-
else partition.train.push(item);
|
|
6291
|
-
}
|
|
6292
|
-
const floors = [
|
|
6293
|
-
["train", options.minTrain ?? 1],
|
|
6294
|
-
["selection", options.minSelection ?? 1],
|
|
6295
|
-
["test", options.minTest ?? 1]
|
|
6296
|
-
];
|
|
6297
|
-
for (const [name, floor] of floors) if (partition[name].length < floor) throw new ValidationError(`partitionTrainSelectionTest: ${name} has ${partition[name].length} item(s), below the floor of ${floor} (n=${items.length}, selectionFraction=${selectionFraction}, testFraction=${testFraction}, seed=${seed})`);
|
|
6298
|
-
return partition;
|
|
6299
|
-
}
|
|
6300
|
-
function assertSplitFraction(name, value) {
|
|
6301
|
-
if (!Number.isFinite(value) || value <= 0 || value >= 1) throw new ValidationError(`partitionTrainSelectionTest: ${name} must be in (0, 1), got ${value}`);
|
|
6302
|
-
}
|
|
6303
5486
|
//#endregion
|
|
6304
5487
|
//#region src/product-benchmark/export.ts
|
|
6305
5488
|
/**
|
|
@@ -7264,9 +6447,6 @@ function parseRuntimeTrajectoryHookEvent(input) {
|
|
|
7264
6447
|
function runtimeTrajectoryRecordKey(record, index, recordIdOf) {
|
|
7265
6448
|
return stringOrUndefined(recordIdOf?.(record, index)) ?? stringOrUndefined(record.id) ?? `record[${index}]`;
|
|
7266
6449
|
}
|
|
7267
|
-
function isRecord(value) {
|
|
7268
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
7269
|
-
}
|
|
7270
6450
|
function stringOrUndefined(value) {
|
|
7271
6451
|
return typeof value === "string" && value.length > 0 ? value : void 0;
|
|
7272
6452
|
}
|
|
@@ -7959,6 +7139,40 @@ function fmt(x) {
|
|
|
7959
7139
|
}
|
|
7960
7140
|
//#endregion
|
|
7961
7141
|
//#region src/hidden-criteria-grading.ts
|
|
7142
|
+
/**
|
|
7143
|
+
* Hidden-criteria grading firewall — grade an agent on criteria it never saw.
|
|
7144
|
+
*
|
|
7145
|
+
* A trustworthy benchmark splits every scenario's data by WHERE each field is
|
|
7146
|
+
* allowed to flow, then proves the held-out / judge-only fields never reach the
|
|
7147
|
+
* agent during the run. The coding bench expresses this with four destinations
|
|
7148
|
+
* (prompt / develop-against / held-out suite / rubric); this module lifts the
|
|
7149
|
+
* domain-FREE core out of it so research, legal, tax, content — any domain —
|
|
7150
|
+
* can declare the same routing and get the same firewall enforcement and the
|
|
7151
|
+
* same held-out-weighted composite, plugging in its OWN grader.
|
|
7152
|
+
*
|
|
7153
|
+
* Two reusable pieces, both domain-agnostic:
|
|
7154
|
+
*
|
|
7155
|
+
* 1. FIELD ROUTING BY DESTINATION. A scenario declares each field's
|
|
7156
|
+
* `FieldDestination`; `assertNoHiddenLeak` is a pure checker that throws if
|
|
7157
|
+
* a grading-only or judge-only field's value appears in what reaches the
|
|
7158
|
+
* agent. The domain decides which fields exist and where they go — the
|
|
7159
|
+
* substrate only enforces "hidden stays hidden".
|
|
7160
|
+
*
|
|
7161
|
+
* 2. HIDDEN-CRITERIA GRADING. The domain supplies its own grader
|
|
7162
|
+
* `(artifact, hiddenCriteria) => { passRate, total }` — the coding
|
|
7163
|
+
* node-test executor is ONE such grader a consumer plugs in; the substrate
|
|
7164
|
+
* bakes in NO node/test/TS/exec/regex. `gradeOnHidden` runs that grader
|
|
7165
|
+
* behind the firewall and `blendHeldout` composes its pass rate with a
|
|
7166
|
+
* judge score into the final number the leaderboard ranks on.
|
|
7167
|
+
*
|
|
7168
|
+
* Shape mirrors `treatment-gate`/`authenticity`: pure predicates and pure
|
|
7169
|
+
* composition over already-computed values, fail-loud, with the
|
|
7170
|
+
* "which field / which weight / which grader" decisions left as parameters and
|
|
7171
|
+
* no domain literal anywhere in the module.
|
|
7172
|
+
*
|
|
7173
|
+
* Lives next to `test-graded-scenario` and `partition-held-out` — it is a
|
|
7174
|
+
* scorecard/grading concept that makes sense without a running agent loop.
|
|
7175
|
+
*/
|
|
7962
7176
|
/** The destinations a value must be kept OUT of the agent context for. */
|
|
7963
7177
|
const hiddenDestinations = /* @__PURE__ */ new Set(["grading-only", "judge-only"]);
|
|
7964
7178
|
/** True for the destinations whose values must never reach the agent context. */
|
|
@@ -7966,26 +7180,6 @@ function isHiddenDestination(destination) {
|
|
|
7966
7180
|
return hiddenDestinations.has(destination);
|
|
7967
7181
|
}
|
|
7968
7182
|
/**
|
|
7969
|
-
* Route a domain's named fields by destination into the firewall's input shape.
|
|
7970
|
-
* The `routing` declares each field's destination; the `values` carry each
|
|
7971
|
-
* field's renderable string. A field present in `routing` but missing from
|
|
7972
|
-
* `values` is an authoring error (fail loud) — every routed field must have a
|
|
7973
|
-
* value the firewall can check.
|
|
7974
|
-
*/
|
|
7975
|
-
function routeFields(routing, values) {
|
|
7976
|
-
const out = [];
|
|
7977
|
-
for (const name of Object.keys(routing)) {
|
|
7978
|
-
const value = values[name];
|
|
7979
|
-
if (value === void 0) throw new ValidationError(`routed field "${name}" has a destination but no value — every routed field must carry its value`);
|
|
7980
|
-
out.push({
|
|
7981
|
-
name,
|
|
7982
|
-
value,
|
|
7983
|
-
destination: routing[name]
|
|
7984
|
-
});
|
|
7985
|
-
}
|
|
7986
|
-
return out;
|
|
7987
|
-
}
|
|
7988
|
-
/**
|
|
7989
7183
|
* The FIREWALL. Throws `ValidationError` if any `grading-only`/`judge-only`
|
|
7990
7184
|
* field's value is found inside `agentContext` — the exact text that reaches the
|
|
7991
7185
|
* agent during the run (its prompt, its seeded files concatenated, whatever the
|
|
@@ -8014,34 +7208,6 @@ function assertNoHiddenLeak(fields, agentContext, opts = {}) {
|
|
|
8014
7208
|
}
|
|
8015
7209
|
return fields;
|
|
8016
7210
|
}
|
|
8017
|
-
/** Normalize a grader's raw `{passed, total}` into a `HiddenGradeResult` with a
|
|
8018
|
-
* consistent, fail-loud `passRate` — the canonical "honest zero on no-run"
|
|
8019
|
-
* rule, single-sourced so every domain grader gets it. */
|
|
8020
|
-
function hiddenGrade(passed, total, notes) {
|
|
8021
|
-
const p = Number.isFinite(passed) && passed > 0 ? Math.floor(passed) : 0;
|
|
8022
|
-
const t = Number.isFinite(total) && total > 0 ? Math.floor(total) : 0;
|
|
8023
|
-
const passRate = t > 0 ? Math.min(1, p / t) : 0;
|
|
8024
|
-
return {
|
|
8025
|
-
passed: Math.min(p, t > 0 ? t : p),
|
|
8026
|
-
total: t,
|
|
8027
|
-
passRate,
|
|
8028
|
-
notes
|
|
8029
|
-
};
|
|
8030
|
-
}
|
|
8031
|
-
/**
|
|
8032
|
-
* Run a domain's hidden-criteria grader behind the firewall. Before grading, it
|
|
8033
|
-
* re-asserts the firewall against the agent context the run actually used —
|
|
8034
|
-
* proving (at grading time, on real data) that the hidden criteria never
|
|
8035
|
-
* reached the agent — then invokes the grader and returns its pass rate. A
|
|
8036
|
-
* domain that wants the firewall and the grader wired together in one call uses
|
|
8037
|
-
* this; a domain that already asserted the firewall at dispatch time can call
|
|
8038
|
-
* its grader directly and feed the result to `blendHeldout`.
|
|
8039
|
-
*/
|
|
8040
|
-
async function gradeOnHidden(args) {
|
|
8041
|
-
assertNoHiddenLeak(args.firewall.fields, args.firewall.agentContext, args.firewall.options);
|
|
8042
|
-
const result = await args.grader(args.artifact, args.hiddenCriteria, args.signal);
|
|
8043
|
-
return hiddenGrade(result.passed, result.total, result.notes);
|
|
8044
|
-
}
|
|
8045
7211
|
/** Default blend: 0.7 hidden correctness, 0.3 judge quality. The coding bench's
|
|
8046
7212
|
* long-standing split — execution truth dominates, style refines. */
|
|
8047
7213
|
const defaultBlendWeights = {
|
|
@@ -8072,33 +7238,6 @@ function blendHeldout(heldoutPassRate, judgeScore, weights = defaultBlendWeights
|
|
|
8072
7238
|
const judge = clampUnit(judgeScore);
|
|
8073
7239
|
return w.heldout * heldout + w.judge * judge;
|
|
8074
7240
|
}
|
|
8075
|
-
/**
|
|
8076
|
-
* Wrap a judge's `score` so the `composite` it REPORTS is the held-out-weighted
|
|
8077
|
-
* blend. The judge still scores its quality dimensions (recorded, secondary),
|
|
8078
|
-
* but the composite that downstream selection/scorecard reads becomes
|
|
8079
|
-
* `blendHeldout(heldoutPassRate(artifact), judgeComposite, weights)`. The held-
|
|
8080
|
-
* out pass rate is read off the artifact via `heldoutPassRate` — already
|
|
8081
|
-
* computed before the judge runs — so no second grading pass is needed.
|
|
8082
|
-
*
|
|
8083
|
-
* Generic over the artifact type, inferred from `heldoutPassRate`, so it
|
|
8084
|
-
* composes with both a `campaign` `JudgeConfig.score` and a bare scoring
|
|
8085
|
-
* function. The input is the judge's `{ artifact, ... }` — any extra fields
|
|
8086
|
-
* (`scenario`, `signal`) ride through untouched via the index signature.
|
|
8087
|
-
*/
|
|
8088
|
-
function withHeldoutBlend(score, heldoutPassRate, weights = defaultBlendWeights) {
|
|
8089
|
-
return async (input) => {
|
|
8090
|
-
const base = await score(input);
|
|
8091
|
-
if (base.failed) return base;
|
|
8092
|
-
const rate = clampUnit(heldoutPassRate(input.artifact));
|
|
8093
|
-
const composite = blendHeldout(rate, base.composite, weights);
|
|
8094
|
-
const w = normalizeWeights(weights);
|
|
8095
|
-
return {
|
|
8096
|
-
...base,
|
|
8097
|
-
composite,
|
|
8098
|
-
notes: `composite=${composite.toFixed(3)} (held-out ${(rate * 100).toFixed(0)}% × ${w.heldout.toFixed(2)} + quality ${base.composite.toFixed(3)} × ${w.judge.toFixed(2)})` + (base.notes ? ` — ${base.notes}` : "")
|
|
8099
|
-
};
|
|
8100
|
-
};
|
|
8101
|
-
}
|
|
8102
7241
|
function clampUnit(value) {
|
|
8103
7242
|
if (!Number.isFinite(value)) return 0;
|
|
8104
7243
|
return Math.max(0, Math.min(1, value));
|
|
@@ -8181,12 +7320,12 @@ function assertSingleBackend(agent, judge, opts = {}) {
|
|
|
8181
7320
|
//#endregion
|
|
8182
7321
|
//#region src/intent-match-judge.ts
|
|
8183
7322
|
const INTENT_MATCH_JUDGE_VERSION = "intent-match-judge-v1-2026-04-24";
|
|
8184
|
-
const DEFAULT_MODEL = "claude-sonnet-4-6";
|
|
8185
|
-
const DEFAULT_TIMEOUT = 3e5;
|
|
8186
|
-
const DEFAULT_MAX_TOKENS = 800;
|
|
8187
|
-
const DEFAULT_MAX_SOURCE = 25e3;
|
|
8188
|
-
const DEFAULT_MAX_PER_FILE = 12e3;
|
|
8189
|
-
const DEFAULT_MAX_HTML = 2e4;
|
|
7323
|
+
const DEFAULT_MODEL$1 = "claude-sonnet-4-6";
|
|
7324
|
+
const DEFAULT_TIMEOUT$1 = 3e5;
|
|
7325
|
+
const DEFAULT_MAX_TOKENS$1 = 800;
|
|
7326
|
+
const DEFAULT_MAX_SOURCE$1 = 25e3;
|
|
7327
|
+
const DEFAULT_MAX_PER_FILE$1 = 12e3;
|
|
7328
|
+
const DEFAULT_MAX_HTML$1 = 2e4;
|
|
8190
7329
|
const INTENT_SCHEMA = {
|
|
8191
7330
|
type: "object",
|
|
8192
7331
|
additionalProperties: false,
|
|
@@ -8204,11 +7343,11 @@ const INTENT_SCHEMA = {
|
|
|
8204
7343
|
}
|
|
8205
7344
|
}
|
|
8206
7345
|
};
|
|
8207
|
-
function truncate(body, cap, label) {
|
|
7346
|
+
function truncate$1(body, cap, label) {
|
|
8208
7347
|
if (body.length <= cap) return body;
|
|
8209
7348
|
return `${body.slice(0, cap)}\n… [truncated ${body.length - cap} chars of ${label}]`;
|
|
8210
7349
|
}
|
|
8211
|
-
function buildPrompt(input, opts) {
|
|
7350
|
+
function buildPrompt$1(input, opts) {
|
|
8212
7351
|
const sourceBlob = input.sourceFiles.filter((f) => f.content.length <= opts.maxPerFileChars).map((f) => `--- FILE: ${f.path} ---\n${f.content}`).join("\n\n");
|
|
8213
7352
|
const html = input.servedHtml ?? "";
|
|
8214
7353
|
return `You are evaluating whether an agent built THE RIGHT APP for a user request.
|
|
@@ -8221,8 +7360,8 @@ project entirely?
|
|
|
8221
7360
|
USER REQUEST:
|
|
8222
7361
|
${input.userRequest}
|
|
8223
7362
|
|
|
8224
|
-
${input.artifactLabel ? `ARTIFACT METADATA:\n name: ${input.artifactLabel}\n description: ${input.artifactDescription ?? ""}\n\n` : ""}${html ? `SERVED HTML (what the preview returns):\n${truncate(html, opts.maxHtmlChars, "HTML")}\n\n` : ""}SOURCE FILES (the agent's workdir):
|
|
8225
|
-
${truncate(sourceBlob, opts.maxSourceChars, "source")}
|
|
7363
|
+
${input.artifactLabel ? `ARTIFACT METADATA:\n name: ${input.artifactLabel}\n description: ${input.artifactDescription ?? ""}\n\n` : ""}${html ? `SERVED HTML (what the preview returns):\n${truncate$1(html, opts.maxHtmlChars, "HTML")}\n\n` : ""}SOURCE FILES (the agent's workdir):
|
|
7364
|
+
${truncate$1(sourceBlob, opts.maxSourceChars, "source")}
|
|
8226
7365
|
|
|
8227
7366
|
Score 0–1:
|
|
8228
7367
|
1.0 — unmistakably the right app. Even with bugs, gaps, or missing
|
|
@@ -8254,12 +7393,12 @@ async function runIntentMatchJudge(input, options) {
|
|
|
8254
7393
|
const start = Date.now();
|
|
8255
7394
|
const opts = {
|
|
8256
7395
|
chat: options.chat,
|
|
8257
|
-
model: options.model ?? options.chat.defaultModel ?? DEFAULT_MODEL,
|
|
8258
|
-
timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT,
|
|
8259
|
-
maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS,
|
|
8260
|
-
maxSourceChars: options.maxSourceChars ?? DEFAULT_MAX_SOURCE,
|
|
8261
|
-
maxPerFileChars: options.maxPerFileChars ?? DEFAULT_MAX_PER_FILE,
|
|
8262
|
-
maxHtmlChars: options.maxHtmlChars ?? DEFAULT_MAX_HTML,
|
|
7396
|
+
model: options.model ?? options.chat.defaultModel ?? DEFAULT_MODEL$1,
|
|
7397
|
+
timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT$1,
|
|
7398
|
+
maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS$1,
|
|
7399
|
+
maxSourceChars: options.maxSourceChars ?? DEFAULT_MAX_SOURCE$1,
|
|
7400
|
+
maxPerFileChars: options.maxPerFileChars ?? DEFAULT_MAX_PER_FILE$1,
|
|
7401
|
+
maxHtmlChars: options.maxHtmlChars ?? DEFAULT_MAX_HTML$1,
|
|
8263
7402
|
...options.pricing ? { pricing: options.pricing } : {},
|
|
8264
7403
|
costLedger: options.costLedger ?? new CostLedger(),
|
|
8265
7404
|
costPhase: options.costPhase ?? "judge.intent-match",
|
|
@@ -8285,7 +7424,7 @@ async function runIntentMatchJudge(input, options) {
|
|
|
8285
7424
|
content: "You are a holistic code reviewer answering one question: did the agent build the right app for the user. Return strict JSON. No prose outside."
|
|
8286
7425
|
}, {
|
|
8287
7426
|
role: "user",
|
|
8288
|
-
content: buildPrompt(input, opts)
|
|
7427
|
+
content: buildPrompt$1(input, opts)
|
|
8289
7428
|
}],
|
|
8290
7429
|
jsonSchema: {
|
|
8291
7430
|
name: "intent_match_judge",
|
|
@@ -8960,38 +8099,6 @@ function mergeSignals(a, b) {
|
|
|
8960
8099
|
* `'test'`-strategy certification: an oracle set IS a small answer key,
|
|
8961
8100
|
* and certifies nothing outside its own coverage.
|
|
8962
8101
|
*/
|
|
8963
|
-
function textInSnapshot(needle, opts = {}) {
|
|
8964
|
-
const id = `text-in-snapshot(${needle})`;
|
|
8965
|
-
return {
|
|
8966
|
-
id,
|
|
8967
|
-
check(obs) {
|
|
8968
|
-
const hay = obs.text ?? "";
|
|
8969
|
-
const found = opts.caseSensitive ? hay.includes(needle) : hay.toLowerCase().includes(needle.toLowerCase());
|
|
8970
|
-
return {
|
|
8971
|
-
id,
|
|
8972
|
-
pass: found,
|
|
8973
|
-
detail: found ? `"${needle}" found` : `"${needle}" not present in observation`,
|
|
8974
|
-
evidence: found ? excerpt(hay, needle, opts.caseSensitive) : void 0
|
|
8975
|
-
};
|
|
8976
|
-
}
|
|
8977
|
-
};
|
|
8978
|
-
}
|
|
8979
|
-
function urlContains(fragment) {
|
|
8980
|
-
const id = `url-contains(${fragment})`;
|
|
8981
|
-
return {
|
|
8982
|
-
id,
|
|
8983
|
-
check(obs) {
|
|
8984
|
-
const url = obs.url ?? "";
|
|
8985
|
-
const pass = url.toLowerCase().includes(fragment.toLowerCase());
|
|
8986
|
-
return {
|
|
8987
|
-
id,
|
|
8988
|
-
pass,
|
|
8989
|
-
detail: pass ? `url ok (${url})` : `url "${url}" missing "${fragment}"`,
|
|
8990
|
-
evidence: url
|
|
8991
|
-
};
|
|
8992
|
-
}
|
|
8993
|
-
};
|
|
8994
|
-
}
|
|
8995
8102
|
function jsonShape(expected) {
|
|
8996
8103
|
const id = `json-shape(${Object.keys(expected).join(",")})`;
|
|
8997
8104
|
return {
|
|
@@ -9031,77 +8138,6 @@ function jsonShape(expected) {
|
|
|
9031
8138
|
}
|
|
9032
8139
|
};
|
|
9033
8140
|
}
|
|
9034
|
-
function regexMatches(pattern) {
|
|
9035
|
-
const id = `regex(${pattern.source})`;
|
|
9036
|
-
return {
|
|
9037
|
-
id,
|
|
9038
|
-
check(obs) {
|
|
9039
|
-
const m = (obs.text ?? "").match(pattern);
|
|
9040
|
-
return {
|
|
9041
|
-
id,
|
|
9042
|
-
pass: m !== null,
|
|
9043
|
-
detail: m ? `matched "${m[0]}"` : `pattern ${pattern.source} not matched`,
|
|
9044
|
-
evidence: m?.[0]
|
|
9045
|
-
};
|
|
9046
|
-
}
|
|
9047
|
-
};
|
|
9048
|
-
}
|
|
9049
|
-
/**
|
|
9050
|
-
* Anti-bot detector — distinguishes genuine failures from blocked navigation
|
|
9051
|
-
* (cloudflare, recaptcha, etc). Returns an Oracle that PASSES when no block
|
|
9052
|
-
* marker is present; on block, detail names the blocker so runners can tag
|
|
9053
|
-
* results as "blocked" rather than "failed". Lifted from browser-agent-driver.
|
|
9054
|
-
*/
|
|
9055
|
-
function notBlocked() {
|
|
9056
|
-
const id = "not-blocked";
|
|
9057
|
-
const markers = [
|
|
9058
|
-
{
|
|
9059
|
-
name: "cloudflare",
|
|
9060
|
-
re: /just a moment|verifying you are human|cf-chl-|cloudflare/i
|
|
9061
|
-
},
|
|
9062
|
-
{
|
|
9063
|
-
name: "recaptcha",
|
|
9064
|
-
re: /recaptcha|i'?m not a robot|challenge.?form/i
|
|
9065
|
-
},
|
|
9066
|
-
{
|
|
9067
|
-
name: "hcaptcha",
|
|
9068
|
-
re: /hcaptcha/i
|
|
9069
|
-
},
|
|
9070
|
-
{
|
|
9071
|
-
name: "akamai",
|
|
9072
|
-
re: /akamai|pragma: no-cache/i
|
|
9073
|
-
},
|
|
9074
|
-
{
|
|
9075
|
-
name: "perimeterx",
|
|
9076
|
-
re: /perimeterx|px-captcha/i
|
|
9077
|
-
},
|
|
9078
|
-
{
|
|
9079
|
-
name: "rate-limit",
|
|
9080
|
-
re: /rate.?limit|429 too many requests/i
|
|
9081
|
-
},
|
|
9082
|
-
{
|
|
9083
|
-
name: "access-denied",
|
|
9084
|
-
re: /access denied|403 forbidden/i
|
|
9085
|
-
}
|
|
9086
|
-
];
|
|
9087
|
-
return {
|
|
9088
|
-
id,
|
|
9089
|
-
check(obs) {
|
|
9090
|
-
const hay = obs.text ?? "";
|
|
9091
|
-
for (const { name, re } of markers) if (re.test(hay)) return {
|
|
9092
|
-
id,
|
|
9093
|
-
pass: false,
|
|
9094
|
-
detail: `blocked by ${name}`,
|
|
9095
|
-
evidence: (hay.match(re) ?? [])[0]
|
|
9096
|
-
};
|
|
9097
|
-
return {
|
|
9098
|
-
id,
|
|
9099
|
-
pass: true,
|
|
9100
|
-
detail: "no anti-bot block detected"
|
|
9101
|
-
};
|
|
9102
|
-
}
|
|
9103
|
-
};
|
|
9104
|
-
}
|
|
9105
8141
|
/** Run all oracles against one observation and aggregate. */
|
|
9106
8142
|
function evaluateOracles(obs, oracles) {
|
|
9107
8143
|
const results = oracles.map((o) => o.check(obs));
|
|
@@ -9128,19 +8164,305 @@ function evaluateOracles(obs, oracles) {
|
|
|
9128
8164
|
}
|
|
9129
8165
|
};
|
|
9130
8166
|
}
|
|
9131
|
-
function excerpt(hay, needle, caseSensitive = false) {
|
|
9132
|
-
const haySearch = caseSensitive ? hay : hay.toLowerCase();
|
|
9133
|
-
const needleSearch = caseSensitive ? needle : needle.toLowerCase();
|
|
9134
|
-
const idx = haySearch.indexOf(needleSearch);
|
|
9135
|
-
if (idx === -1) return "";
|
|
9136
|
-
const start = Math.max(0, idx - 20);
|
|
9137
|
-
const end = Math.min(hay.length, idx + needle.length + 20);
|
|
9138
|
-
return (start > 0 ? "…" : "") + hay.slice(start, end) + (end < hay.length ? "…" : "");
|
|
9139
|
-
}
|
|
9140
8167
|
function isObject(v) {
|
|
9141
8168
|
return typeof v === "object" && v !== null && !Array.isArray(v);
|
|
9142
8169
|
}
|
|
9143
8170
|
//#endregion
|
|
8171
|
+
//#region src/produced-state.ts
|
|
8172
|
+
function artifactKind(mimeType) {
|
|
8173
|
+
if (!mimeType) return "file";
|
|
8174
|
+
if (mimeType.includes("json")) return "json";
|
|
8175
|
+
if (mimeType.startsWith("text/")) return "text";
|
|
8176
|
+
return "file";
|
|
8177
|
+
}
|
|
8178
|
+
/**
|
|
8179
|
+
* Normalize an emission-ordered stream into its latest observed produced state.
|
|
8180
|
+
* Artifacts are keyed by their exact output path (name, then URI, then id);
|
|
8181
|
+
* proposals by id. Later observations replace earlier ones, including missing
|
|
8182
|
+
* content or a rejected status. An obsolete version cannot prove completion.
|
|
8183
|
+
* Distinct paths remain distinct; no path normalization or aliasing is inferred.
|
|
8184
|
+
* Results retain first-seen identity order. Tool names describe invocation,
|
|
8185
|
+
* not success, and are deduplicated in first-seen order.
|
|
8186
|
+
*
|
|
8187
|
+
* An artifact without observed content yields empty content, which the
|
|
8188
|
+
* completion oracle rejects. This projection does not verify external effects,
|
|
8189
|
+
* recover dropped events, or establish freshness beyond the supplied stream.
|
|
8190
|
+
*/
|
|
8191
|
+
function extractProducedState(events) {
|
|
8192
|
+
const artifacts = /* @__PURE__ */ new Map();
|
|
8193
|
+
const proposals = /* @__PURE__ */ new Map();
|
|
8194
|
+
const toolCalls = [];
|
|
8195
|
+
const seenTools = /* @__PURE__ */ new Set();
|
|
8196
|
+
for (const ev of events) if (ev.type === "tool_call") {
|
|
8197
|
+
const name = ev.toolName;
|
|
8198
|
+
if (name && !seenTools.has(name)) {
|
|
8199
|
+
seenTools.add(name);
|
|
8200
|
+
toolCalls.push(name);
|
|
8201
|
+
}
|
|
8202
|
+
} else if (ev.type === "artifact") {
|
|
8203
|
+
const a = ev;
|
|
8204
|
+
const path = a.name ?? a.uri ?? a.artifactId;
|
|
8205
|
+
artifacts.set(path, {
|
|
8206
|
+
kind: artifactKind(a.mimeType),
|
|
8207
|
+
path,
|
|
8208
|
+
content: a.content ?? ""
|
|
8209
|
+
});
|
|
8210
|
+
} else if (ev.type === "proposal_created") {
|
|
8211
|
+
const p = ev;
|
|
8212
|
+
proposals.set(p.proposalId, {
|
|
8213
|
+
id: p.proposalId,
|
|
8214
|
+
title: p.title,
|
|
8215
|
+
status: p.status ?? "pending",
|
|
8216
|
+
...p.content !== void 0 ? { content: p.content } : {}
|
|
8217
|
+
});
|
|
8218
|
+
}
|
|
8219
|
+
return {
|
|
8220
|
+
artifacts: [...artifacts.values()],
|
|
8221
|
+
proposals: [...proposals.values()],
|
|
8222
|
+
toolCalls
|
|
8223
|
+
};
|
|
8224
|
+
}
|
|
8225
|
+
//#endregion
|
|
8226
|
+
//#region src/semantic-concept-judge.ts
|
|
8227
|
+
const DEFAULT_COMPLEXITY_WEIGHTS = {
|
|
8228
|
+
render: 1,
|
|
8229
|
+
integrate: 2,
|
|
8230
|
+
compute: 2.5
|
|
8231
|
+
};
|
|
8232
|
+
const SEMANTIC_CONCEPT_JUDGE_VERSION = "semantic-concept-judge-v1-2026-04-24";
|
|
8233
|
+
const DEFAULT_MAX_SOURCE = 45e3;
|
|
8234
|
+
const DEFAULT_MAX_HTML = 3e4;
|
|
8235
|
+
const DEFAULT_MAX_PER_FILE = 2e4;
|
|
8236
|
+
const DEFAULT_TIMEOUT = 3e5;
|
|
8237
|
+
const DEFAULT_MAX_TOKENS = 16e3;
|
|
8238
|
+
const DEFAULT_MODEL = "claude-sonnet-4-6";
|
|
8239
|
+
const SEMANTIC_SCHEMA = {
|
|
8240
|
+
type: "object",
|
|
8241
|
+
additionalProperties: false,
|
|
8242
|
+
required: ["summary", "concepts"],
|
|
8243
|
+
properties: {
|
|
8244
|
+
summary: {
|
|
8245
|
+
type: "string",
|
|
8246
|
+
minLength: 20,
|
|
8247
|
+
maxLength: 600
|
|
8248
|
+
},
|
|
8249
|
+
concepts: {
|
|
8250
|
+
type: "array",
|
|
8251
|
+
minItems: 1,
|
|
8252
|
+
items: {
|
|
8253
|
+
type: "object",
|
|
8254
|
+
additionalProperties: false,
|
|
8255
|
+
required: [
|
|
8256
|
+
"concept",
|
|
8257
|
+
"present",
|
|
8258
|
+
"score",
|
|
8259
|
+
"evidence",
|
|
8260
|
+
"severity"
|
|
8261
|
+
],
|
|
8262
|
+
properties: {
|
|
8263
|
+
concept: {
|
|
8264
|
+
type: "string",
|
|
8265
|
+
minLength: 1,
|
|
8266
|
+
maxLength: 120
|
|
8267
|
+
},
|
|
8268
|
+
present: { type: "boolean" },
|
|
8269
|
+
score: {
|
|
8270
|
+
type: "number",
|
|
8271
|
+
minimum: 0,
|
|
8272
|
+
maximum: 10
|
|
8273
|
+
},
|
|
8274
|
+
evidence: {
|
|
8275
|
+
type: "string",
|
|
8276
|
+
minLength: 5,
|
|
8277
|
+
maxLength: 400
|
|
8278
|
+
},
|
|
8279
|
+
severity: {
|
|
8280
|
+
type: "string",
|
|
8281
|
+
enum: [
|
|
8282
|
+
"critical",
|
|
8283
|
+
"major",
|
|
8284
|
+
"minor",
|
|
8285
|
+
"info"
|
|
8286
|
+
]
|
|
8287
|
+
}
|
|
8288
|
+
}
|
|
8289
|
+
}
|
|
8290
|
+
}
|
|
8291
|
+
}
|
|
8292
|
+
};
|
|
8293
|
+
function truncate(body, cap, label) {
|
|
8294
|
+
if (body.length <= cap) return body;
|
|
8295
|
+
return `${body.slice(0, cap)}\n… [truncated ${body.length - cap} chars of ${label}]`;
|
|
8296
|
+
}
|
|
8297
|
+
function buildPrompt(input, opts) {
|
|
8298
|
+
const sourceBlob = input.sourceFiles.filter((f) => f.content.length <= opts.maxPerFileChars).map((f) => `--- FILE: ${f.path} ---\n${f.content}`).join("\n\n");
|
|
8299
|
+
const html = input.servedHtml ?? "";
|
|
8300
|
+
return `You are a strict code-review judge evaluating whether an agent's 0-to-1 build actually implements the features the user asked for.
|
|
8301
|
+
|
|
8302
|
+
You MUST distinguish:
|
|
8303
|
+
(a) WORKING code that implements the concept (rendered UI, wired handler, real API call),
|
|
8304
|
+
(b) KEYWORD-PRESENT stub (comments mentioning the concept, variable names, TODOs),
|
|
8305
|
+
(c) ABSENT (concept nowhere).
|
|
8306
|
+
|
|
8307
|
+
A comment like "// TODO: add mint button" is NOT present — score 2-3. Only count a concept as present if there is real functional code: a rendered component, a call handler wired to state or a network call, a computed value actually used.
|
|
8308
|
+
|
|
8309
|
+
USER REQUEST (what the agent was asked to build):
|
|
8310
|
+
${input.userRequest}
|
|
8311
|
+
|
|
8312
|
+
${input.artifactLabel ? `ARTIFACT METADATA:\n name: ${input.artifactLabel}\n description: ${input.artifactDescription ?? ""}\n\n` : ""}EXPECTED CONCEPTS (each must be graded independently):
|
|
8313
|
+
${input.expectedConcepts.map((c, i) => ` ${i + 1}. "${c.name}"${c.keywords?.length ? ` — hints: [${c.keywords.slice(0, 6).join(" | ")}]` : ""}`).join("\n")}
|
|
8314
|
+
|
|
8315
|
+
${html ? `SERVED HTML (what the preview returns when hit):\n${truncate(html, opts.maxHtmlChars, "HTML")}\n\n` : ""}SOURCE FILES (the agent's workdir):
|
|
8316
|
+
${truncate(sourceBlob, opts.maxSourceChars, "source")}
|
|
8317
|
+
|
|
8318
|
+
For EACH concept, return:
|
|
8319
|
+
- concept: the concept name as given (match exactly)
|
|
8320
|
+
- present: boolean — does a working implementation exist?
|
|
8321
|
+
- score: 0-10 — 10 = production-ready; 7 = functional but thin; 4 = partial/stubbed; 2 = keyword-only comment; 0 = absent
|
|
8322
|
+
- evidence: cite "<file>:<line>" or "served-html:<selector>" pointing at the strongest supporting code. If the concept is absent or stubbed, explain what's missing.
|
|
8323
|
+
- severity:
|
|
8324
|
+
"info" when present: true AND score >= 7
|
|
8325
|
+
"minor" when present: true AND 4 <= score < 7
|
|
8326
|
+
"major" when present: false OR score < 4
|
|
8327
|
+
"critical" when the concept is not only absent but a core user flow depends on it
|
|
8328
|
+
|
|
8329
|
+
Also produce a "summary" (one sentence, 20-600 chars): overall verdict on whether this is a shippable implementation of the user request vs a keyword-dense placeholder.
|
|
8330
|
+
|
|
8331
|
+
BE SKEPTICAL. Keyword matching already passed — your job is to catch what keyword matching misses. If the agent shipped a working build, say so. If it shipped a stub, say so. Don't grade on effort.
|
|
8332
|
+
|
|
8333
|
+
Return STRICT JSON. No prose outside the JSON.`;
|
|
8334
|
+
}
|
|
8335
|
+
/**
|
|
8336
|
+
* Run the semantic concept judge. Soft-fails to available=false on
|
|
8337
|
+
* LLM/JSON errors — callers in a MultiLayerVerifier pipeline can treat
|
|
8338
|
+
* that as "skip" rather than "fail."
|
|
8339
|
+
*/
|
|
8340
|
+
async function runSemanticConceptJudge(input, options) {
|
|
8341
|
+
const start = Date.now();
|
|
8342
|
+
const totalCount = input.expectedConcepts.length;
|
|
8343
|
+
if (totalCount === 0) return {
|
|
8344
|
+
kind: "semantic-concept",
|
|
8345
|
+
version: SEMANTIC_CONCEPT_JUDGE_VERSION,
|
|
8346
|
+
score: 0,
|
|
8347
|
+
presentCount: 0,
|
|
8348
|
+
totalCount: 0,
|
|
8349
|
+
findings: [],
|
|
8350
|
+
summary: "no expected concepts declared",
|
|
8351
|
+
durationMs: 0,
|
|
8352
|
+
costUsd: null,
|
|
8353
|
+
available: false,
|
|
8354
|
+
error: "no expected concepts declared"
|
|
8355
|
+
};
|
|
8356
|
+
const opts = {
|
|
8357
|
+
chat: options.chat,
|
|
8358
|
+
model: options.model ?? options.chat.defaultModel ?? DEFAULT_MODEL,
|
|
8359
|
+
timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT,
|
|
8360
|
+
maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS,
|
|
8361
|
+
maxSourceChars: options.maxSourceChars ?? DEFAULT_MAX_SOURCE,
|
|
8362
|
+
maxPerFileChars: options.maxPerFileChars ?? DEFAULT_MAX_PER_FILE,
|
|
8363
|
+
maxHtmlChars: options.maxHtmlChars ?? DEFAULT_MAX_HTML,
|
|
8364
|
+
...options.pricing ? { pricing: options.pricing } : {},
|
|
8365
|
+
costLedger: options.costLedger ?? new CostLedger(),
|
|
8366
|
+
costPhase: options.costPhase ?? "judge.semantic-concept",
|
|
8367
|
+
costTags: options.costTags ?? {},
|
|
8368
|
+
signal: options.signal ?? new AbortController().signal,
|
|
8369
|
+
weightConcepts: options.weightConcepts ?? "mean",
|
|
8370
|
+
complexityWeights: {
|
|
8371
|
+
...DEFAULT_COMPLEXITY_WEIGHTS,
|
|
8372
|
+
...options.complexityWeights ?? {}
|
|
8373
|
+
}
|
|
8374
|
+
};
|
|
8375
|
+
const weightForConcept = (spec) => {
|
|
8376
|
+
if (opts.weightConcepts === "mean") return 1;
|
|
8377
|
+
if (spec.weight != null) return spec.weight;
|
|
8378
|
+
if (opts.weightConcepts === "complexity") return opts.complexityWeights[spec.complexity ?? "render"] ?? 1;
|
|
8379
|
+
return 1;
|
|
8380
|
+
};
|
|
8381
|
+
const weightByName = new Map(input.expectedConcepts.map((c) => [c.name, weightForConcept(c)]));
|
|
8382
|
+
let receipt;
|
|
8383
|
+
try {
|
|
8384
|
+
const request = {
|
|
8385
|
+
model: opts.model,
|
|
8386
|
+
messages: [{
|
|
8387
|
+
role: "system",
|
|
8388
|
+
content: "You are a strict code-review judge. Return strict JSON only. No prose outside the JSON. A keyword in a comment is NOT a working implementation."
|
|
8389
|
+
}, {
|
|
8390
|
+
role: "user",
|
|
8391
|
+
content: buildPrompt(input, opts)
|
|
8392
|
+
}],
|
|
8393
|
+
jsonSchema: {
|
|
8394
|
+
name: "semantic_concept_judge",
|
|
8395
|
+
schema: SEMANTIC_SCHEMA
|
|
8396
|
+
},
|
|
8397
|
+
temperature: 0,
|
|
8398
|
+
maxTokens: opts.maxTokens,
|
|
8399
|
+
timeoutMs: opts.timeoutMs
|
|
8400
|
+
};
|
|
8401
|
+
const paid = await paidJsonChat({
|
|
8402
|
+
chat: opts.chat,
|
|
8403
|
+
request,
|
|
8404
|
+
ledger: opts.costLedger,
|
|
8405
|
+
channel: "judge",
|
|
8406
|
+
phase: opts.costPhase,
|
|
8407
|
+
actor: "semantic-concept",
|
|
8408
|
+
tags: opts.costTags,
|
|
8409
|
+
signal: opts.signal,
|
|
8410
|
+
...opts.pricing ? { pricing: opts.pricing } : {}
|
|
8411
|
+
});
|
|
8412
|
+
receipt = paid.receipt;
|
|
8413
|
+
if (!paid.succeeded) throw paid.error;
|
|
8414
|
+
const { value } = paid;
|
|
8415
|
+
if (!value?.concepts || !Array.isArray(value.concepts)) throw new Error("judge returned malformed response — expected array under \"concepts\"");
|
|
8416
|
+
const findings = value.concepts.map((c) => ({
|
|
8417
|
+
concept: String(c.concept),
|
|
8418
|
+
present: Boolean(c.present),
|
|
8419
|
+
score: Math.max(0, Math.min(10, Number(c.score ?? 0))),
|
|
8420
|
+
evidence: String(c.evidence ?? ""),
|
|
8421
|
+
severity: [
|
|
8422
|
+
"critical",
|
|
8423
|
+
"major",
|
|
8424
|
+
"minor",
|
|
8425
|
+
"info"
|
|
8426
|
+
].includes(c.severity) ? c.severity : "info"
|
|
8427
|
+
}));
|
|
8428
|
+
const presentCount = findings.filter((f) => f.present && f.score >= 7).length;
|
|
8429
|
+
let weightSum = 0;
|
|
8430
|
+
let weightedScoreSum = 0;
|
|
8431
|
+
for (const f of findings) {
|
|
8432
|
+
const w = weightByName.get(f.concept) ?? 1;
|
|
8433
|
+
weightSum += w;
|
|
8434
|
+
weightedScoreSum += w * f.score;
|
|
8435
|
+
}
|
|
8436
|
+
const scoreAvg = weightSum > 0 ? weightedScoreSum / weightSum : findings.reduce((a, f) => a + f.score, 0) / Math.max(1, findings.length);
|
|
8437
|
+
return {
|
|
8438
|
+
kind: "semantic-concept",
|
|
8439
|
+
version: SEMANTIC_CONCEPT_JUDGE_VERSION,
|
|
8440
|
+
score: Number((scoreAvg / 10).toFixed(3)),
|
|
8441
|
+
presentCount,
|
|
8442
|
+
totalCount,
|
|
8443
|
+
findings,
|
|
8444
|
+
summary: String(value.summary ?? ""),
|
|
8445
|
+
durationMs: Date.now() - start,
|
|
8446
|
+
costUsd: paid.receipt.costUnknown ? null : paid.receipt.costUsd,
|
|
8447
|
+
available: true
|
|
8448
|
+
};
|
|
8449
|
+
} catch (err) {
|
|
8450
|
+
return {
|
|
8451
|
+
kind: "semantic-concept",
|
|
8452
|
+
version: SEMANTIC_CONCEPT_JUDGE_VERSION,
|
|
8453
|
+
score: 0,
|
|
8454
|
+
presentCount: 0,
|
|
8455
|
+
totalCount,
|
|
8456
|
+
findings: [],
|
|
8457
|
+
summary: "",
|
|
8458
|
+
durationMs: Date.now() - start,
|
|
8459
|
+
costUsd: receipt && !receipt.costUnknown ? receipt.costUsd : null,
|
|
8460
|
+
available: false,
|
|
8461
|
+
error: err instanceof Error ? err.message : String(err)
|
|
8462
|
+
};
|
|
8463
|
+
}
|
|
8464
|
+
}
|
|
8465
|
+
//#endregion
|
|
9144
8466
|
//#region src/treatment-gate.ts
|
|
9145
8467
|
/**
|
|
9146
8468
|
* Core predicate. Gated (treatment-not-applied) ONLY when telemetry was
|
|
@@ -9710,6 +9032,6 @@ function rankRows(rows, weights) {
|
|
|
9710
9032
|
})).sort((a, b) => b.mean - a.mean);
|
|
9711
9033
|
}
|
|
9712
9034
|
//#endregion
|
|
9713
|
-
export { AGENT_PROFILE_KINDS, ATTESTATION_ALGORITHM, AgentEvalError, AnalystRegistry, BOOTSTRAP_GATE_MIN_N, BackendIntegrityError, BudgetBreachError, BudgetGuard, CODING_HARNESSES, ConfigError, CostAccountingIncompleteError,
|
|
9035
|
+
export { AGENT_PROFILE_KINDS, ATTESTATION_ALGORITHM, AgentEvalError, AnalystRegistry, BOOTSTRAP_GATE_MIN_N, BackendIntegrityError, BudgetBreachError, BudgetGuard, CODING_HARNESSES, ConfigError, CostAccountingIncompleteError, CostCeilingReachedError, CostLedger, CostTracker, CrossFamilyError, DEFAULT_BOUNDED_PROCESS_TIMEOUT_MS, DEFAULT_JUDGE_GATE_POLICY, DEFAULT_RED_TEAM_CORPUS, DEFAULT_TRACE_ANALYST_BUDGETS, DEFAULT_TRACE_ANALYST_KINDS, ERROR_COUNT_PATTERNS, EquivalenceProtocolError, FAILURE_CLASSES, FAILURE_MODE_KIND_SPEC, FileSystemFeedbackTrajectoryStore, FileSystemRawProviderSink, FileSystemTraceStore, FindingsStore, HARNESS_NATIVE_MODEL, HeldOutGate, InMemoryFeedbackTrajectoryStore, InMemoryRawProviderSink, InMemoryTraceStore, JudgeError, JudgeGateRefusedError, LlmCallError, LlmResponseError, MODEL_PRICING, ModelSubstitutionError, MultiLayerVerifier, NoopRawProviderSink, NotFoundError, OUTPUT_VALUE, OtlpFileTraceStore, PairwiseSteeringOptimizer, ProductClient, PromptRegistry, REDACTION_VERSION, RunIntegrityError, RunRecordValidationError, SEMANTIC_CONCEPT_JUDGE_VERSION, TraceEmitter, UNKNOWN_MODEL, VERIFICATION_STRATEGIES, VERIFICATION_STRATEGY_SOURCES, ValidationError, acquisitionPlansForKnowledgeGaps, addCalibrationVerdicts, agentProfileCellKey, agentProfileHash, agentProfileId, aggregateJudgeVerdicts, aggregateRunScore, analystRunDigest, analystRunToFeedbackTrajectory, analystRunToReviewRequests, analyzeAntiSlop, analyzeRuns, analyzeSeries, analyzeTraces, argHash, assertCapabilityHeadroom, assertCrossFamily, assertCrossFamilyServed, assertJudgeMayGate, assertNoHiddenLeak, assertProductBenchmarkRun, assertRealBackend, assertRunCaptured, assertServedModel, assertServedModels, assertSingleBackend, assessShareSafety, assignFeedbackSplit, attest, benjaminiHochberg, blendHeldout, blockingKnowledgeEval, bonferroni, bootstrapCi, budgetBreachView, buildAgentProfileCell, buildDefaultAnalystRegistry, buildReflectionPrompt, buildTraceInsightContext, buildTraceInsightPrompt, buildTrajectory, calibrateJudge, calibrateJudgeContinuous, canonicalJson, capabilityHeadroom, captureFetchToRawSink, certificationEvidenceDigest, checkCanaries, checkServedModel, checkTraceContracts, chiSquared, chiSquaredPValue, clamp01, classifyFailure, classifyFailureReason, cliffsDelta, cohensD, comparePairedArms, compileTraceContractSpec, completionVerdict, computeFindingId, computeToolUseMetrics, confidenceInterval, contentHash, continuousAgreement, controlRunToFeedbackTrajectory, corpusInterRaterAgreement, corpusInterRaterAgreementFromJudgeScores, costForTokenPricing, costForUsage, costReceiptFromLlm, costReceiptFromLlmError, createAntiSlopJudge, createBoundedTraceAnalysisStore, createChatClient, createChatTraceEngine, createDspyRlmTraceEngine, createFeedbackTrajectory, createLlmCorrectnessChecker, createLlmReviewer, createTraceAnalyst, decidePairedPromotion, defineAgentEval, defineEquivalenceCheck, deployGateLayer, describeTraceInsightScope, diffFindings, diffScorecard, discoverPersonas, domainEvidencePattern, dominates, eProcess, ensembleJudge, equivalenceVerdict, errorStreakDetector, estimateCost, estimateTokens, evaluateActionPolicy, evaluateInterimReleaseConfidence, evaluateOracles, evaluateReleaseConfidence, evaluateTraceContract, expandProfileAxes, explainTraceContract, exportProductBenchmark, exportProductBenchmarkRuns, exportRunAsOtlp, extractErrorCount, extractProducedState, extractUsage, extractUsageFromSse, failureBlame, failureClusterView, feedbackTrajectoriesToDatasetScenarios, feedbackTrajectoriesToOptimizerRows, feedbackTrajectoryToOptimizerRow, fileVerdictCache, fisherExactTwoSided, formatScorecardDiff, gainHistogram, gateTreatmentApplied, gradeSemanticStatus, groupRunsByAgentProfileCell, harnessAxisOf, hashContent, hashJson, holm, inMemoryReviewStore, inferDomainKeywords, interRaterReliability, interpretCliffs, iqr, isJudgeSpan, isLlmSpan, isModelPriced, isRunRecord, isToolSpan, isTransientLlmError, jsonShape, jsonlReviewStore, jsonlRunRecordBackend, judgeAgreementView, judgeFamily, judgeGateDecision, judgeSpans, knowledgeReadinessTracePayload, leaderboard, linearRegressionSlope, lintTraceContractSpec, llmJudge, loadScorecard, localCommandRunner, makeFinding, makePolicyEditCandidateRecord, makeProposalFinding, manifestContentDigest, mannWhitneyU, maximumChargeForLlmRequest, mcnemar, mcnemarPower, mcnemarRequiredN, measureJudgeAgreement, minimumPairsForPairedDeltaTest, mintRolloutRows, modelHasSnapshot, mulberry32, normalTwoSidedPValue, objectiveEval, observeAll, otlpTextToTraceAnalysisStore, pairArms, pairRunRecords, pairedBootstrap, pairedCohensDz, pairedDeltaTest, pairedEvalueSequence, pairedMde, pairedPromotionPower, pairedRiskDifference, pairedRiskDifferenceExact, pairedRiskDifferenceScore, pairedSignTest, pairedTTest, paretoFrontier, parseReflectionResponse, parseRunRecordSafe, partialCredit, partitionHeldOut, passAtK, pearsonR, preflightModels, productBenchmarkRepoIdentity, profile_exports as profile, projectRuntimeTrajectoryEvidence, proposeSynthesisTargets, ranks, readProductBenchmarkManifest, recordRuns, recordRunsToScorecard, redTeamDataset, redTeamReport, redact, redactForShare, redactText, registerCalibrationSet, renderPreferenceMemoryMarkdown, repeatedActionDetector, requiredPairedSampleSize, requiredPairsForPairedPromotion, requiredSampleSize, resolveModelPricing, resolveSeat, runAgentControlLoop, runBoundedProcess, runCampaign, runCanaries, runEquivalenceCheck, runEvalCampaign, runIntentMatchJudge, runKeywordCoverageJudge, runKeywordCoverageJudgeUrl, runProposeReview, runSemanticConceptJudge, runTaskScore, runsForScenario, sampleMedian, scoreKnowledgeReadiness, scoreRedTeamOutput, scoreTraceInsightReadiness, seatPresets, selfImprove, shareAllowed, snapshotFromJudgeAgreement, spanRecordsToTraceAnalysisStore, spearmanR, stripFencedJson, studentTTwoSidedPValue, subjectiveEval, summarizeBackendIntegrity, summarizeNumberSeries, summarizePreferenceMemory, summarizeSample, summaryTable, toAgentProfileJson, tokenizeDomainWords, toolSpansToTraceAnalysisStore, toolWasteView, traceContract, transientDispatchFailure, twoByTwoPValue, userQuestionsForKnowledgeGaps, validateRunRecord, verbosityBias, verifyAgentProfileCell, verifyAttestation, verifyCompletion, viteDeployRunner, weightedComposite, weightedMean, welchsTTest, wilcoxonSignedRank, wilson, withAssignedFeedbackSplit, withJudgeRetry, wranglerDeployRunner };
|
|
9714
9036
|
|
|
9715
9037
|
//# sourceMappingURL=index.js.map
|