@tangle-network/agent-eval 0.216.1 → 0.217.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +102 -57
- package/README.md +2 -2
- package/dist/adapters/http.d.ts +2 -2
- package/dist/{agent-profile-VFcQncbW.d.ts → agent-profile-DTt8uGP9.d.ts} +4 -17
- package/dist/agent-profile-DTt8uGP9.d.ts.map +1 -0
- package/dist/{agent-profile-cell-BK8mlqlq.js → agent-profile-cell-B1V1gUZs.js} +4 -53
- package/dist/agent-profile-cell-B1V1gUZs.js.map +1 -0
- package/dist/{agent-profile-cell-BNp-Js9F.d.ts → agent-profile-cell-BcpFbNd5.d.ts} +2 -22
- package/dist/agent-profile-cell-BcpFbNd5.d.ts.map +1 -0
- package/dist/{agreement-irr-BEWLIF3_.js → agreement-irr-BhAqjluA.js} +5 -5
- package/dist/agreement-irr-BhAqjluA.js.map +1 -0
- package/dist/analyst/index.d.ts +36 -169
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +17 -192
- package/dist/analyst/index.js.map +1 -1
- package/dist/{analyze-runs-C84irm4i.js → analyze-runs-DqPJaJ4X.js} +225 -21
- package/dist/analyze-runs-DqPJaJ4X.js.map +1 -0
- package/dist/{analyze-runs-D68fd-su.d.ts → analyze-runs-DyeNcenW.d.ts} +5 -5
- package/dist/{analyze-runs-D68fd-su.d.ts.map → analyze-runs-DyeNcenW.d.ts.map} +1 -1
- package/dist/authenticity/index.d.ts +0 -23
- package/dist/authenticity/index.d.ts.map +1 -1
- package/dist/authenticity/index.js +1 -34
- package/dist/authenticity/index.js.map +1 -1
- package/dist/{benchmark-command-By0vh8q7.js → benchmark-command-xQK8nOI6.js} +554 -89
- package/dist/benchmark-command-xQK8nOI6.js.map +1 -0
- package/dist/benchmarks/index.d.ts +5 -6
- package/dist/benchmarks/index.d.ts.map +1 -1
- package/dist/benchmarks/index.js +4 -4
- package/dist/benchmarks/index.js.map +1 -1
- package/dist/builder-eval/index.d.ts +3 -3
- package/dist/builder-eval/index.js +2 -2
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/campaign/index.d.ts +7 -7
- package/dist/campaign/index.js +15 -15
- package/dist/{campaign-BudZ70zo.js → campaign-CJGkxAQ6.js} +23 -530
- package/dist/campaign-CJGkxAQ6.js.map +1 -0
- package/dist/{campaign-evidence-Clrn4hBV.js → campaign-evidence-CQYw8_0H.js} +10 -18
- package/dist/{campaign-evidence-Clrn4hBV.js.map → campaign-evidence-CQYw8_0H.js.map} +1 -1
- package/dist/{canonical-B2RqLofv.js → canonical-Dq2hOQ8L.js} +2 -2
- package/dist/{canonical-B2RqLofv.js.map → canonical-Dq2hOQ8L.js.map} +1 -1
- package/dist/cell-spend-DEWR3nj-.js +90 -0
- package/dist/cell-spend-DEWR3nj-.js.map +1 -0
- package/dist/{chat-client-l-MVDlDk.js → chat-client-ldqxYYTN.js} +2 -2
- package/dist/{chat-client-l-MVDlDk.js.map → chat-client-ldqxYYTN.js.map} +1 -1
- package/dist/{chat-json-call-puMArmtW.js → chat-json-call-DavzexSp.js} +3 -2
- package/dist/chat-json-call-DavzexSp.js.map +1 -0
- package/dist/{claim-q4HQhK1G.js → claim-BPVuyHuy.js} +3 -3
- package/dist/{claim-q4HQhK1G.js.map → claim-BPVuyHuy.js.map} +1 -1
- package/dist/cli.js +113 -12
- package/dist/cli.js.map +1 -1
- package/dist/{client-BzL9niqi.d.ts → client-hwo3cJjR.d.ts} +3 -16
- package/dist/client-hwo3cJjR.d.ts.map +1 -0
- package/dist/{cluster-bootstrap-qcUWYFl-.js → cluster-bootstrap-DEwwhl0C.js} +3 -3
- package/dist/{cluster-bootstrap-qcUWYFl-.js.map → cluster-bootstrap-DEwwhl0C.js.map} +1 -1
- package/dist/{produced-state-BsDmnwxc.js → completion-verifier-B_jw6C9p.js} +9 -64
- package/dist/completion-verifier-B_jw6C9p.js.map +1 -0
- package/dist/contract/index.d.ts +12 -12
- package/dist/contract/index.js +25 -401
- package/dist/contract/index.js.map +1 -1
- package/dist/{cost-ledger-BP8V4kUQ.js → cost-ledger-D0352Z4d.js} +4 -8
- package/dist/cost-ledger-D0352Z4d.js.map +1 -0
- package/dist/dataset-DQqhOCPt.d.ts.map +1 -1
- package/dist/{default-registry-COhk9g36.js → default-registry-ClYjq3QE.js} +8 -11
- package/dist/default-registry-ClYjq3QE.js.map +1 -0
- package/dist/{default-registry-DzHiGruW.d.ts → default-registry-DXgLmoDI.d.ts} +7 -50
- package/dist/default-registry-DXgLmoDI.d.ts.map +1 -0
- package/dist/{define-agent-eval-CNTPzIkp.js → define-agent-eval-BDxNlP3D.js} +10 -10
- package/dist/{define-agent-eval-CNTPzIkp.js.map → define-agent-eval-BDxNlP3D.js.map} +1 -1
- package/dist/{define-agent-eval-WSck0CZJ.d.ts → define-agent-eval-BIkH_gAR.d.ts} +8 -8
- package/dist/define-agent-eval-BIkH_gAR.d.ts.map +1 -0
- package/dist/{descriptive-DboxPZrG.js → descriptive-CVNy48rR.js} +2 -2
- package/dist/{descriptive-DboxPZrG.js.map → descriptive-CVNy48rR.js.map} +1 -1
- package/dist/diagnosis/index.d.ts +4 -14
- package/dist/diagnosis/index.d.ts.map +1 -1
- package/dist/diagnosis/index.js +67 -28
- package/dist/diagnosis/index.js.map +1 -1
- package/dist/{dspy-rlm-engine-LRCu61m5.js → dspy-rlm-engine-DhdCr1Dc.js} +5 -10
- package/dist/dspy-rlm-engine-DhdCr1Dc.js.map +1 -0
- package/dist/{emitter-HcCmFQl_.d.ts → emitter-DABYYTcH.d.ts} +5 -5
- package/dist/emitter-DABYYTcH.d.ts.map +1 -0
- package/dist/emitter-DbMXUIVP.js.map +1 -1
- package/dist/{engine-3A_4jOlI.d.ts → engine-BvZ0R-36.d.ts} +17 -21
- package/dist/engine-BvZ0R-36.d.ts.map +1 -0
- package/dist/{equal-terms-BtdXayCJ.js → equal-terms-DbPGcVtI.js} +2 -2
- package/dist/equal-terms-DbPGcVtI.js.map +1 -0
- package/dist/{errors-DVZiqS7K.js → errors-BcOs5MKW.js} +2 -8
- package/dist/{errors-DVZiqS7K.js.map → errors-BcOs5MKW.js.map} +1 -1
- package/dist/{eval-campaign-90jeaCAI.js → eval-campaign-BLEfBGbA.js} +6 -6
- package/dist/{eval-campaign-90jeaCAI.js.map → eval-campaign-BLEfBGbA.js.map} +1 -1
- package/dist/{evaluation-6Yoi74gM.d.ts → evaluation-jTjC_QQD.d.ts} +7 -7
- package/dist/evaluation-jTjC_QQD.d.ts.map +1 -0
- package/dist/evaluation.d.ts +2 -2
- package/dist/evaluation.js +1 -1
- package/dist/evaluation.js.map +1 -1
- package/dist/{exact-types-ClugMJfI.d.ts → exact-types-CflbJ9f3.d.ts} +2 -2
- package/dist/{exact-types-ClugMJfI.d.ts.map → exact-types-CflbJ9f3.d.ts.map} +1 -1
- package/dist/{exec-ChJ6d5if.js → exec-CpbtOXVt.js} +2 -24
- package/dist/{exec-ChJ6d5if.js.map → exec-CpbtOXVt.js.map} +1 -1
- package/dist/execution-tracks-CpgFPpS5.js.map +1 -1
- package/dist/experiment/index.d.ts +9 -112
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +149 -241
- package/dist/experiment/index.js.map +1 -1
- package/dist/{exporters-X4xwla8A.js → exporters-Cl6IeNAu.js} +3 -3
- package/dist/exporters-Cl6IeNAu.js.map +1 -0
- package/dist/{exporters-ChTnR90l.d.ts → exporters-Dk3oEbK_.d.ts} +2 -4
- package/dist/{exporters-ChTnR90l.d.ts.map → exporters-Dk3oEbK_.d.ts.map} +1 -1
- package/dist/{external-optimizer-process-BjMGUuds.js → external-optimizer-process-KVFGvYX1.js} +4 -3
- package/dist/external-optimizer-process-KVFGvYX1.js.map +1 -0
- package/dist/{external-optimizer-subprocess-BqW0hHcI.js → external-optimizer-subprocess-XIHBOSKt.js} +9 -11
- package/dist/external-optimizer-subprocess-XIHBOSKt.js.map +1 -0
- package/dist/{extract-usage-BrnzRJ6H.js → extract-usage-BZLU01kR.js} +2 -23
- package/dist/{extract-usage-BrnzRJ6H.js.map → extract-usage-BZLU01kR.js.map} +1 -1
- package/dist/{failure-cluster-CbgEDcFJ.d.ts → failure-cluster-DN0t63Z0.d.ts} +4 -43
- package/dist/failure-cluster-DN0t63Z0.d.ts.map +1 -0
- package/dist/{failure-taxonomy-CjabTATj.js → failure-taxonomy-Bx5oWqaW.js} +2 -19
- package/dist/{failure-taxonomy-CjabTATj.js.map → failure-taxonomy-Bx5oWqaW.js.map} +1 -1
- package/dist/{feedback-trajectory-C4EG0nNH.d.ts → feedback-trajectory-BWmhRBGu.d.ts} +7 -9
- package/dist/feedback-trajectory-BWmhRBGu.d.ts.map +1 -0
- package/dist/{semantic-concept-judge-BQF8te0T.js → findings-store-BMVQtEsm.js} +9 -249
- package/dist/findings-store-BMVQtEsm.js.map +1 -0
- package/dist/fuzz.d.ts +19 -62
- package/dist/fuzz.d.ts.map +1 -1
- package/dist/fuzz.js +62 -108
- package/dist/fuzz.js.map +1 -1
- package/dist/{hf-dataset-BR2yhuNn.js → hf-dataset-D9f7-oX7.js} +4 -24
- package/dist/hf-dataset-D9f7-oX7.js.map +1 -0
- package/dist/hosted/index.d.ts +2 -2
- package/dist/hosted/index.js +2 -2
- package/dist/{index-DHKEAv3s.d.ts → index-9jJ2fKSX.d.ts} +2 -64
- package/dist/index-9jJ2fKSX.d.ts.map +1 -0
- package/dist/{index-DV-jSzxF.d.ts → index-BLGqZ81N.d.ts} +10 -15
- package/dist/{index-DV-jSzxF.d.ts.map → index-BLGqZ81N.d.ts.map} +1 -1
- package/dist/{index-DPcngtwt.d.ts → index-BiT6RtNH.d.ts} +11 -36
- package/dist/index-BiT6RtNH.d.ts.map +1 -0
- package/dist/{index-BO41eGj5.d.ts → index-CX-GAUpw.d.ts} +3 -7
- package/dist/index-CX-GAUpw.d.ts.map +1 -0
- package/dist/{index-BFRO6chs.d.ts → index-CiRwLv2R.d.ts} +5 -69
- package/dist/index-CiRwLv2R.d.ts.map +1 -0
- package/dist/{index-CyszSg4O.d.ts → index-ZuWx52WG.d.ts} +115 -347
- package/dist/index-ZuWx52WG.d.ts.map +1 -0
- package/dist/index.d.ts +211 -218
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +402 -1040
- package/dist/index.js.map +1 -1
- package/dist/{integrity-dLiDe2kD.js → integrity-DOd9Q86O.js} +7 -9
- package/dist/integrity-DOd9Q86O.js.map +1 -0
- package/dist/{integrity-BXOsAb3i.js → integrity-W4CADcFc.js} +3 -7
- package/dist/{integrity-BXOsAb3i.js.map → integrity-W4CADcFc.js.map} +1 -1
- package/dist/{integrity-DAMEDFaC.d.ts → integrity-_qgZLA5p.d.ts} +4 -6
- package/dist/{integrity-DAMEDFaC.d.ts.map → integrity-_qgZLA5p.d.ts.map} +1 -1
- package/dist/{internal-Epl-YBZW.js → internal-BTK91Bqt.js} +3 -3
- package/dist/{internal-Epl-YBZW.js.map → internal-BTK91Bqt.js.map} +1 -1
- package/dist/{interval-CELpye1n.js → interval-Dq9gt7jx.js} +3 -3
- package/dist/{interval-CELpye1n.js.map → interval-Dq9gt7jx.js.map} +1 -1
- package/dist/{journal-DfRz9kps.js → journal-CR4QikYp.js} +110 -110
- package/dist/journal-CR4QikYp.js.map +1 -0
- package/dist/json-BhdsJaOQ.js +111 -0
- package/dist/json-BhdsJaOQ.js.map +1 -0
- package/dist/{judge-calibration-BpCy5XQl.js → judge-calibration-BkXMIWA4.js} +3 -3
- package/dist/{judge-calibration-BpCy5XQl.js.map → judge-calibration-BkXMIWA4.js.map} +1 -1
- package/dist/{judge-gate-iY9s4L44.js → judge-gate-Co5tuh57.js} +4 -27
- package/dist/{judge-gate-iY9s4L44.js.map → judge-gate-Co5tuh57.js.map} +1 -1
- package/dist/{judge-gate-CE7CQAgt.d.ts → judge-gate-DsL0yFcb.d.ts} +8 -9
- package/dist/judge-gate-DsL0yFcb.d.ts.map +1 -0
- package/dist/{kind-factory-BobI5Jy3.js → kind-factory-Gy4fXixd.js} +88 -17
- package/dist/kind-factory-Gy4fXixd.js.map +1 -0
- package/dist/ledger-core/index.d.ts +2 -2
- package/dist/ledger-core/index.js +4 -4
- package/dist/{ledger-core-Gq7yp8lP.js → ledger-core-DrJhbms9.js} +5 -84
- package/dist/ledger-core-DrJhbms9.js.map +1 -0
- package/dist/{llm-client-CyBAoYiX.js → llm-client-BshCREn6.js} +6 -108
- package/dist/llm-client-BshCREn6.js.map +1 -0
- package/dist/{llm-judge-CG12SQJq.js → llm-judge-C8jMTNNG.js} +24 -32
- package/dist/llm-judge-C8jMTNNG.js.map +1 -0
- package/dist/matrix/index.d.ts +3 -3
- package/dist/matrix/index.js +3 -2
- package/dist/{matrix-Dm0BQW7U.js → matrix-B8l-Nsxd.js} +3 -88
- package/dist/matrix-B8l-Nsxd.js.map +1 -0
- package/dist/{matrix-DSrSxjjy.d.ts → matrix-DHe-dwVJ.d.ts} +9 -18
- package/dist/matrix-DHe-dwVJ.d.ts.map +1 -0
- package/dist/{mean-DHoMiudD.js → mean-BbA_JCzW.js} +39 -5
- package/dist/mean-BbA_JCzW.js.map +1 -0
- package/dist/meta-eval/index.d.ts +5 -5
- package/dist/meta-eval/index.js +10 -10
- package/dist/metrics-BKaPBtIG.js.map +1 -1
- package/dist/{mint-DNU8QPG4.d.ts → mint-BRZXuwRi.d.ts} +4 -4
- package/dist/{mint-DNU8QPG4.d.ts.map → mint-BRZXuwRi.d.ts.map} +1 -1
- package/dist/{mint-QaD5cA0u.js → mint-zszyYlpA.js} +4 -4
- package/dist/{mint-QaD5cA0u.js.map → mint-zszyYlpA.js.map} +1 -1
- package/dist/{multi-layer-verifier-BUaQ4C17.d.ts → multi-layer-verifier-DEgMUk6v.d.ts} +2 -2
- package/dist/{multi-layer-verifier-BUaQ4C17.d.ts.map → multi-layer-verifier-DEgMUk6v.d.ts.map} +1 -1
- package/dist/{multiplicity-DQUFcbW4.js → multiplicity-Sg0U7h8j.js} +2 -2
- package/dist/{multiplicity-DQUFcbW4.js.map → multiplicity-Sg0U7h8j.js.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -23
- package/dist/multishot/golden/index.d.ts.map +1 -1
- package/dist/multishot/golden/index.js +3 -3
- package/dist/multishot/golden/index.js.map +1 -1
- package/dist/multishot/index.d.ts +3 -12
- package/dist/multishot/index.d.ts.map +1 -1
- package/dist/multishot/index.js +4 -3
- package/dist/multishot/index.js.map +1 -1
- package/dist/openapi.json +114 -10
- package/dist/{otel-spans-B9tUXRHF.js → otel-spans-CdN909k1.js} +3 -3
- package/dist/{otel-spans-B9tUXRHF.js.map → otel-spans-CdN909k1.js.map} +1 -1
- package/dist/{paired-arms-CwMGJsrS.js → paired-arms-Ct4LTKc1.js} +32 -9
- package/dist/paired-arms-Ct4LTKc1.js.map +1 -0
- package/dist/{paired-binary-CvpFHfKZ.js → paired-binary-B_nejA44.js} +3 -36
- package/dist/paired-binary-B_nejA44.js.map +1 -0
- package/dist/{paired-promotion-decision-DPKz8ZnF.d.ts → paired-promotion-decision-CQMJ5xGN.d.ts} +2 -34
- package/dist/paired-promotion-decision-CQMJ5xGN.d.ts.map +1 -0
- package/dist/{paired-promotion-decision-DsAPYGCm.js → paired-promotion-decision-D0SHb-ZL.js} +3 -3
- package/dist/{paired-promotion-decision-DsAPYGCm.js.map → paired-promotion-decision-D0SHb-ZL.js.map} +1 -1
- package/dist/{pareto-Cm6ndrWN.js → pareto-BzsxidFD.js} +11 -11
- package/dist/pareto-BzsxidFD.js.map +1 -0
- package/dist/pipelines/index.d.ts +16 -19
- package/dist/pipelines/index.d.ts.map +1 -1
- package/dist/pipelines/index.js +4 -68
- package/dist/pipelines/index.js.map +1 -1
- package/dist/{power-and-mde-Hj2ubb4B.js → power-and-mde-N5nU4_Ly.js} +3 -3
- package/dist/{power-and-mde-Hj2ubb4B.js.map → power-and-mde-N5nU4_Ly.js.map} +1 -1
- package/dist/{prime-protocol-CHKgA3a6.js → prime-protocol-BFgGrEMP.js} +3 -3
- package/dist/prime-protocol-BFgGrEMP.js.map +1 -0
- package/dist/{prime-protocol-HiB9HX7E.d.ts → prime-protocol-BWG2DQIk.d.ts} +3 -21
- package/dist/prime-protocol-BWG2DQIk.d.ts.map +1 -0
- package/dist/profile-cell.d.ts +2 -2
- package/dist/profile-cell.js +2 -2
- package/dist/{promotion-gate-BawrdPYr.d.ts → promotion-gate-DaY95yU6.d.ts} +7 -8
- package/dist/{promotion-gate-BawrdPYr.d.ts.map → promotion-gate-DaY95yU6.d.ts.map} +1 -1
- package/dist/{promotion-policy-KO57gbyV.js → promotion-policy-6A8osGr_.js} +4 -4
- package/dist/{promotion-policy-KO57gbyV.js.map → promotion-policy-6A8osGr_.js.map} +1 -1
- package/dist/{proposal-findings-CY1xKn_a.js → proposal-findings-BbWidvUk.js} +3 -3
- package/dist/proposal-findings-BbWidvUk.js.map +1 -0
- package/dist/{query-BUCZCK7F.js → query-B_RZ36an.js} +5 -5
- package/dist/query-B_RZ36an.js.map +1 -0
- package/dist/{query-DK1nWcmp.d.ts → query-Cmv_U8vf.d.ts} +4 -24
- package/dist/{query-DK1nWcmp.d.ts.map → query-Cmv_U8vf.d.ts.map} +1 -1
- package/dist/{random-ByUSnV9w.js → random-DDDcC7YU.js} +2 -2
- package/dist/{random-ByUSnV9w.js.map → random-DDDcC7YU.js.map} +1 -1
- package/dist/{raw-provider-sink-DoHz4JNG.js → raw-provider-sink-75w4FWBj.js} +2 -2
- package/dist/{raw-provider-sink-DoHz4JNG.js.map → raw-provider-sink-75w4FWBj.js.map} +1 -1
- package/dist/{redact-mtCCcT8z.d.ts → redact-DXdlFb_U.d.ts} +2 -3
- package/dist/redact-DXdlFb_U.d.ts.map +1 -0
- package/dist/{redact-B5xXKqQ3.js → redact-mtkTiJVk.js} +2 -7
- package/dist/{redact-B5xXKqQ3.js.map → redact-mtkTiJVk.js.map} +1 -1
- package/dist/{registry-D7_hm2fK.d.ts → registry-ssNoHzwD.d.ts} +4 -4
- package/dist/{registry-D7_hm2fK.d.ts.map → registry-ssNoHzwD.d.ts.map} +1 -1
- package/dist/{release-confidence-CpuP65S5.js → release-confidence-CYIFH-fp.js} +6 -11
- package/dist/release-confidence-CYIFH-fp.js.map +1 -0
- package/dist/{report-command-SC9nqb4M.js → render-C-w2EZ6p.js} +205 -363
- package/dist/render-C-w2EZ6p.js.map +1 -0
- package/dist/reporting.d.ts +6 -6
- package/dist/reporting.js +8 -8
- package/dist/{researcher-CEcfku-T.d.ts → researcher-0ABVIdGa.d.ts} +9 -9
- package/dist/{researcher-CEcfku-T.d.ts.map → researcher-0ABVIdGa.d.ts.map} +1 -1
- package/dist/{reward-CpCmAM8L.d.ts → reward-DrZ71vnf.d.ts} +3 -26
- package/dist/reward-DrZ71vnf.d.ts.map +1 -0
- package/dist/{reward-hacking-E0d-WVcP.js → reward-hacking-ByuETaIb.js} +3 -3
- package/dist/{reward-hacking-E0d-WVcP.js.map → reward-hacking-ByuETaIb.js.map} +1 -1
- package/dist/{reward-hacking-RVdZ6Kkt.d.ts → reward-hacking-E52wq86n.d.ts} +5 -18
- package/dist/reward-hacking-E52wq86n.d.ts.map +1 -0
- package/dist/reward-nw2xZGZG.js.map +1 -1
- package/dist/rl.d.ts +117 -295
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +27 -747
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +5 -5
- package/dist/rollout/index.js +7 -8
- package/dist/{rollout-DZV1t2Wg.js → rollout-D6C50mfA.js} +88 -13
- package/dist/rollout-D6C50mfA.js.map +1 -0
- package/dist/{rubric-predictive-validity-DaFW70y9.js → rubric-predictive-validity-D3_oI72N.js} +3 -3
- package/dist/{rubric-predictive-validity-DaFW70y9.js.map → rubric-predictive-validity-D3_oI72N.js.map} +1 -1
- package/dist/{rubric-predictive-validity-C03EC7ZE.d.ts → rubric-predictive-validity-DiwT8PbL.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-C03EC7ZE.d.ts.map → rubric-predictive-validity-DiwT8PbL.d.ts.map} +1 -1
- package/dist/{run-record-C1E24G2A.d.ts → run-record-CaxbLq6Y.d.ts} +4 -10
- package/dist/run-record-CaxbLq6Y.d.ts.map +1 -0
- package/dist/{run-record-C5e7FjhJ.js → run-record-CysKLB93.js} +3 -3
- package/dist/run-record-CysKLB93.js.map +1 -0
- package/dist/{run-record-Boc5IUTp.js → run-record-DhEJgoPz.js} +4 -4
- package/dist/{run-record-Boc5IUTp.js.map → run-record-DhEJgoPz.js.map} +1 -1
- package/dist/sandbox-harness-BlSOu4LX.d.ts.map +1 -1
- package/dist/{schema-CSf6qWgZ.js → schema-Bt2vLFm3.js} +2 -14
- package/dist/schema-Bt2vLFm3.js.map +1 -0
- package/dist/schema-C-ziyJD7.d.ts.map +1 -1
- package/dist/{schema-2g7trUwH.d.ts → schema-D6EVKzTB.d.ts} +4 -5
- package/dist/schema-D6EVKzTB.d.ts.map +1 -0
- package/dist/{schema-BuiujZ-g.js → schema-DxcKTzbi.js} +2 -2
- package/dist/schema-DxcKTzbi.js.map +1 -0
- package/dist/search/index.d.ts +3 -31
- package/dist/search/index.d.ts.map +1 -1
- package/dist/search/index.js +4 -4
- package/dist/{search-kernel-DyNEa6IU.d.ts → search-kernel-DxqjBxA-.d.ts} +10 -27
- package/dist/search-kernel-DxqjBxA-.d.ts.map +1 -0
- package/dist/{search-ledger-CVUGLDZp.js → search-ledger-TUvB5gWH.js} +7 -7
- package/dist/{search-ledger-CVUGLDZp.js.map → search-ledger-TUvB5gWH.js.map} +1 -1
- package/dist/{search-ledger-file-DDQUgvCg.js → search-ledger-file-UKCO2r8a.js} +4 -4
- package/dist/search-ledger-file-UKCO2r8a.js.map +1 -0
- package/dist/{search-shipper-BOSD46qv.js → search-shipper-Cw2z3-td.js} +4 -4
- package/dist/search-shipper-Cw2z3-td.js.map +1 -0
- package/dist/{search-summary-BQCSWf1g.js → search-summary-DQIvJsSB.js} +5 -5
- package/dist/{search-summary-BQCSWf1g.js.map → search-summary-DQIvJsSB.js.map} +1 -1
- package/dist/{sequential-CzK5DarL.js → sequential-BVv1sjsw.js} +2 -32
- package/dist/{sequential-CzK5DarL.js.map → sequential-BVv1sjsw.js.map} +1 -1
- package/dist/{sequential-Bf_pbE50.js → sequential-CimQrbjv.js} +5 -5
- package/dist/sequential-CimQrbjv.js.map +1 -0
- package/dist/{sequential-BhsrMupG.d.ts → sequential-KwGspfeR.d.ts} +2 -9
- package/dist/{sequential-BhsrMupG.d.ts.map → sequential-KwGspfeR.d.ts.map} +1 -1
- package/dist/{sequential-eprocess-DM_qENCl.js → sequential-eprocess-DodI6RoF.js} +2 -2
- package/dist/{sequential-eprocess-DM_qENCl.js.map → sequential-eprocess-DodI6RoF.js.map} +1 -1
- package/dist/{server-e9oWO_bN.js → server-DoVbqqtJ.js} +101 -101
- package/dist/server-DoVbqqtJ.js.map +1 -0
- package/dist/{single-run-lock-CV9Eyd_q.js → single-run-lock-U58yf8N6.js} +2 -2
- package/dist/{single-run-lock-CV9Eyd_q.js.map → single-run-lock-U58yf8N6.js.map} +1 -1
- package/dist/{skillopt-optimization-method-BiwA8nfX.js → skillopt-optimization-method-kyL94-ri.js} +17 -95
- package/dist/skillopt-optimization-method-kyL94-ri.js.map +1 -0
- package/dist/{spans-BHAUXMGN.d.ts → spans-CVngGij3.d.ts} +2 -2
- package/dist/{spans-BHAUXMGN.d.ts.map → spans-CVngGij3.d.ts.map} +1 -1
- package/dist/{statistical-heldout-42NlCNDe.js → statistical-heldout-Bksv92Th.js} +3 -3
- package/dist/statistical-heldout-Bksv92Th.js.map +1 -0
- package/dist/{statistical-heldout-CYpGaoDH.d.ts → statistical-heldout-C9AbgREN.d.ts} +8 -67
- package/dist/statistical-heldout-C9AbgREN.d.ts.map +1 -0
- package/dist/{steps-Bg5-KVx1.d.ts → steps-DlepJPwD.d.ts} +2 -56
- package/dist/steps-DlepJPwD.d.ts.map +1 -0
- package/dist/{store-C4avFjIi.d.ts → store-B3nWRy3B.d.ts} +2 -2
- package/dist/{store-C4avFjIi.d.ts.map → store-B3nWRy3B.d.ts.map} +1 -1
- package/dist/{store-otlp-DJ9dtA5x.js → store-otlp-BteDAbHK.js} +3 -3
- package/dist/{store-otlp-DJ9dtA5x.js.map → store-otlp-BteDAbHK.js.map} +1 -1
- package/dist/{store-tool-spans-CIX5arAG.js → store-tool-spans-D5l_OfTJ.js} +8 -8
- package/dist/store-tool-spans-D5l_OfTJ.js.map +1 -0
- package/dist/{store-tool-spans-DdCSPa7C.d.ts → store-tool-spans-DZS0HWVf.d.ts} +9 -18
- package/dist/store-tool-spans-DZS0HWVf.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts +4 -11
- package/dist/storyboard/index.d.ts.map +1 -1
- package/dist/storyboard/index.js +1 -16
- package/dist/storyboard/index.js.map +1 -1
- package/dist/{summary-report-V_JACR4Y.js → summary-report-BsXJoYX5.js} +88 -10
- package/dist/summary-report-BsXJoYX5.js.map +1 -0
- package/dist/{summary-report-bdFDXq0b.d.ts → summary-report-DM7tlml1.d.ts} +10 -19
- package/dist/summary-report-DM7tlml1.d.ts.map +1 -0
- package/dist/supervisor-run/index.d.ts +60 -28
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +4 -458
- package/dist/supervisor-run/index.js.map +1 -1
- package/dist/{systemone-BomhHr2c.d.ts → systemone-Bp_Ac0p1.d.ts} +5 -5
- package/dist/{systemone-BomhHr2c.d.ts.map → systemone-Bp_Ac0p1.d.ts.map} +1 -1
- package/dist/systemone-protocol.js +1 -1
- package/dist/{systemone-review-CUVcQJu6.d.ts → systemone-review-CwE6JGIp.d.ts} +2 -2
- package/dist/{systemone-review-CUVcQJu6.d.ts.map → systemone-review-CwE6JGIp.d.ts.map} +1 -1
- package/dist/{systemone-review-D7sXwsqq.js → systemone-review-DzdOJ8I6.js} +3 -3
- package/dist/{systemone-review-D7sXwsqq.js.map → systemone-review-DzdOJ8I6.js.map} +1 -1
- package/dist/systemone.d.ts +5 -5
- package/dist/systemone.js +5 -5
- package/dist/systemone.js.map +1 -1
- package/dist/{task-failure-attributes-hEmASjix.js → task-failure-attributes-BMsblaDh.js} +3 -3
- package/dist/task-failure-attributes-BMsblaDh.js.map +1 -0
- package/dist/{terminal-record-DZyQZ3pz.js → terminal-record-78g6E4vh.js} +5 -5
- package/dist/terminal-record-78g6E4vh.js.map +1 -0
- package/dist/{tool-groups-C3NEcdCM.d.ts → tool-groups-DyNz2JZ_.d.ts} +3 -3
- package/dist/tool-groups-DyNz2JZ_.d.ts.map +1 -0
- package/dist/{tool-waste-BKgvd-TP.d.ts → tool-waste-CkjzWmeY.d.ts} +25 -28
- package/dist/tool-waste-CkjzWmeY.d.ts.map +1 -0
- package/dist/{tool-waste-BlesqTqW.js → tool-waste-DLsjS_p-.js} +5 -5
- package/dist/{tool-waste-BlesqTqW.js.map → tool-waste-DLsjS_p-.js.map} +1 -1
- package/dist/{trace-questions-CM2CAocU.d.ts → trace-questions-xxwgsuMi.d.ts} +4 -9
- package/dist/{trace-questions-CM2CAocU.d.ts.map → trace-questions-xxwgsuMi.d.ts.map} +1 -1
- package/dist/trace-repair/index.d.ts +14 -211
- package/dist/trace-repair/index.d.ts.map +1 -1
- package/dist/trace-repair/index.js +865 -981
- package/dist/trace-repair/index.js.map +1 -1
- package/dist/traces.d.ts +18 -46
- package/dist/traces.d.ts.map +1 -1
- package/dist/traces.js +13 -13
- package/dist/traces.js.map +1 -1
- package/dist/trajectory-replay/index.d.ts +3 -117
- package/dist/trajectory-replay/index.d.ts.map +1 -1
- package/dist/trajectory-replay/index.js +98 -56
- package/dist/trajectory-replay/index.js.map +1 -1
- package/dist/{tree-C1XYeXvy.js → tree-BGRglkKd.js} +13 -10
- package/dist/tree-BGRglkKd.js.map +1 -0
- package/dist/{types-B3jzCp0p.js → types-BXgASBCL.js} +2 -2
- package/dist/{types-B3jzCp0p.js.map → types-BXgASBCL.js.map} +1 -1
- package/dist/{types-C1tnkEgM.d.ts → types-CW8ewATZ.d.ts} +4 -113
- package/dist/types-CW8ewATZ.d.ts.map +1 -0
- package/dist/{types-BqkVmHfv.d.ts → types-CmXY0PQf.d.ts} +2 -4
- package/dist/types-CmXY0PQf.d.ts.map +1 -0
- package/dist/{types-C02qtnG8.d.ts → types-D-mgRzlh.d.ts} +7 -28
- package/dist/types-D-mgRzlh.d.ts.map +1 -0
- package/dist/{types-Cq8mSbq8.d.ts → types-D8GhJoHk.d.ts} +7 -9
- package/dist/{types-Cq8mSbq8.d.ts.map → types-D8GhJoHk.d.ts.map} +1 -1
- package/dist/{types-IXD2HJzu.js → types-DqFMNTgK.js} +2 -4
- package/dist/types-DqFMNTgK.js.map +1 -0
- package/dist/{types-Cbe2fWBa.d.ts → types-DtMbFw6U.d.ts} +3 -3
- package/dist/{types-Cbe2fWBa.d.ts.map → types-DtMbFw6U.d.ts.map} +1 -1
- package/dist/{verdict-Dl0tiT1X.js → verdict-BGQmxQ8A.js} +3 -3
- package/dist/verdict-BGQmxQ8A.js.map +1 -0
- package/dist/{verdict-E4eRNf7-.d.ts → verdict-C24NELtN.d.ts} +2 -15
- package/dist/{verdict-E4eRNf7-.d.ts.map → verdict-C24NELtN.d.ts.map} +1 -1
- package/dist/{verdict-cache-Bm8EKL2s.js → verdict-cache-CL_7tePD.js} +2 -2
- package/dist/verdict-cache-CL_7tePD.js.map +1 -0
- package/dist/wire/index.d.ts +31 -410
- package/dist/wire/index.d.ts.map +1 -1
- package/dist/wire/index.js +2 -2
- package/docs/changes/jev-evidence-review.md +1 -1
- package/docs/changes/jev-product-adoption.md +1 -1
- package/docs/changes/jev-recorded-evidence.md +1 -1
- package/docs/design/statistics-decisions.md +17 -17
- package/docs/design.md +1 -1
- package/docs/evaluation-integrity.md +1 -1
- package/docs/proofs/campaign-attempt-retention-20261001/README.md +2 -2
- package/docs/public-api.md +138 -529
- package/docs/rollout.md +1 -1
- package/docs/trace-repair-admission.md +1 -1
- package/docs/wire-protocol.md +1 -1
- package/package.json +1 -1
- package/dist/active-curriculum-B_Lc9kfO.js +0 -61
- package/dist/active-curriculum-B_Lc9kfO.js.map +0 -1
- package/dist/agent-profile-VFcQncbW.d.ts.map +0 -1
- package/dist/agent-profile-cell-BK8mlqlq.js.map +0 -1
- package/dist/agent-profile-cell-BNp-Js9F.d.ts.map +0 -1
- package/dist/agreement-irr-BEWLIF3_.js.map +0 -1
- package/dist/analyze-runs-C84irm4i.js.map +0 -1
- package/dist/baseline-Udi_PLuT.js +0 -259
- package/dist/baseline-Udi_PLuT.js.map +0 -1
- package/dist/benchmark-CXmVVHH2.js +0 -562
- package/dist/benchmark-CXmVVHH2.js.map +0 -1
- package/dist/benchmark-command-By0vh8q7.js.map +0 -1
- package/dist/campaign-BudZ70zo.js.map +0 -1
- package/dist/chat-json-call-puMArmtW.js.map +0 -1
- package/dist/claude-jsonl-dZir12m6.js +0 -86
- package/dist/claude-jsonl-dZir12m6.js.map +0 -1
- package/dist/client-BzL9niqi.d.ts.map +0 -1
- package/dist/cost-ledger-BP8V4kUQ.js.map +0 -1
- package/dist/counterfactual-CPaWN99-.d.ts +0 -60
- package/dist/counterfactual-CPaWN99-.d.ts.map +0 -1
- package/dist/counterfactual-Dn3veueY.js +0 -98
- package/dist/counterfactual-Dn3veueY.js.map +0 -1
- package/dist/default-registry-COhk9g36.js.map +0 -1
- package/dist/default-registry-DzHiGruW.d.ts.map +0 -1
- package/dist/define-agent-eval-WSck0CZJ.d.ts.map +0 -1
- package/dist/dspy-rlm-engine-LRCu61m5.js.map +0 -1
- package/dist/effect-sizes-BhzEpXMc.js +0 -82
- package/dist/effect-sizes-BhzEpXMc.js.map +0 -1
- package/dist/emitter-HcCmFQl_.d.ts.map +0 -1
- package/dist/engine-3A_4jOlI.d.ts.map +0 -1
- package/dist/equal-terms-BtdXayCJ.js.map +0 -1
- package/dist/evaluation-6Yoi74gM.d.ts.map +0 -1
- package/dist/exporters-X4xwla8A.js.map +0 -1
- package/dist/external-optimizer-process-BjMGUuds.js.map +0 -1
- package/dist/external-optimizer-subprocess-BqW0hHcI.js.map +0 -1
- package/dist/failure-cluster-CbgEDcFJ.d.ts.map +0 -1
- package/dist/feedback-trajectory-C4EG0nNH.d.ts.map +0 -1
- package/dist/hf-dataset-BR2yhuNn.js.map +0 -1
- package/dist/index-BFRO6chs.d.ts.map +0 -1
- package/dist/index-BO41eGj5.d.ts.map +0 -1
- package/dist/index-CyszSg4O.d.ts.map +0 -1
- package/dist/index-DHKEAv3s.d.ts.map +0 -1
- package/dist/index-DPcngtwt.d.ts.map +0 -1
- package/dist/integrity-dLiDe2kD.js.map +0 -1
- package/dist/journal-DfRz9kps.js.map +0 -1
- package/dist/judge-gate-CE7CQAgt.d.ts.map +0 -1
- package/dist/kind-factory-BobI5Jy3.js.map +0 -1
- package/dist/ledger-core-Gq7yp8lP.js.map +0 -1
- package/dist/llm-client-CyBAoYiX.js.map +0 -1
- package/dist/llm-judge-CG12SQJq.js.map +0 -1
- package/dist/matrix-DSrSxjjy.d.ts.map +0 -1
- package/dist/matrix-Dm0BQW7U.js.map +0 -1
- package/dist/mean-DHoMiudD.js.map +0 -1
- package/dist/normal-DsJZtrJc.js +0 -25
- package/dist/normal-DsJZtrJc.js.map +0 -1
- package/dist/paired-arms-CwMGJsrS.js.map +0 -1
- package/dist/paired-binary-CvpFHfKZ.js.map +0 -1
- package/dist/paired-promotion-decision-DPKz8ZnF.d.ts.map +0 -1
- package/dist/pareto-Cm6ndrWN.js.map +0 -1
- package/dist/prime-protocol-CHKgA3a6.js.map +0 -1
- package/dist/prime-protocol-HiB9HX7E.d.ts.map +0 -1
- package/dist/produced-state-BsDmnwxc.js.map +0 -1
- package/dist/proposal-findings-CY1xKn_a.js.map +0 -1
- package/dist/query-BUCZCK7F.js.map +0 -1
- package/dist/redact-mtCCcT8z.d.ts.map +0 -1
- package/dist/release-confidence-CpuP65S5.js.map +0 -1
- package/dist/report-command-SC9nqb4M.js.map +0 -1
- package/dist/reward-CpCmAM8L.d.ts.map +0 -1
- package/dist/reward-hacking-RVdZ6Kkt.d.ts.map +0 -1
- package/dist/rollout-DZV1t2Wg.js.map +0 -1
- package/dist/run-record-C1E24G2A.d.ts.map +0 -1
- package/dist/run-record-C5e7FjhJ.js.map +0 -1
- package/dist/schema-2g7trUwH.d.ts.map +0 -1
- package/dist/schema-BuiujZ-g.js.map +0 -1
- package/dist/schema-CSf6qWgZ.js.map +0 -1
- package/dist/search-kernel-DyNEa6IU.d.ts.map +0 -1
- package/dist/search-ledger-file-DDQUgvCg.js.map +0 -1
- package/dist/search-shipper-BOSD46qv.js.map +0 -1
- package/dist/semantic-concept-judge-BQF8te0T.js.map +0 -1
- package/dist/sequential-Bf_pbE50.js.map +0 -1
- package/dist/server-e9oWO_bN.js.map +0 -1
- package/dist/skillopt-optimization-method-BiwA8nfX.js.map +0 -1
- package/dist/statistical-heldout-42NlCNDe.js.map +0 -1
- package/dist/statistical-heldout-CYpGaoDH.d.ts.map +0 -1
- package/dist/steps-Bg5-KVx1.d.ts.map +0 -1
- package/dist/store-tool-spans-CIX5arAG.js.map +0 -1
- package/dist/store-tool-spans-DdCSPa7C.d.ts.map +0 -1
- package/dist/student-t-DVPW7CdL.js +0 -39
- package/dist/student-t-DVPW7CdL.js.map +0 -1
- package/dist/summary-report-V_JACR4Y.js.map +0 -1
- package/dist/summary-report-bdFDXq0b.d.ts.map +0 -1
- package/dist/task-failure-attributes-hEmASjix.js.map +0 -1
- package/dist/terminal-record-DZyQZ3pz.js.map +0 -1
- package/dist/tool-groups-C3NEcdCM.d.ts.map +0 -1
- package/dist/tool-waste-BKgvd-TP.d.ts.map +0 -1
- package/dist/trajectory-Y1fmdlw7.d.ts +0 -24
- package/dist/trajectory-Y1fmdlw7.d.ts.map +0 -1
- package/dist/tree-C1XYeXvy.js.map +0 -1
- package/dist/types-BqkVmHfv.d.ts.map +0 -1
- package/dist/types-C02qtnG8.d.ts.map +0 -1
- package/dist/types-C1tnkEgM.d.ts.map +0 -1
- package/dist/types-IXD2HJzu.js.map +0 -1
- package/dist/usage-receipt-Cx3xNeyB.js +0 -76
- package/dist/usage-receipt-Cx3xNeyB.js.map +0 -1
- package/dist/verdict-Dl0tiT1X.js.map +0 -1
- package/dist/verdict-cache-Bm8EKL2s.js.map +0 -1
- package/docs/design/self-improvement-evidence-audit.md +0 -263
|
@@ -1,15 +1,162 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { i as compareCodeUnits } from "../canonical-
|
|
3
|
-
import { i as pairedBootstrap, t as mean } from "../mean-
|
|
4
|
-
import { n as contentHash } from "../verdict-cache-
|
|
5
|
-
import {
|
|
1
|
+
import { c as errorMessage, n as CaptureIntegrityError, s as ValidationError } from "../errors-BcOs5MKW.js";
|
|
2
|
+
import { i as compareCodeUnits } from "../canonical-Dq2hOQ8L.js";
|
|
3
|
+
import { i as pairedBootstrap, t as mean } from "../mean-BbA_JCzW.js";
|
|
4
|
+
import { n as contentHash } from "../verdict-cache-CL_7tePD.js";
|
|
5
|
+
import { n as isRecord } from "../json-BhdsJaOQ.js";
|
|
6
|
+
import { t as certificationEvidenceDigest } from "../verdict-BGQmxQ8A.js";
|
|
6
7
|
import { t as packageVersion } from "../package-version-D7lQHt_-.js";
|
|
7
|
-
import { t as assertEqualDeclarativeTerms } from "../equal-terms-
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
8
|
+
import { t as assertEqualDeclarativeTerms } from "../equal-terms-DbPGcVtI.js";
|
|
9
|
+
import { c as runPrimeExchange, n as buildPrimePrompt, t as analystUsageReceiptFromPrimeUsage } from "../prime-protocol-BFgGrEMP.js";
|
|
10
|
+
import { i as TIMEOUT_OBSERVATION_MARKER, l as isRecordedTimeout, n as FORMAT_ERROR_OBSERVATION_PREFIX, p as parseRecordedReturncode, r as SUBMIT_ACTION_SIGNATURE, s as deriveFailureSignature, t as wrapActionForExec } from "../exec-CpbtOXVt.js";
|
|
10
11
|
import { createHash } from "node:crypto";
|
|
11
12
|
import { constants } from "node:os";
|
|
12
13
|
import { spawn } from "node:child_process";
|
|
14
|
+
//#region src/trace-repair/admission-records.ts
|
|
15
|
+
/**
|
|
16
|
+
* The vocabulary of the TB-Repair admission pre-pass: what a corpus row is,
|
|
17
|
+
* which failure population it belongs to, why it can leave the funnel, and the
|
|
18
|
+
* denominator chain those exclusions add up to.
|
|
19
|
+
*
|
|
20
|
+
* Pure data and pure functions. The gate that spends containers on these types
|
|
21
|
+
* lives in `admission.ts`, and the artifact that publishes them lives in
|
|
22
|
+
* `admission-report.ts`.
|
|
23
|
+
*/
|
|
24
|
+
/** A campaign scored rows the pre-pass did not admit, or dropped rows it did. */
|
|
25
|
+
var AdmissionDenominatorError = class extends CaptureIntegrityError {};
|
|
26
|
+
/** A row handed to the pre-pass carried a field only an analyst could produce. */
|
|
27
|
+
var AdmissionIndependenceError = class extends CaptureIntegrityError {};
|
|
28
|
+
const ADMISSION_ROW_KEYS = [
|
|
29
|
+
"rowId",
|
|
30
|
+
"taskName",
|
|
31
|
+
"recordedModel",
|
|
32
|
+
"recordedCommands",
|
|
33
|
+
"finalReturncode"
|
|
34
|
+
];
|
|
35
|
+
/**
|
|
36
|
+
* Reject rows that carry anything beyond the recording.
|
|
37
|
+
*
|
|
38
|
+
* The check is a closed key list rather than a list of known analyst field
|
|
39
|
+
* names, because the failure to catch is "some new analyst output leaked into
|
|
40
|
+
* the gate", and only a closed shape catches the fields nobody thought of.
|
|
41
|
+
*/
|
|
42
|
+
function assertAnalystIndependent(rows) {
|
|
43
|
+
const allowed = new Set(ADMISSION_ROW_KEYS);
|
|
44
|
+
for (const row of rows) {
|
|
45
|
+
const extra = Object.keys(row).filter((key) => !allowed.has(key));
|
|
46
|
+
if (extra.length > 0) throw new AdmissionIndependenceError(`admission row ${String(row.rowId)} carries analyst-side fields: ${extra.sort().join(", ")}`);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
const ADMISSION_STRATA = [
|
|
50
|
+
"clean-exit",
|
|
51
|
+
"command-error",
|
|
52
|
+
"signal-kill"
|
|
53
|
+
];
|
|
54
|
+
/** `null` when the recording holds no parseable final return code. */
|
|
55
|
+
function stratumOf(finalReturncode) {
|
|
56
|
+
if (finalReturncode === null || !Number.isInteger(finalReturncode)) return null;
|
|
57
|
+
if (finalReturncode === 0) return "clean-exit";
|
|
58
|
+
return finalReturncode < 0 ? "signal-kill" : "command-error";
|
|
59
|
+
}
|
|
60
|
+
const ADMISSION_EXCLUSION_ORDER = [
|
|
61
|
+
"no-recorded-commands",
|
|
62
|
+
"unparseable-final-returncode",
|
|
63
|
+
"stratum-not-admitted",
|
|
64
|
+
"task-oracle-uncertified",
|
|
65
|
+
"task-oracle-nondeterministic",
|
|
66
|
+
"prefix-replay-error",
|
|
67
|
+
"prefix-replay-empty",
|
|
68
|
+
"prefix-replay-truncated",
|
|
69
|
+
"prefix-divergence-above-threshold",
|
|
70
|
+
"end-state-oracle-error",
|
|
71
|
+
"end-state-tests-pass",
|
|
72
|
+
"no-fix-control-error",
|
|
73
|
+
"no-fix-control-rescued",
|
|
74
|
+
"no-op-control-error",
|
|
75
|
+
"no-op-control-rescued"
|
|
76
|
+
];
|
|
77
|
+
/** Reasons decided from the recording alone, before a row can be stratified. */
|
|
78
|
+
const PRE_STRATUM_REASONS = ["no-recorded-commands", "unparseable-final-returncode"];
|
|
79
|
+
function isPreStratumReason(reason) {
|
|
80
|
+
return PRE_STRATUM_REASONS.includes(reason);
|
|
81
|
+
}
|
|
82
|
+
/** One line of prose per reason, for the rendered chain. */
|
|
83
|
+
const ADMISSION_EXCLUSION_MEANING = Object.freeze({
|
|
84
|
+
"no-recorded-commands": "the recording holds no command to substitute",
|
|
85
|
+
"unparseable-final-returncode": "the last observation carries no return code",
|
|
86
|
+
"stratum-not-admitted": "the row belongs to a population this campaign excluded",
|
|
87
|
+
"task-oracle-uncertified": "the task grader has no determinism certification on file",
|
|
88
|
+
"task-oracle-nondeterministic": "the task grader returns different verdicts on byte-identical state",
|
|
89
|
+
"prefix-replay-error": "the replay boundary failed, so divergence is unmeasured",
|
|
90
|
+
"prefix-replay-empty": "the replay executed no recorded step",
|
|
91
|
+
"prefix-replay-truncated": "the replay stopped short of the recorded end state",
|
|
92
|
+
"prefix-divergence-above-threshold": "replaying the prefix did not reproduce the recording",
|
|
93
|
+
"end-state-oracle-error": "the task grader failed, so the end state is unjudged",
|
|
94
|
+
"end-state-tests-pass": "the task tests pass on the recorded end state, so nothing failed",
|
|
95
|
+
"no-fix-control-error": "a no-fix rollout failed to run, so the control is unmeasured",
|
|
96
|
+
"no-fix-control-rescued": "the continuation policy passes the task with no intervention",
|
|
97
|
+
"no-op-control-error": "a no-op rollout failed to run, so the control is unmeasured",
|
|
98
|
+
"no-op-control-rescued": "an inert action plus continuation passes the task"
|
|
99
|
+
});
|
|
100
|
+
/**
|
|
101
|
+
* Build the funnel every campaign report publishes.
|
|
102
|
+
*
|
|
103
|
+
* Each stage names the reason, the rows that reached it, the rows it removed,
|
|
104
|
+
* and the rows that survived, so `input = admitted + sum(excluded)` can be read
|
|
105
|
+
* off the table instead of trusted.
|
|
106
|
+
*/
|
|
107
|
+
function buildDenominatorChain(verdicts, admitStrata) {
|
|
108
|
+
const reasonTotals = emptyReasonTotals();
|
|
109
|
+
for (const verdict of verdicts) if (verdict.excludedBy !== null) reasonTotals[verdict.excludedBy] += 1;
|
|
110
|
+
const stratumReasons = ADMISSION_EXCLUSION_ORDER.filter((reason) => !isPreStratumReason(reason));
|
|
111
|
+
const artifact = {
|
|
112
|
+
version: 1,
|
|
113
|
+
overall: chainOf("all", verdicts, ADMISSION_EXCLUSION_ORDER),
|
|
114
|
+
byStratum: ADMISSION_STRATA.filter((stratum) => verdicts.some((verdict) => verdict.stratum === stratum)).map((stratum) => chainOf(stratum, verdicts.filter((verdict) => verdict.stratum === stratum), stratumReasons)),
|
|
115
|
+
reasonTotals,
|
|
116
|
+
unstratified: verdicts.filter((verdict) => verdict.stratum === null).length,
|
|
117
|
+
admitStrata: [...admitStrata]
|
|
118
|
+
};
|
|
119
|
+
assertChainReconciles(artifact);
|
|
120
|
+
return artifact;
|
|
121
|
+
}
|
|
122
|
+
function chainOf(scope, verdicts, reasons) {
|
|
123
|
+
let remaining = verdicts.length;
|
|
124
|
+
const stages = [];
|
|
125
|
+
for (const reason of reasons) {
|
|
126
|
+
const excluded = verdicts.filter((verdict) => verdict.excludedBy === reason).length;
|
|
127
|
+
const entering = remaining;
|
|
128
|
+
remaining -= excluded;
|
|
129
|
+
stages.push({
|
|
130
|
+
reason,
|
|
131
|
+
entering,
|
|
132
|
+
excluded,
|
|
133
|
+
remaining
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
return {
|
|
137
|
+
scope,
|
|
138
|
+
input: verdicts.length,
|
|
139
|
+
stages,
|
|
140
|
+
admitted: verdicts.filter((verdict) => verdict.admitted).length
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
function emptyReasonTotals() {
|
|
144
|
+
const totals = {};
|
|
145
|
+
for (const reason of ADMISSION_EXCLUSION_ORDER) totals[reason] = 0;
|
|
146
|
+
return totals;
|
|
147
|
+
}
|
|
148
|
+
/** A chain that does not add up is a broken denominator, so this throws. */
|
|
149
|
+
function assertChainReconciles(artifact) {
|
|
150
|
+
for (const chain of [artifact.overall, ...artifact.byStratum]) {
|
|
151
|
+
const excluded = chain.stages.reduce((total, stage) => total + stage.excluded, 0);
|
|
152
|
+
if (chain.input !== chain.admitted + excluded) throw new AdmissionDenominatorError(`denominator chain (${chain.scope}) does not reconcile: input ${chain.input} != admitted ${chain.admitted} + excluded ${excluded}`);
|
|
153
|
+
const last = chain.stages[chain.stages.length - 1];
|
|
154
|
+
if (last && last.remaining !== chain.admitted) throw new AdmissionDenominatorError(`denominator chain (${chain.scope}) ends at ${last.remaining} rows but reports ${chain.admitted} admitted`);
|
|
155
|
+
}
|
|
156
|
+
const stratumInputs = artifact.byStratum.reduce((total, chain) => total + chain.input, 0);
|
|
157
|
+
if (stratumInputs + artifact.unstratified !== artifact.overall.input) throw new AdmissionDenominatorError(`stratum inputs ${stratumInputs} plus ${artifact.unstratified} unstratified do not cover ${artifact.overall.input} input rows`);
|
|
158
|
+
}
|
|
159
|
+
//#endregion
|
|
13
160
|
//#region src/trace-repair/mini-swe-scaffold.ts
|
|
14
161
|
/**
|
|
15
162
|
* The mini-swe-agent scaffold as the Terminal-Bench-2 trajectory corpus
|
|
@@ -154,7 +301,7 @@ function parseAction(assistantMessage) {
|
|
|
154
301
|
function renderObservation(output) {
|
|
155
302
|
const head = output.exceptionInfo ? `<exception>${output.exceptionInfo}</exception>\n` : "";
|
|
156
303
|
const returncode = `<returncode>${output.returncode}</returncode>\n`;
|
|
157
|
-
if (output.output.length <
|
|
304
|
+
if (output.output.length < OUTPUT_ELISION_THRESHOLD) return `${head}${returncode}<output>\n${output.output}</output>`;
|
|
158
305
|
const elided = output.output.length - OUTPUT_ELISION_THRESHOLD;
|
|
159
306
|
return `${head}${returncode}<warning>\nThe output of your last command was too long.
|
|
160
307
|
Please try a different command that produces less output.
|
|
@@ -196,582 +343,69 @@ function submissionOf(output) {
|
|
|
196
343
|
return lines.slice(1).join("");
|
|
197
344
|
}
|
|
198
345
|
//#endregion
|
|
199
|
-
//#region src/trace-repair/
|
|
346
|
+
//#region src/trace-repair/continuation-policy.ts
|
|
347
|
+
/** A rollout ran outside the pinned policy, so its evidence cannot be used. */
|
|
348
|
+
var ContinuationPolicyViolationError = class extends CaptureIntegrityError {};
|
|
349
|
+
/** Two arms did not run the same policy, so their difference is not the intervention. */
|
|
350
|
+
var ContinuationSymmetryError = class extends CaptureIntegrityError {};
|
|
200
351
|
/**
|
|
201
|
-
*
|
|
202
|
-
*
|
|
203
|
-
* The analyst answers with one action, applied at step k, drawn from the same
|
|
204
|
-
* action space the scaffold had: one shell command or one edit, at most 4 KB.
|
|
205
|
-
* An answer that buys a bigger action than the scaffold could take is not a
|
|
206
|
-
* counterfactual about the recorded run, so the budget is enforced before any
|
|
207
|
-
* container is opened and a violation never reaches the reproduction gate.
|
|
352
|
+
* Everything the policy fixes except the two a campaign must choose.
|
|
208
353
|
*
|
|
209
|
-
*
|
|
210
|
-
*
|
|
211
|
-
*
|
|
212
|
-
* Two statements separated by a newline or `;` are two actions and are
|
|
213
|
-
* rejected. Heredoc bodies, comments and compound blocks (`if`, `for`,
|
|
214
|
-
* `while`, `until`, `case`, `{ … }`) are inside a statement, never separators.
|
|
354
|
+
* `commandTimeoutSeconds` is 30 because the recorded runs used the scaffold's
|
|
355
|
+
* own 30-second environment timeout; a longer limit would let the continuation
|
|
356
|
+
* finish commands the recorded agent could not.
|
|
215
357
|
*/
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
358
|
+
const CONTINUATION_POLICY_DEFAULTS = {
|
|
359
|
+
id: "tb-repair-continuation-v1",
|
|
360
|
+
stepBudget: 20,
|
|
361
|
+
temperature: 0,
|
|
362
|
+
maxTokens: 4096,
|
|
363
|
+
commandTimeoutSeconds: 30,
|
|
364
|
+
maxConsecutiveFormatErrors: 3,
|
|
365
|
+
networkMode: "none",
|
|
366
|
+
scaffold: "mini-swe-agent"
|
|
367
|
+
};
|
|
368
|
+
function definePinnedContinuationPolicy(input) {
|
|
369
|
+
const policy = {
|
|
370
|
+
...CONTINUATION_POLICY_DEFAULTS,
|
|
371
|
+
...input
|
|
372
|
+
};
|
|
373
|
+
if (!policy.model.trim()) throw new ValidationError("continuation policy requires a model id");
|
|
374
|
+
if (!Number.isInteger(policy.seed)) throw new ValidationError(`continuation policy seed must be an integer, got ${policy.seed}`);
|
|
375
|
+
requirePositiveInteger$1(policy.stepBudget, "stepBudget");
|
|
376
|
+
requirePositiveInteger$1(policy.maxTokens, "maxTokens");
|
|
377
|
+
requirePositiveInteger$1(policy.commandTimeoutSeconds, "commandTimeoutSeconds");
|
|
378
|
+
requirePositiveInteger$1(policy.maxConsecutiveFormatErrors, "maxConsecutiveFormatErrors");
|
|
379
|
+
if (!Number.isFinite(policy.temperature) || policy.temperature < 0) throw new ValidationError(`continuation policy temperature must be a non-negative number, got ${policy.temperature}`);
|
|
380
|
+
return Object.freeze(policy);
|
|
381
|
+
}
|
|
382
|
+
function requirePositiveInteger$1(value, field) {
|
|
383
|
+
if (!Number.isInteger(value) || value <= 0) throw new ValidationError(`continuation policy ${field} must be a positive integer, got ${value}`);
|
|
384
|
+
}
|
|
222
385
|
/**
|
|
223
|
-
*
|
|
224
|
-
*
|
|
225
|
-
*
|
|
226
|
-
* wastes the corpus.
|
|
386
|
+
* Hash over the policy and the scaffold text it renders. A changed template
|
|
387
|
+
* changes the digest, so rollouts recorded before and after an edit cannot be
|
|
388
|
+
* pooled by accident.
|
|
227
389
|
*/
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
]);
|
|
245
|
-
const BLOCK_CLOSERS = /* @__PURE__ */ new Set([
|
|
246
|
-
"fi",
|
|
247
|
-
"done",
|
|
248
|
-
"esac"
|
|
249
|
-
]);
|
|
390
|
+
function continuationPolicyDigest(policy) {
|
|
391
|
+
return contentHash({
|
|
392
|
+
policy: { ...policy },
|
|
393
|
+
systemMessage: MINI_SWE_SYSTEM_MESSAGE,
|
|
394
|
+
instanceMessage: renderInstanceMessage({
|
|
395
|
+
task: "<task>",
|
|
396
|
+
systemInformation: "<system>"
|
|
397
|
+
}),
|
|
398
|
+
formatErrorObservation: renderFormatErrorObservation(0),
|
|
399
|
+
timeoutObservation: renderTimeoutObservation("<command>", "<output>"),
|
|
400
|
+
observation: renderObservation({
|
|
401
|
+
returncode: 0,
|
|
402
|
+
output: ""
|
|
403
|
+
})
|
|
404
|
+
});
|
|
405
|
+
}
|
|
250
406
|
/**
|
|
251
|
-
*
|
|
252
|
-
*
|
|
253
|
-
* The scan tracks quoting, escapes, command substitution, brace and paren
|
|
254
|
-
* grouping, comments, compound-block keywords, and heredoc bodies. Anything
|
|
255
|
-
* it cannot resolve stays inside the current statement, so an unparseable
|
|
256
|
-
* action reads as one oversized statement and is rejected on bytes rather
|
|
257
|
-
* than silently accepted as one clean action.
|
|
258
|
-
*/
|
|
259
|
-
function scanShellAction(script) {
|
|
260
|
-
const state = {
|
|
261
|
-
statements: [],
|
|
262
|
-
current: "",
|
|
263
|
-
heredocs: 0
|
|
264
|
-
};
|
|
265
|
-
let index = 0;
|
|
266
|
-
let parenDepth = 0;
|
|
267
|
-
let braceDepth = 0;
|
|
268
|
-
let blockDepth = 0;
|
|
269
|
-
let inSingle = false;
|
|
270
|
-
let inDouble = false;
|
|
271
|
-
let inBacktick = false;
|
|
272
|
-
let pendingHeredocs = [];
|
|
273
|
-
let word = "";
|
|
274
|
-
let atStatementStart = true;
|
|
275
|
-
const flushWord = () => {
|
|
276
|
-
if (word.length === 0) return;
|
|
277
|
-
if (BLOCK_OPENERS.has(word)) blockDepth += 1;
|
|
278
|
-
else if (BLOCK_CLOSERS.has(word)) blockDepth = Math.max(0, blockDepth - 1);
|
|
279
|
-
word = "";
|
|
280
|
-
};
|
|
281
|
-
const endStatement = () => {
|
|
282
|
-
flushWord();
|
|
283
|
-
const text = state.current.trim();
|
|
284
|
-
if (text.length > 0 && !isCommentOnly(text)) state.statements.push(text);
|
|
285
|
-
state.current = "";
|
|
286
|
-
atStatementStart = true;
|
|
287
|
-
};
|
|
288
|
-
while (index < script.length) {
|
|
289
|
-
const char = script[index];
|
|
290
|
-
if (inSingle) {
|
|
291
|
-
state.current += char;
|
|
292
|
-
if (char === "'") inSingle = false;
|
|
293
|
-
index += 1;
|
|
294
|
-
continue;
|
|
295
|
-
}
|
|
296
|
-
if (char === "\\" && index + 1 < script.length) {
|
|
297
|
-
const next = script[index + 1];
|
|
298
|
-
if (next === "\n" && !inDouble) {
|
|
299
|
-
state.current += char + next;
|
|
300
|
-
index += 2;
|
|
301
|
-
continue;
|
|
302
|
-
}
|
|
303
|
-
state.current += char + next;
|
|
304
|
-
index += 2;
|
|
305
|
-
continue;
|
|
306
|
-
}
|
|
307
|
-
if (inDouble) {
|
|
308
|
-
state.current += char;
|
|
309
|
-
if (char === "\"") inDouble = false;
|
|
310
|
-
index += 1;
|
|
311
|
-
continue;
|
|
312
|
-
}
|
|
313
|
-
if (char === "'") {
|
|
314
|
-
flushWord();
|
|
315
|
-
inSingle = true;
|
|
316
|
-
state.current += char;
|
|
317
|
-
index += 1;
|
|
318
|
-
atStatementStart = false;
|
|
319
|
-
continue;
|
|
320
|
-
}
|
|
321
|
-
if (char === "\"") {
|
|
322
|
-
flushWord();
|
|
323
|
-
inDouble = true;
|
|
324
|
-
state.current += char;
|
|
325
|
-
index += 1;
|
|
326
|
-
atStatementStart = false;
|
|
327
|
-
continue;
|
|
328
|
-
}
|
|
329
|
-
if (char === "`") {
|
|
330
|
-
inBacktick = !inBacktick;
|
|
331
|
-
state.current += char;
|
|
332
|
-
index += 1;
|
|
333
|
-
atStatementStart = false;
|
|
334
|
-
continue;
|
|
335
|
-
}
|
|
336
|
-
if (char === "#" && (atStatementStart || /\s/.test(script[index - 1] ?? " "))) {
|
|
337
|
-
const end = script.indexOf("\n", index);
|
|
338
|
-
const stop = end === -1 ? script.length : end;
|
|
339
|
-
state.current += script.slice(index, stop);
|
|
340
|
-
index = stop;
|
|
341
|
-
continue;
|
|
342
|
-
}
|
|
343
|
-
if (char === "<" && script[index + 1] === "<" && script[index + 2] !== "<" && script[index - 1] !== "<") {
|
|
344
|
-
const mark = readHeredocMark(script, index);
|
|
345
|
-
if (mark) {
|
|
346
|
-
pendingHeredocs.push(mark.mark);
|
|
347
|
-
state.heredocs += 1;
|
|
348
|
-
state.current += script.slice(index, mark.nextIndex);
|
|
349
|
-
index = mark.nextIndex;
|
|
350
|
-
atStatementStart = false;
|
|
351
|
-
continue;
|
|
352
|
-
}
|
|
353
|
-
}
|
|
354
|
-
if (char === "(") {
|
|
355
|
-
flushWord();
|
|
356
|
-
parenDepth += 1;
|
|
357
|
-
state.current += char;
|
|
358
|
-
index += 1;
|
|
359
|
-
atStatementStart = true;
|
|
360
|
-
continue;
|
|
361
|
-
}
|
|
362
|
-
if (char === ")") {
|
|
363
|
-
flushWord();
|
|
364
|
-
parenDepth = Math.max(0, parenDepth - 1);
|
|
365
|
-
state.current += char;
|
|
366
|
-
index += 1;
|
|
367
|
-
atStatementStart = false;
|
|
368
|
-
continue;
|
|
369
|
-
}
|
|
370
|
-
if (char === "{") {
|
|
371
|
-
flushWord();
|
|
372
|
-
braceDepth += 1;
|
|
373
|
-
state.current += char;
|
|
374
|
-
index += 1;
|
|
375
|
-
atStatementStart = true;
|
|
376
|
-
continue;
|
|
377
|
-
}
|
|
378
|
-
if (char === "}") {
|
|
379
|
-
flushWord();
|
|
380
|
-
braceDepth = Math.max(0, braceDepth - 1);
|
|
381
|
-
state.current += char;
|
|
382
|
-
index += 1;
|
|
383
|
-
atStatementStart = false;
|
|
384
|
-
continue;
|
|
385
|
-
}
|
|
386
|
-
if (char === "\n") {
|
|
387
|
-
flushWord();
|
|
388
|
-
if (pendingHeredocs.length > 0) {
|
|
389
|
-
const consumed = consumeHeredocBodies(script, index + 1, pendingHeredocs);
|
|
390
|
-
state.current += script.slice(index, consumed);
|
|
391
|
-
pendingHeredocs = [];
|
|
392
|
-
index = consumed;
|
|
393
|
-
continue;
|
|
394
|
-
}
|
|
395
|
-
if (parenDepth > 0 || braceDepth > 0 || blockDepth > 0 || inBacktick || endsWithContinuation(state.current)) {
|
|
396
|
-
state.current += char;
|
|
397
|
-
index += 1;
|
|
398
|
-
atStatementStart = true;
|
|
399
|
-
continue;
|
|
400
|
-
}
|
|
401
|
-
endStatement();
|
|
402
|
-
index += 1;
|
|
403
|
-
continue;
|
|
404
|
-
}
|
|
405
|
-
if (char === ";") {
|
|
406
|
-
flushWord();
|
|
407
|
-
if (parenDepth > 0 || braceDepth > 0 || blockDepth > 0 || inBacktick) {
|
|
408
|
-
state.current += char;
|
|
409
|
-
index += 1;
|
|
410
|
-
atStatementStart = true;
|
|
411
|
-
continue;
|
|
412
|
-
}
|
|
413
|
-
endStatement();
|
|
414
|
-
index += 1;
|
|
415
|
-
continue;
|
|
416
|
-
}
|
|
417
|
-
if (/\s/.test(char)) {
|
|
418
|
-
flushWord();
|
|
419
|
-
state.current += char;
|
|
420
|
-
index += 1;
|
|
421
|
-
continue;
|
|
422
|
-
}
|
|
423
|
-
if (char === "&" || char === "|") {
|
|
424
|
-
flushWord();
|
|
425
|
-
state.current += char;
|
|
426
|
-
index += 1;
|
|
427
|
-
atStatementStart = true;
|
|
428
|
-
continue;
|
|
429
|
-
}
|
|
430
|
-
word += char;
|
|
431
|
-
state.current += char;
|
|
432
|
-
atStatementStart = false;
|
|
433
|
-
index += 1;
|
|
434
|
-
}
|
|
435
|
-
endStatement();
|
|
436
|
-
return {
|
|
437
|
-
statements: state.statements,
|
|
438
|
-
heredocs: state.heredocs
|
|
439
|
-
};
|
|
440
|
-
}
|
|
441
|
-
function isCommentOnly(text) {
|
|
442
|
-
return text.split("\n").every((line) => line.trim().length === 0 || line.trim().startsWith("#"));
|
|
443
|
-
}
|
|
444
|
-
function endsWithContinuation(current) {
|
|
445
|
-
const trimmed = current.trimEnd();
|
|
446
|
-
return /(&&|\|\||\||&|\\)$/.test(trimmed);
|
|
447
|
-
}
|
|
448
|
-
function readHeredocMark(script, index) {
|
|
449
|
-
let cursor = index + 2;
|
|
450
|
-
let stripTabs = false;
|
|
451
|
-
if (script[cursor] === "-") {
|
|
452
|
-
stripTabs = true;
|
|
453
|
-
cursor += 1;
|
|
454
|
-
}
|
|
455
|
-
while (script[cursor] === " " || script[cursor] === " ") cursor += 1;
|
|
456
|
-
const quote = script[cursor];
|
|
457
|
-
if (quote === "'" || quote === "\"") {
|
|
458
|
-
const close = script.indexOf(quote, cursor + 1);
|
|
459
|
-
if (close === -1) return null;
|
|
460
|
-
return {
|
|
461
|
-
mark: {
|
|
462
|
-
delimiter: script.slice(cursor + 1, close),
|
|
463
|
-
stripTabs,
|
|
464
|
-
quoted: true
|
|
465
|
-
},
|
|
466
|
-
nextIndex: close + 1
|
|
467
|
-
};
|
|
468
|
-
}
|
|
469
|
-
const match = /^[A-Za-z_][A-Za-z0-9_]*/.exec(script.slice(cursor));
|
|
470
|
-
if (!match) return null;
|
|
471
|
-
return {
|
|
472
|
-
mark: {
|
|
473
|
-
delimiter: match[0],
|
|
474
|
-
stripTabs,
|
|
475
|
-
quoted: false
|
|
476
|
-
},
|
|
477
|
-
nextIndex: cursor + match[0].length
|
|
478
|
-
};
|
|
479
|
-
}
|
|
480
|
-
/** Consume every pending heredoc body; returns the offset just past the last
|
|
481
|
-
* terminator, or the end of the script when a terminator never arrives. */
|
|
482
|
-
function consumeHeredocBodies(script, from, marks) {
|
|
483
|
-
let cursor = from;
|
|
484
|
-
for (const mark of marks) {
|
|
485
|
-
let closed = false;
|
|
486
|
-
while (cursor < script.length) {
|
|
487
|
-
const lineEnd = script.indexOf("\n", cursor);
|
|
488
|
-
const stop = lineEnd === -1 ? script.length : lineEnd;
|
|
489
|
-
const line = script.slice(cursor, stop);
|
|
490
|
-
const candidate = mark.stripTabs ? line.replace(/^\t+/, "") : line;
|
|
491
|
-
cursor = lineEnd === -1 ? script.length : lineEnd + 1;
|
|
492
|
-
if (candidate === mark.delimiter) {
|
|
493
|
-
closed = true;
|
|
494
|
-
break;
|
|
495
|
-
}
|
|
496
|
-
}
|
|
497
|
-
if (!closed) return script.length;
|
|
498
|
-
}
|
|
499
|
-
return cursor;
|
|
500
|
-
}
|
|
501
|
-
/** Payload shape of an action: `edit` when it authors file content through a
|
|
502
|
-
* heredoc, `shell` otherwise. */
|
|
503
|
-
function classifyActionPayload(action) {
|
|
504
|
-
return scanShellAction(action).heredocs > 0 ? "edit" : "shell";
|
|
505
|
-
}
|
|
506
|
-
/**
|
|
507
|
-
* Measure an action against the budget.
|
|
508
|
-
*
|
|
509
|
-
* The budget bounds what the scaffold can execute: one top-level statement,
|
|
510
|
-
* one authored file, a byte cap, and neither a no-op nor a submit. Every
|
|
511
|
-
* rejection here is one of those.
|
|
512
|
-
*
|
|
513
|
-
* `declaredKind` is what the analyst called its own action. It is recorded
|
|
514
|
-
* beside the measured payload and never rejected on, because the scaffold runs
|
|
515
|
-
* the action identically either way — so rejecting the label scores an arm on
|
|
516
|
-
* how it described a repair rather than on the repair. A reader who wants the
|
|
517
|
-
* mismatch counts `declared` against `payload`.
|
|
518
|
-
*/
|
|
519
|
-
function checkInterventionBudget(action, declaredKind, budget = SCAFFOLD_INTERVENTION_BUDGET) {
|
|
520
|
-
assertBudget(budget);
|
|
521
|
-
const scan = scanShellAction(action);
|
|
522
|
-
const payload = scan.heredocs > 0 ? "edit" : "shell";
|
|
523
|
-
const measurement = {
|
|
524
|
-
bytes: Buffer.byteLength(action, "utf8"),
|
|
525
|
-
statements: scan.statements.length,
|
|
526
|
-
heredocs: scan.heredocs,
|
|
527
|
-
payload,
|
|
528
|
-
declared: declaredKind
|
|
529
|
-
};
|
|
530
|
-
const reject = (violation, detail) => ({
|
|
531
|
-
admissible: false,
|
|
532
|
-
violation,
|
|
533
|
-
detail,
|
|
534
|
-
measurement
|
|
535
|
-
});
|
|
536
|
-
if (action.trim().length === 0) return reject("empty", "the intervention is empty");
|
|
537
|
-
if (measurement.bytes > budget.maxBytes) return reject("over-byte-cap", `${measurement.bytes} bytes exceeds the ${budget.maxBytes}-byte action budget`);
|
|
538
|
-
if (measurement.statements > budget.maxStatements) return reject("multiple-statements", `${measurement.statements} top-level statements exceeds the ${budget.maxStatements} the scaffold takes per action`);
|
|
539
|
-
if (measurement.heredocs > budget.maxHeredocs) return reject("multiple-heredocs", `${measurement.heredocs} heredocs exceeds the ${budget.maxHeredocs} one edit may author`);
|
|
540
|
-
if (action.includes(SUBMIT_SENTINEL)) return reject("submit-instead-of-repair", "the action submits the run instead of repairing it");
|
|
541
|
-
if (NO_OP_ACTIONS.includes(action.trim())) return reject("no-op-action", `"${action.trim()}" changes nothing`);
|
|
542
|
-
return {
|
|
543
|
-
admissible: true,
|
|
544
|
-
measurement
|
|
545
|
-
};
|
|
546
|
-
}
|
|
547
|
-
function assertBudget(budget) {
|
|
548
|
-
for (const field of [
|
|
549
|
-
"maxBytes",
|
|
550
|
-
"maxStatements",
|
|
551
|
-
"maxHeredocs"
|
|
552
|
-
]) {
|
|
553
|
-
const value = budget[field];
|
|
554
|
-
if (!Number.isInteger(value) || value <= 0) throw new ValidationError(`intervention budget ${field} must be a positive integer, got ${value}`);
|
|
555
|
-
}
|
|
556
|
-
}
|
|
557
|
-
/**
|
|
558
|
-
* Whitespace-insensitive comparison used to reject an intervention that is
|
|
559
|
-
* the recorded action again. Trailing whitespace and blank lines are the only
|
|
560
|
-
* differences a re-proposal can carry without changing what runs.
|
|
561
|
-
*/
|
|
562
|
-
function normalizeActionForComparison(action) {
|
|
563
|
-
return action.split("\n").map((line) => line.trimEnd()).filter((line, index, lines) => line.length > 0 || index > 0 && index < lines.length - 1).join("\n").trim();
|
|
564
|
-
}
|
|
565
|
-
//#endregion
|
|
566
|
-
//#region src/trace-repair/admission-records.ts
|
|
567
|
-
/**
|
|
568
|
-
* The vocabulary of the TB-Repair admission pre-pass: what a corpus row is,
|
|
569
|
-
* which failure population it belongs to, why it can leave the funnel, and the
|
|
570
|
-
* denominator chain those exclusions add up to.
|
|
571
|
-
*
|
|
572
|
-
* Pure data and pure functions. The gate that spends containers on these types
|
|
573
|
-
* lives in `admission.ts`, and the artifact that publishes them lives in
|
|
574
|
-
* `admission-report.ts`.
|
|
575
|
-
*/
|
|
576
|
-
/** A campaign scored rows the pre-pass did not admit, or dropped rows it did. */
|
|
577
|
-
var AdmissionDenominatorError = class extends CaptureIntegrityError {};
|
|
578
|
-
/** A row handed to the pre-pass carried a field only an analyst could produce. */
|
|
579
|
-
var AdmissionIndependenceError = class extends CaptureIntegrityError {};
|
|
580
|
-
const ADMISSION_ROW_KEYS = [
|
|
581
|
-
"rowId",
|
|
582
|
-
"taskName",
|
|
583
|
-
"recordedModel",
|
|
584
|
-
"recordedCommands",
|
|
585
|
-
"finalReturncode"
|
|
586
|
-
];
|
|
587
|
-
/**
|
|
588
|
-
* Reject rows that carry anything beyond the recording.
|
|
589
|
-
*
|
|
590
|
-
* The check is a closed key list rather than a list of known analyst field
|
|
591
|
-
* names, because the failure to catch is "some new analyst output leaked into
|
|
592
|
-
* the gate", and only a closed shape catches the fields nobody thought of.
|
|
593
|
-
*/
|
|
594
|
-
function assertAnalystIndependent(rows) {
|
|
595
|
-
const allowed = new Set(ADMISSION_ROW_KEYS);
|
|
596
|
-
for (const row of rows) {
|
|
597
|
-
const extra = Object.keys(row).filter((key) => !allowed.has(key));
|
|
598
|
-
if (extra.length > 0) throw new AdmissionIndependenceError(`admission row ${String(row.rowId)} carries analyst-side fields: ${extra.sort().join(", ")}`);
|
|
599
|
-
}
|
|
600
|
-
}
|
|
601
|
-
const ADMISSION_STRATA = [
|
|
602
|
-
"clean-exit",
|
|
603
|
-
"command-error",
|
|
604
|
-
"signal-kill"
|
|
605
|
-
];
|
|
606
|
-
/** `null` when the recording holds no parseable final return code. */
|
|
607
|
-
function stratumOf(finalReturncode) {
|
|
608
|
-
if (finalReturncode === null || !Number.isInteger(finalReturncode)) return null;
|
|
609
|
-
if (finalReturncode === 0) return "clean-exit";
|
|
610
|
-
return finalReturncode < 0 ? "signal-kill" : "command-error";
|
|
611
|
-
}
|
|
612
|
-
const ADMISSION_EXCLUSION_ORDER = [
|
|
613
|
-
"no-recorded-commands",
|
|
614
|
-
"unparseable-final-returncode",
|
|
615
|
-
"stratum-not-admitted",
|
|
616
|
-
"task-oracle-uncertified",
|
|
617
|
-
"task-oracle-nondeterministic",
|
|
618
|
-
"prefix-replay-error",
|
|
619
|
-
"prefix-replay-empty",
|
|
620
|
-
"prefix-replay-truncated",
|
|
621
|
-
"prefix-divergence-above-threshold",
|
|
622
|
-
"end-state-oracle-error",
|
|
623
|
-
"end-state-tests-pass",
|
|
624
|
-
"no-fix-control-error",
|
|
625
|
-
"no-fix-control-rescued",
|
|
626
|
-
"no-op-control-error",
|
|
627
|
-
"no-op-control-rescued"
|
|
628
|
-
];
|
|
629
|
-
/** Reasons decided from the recording alone, before a row can be stratified. */
|
|
630
|
-
const PRE_STRATUM_REASONS = ["no-recorded-commands", "unparseable-final-returncode"];
|
|
631
|
-
function isPreStratumReason(reason) {
|
|
632
|
-
return PRE_STRATUM_REASONS.includes(reason);
|
|
633
|
-
}
|
|
634
|
-
/** One line of prose per reason, for the rendered chain. */
|
|
635
|
-
const ADMISSION_EXCLUSION_MEANING = Object.freeze({
|
|
636
|
-
"no-recorded-commands": "the recording holds no command to substitute",
|
|
637
|
-
"unparseable-final-returncode": "the last observation carries no return code",
|
|
638
|
-
"stratum-not-admitted": "the row belongs to a population this campaign excluded",
|
|
639
|
-
"task-oracle-uncertified": "the task grader has no determinism certification on file",
|
|
640
|
-
"task-oracle-nondeterministic": "the task grader returns different verdicts on byte-identical state",
|
|
641
|
-
"prefix-replay-error": "the replay boundary failed, so divergence is unmeasured",
|
|
642
|
-
"prefix-replay-empty": "the replay executed no recorded step",
|
|
643
|
-
"prefix-replay-truncated": "the replay stopped short of the recorded end state",
|
|
644
|
-
"prefix-divergence-above-threshold": "replaying the prefix did not reproduce the recording",
|
|
645
|
-
"end-state-oracle-error": "the task grader failed, so the end state is unjudged",
|
|
646
|
-
"end-state-tests-pass": "the task tests pass on the recorded end state, so nothing failed",
|
|
647
|
-
"no-fix-control-error": "a no-fix rollout failed to run, so the control is unmeasured",
|
|
648
|
-
"no-fix-control-rescued": "the continuation policy passes the task with no intervention",
|
|
649
|
-
"no-op-control-error": "a no-op rollout failed to run, so the control is unmeasured",
|
|
650
|
-
"no-op-control-rescued": "an inert action plus continuation passes the task"
|
|
651
|
-
});
|
|
652
|
-
/**
|
|
653
|
-
* Build the funnel every campaign report publishes.
|
|
654
|
-
*
|
|
655
|
-
* Each stage names the reason, the rows that reached it, the rows it removed,
|
|
656
|
-
* and the rows that survived, so `input = admitted + sum(excluded)` can be read
|
|
657
|
-
* off the table instead of trusted.
|
|
658
|
-
*/
|
|
659
|
-
function buildDenominatorChain(verdicts, admitStrata) {
|
|
660
|
-
const reasonTotals = emptyReasonTotals();
|
|
661
|
-
for (const verdict of verdicts) if (verdict.excludedBy !== null) reasonTotals[verdict.excludedBy] += 1;
|
|
662
|
-
const stratumReasons = ADMISSION_EXCLUSION_ORDER.filter((reason) => !isPreStratumReason(reason));
|
|
663
|
-
const artifact = {
|
|
664
|
-
version: 1,
|
|
665
|
-
overall: chainOf("all", verdicts, ADMISSION_EXCLUSION_ORDER),
|
|
666
|
-
byStratum: ADMISSION_STRATA.filter((stratum) => verdicts.some((verdict) => verdict.stratum === stratum)).map((stratum) => chainOf(stratum, verdicts.filter((verdict) => verdict.stratum === stratum), stratumReasons)),
|
|
667
|
-
reasonTotals,
|
|
668
|
-
unstratified: verdicts.filter((verdict) => verdict.stratum === null).length,
|
|
669
|
-
admitStrata: [...admitStrata]
|
|
670
|
-
};
|
|
671
|
-
assertChainReconciles(artifact);
|
|
672
|
-
return artifact;
|
|
673
|
-
}
|
|
674
|
-
function chainOf(scope, verdicts, reasons) {
|
|
675
|
-
let remaining = verdicts.length;
|
|
676
|
-
const stages = [];
|
|
677
|
-
for (const reason of reasons) {
|
|
678
|
-
const excluded = verdicts.filter((verdict) => verdict.excludedBy === reason).length;
|
|
679
|
-
const entering = remaining;
|
|
680
|
-
remaining -= excluded;
|
|
681
|
-
stages.push({
|
|
682
|
-
reason,
|
|
683
|
-
entering,
|
|
684
|
-
excluded,
|
|
685
|
-
remaining
|
|
686
|
-
});
|
|
687
|
-
}
|
|
688
|
-
return {
|
|
689
|
-
scope,
|
|
690
|
-
input: verdicts.length,
|
|
691
|
-
stages,
|
|
692
|
-
admitted: verdicts.filter((verdict) => verdict.admitted).length
|
|
693
|
-
};
|
|
694
|
-
}
|
|
695
|
-
function emptyReasonTotals() {
|
|
696
|
-
const totals = {};
|
|
697
|
-
for (const reason of ADMISSION_EXCLUSION_ORDER) totals[reason] = 0;
|
|
698
|
-
return totals;
|
|
699
|
-
}
|
|
700
|
-
/** A chain that does not add up is a broken denominator, so this throws. */
|
|
701
|
-
function assertChainReconciles(artifact) {
|
|
702
|
-
for (const chain of [artifact.overall, ...artifact.byStratum]) {
|
|
703
|
-
const excluded = chain.stages.reduce((total, stage) => total + stage.excluded, 0);
|
|
704
|
-
if (chain.input !== chain.admitted + excluded) throw new AdmissionDenominatorError(`denominator chain (${chain.scope}) does not reconcile: input ${chain.input} != admitted ${chain.admitted} + excluded ${excluded}`);
|
|
705
|
-
const last = chain.stages[chain.stages.length - 1];
|
|
706
|
-
if (last && last.remaining !== chain.admitted) throw new AdmissionDenominatorError(`denominator chain (${chain.scope}) ends at ${last.remaining} rows but reports ${chain.admitted} admitted`);
|
|
707
|
-
}
|
|
708
|
-
const stratumInputs = artifact.byStratum.reduce((total, chain) => total + chain.input, 0);
|
|
709
|
-
if (stratumInputs + artifact.unstratified !== artifact.overall.input) throw new AdmissionDenominatorError(`stratum inputs ${stratumInputs} plus ${artifact.unstratified} unstratified do not cover ${artifact.overall.input} input rows`);
|
|
710
|
-
}
|
|
711
|
-
//#endregion
|
|
712
|
-
//#region src/trace-repair/continuation-policy.ts
|
|
713
|
-
/** A rollout ran outside the pinned policy, so its evidence cannot be used. */
|
|
714
|
-
var ContinuationPolicyViolationError = class extends CaptureIntegrityError {};
|
|
715
|
-
/** Two arms did not run the same policy, so their difference is not the intervention. */
|
|
716
|
-
var ContinuationSymmetryError = class extends CaptureIntegrityError {};
|
|
717
|
-
/**
|
|
718
|
-
* Everything the policy fixes except the two a campaign must choose.
|
|
719
|
-
*
|
|
720
|
-
* `commandTimeoutSeconds` is 30 because the recorded runs used the scaffold's
|
|
721
|
-
* own 30-second environment timeout; a longer limit would let the continuation
|
|
722
|
-
* finish commands the recorded agent could not.
|
|
723
|
-
*/
|
|
724
|
-
const CONTINUATION_POLICY_DEFAULTS = {
|
|
725
|
-
id: "tb-repair-continuation-v1",
|
|
726
|
-
stepBudget: 20,
|
|
727
|
-
temperature: 0,
|
|
728
|
-
maxTokens: 4096,
|
|
729
|
-
commandTimeoutSeconds: 30,
|
|
730
|
-
maxConsecutiveFormatErrors: 3,
|
|
731
|
-
networkMode: "none",
|
|
732
|
-
scaffold: "mini-swe-agent"
|
|
733
|
-
};
|
|
734
|
-
function definePinnedContinuationPolicy(input) {
|
|
735
|
-
const policy = {
|
|
736
|
-
...CONTINUATION_POLICY_DEFAULTS,
|
|
737
|
-
...input
|
|
738
|
-
};
|
|
739
|
-
if (!policy.model.trim()) throw new ValidationError("continuation policy requires a model id");
|
|
740
|
-
if (!Number.isInteger(policy.seed)) throw new ValidationError(`continuation policy seed must be an integer, got ${policy.seed}`);
|
|
741
|
-
requirePositiveInteger$1(policy.stepBudget, "stepBudget");
|
|
742
|
-
requirePositiveInteger$1(policy.maxTokens, "maxTokens");
|
|
743
|
-
requirePositiveInteger$1(policy.commandTimeoutSeconds, "commandTimeoutSeconds");
|
|
744
|
-
requirePositiveInteger$1(policy.maxConsecutiveFormatErrors, "maxConsecutiveFormatErrors");
|
|
745
|
-
if (!Number.isFinite(policy.temperature) || policy.temperature < 0) throw new ValidationError(`continuation policy temperature must be a non-negative number, got ${policy.temperature}`);
|
|
746
|
-
return Object.freeze(policy);
|
|
747
|
-
}
|
|
748
|
-
function requirePositiveInteger$1(value, field) {
|
|
749
|
-
if (!Number.isInteger(value) || value <= 0) throw new ValidationError(`continuation policy ${field} must be a positive integer, got ${value}`);
|
|
750
|
-
}
|
|
751
|
-
/**
|
|
752
|
-
* Hash over the policy and the scaffold text it renders. A changed template
|
|
753
|
-
* changes the digest, so rollouts recorded before and after an edit cannot be
|
|
754
|
-
* pooled by accident.
|
|
755
|
-
*/
|
|
756
|
-
function continuationPolicyDigest(policy) {
|
|
757
|
-
return contentHash({
|
|
758
|
-
policy: { ...policy },
|
|
759
|
-
systemMessage: MINI_SWE_SYSTEM_MESSAGE,
|
|
760
|
-
instanceMessage: renderInstanceMessage({
|
|
761
|
-
task: "<task>",
|
|
762
|
-
systemInformation: "<system>"
|
|
763
|
-
}),
|
|
764
|
-
formatErrorObservation: renderFormatErrorObservation(0),
|
|
765
|
-
timeoutObservation: renderTimeoutObservation("<command>", "<output>"),
|
|
766
|
-
observation: renderObservation({
|
|
767
|
-
returncode: 0,
|
|
768
|
-
output: ""
|
|
769
|
-
})
|
|
770
|
-
});
|
|
771
|
-
}
|
|
772
|
-
/**
|
|
773
|
-
* Per-rollout seed. It reads the policy seed, the row, and the rollout index —
|
|
774
|
-
* deliberately not the arm, so paired rollouts across arms draw identically.
|
|
407
|
+
* Per-rollout seed. It reads the policy seed, the row, and the rollout index —
|
|
408
|
+
* deliberately not the arm, so paired rollouts across arms draw identically.
|
|
775
409
|
*/
|
|
776
410
|
function continuationSeed(policySeed, rowId, rolloutIndex) {
|
|
777
411
|
const input = `${policySeed}:${rowId}:${rolloutIndex}`;
|
|
@@ -1448,333 +1082,688 @@ function summarizeControlCost(verdicts) {
|
|
|
1448
1082
|
usd
|
|
1449
1083
|
};
|
|
1450
1084
|
}
|
|
1451
|
-
function groupAdmittedByStratum(verdicts) {
|
|
1452
|
-
const grouped = {
|
|
1453
|
-
"clean-exit": [],
|
|
1454
|
-
"command-error": [],
|
|
1455
|
-
"signal-kill": []
|
|
1456
|
-
};
|
|
1457
|
-
for (const verdict of verdicts) {
|
|
1458
|
-
if (!verdict.admitted || verdict.stratum === null) continue;
|
|
1459
|
-
grouped[verdict.stratum].push(verdict.rowId);
|
|
1460
|
-
}
|
|
1461
|
-
return Object.freeze({
|
|
1462
|
-
"clean-exit": Object.freeze(grouped["clean-exit"]),
|
|
1463
|
-
"command-error": Object.freeze(grouped["command-error"]),
|
|
1464
|
-
"signal-kill": Object.freeze(grouped["signal-kill"])
|
|
1465
|
-
});
|
|
1085
|
+
function groupAdmittedByStratum(verdicts) {
|
|
1086
|
+
const grouped = {
|
|
1087
|
+
"clean-exit": [],
|
|
1088
|
+
"command-error": [],
|
|
1089
|
+
"signal-kill": []
|
|
1090
|
+
};
|
|
1091
|
+
for (const verdict of verdicts) {
|
|
1092
|
+
if (!verdict.admitted || verdict.stratum === null) continue;
|
|
1093
|
+
grouped[verdict.stratum].push(verdict.rowId);
|
|
1094
|
+
}
|
|
1095
|
+
return Object.freeze({
|
|
1096
|
+
"clean-exit": Object.freeze(grouped["clean-exit"]),
|
|
1097
|
+
"command-error": Object.freeze(grouped["command-error"]),
|
|
1098
|
+
"signal-kill": Object.freeze(grouped["signal-kill"])
|
|
1099
|
+
});
|
|
1100
|
+
}
|
|
1101
|
+
/** Bounded worker pool that keeps results in input order. */
|
|
1102
|
+
async function mapOrdered(items, concurrency, fn) {
|
|
1103
|
+
const results = new Array(items.length);
|
|
1104
|
+
let next = 0;
|
|
1105
|
+
const workerCount = Math.max(1, Math.min(concurrency, items.length));
|
|
1106
|
+
const workers = Array.from({ length: workerCount }, async () => {
|
|
1107
|
+
for (;;) {
|
|
1108
|
+
const index = next;
|
|
1109
|
+
next += 1;
|
|
1110
|
+
if (index >= items.length) return;
|
|
1111
|
+
const item = items[index];
|
|
1112
|
+
if (item === void 0) return;
|
|
1113
|
+
results[index] = await fn(item);
|
|
1114
|
+
}
|
|
1115
|
+
});
|
|
1116
|
+
await Promise.all(workers);
|
|
1117
|
+
return results;
|
|
1118
|
+
}
|
|
1119
|
+
/**
|
|
1120
|
+
* Prove a campaign measured the denominator admission published.
|
|
1121
|
+
*
|
|
1122
|
+
* Three ways a denominator moves after the fact, each rejected here: scoring a
|
|
1123
|
+
* row that was never admitted, scoring a row that was never sampled, and
|
|
1124
|
+
* dropping a sampled row instead of scoring it. The third is the one an analyst
|
|
1125
|
+
* can cause on its own — declining a row it cannot solve — so a sampled row
|
|
1126
|
+
* with no outcome is an error, not a smaller `n`.
|
|
1127
|
+
*/
|
|
1128
|
+
function assertDenominatorIntact(input) {
|
|
1129
|
+
const { report, strata, sampled, scored } = input;
|
|
1130
|
+
for (const stratum of strata) if (!ADMISSION_STRATA.includes(stratum)) throw new ValidationError(`unknown stratum in denominator check: ${stratum}`);
|
|
1131
|
+
assertNoDuplicates(sampled, "sampled");
|
|
1132
|
+
assertNoDuplicates(scored, "scored");
|
|
1133
|
+
const eligible = new Set(strata.flatMap((stratum) => [...report.strata[stratum]]));
|
|
1134
|
+
const notAdmitted = sampled.filter((rowId) => !eligible.has(rowId));
|
|
1135
|
+
if (notAdmitted.length > 0) throw new AdmissionDenominatorError(`${notAdmitted.length} sampled row(s) are not admitted in strata ${strata.join(", ")}: ${preview(notAdmitted)}`);
|
|
1136
|
+
const sampledSet = new Set(sampled);
|
|
1137
|
+
const unsampled = scored.filter((rowId) => !sampledSet.has(rowId));
|
|
1138
|
+
if (unsampled.length > 0) throw new AdmissionDenominatorError(`${unsampled.length} scored row(s) were never sampled: ${preview(unsampled)}`);
|
|
1139
|
+
const scoredSet = new Set(scored);
|
|
1140
|
+
const dropped = sampled.filter((rowId) => !scoredSet.has(rowId));
|
|
1141
|
+
if (dropped.length > 0) throw new AdmissionDenominatorError(`denominator shrank by ${dropped.length} row(s): sampled but never scored: ${preview(dropped)}`);
|
|
1142
|
+
}
|
|
1143
|
+
/** A row counted twice raises `n` without measuring anything twice. */
|
|
1144
|
+
function assertNoDuplicates(rowIds, field) {
|
|
1145
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1146
|
+
const duplicates = rowIds.filter((rowId) => {
|
|
1147
|
+
if (seen.has(rowId)) return true;
|
|
1148
|
+
seen.add(rowId);
|
|
1149
|
+
return false;
|
|
1150
|
+
});
|
|
1151
|
+
if (duplicates.length > 0) throw new AdmissionDenominatorError(`${field} lists ${duplicates.length} duplicate row id(s): ${preview([...new Set(duplicates)])}`);
|
|
1152
|
+
}
|
|
1153
|
+
function preview(rowIds) {
|
|
1154
|
+
const shown = rowIds.slice(0, 5).join(", ");
|
|
1155
|
+
return rowIds.length > 5 ? `${shown}, +${rowIds.length - 5} more` : shown;
|
|
1156
|
+
}
|
|
1157
|
+
//#endregion
|
|
1158
|
+
//#region src/trace-repair/admission-contract.ts
|
|
1159
|
+
/**
|
|
1160
|
+
* Admission: the executed checks a row must pass before an analyst is allowed
|
|
1161
|
+
* to see it, and the brand that proves it did.
|
|
1162
|
+
*
|
|
1163
|
+
* Every check is analyst-independent. None of them reads a finding, a label,
|
|
1164
|
+
* or a k, and all of them are anchored at the recorded end state — the point
|
|
1165
|
+
* the agent actually stopped at — so the same evidence admits a row no matter
|
|
1166
|
+
* which step an analyst later blames:
|
|
1167
|
+
*
|
|
1168
|
+
* oracle-determinism the task's own suite returns one verdict on identical
|
|
1169
|
+
* bytes, so its answer is about the state
|
|
1170
|
+
* prefix-fidelity the recorded trajectory replays with at most 10 % of
|
|
1171
|
+
* its steps diverging from their recorded returncode
|
|
1172
|
+
* end-state-fails the held-out suite fails on the recorded end state
|
|
1173
|
+
* no-fix-control 3 of 3 continuations from the end state fail
|
|
1174
|
+
* no-op-control 3 of 3 continuations from the end state, after an
|
|
1175
|
+
* action that changes nothing, fail
|
|
1176
|
+
*
|
|
1177
|
+
* The two controls are what make `Delta-repair` a difference rather than a
|
|
1178
|
+
* rate, and what stops a row where the agent was one free step from success
|
|
1179
|
+
* from counting as a repair the analyst caused. They only do that under a
|
|
1180
|
+
* control that can act: `control-policy.ts` holds the declaration, and the
|
|
1181
|
+
* criteria name which reading of a control pass applies.
|
|
1182
|
+
*
|
|
1183
|
+
* The determinism check sits in front of all of it. Every downstream check
|
|
1184
|
+
* reads the same suite, so a suite whose verdict is not a function of the state
|
|
1185
|
+
* makes each of them a coin flip rather than a measurement.
|
|
1186
|
+
*
|
|
1187
|
+
* This module owns the CONTRACT, not the execution. `runAdmission` in
|
|
1188
|
+
* `./admission` executes the checks against real containers and hands the
|
|
1189
|
+
* measured evidence to `admitRow`, which decides and brands. Splitting it that
|
|
1190
|
+
* way keeps the decision auditable from the recorded numbers alone: a reviewer
|
|
1191
|
+
* can re-derive every admission from the evidence file without re-running a
|
|
1192
|
+
* container.
|
|
1193
|
+
*/
|
|
1194
|
+
const TB_REPAIR_ADMISSION_CRITERIA = Object.freeze({
|
|
1195
|
+
maxPrefixDivergenceRatio: .1,
|
|
1196
|
+
controlRollouts: 3,
|
|
1197
|
+
controlScreening: "enforced"
|
|
1198
|
+
});
|
|
1199
|
+
/**
|
|
1200
|
+
* Decide admission from executed evidence.
|
|
1201
|
+
*
|
|
1202
|
+
* Pure: it opens no container and calls no model. Every threshold it applies
|
|
1203
|
+
* is in `criteria`, and every number it reads is in `evidence`, so an
|
|
1204
|
+
* admission decision is reproducible from the recorded evidence alone.
|
|
1205
|
+
*
|
|
1206
|
+
* It throws, rather than rejecting, when the criteria and the declared control
|
|
1207
|
+
* contradict. A contradiction there is a property of the configuration and not
|
|
1208
|
+
* of the row, so it must stop the run instead of producing one verdict per row
|
|
1209
|
+
* that reads as if a check had been applied.
|
|
1210
|
+
*/
|
|
1211
|
+
function admitRow(evidence, criteria = TB_REPAIR_ADMISSION_CRITERIA) {
|
|
1212
|
+
assertCriteria(criteria);
|
|
1213
|
+
assertControlCalibrated(evidence.controlPolicy, criteria.controlScreening);
|
|
1214
|
+
if (evidence.oracleDeterminism.taskName !== evidence.taskName) throw new ValidationError(`row ${evidence.rowId} is from task ${evidence.taskName} but carries the oracle certification for ${evidence.oracleDeterminism.taskName}`);
|
|
1215
|
+
const screening = {
|
|
1216
|
+
controlPolicy: evidence.controlPolicy,
|
|
1217
|
+
controlScreening: criteria.controlScreening,
|
|
1218
|
+
controlPolicyDigest: evidence.controlPolicy.digest,
|
|
1219
|
+
taskName: evidence.oracleDeterminism.taskName,
|
|
1220
|
+
oracleStable: evidence.oracleDeterminism.stable,
|
|
1221
|
+
oracleFlipRate: evidence.oracleDeterminism.flipRate
|
|
1222
|
+
};
|
|
1223
|
+
const reject = (rejection, detail) => ({
|
|
1224
|
+
admitted: false,
|
|
1225
|
+
rowId: evidence.rowId,
|
|
1226
|
+
screening,
|
|
1227
|
+
rejection,
|
|
1228
|
+
detail
|
|
1229
|
+
});
|
|
1230
|
+
const oracle = evidence.oracleDeterminism;
|
|
1231
|
+
if (!oracle.stable) return reject("task-oracle-nondeterministic", `task ${oracle.taskName} graded byte-identical state inconsistently: flip rate ${(oracle.flipRate * 100).toFixed(1)} % over ${oracle.replicates} replicates (${oracle.detail}). Every check below reads that suite, so none of them measures state.`);
|
|
1232
|
+
if (evidence.steps.length === 0) return reject("empty-trajectory", "the row records no steps");
|
|
1233
|
+
const { stepsReplayed, divergences } = evidence.prefixFidelity;
|
|
1234
|
+
if (stepsReplayed <= 0) return reject("empty-trajectory", "no recorded step was replayed");
|
|
1235
|
+
const prefixDivergenceRatio = divergences / stepsReplayed;
|
|
1236
|
+
if (prefixDivergenceRatio > criteria.maxPrefixDivergenceRatio) return reject("prefix-divergence-too-high", `${divergences}/${stepsReplayed} replayed steps diverged (${(prefixDivergenceRatio * 100).toFixed(1)} %), above the ${(criteria.maxPrefixDivergenceRatio * 100).toFixed(1)} % ceiling`);
|
|
1237
|
+
if (evidence.endStatePassed) return reject("end-state-already-passes", "the held-out suite passes on the recorded end state, so the row records no failure to repair");
|
|
1238
|
+
for (const [name, arm, rescued] of [[
|
|
1239
|
+
"no-fix control",
|
|
1240
|
+
evidence.noFixControl,
|
|
1241
|
+
"no-fix-control-passed"
|
|
1242
|
+
], [
|
|
1243
|
+
"no-op control",
|
|
1244
|
+
evidence.noOpControl,
|
|
1245
|
+
"no-op-control-passed"
|
|
1246
|
+
]]) {
|
|
1247
|
+
if (arm.rollouts !== criteria.controlRollouts) return reject("control-rollouts-short", `${name} ran ${arm.rollouts} rollouts, the criteria pre-register ${criteria.controlRollouts}`);
|
|
1248
|
+
if (arm.policyDigest !== evidence.controlPolicy.digest) return reject("control-policy-mismatch", `${name} reported policy ${arm.policyDigest} but the criteria screen under ${evidence.controlPolicy.id} (${evidence.controlPolicy.digest}); the arm did not run the declared control`);
|
|
1249
|
+
if (arm.passes !== 0) return criteria.controlScreening === "enforced" ? reject(rescued, `${name} passed ${arm.passes}/${arm.rollouts}; the row is repairable by continuing alone`) : reject("control-passed-on-identical-state", `${name} passed ${arm.passes}/${arm.rollouts} under ${evidence.controlPolicy.id}, which executes no command, so it graded the same bytes the end-state check graded as failing. The task's certification reports flip rate ${(oracle.flipRate * 100).toFixed(1)} %; this row is a further flip, not a rescue.`);
|
|
1250
|
+
}
|
|
1251
|
+
return {
|
|
1252
|
+
admitted: true,
|
|
1253
|
+
screening,
|
|
1254
|
+
row: {
|
|
1255
|
+
rowId: evidence.rowId,
|
|
1256
|
+
taskName: evidence.taskName,
|
|
1257
|
+
image: evidence.image,
|
|
1258
|
+
cwd: evidence.cwd,
|
|
1259
|
+
taskStatement: evidence.taskStatement,
|
|
1260
|
+
steps: evidence.steps,
|
|
1261
|
+
criteria,
|
|
1262
|
+
prefixDivergenceRatio,
|
|
1263
|
+
suiteDigest: evidence.suiteDigest,
|
|
1264
|
+
controlPolicy: evidence.controlPolicy,
|
|
1265
|
+
controlScreening: criteria.controlScreening,
|
|
1266
|
+
policyDigest: evidence.controlPolicy.digest,
|
|
1267
|
+
controlRate: evidence.noFixControl.passes / evidence.noFixControl.rollouts,
|
|
1268
|
+
controlRollouts: evidence.noFixControl.rollouts
|
|
1269
|
+
}
|
|
1270
|
+
};
|
|
1271
|
+
}
|
|
1272
|
+
function assertCriteria(criteria) {
|
|
1273
|
+
const { maxPrefixDivergenceRatio, controlRollouts } = criteria;
|
|
1274
|
+
if (!(maxPrefixDivergenceRatio >= 0 && maxPrefixDivergenceRatio <= 1)) throw new ValidationError(`admission maxPrefixDivergenceRatio must be within [0,1], got ${maxPrefixDivergenceRatio}`);
|
|
1275
|
+
if (!Number.isInteger(controlRollouts) || controlRollouts <= 0) throw new ValidationError(`admission controlRollouts must be a positive integer, got ${controlRollouts}`);
|
|
1276
|
+
}
|
|
1277
|
+
//#endregion
|
|
1278
|
+
//#region src/trace-repair/admission-report.ts
|
|
1279
|
+
/** Plain JSON. `JSON.stringify` of this object is the machine-readable artifact. */
|
|
1280
|
+
function admissionArtifact(report) {
|
|
1281
|
+
assertChainReconciles(report.chain);
|
|
1282
|
+
return {
|
|
1283
|
+
version: 1,
|
|
1284
|
+
kind: "tb-repair-admission",
|
|
1285
|
+
generatedAt: report.generatedAt,
|
|
1286
|
+
digest: report.digest,
|
|
1287
|
+
config: report.config,
|
|
1288
|
+
provenance: report.provenance,
|
|
1289
|
+
chain: report.chain,
|
|
1290
|
+
admitted: {
|
|
1291
|
+
"clean-exit": report.strata["clean-exit"],
|
|
1292
|
+
"command-error": report.strata["command-error"],
|
|
1293
|
+
"signal-kill": report.strata["signal-kill"]
|
|
1294
|
+
},
|
|
1295
|
+
controlCost: {
|
|
1296
|
+
kind: report.controlCost.kind,
|
|
1297
|
+
usd: report.controlCost.usd
|
|
1298
|
+
},
|
|
1299
|
+
rows: report.rows
|
|
1300
|
+
};
|
|
1301
|
+
}
|
|
1302
|
+
/** Markdown for the campaign report. Every number here is also in the artifact. */
|
|
1303
|
+
function renderAdmissionReport(artifact, options = {}) {
|
|
1304
|
+
const lines = [];
|
|
1305
|
+
lines.push("# TB-Repair admission", "");
|
|
1306
|
+
lines.push(`${artifact.chain.overall.admitted} of ${artifact.chain.overall.input} rows admitted. Digest \`${artifact.digest}\`.`, "");
|
|
1307
|
+
lines.push(...provenanceSection(artifact));
|
|
1308
|
+
lines.push(...chainSection(artifact.chain.overall, "Denominator chain"));
|
|
1309
|
+
for (const chain of artifact.chain.byStratum) lines.push(...chainSection(chain, `Denominator chain — ${chain.scope}`));
|
|
1310
|
+
lines.push(...stratumSection(artifact));
|
|
1311
|
+
lines.push(...reasonSection(artifact.chain.reasonTotals));
|
|
1312
|
+
const rowLimit = options.rowLimit ?? 0;
|
|
1313
|
+
if (rowLimit > 0) lines.push(...rowSection(artifact.rows, rowLimit));
|
|
1314
|
+
return `${lines.join("\n").trimEnd()}\n`;
|
|
1466
1315
|
}
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
const
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
}
|
|
1316
|
+
function provenanceSection(artifact) {
|
|
1317
|
+
const { provenance, config, controlCost } = artifact;
|
|
1318
|
+
const cost = controlCost.usd === null ? "uncaptured" : `$${controlCost.usd.toFixed(4)}`;
|
|
1319
|
+
return [
|
|
1320
|
+
"## Provenance",
|
|
1321
|
+
"",
|
|
1322
|
+
"| field | value |",
|
|
1323
|
+
"| --- | --- |",
|
|
1324
|
+
`| generated | ${artifact.generatedAt} |`,
|
|
1325
|
+
`| prefix replayer | \`${provenance.replayerId}\` |`,
|
|
1326
|
+
`| end-state oracle | \`${provenance.oracleId}\` |`,
|
|
1327
|
+
`| control runner | \`${provenance.controlRunnerId}\` |`,
|
|
1328
|
+
`| continuation policy | \`${provenance.policyId}\` |`,
|
|
1329
|
+
`| policy model | \`${provenance.policyModel}\` |`,
|
|
1330
|
+
`| policy seed | ${provenance.policySeed} |`,
|
|
1331
|
+
`| policy digest | \`${provenance.policyDigest}\` |`,
|
|
1332
|
+
`| policy step budget | ${provenance.policyStepBudget} model call(s) per control rollout |`,
|
|
1333
|
+
`| control screening | \`${provenance.controlScreening}\` |`,
|
|
1334
|
+
`| certified task oracles | ${formatCertifiedTasks(provenance.certifiedTasks)} |`,
|
|
1335
|
+
`| max prefix divergence | ${formatShare(config.maxPrefixDivergence)} |`,
|
|
1336
|
+
`| control rollouts per arm | ${config.controlRollouts} |`,
|
|
1337
|
+
`| inert action | \`${config.inertAction}\` |`,
|
|
1338
|
+
`| strata admitted | ${config.admitStrata.join(", ")} |`,
|
|
1339
|
+
`| control model cost | ${cost} (${controlCost.kind}) |`,
|
|
1340
|
+
""
|
|
1341
|
+
];
|
|
1342
|
+
}
|
|
1343
|
+
function formatCertifiedTasks(certified) {
|
|
1344
|
+
const entries = Object.entries(certified).sort(([a], [b]) => a.localeCompare(b));
|
|
1345
|
+
if (entries.length === 0) return "none";
|
|
1346
|
+
return entries.map(([task, flipRate]) => `${task} (flip ${formatShare(flipRate)})`).join(", ");
|
|
1347
|
+
}
|
|
1348
|
+
function chainSection(chain, heading) {
|
|
1349
|
+
const lines = [
|
|
1350
|
+
`## ${heading}`,
|
|
1351
|
+
"",
|
|
1352
|
+
"| stage | exclusion reason | entering | excluded | remaining |",
|
|
1353
|
+
"| --- | --- | --- | --- | --- |"
|
|
1354
|
+
];
|
|
1355
|
+
chain.stages.forEach((stage, index) => {
|
|
1356
|
+
lines.push(`| ${index + 1} | \`${stage.reason}\` | ${stage.entering} | ${stage.excluded} | ${stage.remaining} |`);
|
|
1481
1357
|
});
|
|
1482
|
-
|
|
1483
|
-
|
|
1358
|
+
const excluded = chain.input - chain.admitted;
|
|
1359
|
+
lines.push("", `Input ${chain.input} = admitted ${chain.admitted} + excluded ${excluded}. Admission rate ${formatShare(rate(chain.admitted, chain.input))}.`, "");
|
|
1360
|
+
return lines;
|
|
1484
1361
|
}
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1362
|
+
function stratumSection(artifact) {
|
|
1363
|
+
const lines = [
|
|
1364
|
+
"## Strata",
|
|
1365
|
+
"",
|
|
1366
|
+
"| stratum | admitted rows | in this campaign |",
|
|
1367
|
+
"| --- | --- | --- |"
|
|
1368
|
+
];
|
|
1369
|
+
for (const stratum of ADMISSION_STRATA) {
|
|
1370
|
+
const admitted = artifact.admitted[stratum].length;
|
|
1371
|
+
const eligible = artifact.chain.admitStrata.includes(stratum) ? "yes" : "no";
|
|
1372
|
+
lines.push(`| ${stratum} | ${admitted} | ${eligible} |`);
|
|
1373
|
+
}
|
|
1374
|
+
lines.push("", "Sample within a stratum. A command-level repair cannot address a signal kill, so pooling that population with the others averages an addressable class with an unaddressable one.", "");
|
|
1375
|
+
return lines;
|
|
1488
1376
|
}
|
|
1489
|
-
function
|
|
1490
|
-
|
|
1377
|
+
function reasonSection(totals) {
|
|
1378
|
+
const lines = [
|
|
1379
|
+
"## Exclusions",
|
|
1380
|
+
"",
|
|
1381
|
+
"| reason | rows | what it means |",
|
|
1382
|
+
"| --- | --- | --- |"
|
|
1383
|
+
];
|
|
1384
|
+
for (const reason of ADMISSION_EXCLUSION_ORDER) lines.push(`| \`${reason}\` | ${totals[reason]} | ${ADMISSION_EXCLUSION_MEANING[reason]} |`);
|
|
1385
|
+
lines.push("");
|
|
1386
|
+
return lines;
|
|
1491
1387
|
}
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
for (const stratum of strata) if (!ADMISSION_STRATA.includes(stratum)) throw new ValidationError(`unknown stratum in denominator check: ${stratum}`);
|
|
1504
|
-
assertNoDuplicates(sampled, "sampled");
|
|
1505
|
-
assertNoDuplicates(scored, "scored");
|
|
1506
|
-
const eligible = new Set(strata.flatMap((stratum) => [...report.strata[stratum]]));
|
|
1507
|
-
const notAdmitted = sampled.filter((rowId) => !eligible.has(rowId));
|
|
1508
|
-
if (notAdmitted.length > 0) throw new AdmissionDenominatorError(`${notAdmitted.length} sampled row(s) are not admitted in strata ${strata.join(", ")}: ${preview(notAdmitted)}`);
|
|
1509
|
-
const sampledSet = new Set(sampled);
|
|
1510
|
-
const unsampled = scored.filter((rowId) => !sampledSet.has(rowId));
|
|
1511
|
-
if (unsampled.length > 0) throw new AdmissionDenominatorError(`${unsampled.length} scored row(s) were never sampled: ${preview(unsampled)}`);
|
|
1512
|
-
const scoredSet = new Set(scored);
|
|
1513
|
-
const dropped = sampled.filter((rowId) => !scoredSet.has(rowId));
|
|
1514
|
-
if (dropped.length > 0) throw new AdmissionDenominatorError(`denominator shrank by ${dropped.length} row(s): sampled but never scored: ${preview(dropped)}`);
|
|
1388
|
+
function rowSection(rows, limit) {
|
|
1389
|
+
const lines = [
|
|
1390
|
+
"## Rows",
|
|
1391
|
+
"",
|
|
1392
|
+
"| row | task | stratum | final rc | admitted | excluded by |",
|
|
1393
|
+
"| --- | --- | --- | --- | --- | --- |"
|
|
1394
|
+
];
|
|
1395
|
+
for (const row of rows.slice(0, limit)) lines.push(`| \`${row.rowId}\` | ${row.taskName} | ${row.stratum ?? "—"} | ${row.finalReturncode ?? "—"} | ${row.admitted ? "yes" : "no"} | ${row.excludedBy === null ? "—" : `\`${row.excludedBy}\``} |`);
|
|
1396
|
+
if (rows.length > limit) lines.push("", `${rows.length - limit} further rows in the artifact.`);
|
|
1397
|
+
lines.push("");
|
|
1398
|
+
return lines;
|
|
1515
1399
|
}
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
const seen = /* @__PURE__ */ new Set();
|
|
1519
|
-
const duplicates = rowIds.filter((rowId) => {
|
|
1520
|
-
if (seen.has(rowId)) return true;
|
|
1521
|
-
seen.add(rowId);
|
|
1522
|
-
return false;
|
|
1523
|
-
});
|
|
1524
|
-
if (duplicates.length > 0) throw new AdmissionDenominatorError(`${field} lists ${duplicates.length} duplicate row id(s): ${preview([...new Set(duplicates)])}`);
|
|
1400
|
+
function rate(part, total) {
|
|
1401
|
+
return total === 0 ? 0 : part / total;
|
|
1525
1402
|
}
|
|
1526
|
-
function
|
|
1527
|
-
|
|
1528
|
-
return rowIds.length > 5 ? `${shown}, +${rowIds.length - 5} more` : shown;
|
|
1403
|
+
function formatShare(share) {
|
|
1404
|
+
return `${(share * 100).toFixed(2)}%`;
|
|
1529
1405
|
}
|
|
1530
1406
|
//#endregion
|
|
1531
|
-
//#region src/trace-repair/
|
|
1407
|
+
//#region src/trace-repair/action-budget.ts
|
|
1532
1408
|
/**
|
|
1533
|
-
*
|
|
1534
|
-
* to see it, and the brand that proves it did.
|
|
1535
|
-
*
|
|
1536
|
-
* Every check is analyst-independent. None of them reads a finding, a label,
|
|
1537
|
-
* or a k, and all of them are anchored at the recorded end state — the point
|
|
1538
|
-
* the agent actually stopped at — so the same evidence admits a row no matter
|
|
1539
|
-
* which step an analyst later blames:
|
|
1540
|
-
*
|
|
1541
|
-
* oracle-determinism the task's own suite returns one verdict on identical
|
|
1542
|
-
* bytes, so its answer is about the state
|
|
1543
|
-
* prefix-fidelity the recorded trajectory replays with at most 10 % of
|
|
1544
|
-
* its steps diverging from their recorded returncode
|
|
1545
|
-
* end-state-fails the held-out suite fails on the recorded end state
|
|
1546
|
-
* no-fix-control 3 of 3 continuations from the end state fail
|
|
1547
|
-
* no-op-control 3 of 3 continuations from the end state, after an
|
|
1548
|
-
* action that changes nothing, fail
|
|
1549
|
-
*
|
|
1550
|
-
* The two controls are what make `Delta-repair` a difference rather than a
|
|
1551
|
-
* rate, and what stops a row where the agent was one free step from success
|
|
1552
|
-
* from counting as a repair the analyst caused. They only do that under a
|
|
1553
|
-
* control that can act: `control-policy.ts` holds the declaration, and the
|
|
1554
|
-
* criteria name which reading of a control pass applies.
|
|
1409
|
+
* The action budget an intervention must fit inside.
|
|
1555
1410
|
*
|
|
1556
|
-
* The
|
|
1557
|
-
*
|
|
1558
|
-
*
|
|
1411
|
+
* The analyst answers with one action, applied at step k, drawn from the same
|
|
1412
|
+
* action space the scaffold had: one shell command or one edit, at most 4 KB.
|
|
1413
|
+
* An answer that buys a bigger action than the scaffold could take is not a
|
|
1414
|
+
* counterfactual about the recorded run, so the budget is enforced before any
|
|
1415
|
+
* container is opened and a violation never reaches the reproduction gate.
|
|
1559
1416
|
*
|
|
1560
|
-
*
|
|
1561
|
-
*
|
|
1562
|
-
*
|
|
1563
|
-
*
|
|
1564
|
-
*
|
|
1565
|
-
*
|
|
1417
|
+
* "One action" is decided by counting top-level statements, not lines. A
|
|
1418
|
+
* command list joined by `&&`, `||` or a pipe is one statement, because that
|
|
1419
|
+
* is one thing the shell runs and one thing the scaffold could have typed.
|
|
1420
|
+
* Two statements separated by a newline or `;` are two actions and are
|
|
1421
|
+
* rejected. Heredoc bodies, comments and compound blocks (`if`, `for`,
|
|
1422
|
+
* `while`, `until`, `case`, `{ … }`) are inside a statement, never separators.
|
|
1566
1423
|
*/
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1424
|
+
/** The scaffold's own per-action budget, pre-registered for the campaign. */
|
|
1425
|
+
const SCAFFOLD_INTERVENTION_BUDGET = Object.freeze({
|
|
1426
|
+
maxBytes: 4096,
|
|
1427
|
+
maxStatements: 1,
|
|
1428
|
+
maxHeredocs: 1
|
|
1571
1429
|
});
|
|
1572
1430
|
/**
|
|
1573
|
-
*
|
|
1574
|
-
*
|
|
1575
|
-
*
|
|
1576
|
-
*
|
|
1577
|
-
|
|
1431
|
+
* Actions whose only effect is to consume a turn. They are rejected before a
|
|
1432
|
+
* container opens: an intervention that changes nothing is measurably
|
|
1433
|
+
* identical to the no-op control, and paying rollouts to rediscover that
|
|
1434
|
+
* wastes the corpus.
|
|
1435
|
+
*/
|
|
1436
|
+
const NO_OP_ACTIONS = Object.freeze([
|
|
1437
|
+
":",
|
|
1438
|
+
"true",
|
|
1439
|
+
"/bin/true",
|
|
1440
|
+
"exit",
|
|
1441
|
+
"exit 0"
|
|
1442
|
+
]);
|
|
1443
|
+
const BLOCK_OPENERS = /* @__PURE__ */ new Set([
|
|
1444
|
+
"if",
|
|
1445
|
+
"for",
|
|
1446
|
+
"while",
|
|
1447
|
+
"until",
|
|
1448
|
+
"case",
|
|
1449
|
+
"select",
|
|
1450
|
+
"do",
|
|
1451
|
+
"then"
|
|
1452
|
+
]);
|
|
1453
|
+
const BLOCK_CLOSERS = /* @__PURE__ */ new Set([
|
|
1454
|
+
"fi",
|
|
1455
|
+
"done",
|
|
1456
|
+
"esac"
|
|
1457
|
+
]);
|
|
1458
|
+
/**
|
|
1459
|
+
* Split a shell script into top-level statements and count its heredocs.
|
|
1578
1460
|
*
|
|
1579
|
-
*
|
|
1580
|
-
*
|
|
1581
|
-
*
|
|
1582
|
-
*
|
|
1461
|
+
* The scan tracks quoting, escapes, command substitution, brace and paren
|
|
1462
|
+
* grouping, comments, compound-block keywords, and heredoc bodies. Anything
|
|
1463
|
+
* it cannot resolve stays inside the current statement, so an unparseable
|
|
1464
|
+
* action reads as one oversized statement and is rejected on bytes rather
|
|
1465
|
+
* than silently accepted as one clean action.
|
|
1583
1466
|
*/
|
|
1584
|
-
function
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
controlPolicy: evidence.controlPolicy,
|
|
1590
|
-
controlScreening: criteria.controlScreening,
|
|
1591
|
-
controlPolicyDigest: evidence.controlPolicy.digest,
|
|
1592
|
-
taskName: evidence.oracleDeterminism.taskName,
|
|
1593
|
-
oracleStable: evidence.oracleDeterminism.stable,
|
|
1594
|
-
oracleFlipRate: evidence.oracleDeterminism.flipRate
|
|
1467
|
+
function scanShellAction(script) {
|
|
1468
|
+
const state = {
|
|
1469
|
+
statements: [],
|
|
1470
|
+
current: "",
|
|
1471
|
+
heredocs: 0
|
|
1595
1472
|
};
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
const
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
if (
|
|
1622
|
-
|
|
1473
|
+
let index = 0;
|
|
1474
|
+
let parenDepth = 0;
|
|
1475
|
+
let braceDepth = 0;
|
|
1476
|
+
let blockDepth = 0;
|
|
1477
|
+
let inSingle = false;
|
|
1478
|
+
let inDouble = false;
|
|
1479
|
+
let inBacktick = false;
|
|
1480
|
+
let pendingHeredocs = [];
|
|
1481
|
+
let word = "";
|
|
1482
|
+
let atStatementStart = true;
|
|
1483
|
+
const flushWord = () => {
|
|
1484
|
+
if (word.length === 0) return;
|
|
1485
|
+
if (BLOCK_OPENERS.has(word)) blockDepth += 1;
|
|
1486
|
+
else if (BLOCK_CLOSERS.has(word)) blockDepth = Math.max(0, blockDepth - 1);
|
|
1487
|
+
word = "";
|
|
1488
|
+
};
|
|
1489
|
+
const endStatement = () => {
|
|
1490
|
+
flushWord();
|
|
1491
|
+
const text = state.current.trim();
|
|
1492
|
+
if (text.length > 0 && !isCommentOnly(text)) state.statements.push(text);
|
|
1493
|
+
state.current = "";
|
|
1494
|
+
atStatementStart = true;
|
|
1495
|
+
};
|
|
1496
|
+
while (index < script.length) {
|
|
1497
|
+
const char = script[index];
|
|
1498
|
+
if (inSingle) {
|
|
1499
|
+
state.current += char;
|
|
1500
|
+
if (char === "'") inSingle = false;
|
|
1501
|
+
index += 1;
|
|
1502
|
+
continue;
|
|
1503
|
+
}
|
|
1504
|
+
if (char === "\\" && index + 1 < script.length) {
|
|
1505
|
+
const next = script[index + 1];
|
|
1506
|
+
if (next === "\n" && !inDouble) {
|
|
1507
|
+
state.current += char + next;
|
|
1508
|
+
index += 2;
|
|
1509
|
+
continue;
|
|
1510
|
+
}
|
|
1511
|
+
state.current += char + next;
|
|
1512
|
+
index += 2;
|
|
1513
|
+
continue;
|
|
1514
|
+
}
|
|
1515
|
+
if (inDouble) {
|
|
1516
|
+
state.current += char;
|
|
1517
|
+
if (char === "\"") inDouble = false;
|
|
1518
|
+
index += 1;
|
|
1519
|
+
continue;
|
|
1520
|
+
}
|
|
1521
|
+
if (char === "'") {
|
|
1522
|
+
flushWord();
|
|
1523
|
+
inSingle = true;
|
|
1524
|
+
state.current += char;
|
|
1525
|
+
index += 1;
|
|
1526
|
+
atStatementStart = false;
|
|
1527
|
+
continue;
|
|
1528
|
+
}
|
|
1529
|
+
if (char === "\"") {
|
|
1530
|
+
flushWord();
|
|
1531
|
+
inDouble = true;
|
|
1532
|
+
state.current += char;
|
|
1533
|
+
index += 1;
|
|
1534
|
+
atStatementStart = false;
|
|
1535
|
+
continue;
|
|
1536
|
+
}
|
|
1537
|
+
if (char === "`") {
|
|
1538
|
+
inBacktick = !inBacktick;
|
|
1539
|
+
state.current += char;
|
|
1540
|
+
index += 1;
|
|
1541
|
+
atStatementStart = false;
|
|
1542
|
+
continue;
|
|
1543
|
+
}
|
|
1544
|
+
if (char === "#" && (atStatementStart || /\s/.test(script[index - 1] ?? " "))) {
|
|
1545
|
+
const end = script.indexOf("\n", index);
|
|
1546
|
+
const stop = end === -1 ? script.length : end;
|
|
1547
|
+
state.current += script.slice(index, stop);
|
|
1548
|
+
index = stop;
|
|
1549
|
+
continue;
|
|
1550
|
+
}
|
|
1551
|
+
if (char === "<" && script[index + 1] === "<" && script[index + 2] !== "<" && script[index - 1] !== "<") {
|
|
1552
|
+
const mark = readHeredocMark(script, index);
|
|
1553
|
+
if (mark) {
|
|
1554
|
+
pendingHeredocs.push(mark.mark);
|
|
1555
|
+
state.heredocs += 1;
|
|
1556
|
+
state.current += script.slice(index, mark.nextIndex);
|
|
1557
|
+
index = mark.nextIndex;
|
|
1558
|
+
atStatementStart = false;
|
|
1559
|
+
continue;
|
|
1560
|
+
}
|
|
1561
|
+
}
|
|
1562
|
+
if (char === "(") {
|
|
1563
|
+
flushWord();
|
|
1564
|
+
parenDepth += 1;
|
|
1565
|
+
state.current += char;
|
|
1566
|
+
index += 1;
|
|
1567
|
+
atStatementStart = true;
|
|
1568
|
+
continue;
|
|
1569
|
+
}
|
|
1570
|
+
if (char === ")") {
|
|
1571
|
+
flushWord();
|
|
1572
|
+
parenDepth = Math.max(0, parenDepth - 1);
|
|
1573
|
+
state.current += char;
|
|
1574
|
+
index += 1;
|
|
1575
|
+
atStatementStart = false;
|
|
1576
|
+
continue;
|
|
1577
|
+
}
|
|
1578
|
+
if (char === "{") {
|
|
1579
|
+
flushWord();
|
|
1580
|
+
braceDepth += 1;
|
|
1581
|
+
state.current += char;
|
|
1582
|
+
index += 1;
|
|
1583
|
+
atStatementStart = true;
|
|
1584
|
+
continue;
|
|
1585
|
+
}
|
|
1586
|
+
if (char === "}") {
|
|
1587
|
+
flushWord();
|
|
1588
|
+
braceDepth = Math.max(0, braceDepth - 1);
|
|
1589
|
+
state.current += char;
|
|
1590
|
+
index += 1;
|
|
1591
|
+
atStatementStart = false;
|
|
1592
|
+
continue;
|
|
1593
|
+
}
|
|
1594
|
+
if (char === "\n") {
|
|
1595
|
+
flushWord();
|
|
1596
|
+
if (pendingHeredocs.length > 0) {
|
|
1597
|
+
const consumed = consumeHeredocBodies(script, index + 1, pendingHeredocs);
|
|
1598
|
+
state.current += script.slice(index, consumed);
|
|
1599
|
+
pendingHeredocs = [];
|
|
1600
|
+
index = consumed;
|
|
1601
|
+
continue;
|
|
1602
|
+
}
|
|
1603
|
+
if (parenDepth > 0 || braceDepth > 0 || blockDepth > 0 || inBacktick || endsWithContinuation(state.current)) {
|
|
1604
|
+
state.current += char;
|
|
1605
|
+
index += 1;
|
|
1606
|
+
atStatementStart = true;
|
|
1607
|
+
continue;
|
|
1608
|
+
}
|
|
1609
|
+
endStatement();
|
|
1610
|
+
index += 1;
|
|
1611
|
+
continue;
|
|
1612
|
+
}
|
|
1613
|
+
if (char === ";") {
|
|
1614
|
+
flushWord();
|
|
1615
|
+
if (parenDepth > 0 || braceDepth > 0 || blockDepth > 0 || inBacktick) {
|
|
1616
|
+
state.current += char;
|
|
1617
|
+
index += 1;
|
|
1618
|
+
atStatementStart = true;
|
|
1619
|
+
continue;
|
|
1620
|
+
}
|
|
1621
|
+
endStatement();
|
|
1622
|
+
index += 1;
|
|
1623
|
+
continue;
|
|
1624
|
+
}
|
|
1625
|
+
if (/\s/.test(char)) {
|
|
1626
|
+
flushWord();
|
|
1627
|
+
state.current += char;
|
|
1628
|
+
index += 1;
|
|
1629
|
+
continue;
|
|
1630
|
+
}
|
|
1631
|
+
if (char === "&" || char === "|") {
|
|
1632
|
+
flushWord();
|
|
1633
|
+
state.current += char;
|
|
1634
|
+
index += 1;
|
|
1635
|
+
atStatementStart = true;
|
|
1636
|
+
continue;
|
|
1637
|
+
}
|
|
1638
|
+
word += char;
|
|
1639
|
+
state.current += char;
|
|
1640
|
+
atStatementStart = false;
|
|
1641
|
+
index += 1;
|
|
1623
1642
|
}
|
|
1643
|
+
endStatement();
|
|
1624
1644
|
return {
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
row: {
|
|
1628
|
-
rowId: evidence.rowId,
|
|
1629
|
-
taskName: evidence.taskName,
|
|
1630
|
-
image: evidence.image,
|
|
1631
|
-
cwd: evidence.cwd,
|
|
1632
|
-
taskStatement: evidence.taskStatement,
|
|
1633
|
-
steps: evidence.steps,
|
|
1634
|
-
criteria,
|
|
1635
|
-
prefixDivergenceRatio,
|
|
1636
|
-
suiteDigest: evidence.suiteDigest,
|
|
1637
|
-
controlPolicy: evidence.controlPolicy,
|
|
1638
|
-
controlScreening: criteria.controlScreening,
|
|
1639
|
-
policyDigest: evidence.controlPolicy.digest,
|
|
1640
|
-
controlRate: evidence.noFixControl.passes / evidence.noFixControl.rollouts,
|
|
1641
|
-
controlRollouts: evidence.noFixControl.rollouts
|
|
1642
|
-
}
|
|
1645
|
+
statements: state.statements,
|
|
1646
|
+
heredocs: state.heredocs
|
|
1643
1647
|
};
|
|
1644
1648
|
}
|
|
1645
|
-
function
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
+
function isCommentOnly(text) {
|
|
1650
|
+
return text.split("\n").every((line) => line.trim().length === 0 || line.trim().startsWith("#"));
|
|
1651
|
+
}
|
|
1652
|
+
function endsWithContinuation(current) {
|
|
1653
|
+
const trimmed = current.trimEnd();
|
|
1654
|
+
return /(&&|\|\||\||&|\\)$/.test(trimmed);
|
|
1649
1655
|
}
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1656
|
+
function readHeredocMark(script, index) {
|
|
1657
|
+
let cursor = index + 2;
|
|
1658
|
+
let stripTabs = false;
|
|
1659
|
+
if (script[cursor] === "-") {
|
|
1660
|
+
stripTabs = true;
|
|
1661
|
+
cursor += 1;
|
|
1662
|
+
}
|
|
1663
|
+
while (script[cursor] === " " || script[cursor] === " ") cursor += 1;
|
|
1664
|
+
const quote = script[cursor];
|
|
1665
|
+
if (quote === "'" || quote === "\"") {
|
|
1666
|
+
const close = script.indexOf(quote, cursor + 1);
|
|
1667
|
+
if (close === -1) return null;
|
|
1668
|
+
return {
|
|
1669
|
+
mark: {
|
|
1670
|
+
delimiter: script.slice(cursor + 1, close),
|
|
1671
|
+
stripTabs,
|
|
1672
|
+
quoted: true
|
|
1673
|
+
},
|
|
1674
|
+
nextIndex: close + 1
|
|
1675
|
+
};
|
|
1676
|
+
}
|
|
1677
|
+
const match = /^[A-Za-z_][A-Za-z0-9_]*/.exec(script.slice(cursor));
|
|
1678
|
+
if (!match) return null;
|
|
1655
1679
|
return {
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
config: report.config,
|
|
1661
|
-
provenance: report.provenance,
|
|
1662
|
-
chain: report.chain,
|
|
1663
|
-
admitted: {
|
|
1664
|
-
"clean-exit": report.strata["clean-exit"],
|
|
1665
|
-
"command-error": report.strata["command-error"],
|
|
1666
|
-
"signal-kill": report.strata["signal-kill"]
|
|
1667
|
-
},
|
|
1668
|
-
controlCost: {
|
|
1669
|
-
kind: report.controlCost.kind,
|
|
1670
|
-
usd: report.controlCost.usd
|
|
1680
|
+
mark: {
|
|
1681
|
+
delimiter: match[0],
|
|
1682
|
+
stripTabs,
|
|
1683
|
+
quoted: false
|
|
1671
1684
|
},
|
|
1672
|
-
|
|
1685
|
+
nextIndex: cursor + match[0].length
|
|
1673
1686
|
};
|
|
1674
1687
|
}
|
|
1675
|
-
/**
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
"| field | value |",
|
|
1696
|
-
"| --- | --- |",
|
|
1697
|
-
`| generated | ${artifact.generatedAt} |`,
|
|
1698
|
-
`| prefix replayer | \`${provenance.replayerId}\` |`,
|
|
1699
|
-
`| end-state oracle | \`${provenance.oracleId}\` |`,
|
|
1700
|
-
`| control runner | \`${provenance.controlRunnerId}\` |`,
|
|
1701
|
-
`| continuation policy | \`${provenance.policyId}\` |`,
|
|
1702
|
-
`| policy model | \`${provenance.policyModel}\` |`,
|
|
1703
|
-
`| policy seed | ${provenance.policySeed} |`,
|
|
1704
|
-
`| policy digest | \`${provenance.policyDigest}\` |`,
|
|
1705
|
-
`| policy step budget | ${provenance.policyStepBudget} model call(s) per control rollout |`,
|
|
1706
|
-
`| control screening | \`${provenance.controlScreening}\` |`,
|
|
1707
|
-
`| certified task oracles | ${formatCertifiedTasks(provenance.certifiedTasks)} |`,
|
|
1708
|
-
`| max prefix divergence | ${formatShare(config.maxPrefixDivergence)} |`,
|
|
1709
|
-
`| control rollouts per arm | ${config.controlRollouts} |`,
|
|
1710
|
-
`| inert action | \`${config.inertAction}\` |`,
|
|
1711
|
-
`| strata admitted | ${config.admitStrata.join(", ")} |`,
|
|
1712
|
-
`| control model cost | ${cost} (${controlCost.kind}) |`,
|
|
1713
|
-
""
|
|
1714
|
-
];
|
|
1715
|
-
}
|
|
1716
|
-
function formatCertifiedTasks(certified) {
|
|
1717
|
-
const entries = Object.entries(certified).sort(([a], [b]) => a.localeCompare(b));
|
|
1718
|
-
if (entries.length === 0) return "none";
|
|
1719
|
-
return entries.map(([task, flipRate]) => `${task} (flip ${formatShare(flipRate)})`).join(", ");
|
|
1688
|
+
/** Consume every pending heredoc body; returns the offset just past the last
|
|
1689
|
+
* terminator, or the end of the script when a terminator never arrives. */
|
|
1690
|
+
function consumeHeredocBodies(script, from, marks) {
|
|
1691
|
+
let cursor = from;
|
|
1692
|
+
for (const mark of marks) {
|
|
1693
|
+
let closed = false;
|
|
1694
|
+
while (cursor < script.length) {
|
|
1695
|
+
const lineEnd = script.indexOf("\n", cursor);
|
|
1696
|
+
const stop = lineEnd === -1 ? script.length : lineEnd;
|
|
1697
|
+
const line = script.slice(cursor, stop);
|
|
1698
|
+
const candidate = mark.stripTabs ? line.replace(/^\t+/, "") : line;
|
|
1699
|
+
cursor = lineEnd === -1 ? script.length : lineEnd + 1;
|
|
1700
|
+
if (candidate === mark.delimiter) {
|
|
1701
|
+
closed = true;
|
|
1702
|
+
break;
|
|
1703
|
+
}
|
|
1704
|
+
}
|
|
1705
|
+
if (!closed) return script.length;
|
|
1706
|
+
}
|
|
1707
|
+
return cursor;
|
|
1720
1708
|
}
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1709
|
+
/**
|
|
1710
|
+
* Measure an action against the budget.
|
|
1711
|
+
*
|
|
1712
|
+
* The budget bounds what the scaffold can execute: one top-level statement,
|
|
1713
|
+
* one authored file, a byte cap, and neither a no-op nor a submit. Every
|
|
1714
|
+
* rejection here is one of those.
|
|
1715
|
+
*
|
|
1716
|
+
* `declaredKind` is what the analyst called its own action. It is recorded
|
|
1717
|
+
* beside the measured payload and never rejected on, because the scaffold runs
|
|
1718
|
+
* the action identically either way — so rejecting the label scores an arm on
|
|
1719
|
+
* how it described a repair rather than on the repair. A reader who wants the
|
|
1720
|
+
* mismatch counts `declared` against `payload`.
|
|
1721
|
+
*/
|
|
1722
|
+
function checkInterventionBudget(action, declaredKind, budget = SCAFFOLD_INTERVENTION_BUDGET) {
|
|
1723
|
+
assertBudget(budget);
|
|
1724
|
+
const scan = scanShellAction(action);
|
|
1725
|
+
const payload = scan.heredocs > 0 ? "edit" : "shell";
|
|
1726
|
+
const measurement = {
|
|
1727
|
+
bytes: Buffer.byteLength(action, "utf8"),
|
|
1728
|
+
statements: scan.statements.length,
|
|
1729
|
+
heredocs: scan.heredocs,
|
|
1730
|
+
payload,
|
|
1731
|
+
declared: declaredKind
|
|
1732
|
+
};
|
|
1733
|
+
const reject = (violation, detail) => ({
|
|
1734
|
+
admissible: false,
|
|
1735
|
+
violation,
|
|
1736
|
+
detail,
|
|
1737
|
+
measurement
|
|
1730
1738
|
});
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
return
|
|
1739
|
+
if (action.trim().length === 0) return reject("empty", "the intervention is empty");
|
|
1740
|
+
if (measurement.bytes > budget.maxBytes) return reject("over-byte-cap", `${measurement.bytes} bytes exceeds the ${budget.maxBytes}-byte action budget`);
|
|
1741
|
+
if (measurement.statements > budget.maxStatements) return reject("multiple-statements", `${measurement.statements} top-level statements exceeds the ${budget.maxStatements} the scaffold takes per action`);
|
|
1742
|
+
if (measurement.heredocs > budget.maxHeredocs) return reject("multiple-heredocs", `${measurement.heredocs} heredocs exceeds the ${budget.maxHeredocs} one edit may author`);
|
|
1743
|
+
if (action.includes(SUBMIT_SENTINEL)) return reject("submit-instead-of-repair", "the action submits the run instead of repairing it");
|
|
1744
|
+
if (NO_OP_ACTIONS.includes(action.trim())) return reject("no-op-action", `"${action.trim()}" changes nothing`);
|
|
1745
|
+
return {
|
|
1746
|
+
admissible: true,
|
|
1747
|
+
measurement
|
|
1748
|
+
};
|
|
1734
1749
|
}
|
|
1735
|
-
function
|
|
1736
|
-
const
|
|
1737
|
-
"
|
|
1738
|
-
"",
|
|
1739
|
-
"
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
const admitted = artifact.admitted[stratum].length;
|
|
1744
|
-
const eligible = artifact.chain.admitStrata.includes(stratum) ? "yes" : "no";
|
|
1745
|
-
lines.push(`| ${stratum} | ${admitted} | ${eligible} |`);
|
|
1750
|
+
function assertBudget(budget) {
|
|
1751
|
+
for (const field of [
|
|
1752
|
+
"maxBytes",
|
|
1753
|
+
"maxStatements",
|
|
1754
|
+
"maxHeredocs"
|
|
1755
|
+
]) {
|
|
1756
|
+
const value = budget[field];
|
|
1757
|
+
if (!Number.isInteger(value) || value <= 0) throw new ValidationError(`intervention budget ${field} must be a positive integer, got ${value}`);
|
|
1746
1758
|
}
|
|
1747
|
-
lines.push("", "Sample within a stratum. A command-level repair cannot address a signal kill, so pooling that population with the others averages an addressable class with an unaddressable one.", "");
|
|
1748
|
-
return lines;
|
|
1749
|
-
}
|
|
1750
|
-
function reasonSection(totals) {
|
|
1751
|
-
const lines = [
|
|
1752
|
-
"## Exclusions",
|
|
1753
|
-
"",
|
|
1754
|
-
"| reason | rows | what it means |",
|
|
1755
|
-
"| --- | --- | --- |"
|
|
1756
|
-
];
|
|
1757
|
-
for (const reason of ADMISSION_EXCLUSION_ORDER) lines.push(`| \`${reason}\` | ${totals[reason]} | ${ADMISSION_EXCLUSION_MEANING[reason]} |`);
|
|
1758
|
-
lines.push("");
|
|
1759
|
-
return lines;
|
|
1760
|
-
}
|
|
1761
|
-
function rowSection(rows, limit) {
|
|
1762
|
-
const lines = [
|
|
1763
|
-
"## Rows",
|
|
1764
|
-
"",
|
|
1765
|
-
"| row | task | stratum | final rc | admitted | excluded by |",
|
|
1766
|
-
"| --- | --- | --- | --- | --- | --- |"
|
|
1767
|
-
];
|
|
1768
|
-
for (const row of rows.slice(0, limit)) lines.push(`| \`${row.rowId}\` | ${row.taskName} | ${row.stratum ?? "—"} | ${row.finalReturncode ?? "—"} | ${row.admitted ? "yes" : "no"} | ${row.excludedBy === null ? "—" : `\`${row.excludedBy}\``} |`);
|
|
1769
|
-
if (rows.length > limit) lines.push("", `${rows.length - limit} further rows in the artifact.`);
|
|
1770
|
-
lines.push("");
|
|
1771
|
-
return lines;
|
|
1772
1759
|
}
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1760
|
+
/**
|
|
1761
|
+
* Whitespace-insensitive comparison used to reject an intervention that is
|
|
1762
|
+
* the recorded action again. Trailing whitespace and blank lines are the only
|
|
1763
|
+
* differences a re-proposal can carry without changing what runs.
|
|
1764
|
+
*/
|
|
1765
|
+
function normalizeActionForComparison(action) {
|
|
1766
|
+
return action.split("\n").map((line) => line.trimEnd()).filter((line, index, lines) => line.length > 0 || index > 0 && index < lines.length - 1).join("\n").trim();
|
|
1778
1767
|
}
|
|
1779
1768
|
//#endregion
|
|
1780
1769
|
//#region src/trace-repair/blinding.ts
|
|
@@ -1811,11 +1800,7 @@ function blindTrajectory(row, options = {}) {
|
|
|
1811
1800
|
maxK: through
|
|
1812
1801
|
};
|
|
1813
1802
|
}
|
|
1814
|
-
|
|
1815
|
-
* Fields an analyst prompt must never carry. Exported so a consumer that
|
|
1816
|
-
* builds its own prompt can assert against the same list this module honours.
|
|
1817
|
-
*/
|
|
1818
|
-
const BLINDED_FIELDS = Object.freeze([
|
|
1803
|
+
Object.freeze([
|
|
1819
1804
|
"criteria",
|
|
1820
1805
|
"controlPolicy",
|
|
1821
1806
|
"controlRate",
|
|
@@ -2091,20 +2076,6 @@ function assertReplyWithinDeclaration(declaration, reply) {
|
|
|
2091
2076
|
}
|
|
2092
2077
|
//#endregion
|
|
2093
2078
|
//#region src/trace-repair/analyst-response.ts
|
|
2094
|
-
/**
|
|
2095
|
-
* What an analyst is allowed to say about a blinded trajectory prefix.
|
|
2096
|
-
*
|
|
2097
|
-
* Exactly one of two answers:
|
|
2098
|
-
*
|
|
2099
|
-
* a finding {k, failureClaim, intervention}
|
|
2100
|
-
* the literal string no-decisive-failure
|
|
2101
|
-
*
|
|
2102
|
-
* Nothing else parses. A finding names one step, states what went wrong
|
|
2103
|
-
* there, and supplies one action to run instead. Declining is a real answer
|
|
2104
|
-
* with its own cell in the funnel, not a parse failure — an admitted row has
|
|
2105
|
-
* a failure by construction, but it need not have a single-action repair, and
|
|
2106
|
-
* an analyst that says so is answering the question it was asked.
|
|
2107
|
-
*/
|
|
2108
2079
|
/** The literal an analyst returns when no single step carries the failure. */
|
|
2109
2080
|
const NO_DECISIVE_FAILURE = "no-decisive-failure";
|
|
2110
2081
|
/**
|
|
@@ -2185,17 +2156,6 @@ function parseAnalystResponse(reply) {
|
|
|
2185
2156
|
}
|
|
2186
2157
|
};
|
|
2187
2158
|
}
|
|
2188
|
-
/** Build a finding directly, for callers that already hold typed fields. */
|
|
2189
|
-
function repairFinding(input) {
|
|
2190
|
-
if (!Number.isInteger(input.k) || input.k < 1) throw new ValidationError(`repair finding k must be a positive integer, got ${input.k}`);
|
|
2191
|
-
if (input.failureClaim.trim().length === 0) throw new ValidationError("repair finding requires a non-empty failure claim");
|
|
2192
|
-
return {
|
|
2193
|
-
kind: "finding",
|
|
2194
|
-
k: input.k,
|
|
2195
|
-
failureClaim: input.failureClaim.trim(),
|
|
2196
|
-
intervention: input.intervention
|
|
2197
|
-
};
|
|
2198
|
-
}
|
|
2199
2159
|
function describe$2(value) {
|
|
2200
2160
|
if (value === null) return "null";
|
|
2201
2161
|
if (value === void 0) return "undefined";
|
|
@@ -2700,9 +2660,6 @@ function engineUsage(modelCalls) {
|
|
|
2700
2660
|
costUsd: null
|
|
2701
2661
|
};
|
|
2702
2662
|
}
|
|
2703
|
-
function isRecord(value) {
|
|
2704
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
2705
|
-
}
|
|
2706
2663
|
function describe(value) {
|
|
2707
2664
|
if (value === null) return "null";
|
|
2708
2665
|
if (value === void 0) return "undefined";
|
|
@@ -2714,51 +2671,6 @@ function assertDspyRepairEngine(engine) {
|
|
|
2714
2671
|
}
|
|
2715
2672
|
//#endregion
|
|
2716
2673
|
//#region src/trace-repair/continuation-records.ts
|
|
2717
|
-
/**
|
|
2718
|
-
* Project a full message list into corpus steps: the system and task messages
|
|
2719
|
-
* become their own steps, and every assistant turn carries the command it
|
|
2720
|
-
* requested plus the observation that answered it. A trailing assistant turn
|
|
2721
|
-
* with no answer keeps `obs: null`, which is how the corpus records a run that
|
|
2722
|
-
* ended on its last command.
|
|
2723
|
-
*/
|
|
2724
|
-
function toRecordedSteps(messages) {
|
|
2725
|
-
const steps = [];
|
|
2726
|
-
for (let i = 0; i < messages.length; i += 1) {
|
|
2727
|
-
const message = messages[i];
|
|
2728
|
-
if (!message) continue;
|
|
2729
|
-
if (message.role === "assistant") {
|
|
2730
|
-
const parsed = parseAction(message.content);
|
|
2731
|
-
const next = messages[i + 1];
|
|
2732
|
-
steps.push({
|
|
2733
|
-
src: "agent",
|
|
2734
|
-
msg: message.content,
|
|
2735
|
-
tools: parsed.kind === "action" ? [{
|
|
2736
|
-
fn: "bash_command",
|
|
2737
|
-
cmd: parsed.command
|
|
2738
|
-
}] : null,
|
|
2739
|
-
obs: next && next.role === "user" ? next.content : null
|
|
2740
|
-
});
|
|
2741
|
-
continue;
|
|
2742
|
-
}
|
|
2743
|
-
if (message.role === "system") {
|
|
2744
|
-
steps.push({
|
|
2745
|
-
src: "system",
|
|
2746
|
-
msg: message.content,
|
|
2747
|
-
tools: null,
|
|
2748
|
-
obs: null
|
|
2749
|
-
});
|
|
2750
|
-
continue;
|
|
2751
|
-
}
|
|
2752
|
-
const previous = messages[i - 1];
|
|
2753
|
-
if (!previous || previous.role === "system") steps.push({
|
|
2754
|
-
src: "user",
|
|
2755
|
-
msg: message.content,
|
|
2756
|
-
tools: null,
|
|
2757
|
-
obs: null
|
|
2758
|
-
});
|
|
2759
|
-
}
|
|
2760
|
-
return steps;
|
|
2761
|
-
}
|
|
2762
2674
|
/** Corpus steps for the continuation alone, excluding the prefix it inherited. */
|
|
2763
2675
|
function rolloutRecordedSteps(rollout) {
|
|
2764
2676
|
return rollout.steps.map((step) => ({
|
|
@@ -2867,11 +2779,6 @@ const DEGENERATE_STRATEGIES = [
|
|
|
2867
2779
|
];
|
|
2868
2780
|
//#endregion
|
|
2869
2781
|
//#region src/trace-repair/funnel.ts
|
|
2870
|
-
const CREDIT_TERMS = [
|
|
2871
|
-
"executes",
|
|
2872
|
-
"localFlip",
|
|
2873
|
-
"repairRate"
|
|
2874
|
-
];
|
|
2875
2782
|
const ZERO_CREDIT = Object.freeze({
|
|
2876
2783
|
executes: 0,
|
|
2877
2784
|
localFlip: 0,
|
|
@@ -3219,29 +3126,6 @@ const KILL_GRACE_SECONDS = 5;
|
|
|
3219
3126
|
const BACKSTOP_SECONDS = 10;
|
|
3220
3127
|
/** `timeout` reports this when it stopped the command. */
|
|
3221
3128
|
const TIMEOUT_EXIT_CODE = 124;
|
|
3222
|
-
/**
|
|
3223
|
-
* Arguments that create a container the policy accepts. `--network none` is
|
|
3224
|
-
* not optional: a continuation with network access could install what the
|
|
3225
|
-
* recorded run could not, and the arms would no longer differ only by the
|
|
3226
|
-
* intervention.
|
|
3227
|
-
*/
|
|
3228
|
-
function dockerRunArgs(input) {
|
|
3229
|
-
return [
|
|
3230
|
-
input.executable ?? DEFAULT_EXECUTABLE,
|
|
3231
|
-
"run",
|
|
3232
|
-
"-d",
|
|
3233
|
-
"--name",
|
|
3234
|
-
input.name,
|
|
3235
|
-
"--network",
|
|
3236
|
-
"none",
|
|
3237
|
-
"-w",
|
|
3238
|
-
input.cwd,
|
|
3239
|
-
"--rm",
|
|
3240
|
-
input.image,
|
|
3241
|
-
"sleep",
|
|
3242
|
-
input.containerLifetime ?? "2h"
|
|
3243
|
-
];
|
|
3244
|
-
}
|
|
3245
3129
|
function createDockerContinuationEnvironment(options) {
|
|
3246
3130
|
if (!options.containerRef.trim()) throw new ValidationError("docker continuation environment requires a container reference");
|
|
3247
3131
|
const executable = options.executable ?? DEFAULT_EXECUTABLE;
|
|
@@ -3880,6 +3764,6 @@ function shellQuote(value) {
|
|
|
3880
3764
|
return `'${value.replaceAll("'", `'\\''`)}'`;
|
|
3881
3765
|
}
|
|
3882
3766
|
//#endregion
|
|
3883
|
-
export { ADMISSION_CONFIG_DEFAULTS,
|
|
3767
|
+
export { ADMISSION_CONFIG_DEFAULTS, DEGENERATE_STRATEGIES, DSPY_REPAIR_SIGNATURE, TestSuiteTamperedError, UncalibratedControlError, admissionArtifact, admitRow, askRepairArm, assertAnalystIndependent, assertArmSymmetry, assertChainReconciles, assertControlCalibrated, assertDenominatorIntact, assertDeterministicOracle, blindTrajectory, buildDenominatorChain, continuationPolicyDigest, continuationSeed, createCompletionRepairArm, createDockerContinuationEnvironment, createDspyRepairArm, defineControlPolicy, definePinnedContinuationPolicy, deltaRepair, gradeRepairRow, injectedTestOracle, nodeProcessRunner, parseAction, parseAnalystResponse, parseTaskOracleRegistry, renderAdmissionReport, renderDeltaRepairReport, repairArmAsymmetries, repairArmPromptSha256, repairArmResponse, repairQuestionSha256, rolloutDigest, rolloutRecordedSteps, runAdmission, runContinuation, totalCost, totalUsage };
|
|
3884
3768
|
|
|
3885
3769
|
//# sourceMappingURL=index.js.map
|