@tangle-network/agent-eval 0.216.1 → 0.217.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +102 -57
- package/README.md +2 -2
- package/dist/adapters/http.d.ts +2 -2
- package/dist/{agent-profile-VFcQncbW.d.ts → agent-profile-DTt8uGP9.d.ts} +4 -17
- package/dist/agent-profile-DTt8uGP9.d.ts.map +1 -0
- package/dist/{agent-profile-cell-BK8mlqlq.js → agent-profile-cell-B1V1gUZs.js} +4 -53
- package/dist/agent-profile-cell-B1V1gUZs.js.map +1 -0
- package/dist/{agent-profile-cell-BNp-Js9F.d.ts → agent-profile-cell-BcpFbNd5.d.ts} +2 -22
- package/dist/agent-profile-cell-BcpFbNd5.d.ts.map +1 -0
- package/dist/{agreement-irr-BEWLIF3_.js → agreement-irr-BhAqjluA.js} +5 -5
- package/dist/agreement-irr-BhAqjluA.js.map +1 -0
- package/dist/analyst/index.d.ts +36 -169
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +17 -192
- package/dist/analyst/index.js.map +1 -1
- package/dist/{analyze-runs-C84irm4i.js → analyze-runs-DqPJaJ4X.js} +225 -21
- package/dist/analyze-runs-DqPJaJ4X.js.map +1 -0
- package/dist/{analyze-runs-D68fd-su.d.ts → analyze-runs-DyeNcenW.d.ts} +5 -5
- package/dist/{analyze-runs-D68fd-su.d.ts.map → analyze-runs-DyeNcenW.d.ts.map} +1 -1
- package/dist/authenticity/index.d.ts +0 -23
- package/dist/authenticity/index.d.ts.map +1 -1
- package/dist/authenticity/index.js +1 -34
- package/dist/authenticity/index.js.map +1 -1
- package/dist/{benchmark-command-By0vh8q7.js → benchmark-command-xQK8nOI6.js} +554 -89
- package/dist/benchmark-command-xQK8nOI6.js.map +1 -0
- package/dist/benchmarks/index.d.ts +5 -6
- package/dist/benchmarks/index.d.ts.map +1 -1
- package/dist/benchmarks/index.js +4 -4
- package/dist/benchmarks/index.js.map +1 -1
- package/dist/builder-eval/index.d.ts +3 -3
- package/dist/builder-eval/index.js +2 -2
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/campaign/index.d.ts +7 -7
- package/dist/campaign/index.js +15 -15
- package/dist/{campaign-BudZ70zo.js → campaign-CJGkxAQ6.js} +23 -530
- package/dist/campaign-CJGkxAQ6.js.map +1 -0
- package/dist/{campaign-evidence-Clrn4hBV.js → campaign-evidence-CQYw8_0H.js} +10 -18
- package/dist/{campaign-evidence-Clrn4hBV.js.map → campaign-evidence-CQYw8_0H.js.map} +1 -1
- package/dist/{canonical-B2RqLofv.js → canonical-Dq2hOQ8L.js} +2 -2
- package/dist/{canonical-B2RqLofv.js.map → canonical-Dq2hOQ8L.js.map} +1 -1
- package/dist/cell-spend-DEWR3nj-.js +90 -0
- package/dist/cell-spend-DEWR3nj-.js.map +1 -0
- package/dist/{chat-client-l-MVDlDk.js → chat-client-ldqxYYTN.js} +2 -2
- package/dist/{chat-client-l-MVDlDk.js.map → chat-client-ldqxYYTN.js.map} +1 -1
- package/dist/{chat-json-call-puMArmtW.js → chat-json-call-DavzexSp.js} +3 -2
- package/dist/chat-json-call-DavzexSp.js.map +1 -0
- package/dist/{claim-q4HQhK1G.js → claim-BPVuyHuy.js} +3 -3
- package/dist/{claim-q4HQhK1G.js.map → claim-BPVuyHuy.js.map} +1 -1
- package/dist/cli.js +113 -12
- package/dist/cli.js.map +1 -1
- package/dist/{client-BzL9niqi.d.ts → client-hwo3cJjR.d.ts} +3 -16
- package/dist/client-hwo3cJjR.d.ts.map +1 -0
- package/dist/{cluster-bootstrap-qcUWYFl-.js → cluster-bootstrap-DEwwhl0C.js} +3 -3
- package/dist/{cluster-bootstrap-qcUWYFl-.js.map → cluster-bootstrap-DEwwhl0C.js.map} +1 -1
- package/dist/{produced-state-BsDmnwxc.js → completion-verifier-B_jw6C9p.js} +9 -64
- package/dist/completion-verifier-B_jw6C9p.js.map +1 -0
- package/dist/contract/index.d.ts +12 -12
- package/dist/contract/index.js +25 -401
- package/dist/contract/index.js.map +1 -1
- package/dist/{cost-ledger-BP8V4kUQ.js → cost-ledger-D0352Z4d.js} +4 -8
- package/dist/cost-ledger-D0352Z4d.js.map +1 -0
- package/dist/dataset-DQqhOCPt.d.ts.map +1 -1
- package/dist/{default-registry-COhk9g36.js → default-registry-ClYjq3QE.js} +8 -11
- package/dist/default-registry-ClYjq3QE.js.map +1 -0
- package/dist/{default-registry-DzHiGruW.d.ts → default-registry-DXgLmoDI.d.ts} +7 -50
- package/dist/default-registry-DXgLmoDI.d.ts.map +1 -0
- package/dist/{define-agent-eval-CNTPzIkp.js → define-agent-eval-BDxNlP3D.js} +10 -10
- package/dist/{define-agent-eval-CNTPzIkp.js.map → define-agent-eval-BDxNlP3D.js.map} +1 -1
- package/dist/{define-agent-eval-WSck0CZJ.d.ts → define-agent-eval-BIkH_gAR.d.ts} +8 -8
- package/dist/define-agent-eval-BIkH_gAR.d.ts.map +1 -0
- package/dist/{descriptive-DboxPZrG.js → descriptive-CVNy48rR.js} +2 -2
- package/dist/{descriptive-DboxPZrG.js.map → descriptive-CVNy48rR.js.map} +1 -1
- package/dist/diagnosis/index.d.ts +4 -14
- package/dist/diagnosis/index.d.ts.map +1 -1
- package/dist/diagnosis/index.js +67 -28
- package/dist/diagnosis/index.js.map +1 -1
- package/dist/{dspy-rlm-engine-LRCu61m5.js → dspy-rlm-engine-DhdCr1Dc.js} +5 -10
- package/dist/dspy-rlm-engine-DhdCr1Dc.js.map +1 -0
- package/dist/{emitter-HcCmFQl_.d.ts → emitter-DABYYTcH.d.ts} +5 -5
- package/dist/emitter-DABYYTcH.d.ts.map +1 -0
- package/dist/emitter-DbMXUIVP.js.map +1 -1
- package/dist/{engine-3A_4jOlI.d.ts → engine-BvZ0R-36.d.ts} +17 -21
- package/dist/engine-BvZ0R-36.d.ts.map +1 -0
- package/dist/{equal-terms-BtdXayCJ.js → equal-terms-DbPGcVtI.js} +2 -2
- package/dist/equal-terms-DbPGcVtI.js.map +1 -0
- package/dist/{errors-DVZiqS7K.js → errors-BcOs5MKW.js} +2 -8
- package/dist/{errors-DVZiqS7K.js.map → errors-BcOs5MKW.js.map} +1 -1
- package/dist/{eval-campaign-90jeaCAI.js → eval-campaign-BLEfBGbA.js} +6 -6
- package/dist/{eval-campaign-90jeaCAI.js.map → eval-campaign-BLEfBGbA.js.map} +1 -1
- package/dist/{evaluation-6Yoi74gM.d.ts → evaluation-jTjC_QQD.d.ts} +7 -7
- package/dist/evaluation-jTjC_QQD.d.ts.map +1 -0
- package/dist/evaluation.d.ts +2 -2
- package/dist/evaluation.js +1 -1
- package/dist/evaluation.js.map +1 -1
- package/dist/{exact-types-ClugMJfI.d.ts → exact-types-CflbJ9f3.d.ts} +2 -2
- package/dist/{exact-types-ClugMJfI.d.ts.map → exact-types-CflbJ9f3.d.ts.map} +1 -1
- package/dist/{exec-ChJ6d5if.js → exec-CpbtOXVt.js} +2 -24
- package/dist/{exec-ChJ6d5if.js.map → exec-CpbtOXVt.js.map} +1 -1
- package/dist/execution-tracks-CpgFPpS5.js.map +1 -1
- package/dist/experiment/index.d.ts +9 -112
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +149 -241
- package/dist/experiment/index.js.map +1 -1
- package/dist/{exporters-X4xwla8A.js → exporters-Cl6IeNAu.js} +3 -3
- package/dist/exporters-Cl6IeNAu.js.map +1 -0
- package/dist/{exporters-ChTnR90l.d.ts → exporters-Dk3oEbK_.d.ts} +2 -4
- package/dist/{exporters-ChTnR90l.d.ts.map → exporters-Dk3oEbK_.d.ts.map} +1 -1
- package/dist/{external-optimizer-process-BjMGUuds.js → external-optimizer-process-KVFGvYX1.js} +4 -3
- package/dist/external-optimizer-process-KVFGvYX1.js.map +1 -0
- package/dist/{external-optimizer-subprocess-BqW0hHcI.js → external-optimizer-subprocess-XIHBOSKt.js} +9 -11
- package/dist/external-optimizer-subprocess-XIHBOSKt.js.map +1 -0
- package/dist/{extract-usage-BrnzRJ6H.js → extract-usage-BZLU01kR.js} +2 -23
- package/dist/{extract-usage-BrnzRJ6H.js.map → extract-usage-BZLU01kR.js.map} +1 -1
- package/dist/{failure-cluster-CbgEDcFJ.d.ts → failure-cluster-DN0t63Z0.d.ts} +4 -43
- package/dist/failure-cluster-DN0t63Z0.d.ts.map +1 -0
- package/dist/{failure-taxonomy-CjabTATj.js → failure-taxonomy-Bx5oWqaW.js} +2 -19
- package/dist/{failure-taxonomy-CjabTATj.js.map → failure-taxonomy-Bx5oWqaW.js.map} +1 -1
- package/dist/{feedback-trajectory-C4EG0nNH.d.ts → feedback-trajectory-BWmhRBGu.d.ts} +7 -9
- package/dist/feedback-trajectory-BWmhRBGu.d.ts.map +1 -0
- package/dist/{semantic-concept-judge-BQF8te0T.js → findings-store-BMVQtEsm.js} +9 -249
- package/dist/findings-store-BMVQtEsm.js.map +1 -0
- package/dist/fuzz.d.ts +19 -62
- package/dist/fuzz.d.ts.map +1 -1
- package/dist/fuzz.js +62 -108
- package/dist/fuzz.js.map +1 -1
- package/dist/{hf-dataset-BR2yhuNn.js → hf-dataset-D9f7-oX7.js} +4 -24
- package/dist/hf-dataset-D9f7-oX7.js.map +1 -0
- package/dist/hosted/index.d.ts +2 -2
- package/dist/hosted/index.js +2 -2
- package/dist/{index-DHKEAv3s.d.ts → index-9jJ2fKSX.d.ts} +2 -64
- package/dist/index-9jJ2fKSX.d.ts.map +1 -0
- package/dist/{index-DV-jSzxF.d.ts → index-BLGqZ81N.d.ts} +10 -15
- package/dist/{index-DV-jSzxF.d.ts.map → index-BLGqZ81N.d.ts.map} +1 -1
- package/dist/{index-DPcngtwt.d.ts → index-BiT6RtNH.d.ts} +11 -36
- package/dist/index-BiT6RtNH.d.ts.map +1 -0
- package/dist/{index-BO41eGj5.d.ts → index-CX-GAUpw.d.ts} +3 -7
- package/dist/index-CX-GAUpw.d.ts.map +1 -0
- package/dist/{index-BFRO6chs.d.ts → index-CiRwLv2R.d.ts} +5 -69
- package/dist/index-CiRwLv2R.d.ts.map +1 -0
- package/dist/{index-CyszSg4O.d.ts → index-ZuWx52WG.d.ts} +115 -347
- package/dist/index-ZuWx52WG.d.ts.map +1 -0
- package/dist/index.d.ts +211 -218
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +402 -1040
- package/dist/index.js.map +1 -1
- package/dist/{integrity-dLiDe2kD.js → integrity-DOd9Q86O.js} +7 -9
- package/dist/integrity-DOd9Q86O.js.map +1 -0
- package/dist/{integrity-BXOsAb3i.js → integrity-W4CADcFc.js} +3 -7
- package/dist/{integrity-BXOsAb3i.js.map → integrity-W4CADcFc.js.map} +1 -1
- package/dist/{integrity-DAMEDFaC.d.ts → integrity-_qgZLA5p.d.ts} +4 -6
- package/dist/{integrity-DAMEDFaC.d.ts.map → integrity-_qgZLA5p.d.ts.map} +1 -1
- package/dist/{internal-Epl-YBZW.js → internal-BTK91Bqt.js} +3 -3
- package/dist/{internal-Epl-YBZW.js.map → internal-BTK91Bqt.js.map} +1 -1
- package/dist/{interval-CELpye1n.js → interval-Dq9gt7jx.js} +3 -3
- package/dist/{interval-CELpye1n.js.map → interval-Dq9gt7jx.js.map} +1 -1
- package/dist/{journal-DfRz9kps.js → journal-CR4QikYp.js} +110 -110
- package/dist/journal-CR4QikYp.js.map +1 -0
- package/dist/json-BhdsJaOQ.js +111 -0
- package/dist/json-BhdsJaOQ.js.map +1 -0
- package/dist/{judge-calibration-BpCy5XQl.js → judge-calibration-BkXMIWA4.js} +3 -3
- package/dist/{judge-calibration-BpCy5XQl.js.map → judge-calibration-BkXMIWA4.js.map} +1 -1
- package/dist/{judge-gate-iY9s4L44.js → judge-gate-Co5tuh57.js} +4 -27
- package/dist/{judge-gate-iY9s4L44.js.map → judge-gate-Co5tuh57.js.map} +1 -1
- package/dist/{judge-gate-CE7CQAgt.d.ts → judge-gate-DsL0yFcb.d.ts} +8 -9
- package/dist/judge-gate-DsL0yFcb.d.ts.map +1 -0
- package/dist/{kind-factory-BobI5Jy3.js → kind-factory-Gy4fXixd.js} +88 -17
- package/dist/kind-factory-Gy4fXixd.js.map +1 -0
- package/dist/ledger-core/index.d.ts +2 -2
- package/dist/ledger-core/index.js +4 -4
- package/dist/{ledger-core-Gq7yp8lP.js → ledger-core-DrJhbms9.js} +5 -84
- package/dist/ledger-core-DrJhbms9.js.map +1 -0
- package/dist/{llm-client-CyBAoYiX.js → llm-client-BshCREn6.js} +6 -108
- package/dist/llm-client-BshCREn6.js.map +1 -0
- package/dist/{llm-judge-CG12SQJq.js → llm-judge-C8jMTNNG.js} +24 -32
- package/dist/llm-judge-C8jMTNNG.js.map +1 -0
- package/dist/matrix/index.d.ts +3 -3
- package/dist/matrix/index.js +3 -2
- package/dist/{matrix-Dm0BQW7U.js → matrix-B8l-Nsxd.js} +3 -88
- package/dist/matrix-B8l-Nsxd.js.map +1 -0
- package/dist/{matrix-DSrSxjjy.d.ts → matrix-DHe-dwVJ.d.ts} +9 -18
- package/dist/matrix-DHe-dwVJ.d.ts.map +1 -0
- package/dist/{mean-DHoMiudD.js → mean-BbA_JCzW.js} +39 -5
- package/dist/mean-BbA_JCzW.js.map +1 -0
- package/dist/meta-eval/index.d.ts +5 -5
- package/dist/meta-eval/index.js +10 -10
- package/dist/metrics-BKaPBtIG.js.map +1 -1
- package/dist/{mint-DNU8QPG4.d.ts → mint-BRZXuwRi.d.ts} +4 -4
- package/dist/{mint-DNU8QPG4.d.ts.map → mint-BRZXuwRi.d.ts.map} +1 -1
- package/dist/{mint-QaD5cA0u.js → mint-zszyYlpA.js} +4 -4
- package/dist/{mint-QaD5cA0u.js.map → mint-zszyYlpA.js.map} +1 -1
- package/dist/{multi-layer-verifier-BUaQ4C17.d.ts → multi-layer-verifier-DEgMUk6v.d.ts} +2 -2
- package/dist/{multi-layer-verifier-BUaQ4C17.d.ts.map → multi-layer-verifier-DEgMUk6v.d.ts.map} +1 -1
- package/dist/{multiplicity-DQUFcbW4.js → multiplicity-Sg0U7h8j.js} +2 -2
- package/dist/{multiplicity-DQUFcbW4.js.map → multiplicity-Sg0U7h8j.js.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -23
- package/dist/multishot/golden/index.d.ts.map +1 -1
- package/dist/multishot/golden/index.js +3 -3
- package/dist/multishot/golden/index.js.map +1 -1
- package/dist/multishot/index.d.ts +3 -12
- package/dist/multishot/index.d.ts.map +1 -1
- package/dist/multishot/index.js +4 -3
- package/dist/multishot/index.js.map +1 -1
- package/dist/openapi.json +114 -10
- package/dist/{otel-spans-B9tUXRHF.js → otel-spans-CdN909k1.js} +3 -3
- package/dist/{otel-spans-B9tUXRHF.js.map → otel-spans-CdN909k1.js.map} +1 -1
- package/dist/{paired-arms-CwMGJsrS.js → paired-arms-Ct4LTKc1.js} +32 -9
- package/dist/paired-arms-Ct4LTKc1.js.map +1 -0
- package/dist/{paired-binary-CvpFHfKZ.js → paired-binary-B_nejA44.js} +3 -36
- package/dist/paired-binary-B_nejA44.js.map +1 -0
- package/dist/{paired-promotion-decision-DPKz8ZnF.d.ts → paired-promotion-decision-CQMJ5xGN.d.ts} +2 -34
- package/dist/paired-promotion-decision-CQMJ5xGN.d.ts.map +1 -0
- package/dist/{paired-promotion-decision-DsAPYGCm.js → paired-promotion-decision-D0SHb-ZL.js} +3 -3
- package/dist/{paired-promotion-decision-DsAPYGCm.js.map → paired-promotion-decision-D0SHb-ZL.js.map} +1 -1
- package/dist/{pareto-Cm6ndrWN.js → pareto-BzsxidFD.js} +11 -11
- package/dist/pareto-BzsxidFD.js.map +1 -0
- package/dist/pipelines/index.d.ts +16 -19
- package/dist/pipelines/index.d.ts.map +1 -1
- package/dist/pipelines/index.js +4 -68
- package/dist/pipelines/index.js.map +1 -1
- package/dist/{power-and-mde-Hj2ubb4B.js → power-and-mde-N5nU4_Ly.js} +3 -3
- package/dist/{power-and-mde-Hj2ubb4B.js.map → power-and-mde-N5nU4_Ly.js.map} +1 -1
- package/dist/{prime-protocol-CHKgA3a6.js → prime-protocol-BFgGrEMP.js} +3 -3
- package/dist/prime-protocol-BFgGrEMP.js.map +1 -0
- package/dist/{prime-protocol-HiB9HX7E.d.ts → prime-protocol-BWG2DQIk.d.ts} +3 -21
- package/dist/prime-protocol-BWG2DQIk.d.ts.map +1 -0
- package/dist/profile-cell.d.ts +2 -2
- package/dist/profile-cell.js +2 -2
- package/dist/{promotion-gate-BawrdPYr.d.ts → promotion-gate-DaY95yU6.d.ts} +7 -8
- package/dist/{promotion-gate-BawrdPYr.d.ts.map → promotion-gate-DaY95yU6.d.ts.map} +1 -1
- package/dist/{promotion-policy-KO57gbyV.js → promotion-policy-6A8osGr_.js} +4 -4
- package/dist/{promotion-policy-KO57gbyV.js.map → promotion-policy-6A8osGr_.js.map} +1 -1
- package/dist/{proposal-findings-CY1xKn_a.js → proposal-findings-BbWidvUk.js} +3 -3
- package/dist/proposal-findings-BbWidvUk.js.map +1 -0
- package/dist/{query-BUCZCK7F.js → query-B_RZ36an.js} +5 -5
- package/dist/query-B_RZ36an.js.map +1 -0
- package/dist/{query-DK1nWcmp.d.ts → query-Cmv_U8vf.d.ts} +4 -24
- package/dist/{query-DK1nWcmp.d.ts.map → query-Cmv_U8vf.d.ts.map} +1 -1
- package/dist/{random-ByUSnV9w.js → random-DDDcC7YU.js} +2 -2
- package/dist/{random-ByUSnV9w.js.map → random-DDDcC7YU.js.map} +1 -1
- package/dist/{raw-provider-sink-DoHz4JNG.js → raw-provider-sink-75w4FWBj.js} +2 -2
- package/dist/{raw-provider-sink-DoHz4JNG.js.map → raw-provider-sink-75w4FWBj.js.map} +1 -1
- package/dist/{redact-mtCCcT8z.d.ts → redact-DXdlFb_U.d.ts} +2 -3
- package/dist/redact-DXdlFb_U.d.ts.map +1 -0
- package/dist/{redact-B5xXKqQ3.js → redact-mtkTiJVk.js} +2 -7
- package/dist/{redact-B5xXKqQ3.js.map → redact-mtkTiJVk.js.map} +1 -1
- package/dist/{registry-D7_hm2fK.d.ts → registry-ssNoHzwD.d.ts} +4 -4
- package/dist/{registry-D7_hm2fK.d.ts.map → registry-ssNoHzwD.d.ts.map} +1 -1
- package/dist/{release-confidence-CpuP65S5.js → release-confidence-CYIFH-fp.js} +6 -11
- package/dist/release-confidence-CYIFH-fp.js.map +1 -0
- package/dist/{report-command-SC9nqb4M.js → render-C-w2EZ6p.js} +205 -363
- package/dist/render-C-w2EZ6p.js.map +1 -0
- package/dist/reporting.d.ts +6 -6
- package/dist/reporting.js +8 -8
- package/dist/{researcher-CEcfku-T.d.ts → researcher-0ABVIdGa.d.ts} +9 -9
- package/dist/{researcher-CEcfku-T.d.ts.map → researcher-0ABVIdGa.d.ts.map} +1 -1
- package/dist/{reward-CpCmAM8L.d.ts → reward-DrZ71vnf.d.ts} +3 -26
- package/dist/reward-DrZ71vnf.d.ts.map +1 -0
- package/dist/{reward-hacking-E0d-WVcP.js → reward-hacking-ByuETaIb.js} +3 -3
- package/dist/{reward-hacking-E0d-WVcP.js.map → reward-hacking-ByuETaIb.js.map} +1 -1
- package/dist/{reward-hacking-RVdZ6Kkt.d.ts → reward-hacking-E52wq86n.d.ts} +5 -18
- package/dist/reward-hacking-E52wq86n.d.ts.map +1 -0
- package/dist/reward-nw2xZGZG.js.map +1 -1
- package/dist/rl.d.ts +117 -295
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +27 -747
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +5 -5
- package/dist/rollout/index.js +7 -8
- package/dist/{rollout-DZV1t2Wg.js → rollout-D6C50mfA.js} +88 -13
- package/dist/rollout-D6C50mfA.js.map +1 -0
- package/dist/{rubric-predictive-validity-DaFW70y9.js → rubric-predictive-validity-D3_oI72N.js} +3 -3
- package/dist/{rubric-predictive-validity-DaFW70y9.js.map → rubric-predictive-validity-D3_oI72N.js.map} +1 -1
- package/dist/{rubric-predictive-validity-C03EC7ZE.d.ts → rubric-predictive-validity-DiwT8PbL.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-C03EC7ZE.d.ts.map → rubric-predictive-validity-DiwT8PbL.d.ts.map} +1 -1
- package/dist/{run-record-C1E24G2A.d.ts → run-record-CaxbLq6Y.d.ts} +4 -10
- package/dist/run-record-CaxbLq6Y.d.ts.map +1 -0
- package/dist/{run-record-C5e7FjhJ.js → run-record-CysKLB93.js} +3 -3
- package/dist/run-record-CysKLB93.js.map +1 -0
- package/dist/{run-record-Boc5IUTp.js → run-record-DhEJgoPz.js} +4 -4
- package/dist/{run-record-Boc5IUTp.js.map → run-record-DhEJgoPz.js.map} +1 -1
- package/dist/sandbox-harness-BlSOu4LX.d.ts.map +1 -1
- package/dist/{schema-CSf6qWgZ.js → schema-Bt2vLFm3.js} +2 -14
- package/dist/schema-Bt2vLFm3.js.map +1 -0
- package/dist/schema-C-ziyJD7.d.ts.map +1 -1
- package/dist/{schema-2g7trUwH.d.ts → schema-D6EVKzTB.d.ts} +4 -5
- package/dist/schema-D6EVKzTB.d.ts.map +1 -0
- package/dist/{schema-BuiujZ-g.js → schema-DxcKTzbi.js} +2 -2
- package/dist/schema-DxcKTzbi.js.map +1 -0
- package/dist/search/index.d.ts +3 -31
- package/dist/search/index.d.ts.map +1 -1
- package/dist/search/index.js +4 -4
- package/dist/{search-kernel-DyNEa6IU.d.ts → search-kernel-DxqjBxA-.d.ts} +10 -27
- package/dist/search-kernel-DxqjBxA-.d.ts.map +1 -0
- package/dist/{search-ledger-CVUGLDZp.js → search-ledger-TUvB5gWH.js} +7 -7
- package/dist/{search-ledger-CVUGLDZp.js.map → search-ledger-TUvB5gWH.js.map} +1 -1
- package/dist/{search-ledger-file-DDQUgvCg.js → search-ledger-file-UKCO2r8a.js} +4 -4
- package/dist/search-ledger-file-UKCO2r8a.js.map +1 -0
- package/dist/{search-shipper-BOSD46qv.js → search-shipper-Cw2z3-td.js} +4 -4
- package/dist/search-shipper-Cw2z3-td.js.map +1 -0
- package/dist/{search-summary-BQCSWf1g.js → search-summary-DQIvJsSB.js} +5 -5
- package/dist/{search-summary-BQCSWf1g.js.map → search-summary-DQIvJsSB.js.map} +1 -1
- package/dist/{sequential-CzK5DarL.js → sequential-BVv1sjsw.js} +2 -32
- package/dist/{sequential-CzK5DarL.js.map → sequential-BVv1sjsw.js.map} +1 -1
- package/dist/{sequential-Bf_pbE50.js → sequential-CimQrbjv.js} +5 -5
- package/dist/sequential-CimQrbjv.js.map +1 -0
- package/dist/{sequential-BhsrMupG.d.ts → sequential-KwGspfeR.d.ts} +2 -9
- package/dist/{sequential-BhsrMupG.d.ts.map → sequential-KwGspfeR.d.ts.map} +1 -1
- package/dist/{sequential-eprocess-DM_qENCl.js → sequential-eprocess-DodI6RoF.js} +2 -2
- package/dist/{sequential-eprocess-DM_qENCl.js.map → sequential-eprocess-DodI6RoF.js.map} +1 -1
- package/dist/{server-e9oWO_bN.js → server-DoVbqqtJ.js} +101 -101
- package/dist/server-DoVbqqtJ.js.map +1 -0
- package/dist/{single-run-lock-CV9Eyd_q.js → single-run-lock-U58yf8N6.js} +2 -2
- package/dist/{single-run-lock-CV9Eyd_q.js.map → single-run-lock-U58yf8N6.js.map} +1 -1
- package/dist/{skillopt-optimization-method-BiwA8nfX.js → skillopt-optimization-method-kyL94-ri.js} +17 -95
- package/dist/skillopt-optimization-method-kyL94-ri.js.map +1 -0
- package/dist/{spans-BHAUXMGN.d.ts → spans-CVngGij3.d.ts} +2 -2
- package/dist/{spans-BHAUXMGN.d.ts.map → spans-CVngGij3.d.ts.map} +1 -1
- package/dist/{statistical-heldout-42NlCNDe.js → statistical-heldout-Bksv92Th.js} +3 -3
- package/dist/statistical-heldout-Bksv92Th.js.map +1 -0
- package/dist/{statistical-heldout-CYpGaoDH.d.ts → statistical-heldout-C9AbgREN.d.ts} +8 -67
- package/dist/statistical-heldout-C9AbgREN.d.ts.map +1 -0
- package/dist/{steps-Bg5-KVx1.d.ts → steps-DlepJPwD.d.ts} +2 -56
- package/dist/steps-DlepJPwD.d.ts.map +1 -0
- package/dist/{store-C4avFjIi.d.ts → store-B3nWRy3B.d.ts} +2 -2
- package/dist/{store-C4avFjIi.d.ts.map → store-B3nWRy3B.d.ts.map} +1 -1
- package/dist/{store-otlp-DJ9dtA5x.js → store-otlp-BteDAbHK.js} +3 -3
- package/dist/{store-otlp-DJ9dtA5x.js.map → store-otlp-BteDAbHK.js.map} +1 -1
- package/dist/{store-tool-spans-CIX5arAG.js → store-tool-spans-D5l_OfTJ.js} +8 -8
- package/dist/store-tool-spans-D5l_OfTJ.js.map +1 -0
- package/dist/{store-tool-spans-DdCSPa7C.d.ts → store-tool-spans-DZS0HWVf.d.ts} +9 -18
- package/dist/store-tool-spans-DZS0HWVf.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts +4 -11
- package/dist/storyboard/index.d.ts.map +1 -1
- package/dist/storyboard/index.js +1 -16
- package/dist/storyboard/index.js.map +1 -1
- package/dist/{summary-report-V_JACR4Y.js → summary-report-BsXJoYX5.js} +88 -10
- package/dist/summary-report-BsXJoYX5.js.map +1 -0
- package/dist/{summary-report-bdFDXq0b.d.ts → summary-report-DM7tlml1.d.ts} +10 -19
- package/dist/summary-report-DM7tlml1.d.ts.map +1 -0
- package/dist/supervisor-run/index.d.ts +60 -28
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +4 -458
- package/dist/supervisor-run/index.js.map +1 -1
- package/dist/{systemone-BomhHr2c.d.ts → systemone-Bp_Ac0p1.d.ts} +5 -5
- package/dist/{systemone-BomhHr2c.d.ts.map → systemone-Bp_Ac0p1.d.ts.map} +1 -1
- package/dist/systemone-protocol.js +1 -1
- package/dist/{systemone-review-CUVcQJu6.d.ts → systemone-review-CwE6JGIp.d.ts} +2 -2
- package/dist/{systemone-review-CUVcQJu6.d.ts.map → systemone-review-CwE6JGIp.d.ts.map} +1 -1
- package/dist/{systemone-review-D7sXwsqq.js → systemone-review-DzdOJ8I6.js} +3 -3
- package/dist/{systemone-review-D7sXwsqq.js.map → systemone-review-DzdOJ8I6.js.map} +1 -1
- package/dist/systemone.d.ts +5 -5
- package/dist/systemone.js +5 -5
- package/dist/systemone.js.map +1 -1
- package/dist/{task-failure-attributes-hEmASjix.js → task-failure-attributes-BMsblaDh.js} +3 -3
- package/dist/task-failure-attributes-BMsblaDh.js.map +1 -0
- package/dist/{terminal-record-DZyQZ3pz.js → terminal-record-78g6E4vh.js} +5 -5
- package/dist/terminal-record-78g6E4vh.js.map +1 -0
- package/dist/{tool-groups-C3NEcdCM.d.ts → tool-groups-DyNz2JZ_.d.ts} +3 -3
- package/dist/tool-groups-DyNz2JZ_.d.ts.map +1 -0
- package/dist/{tool-waste-BKgvd-TP.d.ts → tool-waste-CkjzWmeY.d.ts} +25 -28
- package/dist/tool-waste-CkjzWmeY.d.ts.map +1 -0
- package/dist/{tool-waste-BlesqTqW.js → tool-waste-DLsjS_p-.js} +5 -5
- package/dist/{tool-waste-BlesqTqW.js.map → tool-waste-DLsjS_p-.js.map} +1 -1
- package/dist/{trace-questions-CM2CAocU.d.ts → trace-questions-xxwgsuMi.d.ts} +4 -9
- package/dist/{trace-questions-CM2CAocU.d.ts.map → trace-questions-xxwgsuMi.d.ts.map} +1 -1
- package/dist/trace-repair/index.d.ts +14 -211
- package/dist/trace-repair/index.d.ts.map +1 -1
- package/dist/trace-repair/index.js +865 -981
- package/dist/trace-repair/index.js.map +1 -1
- package/dist/traces.d.ts +18 -46
- package/dist/traces.d.ts.map +1 -1
- package/dist/traces.js +13 -13
- package/dist/traces.js.map +1 -1
- package/dist/trajectory-replay/index.d.ts +3 -117
- package/dist/trajectory-replay/index.d.ts.map +1 -1
- package/dist/trajectory-replay/index.js +98 -56
- package/dist/trajectory-replay/index.js.map +1 -1
- package/dist/{tree-C1XYeXvy.js → tree-BGRglkKd.js} +13 -10
- package/dist/tree-BGRglkKd.js.map +1 -0
- package/dist/{types-B3jzCp0p.js → types-BXgASBCL.js} +2 -2
- package/dist/{types-B3jzCp0p.js.map → types-BXgASBCL.js.map} +1 -1
- package/dist/{types-C1tnkEgM.d.ts → types-CW8ewATZ.d.ts} +4 -113
- package/dist/types-CW8ewATZ.d.ts.map +1 -0
- package/dist/{types-BqkVmHfv.d.ts → types-CmXY0PQf.d.ts} +2 -4
- package/dist/types-CmXY0PQf.d.ts.map +1 -0
- package/dist/{types-C02qtnG8.d.ts → types-D-mgRzlh.d.ts} +7 -28
- package/dist/types-D-mgRzlh.d.ts.map +1 -0
- package/dist/{types-Cq8mSbq8.d.ts → types-D8GhJoHk.d.ts} +7 -9
- package/dist/{types-Cq8mSbq8.d.ts.map → types-D8GhJoHk.d.ts.map} +1 -1
- package/dist/{types-IXD2HJzu.js → types-DqFMNTgK.js} +2 -4
- package/dist/types-DqFMNTgK.js.map +1 -0
- package/dist/{types-Cbe2fWBa.d.ts → types-DtMbFw6U.d.ts} +3 -3
- package/dist/{types-Cbe2fWBa.d.ts.map → types-DtMbFw6U.d.ts.map} +1 -1
- package/dist/{verdict-Dl0tiT1X.js → verdict-BGQmxQ8A.js} +3 -3
- package/dist/verdict-BGQmxQ8A.js.map +1 -0
- package/dist/{verdict-E4eRNf7-.d.ts → verdict-C24NELtN.d.ts} +2 -15
- package/dist/{verdict-E4eRNf7-.d.ts.map → verdict-C24NELtN.d.ts.map} +1 -1
- package/dist/{verdict-cache-Bm8EKL2s.js → verdict-cache-CL_7tePD.js} +2 -2
- package/dist/verdict-cache-CL_7tePD.js.map +1 -0
- package/dist/wire/index.d.ts +31 -410
- package/dist/wire/index.d.ts.map +1 -1
- package/dist/wire/index.js +2 -2
- package/docs/changes/jev-evidence-review.md +1 -1
- package/docs/changes/jev-product-adoption.md +1 -1
- package/docs/changes/jev-recorded-evidence.md +1 -1
- package/docs/design/statistics-decisions.md +17 -17
- package/docs/design.md +1 -1
- package/docs/evaluation-integrity.md +1 -1
- package/docs/proofs/campaign-attempt-retention-20261001/README.md +2 -2
- package/docs/public-api.md +138 -529
- package/docs/rollout.md +1 -1
- package/docs/trace-repair-admission.md +1 -1
- package/docs/wire-protocol.md +1 -1
- package/package.json +1 -1
- package/dist/active-curriculum-B_Lc9kfO.js +0 -61
- package/dist/active-curriculum-B_Lc9kfO.js.map +0 -1
- package/dist/agent-profile-VFcQncbW.d.ts.map +0 -1
- package/dist/agent-profile-cell-BK8mlqlq.js.map +0 -1
- package/dist/agent-profile-cell-BNp-Js9F.d.ts.map +0 -1
- package/dist/agreement-irr-BEWLIF3_.js.map +0 -1
- package/dist/analyze-runs-C84irm4i.js.map +0 -1
- package/dist/baseline-Udi_PLuT.js +0 -259
- package/dist/baseline-Udi_PLuT.js.map +0 -1
- package/dist/benchmark-CXmVVHH2.js +0 -562
- package/dist/benchmark-CXmVVHH2.js.map +0 -1
- package/dist/benchmark-command-By0vh8q7.js.map +0 -1
- package/dist/campaign-BudZ70zo.js.map +0 -1
- package/dist/chat-json-call-puMArmtW.js.map +0 -1
- package/dist/claude-jsonl-dZir12m6.js +0 -86
- package/dist/claude-jsonl-dZir12m6.js.map +0 -1
- package/dist/client-BzL9niqi.d.ts.map +0 -1
- package/dist/cost-ledger-BP8V4kUQ.js.map +0 -1
- package/dist/counterfactual-CPaWN99-.d.ts +0 -60
- package/dist/counterfactual-CPaWN99-.d.ts.map +0 -1
- package/dist/counterfactual-Dn3veueY.js +0 -98
- package/dist/counterfactual-Dn3veueY.js.map +0 -1
- package/dist/default-registry-COhk9g36.js.map +0 -1
- package/dist/default-registry-DzHiGruW.d.ts.map +0 -1
- package/dist/define-agent-eval-WSck0CZJ.d.ts.map +0 -1
- package/dist/dspy-rlm-engine-LRCu61m5.js.map +0 -1
- package/dist/effect-sizes-BhzEpXMc.js +0 -82
- package/dist/effect-sizes-BhzEpXMc.js.map +0 -1
- package/dist/emitter-HcCmFQl_.d.ts.map +0 -1
- package/dist/engine-3A_4jOlI.d.ts.map +0 -1
- package/dist/equal-terms-BtdXayCJ.js.map +0 -1
- package/dist/evaluation-6Yoi74gM.d.ts.map +0 -1
- package/dist/exporters-X4xwla8A.js.map +0 -1
- package/dist/external-optimizer-process-BjMGUuds.js.map +0 -1
- package/dist/external-optimizer-subprocess-BqW0hHcI.js.map +0 -1
- package/dist/failure-cluster-CbgEDcFJ.d.ts.map +0 -1
- package/dist/feedback-trajectory-C4EG0nNH.d.ts.map +0 -1
- package/dist/hf-dataset-BR2yhuNn.js.map +0 -1
- package/dist/index-BFRO6chs.d.ts.map +0 -1
- package/dist/index-BO41eGj5.d.ts.map +0 -1
- package/dist/index-CyszSg4O.d.ts.map +0 -1
- package/dist/index-DHKEAv3s.d.ts.map +0 -1
- package/dist/index-DPcngtwt.d.ts.map +0 -1
- package/dist/integrity-dLiDe2kD.js.map +0 -1
- package/dist/journal-DfRz9kps.js.map +0 -1
- package/dist/judge-gate-CE7CQAgt.d.ts.map +0 -1
- package/dist/kind-factory-BobI5Jy3.js.map +0 -1
- package/dist/ledger-core-Gq7yp8lP.js.map +0 -1
- package/dist/llm-client-CyBAoYiX.js.map +0 -1
- package/dist/llm-judge-CG12SQJq.js.map +0 -1
- package/dist/matrix-DSrSxjjy.d.ts.map +0 -1
- package/dist/matrix-Dm0BQW7U.js.map +0 -1
- package/dist/mean-DHoMiudD.js.map +0 -1
- package/dist/normal-DsJZtrJc.js +0 -25
- package/dist/normal-DsJZtrJc.js.map +0 -1
- package/dist/paired-arms-CwMGJsrS.js.map +0 -1
- package/dist/paired-binary-CvpFHfKZ.js.map +0 -1
- package/dist/paired-promotion-decision-DPKz8ZnF.d.ts.map +0 -1
- package/dist/pareto-Cm6ndrWN.js.map +0 -1
- package/dist/prime-protocol-CHKgA3a6.js.map +0 -1
- package/dist/prime-protocol-HiB9HX7E.d.ts.map +0 -1
- package/dist/produced-state-BsDmnwxc.js.map +0 -1
- package/dist/proposal-findings-CY1xKn_a.js.map +0 -1
- package/dist/query-BUCZCK7F.js.map +0 -1
- package/dist/redact-mtCCcT8z.d.ts.map +0 -1
- package/dist/release-confidence-CpuP65S5.js.map +0 -1
- package/dist/report-command-SC9nqb4M.js.map +0 -1
- package/dist/reward-CpCmAM8L.d.ts.map +0 -1
- package/dist/reward-hacking-RVdZ6Kkt.d.ts.map +0 -1
- package/dist/rollout-DZV1t2Wg.js.map +0 -1
- package/dist/run-record-C1E24G2A.d.ts.map +0 -1
- package/dist/run-record-C5e7FjhJ.js.map +0 -1
- package/dist/schema-2g7trUwH.d.ts.map +0 -1
- package/dist/schema-BuiujZ-g.js.map +0 -1
- package/dist/schema-CSf6qWgZ.js.map +0 -1
- package/dist/search-kernel-DyNEa6IU.d.ts.map +0 -1
- package/dist/search-ledger-file-DDQUgvCg.js.map +0 -1
- package/dist/search-shipper-BOSD46qv.js.map +0 -1
- package/dist/semantic-concept-judge-BQF8te0T.js.map +0 -1
- package/dist/sequential-Bf_pbE50.js.map +0 -1
- package/dist/server-e9oWO_bN.js.map +0 -1
- package/dist/skillopt-optimization-method-BiwA8nfX.js.map +0 -1
- package/dist/statistical-heldout-42NlCNDe.js.map +0 -1
- package/dist/statistical-heldout-CYpGaoDH.d.ts.map +0 -1
- package/dist/steps-Bg5-KVx1.d.ts.map +0 -1
- package/dist/store-tool-spans-CIX5arAG.js.map +0 -1
- package/dist/store-tool-spans-DdCSPa7C.d.ts.map +0 -1
- package/dist/student-t-DVPW7CdL.js +0 -39
- package/dist/student-t-DVPW7CdL.js.map +0 -1
- package/dist/summary-report-V_JACR4Y.js.map +0 -1
- package/dist/summary-report-bdFDXq0b.d.ts.map +0 -1
- package/dist/task-failure-attributes-hEmASjix.js.map +0 -1
- package/dist/terminal-record-DZyQZ3pz.js.map +0 -1
- package/dist/tool-groups-C3NEcdCM.d.ts.map +0 -1
- package/dist/tool-waste-BKgvd-TP.d.ts.map +0 -1
- package/dist/trajectory-Y1fmdlw7.d.ts +0 -24
- package/dist/trajectory-Y1fmdlw7.d.ts.map +0 -1
- package/dist/tree-C1XYeXvy.js.map +0 -1
- package/dist/types-BqkVmHfv.d.ts.map +0 -1
- package/dist/types-C02qtnG8.d.ts.map +0 -1
- package/dist/types-C1tnkEgM.d.ts.map +0 -1
- package/dist/types-IXD2HJzu.js.map +0 -1
- package/dist/usage-receipt-Cx3xNeyB.js +0 -76
- package/dist/usage-receipt-Cx3xNeyB.js.map +0 -1
- package/dist/verdict-Dl0tiT1X.js.map +0 -1
- package/dist/verdict-cache-Bm8EKL2s.js.map +0 -1
- package/docs/design/self-improvement-evidence-audit.md +0 -263
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { a as CostLedgerHandle, d as CostReceiptInput, m as MaximumCharge, r as CostChannel, u as CostReceipt } from "./cost-ledger-Jk62OBfF.js";
|
|
2
|
-
import {
|
|
3
|
-
import { R as Scenario, S as JudgeConfig, w as JudgeScore } from "./types-
|
|
2
|
+
import { i as AnalystFinding, n as AnalystContext, t as Analyst } from "./types-CW8ewATZ.js";
|
|
3
|
+
import { R as Scenario, S as JudgeConfig, w as JudgeScore } from "./types-D8GhJoHk.js";
|
|
4
4
|
//#region src/evaluation.d.ts
|
|
5
5
|
interface EvaluationResult<T> {
|
|
6
6
|
value: T;
|
|
@@ -21,7 +21,7 @@ interface EvaluationContext {
|
|
|
21
21
|
/** Awaited after accounting and before validation or result delivery. */
|
|
22
22
|
onReceipt?: (receipt: CostReceipt) => void | Promise<void>;
|
|
23
23
|
}
|
|
24
|
-
type
|
|
24
|
+
type CostedEvaluator<I, O> = (input: I, context?: EvaluationContext) => Promise<EvaluationResult<O>>;
|
|
25
25
|
interface EvaluatorOptions<I, O> {
|
|
26
26
|
execute: (input: I, context: {
|
|
27
27
|
signal: AbortSignal;
|
|
@@ -36,12 +36,12 @@ interface EvaluatorOptions<I, O> {
|
|
|
36
36
|
validate?: (value: O, input: I) => void | Promise<void>;
|
|
37
37
|
}
|
|
38
38
|
/** A metered function, independent of providers, question formats, and application policy. */
|
|
39
|
-
declare function createEvaluator<I, O>(options: EvaluatorOptions<I, O>):
|
|
39
|
+
declare function createEvaluator<I, O>(options: EvaluatorOptions<I, O>): CostedEvaluator<I, O>;
|
|
40
40
|
interface EvaluationJudgeOptions<A, S extends Scenario, O> {
|
|
41
41
|
name: string;
|
|
42
42
|
version: string;
|
|
43
43
|
dimensions: JudgeConfig<A, S>['dimensions'];
|
|
44
|
-
evaluate:
|
|
44
|
+
evaluate: CostedEvaluator<{
|
|
45
45
|
artifact: A;
|
|
46
46
|
scenario: S;
|
|
47
47
|
}, O>;
|
|
@@ -71,5 +71,5 @@ interface EvaluationAnalystOptions<I, O> {
|
|
|
71
71
|
/** Registry, graph, and trace consumers keep the same Analyst interface. */
|
|
72
72
|
declare function asAnalyst<I, O>(options: EvaluationAnalystOptions<I, O>): Analyst<I>;
|
|
73
73
|
//#endregion
|
|
74
|
-
export {
|
|
75
|
-
//# sourceMappingURL=evaluation-
|
|
74
|
+
export { EvaluationResult as a, asJudge as c, EvaluationJudgeOptions as i, createEvaluator as l, EvaluationAnalystOptions as n, EvaluatorOptions as o, EvaluationContext as r, asAnalyst as s, CostedEvaluator as t };
|
|
75
|
+
//# sourceMappingURL=evaluation-jTjC_QQD.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"evaluation-jTjC_QQD.d.ts","names":[],"sources":["../src/evaluation.ts"],"mappings":";;;;UAWiB,iBAAiB;EAChC,OAAO;EACP,SAAS;EACT;;UAGe;;;EAGf;EACA,SAAS;EACT,aAAa;EACb;EACA,UAAU;EACV;EACA,WAAW;EACX,gBAAgB;;EAEhB,aAAa,SAAS,uBAAuB;;KAGnC,gBAAgB,GAAG,MAC7B,OAAO,GACP,UAAU,sBACP,QAAQ,iBAAiB;UAEb,iBAAiB,GAAG;EACnC,UAAU,OAAO,GAAG;IAAW,QAAQ;IAAa;QAA6B,QAAQ;EACzF,UAAU,OAAO,MAAM;EACvB,oBAAoB,OAAO,UAAU;EACrC,mBAAmB,OAAO;EAC1B,aAAa;EACb,gBAAgB;;EAEhB,YAAY,OAAO,GAAG,OAAO,aAAa;;;iBAI5B,gBAAgB,GAAG,GAAG,SAAS,iBAAiB,GAAG,KAAK,gBAAgB,GAAG;UA6B1E,uBAAuB,GAAG,UAAU,UAAU;EAC7D;EACA;EACA,YAAY,YAAY,GAAG;EAC3B,UAAU;IAAkB,UAAU;IAAG,UAAU;KAAK;EACxD,MAAM,OAAO,GAAG;IAAS,UAAU;IAAG,UAAU;QAAQ,aAAa,QAAQ;;EAE7E,UACE,QAAQ,iBAAiB,IACzB;IAAS,UAAU;IAAG,UAAU;eACtB;EACZ,aAAa,UAAU;;;iBAIT,QAAQ,GAAG,UAAU,UAAU,GAC7C,SAAS,uBAAuB,GAAG,GAAG,KACrC,YAAY,GAAG;UA4BD,yBAAyB,GAAG;EAC3C;EACA;EACA;EACA,WAAW,QAAQ;EACnB,MAAM,QAAQ;EACd,WACE,OAAO,GACP,SAAS,mBACT,gBAAgB,mBACb,QAAQ,iBAAiB;EAC9B,MAAM,OAAO,GAAG,OAAO,GAAG,SAAS,mBAAmB,mBAAmB,QAAQ;EACjF,UAAU,QAAQ,iBAAiB,IAAI,OAAO,GAAG,SAAS,0BAA0B;;;iBAItE,UAAU,GAAG,GAAG,SAAS,yBAAyB,GAAG,KAAK,QAAQ"}
|
package/dist/evaluation.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import { a as
|
|
2
|
-
export { EvaluationAnalystOptions, EvaluationContext, EvaluationJudgeOptions, EvaluationResult,
|
|
1
|
+
import { a as EvaluationResult, c as asJudge, i as EvaluationJudgeOptions, l as createEvaluator, n as EvaluationAnalystOptions, o as EvaluatorOptions, r as EvaluationContext, s as asAnalyst, t as CostedEvaluator } from "./evaluation-jTjC_QQD.js";
|
|
2
|
+
export { CostedEvaluator, EvaluationAnalystOptions, EvaluationContext, EvaluationJudgeOptions, EvaluationResult, EvaluatorOptions, asAnalyst, asJudge, createEvaluator };
|
package/dist/evaluation.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { t as CostLedger } from "./cost-ledger-
|
|
1
|
+
import { t as CostLedger } from "./cost-ledger-D0352Z4d.js";
|
|
2
2
|
//#region src/evaluation.ts
|
|
3
3
|
/** A metered function, independent of providers, question formats, and application policy. */
|
|
4
4
|
function createEvaluator(options) {
|
package/dist/evaluation.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"evaluation.js","names":[],"sources":["../src/evaluation.ts"],"sourcesContent":["import type { Analyst, AnalystContext, AnalystFinding } from './analyst/types'\nimport type { JudgeConfig, JudgeScore, Scenario } from './campaign/types'\nimport type {\n CostChannel,\n CostLedgerHandle,\n CostReceipt,\n CostReceiptInput,\n MaximumCharge,\n} from './cost-ledger'\nimport { CostLedger } from './cost-ledger'\n\nexport interface EvaluationResult<T> {\n value: T\n receipt: CostReceipt\n durationMs: number\n}\n\nexport interface EvaluationContext {\n /** Physical paid-call identity, forwarded through the existing ledger and transport.\n * Reusing it in one ledger conflicts; it is not an automatic result-replay policy. */\n callId?: string\n signal?: AbortSignal\n costLedger?: CostLedgerHandle\n actor?: string\n channel?: CostChannel\n costPhase?: string\n costTags?: Record<string, string>\n maximumCharge?: MaximumCharge\n /** Awaited after accounting and before validation or result delivery. */\n onReceipt?: (receipt: CostReceipt) => void | Promise<void>\n}\n\nexport type Evaluator<I, O> = (\n input: I,\n context?: EvaluationContext,\n) => Promise<EvaluationResult<O>>\n\nexport interface EvaluatorOptions<I, O> {\n execute: (input: I, context: { signal: AbortSignal; idempotencyKey: string }) => Promise<O>\n receipt: (value: O) => CostReceiptInput\n receiptFromError?: (error: Error) => CostReceiptInput | undefined\n model?: string | ((input: I) => string)\n costLedger?: CostLedgerHandle\n maximumCharge?: MaximumCharge\n /** Awaited after accounting, so invalid output never erases paid work. */\n validate?: (value: O, input: I) => void | Promise<void>\n}\n\n/** A metered function, independent of providers, question formats, and application policy. */\nexport function createEvaluator<I, O>(options: EvaluatorOptions<I, O>): Evaluator<I, O> {\n const defaults = { ...options }\n const ledger = defaults.costLedger ?? new CostLedger()\n return async (input, context = {}) => {\n context.signal?.throwIfAborted()\n const started = performance.now()\n const paid = await (context.costLedger ?? ledger).runPaidCall({\n callId: context.callId,\n actor: context.actor ?? 'evaluation',\n channel: context.channel ?? 'evaluation',\n phase: context.costPhase ?? 'evaluation',\n tags: context.costTags,\n model: typeof defaults.model === 'function' ? defaults.model(input) : defaults.model,\n signal: context.signal,\n maximumCharge: context.maximumCharge ?? defaults.maximumCharge,\n execute: (signal, idempotencyKey) => defaults.execute(input, { signal, idempotencyKey }),\n receipt: defaults.receipt,\n receiptFromError: defaults.receiptFromError,\n })\n if (paid.receipt) await context.onReceipt?.(paid.receipt)\n if (!paid.succeeded) throw paid.error\n // A transport can complete after cancellation. Keep its paid receipt, not a live decision.\n context.signal?.throwIfAborted()\n await defaults.validate?.(paid.value, input)\n context.signal?.throwIfAborted()\n return { value: paid.value, receipt: paid.receipt, durationMs: performance.now() - started }\n }\n}\n\nexport interface EvaluationJudgeOptions<A, S extends Scenario, O> {\n name: string\n version: string\n dimensions: JudgeConfig<A, S>['dimensions']\n evaluate: Evaluator<{ artifact: A; scenario: S }, O>\n map: (value: O, input: { artifact: A; scenario: S }) => JudgeScore | Promise<JudgeScore>\n /** Persist the complete observation before reducing it to a verdict. */\n record?: (\n result: EvaluationResult<O>,\n input: { artifact: A; scenario: S },\n ) => void | Promise<void>\n appliesTo?: (scenario: S) => boolean\n}\n\n/** Return the existing contract. Mapping never performs another inference call. */\nexport function asJudge<A, S extends Scenario, O>(\n options: EvaluationJudgeOptions<A, S, O>,\n): JudgeConfig<A, S> {\n // The version describes these callbacks, not later mutations of the caller's options.\n const config = { ...options, dimensions: structuredClone(options.dimensions) }\n return {\n name: config.name,\n judgeVersion: config.version,\n dimensions: config.dimensions,\n appliesTo: config.appliesTo,\n async score({ artifact, scenario, signal, costLedger, costPhase, costTags }) {\n signal.throwIfAborted()\n const input = { artifact, scenario }\n const result = await config.evaluate(input, {\n signal,\n costLedger,\n actor: config.name,\n channel: 'judge',\n costPhase,\n costTags: { ...costTags, scenarioId: scenario.id },\n })\n await config.record?.(result, input)\n signal.throwIfAborted()\n const score = await config.map(result.value, input)\n signal.throwIfAborted()\n return score\n },\n }\n}\n\nexport interface EvaluationAnalystOptions<I, O> {\n id: string\n version: string\n description: string\n inputKind: Analyst<I>['inputKind']\n cost: Analyst<I>['cost']\n evaluate: (\n input: I,\n context: EvaluationContext,\n analystContext: AnalystContext,\n ) => Promise<EvaluationResult<O>>\n map: (value: O, input: I, context: AnalystContext) => AnalystFinding[] | Promise<AnalystFinding[]>\n record?: (result: EvaluationResult<O>, input: I, context: AnalystContext) => void | Promise<void>\n}\n\n/** Registry, graph, and trace consumers keep the same Analyst interface. */\nexport function asAnalyst<I, O>(options: EvaluationAnalystOptions<I, O>): Analyst<I> {\n const config = { ...options, cost: structuredClone(options.cost) }\n return {\n id: config.id,\n version: config.version,\n description: config.description,\n inputKind: config.inputKind,\n cost: config.cost,\n async analyze(input, context) {\n const signals = context.signal ? [context.signal] : []\n if (context.deadlineMs !== undefined) {\n const remaining = context.deadlineMs - Date.now()\n if (!Number.isFinite(remaining)) throw new TypeError('Analyst deadline must be finite')\n if (remaining <= 0) throw new DOMException('Analyst deadline exceeded', 'TimeoutError')\n signals.push(AbortSignal.timeout(Math.ceil(remaining)))\n }\n const signal = AbortSignal.any(signals)\n signal.throwIfAborted()\n const analystContext = { ...context, signal }\n const result = await config.evaluate(\n input,\n {\n signal,\n // No implicit account may override the evaluator's configured spending authority.\n costLedger:\n context.costLedger ??\n (context.budgetUsd === undefined\n ? undefined\n : new CostLedger({ costCeilingUsd: context.budgetUsd })),\n actor: config.id,\n channel: 'analyst',\n costPhase: context.costPhase,\n costTags: { ...context.tags, runId: context.runId, correlationId: context.correlationId },\n onReceipt: (receipt) =>\n context.recordUsage?.({\n calls: 1,\n tokens: receipt.usageUnknown\n ? null\n : { input: receipt.inputTokens, output: receipt.outputTokens },\n cost: receipt.costUnknown\n ? { kind: 'uncaptured', usd: null }\n : {\n kind: receipt.actualCostUsd === undefined ? 'estimated' : 'observed',\n usd: receipt.costUsd,\n },\n }),\n },\n analystContext,\n )\n await config.record?.(result, input, analystContext)\n signal.throwIfAborted()\n const findings = await config.map(result.value, input, analystContext)\n signal.throwIfAborted()\n return findings\n },\n }\n}\n"],"mappings":";;;AAiDA,SAAgB,gBAAsB,SAAkD;CACtF,MAAM,WAAW,EAAE,GAAG,QAAQ;CAC9B,MAAM,SAAS,SAAS,cAAc,IAAI,WAAW;CACrD,OAAO,OAAO,OAAO,UAAU,CAAC,MAAM;EACpC,QAAQ,QAAQ,eAAe;EAC/B,MAAM,UAAU,YAAY,IAAI;EAChC,MAAM,OAAO,OAAO,QAAQ,cAAc,OAAA,CAAQ,YAAY;GAC5D,QAAQ,QAAQ;GAChB,OAAO,QAAQ,SAAS;GACxB,SAAS,QAAQ,WAAW;GAC5B,OAAO,QAAQ,aAAa;GAC5B,MAAM,QAAQ;GACd,OAAO,OAAO,SAAS,UAAU,aAAa,SAAS,MAAM,KAAK,IAAI,SAAS;GAC/E,QAAQ,QAAQ;GAChB,eAAe,QAAQ,iBAAiB,SAAS;GACjD,UAAU,QAAQ,mBAAmB,SAAS,QAAQ,OAAO;IAAE;IAAQ;GAAe,CAAC;GACvF,SAAS,SAAS;GAClB,kBAAkB,SAAS;EAC7B,CAAC;EACD,IAAI,KAAK,SAAS,MAAM,QAAQ,YAAY,KAAK,OAAO;EACxD,IAAI,CAAC,KAAK,WAAW,MAAM,KAAK;EAEhC,QAAQ,QAAQ,eAAe;EAC/B,MAAM,SAAS,WAAW,KAAK,OAAO,KAAK;EAC3C,QAAQ,QAAQ,eAAe;EAC/B,OAAO;GAAE,OAAO,KAAK;GAAO,SAAS,KAAK;GAAS,YAAY,YAAY,IAAI,IAAI;EAAQ;CAC7F;AACF;;AAiBA,SAAgB,QACd,SACmB;CAEnB,MAAM,SAAS;EAAE,GAAG;EAAS,YAAY,gBAAgB,QAAQ,UAAU;CAAE;CAC7E,OAAO;EACL,MAAM,OAAO;EACb,cAAc,OAAO;EACrB,YAAY,OAAO;EACnB,WAAW,OAAO;EAClB,MAAM,MAAM,EAAE,UAAU,UAAU,QAAQ,YAAY,WAAW,YAAY;GAC3E,OAAO,eAAe;GACtB,MAAM,QAAQ;IAAE;IAAU;GAAS;GACnC,MAAM,SAAS,MAAM,OAAO,SAAS,OAAO;IAC1C;IACA;IACA,OAAO,OAAO;IACd,SAAS;IACT;IACA,UAAU;KAAE,GAAG;KAAU,YAAY,SAAS;IAAG;GACnD,CAAC;GACD,MAAM,OAAO,SAAS,QAAQ,KAAK;GACnC,OAAO,eAAe;GACtB,MAAM,QAAQ,MAAM,OAAO,IAAI,OAAO,OAAO,KAAK;GAClD,OAAO,eAAe;GACtB,OAAO;EACT;CACF;AACF;;AAkBA,SAAgB,UAAgB,SAAqD;CACnF,MAAM,SAAS;EAAE,GAAG;EAAS,MAAM,gBAAgB,QAAQ,IAAI;CAAE;CACjE,OAAO;EACL,IAAI,OAAO;EACX,SAAS,OAAO;EAChB,aAAa,OAAO;EACpB,WAAW,OAAO;EAClB,MAAM,OAAO;EACb,MAAM,QAAQ,OAAO,SAAS;GAC5B,MAAM,UAAU,QAAQ,SAAS,CAAC,QAAQ,MAAM,IAAI,CAAC;GACrD,IAAI,QAAQ,eAAe,KAAA,GAAW;IACpC,MAAM,YAAY,QAAQ,aAAa,KAAK,IAAI;IAChD,IAAI,CAAC,OAAO,SAAS,SAAS,GAAG,MAAM,IAAI,UAAU,iCAAiC;IACtF,IAAI,aAAa,GAAG,MAAM,IAAI,aAAa,6BAA6B,cAAc;IACtF,QAAQ,KAAK,YAAY,QAAQ,KAAK,KAAK,SAAS,CAAC,CAAC;GACxD;GACA,MAAM,SAAS,YAAY,IAAI,OAAO;GACtC,OAAO,eAAe;GACtB,MAAM,iBAAiB;IAAE,GAAG;IAAS;GAAO;GAC5C,MAAM,SAAS,MAAM,OAAO,SAC1B,OACA;IACE;IAEA,YACE,QAAQ,eACP,QAAQ,cAAc,KAAA,IACnB,KAAA,IACA,IAAI,WAAW,EAAE,gBAAgB,QAAQ,UAAU,CAAC;IAC1D,OAAO,OAAO;IACd,SAAS;IACT,WAAW,QAAQ;IACnB,UAAU;KAAE,GAAG,QAAQ;KAAM,OAAO,QAAQ;KAAO,eAAe,QAAQ;IAAc;IACxF,YAAY,YACV,QAAQ,cAAc;KACpB,OAAO;KACP,QAAQ,QAAQ,eACZ,OACA;MAAE,OAAO,QAAQ;MAAa,QAAQ,QAAQ;KAAa;KAC/D,MAAM,QAAQ,cACV;MAAE,MAAM;MAAc,KAAK;KAAK,IAChC;MACE,MAAM,QAAQ,kBAAkB,KAAA,IAAY,cAAc;MAC1D,KAAK,QAAQ;KACf;IACN,CAAC;GACL,GACA,cACF;GACA,MAAM,OAAO,SAAS,QAAQ,OAAO,cAAc;GACnD,OAAO,eAAe;GACtB,MAAM,WAAW,MAAM,OAAO,IAAI,OAAO,OAAO,OAAO,cAAc;GACrE,OAAO,eAAe;GACtB,OAAO;EACT;CACF;AACF"}
|
|
1
|
+
{"version":3,"file":"evaluation.js","names":[],"sources":["../src/evaluation.ts"],"sourcesContent":["import type { Analyst, AnalystContext, AnalystFinding } from './analyst/types'\nimport type { JudgeConfig, JudgeScore, Scenario } from './campaign/types'\nimport type {\n CostChannel,\n CostLedgerHandle,\n CostReceipt,\n CostReceiptInput,\n MaximumCharge,\n} from './cost-ledger'\nimport { CostLedger } from './cost-ledger'\n\nexport interface EvaluationResult<T> {\n value: T\n receipt: CostReceipt\n durationMs: number\n}\n\nexport interface EvaluationContext {\n /** Physical paid-call identity, forwarded through the existing ledger and transport.\n * Reusing it in one ledger conflicts; it is not an automatic result-replay policy. */\n callId?: string\n signal?: AbortSignal\n costLedger?: CostLedgerHandle\n actor?: string\n channel?: CostChannel\n costPhase?: string\n costTags?: Record<string, string>\n maximumCharge?: MaximumCharge\n /** Awaited after accounting and before validation or result delivery. */\n onReceipt?: (receipt: CostReceipt) => void | Promise<void>\n}\n\nexport type CostedEvaluator<I, O> = (\n input: I,\n context?: EvaluationContext,\n) => Promise<EvaluationResult<O>>\n\nexport interface EvaluatorOptions<I, O> {\n execute: (input: I, context: { signal: AbortSignal; idempotencyKey: string }) => Promise<O>\n receipt: (value: O) => CostReceiptInput\n receiptFromError?: (error: Error) => CostReceiptInput | undefined\n model?: string | ((input: I) => string)\n costLedger?: CostLedgerHandle\n maximumCharge?: MaximumCharge\n /** Awaited after accounting, so invalid output never erases paid work. */\n validate?: (value: O, input: I) => void | Promise<void>\n}\n\n/** A metered function, independent of providers, question formats, and application policy. */\nexport function createEvaluator<I, O>(options: EvaluatorOptions<I, O>): CostedEvaluator<I, O> {\n const defaults = { ...options }\n const ledger = defaults.costLedger ?? new CostLedger()\n return async (input, context = {}) => {\n context.signal?.throwIfAborted()\n const started = performance.now()\n const paid = await (context.costLedger ?? ledger).runPaidCall({\n callId: context.callId,\n actor: context.actor ?? 'evaluation',\n channel: context.channel ?? 'evaluation',\n phase: context.costPhase ?? 'evaluation',\n tags: context.costTags,\n model: typeof defaults.model === 'function' ? defaults.model(input) : defaults.model,\n signal: context.signal,\n maximumCharge: context.maximumCharge ?? defaults.maximumCharge,\n execute: (signal, idempotencyKey) => defaults.execute(input, { signal, idempotencyKey }),\n receipt: defaults.receipt,\n receiptFromError: defaults.receiptFromError,\n })\n if (paid.receipt) await context.onReceipt?.(paid.receipt)\n if (!paid.succeeded) throw paid.error\n // A transport can complete after cancellation. Keep its paid receipt, not a live decision.\n context.signal?.throwIfAborted()\n await defaults.validate?.(paid.value, input)\n context.signal?.throwIfAborted()\n return { value: paid.value, receipt: paid.receipt, durationMs: performance.now() - started }\n }\n}\n\nexport interface EvaluationJudgeOptions<A, S extends Scenario, O> {\n name: string\n version: string\n dimensions: JudgeConfig<A, S>['dimensions']\n evaluate: CostedEvaluator<{ artifact: A; scenario: S }, O>\n map: (value: O, input: { artifact: A; scenario: S }) => JudgeScore | Promise<JudgeScore>\n /** Persist the complete observation before reducing it to a verdict. */\n record?: (\n result: EvaluationResult<O>,\n input: { artifact: A; scenario: S },\n ) => void | Promise<void>\n appliesTo?: (scenario: S) => boolean\n}\n\n/** Return the existing contract. Mapping never performs another inference call. */\nexport function asJudge<A, S extends Scenario, O>(\n options: EvaluationJudgeOptions<A, S, O>,\n): JudgeConfig<A, S> {\n // The version describes these callbacks, not later mutations of the caller's options.\n const config = { ...options, dimensions: structuredClone(options.dimensions) }\n return {\n name: config.name,\n judgeVersion: config.version,\n dimensions: config.dimensions,\n appliesTo: config.appliesTo,\n async score({ artifact, scenario, signal, costLedger, costPhase, costTags }) {\n signal.throwIfAborted()\n const input = { artifact, scenario }\n const result = await config.evaluate(input, {\n signal,\n costLedger,\n actor: config.name,\n channel: 'judge',\n costPhase,\n costTags: { ...costTags, scenarioId: scenario.id },\n })\n await config.record?.(result, input)\n signal.throwIfAborted()\n const score = await config.map(result.value, input)\n signal.throwIfAborted()\n return score\n },\n }\n}\n\nexport interface EvaluationAnalystOptions<I, O> {\n id: string\n version: string\n description: string\n inputKind: Analyst<I>['inputKind']\n cost: Analyst<I>['cost']\n evaluate: (\n input: I,\n context: EvaluationContext,\n analystContext: AnalystContext,\n ) => Promise<EvaluationResult<O>>\n map: (value: O, input: I, context: AnalystContext) => AnalystFinding[] | Promise<AnalystFinding[]>\n record?: (result: EvaluationResult<O>, input: I, context: AnalystContext) => void | Promise<void>\n}\n\n/** Registry, graph, and trace consumers keep the same Analyst interface. */\nexport function asAnalyst<I, O>(options: EvaluationAnalystOptions<I, O>): Analyst<I> {\n const config = { ...options, cost: structuredClone(options.cost) }\n return {\n id: config.id,\n version: config.version,\n description: config.description,\n inputKind: config.inputKind,\n cost: config.cost,\n async analyze(input, context) {\n const signals = context.signal ? [context.signal] : []\n if (context.deadlineMs !== undefined) {\n const remaining = context.deadlineMs - Date.now()\n if (!Number.isFinite(remaining)) throw new TypeError('Analyst deadline must be finite')\n if (remaining <= 0) throw new DOMException('Analyst deadline exceeded', 'TimeoutError')\n signals.push(AbortSignal.timeout(Math.ceil(remaining)))\n }\n const signal = AbortSignal.any(signals)\n signal.throwIfAborted()\n const analystContext = { ...context, signal }\n const result = await config.evaluate(\n input,\n {\n signal,\n // No implicit account may override the evaluator's configured spending authority.\n costLedger:\n context.costLedger ??\n (context.budgetUsd === undefined\n ? undefined\n : new CostLedger({ costCeilingUsd: context.budgetUsd })),\n actor: config.id,\n channel: 'analyst',\n costPhase: context.costPhase,\n costTags: { ...context.tags, runId: context.runId, correlationId: context.correlationId },\n onReceipt: (receipt) =>\n context.recordUsage?.({\n calls: 1,\n tokens: receipt.usageUnknown\n ? null\n : { input: receipt.inputTokens, output: receipt.outputTokens },\n cost: receipt.costUnknown\n ? { kind: 'uncaptured', usd: null }\n : {\n kind: receipt.actualCostUsd === undefined ? 'estimated' : 'observed',\n usd: receipt.costUsd,\n },\n }),\n },\n analystContext,\n )\n await config.record?.(result, input, analystContext)\n signal.throwIfAborted()\n const findings = await config.map(result.value, input, analystContext)\n signal.throwIfAborted()\n return findings\n },\n }\n}\n"],"mappings":";;;AAiDA,SAAgB,gBAAsB,SAAwD;CAC5F,MAAM,WAAW,EAAE,GAAG,QAAQ;CAC9B,MAAM,SAAS,SAAS,cAAc,IAAI,WAAW;CACrD,OAAO,OAAO,OAAO,UAAU,CAAC,MAAM;EACpC,QAAQ,QAAQ,eAAe;EAC/B,MAAM,UAAU,YAAY,IAAI;EAChC,MAAM,OAAO,OAAO,QAAQ,cAAc,OAAA,CAAQ,YAAY;GAC5D,QAAQ,QAAQ;GAChB,OAAO,QAAQ,SAAS;GACxB,SAAS,QAAQ,WAAW;GAC5B,OAAO,QAAQ,aAAa;GAC5B,MAAM,QAAQ;GACd,OAAO,OAAO,SAAS,UAAU,aAAa,SAAS,MAAM,KAAK,IAAI,SAAS;GAC/E,QAAQ,QAAQ;GAChB,eAAe,QAAQ,iBAAiB,SAAS;GACjD,UAAU,QAAQ,mBAAmB,SAAS,QAAQ,OAAO;IAAE;IAAQ;GAAe,CAAC;GACvF,SAAS,SAAS;GAClB,kBAAkB,SAAS;EAC7B,CAAC;EACD,IAAI,KAAK,SAAS,MAAM,QAAQ,YAAY,KAAK,OAAO;EACxD,IAAI,CAAC,KAAK,WAAW,MAAM,KAAK;EAEhC,QAAQ,QAAQ,eAAe;EAC/B,MAAM,SAAS,WAAW,KAAK,OAAO,KAAK;EAC3C,QAAQ,QAAQ,eAAe;EAC/B,OAAO;GAAE,OAAO,KAAK;GAAO,SAAS,KAAK;GAAS,YAAY,YAAY,IAAI,IAAI;EAAQ;CAC7F;AACF;;AAiBA,SAAgB,QACd,SACmB;CAEnB,MAAM,SAAS;EAAE,GAAG;EAAS,YAAY,gBAAgB,QAAQ,UAAU;CAAE;CAC7E,OAAO;EACL,MAAM,OAAO;EACb,cAAc,OAAO;EACrB,YAAY,OAAO;EACnB,WAAW,OAAO;EAClB,MAAM,MAAM,EAAE,UAAU,UAAU,QAAQ,YAAY,WAAW,YAAY;GAC3E,OAAO,eAAe;GACtB,MAAM,QAAQ;IAAE;IAAU;GAAS;GACnC,MAAM,SAAS,MAAM,OAAO,SAAS,OAAO;IAC1C;IACA;IACA,OAAO,OAAO;IACd,SAAS;IACT;IACA,UAAU;KAAE,GAAG;KAAU,YAAY,SAAS;IAAG;GACnD,CAAC;GACD,MAAM,OAAO,SAAS,QAAQ,KAAK;GACnC,OAAO,eAAe;GACtB,MAAM,QAAQ,MAAM,OAAO,IAAI,OAAO,OAAO,KAAK;GAClD,OAAO,eAAe;GACtB,OAAO;EACT;CACF;AACF;;AAkBA,SAAgB,UAAgB,SAAqD;CACnF,MAAM,SAAS;EAAE,GAAG;EAAS,MAAM,gBAAgB,QAAQ,IAAI;CAAE;CACjE,OAAO;EACL,IAAI,OAAO;EACX,SAAS,OAAO;EAChB,aAAa,OAAO;EACpB,WAAW,OAAO;EAClB,MAAM,OAAO;EACb,MAAM,QAAQ,OAAO,SAAS;GAC5B,MAAM,UAAU,QAAQ,SAAS,CAAC,QAAQ,MAAM,IAAI,CAAC;GACrD,IAAI,QAAQ,eAAe,KAAA,GAAW;IACpC,MAAM,YAAY,QAAQ,aAAa,KAAK,IAAI;IAChD,IAAI,CAAC,OAAO,SAAS,SAAS,GAAG,MAAM,IAAI,UAAU,iCAAiC;IACtF,IAAI,aAAa,GAAG,MAAM,IAAI,aAAa,6BAA6B,cAAc;IACtF,QAAQ,KAAK,YAAY,QAAQ,KAAK,KAAK,SAAS,CAAC,CAAC;GACxD;GACA,MAAM,SAAS,YAAY,IAAI,OAAO;GACtC,OAAO,eAAe;GACtB,MAAM,iBAAiB;IAAE,GAAG;IAAS;GAAO;GAC5C,MAAM,SAAS,MAAM,OAAO,SAC1B,OACA;IACE;IAEA,YACE,QAAQ,eACP,QAAQ,cAAc,KAAA,IACnB,KAAA,IACA,IAAI,WAAW,EAAE,gBAAgB,QAAQ,UAAU,CAAC;IAC1D,OAAO,OAAO;IACd,SAAS;IACT,WAAW,QAAQ;IACnB,UAAU;KAAE,GAAG,QAAQ;KAAM,OAAO,QAAQ;KAAO,eAAe,QAAQ;IAAc;IACxF,YAAY,YACV,QAAQ,cAAc;KACpB,OAAO;KACP,QAAQ,QAAQ,eACZ,OACA;MAAE,OAAO,QAAQ;MAAa,QAAQ,QAAQ;KAAa;KAC/D,MAAM,QAAQ,cACV;MAAE,MAAM;MAAc,KAAK;KAAK,IAChC;MACE,MAAM,QAAQ,kBAAkB,KAAA,IAAY,cAAc;MAC1D,KAAK,QAAQ;KACf;IACN,CAAC;GACL,GACA,cACF;GACA,MAAM,OAAO,SAAS,QAAQ,OAAO,cAAc;GACnD,OAAO,eAAe;GACtB,MAAM,WAAW,MAAM,OAAO,IAAI,OAAO,OAAO,OAAO,cAAc;GACrE,OAAO,eAAe;GACtB,OAAO;EACT;CACF;AACF"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { c as AnalystRunEvent, d as AnalystRunSummary, t as Analyst, u as AnalystRunResult } from "./types-CW8ewATZ.js";
|
|
2
2
|
import { z } from "zod";
|
|
3
3
|
//#region src/analyst/exact-types.d.ts
|
|
4
4
|
/** Analyst metadata required before the exact registry path will execute it. */
|
|
@@ -231,4 +231,4 @@ declare const exactExecutionPlanSchema: z.ZodObject<{
|
|
|
231
231
|
}, z.core.$strict>;
|
|
232
232
|
//#endregion
|
|
233
233
|
export { ExactAnalystRunPolicySnapshot as a, ExactAnalystSnapshot as c, ExactExecutionComponentSnapshot as d, ExactAnalystRunEvent as i, ExactCapableAnalyst as l, ExactAnalystExecutionPlanSnapshot as n, ExactAnalystRunResult as o, ExactAnalystRunCompletion as r, ExactAnalystRunSummary as s, ExactAnalystBudgetSnapshot as t, ExactExecutionComponentIdentity as u };
|
|
234
|
-
//# sourceMappingURL=exact-types-
|
|
234
|
+
//# sourceMappingURL=exact-types-CflbJ9f3.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"exact-types-
|
|
1
|
+
{"version":3,"file":"exact-types-CflbJ9f3.d.ts","names":[],"sources":["../src/analyst/exact-types.ts"],"mappings":";;;;UAMiB,oBAAoB,0BAA0B,QAAQ;;WAE5D,iBAAiB,SAAS;;KAGhC,aAAa,KAAK,0BAA0B,mBACpC,aAAa,UACtB,+BACc,aAAa,IAAI,aAAa,EAAE,WAC5C;KAEM,uBAAuB,aAAa,EAAE,aAAa;KACnD,6BAA6B,aAAa,EAAE,aAAa;;UAGpD;EACf;EACA;EACA,QAAQ,SAAS;;;UAIF;EACf;EACA;EACA;;KAyBU,gCAAgC,aAAa,EAAE,aAAa;KAC5D,oCAAoC,aAC9C,EAAE,aAAa;KAGL,yBAAyB;;EAEnC;;KAGU;EACN;;EAEA;EACA;IAAS;IAAe;;;UAGb,8BAA8B;EAC7C,aAAa;EACb,gBAAgB;EAChB,YAAY;;;KAIF,wBACP,QAAQ;EAAmB;;EAC1B,gBAAgB;MAEjB,KAAK,QAAQ;EAAmB;;EAC/B,SAAS;KAEX,QAAQ;EAAmB;MAC1B,KAAK,QAAQ;EAAmB;;EAC/B,SAAS;MAEV,KAAK,QAAQ;EAAmB;;EAC/B,QAAQ;;cAwCR,uBAAqB,EAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAOzB,EAAA,KAAA;cAII,sBAAoB,EAAA,uBAAA,EAAA;;;;;;;;;;;GAaxB,EAAA,KAAA;cAiBI,sBAAoB,EAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAYxB,EAAA,KAAA;cAEI,0BAAwB,EAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+E1B,EAAA,KAAA"}
|
|
@@ -168,28 +168,6 @@ function finalRecordedOutcome(steps) {
|
|
|
168
168
|
reason: kind
|
|
169
169
|
};
|
|
170
170
|
}
|
|
171
|
-
/**
|
|
172
|
-
* Steps whose recorded exit a replay cannot check.
|
|
173
|
-
*
|
|
174
|
-
* A killed step counts: the recording holds no exit status to compare a replay
|
|
175
|
-
* against, so agreement on it cannot be measured either way.
|
|
176
|
-
*/
|
|
177
|
-
function unreadableExitCount(steps) {
|
|
178
|
-
return steps.filter((step) => classifyObservation(step.observation) !== "command-result").length;
|
|
179
|
-
}
|
|
180
|
-
/**
|
|
181
|
-
* Throw unless every recorded command survived the dump.
|
|
182
|
-
*
|
|
183
|
-
* A replay executes `steps` verbatim, so one elision marker in an action means
|
|
184
|
-
* the replay runs the literal two-to-four characters of the marker instead of
|
|
185
|
-
* the command the run executed. The guard is here so a replayer needs one call
|
|
186
|
-
* rather than a field check it can forget.
|
|
187
|
-
*/
|
|
188
|
-
function assertReplayableTrajectory(decoded) {
|
|
189
|
-
if (decoded.elidedCommands === 0) return;
|
|
190
|
-
const markers = decoded.steps.filter((step) => isElidedField(step.action)).map((step) => step.action);
|
|
191
|
-
throw new Error(`trajectory holds ${decoded.elidedCommands} command(s) the dump elided (${markers.slice(0, 5).join(", ")}); no dictionary maps a marker back to its text, so this trajectory cannot be replayed`);
|
|
192
|
-
}
|
|
193
171
|
//#endregion
|
|
194
172
|
//#region src/trajectory-replay/exec.ts
|
|
195
173
|
/**
|
|
@@ -203,6 +181,6 @@ function wrapActionForExec(action, cwd) {
|
|
|
203
181
|
return `cd ${`'${cwd.replaceAll("'", `'\\''`)}'`} && printf %s ${b64} | base64 -d | sh`;
|
|
204
182
|
}
|
|
205
183
|
//#endregion
|
|
206
|
-
export {
|
|
184
|
+
export { classifyObservation as a, finalRecordedOutcome as c, isSubmitOnlyAction as d, parseObservationOutput as f, TIMEOUT_OBSERVATION_MARKER as i, isRecordedTimeout as l, FORMAT_ERROR_OBSERVATION_PREFIX as n, decodeRecordedTurns as o, parseRecordedReturncode as p, SUBMIT_ACTION_SIGNATURE as r, deriveFailureSignature as s, wrapActionForExec as t, isSubmitAction as u };
|
|
207
185
|
|
|
208
|
-
//# sourceMappingURL=exec-
|
|
186
|
+
//# sourceMappingURL=exec-CpbtOXVt.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"exec-ChJ6d5if.js","names":[],"sources":["../src/trajectory-replay/steps.ts","../src/trajectory-replay/exec.ts"],"sourcesContent":["/**\n * Recorded shell-trajectory steps and the observation grammar they carry.\n *\n * A recorded trajectory is the action/observation sequence an agent actually\n * ran. Scaffolds that execute one shell command per step (mini-SWE and the\n * CodeTracer-normalized corpora built from it) tag each observation with the\n * command's returncode and its combined output:\n *\n * <returncode>2</returncode>\n * <output>\n * …command output…\n * </output>\n *\n * That is one of four shapes a recorded turn carries, and the other three are\n * not command results at all:\n *\n * - a timeout notice, when the environment killed the command at its bound;\n * - a format-error notice, when the scaffold rejected the turn and ran\n * nothing;\n * - an elision marker `$<hex>`, when the published dump dropped the string.\n *\n * A turn also carries no observation when it is the run's last turn, because\n * the scaffold records an observation only when it hands one back to the model.\n *\n * The parsers here are the only place that grammar is decoded. Everything\n * downstream — replay verdicts, corpus enumeration, admission funnels, fix\n * prompts — reads the returncode, the output, and the failure signature through\n * these functions. A second decoder elsewhere is how a corpus reads as\n * unreplayable when it is not.\n */\n\n/**\n * One step of a recorded shell trajectory. Structural: any richer step record\n * (file refs, thinking text, tool type) satisfies it.\n */\nexport interface RecordedTrajectoryStep {\n /** 1-based position in the trajectory. */\n readonly step_id: number\n readonly action: string\n /** Null when the step recorded no observation (terminal submit steps). */\n readonly observation: string | null\n}\n\n/** Recorded returncode of a step, or null when the observation carries none. */\nexport function parseRecordedReturncode(observation: string | null): number | null {\n if (!observation) return null\n const m = /<returncode>(-?\\d+)<\\/returncode>/.exec(observation)\n return m ? Number(m[1]) : null\n}\n\n/** Text between the observation's <output> tags, or the raw observation when\n * the tags are absent. */\nexport function parseObservationOutput(observation: string | null): string {\n if (!observation) return ''\n const m = /<output>\\n?([\\s\\S]*?)\\n?<\\/output>/.exec(observation)\n return m ? m[1]! : observation\n}\n\n/**\n * Stable failure-signature candidate: the first error line, or a shell\n * command-not-found suffix. Null when neither exists — a verdict then falls\n * back to returncode-only matching and says so.\n * Pass an explicit signature to override (compiler quote glyphs vary with\n * locale, so a hand-picked ASCII substring is often more robust).\n */\nexport function deriveFailureSignature(observation: string | null): string | null {\n for (const line of parseObservationOutput(observation).split('\\n')) {\n if (/\\berror\\b/i.test(line)) return line.trim().slice(0, 200)\n // The shell prefix differs between recorded bash and replayed /bin/sh.\n const missingCommand = /([A-Za-z0-9_.+/-]+: (?:command )?not found)\\b/.exec(line)\n if (missingCommand) return missingCommand[1]!.slice(0, 200)\n }\n return null\n}\n\n/** mini-SWE's end-of-run submit convention: the agent echoes this sentinel\n * and dumps the diff. A label on this step marks a bad SUBMIT DECISION, not a\n * failed command — there is no executable failure to reproduce, so it is\n * never a counterfactual replay target. */\nexport const SUBMIT_ACTION_SIGNATURE = 'COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT'\n\nexport function isSubmitAction(action: string): boolean {\n return action.includes(SUBMIT_ACTION_SIGNATURE)\n}\n\n/**\n * True when the action is the sentinel echo and nothing else.\n *\n * The distinction decides whether a step may be dropped. An agent is told to\n * issue the sentinel alone, and 5.7% of recorded runs end on a command that\n * writes files or edits them and then echoes it. Dropping such a step because\n * it holds the sentinel would remove the run's last state change from the\n * replay, so the recorded end state and the replayed one would differ.\n */\nexport function isSubmitOnlyAction(action: string): boolean {\n return /^echo\\s+(?:\"COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT\"|'COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT'|COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT)$/.test(\n action.trim(),\n )\n}\n\n// ── Fields the published dump dropped ────────────────────────────────\n\n/**\n * A field the dump replaced with a counter instead of its text.\n *\n * The counter is hexadecimal and rises by one per dropped string in document\n * order, so a decimal-only reading (`$12`) accepts every marker that carries a\n * letter (`$3a`) as if it were real text. A command read that way replays as\n * the literal two-to-four characters `$3a`, which is not the command the run\n * executed.\n *\n * Nothing in the dump maps a marker back to its text: the same marker carries\n * different content in different rows, so there is no dictionary to read. A\n * field that matches is unrecoverable, so a caller rejects the row that holds\n * it rather than replaying the marker.\n */\nconst RECORDED_ELISION_PATTERN = /^\\$[0-9a-f]+$/\n\nexport function isElidedField(value: string | null | undefined): boolean {\n return typeof value === 'string' && RECORDED_ELISION_PATTERN.test(value)\n}\n\n// ── Observation grammar ──────────────────────────────────────────────\n\n/** Substring the timeout notice always carries, whatever the command was. */\nexport const TIMEOUT_OBSERVATION_MARKER = 'timed out and has been killed'\n\n/** Opening of the notice the scaffold writes when a turn held no single action. */\nexport const FORMAT_ERROR_OBSERVATION_PREFIX =\n 'Please always provide EXACTLY ONE action in triple backticks, found '\n\n/**\n * What a recorded observation is.\n *\n * `command-result` is the only kind that carries an exit status. `timeout` and\n * `format-error` are the scaffold speaking rather than a command: the first\n * says the environment killed the command, the second says no command ran at\n * all. `elided` and `absent` carry no information about the step, and\n * `unreadable` is a shape this grammar does not know — never assumed to be\n * anything.\n */\nexport type RecordedObservationKind =\n | 'command-result'\n | 'timeout'\n | 'format-error'\n | 'elided'\n | 'absent'\n | 'unreadable'\n\nexport function classifyObservation(observation: string | null): RecordedObservationKind {\n if (observation === null) return 'absent'\n if (isElidedField(observation)) return 'elided'\n if (parseRecordedReturncode(observation) !== null) return 'command-result'\n if (observation.startsWith(FORMAT_ERROR_OBSERVATION_PREFIX)) return 'format-error'\n if (observation.includes(TIMEOUT_OBSERVATION_MARKER)) return 'timeout'\n return 'unreadable'\n}\n\n/** True when the recording shows the environment killed this step at its\n * wall-clock bound. Such a step carries no returncode, so no replay can\n * confirm or contradict it. */\nexport function isRecordedTimeout(observation: string | null): boolean {\n return classifyObservation(observation) === 'timeout'\n}\n\n// ── Turns to steps ───────────────────────────────────────────────────\n\n/**\n * One turn as the published trajectory dump holds it.\n *\n * A turn is not a step: the system prompt, the task statement and every turn\n * the scaffold rejected are turns that executed nothing.\n */\nexport interface RecordedTrajectoryTurn {\n readonly src?: string | null\n readonly msg?: string | null\n readonly tools?: readonly { readonly cmd?: string | null }[] | null\n readonly obs?: string | null\n}\n\nexport interface DecodedTrajectory {\n /** Executed commands in recorded order, each holding its OWN observation. */\n readonly steps: readonly RecordedTrajectoryStep[]\n /** Turns the scaffold rejected before anything ran. */\n readonly formatErrorTurns: number\n /**\n * Executed commands whose text the dump dropped.\n *\n * The step stays in `steps` carrying the marker, because the run did execute\n * a command there and a shorter list would misreport the trajectory. The\n * marker is not a command, so any count above zero means this trajectory\n * cannot be replayed: gate on it, or call `assertReplayableTrajectory`.\n */\n readonly elidedCommands: number\n /** True when the run's last turn was the submit sentinel with no observation. */\n readonly endedOnSubmitSentinel: boolean\n /** Turns carrying an observation this grammar cannot read. */\n readonly unreadableTurns: number\n}\n\n/**\n * Pair every recorded command with the observation of its own turn.\n *\n * Collecting commands and observations into two lists and zipping them is the\n * decode that looks right and is not: a turn the scaffold rejected carries an\n * observation of its own, so from the first such turn onward every observation\n * belongs to a different command than the one it is read against.\n *\n * A rejected turn executed nothing, so it is never a step — including when the\n * dump recorded a command for it. The scaffold rejects a turn holding several\n * bash blocks and runs none of them, while the dump keeps one of the blocks in\n * the command field. Replaying that field would execute a command the recorded\n * run did not, which is a worse corpus than a smaller one.\n *\n * A rejected turn is recognised by its observation, so a rejected turn whose\n * observation the dump elided is indistinguishable from an executed command\n * whose observation it elided. Both read as a step. Nothing in the dump\n * separates them, and the row-level defence is the share of unreadable exits a\n * caller admits: a row with no elided observation cannot hold this case at all.\n *\n * A trailing step that echoes the sentinel and nothing else, with no\n * observation, is dropped from `steps` and reported as `endedOnSubmitSentinel`.\n * The scaffold records an observation only when it hands one to the model, and\n * the sentinel ends the run, so the missing observation is the end of the\n * transcript rather than a gap in it. Echoing the sentinel changes no state, so\n * the recorded end state is the state the step before it left.\n *\n * A step that DID get an observation stays a step: the run continued past it.\n * So does a step that echoes the sentinel after doing real work — its state\n * change is part of the recorded end state, and with no observation its exit is\n * unknown, which `finalRecordedOutcome` reports rather than hides.\n */\nexport function decodeRecordedTurns(turns: readonly RecordedTrajectoryTurn[]): DecodedTrajectory {\n const steps: RecordedTrajectoryStep[] = []\n let formatErrorTurns = 0\n let unreadableTurns = 0\n for (const turn of turns) {\n const command = turn.tools?.[0]?.cmd ?? null\n const observation = turn.obs ?? null\n const kind = classifyObservation(observation)\n if (kind === 'format-error') {\n formatErrorTurns += 1\n continue\n }\n if (command === null) {\n if (kind === 'unreadable' || kind === 'elided') unreadableTurns += 1\n continue\n }\n steps.push({ step_id: steps.length + 1, action: command, observation })\n }\n const last = steps[steps.length - 1]\n const endedOnSubmitSentinel =\n last !== undefined && last.observation === null && isSubmitOnlyAction(last.action)\n if (endedOnSubmitSentinel) steps.pop()\n return {\n steps,\n formatErrorTurns,\n elidedCommands: steps.filter((step) => isElidedField(step.action)).length,\n endedOnSubmitSentinel,\n unreadableTurns,\n }\n}\n\n// ── The state a trajectory ended in ──────────────────────────────────\n\n/**\n * How the recorded run's last executed command ended.\n *\n * `killed` is a measured outcome, not a missing one: the environment stopped\n * the command at its bound and wrote a notice instead of an exit status.\n * `unreadable` names the observation kind that blocked the read, so a funnel\n * can report which shape cost it the row.\n */\nexport type RecordedFinalOutcome =\n | { readonly kind: 'returncode'; readonly value: number }\n | { readonly kind: 'killed' }\n | { readonly kind: 'unreadable'; readonly reason: RecordedObservationKind }\n\n/**\n * The outcome of the last step, or `null` when the trajectory has no steps.\n *\n * Reads only the last step. `decodeRecordedTurns` has already removed the turns\n * that executed nothing, so the last step is the last command the run ran.\n */\nexport function finalRecordedOutcome(\n steps: readonly RecordedTrajectoryStep[],\n): RecordedFinalOutcome | null {\n const last = steps[steps.length - 1]\n if (last === undefined) return null\n const kind = classifyObservation(last.observation)\n if (kind === 'command-result') {\n return { kind: 'returncode', value: parseRecordedReturncode(last.observation)! }\n }\n if (kind === 'timeout') return { kind: 'killed' }\n return { kind: 'unreadable', reason: kind }\n}\n\n/**\n * Steps whose recorded exit a replay cannot check.\n *\n * A killed step counts: the recording holds no exit status to compare a replay\n * against, so agreement on it cannot be measured either way.\n */\nexport function unreadableExitCount(steps: readonly RecordedTrajectoryStep[]): number {\n return steps.filter((step) => classifyObservation(step.observation) !== 'command-result').length\n}\n\n/**\n * Throw unless every recorded command survived the dump.\n *\n * A replay executes `steps` verbatim, so one elision marker in an action means\n * the replay runs the literal two-to-four characters of the marker instead of\n * the command the run executed. The guard is here so a replayer needs one call\n * rather than a field check it can forget.\n */\nexport function assertReplayableTrajectory(decoded: DecodedTrajectory): void {\n if (decoded.elidedCommands === 0) return\n const markers = decoded.steps\n .filter((step) => isElidedField(step.action))\n .map((step) => step.action)\n throw new Error(\n `trajectory holds ${decoded.elidedCommands} command(s) the dump elided (${markers.slice(0, 5).join(', ')}); ` +\n 'no dictionary maps a marker back to its text, so this trajectory cannot be replayed',\n )\n}\n","/**\n * The execution boundary replay runs across.\n *\n * A replay needs one thing from its environment: a session that runs a shell\n * command inside the trajectory's own image and reports the exit code and\n * output. That is the whole contract. Concrete backends — a sandbox platform\n * client, a docker exec, an SSH shell — live with the consumer that owns the\n * infrastructure, so this package depends on no sandbox client.\n */\n\nexport interface ReplayExecResult {\n exitCode: number\n stdout: string\n stderr: string\n}\n\nexport interface ReplayExecSession {\n exec(command: string, timeoutMs: number): Promise<ReplayExecResult>\n close(): Promise<void>\n}\n\nexport interface ReplayExecBackend {\n /** One fresh execution environment per call; the caller closes it. */\n open(): Promise<ReplayExecSession>\n}\n\n/** Builds a backend pinned to one image. Callers that resolve images\n * internally (batch, corpus wire, finding verification) take this instead of\n * a backend, so every case runs against its own image. */\nexport type ReplayExecBackendFactory = (image: string) => ReplayExecBackend\n\n/**\n * mini-SWE runs every action as a fresh /bin/sh subshell from a fixed\n * workdir. Reproduce that exactly — and stay quote-proof for arbitrary\n * recorded actions — by piping the base64 of the action into `sh` after\n * cd-ing to the workdir. Exit code is sh's, i.e. the action's.\n */\nexport function wrapActionForExec(action: string, cwd: string): string {\n const b64 = Buffer.from(action, 'utf8').toString('base64')\n const quotedCwd = `'${cwd.replaceAll(\"'\", `'\\\\''`)}'`\n return `cd ${quotedCwd} && printf %s ${b64} | base64 -d | sh`\n}\n"],"mappings":";;AA4CA,SAAgB,wBAAwB,aAA2C;CACjF,IAAI,CAAC,aAAa,OAAO;CACzB,MAAM,IAAI,oCAAoC,KAAK,WAAW;CAC9D,OAAO,IAAI,OAAO,EAAE,EAAE,IAAI;AAC5B;;;AAIA,SAAgB,uBAAuB,aAAoC;CACzE,IAAI,CAAC,aAAa,OAAO;CACzB,MAAM,IAAI,qCAAqC,KAAK,WAAW;CAC/D,OAAO,IAAI,EAAE,KAAM;AACrB;;;;;;;;AASA,SAAgB,uBAAuB,aAA2C;CAChF,KAAK,MAAM,QAAQ,uBAAuB,WAAW,CAAC,CAAC,MAAM,IAAI,GAAG;EAClE,IAAI,aAAa,KAAK,IAAI,GAAG,OAAO,KAAK,KAAK,CAAC,CAAC,MAAM,GAAG,GAAG;EAE5D,MAAM,iBAAiB,gDAAgD,KAAK,IAAI;EAChF,IAAI,gBAAgB,OAAO,eAAe,EAAE,CAAE,MAAM,GAAG,GAAG;CAC5D;CACA,OAAO;AACT;;;;;AAMA,MAAa,0BAA0B;AAEvC,SAAgB,eAAe,QAAyB;CACtD,OAAO,OAAO,SAAS,uBAAuB;AAChD;;;;;;;;;;AAWA,SAAgB,mBAAmB,QAAyB;CAC1D,OAAO,qIAAqI,KAC1I,OAAO,KAAK,CACd;AACF;;;;;;;;;;;;;;;AAkBA,MAAM,2BAA2B;AAEjC,SAAgB,cAAc,OAA2C;CACvE,OAAO,OAAO,UAAU,YAAY,yBAAyB,KAAK,KAAK;AACzE;;AAKA,MAAa,6BAA6B;;AAG1C,MAAa,kCACX;AAoBF,SAAgB,oBAAoB,aAAqD;CACvF,IAAI,gBAAgB,MAAM,OAAO;CACjC,IAAI,cAAc,WAAW,GAAG,OAAO;CACvC,IAAI,wBAAwB,WAAW,MAAM,MAAM,OAAO;CAC1D,IAAI,YAAY,WAAA,sEAA0C,GAAG,OAAO;CACpE,IAAI,YAAY,SAAA,+BAAmC,GAAG,OAAO;CAC7D,OAAO;AACT;;;;AAKA,SAAgB,kBAAkB,aAAqC;CACrE,OAAO,oBAAoB,WAAW,MAAM;AAC9C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqEA,SAAgB,oBAAoB,OAA6D;CAC/F,MAAM,QAAkC,CAAC;CACzC,IAAI,mBAAmB;CACvB,IAAI,kBAAkB;CACtB,KAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,UAAU,KAAK,QAAQ,EAAE,EAAE,OAAO;EACxC,MAAM,cAAc,KAAK,OAAO;EAChC,MAAM,OAAO,oBAAoB,WAAW;EAC5C,IAAI,SAAS,gBAAgB;GAC3B,oBAAoB;GACpB;EACF;EACA,IAAI,YAAY,MAAM;GACpB,IAAI,SAAS,gBAAgB,SAAS,UAAU,mBAAmB;GACnE;EACF;EACA,MAAM,KAAK;GAAE,SAAS,MAAM,SAAS;GAAG,QAAQ;GAAS;EAAY,CAAC;CACxE;CACA,MAAM,OAAO,MAAM,MAAM,SAAS;CAClC,MAAM,wBACJ,SAAS,KAAA,KAAa,KAAK,gBAAgB,QAAQ,mBAAmB,KAAK,MAAM;CACnF,IAAI,uBAAuB,MAAM,IAAI;CACrC,OAAO;EACL;EACA;EACA,gBAAgB,MAAM,QAAQ,SAAS,cAAc,KAAK,MAAM,CAAC,CAAC,CAAC;EACnE;EACA;CACF;AACF;;;;;;;AAuBA,SAAgB,qBACd,OAC6B;CAC7B,MAAM,OAAO,MAAM,MAAM,SAAS;CAClC,IAAI,SAAS,KAAA,GAAW,OAAO;CAC/B,MAAM,OAAO,oBAAoB,KAAK,WAAW;CACjD,IAAI,SAAS,kBACX,OAAO;EAAE,MAAM;EAAc,OAAO,wBAAwB,KAAK,WAAW;CAAG;CAEjF,IAAI,SAAS,WAAW,OAAO,EAAE,MAAM,SAAS;CAChD,OAAO;EAAE,MAAM;EAAc,QAAQ;CAAK;AAC5C;;;;;;;AAQA,SAAgB,oBAAoB,OAAkD;CACpF,OAAO,MAAM,QAAQ,SAAS,oBAAoB,KAAK,WAAW,MAAM,gBAAgB,CAAC,CAAC;AAC5F;;;;;;;;;AAUA,SAAgB,2BAA2B,SAAkC;CAC3E,IAAI,QAAQ,mBAAmB,GAAG;CAClC,MAAM,UAAU,QAAQ,MACrB,QAAQ,SAAS,cAAc,KAAK,MAAM,CAAC,CAAC,CAC5C,KAAK,SAAS,KAAK,MAAM;CAC5B,MAAM,IAAI,MACR,oBAAoB,QAAQ,eAAe,+BAA+B,QAAQ,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,uFAE3G;AACF;;;;;;;;;AC/RA,SAAgB,kBAAkB,QAAgB,KAAqB;CACrE,MAAM,MAAM,OAAO,KAAK,QAAQ,MAAM,CAAC,CAAC,SAAS,QAAQ;CAEzD,OAAO,MAAM,IADS,IAAI,WAAW,KAAK,OAAO,EAAE,GAC5B,gBAAgB,IAAI;AAC7C"}
|
|
1
|
+
{"version":3,"file":"exec-CpbtOXVt.js","names":[],"sources":["../src/trajectory-replay/steps.ts","../src/trajectory-replay/exec.ts"],"sourcesContent":["/**\n * Recorded shell-trajectory steps and the observation grammar they carry.\n *\n * A recorded trajectory is the action/observation sequence an agent actually\n * ran. Scaffolds that execute one shell command per step (mini-SWE and the\n * CodeTracer-normalized corpora built from it) tag each observation with the\n * command's returncode and its combined output:\n *\n * <returncode>2</returncode>\n * <output>\n * …command output…\n * </output>\n *\n * That is one of four shapes a recorded turn carries, and the other three are\n * not command results at all:\n *\n * - a timeout notice, when the environment killed the command at its bound;\n * - a format-error notice, when the scaffold rejected the turn and ran\n * nothing;\n * - an elision marker `$<hex>`, when the published dump dropped the string.\n *\n * A turn also carries no observation when it is the run's last turn, because\n * the scaffold records an observation only when it hands one back to the model.\n *\n * The parsers here are the only place that grammar is decoded. Everything\n * downstream — replay verdicts, corpus enumeration, admission funnels, fix\n * prompts — reads the returncode, the output, and the failure signature through\n * these functions. A second decoder elsewhere is how a corpus reads as\n * unreplayable when it is not.\n */\n\n/**\n * One step of a recorded shell trajectory. Structural: any richer step record\n * (file refs, thinking text, tool type) satisfies it.\n */\nexport interface RecordedTrajectoryStep {\n /** 1-based position in the trajectory. */\n readonly step_id: number\n readonly action: string\n /** Null when the step recorded no observation (terminal submit steps). */\n readonly observation: string | null\n}\n\n/** Recorded returncode of a step, or null when the observation carries none. */\nexport function parseRecordedReturncode(observation: string | null): number | null {\n if (!observation) return null\n const m = /<returncode>(-?\\d+)<\\/returncode>/.exec(observation)\n return m ? Number(m[1]) : null\n}\n\n/** Text between the observation's <output> tags, or the raw observation when\n * the tags are absent. */\nexport function parseObservationOutput(observation: string | null): string {\n if (!observation) return ''\n const m = /<output>\\n?([\\s\\S]*?)\\n?<\\/output>/.exec(observation)\n return m ? m[1]! : observation\n}\n\n/**\n * Stable failure-signature candidate: the first error line, or a shell\n * command-not-found suffix. Null when neither exists — a verdict then falls\n * back to returncode-only matching and says so.\n * Pass an explicit signature to override (compiler quote glyphs vary with\n * locale, so a hand-picked ASCII substring is often more robust).\n */\nexport function deriveFailureSignature(observation: string | null): string | null {\n for (const line of parseObservationOutput(observation).split('\\n')) {\n if (/\\berror\\b/i.test(line)) return line.trim().slice(0, 200)\n // The shell prefix differs between recorded bash and replayed /bin/sh.\n const missingCommand = /([A-Za-z0-9_.+/-]+: (?:command )?not found)\\b/.exec(line)\n if (missingCommand) return missingCommand[1]!.slice(0, 200)\n }\n return null\n}\n\n/** mini-SWE's end-of-run submit convention: the agent echoes this sentinel\n * and dumps the diff. A label on this step marks a bad SUBMIT DECISION, not a\n * failed command — there is no executable failure to reproduce, so it is\n * never a counterfactual replay target. */\nexport const SUBMIT_ACTION_SIGNATURE = 'COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT'\n\nexport function isSubmitAction(action: string): boolean {\n return action.includes(SUBMIT_ACTION_SIGNATURE)\n}\n\n/**\n * True when the action is the sentinel echo and nothing else.\n *\n * The distinction decides whether a step may be dropped. An agent is told to\n * issue the sentinel alone, and 5.7% of recorded runs end on a command that\n * writes files or edits them and then echoes it. Dropping such a step because\n * it holds the sentinel would remove the run's last state change from the\n * replay, so the recorded end state and the replayed one would differ.\n */\nexport function isSubmitOnlyAction(action: string): boolean {\n return /^echo\\s+(?:\"COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT\"|'COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT'|COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT)$/.test(\n action.trim(),\n )\n}\n\n// ── Fields the published dump dropped ────────────────────────────────\n\n/**\n * A field the dump replaced with a counter instead of its text.\n *\n * The counter is hexadecimal and rises by one per dropped string in document\n * order, so a decimal-only reading (`$12`) accepts every marker that carries a\n * letter (`$3a`) as if it were real text. A command read that way replays as\n * the literal two-to-four characters `$3a`, which is not the command the run\n * executed.\n *\n * Nothing in the dump maps a marker back to its text: the same marker carries\n * different content in different rows, so there is no dictionary to read. A\n * field that matches is unrecoverable, so a caller rejects the row that holds\n * it rather than replaying the marker.\n */\nconst RECORDED_ELISION_PATTERN = /^\\$[0-9a-f]+$/\n\nexport function isElidedField(value: string | null | undefined): boolean {\n return typeof value === 'string' && RECORDED_ELISION_PATTERN.test(value)\n}\n\n// ── Observation grammar ──────────────────────────────────────────────\n\n/** Substring the timeout notice always carries, whatever the command was. */\nexport const TIMEOUT_OBSERVATION_MARKER = 'timed out and has been killed'\n\n/** Opening of the notice the scaffold writes when a turn held no single action. */\nexport const FORMAT_ERROR_OBSERVATION_PREFIX =\n 'Please always provide EXACTLY ONE action in triple backticks, found '\n\n/**\n * What a recorded observation is.\n *\n * `command-result` is the only kind that carries an exit status. `timeout` and\n * `format-error` are the scaffold speaking rather than a command: the first\n * says the environment killed the command, the second says no command ran at\n * all. `elided` and `absent` carry no information about the step, and\n * `unreadable` is a shape this grammar does not know — never assumed to be\n * anything.\n */\nexport type RecordedObservationKind =\n | 'command-result'\n | 'timeout'\n | 'format-error'\n | 'elided'\n | 'absent'\n | 'unreadable'\n\nexport function classifyObservation(observation: string | null): RecordedObservationKind {\n if (observation === null) return 'absent'\n if (isElidedField(observation)) return 'elided'\n if (parseRecordedReturncode(observation) !== null) return 'command-result'\n if (observation.startsWith(FORMAT_ERROR_OBSERVATION_PREFIX)) return 'format-error'\n if (observation.includes(TIMEOUT_OBSERVATION_MARKER)) return 'timeout'\n return 'unreadable'\n}\n\n/** True when the recording shows the environment killed this step at its\n * wall-clock bound. Such a step carries no returncode, so no replay can\n * confirm or contradict it. */\nexport function isRecordedTimeout(observation: string | null): boolean {\n return classifyObservation(observation) === 'timeout'\n}\n\n// ── Turns to steps ───────────────────────────────────────────────────\n\n/**\n * One turn as the published trajectory dump holds it.\n *\n * A turn is not a step: the system prompt, the task statement and every turn\n * the scaffold rejected are turns that executed nothing.\n */\nexport interface RecordedTrajectoryTurn {\n readonly src?: string | null\n readonly msg?: string | null\n readonly tools?: readonly { readonly cmd?: string | null }[] | null\n readonly obs?: string | null\n}\n\nexport interface DecodedTrajectory {\n /** Executed commands in recorded order, each holding its OWN observation. */\n readonly steps: readonly RecordedTrajectoryStep[]\n /** Turns the scaffold rejected before anything ran. */\n readonly formatErrorTurns: number\n /**\n * Executed commands whose text the dump dropped.\n *\n * The step stays in `steps` carrying the marker, because the run did execute\n * a command there and a shorter list would misreport the trajectory. The\n * marker is not a command, so any count above zero means this trajectory\n * cannot be replayed: gate on it, or call `assertReplayableTrajectory`.\n */\n readonly elidedCommands: number\n /** True when the run's last turn was the submit sentinel with no observation. */\n readonly endedOnSubmitSentinel: boolean\n /** Turns carrying an observation this grammar cannot read. */\n readonly unreadableTurns: number\n}\n\n/**\n * Pair every recorded command with the observation of its own turn.\n *\n * Collecting commands and observations into two lists and zipping them is the\n * decode that looks right and is not: a turn the scaffold rejected carries an\n * observation of its own, so from the first such turn onward every observation\n * belongs to a different command than the one it is read against.\n *\n * A rejected turn executed nothing, so it is never a step — including when the\n * dump recorded a command for it. The scaffold rejects a turn holding several\n * bash blocks and runs none of them, while the dump keeps one of the blocks in\n * the command field. Replaying that field would execute a command the recorded\n * run did not, which is a worse corpus than a smaller one.\n *\n * A rejected turn is recognised by its observation, so a rejected turn whose\n * observation the dump elided is indistinguishable from an executed command\n * whose observation it elided. Both read as a step. Nothing in the dump\n * separates them, and the row-level defence is the share of unreadable exits a\n * caller admits: a row with no elided observation cannot hold this case at all.\n *\n * A trailing step that echoes the sentinel and nothing else, with no\n * observation, is dropped from `steps` and reported as `endedOnSubmitSentinel`.\n * The scaffold records an observation only when it hands one to the model, and\n * the sentinel ends the run, so the missing observation is the end of the\n * transcript rather than a gap in it. Echoing the sentinel changes no state, so\n * the recorded end state is the state the step before it left.\n *\n * A step that DID get an observation stays a step: the run continued past it.\n * So does a step that echoes the sentinel after doing real work — its state\n * change is part of the recorded end state, and with no observation its exit is\n * unknown, which `finalRecordedOutcome` reports rather than hides.\n */\nexport function decodeRecordedTurns(turns: readonly RecordedTrajectoryTurn[]): DecodedTrajectory {\n const steps: RecordedTrajectoryStep[] = []\n let formatErrorTurns = 0\n let unreadableTurns = 0\n for (const turn of turns) {\n const command = turn.tools?.[0]?.cmd ?? null\n const observation = turn.obs ?? null\n const kind = classifyObservation(observation)\n if (kind === 'format-error') {\n formatErrorTurns += 1\n continue\n }\n if (command === null) {\n if (kind === 'unreadable' || kind === 'elided') unreadableTurns += 1\n continue\n }\n steps.push({ step_id: steps.length + 1, action: command, observation })\n }\n const last = steps[steps.length - 1]\n const endedOnSubmitSentinel =\n last !== undefined && last.observation === null && isSubmitOnlyAction(last.action)\n if (endedOnSubmitSentinel) steps.pop()\n return {\n steps,\n formatErrorTurns,\n elidedCommands: steps.filter((step) => isElidedField(step.action)).length,\n endedOnSubmitSentinel,\n unreadableTurns,\n }\n}\n\n// ── The state a trajectory ended in ──────────────────────────────────\n\n/**\n * How the recorded run's last executed command ended.\n *\n * `killed` is a measured outcome, not a missing one: the environment stopped\n * the command at its bound and wrote a notice instead of an exit status.\n * `unreadable` names the observation kind that blocked the read, so a funnel\n * can report which shape cost it the row.\n */\nexport type RecordedFinalOutcome =\n | { readonly kind: 'returncode'; readonly value: number }\n | { readonly kind: 'killed' }\n | { readonly kind: 'unreadable'; readonly reason: RecordedObservationKind }\n\n/**\n * The outcome of the last step, or `null` when the trajectory has no steps.\n *\n * Reads only the last step. `decodeRecordedTurns` has already removed the turns\n * that executed nothing, so the last step is the last command the run ran.\n */\nexport function finalRecordedOutcome(\n steps: readonly RecordedTrajectoryStep[],\n): RecordedFinalOutcome | null {\n const last = steps[steps.length - 1]\n if (last === undefined) return null\n const kind = classifyObservation(last.observation)\n if (kind === 'command-result') {\n return { kind: 'returncode', value: parseRecordedReturncode(last.observation)! }\n }\n if (kind === 'timeout') return { kind: 'killed' }\n return { kind: 'unreadable', reason: kind }\n}\n\n/**\n * Steps whose recorded exit a replay cannot check.\n *\n * A killed step counts: the recording holds no exit status to compare a replay\n * against, so agreement on it cannot be measured either way.\n */\nexport function unreadableExitCount(steps: readonly RecordedTrajectoryStep[]): number {\n return steps.filter((step) => classifyObservation(step.observation) !== 'command-result').length\n}\n\n/**\n * Throw unless every recorded command survived the dump.\n *\n * A replay executes `steps` verbatim, so one elision marker in an action means\n * the replay runs the literal two-to-four characters of the marker instead of\n * the command the run executed. The guard is here so a replayer needs one call\n * rather than a field check it can forget.\n */\nexport function assertReplayableTrajectory(decoded: DecodedTrajectory): void {\n if (decoded.elidedCommands === 0) return\n const markers = decoded.steps\n .filter((step) => isElidedField(step.action))\n .map((step) => step.action)\n throw new Error(\n `trajectory holds ${decoded.elidedCommands} command(s) the dump elided (${markers.slice(0, 5).join(', ')}); ` +\n 'no dictionary maps a marker back to its text, so this trajectory cannot be replayed',\n )\n}\n","/**\n * The execution boundary replay runs across.\n *\n * A replay needs one thing from its environment: a session that runs a shell\n * command inside the trajectory's own image and reports the exit code and\n * output. That is the whole contract. Concrete backends — a sandbox platform\n * client, a docker exec, an SSH shell — live with the consumer that owns the\n * infrastructure, so this package depends on no sandbox client.\n */\n\nexport interface ReplayExecResult {\n exitCode: number\n stdout: string\n stderr: string\n}\n\nexport interface ReplayExecSession {\n exec(command: string, timeoutMs: number): Promise<ReplayExecResult>\n close(): Promise<void>\n}\n\nexport interface ReplayExecBackend {\n /** One fresh execution environment per call; the caller closes it. */\n open(): Promise<ReplayExecSession>\n}\n\n/** Builds a backend pinned to one image. Callers that resolve images\n * internally (batch, corpus wire, finding verification) take this instead of\n * a backend, so every case runs against its own image. */\nexport type ReplayExecBackendFactory = (image: string) => ReplayExecBackend\n\n/**\n * mini-SWE runs every action as a fresh /bin/sh subshell from a fixed\n * workdir. Reproduce that exactly — and stay quote-proof for arbitrary\n * recorded actions — by piping the base64 of the action into `sh` after\n * cd-ing to the workdir. Exit code is sh's, i.e. the action's.\n */\nexport function wrapActionForExec(action: string, cwd: string): string {\n const b64 = Buffer.from(action, 'utf8').toString('base64')\n const quotedCwd = `'${cwd.replaceAll(\"'\", `'\\\\''`)}'`\n return `cd ${quotedCwd} && printf %s ${b64} | base64 -d | sh`\n}\n"],"mappings":";;AA4CA,SAAgB,wBAAwB,aAA2C;CACjF,IAAI,CAAC,aAAa,OAAO;CACzB,MAAM,IAAI,oCAAoC,KAAK,WAAW;CAC9D,OAAO,IAAI,OAAO,EAAE,EAAE,IAAI;AAC5B;;;AAIA,SAAgB,uBAAuB,aAAoC;CACzE,IAAI,CAAC,aAAa,OAAO;CACzB,MAAM,IAAI,qCAAqC,KAAK,WAAW;CAC/D,OAAO,IAAI,EAAE,KAAM;AACrB;;;;;;;;AASA,SAAgB,uBAAuB,aAA2C;CAChF,KAAK,MAAM,QAAQ,uBAAuB,WAAW,CAAC,CAAC,MAAM,IAAI,GAAG;EAClE,IAAI,aAAa,KAAK,IAAI,GAAG,OAAO,KAAK,KAAK,CAAC,CAAC,MAAM,GAAG,GAAG;EAE5D,MAAM,iBAAiB,gDAAgD,KAAK,IAAI;EAChF,IAAI,gBAAgB,OAAO,eAAe,EAAE,CAAE,MAAM,GAAG,GAAG;CAC5D;CACA,OAAO;AACT;;;;;AAMA,MAAa,0BAA0B;AAEvC,SAAgB,eAAe,QAAyB;CACtD,OAAO,OAAO,SAAS,uBAAuB;AAChD;;;;;;;;;;AAWA,SAAgB,mBAAmB,QAAyB;CAC1D,OAAO,qIAAqI,KAC1I,OAAO,KAAK,CACd;AACF;;;;;;;;;;;;;;;AAkBA,MAAM,2BAA2B;AAEjC,SAAgB,cAAc,OAA2C;CACvE,OAAO,OAAO,UAAU,YAAY,yBAAyB,KAAK,KAAK;AACzE;;AAKA,MAAa,6BAA6B;;AAG1C,MAAa,kCACX;AAoBF,SAAgB,oBAAoB,aAAqD;CACvF,IAAI,gBAAgB,MAAM,OAAO;CACjC,IAAI,cAAc,WAAW,GAAG,OAAO;CACvC,IAAI,wBAAwB,WAAW,MAAM,MAAM,OAAO;CAC1D,IAAI,YAAY,WAAA,sEAA0C,GAAG,OAAO;CACpE,IAAI,YAAY,SAAA,+BAAmC,GAAG,OAAO;CAC7D,OAAO;AACT;;;;AAKA,SAAgB,kBAAkB,aAAqC;CACrE,OAAO,oBAAoB,WAAW,MAAM;AAC9C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqEA,SAAgB,oBAAoB,OAA6D;CAC/F,MAAM,QAAkC,CAAC;CACzC,IAAI,mBAAmB;CACvB,IAAI,kBAAkB;CACtB,KAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,UAAU,KAAK,QAAQ,EAAE,EAAE,OAAO;EACxC,MAAM,cAAc,KAAK,OAAO;EAChC,MAAM,OAAO,oBAAoB,WAAW;EAC5C,IAAI,SAAS,gBAAgB;GAC3B,oBAAoB;GACpB;EACF;EACA,IAAI,YAAY,MAAM;GACpB,IAAI,SAAS,gBAAgB,SAAS,UAAU,mBAAmB;GACnE;EACF;EACA,MAAM,KAAK;GAAE,SAAS,MAAM,SAAS;GAAG,QAAQ;GAAS;EAAY,CAAC;CACxE;CACA,MAAM,OAAO,MAAM,MAAM,SAAS;CAClC,MAAM,wBACJ,SAAS,KAAA,KAAa,KAAK,gBAAgB,QAAQ,mBAAmB,KAAK,MAAM;CACnF,IAAI,uBAAuB,MAAM,IAAI;CACrC,OAAO;EACL;EACA;EACA,gBAAgB,MAAM,QAAQ,SAAS,cAAc,KAAK,MAAM,CAAC,CAAC,CAAC;EACnE;EACA;CACF;AACF;;;;;;;AAuBA,SAAgB,qBACd,OAC6B;CAC7B,MAAM,OAAO,MAAM,MAAM,SAAS;CAClC,IAAI,SAAS,KAAA,GAAW,OAAO;CAC/B,MAAM,OAAO,oBAAoB,KAAK,WAAW;CACjD,IAAI,SAAS,kBACX,OAAO;EAAE,MAAM;EAAc,OAAO,wBAAwB,KAAK,WAAW;CAAG;CAEjF,IAAI,SAAS,WAAW,OAAO,EAAE,MAAM,SAAS;CAChD,OAAO;EAAE,MAAM;EAAc,QAAQ;CAAK;AAC5C;;;;;;;;;AClQA,SAAgB,kBAAkB,QAAgB,KAAqB;CACrE,MAAM,MAAM,OAAO,KAAK,QAAQ,MAAM,CAAC,CAAC,SAAS,QAAQ;CAEzD,OAAO,MAAM,IADS,IAAI,WAAW,KAAK,OAAO,EAAE,GAC5B,gBAAgB,IAAI;AAC7C"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"execution-tracks-CpgFPpS5.js","names":[],"sources":["../src/trace/execution-tracks.ts"],"sourcesContent":["
|
|
1
|
+
{"version":3,"file":"execution-tracks-CpgFPpS5.js","names":[],"sources":["../src/trace/execution-tracks.ts"],"sourcesContent":["interface ExecutionLane {\n key: string\n scopeKey: string\n start: number | null\n end: number | null\n}\n\n/** Assign lanes to serial tracks without joining concurrent or ambiguously timed work. */\nexport function executionTrackByLane(lanes: readonly ExecutionLane[]): Map<string, string> {\n const unique = new Map<string, ExecutionLane>()\n for (const lane of lanes) {\n const existing = unique.get(lane.key)\n if (existing && !sameLane(existing, lane)) {\n throw new Error(`execution lane '${lane.key}' has conflicting scope or timing`)\n }\n unique.set(lane.key, lane)\n }\n\n const trackByLane = new Map<string, string>()\n const lanesByScope = new Map<string, ExecutionLane[]>()\n for (const lane of unique.values()) {\n const scoped = lanesByScope.get(lane.scopeKey) ?? []\n scoped.push(lane)\n lanesByScope.set(lane.scopeKey, scoped)\n }\n\n for (const [scopeKey, scoped] of lanesByScope) {\n if (scoped.some((lane) => lane.start === null && lane.end === null)) {\n for (const lane of scoped) trackByLane.set(lane.key, trackId('lane', lane.key))\n continue\n }\n const incompleteKeys = new Set<string>()\n const boundarySet = new Set<number>()\n for (const lane of scoped) {\n if (lane.start !== null && lane.end !== null && lane.end > lane.start) continue\n incompleteKeys.add(lane.key)\n const boundary = lane.start ?? lane.end\n if (boundary !== null) boundarySet.add(boundary)\n trackByLane.set(lane.key, trackId('lane', lane.key))\n }\n const boundaries = [...boundarySet].sort((a, b) => a - b)\n const completeBySegment = new Map<number, ExecutionLane[]>()\n for (const lane of scoped) {\n if (incompleteKeys.has(lane.key)) continue\n const segment = upperBound(boundaries, lane.start!)\n const nextBoundary = boundaries[segment]\n if (nextBoundary !== undefined && nextBoundary < lane.end!) {\n trackByLane.set(lane.key, trackId('lane', lane.key))\n continue\n }\n const complete = completeBySegment.get(segment) ?? []\n complete.push(lane)\n completeBySegment.set(segment, complete)\n }\n\n let serialTrack = 0\n for (const [, segment] of [...completeBySegment].sort(([a], [b]) => a - b)) {\n const ordered = [...segment].sort(\n (a, b) => a.start! - b.start! || a.end! - b.end! || a.key.localeCompare(b.key),\n )\n let component: ExecutionLane[] = []\n let componentEnd = Number.NEGATIVE_INFINITY\n const flush = () => {\n if (component.length === 1) {\n trackByLane.set(component[0]!.key, trackId('serial', scopeKey, serialTrack))\n } else if (component.length > 1) {\n for (const lane of component) trackByLane.set(lane.key, trackId('lane', lane.key))\n serialTrack += 1\n }\n component = []\n componentEnd = Number.NEGATIVE_INFINITY\n }\n\n for (const lane of ordered) {\n if (component.length > 0 && lane.start! >= componentEnd) flush()\n component.push(lane)\n componentEnd = Math.max(componentEnd, lane.end!)\n }\n flush()\n serialTrack += 1\n }\n }\n\n return trackByLane\n}\n\nfunction sameLane(a: ExecutionLane, b: ExecutionLane): boolean {\n return a.scopeKey === b.scopeKey && a.start === b.start && a.end === b.end\n}\n\nfunction trackId(kind: string, key: string, index?: number): string {\n return JSON.stringify(index === undefined ? [kind, key] : [kind, key, index])\n}\n\nfunction upperBound(sorted: readonly number[], target: number): number {\n let low = 0\n let high = sorted.length\n while (low < high) {\n const middle = (low + high) >>> 1\n if (sorted[middle]! <= target) low = middle + 1\n else high = middle\n }\n return low\n}\n"],"mappings":";;AAQA,SAAgB,qBAAqB,OAAsD;CACzF,MAAM,yBAAS,IAAI,IAA2B;CAC9C,KAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,WAAW,OAAO,IAAI,KAAK,GAAG;EACpC,IAAI,YAAY,CAAC,SAAS,UAAU,IAAI,GACtC,MAAM,IAAI,MAAM,mBAAmB,KAAK,IAAI,kCAAkC;EAEhF,OAAO,IAAI,KAAK,KAAK,IAAI;CAC3B;CAEA,MAAM,8BAAc,IAAI,IAAoB;CAC5C,MAAM,+BAAe,IAAI,IAA6B;CACtD,KAAK,MAAM,QAAQ,OAAO,OAAO,GAAG;EAClC,MAAM,SAAS,aAAa,IAAI,KAAK,QAAQ,KAAK,CAAC;EACnD,OAAO,KAAK,IAAI;EAChB,aAAa,IAAI,KAAK,UAAU,MAAM;CACxC;CAEA,KAAK,MAAM,CAAC,UAAU,WAAW,cAAc;EAC7C,IAAI,OAAO,MAAM,SAAS,KAAK,UAAU,QAAQ,KAAK,QAAQ,IAAI,GAAG;GACnE,KAAK,MAAM,QAAQ,QAAQ,YAAY,IAAI,KAAK,KAAK,QAAQ,QAAQ,KAAK,GAAG,CAAC;GAC9E;EACF;EACA,MAAM,iCAAiB,IAAI,IAAY;EACvC,MAAM,8BAAc,IAAI,IAAY;EACpC,KAAK,MAAM,QAAQ,QAAQ;GACzB,IAAI,KAAK,UAAU,QAAQ,KAAK,QAAQ,QAAQ,KAAK,MAAM,KAAK,OAAO;GACvE,eAAe,IAAI,KAAK,GAAG;GAC3B,MAAM,WAAW,KAAK,SAAS,KAAK;GACpC,IAAI,aAAa,MAAM,YAAY,IAAI,QAAQ;GAC/C,YAAY,IAAI,KAAK,KAAK,QAAQ,QAAQ,KAAK,GAAG,CAAC;EACrD;EACA,MAAM,aAAa,CAAC,GAAG,WAAW,CAAC,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC;EACxD,MAAM,oCAAoB,IAAI,IAA6B;EAC3D,KAAK,MAAM,QAAQ,QAAQ;GACzB,IAAI,eAAe,IAAI,KAAK,GAAG,GAAG;GAClC,MAAM,UAAU,WAAW,YAAY,KAAK,KAAM;GAClD,MAAM,eAAe,WAAW;GAChC,IAAI,iBAAiB,KAAA,KAAa,eAAe,KAAK,KAAM;IAC1D,YAAY,IAAI,KAAK,KAAK,QAAQ,QAAQ,KAAK,GAAG,CAAC;IACnD;GACF;GACA,MAAM,WAAW,kBAAkB,IAAI,OAAO,KAAK,CAAC;GACpD,SAAS,KAAK,IAAI;GAClB,kBAAkB,IAAI,SAAS,QAAQ;EACzC;EAEA,IAAI,cAAc;EAClB,KAAK,MAAM,GAAG,YAAY,CAAC,GAAG,iBAAiB,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,GAAG;GAC1E,MAAM,UAAU,CAAC,GAAG,OAAO,CAAC,CAAC,MAC1B,GAAG,MAAM,EAAE,QAAS,EAAE,SAAU,EAAE,MAAO,EAAE,OAAQ,EAAE,IAAI,cAAc,EAAE,GAAG,CAC/E;GACA,IAAI,YAA6B,CAAC;GAClC,IAAI,eAAe,OAAO;GAC1B,MAAM,cAAc;IAClB,IAAI,UAAU,WAAW,GACvB,YAAY,IAAI,UAAU,EAAE,CAAE,KAAK,QAAQ,UAAU,UAAU,WAAW,CAAC;SACtE,IAAI,UAAU,SAAS,GAAG;KAC/B,KAAK,MAAM,QAAQ,WAAW,YAAY,IAAI,KAAK,KAAK,QAAQ,QAAQ,KAAK,GAAG,CAAC;KACjF,eAAe;IACjB;IACA,YAAY,CAAC;IACb,eAAe,OAAO;GACxB;GAEA,KAAK,MAAM,QAAQ,SAAS;IAC1B,IAAI,UAAU,SAAS,KAAK,KAAK,SAAU,cAAc,MAAM;IAC/D,UAAU,KAAK,IAAI;IACnB,eAAe,KAAK,IAAI,cAAc,KAAK,GAAI;GACjD;GACA,MAAM;GACN,eAAe;EACjB;CACF;CAEA,OAAO;AACT;AAEA,SAAS,SAAS,GAAkB,GAA2B;CAC7D,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE;AACzE;AAEA,SAAS,QAAQ,MAAc,KAAa,OAAwB;CAClE,OAAO,KAAK,UAAU,UAAU,KAAA,IAAY,CAAC,MAAM,GAAG,IAAI;EAAC;EAAM;EAAK;CAAK,CAAC;AAC9E;AAEA,SAAS,WAAW,QAA2B,QAAwB;CACrE,IAAI,MAAM;CACV,IAAI,OAAO,OAAO;CAClB,OAAO,MAAM,MAAM;EACjB,MAAM,SAAU,MAAM,SAAU;EAChC,IAAI,OAAO,WAAY,QAAQ,MAAM,SAAS;OACzC,OAAO;CACd;CACA,OAAO;AACT"}
|
|
@@ -1,10 +1,9 @@
|
|
|
1
|
-
import { c as ValidationError, r as CaptureIntegrityError } from "../errors-DEE6u6ot.js";
|
|
2
1
|
import { n as bonferroni, r as holm, t as benjaminiHochberg } from "../multiplicity-DIWHvysC.js";
|
|
3
|
-
import {
|
|
4
|
-
import { c as BOOTSTRAP_GATE_MIN_N,
|
|
2
|
+
import { At as EProcessStep, B as powerPreflight, Bt as pairedRiskDifferenceScore, C as signManifest, Ct as FinalEvidenceOutcome, Dt as EProcess, E as MatchedPair, Et as openFinalEvidenceLedger, F as comparePairedArms, Ht as wilson, I as pairArms, J as PromotionPolicy, L as pairRunRecords, Lt as mcnemar, O as PairArmsOptions, Ot as EProcessOptions, Pt as ProportionInterval, R as PowerPreflight, Rt as pairedRiskDifference, S as manifestContentDigest, St as FinalEvidenceMeasurement, Tt as FinalEvidenceReservation, W as EvidenceVector, X as paretoPolicy, Y as buildEvidenceVector, Z as paretoSignificanceGate, _ as HypothesisManifest, at as EvidenceAuthority, b as SignedManifestAlgo, bt as FinalEvidenceError, ct as EvidenceReceipt, d as SequentialObservation, dt as isIndependentEvidence, f as SequentialPairedGate, ft as verifyEvidenceReceipt, g as sequentialPairedGate, h as sequentialDecide, i as PairedHoldout, it as CreateEvidenceReceiptInput, j as PairedArmRow, jt as eProcess, k as PairArmsResult, kt as EProcessState, lt as EvidenceReceiptVerification, n as HeldoutSignificance, nt as CampaignEvidenceContext, o as heldoutSignificance, ot as EvidenceAuthorityKind, p as SequentialPairedGateOptions, r as HeldoutSignificanceOptions, rt as createCampaignEvidenceReceipt, s as pairHoldout, st as EvidenceBinding, u as SequentialGateDecision, ut as createEvidenceReceipt, v as HypothesisResult, w as verifyManifest, wt as FinalEvidenceRecord, x as hashJson, xt as FinalEvidenceLedger, y as SignedManifest, yt as FinalEvidenceConflictError, z as PowerPreflightOptions, zt as pairedRiskDifferenceExact } from "../statistical-heldout-C9AbgREN.js";
|
|
3
|
+
import { c as BOOTSTRAP_GATE_MIN_N, l as PairedBootstrapOptions, m as pairedBootstrap, u as PairedBootstrapResult } from "../paired-promotion-decision-CQMJ5xGN.js";
|
|
5
4
|
import { a as requiredPairedSampleSize, i as pairedMde, n as mcnemarPower, o as requiredSampleSize, r as mcnemarRequiredN, t as mulberry32 } from "../random-DGEyr3zG.js";
|
|
6
5
|
import { at as summarizeEvaluationUnits, it as defineEvaluationClaim, nt as EvaluationClaim, rt as EvaluationUnitSummary } from "../search-ledger-types-p2xKfiDJ.js";
|
|
7
|
-
import { a as PairedEvalueStep,
|
|
6
|
+
import { a as PairedEvalueStep, i as PairedEvalueSequence, o as SequentialCrossingHorizon, r as PairedEvalueOptions, s as SequentialCrossingHorizonOptions, u as pairedEvalueSequence } from "../sequential-KwGspfeR.js";
|
|
8
7
|
import { a as JsonValue, i as EvidenceRecord, n as IntervalSpec, r as computeInterval, t as ComputedInterval } from "../interval-ohuOJurX.js";
|
|
9
8
|
import { z } from "zod";
|
|
10
9
|
//#region src/experiment/ast/predicate.d.ts
|
|
@@ -33,8 +32,6 @@ type Predicate = {
|
|
|
33
32
|
};
|
|
34
33
|
/** Read a dot-separated field path. Missing segments yield `undefined`. */
|
|
35
34
|
export declare function readField(record: EvidenceRecord, path: string): unknown;
|
|
36
|
-
/** Evaluate a predicate against one evidence record. */
|
|
37
|
-
export declare function evaluatePredicate(predicate: Predicate, record: EvidenceRecord): boolean;
|
|
38
35
|
//#endregion
|
|
39
36
|
//#region src/experiment/ast/validity.d.ts
|
|
40
37
|
/** A registered control that must exist before a class of verdicts is read. */
|
|
@@ -74,40 +71,11 @@ type ValidityGate = {
|
|
|
74
71
|
op: 'basename-eq';
|
|
75
72
|
onFail: 'abort';
|
|
76
73
|
};
|
|
77
|
-
interface
|
|
74
|
+
interface ValidityGateResult {
|
|
78
75
|
id: string;
|
|
79
76
|
passed: boolean;
|
|
80
77
|
evidence: JsonValue;
|
|
81
78
|
}
|
|
82
|
-
/**
|
|
83
|
-
* Replicate-flip counting over graded states. A state whose replicates split
|
|
84
|
-
* between pass and fail is flipping; its flip rate is the minority share.
|
|
85
|
-
*/
|
|
86
|
-
export declare function evaluateOracleDeterminismGate(id: string, gate: Extract<ValidityGate, {
|
|
87
|
-
kind: 'oracle-determinism';
|
|
88
|
-
}>, repsByState: Record<string, readonly boolean[]>): GateResult;
|
|
89
|
-
/**
|
|
90
|
-
* Join two population snapshots on `joinOn` and compare the registered fields.
|
|
91
|
-
* Only rows present in both snapshots are compared; a presence change is a
|
|
92
|
-
* different failure and needs its own gate.
|
|
93
|
-
*/
|
|
94
|
-
export declare function evaluatePopulationReproducibilityGate(id: string, gate: Extract<ValidityGate, {
|
|
95
|
-
kind: 'population-reproducibility';
|
|
96
|
-
}>, populations: {
|
|
97
|
-
left: readonly EvidenceRecord[];
|
|
98
|
-
right: readonly EvidenceRecord[];
|
|
99
|
-
}): GateResult;
|
|
100
|
-
/** The registered claim about provenance must hold on the provenance record. */
|
|
101
|
-
export declare function evaluateProvenanceGate(id: string, gate: Extract<ValidityGate, {
|
|
102
|
-
kind: 'provenance-assertion';
|
|
103
|
-
}>, provenance: EvidenceRecord): GateResult;
|
|
104
|
-
/** Final path segment equality between the pinned and the served identity. */
|
|
105
|
-
export declare function evaluateIdentityGate(id: string, _gate: Extract<ValidityGate, {
|
|
106
|
-
kind: 'identity';
|
|
107
|
-
}>, identities: {
|
|
108
|
-
pinned: string;
|
|
109
|
-
served: string;
|
|
110
|
-
}): GateResult;
|
|
111
79
|
/**
|
|
112
80
|
* The design's power at minimumEffect must reach the registered target.
|
|
113
81
|
* The curve must cover the registered effect grid exactly — a curve
|
|
@@ -118,7 +86,7 @@ export declare function evaluatePowerFloorGate(id: string, gate: Extract<Validit
|
|
|
118
86
|
}>, curve: readonly {
|
|
119
87
|
effect: number;
|
|
120
88
|
power: number;
|
|
121
|
-
}[]):
|
|
89
|
+
}[]): ValidityGateResult;
|
|
122
90
|
//#endregion
|
|
123
91
|
//#region src/experiment/ast/budget.d.ts
|
|
124
92
|
/** Checks as prerequisites: any named gate failing refuses the spend. */
|
|
@@ -135,7 +103,6 @@ interface HaltOutcome {
|
|
|
135
103
|
action: 'refuse-spend' | null;
|
|
136
104
|
failedGates: string[];
|
|
137
105
|
}
|
|
138
|
-
export declare function evaluateHaltRule(halt: HaltRule, gates: readonly GateResult[]): HaltOutcome;
|
|
139
106
|
/**
|
|
140
107
|
* Spend schedules as data. `ledger` names every entry the ceiling counts, so
|
|
141
108
|
* what the gate reads is registered, not improvised at gate time.
|
|
@@ -168,15 +135,6 @@ interface UniformPassSchedule {
|
|
|
168
135
|
decisions: UniformPassDecision[];
|
|
169
136
|
uniformN: number;
|
|
170
137
|
}
|
|
171
|
-
/**
|
|
172
|
-
* Execute the uniform-pass schedule against measured pass costs. Pass 1 always
|
|
173
|
-
* runs; each later pass runs only when the cumulative spend plus the last
|
|
174
|
-
* measured pass cost stays at or under the registered ceiling. The registered
|
|
175
|
-
* ledger is the pre-spend the ceiling counts.
|
|
176
|
-
*/
|
|
177
|
-
export declare function runUniformPassBudget(rule: Extract<BudgetRule, {
|
|
178
|
-
kind: 'uniform-pass';
|
|
179
|
-
}>, measuredPassCosts: readonly number[]): UniformPassSchedule;
|
|
180
138
|
interface NLadderProjection {
|
|
181
139
|
chosenN: number | null;
|
|
182
140
|
projections: {
|
|
@@ -189,17 +147,6 @@ interface NLadderProjection {
|
|
|
189
147
|
reason: string;
|
|
190
148
|
};
|
|
191
149
|
}
|
|
192
|
-
/**
|
|
193
|
-
* Walk the registered n-ladder and pick the first affordable step. When no
|
|
194
|
-
* step fits the ceiling, the rule refuses and reports the projection instead
|
|
195
|
-
* of shrinking the row set — "never subset rows" is the registered invariant.
|
|
196
|
-
*/
|
|
197
|
-
export declare function projectNLadderBudget(rule: Extract<BudgetRule, {
|
|
198
|
-
kind: 'n-ladder';
|
|
199
|
-
}>, measured: {
|
|
200
|
-
unitCostUsd: number;
|
|
201
|
-
rows: number;
|
|
202
|
-
}): NLadderProjection;
|
|
203
150
|
/** Arm budget matching as a refusal, not prose. Verified by `verifyMatchedBudgets`. */
|
|
204
151
|
interface MatchedBudgetRule {
|
|
205
152
|
measure: 'realized-tokens';
|
|
@@ -322,8 +269,6 @@ interface EstimandResult {
|
|
|
322
269
|
numerator: number;
|
|
323
270
|
denominator: number;
|
|
324
271
|
}
|
|
325
|
-
/** Compute an estimand over evidence rows. Pure; reads only registered fields. */
|
|
326
|
-
export declare function computeEstimand(estimand: Estimand, rows: readonly EvidenceRecord[]): EstimandResult;
|
|
327
272
|
//#endregion
|
|
328
273
|
//#region src/experiment/ast/reissue.d.ts
|
|
329
274
|
/** Carrier faults are reissued; model outcomes stand. Closed enumeration. */
|
|
@@ -333,12 +278,6 @@ interface ReissuePolicy {
|
|
|
333
278
|
maxIssues: number;
|
|
334
279
|
}
|
|
335
280
|
type ReissueVerdict = 'reissue' | 'stands' | 'exhausted';
|
|
336
|
-
/**
|
|
337
|
-
* Classify one rollout event under the registered reissue policy. A carrier
|
|
338
|
-
* event within the issue budget is reissued; a model outcome always stands;
|
|
339
|
-
* a carrier event past `maxIssues` is exhausted and reported, never retried.
|
|
340
|
-
*/
|
|
341
|
-
export declare function classifyReissue(policy: ReissuePolicy, event: string, issuesSoFar: number): ReissueVerdict;
|
|
342
281
|
//#endregion
|
|
343
282
|
//#region src/experiment/ast/selection.d.ts
|
|
344
283
|
/**
|
|
@@ -389,28 +328,8 @@ type SelectionRule = {
|
|
|
389
328
|
dir: 'asc' | 'desc';
|
|
390
329
|
};
|
|
391
330
|
};
|
|
392
|
-
/**
|
|
393
|
-
* Execute a selection rule.
|
|
394
|
-
*
|
|
395
|
-
* Round-robin walks groups in lexicographic order and takes ids in
|
|
396
|
-
* within-group order until `take` ids are chosen. Filter-of keeps the ids of
|
|
397
|
-
* `bases[rule.base]` whose record satisfies the predicate, in the registered
|
|
398
|
-
* order. Ids absent from `records` are evaluated on their id alone (fields
|
|
399
|
-
* derived from the id via `idFields`), so a sealed base outlives its source
|
|
400
|
-
* records.
|
|
401
|
-
*/
|
|
402
|
-
export declare function runSelectionRule(rule: SelectionRule, records: readonly EvidenceRecord[], options: {
|
|
403
|
-
/** Field carrying a row's identity. */
|
|
404
|
-
idField: string;
|
|
405
|
-
/** Sealed or previously-computed subsets, by name. */
|
|
406
|
-
bases?: Record<string, readonly string[]>;
|
|
407
|
-
/** Derive predicate-readable fields from a bare id when its record is absent. */
|
|
408
|
-
idFields?: (id: string) => EvidenceRecord;
|
|
409
|
-
}): string[];
|
|
410
331
|
//#endregion
|
|
411
332
|
//#region src/experiment/budget.d.ts
|
|
412
|
-
/** Arms whose realized budgets diverge past the registered tolerance. */
|
|
413
|
-
export declare class MatchedBudgetError extends CaptureIntegrityError {}
|
|
414
333
|
interface ArmRealizedBudget {
|
|
415
334
|
armId: string;
|
|
416
335
|
/** Realized prompt + completion tokens the arm actually consumed. */
|
|
@@ -440,8 +359,6 @@ export declare function verifyMatchedBudgets(rule: MatchedBudgetRule, arms: read
|
|
|
440
359
|
export declare function assertMatchedBudgets(rule: MatchedBudgetRule, arms: readonly ArmRealizedBudget[]): MatchedBudgetVerdict;
|
|
441
360
|
//#endregion
|
|
442
361
|
//#region src/experiment/funnel.d.ts
|
|
443
|
-
/** A funnel stage gained rows, double-counted them, or failed to reconcile. */
|
|
444
|
-
export declare class FunnelIntegrityError extends CaptureIntegrityError {}
|
|
445
362
|
interface FunnelStageCount {
|
|
446
363
|
id: string;
|
|
447
364
|
entering: number;
|
|
@@ -493,8 +410,6 @@ export declare function buildFunnel(input: {
|
|
|
493
410
|
count: number;
|
|
494
411
|
}[];
|
|
495
412
|
}): ExperimentFunnel;
|
|
496
|
-
/** A chain that does not add up is a broken denominator, so this throws. */
|
|
497
|
-
export declare function assertFunnelReconciles(funnel: ExperimentFunnel): void;
|
|
498
413
|
interface AdmissionExecution {
|
|
499
414
|
funnel: ExperimentFunnel;
|
|
500
415
|
survivors: EvidenceRecord[];
|
|
@@ -522,8 +437,6 @@ export declare function composeFunnels(first: ExperimentFunnel, second: Experime
|
|
|
522
437
|
export declare function renderFunnelTable(funnel: ExperimentFunnel): string;
|
|
523
438
|
//#endregion
|
|
524
439
|
//#region src/experiment/define.d.ts
|
|
525
|
-
/** A sealed experiment whose digest no longer matches its spec. */
|
|
526
|
-
export declare class SealIntegrityError extends ValidationError {}
|
|
527
440
|
interface ArmSpec {
|
|
528
441
|
id: string;
|
|
529
442
|
role: 'treatment' | 'control';
|
|
@@ -634,7 +547,7 @@ export declare function amendExperiment(sealed: SealedExperiment, amendment: {
|
|
|
634
547
|
/** True when the supported seal matches its canonical spec. */
|
|
635
548
|
export declare function verifySealedExperiment(sealed: SealedExperiment): Promise<boolean>;
|
|
636
549
|
/** Evidence a gate executor consumes, discriminated to match the gate kind. */
|
|
637
|
-
type
|
|
550
|
+
type ExperimentGateEvidence = {
|
|
638
551
|
kind: 'oracle-determinism';
|
|
639
552
|
repsByState: Record<string, readonly boolean[]>;
|
|
640
553
|
} | {
|
|
@@ -672,9 +585,9 @@ interface RegisteredExperiment {
|
|
|
672
585
|
idField: string;
|
|
673
586
|
}): string[];
|
|
674
587
|
/** Evaluate a registered validity gate on evidence of the matching kind. */
|
|
675
|
-
gate(name: string, evidence:
|
|
588
|
+
gate(name: string, evidence: ExperimentGateEvidence): ValidityGateResult;
|
|
676
589
|
/** Evaluate the registered halt rule over gate results. */
|
|
677
|
-
halt(gates: readonly
|
|
590
|
+
halt(gates: readonly ValidityGateResult[]): HaltOutcome;
|
|
678
591
|
/** Execute the registered uniform-pass budget schedule. */
|
|
679
592
|
runUniformPassBudget(measuredPassCosts: readonly number[]): UniformPassSchedule;
|
|
680
593
|
/** Project the registered n-ladder budget. */
|
|
@@ -758,24 +671,8 @@ export declare const evidenceRegistryRecordSchema: z.ZodObject<{
|
|
|
758
671
|
notes: z.ZodOptional<z.ZodString>;
|
|
759
672
|
}, z.core.$strict>;
|
|
760
673
|
type EvidenceRegistryRecord = z.infer<typeof evidenceRegistryRecordSchema>;
|
|
761
|
-
/** Parse one record; throws with the zod issue list on any violation. */
|
|
762
|
-
export declare function parseEvidenceRegistryRecord(raw: unknown): EvidenceRegistryRecord;
|
|
763
|
-
/**
|
|
764
|
-
* Validate a set of records as one registry: every record parses, ids are
|
|
765
|
-
* unique, and every `supersedes` target exists. Returns records sorted in
|
|
766
|
-
* the registry's canonical order (trust ladder, then date desc, then id).
|
|
767
|
-
*/
|
|
768
|
-
export declare function validateEvidenceRegistry(raws: readonly unknown[]): EvidenceRegistryRecord[];
|
|
769
|
-
/**
|
|
770
|
-
* Render the registry's human index from records. Deterministic: identical
|
|
771
|
-
* records always produce identical output, so a byte diff against the
|
|
772
|
-
* committed index is the freshness check. No clocks, no environment.
|
|
773
|
-
*/
|
|
774
|
-
export declare function renderEvidenceIndex(raws: readonly unknown[]): string;
|
|
775
674
|
//#endregion
|
|
776
675
|
//#region src/experiment/power.d.ts
|
|
777
|
-
/** A design refused at configuration time, before any spend. */
|
|
778
|
-
export declare class DesignRefusalError extends ValidationError {}
|
|
779
676
|
interface ClusteredPowerOptions {
|
|
780
677
|
/** Rows per independent cluster, e.g. [6, 3, 3, 2]. */
|
|
781
678
|
clusterSizes: number[];
|
|
@@ -1065,5 +962,5 @@ interface ProspectiveCanaryJoinInput {
|
|
|
1065
962
|
*/
|
|
1066
963
|
export declare function joinProspectiveCanaryReceipts(input: ProspectiveCanaryJoinInput): CanaryObservation[];
|
|
1067
964
|
//#endregion
|
|
1068
|
-
export { type AdmissionExecution, type AdmissionPartition, type AdmissionRule, type AdmissionStage, type ArmRealizedBudget, type ArmSpec, BOOTSTRAP_GATE_MIN_N, type BudgetRule, type CampaignEvidenceContext, type CanaryArm, type CanaryAssignmentReceipt, type CanaryCohortReceipt, type CanaryCohortVerifier, type CanaryObservation, type CanaryRefusal, type ClusteredPowerOptions, type ClusteredPowerPoint, type ClusteredPowerRefusal, type ClusteredPowerResult, type ComputedInterval, type Condition, type CreateEvidenceReceiptInput, type DecisionBranch, type DecisionOutcome, type DecisionRule, type DerivedQuantities, type EProcess, type EProcessOptions, type EProcessState, type EProcessStep,
|
|
965
|
+
export { type AdmissionExecution, type AdmissionPartition, type AdmissionRule, type AdmissionStage, type ArmRealizedBudget, type ArmSpec, BOOTSTRAP_GATE_MIN_N, type BudgetRule, type CampaignEvidenceContext, type CanaryArm, type CanaryAssignmentReceipt, type CanaryCohortReceipt, type CanaryCohortVerifier, type CanaryObservation, type CanaryRefusal, type ClusteredPowerOptions, type ClusteredPowerPoint, type ClusteredPowerRefusal, type ClusteredPowerResult, type ComputedInterval, type Condition, type CreateEvidenceReceiptInput, type DecisionBranch, type DecisionOutcome, type DecisionRule, type DerivedQuantities, type EProcess, type EProcessOptions, type EProcessState, type EProcessStep, type Estimand, type EstimandResult, type EvaluationClaim, type EvaluationUnitSummary, type EvidenceAuthority, type EvidenceAuthorityKind, type EvidenceBinding, type EvidenceDenominator, type EvidenceReceipt, type EvidenceReceiptVerification, type EvidenceRecord, type EvidenceRegistryRecord, type EvidenceState, type EvidenceVector, type ExperimentFunnel, type ExperimentGateEvidence, type ExperimentSpec, FinalEvidenceConflictError, FinalEvidenceError, type FinalEvidenceLedger, type FinalEvidenceMeasurement, type FinalEvidenceOutcome, type FinalEvidenceRecord, type FinalEvidenceReservation, type FunnelPartitionCount, type FunnelStageCount, type FunnelStageInput, type HaltOutcome, type HaltRule, type HeldoutSignificance, type HeldoutSignificanceOptions, type HypothesisManifest, type HypothesisResult, type IntervalSpec, type JsonValue, type MatchedBudgetRule, type MatchedBudgetVerdict, type MatchedPair, type NLadderProjection, type Obligation, type OutcomeSpec, type PairArmsOptions, type PairArmsResult, type PairedArmRow, type PairedBootstrapOptions, type PairedBootstrapResult, type PairedEvalueOptions, type PairedEvalueSequence, type PairedEvalueStep, type PairedHoldout, type PowerPreflight, type PowerPreflightOptions, type Predicate, type PromotionPolicy, type ProportionInterval, type ProspectiveCanaryAssignment, type ProspectiveCanaryBilling, type ProspectiveCanaryExecution, type ProspectiveCanaryJoinInput, type ProspectiveCanaryOutcome, type RandomizedCanaryDecision, type RandomizedCanaryRule, type RegisteredExperiment, type ReissuePolicy, type ReissueVerdict, type SealAmendment, type SealedExperiment, type SealedRandomizedCanaryRule, type SeedDerivation, type SelectionRule, type SequentialCrossingHorizon, type SequentialCrossingHorizonOptions, type SequentialGateDecision, type SequentialObservation, type SequentialPairedGate, type SequentialPairedGateOptions, type SetExpr, type SignFlipFloor, type SignedManifest, type SignedManifestAlgo, type UniformPassDecision, type UniformPassSchedule, type ValidityGate, type ValidityGateResult, benjaminiHochberg, bonferroni, buildEvidenceVector, comparePairedArms, computeInterval, createCampaignEvidenceReceipt, createEvidenceReceipt, defineEvaluationClaim, eProcess, hashJson, heldoutSignificance, holm, isIndependentEvidence, manifestContentDigest, mcnemar, mcnemarPower, mcnemarRequiredN, mulberry32, openFinalEvidenceLedger, pairArms, pairHoldout, pairRunRecords, pairedBootstrap, pairedEvalueSequence, pairedMde, pairedRiskDifference, pairedRiskDifferenceExact, pairedRiskDifferenceScore, paretoPolicy, paretoSignificanceGate, powerPreflight, requiredPairedSampleSize, requiredSampleSize, sequentialDecide, sequentialPairedGate, signManifest, summarizeEvaluationUnits, verifyEvidenceReceipt, verifyManifest, wilson };
|
|
1069
966
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","names":[],"sources":["../../src/experiment/ast/predicate.ts","../../src/experiment/ast/validity.ts","../../src/experiment/ast/budget.ts","../../src/experiment/ast/decision.ts","../../src/experiment/ast/estimand.ts","../../src/experiment/ast/reissue.ts","../../src/experiment/ast/selection.ts","../../src/experiment/budget.ts","../../src/experiment/funnel.ts","../../src/experiment/define.ts","../../src/experiment/evidence-record.ts","../../src/experiment/power.ts","../../src/experiment/randomized-canary.ts","../../src/experiment/prospective-canary-join.ts"],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","names":[],"sources":["../../src/experiment/ast/predicate.ts","../../src/experiment/ast/validity.ts","../../src/experiment/ast/budget.ts","../../src/experiment/ast/decision.ts","../../src/experiment/ast/estimand.ts","../../src/experiment/ast/reissue.ts","../../src/experiment/ast/selection.ts","../../src/experiment/budget.ts","../../src/experiment/funnel.ts","../../src/experiment/define.ts","../../src/experiment/evidence-record.ts","../../src/experiment/power.ts","../../src/experiment/randomized-canary.ts","../../src/experiment/prospective-canary-join.ts"],"mappings":";;;;;;;;;;;;;KAQY;EAEN;EACA;EACA;EACA,OAAO;;EAEP;EAAY;EAAe,QAAQ;;EACnC;EAAa,IAAI;;EACjB;EAAa,IAAI;;EACjB;EAAa,IAAI;;;wBAGP,UAAU,QAAQ,gBAAgB;;;;UChBjC;EACf;EACA;EACA;;;KAMU;EAEN;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;;EAEA;EAA8B,OAAO;;EAErC;EACA;;EAEA;EACA;EACA;IAAO;IAAgB;IAAmB;;;EAE1C;EAAkB;EAAuB;EAAmB;;UAEjD;EACf;EACA;EACA,UAAU;;;;;;;wBA+EI,uBACd,YACA,MAAM,QAAQ;EAAgB;IAC9B;EAAkB;EAAgB;MACjC;;;;UCxHc;EACf;IAAQ;IAAyB;;EACjC;EACA;;UAGe;EACf;EACA;EACA;;;;;;KAuBU;EAEN;EACA;EACA;EACA;EACA;EACA;IAAU;IAAY;;EACtB;;EAGA;EACA;EACA;EACA;EACA;;UAGW;EACf;EACA;EACA;EACA;;UAGe;EACf,WAAW;EACX;;UAmCe;EACf;EACA;IAAe;IAAW;IAAsB;;EAChD;IAAkB;IAAuC;;;;UA+B1C;EACf;EACA;EACA;;;;;KC9HU;EACN;EAAgC;EAAkB;;EAClD;EAAgC;;EAChC;EAA4B;EAAkB;EAAiC;;EAC/E;EAAwB;;EACxB;EAAa,IAAI;;EACjB;EAAa,IAAI;;EACjB;EAAa,IAAI;;;UAGN;EACf,WAAW;IAAiB;IAAe;;EAC3C,YAAY;EACZ,gBAAgB;;wBAGF,kBAAkB,WAAW,WAAW,UAAU;UAyCjD;EACf,MAAM;EACN;EACA;;;;;;;;;KAUU;EACN;EAAe,UAAU;;EAEzB;EACA;EACA;EACA;EACA;IAAmB;IAAuB;;;UAG/B;EACf;EACA;;wBAGc,oBACd,MAAM,cACN,UAAU,oBACT;;;;KC1FS;EACN;EAAoB;EAAa,OAAO;;EACxC;EAAmB,IAAI;;;;;;KAMjB;EACN;EAAc,OAAO;EAAW;;EAChC;EAA4B,OAAO;EAAW;;EAE9C;EACA;EACA;EACA;EACA;;;;;EAKA;;EAEA;;EAGA;EACA;EACA;EACA,WAAW;EACX,aAAa;;UAGF;EACf;EACA;EACA;;;;;UC1Ce;EACf;EACA;EACA;;KAGU;;;;;;;UCGK;EACf;EACA,MAAM;EACN;;;UAIe;EACf;;EAEA;EACA,MAAM;EACN;;;UAIe;EACf;EACA,QAAQ;EACR,aAAa;;;;;;;KAUH;EAEN;EACA;EACA;EACA;IAAe;IAAe;;EAC9B;EACA;;EAGA;;EAEA;EACA,MAAM;EACN;IAAS;IAAe;;;;;UCtCb;EACf;;EAEA;;UAGe;EACf,MAAM;EACN,MAAM;;EAEN;;EAEA;EACA;;EAEA;IAAkB;IAA2B;;;;;;;;wBAQ/B,qBACd,MAAM,mBACN,eAAe,sBACd;;wBAsCa,qBACd,MAAM,mBACN,eAAe,sBACd;;;UC3Dc;EACf;EACA;EACA;EACA;;EAEA,aAAa;;EAEb;;UAGe;EACf;;EAEA;EACA;;EAEA;;UAGe;EACf;EACA;EACA,QAAQ;EACR;EACA,YAAY;;UAGG;EACf;;EAEA;EACA,aAAa;EACb;;;;;;;;;;;wBAYc,YAAY;EAC1B;EACA;EACA,QAAQ;EACR;IAAe;IAAY;IAAc;;IACvC;UA8Fa;EACf,QAAQ;EACR,WAAW;;EAEX,eAAe,eAAe;;;;;;;;wBAShB,qBACd,MAAM,eACN,kBAAkB,mBACjB;;;;;;wBA8Ca,eACd,OAAO,kBACP,QAAQ,mBACP;;;;;;wBA4Ba,kBAAkB,QAAQ;;;UC5LzB;EACf;EACA;;EAEA;;EAEA;;EAEA,OAAO,eAAe;;KAGZ;EAEN;;EAEA;EACA;;EAEA;;EAEA;;EAGA;EACA;EACA;EACA;;;;;;UAOW;EACf;;UAGe;EACf;;EAEA,QAAQ;;EAER;EACA,MAAM;EACN,SAAS;EACT,YAAY;;EAEZ,aAAa,eAAe;;;;;EAK5B,gBAAgB;EAChB,YAAY,eAAe;EAC3B,YAAY,eAAe;EAC3B,UAAU;EACV,cAAc;;EAEd,QAAQ,eAAe;EACvB,OAAO;EACP,SAAS;EACT,gBAAgB;EAChB,UAAU;EACV,iBAAiB;EACjB;;;;;;;;;;wBAwCc,iBAAiB,MAAM,iBAAiB;UA0IvC;;EAEf;EACA;;EAEA;;EAEA;;;KAIG;UAEY;EACf,MAAM;;EAEN;;EAEA,MAAM;EACN;;EAEA;EACA,YAAY;;;wBAIQ,eACpB,MAAM,gBACN;EAAW;IACV,QAAQ;;;;;;wBAkBW,gBACpB,QAAQ,kBACR;EAAa,MAAM;EAAgB;EAAgB;EAAiB;IACnE,QAAQ;;wBAyBW,uBAAuB,QAAQ,mBAAmB;;KAwB5D;EACN;EAA4B,aAAa;;EAEzC;EACA,eAAe;EACf,gBAAgB;;EAEhB;EAA8B,YAAY;;EAC1C;EAAkB;EAAgB;;EAClC;EAAqB;IAAkB;IAAgB;;;;;;;UAM5C;WACN,QAAQ;;EAEjB,MAAM,kBAAkB,mBAAmB;;EAE3C,OAAO,UAAU,oBAAoB;;EAErC,MAAM,kBAAkB,mBAAmB;;EAE3C,OAAO,cAAc,kBAAkB,kBAAkB;IAAW;;;EAEpE,KAAK,cAAc,UAAU,yBAAyB;;EAEtD,KAAK,gBAAgB,uBAAuB;;EAE5C,qBAAqB,uCAAuC;;EAE5D,qBAAqB;IAAY;IAAqB;MAAiB;;EAEvE,eAAe,eAAe,sBAAsB;;EAEpD,SAAS,cAAc,eAAe,mBAAmB;;EAEzD,SACE,cACA;IACM;IAAc,eAAe;;IAC7B;IAAkB;IAAmB;IAAgB;MAC1D;IAAqB,QAAQ;;;;;;;;wBAQZ,qBACpB,QAAQ,mBACP,QAAQ;;;;;;;;;;;;;;;qBC7aE;KAQD,wBAAwB;;cAO9B,2BAAyB,EAAA;;;;GAM7B,EAAA,KAAA;KAEU,sBAAsB,EAAE,aAAa;qBAEpC,8BAA4B,EAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCvC,EAAA,KAAA;KAEU,yBAAyB,EAAE,aAAa;;;UCxEnC;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;EAMA;IAAkB;IAAe;;;UAGlB;EACf;EACA;EACA;;UAGe;;EAEf;;EAEA;EACA;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,OAAO;;EAEP;EACA,eAAe;;EAEf;;EAEA,SAAS;;;;;;wBAOK,eAAe,SAAS,wBAAwB;;wBAgKhD,qBAAqB,QAAQ;;;;KCvQjC;;UAGK;EACf;EACA;EACA;EACA,KAAK;;UAGU,0BAA0B;;EAEzC;EACA;;EAEA;EACA;;EAEA;;EAEA;;EAEA;EACA;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;EAEA;EACA;EACA;EACA;EACA;;EAEA;EACA;;EAEA;;EAEA;;EAEA;;EAEA;;;UAIe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;EAEA;EACA;EACA;EACA;EACA;;;KAIU,wBACV,SAAS,qBACT,UAAU;EAGN;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;EAEA;EAAiB;;UAEN;EACf,MAAM;EACN;EACA;;KAGU;UASK;EACf;EACA;EACA;EACA;EACA;;EAEA;;EAEA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,UAAU,OACR;IAEE;IACA;IACA;IACA;;EAGJ;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;IAAqB;IAAa;;EAClC;EACA;IAA0B;IAAa;;EACvC;;EAEA;EACA;EACA;EACA;EACA;EACA,SAAS;;EAET;;;wBAwBc,6BAA6B,eAAe;;wBA6B5C,kCAAkC,eAAe;;wBAiFjD,yBAAyB,MAAM,uBAAuB;;wBAOtD,uBACd,QAAQ,4BACR,QAAQ,qBACR,eAAe,qBACf,cAAc,uBACb;;;;UC3Sc,oCAAoC;EACnD;EACA;;;UAIe;EACf;EACA;EACA;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA;EACA;EACA;;UAGe;;EAEf;EACA,QAAQ;;EAER;;EAEA;EACA,sBAAsB;EACtB,mBAAmB;EACnB,qBAAqB;EACrB,kBAAkB;;;;;;;wBA4DJ,8BACd,OAAO,6BACN"}
|