@tangle-network/agent-eval 0.216.0 → 0.217.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +141 -57
- package/README.md +2 -2
- package/dist/adapters/http.d.ts +3 -3
- package/dist/{agent-profile-CIA2Atnt.d.ts → agent-profile-DTt8uGP9.d.ts} +5 -18
- package/dist/agent-profile-DTt8uGP9.d.ts.map +1 -0
- package/dist/{agent-profile-cell-BYc0bxsL.js → agent-profile-cell-B1V1gUZs.js} +4 -53
- package/dist/agent-profile-cell-B1V1gUZs.js.map +1 -0
- package/dist/{agent-profile-cell-BNp-Js9F.d.ts → agent-profile-cell-BcpFbNd5.d.ts} +2 -22
- package/dist/agent-profile-cell-BcpFbNd5.d.ts.map +1 -0
- package/dist/{agreement-irr-DsTURxdr.js → agreement-irr-BhAqjluA.js} +5 -5
- package/dist/agreement-irr-BhAqjluA.js.map +1 -0
- package/dist/analyst/index.d.ts +37 -170
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +17 -192
- package/dist/analyst/index.js.map +1 -1
- package/dist/{analyze-runs-3xpby3p2.js → analyze-runs-DqPJaJ4X.js} +763 -555
- package/dist/analyze-runs-DqPJaJ4X.js.map +1 -0
- package/dist/{analyze-runs-CSDAh8A7.d.ts → analyze-runs-DyeNcenW.d.ts} +18 -16
- package/dist/analyze-runs-DyeNcenW.d.ts.map +1 -0
- package/dist/authenticity/index.d.ts +0 -23
- package/dist/authenticity/index.d.ts.map +1 -1
- package/dist/authenticity/index.js +1 -34
- package/dist/authenticity/index.js.map +1 -1
- package/dist/{benchmark-command-CByl9dxG.js → benchmark-command-xQK8nOI6.js} +591 -109
- package/dist/benchmark-command-xQK8nOI6.js.map +1 -0
- package/dist/benchmarks/index.d.ts +5 -6
- package/dist/benchmarks/index.d.ts.map +1 -1
- package/dist/benchmarks/index.js +4 -4
- package/dist/benchmarks/index.js.map +1 -1
- package/dist/builder-eval/index.d.ts +3 -3
- package/dist/builder-eval/index.js +2 -2
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/campaign/index.d.ts +8 -8
- package/dist/campaign/index.js +15 -15
- package/dist/{campaign-BQ-Jujuj.js → campaign-CJGkxAQ6.js} +256 -765
- package/dist/campaign-CJGkxAQ6.js.map +1 -0
- package/dist/{campaign-evidence-OcUciD-N.js → campaign-evidence-CQYw8_0H.js} +10 -18
- package/dist/{campaign-evidence-OcUciD-N.js.map → campaign-evidence-CQYw8_0H.js.map} +1 -1
- package/dist/{canonical-BsZVgBhW.js → canonical-Dq2hOQ8L.js} +2 -2
- package/dist/{canonical-BsZVgBhW.js.map → canonical-Dq2hOQ8L.js.map} +1 -1
- package/dist/cell-spend-DEWR3nj-.js +90 -0
- package/dist/cell-spend-DEWR3nj-.js.map +1 -0
- package/dist/{chat-client-DthPOYGd.js → chat-client-ldqxYYTN.js} +2 -2
- package/dist/{chat-client-DthPOYGd.js.map → chat-client-ldqxYYTN.js.map} +1 -1
- package/dist/{chat-json-call-BrKsyspg.js → chat-json-call-DavzexSp.js} +3 -2
- package/dist/chat-json-call-DavzexSp.js.map +1 -0
- package/dist/{claim-BXcZw-jo.js → claim-BPVuyHuy.js} +4 -4
- package/dist/claim-BPVuyHuy.js.map +1 -0
- package/dist/cli.js +113 -12
- package/dist/cli.js.map +1 -1
- package/dist/{client-BzL9niqi.d.ts → client-hwo3cJjR.d.ts} +3 -16
- package/dist/client-hwo3cJjR.d.ts.map +1 -0
- package/dist/{cluster-bootstrap-B3JywQQn.js → cluster-bootstrap-DEwwhl0C.js} +3 -3
- package/dist/{cluster-bootstrap-B3JywQQn.js.map → cluster-bootstrap-DEwwhl0C.js.map} +1 -1
- package/dist/{produced-state-ZCtDBxAE.js → completion-verifier-B_jw6C9p.js} +9 -64
- package/dist/completion-verifier-B_jw6C9p.js.map +1 -0
- package/dist/contract/index.d.ts +13 -13
- package/dist/contract/index.js +1041 -1473
- package/dist/contract/index.js.map +1 -1
- package/dist/{cost-ledger-D2FnJPBF.js → cost-ledger-D0352Z4d.js} +832 -817
- package/dist/cost-ledger-D0352Z4d.js.map +1 -0
- package/dist/{cost-ledger-CO7UyEcQ.d.ts → cost-ledger-Jk62OBfF.d.ts} +12 -47
- package/dist/cost-ledger-Jk62OBfF.d.ts.map +1 -0
- package/dist/dataset-DQqhOCPt.d.ts.map +1 -1
- package/dist/{default-registry-Cn1LQ2Ve.js → default-registry-ClYjq3QE.js} +8 -11
- package/dist/default-registry-ClYjq3QE.js.map +1 -0
- package/dist/{default-registry-BrWh62zf.d.ts → default-registry-DXgLmoDI.d.ts} +7 -50
- package/dist/default-registry-DXgLmoDI.d.ts.map +1 -0
- package/dist/{define-agent-eval-Ch3MWX6k.js → define-agent-eval-BDxNlP3D.js} +10 -10
- package/dist/define-agent-eval-BDxNlP3D.js.map +1 -0
- package/dist/{define-agent-eval-CEWkX5wZ.d.ts → define-agent-eval-BIkH_gAR.d.ts} +9 -9
- package/dist/define-agent-eval-BIkH_gAR.d.ts.map +1 -0
- package/dist/{descriptive-CnU8Yq29.js → descriptive-CVNy48rR.js} +2 -2
- package/dist/{descriptive-CnU8Yq29.js.map → descriptive-CVNy48rR.js.map} +1 -1
- package/dist/diagnosis/index.d.ts +4 -14
- package/dist/diagnosis/index.d.ts.map +1 -1
- package/dist/diagnosis/index.js +67 -28
- package/dist/diagnosis/index.js.map +1 -1
- package/dist/{dspy-rlm-engine-D-KX_e64.js → dspy-rlm-engine-DhdCr1Dc.js} +5 -10
- package/dist/dspy-rlm-engine-DhdCr1Dc.js.map +1 -0
- package/dist/{emitter-HcCmFQl_.d.ts → emitter-DABYYTcH.d.ts} +5 -5
- package/dist/emitter-DABYYTcH.d.ts.map +1 -0
- package/dist/emitter-DbMXUIVP.js.map +1 -1
- package/dist/{engine-C_zNWfM8.d.ts → engine-BvZ0R-36.d.ts} +18 -22
- package/dist/engine-BvZ0R-36.d.ts.map +1 -0
- package/dist/{equal-terms-Blx9Zk5I.js → equal-terms-DbPGcVtI.js} +2 -2
- package/dist/equal-terms-DbPGcVtI.js.map +1 -0
- package/dist/{errors-Dngq5h35.js → errors-BcOs5MKW.js} +6 -8
- package/dist/{errors-Dngq5h35.js.map → errors-BcOs5MKW.js.map} +1 -1
- package/dist/{eval-campaign-4byTtqrf.js → eval-campaign-BLEfBGbA.js} +6 -6
- package/dist/{eval-campaign-4byTtqrf.js.map → eval-campaign-BLEfBGbA.js.map} +1 -1
- package/dist/{evaluation-Cn2dRZIz.d.ts → evaluation-jTjC_QQD.d.ts} +8 -8
- package/dist/evaluation-jTjC_QQD.d.ts.map +1 -0
- package/dist/evaluation.d.ts +2 -2
- package/dist/evaluation.js +1 -1
- package/dist/evaluation.js.map +1 -1
- package/dist/{exact-types-CD0lf5Qd.d.ts → exact-types-CflbJ9f3.d.ts} +2 -2
- package/dist/{exact-types-CD0lf5Qd.d.ts.map → exact-types-CflbJ9f3.d.ts.map} +1 -1
- package/dist/{exec-ChJ6d5if.js → exec-CpbtOXVt.js} +2 -24
- package/dist/{exec-ChJ6d5if.js.map → exec-CpbtOXVt.js.map} +1 -1
- package/dist/execution-tracks-CpgFPpS5.js.map +1 -1
- package/dist/experiment/index.d.ts +331 -33
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +303 -101
- package/dist/experiment/index.js.map +1 -1
- package/dist/{exporters-X4xwla8A.js → exporters-Cl6IeNAu.js} +3 -3
- package/dist/exporters-Cl6IeNAu.js.map +1 -0
- package/dist/{exporters-ChTnR90l.d.ts → exporters-Dk3oEbK_.d.ts} +2 -4
- package/dist/{exporters-ChTnR90l.d.ts.map → exporters-Dk3oEbK_.d.ts.map} +1 -1
- package/dist/{external-optimizer-process-nnsY5D_4.js → external-optimizer-process-KVFGvYX1.js} +4 -3
- package/dist/external-optimizer-process-KVFGvYX1.js.map +1 -0
- package/dist/{external-optimizer-subprocess-DcerlZSG.js → external-optimizer-subprocess-XIHBOSKt.js} +551 -553
- package/dist/external-optimizer-subprocess-XIHBOSKt.js.map +1 -0
- package/dist/{extract-usage-BrnzRJ6H.js → extract-usage-BZLU01kR.js} +2 -23
- package/dist/{extract-usage-BrnzRJ6H.js.map → extract-usage-BZLU01kR.js.map} +1 -1
- package/dist/{failure-cluster-CbgEDcFJ.d.ts → failure-cluster-DN0t63Z0.d.ts} +4 -43
- package/dist/failure-cluster-DN0t63Z0.d.ts.map +1 -0
- package/dist/{failure-taxonomy-CjabTATj.js → failure-taxonomy-Bx5oWqaW.js} +2 -19
- package/dist/{failure-taxonomy-CjabTATj.js.map → failure-taxonomy-Bx5oWqaW.js.map} +1 -1
- package/dist/{feedback-trajectory-DkaDj0dj.d.ts → feedback-trajectory-BWmhRBGu.d.ts} +8 -13
- package/dist/feedback-trajectory-BWmhRBGu.d.ts.map +1 -0
- package/dist/{semantic-concept-judge-zQBG9hHU.js → findings-store-BMVQtEsm.js} +13 -255
- package/dist/findings-store-BMVQtEsm.js.map +1 -0
- package/dist/fuzz.d.ts +20 -63
- package/dist/fuzz.d.ts.map +1 -1
- package/dist/fuzz.js +62 -108
- package/dist/fuzz.js.map +1 -1
- package/dist/{hf-dataset-BR2yhuNn.js → hf-dataset-D9f7-oX7.js} +4 -24
- package/dist/hf-dataset-D9f7-oX7.js.map +1 -0
- package/dist/hosted/index.d.ts +2 -2
- package/dist/hosted/index.js +2 -2
- package/dist/{index-DHKEAv3s.d.ts → index-9jJ2fKSX.d.ts} +2 -64
- package/dist/index-9jJ2fKSX.d.ts.map +1 -0
- package/dist/{index-oQ4aHziK.d.ts → index-BLGqZ81N.d.ts} +11 -16
- package/dist/{index-oQ4aHziK.d.ts.map → index-BLGqZ81N.d.ts.map} +1 -1
- package/dist/{index-DlH0Y9gI.d.ts → index-BiT6RtNH.d.ts} +31 -43
- package/dist/index-BiT6RtNH.d.ts.map +1 -0
- package/dist/{index-Ch9VxCpK.d.ts → index-CX-GAUpw.d.ts} +4 -8
- package/dist/index-CX-GAUpw.d.ts.map +1 -0
- package/dist/{index-CPJJr98o.d.ts → index-CiRwLv2R.d.ts} +5 -68
- package/dist/index-CiRwLv2R.d.ts.map +1 -0
- package/dist/{index-NvGBiO8H.d.ts → index-ZuWx52WG.d.ts} +266 -495
- package/dist/index-ZuWx52WG.d.ts.map +1 -0
- package/dist/index.d.ts +2256 -2165
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +514 -1192
- package/dist/index.js.map +1 -1
- package/dist/{integrity-BV1ZDf42.js → integrity-DOd9Q86O.js} +7 -12
- package/dist/integrity-DOd9Q86O.js.map +1 -0
- package/dist/{integrity-BvnVARVl.js → integrity-W4CADcFc.js} +3 -7
- package/dist/{integrity-BvnVARVl.js.map → integrity-W4CADcFc.js.map} +1 -1
- package/dist/{integrity-CCAxGVK6.d.ts → integrity-_qgZLA5p.d.ts} +4 -6
- package/dist/{integrity-CCAxGVK6.d.ts.map → integrity-_qgZLA5p.d.ts.map} +1 -1
- package/dist/{internal-CRHwS9iz.js → internal-BTK91Bqt.js} +3 -3
- package/dist/{internal-CRHwS9iz.js.map → internal-BTK91Bqt.js.map} +1 -1
- package/dist/interval-Dq9gt7jx.js +219 -0
- package/dist/interval-Dq9gt7jx.js.map +1 -0
- package/dist/interval-ohuOJurX.d.ts +47 -0
- package/dist/interval-ohuOJurX.d.ts.map +1 -0
- package/dist/{journal-DvKCvjty.js → journal-CR4QikYp.js} +110 -110
- package/dist/journal-CR4QikYp.js.map +1 -0
- package/dist/json-BhdsJaOQ.js +111 -0
- package/dist/json-BhdsJaOQ.js.map +1 -0
- package/dist/{judge-calibration-N44V04Se.js → judge-calibration-BkXMIWA4.js} +3 -3
- package/dist/{judge-calibration-N44V04Se.js.map → judge-calibration-BkXMIWA4.js.map} +1 -1
- package/dist/{judge-gate-B1qYnkpi.js → judge-gate-Co5tuh57.js} +4 -27
- package/dist/judge-gate-Co5tuh57.js.map +1 -0
- package/dist/{judge-gate-DNcUrRFA.d.ts → judge-gate-DsL0yFcb.d.ts} +9 -10
- package/dist/judge-gate-DsL0yFcb.d.ts.map +1 -0
- package/dist/{kind-factory-CbZ-6tvu.js → kind-factory-Gy4fXixd.js} +88 -17
- package/dist/kind-factory-Gy4fXixd.js.map +1 -0
- package/dist/ledger-core/index.d.ts +2 -2
- package/dist/ledger-core/index.js +4 -4
- package/dist/{ledger-core-B2hahzrt.js → ledger-core-DrJhbms9.js} +5 -84
- package/dist/ledger-core-DrJhbms9.js.map +1 -0
- package/dist/{llm-client-DQveoxSM.js → llm-client-BshCREn6.js} +90 -219
- package/dist/llm-client-BshCREn6.js.map +1 -0
- package/dist/{llm-judge-PXV7Xljl.js → llm-judge-C8jMTNNG.js} +501 -931
- package/dist/llm-judge-C8jMTNNG.js.map +1 -0
- package/dist/matrix/index.d.ts +4 -4
- package/dist/matrix/index.js +3 -2
- package/dist/{matrix-_BV9-Lk5.js → matrix-B8l-Nsxd.js} +7 -96
- package/dist/matrix-B8l-Nsxd.js.map +1 -0
- package/dist/{matrix-CHMiebFx.d.ts → matrix-DHe-dwVJ.d.ts} +10 -19
- package/dist/matrix-DHe-dwVJ.d.ts.map +1 -0
- package/dist/{paired-tests-jbwPdSz5.js → mean-BbA_JCzW.js} +45 -5
- package/dist/mean-BbA_JCzW.js.map +1 -0
- package/dist/meta-eval/index.d.ts +6 -6
- package/dist/meta-eval/index.js +10 -10
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/metrics-BKaPBtIG.js.map +1 -1
- package/dist/mint-BRZXuwRi.d.ts +81 -0
- package/dist/mint-BRZXuwRi.d.ts.map +1 -0
- package/dist/{mint-Csor1BLJ.js → mint-zszyYlpA.js} +4 -4
- package/dist/{mint-Csor1BLJ.js.map → mint-zszyYlpA.js.map} +1 -1
- package/dist/{multi-layer-verifier-BUaQ4C17.d.ts → multi-layer-verifier-DEgMUk6v.d.ts} +2 -2
- package/dist/{multi-layer-verifier-BUaQ4C17.d.ts.map → multi-layer-verifier-DEgMUk6v.d.ts.map} +1 -1
- package/dist/{multiplicity-D9cjmStq.js → multiplicity-Sg0U7h8j.js} +2 -2
- package/dist/{multiplicity-D9cjmStq.js.map → multiplicity-Sg0U7h8j.js.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -23
- package/dist/multishot/golden/index.d.ts.map +1 -1
- package/dist/multishot/golden/index.js +5 -8
- package/dist/multishot/golden/index.js.map +1 -1
- package/dist/multishot/index.d.ts +3 -12
- package/dist/multishot/index.d.ts.map +1 -1
- package/dist/multishot/index.js +4 -3
- package/dist/multishot/index.js.map +1 -1
- package/dist/nonnegative-number-DvdMxg_f.js +9 -0
- package/dist/nonnegative-number-DvdMxg_f.js.map +1 -0
- package/dist/openapi.json +114 -10
- package/dist/{otel-spans-C8YIJWy1.js → otel-spans-CdN909k1.js} +3 -3
- package/dist/{otel-spans-C8YIJWy1.js.map → otel-spans-CdN909k1.js.map} +1 -1
- package/dist/{paired-arms-DzKn0TfY.js → paired-arms-Ct4LTKc1.js} +32 -9
- package/dist/paired-arms-Ct4LTKc1.js.map +1 -0
- package/dist/{paired-binary-Q36bhn2R.js → paired-binary-B_nejA44.js} +3 -36
- package/dist/paired-binary-B_nejA44.js.map +1 -0
- package/dist/{paired-promotion-decision-DPKz8ZnF.d.ts → paired-promotion-decision-CQMJ5xGN.d.ts} +2 -34
- package/dist/paired-promotion-decision-CQMJ5xGN.d.ts.map +1 -0
- package/dist/{paired-promotion-decision-BN6IQTiu.js → paired-promotion-decision-D0SHb-ZL.js} +3 -3
- package/dist/{paired-promotion-decision-BN6IQTiu.js.map → paired-promotion-decision-D0SHb-ZL.js.map} +1 -1
- package/dist/{search-kernel-BztlERLv.js → pareto-BzsxidFD.js} +883 -861
- package/dist/pareto-BzsxidFD.js.map +1 -0
- package/dist/pipelines/index.d.ts +16 -19
- package/dist/pipelines/index.d.ts.map +1 -1
- package/dist/pipelines/index.js +4 -68
- package/dist/pipelines/index.js.map +1 -1
- package/dist/{power-and-mde-fK5Qocwo.js → power-and-mde-N5nU4_Ly.js} +3 -3
- package/dist/{power-and-mde-fK5Qocwo.js.map → power-and-mde-N5nU4_Ly.js.map} +1 -1
- package/dist/predicate-FdKmElA_.js +38 -0
- package/dist/predicate-FdKmElA_.js.map +1 -0
- package/dist/{prime-protocol-DcAK9pC8.js → prime-protocol-BFgGrEMP.js} +3 -3
- package/dist/prime-protocol-BFgGrEMP.js.map +1 -0
- package/dist/{prime-protocol-CEE4BR9N.d.ts → prime-protocol-BWG2DQIk.d.ts} +4 -22
- package/dist/prime-protocol-BWG2DQIk.d.ts.map +1 -0
- package/dist/profile-cell.d.ts +2 -2
- package/dist/profile-cell.js +2 -2
- package/dist/{release-confidence-BK2QCB5M.d.ts → promotion-gate-DaY95yU6.d.ts} +105 -106
- package/dist/promotion-gate-DaY95yU6.d.ts.map +1 -0
- package/dist/{promotion-policy-CWW35UWq.js → promotion-policy-6A8osGr_.js} +4 -4
- package/dist/{promotion-policy-CWW35UWq.js.map → promotion-policy-6A8osGr_.js.map} +1 -1
- package/dist/{proposal-findings-BZiMTzAF.js → proposal-findings-BbWidvUk.js} +3 -3
- package/dist/proposal-findings-BbWidvUk.js.map +1 -0
- package/dist/{query-B1D1qp2Z.js → query-B_RZ36an.js} +5 -5
- package/dist/query-B_RZ36an.js.map +1 -0
- package/dist/{query-DK1nWcmp.d.ts → query-Cmv_U8vf.d.ts} +4 -24
- package/dist/{query-DK1nWcmp.d.ts.map → query-Cmv_U8vf.d.ts.map} +1 -1
- package/dist/{random-Dn5fPWkt.js → random-DDDcC7YU.js} +2 -2
- package/dist/{random-Dn5fPWkt.js.map → random-DDDcC7YU.js.map} +1 -1
- package/dist/{raw-provider-sink-rB2DYxr6.js → raw-provider-sink-75w4FWBj.js} +2 -2
- package/dist/{raw-provider-sink-rB2DYxr6.js.map → raw-provider-sink-75w4FWBj.js.map} +1 -1
- package/dist/{redact-Cveoq6fJ.d.ts → redact-DXdlFb_U.d.ts} +4 -21
- package/dist/redact-DXdlFb_U.d.ts.map +1 -0
- package/dist/{redact-Dk1ZdJEm.js → redact-mtkTiJVk.js} +92 -98
- package/dist/redact-mtkTiJVk.js.map +1 -0
- package/dist/{registry-JYfPxcVz.d.ts → registry-ssNoHzwD.d.ts} +5 -5
- package/dist/{registry-JYfPxcVz.d.ts.map → registry-ssNoHzwD.d.ts.map} +1 -1
- package/dist/{release-confidence-Dbe9AS_X.js → release-confidence-CYIFH-fp.js} +6 -11
- package/dist/release-confidence-CYIFH-fp.js.map +1 -0
- package/dist/{report-command-y8H6RSX8.js → render-C-w2EZ6p.js} +331 -554
- package/dist/render-C-w2EZ6p.js.map +1 -0
- package/dist/reporting-html-Kvlc8oMt.js +100 -0
- package/dist/reporting-html-Kvlc8oMt.js.map +1 -0
- package/dist/reporting.d.ts +6 -6
- package/dist/reporting.js +8 -8
- package/dist/{researcher-f6mZhh58.d.ts → researcher-0ABVIdGa.d.ts} +9 -9
- package/dist/{researcher-f6mZhh58.d.ts.map → researcher-0ABVIdGa.d.ts.map} +1 -1
- package/dist/reward-DrZ71vnf.d.ts +55 -0
- package/dist/reward-DrZ71vnf.d.ts.map +1 -0
- package/dist/{reward-hacking-BFcQgkux.js → reward-hacking-ByuETaIb.js} +3 -3
- package/dist/{reward-hacking-BFcQgkux.js.map → reward-hacking-ByuETaIb.js.map} +1 -1
- package/dist/{reward-hacking-DHA7c1l8.d.ts → reward-hacking-E52wq86n.d.ts} +5 -18
- package/dist/reward-hacking-E52wq86n.d.ts.map +1 -0
- package/dist/reward-nw2xZGZG.js.map +1 -1
- package/dist/rl.d.ts +117 -295
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +27 -747
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +5 -4
- package/dist/rollout/index.js +7 -8
- package/dist/{rollout-D9bKKBnZ.js → rollout-D6C50mfA.js} +88 -13
- package/dist/rollout-D6C50mfA.js.map +1 -0
- package/dist/round-decimal-COrKglMh.js +10 -0
- package/dist/round-decimal-COrKglMh.js.map +1 -0
- package/dist/{rubric-predictive-validity-BsgkBfT0.js → rubric-predictive-validity-D3_oI72N.js} +3 -3
- package/dist/{rubric-predictive-validity-BsgkBfT0.js.map → rubric-predictive-validity-D3_oI72N.js.map} +1 -1
- package/dist/{rubric-predictive-validity-2SgHxoa5.d.ts → rubric-predictive-validity-DiwT8PbL.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-2SgHxoa5.d.ts.map → rubric-predictive-validity-DiwT8PbL.d.ts.map} +1 -1
- package/dist/{run-record-D1Hw-Jml.d.ts → run-record-CaxbLq6Y.d.ts} +5 -11
- package/dist/run-record-CaxbLq6Y.d.ts.map +1 -0
- package/dist/{run-record-ES07wfz9.js → run-record-CysKLB93.js} +3 -5
- package/dist/run-record-CysKLB93.js.map +1 -0
- package/dist/{run-record-DlTZ_lqe.js → run-record-DhEJgoPz.js} +4 -4
- package/dist/{run-record-DlTZ_lqe.js.map → run-record-DhEJgoPz.js.map} +1 -1
- package/dist/sandbox-harness-BlSOu4LX.d.ts.map +1 -1
- package/dist/{schema-CSf6qWgZ.js → schema-Bt2vLFm3.js} +2 -14
- package/dist/schema-Bt2vLFm3.js.map +1 -0
- package/dist/schema-C-ziyJD7.d.ts.map +1 -1
- package/dist/{schema-2g7trUwH.d.ts → schema-D6EVKzTB.d.ts} +4 -5
- package/dist/schema-D6EVKzTB.d.ts.map +1 -0
- package/dist/{schema-BuiujZ-g.js → schema-DxcKTzbi.js} +2 -2
- package/dist/schema-DxcKTzbi.js.map +1 -0
- package/dist/search/index.d.ts +39 -53
- package/dist/search/index.d.ts.map +1 -1
- package/dist/search/index.js +4 -4
- package/dist/{search-kernel-CpcKH5Ih.d.ts → search-kernel-DxqjBxA-.d.ts} +18 -31
- package/dist/search-kernel-DxqjBxA-.d.ts.map +1 -0
- package/dist/{search-ledger-d7V8o9Lk.js → search-ledger-TUvB5gWH.js} +189 -199
- package/dist/search-ledger-TUvB5gWH.js.map +1 -0
- package/dist/{search-ledger-file-BqOo-bVo.js → search-ledger-file-UKCO2r8a.js} +4 -4
- package/dist/search-ledger-file-UKCO2r8a.js.map +1 -0
- package/dist/search-ledger-types-p2xKfiDJ.d.ts.map +1 -1
- package/dist/{search-shipper-er4pK5_Y.js → search-shipper-Cw2z3-td.js} +4 -4
- package/dist/search-shipper-Cw2z3-td.js.map +1 -0
- package/dist/{search-summary-CBvz3u_S.js → search-summary-DQIvJsSB.js} +5 -5
- package/dist/{search-summary-CBvz3u_S.js.map → search-summary-DQIvJsSB.js.map} +1 -1
- package/dist/{sequential-CzK5DarL.js → sequential-BVv1sjsw.js} +2 -32
- package/dist/{sequential-CzK5DarL.js.map → sequential-BVv1sjsw.js.map} +1 -1
- package/dist/{sequential-D-1PPCmM.js → sequential-CimQrbjv.js} +5 -5
- package/dist/sequential-CimQrbjv.js.map +1 -0
- package/dist/{sequential-BhsrMupG.d.ts → sequential-KwGspfeR.d.ts} +2 -9
- package/dist/{sequential-BhsrMupG.d.ts.map → sequential-KwGspfeR.d.ts.map} +1 -1
- package/dist/{sequential-eprocess-D1jKoihe.js → sequential-eprocess-DodI6RoF.js} +2 -2
- package/dist/{sequential-eprocess-D1jKoihe.js.map → sequential-eprocess-DodI6RoF.js.map} +1 -1
- package/dist/{server-DbcC7xMl.js → server-DoVbqqtJ.js} +101 -101
- package/dist/server-DoVbqqtJ.js.map +1 -0
- package/dist/{single-run-lock-fVu3MZWH.js → single-run-lock-U58yf8N6.js} +2 -2
- package/dist/{single-run-lock-fVu3MZWH.js.map → single-run-lock-U58yf8N6.js.map} +1 -1
- package/dist/{skillopt-optimization-method-CCtzralq.js → skillopt-optimization-method-kyL94-ri.js} +428 -91
- package/dist/skillopt-optimization-method-kyL94-ri.js.map +1 -0
- package/dist/{spans-D9_ehFuk.d.ts → spans-CVngGij3.d.ts} +2 -2
- package/dist/{spans-D9_ehFuk.d.ts.map → spans-CVngGij3.d.ts.map} +1 -1
- package/dist/{statistical-heldout-1Gbmex7v.js → statistical-heldout-Bksv92Th.js} +3 -3
- package/dist/statistical-heldout-Bksv92Th.js.map +1 -0
- package/dist/{statistical-heldout-BeR4bOdg.d.ts → statistical-heldout-C9AbgREN.d.ts} +8 -67
- package/dist/statistical-heldout-C9AbgREN.d.ts.map +1 -0
- package/dist/{steps-Bg5-KVx1.d.ts → steps-DlepJPwD.d.ts} +2 -56
- package/dist/steps-DlepJPwD.d.ts.map +1 -0
- package/dist/{store-C4avFjIi.d.ts → store-B3nWRy3B.d.ts} +2 -2
- package/dist/{store-C4avFjIi.d.ts.map → store-B3nWRy3B.d.ts.map} +1 -1
- package/dist/{store-otlp-DtB4Slsr.js → store-otlp-BteDAbHK.js} +135 -132
- package/dist/store-otlp-BteDAbHK.js.map +1 -0
- package/dist/{store-tool-spans-fZIigrpP.js → store-tool-spans-D5l_OfTJ.js} +9 -9
- package/dist/store-tool-spans-D5l_OfTJ.js.map +1 -0
- package/dist/{store-tool-spans-DR1SrCLK.d.ts → store-tool-spans-DZS0HWVf.d.ts} +157 -165
- package/dist/store-tool-spans-DZS0HWVf.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts +4 -11
- package/dist/storyboard/index.d.ts.map +1 -1
- package/dist/storyboard/index.js +21 -19
- package/dist/storyboard/index.js.map +1 -1
- package/dist/{summary-report-DtdRucM0.js → summary-report-BsXJoYX5.js} +588 -598
- package/dist/summary-report-BsXJoYX5.js.map +1 -0
- package/dist/{summary-report-C-CtItCN.d.ts → summary-report-DM7tlml1.d.ts} +16 -20
- package/dist/summary-report-DM7tlml1.d.ts.map +1 -0
- package/dist/supervisor-run/index.d.ts +189 -153
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +58 -450
- package/dist/supervisor-run/index.js.map +1 -1
- package/dist/{systemone-CpTLAlqb.d.ts → systemone-Bp_Ac0p1.d.ts} +6 -6
- package/dist/{systemone-CpTLAlqb.d.ts.map → systemone-Bp_Ac0p1.d.ts.map} +1 -1
- package/dist/systemone-protocol.js +1 -1
- package/dist/{systemone-review-B_yo1xsd.d.ts → systemone-review-CwE6JGIp.d.ts} +2 -2
- package/dist/{systemone-review-B_yo1xsd.d.ts.map → systemone-review-CwE6JGIp.d.ts.map} +1 -1
- package/dist/{systemone-review-BZWfrtq1.js → systemone-review-DzdOJ8I6.js} +3 -3
- package/dist/{systemone-review-BZWfrtq1.js.map → systemone-review-DzdOJ8I6.js.map} +1 -1
- package/dist/systemone.d.ts +5 -5
- package/dist/systemone.js +5 -5
- package/dist/systemone.js.map +1 -1
- package/dist/{task-failure-attributes-DngaLdE4.js → task-failure-attributes-BMsblaDh.js} +3 -3
- package/dist/task-failure-attributes-BMsblaDh.js.map +1 -0
- package/dist/{terminal-record-BHocwFw1.js → terminal-record-78g6E4vh.js} +17 -13
- package/dist/terminal-record-78g6E4vh.js.map +1 -0
- package/dist/{tool-groups-mcxOXVlb.d.ts → tool-groups-DyNz2JZ_.d.ts} +3 -3
- package/dist/tool-groups-DyNz2JZ_.d.ts.map +1 -0
- package/dist/{tool-waste-Wi4g6Ekb.d.ts → tool-waste-CkjzWmeY.d.ts} +53 -56
- package/dist/tool-waste-CkjzWmeY.d.ts.map +1 -0
- package/dist/{tool-waste-D52PqU9Q.js → tool-waste-DLsjS_p-.js} +5 -5
- package/dist/{tool-waste-D52PqU9Q.js.map → tool-waste-DLsjS_p-.js.map} +1 -1
- package/dist/{trace-questions-DG7nqLQ7.d.ts → trace-questions-xxwgsuMi.d.ts} +5 -10
- package/dist/{trace-questions-DG7nqLQ7.d.ts.map → trace-questions-xxwgsuMi.d.ts.map} +1 -1
- package/dist/trace-repair/index.d.ts +15 -212
- package/dist/trace-repair/index.d.ts.map +1 -1
- package/dist/trace-repair/index.js +865 -987
- package/dist/trace-repair/index.js.map +1 -1
- package/dist/traces.d.ts +76 -86
- package/dist/traces.d.ts.map +1 -1
- package/dist/traces.js +13 -13
- package/dist/traces.js.map +1 -1
- package/dist/trajectory-replay/index.d.ts +3 -117
- package/dist/trajectory-replay/index.d.ts.map +1 -1
- package/dist/trajectory-replay/index.js +98 -56
- package/dist/trajectory-replay/index.js.map +1 -1
- package/dist/{tree-rXLW3M_4.js → tree-BGRglkKd.js} +2355 -2414
- package/dist/tree-BGRglkKd.js.map +1 -0
- package/dist/{types-B3jzCp0p.js → types-BXgASBCL.js} +2 -2
- package/dist/{types-B3jzCp0p.js.map → types-BXgASBCL.js.map} +1 -1
- package/dist/{types-oKRhQAde.d.ts → types-CW8ewATZ.d.ts} +5 -114
- package/dist/types-CW8ewATZ.d.ts.map +1 -0
- package/dist/{types-BqkVmHfv.d.ts → types-CmXY0PQf.d.ts} +2 -4
- package/dist/types-CmXY0PQf.d.ts.map +1 -0
- package/dist/{types-DBZ7keha.d.ts → types-D-mgRzlh.d.ts} +9 -72
- package/dist/types-D-mgRzlh.d.ts.map +1 -0
- package/dist/{types-B4C5nFw6.d.ts → types-D8GhJoHk.d.ts} +79 -75
- package/dist/types-D8GhJoHk.d.ts.map +1 -0
- package/dist/{types-IXD2HJzu.js → types-DqFMNTgK.js} +2 -4
- package/dist/types-DqFMNTgK.js.map +1 -0
- package/dist/{types-CfZMIzB_.d.ts → types-DtMbFw6U.d.ts} +3 -3
- package/dist/{types-CfZMIzB_.d.ts.map → types-DtMbFw6U.d.ts.map} +1 -1
- package/dist/{verdict-DhPoXApk.js → verdict-BGQmxQ8A.js} +3 -3
- package/dist/verdict-BGQmxQ8A.js.map +1 -0
- package/dist/{verdict-E4eRNf7-.d.ts → verdict-C24NELtN.d.ts} +2 -15
- package/dist/{verdict-E4eRNf7-.d.ts.map → verdict-C24NELtN.d.ts.map} +1 -1
- package/dist/{verdict-cache-BQEsVy9n.js → verdict-cache-CL_7tePD.js} +2 -2
- package/dist/verdict-cache-CL_7tePD.js.map +1 -0
- package/dist/wire/index.d.ts +32 -411
- package/dist/wire/index.d.ts.map +1 -1
- package/dist/wire/index.js +2 -2
- package/docs/changes/jev-evidence-review.md +1 -1
- package/docs/changes/jev-product-adoption.md +1 -1
- package/docs/changes/jev-recorded-evidence.md +1 -1
- package/docs/design/mlbenchmarks-book-review.md +1 -1
- package/docs/design/mlbenchmarks-review/probes.mts +23 -31
- package/docs/design/statistics-decisions.md +17 -17
- package/docs/design.md +1 -1
- package/docs/evaluation-integrity.md +1 -1
- package/docs/experiment.md +1 -1
- package/docs/proofs/campaign-attempt-retention-20261001/README.md +2 -2
- package/docs/public-api.md +138 -529
- package/docs/rollout.md +1 -1
- package/docs/trace-repair-admission.md +1 -1
- package/docs/wire-protocol.md +1 -1
- package/package.json +1 -1
- package/dist/active-curriculum-B_Lc9kfO.js +0 -61
- package/dist/active-curriculum-B_Lc9kfO.js.map +0 -1
- package/dist/agent-profile-CIA2Atnt.d.ts.map +0 -1
- package/dist/agent-profile-cell-BNp-Js9F.d.ts.map +0 -1
- package/dist/agent-profile-cell-BYc0bxsL.js.map +0 -1
- package/dist/agreement-irr-DsTURxdr.js.map +0 -1
- package/dist/analyze-runs-3xpby3p2.js.map +0 -1
- package/dist/analyze-runs-CSDAh8A7.d.ts.map +0 -1
- package/dist/ast-DJ6Bc7ab.js +0 -558
- package/dist/ast-DJ6Bc7ab.js.map +0 -1
- package/dist/ast-hI-vjW6J.d.ts +0 -457
- package/dist/ast-hI-vjW6J.d.ts.map +0 -1
- package/dist/baseline-Dm8966du.js +0 -259
- package/dist/baseline-Dm8966du.js.map +0 -1
- package/dist/benchmark-BywPeAUo.js +0 -562
- package/dist/benchmark-BywPeAUo.js.map +0 -1
- package/dist/benchmark-command-CByl9dxG.js.map +0 -1
- package/dist/campaign-BQ-Jujuj.js.map +0 -1
- package/dist/chat-json-call-BrKsyspg.js.map +0 -1
- package/dist/claim-BXcZw-jo.js.map +0 -1
- package/dist/claude-jsonl-dZir12m6.js +0 -86
- package/dist/claude-jsonl-dZir12m6.js.map +0 -1
- package/dist/client-BzL9niqi.d.ts.map +0 -1
- package/dist/cost-ledger-CO7UyEcQ.d.ts.map +0 -1
- package/dist/cost-ledger-D2FnJPBF.js.map +0 -1
- package/dist/counterfactual-CPaWN99-.d.ts +0 -60
- package/dist/counterfactual-CPaWN99-.d.ts.map +0 -1
- package/dist/counterfactual-DtMWvn7D.js +0 -98
- package/dist/counterfactual-DtMWvn7D.js.map +0 -1
- package/dist/default-registry-BrWh62zf.d.ts.map +0 -1
- package/dist/default-registry-Cn1LQ2Ve.js.map +0 -1
- package/dist/define-agent-eval-CEWkX5wZ.d.ts.map +0 -1
- package/dist/define-agent-eval-Ch3MWX6k.js.map +0 -1
- package/dist/dspy-rlm-engine-D-KX_e64.js.map +0 -1
- package/dist/effect-sizes-DiH8MGOH.js +0 -82
- package/dist/effect-sizes-DiH8MGOH.js.map +0 -1
- package/dist/emitter-HcCmFQl_.d.ts.map +0 -1
- package/dist/engine-C_zNWfM8.d.ts.map +0 -1
- package/dist/equal-terms-Blx9Zk5I.js.map +0 -1
- package/dist/evaluation-Cn2dRZIz.d.ts.map +0 -1
- package/dist/exporters-X4xwla8A.js.map +0 -1
- package/dist/external-optimizer-process-nnsY5D_4.js.map +0 -1
- package/dist/external-optimizer-subprocess-DcerlZSG.js.map +0 -1
- package/dist/failure-cluster-CbgEDcFJ.d.ts.map +0 -1
- package/dist/feedback-trajectory-DkaDj0dj.d.ts.map +0 -1
- package/dist/hf-dataset-BR2yhuNn.js.map +0 -1
- package/dist/index-CPJJr98o.d.ts.map +0 -1
- package/dist/index-Ch9VxCpK.d.ts.map +0 -1
- package/dist/index-DHKEAv3s.d.ts.map +0 -1
- package/dist/index-DlH0Y9gI.d.ts.map +0 -1
- package/dist/index-NvGBiO8H.d.ts.map +0 -1
- package/dist/integrity-BV1ZDf42.js.map +0 -1
- package/dist/journal-DvKCvjty.js.map +0 -1
- package/dist/judge-gate-B1qYnkpi.js.map +0 -1
- package/dist/judge-gate-DNcUrRFA.d.ts.map +0 -1
- package/dist/kind-factory-CbZ-6tvu.js.map +0 -1
- package/dist/ledger-core-B2hahzrt.js.map +0 -1
- package/dist/llm-client-DQveoxSM.js.map +0 -1
- package/dist/llm-judge-PXV7Xljl.js.map +0 -1
- package/dist/matrix-CHMiebFx.d.ts.map +0 -1
- package/dist/matrix-_BV9-Lk5.js.map +0 -1
- package/dist/mint-DCGoBYhB.d.ts +0 -155
- package/dist/mint-DCGoBYhB.d.ts.map +0 -1
- package/dist/normal-DsJZtrJc.js +0 -25
- package/dist/normal-DsJZtrJc.js.map +0 -1
- package/dist/paired-arms-DzKn0TfY.js.map +0 -1
- package/dist/paired-binary-Q36bhn2R.js.map +0 -1
- package/dist/paired-promotion-decision-DPKz8ZnF.d.ts.map +0 -1
- package/dist/paired-tests-jbwPdSz5.js.map +0 -1
- package/dist/prime-protocol-CEE4BR9N.d.ts.map +0 -1
- package/dist/prime-protocol-DcAK9pC8.js.map +0 -1
- package/dist/produced-state-ZCtDBxAE.js.map +0 -1
- package/dist/proposal-findings-BZiMTzAF.js.map +0 -1
- package/dist/query-B1D1qp2Z.js.map +0 -1
- package/dist/redact-Cveoq6fJ.d.ts.map +0 -1
- package/dist/redact-Dk1ZdJEm.js.map +0 -1
- package/dist/release-confidence-BK2QCB5M.d.ts.map +0 -1
- package/dist/release-confidence-Dbe9AS_X.js.map +0 -1
- package/dist/report-command-y8H6RSX8.js.map +0 -1
- package/dist/reward-hacking-DHA7c1l8.d.ts.map +0 -1
- package/dist/rollout-D9bKKBnZ.js.map +0 -1
- package/dist/run-record-D1Hw-Jml.d.ts.map +0 -1
- package/dist/run-record-ES07wfz9.js.map +0 -1
- package/dist/schema-2g7trUwH.d.ts.map +0 -1
- package/dist/schema-BuiujZ-g.js.map +0 -1
- package/dist/schema-CSf6qWgZ.js.map +0 -1
- package/dist/search-kernel-BztlERLv.js.map +0 -1
- package/dist/search-kernel-CpcKH5Ih.d.ts.map +0 -1
- package/dist/search-ledger-d7V8o9Lk.js.map +0 -1
- package/dist/search-ledger-file-BqOo-bVo.js.map +0 -1
- package/dist/search-shipper-er4pK5_Y.js.map +0 -1
- package/dist/semantic-concept-judge-zQBG9hHU.js.map +0 -1
- package/dist/sequential-D-1PPCmM.js.map +0 -1
- package/dist/server-DbcC7xMl.js.map +0 -1
- package/dist/skillopt-optimization-method-CCtzralq.js.map +0 -1
- package/dist/statistical-heldout-1Gbmex7v.js.map +0 -1
- package/dist/statistical-heldout-BeR4bOdg.d.ts.map +0 -1
- package/dist/steps-Bg5-KVx1.d.ts.map +0 -1
- package/dist/store-otlp-DtB4Slsr.js.map +0 -1
- package/dist/store-tool-spans-DR1SrCLK.d.ts.map +0 -1
- package/dist/store-tool-spans-fZIigrpP.js.map +0 -1
- package/dist/student-t-BQeKJx2N.js +0 -39
- package/dist/student-t-BQeKJx2N.js.map +0 -1
- package/dist/summary-report-C-CtItCN.d.ts.map +0 -1
- package/dist/summary-report-DtdRucM0.js.map +0 -1
- package/dist/task-failure-attributes-DngaLdE4.js.map +0 -1
- package/dist/terminal-record-BHocwFw1.js.map +0 -1
- package/dist/tool-groups-mcxOXVlb.d.ts.map +0 -1
- package/dist/tool-waste-Wi4g6Ekb.d.ts.map +0 -1
- package/dist/trajectory-Y1fmdlw7.d.ts +0 -24
- package/dist/trajectory-Y1fmdlw7.d.ts.map +0 -1
- package/dist/tree-rXLW3M_4.js.map +0 -1
- package/dist/types-B4C5nFw6.d.ts.map +0 -1
- package/dist/types-BqkVmHfv.d.ts.map +0 -1
- package/dist/types-DBZ7keha.d.ts.map +0 -1
- package/dist/types-IXD2HJzu.js.map +0 -1
- package/dist/types-oKRhQAde.d.ts.map +0 -1
- package/dist/usage-receipt-Cx3xNeyB.js +0 -76
- package/dist/usage-receipt-Cx3xNeyB.js.map +0 -1
- package/dist/verdict-DhPoXApk.js.map +0 -1
- package/dist/verdict-cache-BQEsVy9n.js.map +0 -1
- package/docs/design/self-improvement-evidence-audit.md +0 -263
package/dist/contract/index.js
CHANGED
|
@@ -1,29 +1,30 @@
|
|
|
1
|
-
import { a as hashCanonical } from "../canonical-
|
|
2
|
-
import
|
|
3
|
-
import { t as
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import { a as
|
|
7
|
-
import {
|
|
1
|
+
import { a as hashCanonical } from "../canonical-Dq2hOQ8L.js";
|
|
2
|
+
import "../run-record-DhEJgoPz.js";
|
|
3
|
+
import { r as summarizeExecution, t as analyzeRuns } from "../analyze-runs-DqPJaJ4X.js";
|
|
4
|
+
import { i as pairedBootstrap } from "../mean-BbA_JCzW.js";
|
|
5
|
+
import { t as decidePairedPromotion } from "../paired-promotion-decision-D0SHb-ZL.js";
|
|
6
|
+
import { a as heldoutSignificance } from "../statistical-heldout-Bksv92Th.js";
|
|
7
|
+
import { B as defaultProductionGate, Q as runCampaign, Z as runEval, i as transientDispatchFailure, o as clamp01, p as runImprovementLoop, t as llmJudge } from "../llm-judge-C8jMTNNG.js";
|
|
8
8
|
import { i as isModelPriced, n as estimateCost } from "../metrics-BKaPBtIG.js";
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
import { n as
|
|
15
|
-
import { t as
|
|
16
|
-
import { t as
|
|
17
|
-
import { t as
|
|
9
|
+
import { t as CostLedger } from "../cost-ledger-D0352Z4d.js";
|
|
10
|
+
import { z as campaignSplitDigest } from "../campaign-evidence-CQYw8_0H.js";
|
|
11
|
+
import { o as mapConcurrentRange } from "../journal-CR4QikYp.js";
|
|
12
|
+
import { M as fsCampaignStorage, N as inMemoryCampaignStorage } from "../external-optimizer-subprocess-XIHBOSKt.js";
|
|
13
|
+
import { a as makeProposalFinding } from "../types-DqFMNTgK.js";
|
|
14
|
+
import { n as isRecord } from "../json-BhdsJaOQ.js";
|
|
15
|
+
import { n as SelfImproveRunError, r as selfImprove, t as defineAgentEval } from "../define-agent-eval-BDxNlP3D.js";
|
|
16
|
+
import { t as buildDefaultAnalystRegistry } from "../default-registry-ClYjq3QE.js";
|
|
17
|
+
import { t as extractUsage } from "../extract-usage-BZLU01kR.js";
|
|
18
|
+
import { t as createChatClient } from "../chat-client-ldqxYYTN.js";
|
|
18
19
|
import { n as InMemoryOutcomeStore, t as FileSystemOutcomeStore } from "../outcome-store-DUcMj-3D.js";
|
|
19
|
-
import { c as
|
|
20
|
-
import { n as paretoPolicy, r as paretoSignificanceGate, t as buildEvidenceVector } from "../promotion-policy-
|
|
21
|
-
import { t as fromOtelSpans } from "../otel-spans-
|
|
20
|
+
import { c as composeGate, l as externalTextOptimizationMethod, n as compareOptimizationMethods, r as gepaOptimizationMethod, s as heldOutGate, t as skillOptOptimizationMethod } from "../skillopt-optimization-method-kyL94-ri.js";
|
|
21
|
+
import { n as paretoPolicy, r as paretoSignificanceGate, t as buildEvidenceVector } from "../promotion-policy-6A8osGr_.js";
|
|
22
|
+
import { t as fromOtelSpans } from "../otel-spans-CdN909k1.js";
|
|
22
23
|
import { createHash } from "node:crypto";
|
|
23
24
|
import { agentCandidateBenchmarkSuiteSchema, agentCandidateBenchmarkTaskSchema, agentCandidateBundleSchema, agentCandidateEvaluationPolicySchema, agentCandidateExperimentSchema, agentImprovementMeasuredComparisonSchema, agentProfileImprovementExperimentSchema, agentProfileImprovementMeasuredComparisonSchema, agentProfileImprovementRunCellSchema, agentProfileImprovementRunReceiptSchema, agentProfileImprovementSuiteInputsSchema, agentProfileImprovementSuiteSchema, agentProfileImprovementTaskSchema, candidateExecutionEvidenceSchema, canonicalCandidateDigest, canonicalCandidateJson, numbersApproximatelyEqual, omitTopLevelDigest } from "@tangle-network/agent-interface";
|
|
24
25
|
import { createReadStream } from "node:fs";
|
|
25
|
-
import
|
|
26
|
-
import
|
|
26
|
+
import "node:path";
|
|
27
|
+
import "node:fs/promises";
|
|
27
28
|
import { readSessionInput, summarizeSessionInput } from "@tangle-network/harness-sessions";
|
|
28
29
|
//#region src/contract/fixed-spend.ts
|
|
29
30
|
function addFixedSpend(left, right) {
|
|
@@ -37,6 +38,189 @@ function addFixedSpend(left, right) {
|
|
|
37
38
|
costProvenance: left.costProvenance === "observed" && right.costProvenance === "observed" ? "observed" : "estimated"
|
|
38
39
|
};
|
|
39
40
|
}
|
|
41
|
+
/** Sum both arms in receipt order while retaining the observed/estimated provenance. */
|
|
42
|
+
function pairedFixedSpendReceipt(measurements, model, usageOf) {
|
|
43
|
+
const usage = measurements.reduce((sum, measurement) => addFixedSpend(addFixedSpend(sum, usageOf(measurement.baseline)), usageOf(measurement.candidate)), {
|
|
44
|
+
inputTokens: 0,
|
|
45
|
+
outputTokens: 0,
|
|
46
|
+
cachedInputTokens: 0,
|
|
47
|
+
reasoningTokens: 0,
|
|
48
|
+
modelCalls: 0,
|
|
49
|
+
costUsdNanos: 0,
|
|
50
|
+
costProvenance: "observed"
|
|
51
|
+
});
|
|
52
|
+
const costUsd = usage.costUsdNanos / 1e9;
|
|
53
|
+
return {
|
|
54
|
+
model,
|
|
55
|
+
inputTokens: usage.inputTokens,
|
|
56
|
+
outputTokens: usage.outputTokens,
|
|
57
|
+
cachedTokens: usage.cachedInputTokens,
|
|
58
|
+
reasoningTokens: usage.reasoningTokens,
|
|
59
|
+
...usage.costProvenance === "observed" ? { actualCostUsd: costUsd } : { estimatedCostUsd: costUsd }
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
//#endregion
|
|
63
|
+
//#region src/contract/candidate-experiment-evidence.ts
|
|
64
|
+
/** Verify signed candidate execution receipts and read their measured usage. */
|
|
65
|
+
const candidateExecutionEvidenceAdapter = {
|
|
66
|
+
score: (evidence) => evidence.receipt.benchmarkResult.material.score,
|
|
67
|
+
dimensions: (evidence) => evidence.receipt.benchmarkResult.material.dimensions,
|
|
68
|
+
costUsd: costFromEvidence,
|
|
69
|
+
costProvenance: (evidence) => combinedUsage(evidence).costProvenance,
|
|
70
|
+
latencyMs: latencyFromEvidence,
|
|
71
|
+
completed: completedSuccessfully,
|
|
72
|
+
passed: (evidence) => evidence.receipt.benchmarkResult.material.passed
|
|
73
|
+
};
|
|
74
|
+
function verifyMeasurement(experiment, input, index) {
|
|
75
|
+
const suite = experiment.benchmark.suite;
|
|
76
|
+
const taskIndex = Math.floor(index / suite.reps);
|
|
77
|
+
const repetition = index % suite.reps;
|
|
78
|
+
const task = experiment.benchmark.tasks[taskIndex];
|
|
79
|
+
const seed = suite.seeds[index];
|
|
80
|
+
if (!task || seed === void 0) throw new Error(`candidate experiment measurement ${index} is outside the signed suite`);
|
|
81
|
+
const baseline = verifyExecutionEvidence(input.baseline);
|
|
82
|
+
const candidate = verifyExecutionEvidence(input.candidate);
|
|
83
|
+
for (const [arm, evidence] of [["baseline", baseline], ["candidate", candidate]]) {
|
|
84
|
+
const bundle = experiment[arm];
|
|
85
|
+
const materialization = evidence.materializationReceipt;
|
|
86
|
+
const runCell = materialization.executionPlan.material.runCell;
|
|
87
|
+
verifySelfAddressed(runCell, "candidate run cell");
|
|
88
|
+
if (runCell.experimentDigest !== experiment.digest || runCell.arm !== arm || runCell.bundleDigest !== bundle.digest || runCell.suiteDigest !== suite.digest || runCell.taskDigest !== task.digest || runCell.taskIndex !== taskIndex || runCell.repetition !== repetition || runCell.seed !== seed || runCell.attempt > task.attempt.maxAttempts || materialization.bundleDigest !== bundle.digest || materialization.benchmark.suite.digest !== suite.digest || materialization.benchmark.task.digest !== task.digest || materialization.codeKind !== bundle.code.kind || materialization.profileActivation.profilePlan.material.sourceProfileDigest !== canonicalCandidateDigest(bundle.profile) || evidence.receipt.runCellDigest !== runCell.digest || JSON.stringify(materialization.resolvedModel) !== JSON.stringify(task.model)) throw new Error(`candidate experiment measurement ${index} substituted its ${arm} arm`);
|
|
89
|
+
verifyTaskOutcome(task, evidence, index, arm);
|
|
90
|
+
}
|
|
91
|
+
const baselinePlan = baseline.materializationReceipt.executionPlan.material;
|
|
92
|
+
const candidatePlan = candidate.materializationReceipt.executionPlan.material;
|
|
93
|
+
if (baselinePlan.executionId === candidatePlan.executionId || baselinePlan.runCell.digest === candidatePlan.runCell.digest || baseline.materializationReceipt.digest === candidate.materializationReceipt.digest || baseline.receipt.digest === candidate.receipt.digest || baseline.digest === candidate.digest) throw new Error(`candidate experiment measurement ${index} reused one execution across arms`);
|
|
94
|
+
return {
|
|
95
|
+
baseline,
|
|
96
|
+
candidate
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
function verifyExecutionEvidence(input) {
|
|
100
|
+
const evidence = candidateExecutionEvidenceSchema.parse(input);
|
|
101
|
+
verifySelfAddressed(evidence, "candidate execution evidence");
|
|
102
|
+
verifySelfAddressed(evidence.materializationReceipt, "candidate materialization receipt");
|
|
103
|
+
verifySelfAddressed(evidence.materializationReceipt.profileActivation, "candidate profile activation");
|
|
104
|
+
verifyMaterialAddressed(evidence.materializationReceipt.profileActivation.profilePlan, "candidate profile plan");
|
|
105
|
+
verifyMaterialAddressed(evidence.materializationReceipt.executionPlan, "candidate execution plan");
|
|
106
|
+
verifySelfAddressed(evidence.receipt, "candidate run receipt");
|
|
107
|
+
verifyMaterialAddressed(evidence.receipt.modelSettlement, "candidate model settlement");
|
|
108
|
+
verifyMaterialAddressed(evidence.receipt.taskOutcome, "candidate task outcome");
|
|
109
|
+
verifyMaterialAddressed(evidence.receipt.benchmarkResult, "candidate benchmark result");
|
|
110
|
+
return evidence;
|
|
111
|
+
}
|
|
112
|
+
function verifySelfAddressed(document, label) {
|
|
113
|
+
if (canonicalCandidateDigest(omitTopLevelDigest(document)) !== document.digest) throw new Error(`${label} digest is invalid`);
|
|
114
|
+
}
|
|
115
|
+
function verifyTaskOutcome(task, evidence, index, arm) {
|
|
116
|
+
const outcome = evidence.receipt.taskOutcome.material.outcome;
|
|
117
|
+
const result = evidence.receipt.benchmarkResult.material;
|
|
118
|
+
const prefix = `candidate experiment measurement ${index} ${arm}`;
|
|
119
|
+
if (result.evidence.sha256 === task.grader.artifact.sha256) throw new Error(`${prefix} reused grader bytes as grading evidence`);
|
|
120
|
+
const usage = combinedUsage(evidence);
|
|
121
|
+
const usageChecks = [
|
|
122
|
+
[
|
|
123
|
+
usage.modelCalls,
|
|
124
|
+
task.limits.maxModelCalls,
|
|
125
|
+
"model calls"
|
|
126
|
+
],
|
|
127
|
+
[
|
|
128
|
+
usage.inputTokens,
|
|
129
|
+
task.limits.maxInputTokens,
|
|
130
|
+
"input tokens"
|
|
131
|
+
],
|
|
132
|
+
[
|
|
133
|
+
usage.outputTokens,
|
|
134
|
+
task.limits.maxOutputTokens,
|
|
135
|
+
"output tokens"
|
|
136
|
+
],
|
|
137
|
+
[
|
|
138
|
+
usage.costUsdNanos,
|
|
139
|
+
Math.round(task.limits.maxCostUsd * 1e9),
|
|
140
|
+
"cost"
|
|
141
|
+
]
|
|
142
|
+
];
|
|
143
|
+
for (const [actual, maximum, label] of usageChecks) if (actual > maximum) throw new Error(`${prefix} ${label} ${actual} exceeds the signed limit ${maximum}`);
|
|
144
|
+
if (outcome.kind !== task.outcome.kind) throw new Error(`${prefix} returned an outcome outside the signed task contract`);
|
|
145
|
+
if (task.outcome.kind === "output") {
|
|
146
|
+
if (outcome.kind !== "output" || outcome.spec.mediaType !== task.outcome.mediaType || outcome.spec.maxBytes !== task.outcome.maxBytes) throw new Error(`${prefix} changed the signed output contract`);
|
|
147
|
+
return;
|
|
148
|
+
}
|
|
149
|
+
const repository = task.repository;
|
|
150
|
+
if (outcome.kind !== "workspace" || repository === void 0 || outcome.baseRepository.identity !== repository.identity || outcome.baseRepository.rootIdentity !== repository.rootIdentity || outcome.baseRepository.commit !== repository.baseCommit || outcome.baseRepository.tree !== repository.baseTree) throw new Error(`${prefix} did not start from the signed repository state`);
|
|
151
|
+
}
|
|
152
|
+
function verifyStableProfileMaterialization(measurements) {
|
|
153
|
+
for (const arm of ["baseline", "candidate"]) {
|
|
154
|
+
const expected = measurements[0]?.[arm].materializationReceipt.profileActivation;
|
|
155
|
+
if (!expected) throw new Error("candidate experiment contains no profile materialization");
|
|
156
|
+
const expectedDigest = canonicalCandidateDigest({
|
|
157
|
+
profilePlanDigest: expected.profilePlan.digest,
|
|
158
|
+
files: expected.files
|
|
159
|
+
});
|
|
160
|
+
for (const [index, measurement] of measurements.entries()) {
|
|
161
|
+
const activation = measurement[arm].materializationReceipt.profileActivation;
|
|
162
|
+
if (canonicalCandidateDigest({
|
|
163
|
+
profilePlanDigest: activation.profilePlan.digest,
|
|
164
|
+
files: activation.files
|
|
165
|
+
}) !== expectedDigest) throw new Error(`candidate experiment measurement ${index} ${arm} materialized a different profile`);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
function completedSuccessfully(evidence) {
|
|
170
|
+
const termination = evidence.receipt.termination;
|
|
171
|
+
return termination.kind === "exit" && termination.exitCode === 0;
|
|
172
|
+
}
|
|
173
|
+
function verifyMaterialAddressed(evidence, label) {
|
|
174
|
+
if (canonicalCandidateDigest(evidence.material) !== evidence.digest) throw new Error(`${label} digest is invalid`);
|
|
175
|
+
}
|
|
176
|
+
function costFromEvidence(evidence) {
|
|
177
|
+
return combinedUsage(evidence).costUsdNanos / 1e9;
|
|
178
|
+
}
|
|
179
|
+
function latencyFromEvidence(evidence) {
|
|
180
|
+
return evidence.receipt.timing.durationMs + evidence.receipt.benchmarkResult.material.grading.timing.durationMs;
|
|
181
|
+
}
|
|
182
|
+
function combinedUsage(evidence) {
|
|
183
|
+
const candidate = evidence.receipt.modelSettlement.material.usage;
|
|
184
|
+
const grader = evidence.receipt.benchmarkResult.material.grading.usage;
|
|
185
|
+
return addFixedSpend(candidate, grader);
|
|
186
|
+
}
|
|
187
|
+
function candidateMeasurementCostReceipt(measurements, model) {
|
|
188
|
+
return pairedFixedSpendReceipt(measurements, model, combinedUsage);
|
|
189
|
+
}
|
|
190
|
+
//#endregion
|
|
191
|
+
//#region src/contract/measured-comparison-record.ts
|
|
192
|
+
/** Keep published timing and cost accounting identical across experiment domains. */
|
|
193
|
+
function comparisonEvaluation(options, evaluation) {
|
|
194
|
+
return {
|
|
195
|
+
generationsExplored: options.generationsExplored ?? 0,
|
|
196
|
+
preparation: options.preparation,
|
|
197
|
+
measurement: {
|
|
198
|
+
wallDurationMs: options.measurement.wallDurationMs,
|
|
199
|
+
workDurationMs: evaluation.measurementWorkDurationMs,
|
|
200
|
+
cost: evaluation.measurementCost
|
|
201
|
+
},
|
|
202
|
+
total: {
|
|
203
|
+
wallDurationMs: options.preparation.wallDurationMs + options.measurement.wallDurationMs,
|
|
204
|
+
cost: evaluation.totalCost
|
|
205
|
+
}
|
|
206
|
+
};
|
|
207
|
+
}
|
|
208
|
+
/** Replay only the original inputs; derived timing and statistics are recomputed. */
|
|
209
|
+
function comparisonRecomputationInput(comparison) {
|
|
210
|
+
return {
|
|
211
|
+
experiment: comparison.experiment,
|
|
212
|
+
measurements: comparison.measurements,
|
|
213
|
+
runId: comparison.provenance.runId,
|
|
214
|
+
...comparison.candidate ? { candidate: comparison.candidate } : {},
|
|
215
|
+
generationsExplored: comparison.evaluation.generationsExplored,
|
|
216
|
+
preparation: comparison.evaluation.preparation,
|
|
217
|
+
measurement: {
|
|
218
|
+
wallDurationMs: comparison.evaluation.measurement.wallDurationMs,
|
|
219
|
+
cost: comparison.evaluation.measurement.cost
|
|
220
|
+
},
|
|
221
|
+
...comparison.metadata ? { metadata: comparison.metadata } : {}
|
|
222
|
+
};
|
|
223
|
+
}
|
|
40
224
|
//#endregion
|
|
41
225
|
//#region src/contract/concurrent-map.ts
|
|
42
226
|
/** Map both arms while bounding the actual executions, not the number of pairs. */
|
|
@@ -116,104 +300,67 @@ function assertNonNegativeFinite(value, label) {
|
|
|
116
300
|
if (!Number.isFinite(value) || value < 0) throw new Error(`${label} must be a non-negative finite number`);
|
|
117
301
|
}
|
|
118
302
|
//#endregion
|
|
119
|
-
//#region src/contract/
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
return
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
}
|
|
127
|
-
/** Freeze task order, repetitions, and every seed before either arm runs. */
|
|
128
|
-
function sealCandidateBenchmarkSuite(options) {
|
|
129
|
-
for (const task of options.tasks) verifyCandidateBenchmarkTask(task);
|
|
130
|
-
const material = {
|
|
131
|
-
kind: "agent-candidate-benchmark-suite",
|
|
132
|
-
digestAlgorithm: "rfc8785-sha256",
|
|
133
|
-
taskDigests: options.tasks.map((task) => task.digest),
|
|
134
|
-
reps: options.reps,
|
|
135
|
-
seeds: options.seeds
|
|
303
|
+
//#region src/contract/paired-measurement-input.ts
|
|
304
|
+
function projectPairedMeasurement(measurement, index, adapter) {
|
|
305
|
+
if (typeof measurement.cellId !== "string" || !measurement.cellId.trim()) throw new Error(`paired measurement ${index} requires a cell id`);
|
|
306
|
+
return {
|
|
307
|
+
cellId: measurement.cellId,
|
|
308
|
+
baseline: projectRun(measurement.baseline, adapter, `paired measurement ${index} baseline`),
|
|
309
|
+
candidate: projectRun(measurement.candidate, adapter, `paired measurement ${index} candidate`)
|
|
136
310
|
};
|
|
311
|
+
}
|
|
312
|
+
function projectRun(run, adapter, label) {
|
|
313
|
+
const suppliedDimensions = adapter.dimensions(run);
|
|
314
|
+
if (!Array.isArray(suppliedDimensions)) throw new Error(`${label} dimensions must be an array`);
|
|
315
|
+
const dimensions = /* @__PURE__ */ new Map();
|
|
316
|
+
for (const dimension of suppliedDimensions) {
|
|
317
|
+
if (typeof dimension.name !== "string" || !dimension.name.trim()) throw new Error(`${label} contains an unnamed dimension`);
|
|
318
|
+
if (dimensions.has(dimension.name)) throw new Error(`${label} repeats dimension '${dimension.name}'`);
|
|
319
|
+
dimensions.set(dimension.name, finiteMeasurement(dimension.score, `${label} ${dimension.name}`));
|
|
320
|
+
}
|
|
321
|
+
const completed = adapter.completed(run);
|
|
322
|
+
const passed = adapter.passed(run);
|
|
323
|
+
if (typeof completed !== "boolean" || typeof passed !== "boolean") throw new Error(`${label} completion and pass values must be booleans`);
|
|
324
|
+
const costProvenance = adapter.costProvenance(run);
|
|
325
|
+
if (costProvenance !== "observed" && costProvenance !== "estimated") throw new Error(`${label} cost provenance must be observed or estimated`);
|
|
137
326
|
return {
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
327
|
+
score: finiteMeasurement(adapter.score(run), `${label} score`),
|
|
328
|
+
dimensions,
|
|
329
|
+
costUsd: nonNegativeMeasurement(adapter.costUsd(run), `${label} cost`),
|
|
330
|
+
costProvenance,
|
|
331
|
+
latencyMs: nonNegativeMeasurement(adapter.latencyMs(run), `${label} latency`),
|
|
332
|
+
completed,
|
|
333
|
+
passed
|
|
143
334
|
};
|
|
144
335
|
}
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
...
|
|
149
|
-
|
|
150
|
-
}
|
|
336
|
+
function sharedProjectedDimensions(measurements) {
|
|
337
|
+
const expected = [...measurements[0].baseline.dimensions.keys()];
|
|
338
|
+
for (const [index, measurement] of measurements.entries()) for (const [arm, run] of [["baseline", measurement.baseline], ["candidate", measurement.candidate]]) {
|
|
339
|
+
const actual = [...run.dimensions.keys()];
|
|
340
|
+
if (JSON.stringify(actual) !== JSON.stringify(expected)) throw new Error(`paired measurement ${index} ${arm} dimensions do not match the suite`);
|
|
341
|
+
}
|
|
342
|
+
return expected;
|
|
151
343
|
}
|
|
152
|
-
function
|
|
153
|
-
const
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
verifyBundle(experiment.candidate, "candidate bundle");
|
|
157
|
-
if (experiment.baseline.digest === experiment.candidate.digest) throw new Error("candidate experiment baseline and candidate bundles are identical");
|
|
158
|
-
verifyCandidateBenchmarkSuiteInputs(experiment.benchmark);
|
|
159
|
-
return experiment;
|
|
344
|
+
function dimensionScore(run, name) {
|
|
345
|
+
const value = run.dimensions.get(name);
|
|
346
|
+
if (value === void 0) throw new Error(`paired measurement is missing dimension '${name}'`);
|
|
347
|
+
return value;
|
|
160
348
|
}
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
call: {
|
|
173
|
-
callId: `candidate-measurement:${experiment.digest}`,
|
|
174
|
-
channel: "measurement",
|
|
175
|
-
phase: "heldout",
|
|
176
|
-
actor: experiment.digest,
|
|
177
|
-
model: candidateMeasurementModel(experiment),
|
|
178
|
-
tags: { experimentDigest: experiment.digest }
|
|
179
|
-
},
|
|
180
|
-
...options.signal ? { signal: options.signal } : {},
|
|
181
|
-
async execute(index, arm, signal) {
|
|
182
|
-
const taskIndex = Math.floor(index / suite.reps);
|
|
183
|
-
const repetition = index % suite.reps;
|
|
184
|
-
const task = tasks[taskIndex];
|
|
185
|
-
const seed = suite.seeds[index];
|
|
186
|
-
if (!task || seed === void 0) throw new Error(`candidate experiment cell ${index} has no signed task or seed`);
|
|
187
|
-
const benchmarkCell = {
|
|
188
|
-
suiteDigest: suite.digest,
|
|
189
|
-
taskIndex,
|
|
190
|
-
repetition
|
|
191
|
-
};
|
|
192
|
-
return verifyExecutionEvidence(await options.execute({
|
|
193
|
-
experiment,
|
|
194
|
-
arm,
|
|
195
|
-
bundle: experiment[arm],
|
|
196
|
-
task,
|
|
197
|
-
benchmarkCell,
|
|
198
|
-
seed,
|
|
199
|
-
signal
|
|
200
|
-
}));
|
|
201
|
-
},
|
|
202
|
-
receipt(measurements) {
|
|
203
|
-
return candidateMeasurementCostReceipt(measurements, candidateMeasurementModel(experiment));
|
|
204
|
-
}
|
|
205
|
-
});
|
|
206
|
-
return {
|
|
207
|
-
measurements: run.measurements.map((measurement, index) => verifyMeasurement(experiment, measurement, index)),
|
|
208
|
-
measurement: {
|
|
209
|
-
wallDurationMs: run.wallDurationMs,
|
|
210
|
-
cost: {
|
|
211
|
-
usd: run.cost.usd,
|
|
212
|
-
provenance: run.cost.kind
|
|
213
|
-
}
|
|
214
|
-
}
|
|
215
|
-
};
|
|
349
|
+
function finiteMeasurement(value, label) {
|
|
350
|
+
if (!Number.isFinite(value)) throw new Error(`${label} must be finite`);
|
|
351
|
+
return value;
|
|
352
|
+
}
|
|
353
|
+
function nonNegativeMeasurement(value, label) {
|
|
354
|
+
if (!Number.isFinite(value) || value < 0) throw new Error(`${label} must be non-negative`);
|
|
355
|
+
return value;
|
|
356
|
+
}
|
|
357
|
+
function assertKnownCost(cost, label) {
|
|
358
|
+
if (!Number.isFinite(cost.usd) || cost.usd < 0) throw new Error(`${label} must be a non-negative number`);
|
|
359
|
+
if (cost.provenance !== "observed" && cost.provenance !== "estimated") throw new Error(`${label} provenance must be observed or estimated`);
|
|
216
360
|
}
|
|
361
|
+
//#endregion
|
|
362
|
+
//#region src/contract/paired-measurement-evaluation.ts
|
|
363
|
+
/** Shared significance, regression, precision, and budget decisions on paired receipts. */
|
|
217
364
|
/**
|
|
218
365
|
* Calculate the shared paired decision from any complete receipt shape.
|
|
219
366
|
*
|
|
@@ -449,6 +596,164 @@ function significanceReason(significance, deltaThreshold) {
|
|
|
449
596
|
if (decision.exactTestVetoes && decision.mcnemar !== null) return `McNemar's exact test refuses at threshold ${deltaThreshold}: b=${decision.mcnemar.b}, c=${decision.mcnemar.c}, p=${decision.mcnemar.pValue}`;
|
|
450
597
|
return `paired ${decision.statistic} interval lower bound ${decision.low} did not clear ${deltaThreshold}.${decision.methodDetail}`;
|
|
451
598
|
}
|
|
599
|
+
function measuredEstimate(baseline, candidate, options) {
|
|
600
|
+
const bootstrap = pairedBootstrap(baseline, candidate, {
|
|
601
|
+
confidence: options.confidence,
|
|
602
|
+
resamples: options.resamples,
|
|
603
|
+
statistic: "mean",
|
|
604
|
+
seed: options.seed
|
|
605
|
+
});
|
|
606
|
+
const baselineMean = mean(baseline);
|
|
607
|
+
const candidateMean = mean(candidate);
|
|
608
|
+
const delta = candidateMean - baselineMean;
|
|
609
|
+
return {
|
|
610
|
+
baseline: baselineMean,
|
|
611
|
+
candidate: candidateMean,
|
|
612
|
+
delta,
|
|
613
|
+
confidenceInterval: {
|
|
614
|
+
level: bootstrap.confidence,
|
|
615
|
+
lower: Math.min(bootstrap.low, delta),
|
|
616
|
+
upper: Math.max(bootstrap.high, delta),
|
|
617
|
+
method: "paired-bootstrap",
|
|
618
|
+
statistic: "mean",
|
|
619
|
+
resamples: bootstrap.resamples
|
|
620
|
+
},
|
|
621
|
+
n: bootstrap.n
|
|
622
|
+
};
|
|
623
|
+
}
|
|
624
|
+
function observedPairedPrecision(options) {
|
|
625
|
+
const n = options.baselineScores.length;
|
|
626
|
+
const lowerMargin = Math.max(0, options.delta - options.lowerBound);
|
|
627
|
+
const minimumDetectableDelta = options.deltaThreshold + lowerMargin;
|
|
628
|
+
const scaleAssumed = [...options.baselineScores, ...options.candidateScores].every((score) => score >= -.001 && score <= 1.001);
|
|
629
|
+
const headroom = Math.max(0, 1 - mean(options.baselineScores));
|
|
630
|
+
const enoughRuns = n >= options.minProductiveRuns;
|
|
631
|
+
const attainable = !scaleAssumed || minimumDetectableDelta <= headroom;
|
|
632
|
+
const sufficient = enoughRuns && attainable;
|
|
633
|
+
const scorerNote = options.sharedScorerChannel ? " The same scorer evaluated both arms, so scorer bias is not measured." : "";
|
|
634
|
+
if (!enoughRuns) return {
|
|
635
|
+
sufficient,
|
|
636
|
+
minimumDetectableDelta,
|
|
637
|
+
scaleAssumed,
|
|
638
|
+
reason: `only ${n} paired runs; ${options.minProductiveRuns} required.${scorerNote}`
|
|
639
|
+
};
|
|
640
|
+
if (!attainable) return {
|
|
641
|
+
sufficient,
|
|
642
|
+
minimumDetectableDelta,
|
|
643
|
+
scaleAssumed,
|
|
644
|
+
reason: `observed paired uncertainty gives a minimum detectable delta of ${minimumDetectableDelta.toFixed(3)}, above the remaining score headroom ${headroom.toFixed(3)}.${scorerNote}`
|
|
645
|
+
};
|
|
646
|
+
return {
|
|
647
|
+
sufficient,
|
|
648
|
+
minimumDetectableDelta,
|
|
649
|
+
scaleAssumed,
|
|
650
|
+
reason: `observed paired uncertainty gives a minimum detectable delta of ${minimumDetectableDelta.toFixed(3)} at ${options.confidence.toFixed(2)} confidence from ${n} runs.${scorerNote}`
|
|
651
|
+
};
|
|
652
|
+
}
|
|
653
|
+
function mean(values) {
|
|
654
|
+
if (values.length === 0) throw new Error("candidate experiment requires measured values");
|
|
655
|
+
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
656
|
+
}
|
|
657
|
+
//#endregion
|
|
658
|
+
//#region src/contract/candidate-experiment.ts
|
|
659
|
+
/** Seal held-out experiments, execute their signed cells, and publish verified comparisons. */
|
|
660
|
+
/** Content-address one task before any measured execution can see it. */
|
|
661
|
+
function sealCandidateBenchmarkTask(material) {
|
|
662
|
+
return agentCandidateBenchmarkTaskSchema.parse({
|
|
663
|
+
...material,
|
|
664
|
+
digest: canonicalCandidateDigest(material)
|
|
665
|
+
});
|
|
666
|
+
}
|
|
667
|
+
/** Freeze task order, repetitions, and every seed before either arm runs. */
|
|
668
|
+
function sealCandidateBenchmarkSuite(options) {
|
|
669
|
+
for (const task of options.tasks) verifyCandidateBenchmarkTask(task);
|
|
670
|
+
const material = {
|
|
671
|
+
kind: "agent-candidate-benchmark-suite",
|
|
672
|
+
digestAlgorithm: "rfc8785-sha256",
|
|
673
|
+
taskDigests: options.tasks.map((task) => task.digest),
|
|
674
|
+
reps: options.reps,
|
|
675
|
+
seeds: options.seeds
|
|
676
|
+
};
|
|
677
|
+
return {
|
|
678
|
+
suite: agentCandidateBenchmarkSuiteSchema.parse({
|
|
679
|
+
...material,
|
|
680
|
+
digest: canonicalCandidateDigest(material)
|
|
681
|
+
}),
|
|
682
|
+
tasks: options.tasks
|
|
683
|
+
};
|
|
684
|
+
}
|
|
685
|
+
/** Freeze both complete agent states and their exact held-out work. */
|
|
686
|
+
function sealCandidateExperiment(material) {
|
|
687
|
+
return verifyCandidateExperiment(agentCandidateExperimentSchema.parse({
|
|
688
|
+
...material,
|
|
689
|
+
digest: canonicalCandidateDigest(material)
|
|
690
|
+
}));
|
|
691
|
+
}
|
|
692
|
+
function verifyCandidateExperiment(input) {
|
|
693
|
+
const experiment = agentCandidateExperimentSchema.parse(input);
|
|
694
|
+
verifySelfAddressed(experiment, "candidate experiment");
|
|
695
|
+
verifyBundle(experiment.baseline, "baseline bundle");
|
|
696
|
+
verifyBundle(experiment.candidate, "candidate bundle");
|
|
697
|
+
if (experiment.baseline.digest === experiment.candidate.digest) throw new Error("candidate experiment baseline and candidate bundles are identical");
|
|
698
|
+
verifyCandidateBenchmarkSuiteInputs(experiment.benchmark);
|
|
699
|
+
return experiment;
|
|
700
|
+
}
|
|
701
|
+
/** Execute each signed cell for both arms. The callback is Runtime's one executor. */
|
|
702
|
+
async function runCandidateExperiment(options) {
|
|
703
|
+
const experiment = verifyCandidateExperiment(options.experiment);
|
|
704
|
+
const { suite, tasks } = experiment.benchmark;
|
|
705
|
+
const run = await runPaidPairedMeasurement({
|
|
706
|
+
count: suite.taskDigests.length * suite.reps,
|
|
707
|
+
maxConcurrency: options.maxConcurrency ?? 2,
|
|
708
|
+
label: "candidate experiment",
|
|
709
|
+
budgetUsd: experiment.policy.budgetUsd,
|
|
710
|
+
...options.costLedger ? { costLedger: options.costLedger } : {},
|
|
711
|
+
maximumCostUsd: candidateMeasurementMaximumCostUsd(experiment),
|
|
712
|
+
call: {
|
|
713
|
+
callId: `candidate-measurement:${experiment.digest}`,
|
|
714
|
+
channel: "measurement",
|
|
715
|
+
phase: "heldout",
|
|
716
|
+
actor: experiment.digest,
|
|
717
|
+
model: candidateMeasurementModel(experiment),
|
|
718
|
+
tags: { experimentDigest: experiment.digest }
|
|
719
|
+
},
|
|
720
|
+
...options.signal ? { signal: options.signal } : {},
|
|
721
|
+
async execute(index, arm, signal) {
|
|
722
|
+
const taskIndex = Math.floor(index / suite.reps);
|
|
723
|
+
const repetition = index % suite.reps;
|
|
724
|
+
const task = tasks[taskIndex];
|
|
725
|
+
const seed = suite.seeds[index];
|
|
726
|
+
if (!task || seed === void 0) throw new Error(`candidate experiment cell ${index} has no signed task or seed`);
|
|
727
|
+
const benchmarkCell = {
|
|
728
|
+
suiteDigest: suite.digest,
|
|
729
|
+
taskIndex,
|
|
730
|
+
repetition
|
|
731
|
+
};
|
|
732
|
+
return verifyExecutionEvidence(await options.execute({
|
|
733
|
+
experiment,
|
|
734
|
+
arm,
|
|
735
|
+
bundle: experiment[arm],
|
|
736
|
+
task,
|
|
737
|
+
benchmarkCell,
|
|
738
|
+
seed,
|
|
739
|
+
signal
|
|
740
|
+
}));
|
|
741
|
+
},
|
|
742
|
+
receipt(measurements) {
|
|
743
|
+
return candidateMeasurementCostReceipt(measurements, candidateMeasurementModel(experiment));
|
|
744
|
+
}
|
|
745
|
+
});
|
|
746
|
+
return {
|
|
747
|
+
measurements: run.measurements.map((measurement, index) => verifyMeasurement(experiment, measurement, index)),
|
|
748
|
+
measurement: {
|
|
749
|
+
wallDurationMs: run.wallDurationMs,
|
|
750
|
+
cost: {
|
|
751
|
+
usd: run.cost.usd,
|
|
752
|
+
provenance: run.cost.kind
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
};
|
|
756
|
+
}
|
|
452
757
|
/** Build the only publishable comparison: paired statistics over Runtime receipts. */
|
|
453
758
|
function measuredComparisonFromCandidateExperiment(options) {
|
|
454
759
|
const experiment = verifyCandidateExperiment(options.experiment);
|
|
@@ -489,19 +794,7 @@ function measuredComparisonFromCandidateExperiment(options) {
|
|
|
489
794
|
candidateContentHash: experiment.candidate.digest
|
|
490
795
|
},
|
|
491
796
|
diff,
|
|
492
|
-
evaluation:
|
|
493
|
-
generationsExplored: options.generationsExplored ?? 0,
|
|
494
|
-
preparation: options.preparation,
|
|
495
|
-
measurement: {
|
|
496
|
-
wallDurationMs: options.measurement.wallDurationMs,
|
|
497
|
-
workDurationMs: evaluation.measurementWorkDurationMs,
|
|
498
|
-
cost: evaluation.measurementCost
|
|
499
|
-
},
|
|
500
|
-
total: {
|
|
501
|
-
wallDurationMs: options.preparation.wallDurationMs + options.measurement.wallDurationMs,
|
|
502
|
-
cost: evaluation.totalCost
|
|
503
|
-
}
|
|
504
|
-
},
|
|
797
|
+
evaluation: comparisonEvaluation(options, evaluation),
|
|
505
798
|
...options.metadata ? { metadata: options.metadata } : {}
|
|
506
799
|
});
|
|
507
800
|
const { recordDigest: _recordDigest, ...provenance } = provisional.provenance;
|
|
@@ -519,19 +812,7 @@ function measuredComparisonFromCandidateExperiment(options) {
|
|
|
519
812
|
/** Recompute every statistic and decision from the signed experiment receipts. */
|
|
520
813
|
function verifyCandidateExperimentComparison(input) {
|
|
521
814
|
const comparison = agentImprovementMeasuredComparisonSchema.parse(input);
|
|
522
|
-
const recomputed = measuredComparisonFromCandidateExperiment(
|
|
523
|
-
experiment: comparison.experiment,
|
|
524
|
-
measurements: comparison.measurements,
|
|
525
|
-
runId: comparison.provenance.runId,
|
|
526
|
-
...comparison.candidate ? { candidate: comparison.candidate } : {},
|
|
527
|
-
generationsExplored: comparison.evaluation.generationsExplored,
|
|
528
|
-
preparation: comparison.evaluation.preparation,
|
|
529
|
-
measurement: {
|
|
530
|
-
wallDurationMs: comparison.evaluation.measurement.wallDurationMs,
|
|
531
|
-
cost: comparison.evaluation.measurement.cost
|
|
532
|
-
},
|
|
533
|
-
...comparison.metadata ? { metadata: comparison.metadata } : {}
|
|
534
|
-
});
|
|
815
|
+
const recomputed = measuredComparisonFromCandidateExperiment(comparisonRecomputationInput(comparison));
|
|
535
816
|
if (canonicalCandidateDigest(recomputed) !== canonicalCandidateDigest(comparison)) throw new Error("candidate experiment comparison does not match its Runtime receipts");
|
|
536
817
|
return comparison;
|
|
537
818
|
}
|
|
@@ -544,34 +825,10 @@ function candidateMeasurementModel(experiment) {
|
|
|
544
825
|
const models = [...new Set(experiment.benchmark.tasks.map((task) => task.model.model))];
|
|
545
826
|
return models.length === 1 ? models[0] : "multiple-models";
|
|
546
827
|
}
|
|
547
|
-
function candidateMeasurementCostReceipt(measurements, model) {
|
|
548
|
-
const usage = measurements.reduce((sum, measurement) => addFixedSpend(addFixedSpend(sum, combinedUsage(measurement.baseline)), combinedUsage(measurement.candidate)), {
|
|
549
|
-
inputTokens: 0,
|
|
550
|
-
outputTokens: 0,
|
|
551
|
-
cachedInputTokens: 0,
|
|
552
|
-
reasoningTokens: 0,
|
|
553
|
-
modelCalls: 0,
|
|
554
|
-
costUsdNanos: 0,
|
|
555
|
-
costProvenance: "observed"
|
|
556
|
-
});
|
|
557
|
-
const costUsd = usage.costUsdNanos / 1e9;
|
|
558
|
-
return {
|
|
559
|
-
model,
|
|
560
|
-
inputTokens: usage.inputTokens,
|
|
561
|
-
outputTokens: usage.outputTokens,
|
|
562
|
-
cachedTokens: usage.cachedInputTokens,
|
|
563
|
-
reasoningTokens: usage.reasoningTokens,
|
|
564
|
-
...usage.costProvenance === "observed" ? { actualCostUsd: costUsd } : { estimatedCostUsd: costUsd }
|
|
565
|
-
};
|
|
566
|
-
}
|
|
567
828
|
function assertPhaseAccounting(accounting, label) {
|
|
568
829
|
if (!Number.isFinite(accounting.wallDurationMs) || accounting.wallDurationMs < 0) throw new Error(`${label} wall duration must be a non-negative number`);
|
|
569
830
|
assertKnownCost(accounting.cost, `${label} cost`);
|
|
570
831
|
}
|
|
571
|
-
function assertKnownCost(cost, label) {
|
|
572
|
-
if (!Number.isFinite(cost.usd) || cost.usd < 0) throw new Error(`${label} must be a non-negative number`);
|
|
573
|
-
if (cost.provenance !== "observed" && cost.provenance !== "estimated") throw new Error(`${label} provenance must be observed or estimated`);
|
|
574
|
-
}
|
|
575
832
|
function deriveCandidateBundleDiff(experiment) {
|
|
576
833
|
const changed = [
|
|
577
834
|
"profile",
|
|
@@ -625,235 +882,6 @@ function verifyBundle(input, label) {
|
|
|
625
882
|
verifySelfAddressed(bundle, label);
|
|
626
883
|
return bundle;
|
|
627
884
|
}
|
|
628
|
-
function verifyMeasurement(experiment, input, index) {
|
|
629
|
-
const suite = experiment.benchmark.suite;
|
|
630
|
-
const taskIndex = Math.floor(index / suite.reps);
|
|
631
|
-
const repetition = index % suite.reps;
|
|
632
|
-
const task = experiment.benchmark.tasks[taskIndex];
|
|
633
|
-
const seed = suite.seeds[index];
|
|
634
|
-
if (!task || seed === void 0) throw new Error(`candidate experiment measurement ${index} is outside the signed suite`);
|
|
635
|
-
const baseline = verifyExecutionEvidence(input.baseline);
|
|
636
|
-
const candidate = verifyExecutionEvidence(input.candidate);
|
|
637
|
-
for (const [arm, evidence] of [["baseline", baseline], ["candidate", candidate]]) {
|
|
638
|
-
const bundle = experiment[arm];
|
|
639
|
-
const materialization = evidence.materializationReceipt;
|
|
640
|
-
const runCell = materialization.executionPlan.material.runCell;
|
|
641
|
-
verifySelfAddressed(runCell, "candidate run cell");
|
|
642
|
-
if (runCell.experimentDigest !== experiment.digest || runCell.arm !== arm || runCell.bundleDigest !== bundle.digest || runCell.suiteDigest !== suite.digest || runCell.taskDigest !== task.digest || runCell.taskIndex !== taskIndex || runCell.repetition !== repetition || runCell.seed !== seed || runCell.attempt > task.attempt.maxAttempts || materialization.bundleDigest !== bundle.digest || materialization.benchmark.suite.digest !== suite.digest || materialization.benchmark.task.digest !== task.digest || materialization.codeKind !== bundle.code.kind || materialization.profileActivation.profilePlan.material.sourceProfileDigest !== canonicalCandidateDigest(bundle.profile) || evidence.receipt.runCellDigest !== runCell.digest || JSON.stringify(materialization.resolvedModel) !== JSON.stringify(task.model)) throw new Error(`candidate experiment measurement ${index} substituted its ${arm} arm`);
|
|
643
|
-
verifyTaskOutcome(task, evidence, index, arm);
|
|
644
|
-
}
|
|
645
|
-
const baselinePlan = baseline.materializationReceipt.executionPlan.material;
|
|
646
|
-
const candidatePlan = candidate.materializationReceipt.executionPlan.material;
|
|
647
|
-
if (baselinePlan.executionId === candidatePlan.executionId || baselinePlan.runCell.digest === candidatePlan.runCell.digest || baseline.materializationReceipt.digest === candidate.materializationReceipt.digest || baseline.receipt.digest === candidate.receipt.digest || baseline.digest === candidate.digest) throw new Error(`candidate experiment measurement ${index} reused one execution across arms`);
|
|
648
|
-
return {
|
|
649
|
-
baseline,
|
|
650
|
-
candidate
|
|
651
|
-
};
|
|
652
|
-
}
|
|
653
|
-
function verifyExecutionEvidence(input) {
|
|
654
|
-
const evidence = candidateExecutionEvidenceSchema.parse(input);
|
|
655
|
-
verifySelfAddressed(evidence, "candidate execution evidence");
|
|
656
|
-
verifySelfAddressed(evidence.materializationReceipt, "candidate materialization receipt");
|
|
657
|
-
verifySelfAddressed(evidence.materializationReceipt.profileActivation, "candidate profile activation");
|
|
658
|
-
verifyMaterialAddressed(evidence.materializationReceipt.profileActivation.profilePlan, "candidate profile plan");
|
|
659
|
-
verifyMaterialAddressed(evidence.materializationReceipt.executionPlan, "candidate execution plan");
|
|
660
|
-
verifySelfAddressed(evidence.receipt, "candidate run receipt");
|
|
661
|
-
verifyMaterialAddressed(evidence.receipt.modelSettlement, "candidate model settlement");
|
|
662
|
-
verifyMaterialAddressed(evidence.receipt.taskOutcome, "candidate task outcome");
|
|
663
|
-
verifyMaterialAddressed(evidence.receipt.benchmarkResult, "candidate benchmark result");
|
|
664
|
-
return evidence;
|
|
665
|
-
}
|
|
666
|
-
function verifySelfAddressed(document, label) {
|
|
667
|
-
if (canonicalCandidateDigest(omitTopLevelDigest(document)) !== document.digest) throw new Error(`${label} digest is invalid`);
|
|
668
|
-
}
|
|
669
|
-
function verifyTaskOutcome(task, evidence, index, arm) {
|
|
670
|
-
const outcome = evidence.receipt.taskOutcome.material.outcome;
|
|
671
|
-
const result = evidence.receipt.benchmarkResult.material;
|
|
672
|
-
const prefix = `candidate experiment measurement ${index} ${arm}`;
|
|
673
|
-
if (result.evidence.sha256 === task.grader.artifact.sha256) throw new Error(`${prefix} reused grader bytes as grading evidence`);
|
|
674
|
-
const usage = combinedUsage(evidence);
|
|
675
|
-
const usageChecks = [
|
|
676
|
-
[
|
|
677
|
-
usage.modelCalls,
|
|
678
|
-
task.limits.maxModelCalls,
|
|
679
|
-
"model calls"
|
|
680
|
-
],
|
|
681
|
-
[
|
|
682
|
-
usage.inputTokens,
|
|
683
|
-
task.limits.maxInputTokens,
|
|
684
|
-
"input tokens"
|
|
685
|
-
],
|
|
686
|
-
[
|
|
687
|
-
usage.outputTokens,
|
|
688
|
-
task.limits.maxOutputTokens,
|
|
689
|
-
"output tokens"
|
|
690
|
-
],
|
|
691
|
-
[
|
|
692
|
-
usage.costUsdNanos,
|
|
693
|
-
Math.round(task.limits.maxCostUsd * 1e9),
|
|
694
|
-
"cost"
|
|
695
|
-
]
|
|
696
|
-
];
|
|
697
|
-
for (const [actual, maximum, label] of usageChecks) if (actual > maximum) throw new Error(`${prefix} ${label} ${actual} exceeds the signed limit ${maximum}`);
|
|
698
|
-
if (outcome.kind !== task.outcome.kind) throw new Error(`${prefix} returned an outcome outside the signed task contract`);
|
|
699
|
-
if (task.outcome.kind === "output") {
|
|
700
|
-
if (outcome.kind !== "output" || outcome.spec.mediaType !== task.outcome.mediaType || outcome.spec.maxBytes !== task.outcome.maxBytes) throw new Error(`${prefix} changed the signed output contract`);
|
|
701
|
-
return;
|
|
702
|
-
}
|
|
703
|
-
const repository = task.repository;
|
|
704
|
-
if (outcome.kind !== "workspace" || repository === void 0 || outcome.baseRepository.identity !== repository.identity || outcome.baseRepository.rootIdentity !== repository.rootIdentity || outcome.baseRepository.commit !== repository.baseCommit || outcome.baseRepository.tree !== repository.baseTree) throw new Error(`${prefix} did not start from the signed repository state`);
|
|
705
|
-
}
|
|
706
|
-
function verifyStableProfileMaterialization(measurements) {
|
|
707
|
-
for (const arm of ["baseline", "candidate"]) {
|
|
708
|
-
const expected = measurements[0]?.[arm].materializationReceipt.profileActivation;
|
|
709
|
-
if (!expected) throw new Error("candidate experiment contains no profile materialization");
|
|
710
|
-
const expectedDigest = canonicalCandidateDigest({
|
|
711
|
-
profilePlanDigest: expected.profilePlan.digest,
|
|
712
|
-
files: expected.files
|
|
713
|
-
});
|
|
714
|
-
for (const [index, measurement] of measurements.entries()) {
|
|
715
|
-
const activation = measurement[arm].materializationReceipt.profileActivation;
|
|
716
|
-
if (canonicalCandidateDigest({
|
|
717
|
-
profilePlanDigest: activation.profilePlan.digest,
|
|
718
|
-
files: activation.files
|
|
719
|
-
}) !== expectedDigest) throw new Error(`candidate experiment measurement ${index} ${arm} materialized a different profile`);
|
|
720
|
-
}
|
|
721
|
-
}
|
|
722
|
-
}
|
|
723
|
-
function completedSuccessfully(evidence) {
|
|
724
|
-
const termination = evidence.receipt.termination;
|
|
725
|
-
return termination.kind === "exit" && termination.exitCode === 0;
|
|
726
|
-
}
|
|
727
|
-
function verifyMaterialAddressed(evidence, label) {
|
|
728
|
-
if (canonicalCandidateDigest(evidence.material) !== evidence.digest) throw new Error(`${label} digest is invalid`);
|
|
729
|
-
}
|
|
730
|
-
function projectPairedMeasurement(measurement, index, adapter) {
|
|
731
|
-
if (typeof measurement.cellId !== "string" || !measurement.cellId.trim()) throw new Error(`paired measurement ${index} requires a cell id`);
|
|
732
|
-
return {
|
|
733
|
-
cellId: measurement.cellId,
|
|
734
|
-
baseline: projectRun(measurement.baseline, adapter, `paired measurement ${index} baseline`),
|
|
735
|
-
candidate: projectRun(measurement.candidate, adapter, `paired measurement ${index} candidate`)
|
|
736
|
-
};
|
|
737
|
-
}
|
|
738
|
-
function projectRun(run, adapter, label) {
|
|
739
|
-
const suppliedDimensions = adapter.dimensions(run);
|
|
740
|
-
if (!Array.isArray(suppliedDimensions)) throw new Error(`${label} dimensions must be an array`);
|
|
741
|
-
const dimensions = /* @__PURE__ */ new Map();
|
|
742
|
-
for (const dimension of suppliedDimensions) {
|
|
743
|
-
if (typeof dimension.name !== "string" || !dimension.name.trim()) throw new Error(`${label} contains an unnamed dimension`);
|
|
744
|
-
if (dimensions.has(dimension.name)) throw new Error(`${label} repeats dimension '${dimension.name}'`);
|
|
745
|
-
dimensions.set(dimension.name, finiteMeasurement(dimension.score, `${label} ${dimension.name}`));
|
|
746
|
-
}
|
|
747
|
-
const completed = adapter.completed(run);
|
|
748
|
-
const passed = adapter.passed(run);
|
|
749
|
-
if (typeof completed !== "boolean" || typeof passed !== "boolean") throw new Error(`${label} completion and pass values must be booleans`);
|
|
750
|
-
const costProvenance = adapter.costProvenance(run);
|
|
751
|
-
if (costProvenance !== "observed" && costProvenance !== "estimated") throw new Error(`${label} cost provenance must be observed or estimated`);
|
|
752
|
-
return {
|
|
753
|
-
score: finiteMeasurement(adapter.score(run), `${label} score`),
|
|
754
|
-
dimensions,
|
|
755
|
-
costUsd: nonNegativeMeasurement(adapter.costUsd(run), `${label} cost`),
|
|
756
|
-
costProvenance,
|
|
757
|
-
latencyMs: nonNegativeMeasurement(adapter.latencyMs(run), `${label} latency`),
|
|
758
|
-
completed,
|
|
759
|
-
passed
|
|
760
|
-
};
|
|
761
|
-
}
|
|
762
|
-
function sharedProjectedDimensions(measurements) {
|
|
763
|
-
const expected = [...measurements[0].baseline.dimensions.keys()];
|
|
764
|
-
for (const [index, measurement] of measurements.entries()) for (const [arm, run] of [["baseline", measurement.baseline], ["candidate", measurement.candidate]]) {
|
|
765
|
-
const actual = [...run.dimensions.keys()];
|
|
766
|
-
if (JSON.stringify(actual) !== JSON.stringify(expected)) throw new Error(`paired measurement ${index} ${arm} dimensions do not match the suite`);
|
|
767
|
-
}
|
|
768
|
-
return expected;
|
|
769
|
-
}
|
|
770
|
-
function dimensionScore(run, name) {
|
|
771
|
-
const value = run.dimensions.get(name);
|
|
772
|
-
if (value === void 0) throw new Error(`paired measurement is missing dimension '${name}'`);
|
|
773
|
-
return value;
|
|
774
|
-
}
|
|
775
|
-
function measuredEstimate(baseline, candidate, options) {
|
|
776
|
-
const bootstrap = pairedBootstrap(baseline, candidate, {
|
|
777
|
-
confidence: options.confidence,
|
|
778
|
-
resamples: options.resamples,
|
|
779
|
-
statistic: "mean",
|
|
780
|
-
seed: options.seed
|
|
781
|
-
});
|
|
782
|
-
const baselineMean = mean(baseline);
|
|
783
|
-
const candidateMean = mean(candidate);
|
|
784
|
-
const delta = candidateMean - baselineMean;
|
|
785
|
-
return {
|
|
786
|
-
baseline: baselineMean,
|
|
787
|
-
candidate: candidateMean,
|
|
788
|
-
delta,
|
|
789
|
-
confidenceInterval: {
|
|
790
|
-
level: bootstrap.confidence,
|
|
791
|
-
lower: Math.min(bootstrap.low, delta),
|
|
792
|
-
upper: Math.max(bootstrap.high, delta),
|
|
793
|
-
method: "paired-bootstrap",
|
|
794
|
-
statistic: "mean",
|
|
795
|
-
resamples: bootstrap.resamples
|
|
796
|
-
},
|
|
797
|
-
n: bootstrap.n
|
|
798
|
-
};
|
|
799
|
-
}
|
|
800
|
-
function observedPairedPrecision(options) {
|
|
801
|
-
const n = options.baselineScores.length;
|
|
802
|
-
const lowerMargin = Math.max(0, options.delta - options.lowerBound);
|
|
803
|
-
const minimumDetectableDelta = options.deltaThreshold + lowerMargin;
|
|
804
|
-
const scaleAssumed = [...options.baselineScores, ...options.candidateScores].every((score) => score >= -.001 && score <= 1.001);
|
|
805
|
-
const headroom = Math.max(0, 1 - mean(options.baselineScores));
|
|
806
|
-
const enoughRuns = n >= options.minProductiveRuns;
|
|
807
|
-
const attainable = !scaleAssumed || minimumDetectableDelta <= headroom;
|
|
808
|
-
const sufficient = enoughRuns && attainable;
|
|
809
|
-
const scorerNote = options.sharedScorerChannel ? " The same scorer evaluated both arms, so scorer bias is not measured." : "";
|
|
810
|
-
if (!enoughRuns) return {
|
|
811
|
-
sufficient,
|
|
812
|
-
minimumDetectableDelta,
|
|
813
|
-
scaleAssumed,
|
|
814
|
-
reason: `only ${n} paired runs; ${options.minProductiveRuns} required.${scorerNote}`
|
|
815
|
-
};
|
|
816
|
-
if (!attainable) return {
|
|
817
|
-
sufficient,
|
|
818
|
-
minimumDetectableDelta,
|
|
819
|
-
scaleAssumed,
|
|
820
|
-
reason: `observed paired uncertainty gives a minimum detectable delta of ${minimumDetectableDelta.toFixed(3)}, above the remaining score headroom ${headroom.toFixed(3)}.${scorerNote}`
|
|
821
|
-
};
|
|
822
|
-
return {
|
|
823
|
-
sufficient,
|
|
824
|
-
minimumDetectableDelta,
|
|
825
|
-
scaleAssumed,
|
|
826
|
-
reason: `observed paired uncertainty gives a minimum detectable delta of ${minimumDetectableDelta.toFixed(3)} at ${options.confidence.toFixed(2)} confidence from ${n} runs.${scorerNote}`
|
|
827
|
-
};
|
|
828
|
-
}
|
|
829
|
-
function finiteMeasurement(value, label) {
|
|
830
|
-
if (!Number.isFinite(value)) throw new Error(`${label} must be finite`);
|
|
831
|
-
return value;
|
|
832
|
-
}
|
|
833
|
-
function nonNegativeMeasurement(value, label) {
|
|
834
|
-
if (!Number.isFinite(value) || value < 0) throw new Error(`${label} must be non-negative`);
|
|
835
|
-
return value;
|
|
836
|
-
}
|
|
837
|
-
const candidateExecutionEvidenceAdapter = {
|
|
838
|
-
score: (evidence) => evidence.receipt.benchmarkResult.material.score,
|
|
839
|
-
dimensions: (evidence) => evidence.receipt.benchmarkResult.material.dimensions,
|
|
840
|
-
costUsd: costFromEvidence,
|
|
841
|
-
costProvenance: (evidence) => combinedUsage(evidence).costProvenance,
|
|
842
|
-
latencyMs: latencyFromEvidence,
|
|
843
|
-
completed: completedSuccessfully,
|
|
844
|
-
passed: (evidence) => evidence.receipt.benchmarkResult.material.passed
|
|
845
|
-
};
|
|
846
|
-
function costFromEvidence(evidence) {
|
|
847
|
-
return combinedUsage(evidence).costUsdNanos / 1e9;
|
|
848
|
-
}
|
|
849
|
-
function latencyFromEvidence(evidence) {
|
|
850
|
-
return evidence.receipt.timing.durationMs + evidence.receipt.benchmarkResult.material.grading.timing.durationMs;
|
|
851
|
-
}
|
|
852
|
-
function combinedUsage(evidence) {
|
|
853
|
-
const candidate = evidence.receipt.modelSettlement.material.usage;
|
|
854
|
-
const grader = evidence.receipt.benchmarkResult.material.grading.usage;
|
|
855
|
-
return addFixedSpend(candidate, grader);
|
|
856
|
-
}
|
|
857
885
|
function cellIds(experiment) {
|
|
858
886
|
const { suite, tasks } = experiment.benchmark;
|
|
859
887
|
return suite.seeds.map((_, index) => {
|
|
@@ -862,10 +890,6 @@ function cellIds(experiment) {
|
|
|
862
890
|
return `${tasks[taskIndex]?.scenario.id ?? taskIndex}:${repetition}`;
|
|
863
891
|
});
|
|
864
892
|
}
|
|
865
|
-
function mean(values) {
|
|
866
|
-
if (values.length === 0) throw new Error("candidate experiment requires measured values");
|
|
867
|
-
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
868
|
-
}
|
|
869
893
|
//#endregion
|
|
870
894
|
//#region src/contract/profile-measured-comparison.ts
|
|
871
895
|
/** Content-address one held-out profile task before either state can execute it. */
|
|
@@ -991,19 +1015,7 @@ function measuredComparisonFromAgentProfileImprovementExperiment(options) {
|
|
|
991
1015
|
candidateContentHash: experiment.candidate.stateDigest
|
|
992
1016
|
},
|
|
993
1017
|
diff: canonicalCandidateJson(experiment.change),
|
|
994
|
-
evaluation:
|
|
995
|
-
generationsExplored: options.generationsExplored ?? 0,
|
|
996
|
-
preparation: options.preparation,
|
|
997
|
-
measurement: {
|
|
998
|
-
wallDurationMs: options.measurement.wallDurationMs,
|
|
999
|
-
workDurationMs: evaluation.measurementWorkDurationMs,
|
|
1000
|
-
cost: evaluation.measurementCost
|
|
1001
|
-
},
|
|
1002
|
-
total: {
|
|
1003
|
-
wallDurationMs: options.preparation.wallDurationMs + options.measurement.wallDurationMs,
|
|
1004
|
-
cost: evaluation.totalCost
|
|
1005
|
-
}
|
|
1006
|
-
},
|
|
1018
|
+
evaluation: comparisonEvaluation(options, evaluation),
|
|
1007
1019
|
...options.metadata ? { metadata: options.metadata } : {}
|
|
1008
1020
|
});
|
|
1009
1021
|
const { recordDigest: _recordDigest, ...provenance } = provisional.provenance;
|
|
@@ -1021,19 +1033,7 @@ function measuredComparisonFromAgentProfileImprovementExperiment(options) {
|
|
|
1021
1033
|
/** Recompute a profile comparison from the exact sealed experiment and receipts. */
|
|
1022
1034
|
function verifyAgentProfileImprovementExperimentComparison(input) {
|
|
1023
1035
|
const comparison = agentProfileImprovementMeasuredComparisonSchema.parse(input);
|
|
1024
|
-
const recomputed = measuredComparisonFromAgentProfileImprovementExperiment(
|
|
1025
|
-
experiment: comparison.experiment,
|
|
1026
|
-
measurements: comparison.measurements,
|
|
1027
|
-
runId: comparison.provenance.runId,
|
|
1028
|
-
...comparison.candidate ? { candidate: comparison.candidate } : {},
|
|
1029
|
-
generationsExplored: comparison.evaluation.generationsExplored,
|
|
1030
|
-
preparation: comparison.evaluation.preparation,
|
|
1031
|
-
measurement: {
|
|
1032
|
-
wallDurationMs: comparison.evaluation.measurement.wallDurationMs,
|
|
1033
|
-
cost: comparison.evaluation.measurement.cost
|
|
1034
|
-
},
|
|
1035
|
-
...comparison.metadata ? { metadata: comparison.metadata } : {}
|
|
1036
|
-
});
|
|
1036
|
+
const recomputed = measuredComparisonFromAgentProfileImprovementExperiment(comparisonRecomputationInput(comparison));
|
|
1037
1037
|
if (canonicalCandidateDigest(recomputed) !== canonicalCandidateDigest(comparison)) throw new Error("profile improvement comparison does not match its Runtime receipts");
|
|
1038
1038
|
return comparison;
|
|
1039
1039
|
}
|
|
@@ -1048,24 +1048,7 @@ function profileMeasurementModel(experiment) {
|
|
|
1048
1048
|
return models.length === 1 ? models[0] : "multiple-models";
|
|
1049
1049
|
}
|
|
1050
1050
|
function profileMeasurementCostReceipt(measurements, model) {
|
|
1051
|
-
|
|
1052
|
-
inputTokens: 0,
|
|
1053
|
-
outputTokens: 0,
|
|
1054
|
-
cachedInputTokens: 0,
|
|
1055
|
-
reasoningTokens: 0,
|
|
1056
|
-
modelCalls: 0,
|
|
1057
|
-
costUsdNanos: 0,
|
|
1058
|
-
costProvenance: "observed"
|
|
1059
|
-
});
|
|
1060
|
-
const costUsd = usage.costUsdNanos / 1e9;
|
|
1061
|
-
return {
|
|
1062
|
-
model,
|
|
1063
|
-
inputTokens: usage.inputTokens,
|
|
1064
|
-
outputTokens: usage.outputTokens,
|
|
1065
|
-
cachedTokens: usage.cachedInputTokens,
|
|
1066
|
-
reasoningTokens: usage.reasoningTokens,
|
|
1067
|
-
...usage.costProvenance === "observed" ? { actualCostUsd: costUsd } : { estimatedCostUsd: costUsd }
|
|
1068
|
-
};
|
|
1051
|
+
return pairedFixedSpendReceipt(measurements, model, combinedProfileUsage);
|
|
1069
1052
|
}
|
|
1070
1053
|
function combinedProfileUsage(receipt) {
|
|
1071
1054
|
return addFixedSpend(receipt.usage, receipt.grading.usage);
|
|
@@ -1146,186 +1129,10 @@ const profileReceiptAdapter = {
|
|
|
1146
1129
|
passed: (receipt) => receipt.grading.passed
|
|
1147
1130
|
};
|
|
1148
1131
|
//#endregion
|
|
1149
|
-
//#region src/contract/intake/
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
* The on-disk counterpart to the in-memory intake adapters: point it at a
|
|
1154
|
-
* single `.json` (array) / `.jsonl` (one record per line) file or at a
|
|
1155
|
-
* directory of such files, and it returns the substrate-canonical
|
|
1156
|
-
* `RunRecord[]` ready for `analyzeRuns({ runs })`.
|
|
1157
|
-
*
|
|
1158
|
-
* Validation is at the boundary: each parsed object goes through
|
|
1159
|
-
* `parseRunRecordSafe`. By default an invalid record fails loud with its
|
|
1160
|
-
* file + index; pass `onInvalid: 'collect'` to keep the valid records and
|
|
1161
|
-
* receive the rejects as structured diagnostics instead.
|
|
1162
|
-
*/
|
|
1163
|
-
const ANALYSIS_ARTIFACT$1 = "analysis.json";
|
|
1164
|
-
function defaultInclude(fileName) {
|
|
1165
|
-
if (fileName === ANALYSIS_ARTIFACT$1) return false;
|
|
1166
|
-
return fileName.endsWith(".json") || fileName.endsWith(".jsonl");
|
|
1167
|
-
}
|
|
1168
|
-
/**
|
|
1169
|
-
* Resolve a file or directory path into validated `RunRecord[]`.
|
|
1170
|
-
*
|
|
1171
|
-
* A `.json` file must parse to a top-level array; a `.jsonl` file is one
|
|
1172
|
-
* record per non-empty line. Directories are read shallowly by default
|
|
1173
|
-
* (set `recursive` to descend); the `analysis.json` output artifact is
|
|
1174
|
-
* always excluded.
|
|
1175
|
-
*/
|
|
1176
|
-
async function fromRunRecordDir(path, options = {}) {
|
|
1177
|
-
const onInvalid = options.onInvalid ?? "throw";
|
|
1178
|
-
const include = options.include ?? defaultInclude;
|
|
1179
|
-
const filePaths = (await stat(path)).isDirectory() ? await collectFiles(path, include, options.recursive ?? false) : [path];
|
|
1180
|
-
const runs = [];
|
|
1181
|
-
const rejected = [];
|
|
1182
|
-
for (const file of filePaths) {
|
|
1183
|
-
const raw = await parseRecordFile(file);
|
|
1184
|
-
for (const { index, value } of raw) {
|
|
1185
|
-
const parsed = parseRunRecordSafe(value);
|
|
1186
|
-
if (parsed.ok) {
|
|
1187
|
-
runs.push(parsed.value);
|
|
1188
|
-
continue;
|
|
1189
|
-
}
|
|
1190
|
-
const rejection = {
|
|
1191
|
-
file,
|
|
1192
|
-
index,
|
|
1193
|
-
reason: parsed.error.message
|
|
1194
|
-
};
|
|
1195
|
-
if (onInvalid === "throw") throw new Error(`fromRunRecordDir: invalid RunRecord in '${file}' at index ${index}: ${parsed.error.message}`);
|
|
1196
|
-
rejected.push(rejection);
|
|
1197
|
-
}
|
|
1198
|
-
}
|
|
1199
|
-
return {
|
|
1200
|
-
runs,
|
|
1201
|
-
rejected,
|
|
1202
|
-
files: filePaths
|
|
1203
|
-
};
|
|
1204
|
-
}
|
|
1205
|
-
/** Read a single `.json` / `.jsonl` file into `{ index, value }` pairs. A
|
|
1206
|
-
* malformed JSONL line throws with its line number rather than being skipped —
|
|
1207
|
-
* silent line-dropping is how corpora quietly shrink. */
|
|
1208
|
-
async function parseRecordFile(file) {
|
|
1209
|
-
const trimmed = (await readFile(file, "utf8")).trim();
|
|
1210
|
-
if (trimmed.length === 0) return [];
|
|
1211
|
-
if (trimmed.startsWith("[")) {
|
|
1212
|
-
const parsed = JSON.parse(trimmed);
|
|
1213
|
-
if (!Array.isArray(parsed)) throw new Error(`fromRunRecordDir: file '${file}' did not parse to an array`);
|
|
1214
|
-
return parsed.map((value, index) => ({
|
|
1215
|
-
index,
|
|
1216
|
-
value
|
|
1217
|
-
}));
|
|
1218
|
-
}
|
|
1219
|
-
const out = [];
|
|
1220
|
-
const lines = trimmed.split("\n");
|
|
1221
|
-
for (let i = 0; i < lines.length; i++) {
|
|
1222
|
-
const line = lines[i].trim();
|
|
1223
|
-
if (line.length === 0) continue;
|
|
1224
|
-
try {
|
|
1225
|
-
out.push({
|
|
1226
|
-
index: i,
|
|
1227
|
-
value: JSON.parse(line)
|
|
1228
|
-
});
|
|
1229
|
-
} catch (err) {
|
|
1230
|
-
throw new Error(`fromRunRecordDir: file '${file}' line ${i + 1} is not valid JSON: ${err instanceof Error ? err.message : String(err)}`);
|
|
1231
|
-
}
|
|
1232
|
-
}
|
|
1233
|
-
return out;
|
|
1234
|
-
}
|
|
1235
|
-
/** Sorted file list under a directory, filtered by `include`. Sorted so the
|
|
1236
|
-
* resulting `RunRecord` order — and any downstream fingerprint — is stable
|
|
1237
|
-
* across filesystems. */
|
|
1238
|
-
async function collectFiles(dir, include, recursive) {
|
|
1239
|
-
const entries = await readdir(dir, { withFileTypes: true });
|
|
1240
|
-
const files = [];
|
|
1241
|
-
const subdirs = [];
|
|
1242
|
-
for (const entry of entries) {
|
|
1243
|
-
if (entry.isDirectory()) {
|
|
1244
|
-
if (recursive) subdirs.push(join(dir, entry.name));
|
|
1245
|
-
continue;
|
|
1246
|
-
}
|
|
1247
|
-
if (include(entry.name)) files.push(join(dir, entry.name));
|
|
1248
|
-
}
|
|
1249
|
-
files.sort();
|
|
1250
|
-
subdirs.sort();
|
|
1251
|
-
for (const sub of subdirs) files.push(...await collectFiles(sub, include, recursive));
|
|
1252
|
-
return files;
|
|
1253
|
-
}
|
|
1254
|
-
//#endregion
|
|
1255
|
-
//#region src/contract/eval-reporting-suite.ts
|
|
1256
|
-
/**
|
|
1257
|
-
* # `evalReportingSuite` — one call from runs (or a run dir) to `analysis.json`.
|
|
1258
|
-
*
|
|
1259
|
-
* A thin wrapper over the analysis primitive (`analyzeRuns`) and the on-disk
|
|
1260
|
-
* intake adapter (`fromRunRecordDir`). It does NOT reimplement any statistics,
|
|
1261
|
-
* distributions, or clustering — it resolves the input into validated
|
|
1262
|
-
* `RunRecord[]`, calls `analyzeRuns` with the options you'd pass it directly,
|
|
1263
|
-
* wraps the result in a small provenance envelope, and (optionally) writes a
|
|
1264
|
-
* single `analysis.json` artifact.
|
|
1265
|
-
*
|
|
1266
|
-
* ```ts
|
|
1267
|
-
* // From a directory of run files, write ./runs/analysis.json:
|
|
1268
|
-
* const suite = await evalReportingSuite('./runs', { write: true })
|
|
1269
|
-
* // From records already in memory, no write:
|
|
1270
|
-
* const suite = await evalReportingSuite(records, { analyze: { decisionThreshold: 0.03 } })
|
|
1271
|
-
* suite.report // the InsightReport — distributions, paired lift, findings rollup
|
|
1272
|
-
* ```
|
|
1273
|
-
*/
|
|
1274
|
-
const ANALYSIS_ARTIFACT = "analysis.json";
|
|
1275
|
-
/**
|
|
1276
|
-
* Resolve runs (or a run dir/file), run `analyzeRuns`, and optionally persist a
|
|
1277
|
-
* single `analysis.json`. The only analysis logic lives in `analyzeRuns`; this
|
|
1278
|
-
* function is composition + I/O.
|
|
1279
|
-
*/
|
|
1280
|
-
async function evalReportingSuite(input, options = {}) {
|
|
1281
|
-
const fromPath = typeof input === "string";
|
|
1282
|
-
let runs;
|
|
1283
|
-
let files = [];
|
|
1284
|
-
let rejected = [];
|
|
1285
|
-
if (fromPath) {
|
|
1286
|
-
const loaded = await fromRunRecordDir(input, options.load);
|
|
1287
|
-
runs = loaded.runs;
|
|
1288
|
-
files = loaded.files;
|
|
1289
|
-
rejected = loaded.rejected;
|
|
1290
|
-
} else runs = input;
|
|
1291
|
-
if (runs.length === 0) throw new Error(fromPath ? `evalReportingSuite: no RunRecords found at '${input}'` : "evalReportingSuite: no RunRecords to analyze");
|
|
1292
|
-
const result = {
|
|
1293
|
-
report: await analyzeRuns({
|
|
1294
|
-
...options.analyze,
|
|
1295
|
-
runs
|
|
1296
|
-
}),
|
|
1297
|
-
provenance: {
|
|
1298
|
-
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1299
|
-
runCount: runs.length,
|
|
1300
|
-
sourcePath: fromPath ? input : null,
|
|
1301
|
-
files,
|
|
1302
|
-
rejected
|
|
1303
|
-
},
|
|
1304
|
-
writtenTo: null
|
|
1305
|
-
};
|
|
1306
|
-
const target = resolveWriteTarget(options.write, fromPath ? input : null);
|
|
1307
|
-
if (target) {
|
|
1308
|
-
await mkdir(dirname(target), { recursive: true });
|
|
1309
|
-
await writeFile(target, `${JSON.stringify(result, null, 2)}\n`, "utf8");
|
|
1310
|
-
result.writtenTo = target;
|
|
1311
|
-
}
|
|
1312
|
-
return result;
|
|
1313
|
-
}
|
|
1314
|
-
/** Resolve where (if anywhere) to write `analysis.json`. Returns null when
|
|
1315
|
-
* writing is disabled. Throws on `write: true` with in-memory input — there is
|
|
1316
|
-
* no directory to anchor the artifact to, and silently inventing `cwd` would
|
|
1317
|
-
* scatter files. */
|
|
1318
|
-
function resolveWriteTarget(write, sourcePath) {
|
|
1319
|
-
if (!write) return null;
|
|
1320
|
-
if (typeof write === "string") return write.endsWith("/") || !write.endsWith(".json") && !write.endsWith(".jsonl") ? join(write, ANALYSIS_ARTIFACT) : write;
|
|
1321
|
-
if (sourcePath === null) throw new Error("evalReportingSuite: write:true needs a source path to anchor analysis.json — pass an explicit output path when analyzing in-memory records");
|
|
1322
|
-
return sourcePath.endsWith(".json") || sourcePath.endsWith(".jsonl") ? join(dirname(sourcePath), ANALYSIS_ARTIFACT) : join(sourcePath, ANALYSIS_ARTIFACT);
|
|
1323
|
-
}
|
|
1324
|
-
//#endregion
|
|
1325
|
-
//#region src/contract/intake/agent-trace.ts
|
|
1326
|
-
function rangeLines(r) {
|
|
1327
|
-
return Math.max(0, r.end_line - r.start_line + 1);
|
|
1328
|
-
}
|
|
1132
|
+
//#region src/contract/intake/agent-trace.ts
|
|
1133
|
+
function rangeLines(r) {
|
|
1134
|
+
return Math.max(0, r.end_line - r.start_line + 1);
|
|
1135
|
+
}
|
|
1329
1136
|
/**
|
|
1330
1137
|
* Build a commit → provenance index from Agent Trace records. Multiple records
|
|
1331
1138
|
* for the same revision are merged. Records without `vcs.revision` are skipped
|
|
@@ -1402,6 +1209,94 @@ function partitionRunsByAuthoringModel(runs, index) {
|
|
|
1402
1209
|
};
|
|
1403
1210
|
}
|
|
1404
1211
|
//#endregion
|
|
1212
|
+
//#region src/contract/intake/code-agent-fields.ts
|
|
1213
|
+
function record(value) {
|
|
1214
|
+
return isRecord(value) ? value : null;
|
|
1215
|
+
}
|
|
1216
|
+
function stringField(obj, key) {
|
|
1217
|
+
const value = obj[key];
|
|
1218
|
+
return typeof value === "string" && value.length > 0 ? value : void 0;
|
|
1219
|
+
}
|
|
1220
|
+
function numberField(obj, key) {
|
|
1221
|
+
const value = obj[key];
|
|
1222
|
+
return typeof value === "number" && Number.isFinite(value) ? value : void 0;
|
|
1223
|
+
}
|
|
1224
|
+
function timestampMs(value) {
|
|
1225
|
+
if (typeof value === "number" && Number.isFinite(value)) return value > 0xe8d4a51000 ? value : value * 1e3;
|
|
1226
|
+
if (typeof value === "string" && value.length > 0) {
|
|
1227
|
+
const parsed = Date.parse(value);
|
|
1228
|
+
return Number.isFinite(parsed) ? parsed : void 0;
|
|
1229
|
+
}
|
|
1230
|
+
}
|
|
1231
|
+
//#endregion
|
|
1232
|
+
//#region src/contract/intake/code-agent-identity.ts
|
|
1233
|
+
function sessionIdFromEntries(source, entries) {
|
|
1234
|
+
for (const entry of entries) {
|
|
1235
|
+
if (source === "pi" && entry.type === "session") {
|
|
1236
|
+
const id = stringField(entry, "id");
|
|
1237
|
+
if (id) return id;
|
|
1238
|
+
}
|
|
1239
|
+
if (source === "codex") {
|
|
1240
|
+
const threadId = stringField(entry, "thread_id");
|
|
1241
|
+
if (threadId) return threadId;
|
|
1242
|
+
const payload = record(entry.payload);
|
|
1243
|
+
const id = payload ? stringField(payload, "id") : void 0;
|
|
1244
|
+
if (id) return id;
|
|
1245
|
+
}
|
|
1246
|
+
const payload = record(record(entry.message)?.payload);
|
|
1247
|
+
const sessionId = stringField(entry, "sessionID") ?? stringField(entry, "sessionId") ?? stringField(entry, "session_id") ?? stringField(payload ?? {}, "session_id");
|
|
1248
|
+
if (sessionId) return sessionId;
|
|
1249
|
+
}
|
|
1250
|
+
}
|
|
1251
|
+
function modelFromEntries(source, entries) {
|
|
1252
|
+
if (source === "pi") return summarizeSessionInput("pi", { records: entries }).lastServedModel ?? void 0;
|
|
1253
|
+
let providerModel;
|
|
1254
|
+
for (const entry of entries) {
|
|
1255
|
+
if (source === "codex") {
|
|
1256
|
+
const payload = record(entry.payload);
|
|
1257
|
+
const model = payload ? stringField(payload, "model") : void 0;
|
|
1258
|
+
if (model) return model;
|
|
1259
|
+
providerModel = providerModel ?? (payload ? stringField(payload, "model_provider") : void 0);
|
|
1260
|
+
}
|
|
1261
|
+
const openCodeModel = stringField(entry, "modelID");
|
|
1262
|
+
if (openCodeModel) {
|
|
1263
|
+
const provider = stringField(entry, "providerID");
|
|
1264
|
+
return provider ? `${provider}/${openCodeModel}` : openCodeModel;
|
|
1265
|
+
}
|
|
1266
|
+
const modelObject = record(entry.model);
|
|
1267
|
+
const nestedModel = stringField(modelObject ?? {}, "modelID");
|
|
1268
|
+
if (nestedModel) {
|
|
1269
|
+
const provider = stringField(modelObject ?? {}, "providerID");
|
|
1270
|
+
return provider ? `${provider}/${nestedModel}` : nestedModel;
|
|
1271
|
+
}
|
|
1272
|
+
const message = record(entry.message);
|
|
1273
|
+
const model = message ? stringField(message, "model") : void 0;
|
|
1274
|
+
if (model) return model;
|
|
1275
|
+
}
|
|
1276
|
+
return providerModel;
|
|
1277
|
+
}
|
|
1278
|
+
function cwdFromEntries(entries) {
|
|
1279
|
+
for (const entry of entries) {
|
|
1280
|
+
const path = record(entry.path);
|
|
1281
|
+
const cwd = stringField(entry, "cwd") ?? stringField(path ?? {}, "cwd") ?? stringField(path ?? {}, "root") ?? (record(entry.payload) ? stringField(record(entry.payload), "cwd") : void 0);
|
|
1282
|
+
if (cwd) return cwd;
|
|
1283
|
+
}
|
|
1284
|
+
}
|
|
1285
|
+
function fallbackSessionId$1(source, sourcePath) {
|
|
1286
|
+
return hashString(`${source}:${sourcePath ?? "unknown-session"}`).slice(7, 27);
|
|
1287
|
+
}
|
|
1288
|
+
function withSnapshot(model) {
|
|
1289
|
+
const trimmed = model.trim() || "unknown";
|
|
1290
|
+
if (trimmed.includes("@") || /-\d{8}$/.test(trimmed) || /-\d{4}-\d{2}-\d{2}$/.test(trimmed)) return trimmed;
|
|
1291
|
+
return `${trimmed}@observed-local`;
|
|
1292
|
+
}
|
|
1293
|
+
function hashString(value) {
|
|
1294
|
+
return `sha256:${createHash("sha256").update(value).digest("hex")}`;
|
|
1295
|
+
}
|
|
1296
|
+
function stableSeed(value) {
|
|
1297
|
+
return createHash("sha256").update(value).digest().readUInt32BE(0);
|
|
1298
|
+
}
|
|
1299
|
+
//#endregion
|
|
1405
1300
|
//#region src/contract/intake/code-agent-observation.ts
|
|
1406
1301
|
/**
|
|
1407
1302
|
* Where a session came from. `pi` is a native Pi session (`~/.pi/agent/sessions/*.jsonl`), read
|
|
@@ -1414,12 +1309,12 @@ function partitionRunsByAuthoringModel(runs, index) {
|
|
|
1414
1309
|
* stay out of this projection; callers retain the source JSONL as evidence.
|
|
1415
1310
|
*/
|
|
1416
1311
|
function observeCodeAgentSession(options) {
|
|
1417
|
-
const entries = options.entries.filter(isRecord
|
|
1312
|
+
const entries = options.entries.filter(isRecord);
|
|
1418
1313
|
const projection = projectionFor(options.source, entries);
|
|
1419
1314
|
const terminal = terminalWithExecution(projection, options.execution);
|
|
1420
1315
|
return {
|
|
1421
1316
|
source: options.source,
|
|
1422
|
-
sessionId: sessionIdFromEntries
|
|
1317
|
+
sessionId: sessionIdFromEntries(options.source, entries) ?? fallbackSessionId(options.source, options.sourcePath),
|
|
1423
1318
|
finalText: projection.finalText,
|
|
1424
1319
|
terminal,
|
|
1425
1320
|
actions: projection.actions
|
|
@@ -1443,16 +1338,16 @@ function codexProjection(entries) {
|
|
|
1443
1338
|
let terminal = "unknown";
|
|
1444
1339
|
let explicitTerminal = false;
|
|
1445
1340
|
for (const entry of entries) {
|
|
1446
|
-
const entryType = stringField
|
|
1447
|
-
const payload = record
|
|
1448
|
-
const payloadType = stringField
|
|
1449
|
-
const timestampMs =
|
|
1341
|
+
const entryType = stringField(entry, "type");
|
|
1342
|
+
const payload = record(entry.payload) ?? {};
|
|
1343
|
+
const payloadType = stringField(payload, "type");
|
|
1344
|
+
const timestampMs$1 = timestampMs(entry.timestamp);
|
|
1450
1345
|
const itemEvent = admittedCodexItem(entry, transcriptStream);
|
|
1451
1346
|
const item = itemEvent?.item ?? null;
|
|
1452
1347
|
if (itemEvent?.eventType === "item.started" && item) {
|
|
1453
|
-
const itemType = stringField
|
|
1348
|
+
const itemType = stringField(item, "type");
|
|
1454
1349
|
if (isCodexActionItem(itemType)) {
|
|
1455
|
-
const id = stringField
|
|
1350
|
+
const id = stringField(item, "id") ?? `item-${actions.length}`;
|
|
1456
1351
|
const action = actionFor({
|
|
1457
1352
|
id,
|
|
1458
1353
|
stepIndex: actions.length,
|
|
@@ -1460,7 +1355,7 @@ function codexProjection(entries) {
|
|
|
1460
1355
|
surface: codexItemSurface(itemType, item),
|
|
1461
1356
|
name: codexItemName(itemType, item),
|
|
1462
1357
|
status: "started",
|
|
1463
|
-
timestampMs,
|
|
1358
|
+
timestampMs: timestampMs$1,
|
|
1464
1359
|
metadata: compactMetadata({
|
|
1465
1360
|
sourceEventType: itemEvent.eventType,
|
|
1466
1361
|
itemType
|
|
@@ -1471,10 +1366,10 @@ function codexProjection(entries) {
|
|
|
1471
1366
|
}
|
|
1472
1367
|
}
|
|
1473
1368
|
if (itemEvent?.eventType === "item.completed" && item) {
|
|
1474
|
-
const itemType = stringField
|
|
1475
|
-
if (itemType === "agent_message" || itemType === "message") finalText = nonEmpty(stringField
|
|
1369
|
+
const itemType = stringField(item, "type");
|
|
1370
|
+
if (itemType === "agent_message" || itemType === "message") finalText = nonEmpty(stringField(item, "text")) ?? finalText;
|
|
1476
1371
|
if (isCodexActionItem(itemType)) {
|
|
1477
|
-
const id = stringField
|
|
1372
|
+
const id = stringField(item, "id") ?? `item-${actions.length}`;
|
|
1478
1373
|
const status = codexItemStatus(itemType, item);
|
|
1479
1374
|
const existing = calls.get(id);
|
|
1480
1375
|
if (existing) existing.status = status;
|
|
@@ -1486,7 +1381,7 @@ function codexProjection(entries) {
|
|
|
1486
1381
|
surface: codexItemSurface(itemType, item),
|
|
1487
1382
|
name: codexItemName(itemType, item),
|
|
1488
1383
|
status,
|
|
1489
|
-
timestampMs,
|
|
1384
|
+
timestampMs: timestampMs$1,
|
|
1490
1385
|
metadata: compactMetadata({
|
|
1491
1386
|
sourceEventType: itemEvent.eventType,
|
|
1492
1387
|
itemType
|
|
@@ -1500,8 +1395,8 @@ function codexProjection(entries) {
|
|
|
1500
1395
|
if (entryType === "response_item") {
|
|
1501
1396
|
if (payloadType === "message" || payloadType === "agent_message") finalText = textFromMessagePayload(payload) ?? finalText;
|
|
1502
1397
|
if (payloadType === "function_call" || payloadType === "custom_tool_call") {
|
|
1503
|
-
const id = stringField
|
|
1504
|
-
const name = stringField
|
|
1398
|
+
const id = stringField(payload, "call_id") ?? stringField(payload, "id") ?? `call-${actions.length}`;
|
|
1399
|
+
const name = stringField(payload, "name") ?? payloadType;
|
|
1505
1400
|
const action = actionFor({
|
|
1506
1401
|
id,
|
|
1507
1402
|
stepIndex: actions.length,
|
|
@@ -1509,7 +1404,7 @@ function codexProjection(entries) {
|
|
|
1509
1404
|
surface: surfaceForTool(name),
|
|
1510
1405
|
name,
|
|
1511
1406
|
status: "started",
|
|
1512
|
-
timestampMs,
|
|
1407
|
+
timestampMs: timestampMs$1,
|
|
1513
1408
|
metadata: compactMetadata({
|
|
1514
1409
|
sourceEventType: entryType,
|
|
1515
1410
|
payloadType
|
|
@@ -1519,23 +1414,23 @@ function codexProjection(entries) {
|
|
|
1519
1414
|
actions.push(action);
|
|
1520
1415
|
}
|
|
1521
1416
|
if (payloadType === "function_call_output" || payloadType === "custom_tool_call_output") {
|
|
1522
|
-
const id = stringField
|
|
1417
|
+
const id = stringField(payload, "call_id") ?? stringField(payload, "id");
|
|
1523
1418
|
const existing = id ? calls.get(id) : void 0;
|
|
1524
1419
|
if (existing) existing.status = looksLikeError(payload.output) ? "failed" : "completed";
|
|
1525
1420
|
}
|
|
1526
1421
|
}
|
|
1527
1422
|
if (entryType === "event_msg") {
|
|
1528
|
-
if (payloadType === "agent_message") finalText = nonEmpty(stringField
|
|
1423
|
+
if (payloadType === "agent_message") finalText = nonEmpty(stringField(payload, "message")) ?? nonEmpty(stringField(payload, "text")) ?? finalText;
|
|
1529
1424
|
if (payloadType === "patch_apply_end") {
|
|
1530
1425
|
const patchStatus = typeof payload.success === "boolean" ? payload.success ? "completed" : "failed" : "unknown";
|
|
1531
1426
|
actions.push(actionFor({
|
|
1532
|
-
id: stringField
|
|
1427
|
+
id: stringField(payload, "call_id") ?? `patch-${actions.length}`,
|
|
1533
1428
|
stepIndex: actions.length,
|
|
1534
1429
|
kind: "patch",
|
|
1535
1430
|
surface: "code",
|
|
1536
1431
|
name: "patch",
|
|
1537
1432
|
status: patchStatus,
|
|
1538
|
-
timestampMs,
|
|
1433
|
+
timestampMs: timestampMs$1,
|
|
1539
1434
|
metadata: compactMetadata({
|
|
1540
1435
|
sourceEventType: entryType,
|
|
1541
1436
|
payloadType
|
|
@@ -1543,15 +1438,15 @@ function codexProjection(entries) {
|
|
|
1543
1438
|
}));
|
|
1544
1439
|
}
|
|
1545
1440
|
if (payloadType === "sub_agent_activity") {
|
|
1546
|
-
const name = stringField
|
|
1441
|
+
const name = stringField(payload, "kind") ?? "subagent";
|
|
1547
1442
|
actions.push(actionFor({
|
|
1548
|
-
id: stringField
|
|
1443
|
+
id: stringField(payload, "event_id") ?? `subagent-${actions.length}`,
|
|
1549
1444
|
stepIndex: actions.length,
|
|
1550
1445
|
kind: "tool",
|
|
1551
1446
|
surface: "subagent",
|
|
1552
1447
|
name,
|
|
1553
1448
|
status: statusFrom(payload),
|
|
1554
|
-
timestampMs,
|
|
1449
|
+
timestampMs: timestampMs$1,
|
|
1555
1450
|
metadata: compactMetadata({
|
|
1556
1451
|
sourceEventType: entryType,
|
|
1557
1452
|
payloadType
|
|
@@ -1562,20 +1457,20 @@ function codexProjection(entries) {
|
|
|
1562
1457
|
const completed = payloadType === "task_complete";
|
|
1563
1458
|
terminal = completed ? "completed" : "failed";
|
|
1564
1459
|
explicitTerminal = true;
|
|
1565
|
-
actions.push(terminalAction(actions.length, payloadType, completed, timestampMs));
|
|
1460
|
+
actions.push(terminalAction(actions.length, payloadType, completed, timestampMs$1));
|
|
1566
1461
|
}
|
|
1567
1462
|
}
|
|
1568
1463
|
if (entryType === "turn.completed" || entryType === "exec_done") {
|
|
1569
1464
|
const completed = entry.success !== false;
|
|
1570
1465
|
terminal = completed ? "completed" : "failed";
|
|
1571
1466
|
explicitTerminal = true;
|
|
1572
|
-
finalText = nonEmpty(stringField
|
|
1573
|
-
actions.push(terminalAction(actions.length, entryType, completed, timestampMs));
|
|
1467
|
+
finalText = nonEmpty(stringField(entry, "message")) ?? finalText;
|
|
1468
|
+
actions.push(terminalAction(actions.length, entryType, completed, timestampMs$1));
|
|
1574
1469
|
}
|
|
1575
1470
|
if (entryType === "turn.failed" || entryType === "error") {
|
|
1576
1471
|
terminal = "failed";
|
|
1577
1472
|
explicitTerminal = true;
|
|
1578
|
-
actions.push(terminalAction(actions.length, entryType, false, timestampMs));
|
|
1473
|
+
actions.push(terminalAction(actions.length, entryType, false, timestampMs$1));
|
|
1579
1474
|
}
|
|
1580
1475
|
}
|
|
1581
1476
|
return {
|
|
@@ -1593,20 +1488,20 @@ function claudeProjection(entries) {
|
|
|
1593
1488
|
let terminal = "unknown";
|
|
1594
1489
|
let explicitTerminal = false;
|
|
1595
1490
|
for (const entry of entries) {
|
|
1596
|
-
const entryType = stringField
|
|
1597
|
-
const timestampMs =
|
|
1598
|
-
const message = record
|
|
1491
|
+
const entryType = stringField(entry, "type");
|
|
1492
|
+
const timestampMs$2 = timestampMs(entry.timestamp);
|
|
1493
|
+
const message = record(entry.message);
|
|
1599
1494
|
const content = Array.isArray(message?.content) ? message.content : [];
|
|
1600
1495
|
if (entryType === "assistant") {
|
|
1601
1496
|
const text = textBlocks(content);
|
|
1602
1497
|
if (text !== null) finalText = text;
|
|
1603
1498
|
for (const value of content) {
|
|
1604
|
-
const part = record
|
|
1499
|
+
const part = record(value);
|
|
1605
1500
|
if (!part) continue;
|
|
1606
|
-
const partType = stringField
|
|
1501
|
+
const partType = stringField(part, "type");
|
|
1607
1502
|
if (partType === "tool_use") {
|
|
1608
|
-
const id = stringField
|
|
1609
|
-
const name = stringField
|
|
1503
|
+
const id = stringField(part, "id") ?? `tool-${actions.length}`;
|
|
1504
|
+
const name = stringField(part, "name") ?? "tool";
|
|
1610
1505
|
const action = actionFor({
|
|
1611
1506
|
id,
|
|
1612
1507
|
stepIndex: actions.length,
|
|
@@ -1614,7 +1509,7 @@ function claudeProjection(entries) {
|
|
|
1614
1509
|
surface: surfaceForTool(name),
|
|
1615
1510
|
name,
|
|
1616
1511
|
status: "started",
|
|
1617
|
-
timestampMs,
|
|
1512
|
+
timestampMs: timestampMs$2,
|
|
1618
1513
|
metadata: compactMetadata({
|
|
1619
1514
|
sourceEventType: entryType,
|
|
1620
1515
|
partType
|
|
@@ -1625,17 +1520,17 @@ function claudeProjection(entries) {
|
|
|
1625
1520
|
}
|
|
1626
1521
|
}
|
|
1627
1522
|
completeClaudeTools(content, calls);
|
|
1628
|
-
const stopReason = stringField
|
|
1523
|
+
const stopReason = stringField(message ?? {}, "stop_reason");
|
|
1629
1524
|
if (stopReason === "end_turn") {
|
|
1630
1525
|
terminal = "completed";
|
|
1631
1526
|
explicitTerminal = true;
|
|
1632
|
-
actions.push(terminalAction(actions.length, stopReason, true, timestampMs));
|
|
1527
|
+
actions.push(terminalAction(actions.length, stopReason, true, timestampMs$2));
|
|
1633
1528
|
}
|
|
1634
1529
|
}
|
|
1635
1530
|
if (entryType === "user") completeClaudeTools(content, calls);
|
|
1636
1531
|
if (entryType === "tool_use") {
|
|
1637
|
-
const id = stringField
|
|
1638
|
-
const name = stringField
|
|
1532
|
+
const id = stringField(entry, "id") ?? `tool-${actions.length}`;
|
|
1533
|
+
const name = stringField(entry, "name") ?? "tool";
|
|
1639
1534
|
const action = actionFor({
|
|
1640
1535
|
id,
|
|
1641
1536
|
stepIndex: actions.length,
|
|
@@ -1643,47 +1538,47 @@ function claudeProjection(entries) {
|
|
|
1643
1538
|
surface: surfaceForTool(name),
|
|
1644
1539
|
name,
|
|
1645
1540
|
status: "started",
|
|
1646
|
-
timestampMs,
|
|
1541
|
+
timestampMs: timestampMs$2,
|
|
1647
1542
|
metadata: { sourceEventType: entryType }
|
|
1648
1543
|
});
|
|
1649
1544
|
calls.set(id, action);
|
|
1650
1545
|
actions.push(action);
|
|
1651
1546
|
}
|
|
1652
1547
|
if (entryType === "tool_result") {
|
|
1653
|
-
const id = stringField
|
|
1548
|
+
const id = stringField(entry, "tool_use_id");
|
|
1654
1549
|
const action = id ? calls.get(id) : void 0;
|
|
1655
1550
|
if (action) action.status = entry.is_error === true ? "failed" : "completed";
|
|
1656
1551
|
}
|
|
1657
1552
|
if (entryType === "system") {
|
|
1658
|
-
const subtype = stringField
|
|
1659
|
-
const hookName = stringField
|
|
1553
|
+
const subtype = stringField(entry, "subtype");
|
|
1554
|
+
const hookName = stringField(entry, "hook_name") ?? stringField(entry, "hookName");
|
|
1660
1555
|
if (subtype?.toLowerCase().includes("hook") || hookName) actions.push(actionFor({
|
|
1661
|
-
id: stringField
|
|
1556
|
+
id: stringField(entry, "uuid") ?? `hook-${actions.length}`,
|
|
1662
1557
|
stepIndex: actions.length,
|
|
1663
1558
|
kind: "tool",
|
|
1664
1559
|
surface: "hook",
|
|
1665
1560
|
name: hookName ?? subtype ?? "hook",
|
|
1666
1561
|
status: statusFrom(entry),
|
|
1667
|
-
timestampMs,
|
|
1562
|
+
timestampMs: timestampMs$2,
|
|
1668
1563
|
metadata: compactMetadata({
|
|
1669
1564
|
sourceEventType: entryType,
|
|
1670
1565
|
subtype
|
|
1671
1566
|
})
|
|
1672
1567
|
}));
|
|
1673
1568
|
}
|
|
1674
|
-
if (entryType === "pr-link") actions.push(terminalAction(actions.length, entryType, true, timestampMs));
|
|
1569
|
+
if (entryType === "pr-link") actions.push(terminalAction(actions.length, entryType, true, timestampMs$2));
|
|
1675
1570
|
if (entryType === "result") {
|
|
1676
|
-
resultText = nonEmpty(stringField
|
|
1677
|
-
const subtype = stringField
|
|
1571
|
+
resultText = nonEmpty(stringField(entry, "result")) ?? resultText;
|
|
1572
|
+
const subtype = stringField(entry, "subtype");
|
|
1678
1573
|
const completed = entry.is_error !== true && !subtype?.toLowerCase().includes("error");
|
|
1679
1574
|
terminal = completed ? "completed" : "failed";
|
|
1680
1575
|
explicitTerminal = true;
|
|
1681
|
-
actions.push(terminalAction(actions.length, subtype ?? entryType, completed, timestampMs));
|
|
1576
|
+
actions.push(terminalAction(actions.length, subtype ?? entryType, completed, timestampMs$2));
|
|
1682
1577
|
}
|
|
1683
1578
|
if (entryType === "error") {
|
|
1684
1579
|
terminal = "failed";
|
|
1685
1580
|
explicitTerminal = true;
|
|
1686
|
-
actions.push(terminalAction(actions.length, entryType, false, timestampMs));
|
|
1581
|
+
actions.push(terminalAction(actions.length, entryType, false, timestampMs$2));
|
|
1687
1582
|
}
|
|
1688
1583
|
}
|
|
1689
1584
|
return {
|
|
@@ -1695,28 +1590,28 @@ function claudeProjection(entries) {
|
|
|
1695
1590
|
}
|
|
1696
1591
|
function openCodeProjection(entries) {
|
|
1697
1592
|
const actions = [];
|
|
1698
|
-
const assistantMessageIds = new Set(entries.filter((entry) => stringField
|
|
1593
|
+
const assistantMessageIds = new Set(entries.filter((entry) => stringField(entry, "role") === "assistant").map((entry) => stringField(entry, "id")).filter((id) => id !== void 0));
|
|
1699
1594
|
let finalText = null;
|
|
1700
1595
|
let terminal = "unknown";
|
|
1701
1596
|
let explicitTerminal = false;
|
|
1702
1597
|
for (const entry of entries) {
|
|
1703
|
-
const entryType = stringField
|
|
1704
|
-
const part = record
|
|
1705
|
-
const partType = stringField
|
|
1706
|
-
const timestampMs =
|
|
1707
|
-
if (entryType === "text" && record
|
|
1708
|
-
else if (partType === "text" && (assistantMessageIds.size === 0 || assistantMessageIds.has(stringField
|
|
1598
|
+
const entryType = stringField(entry, "type");
|
|
1599
|
+
const part = record(entry.part) ?? entry;
|
|
1600
|
+
const partType = stringField(part, "type");
|
|
1601
|
+
const timestampMs$3 = timestampMs(entry.timestamp) ?? timestampMs(record(entry.time)?.created);
|
|
1602
|
+
if (entryType === "text" && record(entry.part)) finalText = nonEmpty(stringField(part, "text")) ?? finalText;
|
|
1603
|
+
else if (partType === "text" && (assistantMessageIds.size === 0 || assistantMessageIds.has(stringField(part, "messageID") ?? ""))) finalText = nonEmpty(stringField(part, "text")) ?? finalText;
|
|
1709
1604
|
if (entryType === "tool_use" || partType === "tool") {
|
|
1710
|
-
const state = record
|
|
1711
|
-
const name = stringField
|
|
1605
|
+
const state = record(part.state) ?? {};
|
|
1606
|
+
const name = stringField(part, "tool") ?? "tool";
|
|
1712
1607
|
actions.push(actionFor({
|
|
1713
|
-
id: stringField
|
|
1608
|
+
id: stringField(part, "callID") ?? stringField(part, "id") ?? `tool-${actions.length}`,
|
|
1714
1609
|
stepIndex: actions.length,
|
|
1715
1610
|
kind: "tool",
|
|
1716
1611
|
surface: surfaceForTool(name),
|
|
1717
1612
|
name,
|
|
1718
1613
|
status: statusFrom(state),
|
|
1719
|
-
timestampMs,
|
|
1614
|
+
timestampMs: timestampMs$3,
|
|
1720
1615
|
metadata: compactMetadata({
|
|
1721
1616
|
sourceEventType: entryType,
|
|
1722
1617
|
partType
|
|
@@ -1724,31 +1619,31 @@ function openCodeProjection(entries) {
|
|
|
1724
1619
|
}));
|
|
1725
1620
|
}
|
|
1726
1621
|
if (partType === "patch") actions.push(actionFor({
|
|
1727
|
-
id: stringField
|
|
1622
|
+
id: stringField(part, "id") ?? `patch-${actions.length}`,
|
|
1728
1623
|
stepIndex: actions.length,
|
|
1729
1624
|
kind: "patch",
|
|
1730
1625
|
surface: "code",
|
|
1731
1626
|
name: "patch",
|
|
1732
1627
|
status: "completed",
|
|
1733
|
-
timestampMs,
|
|
1628
|
+
timestampMs: timestampMs$3,
|
|
1734
1629
|
metadata: compactMetadata({
|
|
1735
1630
|
sourceEventType: entryType,
|
|
1736
1631
|
partType
|
|
1737
1632
|
})
|
|
1738
1633
|
}));
|
|
1739
|
-
if (stringField
|
|
1740
|
-
const finish = stringField
|
|
1634
|
+
if (stringField(entry, "role") === "assistant") {
|
|
1635
|
+
const finish = stringField(entry, "finish");
|
|
1741
1636
|
if (finish === "stop" || finish === "error") {
|
|
1742
1637
|
const completed = finish === "stop";
|
|
1743
1638
|
terminal = completed ? "completed" : "failed";
|
|
1744
1639
|
explicitTerminal = true;
|
|
1745
|
-
actions.push(terminalAction(actions.length, finish, completed, timestampMs));
|
|
1640
|
+
actions.push(terminalAction(actions.length, finish, completed, timestampMs$3));
|
|
1746
1641
|
}
|
|
1747
1642
|
}
|
|
1748
1643
|
if (entryType === "error") {
|
|
1749
1644
|
terminal = "failed";
|
|
1750
1645
|
explicitTerminal = true;
|
|
1751
|
-
actions.push(terminalAction(actions.length, entryType, false, timestampMs));
|
|
1646
|
+
actions.push(terminalAction(actions.length, entryType, false, timestampMs$3));
|
|
1752
1647
|
}
|
|
1753
1648
|
}
|
|
1754
1649
|
return {
|
|
@@ -1765,15 +1660,15 @@ function kimiProjection(entries) {
|
|
|
1765
1660
|
let terminal = "unknown";
|
|
1766
1661
|
let explicitTerminal = false;
|
|
1767
1662
|
for (const entry of entries) {
|
|
1768
|
-
const timestampMs =
|
|
1769
|
-
const message = record
|
|
1770
|
-
const messageType = stringField
|
|
1771
|
-
const payload = record
|
|
1772
|
-
if (messageType === "ContentPart" && stringField
|
|
1663
|
+
const timestampMs$4 = timestampMs(entry.timestamp);
|
|
1664
|
+
const message = record(entry.message);
|
|
1665
|
+
const messageType = stringField(message ?? {}, "type");
|
|
1666
|
+
const payload = record(message?.payload) ?? {};
|
|
1667
|
+
if (messageType === "ContentPart" && stringField(payload, "type") === "text") finalText = nonEmpty(stringField(payload, "text")) ?? nonEmpty(stringField(payload, "content")) ?? finalText;
|
|
1773
1668
|
if (messageType === "ToolCall") {
|
|
1774
|
-
const fn = record
|
|
1775
|
-
const id = stringField
|
|
1776
|
-
const name = stringField
|
|
1669
|
+
const fn = record(payload.function) ?? {};
|
|
1670
|
+
const id = stringField(payload, "id") ?? `tool-${actions.length}`;
|
|
1671
|
+
const name = stringField(fn, "name") ?? "tool";
|
|
1777
1672
|
const action = actionFor({
|
|
1778
1673
|
id,
|
|
1779
1674
|
stepIndex: actions.length,
|
|
@@ -1781,22 +1676,22 @@ function kimiProjection(entries) {
|
|
|
1781
1676
|
surface: surfaceForTool(name),
|
|
1782
1677
|
name,
|
|
1783
1678
|
status: "started",
|
|
1784
|
-
timestampMs,
|
|
1679
|
+
timestampMs: timestampMs$4,
|
|
1785
1680
|
metadata: { sourceEventType: messageType }
|
|
1786
1681
|
});
|
|
1787
1682
|
calls.set(id, action);
|
|
1788
1683
|
actions.push(action);
|
|
1789
1684
|
}
|
|
1790
1685
|
if (messageType === "ToolResult") {
|
|
1791
|
-
const id = stringField
|
|
1686
|
+
const id = stringField(payload, "tool_call_id");
|
|
1792
1687
|
const action = id ? calls.get(id) : void 0;
|
|
1793
|
-
if (action) action.status = record
|
|
1688
|
+
if (action) action.status = record(payload.return_value)?.is_error === true ? "failed" : "completed";
|
|
1794
1689
|
}
|
|
1795
1690
|
if (messageType === "TurnEnd" || messageType === "StepInterrupted") {
|
|
1796
1691
|
const completed = messageType === "TurnEnd";
|
|
1797
1692
|
terminal = completed ? "completed" : "failed";
|
|
1798
1693
|
explicitTerminal = true;
|
|
1799
|
-
actions.push(terminalAction(actions.length, messageType, completed, timestampMs));
|
|
1694
|
+
actions.push(terminalAction(actions.length, messageType, completed, timestampMs$4));
|
|
1800
1695
|
}
|
|
1801
1696
|
}
|
|
1802
1697
|
return {
|
|
@@ -1816,7 +1711,7 @@ function piSessionProjection(entries) {
|
|
|
1816
1711
|
surface: surfaceForTool(call.name),
|
|
1817
1712
|
name: call.name,
|
|
1818
1713
|
status: call.status === "completed" ? "completed" : call.status === "error" ? "failed" : call.status === "pending" ? "started" : "unknown",
|
|
1819
|
-
timestampMs:
|
|
1714
|
+
timestampMs: timestampMs(call.startedAt),
|
|
1820
1715
|
metadata: { sourceEventType: "pi-tool-call" }
|
|
1821
1716
|
}));
|
|
1822
1717
|
const answer = [...session.messages].reverse().find((m) => m.role === "assistant" && m.actor === "agent" && m.parts.some((p) => p.type === "text"));
|
|
@@ -1836,17 +1731,17 @@ function piGraphProjection(entries) {
|
|
|
1836
1731
|
for (const entry of entries) {
|
|
1837
1732
|
const nodes = Array.isArray(entry.nodes) ? entry.nodes : [];
|
|
1838
1733
|
for (const value of nodes) {
|
|
1839
|
-
const node = record
|
|
1840
|
-
const ir = record
|
|
1841
|
-
const kind = stringField
|
|
1734
|
+
const node = record(value);
|
|
1735
|
+
const ir = record(node?.ir) ?? node;
|
|
1736
|
+
const kind = stringField(ir ?? {}, "kind");
|
|
1842
1737
|
if (kind === "ToolInvocation") actions.push(actionFor({
|
|
1843
|
-
id: stringField
|
|
1738
|
+
id: stringField(ir ?? {}, "id") ?? stringField(node ?? {}, "id") ?? `tool-${actions.length}`,
|
|
1844
1739
|
stepIndex: actions.length,
|
|
1845
1740
|
kind: "tool",
|
|
1846
1741
|
surface: "tool",
|
|
1847
|
-
name: stringField
|
|
1742
|
+
name: stringField(ir ?? {}, "name") ?? "graph-tool",
|
|
1848
1743
|
status: "started",
|
|
1849
|
-
timestampMs:
|
|
1744
|
+
timestampMs: timestampMs(ir?.createdAt),
|
|
1850
1745
|
metadata: {
|
|
1851
1746
|
sourceEventType: "graph-node",
|
|
1852
1747
|
graphKind: kind
|
|
@@ -1860,13 +1755,13 @@ function piGraphProjection(entries) {
|
|
|
1860
1755
|
terminal = "completed";
|
|
1861
1756
|
explicitTerminal = true;
|
|
1862
1757
|
actions.push(actionFor({
|
|
1863
|
-
id: stringField
|
|
1758
|
+
id: stringField(ir ?? {}, "id") ?? stringField(node ?? {}, "id") ?? `completion-${actions.length}`,
|
|
1864
1759
|
stepIndex: actions.length,
|
|
1865
1760
|
kind: "graph-completion",
|
|
1866
1761
|
surface: "tool",
|
|
1867
1762
|
name: "complete",
|
|
1868
1763
|
status: "completed",
|
|
1869
|
-
timestampMs:
|
|
1764
|
+
timestampMs: timestampMs(ir?.createdAt),
|
|
1870
1765
|
metadata: {
|
|
1871
1766
|
sourceEventType: "graph-node",
|
|
1872
1767
|
graphKind: kind
|
|
@@ -1884,9 +1779,9 @@ function piGraphProjection(entries) {
|
|
|
1884
1779
|
}
|
|
1885
1780
|
function completeClaudeTools(content, calls) {
|
|
1886
1781
|
for (const value of content) {
|
|
1887
|
-
const part = record
|
|
1888
|
-
if (!part || stringField
|
|
1889
|
-
const id = stringField
|
|
1782
|
+
const part = record(value);
|
|
1783
|
+
if (!part || stringField(part, "type") !== "tool_result") continue;
|
|
1784
|
+
const id = stringField(part, "tool_use_id");
|
|
1890
1785
|
const action = id ? calls.get(id) : void 0;
|
|
1891
1786
|
if (action) action.status = part.is_error === true ? "failed" : "completed";
|
|
1892
1787
|
}
|
|
@@ -1900,21 +1795,21 @@ function completeClaudeTools(content, calls) {
|
|
|
1900
1795
|
* PascalCase. Read both here so one branch serves both transports.
|
|
1901
1796
|
*/
|
|
1902
1797
|
function codexItemEvent(entry) {
|
|
1903
|
-
const entryType = stringField
|
|
1798
|
+
const entryType = stringField(entry, "type");
|
|
1904
1799
|
if (entryType === "item.started" || entryType === "item.completed") {
|
|
1905
|
-
const item = record
|
|
1800
|
+
const item = record(entry.item);
|
|
1906
1801
|
return item ? {
|
|
1907
1802
|
eventType: entryType,
|
|
1908
1803
|
item
|
|
1909
1804
|
} : null;
|
|
1910
1805
|
}
|
|
1911
1806
|
if (entryType !== "event_msg") return null;
|
|
1912
|
-
const payload = record
|
|
1913
|
-
const payloadType = payload ? stringField
|
|
1807
|
+
const payload = record(entry.payload);
|
|
1808
|
+
const payloadType = payload ? stringField(payload, "type") : void 0;
|
|
1914
1809
|
if (payloadType !== "item_started" && payloadType !== "item_completed") return null;
|
|
1915
|
-
const item = record
|
|
1810
|
+
const item = record(payload?.item);
|
|
1916
1811
|
if (!item) return null;
|
|
1917
|
-
const itemType = codexItemType(stringField
|
|
1812
|
+
const itemType = codexItemType(stringField(item, "type"));
|
|
1918
1813
|
return {
|
|
1919
1814
|
eventType: payloadType === "item_started" ? "item.started" : "item.completed",
|
|
1920
1815
|
item: itemType === void 0 ? item : {
|
|
@@ -1934,7 +1829,7 @@ function codexItemType(itemType) {
|
|
|
1934
1829
|
return itemType.replace(/([a-z0-9])([A-Z])/g, "$1_$2").toLowerCase();
|
|
1935
1830
|
}
|
|
1936
1831
|
function hasCodexTranscriptStream(entries) {
|
|
1937
|
-
return entries.some((entry) => stringField
|
|
1832
|
+
return entries.some((entry) => stringField(entry, "type") === "response_item");
|
|
1938
1833
|
}
|
|
1939
1834
|
/**
|
|
1940
1835
|
* A rollout file records each turn twice: `response_item` holds the model
|
|
@@ -1962,7 +1857,7 @@ function admittedCodexItem(entry, transcriptStream) {
|
|
|
1962
1857
|
const event = codexItemEvent(entry);
|
|
1963
1858
|
if (!event) return null;
|
|
1964
1859
|
if (!transcriptStream) return event;
|
|
1965
|
-
return isCodexExecutionOnlyItem(stringField
|
|
1860
|
+
return isCodexExecutionOnlyItem(stringField(event.item, "type")) ? event : null;
|
|
1966
1861
|
}
|
|
1967
1862
|
function isCodexActionItem(itemType) {
|
|
1968
1863
|
return itemType === "command_execution" || itemType === "mcp_tool_call" || itemType === "collab_tool_call" || itemType === "web_search" || itemType === "file_change";
|
|
@@ -1972,27 +1867,27 @@ function codexItemSurface(itemType, item) {
|
|
|
1972
1867
|
if (itemType === "collab_tool_call") return "subagent";
|
|
1973
1868
|
if (itemType === "web_search") return "web";
|
|
1974
1869
|
if (itemType === "file_change") return "code";
|
|
1975
|
-
return surfaceForTool(stringField
|
|
1870
|
+
return surfaceForTool(stringField(item, "name") ?? itemType ?? "tool");
|
|
1976
1871
|
}
|
|
1977
1872
|
function codexItemName(itemType, item) {
|
|
1978
|
-
if (itemType === "mcp_tool_call") return [stringField
|
|
1979
|
-
if (itemType === "collab_tool_call") return stringField
|
|
1873
|
+
if (itemType === "mcp_tool_call") return [stringField(item, "server"), stringField(item, "tool")].filter(Boolean).join("/") || "mcp";
|
|
1874
|
+
if (itemType === "collab_tool_call") return stringField(item, "tool") ?? "subagent";
|
|
1980
1875
|
if (itemType === "web_search") return "web_search";
|
|
1981
1876
|
if (itemType === "file_change") return "file_change";
|
|
1982
|
-
return stringField
|
|
1877
|
+
return stringField(item, "name") ?? itemType ?? "tool";
|
|
1983
1878
|
}
|
|
1984
1879
|
function codexItemStatus(itemType, item) {
|
|
1985
1880
|
const status = statusFrom(item);
|
|
1986
1881
|
if (status === "failed") return status;
|
|
1987
1882
|
if (itemType === "command_execution") {
|
|
1988
|
-
const exitCode = numberField
|
|
1883
|
+
const exitCode = numberField(item, "exit_code");
|
|
1989
1884
|
if (exitCode !== void 0) return exitCode === 0 ? "completed" : "failed";
|
|
1990
1885
|
}
|
|
1991
|
-
if (itemType === "mcp_tool_call" && record
|
|
1886
|
+
if (itemType === "mcp_tool_call" && record(item.error)) return "failed";
|
|
1992
1887
|
return status === "unknown" || status === "started" ? "completed" : status;
|
|
1993
1888
|
}
|
|
1994
1889
|
function textFromMessagePayload(payload) {
|
|
1995
|
-
const direct = nonEmpty(stringField
|
|
1890
|
+
const direct = nonEmpty(stringField(payload, "text"));
|
|
1996
1891
|
if (direct) return direct;
|
|
1997
1892
|
const content = payload.content;
|
|
1998
1893
|
if (typeof content === "string") return nonEmpty(content);
|
|
@@ -2000,7 +1895,7 @@ function textFromMessagePayload(payload) {
|
|
|
2000
1895
|
return null;
|
|
2001
1896
|
}
|
|
2002
1897
|
function textBlocks(values) {
|
|
2003
|
-
return nonEmpty(values.map((value) => record
|
|
1898
|
+
return nonEmpty(values.map((value) => record(value)).filter((value) => value !== null).filter((value) => stringField(value, "type") === "text").map((value) => stringField(value, "text")).filter((value) => value !== void 0).join("\n"));
|
|
2004
1899
|
}
|
|
2005
1900
|
function terminalWithExecution(projection, execution) {
|
|
2006
1901
|
if (!execution) return {
|
|
@@ -2056,7 +1951,7 @@ function isCodexCollaborationTool(normalized) {
|
|
|
2056
1951
|
return /^[a-z_]+_agents?$/.test(verb) || verb === "send_message" || verb === "send_input" || verb === "followup_task";
|
|
2057
1952
|
}
|
|
2058
1953
|
function statusFrom(value) {
|
|
2059
|
-
const status = (stringField
|
|
1954
|
+
const status = (stringField(value, "status") ?? "").toLowerCase();
|
|
2060
1955
|
if (status === "completed" || status === "success" || status === "succeeded") return "completed";
|
|
2061
1956
|
if (status === "error" || status === "failed" || status === "declined") return "failed";
|
|
2062
1957
|
if (status === "running" || status === "started" || status === "in_progress") return "started";
|
|
@@ -2065,31 +1960,13 @@ function statusFrom(value) {
|
|
|
2065
1960
|
function compactMetadata(input) {
|
|
2066
1961
|
return Object.fromEntries(Object.entries(input).filter((entry) => entry[1] !== void 0));
|
|
2067
1962
|
}
|
|
2068
|
-
function
|
|
2069
|
-
for (const entry of entries) {
|
|
2070
|
-
if (source === "pi" && entry.type === "session") {
|
|
2071
|
-
const id = stringField$1(entry, "id");
|
|
2072
|
-
if (id) return id;
|
|
2073
|
-
}
|
|
2074
|
-
if (source === "codex") {
|
|
2075
|
-
const threadId = stringField$1(entry, "thread_id");
|
|
2076
|
-
if (threadId) return threadId;
|
|
2077
|
-
const payload = record$1(entry.payload);
|
|
2078
|
-
const id = payload ? stringField$1(payload, "id") : void 0;
|
|
2079
|
-
if (id) return id;
|
|
2080
|
-
}
|
|
2081
|
-
const payload = record$1(record$1(entry.message)?.payload);
|
|
2082
|
-
const sessionId = stringField$1(entry, "sessionID") ?? stringField$1(entry, "sessionId") ?? stringField$1(entry, "session_id") ?? stringField$1(payload ?? {}, "session_id");
|
|
2083
|
-
if (sessionId) return sessionId;
|
|
2084
|
-
}
|
|
2085
|
-
}
|
|
2086
|
-
function fallbackSessionId$1(source, sourcePath) {
|
|
1963
|
+
function fallbackSessionId(source, sourcePath) {
|
|
2087
1964
|
return `${source}:${sourcePath ?? "unknown-session"}`;
|
|
2088
1965
|
}
|
|
2089
1966
|
function looksLikeError(value) {
|
|
2090
|
-
if (isRecord
|
|
1967
|
+
if (isRecord(value)) {
|
|
2091
1968
|
if (value.success === false || value.is_error === true) return true;
|
|
2092
|
-
const exitCode = numberField
|
|
1969
|
+
const exitCode = numberField(value, "exit_code");
|
|
2093
1970
|
if (exitCode !== void 0 && exitCode !== 0) return true;
|
|
2094
1971
|
}
|
|
2095
1972
|
const text = typeof value === "string" ? value : JSON.stringify(value ?? "");
|
|
@@ -2098,298 +1975,147 @@ function looksLikeError(value) {
|
|
|
2098
1975
|
function nonEmpty(value) {
|
|
2099
1976
|
return value !== void 0 && value.length > 0 ? value : null;
|
|
2100
1977
|
}
|
|
2101
|
-
|
|
2102
|
-
|
|
1978
|
+
//#endregion
|
|
1979
|
+
//#region src/contract/intake/code-agent-session-metrics.ts
|
|
1980
|
+
function emptyMetrics(entries) {
|
|
1981
|
+
return {
|
|
1982
|
+
entries,
|
|
1983
|
+
userMessages: 0,
|
|
1984
|
+
assistantMessages: 0,
|
|
1985
|
+
reasoningItems: 0,
|
|
1986
|
+
toolCalls: 0,
|
|
1987
|
+
toolOutputs: 0,
|
|
1988
|
+
toolErrors: 0,
|
|
1989
|
+
unclassifiedErrors: 0,
|
|
1990
|
+
patchAttempts: 0,
|
|
1991
|
+
patchSuccesses: 0,
|
|
1992
|
+
patchFailures: 0,
|
|
1993
|
+
turnsStarted: 0,
|
|
1994
|
+
turnsCompleted: 0,
|
|
1995
|
+
turnsAborted: 0,
|
|
1996
|
+
contextCompactions: 0,
|
|
1997
|
+
mcpCalls: 0,
|
|
1998
|
+
subagentCalls: 0,
|
|
1999
|
+
skillCalls: 0,
|
|
2000
|
+
hookCalls: 0,
|
|
2001
|
+
webCalls: 0,
|
|
2002
|
+
codeActions: 0,
|
|
2003
|
+
prLinks: 0,
|
|
2004
|
+
fileSnapshots: 0,
|
|
2005
|
+
graphNodes: 0,
|
|
2006
|
+
graphEdges: 0,
|
|
2007
|
+
actionCandidates: 0,
|
|
2008
|
+
verificationReports: 0,
|
|
2009
|
+
completionDecisions: 0,
|
|
2010
|
+
reliabilityRows: 0,
|
|
2011
|
+
reliabilityLift: 0,
|
|
2012
|
+
inputTokens: 0,
|
|
2013
|
+
outputTokens: 0,
|
|
2014
|
+
reasoningTokens: 0,
|
|
2015
|
+
cachedTokens: 0,
|
|
2016
|
+
cacheWriteTokens: 0,
|
|
2017
|
+
observedCostUsd: 0,
|
|
2018
|
+
observedCostCaptured: false,
|
|
2019
|
+
wallMs: 0,
|
|
2020
|
+
processScore: 0
|
|
2021
|
+
};
|
|
2103
2022
|
}
|
|
2104
|
-
function
|
|
2105
|
-
|
|
2023
|
+
function penalizeErrors(base, metrics) {
|
|
2024
|
+
const operations = Math.max(1, metrics.toolCalls + metrics.patchAttempts);
|
|
2025
|
+
const errorRate = Math.min(1, (metrics.toolErrors + metrics.patchFailures) / operations);
|
|
2026
|
+
return clamp01(base * (1 - .5 * errorRate));
|
|
2106
2027
|
}
|
|
2107
|
-
function
|
|
2108
|
-
|
|
2109
|
-
|
|
2028
|
+
function emptyTokenTotals() {
|
|
2029
|
+
return {
|
|
2030
|
+
input: 0,
|
|
2031
|
+
output: 0,
|
|
2032
|
+
reasoning: 0,
|
|
2033
|
+
cached: 0,
|
|
2034
|
+
cacheWrite: 0
|
|
2035
|
+
};
|
|
2110
2036
|
}
|
|
2111
|
-
function
|
|
2112
|
-
|
|
2113
|
-
return typeof field === "number" && Number.isFinite(field) ? field : void 0;
|
|
2037
|
+
function hasTokenTotals(usage) {
|
|
2038
|
+
return usage.input > 0 || usage.output > 0 || usage.reasoning > 0 || usage.cached > 0 || usage.cacheWrite > 0;
|
|
2114
2039
|
}
|
|
2115
|
-
function
|
|
2116
|
-
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
|
|
2040
|
+
function addUsage(metrics, usage) {
|
|
2041
|
+
const parsed = readUsage(usage);
|
|
2042
|
+
metrics.inputTokens += parsed.input;
|
|
2043
|
+
metrics.outputTokens += parsed.output;
|
|
2044
|
+
metrics.reasoningTokens += parsed.reasoning;
|
|
2045
|
+
metrics.cachedTokens += parsed.cached;
|
|
2046
|
+
metrics.cacheWriteTokens += parsed.cacheWrite;
|
|
2121
2047
|
}
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
try {
|
|
2130
|
-
return {
|
|
2131
|
-
kind: "entry",
|
|
2132
|
-
lineNumber,
|
|
2133
|
-
entry: JSON.parse(trimmed)
|
|
2134
|
-
};
|
|
2135
|
-
} catch {
|
|
2136
|
-
return {
|
|
2137
|
-
kind: "malformed",
|
|
2138
|
-
lineNumber
|
|
2139
|
-
};
|
|
2140
|
-
}
|
|
2048
|
+
function addUsageTo(totals, usage) {
|
|
2049
|
+
const parsed = readUsage(usage);
|
|
2050
|
+
totals.input += parsed.input;
|
|
2051
|
+
totals.output += parsed.output;
|
|
2052
|
+
totals.reasoning += parsed.reasoning;
|
|
2053
|
+
totals.cached += parsed.cached;
|
|
2054
|
+
totals.cacheWrite += parsed.cacheWrite;
|
|
2141
2055
|
}
|
|
2142
|
-
function
|
|
2143
|
-
const
|
|
2144
|
-
let malformedLines = 0;
|
|
2145
|
-
let lineNumber = 0;
|
|
2146
|
-
for (const line of jsonl.split("\n")) {
|
|
2147
|
-
lineNumber += 1;
|
|
2148
|
-
const read = readCodeAgentJsonlLine(line, lineNumber);
|
|
2149
|
-
if (!read) continue;
|
|
2150
|
-
if (read.kind === "malformed") malformedLines += 1;
|
|
2151
|
-
else entries.push(read.entry);
|
|
2152
|
-
}
|
|
2056
|
+
function readUsage(usage) {
|
|
2057
|
+
const parsed = extractUsage(usage);
|
|
2153
2058
|
return {
|
|
2154
|
-
|
|
2155
|
-
|
|
2059
|
+
input: parsed?.input ?? 0,
|
|
2060
|
+
output: parsed?.output ?? 0,
|
|
2061
|
+
reasoning: parsed?.reasoning ?? 0,
|
|
2062
|
+
cached: parsed?.cached ?? 0,
|
|
2063
|
+
cacheWrite: parsed?.cacheWrite ?? 0
|
|
2156
2064
|
};
|
|
2157
2065
|
}
|
|
2158
|
-
|
|
2159
|
-
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
|
|
2165
|
-
|
|
2166
|
-
* two paths to report the same malformed count. */
|
|
2167
|
-
async function* streamCodeAgentJsonlFile(path) {
|
|
2168
|
-
const stream = createReadStream(path, { encoding: "utf8" });
|
|
2169
|
-
let pending = "";
|
|
2170
|
-
let lineNumber = 0;
|
|
2171
|
-
try {
|
|
2172
|
-
for await (const chunk of stream) {
|
|
2173
|
-
pending += chunk;
|
|
2174
|
-
let start = 0;
|
|
2175
|
-
for (let at = pending.indexOf("\n"); at !== -1; at = pending.indexOf("\n", start)) {
|
|
2176
|
-
lineNumber += 1;
|
|
2177
|
-
const read = readCodeAgentJsonlLine(pending.slice(start, at), lineNumber);
|
|
2178
|
-
if (read) yield read;
|
|
2179
|
-
start = at + 1;
|
|
2180
|
-
}
|
|
2181
|
-
pending = pending.slice(start);
|
|
2066
|
+
function resolveSessionCost(explicit, metrics, model) {
|
|
2067
|
+
if (explicit) {
|
|
2068
|
+
if (explicit.kind === "uncaptured") {
|
|
2069
|
+
if (explicit.usd !== null) throw new Error("uncaptured cost must have usd: null");
|
|
2070
|
+
return {
|
|
2071
|
+
costUsd: null,
|
|
2072
|
+
costProvenance: explicit
|
|
2073
|
+
};
|
|
2182
2074
|
}
|
|
2183
|
-
|
|
2184
|
-
|
|
2075
|
+
if (explicit.kind === "lower-bound") {
|
|
2076
|
+
if (explicit.usd !== null) throw new Error("lower-bound cost must have usd: null");
|
|
2077
|
+
if (!Number.isFinite(explicit.knownLowerBoundUsd) || explicit.knownLowerBoundUsd <= 0) throw new Error("lower-bound cost must have a finite, positive knownLowerBoundUsd");
|
|
2078
|
+
return {
|
|
2079
|
+
costUsd: null,
|
|
2080
|
+
costProvenance: explicit
|
|
2081
|
+
};
|
|
2082
|
+
}
|
|
2083
|
+
if (!Number.isFinite(explicit.usd) || explicit.usd < 0) throw new Error(`${explicit.kind} cost must be a finite, non-negative USD amount`);
|
|
2084
|
+
return {
|
|
2085
|
+
costUsd: explicit.usd,
|
|
2086
|
+
costProvenance: explicit
|
|
2087
|
+
};
|
|
2185
2088
|
}
|
|
2186
|
-
|
|
2187
|
-
|
|
2188
|
-
|
|
2189
|
-
|
|
2190
|
-
|
|
2191
|
-
|
|
2192
|
-
|
|
2193
|
-
|
|
2194
|
-
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
|
|
2199
|
-
entries,
|
|
2200
|
-
malformedLines
|
|
2201
|
-
};
|
|
2202
|
-
}
|
|
2203
|
-
function fromCodexSession(options) {
|
|
2204
|
-
return fromCodeAgentSession("codex", options);
|
|
2205
|
-
}
|
|
2206
|
-
function fromClaudeCodeSession(options) {
|
|
2207
|
-
return fromCodeAgentSession("claude-code", options);
|
|
2208
|
-
}
|
|
2209
|
-
function fromOpenCodeSession(options) {
|
|
2210
|
-
return fromCodeAgentSession("opencode", options);
|
|
2211
|
-
}
|
|
2212
|
-
function fromKimiCodeSession(options) {
|
|
2213
|
-
return fromCodeAgentSession("kimi-code", options);
|
|
2214
|
-
}
|
|
2215
|
-
/** A native Pi session (`~/.pi/agent/sessions/<cwd>/<ts>_<id>.jsonl`), parsed entries in order. */
|
|
2216
|
-
function fromPiSession(options) {
|
|
2217
|
-
return fromCodeAgentSession("pi", options);
|
|
2218
|
-
}
|
|
2219
|
-
/** The graph IR a Pi graph run exports (`nodes[].ir`), which is not a Pi session. */
|
|
2220
|
-
function fromPiGraphSession(options) {
|
|
2221
|
-
return fromCodeAgentSession("pi-graph", options);
|
|
2222
|
-
}
|
|
2223
|
-
function fromCodeAgentSession(source, options) {
|
|
2224
|
-
const entries = options.entries.filter(isRecord);
|
|
2225
|
-
if (entries.length === 0) return {
|
|
2226
|
-
runs: [],
|
|
2227
|
-
diagnostics: [{
|
|
2228
|
-
source,
|
|
2229
|
-
sessionId: fallbackSessionId(source, options.sourcePath),
|
|
2230
|
-
sourcePath: options.sourcePath,
|
|
2231
|
-
entries: 0,
|
|
2232
|
-
malformedLines: options.malformedLines ?? 0,
|
|
2233
|
-
hasExplicitTerminalSignal: false,
|
|
2234
|
-
hasFinalOutput: false,
|
|
2235
|
-
hasQualityLabel: false,
|
|
2236
|
-
hasTokenUsage: false,
|
|
2237
|
-
hasCost: false,
|
|
2238
|
-
costKind: "uncaptured",
|
|
2239
|
-
warnings: ["no parseable session entries"]
|
|
2240
|
-
}],
|
|
2241
|
-
metrics: [],
|
|
2242
|
-
observations: []
|
|
2243
|
-
};
|
|
2244
|
-
const metrics = metricsFor(source, entries);
|
|
2245
|
-
const sessionId = sessionIdFromEntries(source, entries) ?? fallbackSessionId(source, options.sourcePath);
|
|
2246
|
-
const observation = observeCodeAgentSession({
|
|
2247
|
-
source,
|
|
2248
|
-
entries,
|
|
2249
|
-
sourcePath: options.sourcePath,
|
|
2250
|
-
execution: options.execution
|
|
2251
|
-
});
|
|
2252
|
-
applyObservedSurfaceMetrics(metrics, observation);
|
|
2253
|
-
const model = withSnapshot(options.model ?? modelFromEntries(source, entries) ?? source);
|
|
2254
|
-
const tokenUsage = {
|
|
2255
|
-
input: metrics.inputTokens,
|
|
2256
|
-
output: metrics.outputTokens,
|
|
2257
|
-
...metrics.reasoningTokens > 0 ? { reasoning: metrics.reasoningTokens } : {},
|
|
2258
|
-
...metrics.cachedTokens > 0 ? { cached: metrics.cachedTokens } : {},
|
|
2259
|
-
...metrics.cacheWriteTokens > 0 ? { cacheWrite: metrics.cacheWriteTokens } : {}
|
|
2260
|
-
};
|
|
2261
|
-
const { costUsd, costProvenance } = resolveSessionCost(options.costProvenance, metrics, model);
|
|
2262
|
-
const qualityScore = finiteScore(options.score);
|
|
2263
|
-
const invalidQualityScore = options.score !== void 0 && qualityScore === void 0;
|
|
2264
|
-
const promptHash = options.promptHash ?? hashString(`prompt:${source}:${firstUserText(entries) ?? ""}`);
|
|
2265
|
-
const configHash = options.configHash ?? hashJson({
|
|
2266
|
-
source,
|
|
2267
|
-
model,
|
|
2268
|
-
sourcePath: options.sourcePath ?? null,
|
|
2269
|
-
cwd: cwdFromEntries(entries) ?? null,
|
|
2270
|
-
entryCount: entries.length
|
|
2271
|
-
});
|
|
2272
|
-
const explicitTerminal = observation.terminal.explicit;
|
|
2273
|
-
const warnings = diagnosticsFor(metrics, {
|
|
2274
|
-
model,
|
|
2275
|
-
explicitTerminal,
|
|
2276
|
-
malformedLines: options.malformedLines ?? 0,
|
|
2277
|
-
hasQualityLabel: qualityScore !== void 0,
|
|
2278
|
-
invalidQualityScore,
|
|
2279
|
-
costKind: costProvenance.kind
|
|
2280
|
-
});
|
|
2281
|
-
const terminalOutcome = terminalOutcomeFromSession(observation.terminal);
|
|
2282
|
-
const processExitCode = options.execution?.exitCode;
|
|
2283
|
-
const splitTag = options.splitTag ?? "holdout";
|
|
2284
|
-
return {
|
|
2285
|
-
runs: [{
|
|
2286
|
-
runId: `${source}:${sessionId}`,
|
|
2287
|
-
experimentId: options.experimentId ?? `${source}-local-sessions`,
|
|
2288
|
-
candidateId: options.candidateId ?? model,
|
|
2289
|
-
seed: options.seed ?? stableSeed(sessionId),
|
|
2290
|
-
model,
|
|
2291
|
-
promptHash,
|
|
2292
|
-
configHash,
|
|
2293
|
-
commitSha: options.commitSha ?? "local-session",
|
|
2294
|
-
wallMs: metrics.wallMs,
|
|
2089
|
+
if (metrics.observedCostCaptured) {
|
|
2090
|
+
const costProvenance = {
|
|
2091
|
+
kind: "observed",
|
|
2092
|
+
usd: metrics.observedCostUsd
|
|
2093
|
+
};
|
|
2094
|
+
return {
|
|
2095
|
+
costUsd: metrics.observedCostUsd,
|
|
2096
|
+
costProvenance
|
|
2097
|
+
};
|
|
2098
|
+
}
|
|
2099
|
+
if ((metrics.inputTokens > 0 || metrics.outputTokens > 0) && isModelPriced(model)) {
|
|
2100
|
+
const costUsd = estimateCost(metrics.inputTokens, metrics.outputTokens, model);
|
|
2101
|
+
return {
|
|
2295
2102
|
costUsd,
|
|
2296
|
-
costProvenance
|
|
2297
|
-
|
|
2298
|
-
|
|
2299
|
-
|
|
2300
|
-
|
|
2301
|
-
...qualityScore === void 0 ? {} : splitTag === "holdout" ? { holdoutScore: qualityScore } : { searchScore: qualityScore },
|
|
2302
|
-
raw: {
|
|
2303
|
-
entries: metrics.entries,
|
|
2304
|
-
user_messages: metrics.userMessages,
|
|
2305
|
-
assistant_messages: metrics.assistantMessages,
|
|
2306
|
-
reasoning_items: metrics.reasoningItems,
|
|
2307
|
-
tool_calls: metrics.toolCalls,
|
|
2308
|
-
tool_outputs: metrics.toolOutputs,
|
|
2309
|
-
tool_errors: metrics.toolErrors,
|
|
2310
|
-
execution_error_count: metrics.toolErrors,
|
|
2311
|
-
process_error_count: terminalOutcome === "failed" ? 1 : 0,
|
|
2312
|
-
unclassified_error_count: metrics.unclassifiedErrors,
|
|
2313
|
-
...processExitCode !== void 0 ? { process_exit_code: processExitCode } : {},
|
|
2314
|
-
patch_attempts: metrics.patchAttempts,
|
|
2315
|
-
patch_successes: metrics.patchSuccesses,
|
|
2316
|
-
patch_failures: metrics.patchFailures,
|
|
2317
|
-
turns_started: metrics.turnsStarted,
|
|
2318
|
-
turns_completed: metrics.turnsCompleted,
|
|
2319
|
-
turns_aborted: metrics.turnsAborted,
|
|
2320
|
-
context_compactions: metrics.contextCompactions,
|
|
2321
|
-
mcp_calls: metrics.mcpCalls,
|
|
2322
|
-
subagent_calls: metrics.subagentCalls,
|
|
2323
|
-
skill_calls: metrics.skillCalls,
|
|
2324
|
-
hook_calls: metrics.hookCalls,
|
|
2325
|
-
web_calls: metrics.webCalls,
|
|
2326
|
-
code_actions: metrics.codeActions,
|
|
2327
|
-
pr_links: metrics.prLinks,
|
|
2328
|
-
file_snapshots: metrics.fileSnapshots,
|
|
2329
|
-
graph_nodes: metrics.graphNodes,
|
|
2330
|
-
graph_edges: metrics.graphEdges,
|
|
2331
|
-
action_candidates: metrics.actionCandidates,
|
|
2332
|
-
verification_reports: metrics.verificationReports,
|
|
2333
|
-
completion_decisions: metrics.completionDecisions,
|
|
2334
|
-
reliability_rows: metrics.reliabilityRows,
|
|
2335
|
-
reliability_lift: metrics.reliabilityLift,
|
|
2336
|
-
input_tokens: metrics.inputTokens,
|
|
2337
|
-
output_tokens: metrics.outputTokens,
|
|
2338
|
-
reasoning_tokens: metrics.reasoningTokens,
|
|
2339
|
-
cached_tokens: metrics.cachedTokens,
|
|
2340
|
-
cache_write_tokens: metrics.cacheWriteTokens,
|
|
2341
|
-
observed_cost_usd: metrics.observedCostUsd,
|
|
2342
|
-
observed_cost_captured: metrics.observedCostCaptured ? 1 : 0,
|
|
2343
|
-
estimated_cost_usd: costProvenance.kind === "estimated" ? costProvenance.usd : 0,
|
|
2344
|
-
process_score: metrics.processScore,
|
|
2345
|
-
explicit_terminal_signal: explicitTerminal ? 1 : 0,
|
|
2346
|
-
quality_label_present: qualityScore !== void 0 ? 1 : 0,
|
|
2347
|
-
cost_observed: costProvenance.kind === "observed" ? 1 : 0,
|
|
2348
|
-
cost_estimated: costProvenance.kind === "estimated" ? 1 : 0,
|
|
2349
|
-
cost_uncaptured: costProvenance.kind === "uncaptured" ? 1 : 0,
|
|
2350
|
-
cost_unknown: costProvenance.usd === null ? 1 : 0
|
|
2351
|
-
}
|
|
2352
|
-
},
|
|
2353
|
-
splitTag,
|
|
2354
|
-
scenarioId: options.scenarioId ?? sessionId
|
|
2355
|
-
}],
|
|
2356
|
-
diagnostics: [{
|
|
2357
|
-
source,
|
|
2358
|
-
sessionId,
|
|
2359
|
-
sourcePath: options.sourcePath,
|
|
2360
|
-
entries: entries.length,
|
|
2361
|
-
malformedLines: options.malformedLines ?? 0,
|
|
2362
|
-
hasExplicitTerminalSignal: explicitTerminal,
|
|
2363
|
-
hasFinalOutput: observation.finalText !== null,
|
|
2364
|
-
hasQualityLabel: qualityScore !== void 0,
|
|
2365
|
-
hasTokenUsage: metrics.inputTokens > 0 || metrics.outputTokens > 0 || metrics.reasoningTokens > 0 || metrics.cachedTokens > 0 || metrics.cacheWriteTokens > 0,
|
|
2366
|
-
hasCost: costProvenance.usd !== null,
|
|
2367
|
-
costKind: costProvenance.kind,
|
|
2368
|
-
warnings
|
|
2369
|
-
}],
|
|
2370
|
-
metrics: [metrics],
|
|
2371
|
-
observations: [{
|
|
2372
|
-
...observation,
|
|
2373
|
-
sessionId
|
|
2374
|
-
}]
|
|
2375
|
-
};
|
|
2376
|
-
}
|
|
2377
|
-
function terminalOutcomeFromSession(terminal) {
|
|
2378
|
-
if (!terminal.explicit) return "unknown";
|
|
2379
|
-
if (terminal.status === "completed") return "succeeded";
|
|
2380
|
-
if (terminal.status === "failed") return "failed";
|
|
2381
|
-
return "unknown";
|
|
2382
|
-
}
|
|
2383
|
-
function metricsFor(source, entries) {
|
|
2384
|
-
switch (source) {
|
|
2385
|
-
case "codex": return codexMetrics(entries);
|
|
2386
|
-
case "claude-code": return claudeCodeMetrics(entries);
|
|
2387
|
-
case "opencode": return openCodeMetrics(entries);
|
|
2388
|
-
case "kimi-code": return kimiCodeMetrics(entries);
|
|
2389
|
-
case "pi": return piSessionMetrics(entries);
|
|
2390
|
-
case "pi-graph": return piGraphMetrics(entries);
|
|
2103
|
+
costProvenance: {
|
|
2104
|
+
kind: "estimated",
|
|
2105
|
+
usd: costUsd
|
|
2106
|
+
}
|
|
2107
|
+
};
|
|
2391
2108
|
}
|
|
2109
|
+
return {
|
|
2110
|
+
costUsd: null,
|
|
2111
|
+
costProvenance: {
|
|
2112
|
+
kind: "uncaptured",
|
|
2113
|
+
usd: null
|
|
2114
|
+
}
|
|
2115
|
+
};
|
|
2392
2116
|
}
|
|
2117
|
+
//#endregion
|
|
2118
|
+
//#region src/contract/intake/code-agent-codex-metrics.ts
|
|
2393
2119
|
function codexMetrics(entries) {
|
|
2394
2120
|
const metrics = emptyMetrics(entries.length);
|
|
2395
2121
|
const startedToolIds = /* @__PURE__ */ new Set();
|
|
@@ -2408,7 +2134,7 @@ function codexMetrics(entries) {
|
|
|
2408
2134
|
if (entryType === "turn.started") metrics.turnsStarted += 1;
|
|
2409
2135
|
if (entryType === "turn.completed") {
|
|
2410
2136
|
metrics.turnsCompleted += 1;
|
|
2411
|
-
|
|
2137
|
+
addUsage(metrics, record(entry.usage));
|
|
2412
2138
|
}
|
|
2413
2139
|
if (entryType === "turn.failed") metrics.turnsAborted += 1;
|
|
2414
2140
|
if (entryType === "error") metrics.unclassifiedErrors += 1;
|
|
@@ -2496,14 +2222,6 @@ function codexExecToolFailed(itemType, item) {
|
|
|
2496
2222
|
if (itemType === "mcp_tool_call") return status === "failed" || record(item.error) !== null;
|
|
2497
2223
|
return itemType === "collab_tool_call" && status === "failed";
|
|
2498
2224
|
}
|
|
2499
|
-
function addCodexExecUsage(metrics, usage) {
|
|
2500
|
-
const parsed = readUsage(usage);
|
|
2501
|
-
metrics.inputTokens += parsed.input;
|
|
2502
|
-
metrics.outputTokens += parsed.output;
|
|
2503
|
-
metrics.reasoningTokens += parsed.reasoning;
|
|
2504
|
-
metrics.cachedTokens += parsed.cached;
|
|
2505
|
-
metrics.cacheWriteTokens += parsed.cacheWrite;
|
|
2506
|
-
}
|
|
2507
2225
|
function setCodexCumulativeUsage(metrics, usage) {
|
|
2508
2226
|
const parsed = readUsage(usage);
|
|
2509
2227
|
metrics.inputTokens = Math.max(metrics.inputTokens, parsed.input);
|
|
@@ -2512,6 +2230,12 @@ function setCodexCumulativeUsage(metrics, usage) {
|
|
|
2512
2230
|
metrics.cachedTokens = Math.max(metrics.cachedTokens, parsed.cached);
|
|
2513
2231
|
metrics.cacheWriteTokens = Math.max(metrics.cacheWriteTokens, parsed.cacheWrite);
|
|
2514
2232
|
}
|
|
2233
|
+
function codexProcessScore(metrics) {
|
|
2234
|
+
const terminalTurns = metrics.turnsCompleted + metrics.turnsAborted;
|
|
2235
|
+
return penalizeErrors(terminalTurns > 0 ? metrics.turnsCompleted / terminalTurns : metrics.assistantMessages > 0 ? .5 : 0, metrics);
|
|
2236
|
+
}
|
|
2237
|
+
//#endregion
|
|
2238
|
+
//#region src/contract/intake/code-agent-harness-metrics.ts
|
|
2515
2239
|
function claudeCodeMetrics(entries) {
|
|
2516
2240
|
const metrics = emptyMetrics(entries.length);
|
|
2517
2241
|
let startedAt;
|
|
@@ -2708,170 +2432,302 @@ function piGraphMetrics(entries) {
|
|
|
2708
2432
|
metrics.turnsCompleted += 1;
|
|
2709
2433
|
}
|
|
2710
2434
|
}
|
|
2711
|
-
const averageReliability = numberField(entry, "averageNodeReliability");
|
|
2712
|
-
if (averageReliability !== void 0) bestReliabilityScore = maxOptional(bestReliabilityScore, averageReliability);
|
|
2713
|
-
const pessimisticPath = numberField(entry, "pessimisticPathEstimate");
|
|
2714
|
-
if (pessimisticPath !== void 0) bestReliabilityScore = maxOptional(bestReliabilityScore, pessimisticPath);
|
|
2715
|
-
const rows = Array.isArray(entry.rows) ? entry.rows : [];
|
|
2716
|
-
metrics.reliabilityRows += rows.length;
|
|
2717
|
-
for (const row of rows) {
|
|
2718
|
-
const obj = record(row);
|
|
2719
|
-
if (!obj) continue;
|
|
2720
|
-
const lift = numberField(obj, "lift");
|
|
2721
|
-
if (lift !== void 0) metrics.reliabilityLift = Math.max(metrics.reliabilityLift, lift);
|
|
2722
|
-
const validated = numberField(obj, "validatedSuccessEstimate");
|
|
2723
|
-
if (validated !== void 0) bestReliabilityScore = maxOptional(bestReliabilityScore, validated);
|
|
2724
|
-
}
|
|
2435
|
+
const averageReliability = numberField(entry, "averageNodeReliability");
|
|
2436
|
+
if (averageReliability !== void 0) bestReliabilityScore = maxOptional(bestReliabilityScore, averageReliability);
|
|
2437
|
+
const pessimisticPath = numberField(entry, "pessimisticPathEstimate");
|
|
2438
|
+
if (pessimisticPath !== void 0) bestReliabilityScore = maxOptional(bestReliabilityScore, pessimisticPath);
|
|
2439
|
+
const rows = Array.isArray(entry.rows) ? entry.rows : [];
|
|
2440
|
+
metrics.reliabilityRows += rows.length;
|
|
2441
|
+
for (const row of rows) {
|
|
2442
|
+
const obj = record(row);
|
|
2443
|
+
if (!obj) continue;
|
|
2444
|
+
const lift = numberField(obj, "lift");
|
|
2445
|
+
if (lift !== void 0) metrics.reliabilityLift = Math.max(metrics.reliabilityLift, lift);
|
|
2446
|
+
const validated = numberField(obj, "validatedSuccessEstimate");
|
|
2447
|
+
if (validated !== void 0) bestReliabilityScore = maxOptional(bestReliabilityScore, validated);
|
|
2448
|
+
}
|
|
2449
|
+
}
|
|
2450
|
+
metrics.processScore = bestReliabilityScore !== void 0 ? clamp01(bestReliabilityScore) : metrics.completionDecisions > 0 ? 1 : 0;
|
|
2451
|
+
return metrics;
|
|
2452
|
+
}
|
|
2453
|
+
function claudeProcessScore(metrics) {
|
|
2454
|
+
return penalizeErrors(metrics.prLinks > 0 ? 1 : metrics.assistantMessages > 0 ? .75 : 0, metrics);
|
|
2455
|
+
}
|
|
2456
|
+
function terminalProcessScore(metrics) {
|
|
2457
|
+
const terminalTurns = metrics.turnsCompleted + metrics.turnsAborted;
|
|
2458
|
+
return penalizeErrors(terminalTurns > 0 ? metrics.turnsCompleted / terminalTurns : metrics.assistantMessages > 0 ? .75 : metrics.toolCalls > 0 ? .5 : 0, metrics);
|
|
2459
|
+
}
|
|
2460
|
+
function maxOptional(current, next) {
|
|
2461
|
+
return current === void 0 ? next : Math.max(current, next);
|
|
2462
|
+
}
|
|
2463
|
+
//#endregion
|
|
2464
|
+
//#region src/contract/intake/code-agent-jsonl.ts
|
|
2465
|
+
/** The single per-line rule. Both the string path and the file path call this,
|
|
2466
|
+
* so malformed-line handling and entry validation cannot drift apart. */
|
|
2467
|
+
function readCodeAgentJsonlLine(line, lineNumber) {
|
|
2468
|
+
const trimmed = line.trim();
|
|
2469
|
+
if (!trimmed) return void 0;
|
|
2470
|
+
try {
|
|
2471
|
+
return {
|
|
2472
|
+
kind: "entry",
|
|
2473
|
+
lineNumber,
|
|
2474
|
+
entry: JSON.parse(trimmed)
|
|
2475
|
+
};
|
|
2476
|
+
} catch {
|
|
2477
|
+
return {
|
|
2478
|
+
kind: "malformed",
|
|
2479
|
+
lineNumber
|
|
2480
|
+
};
|
|
2481
|
+
}
|
|
2482
|
+
}
|
|
2483
|
+
function parseCodeAgentJsonl(jsonl) {
|
|
2484
|
+
const entries = [];
|
|
2485
|
+
let malformedLines = 0;
|
|
2486
|
+
let lineNumber = 0;
|
|
2487
|
+
for (const line of jsonl.split("\n")) {
|
|
2488
|
+
lineNumber += 1;
|
|
2489
|
+
const read = readCodeAgentJsonlLine(line, lineNumber);
|
|
2490
|
+
if (!read) continue;
|
|
2491
|
+
if (read.kind === "malformed") malformedLines += 1;
|
|
2492
|
+
else entries.push(read.entry);
|
|
2493
|
+
}
|
|
2494
|
+
return {
|
|
2495
|
+
entries,
|
|
2496
|
+
malformedLines
|
|
2497
|
+
};
|
|
2498
|
+
}
|
|
2499
|
+
/** Reads a transcript one line at a time and never holds the file as a single
|
|
2500
|
+
* string. `parseCodeAgentJsonl` needs the whole file in one string, so a
|
|
2501
|
+
* session above V8's ~512MB string ceiling throws `ERR_STRING_TOO_LONG` and
|
|
2502
|
+
* cannot be ingested at all; the largest real Codex rollout on record is 695MB.
|
|
2503
|
+
*
|
|
2504
|
+
* Lines break on `\n` only, which is what the string path's `split('\n')` does.
|
|
2505
|
+
* `node:readline` also breaks on a bare `\r`, so it is deliberately not used
|
|
2506
|
+
* here: a lone carriage return inside a line must stay inside that line for the
|
|
2507
|
+
* two paths to report the same malformed count. */
|
|
2508
|
+
async function* streamCodeAgentJsonlFile(path) {
|
|
2509
|
+
const stream = createReadStream(path, { encoding: "utf8" });
|
|
2510
|
+
let pending = "";
|
|
2511
|
+
let lineNumber = 0;
|
|
2512
|
+
try {
|
|
2513
|
+
for await (const chunk of stream) {
|
|
2514
|
+
pending += chunk;
|
|
2515
|
+
let start = 0;
|
|
2516
|
+
for (let at = pending.indexOf("\n"); at !== -1; at = pending.indexOf("\n", start)) {
|
|
2517
|
+
lineNumber += 1;
|
|
2518
|
+
const read = readCodeAgentJsonlLine(pending.slice(start, at), lineNumber);
|
|
2519
|
+
if (read) yield read;
|
|
2520
|
+
start = at + 1;
|
|
2521
|
+
}
|
|
2522
|
+
pending = pending.slice(start);
|
|
2523
|
+
}
|
|
2524
|
+
} finally {
|
|
2525
|
+
stream.destroy();
|
|
2725
2526
|
}
|
|
2726
|
-
|
|
2727
|
-
|
|
2527
|
+
const last = readCodeAgentJsonlLine(pending, lineNumber + 1);
|
|
2528
|
+
if (last) yield last;
|
|
2728
2529
|
}
|
|
2729
|
-
|
|
2530
|
+
/** Streaming counterpart to `parseCodeAgentJsonl` for a transcript on disk.
|
|
2531
|
+
* It returns the same shape, so a caller that holds every entry keeps working
|
|
2532
|
+
* above the string ceiling. The entry array still grows with the transcript;
|
|
2533
|
+
* consume `streamCodeAgentJsonlFile` directly when memory must stay flat. */
|
|
2534
|
+
async function parseCodeAgentJsonlFile(path) {
|
|
2535
|
+
const entries = [];
|
|
2536
|
+
let malformedLines = 0;
|
|
2537
|
+
for await (const read of streamCodeAgentJsonlFile(path)) if (read.kind === "malformed") malformedLines += 1;
|
|
2538
|
+
else entries.push(read.entry);
|
|
2730
2539
|
return {
|
|
2731
2540
|
entries,
|
|
2732
|
-
|
|
2733
|
-
assistantMessages: 0,
|
|
2734
|
-
reasoningItems: 0,
|
|
2735
|
-
toolCalls: 0,
|
|
2736
|
-
toolOutputs: 0,
|
|
2737
|
-
toolErrors: 0,
|
|
2738
|
-
unclassifiedErrors: 0,
|
|
2739
|
-
patchAttempts: 0,
|
|
2740
|
-
patchSuccesses: 0,
|
|
2741
|
-
patchFailures: 0,
|
|
2742
|
-
turnsStarted: 0,
|
|
2743
|
-
turnsCompleted: 0,
|
|
2744
|
-
turnsAborted: 0,
|
|
2745
|
-
contextCompactions: 0,
|
|
2746
|
-
mcpCalls: 0,
|
|
2747
|
-
subagentCalls: 0,
|
|
2748
|
-
skillCalls: 0,
|
|
2749
|
-
hookCalls: 0,
|
|
2750
|
-
webCalls: 0,
|
|
2751
|
-
codeActions: 0,
|
|
2752
|
-
prLinks: 0,
|
|
2753
|
-
fileSnapshots: 0,
|
|
2754
|
-
graphNodes: 0,
|
|
2755
|
-
graphEdges: 0,
|
|
2756
|
-
actionCandidates: 0,
|
|
2757
|
-
verificationReports: 0,
|
|
2758
|
-
completionDecisions: 0,
|
|
2759
|
-
reliabilityRows: 0,
|
|
2760
|
-
reliabilityLift: 0,
|
|
2761
|
-
inputTokens: 0,
|
|
2762
|
-
outputTokens: 0,
|
|
2763
|
-
reasoningTokens: 0,
|
|
2764
|
-
cachedTokens: 0,
|
|
2765
|
-
cacheWriteTokens: 0,
|
|
2766
|
-
observedCostUsd: 0,
|
|
2767
|
-
observedCostCaptured: false,
|
|
2768
|
-
wallMs: 0,
|
|
2769
|
-
processScore: 0
|
|
2541
|
+
malformedLines
|
|
2770
2542
|
};
|
|
2771
2543
|
}
|
|
2772
|
-
|
|
2773
|
-
|
|
2774
|
-
|
|
2544
|
+
//#endregion
|
|
2545
|
+
//#region src/contract/intake/code-agent-session.ts
|
|
2546
|
+
function fromCodexSession(options) {
|
|
2547
|
+
return fromCodeAgentSession("codex", options);
|
|
2775
2548
|
}
|
|
2776
|
-
function
|
|
2777
|
-
return
|
|
2549
|
+
function fromClaudeCodeSession(options) {
|
|
2550
|
+
return fromCodeAgentSession("claude-code", options);
|
|
2778
2551
|
}
|
|
2779
|
-
function
|
|
2780
|
-
|
|
2781
|
-
return penalizeErrors(terminalTurns > 0 ? metrics.turnsCompleted / terminalTurns : metrics.assistantMessages > 0 ? .75 : metrics.toolCalls > 0 ? .5 : 0, metrics);
|
|
2552
|
+
function fromOpenCodeSession(options) {
|
|
2553
|
+
return fromCodeAgentSession("opencode", options);
|
|
2782
2554
|
}
|
|
2783
|
-
function
|
|
2784
|
-
|
|
2785
|
-
return clamp01(base * (1 - .5 * Math.min(1, (metrics.toolErrors + metrics.patchFailures) / operations)));
|
|
2555
|
+
function fromKimiCodeSession(options) {
|
|
2556
|
+
return fromCodeAgentSession("kimi-code", options);
|
|
2786
2557
|
}
|
|
2787
|
-
|
|
2558
|
+
/** A native Pi session (`~/.pi/agent/sessions/<cwd>/<ts>_<id>.jsonl`), parsed entries in order. */
|
|
2559
|
+
function fromPiSession(options) {
|
|
2560
|
+
return fromCodeAgentSession("pi", options);
|
|
2561
|
+
}
|
|
2562
|
+
function fromCodeAgentSession(source, options) {
|
|
2563
|
+
const entries = options.entries.filter(isRecord);
|
|
2564
|
+
if (entries.length === 0) return {
|
|
2565
|
+
runs: [],
|
|
2566
|
+
diagnostics: [{
|
|
2567
|
+
source,
|
|
2568
|
+
sessionId: fallbackSessionId$1(source, options.sourcePath),
|
|
2569
|
+
sourcePath: options.sourcePath,
|
|
2570
|
+
entries: 0,
|
|
2571
|
+
malformedLines: options.malformedLines ?? 0,
|
|
2572
|
+
hasExplicitTerminalSignal: false,
|
|
2573
|
+
hasFinalOutput: false,
|
|
2574
|
+
hasQualityLabel: false,
|
|
2575
|
+
hasTokenUsage: false,
|
|
2576
|
+
hasCost: false,
|
|
2577
|
+
costKind: "uncaptured",
|
|
2578
|
+
warnings: ["no parseable session entries"]
|
|
2579
|
+
}],
|
|
2580
|
+
metrics: [],
|
|
2581
|
+
observations: []
|
|
2582
|
+
};
|
|
2583
|
+
const metrics = metricsFor(source, entries);
|
|
2584
|
+
const sessionId = sessionIdFromEntries(source, entries) ?? fallbackSessionId$1(source, options.sourcePath);
|
|
2585
|
+
const observation = observeCodeAgentSession({
|
|
2586
|
+
source,
|
|
2587
|
+
entries,
|
|
2588
|
+
sourcePath: options.sourcePath,
|
|
2589
|
+
execution: options.execution
|
|
2590
|
+
});
|
|
2591
|
+
applyObservedSurfaceMetrics(metrics, observation);
|
|
2592
|
+
const model = withSnapshot(options.model ?? modelFromEntries(source, entries) ?? source);
|
|
2593
|
+
const tokenUsage = {
|
|
2594
|
+
input: metrics.inputTokens,
|
|
2595
|
+
output: metrics.outputTokens,
|
|
2596
|
+
...metrics.reasoningTokens > 0 ? { reasoning: metrics.reasoningTokens } : {},
|
|
2597
|
+
...metrics.cachedTokens > 0 ? { cached: metrics.cachedTokens } : {},
|
|
2598
|
+
...metrics.cacheWriteTokens > 0 ? { cacheWrite: metrics.cacheWriteTokens } : {}
|
|
2599
|
+
};
|
|
2600
|
+
const { costUsd, costProvenance } = resolveSessionCost(options.costProvenance, metrics, model);
|
|
2601
|
+
const qualityScore = finiteScore(options.score);
|
|
2602
|
+
const invalidQualityScore = options.score !== void 0 && qualityScore === void 0;
|
|
2603
|
+
const promptHash = options.promptHash ?? hashString(`prompt:${source}:${firstUserText(entries) ?? ""}`);
|
|
2604
|
+
const configHash = options.configHash ?? hashCanonical({
|
|
2605
|
+
source,
|
|
2606
|
+
model,
|
|
2607
|
+
sourcePath: options.sourcePath ?? null,
|
|
2608
|
+
cwd: cwdFromEntries(entries) ?? null,
|
|
2609
|
+
entryCount: entries.length
|
|
2610
|
+
});
|
|
2611
|
+
const explicitTerminal = observation.terminal.explicit;
|
|
2612
|
+
const warnings = diagnosticsFor(metrics, {
|
|
2613
|
+
model,
|
|
2614
|
+
explicitTerminal,
|
|
2615
|
+
malformedLines: options.malformedLines ?? 0,
|
|
2616
|
+
hasQualityLabel: qualityScore !== void 0,
|
|
2617
|
+
invalidQualityScore,
|
|
2618
|
+
costKind: costProvenance.kind
|
|
2619
|
+
});
|
|
2620
|
+
const terminalOutcome = terminalOutcomeFromSession(observation.terminal);
|
|
2621
|
+
const processExitCode = options.execution?.exitCode;
|
|
2622
|
+
const splitTag = options.splitTag ?? "holdout";
|
|
2788
2623
|
return {
|
|
2789
|
-
|
|
2790
|
-
|
|
2791
|
-
|
|
2792
|
-
|
|
2793
|
-
|
|
2624
|
+
runs: [{
|
|
2625
|
+
runId: `${source}:${sessionId}`,
|
|
2626
|
+
experimentId: options.experimentId ?? `${source}-local-sessions`,
|
|
2627
|
+
candidateId: options.candidateId ?? model,
|
|
2628
|
+
seed: options.seed ?? stableSeed(sessionId),
|
|
2629
|
+
model,
|
|
2630
|
+
promptHash,
|
|
2631
|
+
configHash,
|
|
2632
|
+
commitSha: options.commitSha ?? "local-session",
|
|
2633
|
+
wallMs: metrics.wallMs,
|
|
2634
|
+
costUsd,
|
|
2635
|
+
costProvenance,
|
|
2636
|
+
tokenUsage,
|
|
2637
|
+
terminalOutcome,
|
|
2638
|
+
...terminalOutcome === "failed" ? { terminalFailureReason: processExitCode !== void 0 && processExitCode !== 0 ? `process exited with code ${processExitCode}` : metrics.turnsAborted > 0 ? "turn aborted" : "session reported terminal failure" } : {},
|
|
2639
|
+
outcome: {
|
|
2640
|
+
...qualityScore === void 0 ? {} : splitTag === "holdout" ? { holdoutScore: qualityScore } : { searchScore: qualityScore },
|
|
2641
|
+
raw: {
|
|
2642
|
+
entries: metrics.entries,
|
|
2643
|
+
user_messages: metrics.userMessages,
|
|
2644
|
+
assistant_messages: metrics.assistantMessages,
|
|
2645
|
+
reasoning_items: metrics.reasoningItems,
|
|
2646
|
+
tool_calls: metrics.toolCalls,
|
|
2647
|
+
tool_outputs: metrics.toolOutputs,
|
|
2648
|
+
tool_errors: metrics.toolErrors,
|
|
2649
|
+
execution_error_count: metrics.toolErrors,
|
|
2650
|
+
process_error_count: terminalOutcome === "failed" ? 1 : 0,
|
|
2651
|
+
unclassified_error_count: metrics.unclassifiedErrors,
|
|
2652
|
+
...processExitCode !== void 0 ? { process_exit_code: processExitCode } : {},
|
|
2653
|
+
patch_attempts: metrics.patchAttempts,
|
|
2654
|
+
patch_successes: metrics.patchSuccesses,
|
|
2655
|
+
patch_failures: metrics.patchFailures,
|
|
2656
|
+
turns_started: metrics.turnsStarted,
|
|
2657
|
+
turns_completed: metrics.turnsCompleted,
|
|
2658
|
+
turns_aborted: metrics.turnsAborted,
|
|
2659
|
+
context_compactions: metrics.contextCompactions,
|
|
2660
|
+
mcp_calls: metrics.mcpCalls,
|
|
2661
|
+
subagent_calls: metrics.subagentCalls,
|
|
2662
|
+
skill_calls: metrics.skillCalls,
|
|
2663
|
+
hook_calls: metrics.hookCalls,
|
|
2664
|
+
web_calls: metrics.webCalls,
|
|
2665
|
+
code_actions: metrics.codeActions,
|
|
2666
|
+
pr_links: metrics.prLinks,
|
|
2667
|
+
file_snapshots: metrics.fileSnapshots,
|
|
2668
|
+
graph_nodes: metrics.graphNodes,
|
|
2669
|
+
graph_edges: metrics.graphEdges,
|
|
2670
|
+
action_candidates: metrics.actionCandidates,
|
|
2671
|
+
verification_reports: metrics.verificationReports,
|
|
2672
|
+
completion_decisions: metrics.completionDecisions,
|
|
2673
|
+
reliability_rows: metrics.reliabilityRows,
|
|
2674
|
+
reliability_lift: metrics.reliabilityLift,
|
|
2675
|
+
input_tokens: metrics.inputTokens,
|
|
2676
|
+
output_tokens: metrics.outputTokens,
|
|
2677
|
+
reasoning_tokens: metrics.reasoningTokens,
|
|
2678
|
+
cached_tokens: metrics.cachedTokens,
|
|
2679
|
+
cache_write_tokens: metrics.cacheWriteTokens,
|
|
2680
|
+
observed_cost_usd: metrics.observedCostUsd,
|
|
2681
|
+
observed_cost_captured: metrics.observedCostCaptured ? 1 : 0,
|
|
2682
|
+
estimated_cost_usd: costProvenance.kind === "estimated" ? costProvenance.usd : 0,
|
|
2683
|
+
process_score: metrics.processScore,
|
|
2684
|
+
explicit_terminal_signal: explicitTerminal ? 1 : 0,
|
|
2685
|
+
quality_label_present: qualityScore !== void 0 ? 1 : 0,
|
|
2686
|
+
cost_observed: costProvenance.kind === "observed" ? 1 : 0,
|
|
2687
|
+
cost_estimated: costProvenance.kind === "estimated" ? 1 : 0,
|
|
2688
|
+
cost_uncaptured: costProvenance.kind === "uncaptured" ? 1 : 0,
|
|
2689
|
+
cost_unknown: costProvenance.usd === null ? 1 : 0
|
|
2690
|
+
}
|
|
2691
|
+
},
|
|
2692
|
+
splitTag,
|
|
2693
|
+
scenarioId: options.scenarioId ?? sessionId
|
|
2694
|
+
}],
|
|
2695
|
+
diagnostics: [{
|
|
2696
|
+
source,
|
|
2697
|
+
sessionId,
|
|
2698
|
+
sourcePath: options.sourcePath,
|
|
2699
|
+
entries: entries.length,
|
|
2700
|
+
malformedLines: options.malformedLines ?? 0,
|
|
2701
|
+
hasExplicitTerminalSignal: explicitTerminal,
|
|
2702
|
+
hasFinalOutput: observation.finalText !== null,
|
|
2703
|
+
hasQualityLabel: qualityScore !== void 0,
|
|
2704
|
+
hasTokenUsage: metrics.inputTokens > 0 || metrics.outputTokens > 0 || metrics.reasoningTokens > 0 || metrics.cachedTokens > 0 || metrics.cacheWriteTokens > 0,
|
|
2705
|
+
hasCost: costProvenance.usd !== null,
|
|
2706
|
+
costKind: costProvenance.kind,
|
|
2707
|
+
warnings
|
|
2708
|
+
}],
|
|
2709
|
+
metrics: [metrics],
|
|
2710
|
+
observations: [{
|
|
2711
|
+
...observation,
|
|
2712
|
+
sessionId
|
|
2713
|
+
}]
|
|
2794
2714
|
};
|
|
2795
2715
|
}
|
|
2796
|
-
function
|
|
2797
|
-
|
|
2798
|
-
|
|
2799
|
-
|
|
2800
|
-
|
|
2801
|
-
metrics.inputTokens += parsed.input;
|
|
2802
|
-
metrics.outputTokens += parsed.output;
|
|
2803
|
-
metrics.reasoningTokens += parsed.reasoning;
|
|
2804
|
-
metrics.cachedTokens += parsed.cached;
|
|
2805
|
-
metrics.cacheWriteTokens += parsed.cacheWrite;
|
|
2806
|
-
}
|
|
2807
|
-
function addUsageTo(totals, usage) {
|
|
2808
|
-
const parsed = readUsage(usage);
|
|
2809
|
-
totals.input += parsed.input;
|
|
2810
|
-
totals.output += parsed.output;
|
|
2811
|
-
totals.reasoning += parsed.reasoning;
|
|
2812
|
-
totals.cached += parsed.cached;
|
|
2813
|
-
totals.cacheWrite += parsed.cacheWrite;
|
|
2814
|
-
}
|
|
2815
|
-
function readUsage(usage) {
|
|
2816
|
-
const parsed = extractUsage(usage);
|
|
2817
|
-
return {
|
|
2818
|
-
input: parsed?.input ?? 0,
|
|
2819
|
-
output: parsed?.output ?? 0,
|
|
2820
|
-
reasoning: parsed?.reasoning ?? 0,
|
|
2821
|
-
cached: parsed?.cached ?? 0,
|
|
2822
|
-
cacheWrite: parsed?.cacheWrite ?? 0
|
|
2823
|
-
};
|
|
2716
|
+
function terminalOutcomeFromSession(terminal) {
|
|
2717
|
+
if (!terminal.explicit) return "unknown";
|
|
2718
|
+
if (terminal.status === "completed") return "succeeded";
|
|
2719
|
+
if (terminal.status === "failed") return "failed";
|
|
2720
|
+
return "unknown";
|
|
2824
2721
|
}
|
|
2825
|
-
function
|
|
2826
|
-
|
|
2827
|
-
|
|
2828
|
-
|
|
2829
|
-
|
|
2830
|
-
|
|
2831
|
-
|
|
2832
|
-
|
|
2833
|
-
}
|
|
2834
|
-
if (explicit.kind === "lower-bound") {
|
|
2835
|
-
if (explicit.usd !== null) throw new Error("lower-bound cost must have usd: null");
|
|
2836
|
-
if (!Number.isFinite(explicit.knownLowerBoundUsd) || explicit.knownLowerBoundUsd <= 0) throw new Error("lower-bound cost must have a finite, positive knownLowerBoundUsd");
|
|
2837
|
-
return {
|
|
2838
|
-
costUsd: null,
|
|
2839
|
-
costProvenance: explicit
|
|
2840
|
-
};
|
|
2841
|
-
}
|
|
2842
|
-
if (!Number.isFinite(explicit.usd) || explicit.usd < 0) throw new Error(`${explicit.kind} cost must be a finite, non-negative USD amount`);
|
|
2843
|
-
return {
|
|
2844
|
-
costUsd: explicit.usd,
|
|
2845
|
-
costProvenance: explicit
|
|
2846
|
-
};
|
|
2847
|
-
}
|
|
2848
|
-
if (metrics.observedCostCaptured) {
|
|
2849
|
-
const costProvenance = {
|
|
2850
|
-
kind: "observed",
|
|
2851
|
-
usd: metrics.observedCostUsd
|
|
2852
|
-
};
|
|
2853
|
-
return {
|
|
2854
|
-
costUsd: metrics.observedCostUsd,
|
|
2855
|
-
costProvenance
|
|
2856
|
-
};
|
|
2857
|
-
}
|
|
2858
|
-
if ((metrics.inputTokens > 0 || metrics.outputTokens > 0) && isModelPriced(model)) {
|
|
2859
|
-
const costUsd = estimateCost(metrics.inputTokens, metrics.outputTokens, model);
|
|
2860
|
-
return {
|
|
2861
|
-
costUsd,
|
|
2862
|
-
costProvenance: {
|
|
2863
|
-
kind: "estimated",
|
|
2864
|
-
usd: costUsd
|
|
2865
|
-
}
|
|
2866
|
-
};
|
|
2722
|
+
function metricsFor(source, entries) {
|
|
2723
|
+
switch (source) {
|
|
2724
|
+
case "codex": return codexMetrics(entries);
|
|
2725
|
+
case "claude-code": return claudeCodeMetrics(entries);
|
|
2726
|
+
case "opencode": return openCodeMetrics(entries);
|
|
2727
|
+
case "kimi-code": return kimiCodeMetrics(entries);
|
|
2728
|
+
case "pi": return piSessionMetrics(entries);
|
|
2729
|
+
case "pi-graph": return piGraphMetrics(entries);
|
|
2867
2730
|
}
|
|
2868
|
-
return {
|
|
2869
|
-
costUsd: null,
|
|
2870
|
-
costProvenance: {
|
|
2871
|
-
kind: "uncaptured",
|
|
2872
|
-
usd: null
|
|
2873
|
-
}
|
|
2874
|
-
};
|
|
2875
2731
|
}
|
|
2876
2732
|
function diagnosticsFor(metrics, options) {
|
|
2877
2733
|
const warnings = [];
|
|
@@ -2895,58 +2751,6 @@ function applyObservedSurfaceMetrics(metrics, observation) {
|
|
|
2895
2751
|
metrics.webCalls = actions.filter((action) => action.surface === "web").length;
|
|
2896
2752
|
metrics.codeActions = actions.filter((action) => action.surface === "code").length;
|
|
2897
2753
|
}
|
|
2898
|
-
function sessionIdFromEntries(source, entries) {
|
|
2899
|
-
for (const entry of entries) {
|
|
2900
|
-
if (source === "pi" && entry.type === "session") {
|
|
2901
|
-
const id = stringField(entry, "id");
|
|
2902
|
-
if (id) return id;
|
|
2903
|
-
}
|
|
2904
|
-
if (source === "codex") {
|
|
2905
|
-
const threadId = stringField(entry, "thread_id");
|
|
2906
|
-
if (threadId) return threadId;
|
|
2907
|
-
const payload = record(entry.payload);
|
|
2908
|
-
const id = payload ? stringField(payload, "id") : void 0;
|
|
2909
|
-
if (id) return id;
|
|
2910
|
-
}
|
|
2911
|
-
const payload = record(record(entry.message)?.payload);
|
|
2912
|
-
const sessionId = stringField(entry, "sessionID") ?? stringField(entry, "sessionId") ?? stringField(entry, "session_id") ?? stringField(payload ?? {}, "session_id");
|
|
2913
|
-
if (sessionId) return sessionId;
|
|
2914
|
-
}
|
|
2915
|
-
}
|
|
2916
|
-
function modelFromEntries(source, entries) {
|
|
2917
|
-
if (source === "pi") return summarizeSessionInput("pi", { records: entries }).lastServedModel ?? void 0;
|
|
2918
|
-
let providerModel;
|
|
2919
|
-
for (const entry of entries) {
|
|
2920
|
-
if (source === "codex") {
|
|
2921
|
-
const payload = record(entry.payload);
|
|
2922
|
-
const model = payload ? stringField(payload, "model") : void 0;
|
|
2923
|
-
if (model) return model;
|
|
2924
|
-
providerModel = providerModel ?? (payload ? stringField(payload, "model_provider") : void 0);
|
|
2925
|
-
}
|
|
2926
|
-
const openCodeModel = stringField(entry, "modelID");
|
|
2927
|
-
if (openCodeModel) {
|
|
2928
|
-
const provider = stringField(entry, "providerID");
|
|
2929
|
-
return provider ? `${provider}/${openCodeModel}` : openCodeModel;
|
|
2930
|
-
}
|
|
2931
|
-
const modelObject = record(entry.model);
|
|
2932
|
-
const nestedModel = stringField(modelObject ?? {}, "modelID");
|
|
2933
|
-
if (nestedModel) {
|
|
2934
|
-
const provider = stringField(modelObject ?? {}, "providerID");
|
|
2935
|
-
return provider ? `${provider}/${nestedModel}` : nestedModel;
|
|
2936
|
-
}
|
|
2937
|
-
const message = record(entry.message);
|
|
2938
|
-
const model = message ? stringField(message, "model") : void 0;
|
|
2939
|
-
if (model) return model;
|
|
2940
|
-
}
|
|
2941
|
-
return providerModel;
|
|
2942
|
-
}
|
|
2943
|
-
function cwdFromEntries(entries) {
|
|
2944
|
-
for (const entry of entries) {
|
|
2945
|
-
const path = record(entry.path);
|
|
2946
|
-
const cwd = stringField(entry, "cwd") ?? stringField(path ?? {}, "cwd") ?? stringField(path ?? {}, "root") ?? (record(entry.payload) ? stringField(record(entry.payload), "cwd") : void 0);
|
|
2947
|
-
if (cwd) return cwd;
|
|
2948
|
-
}
|
|
2949
|
-
}
|
|
2950
2754
|
/** Join the `text` of every part in a `[{ type: 'text', text }]` content list. */
|
|
2951
2755
|
function textFromContentParts(content) {
|
|
2952
2756
|
if (!Array.isArray(content)) return void 0;
|
|
@@ -2981,245 +2785,9 @@ function firstUserText(entries) {
|
|
|
2981
2785
|
}
|
|
2982
2786
|
}
|
|
2983
2787
|
}
|
|
2984
|
-
function fallbackSessionId(source, sourcePath) {
|
|
2985
|
-
return hashString(`${source}:${sourcePath ?? "unknown-session"}`).slice(7, 27);
|
|
2986
|
-
}
|
|
2987
|
-
function withSnapshot(model) {
|
|
2988
|
-
const trimmed = model.trim() || "unknown";
|
|
2989
|
-
if (trimmed.includes("@") || /-\d{8}$/.test(trimmed) || /-\d{4}-\d{2}-\d{2}$/.test(trimmed)) return trimmed;
|
|
2990
|
-
return `${trimmed}@observed-local`;
|
|
2991
|
-
}
|
|
2992
|
-
function hashJson(value) {
|
|
2993
|
-
return hashCanonical(value);
|
|
2994
|
-
}
|
|
2995
|
-
function hashString(value) {
|
|
2996
|
-
return `sha256:${createHash("sha256").update(value).digest("hex")}`;
|
|
2997
|
-
}
|
|
2998
|
-
function stableSeed(value) {
|
|
2999
|
-
return createHash("sha256").update(value).digest().readUInt32BE(0);
|
|
3000
|
-
}
|
|
3001
|
-
function maxOptional(current, next) {
|
|
3002
|
-
return current === void 0 ? next : Math.max(current, next);
|
|
3003
|
-
}
|
|
3004
|
-
function clamp01(value) {
|
|
3005
|
-
if (!Number.isFinite(value)) return 0;
|
|
3006
|
-
return Math.max(0, Math.min(1, value));
|
|
3007
|
-
}
|
|
3008
2788
|
function finiteScore(value) {
|
|
3009
2789
|
return value !== void 0 && Number.isFinite(value) ? clamp01(value) : void 0;
|
|
3010
2790
|
}
|
|
3011
|
-
function isRecord(value) {
|
|
3012
|
-
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
3013
|
-
}
|
|
3014
|
-
function record(value) {
|
|
3015
|
-
return isRecord(value) ? value : null;
|
|
3016
|
-
}
|
|
3017
|
-
function stringField(obj, key) {
|
|
3018
|
-
const value = obj[key];
|
|
3019
|
-
return typeof value === "string" && value.length > 0 ? value : void 0;
|
|
3020
|
-
}
|
|
3021
|
-
function numberField(obj, key) {
|
|
3022
|
-
const value = obj[key];
|
|
3023
|
-
return typeof value === "number" && Number.isFinite(value) ? value : void 0;
|
|
3024
|
-
}
|
|
3025
|
-
function timestampMs(value) {
|
|
3026
|
-
if (typeof value === "number" && Number.isFinite(value)) return value > 0xe8d4a51000 ? value : value * 1e3;
|
|
3027
|
-
if (typeof value === "string" && value.length > 0) {
|
|
3028
|
-
const parsed = Date.parse(value);
|
|
3029
|
-
return Number.isFinite(parsed) ? parsed : void 0;
|
|
3030
|
-
}
|
|
3031
|
-
}
|
|
3032
|
-
//#endregion
|
|
3033
|
-
//#region src/contract/intake/code-agent-store.ts
|
|
3034
|
-
const SESSION_SUFFIX = ".jsonl";
|
|
3035
|
-
const REJECTION_SAMPLE_LIMIT = 8;
|
|
3036
|
-
/**
|
|
3037
|
-
* Read every session this run captured under `scope.root` and project them.
|
|
3038
|
-
*
|
|
3039
|
-
* A store directory that does not exist, or that holds no session the scope attributes, returns
|
|
3040
|
-
* `status: 'unavailable'`. That is not an error: a run whose harness store was never synced back
|
|
3041
|
-
* has no evidence, and saying so is the correct answer.
|
|
3042
|
-
*/
|
|
3043
|
-
async function observeCodeAgentStore(scope) {
|
|
3044
|
-
if (!isAbsolute(scope.root)) throw new TypeError(`observeCodeAgentStore: scope.root must be an absolute path to this run's own store, received ${JSON.stringify(scope.root)}`);
|
|
3045
|
-
const workspaceRoot = scope.workspaceRoot === void 0 ? void 0 : resolve(scope.workspaceRoot);
|
|
3046
|
-
const paths = await listSessionFiles(resolve(scope.root));
|
|
3047
|
-
const sessions = [];
|
|
3048
|
-
const byReason = /* @__PURE__ */ new Map();
|
|
3049
|
-
const sample = [];
|
|
3050
|
-
const reject = (path, reason) => {
|
|
3051
|
-
byReason.set(reason, (byReason.get(reason) ?? 0) + 1);
|
|
3052
|
-
if (sample.length < REJECTION_SAMPLE_LIMIT) sample.push({
|
|
3053
|
-
path,
|
|
3054
|
-
reason
|
|
3055
|
-
});
|
|
3056
|
-
};
|
|
3057
|
-
for (const path of paths) {
|
|
3058
|
-
let entries;
|
|
3059
|
-
try {
|
|
3060
|
-
entries = (await parseCodeAgentJsonlFile(path)).entries;
|
|
3061
|
-
} catch {
|
|
3062
|
-
reject(path, "unreadable");
|
|
3063
|
-
continue;
|
|
3064
|
-
}
|
|
3065
|
-
if (entries.length === 0) {
|
|
3066
|
-
reject(path, "no-entries");
|
|
3067
|
-
continue;
|
|
3068
|
-
}
|
|
3069
|
-
const workspace = sessionWorkspace(entries);
|
|
3070
|
-
if (workspaceRoot !== void 0) {
|
|
3071
|
-
if (workspace === void 0) {
|
|
3072
|
-
reject(path, "no-workspace-recorded");
|
|
3073
|
-
continue;
|
|
3074
|
-
}
|
|
3075
|
-
if (!isSameOrBelow(workspaceRoot, workspace)) {
|
|
3076
|
-
reject(path, "outside-workspace");
|
|
3077
|
-
continue;
|
|
3078
|
-
}
|
|
3079
|
-
}
|
|
3080
|
-
const startedAtMs = sessionStartedAtMs(entries);
|
|
3081
|
-
if (scope.window !== void 0) {
|
|
3082
|
-
if (startedAtMs === void 0) {
|
|
3083
|
-
reject(path, "no-start-recorded");
|
|
3084
|
-
continue;
|
|
3085
|
-
}
|
|
3086
|
-
if (startedAtMs < scope.window.startedAtMs) {
|
|
3087
|
-
reject(path, "before-run");
|
|
3088
|
-
continue;
|
|
3089
|
-
}
|
|
3090
|
-
if (scope.window.finishedAtMs !== void 0 && startedAtMs > scope.window.finishedAtMs) {
|
|
3091
|
-
reject(path, "after-run");
|
|
3092
|
-
continue;
|
|
3093
|
-
}
|
|
3094
|
-
}
|
|
3095
|
-
const observation = observeCodeAgentSession({
|
|
3096
|
-
source: scope.source,
|
|
3097
|
-
entries,
|
|
3098
|
-
sourcePath: path
|
|
3099
|
-
});
|
|
3100
|
-
sessions.push({
|
|
3101
|
-
path,
|
|
3102
|
-
sessionId: observation.sessionId,
|
|
3103
|
-
...workspace === void 0 ? {} : { workspace },
|
|
3104
|
-
...startedAtMs === void 0 ? {} : { startedAtMs },
|
|
3105
|
-
observation
|
|
3106
|
-
});
|
|
3107
|
-
}
|
|
3108
|
-
const rejections = {
|
|
3109
|
-
total: [...byReason.values()].reduce((sum, count) => sum + count, 0),
|
|
3110
|
-
byReason: Object.fromEntries(byReason),
|
|
3111
|
-
sample
|
|
3112
|
-
};
|
|
3113
|
-
if (sessions.length === 0) return {
|
|
3114
|
-
status: "unavailable",
|
|
3115
|
-
scope,
|
|
3116
|
-
filesScanned: paths.length,
|
|
3117
|
-
reason: unavailableReason(paths.length, rejections),
|
|
3118
|
-
rejections
|
|
3119
|
-
};
|
|
3120
|
-
const bySurface = /* @__PURE__ */ new Map();
|
|
3121
|
-
let actions = 0;
|
|
3122
|
-
let subagentActions = 0;
|
|
3123
|
-
let sessionsWithSubagentActions = 0;
|
|
3124
|
-
for (const session of sessions) {
|
|
3125
|
-
let sessionSubagents = 0;
|
|
3126
|
-
for (const action of session.observation.actions) {
|
|
3127
|
-
actions += 1;
|
|
3128
|
-
bySurface.set(action.surface, (bySurface.get(action.surface) ?? 0) + 1);
|
|
3129
|
-
if (action.surface === "subagent") sessionSubagents += 1;
|
|
3130
|
-
}
|
|
3131
|
-
subagentActions += sessionSubagents;
|
|
3132
|
-
if (sessionSubagents > 0) sessionsWithSubagentActions += 1;
|
|
3133
|
-
}
|
|
3134
|
-
return {
|
|
3135
|
-
status: "observed",
|
|
3136
|
-
scope,
|
|
3137
|
-
filesScanned: paths.length,
|
|
3138
|
-
sessions,
|
|
3139
|
-
actions,
|
|
3140
|
-
subagentActions,
|
|
3141
|
-
sessionsWithSubagentActions,
|
|
3142
|
-
bySurface: Object.fromEntries(bySurface),
|
|
3143
|
-
rejections
|
|
3144
|
-
};
|
|
3145
|
-
}
|
|
3146
|
-
function unavailableReason(filesScanned, rejections) {
|
|
3147
|
-
if (filesScanned === 0) return "the run captured no session files in this store";
|
|
3148
|
-
return `no session in this store belongs to the run: ${filesScanned} file(s) scanned, all rejected (${Object.entries(rejections.byReason).map(([reason, count]) => `${count} ${reason}`).sort().join(", ")})`;
|
|
3149
|
-
}
|
|
3150
|
-
async function listSessionFiles(root) {
|
|
3151
|
-
const found = [];
|
|
3152
|
-
const walk = async (dir) => {
|
|
3153
|
-
let entries;
|
|
3154
|
-
try {
|
|
3155
|
-
entries = await readdir(dir, { withFileTypes: true });
|
|
3156
|
-
} catch {
|
|
3157
|
-
return;
|
|
3158
|
-
}
|
|
3159
|
-
for (const entry of entries) {
|
|
3160
|
-
const path = join(dir, entry.name);
|
|
3161
|
-
if (entry.isDirectory()) await walk(path);
|
|
3162
|
-
else if (entry.isFile() && entry.name.endsWith(SESSION_SUFFIX)) found.push(path);
|
|
3163
|
-
}
|
|
3164
|
-
};
|
|
3165
|
-
await walk(root);
|
|
3166
|
-
if (found.length === 0 && await isSessionFile(root)) found.push(root);
|
|
3167
|
-
return found.sort();
|
|
3168
|
-
}
|
|
3169
|
-
async function isSessionFile(path) {
|
|
3170
|
-
if (!path.endsWith(SESSION_SUFFIX)) return false;
|
|
3171
|
-
try {
|
|
3172
|
-
return (await stat(path)).isFile();
|
|
3173
|
-
} catch {
|
|
3174
|
-
return false;
|
|
3175
|
-
}
|
|
3176
|
-
}
|
|
3177
|
-
/**
|
|
3178
|
-
* The working directory the session RECORDED, from the first entry that states one.
|
|
3179
|
-
*
|
|
3180
|
-
* codex writes it on `session_meta.payload.cwd`; claude-code writes `cwd` on every line. Neither is
|
|
3181
|
-
* inferred from the file's own location: where a captured file was copied to says nothing about
|
|
3182
|
-
* where the agent ran.
|
|
3183
|
-
*/
|
|
3184
|
-
function sessionWorkspace(entries) {
|
|
3185
|
-
for (const entry of entries) {
|
|
3186
|
-
const record = plainRecord(entry);
|
|
3187
|
-
if (record === void 0) continue;
|
|
3188
|
-
const direct = stringOf(record.cwd);
|
|
3189
|
-
if (direct !== void 0) return direct;
|
|
3190
|
-
const nested = stringOf(plainRecord(record.payload)?.cwd);
|
|
3191
|
-
if (nested !== void 0) return nested;
|
|
3192
|
-
}
|
|
3193
|
-
}
|
|
3194
|
-
/** When the session itself says it started, from the first entry carrying a parseable timestamp. */
|
|
3195
|
-
function sessionStartedAtMs(entries) {
|
|
3196
|
-
for (const entry of entries) {
|
|
3197
|
-
const record = plainRecord(entry);
|
|
3198
|
-
if (record === void 0) continue;
|
|
3199
|
-
const payload = plainRecord(record.payload);
|
|
3200
|
-
for (const candidate of [payload?.timestamp, record.timestamp]) {
|
|
3201
|
-
const text = stringOf(candidate);
|
|
3202
|
-
if (text === void 0) continue;
|
|
3203
|
-
const parsed = Date.parse(text);
|
|
3204
|
-
if (Number.isFinite(parsed)) return parsed;
|
|
3205
|
-
}
|
|
3206
|
-
const epochMs = numberOf(record.timestampMs) ?? numberOf(payload?.timestampMs);
|
|
3207
|
-
if (epochMs !== void 0) return epochMs;
|
|
3208
|
-
}
|
|
3209
|
-
}
|
|
3210
|
-
function isSameOrBelow(root, candidate) {
|
|
3211
|
-
const rel = relative(root, resolve(candidate));
|
|
3212
|
-
return rel === "" || rel !== ".." && !rel.startsWith(`..${sep}`) && !isAbsolute(rel);
|
|
3213
|
-
}
|
|
3214
|
-
function plainRecord(value) {
|
|
3215
|
-
return value !== null && typeof value === "object" && !Array.isArray(value) ? value : void 0;
|
|
3216
|
-
}
|
|
3217
|
-
function stringOf(value) {
|
|
3218
|
-
return typeof value === "string" && value.length > 0 ? value : void 0;
|
|
3219
|
-
}
|
|
3220
|
-
function numberOf(value) {
|
|
3221
|
-
return typeof value === "number" && Number.isFinite(value) ? value : void 0;
|
|
3222
|
-
}
|
|
3223
2791
|
//#endregion
|
|
3224
2792
|
//#region src/contract/intake/feedback-table.ts
|
|
3225
2793
|
function fromFeedbackTable(opts) {
|
|
@@ -3300,6 +2868,6 @@ function fromFeedbackTable(opts) {
|
|
|
3300
2868
|
};
|
|
3301
2869
|
}
|
|
3302
2870
|
//#endregion
|
|
3303
|
-
export { FileSystemOutcomeStore, InMemoryOutcomeStore,
|
|
2871
|
+
export { FileSystemOutcomeStore, InMemoryOutcomeStore, SelfImproveRunError, analyzeRuns, buildDefaultAnalystRegistry, buildEvidenceVector, campaignSplitDigest, compareOptimizationMethods, composeGate, createChatClient, defaultProductionGate, defineAgentEval, evaluatePairedMeasurements, externalTextOptimizationMethod, fromClaudeCodeSession, fromCodexSession, fromFeedbackTable, fromKimiCodeSession, fromOpenCodeSession, fromOtelSpans, fromPiSession, fsCampaignStorage, gepaOptimizationMethod, heldOutGate, inMemoryCampaignStorage, llmJudge, makeProposalFinding, measuredComparisonFromAgentProfileImprovementExperiment, measuredComparisonFromCandidateExperiment, observeCodeAgentSession, paretoPolicy, paretoSignificanceGate, parseAgentTrace, parseCodeAgentJsonl, parseCodeAgentJsonlFile, partitionRunsByAuthoringModel, runAgentProfileImprovementExperiment, runCampaign, runCandidateExperiment, runEval, runImprovementLoop, sealAgentProfileImprovementExperiment, sealAgentProfileImprovementSuite, sealAgentProfileImprovementTask, sealCandidateBenchmarkSuite, sealCandidateBenchmarkTask, sealCandidateExperiment, selfImprove, skillOptOptimizationMethod, streamCodeAgentJsonlFile, summarizeExecution, transientDispatchFailure, verifyAgentProfileImprovementExperimentComparison, verifyCandidateExperiment, verifyCandidateExperimentComparison };
|
|
3304
2872
|
|
|
3305
2873
|
//# sourceMappingURL=index.js.map
|