@tangle-network/agent-eval 0.216.0 → 0.216.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -0
- package/dist/adapters/http.d.ts +3 -3
- package/dist/{agent-profile-CIA2Atnt.d.ts → agent-profile-VFcQncbW.d.ts} +5 -5
- package/dist/{agent-profile-CIA2Atnt.d.ts.map → agent-profile-VFcQncbW.d.ts.map} +1 -1
- package/dist/{agent-profile-cell-BYc0bxsL.js → agent-profile-cell-BK8mlqlq.js} +3 -3
- package/dist/{agent-profile-cell-BYc0bxsL.js.map → agent-profile-cell-BK8mlqlq.js.map} +1 -1
- package/dist/{agreement-irr-DsTURxdr.js → agreement-irr-BEWLIF3_.js} +3 -3
- package/dist/{agreement-irr-DsTURxdr.js.map → agreement-irr-BEWLIF3_.js.map} +1 -1
- package/dist/analyst/index.d.ts +14 -14
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +15 -15
- package/dist/{analyze-runs-3xpby3p2.js → analyze-runs-C84irm4i.js} +608 -604
- package/dist/analyze-runs-C84irm4i.js.map +1 -0
- package/dist/{analyze-runs-CSDAh8A7.d.ts → analyze-runs-D68fd-su.d.ts} +17 -15
- package/dist/analyze-runs-D68fd-su.d.ts.map +1 -0
- package/dist/{baseline-Dm8966du.js → baseline-Udi_PLuT.js} +5 -5
- package/dist/{baseline-Dm8966du.js.map → baseline-Udi_PLuT.js.map} +1 -1
- package/dist/{benchmark-BywPeAUo.js → benchmark-CXmVVHH2.js} +2 -2
- package/dist/{benchmark-BywPeAUo.js.map → benchmark-CXmVVHH2.js.map} +1 -1
- package/dist/{benchmark-command-CByl9dxG.js → benchmark-command-By0vh8q7.js} +62 -45
- package/dist/benchmark-command-By0vh8q7.js.map +1 -0
- package/dist/benchmarks/index.d.ts +5 -5
- package/dist/benchmarks/index.js +3 -3
- package/dist/builder-eval/index.js +2 -2
- package/dist/campaign/index.d.ts +7 -7
- package/dist/campaign/index.js +13 -13
- package/dist/{campaign-BQ-Jujuj.js → campaign-BudZ70zo.js} +245 -247
- package/dist/campaign-BudZ70zo.js.map +1 -0
- package/dist/{campaign-evidence-OcUciD-N.js → campaign-evidence-Clrn4hBV.js} +8 -8
- package/dist/campaign-evidence-Clrn4hBV.js.map +1 -0
- package/dist/{canonical-BsZVgBhW.js → canonical-B2RqLofv.js} +2 -2
- package/dist/{canonical-BsZVgBhW.js.map → canonical-B2RqLofv.js.map} +1 -1
- package/dist/{chat-client-DthPOYGd.js → chat-client-l-MVDlDk.js} +2 -2
- package/dist/{chat-client-DthPOYGd.js.map → chat-client-l-MVDlDk.js.map} +1 -1
- package/dist/{chat-json-call-BrKsyspg.js → chat-json-call-puMArmtW.js} +2 -2
- package/dist/{chat-json-call-BrKsyspg.js.map → chat-json-call-puMArmtW.js.map} +1 -1
- package/dist/{claim-BXcZw-jo.js → claim-q4HQhK1G.js} +4 -4
- package/dist/claim-q4HQhK1G.js.map +1 -0
- package/dist/cli.js +9 -9
- package/dist/cli.js.map +1 -1
- package/dist/{cluster-bootstrap-B3JywQQn.js → cluster-bootstrap-qcUWYFl-.js} +3 -3
- package/dist/{cluster-bootstrap-B3JywQQn.js.map → cluster-bootstrap-qcUWYFl-.js.map} +1 -1
- package/dist/contract/index.d.ts +11 -11
- package/dist/contract/index.js +1022 -1078
- package/dist/contract/index.js.map +1 -1
- package/dist/{cost-ledger-D2FnJPBF.js → cost-ledger-BP8V4kUQ.js} +835 -816
- package/dist/cost-ledger-BP8V4kUQ.js.map +1 -0
- package/dist/{cost-ledger-CO7UyEcQ.d.ts → cost-ledger-Jk62OBfF.d.ts} +12 -47
- package/dist/cost-ledger-Jk62OBfF.d.ts.map +1 -0
- package/dist/{counterfactual-DtMWvn7D.js → counterfactual-Dn3veueY.js} +2 -2
- package/dist/{counterfactual-DtMWvn7D.js.map → counterfactual-Dn3veueY.js.map} +1 -1
- package/dist/{default-registry-Cn1LQ2Ve.js → default-registry-COhk9g36.js} +5 -5
- package/dist/{default-registry-Cn1LQ2Ve.js.map → default-registry-COhk9g36.js.map} +1 -1
- package/dist/{default-registry-BrWh62zf.d.ts → default-registry-DzHiGruW.d.ts} +6 -6
- package/dist/{default-registry-BrWh62zf.d.ts.map → default-registry-DzHiGruW.d.ts.map} +1 -1
- package/dist/{define-agent-eval-Ch3MWX6k.js → define-agent-eval-CNTPzIkp.js} +10 -10
- package/dist/define-agent-eval-CNTPzIkp.js.map +1 -0
- package/dist/{define-agent-eval-CEWkX5wZ.d.ts → define-agent-eval-WSck0CZJ.d.ts} +8 -8
- package/dist/{define-agent-eval-CEWkX5wZ.d.ts.map → define-agent-eval-WSck0CZJ.d.ts.map} +1 -1
- package/dist/{descriptive-CnU8Yq29.js → descriptive-DboxPZrG.js} +2 -2
- package/dist/{descriptive-CnU8Yq29.js.map → descriptive-DboxPZrG.js.map} +1 -1
- package/dist/diagnosis/index.d.ts +3 -3
- package/dist/diagnosis/index.js +4 -4
- package/dist/{dspy-rlm-engine-D-KX_e64.js → dspy-rlm-engine-LRCu61m5.js} +3 -3
- package/dist/{dspy-rlm-engine-D-KX_e64.js.map → dspy-rlm-engine-LRCu61m5.js.map} +1 -1
- package/dist/{effect-sizes-DiH8MGOH.js → effect-sizes-BhzEpXMc.js} +2 -2
- package/dist/{effect-sizes-DiH8MGOH.js.map → effect-sizes-BhzEpXMc.js.map} +1 -1
- package/dist/{engine-C_zNWfM8.d.ts → engine-3A_4jOlI.d.ts} +5 -5
- package/dist/{engine-C_zNWfM8.d.ts.map → engine-3A_4jOlI.d.ts.map} +1 -1
- package/dist/{equal-terms-Blx9Zk5I.js → equal-terms-BtdXayCJ.js} +2 -2
- package/dist/{equal-terms-Blx9Zk5I.js.map → equal-terms-BtdXayCJ.js.map} +1 -1
- package/dist/{errors-Dngq5h35.js → errors-DVZiqS7K.js} +6 -2
- package/dist/{errors-Dngq5h35.js.map → errors-DVZiqS7K.js.map} +1 -1
- package/dist/{eval-campaign-4byTtqrf.js → eval-campaign-90jeaCAI.js} +6 -6
- package/dist/{eval-campaign-4byTtqrf.js.map → eval-campaign-90jeaCAI.js.map} +1 -1
- package/dist/{evaluation-Cn2dRZIz.d.ts → evaluation-6Yoi74gM.d.ts} +4 -4
- package/dist/{evaluation-Cn2dRZIz.d.ts.map → evaluation-6Yoi74gM.d.ts.map} +1 -1
- package/dist/evaluation.d.ts +1 -1
- package/dist/evaluation.js +1 -1
- package/dist/{exact-types-CD0lf5Qd.d.ts → exact-types-ClugMJfI.d.ts} +2 -2
- package/dist/{exact-types-CD0lf5Qd.d.ts.map → exact-types-ClugMJfI.d.ts.map} +1 -1
- package/dist/experiment/index.d.ts +404 -3
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +313 -19
- package/dist/experiment/index.js.map +1 -1
- package/dist/{external-optimizer-process-nnsY5D_4.js → external-optimizer-process-BjMGUuds.js} +3 -3
- package/dist/{external-optimizer-process-nnsY5D_4.js.map → external-optimizer-process-BjMGUuds.js.map} +1 -1
- package/dist/{external-optimizer-subprocess-DcerlZSG.js → external-optimizer-subprocess-BqW0hHcI.js} +548 -548
- package/dist/external-optimizer-subprocess-BqW0hHcI.js.map +1 -0
- package/dist/{feedback-trajectory-DkaDj0dj.d.ts → feedback-trajectory-C4EG0nNH.d.ts} +5 -8
- package/dist/feedback-trajectory-C4EG0nNH.d.ts.map +1 -0
- package/dist/fuzz.d.ts +1 -1
- package/dist/fuzz.js +3 -3
- package/dist/hosted/index.js +1 -1
- package/dist/{index-CPJJr98o.d.ts → index-BFRO6chs.d.ts} +3 -2
- package/dist/index-BFRO6chs.d.ts.map +1 -0
- package/dist/{index-Ch9VxCpK.d.ts → index-BO41eGj5.d.ts} +2 -2
- package/dist/{index-Ch9VxCpK.d.ts.map → index-BO41eGj5.d.ts.map} +1 -1
- package/dist/{index-NvGBiO8H.d.ts → index-CyszSg4O.d.ts} +165 -162
- package/dist/index-CyszSg4O.d.ts.map +1 -0
- package/dist/{index-DlH0Y9gI.d.ts → index-DPcngtwt.d.ts} +30 -17
- package/dist/index-DPcngtwt.d.ts.map +1 -0
- package/dist/{index-oQ4aHziK.d.ts → index-DV-jSzxF.d.ts} +8 -8
- package/dist/{index-oQ4aHziK.d.ts.map → index-DV-jSzxF.d.ts.map} +1 -1
- package/dist/index.d.ts +1969 -1871
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +161 -201
- package/dist/index.js.map +1 -1
- package/dist/{integrity-BvnVARVl.js → integrity-BXOsAb3i.js} +2 -2
- package/dist/{integrity-BvnVARVl.js.map → integrity-BXOsAb3i.js.map} +1 -1
- package/dist/{integrity-CCAxGVK6.d.ts → integrity-DAMEDFaC.d.ts} +2 -2
- package/dist/{integrity-CCAxGVK6.d.ts.map → integrity-DAMEDFaC.d.ts.map} +1 -1
- package/dist/{integrity-BV1ZDf42.js → integrity-dLiDe2kD.js} +2 -5
- package/dist/integrity-dLiDe2kD.js.map +1 -0
- package/dist/{internal-CRHwS9iz.js → internal-Epl-YBZW.js} +3 -3
- package/dist/{internal-CRHwS9iz.js.map → internal-Epl-YBZW.js.map} +1 -1
- package/dist/interval-CELpye1n.js +219 -0
- package/dist/interval-CELpye1n.js.map +1 -0
- package/dist/interval-ohuOJurX.d.ts +47 -0
- package/dist/interval-ohuOJurX.d.ts.map +1 -0
- package/dist/{journal-DvKCvjty.js → journal-DfRz9kps.js} +2 -2
- package/dist/{journal-DvKCvjty.js.map → journal-DfRz9kps.js.map} +1 -1
- package/dist/{judge-calibration-N44V04Se.js → judge-calibration-BpCy5XQl.js} +3 -3
- package/dist/{judge-calibration-N44V04Se.js.map → judge-calibration-BpCy5XQl.js.map} +1 -1
- package/dist/{judge-gate-DNcUrRFA.d.ts → judge-gate-CE7CQAgt.d.ts} +3 -3
- package/dist/{judge-gate-DNcUrRFA.d.ts.map → judge-gate-CE7CQAgt.d.ts.map} +1 -1
- package/dist/{judge-gate-B1qYnkpi.js → judge-gate-iY9s4L44.js} +3 -3
- package/dist/judge-gate-iY9s4L44.js.map +1 -0
- package/dist/{kind-factory-CbZ-6tvu.js → kind-factory-BobI5Jy3.js} +4 -4
- package/dist/{kind-factory-CbZ-6tvu.js.map → kind-factory-BobI5Jy3.js.map} +1 -1
- package/dist/ledger-core/index.js +3 -3
- package/dist/{ledger-core-B2hahzrt.js → ledger-core-Gq7yp8lP.js} +4 -4
- package/dist/{ledger-core-B2hahzrt.js.map → ledger-core-Gq7yp8lP.js.map} +1 -1
- package/dist/{llm-client-DQveoxSM.js → llm-client-CyBAoYiX.js} +192 -219
- package/dist/llm-client-CyBAoYiX.js.map +1 -0
- package/dist/{llm-judge-PXV7Xljl.js → llm-judge-CG12SQJq.js} +499 -921
- package/dist/llm-judge-CG12SQJq.js.map +1 -0
- package/dist/matrix/index.d.ts +2 -2
- package/dist/matrix/index.js +1 -1
- package/dist/{matrix-CHMiebFx.d.ts → matrix-DSrSxjjy.d.ts} +4 -4
- package/dist/{matrix-CHMiebFx.d.ts.map → matrix-DSrSxjjy.d.ts.map} +1 -1
- package/dist/{matrix-_BV9-Lk5.js → matrix-Dm0BQW7U.js} +7 -11
- package/dist/matrix-Dm0BQW7U.js.map +1 -0
- package/dist/{paired-tests-jbwPdSz5.js → mean-DHoMiudD.js} +11 -5
- package/dist/mean-DHoMiudD.js.map +1 -0
- package/dist/meta-eval/index.d.ts +3 -3
- package/dist/meta-eval/index.js +9 -9
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/mint-DNU8QPG4.d.ts +81 -0
- package/dist/mint-DNU8QPG4.d.ts.map +1 -0
- package/dist/{mint-Csor1BLJ.js → mint-QaD5cA0u.js} +3 -3
- package/dist/{mint-Csor1BLJ.js.map → mint-QaD5cA0u.js.map} +1 -1
- package/dist/{multiplicity-D9cjmStq.js → multiplicity-DQUFcbW4.js} +2 -2
- package/dist/{multiplicity-D9cjmStq.js.map → multiplicity-DQUFcbW4.js.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/golden/index.d.ts.map +1 -1
- package/dist/multishot/golden/index.js +3 -6
- package/dist/multishot/golden/index.js.map +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/multishot/index.js +1 -1
- package/dist/nonnegative-number-DvdMxg_f.js +9 -0
- package/dist/nonnegative-number-DvdMxg_f.js.map +1 -0
- package/dist/openapi.json +1 -1
- package/dist/{otel-spans-C8YIJWy1.js → otel-spans-B9tUXRHF.js} +3 -3
- package/dist/{otel-spans-C8YIJWy1.js.map → otel-spans-B9tUXRHF.js.map} +1 -1
- package/dist/{paired-arms-DzKn0TfY.js → paired-arms-CwMGJsrS.js} +5 -5
- package/dist/{paired-arms-DzKn0TfY.js.map → paired-arms-CwMGJsrS.js.map} +1 -1
- package/dist/{paired-binary-Q36bhn2R.js → paired-binary-CvpFHfKZ.js} +2 -2
- package/dist/{paired-binary-Q36bhn2R.js.map → paired-binary-CvpFHfKZ.js.map} +1 -1
- package/dist/{paired-promotion-decision-BN6IQTiu.js → paired-promotion-decision-DsAPYGCm.js} +3 -3
- package/dist/{paired-promotion-decision-BN6IQTiu.js.map → paired-promotion-decision-DsAPYGCm.js.map} +1 -1
- package/dist/{search-kernel-BztlERLv.js → pareto-Cm6ndrWN.js} +883 -861
- package/dist/pareto-Cm6ndrWN.js.map +1 -0
- package/dist/pipelines/index.d.ts +2 -2
- package/dist/pipelines/index.js +3 -3
- package/dist/{power-and-mde-fK5Qocwo.js → power-and-mde-Hj2ubb4B.js} +2 -2
- package/dist/{power-and-mde-fK5Qocwo.js.map → power-and-mde-Hj2ubb4B.js.map} +1 -1
- package/dist/predicate-FdKmElA_.js +38 -0
- package/dist/predicate-FdKmElA_.js.map +1 -0
- package/dist/{prime-protocol-DcAK9pC8.js → prime-protocol-CHKgA3a6.js} +2 -2
- package/dist/{prime-protocol-DcAK9pC8.js.map → prime-protocol-CHKgA3a6.js.map} +1 -1
- package/dist/{prime-protocol-CEE4BR9N.d.ts → prime-protocol-HiB9HX7E.d.ts} +3 -3
- package/dist/{prime-protocol-CEE4BR9N.d.ts.map → prime-protocol-HiB9HX7E.d.ts.map} +1 -1
- package/dist/{produced-state-ZCtDBxAE.js → produced-state-BsDmnwxc.js} +8 -8
- package/dist/{produced-state-ZCtDBxAE.js.map → produced-state-BsDmnwxc.js.map} +1 -1
- package/dist/profile-cell.js +1 -1
- package/dist/{release-confidence-BK2QCB5M.d.ts → promotion-gate-BawrdPYr.d.ts} +102 -102
- package/dist/promotion-gate-BawrdPYr.d.ts.map +1 -0
- package/dist/{promotion-policy-CWW35UWq.js → promotion-policy-KO57gbyV.js} +4 -4
- package/dist/{promotion-policy-CWW35UWq.js.map → promotion-policy-KO57gbyV.js.map} +1 -1
- package/dist/{proposal-findings-BZiMTzAF.js → proposal-findings-CY1xKn_a.js} +2 -2
- package/dist/{proposal-findings-BZiMTzAF.js.map → proposal-findings-CY1xKn_a.js.map} +1 -1
- package/dist/{query-B1D1qp2Z.js → query-BUCZCK7F.js} +3 -3
- package/dist/{query-B1D1qp2Z.js.map → query-BUCZCK7F.js.map} +1 -1
- package/dist/{random-Dn5fPWkt.js → random-ByUSnV9w.js} +2 -2
- package/dist/{random-Dn5fPWkt.js.map → random-ByUSnV9w.js.map} +1 -1
- package/dist/{raw-provider-sink-rB2DYxr6.js → raw-provider-sink-DoHz4JNG.js} +2 -2
- package/dist/{raw-provider-sink-rB2DYxr6.js.map → raw-provider-sink-DoHz4JNG.js.map} +1 -1
- package/dist/{redact-Dk1ZdJEm.js → redact-B5xXKqQ3.js} +97 -98
- package/dist/redact-B5xXKqQ3.js.map +1 -0
- package/dist/{redact-Cveoq6fJ.d.ts → redact-mtCCcT8z.d.ts} +4 -20
- package/dist/redact-mtCCcT8z.d.ts.map +1 -0
- package/dist/{registry-JYfPxcVz.d.ts → registry-D7_hm2fK.d.ts} +5 -5
- package/dist/{registry-JYfPxcVz.d.ts.map → registry-D7_hm2fK.d.ts.map} +1 -1
- package/dist/{release-confidence-Dbe9AS_X.js → release-confidence-CpuP65S5.js} +5 -5
- package/dist/{release-confidence-Dbe9AS_X.js.map → release-confidence-CpuP65S5.js.map} +1 -1
- package/dist/{report-command-y8H6RSX8.js → report-command-SC9nqb4M.js} +106 -171
- package/dist/report-command-SC9nqb4M.js.map +1 -0
- package/dist/reporting-html-Kvlc8oMt.js +100 -0
- package/dist/reporting-html-Kvlc8oMt.js.map +1 -0
- package/dist/reporting.d.ts +3 -3
- package/dist/reporting.js +6 -6
- package/dist/{researcher-f6mZhh58.d.ts → researcher-CEcfku-T.d.ts} +5 -5
- package/dist/{researcher-f6mZhh58.d.ts.map → researcher-CEcfku-T.d.ts.map} +1 -1
- package/dist/reward-CpCmAM8L.d.ts +78 -0
- package/dist/reward-CpCmAM8L.d.ts.map +1 -0
- package/dist/{reward-hacking-BFcQgkux.js → reward-hacking-E0d-WVcP.js} +2 -2
- package/dist/{reward-hacking-BFcQgkux.js.map → reward-hacking-E0d-WVcP.js.map} +1 -1
- package/dist/{reward-hacking-DHA7c1l8.d.ts → reward-hacking-RVdZ6Kkt.d.ts} +2 -2
- package/dist/{reward-hacking-DHA7c1l8.d.ts.map → reward-hacking-RVdZ6Kkt.d.ts.map} +1 -1
- package/dist/rl.d.ts +8 -8
- package/dist/rl.js +14 -14
- package/dist/rollout/index.d.ts +3 -2
- package/dist/rollout/index.js +2 -2
- package/dist/{rollout-D9bKKBnZ.js → rollout-DZV1t2Wg.js} +2 -2
- package/dist/{rollout-D9bKKBnZ.js.map → rollout-DZV1t2Wg.js.map} +1 -1
- package/dist/round-decimal-COrKglMh.js +10 -0
- package/dist/round-decimal-COrKglMh.js.map +1 -0
- package/dist/{rubric-predictive-validity-2SgHxoa5.d.ts → rubric-predictive-validity-C03EC7ZE.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-2SgHxoa5.d.ts.map → rubric-predictive-validity-C03EC7ZE.d.ts.map} +1 -1
- package/dist/{rubric-predictive-validity-BsgkBfT0.js → rubric-predictive-validity-DaFW70y9.js} +3 -3
- package/dist/{rubric-predictive-validity-BsgkBfT0.js.map → rubric-predictive-validity-DaFW70y9.js.map} +1 -1
- package/dist/{run-record-DlTZ_lqe.js → run-record-Boc5IUTp.js} +4 -4
- package/dist/{run-record-DlTZ_lqe.js.map → run-record-Boc5IUTp.js.map} +1 -1
- package/dist/{run-record-D1Hw-Jml.d.ts → run-record-C1E24G2A.d.ts} +2 -2
- package/dist/{run-record-D1Hw-Jml.d.ts.map → run-record-C1E24G2A.d.ts.map} +1 -1
- package/dist/{run-record-ES07wfz9.js → run-record-C5e7FjhJ.js} +3 -5
- package/dist/run-record-C5e7FjhJ.js.map +1 -0
- package/dist/search/index.d.ts +46 -32
- package/dist/search/index.d.ts.map +1 -1
- package/dist/search/index.js +3 -3
- package/dist/{search-kernel-CpcKH5Ih.d.ts → search-kernel-DyNEa6IU.d.ts} +14 -10
- package/dist/search-kernel-DyNEa6IU.d.ts.map +1 -0
- package/dist/{search-ledger-d7V8o9Lk.js → search-ledger-CVUGLDZp.js} +189 -199
- package/dist/search-ledger-CVUGLDZp.js.map +1 -0
- package/dist/{search-ledger-file-BqOo-bVo.js → search-ledger-file-DDQUgvCg.js} +4 -4
- package/dist/{search-ledger-file-BqOo-bVo.js.map → search-ledger-file-DDQUgvCg.js.map} +1 -1
- package/dist/search-ledger-types-p2xKfiDJ.d.ts.map +1 -1
- package/dist/{search-shipper-er4pK5_Y.js → search-shipper-BOSD46qv.js} +3 -3
- package/dist/{search-shipper-er4pK5_Y.js.map → search-shipper-BOSD46qv.js.map} +1 -1
- package/dist/{search-summary-CBvz3u_S.js → search-summary-BQCSWf1g.js} +5 -5
- package/dist/{search-summary-CBvz3u_S.js.map → search-summary-BQCSWf1g.js.map} +1 -1
- package/dist/{semantic-concept-judge-zQBG9hHU.js → semantic-concept-judge-BQF8te0T.js} +12 -14
- package/dist/semantic-concept-judge-BQF8te0T.js.map +1 -0
- package/dist/{sequential-D-1PPCmM.js → sequential-Bf_pbE50.js} +5 -5
- package/dist/{sequential-D-1PPCmM.js.map → sequential-Bf_pbE50.js.map} +1 -1
- package/dist/{sequential-eprocess-D1jKoihe.js → sequential-eprocess-DM_qENCl.js} +2 -2
- package/dist/{sequential-eprocess-D1jKoihe.js.map → sequential-eprocess-DM_qENCl.js.map} +1 -1
- package/dist/{server-DbcC7xMl.js → server-e9oWO_bN.js} +4 -4
- package/dist/{server-DbcC7xMl.js.map → server-e9oWO_bN.js.map} +1 -1
- package/dist/{single-run-lock-fVu3MZWH.js → single-run-lock-CV9Eyd_q.js} +2 -2
- package/dist/{single-run-lock-fVu3MZWH.js.map → single-run-lock-CV9Eyd_q.js.map} +1 -1
- package/dist/{skillopt-optimization-method-CCtzralq.js → skillopt-optimization-method-BiwA8nfX.js} +427 -12
- package/dist/skillopt-optimization-method-BiwA8nfX.js.map +1 -0
- package/dist/{spans-D9_ehFuk.d.ts → spans-BHAUXMGN.d.ts} +2 -2
- package/dist/{spans-D9_ehFuk.d.ts.map → spans-BHAUXMGN.d.ts.map} +1 -1
- package/dist/{statistical-heldout-1Gbmex7v.js → statistical-heldout-42NlCNDe.js} +3 -3
- package/dist/{statistical-heldout-1Gbmex7v.js.map → statistical-heldout-42NlCNDe.js.map} +1 -1
- package/dist/{statistical-heldout-BeR4bOdg.d.ts → statistical-heldout-CYpGaoDH.d.ts} +3 -3
- package/dist/{statistical-heldout-BeR4bOdg.d.ts.map → statistical-heldout-CYpGaoDH.d.ts.map} +1 -1
- package/dist/{store-otlp-DtB4Slsr.js → store-otlp-DJ9dtA5x.js} +135 -132
- package/dist/store-otlp-DJ9dtA5x.js.map +1 -0
- package/dist/{store-tool-spans-fZIigrpP.js → store-tool-spans-CIX5arAG.js} +7 -7
- package/dist/{store-tool-spans-fZIigrpP.js.map → store-tool-spans-CIX5arAG.js.map} +1 -1
- package/dist/{store-tool-spans-DR1SrCLK.d.ts → store-tool-spans-DdCSPa7C.d.ts} +162 -161
- package/dist/store-tool-spans-DdCSPa7C.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts.map +1 -1
- package/dist/storyboard/index.js +20 -3
- package/dist/storyboard/index.js.map +1 -1
- package/dist/{student-t-BQeKJx2N.js → student-t-DVPW7CdL.js} +2 -2
- package/dist/{student-t-BQeKJx2N.js.map → student-t-DVPW7CdL.js.map} +1 -1
- package/dist/{summary-report-DtdRucM0.js → summary-report-V_JACR4Y.js} +526 -614
- package/dist/summary-report-V_JACR4Y.js.map +1 -0
- package/dist/{summary-report-C-CtItCN.d.ts → summary-report-bdFDXq0b.d.ts} +10 -5
- package/dist/summary-report-bdFDXq0b.d.ts.map +1 -0
- package/dist/supervisor-run/index.d.ts +153 -149
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +65 -3
- package/dist/supervisor-run/index.js.map +1 -1
- package/dist/{systemone-CpTLAlqb.d.ts → systemone-BomhHr2c.d.ts} +6 -6
- package/dist/{systemone-CpTLAlqb.d.ts.map → systemone-BomhHr2c.d.ts.map} +1 -1
- package/dist/systemone-protocol.js +1 -1
- package/dist/{systemone-review-B_yo1xsd.d.ts → systemone-review-CUVcQJu6.d.ts} +2 -2
- package/dist/{systemone-review-B_yo1xsd.d.ts.map → systemone-review-CUVcQJu6.d.ts.map} +1 -1
- package/dist/{systemone-review-BZWfrtq1.js → systemone-review-D7sXwsqq.js} +2 -2
- package/dist/{systemone-review-BZWfrtq1.js.map → systemone-review-D7sXwsqq.js.map} +1 -1
- package/dist/systemone.d.ts +3 -3
- package/dist/systemone.js +4 -4
- package/dist/{task-failure-attributes-DngaLdE4.js → task-failure-attributes-hEmASjix.js} +2 -2
- package/dist/{task-failure-attributes-DngaLdE4.js.map → task-failure-attributes-hEmASjix.js.map} +1 -1
- package/dist/{terminal-record-BHocwFw1.js → terminal-record-DZyQZ3pz.js} +14 -10
- package/dist/terminal-record-DZyQZ3pz.js.map +1 -0
- package/dist/{tool-groups-mcxOXVlb.d.ts → tool-groups-C3NEcdCM.d.ts} +3 -3
- package/dist/tool-groups-C3NEcdCM.d.ts.map +1 -0
- package/dist/{tool-waste-Wi4g6Ekb.d.ts → tool-waste-BKgvd-TP.d.ts} +30 -30
- package/dist/tool-waste-BKgvd-TP.d.ts.map +1 -0
- package/dist/{tool-waste-D52PqU9Q.js → tool-waste-BlesqTqW.js} +4 -4
- package/dist/{tool-waste-D52PqU9Q.js.map → tool-waste-BlesqTqW.js.map} +1 -1
- package/dist/{trace-questions-DG7nqLQ7.d.ts → trace-questions-CM2CAocU.d.ts} +4 -4
- package/dist/{trace-questions-DG7nqLQ7.d.ts.map → trace-questions-CM2CAocU.d.ts.map} +1 -1
- package/dist/trace-repair/index.d.ts +3 -3
- package/dist/trace-repair/index.d.ts.map +1 -1
- package/dist/trace-repair/index.js +7 -13
- package/dist/trace-repair/index.js.map +1 -1
- package/dist/traces.d.ts +76 -58
- package/dist/traces.d.ts.map +1 -1
- package/dist/traces.js +9 -9
- package/dist/trajectory-replay/index.js +3 -3
- package/dist/{tree-rXLW3M_4.js → tree-C1XYeXvy.js} +2277 -2339
- package/dist/tree-C1XYeXvy.js.map +1 -0
- package/dist/{types-DBZ7keha.d.ts → types-C02qtnG8.d.ts} +4 -46
- package/dist/types-C02qtnG8.d.ts.map +1 -0
- package/dist/{types-oKRhQAde.d.ts → types-C1tnkEgM.d.ts} +4 -4
- package/dist/{types-oKRhQAde.d.ts.map → types-C1tnkEgM.d.ts.map} +1 -1
- package/dist/{types-CfZMIzB_.d.ts → types-Cbe2fWBa.d.ts} +3 -3
- package/dist/{types-CfZMIzB_.d.ts.map → types-Cbe2fWBa.d.ts.map} +1 -1
- package/dist/{types-B4C5nFw6.d.ts → types-Cq8mSbq8.d.ts} +78 -72
- package/dist/types-Cq8mSbq8.d.ts.map +1 -0
- package/dist/{verdict-DhPoXApk.js → verdict-Dl0tiT1X.js} +2 -2
- package/dist/{verdict-DhPoXApk.js.map → verdict-Dl0tiT1X.js.map} +1 -1
- package/dist/{verdict-cache-BQEsVy9n.js → verdict-cache-Bm8EKL2s.js} +2 -2
- package/dist/{verdict-cache-BQEsVy9n.js.map → verdict-cache-Bm8EKL2s.js.map} +1 -1
- package/dist/wire/index.d.ts +3 -3
- package/dist/wire/index.js +1 -1
- package/docs/design/mlbenchmarks-book-review.md +1 -1
- package/docs/design/mlbenchmarks-review/probes.mts +23 -31
- package/docs/experiment.md +1 -1
- package/package.json +1 -1
- package/dist/analyze-runs-3xpby3p2.js.map +0 -1
- package/dist/analyze-runs-CSDAh8A7.d.ts.map +0 -1
- package/dist/ast-DJ6Bc7ab.js +0 -558
- package/dist/ast-DJ6Bc7ab.js.map +0 -1
- package/dist/ast-hI-vjW6J.d.ts +0 -457
- package/dist/ast-hI-vjW6J.d.ts.map +0 -1
- package/dist/benchmark-command-CByl9dxG.js.map +0 -1
- package/dist/campaign-BQ-Jujuj.js.map +0 -1
- package/dist/campaign-evidence-OcUciD-N.js.map +0 -1
- package/dist/claim-BXcZw-jo.js.map +0 -1
- package/dist/cost-ledger-CO7UyEcQ.d.ts.map +0 -1
- package/dist/cost-ledger-D2FnJPBF.js.map +0 -1
- package/dist/define-agent-eval-Ch3MWX6k.js.map +0 -1
- package/dist/external-optimizer-subprocess-DcerlZSG.js.map +0 -1
- package/dist/feedback-trajectory-DkaDj0dj.d.ts.map +0 -1
- package/dist/index-CPJJr98o.d.ts.map +0 -1
- package/dist/index-DlH0Y9gI.d.ts.map +0 -1
- package/dist/index-NvGBiO8H.d.ts.map +0 -1
- package/dist/integrity-BV1ZDf42.js.map +0 -1
- package/dist/judge-gate-B1qYnkpi.js.map +0 -1
- package/dist/llm-client-DQveoxSM.js.map +0 -1
- package/dist/llm-judge-PXV7Xljl.js.map +0 -1
- package/dist/matrix-_BV9-Lk5.js.map +0 -1
- package/dist/mint-DCGoBYhB.d.ts +0 -155
- package/dist/mint-DCGoBYhB.d.ts.map +0 -1
- package/dist/paired-tests-jbwPdSz5.js.map +0 -1
- package/dist/redact-Cveoq6fJ.d.ts.map +0 -1
- package/dist/redact-Dk1ZdJEm.js.map +0 -1
- package/dist/release-confidence-BK2QCB5M.d.ts.map +0 -1
- package/dist/report-command-y8H6RSX8.js.map +0 -1
- package/dist/run-record-ES07wfz9.js.map +0 -1
- package/dist/search-kernel-BztlERLv.js.map +0 -1
- package/dist/search-kernel-CpcKH5Ih.d.ts.map +0 -1
- package/dist/search-ledger-d7V8o9Lk.js.map +0 -1
- package/dist/semantic-concept-judge-zQBG9hHU.js.map +0 -1
- package/dist/skillopt-optimization-method-CCtzralq.js.map +0 -1
- package/dist/store-otlp-DtB4Slsr.js.map +0 -1
- package/dist/store-tool-spans-DR1SrCLK.d.ts.map +0 -1
- package/dist/summary-report-C-CtItCN.d.ts.map +0 -1
- package/dist/summary-report-DtdRucM0.js.map +0 -1
- package/dist/terminal-record-BHocwFw1.js.map +0 -1
- package/dist/tool-groups-mcxOXVlb.d.ts.map +0 -1
- package/dist/tool-waste-Wi4g6Ekb.d.ts.map +0 -1
- package/dist/tree-rXLW3M_4.js.map +0 -1
- package/dist/types-B4C5nFw6.d.ts.map +0 -1
- package/dist/types-DBZ7keha.d.ts.map +0 -1
package/dist/contract/index.js
CHANGED
|
@@ -1,24 +1,24 @@
|
|
|
1
|
-
import { a as hashCanonical } from "../canonical-
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
1
|
+
import { a as hashCanonical } from "../canonical-B2RqLofv.js";
|
|
2
|
+
import { a as parseRunRecordSafe } from "../run-record-Boc5IUTp.js";
|
|
3
|
+
import { i as pairedBootstrap } from "../mean-DHoMiudD.js";
|
|
4
|
+
import { t as decidePairedPromotion } from "../paired-promotion-decision-DsAPYGCm.js";
|
|
5
|
+
import { a as heldoutSignificance } from "../statistical-heldout-42NlCNDe.js";
|
|
6
|
+
import { r as summarizeExecution, t as analyzeRuns } from "../analyze-runs-C84irm4i.js";
|
|
7
|
+
import { G as defaultProductionGate, a as transientDispatchFailure, m as runImprovementLoop, nt as runEval, rt as runCampaign, t as llmJudge } from "../llm-judge-CG12SQJq.js";
|
|
8
8
|
import { i as isModelPriced, n as estimateCost } from "../metrics-BKaPBtIG.js";
|
|
9
|
-
import {
|
|
10
|
-
import { G as campaignSplitDigest } from "../campaign-evidence-
|
|
11
|
-
import { h as mapConcurrentRange } from "../journal-
|
|
12
|
-
import { F as inMemoryCampaignStorage, P as fsCampaignStorage } from "../external-optimizer-subprocess-
|
|
9
|
+
import { t as CostLedger } from "../cost-ledger-BP8V4kUQ.js";
|
|
10
|
+
import { G as campaignSplitDigest } from "../campaign-evidence-Clrn4hBV.js";
|
|
11
|
+
import { h as mapConcurrentRange } from "../journal-DfRz9kps.js";
|
|
12
|
+
import { F as inMemoryCampaignStorage, P as fsCampaignStorage } from "../external-optimizer-subprocess-BqW0hHcI.js";
|
|
13
13
|
import { o as makeProposalFinding } from "../types-IXD2HJzu.js";
|
|
14
|
-
import { n as SelfImproveRunError, r as selfImprove, t as defineAgentEval } from "../define-agent-eval-
|
|
15
|
-
import { t as buildDefaultAnalystRegistry } from "../default-registry-
|
|
14
|
+
import { n as SelfImproveRunError, r as selfImprove, t as defineAgentEval } from "../define-agent-eval-CNTPzIkp.js";
|
|
15
|
+
import { t as buildDefaultAnalystRegistry } from "../default-registry-COhk9g36.js";
|
|
16
16
|
import { t as extractUsage } from "../extract-usage-BrnzRJ6H.js";
|
|
17
|
-
import { t as createChatClient } from "../chat-client-
|
|
17
|
+
import { t as createChatClient } from "../chat-client-l-MVDlDk.js";
|
|
18
18
|
import { n as InMemoryOutcomeStore, t as FileSystemOutcomeStore } from "../outcome-store-DUcMj-3D.js";
|
|
19
|
-
import {
|
|
20
|
-
import { n as paretoPolicy, r as paretoSignificanceGate, t as buildEvidenceVector } from "../promotion-policy-
|
|
21
|
-
import { t as fromOtelSpans } from "../otel-spans-
|
|
19
|
+
import { _ as runReferenceEquivalenceJudge, d as externalTextOptimizationMethod, g as createReferenceEquivalenceJudge, h as REFERENCE_EQUIVALENCE_JUDGE_VERSION, l as heldOutGate, m as REFERENCE_EQUIVALENCE_INPUT_LIMITS, n as compareOptimizationMethods, r as gepaOptimizationMethod, t as skillOptOptimizationMethod, u as composeGate } from "../skillopt-optimization-method-BiwA8nfX.js";
|
|
20
|
+
import { n as paretoPolicy, r as paretoSignificanceGate, t as buildEvidenceVector } from "../promotion-policy-KO57gbyV.js";
|
|
21
|
+
import { t as fromOtelSpans } from "../otel-spans-B9tUXRHF.js";
|
|
22
22
|
import { createHash } from "node:crypto";
|
|
23
23
|
import { agentCandidateBenchmarkSuiteSchema, agentCandidateBenchmarkTaskSchema, agentCandidateBundleSchema, agentCandidateEvaluationPolicySchema, agentCandidateExperimentSchema, agentImprovementMeasuredComparisonSchema, agentProfileImprovementExperimentSchema, agentProfileImprovementMeasuredComparisonSchema, agentProfileImprovementRunCellSchema, agentProfileImprovementRunReceiptSchema, agentProfileImprovementSuiteInputsSchema, agentProfileImprovementSuiteSchema, agentProfileImprovementTaskSchema, candidateExecutionEvidenceSchema, canonicalCandidateDigest, canonicalCandidateJson, numbersApproximatelyEqual, omitTopLevelDigest } from "@tangle-network/agent-interface";
|
|
24
24
|
import { createReadStream } from "node:fs";
|
|
@@ -37,6 +37,189 @@ function addFixedSpend(left, right) {
|
|
|
37
37
|
costProvenance: left.costProvenance === "observed" && right.costProvenance === "observed" ? "observed" : "estimated"
|
|
38
38
|
};
|
|
39
39
|
}
|
|
40
|
+
/** Sum both arms in receipt order while retaining the observed/estimated provenance. */
|
|
41
|
+
function pairedFixedSpendReceipt(measurements, model, usageOf) {
|
|
42
|
+
const usage = measurements.reduce((sum, measurement) => addFixedSpend(addFixedSpend(sum, usageOf(measurement.baseline)), usageOf(measurement.candidate)), {
|
|
43
|
+
inputTokens: 0,
|
|
44
|
+
outputTokens: 0,
|
|
45
|
+
cachedInputTokens: 0,
|
|
46
|
+
reasoningTokens: 0,
|
|
47
|
+
modelCalls: 0,
|
|
48
|
+
costUsdNanos: 0,
|
|
49
|
+
costProvenance: "observed"
|
|
50
|
+
});
|
|
51
|
+
const costUsd = usage.costUsdNanos / 1e9;
|
|
52
|
+
return {
|
|
53
|
+
model,
|
|
54
|
+
inputTokens: usage.inputTokens,
|
|
55
|
+
outputTokens: usage.outputTokens,
|
|
56
|
+
cachedTokens: usage.cachedInputTokens,
|
|
57
|
+
reasoningTokens: usage.reasoningTokens,
|
|
58
|
+
...usage.costProvenance === "observed" ? { actualCostUsd: costUsd } : { estimatedCostUsd: costUsd }
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
//#endregion
|
|
62
|
+
//#region src/contract/candidate-experiment-evidence.ts
|
|
63
|
+
/** Verify signed candidate execution receipts and read their measured usage. */
|
|
64
|
+
const candidateExecutionEvidenceAdapter = {
|
|
65
|
+
score: (evidence) => evidence.receipt.benchmarkResult.material.score,
|
|
66
|
+
dimensions: (evidence) => evidence.receipt.benchmarkResult.material.dimensions,
|
|
67
|
+
costUsd: costFromEvidence,
|
|
68
|
+
costProvenance: (evidence) => combinedUsage(evidence).costProvenance,
|
|
69
|
+
latencyMs: latencyFromEvidence,
|
|
70
|
+
completed: completedSuccessfully,
|
|
71
|
+
passed: (evidence) => evidence.receipt.benchmarkResult.material.passed
|
|
72
|
+
};
|
|
73
|
+
function verifyMeasurement(experiment, input, index) {
|
|
74
|
+
const suite = experiment.benchmark.suite;
|
|
75
|
+
const taskIndex = Math.floor(index / suite.reps);
|
|
76
|
+
const repetition = index % suite.reps;
|
|
77
|
+
const task = experiment.benchmark.tasks[taskIndex];
|
|
78
|
+
const seed = suite.seeds[index];
|
|
79
|
+
if (!task || seed === void 0) throw new Error(`candidate experiment measurement ${index} is outside the signed suite`);
|
|
80
|
+
const baseline = verifyExecutionEvidence(input.baseline);
|
|
81
|
+
const candidate = verifyExecutionEvidence(input.candidate);
|
|
82
|
+
for (const [arm, evidence] of [["baseline", baseline], ["candidate", candidate]]) {
|
|
83
|
+
const bundle = experiment[arm];
|
|
84
|
+
const materialization = evidence.materializationReceipt;
|
|
85
|
+
const runCell = materialization.executionPlan.material.runCell;
|
|
86
|
+
verifySelfAddressed(runCell, "candidate run cell");
|
|
87
|
+
if (runCell.experimentDigest !== experiment.digest || runCell.arm !== arm || runCell.bundleDigest !== bundle.digest || runCell.suiteDigest !== suite.digest || runCell.taskDigest !== task.digest || runCell.taskIndex !== taskIndex || runCell.repetition !== repetition || runCell.seed !== seed || runCell.attempt > task.attempt.maxAttempts || materialization.bundleDigest !== bundle.digest || materialization.benchmark.suite.digest !== suite.digest || materialization.benchmark.task.digest !== task.digest || materialization.codeKind !== bundle.code.kind || materialization.profileActivation.profilePlan.material.sourceProfileDigest !== canonicalCandidateDigest(bundle.profile) || evidence.receipt.runCellDigest !== runCell.digest || JSON.stringify(materialization.resolvedModel) !== JSON.stringify(task.model)) throw new Error(`candidate experiment measurement ${index} substituted its ${arm} arm`);
|
|
88
|
+
verifyTaskOutcome(task, evidence, index, arm);
|
|
89
|
+
}
|
|
90
|
+
const baselinePlan = baseline.materializationReceipt.executionPlan.material;
|
|
91
|
+
const candidatePlan = candidate.materializationReceipt.executionPlan.material;
|
|
92
|
+
if (baselinePlan.executionId === candidatePlan.executionId || baselinePlan.runCell.digest === candidatePlan.runCell.digest || baseline.materializationReceipt.digest === candidate.materializationReceipt.digest || baseline.receipt.digest === candidate.receipt.digest || baseline.digest === candidate.digest) throw new Error(`candidate experiment measurement ${index} reused one execution across arms`);
|
|
93
|
+
return {
|
|
94
|
+
baseline,
|
|
95
|
+
candidate
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
function verifyExecutionEvidence(input) {
|
|
99
|
+
const evidence = candidateExecutionEvidenceSchema.parse(input);
|
|
100
|
+
verifySelfAddressed(evidence, "candidate execution evidence");
|
|
101
|
+
verifySelfAddressed(evidence.materializationReceipt, "candidate materialization receipt");
|
|
102
|
+
verifySelfAddressed(evidence.materializationReceipt.profileActivation, "candidate profile activation");
|
|
103
|
+
verifyMaterialAddressed(evidence.materializationReceipt.profileActivation.profilePlan, "candidate profile plan");
|
|
104
|
+
verifyMaterialAddressed(evidence.materializationReceipt.executionPlan, "candidate execution plan");
|
|
105
|
+
verifySelfAddressed(evidence.receipt, "candidate run receipt");
|
|
106
|
+
verifyMaterialAddressed(evidence.receipt.modelSettlement, "candidate model settlement");
|
|
107
|
+
verifyMaterialAddressed(evidence.receipt.taskOutcome, "candidate task outcome");
|
|
108
|
+
verifyMaterialAddressed(evidence.receipt.benchmarkResult, "candidate benchmark result");
|
|
109
|
+
return evidence;
|
|
110
|
+
}
|
|
111
|
+
function verifySelfAddressed(document, label) {
|
|
112
|
+
if (canonicalCandidateDigest(omitTopLevelDigest(document)) !== document.digest) throw new Error(`${label} digest is invalid`);
|
|
113
|
+
}
|
|
114
|
+
function verifyTaskOutcome(task, evidence, index, arm) {
|
|
115
|
+
const outcome = evidence.receipt.taskOutcome.material.outcome;
|
|
116
|
+
const result = evidence.receipt.benchmarkResult.material;
|
|
117
|
+
const prefix = `candidate experiment measurement ${index} ${arm}`;
|
|
118
|
+
if (result.evidence.sha256 === task.grader.artifact.sha256) throw new Error(`${prefix} reused grader bytes as grading evidence`);
|
|
119
|
+
const usage = combinedUsage(evidence);
|
|
120
|
+
const usageChecks = [
|
|
121
|
+
[
|
|
122
|
+
usage.modelCalls,
|
|
123
|
+
task.limits.maxModelCalls,
|
|
124
|
+
"model calls"
|
|
125
|
+
],
|
|
126
|
+
[
|
|
127
|
+
usage.inputTokens,
|
|
128
|
+
task.limits.maxInputTokens,
|
|
129
|
+
"input tokens"
|
|
130
|
+
],
|
|
131
|
+
[
|
|
132
|
+
usage.outputTokens,
|
|
133
|
+
task.limits.maxOutputTokens,
|
|
134
|
+
"output tokens"
|
|
135
|
+
],
|
|
136
|
+
[
|
|
137
|
+
usage.costUsdNanos,
|
|
138
|
+
Math.round(task.limits.maxCostUsd * 1e9),
|
|
139
|
+
"cost"
|
|
140
|
+
]
|
|
141
|
+
];
|
|
142
|
+
for (const [actual, maximum, label] of usageChecks) if (actual > maximum) throw new Error(`${prefix} ${label} ${actual} exceeds the signed limit ${maximum}`);
|
|
143
|
+
if (outcome.kind !== task.outcome.kind) throw new Error(`${prefix} returned an outcome outside the signed task contract`);
|
|
144
|
+
if (task.outcome.kind === "output") {
|
|
145
|
+
if (outcome.kind !== "output" || outcome.spec.mediaType !== task.outcome.mediaType || outcome.spec.maxBytes !== task.outcome.maxBytes) throw new Error(`${prefix} changed the signed output contract`);
|
|
146
|
+
return;
|
|
147
|
+
}
|
|
148
|
+
const repository = task.repository;
|
|
149
|
+
if (outcome.kind !== "workspace" || repository === void 0 || outcome.baseRepository.identity !== repository.identity || outcome.baseRepository.rootIdentity !== repository.rootIdentity || outcome.baseRepository.commit !== repository.baseCommit || outcome.baseRepository.tree !== repository.baseTree) throw new Error(`${prefix} did not start from the signed repository state`);
|
|
150
|
+
}
|
|
151
|
+
function verifyStableProfileMaterialization(measurements) {
|
|
152
|
+
for (const arm of ["baseline", "candidate"]) {
|
|
153
|
+
const expected = measurements[0]?.[arm].materializationReceipt.profileActivation;
|
|
154
|
+
if (!expected) throw new Error("candidate experiment contains no profile materialization");
|
|
155
|
+
const expectedDigest = canonicalCandidateDigest({
|
|
156
|
+
profilePlanDigest: expected.profilePlan.digest,
|
|
157
|
+
files: expected.files
|
|
158
|
+
});
|
|
159
|
+
for (const [index, measurement] of measurements.entries()) {
|
|
160
|
+
const activation = measurement[arm].materializationReceipt.profileActivation;
|
|
161
|
+
if (canonicalCandidateDigest({
|
|
162
|
+
profilePlanDigest: activation.profilePlan.digest,
|
|
163
|
+
files: activation.files
|
|
164
|
+
}) !== expectedDigest) throw new Error(`candidate experiment measurement ${index} ${arm} materialized a different profile`);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
function completedSuccessfully(evidence) {
|
|
169
|
+
const termination = evidence.receipt.termination;
|
|
170
|
+
return termination.kind === "exit" && termination.exitCode === 0;
|
|
171
|
+
}
|
|
172
|
+
function verifyMaterialAddressed(evidence, label) {
|
|
173
|
+
if (canonicalCandidateDigest(evidence.material) !== evidence.digest) throw new Error(`${label} digest is invalid`);
|
|
174
|
+
}
|
|
175
|
+
function costFromEvidence(evidence) {
|
|
176
|
+
return combinedUsage(evidence).costUsdNanos / 1e9;
|
|
177
|
+
}
|
|
178
|
+
function latencyFromEvidence(evidence) {
|
|
179
|
+
return evidence.receipt.timing.durationMs + evidence.receipt.benchmarkResult.material.grading.timing.durationMs;
|
|
180
|
+
}
|
|
181
|
+
function combinedUsage(evidence) {
|
|
182
|
+
const candidate = evidence.receipt.modelSettlement.material.usage;
|
|
183
|
+
const grader = evidence.receipt.benchmarkResult.material.grading.usage;
|
|
184
|
+
return addFixedSpend(candidate, grader);
|
|
185
|
+
}
|
|
186
|
+
function candidateMeasurementCostReceipt(measurements, model) {
|
|
187
|
+
return pairedFixedSpendReceipt(measurements, model, combinedUsage);
|
|
188
|
+
}
|
|
189
|
+
//#endregion
|
|
190
|
+
//#region src/contract/measured-comparison-record.ts
|
|
191
|
+
/** Keep published timing and cost accounting identical across experiment domains. */
|
|
192
|
+
function comparisonEvaluation(options, evaluation) {
|
|
193
|
+
return {
|
|
194
|
+
generationsExplored: options.generationsExplored ?? 0,
|
|
195
|
+
preparation: options.preparation,
|
|
196
|
+
measurement: {
|
|
197
|
+
wallDurationMs: options.measurement.wallDurationMs,
|
|
198
|
+
workDurationMs: evaluation.measurementWorkDurationMs,
|
|
199
|
+
cost: evaluation.measurementCost
|
|
200
|
+
},
|
|
201
|
+
total: {
|
|
202
|
+
wallDurationMs: options.preparation.wallDurationMs + options.measurement.wallDurationMs,
|
|
203
|
+
cost: evaluation.totalCost
|
|
204
|
+
}
|
|
205
|
+
};
|
|
206
|
+
}
|
|
207
|
+
/** Replay only the original inputs; derived timing and statistics are recomputed. */
|
|
208
|
+
function comparisonRecomputationInput(comparison) {
|
|
209
|
+
return {
|
|
210
|
+
experiment: comparison.experiment,
|
|
211
|
+
measurements: comparison.measurements,
|
|
212
|
+
runId: comparison.provenance.runId,
|
|
213
|
+
...comparison.candidate ? { candidate: comparison.candidate } : {},
|
|
214
|
+
generationsExplored: comparison.evaluation.generationsExplored,
|
|
215
|
+
preparation: comparison.evaluation.preparation,
|
|
216
|
+
measurement: {
|
|
217
|
+
wallDurationMs: comparison.evaluation.measurement.wallDurationMs,
|
|
218
|
+
cost: comparison.evaluation.measurement.cost
|
|
219
|
+
},
|
|
220
|
+
...comparison.metadata ? { metadata: comparison.metadata } : {}
|
|
221
|
+
};
|
|
222
|
+
}
|
|
40
223
|
//#endregion
|
|
41
224
|
//#region src/contract/concurrent-map.ts
|
|
42
225
|
/** Map both arms while bounding the actual executions, not the number of pairs. */
|
|
@@ -116,104 +299,67 @@ function assertNonNegativeFinite(value, label) {
|
|
|
116
299
|
if (!Number.isFinite(value) || value < 0) throw new Error(`${label} must be a non-negative finite number`);
|
|
117
300
|
}
|
|
118
301
|
//#endregion
|
|
119
|
-
//#region src/contract/
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
return
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
}
|
|
127
|
-
/** Freeze task order, repetitions, and every seed before either arm runs. */
|
|
128
|
-
function sealCandidateBenchmarkSuite(options) {
|
|
129
|
-
for (const task of options.tasks) verifyCandidateBenchmarkTask(task);
|
|
130
|
-
const material = {
|
|
131
|
-
kind: "agent-candidate-benchmark-suite",
|
|
132
|
-
digestAlgorithm: "rfc8785-sha256",
|
|
133
|
-
taskDigests: options.tasks.map((task) => task.digest),
|
|
134
|
-
reps: options.reps,
|
|
135
|
-
seeds: options.seeds
|
|
302
|
+
//#region src/contract/paired-measurement-input.ts
|
|
303
|
+
function projectPairedMeasurement(measurement, index, adapter) {
|
|
304
|
+
if (typeof measurement.cellId !== "string" || !measurement.cellId.trim()) throw new Error(`paired measurement ${index} requires a cell id`);
|
|
305
|
+
return {
|
|
306
|
+
cellId: measurement.cellId,
|
|
307
|
+
baseline: projectRun(measurement.baseline, adapter, `paired measurement ${index} baseline`),
|
|
308
|
+
candidate: projectRun(measurement.candidate, adapter, `paired measurement ${index} candidate`)
|
|
136
309
|
};
|
|
310
|
+
}
|
|
311
|
+
function projectRun(run, adapter, label) {
|
|
312
|
+
const suppliedDimensions = adapter.dimensions(run);
|
|
313
|
+
if (!Array.isArray(suppliedDimensions)) throw new Error(`${label} dimensions must be an array`);
|
|
314
|
+
const dimensions = /* @__PURE__ */ new Map();
|
|
315
|
+
for (const dimension of suppliedDimensions) {
|
|
316
|
+
if (typeof dimension.name !== "string" || !dimension.name.trim()) throw new Error(`${label} contains an unnamed dimension`);
|
|
317
|
+
if (dimensions.has(dimension.name)) throw new Error(`${label} repeats dimension '${dimension.name}'`);
|
|
318
|
+
dimensions.set(dimension.name, finiteMeasurement(dimension.score, `${label} ${dimension.name}`));
|
|
319
|
+
}
|
|
320
|
+
const completed = adapter.completed(run);
|
|
321
|
+
const passed = adapter.passed(run);
|
|
322
|
+
if (typeof completed !== "boolean" || typeof passed !== "boolean") throw new Error(`${label} completion and pass values must be booleans`);
|
|
323
|
+
const costProvenance = adapter.costProvenance(run);
|
|
324
|
+
if (costProvenance !== "observed" && costProvenance !== "estimated") throw new Error(`${label} cost provenance must be observed or estimated`);
|
|
137
325
|
return {
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
326
|
+
score: finiteMeasurement(adapter.score(run), `${label} score`),
|
|
327
|
+
dimensions,
|
|
328
|
+
costUsd: nonNegativeMeasurement(adapter.costUsd(run), `${label} cost`),
|
|
329
|
+
costProvenance,
|
|
330
|
+
latencyMs: nonNegativeMeasurement(adapter.latencyMs(run), `${label} latency`),
|
|
331
|
+
completed,
|
|
332
|
+
passed
|
|
143
333
|
};
|
|
144
334
|
}
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
...
|
|
149
|
-
|
|
150
|
-
}
|
|
335
|
+
function sharedProjectedDimensions(measurements) {
|
|
336
|
+
const expected = [...measurements[0].baseline.dimensions.keys()];
|
|
337
|
+
for (const [index, measurement] of measurements.entries()) for (const [arm, run] of [["baseline", measurement.baseline], ["candidate", measurement.candidate]]) {
|
|
338
|
+
const actual = [...run.dimensions.keys()];
|
|
339
|
+
if (JSON.stringify(actual) !== JSON.stringify(expected)) throw new Error(`paired measurement ${index} ${arm} dimensions do not match the suite`);
|
|
340
|
+
}
|
|
341
|
+
return expected;
|
|
151
342
|
}
|
|
152
|
-
function
|
|
153
|
-
const
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
verifyBundle(experiment.candidate, "candidate bundle");
|
|
157
|
-
if (experiment.baseline.digest === experiment.candidate.digest) throw new Error("candidate experiment baseline and candidate bundles are identical");
|
|
158
|
-
verifyCandidateBenchmarkSuiteInputs(experiment.benchmark);
|
|
159
|
-
return experiment;
|
|
343
|
+
function dimensionScore(run, name) {
|
|
344
|
+
const value = run.dimensions.get(name);
|
|
345
|
+
if (value === void 0) throw new Error(`paired measurement is missing dimension '${name}'`);
|
|
346
|
+
return value;
|
|
160
347
|
}
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
call: {
|
|
173
|
-
callId: `candidate-measurement:${experiment.digest}`,
|
|
174
|
-
channel: "measurement",
|
|
175
|
-
phase: "heldout",
|
|
176
|
-
actor: experiment.digest,
|
|
177
|
-
model: candidateMeasurementModel(experiment),
|
|
178
|
-
tags: { experimentDigest: experiment.digest }
|
|
179
|
-
},
|
|
180
|
-
...options.signal ? { signal: options.signal } : {},
|
|
181
|
-
async execute(index, arm, signal) {
|
|
182
|
-
const taskIndex = Math.floor(index / suite.reps);
|
|
183
|
-
const repetition = index % suite.reps;
|
|
184
|
-
const task = tasks[taskIndex];
|
|
185
|
-
const seed = suite.seeds[index];
|
|
186
|
-
if (!task || seed === void 0) throw new Error(`candidate experiment cell ${index} has no signed task or seed`);
|
|
187
|
-
const benchmarkCell = {
|
|
188
|
-
suiteDigest: suite.digest,
|
|
189
|
-
taskIndex,
|
|
190
|
-
repetition
|
|
191
|
-
};
|
|
192
|
-
return verifyExecutionEvidence(await options.execute({
|
|
193
|
-
experiment,
|
|
194
|
-
arm,
|
|
195
|
-
bundle: experiment[arm],
|
|
196
|
-
task,
|
|
197
|
-
benchmarkCell,
|
|
198
|
-
seed,
|
|
199
|
-
signal
|
|
200
|
-
}));
|
|
201
|
-
},
|
|
202
|
-
receipt(measurements) {
|
|
203
|
-
return candidateMeasurementCostReceipt(measurements, candidateMeasurementModel(experiment));
|
|
204
|
-
}
|
|
205
|
-
});
|
|
206
|
-
return {
|
|
207
|
-
measurements: run.measurements.map((measurement, index) => verifyMeasurement(experiment, measurement, index)),
|
|
208
|
-
measurement: {
|
|
209
|
-
wallDurationMs: run.wallDurationMs,
|
|
210
|
-
cost: {
|
|
211
|
-
usd: run.cost.usd,
|
|
212
|
-
provenance: run.cost.kind
|
|
213
|
-
}
|
|
214
|
-
}
|
|
215
|
-
};
|
|
348
|
+
function finiteMeasurement(value, label) {
|
|
349
|
+
if (!Number.isFinite(value)) throw new Error(`${label} must be finite`);
|
|
350
|
+
return value;
|
|
351
|
+
}
|
|
352
|
+
function nonNegativeMeasurement(value, label) {
|
|
353
|
+
if (!Number.isFinite(value) || value < 0) throw new Error(`${label} must be non-negative`);
|
|
354
|
+
return value;
|
|
355
|
+
}
|
|
356
|
+
function assertKnownCost(cost, label) {
|
|
357
|
+
if (!Number.isFinite(cost.usd) || cost.usd < 0) throw new Error(`${label} must be a non-negative number`);
|
|
358
|
+
if (cost.provenance !== "observed" && cost.provenance !== "estimated") throw new Error(`${label} provenance must be observed or estimated`);
|
|
216
359
|
}
|
|
360
|
+
//#endregion
|
|
361
|
+
//#region src/contract/paired-measurement-evaluation.ts
|
|
362
|
+
/** Shared significance, regression, precision, and budget decisions on paired receipts. */
|
|
217
363
|
/**
|
|
218
364
|
* Calculate the shared paired decision from any complete receipt shape.
|
|
219
365
|
*
|
|
@@ -449,6 +595,164 @@ function significanceReason(significance, deltaThreshold) {
|
|
|
449
595
|
if (decision.exactTestVetoes && decision.mcnemar !== null) return `McNemar's exact test refuses at threshold ${deltaThreshold}: b=${decision.mcnemar.b}, c=${decision.mcnemar.c}, p=${decision.mcnemar.pValue}`;
|
|
450
596
|
return `paired ${decision.statistic} interval lower bound ${decision.low} did not clear ${deltaThreshold}.${decision.methodDetail}`;
|
|
451
597
|
}
|
|
598
|
+
function measuredEstimate(baseline, candidate, options) {
|
|
599
|
+
const bootstrap = pairedBootstrap(baseline, candidate, {
|
|
600
|
+
confidence: options.confidence,
|
|
601
|
+
resamples: options.resamples,
|
|
602
|
+
statistic: "mean",
|
|
603
|
+
seed: options.seed
|
|
604
|
+
});
|
|
605
|
+
const baselineMean = mean(baseline);
|
|
606
|
+
const candidateMean = mean(candidate);
|
|
607
|
+
const delta = candidateMean - baselineMean;
|
|
608
|
+
return {
|
|
609
|
+
baseline: baselineMean,
|
|
610
|
+
candidate: candidateMean,
|
|
611
|
+
delta,
|
|
612
|
+
confidenceInterval: {
|
|
613
|
+
level: bootstrap.confidence,
|
|
614
|
+
lower: Math.min(bootstrap.low, delta),
|
|
615
|
+
upper: Math.max(bootstrap.high, delta),
|
|
616
|
+
method: "paired-bootstrap",
|
|
617
|
+
statistic: "mean",
|
|
618
|
+
resamples: bootstrap.resamples
|
|
619
|
+
},
|
|
620
|
+
n: bootstrap.n
|
|
621
|
+
};
|
|
622
|
+
}
|
|
623
|
+
function observedPairedPrecision(options) {
|
|
624
|
+
const n = options.baselineScores.length;
|
|
625
|
+
const lowerMargin = Math.max(0, options.delta - options.lowerBound);
|
|
626
|
+
const minimumDetectableDelta = options.deltaThreshold + lowerMargin;
|
|
627
|
+
const scaleAssumed = [...options.baselineScores, ...options.candidateScores].every((score) => score >= -.001 && score <= 1.001);
|
|
628
|
+
const headroom = Math.max(0, 1 - mean(options.baselineScores));
|
|
629
|
+
const enoughRuns = n >= options.minProductiveRuns;
|
|
630
|
+
const attainable = !scaleAssumed || minimumDetectableDelta <= headroom;
|
|
631
|
+
const sufficient = enoughRuns && attainable;
|
|
632
|
+
const scorerNote = options.sharedScorerChannel ? " The same scorer evaluated both arms, so scorer bias is not measured." : "";
|
|
633
|
+
if (!enoughRuns) return {
|
|
634
|
+
sufficient,
|
|
635
|
+
minimumDetectableDelta,
|
|
636
|
+
scaleAssumed,
|
|
637
|
+
reason: `only ${n} paired runs; ${options.minProductiveRuns} required.${scorerNote}`
|
|
638
|
+
};
|
|
639
|
+
if (!attainable) return {
|
|
640
|
+
sufficient,
|
|
641
|
+
minimumDetectableDelta,
|
|
642
|
+
scaleAssumed,
|
|
643
|
+
reason: `observed paired uncertainty gives a minimum detectable delta of ${minimumDetectableDelta.toFixed(3)}, above the remaining score headroom ${headroom.toFixed(3)}.${scorerNote}`
|
|
644
|
+
};
|
|
645
|
+
return {
|
|
646
|
+
sufficient,
|
|
647
|
+
minimumDetectableDelta,
|
|
648
|
+
scaleAssumed,
|
|
649
|
+
reason: `observed paired uncertainty gives a minimum detectable delta of ${minimumDetectableDelta.toFixed(3)} at ${options.confidence.toFixed(2)} confidence from ${n} runs.${scorerNote}`
|
|
650
|
+
};
|
|
651
|
+
}
|
|
652
|
+
function mean(values) {
|
|
653
|
+
if (values.length === 0) throw new Error("candidate experiment requires measured values");
|
|
654
|
+
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
655
|
+
}
|
|
656
|
+
//#endregion
|
|
657
|
+
//#region src/contract/candidate-experiment.ts
|
|
658
|
+
/** Seal held-out experiments, execute their signed cells, and publish verified comparisons. */
|
|
659
|
+
/** Content-address one task before any measured execution can see it. */
|
|
660
|
+
function sealCandidateBenchmarkTask(material) {
|
|
661
|
+
return agentCandidateBenchmarkTaskSchema.parse({
|
|
662
|
+
...material,
|
|
663
|
+
digest: canonicalCandidateDigest(material)
|
|
664
|
+
});
|
|
665
|
+
}
|
|
666
|
+
/** Freeze task order, repetitions, and every seed before either arm runs. */
|
|
667
|
+
function sealCandidateBenchmarkSuite(options) {
|
|
668
|
+
for (const task of options.tasks) verifyCandidateBenchmarkTask(task);
|
|
669
|
+
const material = {
|
|
670
|
+
kind: "agent-candidate-benchmark-suite",
|
|
671
|
+
digestAlgorithm: "rfc8785-sha256",
|
|
672
|
+
taskDigests: options.tasks.map((task) => task.digest),
|
|
673
|
+
reps: options.reps,
|
|
674
|
+
seeds: options.seeds
|
|
675
|
+
};
|
|
676
|
+
return {
|
|
677
|
+
suite: agentCandidateBenchmarkSuiteSchema.parse({
|
|
678
|
+
...material,
|
|
679
|
+
digest: canonicalCandidateDigest(material)
|
|
680
|
+
}),
|
|
681
|
+
tasks: options.tasks
|
|
682
|
+
};
|
|
683
|
+
}
|
|
684
|
+
/** Freeze both complete agent states and their exact held-out work. */
|
|
685
|
+
function sealCandidateExperiment(material) {
|
|
686
|
+
return verifyCandidateExperiment(agentCandidateExperimentSchema.parse({
|
|
687
|
+
...material,
|
|
688
|
+
digest: canonicalCandidateDigest(material)
|
|
689
|
+
}));
|
|
690
|
+
}
|
|
691
|
+
function verifyCandidateExperiment(input) {
|
|
692
|
+
const experiment = agentCandidateExperimentSchema.parse(input);
|
|
693
|
+
verifySelfAddressed(experiment, "candidate experiment");
|
|
694
|
+
verifyBundle(experiment.baseline, "baseline bundle");
|
|
695
|
+
verifyBundle(experiment.candidate, "candidate bundle");
|
|
696
|
+
if (experiment.baseline.digest === experiment.candidate.digest) throw new Error("candidate experiment baseline and candidate bundles are identical");
|
|
697
|
+
verifyCandidateBenchmarkSuiteInputs(experiment.benchmark);
|
|
698
|
+
return experiment;
|
|
699
|
+
}
|
|
700
|
+
/** Execute each signed cell for both arms. The callback is Runtime's one executor. */
|
|
701
|
+
async function runCandidateExperiment(options) {
|
|
702
|
+
const experiment = verifyCandidateExperiment(options.experiment);
|
|
703
|
+
const { suite, tasks } = experiment.benchmark;
|
|
704
|
+
const run = await runPaidPairedMeasurement({
|
|
705
|
+
count: suite.taskDigests.length * suite.reps,
|
|
706
|
+
maxConcurrency: options.maxConcurrency ?? 2,
|
|
707
|
+
label: "candidate experiment",
|
|
708
|
+
budgetUsd: experiment.policy.budgetUsd,
|
|
709
|
+
...options.costLedger ? { costLedger: options.costLedger } : {},
|
|
710
|
+
maximumCostUsd: candidateMeasurementMaximumCostUsd(experiment),
|
|
711
|
+
call: {
|
|
712
|
+
callId: `candidate-measurement:${experiment.digest}`,
|
|
713
|
+
channel: "measurement",
|
|
714
|
+
phase: "heldout",
|
|
715
|
+
actor: experiment.digest,
|
|
716
|
+
model: candidateMeasurementModel(experiment),
|
|
717
|
+
tags: { experimentDigest: experiment.digest }
|
|
718
|
+
},
|
|
719
|
+
...options.signal ? { signal: options.signal } : {},
|
|
720
|
+
async execute(index, arm, signal) {
|
|
721
|
+
const taskIndex = Math.floor(index / suite.reps);
|
|
722
|
+
const repetition = index % suite.reps;
|
|
723
|
+
const task = tasks[taskIndex];
|
|
724
|
+
const seed = suite.seeds[index];
|
|
725
|
+
if (!task || seed === void 0) throw new Error(`candidate experiment cell ${index} has no signed task or seed`);
|
|
726
|
+
const benchmarkCell = {
|
|
727
|
+
suiteDigest: suite.digest,
|
|
728
|
+
taskIndex,
|
|
729
|
+
repetition
|
|
730
|
+
};
|
|
731
|
+
return verifyExecutionEvidence(await options.execute({
|
|
732
|
+
experiment,
|
|
733
|
+
arm,
|
|
734
|
+
bundle: experiment[arm],
|
|
735
|
+
task,
|
|
736
|
+
benchmarkCell,
|
|
737
|
+
seed,
|
|
738
|
+
signal
|
|
739
|
+
}));
|
|
740
|
+
},
|
|
741
|
+
receipt(measurements) {
|
|
742
|
+
return candidateMeasurementCostReceipt(measurements, candidateMeasurementModel(experiment));
|
|
743
|
+
}
|
|
744
|
+
});
|
|
745
|
+
return {
|
|
746
|
+
measurements: run.measurements.map((measurement, index) => verifyMeasurement(experiment, measurement, index)),
|
|
747
|
+
measurement: {
|
|
748
|
+
wallDurationMs: run.wallDurationMs,
|
|
749
|
+
cost: {
|
|
750
|
+
usd: run.cost.usd,
|
|
751
|
+
provenance: run.cost.kind
|
|
752
|
+
}
|
|
753
|
+
}
|
|
754
|
+
};
|
|
755
|
+
}
|
|
452
756
|
/** Build the only publishable comparison: paired statistics over Runtime receipts. */
|
|
453
757
|
function measuredComparisonFromCandidateExperiment(options) {
|
|
454
758
|
const experiment = verifyCandidateExperiment(options.experiment);
|
|
@@ -489,19 +793,7 @@ function measuredComparisonFromCandidateExperiment(options) {
|
|
|
489
793
|
candidateContentHash: experiment.candidate.digest
|
|
490
794
|
},
|
|
491
795
|
diff,
|
|
492
|
-
evaluation:
|
|
493
|
-
generationsExplored: options.generationsExplored ?? 0,
|
|
494
|
-
preparation: options.preparation,
|
|
495
|
-
measurement: {
|
|
496
|
-
wallDurationMs: options.measurement.wallDurationMs,
|
|
497
|
-
workDurationMs: evaluation.measurementWorkDurationMs,
|
|
498
|
-
cost: evaluation.measurementCost
|
|
499
|
-
},
|
|
500
|
-
total: {
|
|
501
|
-
wallDurationMs: options.preparation.wallDurationMs + options.measurement.wallDurationMs,
|
|
502
|
-
cost: evaluation.totalCost
|
|
503
|
-
}
|
|
504
|
-
},
|
|
796
|
+
evaluation: comparisonEvaluation(options, evaluation),
|
|
505
797
|
...options.metadata ? { metadata: options.metadata } : {}
|
|
506
798
|
});
|
|
507
799
|
const { recordDigest: _recordDigest, ...provenance } = provisional.provenance;
|
|
@@ -519,19 +811,7 @@ function measuredComparisonFromCandidateExperiment(options) {
|
|
|
519
811
|
/** Recompute every statistic and decision from the signed experiment receipts. */
|
|
520
812
|
function verifyCandidateExperimentComparison(input) {
|
|
521
813
|
const comparison = agentImprovementMeasuredComparisonSchema.parse(input);
|
|
522
|
-
const recomputed = measuredComparisonFromCandidateExperiment(
|
|
523
|
-
experiment: comparison.experiment,
|
|
524
|
-
measurements: comparison.measurements,
|
|
525
|
-
runId: comparison.provenance.runId,
|
|
526
|
-
...comparison.candidate ? { candidate: comparison.candidate } : {},
|
|
527
|
-
generationsExplored: comparison.evaluation.generationsExplored,
|
|
528
|
-
preparation: comparison.evaluation.preparation,
|
|
529
|
-
measurement: {
|
|
530
|
-
wallDurationMs: comparison.evaluation.measurement.wallDurationMs,
|
|
531
|
-
cost: comparison.evaluation.measurement.cost
|
|
532
|
-
},
|
|
533
|
-
...comparison.metadata ? { metadata: comparison.metadata } : {}
|
|
534
|
-
});
|
|
814
|
+
const recomputed = measuredComparisonFromCandidateExperiment(comparisonRecomputationInput(comparison));
|
|
535
815
|
if (canonicalCandidateDigest(recomputed) !== canonicalCandidateDigest(comparison)) throw new Error("candidate experiment comparison does not match its Runtime receipts");
|
|
536
816
|
return comparison;
|
|
537
817
|
}
|
|
@@ -544,34 +824,10 @@ function candidateMeasurementModel(experiment) {
|
|
|
544
824
|
const models = [...new Set(experiment.benchmark.tasks.map((task) => task.model.model))];
|
|
545
825
|
return models.length === 1 ? models[0] : "multiple-models";
|
|
546
826
|
}
|
|
547
|
-
function candidateMeasurementCostReceipt(measurements, model) {
|
|
548
|
-
const usage = measurements.reduce((sum, measurement) => addFixedSpend(addFixedSpend(sum, combinedUsage(measurement.baseline)), combinedUsage(measurement.candidate)), {
|
|
549
|
-
inputTokens: 0,
|
|
550
|
-
outputTokens: 0,
|
|
551
|
-
cachedInputTokens: 0,
|
|
552
|
-
reasoningTokens: 0,
|
|
553
|
-
modelCalls: 0,
|
|
554
|
-
costUsdNanos: 0,
|
|
555
|
-
costProvenance: "observed"
|
|
556
|
-
});
|
|
557
|
-
const costUsd = usage.costUsdNanos / 1e9;
|
|
558
|
-
return {
|
|
559
|
-
model,
|
|
560
|
-
inputTokens: usage.inputTokens,
|
|
561
|
-
outputTokens: usage.outputTokens,
|
|
562
|
-
cachedTokens: usage.cachedInputTokens,
|
|
563
|
-
reasoningTokens: usage.reasoningTokens,
|
|
564
|
-
...usage.costProvenance === "observed" ? { actualCostUsd: costUsd } : { estimatedCostUsd: costUsd }
|
|
565
|
-
};
|
|
566
|
-
}
|
|
567
827
|
function assertPhaseAccounting(accounting, label) {
|
|
568
828
|
if (!Number.isFinite(accounting.wallDurationMs) || accounting.wallDurationMs < 0) throw new Error(`${label} wall duration must be a non-negative number`);
|
|
569
829
|
assertKnownCost(accounting.cost, `${label} cost`);
|
|
570
830
|
}
|
|
571
|
-
function assertKnownCost(cost, label) {
|
|
572
|
-
if (!Number.isFinite(cost.usd) || cost.usd < 0) throw new Error(`${label} must be a non-negative number`);
|
|
573
|
-
if (cost.provenance !== "observed" && cost.provenance !== "estimated") throw new Error(`${label} provenance must be observed or estimated`);
|
|
574
|
-
}
|
|
575
831
|
function deriveCandidateBundleDiff(experiment) {
|
|
576
832
|
const changed = [
|
|
577
833
|
"profile",
|
|
@@ -625,235 +881,6 @@ function verifyBundle(input, label) {
|
|
|
625
881
|
verifySelfAddressed(bundle, label);
|
|
626
882
|
return bundle;
|
|
627
883
|
}
|
|
628
|
-
function verifyMeasurement(experiment, input, index) {
|
|
629
|
-
const suite = experiment.benchmark.suite;
|
|
630
|
-
const taskIndex = Math.floor(index / suite.reps);
|
|
631
|
-
const repetition = index % suite.reps;
|
|
632
|
-
const task = experiment.benchmark.tasks[taskIndex];
|
|
633
|
-
const seed = suite.seeds[index];
|
|
634
|
-
if (!task || seed === void 0) throw new Error(`candidate experiment measurement ${index} is outside the signed suite`);
|
|
635
|
-
const baseline = verifyExecutionEvidence(input.baseline);
|
|
636
|
-
const candidate = verifyExecutionEvidence(input.candidate);
|
|
637
|
-
for (const [arm, evidence] of [["baseline", baseline], ["candidate", candidate]]) {
|
|
638
|
-
const bundle = experiment[arm];
|
|
639
|
-
const materialization = evidence.materializationReceipt;
|
|
640
|
-
const runCell = materialization.executionPlan.material.runCell;
|
|
641
|
-
verifySelfAddressed(runCell, "candidate run cell");
|
|
642
|
-
if (runCell.experimentDigest !== experiment.digest || runCell.arm !== arm || runCell.bundleDigest !== bundle.digest || runCell.suiteDigest !== suite.digest || runCell.taskDigest !== task.digest || runCell.taskIndex !== taskIndex || runCell.repetition !== repetition || runCell.seed !== seed || runCell.attempt > task.attempt.maxAttempts || materialization.bundleDigest !== bundle.digest || materialization.benchmark.suite.digest !== suite.digest || materialization.benchmark.task.digest !== task.digest || materialization.codeKind !== bundle.code.kind || materialization.profileActivation.profilePlan.material.sourceProfileDigest !== canonicalCandidateDigest(bundle.profile) || evidence.receipt.runCellDigest !== runCell.digest || JSON.stringify(materialization.resolvedModel) !== JSON.stringify(task.model)) throw new Error(`candidate experiment measurement ${index} substituted its ${arm} arm`);
|
|
643
|
-
verifyTaskOutcome(task, evidence, index, arm);
|
|
644
|
-
}
|
|
645
|
-
const baselinePlan = baseline.materializationReceipt.executionPlan.material;
|
|
646
|
-
const candidatePlan = candidate.materializationReceipt.executionPlan.material;
|
|
647
|
-
if (baselinePlan.executionId === candidatePlan.executionId || baselinePlan.runCell.digest === candidatePlan.runCell.digest || baseline.materializationReceipt.digest === candidate.materializationReceipt.digest || baseline.receipt.digest === candidate.receipt.digest || baseline.digest === candidate.digest) throw new Error(`candidate experiment measurement ${index} reused one execution across arms`);
|
|
648
|
-
return {
|
|
649
|
-
baseline,
|
|
650
|
-
candidate
|
|
651
|
-
};
|
|
652
|
-
}
|
|
653
|
-
function verifyExecutionEvidence(input) {
|
|
654
|
-
const evidence = candidateExecutionEvidenceSchema.parse(input);
|
|
655
|
-
verifySelfAddressed(evidence, "candidate execution evidence");
|
|
656
|
-
verifySelfAddressed(evidence.materializationReceipt, "candidate materialization receipt");
|
|
657
|
-
verifySelfAddressed(evidence.materializationReceipt.profileActivation, "candidate profile activation");
|
|
658
|
-
verifyMaterialAddressed(evidence.materializationReceipt.profileActivation.profilePlan, "candidate profile plan");
|
|
659
|
-
verifyMaterialAddressed(evidence.materializationReceipt.executionPlan, "candidate execution plan");
|
|
660
|
-
verifySelfAddressed(evidence.receipt, "candidate run receipt");
|
|
661
|
-
verifyMaterialAddressed(evidence.receipt.modelSettlement, "candidate model settlement");
|
|
662
|
-
verifyMaterialAddressed(evidence.receipt.taskOutcome, "candidate task outcome");
|
|
663
|
-
verifyMaterialAddressed(evidence.receipt.benchmarkResult, "candidate benchmark result");
|
|
664
|
-
return evidence;
|
|
665
|
-
}
|
|
666
|
-
function verifySelfAddressed(document, label) {
|
|
667
|
-
if (canonicalCandidateDigest(omitTopLevelDigest(document)) !== document.digest) throw new Error(`${label} digest is invalid`);
|
|
668
|
-
}
|
|
669
|
-
function verifyTaskOutcome(task, evidence, index, arm) {
|
|
670
|
-
const outcome = evidence.receipt.taskOutcome.material.outcome;
|
|
671
|
-
const result = evidence.receipt.benchmarkResult.material;
|
|
672
|
-
const prefix = `candidate experiment measurement ${index} ${arm}`;
|
|
673
|
-
if (result.evidence.sha256 === task.grader.artifact.sha256) throw new Error(`${prefix} reused grader bytes as grading evidence`);
|
|
674
|
-
const usage = combinedUsage(evidence);
|
|
675
|
-
const usageChecks = [
|
|
676
|
-
[
|
|
677
|
-
usage.modelCalls,
|
|
678
|
-
task.limits.maxModelCalls,
|
|
679
|
-
"model calls"
|
|
680
|
-
],
|
|
681
|
-
[
|
|
682
|
-
usage.inputTokens,
|
|
683
|
-
task.limits.maxInputTokens,
|
|
684
|
-
"input tokens"
|
|
685
|
-
],
|
|
686
|
-
[
|
|
687
|
-
usage.outputTokens,
|
|
688
|
-
task.limits.maxOutputTokens,
|
|
689
|
-
"output tokens"
|
|
690
|
-
],
|
|
691
|
-
[
|
|
692
|
-
usage.costUsdNanos,
|
|
693
|
-
Math.round(task.limits.maxCostUsd * 1e9),
|
|
694
|
-
"cost"
|
|
695
|
-
]
|
|
696
|
-
];
|
|
697
|
-
for (const [actual, maximum, label] of usageChecks) if (actual > maximum) throw new Error(`${prefix} ${label} ${actual} exceeds the signed limit ${maximum}`);
|
|
698
|
-
if (outcome.kind !== task.outcome.kind) throw new Error(`${prefix} returned an outcome outside the signed task contract`);
|
|
699
|
-
if (task.outcome.kind === "output") {
|
|
700
|
-
if (outcome.kind !== "output" || outcome.spec.mediaType !== task.outcome.mediaType || outcome.spec.maxBytes !== task.outcome.maxBytes) throw new Error(`${prefix} changed the signed output contract`);
|
|
701
|
-
return;
|
|
702
|
-
}
|
|
703
|
-
const repository = task.repository;
|
|
704
|
-
if (outcome.kind !== "workspace" || repository === void 0 || outcome.baseRepository.identity !== repository.identity || outcome.baseRepository.rootIdentity !== repository.rootIdentity || outcome.baseRepository.commit !== repository.baseCommit || outcome.baseRepository.tree !== repository.baseTree) throw new Error(`${prefix} did not start from the signed repository state`);
|
|
705
|
-
}
|
|
706
|
-
function verifyStableProfileMaterialization(measurements) {
|
|
707
|
-
for (const arm of ["baseline", "candidate"]) {
|
|
708
|
-
const expected = measurements[0]?.[arm].materializationReceipt.profileActivation;
|
|
709
|
-
if (!expected) throw new Error("candidate experiment contains no profile materialization");
|
|
710
|
-
const expectedDigest = canonicalCandidateDigest({
|
|
711
|
-
profilePlanDigest: expected.profilePlan.digest,
|
|
712
|
-
files: expected.files
|
|
713
|
-
});
|
|
714
|
-
for (const [index, measurement] of measurements.entries()) {
|
|
715
|
-
const activation = measurement[arm].materializationReceipt.profileActivation;
|
|
716
|
-
if (canonicalCandidateDigest({
|
|
717
|
-
profilePlanDigest: activation.profilePlan.digest,
|
|
718
|
-
files: activation.files
|
|
719
|
-
}) !== expectedDigest) throw new Error(`candidate experiment measurement ${index} ${arm} materialized a different profile`);
|
|
720
|
-
}
|
|
721
|
-
}
|
|
722
|
-
}
|
|
723
|
-
function completedSuccessfully(evidence) {
|
|
724
|
-
const termination = evidence.receipt.termination;
|
|
725
|
-
return termination.kind === "exit" && termination.exitCode === 0;
|
|
726
|
-
}
|
|
727
|
-
function verifyMaterialAddressed(evidence, label) {
|
|
728
|
-
if (canonicalCandidateDigest(evidence.material) !== evidence.digest) throw new Error(`${label} digest is invalid`);
|
|
729
|
-
}
|
|
730
|
-
function projectPairedMeasurement(measurement, index, adapter) {
|
|
731
|
-
if (typeof measurement.cellId !== "string" || !measurement.cellId.trim()) throw new Error(`paired measurement ${index} requires a cell id`);
|
|
732
|
-
return {
|
|
733
|
-
cellId: measurement.cellId,
|
|
734
|
-
baseline: projectRun(measurement.baseline, adapter, `paired measurement ${index} baseline`),
|
|
735
|
-
candidate: projectRun(measurement.candidate, adapter, `paired measurement ${index} candidate`)
|
|
736
|
-
};
|
|
737
|
-
}
|
|
738
|
-
function projectRun(run, adapter, label) {
|
|
739
|
-
const suppliedDimensions = adapter.dimensions(run);
|
|
740
|
-
if (!Array.isArray(suppliedDimensions)) throw new Error(`${label} dimensions must be an array`);
|
|
741
|
-
const dimensions = /* @__PURE__ */ new Map();
|
|
742
|
-
for (const dimension of suppliedDimensions) {
|
|
743
|
-
if (typeof dimension.name !== "string" || !dimension.name.trim()) throw new Error(`${label} contains an unnamed dimension`);
|
|
744
|
-
if (dimensions.has(dimension.name)) throw new Error(`${label} repeats dimension '${dimension.name}'`);
|
|
745
|
-
dimensions.set(dimension.name, finiteMeasurement(dimension.score, `${label} ${dimension.name}`));
|
|
746
|
-
}
|
|
747
|
-
const completed = adapter.completed(run);
|
|
748
|
-
const passed = adapter.passed(run);
|
|
749
|
-
if (typeof completed !== "boolean" || typeof passed !== "boolean") throw new Error(`${label} completion and pass values must be booleans`);
|
|
750
|
-
const costProvenance = adapter.costProvenance(run);
|
|
751
|
-
if (costProvenance !== "observed" && costProvenance !== "estimated") throw new Error(`${label} cost provenance must be observed or estimated`);
|
|
752
|
-
return {
|
|
753
|
-
score: finiteMeasurement(adapter.score(run), `${label} score`),
|
|
754
|
-
dimensions,
|
|
755
|
-
costUsd: nonNegativeMeasurement(adapter.costUsd(run), `${label} cost`),
|
|
756
|
-
costProvenance,
|
|
757
|
-
latencyMs: nonNegativeMeasurement(adapter.latencyMs(run), `${label} latency`),
|
|
758
|
-
completed,
|
|
759
|
-
passed
|
|
760
|
-
};
|
|
761
|
-
}
|
|
762
|
-
function sharedProjectedDimensions(measurements) {
|
|
763
|
-
const expected = [...measurements[0].baseline.dimensions.keys()];
|
|
764
|
-
for (const [index, measurement] of measurements.entries()) for (const [arm, run] of [["baseline", measurement.baseline], ["candidate", measurement.candidate]]) {
|
|
765
|
-
const actual = [...run.dimensions.keys()];
|
|
766
|
-
if (JSON.stringify(actual) !== JSON.stringify(expected)) throw new Error(`paired measurement ${index} ${arm} dimensions do not match the suite`);
|
|
767
|
-
}
|
|
768
|
-
return expected;
|
|
769
|
-
}
|
|
770
|
-
function dimensionScore(run, name) {
|
|
771
|
-
const value = run.dimensions.get(name);
|
|
772
|
-
if (value === void 0) throw new Error(`paired measurement is missing dimension '${name}'`);
|
|
773
|
-
return value;
|
|
774
|
-
}
|
|
775
|
-
function measuredEstimate(baseline, candidate, options) {
|
|
776
|
-
const bootstrap = pairedBootstrap(baseline, candidate, {
|
|
777
|
-
confidence: options.confidence,
|
|
778
|
-
resamples: options.resamples,
|
|
779
|
-
statistic: "mean",
|
|
780
|
-
seed: options.seed
|
|
781
|
-
});
|
|
782
|
-
const baselineMean = mean(baseline);
|
|
783
|
-
const candidateMean = mean(candidate);
|
|
784
|
-
const delta = candidateMean - baselineMean;
|
|
785
|
-
return {
|
|
786
|
-
baseline: baselineMean,
|
|
787
|
-
candidate: candidateMean,
|
|
788
|
-
delta,
|
|
789
|
-
confidenceInterval: {
|
|
790
|
-
level: bootstrap.confidence,
|
|
791
|
-
lower: Math.min(bootstrap.low, delta),
|
|
792
|
-
upper: Math.max(bootstrap.high, delta),
|
|
793
|
-
method: "paired-bootstrap",
|
|
794
|
-
statistic: "mean",
|
|
795
|
-
resamples: bootstrap.resamples
|
|
796
|
-
},
|
|
797
|
-
n: bootstrap.n
|
|
798
|
-
};
|
|
799
|
-
}
|
|
800
|
-
function observedPairedPrecision(options) {
|
|
801
|
-
const n = options.baselineScores.length;
|
|
802
|
-
const lowerMargin = Math.max(0, options.delta - options.lowerBound);
|
|
803
|
-
const minimumDetectableDelta = options.deltaThreshold + lowerMargin;
|
|
804
|
-
const scaleAssumed = [...options.baselineScores, ...options.candidateScores].every((score) => score >= -.001 && score <= 1.001);
|
|
805
|
-
const headroom = Math.max(0, 1 - mean(options.baselineScores));
|
|
806
|
-
const enoughRuns = n >= options.minProductiveRuns;
|
|
807
|
-
const attainable = !scaleAssumed || minimumDetectableDelta <= headroom;
|
|
808
|
-
const sufficient = enoughRuns && attainable;
|
|
809
|
-
const scorerNote = options.sharedScorerChannel ? " The same scorer evaluated both arms, so scorer bias is not measured." : "";
|
|
810
|
-
if (!enoughRuns) return {
|
|
811
|
-
sufficient,
|
|
812
|
-
minimumDetectableDelta,
|
|
813
|
-
scaleAssumed,
|
|
814
|
-
reason: `only ${n} paired runs; ${options.minProductiveRuns} required.${scorerNote}`
|
|
815
|
-
};
|
|
816
|
-
if (!attainable) return {
|
|
817
|
-
sufficient,
|
|
818
|
-
minimumDetectableDelta,
|
|
819
|
-
scaleAssumed,
|
|
820
|
-
reason: `observed paired uncertainty gives a minimum detectable delta of ${minimumDetectableDelta.toFixed(3)}, above the remaining score headroom ${headroom.toFixed(3)}.${scorerNote}`
|
|
821
|
-
};
|
|
822
|
-
return {
|
|
823
|
-
sufficient,
|
|
824
|
-
minimumDetectableDelta,
|
|
825
|
-
scaleAssumed,
|
|
826
|
-
reason: `observed paired uncertainty gives a minimum detectable delta of ${minimumDetectableDelta.toFixed(3)} at ${options.confidence.toFixed(2)} confidence from ${n} runs.${scorerNote}`
|
|
827
|
-
};
|
|
828
|
-
}
|
|
829
|
-
function finiteMeasurement(value, label) {
|
|
830
|
-
if (!Number.isFinite(value)) throw new Error(`${label} must be finite`);
|
|
831
|
-
return value;
|
|
832
|
-
}
|
|
833
|
-
function nonNegativeMeasurement(value, label) {
|
|
834
|
-
if (!Number.isFinite(value) || value < 0) throw new Error(`${label} must be non-negative`);
|
|
835
|
-
return value;
|
|
836
|
-
}
|
|
837
|
-
const candidateExecutionEvidenceAdapter = {
|
|
838
|
-
score: (evidence) => evidence.receipt.benchmarkResult.material.score,
|
|
839
|
-
dimensions: (evidence) => evidence.receipt.benchmarkResult.material.dimensions,
|
|
840
|
-
costUsd: costFromEvidence,
|
|
841
|
-
costProvenance: (evidence) => combinedUsage(evidence).costProvenance,
|
|
842
|
-
latencyMs: latencyFromEvidence,
|
|
843
|
-
completed: completedSuccessfully,
|
|
844
|
-
passed: (evidence) => evidence.receipt.benchmarkResult.material.passed
|
|
845
|
-
};
|
|
846
|
-
function costFromEvidence(evidence) {
|
|
847
|
-
return combinedUsage(evidence).costUsdNanos / 1e9;
|
|
848
|
-
}
|
|
849
|
-
function latencyFromEvidence(evidence) {
|
|
850
|
-
return evidence.receipt.timing.durationMs + evidence.receipt.benchmarkResult.material.grading.timing.durationMs;
|
|
851
|
-
}
|
|
852
|
-
function combinedUsage(evidence) {
|
|
853
|
-
const candidate = evidence.receipt.modelSettlement.material.usage;
|
|
854
|
-
const grader = evidence.receipt.benchmarkResult.material.grading.usage;
|
|
855
|
-
return addFixedSpend(candidate, grader);
|
|
856
|
-
}
|
|
857
884
|
function cellIds(experiment) {
|
|
858
885
|
const { suite, tasks } = experiment.benchmark;
|
|
859
886
|
return suite.seeds.map((_, index) => {
|
|
@@ -862,10 +889,6 @@ function cellIds(experiment) {
|
|
|
862
889
|
return `${tasks[taskIndex]?.scenario.id ?? taskIndex}:${repetition}`;
|
|
863
890
|
});
|
|
864
891
|
}
|
|
865
|
-
function mean(values) {
|
|
866
|
-
if (values.length === 0) throw new Error("candidate experiment requires measured values");
|
|
867
|
-
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
868
|
-
}
|
|
869
892
|
//#endregion
|
|
870
893
|
//#region src/contract/profile-measured-comparison.ts
|
|
871
894
|
/** Content-address one held-out profile task before either state can execute it. */
|
|
@@ -991,19 +1014,7 @@ function measuredComparisonFromAgentProfileImprovementExperiment(options) {
|
|
|
991
1014
|
candidateContentHash: experiment.candidate.stateDigest
|
|
992
1015
|
},
|
|
993
1016
|
diff: canonicalCandidateJson(experiment.change),
|
|
994
|
-
evaluation:
|
|
995
|
-
generationsExplored: options.generationsExplored ?? 0,
|
|
996
|
-
preparation: options.preparation,
|
|
997
|
-
measurement: {
|
|
998
|
-
wallDurationMs: options.measurement.wallDurationMs,
|
|
999
|
-
workDurationMs: evaluation.measurementWorkDurationMs,
|
|
1000
|
-
cost: evaluation.measurementCost
|
|
1001
|
-
},
|
|
1002
|
-
total: {
|
|
1003
|
-
wallDurationMs: options.preparation.wallDurationMs + options.measurement.wallDurationMs,
|
|
1004
|
-
cost: evaluation.totalCost
|
|
1005
|
-
}
|
|
1006
|
-
},
|
|
1017
|
+
evaluation: comparisonEvaluation(options, evaluation),
|
|
1007
1018
|
...options.metadata ? { metadata: options.metadata } : {}
|
|
1008
1019
|
});
|
|
1009
1020
|
const { recordDigest: _recordDigest, ...provenance } = provisional.provenance;
|
|
@@ -1021,19 +1032,7 @@ function measuredComparisonFromAgentProfileImprovementExperiment(options) {
|
|
|
1021
1032
|
/** Recompute a profile comparison from the exact sealed experiment and receipts. */
|
|
1022
1033
|
function verifyAgentProfileImprovementExperimentComparison(input) {
|
|
1023
1034
|
const comparison = agentProfileImprovementMeasuredComparisonSchema.parse(input);
|
|
1024
|
-
const recomputed = measuredComparisonFromAgentProfileImprovementExperiment(
|
|
1025
|
-
experiment: comparison.experiment,
|
|
1026
|
-
measurements: comparison.measurements,
|
|
1027
|
-
runId: comparison.provenance.runId,
|
|
1028
|
-
...comparison.candidate ? { candidate: comparison.candidate } : {},
|
|
1029
|
-
generationsExplored: comparison.evaluation.generationsExplored,
|
|
1030
|
-
preparation: comparison.evaluation.preparation,
|
|
1031
|
-
measurement: {
|
|
1032
|
-
wallDurationMs: comparison.evaluation.measurement.wallDurationMs,
|
|
1033
|
-
cost: comparison.evaluation.measurement.cost
|
|
1034
|
-
},
|
|
1035
|
-
...comparison.metadata ? { metadata: comparison.metadata } : {}
|
|
1036
|
-
});
|
|
1035
|
+
const recomputed = measuredComparisonFromAgentProfileImprovementExperiment(comparisonRecomputationInput(comparison));
|
|
1037
1036
|
if (canonicalCandidateDigest(recomputed) !== canonicalCandidateDigest(comparison)) throw new Error("profile improvement comparison does not match its Runtime receipts");
|
|
1038
1037
|
return comparison;
|
|
1039
1038
|
}
|
|
@@ -1048,24 +1047,7 @@ function profileMeasurementModel(experiment) {
|
|
|
1048
1047
|
return models.length === 1 ? models[0] : "multiple-models";
|
|
1049
1048
|
}
|
|
1050
1049
|
function profileMeasurementCostReceipt(measurements, model) {
|
|
1051
|
-
|
|
1052
|
-
inputTokens: 0,
|
|
1053
|
-
outputTokens: 0,
|
|
1054
|
-
cachedInputTokens: 0,
|
|
1055
|
-
reasoningTokens: 0,
|
|
1056
|
-
modelCalls: 0,
|
|
1057
|
-
costUsdNanos: 0,
|
|
1058
|
-
costProvenance: "observed"
|
|
1059
|
-
});
|
|
1060
|
-
const costUsd = usage.costUsdNanos / 1e9;
|
|
1061
|
-
return {
|
|
1062
|
-
model,
|
|
1063
|
-
inputTokens: usage.inputTokens,
|
|
1064
|
-
outputTokens: usage.outputTokens,
|
|
1065
|
-
cachedTokens: usage.cachedInputTokens,
|
|
1066
|
-
reasoningTokens: usage.reasoningTokens,
|
|
1067
|
-
...usage.costProvenance === "observed" ? { actualCostUsd: costUsd } : { estimatedCostUsd: costUsd }
|
|
1068
|
-
};
|
|
1050
|
+
return pairedFixedSpendReceipt(measurements, model, combinedProfileUsage);
|
|
1069
1051
|
}
|
|
1070
1052
|
function combinedProfileUsage(receipt) {
|
|
1071
1053
|
return addFixedSpend(receipt.usage, receipt.grading.usage);
|
|
@@ -1402,10 +1384,101 @@ function partitionRunsByAuthoringModel(runs, index) {
|
|
|
1402
1384
|
};
|
|
1403
1385
|
}
|
|
1404
1386
|
//#endregion
|
|
1405
|
-
//#region src/contract/intake/code-agent-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1387
|
+
//#region src/contract/intake/code-agent-fields.ts
|
|
1388
|
+
function isRecord(value) {
|
|
1389
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
1390
|
+
}
|
|
1391
|
+
function record(value) {
|
|
1392
|
+
return isRecord(value) ? value : null;
|
|
1393
|
+
}
|
|
1394
|
+
function stringField(obj, key) {
|
|
1395
|
+
const value = obj[key];
|
|
1396
|
+
return typeof value === "string" && value.length > 0 ? value : void 0;
|
|
1397
|
+
}
|
|
1398
|
+
function numberField(obj, key) {
|
|
1399
|
+
const value = obj[key];
|
|
1400
|
+
return typeof value === "number" && Number.isFinite(value) ? value : void 0;
|
|
1401
|
+
}
|
|
1402
|
+
function timestampMs(value) {
|
|
1403
|
+
if (typeof value === "number" && Number.isFinite(value)) return value > 0xe8d4a51000 ? value : value * 1e3;
|
|
1404
|
+
if (typeof value === "string" && value.length > 0) {
|
|
1405
|
+
const parsed = Date.parse(value);
|
|
1406
|
+
return Number.isFinite(parsed) ? parsed : void 0;
|
|
1407
|
+
}
|
|
1408
|
+
}
|
|
1409
|
+
//#endregion
|
|
1410
|
+
//#region src/contract/intake/code-agent-identity.ts
|
|
1411
|
+
function sessionIdFromEntries(source, entries) {
|
|
1412
|
+
for (const entry of entries) {
|
|
1413
|
+
if (source === "pi" && entry.type === "session") {
|
|
1414
|
+
const id = stringField(entry, "id");
|
|
1415
|
+
if (id) return id;
|
|
1416
|
+
}
|
|
1417
|
+
if (source === "codex") {
|
|
1418
|
+
const threadId = stringField(entry, "thread_id");
|
|
1419
|
+
if (threadId) return threadId;
|
|
1420
|
+
const payload = record(entry.payload);
|
|
1421
|
+
const id = payload ? stringField(payload, "id") : void 0;
|
|
1422
|
+
if (id) return id;
|
|
1423
|
+
}
|
|
1424
|
+
const payload = record(record(entry.message)?.payload);
|
|
1425
|
+
const sessionId = stringField(entry, "sessionID") ?? stringField(entry, "sessionId") ?? stringField(entry, "session_id") ?? stringField(payload ?? {}, "session_id");
|
|
1426
|
+
if (sessionId) return sessionId;
|
|
1427
|
+
}
|
|
1428
|
+
}
|
|
1429
|
+
function modelFromEntries(source, entries) {
|
|
1430
|
+
if (source === "pi") return summarizeSessionInput("pi", { records: entries }).lastServedModel ?? void 0;
|
|
1431
|
+
let providerModel;
|
|
1432
|
+
for (const entry of entries) {
|
|
1433
|
+
if (source === "codex") {
|
|
1434
|
+
const payload = record(entry.payload);
|
|
1435
|
+
const model = payload ? stringField(payload, "model") : void 0;
|
|
1436
|
+
if (model) return model;
|
|
1437
|
+
providerModel = providerModel ?? (payload ? stringField(payload, "model_provider") : void 0);
|
|
1438
|
+
}
|
|
1439
|
+
const openCodeModel = stringField(entry, "modelID");
|
|
1440
|
+
if (openCodeModel) {
|
|
1441
|
+
const provider = stringField(entry, "providerID");
|
|
1442
|
+
return provider ? `${provider}/${openCodeModel}` : openCodeModel;
|
|
1443
|
+
}
|
|
1444
|
+
const modelObject = record(entry.model);
|
|
1445
|
+
const nestedModel = stringField(modelObject ?? {}, "modelID");
|
|
1446
|
+
if (nestedModel) {
|
|
1447
|
+
const provider = stringField(modelObject ?? {}, "providerID");
|
|
1448
|
+
return provider ? `${provider}/${nestedModel}` : nestedModel;
|
|
1449
|
+
}
|
|
1450
|
+
const message = record(entry.message);
|
|
1451
|
+
const model = message ? stringField(message, "model") : void 0;
|
|
1452
|
+
if (model) return model;
|
|
1453
|
+
}
|
|
1454
|
+
return providerModel;
|
|
1455
|
+
}
|
|
1456
|
+
function cwdFromEntries(entries) {
|
|
1457
|
+
for (const entry of entries) {
|
|
1458
|
+
const path = record(entry.path);
|
|
1459
|
+
const cwd = stringField(entry, "cwd") ?? stringField(path ?? {}, "cwd") ?? stringField(path ?? {}, "root") ?? (record(entry.payload) ? stringField(record(entry.payload), "cwd") : void 0);
|
|
1460
|
+
if (cwd) return cwd;
|
|
1461
|
+
}
|
|
1462
|
+
}
|
|
1463
|
+
function fallbackSessionId$1(source, sourcePath) {
|
|
1464
|
+
return hashString(`${source}:${sourcePath ?? "unknown-session"}`).slice(7, 27);
|
|
1465
|
+
}
|
|
1466
|
+
function withSnapshot(model) {
|
|
1467
|
+
const trimmed = model.trim() || "unknown";
|
|
1468
|
+
if (trimmed.includes("@") || /-\d{8}$/.test(trimmed) || /-\d{4}-\d{2}-\d{2}$/.test(trimmed)) return trimmed;
|
|
1469
|
+
return `${trimmed}@observed-local`;
|
|
1470
|
+
}
|
|
1471
|
+
function hashString(value) {
|
|
1472
|
+
return `sha256:${createHash("sha256").update(value).digest("hex")}`;
|
|
1473
|
+
}
|
|
1474
|
+
function stableSeed(value) {
|
|
1475
|
+
return createHash("sha256").update(value).digest().readUInt32BE(0);
|
|
1476
|
+
}
|
|
1477
|
+
//#endregion
|
|
1478
|
+
//#region src/contract/intake/code-agent-observation.ts
|
|
1479
|
+
/**
|
|
1480
|
+
* Where a session came from. `pi` is a native Pi session (`~/.pi/agent/sessions/*.jsonl`), read
|
|
1481
|
+
* through @tangle-network/harness-sessions; `pi-graph` is the graph IR a Pi graph run exports
|
|
1409
1482
|
* (`nodes[].ir.kind`), which is not a Pi session.
|
|
1410
1483
|
*/
|
|
1411
1484
|
/**
|
|
@@ -1414,12 +1487,12 @@ function partitionRunsByAuthoringModel(runs, index) {
|
|
|
1414
1487
|
* stay out of this projection; callers retain the source JSONL as evidence.
|
|
1415
1488
|
*/
|
|
1416
1489
|
function observeCodeAgentSession(options) {
|
|
1417
|
-
const entries = options.entries.filter(isRecord
|
|
1490
|
+
const entries = options.entries.filter(isRecord);
|
|
1418
1491
|
const projection = projectionFor(options.source, entries);
|
|
1419
1492
|
const terminal = terminalWithExecution(projection, options.execution);
|
|
1420
1493
|
return {
|
|
1421
1494
|
source: options.source,
|
|
1422
|
-
sessionId: sessionIdFromEntries
|
|
1495
|
+
sessionId: sessionIdFromEntries(options.source, entries) ?? fallbackSessionId(options.source, options.sourcePath),
|
|
1423
1496
|
finalText: projection.finalText,
|
|
1424
1497
|
terminal,
|
|
1425
1498
|
actions: projection.actions
|
|
@@ -1443,16 +1516,16 @@ function codexProjection(entries) {
|
|
|
1443
1516
|
let terminal = "unknown";
|
|
1444
1517
|
let explicitTerminal = false;
|
|
1445
1518
|
for (const entry of entries) {
|
|
1446
|
-
const entryType = stringField
|
|
1447
|
-
const payload = record
|
|
1448
|
-
const payloadType = stringField
|
|
1449
|
-
const timestampMs =
|
|
1519
|
+
const entryType = stringField(entry, "type");
|
|
1520
|
+
const payload = record(entry.payload) ?? {};
|
|
1521
|
+
const payloadType = stringField(payload, "type");
|
|
1522
|
+
const timestampMs$1 = timestampMs(entry.timestamp);
|
|
1450
1523
|
const itemEvent = admittedCodexItem(entry, transcriptStream);
|
|
1451
1524
|
const item = itemEvent?.item ?? null;
|
|
1452
1525
|
if (itemEvent?.eventType === "item.started" && item) {
|
|
1453
|
-
const itemType = stringField
|
|
1526
|
+
const itemType = stringField(item, "type");
|
|
1454
1527
|
if (isCodexActionItem(itemType)) {
|
|
1455
|
-
const id = stringField
|
|
1528
|
+
const id = stringField(item, "id") ?? `item-${actions.length}`;
|
|
1456
1529
|
const action = actionFor({
|
|
1457
1530
|
id,
|
|
1458
1531
|
stepIndex: actions.length,
|
|
@@ -1460,7 +1533,7 @@ function codexProjection(entries) {
|
|
|
1460
1533
|
surface: codexItemSurface(itemType, item),
|
|
1461
1534
|
name: codexItemName(itemType, item),
|
|
1462
1535
|
status: "started",
|
|
1463
|
-
timestampMs,
|
|
1536
|
+
timestampMs: timestampMs$1,
|
|
1464
1537
|
metadata: compactMetadata({
|
|
1465
1538
|
sourceEventType: itemEvent.eventType,
|
|
1466
1539
|
itemType
|
|
@@ -1471,10 +1544,10 @@ function codexProjection(entries) {
|
|
|
1471
1544
|
}
|
|
1472
1545
|
}
|
|
1473
1546
|
if (itemEvent?.eventType === "item.completed" && item) {
|
|
1474
|
-
const itemType = stringField
|
|
1475
|
-
if (itemType === "agent_message" || itemType === "message") finalText = nonEmpty(stringField
|
|
1547
|
+
const itemType = stringField(item, "type");
|
|
1548
|
+
if (itemType === "agent_message" || itemType === "message") finalText = nonEmpty(stringField(item, "text")) ?? finalText;
|
|
1476
1549
|
if (isCodexActionItem(itemType)) {
|
|
1477
|
-
const id = stringField
|
|
1550
|
+
const id = stringField(item, "id") ?? `item-${actions.length}`;
|
|
1478
1551
|
const status = codexItemStatus(itemType, item);
|
|
1479
1552
|
const existing = calls.get(id);
|
|
1480
1553
|
if (existing) existing.status = status;
|
|
@@ -1486,7 +1559,7 @@ function codexProjection(entries) {
|
|
|
1486
1559
|
surface: codexItemSurface(itemType, item),
|
|
1487
1560
|
name: codexItemName(itemType, item),
|
|
1488
1561
|
status,
|
|
1489
|
-
timestampMs,
|
|
1562
|
+
timestampMs: timestampMs$1,
|
|
1490
1563
|
metadata: compactMetadata({
|
|
1491
1564
|
sourceEventType: itemEvent.eventType,
|
|
1492
1565
|
itemType
|
|
@@ -1500,8 +1573,8 @@ function codexProjection(entries) {
|
|
|
1500
1573
|
if (entryType === "response_item") {
|
|
1501
1574
|
if (payloadType === "message" || payloadType === "agent_message") finalText = textFromMessagePayload(payload) ?? finalText;
|
|
1502
1575
|
if (payloadType === "function_call" || payloadType === "custom_tool_call") {
|
|
1503
|
-
const id = stringField
|
|
1504
|
-
const name = stringField
|
|
1576
|
+
const id = stringField(payload, "call_id") ?? stringField(payload, "id") ?? `call-${actions.length}`;
|
|
1577
|
+
const name = stringField(payload, "name") ?? payloadType;
|
|
1505
1578
|
const action = actionFor({
|
|
1506
1579
|
id,
|
|
1507
1580
|
stepIndex: actions.length,
|
|
@@ -1509,7 +1582,7 @@ function codexProjection(entries) {
|
|
|
1509
1582
|
surface: surfaceForTool(name),
|
|
1510
1583
|
name,
|
|
1511
1584
|
status: "started",
|
|
1512
|
-
timestampMs,
|
|
1585
|
+
timestampMs: timestampMs$1,
|
|
1513
1586
|
metadata: compactMetadata({
|
|
1514
1587
|
sourceEventType: entryType,
|
|
1515
1588
|
payloadType
|
|
@@ -1519,23 +1592,23 @@ function codexProjection(entries) {
|
|
|
1519
1592
|
actions.push(action);
|
|
1520
1593
|
}
|
|
1521
1594
|
if (payloadType === "function_call_output" || payloadType === "custom_tool_call_output") {
|
|
1522
|
-
const id = stringField
|
|
1595
|
+
const id = stringField(payload, "call_id") ?? stringField(payload, "id");
|
|
1523
1596
|
const existing = id ? calls.get(id) : void 0;
|
|
1524
1597
|
if (existing) existing.status = looksLikeError(payload.output) ? "failed" : "completed";
|
|
1525
1598
|
}
|
|
1526
1599
|
}
|
|
1527
1600
|
if (entryType === "event_msg") {
|
|
1528
|
-
if (payloadType === "agent_message") finalText = nonEmpty(stringField
|
|
1601
|
+
if (payloadType === "agent_message") finalText = nonEmpty(stringField(payload, "message")) ?? nonEmpty(stringField(payload, "text")) ?? finalText;
|
|
1529
1602
|
if (payloadType === "patch_apply_end") {
|
|
1530
1603
|
const patchStatus = typeof payload.success === "boolean" ? payload.success ? "completed" : "failed" : "unknown";
|
|
1531
1604
|
actions.push(actionFor({
|
|
1532
|
-
id: stringField
|
|
1605
|
+
id: stringField(payload, "call_id") ?? `patch-${actions.length}`,
|
|
1533
1606
|
stepIndex: actions.length,
|
|
1534
1607
|
kind: "patch",
|
|
1535
1608
|
surface: "code",
|
|
1536
1609
|
name: "patch",
|
|
1537
1610
|
status: patchStatus,
|
|
1538
|
-
timestampMs,
|
|
1611
|
+
timestampMs: timestampMs$1,
|
|
1539
1612
|
metadata: compactMetadata({
|
|
1540
1613
|
sourceEventType: entryType,
|
|
1541
1614
|
payloadType
|
|
@@ -1543,15 +1616,15 @@ function codexProjection(entries) {
|
|
|
1543
1616
|
}));
|
|
1544
1617
|
}
|
|
1545
1618
|
if (payloadType === "sub_agent_activity") {
|
|
1546
|
-
const name = stringField
|
|
1619
|
+
const name = stringField(payload, "kind") ?? "subagent";
|
|
1547
1620
|
actions.push(actionFor({
|
|
1548
|
-
id: stringField
|
|
1621
|
+
id: stringField(payload, "event_id") ?? `subagent-${actions.length}`,
|
|
1549
1622
|
stepIndex: actions.length,
|
|
1550
1623
|
kind: "tool",
|
|
1551
1624
|
surface: "subagent",
|
|
1552
1625
|
name,
|
|
1553
1626
|
status: statusFrom(payload),
|
|
1554
|
-
timestampMs,
|
|
1627
|
+
timestampMs: timestampMs$1,
|
|
1555
1628
|
metadata: compactMetadata({
|
|
1556
1629
|
sourceEventType: entryType,
|
|
1557
1630
|
payloadType
|
|
@@ -1562,20 +1635,20 @@ function codexProjection(entries) {
|
|
|
1562
1635
|
const completed = payloadType === "task_complete";
|
|
1563
1636
|
terminal = completed ? "completed" : "failed";
|
|
1564
1637
|
explicitTerminal = true;
|
|
1565
|
-
actions.push(terminalAction(actions.length, payloadType, completed, timestampMs));
|
|
1638
|
+
actions.push(terminalAction(actions.length, payloadType, completed, timestampMs$1));
|
|
1566
1639
|
}
|
|
1567
1640
|
}
|
|
1568
1641
|
if (entryType === "turn.completed" || entryType === "exec_done") {
|
|
1569
1642
|
const completed = entry.success !== false;
|
|
1570
1643
|
terminal = completed ? "completed" : "failed";
|
|
1571
1644
|
explicitTerminal = true;
|
|
1572
|
-
finalText = nonEmpty(stringField
|
|
1573
|
-
actions.push(terminalAction(actions.length, entryType, completed, timestampMs));
|
|
1645
|
+
finalText = nonEmpty(stringField(entry, "message")) ?? finalText;
|
|
1646
|
+
actions.push(terminalAction(actions.length, entryType, completed, timestampMs$1));
|
|
1574
1647
|
}
|
|
1575
1648
|
if (entryType === "turn.failed" || entryType === "error") {
|
|
1576
1649
|
terminal = "failed";
|
|
1577
1650
|
explicitTerminal = true;
|
|
1578
|
-
actions.push(terminalAction(actions.length, entryType, false, timestampMs));
|
|
1651
|
+
actions.push(terminalAction(actions.length, entryType, false, timestampMs$1));
|
|
1579
1652
|
}
|
|
1580
1653
|
}
|
|
1581
1654
|
return {
|
|
@@ -1593,20 +1666,20 @@ function claudeProjection(entries) {
|
|
|
1593
1666
|
let terminal = "unknown";
|
|
1594
1667
|
let explicitTerminal = false;
|
|
1595
1668
|
for (const entry of entries) {
|
|
1596
|
-
const entryType = stringField
|
|
1597
|
-
const timestampMs =
|
|
1598
|
-
const message = record
|
|
1669
|
+
const entryType = stringField(entry, "type");
|
|
1670
|
+
const timestampMs$2 = timestampMs(entry.timestamp);
|
|
1671
|
+
const message = record(entry.message);
|
|
1599
1672
|
const content = Array.isArray(message?.content) ? message.content : [];
|
|
1600
1673
|
if (entryType === "assistant") {
|
|
1601
1674
|
const text = textBlocks(content);
|
|
1602
1675
|
if (text !== null) finalText = text;
|
|
1603
1676
|
for (const value of content) {
|
|
1604
|
-
const part = record
|
|
1677
|
+
const part = record(value);
|
|
1605
1678
|
if (!part) continue;
|
|
1606
|
-
const partType = stringField
|
|
1679
|
+
const partType = stringField(part, "type");
|
|
1607
1680
|
if (partType === "tool_use") {
|
|
1608
|
-
const id = stringField
|
|
1609
|
-
const name = stringField
|
|
1681
|
+
const id = stringField(part, "id") ?? `tool-${actions.length}`;
|
|
1682
|
+
const name = stringField(part, "name") ?? "tool";
|
|
1610
1683
|
const action = actionFor({
|
|
1611
1684
|
id,
|
|
1612
1685
|
stepIndex: actions.length,
|
|
@@ -1614,7 +1687,7 @@ function claudeProjection(entries) {
|
|
|
1614
1687
|
surface: surfaceForTool(name),
|
|
1615
1688
|
name,
|
|
1616
1689
|
status: "started",
|
|
1617
|
-
timestampMs,
|
|
1690
|
+
timestampMs: timestampMs$2,
|
|
1618
1691
|
metadata: compactMetadata({
|
|
1619
1692
|
sourceEventType: entryType,
|
|
1620
1693
|
partType
|
|
@@ -1625,17 +1698,17 @@ function claudeProjection(entries) {
|
|
|
1625
1698
|
}
|
|
1626
1699
|
}
|
|
1627
1700
|
completeClaudeTools(content, calls);
|
|
1628
|
-
const stopReason = stringField
|
|
1701
|
+
const stopReason = stringField(message ?? {}, "stop_reason");
|
|
1629
1702
|
if (stopReason === "end_turn") {
|
|
1630
1703
|
terminal = "completed";
|
|
1631
1704
|
explicitTerminal = true;
|
|
1632
|
-
actions.push(terminalAction(actions.length, stopReason, true, timestampMs));
|
|
1705
|
+
actions.push(terminalAction(actions.length, stopReason, true, timestampMs$2));
|
|
1633
1706
|
}
|
|
1634
1707
|
}
|
|
1635
1708
|
if (entryType === "user") completeClaudeTools(content, calls);
|
|
1636
1709
|
if (entryType === "tool_use") {
|
|
1637
|
-
const id = stringField
|
|
1638
|
-
const name = stringField
|
|
1710
|
+
const id = stringField(entry, "id") ?? `tool-${actions.length}`;
|
|
1711
|
+
const name = stringField(entry, "name") ?? "tool";
|
|
1639
1712
|
const action = actionFor({
|
|
1640
1713
|
id,
|
|
1641
1714
|
stepIndex: actions.length,
|
|
@@ -1643,47 +1716,47 @@ function claudeProjection(entries) {
|
|
|
1643
1716
|
surface: surfaceForTool(name),
|
|
1644
1717
|
name,
|
|
1645
1718
|
status: "started",
|
|
1646
|
-
timestampMs,
|
|
1719
|
+
timestampMs: timestampMs$2,
|
|
1647
1720
|
metadata: { sourceEventType: entryType }
|
|
1648
1721
|
});
|
|
1649
1722
|
calls.set(id, action);
|
|
1650
1723
|
actions.push(action);
|
|
1651
1724
|
}
|
|
1652
1725
|
if (entryType === "tool_result") {
|
|
1653
|
-
const id = stringField
|
|
1726
|
+
const id = stringField(entry, "tool_use_id");
|
|
1654
1727
|
const action = id ? calls.get(id) : void 0;
|
|
1655
1728
|
if (action) action.status = entry.is_error === true ? "failed" : "completed";
|
|
1656
1729
|
}
|
|
1657
1730
|
if (entryType === "system") {
|
|
1658
|
-
const subtype = stringField
|
|
1659
|
-
const hookName = stringField
|
|
1731
|
+
const subtype = stringField(entry, "subtype");
|
|
1732
|
+
const hookName = stringField(entry, "hook_name") ?? stringField(entry, "hookName");
|
|
1660
1733
|
if (subtype?.toLowerCase().includes("hook") || hookName) actions.push(actionFor({
|
|
1661
|
-
id: stringField
|
|
1734
|
+
id: stringField(entry, "uuid") ?? `hook-${actions.length}`,
|
|
1662
1735
|
stepIndex: actions.length,
|
|
1663
1736
|
kind: "tool",
|
|
1664
1737
|
surface: "hook",
|
|
1665
1738
|
name: hookName ?? subtype ?? "hook",
|
|
1666
1739
|
status: statusFrom(entry),
|
|
1667
|
-
timestampMs,
|
|
1740
|
+
timestampMs: timestampMs$2,
|
|
1668
1741
|
metadata: compactMetadata({
|
|
1669
1742
|
sourceEventType: entryType,
|
|
1670
1743
|
subtype
|
|
1671
1744
|
})
|
|
1672
1745
|
}));
|
|
1673
1746
|
}
|
|
1674
|
-
if (entryType === "pr-link") actions.push(terminalAction(actions.length, entryType, true, timestampMs));
|
|
1747
|
+
if (entryType === "pr-link") actions.push(terminalAction(actions.length, entryType, true, timestampMs$2));
|
|
1675
1748
|
if (entryType === "result") {
|
|
1676
|
-
resultText = nonEmpty(stringField
|
|
1677
|
-
const subtype = stringField
|
|
1749
|
+
resultText = nonEmpty(stringField(entry, "result")) ?? resultText;
|
|
1750
|
+
const subtype = stringField(entry, "subtype");
|
|
1678
1751
|
const completed = entry.is_error !== true && !subtype?.toLowerCase().includes("error");
|
|
1679
1752
|
terminal = completed ? "completed" : "failed";
|
|
1680
1753
|
explicitTerminal = true;
|
|
1681
|
-
actions.push(terminalAction(actions.length, subtype ?? entryType, completed, timestampMs));
|
|
1754
|
+
actions.push(terminalAction(actions.length, subtype ?? entryType, completed, timestampMs$2));
|
|
1682
1755
|
}
|
|
1683
1756
|
if (entryType === "error") {
|
|
1684
1757
|
terminal = "failed";
|
|
1685
1758
|
explicitTerminal = true;
|
|
1686
|
-
actions.push(terminalAction(actions.length, entryType, false, timestampMs));
|
|
1759
|
+
actions.push(terminalAction(actions.length, entryType, false, timestampMs$2));
|
|
1687
1760
|
}
|
|
1688
1761
|
}
|
|
1689
1762
|
return {
|
|
@@ -1695,28 +1768,28 @@ function claudeProjection(entries) {
|
|
|
1695
1768
|
}
|
|
1696
1769
|
function openCodeProjection(entries) {
|
|
1697
1770
|
const actions = [];
|
|
1698
|
-
const assistantMessageIds = new Set(entries.filter((entry) => stringField
|
|
1771
|
+
const assistantMessageIds = new Set(entries.filter((entry) => stringField(entry, "role") === "assistant").map((entry) => stringField(entry, "id")).filter((id) => id !== void 0));
|
|
1699
1772
|
let finalText = null;
|
|
1700
1773
|
let terminal = "unknown";
|
|
1701
1774
|
let explicitTerminal = false;
|
|
1702
1775
|
for (const entry of entries) {
|
|
1703
|
-
const entryType = stringField
|
|
1704
|
-
const part = record
|
|
1705
|
-
const partType = stringField
|
|
1706
|
-
const timestampMs =
|
|
1707
|
-
if (entryType === "text" && record
|
|
1708
|
-
else if (partType === "text" && (assistantMessageIds.size === 0 || assistantMessageIds.has(stringField
|
|
1776
|
+
const entryType = stringField(entry, "type");
|
|
1777
|
+
const part = record(entry.part) ?? entry;
|
|
1778
|
+
const partType = stringField(part, "type");
|
|
1779
|
+
const timestampMs$3 = timestampMs(entry.timestamp) ?? timestampMs(record(entry.time)?.created);
|
|
1780
|
+
if (entryType === "text" && record(entry.part)) finalText = nonEmpty(stringField(part, "text")) ?? finalText;
|
|
1781
|
+
else if (partType === "text" && (assistantMessageIds.size === 0 || assistantMessageIds.has(stringField(part, "messageID") ?? ""))) finalText = nonEmpty(stringField(part, "text")) ?? finalText;
|
|
1709
1782
|
if (entryType === "tool_use" || partType === "tool") {
|
|
1710
|
-
const state = record
|
|
1711
|
-
const name = stringField
|
|
1783
|
+
const state = record(part.state) ?? {};
|
|
1784
|
+
const name = stringField(part, "tool") ?? "tool";
|
|
1712
1785
|
actions.push(actionFor({
|
|
1713
|
-
id: stringField
|
|
1786
|
+
id: stringField(part, "callID") ?? stringField(part, "id") ?? `tool-${actions.length}`,
|
|
1714
1787
|
stepIndex: actions.length,
|
|
1715
1788
|
kind: "tool",
|
|
1716
1789
|
surface: surfaceForTool(name),
|
|
1717
1790
|
name,
|
|
1718
1791
|
status: statusFrom(state),
|
|
1719
|
-
timestampMs,
|
|
1792
|
+
timestampMs: timestampMs$3,
|
|
1720
1793
|
metadata: compactMetadata({
|
|
1721
1794
|
sourceEventType: entryType,
|
|
1722
1795
|
partType
|
|
@@ -1724,31 +1797,31 @@ function openCodeProjection(entries) {
|
|
|
1724
1797
|
}));
|
|
1725
1798
|
}
|
|
1726
1799
|
if (partType === "patch") actions.push(actionFor({
|
|
1727
|
-
id: stringField
|
|
1800
|
+
id: stringField(part, "id") ?? `patch-${actions.length}`,
|
|
1728
1801
|
stepIndex: actions.length,
|
|
1729
1802
|
kind: "patch",
|
|
1730
1803
|
surface: "code",
|
|
1731
1804
|
name: "patch",
|
|
1732
1805
|
status: "completed",
|
|
1733
|
-
timestampMs,
|
|
1806
|
+
timestampMs: timestampMs$3,
|
|
1734
1807
|
metadata: compactMetadata({
|
|
1735
1808
|
sourceEventType: entryType,
|
|
1736
1809
|
partType
|
|
1737
1810
|
})
|
|
1738
1811
|
}));
|
|
1739
|
-
if (stringField
|
|
1740
|
-
const finish = stringField
|
|
1812
|
+
if (stringField(entry, "role") === "assistant") {
|
|
1813
|
+
const finish = stringField(entry, "finish");
|
|
1741
1814
|
if (finish === "stop" || finish === "error") {
|
|
1742
1815
|
const completed = finish === "stop";
|
|
1743
1816
|
terminal = completed ? "completed" : "failed";
|
|
1744
1817
|
explicitTerminal = true;
|
|
1745
|
-
actions.push(terminalAction(actions.length, finish, completed, timestampMs));
|
|
1818
|
+
actions.push(terminalAction(actions.length, finish, completed, timestampMs$3));
|
|
1746
1819
|
}
|
|
1747
1820
|
}
|
|
1748
1821
|
if (entryType === "error") {
|
|
1749
1822
|
terminal = "failed";
|
|
1750
1823
|
explicitTerminal = true;
|
|
1751
|
-
actions.push(terminalAction(actions.length, entryType, false, timestampMs));
|
|
1824
|
+
actions.push(terminalAction(actions.length, entryType, false, timestampMs$3));
|
|
1752
1825
|
}
|
|
1753
1826
|
}
|
|
1754
1827
|
return {
|
|
@@ -1765,15 +1838,15 @@ function kimiProjection(entries) {
|
|
|
1765
1838
|
let terminal = "unknown";
|
|
1766
1839
|
let explicitTerminal = false;
|
|
1767
1840
|
for (const entry of entries) {
|
|
1768
|
-
const timestampMs =
|
|
1769
|
-
const message = record
|
|
1770
|
-
const messageType = stringField
|
|
1771
|
-
const payload = record
|
|
1772
|
-
if (messageType === "ContentPart" && stringField
|
|
1841
|
+
const timestampMs$4 = timestampMs(entry.timestamp);
|
|
1842
|
+
const message = record(entry.message);
|
|
1843
|
+
const messageType = stringField(message ?? {}, "type");
|
|
1844
|
+
const payload = record(message?.payload) ?? {};
|
|
1845
|
+
if (messageType === "ContentPart" && stringField(payload, "type") === "text") finalText = nonEmpty(stringField(payload, "text")) ?? nonEmpty(stringField(payload, "content")) ?? finalText;
|
|
1773
1846
|
if (messageType === "ToolCall") {
|
|
1774
|
-
const fn = record
|
|
1775
|
-
const id = stringField
|
|
1776
|
-
const name = stringField
|
|
1847
|
+
const fn = record(payload.function) ?? {};
|
|
1848
|
+
const id = stringField(payload, "id") ?? `tool-${actions.length}`;
|
|
1849
|
+
const name = stringField(fn, "name") ?? "tool";
|
|
1777
1850
|
const action = actionFor({
|
|
1778
1851
|
id,
|
|
1779
1852
|
stepIndex: actions.length,
|
|
@@ -1781,22 +1854,22 @@ function kimiProjection(entries) {
|
|
|
1781
1854
|
surface: surfaceForTool(name),
|
|
1782
1855
|
name,
|
|
1783
1856
|
status: "started",
|
|
1784
|
-
timestampMs,
|
|
1857
|
+
timestampMs: timestampMs$4,
|
|
1785
1858
|
metadata: { sourceEventType: messageType }
|
|
1786
1859
|
});
|
|
1787
1860
|
calls.set(id, action);
|
|
1788
1861
|
actions.push(action);
|
|
1789
1862
|
}
|
|
1790
1863
|
if (messageType === "ToolResult") {
|
|
1791
|
-
const id = stringField
|
|
1864
|
+
const id = stringField(payload, "tool_call_id");
|
|
1792
1865
|
const action = id ? calls.get(id) : void 0;
|
|
1793
|
-
if (action) action.status = record
|
|
1866
|
+
if (action) action.status = record(payload.return_value)?.is_error === true ? "failed" : "completed";
|
|
1794
1867
|
}
|
|
1795
1868
|
if (messageType === "TurnEnd" || messageType === "StepInterrupted") {
|
|
1796
1869
|
const completed = messageType === "TurnEnd";
|
|
1797
1870
|
terminal = completed ? "completed" : "failed";
|
|
1798
1871
|
explicitTerminal = true;
|
|
1799
|
-
actions.push(terminalAction(actions.length, messageType, completed, timestampMs));
|
|
1872
|
+
actions.push(terminalAction(actions.length, messageType, completed, timestampMs$4));
|
|
1800
1873
|
}
|
|
1801
1874
|
}
|
|
1802
1875
|
return {
|
|
@@ -1816,7 +1889,7 @@ function piSessionProjection(entries) {
|
|
|
1816
1889
|
surface: surfaceForTool(call.name),
|
|
1817
1890
|
name: call.name,
|
|
1818
1891
|
status: call.status === "completed" ? "completed" : call.status === "error" ? "failed" : call.status === "pending" ? "started" : "unknown",
|
|
1819
|
-
timestampMs:
|
|
1892
|
+
timestampMs: timestampMs(call.startedAt),
|
|
1820
1893
|
metadata: { sourceEventType: "pi-tool-call" }
|
|
1821
1894
|
}));
|
|
1822
1895
|
const answer = [...session.messages].reverse().find((m) => m.role === "assistant" && m.actor === "agent" && m.parts.some((p) => p.type === "text"));
|
|
@@ -1836,17 +1909,17 @@ function piGraphProjection(entries) {
|
|
|
1836
1909
|
for (const entry of entries) {
|
|
1837
1910
|
const nodes = Array.isArray(entry.nodes) ? entry.nodes : [];
|
|
1838
1911
|
for (const value of nodes) {
|
|
1839
|
-
const node = record
|
|
1840
|
-
const ir = record
|
|
1841
|
-
const kind = stringField
|
|
1912
|
+
const node = record(value);
|
|
1913
|
+
const ir = record(node?.ir) ?? node;
|
|
1914
|
+
const kind = stringField(ir ?? {}, "kind");
|
|
1842
1915
|
if (kind === "ToolInvocation") actions.push(actionFor({
|
|
1843
|
-
id: stringField
|
|
1916
|
+
id: stringField(ir ?? {}, "id") ?? stringField(node ?? {}, "id") ?? `tool-${actions.length}`,
|
|
1844
1917
|
stepIndex: actions.length,
|
|
1845
1918
|
kind: "tool",
|
|
1846
1919
|
surface: "tool",
|
|
1847
|
-
name: stringField
|
|
1920
|
+
name: stringField(ir ?? {}, "name") ?? "graph-tool",
|
|
1848
1921
|
status: "started",
|
|
1849
|
-
timestampMs:
|
|
1922
|
+
timestampMs: timestampMs(ir?.createdAt),
|
|
1850
1923
|
metadata: {
|
|
1851
1924
|
sourceEventType: "graph-node",
|
|
1852
1925
|
graphKind: kind
|
|
@@ -1860,13 +1933,13 @@ function piGraphProjection(entries) {
|
|
|
1860
1933
|
terminal = "completed";
|
|
1861
1934
|
explicitTerminal = true;
|
|
1862
1935
|
actions.push(actionFor({
|
|
1863
|
-
id: stringField
|
|
1936
|
+
id: stringField(ir ?? {}, "id") ?? stringField(node ?? {}, "id") ?? `completion-${actions.length}`,
|
|
1864
1937
|
stepIndex: actions.length,
|
|
1865
1938
|
kind: "graph-completion",
|
|
1866
1939
|
surface: "tool",
|
|
1867
1940
|
name: "complete",
|
|
1868
1941
|
status: "completed",
|
|
1869
|
-
timestampMs:
|
|
1942
|
+
timestampMs: timestampMs(ir?.createdAt),
|
|
1870
1943
|
metadata: {
|
|
1871
1944
|
sourceEventType: "graph-node",
|
|
1872
1945
|
graphKind: kind
|
|
@@ -1884,9 +1957,9 @@ function piGraphProjection(entries) {
|
|
|
1884
1957
|
}
|
|
1885
1958
|
function completeClaudeTools(content, calls) {
|
|
1886
1959
|
for (const value of content) {
|
|
1887
|
-
const part = record
|
|
1888
|
-
if (!part || stringField
|
|
1889
|
-
const id = stringField
|
|
1960
|
+
const part = record(value);
|
|
1961
|
+
if (!part || stringField(part, "type") !== "tool_result") continue;
|
|
1962
|
+
const id = stringField(part, "tool_use_id");
|
|
1890
1963
|
const action = id ? calls.get(id) : void 0;
|
|
1891
1964
|
if (action) action.status = part.is_error === true ? "failed" : "completed";
|
|
1892
1965
|
}
|
|
@@ -1900,21 +1973,21 @@ function completeClaudeTools(content, calls) {
|
|
|
1900
1973
|
* PascalCase. Read both here so one branch serves both transports.
|
|
1901
1974
|
*/
|
|
1902
1975
|
function codexItemEvent(entry) {
|
|
1903
|
-
const entryType = stringField
|
|
1976
|
+
const entryType = stringField(entry, "type");
|
|
1904
1977
|
if (entryType === "item.started" || entryType === "item.completed") {
|
|
1905
|
-
const item = record
|
|
1978
|
+
const item = record(entry.item);
|
|
1906
1979
|
return item ? {
|
|
1907
1980
|
eventType: entryType,
|
|
1908
1981
|
item
|
|
1909
1982
|
} : null;
|
|
1910
1983
|
}
|
|
1911
1984
|
if (entryType !== "event_msg") return null;
|
|
1912
|
-
const payload = record
|
|
1913
|
-
const payloadType = payload ? stringField
|
|
1985
|
+
const payload = record(entry.payload);
|
|
1986
|
+
const payloadType = payload ? stringField(payload, "type") : void 0;
|
|
1914
1987
|
if (payloadType !== "item_started" && payloadType !== "item_completed") return null;
|
|
1915
|
-
const item = record
|
|
1988
|
+
const item = record(payload?.item);
|
|
1916
1989
|
if (!item) return null;
|
|
1917
|
-
const itemType = codexItemType(stringField
|
|
1990
|
+
const itemType = codexItemType(stringField(item, "type"));
|
|
1918
1991
|
return {
|
|
1919
1992
|
eventType: payloadType === "item_started" ? "item.started" : "item.completed",
|
|
1920
1993
|
item: itemType === void 0 ? item : {
|
|
@@ -1934,7 +2007,7 @@ function codexItemType(itemType) {
|
|
|
1934
2007
|
return itemType.replace(/([a-z0-9])([A-Z])/g, "$1_$2").toLowerCase();
|
|
1935
2008
|
}
|
|
1936
2009
|
function hasCodexTranscriptStream(entries) {
|
|
1937
|
-
return entries.some((entry) => stringField
|
|
2010
|
+
return entries.some((entry) => stringField(entry, "type") === "response_item");
|
|
1938
2011
|
}
|
|
1939
2012
|
/**
|
|
1940
2013
|
* A rollout file records each turn twice: `response_item` holds the model
|
|
@@ -1962,7 +2035,7 @@ function admittedCodexItem(entry, transcriptStream) {
|
|
|
1962
2035
|
const event = codexItemEvent(entry);
|
|
1963
2036
|
if (!event) return null;
|
|
1964
2037
|
if (!transcriptStream) return event;
|
|
1965
|
-
return isCodexExecutionOnlyItem(stringField
|
|
2038
|
+
return isCodexExecutionOnlyItem(stringField(event.item, "type")) ? event : null;
|
|
1966
2039
|
}
|
|
1967
2040
|
function isCodexActionItem(itemType) {
|
|
1968
2041
|
return itemType === "command_execution" || itemType === "mcp_tool_call" || itemType === "collab_tool_call" || itemType === "web_search" || itemType === "file_change";
|
|
@@ -1972,27 +2045,27 @@ function codexItemSurface(itemType, item) {
|
|
|
1972
2045
|
if (itemType === "collab_tool_call") return "subagent";
|
|
1973
2046
|
if (itemType === "web_search") return "web";
|
|
1974
2047
|
if (itemType === "file_change") return "code";
|
|
1975
|
-
return surfaceForTool(stringField
|
|
2048
|
+
return surfaceForTool(stringField(item, "name") ?? itemType ?? "tool");
|
|
1976
2049
|
}
|
|
1977
2050
|
function codexItemName(itemType, item) {
|
|
1978
|
-
if (itemType === "mcp_tool_call") return [stringField
|
|
1979
|
-
if (itemType === "collab_tool_call") return stringField
|
|
2051
|
+
if (itemType === "mcp_tool_call") return [stringField(item, "server"), stringField(item, "tool")].filter(Boolean).join("/") || "mcp";
|
|
2052
|
+
if (itemType === "collab_tool_call") return stringField(item, "tool") ?? "subagent";
|
|
1980
2053
|
if (itemType === "web_search") return "web_search";
|
|
1981
2054
|
if (itemType === "file_change") return "file_change";
|
|
1982
|
-
return stringField
|
|
2055
|
+
return stringField(item, "name") ?? itemType ?? "tool";
|
|
1983
2056
|
}
|
|
1984
2057
|
function codexItemStatus(itemType, item) {
|
|
1985
2058
|
const status = statusFrom(item);
|
|
1986
2059
|
if (status === "failed") return status;
|
|
1987
2060
|
if (itemType === "command_execution") {
|
|
1988
|
-
const exitCode = numberField
|
|
2061
|
+
const exitCode = numberField(item, "exit_code");
|
|
1989
2062
|
if (exitCode !== void 0) return exitCode === 0 ? "completed" : "failed";
|
|
1990
2063
|
}
|
|
1991
|
-
if (itemType === "mcp_tool_call" && record
|
|
2064
|
+
if (itemType === "mcp_tool_call" && record(item.error)) return "failed";
|
|
1992
2065
|
return status === "unknown" || status === "started" ? "completed" : status;
|
|
1993
2066
|
}
|
|
1994
2067
|
function textFromMessagePayload(payload) {
|
|
1995
|
-
const direct = nonEmpty(stringField
|
|
2068
|
+
const direct = nonEmpty(stringField(payload, "text"));
|
|
1996
2069
|
if (direct) return direct;
|
|
1997
2070
|
const content = payload.content;
|
|
1998
2071
|
if (typeof content === "string") return nonEmpty(content);
|
|
@@ -2000,7 +2073,7 @@ function textFromMessagePayload(payload) {
|
|
|
2000
2073
|
return null;
|
|
2001
2074
|
}
|
|
2002
2075
|
function textBlocks(values) {
|
|
2003
|
-
return nonEmpty(values.map((value) => record
|
|
2076
|
+
return nonEmpty(values.map((value) => record(value)).filter((value) => value !== null).filter((value) => stringField(value, "type") === "text").map((value) => stringField(value, "text")).filter((value) => value !== void 0).join("\n"));
|
|
2004
2077
|
}
|
|
2005
2078
|
function terminalWithExecution(projection, execution) {
|
|
2006
2079
|
if (!execution) return {
|
|
@@ -2056,7 +2129,7 @@ function isCodexCollaborationTool(normalized) {
|
|
|
2056
2129
|
return /^[a-z_]+_agents?$/.test(verb) || verb === "send_message" || verb === "send_input" || verb === "followup_task";
|
|
2057
2130
|
}
|
|
2058
2131
|
function statusFrom(value) {
|
|
2059
|
-
const status = (stringField
|
|
2132
|
+
const status = (stringField(value, "status") ?? "").toLowerCase();
|
|
2060
2133
|
if (status === "completed" || status === "success" || status === "succeeded") return "completed";
|
|
2061
2134
|
if (status === "error" || status === "failed" || status === "declined") return "failed";
|
|
2062
2135
|
if (status === "running" || status === "started" || status === "in_progress") return "started";
|
|
@@ -2065,31 +2138,13 @@ function statusFrom(value) {
|
|
|
2065
2138
|
function compactMetadata(input) {
|
|
2066
2139
|
return Object.fromEntries(Object.entries(input).filter((entry) => entry[1] !== void 0));
|
|
2067
2140
|
}
|
|
2068
|
-
function
|
|
2069
|
-
for (const entry of entries) {
|
|
2070
|
-
if (source === "pi" && entry.type === "session") {
|
|
2071
|
-
const id = stringField$1(entry, "id");
|
|
2072
|
-
if (id) return id;
|
|
2073
|
-
}
|
|
2074
|
-
if (source === "codex") {
|
|
2075
|
-
const threadId = stringField$1(entry, "thread_id");
|
|
2076
|
-
if (threadId) return threadId;
|
|
2077
|
-
const payload = record$1(entry.payload);
|
|
2078
|
-
const id = payload ? stringField$1(payload, "id") : void 0;
|
|
2079
|
-
if (id) return id;
|
|
2080
|
-
}
|
|
2081
|
-
const payload = record$1(record$1(entry.message)?.payload);
|
|
2082
|
-
const sessionId = stringField$1(entry, "sessionID") ?? stringField$1(entry, "sessionId") ?? stringField$1(entry, "session_id") ?? stringField$1(payload ?? {}, "session_id");
|
|
2083
|
-
if (sessionId) return sessionId;
|
|
2084
|
-
}
|
|
2085
|
-
}
|
|
2086
|
-
function fallbackSessionId$1(source, sourcePath) {
|
|
2141
|
+
function fallbackSessionId(source, sourcePath) {
|
|
2087
2142
|
return `${source}:${sourcePath ?? "unknown-session"}`;
|
|
2088
2143
|
}
|
|
2089
2144
|
function looksLikeError(value) {
|
|
2090
|
-
if (isRecord
|
|
2145
|
+
if (isRecord(value)) {
|
|
2091
2146
|
if (value.success === false || value.is_error === true) return true;
|
|
2092
|
-
const exitCode = numberField
|
|
2147
|
+
const exitCode = numberField(value, "exit_code");
|
|
2093
2148
|
if (exitCode !== void 0 && exitCode !== 0) return true;
|
|
2094
2149
|
}
|
|
2095
2150
|
const text = typeof value === "string" ? value : JSON.stringify(value ?? "");
|
|
@@ -2098,298 +2153,150 @@ function looksLikeError(value) {
|
|
|
2098
2153
|
function nonEmpty(value) {
|
|
2099
2154
|
return value !== void 0 && value.length > 0 ? value : null;
|
|
2100
2155
|
}
|
|
2101
|
-
function isRecord$1(value) {
|
|
2102
|
-
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
2103
|
-
}
|
|
2104
|
-
function record$1(value) {
|
|
2105
|
-
return isRecord$1(value) ? value : null;
|
|
2106
|
-
}
|
|
2107
|
-
function stringField$1(value, key) {
|
|
2108
|
-
const field = value[key];
|
|
2109
|
-
return typeof field === "string" && field.length > 0 ? field : void 0;
|
|
2110
|
-
}
|
|
2111
|
-
function numberField$1(value, key) {
|
|
2112
|
-
const field = value[key];
|
|
2113
|
-
return typeof field === "number" && Number.isFinite(field) ? field : void 0;
|
|
2114
|
-
}
|
|
2115
|
-
function timestamp(value) {
|
|
2116
|
-
if (typeof value === "number" && Number.isFinite(value)) return value > 0xe8d4a51000 ? value : value * 1e3;
|
|
2117
|
-
if (typeof value === "string" && value.length > 0) {
|
|
2118
|
-
const parsed = Date.parse(value);
|
|
2119
|
-
return Number.isFinite(parsed) ? parsed : void 0;
|
|
2120
|
-
}
|
|
2121
|
-
}
|
|
2122
2156
|
//#endregion
|
|
2123
|
-
//#region src/contract/intake/code-agent-session.ts
|
|
2124
|
-
|
|
2125
|
-
* so malformed-line handling and entry validation cannot drift apart. */
|
|
2126
|
-
function readCodeAgentJsonlLine(line, lineNumber) {
|
|
2127
|
-
const trimmed = line.trim();
|
|
2128
|
-
if (!trimmed) return void 0;
|
|
2129
|
-
try {
|
|
2130
|
-
return {
|
|
2131
|
-
kind: "entry",
|
|
2132
|
-
lineNumber,
|
|
2133
|
-
entry: JSON.parse(trimmed)
|
|
2134
|
-
};
|
|
2135
|
-
} catch {
|
|
2136
|
-
return {
|
|
2137
|
-
kind: "malformed",
|
|
2138
|
-
lineNumber
|
|
2139
|
-
};
|
|
2140
|
-
}
|
|
2141
|
-
}
|
|
2142
|
-
function parseCodeAgentJsonl(jsonl) {
|
|
2143
|
-
const entries = [];
|
|
2144
|
-
let malformedLines = 0;
|
|
2145
|
-
let lineNumber = 0;
|
|
2146
|
-
for (const line of jsonl.split("\n")) {
|
|
2147
|
-
lineNumber += 1;
|
|
2148
|
-
const read = readCodeAgentJsonlLine(line, lineNumber);
|
|
2149
|
-
if (!read) continue;
|
|
2150
|
-
if (read.kind === "malformed") malformedLines += 1;
|
|
2151
|
-
else entries.push(read.entry);
|
|
2152
|
-
}
|
|
2157
|
+
//#region src/contract/intake/code-agent-session-metrics.ts
|
|
2158
|
+
function emptyMetrics(entries) {
|
|
2153
2159
|
return {
|
|
2154
2160
|
entries,
|
|
2155
|
-
|
|
2161
|
+
userMessages: 0,
|
|
2162
|
+
assistantMessages: 0,
|
|
2163
|
+
reasoningItems: 0,
|
|
2164
|
+
toolCalls: 0,
|
|
2165
|
+
toolOutputs: 0,
|
|
2166
|
+
toolErrors: 0,
|
|
2167
|
+
unclassifiedErrors: 0,
|
|
2168
|
+
patchAttempts: 0,
|
|
2169
|
+
patchSuccesses: 0,
|
|
2170
|
+
patchFailures: 0,
|
|
2171
|
+
turnsStarted: 0,
|
|
2172
|
+
turnsCompleted: 0,
|
|
2173
|
+
turnsAborted: 0,
|
|
2174
|
+
contextCompactions: 0,
|
|
2175
|
+
mcpCalls: 0,
|
|
2176
|
+
subagentCalls: 0,
|
|
2177
|
+
skillCalls: 0,
|
|
2178
|
+
hookCalls: 0,
|
|
2179
|
+
webCalls: 0,
|
|
2180
|
+
codeActions: 0,
|
|
2181
|
+
prLinks: 0,
|
|
2182
|
+
fileSnapshots: 0,
|
|
2183
|
+
graphNodes: 0,
|
|
2184
|
+
graphEdges: 0,
|
|
2185
|
+
actionCandidates: 0,
|
|
2186
|
+
verificationReports: 0,
|
|
2187
|
+
completionDecisions: 0,
|
|
2188
|
+
reliabilityRows: 0,
|
|
2189
|
+
reliabilityLift: 0,
|
|
2190
|
+
inputTokens: 0,
|
|
2191
|
+
outputTokens: 0,
|
|
2192
|
+
reasoningTokens: 0,
|
|
2193
|
+
cachedTokens: 0,
|
|
2194
|
+
cacheWriteTokens: 0,
|
|
2195
|
+
observedCostUsd: 0,
|
|
2196
|
+
observedCostCaptured: false,
|
|
2197
|
+
wallMs: 0,
|
|
2198
|
+
processScore: 0
|
|
2156
2199
|
};
|
|
2157
2200
|
}
|
|
2158
|
-
|
|
2159
|
-
|
|
2160
|
-
*
|
|
2161
|
-
* cannot be ingested at all; the largest real Codex rollout on record is 695MB.
|
|
2162
|
-
*
|
|
2163
|
-
* Lines break on `\n` only, which is what the string path's `split('\n')` does.
|
|
2164
|
-
* `node:readline` also breaks on a bare `\r`, so it is deliberately not used
|
|
2165
|
-
* here: a lone carriage return inside a line must stay inside that line for the
|
|
2166
|
-
* two paths to report the same malformed count. */
|
|
2167
|
-
async function* streamCodeAgentJsonlFile(path) {
|
|
2168
|
-
const stream = createReadStream(path, { encoding: "utf8" });
|
|
2169
|
-
let pending = "";
|
|
2170
|
-
let lineNumber = 0;
|
|
2171
|
-
try {
|
|
2172
|
-
for await (const chunk of stream) {
|
|
2173
|
-
pending += chunk;
|
|
2174
|
-
let start = 0;
|
|
2175
|
-
for (let at = pending.indexOf("\n"); at !== -1; at = pending.indexOf("\n", start)) {
|
|
2176
|
-
lineNumber += 1;
|
|
2177
|
-
const read = readCodeAgentJsonlLine(pending.slice(start, at), lineNumber);
|
|
2178
|
-
if (read) yield read;
|
|
2179
|
-
start = at + 1;
|
|
2180
|
-
}
|
|
2181
|
-
pending = pending.slice(start);
|
|
2182
|
-
}
|
|
2183
|
-
} finally {
|
|
2184
|
-
stream.destroy();
|
|
2185
|
-
}
|
|
2186
|
-
const last = readCodeAgentJsonlLine(pending, lineNumber + 1);
|
|
2187
|
-
if (last) yield last;
|
|
2201
|
+
function penalizeErrors(base, metrics) {
|
|
2202
|
+
const operations = Math.max(1, metrics.toolCalls + metrics.patchAttempts);
|
|
2203
|
+
return clamp01(base * (1 - .5 * Math.min(1, (metrics.toolErrors + metrics.patchFailures) / operations)));
|
|
2188
2204
|
}
|
|
2189
|
-
|
|
2190
|
-
* It returns the same shape, so a caller that holds every entry keeps working
|
|
2191
|
-
* above the string ceiling. The entry array still grows with the transcript;
|
|
2192
|
-
* consume `streamCodeAgentJsonlFile` directly when memory must stay flat. */
|
|
2193
|
-
async function parseCodeAgentJsonlFile(path) {
|
|
2194
|
-
const entries = [];
|
|
2195
|
-
let malformedLines = 0;
|
|
2196
|
-
for await (const read of streamCodeAgentJsonlFile(path)) if (read.kind === "malformed") malformedLines += 1;
|
|
2197
|
-
else entries.push(read.entry);
|
|
2205
|
+
function emptyTokenTotals() {
|
|
2198
2206
|
return {
|
|
2199
|
-
|
|
2200
|
-
|
|
2207
|
+
input: 0,
|
|
2208
|
+
output: 0,
|
|
2209
|
+
reasoning: 0,
|
|
2210
|
+
cached: 0,
|
|
2211
|
+
cacheWrite: 0
|
|
2201
2212
|
};
|
|
2202
2213
|
}
|
|
2203
|
-
function
|
|
2204
|
-
return
|
|
2205
|
-
}
|
|
2206
|
-
function fromClaudeCodeSession(options) {
|
|
2207
|
-
return fromCodeAgentSession("claude-code", options);
|
|
2214
|
+
function hasTokenTotals(usage) {
|
|
2215
|
+
return usage.input > 0 || usage.output > 0 || usage.reasoning > 0 || usage.cached > 0 || usage.cacheWrite > 0;
|
|
2208
2216
|
}
|
|
2209
|
-
function
|
|
2210
|
-
|
|
2211
|
-
|
|
2212
|
-
|
|
2213
|
-
|
|
2214
|
-
|
|
2215
|
-
|
|
2216
|
-
function fromPiSession(options) {
|
|
2217
|
-
return fromCodeAgentSession("pi", options);
|
|
2217
|
+
function addUsage(metrics, usage) {
|
|
2218
|
+
const parsed = readUsage(usage);
|
|
2219
|
+
metrics.inputTokens += parsed.input;
|
|
2220
|
+
metrics.outputTokens += parsed.output;
|
|
2221
|
+
metrics.reasoningTokens += parsed.reasoning;
|
|
2222
|
+
metrics.cachedTokens += parsed.cached;
|
|
2223
|
+
metrics.cacheWriteTokens += parsed.cacheWrite;
|
|
2218
2224
|
}
|
|
2219
|
-
|
|
2220
|
-
|
|
2221
|
-
|
|
2225
|
+
function addUsageTo(totals, usage) {
|
|
2226
|
+
const parsed = readUsage(usage);
|
|
2227
|
+
totals.input += parsed.input;
|
|
2228
|
+
totals.output += parsed.output;
|
|
2229
|
+
totals.reasoning += parsed.reasoning;
|
|
2230
|
+
totals.cached += parsed.cached;
|
|
2231
|
+
totals.cacheWrite += parsed.cacheWrite;
|
|
2222
2232
|
}
|
|
2223
|
-
function
|
|
2224
|
-
const
|
|
2225
|
-
if (entries.length === 0) return {
|
|
2226
|
-
runs: [],
|
|
2227
|
-
diagnostics: [{
|
|
2228
|
-
source,
|
|
2229
|
-
sessionId: fallbackSessionId(source, options.sourcePath),
|
|
2230
|
-
sourcePath: options.sourcePath,
|
|
2231
|
-
entries: 0,
|
|
2232
|
-
malformedLines: options.malformedLines ?? 0,
|
|
2233
|
-
hasExplicitTerminalSignal: false,
|
|
2234
|
-
hasFinalOutput: false,
|
|
2235
|
-
hasQualityLabel: false,
|
|
2236
|
-
hasTokenUsage: false,
|
|
2237
|
-
hasCost: false,
|
|
2238
|
-
costKind: "uncaptured",
|
|
2239
|
-
warnings: ["no parseable session entries"]
|
|
2240
|
-
}],
|
|
2241
|
-
metrics: [],
|
|
2242
|
-
observations: []
|
|
2243
|
-
};
|
|
2244
|
-
const metrics = metricsFor(source, entries);
|
|
2245
|
-
const sessionId = sessionIdFromEntries(source, entries) ?? fallbackSessionId(source, options.sourcePath);
|
|
2246
|
-
const observation = observeCodeAgentSession({
|
|
2247
|
-
source,
|
|
2248
|
-
entries,
|
|
2249
|
-
sourcePath: options.sourcePath,
|
|
2250
|
-
execution: options.execution
|
|
2251
|
-
});
|
|
2252
|
-
applyObservedSurfaceMetrics(metrics, observation);
|
|
2253
|
-
const model = withSnapshot(options.model ?? modelFromEntries(source, entries) ?? source);
|
|
2254
|
-
const tokenUsage = {
|
|
2255
|
-
input: metrics.inputTokens,
|
|
2256
|
-
output: metrics.outputTokens,
|
|
2257
|
-
...metrics.reasoningTokens > 0 ? { reasoning: metrics.reasoningTokens } : {},
|
|
2258
|
-
...metrics.cachedTokens > 0 ? { cached: metrics.cachedTokens } : {},
|
|
2259
|
-
...metrics.cacheWriteTokens > 0 ? { cacheWrite: metrics.cacheWriteTokens } : {}
|
|
2260
|
-
};
|
|
2261
|
-
const { costUsd, costProvenance } = resolveSessionCost(options.costProvenance, metrics, model);
|
|
2262
|
-
const qualityScore = finiteScore(options.score);
|
|
2263
|
-
const invalidQualityScore = options.score !== void 0 && qualityScore === void 0;
|
|
2264
|
-
const promptHash = options.promptHash ?? hashString(`prompt:${source}:${firstUserText(entries) ?? ""}`);
|
|
2265
|
-
const configHash = options.configHash ?? hashJson({
|
|
2266
|
-
source,
|
|
2267
|
-
model,
|
|
2268
|
-
sourcePath: options.sourcePath ?? null,
|
|
2269
|
-
cwd: cwdFromEntries(entries) ?? null,
|
|
2270
|
-
entryCount: entries.length
|
|
2271
|
-
});
|
|
2272
|
-
const explicitTerminal = observation.terminal.explicit;
|
|
2273
|
-
const warnings = diagnosticsFor(metrics, {
|
|
2274
|
-
model,
|
|
2275
|
-
explicitTerminal,
|
|
2276
|
-
malformedLines: options.malformedLines ?? 0,
|
|
2277
|
-
hasQualityLabel: qualityScore !== void 0,
|
|
2278
|
-
invalidQualityScore,
|
|
2279
|
-
costKind: costProvenance.kind
|
|
2280
|
-
});
|
|
2281
|
-
const terminalOutcome = terminalOutcomeFromSession(observation.terminal);
|
|
2282
|
-
const processExitCode = options.execution?.exitCode;
|
|
2283
|
-
const splitTag = options.splitTag ?? "holdout";
|
|
2233
|
+
function readUsage(usage) {
|
|
2234
|
+
const parsed = extractUsage(usage);
|
|
2284
2235
|
return {
|
|
2285
|
-
|
|
2286
|
-
|
|
2287
|
-
|
|
2288
|
-
|
|
2289
|
-
|
|
2290
|
-
model,
|
|
2291
|
-
promptHash,
|
|
2292
|
-
configHash,
|
|
2293
|
-
commitSha: options.commitSha ?? "local-session",
|
|
2294
|
-
wallMs: metrics.wallMs,
|
|
2295
|
-
costUsd,
|
|
2296
|
-
costProvenance,
|
|
2297
|
-
tokenUsage,
|
|
2298
|
-
terminalOutcome,
|
|
2299
|
-
...terminalOutcome === "failed" ? { terminalFailureReason: processExitCode !== void 0 && processExitCode !== 0 ? `process exited with code ${processExitCode}` : metrics.turnsAborted > 0 ? "turn aborted" : "session reported terminal failure" } : {},
|
|
2300
|
-
outcome: {
|
|
2301
|
-
...qualityScore === void 0 ? {} : splitTag === "holdout" ? { holdoutScore: qualityScore } : { searchScore: qualityScore },
|
|
2302
|
-
raw: {
|
|
2303
|
-
entries: metrics.entries,
|
|
2304
|
-
user_messages: metrics.userMessages,
|
|
2305
|
-
assistant_messages: metrics.assistantMessages,
|
|
2306
|
-
reasoning_items: metrics.reasoningItems,
|
|
2307
|
-
tool_calls: metrics.toolCalls,
|
|
2308
|
-
tool_outputs: metrics.toolOutputs,
|
|
2309
|
-
tool_errors: metrics.toolErrors,
|
|
2310
|
-
execution_error_count: metrics.toolErrors,
|
|
2311
|
-
process_error_count: terminalOutcome === "failed" ? 1 : 0,
|
|
2312
|
-
unclassified_error_count: metrics.unclassifiedErrors,
|
|
2313
|
-
...processExitCode !== void 0 ? { process_exit_code: processExitCode } : {},
|
|
2314
|
-
patch_attempts: metrics.patchAttempts,
|
|
2315
|
-
patch_successes: metrics.patchSuccesses,
|
|
2316
|
-
patch_failures: metrics.patchFailures,
|
|
2317
|
-
turns_started: metrics.turnsStarted,
|
|
2318
|
-
turns_completed: metrics.turnsCompleted,
|
|
2319
|
-
turns_aborted: metrics.turnsAborted,
|
|
2320
|
-
context_compactions: metrics.contextCompactions,
|
|
2321
|
-
mcp_calls: metrics.mcpCalls,
|
|
2322
|
-
subagent_calls: metrics.subagentCalls,
|
|
2323
|
-
skill_calls: metrics.skillCalls,
|
|
2324
|
-
hook_calls: metrics.hookCalls,
|
|
2325
|
-
web_calls: metrics.webCalls,
|
|
2326
|
-
code_actions: metrics.codeActions,
|
|
2327
|
-
pr_links: metrics.prLinks,
|
|
2328
|
-
file_snapshots: metrics.fileSnapshots,
|
|
2329
|
-
graph_nodes: metrics.graphNodes,
|
|
2330
|
-
graph_edges: metrics.graphEdges,
|
|
2331
|
-
action_candidates: metrics.actionCandidates,
|
|
2332
|
-
verification_reports: metrics.verificationReports,
|
|
2333
|
-
completion_decisions: metrics.completionDecisions,
|
|
2334
|
-
reliability_rows: metrics.reliabilityRows,
|
|
2335
|
-
reliability_lift: metrics.reliabilityLift,
|
|
2336
|
-
input_tokens: metrics.inputTokens,
|
|
2337
|
-
output_tokens: metrics.outputTokens,
|
|
2338
|
-
reasoning_tokens: metrics.reasoningTokens,
|
|
2339
|
-
cached_tokens: metrics.cachedTokens,
|
|
2340
|
-
cache_write_tokens: metrics.cacheWriteTokens,
|
|
2341
|
-
observed_cost_usd: metrics.observedCostUsd,
|
|
2342
|
-
observed_cost_captured: metrics.observedCostCaptured ? 1 : 0,
|
|
2343
|
-
estimated_cost_usd: costProvenance.kind === "estimated" ? costProvenance.usd : 0,
|
|
2344
|
-
process_score: metrics.processScore,
|
|
2345
|
-
explicit_terminal_signal: explicitTerminal ? 1 : 0,
|
|
2346
|
-
quality_label_present: qualityScore !== void 0 ? 1 : 0,
|
|
2347
|
-
cost_observed: costProvenance.kind === "observed" ? 1 : 0,
|
|
2348
|
-
cost_estimated: costProvenance.kind === "estimated" ? 1 : 0,
|
|
2349
|
-
cost_uncaptured: costProvenance.kind === "uncaptured" ? 1 : 0,
|
|
2350
|
-
cost_unknown: costProvenance.usd === null ? 1 : 0
|
|
2351
|
-
}
|
|
2352
|
-
},
|
|
2353
|
-
splitTag,
|
|
2354
|
-
scenarioId: options.scenarioId ?? sessionId
|
|
2355
|
-
}],
|
|
2356
|
-
diagnostics: [{
|
|
2357
|
-
source,
|
|
2358
|
-
sessionId,
|
|
2359
|
-
sourcePath: options.sourcePath,
|
|
2360
|
-
entries: entries.length,
|
|
2361
|
-
malformedLines: options.malformedLines ?? 0,
|
|
2362
|
-
hasExplicitTerminalSignal: explicitTerminal,
|
|
2363
|
-
hasFinalOutput: observation.finalText !== null,
|
|
2364
|
-
hasQualityLabel: qualityScore !== void 0,
|
|
2365
|
-
hasTokenUsage: metrics.inputTokens > 0 || metrics.outputTokens > 0 || metrics.reasoningTokens > 0 || metrics.cachedTokens > 0 || metrics.cacheWriteTokens > 0,
|
|
2366
|
-
hasCost: costProvenance.usd !== null,
|
|
2367
|
-
costKind: costProvenance.kind,
|
|
2368
|
-
warnings
|
|
2369
|
-
}],
|
|
2370
|
-
metrics: [metrics],
|
|
2371
|
-
observations: [{
|
|
2372
|
-
...observation,
|
|
2373
|
-
sessionId
|
|
2374
|
-
}]
|
|
2236
|
+
input: parsed?.input ?? 0,
|
|
2237
|
+
output: parsed?.output ?? 0,
|
|
2238
|
+
reasoning: parsed?.reasoning ?? 0,
|
|
2239
|
+
cached: parsed?.cached ?? 0,
|
|
2240
|
+
cacheWrite: parsed?.cacheWrite ?? 0
|
|
2375
2241
|
};
|
|
2376
2242
|
}
|
|
2377
|
-
function
|
|
2378
|
-
if (
|
|
2379
|
-
|
|
2380
|
-
|
|
2381
|
-
|
|
2382
|
-
|
|
2383
|
-
|
|
2384
|
-
|
|
2385
|
-
|
|
2386
|
-
|
|
2387
|
-
|
|
2388
|
-
|
|
2389
|
-
|
|
2390
|
-
|
|
2243
|
+
function resolveSessionCost(explicit, metrics, model) {
|
|
2244
|
+
if (explicit) {
|
|
2245
|
+
if (explicit.kind === "uncaptured") {
|
|
2246
|
+
if (explicit.usd !== null) throw new Error("uncaptured cost must have usd: null");
|
|
2247
|
+
return {
|
|
2248
|
+
costUsd: null,
|
|
2249
|
+
costProvenance: explicit
|
|
2250
|
+
};
|
|
2251
|
+
}
|
|
2252
|
+
if (explicit.kind === "lower-bound") {
|
|
2253
|
+
if (explicit.usd !== null) throw new Error("lower-bound cost must have usd: null");
|
|
2254
|
+
if (!Number.isFinite(explicit.knownLowerBoundUsd) || explicit.knownLowerBoundUsd <= 0) throw new Error("lower-bound cost must have a finite, positive knownLowerBoundUsd");
|
|
2255
|
+
return {
|
|
2256
|
+
costUsd: null,
|
|
2257
|
+
costProvenance: explicit
|
|
2258
|
+
};
|
|
2259
|
+
}
|
|
2260
|
+
if (!Number.isFinite(explicit.usd) || explicit.usd < 0) throw new Error(`${explicit.kind} cost must be a finite, non-negative USD amount`);
|
|
2261
|
+
return {
|
|
2262
|
+
costUsd: explicit.usd,
|
|
2263
|
+
costProvenance: explicit
|
|
2264
|
+
};
|
|
2265
|
+
}
|
|
2266
|
+
if (metrics.observedCostCaptured) {
|
|
2267
|
+
const costProvenance = {
|
|
2268
|
+
kind: "observed",
|
|
2269
|
+
usd: metrics.observedCostUsd
|
|
2270
|
+
};
|
|
2271
|
+
return {
|
|
2272
|
+
costUsd: metrics.observedCostUsd,
|
|
2273
|
+
costProvenance
|
|
2274
|
+
};
|
|
2275
|
+
}
|
|
2276
|
+
if ((metrics.inputTokens > 0 || metrics.outputTokens > 0) && isModelPriced(model)) {
|
|
2277
|
+
const costUsd = estimateCost(metrics.inputTokens, metrics.outputTokens, model);
|
|
2278
|
+
return {
|
|
2279
|
+
costUsd,
|
|
2280
|
+
costProvenance: {
|
|
2281
|
+
kind: "estimated",
|
|
2282
|
+
usd: costUsd
|
|
2283
|
+
}
|
|
2284
|
+
};
|
|
2391
2285
|
}
|
|
2286
|
+
return {
|
|
2287
|
+
costUsd: null,
|
|
2288
|
+
costProvenance: {
|
|
2289
|
+
kind: "uncaptured",
|
|
2290
|
+
usd: null
|
|
2291
|
+
}
|
|
2292
|
+
};
|
|
2293
|
+
}
|
|
2294
|
+
function clamp01(value) {
|
|
2295
|
+
if (!Number.isFinite(value)) return 0;
|
|
2296
|
+
return Math.max(0, Math.min(1, value));
|
|
2392
2297
|
}
|
|
2298
|
+
//#endregion
|
|
2299
|
+
//#region src/contract/intake/code-agent-codex-metrics.ts
|
|
2393
2300
|
function codexMetrics(entries) {
|
|
2394
2301
|
const metrics = emptyMetrics(entries.length);
|
|
2395
2302
|
const startedToolIds = /* @__PURE__ */ new Set();
|
|
@@ -2408,7 +2315,7 @@ function codexMetrics(entries) {
|
|
|
2408
2315
|
if (entryType === "turn.started") metrics.turnsStarted += 1;
|
|
2409
2316
|
if (entryType === "turn.completed") {
|
|
2410
2317
|
metrics.turnsCompleted += 1;
|
|
2411
|
-
|
|
2318
|
+
addUsage(metrics, record(entry.usage));
|
|
2412
2319
|
}
|
|
2413
2320
|
if (entryType === "turn.failed") metrics.turnsAborted += 1;
|
|
2414
2321
|
if (entryType === "error") metrics.unclassifiedErrors += 1;
|
|
@@ -2496,14 +2403,6 @@ function codexExecToolFailed(itemType, item) {
|
|
|
2496
2403
|
if (itemType === "mcp_tool_call") return status === "failed" || record(item.error) !== null;
|
|
2497
2404
|
return itemType === "collab_tool_call" && status === "failed";
|
|
2498
2405
|
}
|
|
2499
|
-
function addCodexExecUsage(metrics, usage) {
|
|
2500
|
-
const parsed = readUsage(usage);
|
|
2501
|
-
metrics.inputTokens += parsed.input;
|
|
2502
|
-
metrics.outputTokens += parsed.output;
|
|
2503
|
-
metrics.reasoningTokens += parsed.reasoning;
|
|
2504
|
-
metrics.cachedTokens += parsed.cached;
|
|
2505
|
-
metrics.cacheWriteTokens += parsed.cacheWrite;
|
|
2506
|
-
}
|
|
2507
2406
|
function setCodexCumulativeUsage(metrics, usage) {
|
|
2508
2407
|
const parsed = readUsage(usage);
|
|
2509
2408
|
metrics.inputTokens = Math.max(metrics.inputTokens, parsed.input);
|
|
@@ -2512,7 +2411,13 @@ function setCodexCumulativeUsage(metrics, usage) {
|
|
|
2512
2411
|
metrics.cachedTokens = Math.max(metrics.cachedTokens, parsed.cached);
|
|
2513
2412
|
metrics.cacheWriteTokens = Math.max(metrics.cacheWriteTokens, parsed.cacheWrite);
|
|
2514
2413
|
}
|
|
2515
|
-
function
|
|
2414
|
+
function codexProcessScore(metrics) {
|
|
2415
|
+
const terminalTurns = metrics.turnsCompleted + metrics.turnsAborted;
|
|
2416
|
+
return penalizeErrors(terminalTurns > 0 ? metrics.turnsCompleted / terminalTurns : metrics.assistantMessages > 0 ? .5 : 0, metrics);
|
|
2417
|
+
}
|
|
2418
|
+
//#endregion
|
|
2419
|
+
//#region src/contract/intake/code-agent-harness-metrics.ts
|
|
2420
|
+
function claudeCodeMetrics(entries) {
|
|
2516
2421
|
const metrics = emptyMetrics(entries.length);
|
|
2517
2422
|
let startedAt;
|
|
2518
2423
|
let completedAt;
|
|
@@ -2726,53 +2631,6 @@ function piGraphMetrics(entries) {
|
|
|
2726
2631
|
metrics.processScore = bestReliabilityScore !== void 0 ? clamp01(bestReliabilityScore) : metrics.completionDecisions > 0 ? 1 : 0;
|
|
2727
2632
|
return metrics;
|
|
2728
2633
|
}
|
|
2729
|
-
function emptyMetrics(entries) {
|
|
2730
|
-
return {
|
|
2731
|
-
entries,
|
|
2732
|
-
userMessages: 0,
|
|
2733
|
-
assistantMessages: 0,
|
|
2734
|
-
reasoningItems: 0,
|
|
2735
|
-
toolCalls: 0,
|
|
2736
|
-
toolOutputs: 0,
|
|
2737
|
-
toolErrors: 0,
|
|
2738
|
-
unclassifiedErrors: 0,
|
|
2739
|
-
patchAttempts: 0,
|
|
2740
|
-
patchSuccesses: 0,
|
|
2741
|
-
patchFailures: 0,
|
|
2742
|
-
turnsStarted: 0,
|
|
2743
|
-
turnsCompleted: 0,
|
|
2744
|
-
turnsAborted: 0,
|
|
2745
|
-
contextCompactions: 0,
|
|
2746
|
-
mcpCalls: 0,
|
|
2747
|
-
subagentCalls: 0,
|
|
2748
|
-
skillCalls: 0,
|
|
2749
|
-
hookCalls: 0,
|
|
2750
|
-
webCalls: 0,
|
|
2751
|
-
codeActions: 0,
|
|
2752
|
-
prLinks: 0,
|
|
2753
|
-
fileSnapshots: 0,
|
|
2754
|
-
graphNodes: 0,
|
|
2755
|
-
graphEdges: 0,
|
|
2756
|
-
actionCandidates: 0,
|
|
2757
|
-
verificationReports: 0,
|
|
2758
|
-
completionDecisions: 0,
|
|
2759
|
-
reliabilityRows: 0,
|
|
2760
|
-
reliabilityLift: 0,
|
|
2761
|
-
inputTokens: 0,
|
|
2762
|
-
outputTokens: 0,
|
|
2763
|
-
reasoningTokens: 0,
|
|
2764
|
-
cachedTokens: 0,
|
|
2765
|
-
cacheWriteTokens: 0,
|
|
2766
|
-
observedCostUsd: 0,
|
|
2767
|
-
observedCostCaptured: false,
|
|
2768
|
-
wallMs: 0,
|
|
2769
|
-
processScore: 0
|
|
2770
|
-
};
|
|
2771
|
-
}
|
|
2772
|
-
function codexProcessScore(metrics) {
|
|
2773
|
-
const terminalTurns = metrics.turnsCompleted + metrics.turnsAborted;
|
|
2774
|
-
return penalizeErrors(terminalTurns > 0 ? metrics.turnsCompleted / terminalTurns : metrics.assistantMessages > 0 ? .5 : 0, metrics);
|
|
2775
|
-
}
|
|
2776
2634
|
function claudeProcessScore(metrics) {
|
|
2777
2635
|
return penalizeErrors(metrics.prLinks > 0 ? 1 : metrics.assistantMessages > 0 ? .75 : 0, metrics);
|
|
2778
2636
|
}
|
|
@@ -2780,98 +2638,281 @@ function terminalProcessScore(metrics) {
|
|
|
2780
2638
|
const terminalTurns = metrics.turnsCompleted + metrics.turnsAborted;
|
|
2781
2639
|
return penalizeErrors(terminalTurns > 0 ? metrics.turnsCompleted / terminalTurns : metrics.assistantMessages > 0 ? .75 : metrics.toolCalls > 0 ? .5 : 0, metrics);
|
|
2782
2640
|
}
|
|
2783
|
-
function
|
|
2784
|
-
|
|
2785
|
-
return clamp01(base * (1 - .5 * Math.min(1, (metrics.toolErrors + metrics.patchFailures) / operations)));
|
|
2641
|
+
function maxOptional(current, next) {
|
|
2642
|
+
return current === void 0 ? next : Math.max(current, next);
|
|
2786
2643
|
}
|
|
2787
|
-
|
|
2644
|
+
//#endregion
|
|
2645
|
+
//#region src/contract/intake/code-agent-jsonl.ts
|
|
2646
|
+
/** The single per-line rule. Both the string path and the file path call this,
|
|
2647
|
+
* so malformed-line handling and entry validation cannot drift apart. */
|
|
2648
|
+
function readCodeAgentJsonlLine(line, lineNumber) {
|
|
2649
|
+
const trimmed = line.trim();
|
|
2650
|
+
if (!trimmed) return void 0;
|
|
2651
|
+
try {
|
|
2652
|
+
return {
|
|
2653
|
+
kind: "entry",
|
|
2654
|
+
lineNumber,
|
|
2655
|
+
entry: JSON.parse(trimmed)
|
|
2656
|
+
};
|
|
2657
|
+
} catch {
|
|
2658
|
+
return {
|
|
2659
|
+
kind: "malformed",
|
|
2660
|
+
lineNumber
|
|
2661
|
+
};
|
|
2662
|
+
}
|
|
2663
|
+
}
|
|
2664
|
+
function parseCodeAgentJsonl(jsonl) {
|
|
2665
|
+
const entries = [];
|
|
2666
|
+
let malformedLines = 0;
|
|
2667
|
+
let lineNumber = 0;
|
|
2668
|
+
for (const line of jsonl.split("\n")) {
|
|
2669
|
+
lineNumber += 1;
|
|
2670
|
+
const read = readCodeAgentJsonlLine(line, lineNumber);
|
|
2671
|
+
if (!read) continue;
|
|
2672
|
+
if (read.kind === "malformed") malformedLines += 1;
|
|
2673
|
+
else entries.push(read.entry);
|
|
2674
|
+
}
|
|
2788
2675
|
return {
|
|
2789
|
-
|
|
2790
|
-
|
|
2791
|
-
reasoning: 0,
|
|
2792
|
-
cached: 0,
|
|
2793
|
-
cacheWrite: 0
|
|
2676
|
+
entries,
|
|
2677
|
+
malformedLines
|
|
2794
2678
|
};
|
|
2795
2679
|
}
|
|
2796
|
-
|
|
2797
|
-
|
|
2680
|
+
/** Reads a transcript one line at a time and never holds the file as a single
|
|
2681
|
+
* string. `parseCodeAgentJsonl` needs the whole file in one string, so a
|
|
2682
|
+
* session above V8's ~512MB string ceiling throws `ERR_STRING_TOO_LONG` and
|
|
2683
|
+
* cannot be ingested at all; the largest real Codex rollout on record is 695MB.
|
|
2684
|
+
*
|
|
2685
|
+
* Lines break on `\n` only, which is what the string path's `split('\n')` does.
|
|
2686
|
+
* `node:readline` also breaks on a bare `\r`, so it is deliberately not used
|
|
2687
|
+
* here: a lone carriage return inside a line must stay inside that line for the
|
|
2688
|
+
* two paths to report the same malformed count. */
|
|
2689
|
+
async function* streamCodeAgentJsonlFile(path) {
|
|
2690
|
+
const stream = createReadStream(path, { encoding: "utf8" });
|
|
2691
|
+
let pending = "";
|
|
2692
|
+
let lineNumber = 0;
|
|
2693
|
+
try {
|
|
2694
|
+
for await (const chunk of stream) {
|
|
2695
|
+
pending += chunk;
|
|
2696
|
+
let start = 0;
|
|
2697
|
+
for (let at = pending.indexOf("\n"); at !== -1; at = pending.indexOf("\n", start)) {
|
|
2698
|
+
lineNumber += 1;
|
|
2699
|
+
const read = readCodeAgentJsonlLine(pending.slice(start, at), lineNumber);
|
|
2700
|
+
if (read) yield read;
|
|
2701
|
+
start = at + 1;
|
|
2702
|
+
}
|
|
2703
|
+
pending = pending.slice(start);
|
|
2704
|
+
}
|
|
2705
|
+
} finally {
|
|
2706
|
+
stream.destroy();
|
|
2707
|
+
}
|
|
2708
|
+
const last = readCodeAgentJsonlLine(pending, lineNumber + 1);
|
|
2709
|
+
if (last) yield last;
|
|
2798
2710
|
}
|
|
2799
|
-
|
|
2800
|
-
|
|
2801
|
-
|
|
2802
|
-
|
|
2803
|
-
|
|
2804
|
-
|
|
2805
|
-
|
|
2711
|
+
/** Streaming counterpart to `parseCodeAgentJsonl` for a transcript on disk.
|
|
2712
|
+
* It returns the same shape, so a caller that holds every entry keeps working
|
|
2713
|
+
* above the string ceiling. The entry array still grows with the transcript;
|
|
2714
|
+
* consume `streamCodeAgentJsonlFile` directly when memory must stay flat. */
|
|
2715
|
+
async function parseCodeAgentJsonlFile(path) {
|
|
2716
|
+
const entries = [];
|
|
2717
|
+
let malformedLines = 0;
|
|
2718
|
+
for await (const read of streamCodeAgentJsonlFile(path)) if (read.kind === "malformed") malformedLines += 1;
|
|
2719
|
+
else entries.push(read.entry);
|
|
2720
|
+
return {
|
|
2721
|
+
entries,
|
|
2722
|
+
malformedLines
|
|
2723
|
+
};
|
|
2806
2724
|
}
|
|
2807
|
-
|
|
2808
|
-
|
|
2809
|
-
|
|
2810
|
-
|
|
2811
|
-
totals.reasoning += parsed.reasoning;
|
|
2812
|
-
totals.cached += parsed.cached;
|
|
2813
|
-
totals.cacheWrite += parsed.cacheWrite;
|
|
2725
|
+
//#endregion
|
|
2726
|
+
//#region src/contract/intake/code-agent-session.ts
|
|
2727
|
+
function fromCodexSession(options) {
|
|
2728
|
+
return fromCodeAgentSession("codex", options);
|
|
2814
2729
|
}
|
|
2815
|
-
function
|
|
2816
|
-
|
|
2730
|
+
function fromClaudeCodeSession(options) {
|
|
2731
|
+
return fromCodeAgentSession("claude-code", options);
|
|
2732
|
+
}
|
|
2733
|
+
function fromOpenCodeSession(options) {
|
|
2734
|
+
return fromCodeAgentSession("opencode", options);
|
|
2735
|
+
}
|
|
2736
|
+
function fromKimiCodeSession(options) {
|
|
2737
|
+
return fromCodeAgentSession("kimi-code", options);
|
|
2738
|
+
}
|
|
2739
|
+
/** A native Pi session (`~/.pi/agent/sessions/<cwd>/<ts>_<id>.jsonl`), parsed entries in order. */
|
|
2740
|
+
function fromPiSession(options) {
|
|
2741
|
+
return fromCodeAgentSession("pi", options);
|
|
2742
|
+
}
|
|
2743
|
+
/** The graph IR a Pi graph run exports (`nodes[].ir`), which is not a Pi session. */
|
|
2744
|
+
function fromPiGraphSession(options) {
|
|
2745
|
+
return fromCodeAgentSession("pi-graph", options);
|
|
2746
|
+
}
|
|
2747
|
+
function fromCodeAgentSession(source, options) {
|
|
2748
|
+
const entries = options.entries.filter(isRecord);
|
|
2749
|
+
if (entries.length === 0) return {
|
|
2750
|
+
runs: [],
|
|
2751
|
+
diagnostics: [{
|
|
2752
|
+
source,
|
|
2753
|
+
sessionId: fallbackSessionId$1(source, options.sourcePath),
|
|
2754
|
+
sourcePath: options.sourcePath,
|
|
2755
|
+
entries: 0,
|
|
2756
|
+
malformedLines: options.malformedLines ?? 0,
|
|
2757
|
+
hasExplicitTerminalSignal: false,
|
|
2758
|
+
hasFinalOutput: false,
|
|
2759
|
+
hasQualityLabel: false,
|
|
2760
|
+
hasTokenUsage: false,
|
|
2761
|
+
hasCost: false,
|
|
2762
|
+
costKind: "uncaptured",
|
|
2763
|
+
warnings: ["no parseable session entries"]
|
|
2764
|
+
}],
|
|
2765
|
+
metrics: [],
|
|
2766
|
+
observations: []
|
|
2767
|
+
};
|
|
2768
|
+
const metrics = metricsFor(source, entries);
|
|
2769
|
+
const sessionId = sessionIdFromEntries(source, entries) ?? fallbackSessionId$1(source, options.sourcePath);
|
|
2770
|
+
const observation = observeCodeAgentSession({
|
|
2771
|
+
source,
|
|
2772
|
+
entries,
|
|
2773
|
+
sourcePath: options.sourcePath,
|
|
2774
|
+
execution: options.execution
|
|
2775
|
+
});
|
|
2776
|
+
applyObservedSurfaceMetrics(metrics, observation);
|
|
2777
|
+
const model = withSnapshot(options.model ?? modelFromEntries(source, entries) ?? source);
|
|
2778
|
+
const tokenUsage = {
|
|
2779
|
+
input: metrics.inputTokens,
|
|
2780
|
+
output: metrics.outputTokens,
|
|
2781
|
+
...metrics.reasoningTokens > 0 ? { reasoning: metrics.reasoningTokens } : {},
|
|
2782
|
+
...metrics.cachedTokens > 0 ? { cached: metrics.cachedTokens } : {},
|
|
2783
|
+
...metrics.cacheWriteTokens > 0 ? { cacheWrite: metrics.cacheWriteTokens } : {}
|
|
2784
|
+
};
|
|
2785
|
+
const { costUsd, costProvenance } = resolveSessionCost(options.costProvenance, metrics, model);
|
|
2786
|
+
const qualityScore = finiteScore(options.score);
|
|
2787
|
+
const invalidQualityScore = options.score !== void 0 && qualityScore === void 0;
|
|
2788
|
+
const promptHash = options.promptHash ?? hashString(`prompt:${source}:${firstUserText(entries) ?? ""}`);
|
|
2789
|
+
const configHash = options.configHash ?? hashCanonical({
|
|
2790
|
+
source,
|
|
2791
|
+
model,
|
|
2792
|
+
sourcePath: options.sourcePath ?? null,
|
|
2793
|
+
cwd: cwdFromEntries(entries) ?? null,
|
|
2794
|
+
entryCount: entries.length
|
|
2795
|
+
});
|
|
2796
|
+
const explicitTerminal = observation.terminal.explicit;
|
|
2797
|
+
const warnings = diagnosticsFor(metrics, {
|
|
2798
|
+
model,
|
|
2799
|
+
explicitTerminal,
|
|
2800
|
+
malformedLines: options.malformedLines ?? 0,
|
|
2801
|
+
hasQualityLabel: qualityScore !== void 0,
|
|
2802
|
+
invalidQualityScore,
|
|
2803
|
+
costKind: costProvenance.kind
|
|
2804
|
+
});
|
|
2805
|
+
const terminalOutcome = terminalOutcomeFromSession(observation.terminal);
|
|
2806
|
+
const processExitCode = options.execution?.exitCode;
|
|
2807
|
+
const splitTag = options.splitTag ?? "holdout";
|
|
2817
2808
|
return {
|
|
2818
|
-
|
|
2819
|
-
|
|
2820
|
-
|
|
2821
|
-
|
|
2822
|
-
|
|
2809
|
+
runs: [{
|
|
2810
|
+
runId: `${source}:${sessionId}`,
|
|
2811
|
+
experimentId: options.experimentId ?? `${source}-local-sessions`,
|
|
2812
|
+
candidateId: options.candidateId ?? model,
|
|
2813
|
+
seed: options.seed ?? stableSeed(sessionId),
|
|
2814
|
+
model,
|
|
2815
|
+
promptHash,
|
|
2816
|
+
configHash,
|
|
2817
|
+
commitSha: options.commitSha ?? "local-session",
|
|
2818
|
+
wallMs: metrics.wallMs,
|
|
2819
|
+
costUsd,
|
|
2820
|
+
costProvenance,
|
|
2821
|
+
tokenUsage,
|
|
2822
|
+
terminalOutcome,
|
|
2823
|
+
...terminalOutcome === "failed" ? { terminalFailureReason: processExitCode !== void 0 && processExitCode !== 0 ? `process exited with code ${processExitCode}` : metrics.turnsAborted > 0 ? "turn aborted" : "session reported terminal failure" } : {},
|
|
2824
|
+
outcome: {
|
|
2825
|
+
...qualityScore === void 0 ? {} : splitTag === "holdout" ? { holdoutScore: qualityScore } : { searchScore: qualityScore },
|
|
2826
|
+
raw: {
|
|
2827
|
+
entries: metrics.entries,
|
|
2828
|
+
user_messages: metrics.userMessages,
|
|
2829
|
+
assistant_messages: metrics.assistantMessages,
|
|
2830
|
+
reasoning_items: metrics.reasoningItems,
|
|
2831
|
+
tool_calls: metrics.toolCalls,
|
|
2832
|
+
tool_outputs: metrics.toolOutputs,
|
|
2833
|
+
tool_errors: metrics.toolErrors,
|
|
2834
|
+
execution_error_count: metrics.toolErrors,
|
|
2835
|
+
process_error_count: terminalOutcome === "failed" ? 1 : 0,
|
|
2836
|
+
unclassified_error_count: metrics.unclassifiedErrors,
|
|
2837
|
+
...processExitCode !== void 0 ? { process_exit_code: processExitCode } : {},
|
|
2838
|
+
patch_attempts: metrics.patchAttempts,
|
|
2839
|
+
patch_successes: metrics.patchSuccesses,
|
|
2840
|
+
patch_failures: metrics.patchFailures,
|
|
2841
|
+
turns_started: metrics.turnsStarted,
|
|
2842
|
+
turns_completed: metrics.turnsCompleted,
|
|
2843
|
+
turns_aborted: metrics.turnsAborted,
|
|
2844
|
+
context_compactions: metrics.contextCompactions,
|
|
2845
|
+
mcp_calls: metrics.mcpCalls,
|
|
2846
|
+
subagent_calls: metrics.subagentCalls,
|
|
2847
|
+
skill_calls: metrics.skillCalls,
|
|
2848
|
+
hook_calls: metrics.hookCalls,
|
|
2849
|
+
web_calls: metrics.webCalls,
|
|
2850
|
+
code_actions: metrics.codeActions,
|
|
2851
|
+
pr_links: metrics.prLinks,
|
|
2852
|
+
file_snapshots: metrics.fileSnapshots,
|
|
2853
|
+
graph_nodes: metrics.graphNodes,
|
|
2854
|
+
graph_edges: metrics.graphEdges,
|
|
2855
|
+
action_candidates: metrics.actionCandidates,
|
|
2856
|
+
verification_reports: metrics.verificationReports,
|
|
2857
|
+
completion_decisions: metrics.completionDecisions,
|
|
2858
|
+
reliability_rows: metrics.reliabilityRows,
|
|
2859
|
+
reliability_lift: metrics.reliabilityLift,
|
|
2860
|
+
input_tokens: metrics.inputTokens,
|
|
2861
|
+
output_tokens: metrics.outputTokens,
|
|
2862
|
+
reasoning_tokens: metrics.reasoningTokens,
|
|
2863
|
+
cached_tokens: metrics.cachedTokens,
|
|
2864
|
+
cache_write_tokens: metrics.cacheWriteTokens,
|
|
2865
|
+
observed_cost_usd: metrics.observedCostUsd,
|
|
2866
|
+
observed_cost_captured: metrics.observedCostCaptured ? 1 : 0,
|
|
2867
|
+
estimated_cost_usd: costProvenance.kind === "estimated" ? costProvenance.usd : 0,
|
|
2868
|
+
process_score: metrics.processScore,
|
|
2869
|
+
explicit_terminal_signal: explicitTerminal ? 1 : 0,
|
|
2870
|
+
quality_label_present: qualityScore !== void 0 ? 1 : 0,
|
|
2871
|
+
cost_observed: costProvenance.kind === "observed" ? 1 : 0,
|
|
2872
|
+
cost_estimated: costProvenance.kind === "estimated" ? 1 : 0,
|
|
2873
|
+
cost_uncaptured: costProvenance.kind === "uncaptured" ? 1 : 0,
|
|
2874
|
+
cost_unknown: costProvenance.usd === null ? 1 : 0
|
|
2875
|
+
}
|
|
2876
|
+
},
|
|
2877
|
+
splitTag,
|
|
2878
|
+
scenarioId: options.scenarioId ?? sessionId
|
|
2879
|
+
}],
|
|
2880
|
+
diagnostics: [{
|
|
2881
|
+
source,
|
|
2882
|
+
sessionId,
|
|
2883
|
+
sourcePath: options.sourcePath,
|
|
2884
|
+
entries: entries.length,
|
|
2885
|
+
malformedLines: options.malformedLines ?? 0,
|
|
2886
|
+
hasExplicitTerminalSignal: explicitTerminal,
|
|
2887
|
+
hasFinalOutput: observation.finalText !== null,
|
|
2888
|
+
hasQualityLabel: qualityScore !== void 0,
|
|
2889
|
+
hasTokenUsage: metrics.inputTokens > 0 || metrics.outputTokens > 0 || metrics.reasoningTokens > 0 || metrics.cachedTokens > 0 || metrics.cacheWriteTokens > 0,
|
|
2890
|
+
hasCost: costProvenance.usd !== null,
|
|
2891
|
+
costKind: costProvenance.kind,
|
|
2892
|
+
warnings
|
|
2893
|
+
}],
|
|
2894
|
+
metrics: [metrics],
|
|
2895
|
+
observations: [{
|
|
2896
|
+
...observation,
|
|
2897
|
+
sessionId
|
|
2898
|
+
}]
|
|
2823
2899
|
};
|
|
2824
2900
|
}
|
|
2825
|
-
function
|
|
2826
|
-
if (explicit)
|
|
2827
|
-
|
|
2828
|
-
|
|
2829
|
-
|
|
2830
|
-
|
|
2831
|
-
|
|
2832
|
-
|
|
2833
|
-
|
|
2834
|
-
|
|
2835
|
-
|
|
2836
|
-
|
|
2837
|
-
|
|
2838
|
-
|
|
2839
|
-
costProvenance: explicit
|
|
2840
|
-
};
|
|
2841
|
-
}
|
|
2842
|
-
if (!Number.isFinite(explicit.usd) || explicit.usd < 0) throw new Error(`${explicit.kind} cost must be a finite, non-negative USD amount`);
|
|
2843
|
-
return {
|
|
2844
|
-
costUsd: explicit.usd,
|
|
2845
|
-
costProvenance: explicit
|
|
2846
|
-
};
|
|
2847
|
-
}
|
|
2848
|
-
if (metrics.observedCostCaptured) {
|
|
2849
|
-
const costProvenance = {
|
|
2850
|
-
kind: "observed",
|
|
2851
|
-
usd: metrics.observedCostUsd
|
|
2852
|
-
};
|
|
2853
|
-
return {
|
|
2854
|
-
costUsd: metrics.observedCostUsd,
|
|
2855
|
-
costProvenance
|
|
2856
|
-
};
|
|
2857
|
-
}
|
|
2858
|
-
if ((metrics.inputTokens > 0 || metrics.outputTokens > 0) && isModelPriced(model)) {
|
|
2859
|
-
const costUsd = estimateCost(metrics.inputTokens, metrics.outputTokens, model);
|
|
2860
|
-
return {
|
|
2861
|
-
costUsd,
|
|
2862
|
-
costProvenance: {
|
|
2863
|
-
kind: "estimated",
|
|
2864
|
-
usd: costUsd
|
|
2865
|
-
}
|
|
2866
|
-
};
|
|
2901
|
+
function terminalOutcomeFromSession(terminal) {
|
|
2902
|
+
if (!terminal.explicit) return "unknown";
|
|
2903
|
+
if (terminal.status === "completed") return "succeeded";
|
|
2904
|
+
if (terminal.status === "failed") return "failed";
|
|
2905
|
+
return "unknown";
|
|
2906
|
+
}
|
|
2907
|
+
function metricsFor(source, entries) {
|
|
2908
|
+
switch (source) {
|
|
2909
|
+
case "codex": return codexMetrics(entries);
|
|
2910
|
+
case "claude-code": return claudeCodeMetrics(entries);
|
|
2911
|
+
case "opencode": return openCodeMetrics(entries);
|
|
2912
|
+
case "kimi-code": return kimiCodeMetrics(entries);
|
|
2913
|
+
case "pi": return piSessionMetrics(entries);
|
|
2914
|
+
case "pi-graph": return piGraphMetrics(entries);
|
|
2867
2915
|
}
|
|
2868
|
-
return {
|
|
2869
|
-
costUsd: null,
|
|
2870
|
-
costProvenance: {
|
|
2871
|
-
kind: "uncaptured",
|
|
2872
|
-
usd: null
|
|
2873
|
-
}
|
|
2874
|
-
};
|
|
2875
2916
|
}
|
|
2876
2917
|
function diagnosticsFor(metrics, options) {
|
|
2877
2918
|
const warnings = [];
|
|
@@ -2895,58 +2936,6 @@ function applyObservedSurfaceMetrics(metrics, observation) {
|
|
|
2895
2936
|
metrics.webCalls = actions.filter((action) => action.surface === "web").length;
|
|
2896
2937
|
metrics.codeActions = actions.filter((action) => action.surface === "code").length;
|
|
2897
2938
|
}
|
|
2898
|
-
function sessionIdFromEntries(source, entries) {
|
|
2899
|
-
for (const entry of entries) {
|
|
2900
|
-
if (source === "pi" && entry.type === "session") {
|
|
2901
|
-
const id = stringField(entry, "id");
|
|
2902
|
-
if (id) return id;
|
|
2903
|
-
}
|
|
2904
|
-
if (source === "codex") {
|
|
2905
|
-
const threadId = stringField(entry, "thread_id");
|
|
2906
|
-
if (threadId) return threadId;
|
|
2907
|
-
const payload = record(entry.payload);
|
|
2908
|
-
const id = payload ? stringField(payload, "id") : void 0;
|
|
2909
|
-
if (id) return id;
|
|
2910
|
-
}
|
|
2911
|
-
const payload = record(record(entry.message)?.payload);
|
|
2912
|
-
const sessionId = stringField(entry, "sessionID") ?? stringField(entry, "sessionId") ?? stringField(entry, "session_id") ?? stringField(payload ?? {}, "session_id");
|
|
2913
|
-
if (sessionId) return sessionId;
|
|
2914
|
-
}
|
|
2915
|
-
}
|
|
2916
|
-
function modelFromEntries(source, entries) {
|
|
2917
|
-
if (source === "pi") return summarizeSessionInput("pi", { records: entries }).lastServedModel ?? void 0;
|
|
2918
|
-
let providerModel;
|
|
2919
|
-
for (const entry of entries) {
|
|
2920
|
-
if (source === "codex") {
|
|
2921
|
-
const payload = record(entry.payload);
|
|
2922
|
-
const model = payload ? stringField(payload, "model") : void 0;
|
|
2923
|
-
if (model) return model;
|
|
2924
|
-
providerModel = providerModel ?? (payload ? stringField(payload, "model_provider") : void 0);
|
|
2925
|
-
}
|
|
2926
|
-
const openCodeModel = stringField(entry, "modelID");
|
|
2927
|
-
if (openCodeModel) {
|
|
2928
|
-
const provider = stringField(entry, "providerID");
|
|
2929
|
-
return provider ? `${provider}/${openCodeModel}` : openCodeModel;
|
|
2930
|
-
}
|
|
2931
|
-
const modelObject = record(entry.model);
|
|
2932
|
-
const nestedModel = stringField(modelObject ?? {}, "modelID");
|
|
2933
|
-
if (nestedModel) {
|
|
2934
|
-
const provider = stringField(modelObject ?? {}, "providerID");
|
|
2935
|
-
return provider ? `${provider}/${nestedModel}` : nestedModel;
|
|
2936
|
-
}
|
|
2937
|
-
const message = record(entry.message);
|
|
2938
|
-
const model = message ? stringField(message, "model") : void 0;
|
|
2939
|
-
if (model) return model;
|
|
2940
|
-
}
|
|
2941
|
-
return providerModel;
|
|
2942
|
-
}
|
|
2943
|
-
function cwdFromEntries(entries) {
|
|
2944
|
-
for (const entry of entries) {
|
|
2945
|
-
const path = record(entry.path);
|
|
2946
|
-
const cwd = stringField(entry, "cwd") ?? stringField(path ?? {}, "cwd") ?? stringField(path ?? {}, "root") ?? (record(entry.payload) ? stringField(record(entry.payload), "cwd") : void 0);
|
|
2947
|
-
if (cwd) return cwd;
|
|
2948
|
-
}
|
|
2949
|
-
}
|
|
2950
2939
|
/** Join the `text` of every part in a `[{ type: 'text', text }]` content list. */
|
|
2951
2940
|
function textFromContentParts(content) {
|
|
2952
2941
|
if (!Array.isArray(content)) return void 0;
|
|
@@ -2981,54 +2970,9 @@ function firstUserText(entries) {
|
|
|
2981
2970
|
}
|
|
2982
2971
|
}
|
|
2983
2972
|
}
|
|
2984
|
-
function fallbackSessionId(source, sourcePath) {
|
|
2985
|
-
return hashString(`${source}:${sourcePath ?? "unknown-session"}`).slice(7, 27);
|
|
2986
|
-
}
|
|
2987
|
-
function withSnapshot(model) {
|
|
2988
|
-
const trimmed = model.trim() || "unknown";
|
|
2989
|
-
if (trimmed.includes("@") || /-\d{8}$/.test(trimmed) || /-\d{4}-\d{2}-\d{2}$/.test(trimmed)) return trimmed;
|
|
2990
|
-
return `${trimmed}@observed-local`;
|
|
2991
|
-
}
|
|
2992
|
-
function hashJson(value) {
|
|
2993
|
-
return hashCanonical(value);
|
|
2994
|
-
}
|
|
2995
|
-
function hashString(value) {
|
|
2996
|
-
return `sha256:${createHash("sha256").update(value).digest("hex")}`;
|
|
2997
|
-
}
|
|
2998
|
-
function stableSeed(value) {
|
|
2999
|
-
return createHash("sha256").update(value).digest().readUInt32BE(0);
|
|
3000
|
-
}
|
|
3001
|
-
function maxOptional(current, next) {
|
|
3002
|
-
return current === void 0 ? next : Math.max(current, next);
|
|
3003
|
-
}
|
|
3004
|
-
function clamp01(value) {
|
|
3005
|
-
if (!Number.isFinite(value)) return 0;
|
|
3006
|
-
return Math.max(0, Math.min(1, value));
|
|
3007
|
-
}
|
|
3008
2973
|
function finiteScore(value) {
|
|
3009
2974
|
return value !== void 0 && Number.isFinite(value) ? clamp01(value) : void 0;
|
|
3010
2975
|
}
|
|
3011
|
-
function isRecord(value) {
|
|
3012
|
-
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
3013
|
-
}
|
|
3014
|
-
function record(value) {
|
|
3015
|
-
return isRecord(value) ? value : null;
|
|
3016
|
-
}
|
|
3017
|
-
function stringField(obj, key) {
|
|
3018
|
-
const value = obj[key];
|
|
3019
|
-
return typeof value === "string" && value.length > 0 ? value : void 0;
|
|
3020
|
-
}
|
|
3021
|
-
function numberField(obj, key) {
|
|
3022
|
-
const value = obj[key];
|
|
3023
|
-
return typeof value === "number" && Number.isFinite(value) ? value : void 0;
|
|
3024
|
-
}
|
|
3025
|
-
function timestampMs(value) {
|
|
3026
|
-
if (typeof value === "number" && Number.isFinite(value)) return value > 0xe8d4a51000 ? value : value * 1e3;
|
|
3027
|
-
if (typeof value === "string" && value.length > 0) {
|
|
3028
|
-
const parsed = Date.parse(value);
|
|
3029
|
-
return Number.isFinite(parsed) ? parsed : void 0;
|
|
3030
|
-
}
|
|
3031
|
-
}
|
|
3032
2976
|
//#endregion
|
|
3033
2977
|
//#region src/contract/intake/code-agent-store.ts
|
|
3034
2978
|
const SESSION_SUFFIX = ".jsonl";
|