@tangle-network/agent-eval 0.216.1 → 0.218.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +137 -57
- package/README.md +2 -2
- package/dist/adapters/http.d.ts +2 -2
- package/dist/{agent-profile-VFcQncbW.d.ts → agent-profile-DTt8uGP9.d.ts} +4 -17
- package/dist/agent-profile-DTt8uGP9.d.ts.map +1 -0
- package/dist/{agent-profile-cell-BK8mlqlq.js → agent-profile-cell-B1V1gUZs.js} +4 -53
- package/dist/agent-profile-cell-B1V1gUZs.js.map +1 -0
- package/dist/{agent-profile-cell-BNp-Js9F.d.ts → agent-profile-cell-BcpFbNd5.d.ts} +2 -22
- package/dist/agent-profile-cell-BcpFbNd5.d.ts.map +1 -0
- package/dist/{agreement-irr-BEWLIF3_.js → agreement-irr-BhAqjluA.js} +5 -5
- package/dist/agreement-irr-BhAqjluA.js.map +1 -0
- package/dist/analyst/index.d.ts +36 -169
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +17 -192
- package/dist/analyst/index.js.map +1 -1
- package/dist/{analyze-runs-D68fd-su.d.ts → analyze-runs-DjlZOVXx.d.ts} +5 -5
- package/dist/{analyze-runs-D68fd-su.d.ts.map → analyze-runs-DjlZOVXx.d.ts.map} +1 -1
- package/dist/{analyze-runs-C84irm4i.js → analyze-runs-DqPJaJ4X.js} +225 -21
- package/dist/analyze-runs-DqPJaJ4X.js.map +1 -0
- package/dist/authenticity/index.d.ts +0 -23
- package/dist/authenticity/index.d.ts.map +1 -1
- package/dist/authenticity/index.js +1 -34
- package/dist/authenticity/index.js.map +1 -1
- package/dist/{benchmark-command-By0vh8q7.js → benchmark-command-BupdHRyf.js} +1225 -661
- package/dist/benchmark-command-BupdHRyf.js.map +1 -0
- package/dist/benchmarks/index.d.ts +5 -6
- package/dist/benchmarks/index.d.ts.map +1 -1
- package/dist/benchmarks/index.js +4 -4
- package/dist/benchmarks/index.js.map +1 -1
- package/dist/builder-eval/index.d.ts +3 -3
- package/dist/builder-eval/index.js +2 -2
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/campaign/index.d.ts +7 -7
- package/dist/campaign/index.js +15 -15
- package/dist/{campaign-BudZ70zo.js → campaign-CGvUTnmc.js} +324 -800
- package/dist/campaign-CGvUTnmc.js.map +1 -0
- package/dist/{campaign-evidence-Clrn4hBV.js → campaign-evidence-DunzcJGv.js} +10 -18
- package/dist/{campaign-evidence-Clrn4hBV.js.map → campaign-evidence-DunzcJGv.js.map} +1 -1
- package/dist/{canonical-B2RqLofv.js → canonical-Dq2hOQ8L.js} +2 -2
- package/dist/{canonical-B2RqLofv.js.map → canonical-Dq2hOQ8L.js.map} +1 -1
- package/dist/cell-spend-DEWR3nj-.js +90 -0
- package/dist/cell-spend-DEWR3nj-.js.map +1 -0
- package/dist/{chat-client-l-MVDlDk.js → chat-client-B2pXlPdx.js} +2 -2
- package/dist/{chat-client-l-MVDlDk.js.map → chat-client-B2pXlPdx.js.map} +1 -1
- package/dist/{chat-json-call-puMArmtW.js → chat-json-call-Dm9c-b41.js} +3 -2
- package/dist/chat-json-call-Dm9c-b41.js.map +1 -0
- package/dist/{claim-q4HQhK1G.js → claim-BPVuyHuy.js} +3 -3
- package/dist/{claim-q4HQhK1G.js.map → claim-BPVuyHuy.js.map} +1 -1
- package/dist/cli.js +113 -12
- package/dist/cli.js.map +1 -1
- package/dist/{client-BzL9niqi.d.ts → client-hwo3cJjR.d.ts} +3 -16
- package/dist/client-hwo3cJjR.d.ts.map +1 -0
- package/dist/{cluster-bootstrap-qcUWYFl-.js → cluster-bootstrap-DEwwhl0C.js} +3 -3
- package/dist/{cluster-bootstrap-qcUWYFl-.js.map → cluster-bootstrap-DEwwhl0C.js.map} +1 -1
- package/dist/{produced-state-BsDmnwxc.js → completion-verifier-DbLVf3i8.js} +9 -64
- package/dist/completion-verifier-DbLVf3i8.js.map +1 -0
- package/dist/contract/index.d.ts +12 -12
- package/dist/contract/index.js +107 -403
- package/dist/contract/index.js.map +1 -1
- package/dist/{cost-ledger-BP8V4kUQ.js → cost-ledger-D0352Z4d.js} +4 -8
- package/dist/cost-ledger-D0352Z4d.js.map +1 -0
- package/dist/dataset-DQqhOCPt.d.ts.map +1 -1
- package/dist/{default-registry-COhk9g36.js → default-registry-BJ-Pq6yL.js} +314 -285
- package/dist/default-registry-BJ-Pq6yL.js.map +1 -0
- package/dist/{default-registry-DzHiGruW.d.ts → default-registry-DXgLmoDI.d.ts} +7 -50
- package/dist/default-registry-DXgLmoDI.d.ts.map +1 -0
- package/dist/{define-agent-eval-CNTPzIkp.js → define-agent-eval-BGdFKlrc.js} +170 -110
- package/dist/define-agent-eval-BGdFKlrc.js.map +1 -0
- package/dist/{define-agent-eval-WSck0CZJ.d.ts → define-agent-eval-BlH10zEw.d.ts} +8 -8
- package/dist/define-agent-eval-BlH10zEw.d.ts.map +1 -0
- package/dist/{descriptive-DboxPZrG.js → descriptive-CVNy48rR.js} +2 -2
- package/dist/{descriptive-DboxPZrG.js.map → descriptive-CVNy48rR.js.map} +1 -1
- package/dist/diagnosis/index.d.ts +4 -14
- package/dist/diagnosis/index.d.ts.map +1 -1
- package/dist/diagnosis/index.js +67 -28
- package/dist/diagnosis/index.js.map +1 -1
- package/dist/{dspy-rlm-engine-LRCu61m5.js → dspy-rlm-engine-CSMwudLO.js} +5 -10
- package/dist/dspy-rlm-engine-CSMwudLO.js.map +1 -0
- package/dist/{emitter-HcCmFQl_.d.ts → emitter-DABYYTcH.d.ts} +5 -5
- package/dist/emitter-DABYYTcH.d.ts.map +1 -0
- package/dist/emitter-DbMXUIVP.js.map +1 -1
- package/dist/{engine-3A_4jOlI.d.ts → engine-BvZ0R-36.d.ts} +17 -21
- package/dist/engine-BvZ0R-36.d.ts.map +1 -0
- package/dist/{equal-terms-BtdXayCJ.js → equal-terms-DbPGcVtI.js} +2 -2
- package/dist/equal-terms-DbPGcVtI.js.map +1 -0
- package/dist/{errors-DVZiqS7K.js → errors-BcOs5MKW.js} +2 -8
- package/dist/{errors-DVZiqS7K.js.map → errors-BcOs5MKW.js.map} +1 -1
- package/dist/eval-campaign-BUMIxKTE.js +393 -0
- package/dist/eval-campaign-BUMIxKTE.js.map +1 -0
- package/dist/{evaluation-6Yoi74gM.d.ts → evaluation-BOZ38vOH.d.ts} +7 -7
- package/dist/evaluation-BOZ38vOH.d.ts.map +1 -0
- package/dist/evaluation.d.ts +2 -2
- package/dist/evaluation.js +1 -1
- package/dist/evaluation.js.map +1 -1
- package/dist/{exact-types-ClugMJfI.d.ts → exact-types-CflbJ9f3.d.ts} +2 -2
- package/dist/{exact-types-ClugMJfI.d.ts.map → exact-types-CflbJ9f3.d.ts.map} +1 -1
- package/dist/{exec-ChJ6d5if.js → exec-CpbtOXVt.js} +2 -24
- package/dist/{exec-ChJ6d5if.js.map → exec-CpbtOXVt.js.map} +1 -1
- package/dist/execution-tracks-CpgFPpS5.js.map +1 -1
- package/dist/experiment/index.d.ts +9 -112
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +258 -311
- package/dist/experiment/index.js.map +1 -1
- package/dist/{exporters-X4xwla8A.js → exporters-DWvM16LE.js} +3 -3
- package/dist/exporters-DWvM16LE.js.map +1 -0
- package/dist/{exporters-ChTnR90l.d.ts → exporters-Dk3oEbK_.d.ts} +2 -4
- package/dist/{exporters-ChTnR90l.d.ts.map → exporters-Dk3oEbK_.d.ts.map} +1 -1
- package/dist/{external-optimizer-process-BjMGUuds.js → external-optimizer-process-C3Wm7Vhn.js} +4 -3
- package/dist/external-optimizer-process-C3Wm7Vhn.js.map +1 -0
- package/dist/{external-optimizer-subprocess-BqW0hHcI.js → external-optimizer-subprocess-B5gdLRZv.js} +9 -11
- package/dist/external-optimizer-subprocess-B5gdLRZv.js.map +1 -0
- package/dist/{extract-usage-BrnzRJ6H.js → extract-usage-BZLU01kR.js} +2 -23
- package/dist/{extract-usage-BrnzRJ6H.js.map → extract-usage-BZLU01kR.js.map} +1 -1
- package/dist/{failure-cluster-CbgEDcFJ.d.ts → failure-cluster-DN0t63Z0.d.ts} +4 -43
- package/dist/failure-cluster-DN0t63Z0.d.ts.map +1 -0
- package/dist/{failure-taxonomy-CjabTATj.js → failure-taxonomy-Bx5oWqaW.js} +2 -19
- package/dist/{failure-taxonomy-CjabTATj.js.map → failure-taxonomy-Bx5oWqaW.js.map} +1 -1
- package/dist/{feedback-trajectory-C4EG0nNH.d.ts → feedback-trajectory-BWmhRBGu.d.ts} +7 -9
- package/dist/feedback-trajectory-BWmhRBGu.d.ts.map +1 -0
- package/dist/{semantic-concept-judge-BQF8te0T.js → findings-store-BtTzPL0d.js} +9 -249
- package/dist/findings-store-BtTzPL0d.js.map +1 -0
- package/dist/fuzz.d.ts +19 -62
- package/dist/fuzz.d.ts.map +1 -1
- package/dist/fuzz.js +62 -108
- package/dist/fuzz.js.map +1 -1
- package/dist/{hf-dataset-BR2yhuNn.js → hf-dataset-CKxIJQba.js} +4 -24
- package/dist/hf-dataset-CKxIJQba.js.map +1 -0
- package/dist/hosted/index.d.ts +2 -2
- package/dist/hosted/index.js +2 -2
- package/dist/{index-CyszSg4O.d.ts → index-B16FNdW5.d.ts} +145 -360
- package/dist/index-B16FNdW5.d.ts.map +1 -0
- package/dist/{index-DV-jSzxF.d.ts → index-BLGqZ81N.d.ts} +10 -15
- package/dist/{index-DV-jSzxF.d.ts.map → index-BLGqZ81N.d.ts.map} +1 -1
- package/dist/{index-BO41eGj5.d.ts → index-CX-GAUpw.d.ts} +3 -7
- package/dist/index-CX-GAUpw.d.ts.map +1 -0
- package/dist/{index-DHKEAv3s.d.ts → index-CalAsXMR.d.ts} +8 -68
- package/dist/index-CalAsXMR.d.ts.map +1 -0
- package/dist/{index-BFRO6chs.d.ts → index-CiRwLv2R.d.ts} +5 -69
- package/dist/index-CiRwLv2R.d.ts.map +1 -0
- package/dist/{index-DPcngtwt.d.ts → index-DbMOXYnP.d.ts} +11 -36
- package/dist/index-DbMOXYnP.d.ts.map +1 -0
- package/dist/index.d.ts +211 -218
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +999 -1714
- package/dist/index.js.map +1 -1
- package/dist/{integrity-dLiDe2kD.js → integrity-C8oUeB0g.js} +326 -203
- package/dist/integrity-C8oUeB0g.js.map +1 -0
- package/dist/{integrity-BXOsAb3i.js → integrity-W4CADcFc.js} +3 -7
- package/dist/{integrity-BXOsAb3i.js.map → integrity-W4CADcFc.js.map} +1 -1
- package/dist/{integrity-DAMEDFaC.d.ts → integrity-_qgZLA5p.d.ts} +4 -6
- package/dist/{integrity-DAMEDFaC.d.ts.map → integrity-_qgZLA5p.d.ts.map} +1 -1
- package/dist/{internal-Epl-YBZW.js → internal-BTK91Bqt.js} +3 -3
- package/dist/{internal-Epl-YBZW.js.map → internal-BTK91Bqt.js.map} +1 -1
- package/dist/{interval-CELpye1n.js → interval-Dq9gt7jx.js} +3 -3
- package/dist/{interval-CELpye1n.js.map → interval-Dq9gt7jx.js.map} +1 -1
- package/dist/{journal-DfRz9kps.js → journal-C7om8Gum.js} +157 -147
- package/dist/journal-C7om8Gum.js.map +1 -0
- package/dist/json-BhdsJaOQ.js +111 -0
- package/dist/json-BhdsJaOQ.js.map +1 -0
- package/dist/{judge-calibration-BpCy5XQl.js → judge-calibration-BkXMIWA4.js} +3 -3
- package/dist/{judge-calibration-BpCy5XQl.js.map → judge-calibration-BkXMIWA4.js.map} +1 -1
- package/dist/{judge-gate-iY9s4L44.js → judge-gate-Co5tuh57.js} +4 -27
- package/dist/{judge-gate-iY9s4L44.js.map → judge-gate-Co5tuh57.js.map} +1 -1
- package/dist/{judge-gate-CE7CQAgt.d.ts → judge-gate-DsL0yFcb.d.ts} +8 -9
- package/dist/judge-gate-DsL0yFcb.d.ts.map +1 -0
- package/dist/{kind-factory-BobI5Jy3.js → kind-factory-Gy4fXixd.js} +88 -17
- package/dist/kind-factory-Gy4fXixd.js.map +1 -0
- package/dist/ledger-core/index.d.ts +2 -2
- package/dist/ledger-core/index.js +4 -4
- package/dist/ledger-core/lock-holder.d.ts +2 -0
- package/dist/ledger-core/lock-holder.js +181 -0
- package/dist/ledger-core/lock-holder.js.map +1 -0
- package/dist/{ledger-core-Gq7yp8lP.js → ledger-core-CAxUdDk0.js} +5 -84
- package/dist/ledger-core-CAxUdDk0.js.map +1 -0
- package/dist/{llm-client-CyBAoYiX.js → llm-client-_Y6ic-tf.js} +243 -355
- package/dist/llm-client-_Y6ic-tf.js.map +1 -0
- package/dist/{llm-judge-CG12SQJq.js → llm-judge-Djxh-Uhs.js} +662 -482
- package/dist/llm-judge-Djxh-Uhs.js.map +1 -0
- package/dist/lock-holder-DaJbfpci.d.ts +86 -0
- package/dist/lock-holder-DaJbfpci.d.ts.map +1 -0
- package/dist/matrix/index.d.ts +3 -3
- package/dist/matrix/index.js +3 -2
- package/dist/{matrix-Dm0BQW7U.js → matrix-B8l-Nsxd.js} +3 -88
- package/dist/matrix-B8l-Nsxd.js.map +1 -0
- package/dist/{matrix-DSrSxjjy.d.ts → matrix-CCSBna4q.d.ts} +9 -18
- package/dist/matrix-CCSBna4q.d.ts.map +1 -0
- package/dist/{mean-DHoMiudD.js → mean-BbA_JCzW.js} +39 -5
- package/dist/mean-BbA_JCzW.js.map +1 -0
- package/dist/meta-eval/index.d.ts +5 -5
- package/dist/meta-eval/index.js +10 -10
- package/dist/metrics-BKaPBtIG.js.map +1 -1
- package/dist/{mint-DNU8QPG4.d.ts → mint-BRZXuwRi.d.ts} +4 -4
- package/dist/{mint-DNU8QPG4.d.ts.map → mint-BRZXuwRi.d.ts.map} +1 -1
- package/dist/{mint-QaD5cA0u.js → mint-CbdwjTPW.js} +4 -4
- package/dist/{mint-QaD5cA0u.js.map → mint-CbdwjTPW.js.map} +1 -1
- package/dist/{multi-layer-verifier-BUaQ4C17.d.ts → multi-layer-verifier-DEgMUk6v.d.ts} +2 -2
- package/dist/{multi-layer-verifier-BUaQ4C17.d.ts.map → multi-layer-verifier-DEgMUk6v.d.ts.map} +1 -1
- package/dist/{multiplicity-DQUFcbW4.js → multiplicity-Sg0U7h8j.js} +2 -2
- package/dist/{multiplicity-DQUFcbW4.js.map → multiplicity-Sg0U7h8j.js.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -23
- package/dist/multishot/golden/index.d.ts.map +1 -1
- package/dist/multishot/golden/index.js +3 -3
- package/dist/multishot/golden/index.js.map +1 -1
- package/dist/multishot/index.d.ts +3 -12
- package/dist/multishot/index.d.ts.map +1 -1
- package/dist/multishot/index.js +4 -3
- package/dist/multishot/index.js.map +1 -1
- package/dist/openapi.json +114 -10
- package/dist/{otel-spans-B9tUXRHF.js → otel-spans-CdN909k1.js} +3 -3
- package/dist/{otel-spans-B9tUXRHF.js.map → otel-spans-CdN909k1.js.map} +1 -1
- package/dist/{paired-arms-CwMGJsrS.js → paired-arms-Ct4LTKc1.js} +32 -9
- package/dist/paired-arms-Ct4LTKc1.js.map +1 -0
- package/dist/{paired-binary-CvpFHfKZ.js → paired-binary-B_nejA44.js} +3 -36
- package/dist/paired-binary-B_nejA44.js.map +1 -0
- package/dist/{paired-promotion-decision-DPKz8ZnF.d.ts → paired-promotion-decision-CQMJ5xGN.d.ts} +2 -34
- package/dist/paired-promotion-decision-CQMJ5xGN.d.ts.map +1 -0
- package/dist/{paired-promotion-decision-DsAPYGCm.js → paired-promotion-decision-D0SHb-ZL.js} +3 -3
- package/dist/{paired-promotion-decision-DsAPYGCm.js.map → paired-promotion-decision-D0SHb-ZL.js.map} +1 -1
- package/dist/{pareto-Cm6ndrWN.js → pareto-CTv8vozZ.js} +11 -11
- package/dist/pareto-CTv8vozZ.js.map +1 -0
- package/dist/pipelines/index.d.ts +16 -19
- package/dist/pipelines/index.d.ts.map +1 -1
- package/dist/pipelines/index.js +4 -68
- package/dist/pipelines/index.js.map +1 -1
- package/dist/{power-and-mde-Hj2ubb4B.js → power-and-mde-N5nU4_Ly.js} +3 -3
- package/dist/{power-and-mde-Hj2ubb4B.js.map → power-and-mde-N5nU4_Ly.js.map} +1 -1
- package/dist/{prime-protocol-CHKgA3a6.js → prime-protocol-BFgGrEMP.js} +3 -3
- package/dist/prime-protocol-BFgGrEMP.js.map +1 -0
- package/dist/{prime-protocol-HiB9HX7E.d.ts → prime-protocol-BWG2DQIk.d.ts} +3 -21
- package/dist/prime-protocol-BWG2DQIk.d.ts.map +1 -0
- package/dist/profile-cell.d.ts +2 -2
- package/dist/profile-cell.js +2 -2
- package/dist/{promotion-gate-BawrdPYr.d.ts → promotion-gate-CxEHRsZW.d.ts} +7 -8
- package/dist/{promotion-gate-BawrdPYr.d.ts.map → promotion-gate-CxEHRsZW.d.ts.map} +1 -1
- package/dist/{promotion-policy-KO57gbyV.js → promotion-policy-6A8osGr_.js} +4 -4
- package/dist/{promotion-policy-KO57gbyV.js.map → promotion-policy-6A8osGr_.js.map} +1 -1
- package/dist/{proposal-findings-CY1xKn_a.js → proposal-findings-n-5WBbEx.js} +3 -3
- package/dist/proposal-findings-n-5WBbEx.js.map +1 -0
- package/dist/{query-BUCZCK7F.js → query-B_RZ36an.js} +5 -5
- package/dist/query-B_RZ36an.js.map +1 -0
- package/dist/{query-DK1nWcmp.d.ts → query-Cmv_U8vf.d.ts} +4 -24
- package/dist/{query-DK1nWcmp.d.ts.map → query-Cmv_U8vf.d.ts.map} +1 -1
- package/dist/{random-ByUSnV9w.js → random-DDDcC7YU.js} +2 -2
- package/dist/{random-ByUSnV9w.js.map → random-DDDcC7YU.js.map} +1 -1
- package/dist/{raw-provider-sink-DoHz4JNG.js → raw-provider-sink-75w4FWBj.js} +2 -2
- package/dist/{raw-provider-sink-DoHz4JNG.js.map → raw-provider-sink-75w4FWBj.js.map} +1 -1
- package/dist/{redact-mtCCcT8z.d.ts → redact-DXdlFb_U.d.ts} +2 -3
- package/dist/redact-DXdlFb_U.d.ts.map +1 -0
- package/dist/{redact-B5xXKqQ3.js → redact-mtkTiJVk.js} +2 -7
- package/dist/{redact-B5xXKqQ3.js.map → redact-mtkTiJVk.js.map} +1 -1
- package/dist/{registry-D7_hm2fK.d.ts → registry-ssNoHzwD.d.ts} +4 -4
- package/dist/{registry-D7_hm2fK.d.ts.map → registry-ssNoHzwD.d.ts.map} +1 -1
- package/dist/{release-confidence-CpuP65S5.js → release-confidence-CYIFH-fp.js} +6 -11
- package/dist/release-confidence-CYIFH-fp.js.map +1 -0
- package/dist/{report-command-SC9nqb4M.js → render-Cs34y0iX.js} +625 -587
- package/dist/render-Cs34y0iX.js.map +1 -0
- package/dist/reporting.d.ts +6 -6
- package/dist/reporting.js +8 -8
- package/dist/{researcher-CEcfku-T.d.ts → researcher-GU1tV4dn.d.ts} +9 -9
- package/dist/{researcher-CEcfku-T.d.ts.map → researcher-GU1tV4dn.d.ts.map} +1 -1
- package/dist/{reward-CpCmAM8L.d.ts → reward-DrZ71vnf.d.ts} +3 -26
- package/dist/reward-DrZ71vnf.d.ts.map +1 -0
- package/dist/{reward-hacking-E0d-WVcP.js → reward-hacking-ByuETaIb.js} +3 -3
- package/dist/{reward-hacking-E0d-WVcP.js.map → reward-hacking-ByuETaIb.js.map} +1 -1
- package/dist/{reward-hacking-RVdZ6Kkt.d.ts → reward-hacking-E52wq86n.d.ts} +5 -18
- package/dist/reward-hacking-E52wq86n.d.ts.map +1 -0
- package/dist/reward-nw2xZGZG.js.map +1 -1
- package/dist/rl.d.ts +117 -295
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +27 -747
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +5 -5
- package/dist/rollout/index.js +7 -8
- package/dist/{rollout-DZV1t2Wg.js → rollout-Dd1ucT_M.js} +88 -13
- package/dist/rollout-Dd1ucT_M.js.map +1 -0
- package/dist/{rubric-predictive-validity-DaFW70y9.js → rubric-predictive-validity-D3_oI72N.js} +3 -3
- package/dist/{rubric-predictive-validity-DaFW70y9.js.map → rubric-predictive-validity-D3_oI72N.js.map} +1 -1
- package/dist/{rubric-predictive-validity-C03EC7ZE.d.ts → rubric-predictive-validity-DiwT8PbL.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-C03EC7ZE.d.ts.map → rubric-predictive-validity-DiwT8PbL.d.ts.map} +1 -1
- package/dist/{run-record-C1E24G2A.d.ts → run-record-CaxbLq6Y.d.ts} +4 -10
- package/dist/run-record-CaxbLq6Y.d.ts.map +1 -0
- package/dist/{run-record-C5e7FjhJ.js → run-record-CysKLB93.js} +3 -3
- package/dist/run-record-CysKLB93.js.map +1 -0
- package/dist/{run-record-Boc5IUTp.js → run-record-DhEJgoPz.js} +4 -4
- package/dist/{run-record-Boc5IUTp.js.map → run-record-DhEJgoPz.js.map} +1 -1
- package/dist/sandbox-harness-BlSOu4LX.d.ts.map +1 -1
- package/dist/{schema-CSf6qWgZ.js → schema-Bt2vLFm3.js} +2 -14
- package/dist/schema-Bt2vLFm3.js.map +1 -0
- package/dist/schema-C-ziyJD7.d.ts.map +1 -1
- package/dist/{schema-BuiujZ-g.js → schema-CvhfLjtZ.js} +78 -48
- package/dist/schema-CvhfLjtZ.js.map +1 -0
- package/dist/{schema-2g7trUwH.d.ts → schema-D6EVKzTB.d.ts} +4 -5
- package/dist/schema-D6EVKzTB.d.ts.map +1 -0
- package/dist/search/index.d.ts +3 -31
- package/dist/search/index.d.ts.map +1 -1
- package/dist/search/index.js +4 -4
- package/dist/{search-kernel-DyNEa6IU.d.ts → search-kernel-p9JB3SmL.d.ts} +10 -27
- package/dist/search-kernel-p9JB3SmL.d.ts.map +1 -0
- package/dist/{search-ledger-CVUGLDZp.js → search-ledger-ClpN4I3e.js} +7 -7
- package/dist/{search-ledger-CVUGLDZp.js.map → search-ledger-ClpN4I3e.js.map} +1 -1
- package/dist/{search-ledger-file-DDQUgvCg.js → search-ledger-file-Bd5Tioq6.js} +4 -4
- package/dist/search-ledger-file-Bd5Tioq6.js.map +1 -0
- package/dist/{search-shipper-BOSD46qv.js → search-shipper-Bg2HW3Uv.js} +4 -4
- package/dist/search-shipper-Bg2HW3Uv.js.map +1 -0
- package/dist/{search-summary-BQCSWf1g.js → search-summary-CrBuvVnf.js} +5 -5
- package/dist/{search-summary-BQCSWf1g.js.map → search-summary-CrBuvVnf.js.map} +1 -1
- package/dist/{sequential-CzK5DarL.js → sequential-BVv1sjsw.js} +2 -32
- package/dist/{sequential-CzK5DarL.js.map → sequential-BVv1sjsw.js.map} +1 -1
- package/dist/{sequential-Bf_pbE50.js → sequential-CimQrbjv.js} +5 -5
- package/dist/sequential-CimQrbjv.js.map +1 -0
- package/dist/{sequential-BhsrMupG.d.ts → sequential-KwGspfeR.d.ts} +2 -9
- package/dist/{sequential-BhsrMupG.d.ts.map → sequential-KwGspfeR.d.ts.map} +1 -1
- package/dist/{sequential-eprocess-DM_qENCl.js → sequential-eprocess-DodI6RoF.js} +2 -2
- package/dist/{sequential-eprocess-DM_qENCl.js.map → sequential-eprocess-DodI6RoF.js.map} +1 -1
- package/dist/{server-e9oWO_bN.js → server-C6M0qR8h.js} +101 -101
- package/dist/server-C6M0qR8h.js.map +1 -0
- package/dist/{single-run-lock-CV9Eyd_q.js → single-run-lock-Dekd8_Wv.js} +7 -7
- package/dist/single-run-lock-Dekd8_Wv.js.map +1 -0
- package/dist/{skillopt-optimization-method-BiwA8nfX.js → skillopt-optimization-method-5dlex7MJ.js} +1386 -1184
- package/dist/skillopt-optimization-method-5dlex7MJ.js.map +1 -0
- package/dist/{spans-BHAUXMGN.d.ts → spans-CVngGij3.d.ts} +2 -2
- package/dist/{spans-BHAUXMGN.d.ts.map → spans-CVngGij3.d.ts.map} +1 -1
- package/dist/{statistical-heldout-CYpGaoDH.d.ts → statistical-heldout-BHfLZHrm.d.ts} +8 -67
- package/dist/statistical-heldout-BHfLZHrm.d.ts.map +1 -0
- package/dist/{statistical-heldout-42NlCNDe.js → statistical-heldout-Bksv92Th.js} +3 -3
- package/dist/statistical-heldout-Bksv92Th.js.map +1 -0
- package/dist/{steps-Bg5-KVx1.d.ts → steps-DlepJPwD.d.ts} +2 -56
- package/dist/steps-DlepJPwD.d.ts.map +1 -0
- package/dist/{store-C4avFjIi.d.ts → store-B3nWRy3B.d.ts} +2 -2
- package/dist/{store-C4avFjIi.d.ts.map → store-B3nWRy3B.d.ts.map} +1 -1
- package/dist/{store-otlp-DJ9dtA5x.js → store-otlp-BteDAbHK.js} +3 -3
- package/dist/{store-otlp-DJ9dtA5x.js.map → store-otlp-BteDAbHK.js.map} +1 -1
- package/dist/{store-tool-spans-CIX5arAG.js → store-tool-spans-D5l_OfTJ.js} +8 -8
- package/dist/store-tool-spans-D5l_OfTJ.js.map +1 -0
- package/dist/{store-tool-spans-DdCSPa7C.d.ts → store-tool-spans-DZS0HWVf.d.ts} +9 -18
- package/dist/store-tool-spans-DZS0HWVf.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts +4 -11
- package/dist/storyboard/index.d.ts.map +1 -1
- package/dist/storyboard/index.js +1 -16
- package/dist/storyboard/index.js.map +1 -1
- package/dist/{summary-report-V_JACR4Y.js → summary-report-BsXJoYX5.js} +88 -10
- package/dist/summary-report-BsXJoYX5.js.map +1 -0
- package/dist/{summary-report-bdFDXq0b.d.ts → summary-report-CkbRcSuu.d.ts} +25 -19
- package/dist/summary-report-CkbRcSuu.d.ts.map +1 -0
- package/dist/supervisor-run/index.d.ts +60 -28
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +4 -458
- package/dist/supervisor-run/index.js.map +1 -1
- package/dist/{systemone-BomhHr2c.d.ts → systemone-BMzDJb1i.d.ts} +5 -5
- package/dist/{systemone-BomhHr2c.d.ts.map → systemone-BMzDJb1i.d.ts.map} +1 -1
- package/dist/systemone-protocol.js +1 -1
- package/dist/{systemone-review-CUVcQJu6.d.ts → systemone-review-CwE6JGIp.d.ts} +2 -2
- package/dist/{systemone-review-CUVcQJu6.d.ts.map → systemone-review-CwE6JGIp.d.ts.map} +1 -1
- package/dist/{systemone-review-D7sXwsqq.js → systemone-review-DzdOJ8I6.js} +3 -3
- package/dist/{systemone-review-D7sXwsqq.js.map → systemone-review-DzdOJ8I6.js.map} +1 -1
- package/dist/systemone.d.ts +5 -5
- package/dist/systemone.js +5 -5
- package/dist/systemone.js.map +1 -1
- package/dist/{task-failure-attributes-hEmASjix.js → task-failure-attributes-BMsblaDh.js} +3 -3
- package/dist/task-failure-attributes-BMsblaDh.js.map +1 -0
- package/dist/{terminal-record-DZyQZ3pz.js → terminal-record-D4Rv-j1W.js} +244 -196
- package/dist/terminal-record-D4Rv-j1W.js.map +1 -0
- package/dist/{tool-groups-C3NEcdCM.d.ts → tool-groups-DyNz2JZ_.d.ts} +3 -3
- package/dist/tool-groups-DyNz2JZ_.d.ts.map +1 -0
- package/dist/{tool-waste-BKgvd-TP.d.ts → tool-waste-CkjzWmeY.d.ts} +25 -28
- package/dist/tool-waste-CkjzWmeY.d.ts.map +1 -0
- package/dist/{tool-waste-BlesqTqW.js → tool-waste-DLsjS_p-.js} +5 -5
- package/dist/{tool-waste-BlesqTqW.js.map → tool-waste-DLsjS_p-.js.map} +1 -1
- package/dist/{trace-questions-CM2CAocU.d.ts → trace-questions-xxwgsuMi.d.ts} +4 -9
- package/dist/{trace-questions-CM2CAocU.d.ts.map → trace-questions-xxwgsuMi.d.ts.map} +1 -1
- package/dist/trace-repair/index.d.ts +14 -211
- package/dist/trace-repair/index.d.ts.map +1 -1
- package/dist/trace-repair/index.js +865 -981
- package/dist/trace-repair/index.js.map +1 -1
- package/dist/traces.d.ts +18 -46
- package/dist/traces.d.ts.map +1 -1
- package/dist/traces.js +13 -13
- package/dist/traces.js.map +1 -1
- package/dist/trajectory-replay/index.d.ts +7 -118
- package/dist/trajectory-replay/index.d.ts.map +1 -1
- package/dist/trajectory-replay/index.js +921 -869
- package/dist/trajectory-replay/index.js.map +1 -1
- package/dist/{tree-C1XYeXvy.js → tree-B5e1uAeL.js} +132 -21
- package/dist/tree-B5e1uAeL.js.map +1 -0
- package/dist/{types-Cbe2fWBa.d.ts → types-BKCzzmit.d.ts} +3 -3
- package/dist/{types-Cbe2fWBa.d.ts.map → types-BKCzzmit.d.ts.map} +1 -1
- package/dist/{types-B3jzCp0p.js → types-BXgASBCL.js} +2 -2
- package/dist/{types-B3jzCp0p.js.map → types-BXgASBCL.js.map} +1 -1
- package/dist/{types-C1tnkEgM.d.ts → types-CW8ewATZ.d.ts} +4 -113
- package/dist/types-CW8ewATZ.d.ts.map +1 -0
- package/dist/{types-BqkVmHfv.d.ts → types-CmXY0PQf.d.ts} +2 -4
- package/dist/types-CmXY0PQf.d.ts.map +1 -0
- package/dist/{types-C02qtnG8.d.ts → types-D-mgRzlh.d.ts} +7 -28
- package/dist/types-D-mgRzlh.d.ts.map +1 -0
- package/dist/{types-IXD2HJzu.js → types-DqFMNTgK.js} +2 -4
- package/dist/types-DqFMNTgK.js.map +1 -0
- package/dist/{types-Cq8mSbq8.d.ts → types-DtkO1G9L.d.ts} +7 -9
- package/dist/{types-Cq8mSbq8.d.ts.map → types-DtkO1G9L.d.ts.map} +1 -1
- package/dist/{verdict-Dl0tiT1X.js → verdict-BGQmxQ8A.js} +3 -3
- package/dist/verdict-BGQmxQ8A.js.map +1 -0
- package/dist/{verdict-E4eRNf7-.d.ts → verdict-C24NELtN.d.ts} +2 -15
- package/dist/{verdict-E4eRNf7-.d.ts.map → verdict-C24NELtN.d.ts.map} +1 -1
- package/dist/{verdict-cache-Bm8EKL2s.js → verdict-cache-CL_7tePD.js} +2 -2
- package/dist/verdict-cache-CL_7tePD.js.map +1 -0
- package/dist/wire/index.d.ts +31 -410
- package/dist/wire/index.d.ts.map +1 -1
- package/dist/wire/index.js +2 -2
- package/docs/changes/jev-evidence-review.md +1 -1
- package/docs/changes/jev-product-adoption.md +1 -1
- package/docs/changes/jev-recorded-evidence.md +1 -1
- package/docs/design/statistics-decisions.md +18 -18
- package/docs/design.md +1 -1
- package/docs/eval-surface-map.md +1 -0
- package/docs/evaluation-integrity.md +1 -1
- package/docs/profile-json-identity.md +1 -2
- package/docs/proofs/campaign-attempt-retention-20261001/README.md +2 -2
- package/docs/public-api.md +158 -538
- package/docs/rollout.md +1 -1
- package/docs/trace-repair-admission.md +1 -1
- package/docs/wire-protocol.md +1 -1
- package/package.json +7 -2
- package/dist/active-curriculum-B_Lc9kfO.js +0 -61
- package/dist/active-curriculum-B_Lc9kfO.js.map +0 -1
- package/dist/agent-profile-VFcQncbW.d.ts.map +0 -1
- package/dist/agent-profile-cell-BK8mlqlq.js.map +0 -1
- package/dist/agent-profile-cell-BNp-Js9F.d.ts.map +0 -1
- package/dist/agreement-irr-BEWLIF3_.js.map +0 -1
- package/dist/analyze-runs-C84irm4i.js.map +0 -1
- package/dist/baseline-Udi_PLuT.js +0 -259
- package/dist/baseline-Udi_PLuT.js.map +0 -1
- package/dist/benchmark-CXmVVHH2.js +0 -562
- package/dist/benchmark-CXmVVHH2.js.map +0 -1
- package/dist/benchmark-command-By0vh8q7.js.map +0 -1
- package/dist/campaign-BudZ70zo.js.map +0 -1
- package/dist/chat-json-call-puMArmtW.js.map +0 -1
- package/dist/claude-jsonl-dZir12m6.js +0 -86
- package/dist/claude-jsonl-dZir12m6.js.map +0 -1
- package/dist/client-BzL9niqi.d.ts.map +0 -1
- package/dist/cost-ledger-BP8V4kUQ.js.map +0 -1
- package/dist/counterfactual-CPaWN99-.d.ts +0 -60
- package/dist/counterfactual-CPaWN99-.d.ts.map +0 -1
- package/dist/counterfactual-Dn3veueY.js +0 -98
- package/dist/counterfactual-Dn3veueY.js.map +0 -1
- package/dist/default-registry-COhk9g36.js.map +0 -1
- package/dist/default-registry-DzHiGruW.d.ts.map +0 -1
- package/dist/define-agent-eval-CNTPzIkp.js.map +0 -1
- package/dist/define-agent-eval-WSck0CZJ.d.ts.map +0 -1
- package/dist/dspy-rlm-engine-LRCu61m5.js.map +0 -1
- package/dist/effect-sizes-BhzEpXMc.js +0 -82
- package/dist/effect-sizes-BhzEpXMc.js.map +0 -1
- package/dist/emitter-HcCmFQl_.d.ts.map +0 -1
- package/dist/engine-3A_4jOlI.d.ts.map +0 -1
- package/dist/equal-terms-BtdXayCJ.js.map +0 -1
- package/dist/eval-campaign-90jeaCAI.js +0 -343
- package/dist/eval-campaign-90jeaCAI.js.map +0 -1
- package/dist/evaluation-6Yoi74gM.d.ts.map +0 -1
- package/dist/exporters-X4xwla8A.js.map +0 -1
- package/dist/external-optimizer-process-BjMGUuds.js.map +0 -1
- package/dist/external-optimizer-subprocess-BqW0hHcI.js.map +0 -1
- package/dist/failure-cluster-CbgEDcFJ.d.ts.map +0 -1
- package/dist/feedback-trajectory-C4EG0nNH.d.ts.map +0 -1
- package/dist/hf-dataset-BR2yhuNn.js.map +0 -1
- package/dist/index-BFRO6chs.d.ts.map +0 -1
- package/dist/index-BO41eGj5.d.ts.map +0 -1
- package/dist/index-CyszSg4O.d.ts.map +0 -1
- package/dist/index-DHKEAv3s.d.ts.map +0 -1
- package/dist/index-DPcngtwt.d.ts.map +0 -1
- package/dist/integrity-dLiDe2kD.js.map +0 -1
- package/dist/journal-DfRz9kps.js.map +0 -1
- package/dist/judge-gate-CE7CQAgt.d.ts.map +0 -1
- package/dist/kind-factory-BobI5Jy3.js.map +0 -1
- package/dist/ledger-core-Gq7yp8lP.js.map +0 -1
- package/dist/llm-client-CyBAoYiX.js.map +0 -1
- package/dist/llm-judge-CG12SQJq.js.map +0 -1
- package/dist/matrix-DSrSxjjy.d.ts.map +0 -1
- package/dist/matrix-Dm0BQW7U.js.map +0 -1
- package/dist/mean-DHoMiudD.js.map +0 -1
- package/dist/normal-DsJZtrJc.js +0 -25
- package/dist/normal-DsJZtrJc.js.map +0 -1
- package/dist/paired-arms-CwMGJsrS.js.map +0 -1
- package/dist/paired-binary-CvpFHfKZ.js.map +0 -1
- package/dist/paired-promotion-decision-DPKz8ZnF.d.ts.map +0 -1
- package/dist/pareto-Cm6ndrWN.js.map +0 -1
- package/dist/prime-protocol-CHKgA3a6.js.map +0 -1
- package/dist/prime-protocol-HiB9HX7E.d.ts.map +0 -1
- package/dist/produced-state-BsDmnwxc.js.map +0 -1
- package/dist/proposal-findings-CY1xKn_a.js.map +0 -1
- package/dist/query-BUCZCK7F.js.map +0 -1
- package/dist/redact-mtCCcT8z.d.ts.map +0 -1
- package/dist/release-confidence-CpuP65S5.js.map +0 -1
- package/dist/report-command-SC9nqb4M.js.map +0 -1
- package/dist/reward-CpCmAM8L.d.ts.map +0 -1
- package/dist/reward-hacking-RVdZ6Kkt.d.ts.map +0 -1
- package/dist/rollout-DZV1t2Wg.js.map +0 -1
- package/dist/run-record-C1E24G2A.d.ts.map +0 -1
- package/dist/run-record-C5e7FjhJ.js.map +0 -1
- package/dist/schema-2g7trUwH.d.ts.map +0 -1
- package/dist/schema-BuiujZ-g.js.map +0 -1
- package/dist/schema-CSf6qWgZ.js.map +0 -1
- package/dist/search-kernel-DyNEa6IU.d.ts.map +0 -1
- package/dist/search-ledger-file-DDQUgvCg.js.map +0 -1
- package/dist/search-shipper-BOSD46qv.js.map +0 -1
- package/dist/semantic-concept-judge-BQF8te0T.js.map +0 -1
- package/dist/sequential-Bf_pbE50.js.map +0 -1
- package/dist/server-e9oWO_bN.js.map +0 -1
- package/dist/single-run-lock-CV9Eyd_q.js.map +0 -1
- package/dist/skillopt-optimization-method-BiwA8nfX.js.map +0 -1
- package/dist/statistical-heldout-42NlCNDe.js.map +0 -1
- package/dist/statistical-heldout-CYpGaoDH.d.ts.map +0 -1
- package/dist/steps-Bg5-KVx1.d.ts.map +0 -1
- package/dist/store-tool-spans-CIX5arAG.js.map +0 -1
- package/dist/store-tool-spans-DdCSPa7C.d.ts.map +0 -1
- package/dist/student-t-DVPW7CdL.js +0 -39
- package/dist/student-t-DVPW7CdL.js.map +0 -1
- package/dist/summary-report-V_JACR4Y.js.map +0 -1
- package/dist/summary-report-bdFDXq0b.d.ts.map +0 -1
- package/dist/task-failure-attributes-hEmASjix.js.map +0 -1
- package/dist/terminal-record-DZyQZ3pz.js.map +0 -1
- package/dist/tool-groups-C3NEcdCM.d.ts.map +0 -1
- package/dist/tool-waste-BKgvd-TP.d.ts.map +0 -1
- package/dist/trajectory-Y1fmdlw7.d.ts +0 -24
- package/dist/trajectory-Y1fmdlw7.d.ts.map +0 -1
- package/dist/tree-C1XYeXvy.js.map +0 -1
- package/dist/types-BqkVmHfv.d.ts.map +0 -1
- package/dist/types-C02qtnG8.d.ts.map +0 -1
- package/dist/types-C1tnkEgM.d.ts.map +0 -1
- package/dist/types-IXD2HJzu.js.map +0 -1
- package/dist/usage-receipt-Cx3xNeyB.js +0 -76
- package/dist/usage-receipt-Cx3xNeyB.js.map +0 -1
- package/dist/verdict-Dl0tiT1X.js.map +0 -1
- package/dist/verdict-cache-Bm8EKL2s.js.map +0 -1
- package/docs/design/self-improvement-evidence-audit.md +0 -263
|
@@ -1,56 +1,12 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import { o as summarizeNumberSeries } from "./descriptive-DboxPZrG.js";
|
|
1
|
+
import { a as hashCanonical } from "./canonical-Dq2hOQ8L.js";
|
|
2
|
+
import { o as summarizeNumberSeries } from "./descriptive-CVNy48rR.js";
|
|
4
3
|
import { t as roundDecimal } from "./round-decimal-COrKglMh.js";
|
|
5
|
-
import { a as nonEmptyString, c as parseSupervisorTree, d as NO_SOURCE_LIMITS, g as unavailable, h as showMeasured, i as asRecord, l as recordOrNull, m as isUnavailable, o as
|
|
6
|
-
import { join
|
|
7
|
-
import {
|
|
8
|
-
//#region src/supervisor-run/
|
|
9
|
-
/**
|
|
10
|
-
function
|
|
11
|
-
const files = /* @__PURE__ */ new Set();
|
|
12
|
-
const testFiles = /* @__PURE__ */ new Set();
|
|
13
|
-
let added = 0;
|
|
14
|
-
let removed = 0;
|
|
15
|
-
for (const line of text.split("\n")) {
|
|
16
|
-
if (line.startsWith("+++ ")) {
|
|
17
|
-
const p = line.slice(4).trim().replace(/^b\//, "");
|
|
18
|
-
if (p !== "/dev/null") {
|
|
19
|
-
files.add(p);
|
|
20
|
-
if (isTestPath(p)) testFiles.add(p);
|
|
21
|
-
}
|
|
22
|
-
continue;
|
|
23
|
-
}
|
|
24
|
-
if (line.startsWith("--- ") || line.startsWith("diff --git") || line.startsWith("index ")) continue;
|
|
25
|
-
if (line.startsWith("+")) added += 1;
|
|
26
|
-
else if (line.startsWith("-")) removed += 1;
|
|
27
|
-
}
|
|
28
|
-
return {
|
|
29
|
-
files: files.size,
|
|
30
|
-
linesAdded: added,
|
|
31
|
-
linesRemoved: removed,
|
|
32
|
-
testFilesTouched: [...testFiles].sort()
|
|
33
|
-
};
|
|
34
|
-
}
|
|
35
|
-
function isTestPath(p) {
|
|
36
|
-
const base = p.split("/").pop() ?? p;
|
|
37
|
-
return /(^|\/)(tests?|__tests__|testing|spec)(\/|$)/.test(p) || /\.(test|spec)\.[cm]?[jt]sx?$/.test(base) || /^test_.*\.py$/.test(base) || /_test\.py$/.test(base);
|
|
38
|
-
}
|
|
39
|
-
//#endregion
|
|
40
|
-
//#region src/supervisor-run/analyze.ts
|
|
41
|
-
/**
|
|
42
|
-
* The pure analyzer. Takes already-read bytes (`SupervisorRunSources`) and
|
|
43
|
-
* returns the report — every metric derivable from a synthetic journal string
|
|
44
|
-
* with no filesystem, no process, and no network. All I/O lives in a reader
|
|
45
|
-
* (`reader.ts` is one).
|
|
46
|
-
*/
|
|
47
|
-
const NO_CACHE_COUNTERS = "the journal carries no cache-token counters for this role";
|
|
48
|
-
const NO_CACHE_BREAKDOWN = "Runtime recorded cacheBreakdownKnown:false — the provider reported a total without splitting cache reads from writes";
|
|
49
|
-
/**
|
|
50
|
-
* Analyze already-read supervisor-run bytes. Pure and synchronous: same bytes
|
|
51
|
-
* in, same report out (modulo `generatedAt`, which `now` pins in tests).
|
|
52
|
-
*/
|
|
53
|
-
function analyzeSupervisorRunSources(src, now = Date.now) {
|
|
4
|
+
import { a as nonEmptyString, c as parseSupervisorTree, d as NO_SOURCE_LIMITS, g as unavailable, h as showMeasured, i as asRecord, l as recordOrNull, m as isUnavailable, o as parseJsonRecord, p as SUPERVISOR_RUN_SCHEMA, r as readTerminalRecord, s as parseJsonl, u as workerSourceKey } from "./terminal-record-D4Rv-j1W.js";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
import { readFile, readdir, stat } from "node:fs/promises";
|
|
7
|
+
//#region src/supervisor-run/analyze-context.ts
|
|
8
|
+
/** The parsed sources and shared lookups every metric group reads. `gap` records an unavailable measurement on the report. */
|
|
9
|
+
function createAnalysis(src) {
|
|
54
10
|
const gaps = [];
|
|
55
11
|
const gap = (what, reason) => {
|
|
56
12
|
gaps.push(`${what}: ${reason}`);
|
|
@@ -60,12 +16,12 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
|
|
|
60
16
|
const haveJournal = src.journal !== null;
|
|
61
17
|
const tree = parseSupervisorTree(src);
|
|
62
18
|
const state = tree.state;
|
|
63
|
-
const result =
|
|
64
|
-
const judge =
|
|
19
|
+
const result = parseJsonRecord(src.result);
|
|
20
|
+
const judge = parseJsonRecord(src.judge);
|
|
65
21
|
const terminal = readTerminalRecord({
|
|
66
22
|
state,
|
|
67
23
|
result,
|
|
68
|
-
failure: src.failure === void 0 ? void 0 :
|
|
24
|
+
failure: src.failure === void 0 ? void 0 : parseJsonRecord(src.failure)
|
|
69
25
|
});
|
|
70
26
|
const { rootId, workerSpawns, workerCloses, startedAt, completedAt } = tree;
|
|
71
27
|
const rootSpawn = rootId === null ? null : tree.spawns.find((spawn) => spawn.id === rootId) ?? null;
|
|
@@ -85,6 +41,182 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
|
|
|
85
41
|
const matches = spawnsForSource(worker);
|
|
86
42
|
return matches.length === 1 ? matches[0] ?? null : null;
|
|
87
43
|
};
|
|
44
|
+
return {
|
|
45
|
+
src,
|
|
46
|
+
gaps,
|
|
47
|
+
gap,
|
|
48
|
+
journalMissing,
|
|
49
|
+
haveJournal,
|
|
50
|
+
tree,
|
|
51
|
+
state,
|
|
52
|
+
result,
|
|
53
|
+
judge,
|
|
54
|
+
terminal,
|
|
55
|
+
rootId,
|
|
56
|
+
rootSpawn,
|
|
57
|
+
workerSpawns,
|
|
58
|
+
workerCloses,
|
|
59
|
+
startedAt,
|
|
60
|
+
completedAt,
|
|
61
|
+
spawnsForSource,
|
|
62
|
+
spawnForSource,
|
|
63
|
+
closeById: new Map(workerCloses.map((close) => [close.id, close])),
|
|
64
|
+
workersGapReason: src.workersMissingReason ?? "workers/ directory absent"
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
//#endregion
|
|
68
|
+
//#region src/supervisor-run/analyze-decision.ts
|
|
69
|
+
/** Settled verdicts, acceptance and review effort, with a named gap wherever the store never said. */
|
|
70
|
+
function measureDecision(a, m) {
|
|
71
|
+
const { src, gap, journalMissing, haveJournal, tree, workerCloses, spawnForSource, closeById, workersGapReason } = a;
|
|
72
|
+
const { steerQueuedTotal, upLegMessages, queuedGapReason, workerEventsGapReason, observeThenRespawn, respawnWithoutEvidence } = m;
|
|
73
|
+
const settledByStatus = {};
|
|
74
|
+
const settledVerdicts = {};
|
|
75
|
+
for (const c of workerCloses) {
|
|
76
|
+
const key = c.status ?? "unknown";
|
|
77
|
+
settledByStatus[key] = (settledByStatus[key] ?? 0) + 1;
|
|
78
|
+
if (c.verdict !== null) settledVerdicts[c.verdict] = (settledVerdicts[c.verdict] ?? 0) + 1;
|
|
79
|
+
}
|
|
80
|
+
const verdictLimit = src.limits.workerVerdicts;
|
|
81
|
+
const deliverablesLimit = src.limits.deliverables;
|
|
82
|
+
let accepted = 0;
|
|
83
|
+
let emptyPass = 0;
|
|
84
|
+
let evidenceBytes = 0;
|
|
85
|
+
const sourceVerdicts = [];
|
|
86
|
+
for (const w of src.workers ?? []) {
|
|
87
|
+
const f = tree.workerLogs.get(workerSourceKey(w));
|
|
88
|
+
if (f?.finished) evidenceBytes += f.evidenceBytes;
|
|
89
|
+
const spawn = spawnForSource(w);
|
|
90
|
+
const passed = (spawn === null ? null : closeById.get(spawn.id) ?? null)?.valid ?? f?.passed ?? null;
|
|
91
|
+
if (passed !== null) sourceVerdicts.push(passed);
|
|
92
|
+
if (passed === true) {
|
|
93
|
+
if (deliverablesLimit !== null || (w.patchBytes ?? f?.finishedPatchBytes ?? 0) > 0) accepted += 1;
|
|
94
|
+
else emptyPass += 1;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
const settledCloses = workerCloses.filter((close) => close.kind === "settled");
|
|
98
|
+
const structuredVerdicts = settledCloses.map((close) => close.valid).filter((valid) => valid !== null);
|
|
99
|
+
const journalVerdictsComplete = settledCloses.length > 0 && structuredVerdicts.length === settledCloses.length;
|
|
100
|
+
const sourceVerdictsComplete = sourceVerdicts.length > 0 && sourceVerdicts.length >= settledCloses.length;
|
|
101
|
+
const rejected = journalVerdictsComplete ? structuredVerdicts.filter((valid) => !valid).length : sourceVerdictsComplete ? sourceVerdicts.filter((valid) => !valid).length : 0;
|
|
102
|
+
const rejectedLimit = journalVerdictsComplete || sourceVerdictsComplete ? null : settledCloses.length > 0 ? verdictLimit ?? "a settled journal verdict has no validity and no matched worker log" : verdictLimit !== null ? verdictLimit : !haveJournal && src.workers === null ? workersGapReason : null;
|
|
103
|
+
return {
|
|
104
|
+
settledByStatus: haveJournal ? settledByStatus : gap("settledByStatus", journalMissing),
|
|
105
|
+
settledVerdicts: verdictLimit !== null ? unavailable(verdictLimit) : haveJournal ? settledVerdicts : unavailable(journalMissing),
|
|
106
|
+
accepted: verdictLimit !== null ? gap("accepted", verdictLimit) : src.workers === null ? unavailable(workersGapReason) : accepted,
|
|
107
|
+
rejected: rejectedLimit === null ? rejected : unavailable(rejectedLimit),
|
|
108
|
+
emptyPass: verdictLimit !== null ? gap("emptyPass", verdictLimit) : deliverablesLimit !== null ? gap("emptyPass", deliverablesLimit) : src.workers === null ? unavailable(workersGapReason) : emptyPass,
|
|
109
|
+
observeThenRespawn: haveJournal ? observeThenRespawn : unavailable(journalMissing),
|
|
110
|
+
respawnWithoutEvidence: haveJournal ? respawnWithoutEvidence : unavailable(journalMissing),
|
|
111
|
+
reviewActions: src.workers === null ? unavailable(workersGapReason) : queuedGapReason === null ? steerQueuedTotal + upLegMessages : unavailable(queuedGapReason),
|
|
112
|
+
workerEvidenceBytes: src.workers === null ? unavailable(workersGapReason) : workerEventsGapReason === null ? evidenceBytes : unavailable(workerEventsGapReason)
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
//#endregion
|
|
116
|
+
//#region src/supervisor-run/analyze-economics.ts
|
|
117
|
+
const NO_CACHE_COUNTERS = "the journal carries no cache-token counters for this role";
|
|
118
|
+
const NO_CACHE_BREAKDOWN = "Runtime recorded cacheBreakdownKnown:false — the provider reported a total without splitting cache reads from writes";
|
|
119
|
+
/** One row per worker log, plus the capture gaps of the root and every worker. */
|
|
120
|
+
function buildPerWorker(a, f, t) {
|
|
121
|
+
const { src, gaps, tree, rootId, spawnsForSource, spawnForSource, closeById } = a;
|
|
122
|
+
const { workerUsdById, workerUsdByLabel } = f;
|
|
123
|
+
const { usdLimit } = t;
|
|
124
|
+
const noteCaptureGaps = (owner, captures) => {
|
|
125
|
+
if (captures === null || captures?.length === 0) gaps.push(`${owner} workspaceCapture: no retained output capture evidence`);
|
|
126
|
+
else if (captures !== void 0) for (const [index, capture] of captures.entries()) {
|
|
127
|
+
const source = `${owner} workspaceCapture[${index + 1}]`;
|
|
128
|
+
if (capture.projectionGap !== null) gaps.push(`${source}: ${capture.projectionGap}`);
|
|
129
|
+
if (capture.coverageComplete === false) gaps.push(`${source}: ${capture.incompleteReason ?? "provider reported incomplete coverage"}`);
|
|
130
|
+
if (capture.coverageComplete === null && capture.projectionGap === null) gaps.push(`${source}: capture coverage was not recorded`);
|
|
131
|
+
}
|
|
132
|
+
};
|
|
133
|
+
noteCaptureGaps(`root ${rootId ?? src.runRef}`, src.rootWorkspaceCaptures);
|
|
134
|
+
return (src.workers ?? []).map((w) => {
|
|
135
|
+
noteCaptureGaps(`worker ${w.workerId ?? w.label}`, w.workspaceCaptures);
|
|
136
|
+
const f = tree.workerLogs.get(workerSourceKey(w));
|
|
137
|
+
const matchingSpawns = spawnsForSource(w);
|
|
138
|
+
const spawn = spawnForSource(w);
|
|
139
|
+
const close = spawn === null ? null : closeById.get(spawn.id) ?? null;
|
|
140
|
+
const passed = close?.valid ?? f?.passed ?? null;
|
|
141
|
+
const matchingRoles = new Set(matchingSpawns.map((candidate) => candidate.role));
|
|
142
|
+
const matchingRuntimes = new Set(matchingSpawns.map((candidate) => candidate.runtime));
|
|
143
|
+
const matchingProfiles = new Set(matchingSpawns.map((candidate) => candidate.profileDigest));
|
|
144
|
+
const journalWallMs = spawn?.at !== null && spawn?.at !== void 0 && close?.at !== null && close?.at !== void 0 && close.at >= spawn.at ? close.at - spawn.at : null;
|
|
145
|
+
return {
|
|
146
|
+
workerId: w.workerId ?? null,
|
|
147
|
+
worker: w.label,
|
|
148
|
+
role: matchingRoles.size === 1 ? matchingSpawns[0]?.role ?? null : null,
|
|
149
|
+
runtime: matchingRuntimes.size === 1 ? matchingSpawns[0]?.runtime ?? null : null,
|
|
150
|
+
profileDigest: matchingProfiles.size === 1 ? matchingSpawns[0]?.profileDigest ?? null : null,
|
|
151
|
+
status: close?.status ?? null,
|
|
152
|
+
failure: close?.reason ?? null,
|
|
153
|
+
infra: close?.infra ?? null,
|
|
154
|
+
wallMs: f?.started != null && f.finishedAt != null ? f.finishedAt - f.started : journalWallMs,
|
|
155
|
+
tokensIn: w.tokensIn ?? (close?.hasSpend === true && close.spend.tokensKnown ? close.spend.tokens.input : null),
|
|
156
|
+
tokensOut: w.tokensOut ?? (close?.hasSpend === true && close.spend.tokensKnown ? close.spend.tokens.output : null),
|
|
157
|
+
usd: usdLimit !== null ? null : w.workerId === void 0 ? workerUsdByLabel.get(w.label) ?? null : workerUsdById.get(w.workerId) ?? null,
|
|
158
|
+
patchBytes: w.patchBytes ?? f?.finishedPatchBytes ?? null,
|
|
159
|
+
passed,
|
|
160
|
+
score: close?.score ?? f?.score ?? null,
|
|
161
|
+
workspaceCaptures: w.workspaceCaptures ?? null
|
|
162
|
+
};
|
|
163
|
+
});
|
|
164
|
+
}
|
|
165
|
+
/** The supervisor brain's tokens, cache and USD. */
|
|
166
|
+
function buildBrainSpend(a, f, t) {
|
|
167
|
+
const { src, gap, journalMissing, haveJournal, tree } = a;
|
|
168
|
+
const { brainTokensUnreported, brainUsdUnreported } = f;
|
|
169
|
+
const { usdLimit } = t;
|
|
170
|
+
const managerTokenLimit = src.limits.managerTokens;
|
|
171
|
+
return {
|
|
172
|
+
tokensIn: managerTokenLimit !== null ? gap("brain.tokensIn", managerTokenLimit) : !haveJournal ? gap("brain.tokensIn", journalMissing) : brainTokensUnreported !== null ? gap("brain.tokensIn", brainTokensUnreported) : tree.brain.tokensIn,
|
|
173
|
+
tokensOut: managerTokenLimit !== null ? unavailable(managerTokenLimit) : !haveJournal ? unavailable(journalMissing) : brainTokensUnreported !== null ? unavailable(brainTokensUnreported) : tree.brain.tokensOut,
|
|
174
|
+
usd: usdLimit !== null ? unavailable(usdLimit) : !haveJournal ? unavailable(journalMissing) : brainUsdUnreported !== null ? unavailable(brainUsdUnreported) : roundDecimal(tree.brain.usd, 6),
|
|
175
|
+
cacheRead: managerTokenLimit !== null ? unavailable(managerTokenLimit) : !haveJournal ? unavailable(journalMissing) : brainTokensUnreported !== null ? unavailable(brainTokensUnreported) : !tree.brain.hasCache ? unavailable(NO_CACHE_COUNTERS) : tree.brain.cacheBreakdownKnown ? tree.brain.cacheRead : unavailable(NO_CACHE_BREAKDOWN),
|
|
176
|
+
cacheWrite: managerTokenLimit !== null ? unavailable(managerTokenLimit) : !haveJournal ? unavailable(journalMissing) : brainTokensUnreported !== null ? unavailable(brainTokensUnreported) : !tree.brain.hasCache ? unavailable(NO_CACHE_COUNTERS) : tree.brain.cacheBreakdownKnown ? tree.brain.cacheWrite : unavailable(NO_CACHE_BREAKDOWN),
|
|
177
|
+
source: managerTokenLimit ?? (haveJournal ? `journal metered events (n=${tree.brain.meteredCount})${tree.brain.usdKnownCount > 0 && tree.brain.usdUnknownCount > 0 ? ` — ${tree.brain.usdUnknownCount} unpriced` : ""}` : journalMissing)
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
/** The workers' tokens, cache and USD. */
|
|
181
|
+
function buildWorkerSpend(a, f, t, w) {
|
|
182
|
+
const { src, journalMissing, haveJournal } = a;
|
|
183
|
+
const { rootChildSpends, workerUsdUnknownNodes, journalWorkerUsd, workerUsdUnreported } = f;
|
|
184
|
+
const { usdLimit } = t;
|
|
185
|
+
const { sq, workerTokenLimit, workerIn, workerOut } = w;
|
|
186
|
+
return {
|
|
187
|
+
tokensIn: workerIn,
|
|
188
|
+
tokensOut: workerOut,
|
|
189
|
+
cacheRead: workerTokenLimit !== null ? unavailable(workerTokenLimit) : sq?.cacheRead !== void 0 ? sq.cacheRead : unavailable(NO_CACHE_COUNTERS),
|
|
190
|
+
cacheWrite: workerTokenLimit !== null ? unavailable(workerTokenLimit) : sq?.cacheWrite !== void 0 ? sq.cacheWrite : unavailable(NO_CACHE_COUNTERS),
|
|
191
|
+
usd: usdLimit !== null ? unavailable(usdLimit) : !haveJournal ? unavailable(journalMissing) : workerUsdUnreported !== null ? unavailable(workerUsdUnreported) : roundDecimal(journalWorkerUsd, 6),
|
|
192
|
+
source: `${workerTokenLimit !== null ? workerTokenLimit : sq !== null ? `journal settled spend + ${sq.store} sessions (n=${sq.sessions})` : `journal settled spend only — ${src.harnessMissingReason ?? "harness session store unavailable"}`}${workerUsdUnknownNodes.length > 0 && workerUsdUnknownNodes.length < rootChildSpends.length ? ` — ${workerUsdUnknownNodes.length} unpriced` : ""}`
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
function buildEconomics(a, f, t, w, e) {
|
|
196
|
+
const { src, gap, journalMissing, haveJournal, tree, workersGapReason } = a;
|
|
197
|
+
const { rootChildCloses, journalWorkerUsd, usdUnknownRecords } = f;
|
|
198
|
+
const { stateUsd, usdLimit, totalUsd, usdUnreportedReason } = t;
|
|
199
|
+
const { decision, spend, perWorker } = e;
|
|
200
|
+
const wallDistribution = summarizeNumberSeries(perWorker.map((w) => w.wallMs).filter((w) => w !== null));
|
|
201
|
+
const brainCalls = parseJsonl(src.brainLog);
|
|
202
|
+
return {
|
|
203
|
+
resourceRecords: src.journal === null && !tree.resourceRecords?.length ? unavailable(journalMissing) : tree.resourceRecords ?? [],
|
|
204
|
+
brain: buildBrainSpend(a, f, t),
|
|
205
|
+
brainTruncations: src.brainLog === null ? gap("brain.brainTruncations", src.brainLogMissingReason ?? (src.supRunDir === null ? "no Runtime supervisor run directory" : "brain.jsonl absent — this source has no brain-call tap, so truncation cannot be ruled out")) : brainCalls.filter((c) => c.finish_reason === "length").length,
|
|
206
|
+
workers: buildWorkerSpend(a, f, t, w),
|
|
207
|
+
spend,
|
|
208
|
+
totalUsd,
|
|
209
|
+
totalUsdSource: usdLimit !== null ? usdLimit : stateUsd !== null ? `state.json result.spentUsd${rootChildCloses.length > 0 && journalWorkerUsd === 0 ? " — brain-priced only; worker CLI inference is unpriced (see worker token counts)" : ""}` : !haveJournal ? journalMissing : usdUnknownRecords > 0 ? usdUnreportedReason : "journal metered + settled usd",
|
|
210
|
+
costPerAcceptedPatchUsd: isUnavailable(totalUsd) ? unavailable(totalUsd.unavailable) : isUnavailable(decision.accepted) ? unavailable(decision.accepted.unavailable) : decision.accepted === 0 ? unavailable("no accepted worker patch (cost has no denominator)") : roundDecimal(totalUsd / decision.accepted, 6),
|
|
211
|
+
workerWallMsDistribution: wallDistribution === null ? unavailable(src.workers === null ? workersGapReason : "no worker start/finish pairs captured") : wallDistribution,
|
|
212
|
+
perWorker: src.workers === null ? unavailable(workersGapReason) : perWorker
|
|
213
|
+
};
|
|
214
|
+
}
|
|
215
|
+
//#endregion
|
|
216
|
+
//#region src/supervisor-run/analyze-orchestration.ts
|
|
217
|
+
/** Supervisor wall time and where it ends. Explicit stamps win; the journal span is a lower bound; an inverted stamp pair stays unavailable. */
|
|
218
|
+
function measureWall(a) {
|
|
219
|
+
const { gap, journalMissing, haveJournal, tree, startedAt, completedAt } = a;
|
|
88
220
|
const wallSpanStart = startedAt ?? tree.firstEventAt;
|
|
89
221
|
const wallSpanEnd = tree.lastEventAt;
|
|
90
222
|
let supervisorWallMs;
|
|
@@ -100,7 +232,15 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
|
|
|
100
232
|
supervisorWallMs = gap("supervisorWallMs", reason);
|
|
101
233
|
supervisorWallSource = unavailable(reason);
|
|
102
234
|
}
|
|
103
|
-
|
|
235
|
+
return {
|
|
236
|
+
supervisorWallMs,
|
|
237
|
+
supervisorWallSource,
|
|
238
|
+
wallEndAt: completedAt ?? (supervisorWallSource === "journal-span" ? wallSpanEnd : null)
|
|
239
|
+
};
|
|
240
|
+
}
|
|
241
|
+
/** Steer counts from worker inboxes and the driver log. */
|
|
242
|
+
function measureSteers(a) {
|
|
243
|
+
const { src, gap, tree, workersGapReason } = a;
|
|
104
244
|
const steerRows = [];
|
|
105
245
|
let steerQueuedTotal = 0;
|
|
106
246
|
let steerDeliveredTotal = 0;
|
|
@@ -119,7 +259,6 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
|
|
|
119
259
|
if (queued !== null) steerQueuedTotal += queued;
|
|
120
260
|
if (delivered !== null) steerDeliveredTotal += delivered;
|
|
121
261
|
}
|
|
122
|
-
const workersGapReason = src.workersMissingReason ?? "workers/ directory absent";
|
|
123
262
|
const unavailableReasons = (pick) => {
|
|
124
263
|
const reasons = tree.workerLogRows.map((facts) => pick(facts)).filter((reason) => reason !== null);
|
|
125
264
|
return reasons.length === 0 ? null : `exact steer accounting unavailable for ${reasons.length} worker row(s): ${[...new Set(reasons)].join(" | ")}`;
|
|
@@ -127,10 +266,20 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
|
|
|
127
266
|
const queuedGapReason = unavailableReasons((facts) => facts.steersQueuedUnavailable);
|
|
128
267
|
const deliveredGapReason = unavailableReasons((facts) => facts.steersDeliveredUnavailable);
|
|
129
268
|
const workerEventsGapReason = unavailableReasons((facts) => !facts.eventsCaptured ? "events absent" : facts.eventsInvalidRows > 0 ? "events contain malformed rows" : null);
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
269
|
+
return {
|
|
270
|
+
steers: src.workers === null ? gap("steers", workersGapReason) : queuedGapReason === null ? steerQueuedTotal : gap("steers", queuedGapReason),
|
|
271
|
+
steersDelivered: src.workers === null ? unavailable(workersGapReason) : deliveredGapReason === null ? steerDeliveredTotal : unavailable(deliveredGapReason),
|
|
272
|
+
steersByWorker: src.workers === null ? unavailable(workersGapReason) : queuedGapReason === null && deliveredGapReason === null ? steerRows : unavailable(queuedGapReason ?? deliveredGapReason ?? workersGapReason),
|
|
273
|
+
driverSteerCalls: src.driverLog === null ? gap("driverSteerCalls", "driver.log absent") : Math.max(0, (src.driverLog.match(/supervisor_steer/g) ?? []).length - registrationMentions(src.driverLog)),
|
|
274
|
+
steerQueuedTotal,
|
|
275
|
+
upLegMessages,
|
|
276
|
+
queuedGapReason,
|
|
277
|
+
workerEventsGapReason
|
|
278
|
+
};
|
|
279
|
+
}
|
|
280
|
+
/** Waves, peak concurrency, idle time and summed worker wall from the spawn/close timeline. */
|
|
281
|
+
function measureConcurrency(a, wallEndAt) {
|
|
282
|
+
const { workerSpawns, workerCloses, startedAt } = a;
|
|
134
283
|
const timeline = [];
|
|
135
284
|
for (const s of workerSpawns) if (s.at !== null) timeline.push({
|
|
136
285
|
at: s.at,
|
|
@@ -172,8 +321,17 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
|
|
|
172
321
|
if (live === 0) idleMs += span;
|
|
173
322
|
sumWorkerWallMs += span * live;
|
|
174
323
|
}
|
|
175
|
-
|
|
176
|
-
|
|
324
|
+
return {
|
|
325
|
+
waves,
|
|
326
|
+
waveSizes,
|
|
327
|
+
maxConcurrency,
|
|
328
|
+
idleMs,
|
|
329
|
+
sumWorkerWallMs
|
|
330
|
+
};
|
|
331
|
+
}
|
|
332
|
+
/** Respawn counts per parent, split by whether a sibling closed in between. */
|
|
333
|
+
function measureRespawns(a) {
|
|
334
|
+
const { workerSpawns, closeById } = a;
|
|
177
335
|
const childSpawnsByParent = /* @__PURE__ */ new Map();
|
|
178
336
|
for (const spawn of workerSpawns) {
|
|
179
337
|
if (spawn.parent === null) continue;
|
|
@@ -206,6 +364,16 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
|
|
|
206
364
|
}
|
|
207
365
|
}
|
|
208
366
|
const repeatedLabels = [...repeatedLabelSet];
|
|
367
|
+
return {
|
|
368
|
+
respawns,
|
|
369
|
+
observeThenRespawn,
|
|
370
|
+
respawnWithoutEvidence,
|
|
371
|
+
repeatedLabels
|
|
372
|
+
};
|
|
373
|
+
}
|
|
374
|
+
/** The deepest chain of spawns below the root. */
|
|
375
|
+
function measureDelegationDepth(a) {
|
|
376
|
+
const { tree, rootId, workerSpawns } = a;
|
|
209
377
|
const parentOf = new Map(tree.spawns.map((s) => [s.id, s.parent]));
|
|
210
378
|
let delegationDepth = 0;
|
|
211
379
|
for (const s of workerSpawns) {
|
|
@@ -219,7 +387,13 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
|
|
|
219
387
|
}
|
|
220
388
|
if (d > delegationDepth) delegationDepth = d;
|
|
221
389
|
}
|
|
222
|
-
|
|
390
|
+
return delegationDepth;
|
|
391
|
+
}
|
|
392
|
+
function buildOrchestration(a, m) {
|
|
393
|
+
const { gap, journalMissing, haveJournal, workerSpawns, workerCloses, startedAt } = a;
|
|
394
|
+
const { supervisorWallMs, supervisorWallSource, steers, steersDelivered, steersByWorker, driverSteerCalls, waves, waveSizes, maxConcurrency, idleMs, sumWorkerWallMs, respawns, repeatedLabels, delegationDepth } = m;
|
|
395
|
+
const firstWorkerSpawnAt = workerSpawns.reduce((acc, s) => s.at === null ? acc : acc === null ? s.at : Math.min(acc, s.at), null);
|
|
396
|
+
return {
|
|
223
397
|
workersSpawned: haveJournal ? workerSpawns.length : gap("workersSpawned", journalMissing),
|
|
224
398
|
workersSettled: haveJournal ? workerCloses.filter((c) => c.kind === "settled").length : unavailable(journalMissing),
|
|
225
399
|
workersCancelled: haveJournal ? workerCloses.filter((c) => c.kind === "cancelled").length : unavailable(journalMissing),
|
|
@@ -240,49 +414,91 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
|
|
|
240
414
|
idlePct: isUnavailable(supervisorWallMs) || supervisorWallMs === 0 ? isUnavailable(supervisorWallMs) ? unavailable(supervisorWallMs.unavailable) : unavailable("supervisor wall is 0ms") : roundDecimal(idleMs / supervisorWallMs * 100, 1),
|
|
241
415
|
workerUtilization: isUnavailable(supervisorWallMs) || supervisorWallMs === 0 ? isUnavailable(supervisorWallMs) ? unavailable(supervisorWallMs.unavailable) : unavailable("supervisor wall is 0ms") : roundDecimal(sumWorkerWallMs / supervisorWallMs, 3)
|
|
242
416
|
};
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
417
|
+
}
|
|
418
|
+
/** Whether sorted values contain one value in the inclusive interval. */
|
|
419
|
+
function hasNumberBetween(sorted, low, high) {
|
|
420
|
+
let left = 0;
|
|
421
|
+
let right = sorted.length;
|
|
422
|
+
while (left < right) {
|
|
423
|
+
const middle = left + Math.floor((right - left) / 2);
|
|
424
|
+
if (sorted[middle] < low) left = middle + 1;
|
|
425
|
+
else right = middle;
|
|
249
426
|
}
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
let
|
|
255
|
-
const
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
427
|
+
return left < sorted.length && sorted[left] <= high;
|
|
428
|
+
}
|
|
429
|
+
/** `[driver] registered tools: …supervisor_steer…` is a banner, not an invocation. */
|
|
430
|
+
function registrationMentions(driverLog) {
|
|
431
|
+
let n = 0;
|
|
432
|
+
for (const line of driverLog.split("\n")) if (line.includes("registered tools:") && line.includes("supervisor_steer")) n += 1;
|
|
433
|
+
return n;
|
|
434
|
+
}
|
|
435
|
+
//#endregion
|
|
436
|
+
//#region src/supervisor-run/patch-stats.ts
|
|
437
|
+
/** Unified-diff stats. Counts `+++ b/<path>` targets, body +/- lines, and test-file touches. */
|
|
438
|
+
function parsePatch(text) {
|
|
439
|
+
const files = /* @__PURE__ */ new Set();
|
|
440
|
+
const testFiles = /* @__PURE__ */ new Set();
|
|
441
|
+
let added = 0;
|
|
442
|
+
let removed = 0;
|
|
443
|
+
for (const line of text.split("\n")) {
|
|
444
|
+
if (line.startsWith("+++ ")) {
|
|
445
|
+
const p = line.slice(4).trim().replace(/^b\//, "");
|
|
446
|
+
if (p !== "/dev/null") {
|
|
447
|
+
files.add(p);
|
|
448
|
+
if (isTestPath(p)) testFiles.add(p);
|
|
449
|
+
}
|
|
450
|
+
continue;
|
|
265
451
|
}
|
|
452
|
+
if (line.startsWith("--- ") || line.startsWith("diff --git") || line.startsWith("index ")) continue;
|
|
453
|
+
if (line.startsWith("+")) added += 1;
|
|
454
|
+
else if (line.startsWith("-")) removed += 1;
|
|
266
455
|
}
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
const rejectedLimit = journalVerdictsComplete || sourceVerdictsComplete ? null : settledCloses.length > 0 ? verdictLimit ?? "a settled journal verdict has no validity and no matched worker log" : verdictLimit !== null ? verdictLimit : !haveJournal && src.workers === null ? workersGapReason : null;
|
|
273
|
-
const decision = {
|
|
274
|
-
settledByStatus: haveJournal ? settledByStatus : gap("settledByStatus", journalMissing),
|
|
275
|
-
settledVerdicts: verdictLimit !== null ? unavailable(verdictLimit) : haveJournal ? settledVerdicts : unavailable(journalMissing),
|
|
276
|
-
accepted: verdictLimit !== null ? gap("accepted", verdictLimit) : src.workers === null ? unavailable(workersGapReason) : accepted,
|
|
277
|
-
rejected: rejectedLimit === null ? rejected : unavailable(rejectedLimit),
|
|
278
|
-
emptyPass: verdictLimit !== null ? gap("emptyPass", verdictLimit) : deliverablesLimit !== null ? gap("emptyPass", deliverablesLimit) : src.workers === null ? unavailable(workersGapReason) : emptyPass,
|
|
279
|
-
observeThenRespawn: haveJournal ? observeThenRespawn : unavailable(journalMissing),
|
|
280
|
-
respawnWithoutEvidence: haveJournal ? respawnWithoutEvidence : unavailable(journalMissing),
|
|
281
|
-
reviewActions: src.workers === null ? unavailable(workersGapReason) : queuedGapReason === null ? steerQueuedTotal + upLegMessages : unavailable(queuedGapReason),
|
|
282
|
-
workerEvidenceBytes: src.workers === null ? unavailable(workersGapReason) : workerEventsGapReason === null ? evidenceBytes : unavailable(workerEventsGapReason)
|
|
456
|
+
return {
|
|
457
|
+
files: files.size,
|
|
458
|
+
linesAdded: added,
|
|
459
|
+
linesRemoved: removed,
|
|
460
|
+
testFilesTouched: [...testFiles].sort()
|
|
283
461
|
};
|
|
284
|
-
|
|
285
|
-
|
|
462
|
+
}
|
|
463
|
+
function isTestPath(p) {
|
|
464
|
+
const base = p.split("/").pop() ?? p;
|
|
465
|
+
return /(^|\/)(tests?|__tests__|testing|spec)(\/|$)/.test(p) || /\.(test|spec)\.[cm]?[jt]sx?$/.test(base) || /^test_.*\.py$/.test(base) || /_test\.py$/.test(base);
|
|
466
|
+
}
|
|
467
|
+
//#endregion
|
|
468
|
+
//#region src/supervisor-run/analyze-outcome.ts
|
|
469
|
+
/** Terminal status, verdicts, judge result and the delivered patch. */
|
|
470
|
+
function buildOutcome(a, t) {
|
|
471
|
+
const { src, gap, state, result, judge, terminal } = a;
|
|
472
|
+
const { stateResult } = t;
|
|
473
|
+
const patchStats = src.patch === null ? gap("patch", src.limits.deliverables ?? "delivered patch file absent") : parsePatch(src.patch);
|
|
474
|
+
return {
|
|
475
|
+
supStatus: isUnavailable(terminal.supStatus) ? gap("supStatus", terminal.supStatus.unavailable) : terminal.supStatus,
|
|
476
|
+
supStatusSource: terminal.supStatusSource,
|
|
477
|
+
supReason: terminal.supReason,
|
|
478
|
+
failure: terminal.failure,
|
|
479
|
+
supVerdict: pickString(state, "verdict") ?? pickString(result, "sup_verdict") ?? unavailable("no state.json / result.json verdict"),
|
|
480
|
+
delivered: typeof stateResult.delivered === "boolean" ? stateResult.delivered : typeof result?.delivered === "boolean" ? result.delivered : unavailable("no delivered flag in state.json or result.json"),
|
|
481
|
+
judgeResolved: judge === null ? gap("judge", "judge.json absent") : typeof judge.resolved === "boolean" ? judge.resolved : null,
|
|
482
|
+
judgeScore: judge === null ? unavailable("judge.json absent") : typeof judge.score === "number" ? judge.score : null,
|
|
483
|
+
judgePassed: judge === null ? unavailable("judge.json absent") : typeof judge.passed === "number" ? judge.passed : null,
|
|
484
|
+
judgeTotal: judge === null ? unavailable("judge.json absent") : typeof judge.total === "number" ? judge.total : null,
|
|
485
|
+
verifyPass: typeof result?.verify_pass === "boolean" ? result.verify_pass : gap("verifyPass", "result.json absent or has no verify_pass"),
|
|
486
|
+
verifyRc: typeof result?.verify_rc === "number" ? result.verify_rc : unavailable("result.json absent or has no verify_rc"),
|
|
487
|
+
patch: patchStats,
|
|
488
|
+
judgeSource: src.judgeSource
|
|
489
|
+
};
|
|
490
|
+
}
|
|
491
|
+
function pickString(rec, key) {
|
|
492
|
+
const v = rec?.[key];
|
|
493
|
+
return typeof v === "string" ? v : null;
|
|
494
|
+
}
|
|
495
|
+
//#endregion
|
|
496
|
+
//#region src/supervisor-run/analyze-spend.ts
|
|
497
|
+
/** Spend records of the root and its children, split into reporting and unreporting. */
|
|
498
|
+
function collectSpendFacts(a) {
|
|
499
|
+
const { tree, rootId, workerSpawns, workerCloses } = a;
|
|
500
|
+
const rootChildIds = new Set(workerSpawns.filter((spawn) => spawn.parent === rootId).map((spawn) => spawn.id));
|
|
501
|
+
const rootChildCloses = workerCloses.filter((close) => rootChildIds.has(close.id));
|
|
286
502
|
const rootChildSpends = rootChildCloses.filter((close) => close.hasSpend);
|
|
287
503
|
const workerUsdUnknownNodes = rootChildSpends.filter((close) => !close.spend.usdKnown).map((close) => close.id);
|
|
288
504
|
const workerTokensUnknownNodes = rootChildSpends.filter((close) => !close.spend.tokensKnown).map((close) => close.id);
|
|
@@ -318,17 +534,49 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
|
|
|
318
534
|
};
|
|
319
535
|
const unpriced = (unknown, total, nodes) => `Runtime recorded usdKnown:false on ${unknown} of ${total} spend record(s)${nameNodes(nodes)}`;
|
|
320
536
|
const unreportedTokens = (unknown, total, nodes) => `Runtime recorded tokensKnown:false on ${unknown} of ${total} spend record(s)${nameNodes(nodes)}`;
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
537
|
+
return {
|
|
538
|
+
rootChildCloses,
|
|
539
|
+
rootChildSpends,
|
|
540
|
+
workerUsdUnknownNodes,
|
|
541
|
+
workerTokensUnknownNodes,
|
|
542
|
+
journalWorkerIn,
|
|
543
|
+
journalWorkerOut,
|
|
544
|
+
journalWorkerUsd,
|
|
545
|
+
workerUsdById,
|
|
546
|
+
workerUsdByLabel,
|
|
547
|
+
usdKnownRecords,
|
|
548
|
+
usdUnknownRecords,
|
|
549
|
+
usdUnknownNodes,
|
|
550
|
+
brainUsdUnknownNodes,
|
|
551
|
+
brainTokensUnknownNodes,
|
|
552
|
+
unpriced,
|
|
553
|
+
unreportedTokens,
|
|
554
|
+
usdPartial: usdKnownRecords > 0 && usdUnknownRecords > 0,
|
|
555
|
+
usdAllUnknown: usdKnownRecords === 0 && usdUnknownRecords > 0,
|
|
556
|
+
brainTokensUnreported: tree.brain.tokensUnknownCount === 0 ? null : unreportedTokens(tree.brain.tokensUnknownCount, tree.brain.meteredCount, brainTokensUnknownNodes),
|
|
557
|
+
brainUsdUnreported: tree.brain.usdKnownCount > 0 || tree.brain.usdUnknownCount === 0 ? null : unpriced(tree.brain.usdUnknownCount, tree.brain.meteredCount, brainUsdUnknownNodes),
|
|
558
|
+
workerUsdUnreported: workerUsdUnknownNodes.length === 0 || workerUsdUnknownNodes.length < rootChildSpends.length ? null : unpriced(workerUsdUnknownNodes.length, rootChildSpends.length, workerUsdUnknownNodes)
|
|
559
|
+
};
|
|
560
|
+
}
|
|
561
|
+
/** Worker token totals: journal spend plus harness session counts, or the reason they are absent. */
|
|
562
|
+
function measureWorkerTokens(a, f) {
|
|
563
|
+
const { src, gap, haveJournal } = a;
|
|
564
|
+
const { rootChildSpends, workerTokensUnknownNodes, journalWorkerIn, journalWorkerOut, unreportedTokens } = f;
|
|
326
565
|
const sq = src.harnessWorkerTokens;
|
|
327
566
|
const harnessGapReason = src.harnessMissingReason ?? "harness session store unavailable and journal settled spend is 0";
|
|
328
567
|
const workerTokenLimit = src.limits.workerTokens;
|
|
329
568
|
const workerTokensUnreported = workerTokensUnknownNodes.length === 0 ? null : unreportedTokens(workerTokensUnknownNodes.length, rootChildSpends.length, workerTokensUnknownNodes);
|
|
330
|
-
|
|
331
|
-
|
|
569
|
+
return {
|
|
570
|
+
sq,
|
|
571
|
+
workerTokenLimit,
|
|
572
|
+
workerIn: workerTokenLimit !== null ? gap("workers.tokensIn", workerTokenLimit) : workerTokensUnreported !== null ? gap("workers.tokensIn", workerTokensUnreported) : sq !== null ? journalWorkerIn + sq.input : haveJournal ? journalWorkerIn : gap("workers.tokensIn", harnessGapReason),
|
|
573
|
+
workerOut: workerTokenLimit !== null ? unavailable(workerTokenLimit) : workerTokensUnreported !== null ? unavailable(workerTokensUnreported) : sq !== null ? journalWorkerOut + sq.output : haveJournal ? journalWorkerOut : unavailable(harnessGapReason)
|
|
574
|
+
};
|
|
575
|
+
}
|
|
576
|
+
/** Total USD and the close record it is read against. */
|
|
577
|
+
function measureTotalUsd(a, f) {
|
|
578
|
+
const { src, gap, journalMissing, haveJournal, tree, state, result } = a;
|
|
579
|
+
const { journalWorkerUsd, usdKnownRecords, usdUnknownRecords, usdUnknownNodes, unpriced } = f;
|
|
332
580
|
const stateResult = asRecord(state?.result);
|
|
333
581
|
const stateUsd = typeof stateResult.spentUsd === "number" ? stateResult.spentUsd : null;
|
|
334
582
|
const resultSpentTotal = asRecord(result?.spentTotal);
|
|
@@ -337,11 +585,24 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
|
|
|
337
585
|
const usdLimit = src.limits.spendUsd;
|
|
338
586
|
const closeUsdUnreported = stateUsd === null && resultCloseUsd !== null && resultSpentTotal.usdKnown === false;
|
|
339
587
|
const usdUnreportedReason = unpriced(usdUnknownRecords, usdKnownRecords + usdUnknownRecords, usdUnknownNodes);
|
|
340
|
-
|
|
588
|
+
return {
|
|
589
|
+
stateResult,
|
|
590
|
+
stateUsd,
|
|
591
|
+
usdLimit,
|
|
592
|
+
totalUsd: usdLimit !== null ? gap("totalUsd", usdLimit) : stateUsd !== null ? roundDecimal(stateUsd, 6) : !haveJournal ? gap("totalUsd", journalMissing) : usdUnknownRecords > 0 ? gap("totalUsd", usdUnreportedReason) : roundDecimal(tree.brain.usd + journalWorkerUsd, 6),
|
|
593
|
+
closeUsd,
|
|
594
|
+
closeUsdUnreported,
|
|
595
|
+
usdUnreportedReason
|
|
596
|
+
};
|
|
597
|
+
}
|
|
598
|
+
function buildSpend(a, f, t) {
|
|
599
|
+
const { journalMissing, haveJournal, tree, rootId } = a;
|
|
600
|
+
const { journalWorkerUsd, usdKnownRecords, usdUnknownRecords, usdUnknownNodes, usdPartial, usdAllUnknown } = f;
|
|
601
|
+
const { usdLimit, closeUsd, closeUsdUnreported, usdUnreportedReason } = t;
|
|
341
602
|
const journalSpendRecords = usdKnownRecords;
|
|
342
603
|
const journalDerivedAvailable = usdLimit === null && haveJournal && !usdAllUnknown;
|
|
343
604
|
const closeRecordAvailable = usdLimit === null && closeUsd !== null && !closeUsdUnreported;
|
|
344
|
-
|
|
605
|
+
return {
|
|
345
606
|
journalDerived: {
|
|
346
607
|
usd: journalDerivedAvailable ? roundDecimal(tree.brain.usd + journalWorkerUsd, 6) : unavailable(usdLimit ?? (haveJournal ? usdUnreportedReason : journalMissing)),
|
|
347
608
|
records: journalDerivedAvailable ? journalSpendRecords : 0,
|
|
@@ -357,335 +618,63 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
|
|
|
357
618
|
unknownNodes: closeUsdUnreported && rootId !== null ? [rootId] : []
|
|
358
619
|
}
|
|
359
620
|
};
|
|
360
|
-
const noteCaptureGaps = (owner, captures) => {
|
|
361
|
-
if (captures === null || captures?.length === 0) gaps.push(`${owner} workspaceCapture: no retained output capture evidence`);
|
|
362
|
-
else if (captures !== void 0) for (const [index, capture] of captures.entries()) {
|
|
363
|
-
const source = `${owner} workspaceCapture[${index + 1}]`;
|
|
364
|
-
if (capture.projectionGap !== null) gaps.push(`${source}: ${capture.projectionGap}`);
|
|
365
|
-
if (capture.coverageComplete === false) gaps.push(`${source}: ${capture.incompleteReason ?? "provider reported incomplete coverage"}`);
|
|
366
|
-
if (capture.coverageComplete === null && capture.projectionGap === null) gaps.push(`${source}: capture coverage was not recorded`);
|
|
367
|
-
}
|
|
368
|
-
};
|
|
369
|
-
noteCaptureGaps(`root ${rootId ?? src.runRef}`, src.rootWorkspaceCaptures);
|
|
370
|
-
const perWorker = (src.workers ?? []).map((w) => {
|
|
371
|
-
noteCaptureGaps(`worker ${w.workerId ?? w.label}`, w.workspaceCaptures);
|
|
372
|
-
const f = tree.workerLogs.get(workerSourceKey(w));
|
|
373
|
-
const matchingSpawns = spawnsForSource(w);
|
|
374
|
-
const spawn = spawnForSource(w);
|
|
375
|
-
const close = spawn === null ? null : closeById.get(spawn.id) ?? null;
|
|
376
|
-
const passed = close?.valid ?? f?.passed ?? null;
|
|
377
|
-
const matchingRoles = new Set(matchingSpawns.map((candidate) => candidate.role));
|
|
378
|
-
const matchingRuntimes = new Set(matchingSpawns.map((candidate) => candidate.runtime));
|
|
379
|
-
const matchingProfiles = new Set(matchingSpawns.map((candidate) => candidate.profileDigest));
|
|
380
|
-
const journalWallMs = spawn?.at !== null && spawn?.at !== void 0 && close?.at !== null && close?.at !== void 0 && close.at >= spawn.at ? close.at - spawn.at : null;
|
|
381
|
-
return {
|
|
382
|
-
workerId: w.workerId ?? null,
|
|
383
|
-
worker: w.label,
|
|
384
|
-
role: matchingRoles.size === 1 ? matchingSpawns[0]?.role ?? null : null,
|
|
385
|
-
runtime: matchingRuntimes.size === 1 ? matchingSpawns[0]?.runtime ?? null : null,
|
|
386
|
-
profileDigest: matchingProfiles.size === 1 ? matchingSpawns[0]?.profileDigest ?? null : null,
|
|
387
|
-
status: close?.status ?? null,
|
|
388
|
-
failure: close?.reason ?? null,
|
|
389
|
-
infra: close?.infra ?? null,
|
|
390
|
-
wallMs: f?.started != null && f.finishedAt != null ? f.finishedAt - f.started : journalWallMs,
|
|
391
|
-
tokensIn: w.tokensIn ?? (close?.hasSpend === true && close.spend.tokensKnown ? close.spend.tokens.input : null),
|
|
392
|
-
tokensOut: w.tokensOut ?? (close?.hasSpend === true && close.spend.tokensKnown ? close.spend.tokens.output : null),
|
|
393
|
-
usd: usdLimit !== null ? null : w.workerId === void 0 ? workerUsdByLabel.get(w.label) ?? null : workerUsdById.get(w.workerId) ?? null,
|
|
394
|
-
patchBytes: w.patchBytes ?? f?.finishedPatchBytes ?? null,
|
|
395
|
-
passed,
|
|
396
|
-
score: close?.score ?? f?.score ?? null,
|
|
397
|
-
workspaceCaptures: w.workspaceCaptures ?? null
|
|
398
|
-
};
|
|
399
|
-
});
|
|
400
|
-
const wallDistribution = summarizeNumberSeries(perWorker.map((w) => w.wallMs).filter((w) => w !== null));
|
|
401
|
-
const brainCalls = parseJsonl(src.brainLog);
|
|
402
|
-
const managerTokenLimit = src.limits.managerTokens;
|
|
403
|
-
const economics = {
|
|
404
|
-
resourceRecords: src.journal === null && !tree.resourceRecords?.length ? unavailable(journalMissing) : tree.resourceRecords ?? [],
|
|
405
|
-
brain: {
|
|
406
|
-
tokensIn: managerTokenLimit !== null ? gap("brain.tokensIn", managerTokenLimit) : !haveJournal ? gap("brain.tokensIn", journalMissing) : brainTokensUnreported !== null ? gap("brain.tokensIn", brainTokensUnreported) : tree.brain.tokensIn,
|
|
407
|
-
tokensOut: managerTokenLimit !== null ? unavailable(managerTokenLimit) : !haveJournal ? unavailable(journalMissing) : brainTokensUnreported !== null ? unavailable(brainTokensUnreported) : tree.brain.tokensOut,
|
|
408
|
-
usd: usdLimit !== null ? unavailable(usdLimit) : !haveJournal ? unavailable(journalMissing) : brainUsdUnreported !== null ? unavailable(brainUsdUnreported) : roundDecimal(tree.brain.usd, 6),
|
|
409
|
-
cacheRead: managerTokenLimit !== null ? unavailable(managerTokenLimit) : !haveJournal ? unavailable(journalMissing) : brainTokensUnreported !== null ? unavailable(brainTokensUnreported) : !tree.brain.hasCache ? unavailable(NO_CACHE_COUNTERS) : tree.brain.cacheBreakdownKnown ? tree.brain.cacheRead : unavailable(NO_CACHE_BREAKDOWN),
|
|
410
|
-
cacheWrite: managerTokenLimit !== null ? unavailable(managerTokenLimit) : !haveJournal ? unavailable(journalMissing) : brainTokensUnreported !== null ? unavailable(brainTokensUnreported) : !tree.brain.hasCache ? unavailable(NO_CACHE_COUNTERS) : tree.brain.cacheBreakdownKnown ? tree.brain.cacheWrite : unavailable(NO_CACHE_BREAKDOWN),
|
|
411
|
-
source: managerTokenLimit ?? (haveJournal ? `journal metered events (n=${tree.brain.meteredCount})${tree.brain.usdKnownCount > 0 && tree.brain.usdUnknownCount > 0 ? ` — ${tree.brain.usdUnknownCount} unpriced` : ""}` : journalMissing)
|
|
412
|
-
},
|
|
413
|
-
brainTruncations: src.brainLog === null ? gap("brain.brainTruncations", src.brainLogMissingReason ?? (src.supRunDir === null ? "no Runtime supervisor run directory" : "brain.jsonl absent — this source has no brain-call tap, so truncation cannot be ruled out")) : brainCalls.filter((c) => c.finish_reason === "length").length,
|
|
414
|
-
workers: {
|
|
415
|
-
tokensIn: workerIn,
|
|
416
|
-
tokensOut: workerOut,
|
|
417
|
-
cacheRead: workerTokenLimit !== null ? unavailable(workerTokenLimit) : sq?.cacheRead !== void 0 ? sq.cacheRead : unavailable(NO_CACHE_COUNTERS),
|
|
418
|
-
cacheWrite: workerTokenLimit !== null ? unavailable(workerTokenLimit) : sq?.cacheWrite !== void 0 ? sq.cacheWrite : unavailable(NO_CACHE_COUNTERS),
|
|
419
|
-
usd: usdLimit !== null ? unavailable(usdLimit) : !haveJournal ? unavailable(journalMissing) : workerUsdUnreported !== null ? unavailable(workerUsdUnreported) : roundDecimal(journalWorkerUsd, 6),
|
|
420
|
-
source: `${workerTokenLimit !== null ? workerTokenLimit : sq !== null ? `journal settled spend + ${sq.store} sessions (n=${sq.sessions})` : `journal settled spend only — ${src.harnessMissingReason ?? "harness session store unavailable"}`}${workerUsdUnknownNodes.length > 0 && workerUsdUnknownNodes.length < rootChildSpends.length ? ` — ${workerUsdUnknownNodes.length} unpriced` : ""}`
|
|
421
|
-
},
|
|
422
|
-
spend,
|
|
423
|
-
totalUsd,
|
|
424
|
-
totalUsdSource: usdLimit !== null ? usdLimit : stateUsd !== null ? `state.json result.spentUsd${rootChildCloses.length > 0 && journalWorkerUsd === 0 ? " — brain-priced only; worker CLI inference is unpriced (see worker token counts)" : ""}` : !haveJournal ? journalMissing : usdUnknownRecords > 0 ? usdUnreportedReason : "journal metered + settled usd",
|
|
425
|
-
costPerAcceptedPatchUsd: isUnavailable(totalUsd) ? unavailable(totalUsd.unavailable) : isUnavailable(decision.accepted) ? unavailable(decision.accepted.unavailable) : decision.accepted === 0 ? unavailable("no accepted worker patch (cost has no denominator)") : roundDecimal(totalUsd / decision.accepted, 6),
|
|
426
|
-
workerWallMsDistribution: wallDistribution === null ? unavailable(src.workers === null ? workersGapReason : "no worker start/finish pairs captured") : wallDistribution,
|
|
427
|
-
perWorker: src.workers === null ? unavailable(workersGapReason) : perWorker
|
|
428
|
-
};
|
|
429
|
-
const patchStats = src.patch === null ? gap("patch", src.limits.deliverables ?? "delivered patch file absent") : parsePatch(src.patch);
|
|
430
|
-
const outcome = {
|
|
431
|
-
supStatus: isUnavailable(terminal.supStatus) ? gap("supStatus", terminal.supStatus.unavailable) : terminal.supStatus,
|
|
432
|
-
supStatusSource: terminal.supStatusSource,
|
|
433
|
-
supReason: terminal.supReason,
|
|
434
|
-
failure: terminal.failure,
|
|
435
|
-
supVerdict: pickString(state, "verdict") ?? pickString(result, "sup_verdict") ?? unavailable("no state.json / result.json verdict"),
|
|
436
|
-
delivered: typeof stateResult.delivered === "boolean" ? stateResult.delivered : typeof result?.delivered === "boolean" ? result.delivered : unavailable("no delivered flag in state.json or result.json"),
|
|
437
|
-
judgeResolved: judge === null ? gap("judge", "judge.json absent") : typeof judge.resolved === "boolean" ? judge.resolved : null,
|
|
438
|
-
judgeScore: judge === null ? unavailable("judge.json absent") : typeof judge.score === "number" ? judge.score : null,
|
|
439
|
-
judgePassed: judge === null ? unavailable("judge.json absent") : typeof judge.passed === "number" ? judge.passed : null,
|
|
440
|
-
judgeTotal: judge === null ? unavailable("judge.json absent") : typeof judge.total === "number" ? judge.total : null,
|
|
441
|
-
verifyPass: typeof result?.verify_pass === "boolean" ? result.verify_pass : gap("verifyPass", "result.json absent or has no verify_pass"),
|
|
442
|
-
verifyRc: typeof result?.verify_rc === "number" ? result.verify_rc : unavailable("result.json absent or has no verify_rc"),
|
|
443
|
-
patch: patchStats,
|
|
444
|
-
judgeSource: src.judgeSource
|
|
445
|
-
};
|
|
446
|
-
return {
|
|
447
|
-
schema: SUPERVISOR_RUN_SCHEMA,
|
|
448
|
-
runRef: src.runRef,
|
|
449
|
-
instanceId: src.instanceId,
|
|
450
|
-
arm: src.arm,
|
|
451
|
-
supervisorId: rootId !== null ? rootId : unavailable(journalMissing),
|
|
452
|
-
supervisorProfileDigest: rootSpawn?.profileDigest !== null && rootSpawn?.profileDigest !== void 0 ? rootSpawn.profileDigest : gap("supervisorProfileDigest", "root spawned event has no profile digest"),
|
|
453
|
-
generatedAt: new Date(now()).toISOString(),
|
|
454
|
-
orchestration,
|
|
455
|
-
decision,
|
|
456
|
-
economics,
|
|
457
|
-
outcome,
|
|
458
|
-
rootWorkspaceCaptures: src.rootWorkspaceCaptures ?? null,
|
|
459
|
-
gaps,
|
|
460
|
-
traceCommand: src.traceCommand ?? "npx --yes @tangle-network/traces@latest analyze --harness opencode --cwd <worker-clone-cwd>"
|
|
461
|
-
};
|
|
462
|
-
}
|
|
463
|
-
/** Whether sorted values contain one value in the inclusive interval. */
|
|
464
|
-
function hasNumberBetween(sorted, low, high) {
|
|
465
|
-
let left = 0;
|
|
466
|
-
let right = sorted.length;
|
|
467
|
-
while (left < right) {
|
|
468
|
-
const middle = left + Math.floor((right - left) / 2);
|
|
469
|
-
if (sorted[middle] < low) left = middle + 1;
|
|
470
|
-
else right = middle;
|
|
471
|
-
}
|
|
472
|
-
return left < sorted.length && sorted[left] <= high;
|
|
473
|
-
}
|
|
474
|
-
/** `[driver] registered tools: …supervisor_steer…` is a banner, not an invocation. */
|
|
475
|
-
function registrationMentions(driverLog) {
|
|
476
|
-
let n = 0;
|
|
477
|
-
for (const line of driverLog.split("\n")) if (line.includes("registered tools:") && line.includes("supervisor_steer")) n += 1;
|
|
478
|
-
return n;
|
|
479
|
-
}
|
|
480
|
-
function pickString(rec, key) {
|
|
481
|
-
const v = rec?.[key];
|
|
482
|
-
return typeof v === "string" ? v : null;
|
|
483
621
|
}
|
|
484
622
|
//#endregion
|
|
485
|
-
//#region src/supervisor-run/
|
|
486
|
-
/**
|
|
487
|
-
*
|
|
488
|
-
*
|
|
489
|
-
*
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
* The
|
|
493
|
-
*
|
|
494
|
-
*/
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
const
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
}
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
"
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
out.push("");
|
|
541
|
-
return out;
|
|
542
|
-
}
|
|
543
|
-
function fmtMs(v) {
|
|
544
|
-
if (isUnavailable(v)) return `unavailable — ${v.unavailable}`;
|
|
545
|
-
if (v < 1e3) return `${v}ms`;
|
|
546
|
-
const s = v / 1e3;
|
|
547
|
-
if (s < 120) return `${roundDecimal(s, 1)}s`;
|
|
548
|
-
return `${roundDecimal(s / 60, 1)}min`;
|
|
549
|
-
}
|
|
550
|
-
function renderSupervisorRunMarkdown(r) {
|
|
551
|
-
const o = r.orchestration;
|
|
552
|
-
const d = r.decision;
|
|
553
|
-
const e = r.economics;
|
|
554
|
-
const out = [];
|
|
555
|
-
out.push(`# Run report — ${r.instanceId ?? "unknown instance"} [${r.arm ?? "unknown arm"}]`);
|
|
556
|
-
out.push("");
|
|
557
|
-
out.push("```");
|
|
558
|
-
out.push(renderSupervisorRunHeadline(r));
|
|
559
|
-
out.push("```");
|
|
560
|
-
out.push("");
|
|
561
|
-
out.push(`- Run: \`${r.runRef}\``);
|
|
562
|
-
out.push(`- Supervisor: \`${showMeasured(r.supervisorId)}\``);
|
|
563
|
-
out.push(`- Supervisor profile: \`${showMeasured(r.supervisorProfileDigest)}\``);
|
|
564
|
-
out.push(`- Generated: ${r.generatedAt}`);
|
|
565
|
-
out.push("");
|
|
566
|
-
out.push("## Orchestration");
|
|
567
|
-
out.push("");
|
|
568
|
-
out.push("| Metric | Value |");
|
|
569
|
-
out.push("|---|---|");
|
|
570
|
-
out.push(`| Workers spawned | ${showMeasured(o.workersSpawned)} |`);
|
|
571
|
-
out.push(`| Workers settled | ${showMeasured(o.workersSettled)} |`);
|
|
572
|
-
out.push(`| Workers cancelled | ${showMeasured(o.workersCancelled)} |`);
|
|
573
|
-
out.push(`| **Steers (mid-task messages to live workers)** | **${showMeasured(o.steers)}** |`);
|
|
574
|
-
out.push(`| Steers delivered | ${showMeasured(o.steersDelivered)} |`);
|
|
575
|
-
out.push(`| Outer-driver \`supervisor_steer\` calls | ${showMeasured(o.driverSteerCalls)} |`);
|
|
576
|
-
out.push(`| Spawn waves | ${showMeasured(o.waves)} |`);
|
|
577
|
-
out.push(`| Wave sizes | ${isUnavailable(o.waveSizes) ? showMeasured(o.waveSizes) : `[${o.waveSizes.join(", ")}]`} |`);
|
|
578
|
-
out.push(`| Max concurrency | ${showMeasured(o.maxConcurrency)} |`);
|
|
579
|
-
out.push(`| Respawns (after same parent's first settle) | ${showMeasured(o.respawns)} |`);
|
|
580
|
-
out.push(`| Repeated labels | ${isUnavailable(o.repeatedLabels) ? showMeasured(o.repeatedLabels) : o.repeatedLabels.length === 0 ? "none" : o.repeatedLabels.join(", ")} |`);
|
|
581
|
-
out.push(`| Delegation depth | ${showMeasured(o.delegationDepth)} |`);
|
|
582
|
-
out.push(`| Time to first spawn | ${fmtMs(o.timeToFirstSpawnMs)} |`);
|
|
583
|
-
out.push(`| Supervisor wall | ${fmtMs(o.supervisorWallMs)}${isUnavailable(o.supervisorWallSource) ? "" : ` (source: ${o.supervisorWallSource}${o.supervisorWallSource === "journal-span" ? ", lower bound" : ""})`} |`);
|
|
584
|
-
out.push(`| Idle (zero live workers) | ${fmtMs(o.idleMs)} (${showMeasured(o.idlePct)}%) |`);
|
|
585
|
-
out.push(`| Worker utilization (Σ worker wall ÷ supervisor wall) | ${showMeasured(o.workerUtilization)} |`);
|
|
586
|
-
out.push("");
|
|
587
|
-
if (!isUnavailable(o.steersByWorker) && o.steersByWorker.length > 0) {
|
|
588
|
-
out.push("### Steers per worker");
|
|
589
|
-
out.push("");
|
|
590
|
-
out.push("| Worker id | Label | Queued | Delivered |");
|
|
591
|
-
out.push("|---|---|---:|---:|");
|
|
592
|
-
for (const s of o.steersByWorker) out.push(`| ${s.workerId === null ? "unavailable — worker id absent; label join used" : `\`${s.workerId}\``} | \`${s.worker}\` | ${s.queued} | ${s.delivered} |`);
|
|
593
|
-
out.push("");
|
|
594
|
-
} else if (isUnavailable(o.steersByWorker)) out.push(`### Steers per worker\n\nunavailable — ${o.steersByWorker.unavailable}\n`);
|
|
595
|
-
out.push("## Decision quality");
|
|
596
|
-
out.push("");
|
|
597
|
-
out.push("| Metric | Value |");
|
|
598
|
-
out.push("|---|---|");
|
|
599
|
-
out.push(`| Settled by status | ${fmtCounts(d.settledByStatus)} |`);
|
|
600
|
-
out.push(`| Settled verdicts | ${fmtCounts(d.settledVerdicts)} |`);
|
|
601
|
-
out.push(`| Accepted (verify green + patch bytes) | ${showMeasured(d.accepted)} |`);
|
|
602
|
-
out.push(`| Rejected (verify red) | ${showMeasured(d.rejected)} |`);
|
|
603
|
-
out.push(`| Empty pass (green, no patch) | ${showMeasured(d.emptyPass)} |`);
|
|
604
|
-
out.push(`| Evidence → respawn sequences | ${showMeasured(d.observeThenRespawn)} |`);
|
|
605
|
-
out.push(`| Respawn with no same-parent settled evidence in front | ${showMeasured(d.respawnWithoutEvidence)} |`);
|
|
606
|
-
out.push(`| Review actions (steers + worker questions) | ${showMeasured(d.reviewActions)} |`);
|
|
607
|
-
out.push(`| Worker evidence returned | ${isUnavailable(d.workerEvidenceBytes) ? showMeasured(d.workerEvidenceBytes) : `${d.workerEvidenceBytes} bytes`} |`);
|
|
608
|
-
out.push("");
|
|
609
|
-
out.push("## Economics");
|
|
610
|
-
out.push("");
|
|
611
|
-
out.push("| Role | Tokens in | Tokens out | Cache read | Cache write | USD | Source |");
|
|
612
|
-
out.push("|---|---:|---:|---:|---:|---:|---|");
|
|
613
|
-
out.push(`| brain | ${showMeasured(e.brain.tokensIn)} | ${showMeasured(e.brain.tokensOut)} | ${showMeasured(e.brain.cacheRead)} | ${showMeasured(e.brain.cacheWrite)} | ${showMeasured(e.brain.usd)} | ${e.brain.source} |`);
|
|
614
|
-
out.push(`| workers | ${showMeasured(e.workers.tokensIn)} | ${showMeasured(e.workers.tokensOut)} | ${showMeasured(e.workers.cacheRead)} | ${showMeasured(e.workers.cacheWrite)} | ${showMeasured(e.workers.usd)} | ${e.workers.source} |`);
|
|
615
|
-
out.push("");
|
|
616
|
-
if (!isUnavailable(e.brainTruncations) && e.brainTruncations > 0) out.push(`- **BRAIN OUTPUT TRUNCATED: ${e.brainTruncations} completion(s) hit \`finish_reason: "length"\`** — the supervisor acted on a half-written plan. Its output ceiling is too low; see \`brain.jsonl\` for the per-call \`req_max_tokens\`.`);
|
|
617
|
-
else out.push(`- Brain completions truncated (finish_reason=length): ${showMeasured(e.brainTruncations)}`);
|
|
618
|
-
out.push(`- Total USD: ${showMeasured(e.totalUsd)} (source: ${e.totalUsdSource})`);
|
|
619
|
-
out.push(`- Spend measured two ways: journal-derived $${showMeasured(e.spend.journalDerived.usd)} over ${e.spend.journalDerived.records} journal record(s) (execution accounting) · close-record $${showMeasured(e.spend.closeRecord.usd)} over ${e.spend.closeRecord.records} close record(s) (billing-shaped). Divergence is a signal, not an error.`);
|
|
620
|
-
out.push(`- Cost per accepted patch: ${showMeasured(e.costPerAcceptedPatchUsd)}`);
|
|
621
|
-
if (isUnavailable(e.workerWallMsDistribution)) out.push(`- Worker wall distribution: unavailable — ${e.workerWallMsDistribution.unavailable}`);
|
|
622
|
-
else {
|
|
623
|
-
const w = e.workerWallMsDistribution;
|
|
624
|
-
out.push(`- Worker wall (n=${w.n}): min ${fmtMs(w.min)} / p50 ${fmtMs(w.p50)} / p90 ${fmtMs(w.p90)} / max ${fmtMs(w.max)} / Σ ${fmtMs(w.sum)}`);
|
|
625
|
-
}
|
|
626
|
-
out.push("");
|
|
627
|
-
if (!isUnavailable(e.perWorker) && e.perWorker.length > 0) {
|
|
628
|
-
out.push("| Worker id | Label | Role | Runtime | Profile digest | Status | Failure | Infra | Wall | Tokens in | Tokens out | Patch bytes | Verify passed | Score |");
|
|
629
|
-
out.push("|---|---|---|---|---|---|---|---|---:|---:|---:|---:|---|---:|");
|
|
630
|
-
for (const w of e.perWorker) out.push(`| ${w.workerId === null ? "unavailable — worker id absent; label join used" : `\`${w.workerId}\``} | \`${w.worker}\` | ${w.role ?? "unavailable — source recorded no role"} | ${w.runtime ?? "unavailable — source recorded no runtime"} | ${w.profileDigest === null ? "unavailable — source recorded no profile digest" : `\`${w.profileDigest}\``} | ${w.status ?? "unavailable — no terminal event"} | ${w.failure ?? "none recorded"} | ${w.infra ?? "unavailable"} | ${w.wallMs === null ? "unavailable — no spawn/finish pair" : fmtMs(w.wallMs)} | ${w.tokensIn ?? "unavailable — store does not attribute tokens per worker"} | ${w.tokensOut ?? "unavailable — store does not attribute tokens per worker"} | ${w.patchBytes ?? "unavailable — no worker patch file"} | ${w.passed === null ? "unavailable — no verdict" : String(w.passed)} | ${w.score ?? "unavailable — no numeric score"} |`);
|
|
631
|
-
out.push("");
|
|
632
|
-
}
|
|
633
|
-
if (e.resourceRecords !== void 0) out.push("## Named resource receipts", "", ...resourceRows(e.resourceRecords));
|
|
634
|
-
out.push("## Outcome");
|
|
635
|
-
out.push("");
|
|
636
|
-
out.push("| Metric | Value |");
|
|
637
|
-
out.push("|---|---|");
|
|
638
|
-
out.push(`| Supervisor status | ${showMeasured(r.outcome.supStatus)} |`);
|
|
639
|
-
out.push(`| Status source | ${showMeasured(r.outcome.supStatusSource)} |`);
|
|
640
|
-
out.push(`| Terminal reason | ${showMeasured(r.outcome.supReason)} |`);
|
|
641
|
-
out.push(`| Failure | ${fmtFailure(r.outcome.failure)} |`);
|
|
642
|
-
out.push(`| Supervisor verdict | ${showMeasured(r.outcome.supVerdict)} |`);
|
|
643
|
-
out.push(`| Delivered | ${showMeasured(r.outcome.delivered)} |`);
|
|
644
|
-
out.push(`| Judge resolved | ${showMeasured(r.outcome.judgeResolved)} |`);
|
|
645
|
-
out.push(`| Judge score | ${showMeasured(r.outcome.judgeScore)} |`);
|
|
646
|
-
out.push(`| Judge passed / total | ${showMeasured(r.outcome.judgePassed)} / ${showMeasured(r.outcome.judgeTotal)} |`);
|
|
647
|
-
out.push(`| Judge source | ${r.outcome.judgeSource ?? "unavailable — no judge.json and no ledger row"} |`);
|
|
648
|
-
out.push(`| Verify gate | pass=${showMeasured(r.outcome.verifyPass)} rc=${showMeasured(r.outcome.verifyRc)} |`);
|
|
649
|
-
if (isUnavailable(r.outcome.patch)) out.push(`| Patch | unavailable — ${r.outcome.patch.unavailable} |`);
|
|
650
|
-
else {
|
|
651
|
-
const p = r.outcome.patch;
|
|
652
|
-
out.push(`| Patch | ${p.files} file(s), +${p.linesAdded}/-${p.linesRemoved}, test files touched: ${p.testFilesTouched.length === 0 ? "none" : p.testFilesTouched.join(", ")} |`);
|
|
653
|
-
}
|
|
654
|
-
out.push("");
|
|
655
|
-
out.push("## Gaps");
|
|
656
|
-
out.push("");
|
|
657
|
-
if (r.gaps.length === 0) out.push("None — every metric above is backed by a present artifact.");
|
|
658
|
-
else for (const g of r.gaps) out.push(`- ${g}`);
|
|
659
|
-
out.push("");
|
|
660
|
-
out.push(`> Harness-session view of the same run (model calls, stuck loops, tool errors): \`${r.traceCommand}\``);
|
|
661
|
-
out.push("");
|
|
662
|
-
return out.join("\n");
|
|
663
|
-
}
|
|
664
|
-
function fmtCounts(v) {
|
|
665
|
-
if (isUnavailable(v)) return showMeasured(v);
|
|
666
|
-
const entries = Object.entries(v);
|
|
667
|
-
return entries.length === 0 ? "none" : entries.map(([k, n]) => `${k}=${n}`).join(", ");
|
|
668
|
-
}
|
|
669
|
-
function renderSupervisorRollupMarkdown(rollup, title = "Round rollup") {
|
|
670
|
-
const out = [];
|
|
671
|
-
out.push(`# ${title}`);
|
|
672
|
-
out.push("");
|
|
673
|
-
out.push(`- Cells: ${rollup.cells}`);
|
|
674
|
-
out.push(`- **Steers across all cells: ${showMeasured(rollup.steersTotal)}** (cells with ≥1 steer: ${showMeasured(rollup.cellsWithSteers)}; cells where the steer count is unavailable: ${rollup.cellsWithUnavailableSteers})`);
|
|
675
|
-
out.push(`- Waves per cell (mean): ${showMeasured(rollup.wavesMean)}`);
|
|
676
|
-
out.push(`- Max concurrency observed: ${showMeasured(rollup.maxConcurrencyMax)}`);
|
|
677
|
-
out.push(`- Worker utilization (mean): ${showMeasured(rollup.utilizationMean)}`);
|
|
678
|
-
out.push(`- Idle share (mean): ${showMeasured(rollup.idlePctMean)}%`);
|
|
679
|
-
out.push(`- Workers spawned: ${showMeasured(rollup.workersSpawnedTotal)} · accepted: ${showMeasured(rollup.acceptedTotal)}`);
|
|
680
|
-
out.push(`- Spend: $${showMeasured(rollup.usdTotal)} · judged resolved: ${showMeasured(rollup.resolvedCount)}/${rollup.cells}`);
|
|
681
|
-
out.push(`- Spend measured two ways: journal-derived $${showMeasured(rollup.spendUsd.journalDerived.value)} over ${rollup.spendUsd.journalDerived.runs}/${rollup.cells} runs (execution accounting) · close-record $${showMeasured(rollup.spendUsd.closeRecord.value)} over ${rollup.spendUsd.closeRecord.runs}/${rollup.cells} runs (billing-shaped)`);
|
|
682
|
-
out.push("");
|
|
683
|
-
out.push("| Instance | Arm | Steers | Waves | Utilization | Idle % | Resolved | USD |");
|
|
684
|
-
out.push("|---|---|---:|---:|---:|---:|---|---:|");
|
|
685
|
-
for (const c of rollup.perCell) out.push(`| ${c.instanceId ?? "?"} | ${c.arm ?? "?"} | ${showMeasured(c.steers)} | ${showMeasured(c.waves)} | ${showMeasured(c.utilization)} | ${showMeasured(c.idlePct)} | ${showMeasured(c.resolved)} | ${showMeasured(c.usd)} |`);
|
|
686
|
-
out.push("");
|
|
687
|
-
for (const cell of rollup.perCell) if (cell.resourceRecords !== void 0) out.push(`## Resource receipts — ${cell.instanceId ?? "?"} [${cell.arm ?? "?"}]`, "", ...resourceRows(cell.resourceRecords));
|
|
688
|
-
return out.join("\n");
|
|
623
|
+
//#region src/supervisor-run/analyze.ts
|
|
624
|
+
/**
|
|
625
|
+
* The pure analyzer. Takes already-read bytes (`SupervisorRunSources`) and
|
|
626
|
+
* returns the report — every metric derivable from a synthetic journal string
|
|
627
|
+
* with no filesystem, no process, and no network. All I/O lives in a reader
|
|
628
|
+
* (`reader.ts` is one).
|
|
629
|
+
*
|
|
630
|
+
* The metric groups live in `analyze-*.ts`; this file only runs them in the
|
|
631
|
+
* order the report's `gaps` list is written.
|
|
632
|
+
*/
|
|
633
|
+
/**
|
|
634
|
+
* Analyze already-read supervisor-run bytes. Pure and synchronous: same bytes
|
|
635
|
+
* in, same report out (modulo `generatedAt`, which `now` pins in tests).
|
|
636
|
+
*/
|
|
637
|
+
function analyzeSupervisorRunSources(src, now = Date.now) {
|
|
638
|
+
const a = createAnalysis(src);
|
|
639
|
+
const { gap, gaps, rootId, rootSpawn, journalMissing } = a;
|
|
640
|
+
const wall = measureWall(a);
|
|
641
|
+
const steerFacts = measureSteers(a);
|
|
642
|
+
const concurrency = measureConcurrency(a, wall.wallEndAt);
|
|
643
|
+
const respawns = measureRespawns(a);
|
|
644
|
+
const steps = {
|
|
645
|
+
...wall,
|
|
646
|
+
...steerFacts,
|
|
647
|
+
...concurrency,
|
|
648
|
+
...respawns,
|
|
649
|
+
delegationDepth: measureDelegationDepth(a)
|
|
650
|
+
};
|
|
651
|
+
const orchestration = buildOrchestration(a, steps);
|
|
652
|
+
const decision = measureDecision(a, steps);
|
|
653
|
+
const spendFacts = collectSpendFacts(a);
|
|
654
|
+
const tokens = measureWorkerTokens(a, spendFacts);
|
|
655
|
+
const totals = measureTotalUsd(a, spendFacts);
|
|
656
|
+
const economics = buildEconomics(a, spendFacts, totals, tokens, {
|
|
657
|
+
decision,
|
|
658
|
+
spend: buildSpend(a, spendFacts, totals),
|
|
659
|
+
perWorker: buildPerWorker(a, spendFacts, totals)
|
|
660
|
+
});
|
|
661
|
+
const outcome = buildOutcome(a, totals);
|
|
662
|
+
return {
|
|
663
|
+
schema: SUPERVISOR_RUN_SCHEMA,
|
|
664
|
+
runRef: src.runRef,
|
|
665
|
+
instanceId: src.instanceId,
|
|
666
|
+
arm: src.arm,
|
|
667
|
+
supervisorId: rootId !== null ? rootId : unavailable(journalMissing),
|
|
668
|
+
supervisorProfileDigest: rootSpawn?.profileDigest !== null && rootSpawn?.profileDigest !== void 0 ? rootSpawn.profileDigest : gap("supervisorProfileDigest", "root spawned event has no profile digest"),
|
|
669
|
+
generatedAt: new Date(now()).toISOString(),
|
|
670
|
+
orchestration,
|
|
671
|
+
decision,
|
|
672
|
+
economics,
|
|
673
|
+
outcome,
|
|
674
|
+
rootWorkspaceCaptures: src.rootWorkspaceCaptures ?? null,
|
|
675
|
+
gaps,
|
|
676
|
+
traceCommand: src.traceCommand ?? "npx --yes @tangle-network/traces@latest analyze --harness opencode --cwd <worker-clone-cwd>"
|
|
677
|
+
};
|
|
689
678
|
}
|
|
690
679
|
//#endregion
|
|
691
680
|
//#region src/supervisor-run/coordination-reader.ts
|
|
@@ -1368,13 +1357,6 @@ async function readRuntimeSupervisorRun(runDir, opts = {}) {
|
|
|
1368
1357
|
traceCommand: nativeTraceSummary(workers, transcripts.get(normalized.root))
|
|
1369
1358
|
};
|
|
1370
1359
|
}
|
|
1371
|
-
/** The agent-runtime file-backed layout as a `SupervisorRunReader`. */
|
|
1372
|
-
function runtimeSupervisorRunReader(runDir, opts = {}) {
|
|
1373
|
-
return {
|
|
1374
|
-
runRef: runDir,
|
|
1375
|
-
read: () => readRuntimeSupervisorRun(runDir, opts)
|
|
1376
|
-
};
|
|
1377
|
-
}
|
|
1378
1360
|
/**
|
|
1379
1361
|
* True when a directory holds any file only Runtime's durable layer writes:
|
|
1380
1362
|
* the spawn journal, the observer journal, or the failure record. A run that
|
|
@@ -1413,56 +1395,6 @@ function isReader(input) {
|
|
|
1413
1395
|
return typeof input.read === "function";
|
|
1414
1396
|
}
|
|
1415
1397
|
/**
|
|
1416
|
-
* Read a completed run, write `run-report.json` + `run-report.md` beside its
|
|
1417
|
-
* artifacts, and append the headline block to the run log. Never throws on a
|
|
1418
|
-
* missing artifact — a run that produced nothing still yields a report whose
|
|
1419
|
-
* every metric says why.
|
|
1420
|
-
*/
|
|
1421
|
-
async function writeSupervisorRunReport(runDir, opts = {}) {
|
|
1422
|
-
const report = analyzeSupervisorRunSources(await readRuntimeSupervisorRun(runDir));
|
|
1423
|
-
const md = renderSupervisorRunMarkdown(report);
|
|
1424
|
-
const dest = opts.reportDir ?? runDir;
|
|
1425
|
-
const stem = opts.reportDir === void 0 ? "run-report" : supervisorReportStem(runDir);
|
|
1426
|
-
if (opts.reportDir !== void 0) await mkdir(opts.reportDir, { recursive: true }).catch(() => {});
|
|
1427
|
-
await writeFile(join(dest, `${stem}.json`), JSON.stringify(report, null, 1)).catch(() => {});
|
|
1428
|
-
await writeFile(join(dest, `${stem}.md`), md).catch(() => {});
|
|
1429
|
-
const headline = renderSupervisorRunHeadline(report);
|
|
1430
|
-
if (opts.appendHeadlineTo !== void 0) await appendFile(opts.appendHeadlineTo, `${headline}\n`).catch(() => {});
|
|
1431
|
-
if (opts.echo !== false) console.log(headline);
|
|
1432
|
-
return report;
|
|
1433
|
-
}
|
|
1434
|
-
/**
|
|
1435
|
-
* File stem for out-of-tree reports. Built from the run path's identifying
|
|
1436
|
-
* segments — candidate tag (the segment under `arm-runs/`), rep, instance, arm
|
|
1437
|
-
* — so two runs of the same instance from different candidates/reps never
|
|
1438
|
-
* overwrite each other.
|
|
1439
|
-
*/
|
|
1440
|
-
function supervisorReportStem(runDir) {
|
|
1441
|
-
const parts = runDir.split("/").filter(Boolean);
|
|
1442
|
-
const arm = parts[parts.length - 1] ?? "cell";
|
|
1443
|
-
const iid = parts[parts.length - 2] ?? "instance";
|
|
1444
|
-
const rep = parts.find((p) => /^rep-\d+$/.test(p));
|
|
1445
|
-
const armRunsIdx = parts.indexOf("arm-runs");
|
|
1446
|
-
return [
|
|
1447
|
-
armRunsIdx >= 0 ? parts[armRunsIdx + 1] : void 0,
|
|
1448
|
-
rep,
|
|
1449
|
-
iid,
|
|
1450
|
-
arm
|
|
1451
|
-
].filter((s) => s !== void 0 && s !== "runs").join(".").replace(/[^A-Za-z0-9._-]/g, "_");
|
|
1452
|
-
}
|
|
1453
|
-
/**
|
|
1454
|
-
* Best-effort wrapper for a hot path: a reporting failure must never kill a run
|
|
1455
|
-
* that already produced real work. Returns null and logs the reason instead.
|
|
1456
|
-
*/
|
|
1457
|
-
async function writeSupervisorRunReportSafe(runDir, opts = {}) {
|
|
1458
|
-
try {
|
|
1459
|
-
return await writeSupervisorRunReport(runDir, opts);
|
|
1460
|
-
} catch (err) {
|
|
1461
|
-
console.log(`RUN-REPORT failed for ${runDir}: ${err instanceof Error ? err.message : String(err)}`);
|
|
1462
|
-
return null;
|
|
1463
|
-
}
|
|
1464
|
-
}
|
|
1465
|
-
/**
|
|
1466
1398
|
* Every Runtime supervisor run below `root`.
|
|
1467
1399
|
*
|
|
1468
1400
|
* When `root` itself is one run, return no children so callers can distinguish
|
|
@@ -1489,106 +1421,212 @@ async function findSupervisorRunDirs(root) {
|
|
|
1489
1421
|
return found.sort();
|
|
1490
1422
|
}
|
|
1491
1423
|
//#endregion
|
|
1492
|
-
//#region src/supervisor-run/
|
|
1424
|
+
//#region src/supervisor-run/render.ts
|
|
1493
1425
|
/**
|
|
1494
|
-
*
|
|
1495
|
-
*
|
|
1496
|
-
*
|
|
1497
|
-
* same path and the status comes from the record `terminal-record.ts` names.
|
|
1498
|
-
*
|
|
1499
|
-
* Exit codes follow the other self-parsing subcommands: 2 for a usage error,
|
|
1500
|
-
* 1 when the directory cannot be read (missing, not a directory, or a corrupt
|
|
1501
|
-
* journal or terminal record), 0 after the report was written to stdout.
|
|
1426
|
+
* Human-readable renderings of a supervisor-run report. Zero and unavailable
|
|
1427
|
+
* render differently on purpose (`0` vs `unavailable — <reason>`), because the
|
|
1428
|
+
* two have driven opposite conclusions about the same architecture.
|
|
1502
1429
|
*/
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
const
|
|
1509
|
-
const
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
}
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
}
|
|
1520
|
-
|
|
1521
|
-
let runDir = null;
|
|
1522
|
-
let format = "headline";
|
|
1523
|
-
for (let i = 0; i < argv.length; i++) {
|
|
1524
|
-
const arg = argv[i];
|
|
1525
|
-
if (arg === "--format") {
|
|
1526
|
-
const raw = argv[++i];
|
|
1527
|
-
if (raw === void 0 || !isFormat(raw)) throw new Error(`--format expects one of ${FORMATS.join("|")}, got ${JSON.stringify(raw ?? "")}`);
|
|
1528
|
-
format = raw;
|
|
1529
|
-
} else if (arg.startsWith("--format=")) {
|
|
1530
|
-
const raw = arg.slice(9);
|
|
1531
|
-
if (!isFormat(raw)) throw new Error(`--format expects one of ${FORMATS.join("|")}, got ${JSON.stringify(raw)}`);
|
|
1532
|
-
format = raw;
|
|
1533
|
-
} else if (arg.startsWith("--")) throw new Error(`unknown flag "${arg}"`);
|
|
1534
|
-
else if (runDir === null) runDir = arg;
|
|
1535
|
-
else throw new Error(`unexpected argument "${arg}"`);
|
|
1536
|
-
}
|
|
1537
|
-
if (runDir === null) throw new Error(SUPERVISOR_RUN_USAGE);
|
|
1538
|
-
return {
|
|
1539
|
-
runDir,
|
|
1540
|
-
format
|
|
1541
|
-
};
|
|
1430
|
+
/**
|
|
1431
|
+
* The block appended to a run log after every run — the answers an operator asks
|
|
1432
|
+
* for, in the log tail, with no extra command.
|
|
1433
|
+
*/
|
|
1434
|
+
function renderSupervisorRunHeadline(r) {
|
|
1435
|
+
const o = r.orchestration;
|
|
1436
|
+
const steerNote = isUnavailable(o.steers) ? `unavailable — ${o.steers.unavailable}` : o.steers === 0 ? "0 (spawn→wait→respawn only; no mid-task steering)" : `${o.steers} queued / ${showMeasured(o.steersDelivered)} delivered`;
|
|
1437
|
+
return [
|
|
1438
|
+
`RUN-REPORT ${r.instanceId ?? "?"} [${r.arm ?? "?"}]`,
|
|
1439
|
+
` status=${showMeasured(r.outcome.supStatus)} source=${showMeasured(r.outcome.supStatusSource)} reason=${showMeasured(r.outcome.supReason)} failure=${fmtFailure(r.outcome.failure)}`,
|
|
1440
|
+
` steers=${steerNote}`,
|
|
1441
|
+
` waves=${showMeasured(o.waves)} sizes=${isUnavailable(o.waveSizes) ? `unavailable — ${o.waveSizes.unavailable}` : `[${o.waveSizes.join(",")}]`} workers=${showMeasured(o.workersSpawned)} settled=${showMeasured(o.workersSettled)} cancelled=${showMeasured(o.workersCancelled)}`,
|
|
1442
|
+
` concurrency max=${showMeasured(o.maxConcurrency)} utilization=${showMeasured(o.workerUtilization)} idle=${fmtMs(o.idleMs)} (${showMeasured(o.idlePct)}%) wall=${fmtMs(o.supervisorWallMs)}${o.supervisorWallSource === "journal-span" ? " (journal-span lower bound)" : ""}`,
|
|
1443
|
+
` respawns=${showMeasured(o.respawns)} evidence→respawn=${showMeasured(r.decision.observeThenRespawn)} blind-respawn=${showMeasured(r.decision.respawnWithoutEvidence)} depth=${showMeasured(o.delegationDepth)}`,
|
|
1444
|
+
` accepted=${showMeasured(r.decision.accepted)} rejected=${showMeasured(r.decision.rejected)} empty-pass=${showMeasured(r.decision.emptyPass)}`,
|
|
1445
|
+
` brain=$${showMeasured(r.economics.brain.usd)} total=$${showMeasured(r.economics.totalUsd)} judge.resolved=${showMeasured(r.outcome.judgeResolved)} score=${showMeasured(r.outcome.judgeScore)} verify=${showMeasured(r.outcome.verifyPass)}`,
|
|
1446
|
+
r.gaps.length > 0 ? ` gaps(${r.gaps.length}): ${r.gaps.join("; ")}` : " gaps: none"
|
|
1447
|
+
].join("\n");
|
|
1542
1448
|
}
|
|
1543
1449
|
/**
|
|
1544
|
-
*
|
|
1545
|
-
*
|
|
1450
|
+
* The recorded error on one line; `none recorded` and `unavailable` stay
|
|
1451
|
+
* distinct, and a throw the run outlived is labeled so it is not read as the
|
|
1452
|
+
* outcome of the settled result beside it.
|
|
1546
1453
|
*/
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
if (
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
}
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1454
|
+
function fmtFailure(v) {
|
|
1455
|
+
if (isUnavailable(v)) return `unavailable — ${v.unavailable}`;
|
|
1456
|
+
if (v === null) return "none recorded";
|
|
1457
|
+
const name = v.name ?? "unavailable — record has no error.name";
|
|
1458
|
+
const message = v.message ?? "unavailable — record has no error.message";
|
|
1459
|
+
const attempt = v.earlierAttempt ? ", earlier attempt" : "";
|
|
1460
|
+
return `${name}: ${message} [${v.source}${v.at === null ? "" : ` at ${v.at}`}${attempt}]`;
|
|
1461
|
+
}
|
|
1462
|
+
function resourceRows(records) {
|
|
1463
|
+
if (isUnavailable(records)) return [`unavailable — ${records.unavailable}`, ""];
|
|
1464
|
+
const out = [
|
|
1465
|
+
"Receipts are source records, not additive totals.",
|
|
1466
|
+
"A false completeness flag means the amount is a recorded subtotal, not complete usage.",
|
|
1467
|
+
"",
|
|
1468
|
+
"| Node | Source | Resource | Unit | Amount | Known |",
|
|
1469
|
+
"|---|---|---|---|---:|---|"
|
|
1470
|
+
];
|
|
1471
|
+
const cell = (value) => value.replace(/\|/g, "\\|").replace(/\r?\n/g, " ");
|
|
1472
|
+
for (const record of records) {
|
|
1473
|
+
const prefix = `| ${cell(record.nodeId ?? "unavailable")} | ${cell(record.source)} |`;
|
|
1474
|
+
if (isUnavailable(record.resources)) out.push(`${prefix} ${cell(showMeasured(record.resources))} | — | — | — |`);
|
|
1475
|
+
else if (record.resources.length === 0) out.push(`${prefix} recorded empty map | — | — | — |`);
|
|
1476
|
+
else for (const receipt of record.resources) out.push(`${prefix} ${cell(showMeasured(receipt.name))} | ${cell(showMeasured(receipt.unit))} | ${cell(showMeasured(receipt.amount))} | ${cell(showMeasured(receipt.known))} |`);
|
|
1560
1477
|
}
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1478
|
+
if (!records.length) out.push("| — | — | no spend records | — | — | — |");
|
|
1479
|
+
out.push("");
|
|
1480
|
+
return out;
|
|
1481
|
+
}
|
|
1482
|
+
function fmtMs(v) {
|
|
1483
|
+
if (isUnavailable(v)) return `unavailable — ${v.unavailable}`;
|
|
1484
|
+
if (v < 1e3) return `${v}ms`;
|
|
1485
|
+
const s = v / 1e3;
|
|
1486
|
+
if (s < 120) return `${roundDecimal(s, 1)}s`;
|
|
1487
|
+
return `${roundDecimal(s / 60, 1)}min`;
|
|
1488
|
+
}
|
|
1489
|
+
function renderSupervisorRunMarkdown(r) {
|
|
1490
|
+
const o = r.orchestration;
|
|
1491
|
+
const d = r.decision;
|
|
1492
|
+
const e = r.economics;
|
|
1493
|
+
const out = [];
|
|
1494
|
+
out.push(`# Run report — ${r.instanceId ?? "unknown instance"} [${r.arm ?? "unknown arm"}]`);
|
|
1495
|
+
out.push("");
|
|
1496
|
+
out.push("```");
|
|
1497
|
+
out.push(renderSupervisorRunHeadline(r));
|
|
1498
|
+
out.push("```");
|
|
1499
|
+
out.push("");
|
|
1500
|
+
out.push(`- Run: \`${r.runRef}\``);
|
|
1501
|
+
out.push(`- Supervisor: \`${showMeasured(r.supervisorId)}\``);
|
|
1502
|
+
out.push(`- Supervisor profile: \`${showMeasured(r.supervisorProfileDigest)}\``);
|
|
1503
|
+
out.push(`- Generated: ${r.generatedAt}`);
|
|
1504
|
+
out.push("");
|
|
1505
|
+
out.push("## Orchestration");
|
|
1506
|
+
out.push("");
|
|
1507
|
+
out.push("| Metric | Value |");
|
|
1508
|
+
out.push("|---|---|");
|
|
1509
|
+
out.push(`| Workers spawned | ${showMeasured(o.workersSpawned)} |`);
|
|
1510
|
+
out.push(`| Workers settled | ${showMeasured(o.workersSettled)} |`);
|
|
1511
|
+
out.push(`| Workers cancelled | ${showMeasured(o.workersCancelled)} |`);
|
|
1512
|
+
out.push(`| **Steers (mid-task messages to live workers)** | **${showMeasured(o.steers)}** |`);
|
|
1513
|
+
out.push(`| Steers delivered | ${showMeasured(o.steersDelivered)} |`);
|
|
1514
|
+
out.push(`| Outer-driver \`supervisor_steer\` calls | ${showMeasured(o.driverSteerCalls)} |`);
|
|
1515
|
+
out.push(`| Spawn waves | ${showMeasured(o.waves)} |`);
|
|
1516
|
+
out.push(`| Wave sizes | ${isUnavailable(o.waveSizes) ? showMeasured(o.waveSizes) : `[${o.waveSizes.join(", ")}]`} |`);
|
|
1517
|
+
out.push(`| Max concurrency | ${showMeasured(o.maxConcurrency)} |`);
|
|
1518
|
+
out.push(`| Respawns (after same parent's first settle) | ${showMeasured(o.respawns)} |`);
|
|
1519
|
+
out.push(`| Repeated labels | ${isUnavailable(o.repeatedLabels) ? showMeasured(o.repeatedLabels) : o.repeatedLabels.length === 0 ? "none" : o.repeatedLabels.join(", ")} |`);
|
|
1520
|
+
out.push(`| Delegation depth | ${showMeasured(o.delegationDepth)} |`);
|
|
1521
|
+
out.push(`| Time to first spawn | ${fmtMs(o.timeToFirstSpawnMs)} |`);
|
|
1522
|
+
out.push(`| Supervisor wall | ${fmtMs(o.supervisorWallMs)}${isUnavailable(o.supervisorWallSource) ? "" : ` (source: ${o.supervisorWallSource}${o.supervisorWallSource === "journal-span" ? ", lower bound" : ""})`} |`);
|
|
1523
|
+
out.push(`| Idle (zero live workers) | ${fmtMs(o.idleMs)} (${showMeasured(o.idlePct)}%) |`);
|
|
1524
|
+
out.push(`| Worker utilization (Σ worker wall ÷ supervisor wall) | ${showMeasured(o.workerUtilization)} |`);
|
|
1525
|
+
out.push("");
|
|
1526
|
+
if (!isUnavailable(o.steersByWorker) && o.steersByWorker.length > 0) {
|
|
1527
|
+
out.push("### Steers per worker");
|
|
1528
|
+
out.push("");
|
|
1529
|
+
out.push("| Worker id | Label | Queued | Delivered |");
|
|
1530
|
+
out.push("|---|---|---:|---:|");
|
|
1531
|
+
for (const s of o.steersByWorker) out.push(`| ${s.workerId === null ? "unavailable — worker id absent; label join used" : `\`${s.workerId}\``} | \`${s.worker}\` | ${s.queued} | ${s.delivered} |`);
|
|
1532
|
+
out.push("");
|
|
1533
|
+
} else if (isUnavailable(o.steersByWorker)) out.push(`### Steers per worker\n\nunavailable — ${o.steersByWorker.unavailable}\n`);
|
|
1534
|
+
out.push("## Decision quality");
|
|
1535
|
+
out.push("");
|
|
1536
|
+
out.push("| Metric | Value |");
|
|
1537
|
+
out.push("|---|---|");
|
|
1538
|
+
out.push(`| Settled by status | ${fmtCounts(d.settledByStatus)} |`);
|
|
1539
|
+
out.push(`| Settled verdicts | ${fmtCounts(d.settledVerdicts)} |`);
|
|
1540
|
+
out.push(`| Accepted (verify green + patch bytes) | ${showMeasured(d.accepted)} |`);
|
|
1541
|
+
out.push(`| Rejected (verify red) | ${showMeasured(d.rejected)} |`);
|
|
1542
|
+
out.push(`| Empty pass (green, no patch) | ${showMeasured(d.emptyPass)} |`);
|
|
1543
|
+
out.push(`| Evidence → respawn sequences | ${showMeasured(d.observeThenRespawn)} |`);
|
|
1544
|
+
out.push(`| Respawn with no same-parent settled evidence in front | ${showMeasured(d.respawnWithoutEvidence)} |`);
|
|
1545
|
+
out.push(`| Review actions (steers + worker questions) | ${showMeasured(d.reviewActions)} |`);
|
|
1546
|
+
out.push(`| Worker evidence returned | ${isUnavailable(d.workerEvidenceBytes) ? showMeasured(d.workerEvidenceBytes) : `${d.workerEvidenceBytes} bytes`} |`);
|
|
1547
|
+
out.push("");
|
|
1548
|
+
out.push("## Economics");
|
|
1549
|
+
out.push("");
|
|
1550
|
+
out.push("| Role | Tokens in | Tokens out | Cache read | Cache write | USD | Source |");
|
|
1551
|
+
out.push("|---|---:|---:|---:|---:|---:|---|");
|
|
1552
|
+
out.push(`| brain | ${showMeasured(e.brain.tokensIn)} | ${showMeasured(e.brain.tokensOut)} | ${showMeasured(e.brain.cacheRead)} | ${showMeasured(e.brain.cacheWrite)} | ${showMeasured(e.brain.usd)} | ${e.brain.source} |`);
|
|
1553
|
+
out.push(`| workers | ${showMeasured(e.workers.tokensIn)} | ${showMeasured(e.workers.tokensOut)} | ${showMeasured(e.workers.cacheRead)} | ${showMeasured(e.workers.cacheWrite)} | ${showMeasured(e.workers.usd)} | ${e.workers.source} |`);
|
|
1554
|
+
out.push("");
|
|
1555
|
+
if (!isUnavailable(e.brainTruncations) && e.brainTruncations > 0) out.push(`- **BRAIN OUTPUT TRUNCATED: ${e.brainTruncations} completion(s) hit \`finish_reason: "length"\`** — the supervisor acted on a half-written plan. Its output ceiling is too low; see \`brain.jsonl\` for the per-call \`req_max_tokens\`.`);
|
|
1556
|
+
else out.push(`- Brain completions truncated (finish_reason=length): ${showMeasured(e.brainTruncations)}`);
|
|
1557
|
+
out.push(`- Total USD: ${showMeasured(e.totalUsd)} (source: ${e.totalUsdSource})`);
|
|
1558
|
+
out.push(`- Spend measured two ways: journal-derived $${showMeasured(e.spend.journalDerived.usd)} over ${e.spend.journalDerived.records} journal record(s) (execution accounting) · close-record $${showMeasured(e.spend.closeRecord.usd)} over ${e.spend.closeRecord.records} close record(s) (billing-shaped). Divergence is a signal, not an error.`);
|
|
1559
|
+
out.push(`- Cost per accepted patch: ${showMeasured(e.costPerAcceptedPatchUsd)}`);
|
|
1560
|
+
if (isUnavailable(e.workerWallMsDistribution)) out.push(`- Worker wall distribution: unavailable — ${e.workerWallMsDistribution.unavailable}`);
|
|
1561
|
+
else {
|
|
1562
|
+
const w = e.workerWallMsDistribution;
|
|
1563
|
+
out.push(`- Worker wall (n=${w.n}): min ${fmtMs(w.min)} / p50 ${fmtMs(w.p50)} / p90 ${fmtMs(w.p90)} / max ${fmtMs(w.max)} / Σ ${fmtMs(w.sum)}`);
|
|
1567
1564
|
}
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1565
|
+
out.push("");
|
|
1566
|
+
if (!isUnavailable(e.perWorker) && e.perWorker.length > 0) {
|
|
1567
|
+
out.push("| Worker id | Label | Role | Runtime | Profile digest | Status | Failure | Infra | Wall | Tokens in | Tokens out | Patch bytes | Verify passed | Score |");
|
|
1568
|
+
out.push("|---|---|---|---|---|---|---|---|---:|---:|---:|---:|---|---:|");
|
|
1569
|
+
for (const w of e.perWorker) out.push(`| ${w.workerId === null ? "unavailable — worker id absent; label join used" : `\`${w.workerId}\``} | \`${w.worker}\` | ${w.role ?? "unavailable — source recorded no role"} | ${w.runtime ?? "unavailable — source recorded no runtime"} | ${w.profileDigest === null ? "unavailable — source recorded no profile digest" : `\`${w.profileDigest}\``} | ${w.status ?? "unavailable — no terminal event"} | ${w.failure ?? "none recorded"} | ${w.infra ?? "unavailable"} | ${w.wallMs === null ? "unavailable — no spawn/finish pair" : fmtMs(w.wallMs)} | ${w.tokensIn ?? "unavailable — store does not attribute tokens per worker"} | ${w.tokensOut ?? "unavailable — store does not attribute tokens per worker"} | ${w.patchBytes ?? "unavailable — no worker patch file"} | ${w.passed === null ? "unavailable — no verdict" : String(w.passed)} | ${w.score ?? "unavailable — no numeric score"} |`);
|
|
1570
|
+
out.push("");
|
|
1573
1571
|
}
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1572
|
+
if (e.resourceRecords !== void 0) out.push("## Named resource receipts", "", ...resourceRows(e.resourceRecords));
|
|
1573
|
+
out.push("## Outcome");
|
|
1574
|
+
out.push("");
|
|
1575
|
+
out.push("| Metric | Value |");
|
|
1576
|
+
out.push("|---|---|");
|
|
1577
|
+
out.push(`| Supervisor status | ${showMeasured(r.outcome.supStatus)} |`);
|
|
1578
|
+
out.push(`| Status source | ${showMeasured(r.outcome.supStatusSource)} |`);
|
|
1579
|
+
out.push(`| Terminal reason | ${showMeasured(r.outcome.supReason)} |`);
|
|
1580
|
+
out.push(`| Failure | ${fmtFailure(r.outcome.failure)} |`);
|
|
1581
|
+
out.push(`| Supervisor verdict | ${showMeasured(r.outcome.supVerdict)} |`);
|
|
1582
|
+
out.push(`| Delivered | ${showMeasured(r.outcome.delivered)} |`);
|
|
1583
|
+
out.push(`| Judge resolved | ${showMeasured(r.outcome.judgeResolved)} |`);
|
|
1584
|
+
out.push(`| Judge score | ${showMeasured(r.outcome.judgeScore)} |`);
|
|
1585
|
+
out.push(`| Judge passed / total | ${showMeasured(r.outcome.judgePassed)} / ${showMeasured(r.outcome.judgeTotal)} |`);
|
|
1586
|
+
out.push(`| Judge source | ${r.outcome.judgeSource ?? "unavailable — no judge.json and no ledger row"} |`);
|
|
1587
|
+
out.push(`| Verify gate | pass=${showMeasured(r.outcome.verifyPass)} rc=${showMeasured(r.outcome.verifyRc)} |`);
|
|
1588
|
+
if (isUnavailable(r.outcome.patch)) out.push(`| Patch | unavailable — ${r.outcome.patch.unavailable} |`);
|
|
1589
|
+
else {
|
|
1590
|
+
const p = r.outcome.patch;
|
|
1591
|
+
out.push(`| Patch | ${p.files} file(s), +${p.linesAdded}/-${p.linesRemoved}, test files touched: ${p.testFilesTouched.length === 0 ? "none" : p.testFilesTouched.join(", ")} |`);
|
|
1589
1592
|
}
|
|
1593
|
+
out.push("");
|
|
1594
|
+
out.push("## Gaps");
|
|
1595
|
+
out.push("");
|
|
1596
|
+
if (r.gaps.length === 0) out.push("None — every metric above is backed by a present artifact.");
|
|
1597
|
+
else for (const g of r.gaps) out.push(`- ${g}`);
|
|
1598
|
+
out.push("");
|
|
1599
|
+
out.push(`> Harness-session view of the same run (model calls, stuck loops, tool errors): \`${r.traceCommand}\``);
|
|
1600
|
+
out.push("");
|
|
1601
|
+
return out.join("\n");
|
|
1602
|
+
}
|
|
1603
|
+
function fmtCounts(v) {
|
|
1604
|
+
if (isUnavailable(v)) return showMeasured(v);
|
|
1605
|
+
const entries = Object.entries(v);
|
|
1606
|
+
return entries.length === 0 ? "none" : entries.map(([k, n]) => `${k}=${n}`).join(", ");
|
|
1607
|
+
}
|
|
1608
|
+
function renderSupervisorRollupMarkdown(rollup, title = "Round rollup") {
|
|
1609
|
+
const out = [];
|
|
1610
|
+
out.push(`# ${title}`);
|
|
1611
|
+
out.push("");
|
|
1612
|
+
out.push(`- Cells: ${rollup.cells}`);
|
|
1613
|
+
out.push(`- **Steers across all cells: ${showMeasured(rollup.steersTotal)}** (cells with ≥1 steer: ${showMeasured(rollup.cellsWithSteers)}; cells where the steer count is unavailable: ${rollup.cellsWithUnavailableSteers})`);
|
|
1614
|
+
out.push(`- Waves per cell (mean): ${showMeasured(rollup.wavesMean)}`);
|
|
1615
|
+
out.push(`- Max concurrency observed: ${showMeasured(rollup.maxConcurrencyMax)}`);
|
|
1616
|
+
out.push(`- Worker utilization (mean): ${showMeasured(rollup.utilizationMean)}`);
|
|
1617
|
+
out.push(`- Idle share (mean): ${showMeasured(rollup.idlePctMean)}%`);
|
|
1618
|
+
out.push(`- Workers spawned: ${showMeasured(rollup.workersSpawnedTotal)} · accepted: ${showMeasured(rollup.acceptedTotal)}`);
|
|
1619
|
+
out.push(`- Spend: $${showMeasured(rollup.usdTotal)} · judged resolved: ${showMeasured(rollup.resolvedCount)}/${rollup.cells}`);
|
|
1620
|
+
out.push(`- Spend measured two ways: journal-derived $${showMeasured(rollup.spendUsd.journalDerived.value)} over ${rollup.spendUsd.journalDerived.runs}/${rollup.cells} runs (execution accounting) · close-record $${showMeasured(rollup.spendUsd.closeRecord.value)} over ${rollup.spendUsd.closeRecord.runs}/${rollup.cells} runs (billing-shaped)`);
|
|
1621
|
+
out.push("");
|
|
1622
|
+
out.push("| Instance | Arm | Steers | Waves | Utilization | Idle % | Resolved | USD |");
|
|
1623
|
+
out.push("|---|---|---:|---:|---:|---:|---|---:|");
|
|
1624
|
+
for (const c of rollup.perCell) out.push(`| ${c.instanceId ?? "?"} | ${c.arm ?? "?"} | ${showMeasured(c.steers)} | ${showMeasured(c.waves)} | ${showMeasured(c.utilization)} | ${showMeasured(c.idlePct)} | ${showMeasured(c.resolved)} | ${showMeasured(c.usd)} |`);
|
|
1625
|
+
out.push("");
|
|
1626
|
+
for (const cell of rollup.perCell) if (cell.resourceRecords !== void 0) out.push(`## Resource receipts — ${cell.instanceId ?? "?"} [${cell.arm ?? "?"}]`, "", ...resourceRows(cell.resourceRecords));
|
|
1627
|
+
return out.join("\n");
|
|
1590
1628
|
}
|
|
1591
1629
|
//#endregion
|
|
1592
|
-
export {
|
|
1630
|
+
export { findSupervisorRunDirs as a, analyzeSupervisorRunSources as c, analyzeSupervisorRun as i, parsePatch as l, renderSupervisorRunHeadline as n, isRuntimeSupervisorRunDir as o, renderSupervisorRunMarkdown as r, readRuntimeSupervisorRun as s, renderSupervisorRollupMarkdown as t };
|
|
1593
1631
|
|
|
1594
|
-
//# sourceMappingURL=
|
|
1632
|
+
//# sourceMappingURL=render-Cs34y0iX.js.map
|