@tangle-network/agent-eval 0.135.4 → 0.137.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +56 -0
- package/README.md +4 -2
- package/dist/{agent-profile-cell-OhuTee9n.js → agent-profile-cell-CbfBm2g6.js} +2 -2
- package/dist/{agent-profile-cell-OhuTee9n.js.map → agent-profile-cell-CbfBm2g6.js.map} +1 -1
- package/dist/{agent-profile-cell-CCm3l2v2.d.ts → agent-profile-cell-Cw0PVwDr.d.ts} +2 -2
- package/dist/{agent-profile-cell-CCm3l2v2.d.ts.map → agent-profile-cell-Cw0PVwDr.d.ts.map} +1 -1
- package/dist/analyst/index.d.ts +139 -17
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +606 -4
- package/dist/analyst/index.js.map +1 -1
- package/dist/{analyze-runs-jjCmF8pU.js → analyze-runs-BScZvqMV.js} +6 -6
- package/dist/{analyze-runs-jjCmF8pU.js.map → analyze-runs-BScZvqMV.js.map} +1 -1
- package/dist/{analyze-runs-Cda5Xkj1.d.ts → analyze-runs-PVtnfjvA.d.ts} +6 -6
- package/dist/{analyze-runs-Cda5Xkj1.d.ts.map → analyze-runs-PVtnfjvA.d.ts.map} +1 -1
- package/dist/{baseline-BUeFcgrn.js → baseline-C-GocmIW.js} +2 -2
- package/dist/{baseline-BUeFcgrn.js.map → baseline-C-GocmIW.js.map} +1 -1
- package/dist/benchmark-CHX4orG7.d.ts +184 -0
- package/dist/benchmark-CHX4orG7.d.ts.map +1 -0
- package/dist/benchmark-YDrpumqB.js +414 -0
- package/dist/benchmark-YDrpumqB.js.map +1 -0
- package/dist/benchmarks/index.d.ts +1 -1
- package/dist/benchmarks/index.js +1 -1
- package/dist/{benchmarks-Dfgm9ts5.js → benchmarks-DCLkQOmc.js} +3 -3
- package/dist/{benchmarks-Dfgm9ts5.js.map → benchmarks-DCLkQOmc.js.map} +1 -1
- package/dist/builder-eval/index.js +2 -2
- package/dist/campaign/index.d.ts +6 -6
- package/dist/campaign/index.js +3 -3
- package/dist/{campaign-Dz8uQnhC.js → campaign-lgObcHFC.js} +212 -78
- package/dist/campaign-lgObcHFC.js.map +1 -0
- package/dist/cli.js +1 -1
- package/dist/client-C8L6h6Wf.d.ts +202 -0
- package/dist/client-C8L6h6Wf.d.ts.map +1 -0
- package/dist/completion-verifier-DSyRNVzU.d.ts +240 -0
- package/dist/completion-verifier-DSyRNVzU.d.ts.map +1 -0
- package/dist/contract/index.d.ts +10 -9
- package/dist/contract/index.d.ts.map +1 -1
- package/dist/contract/index.js +10 -10
- package/dist/control.d.ts +2 -2
- package/dist/control.js +1 -1
- package/dist/{cost-ledger-DHAjwNj7.js → cost-ledger-D-5_-dhi.js} +2 -2
- package/dist/{cost-ledger-DHAjwNj7.js.map → cost-ledger-D-5_-dhi.js.map} +1 -1
- package/dist/{cost-ledger-fGS_u_O1.d.ts → cost-ledger-D2o6JOrL.d.ts} +2 -2
- package/dist/{cost-ledger-fGS_u_O1.d.ts.map → cost-ledger-D2o6JOrL.d.ts.map} +1 -1
- package/dist/{dataset-BvtnC8Dc.d.ts → dataset-v_Y5902-.d.ts} +2 -2
- package/dist/{dataset-BvtnC8Dc.d.ts.map → dataset-v_Y5902-.d.ts.map} +1 -1
- package/dist/{default-registry-CHmdy2An.js → default-registry-CLXbRt0f.js} +119 -38
- package/dist/default-registry-CLXbRt0f.js.map +1 -0
- package/dist/{default-registry-Brxr728w.d.ts → default-registry-Dc5D_Loc.d.ts} +63 -137
- package/dist/default-registry-Dc5D_Loc.d.ts.map +1 -0
- package/dist/{errors-8YnH8WlF.js → errors-D-LKuDhb.js} +8 -2
- package/dist/errors-D-LKuDhb.js.map +1 -0
- package/dist/{errors-CEk209JS.d.ts → errors-DkfjIDvD.d.ts} +9 -3
- package/dist/errors-DkfjIDvD.d.ts.map +1 -0
- package/dist/{eval-campaign-Cc8WZJ6b.js → eval-campaign-CHqfLnff.js} +6 -6
- package/dist/{eval-campaign-Cc8WZJ6b.js.map → eval-campaign-CHqfLnff.js.map} +1 -1
- package/dist/{extract-usage-DIQpN-ww.js → extract-usage-p-56bh8q.js} +3 -3
- package/dist/{extract-usage-DIQpN-ww.js.map → extract-usage-p-56bh8q.js.map} +1 -1
- package/dist/{feedback-trajectory-CVaeREXV.d.ts → feedback-trajectory-N_F0PwHz.d.ts} +90 -3
- package/dist/feedback-trajectory-N_F0PwHz.d.ts.map +1 -0
- package/dist/fuzz.d.ts +1 -1
- package/dist/fuzz.js +2 -2
- package/dist/hosted/index.d.ts +3 -2
- package/dist/hosted/index.d.ts.map +1 -1
- package/dist/index-BipJlj-C.d.ts +316 -0
- package/dist/index-BipJlj-C.d.ts.map +1 -0
- package/dist/{index-CQsJcqch.d.ts → index-BnP1QJUv.d.ts} +5 -5
- package/dist/{index-CQsJcqch.d.ts.map → index-BnP1QJUv.d.ts.map} +1 -1
- package/dist/{index-B4Fjfo5U.d.ts → index-C-Pr4OWg.d.ts} +88 -317
- package/dist/index-C-Pr4OWg.d.ts.map +1 -0
- package/dist/{index-DuhJaaiH.d.ts → index-DEb46kc6.d.ts} +2 -2
- package/dist/{index-DuhJaaiH.d.ts.map → index-DEb46kc6.d.ts.map} +1 -1
- package/dist/{index-C2fkZhv_.d.ts → index-DRNl6g_N.d.ts} +3 -3
- package/dist/{index-C2fkZhv_.d.ts.map → index-DRNl6g_N.d.ts.map} +1 -1
- package/dist/{index-AbhwHp0V.d.ts → index-U3RHOShi.d.ts} +2 -2
- package/dist/{index-AbhwHp0V.d.ts.map → index-U3RHOShi.d.ts.map} +1 -1
- package/dist/index.d.ts +29 -70
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +507 -33
- package/dist/index.js.map +1 -1
- package/dist/{client-DcvgkaZi.d.ts → insight-report-B9ooYH_g.d.ts} +5 -203
- package/dist/insight-report-B9ooYH_g.d.ts.map +1 -0
- package/dist/integrity-CCXTftiL.js +1360 -0
- package/dist/integrity-CCXTftiL.js.map +1 -0
- package/dist/{integrity-rmVhXWA7.d.ts → integrity-CKxosZ5Z.d.ts} +3 -3
- package/dist/{integrity-rmVhXWA7.d.ts.map → integrity-CKxosZ5Z.d.ts.map} +1 -1
- package/dist/{integrity-BzRbCHzi.js → integrity-fdt8XPAv.js} +2 -2
- package/dist/{integrity-BzRbCHzi.js.map → integrity-fdt8XPAv.js.map} +1 -1
- package/dist/ledger-core/index.d.ts +1 -1
- package/dist/ledger-core/index.js +1 -1
- package/dist/{ledger-core-DAKFKRzi.js → ledger-core-t6sItivm.js} +85 -85
- package/dist/{ledger-core-DAKFKRzi.js.map → ledger-core-t6sItivm.js.map} +1 -1
- package/dist/{llm-client-DHx8pzyJ.js → llm-client-DKB25jV8.js} +3 -3
- package/dist/{llm-client-DHx8pzyJ.js.map → llm-client-DKB25jV8.js.map} +1 -1
- package/dist/meta-eval/index.d.ts +2 -2
- package/dist/meta-eval/index.js +3 -3
- package/dist/{mint-DyRUc9k6.js → mint-Ctwk079K.js} +4 -4
- package/dist/{mint-DyRUc9k6.js.map → mint-Ctwk079K.js.map} +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/{paired-arms-BbFKrAU-.js → paired-arms-iZ08VFMN.js} +3 -3
- package/dist/{paired-arms-BbFKrAU-.js.map → paired-arms-iZ08VFMN.js.map} +1 -1
- package/dist/pipelines/index.js +2 -2
- package/dist/profile-cell.d.ts +1 -1
- package/dist/profile-cell.js +1 -1
- package/dist/{propose-review-control-SQ-n9-We.js → propose-review-control-DLXz4FCX.js} +2 -2
- package/dist/{propose-review-control-SQ-n9-We.js.map → propose-review-control-DLXz4FCX.js.map} +1 -1
- package/dist/registry-BdM7SuTr.d.ts +124 -0
- package/dist/registry-BdM7SuTr.d.ts.map +1 -0
- package/dist/{release-report-DooPguBc.js → release-report-B5XPBvAU.js} +4 -4
- package/dist/{release-report-DooPguBc.js.map → release-report-B5XPBvAU.js.map} +1 -1
- package/dist/{release-report-DpBxGGI1.d.ts → release-report-CofgVNZt.d.ts} +4 -4
- package/dist/{release-report-DpBxGGI1.d.ts.map → release-report-CofgVNZt.d.ts.map} +1 -1
- package/dist/{replay-C6wRg47C.js → replay-Bju0T8Ls.js} +248 -8
- package/dist/replay-Bju0T8Ls.js.map +1 -0
- package/dist/{replay-BRfMIs81.d.ts → replay-K8FaC0CB.d.ts} +227 -52
- package/dist/replay-K8FaC0CB.d.ts.map +1 -0
- package/dist/reporting.d.ts +4 -4
- package/dist/reporting.js +4 -4
- package/dist/{researcher-Doo95b50.d.ts → researcher-Da0Wj-bt.d.ts} +6 -7
- package/dist/researcher-Da0Wj-bt.d.ts.map +1 -0
- package/dist/{reward-hacking-D-QqXvg-.d.ts → reward-hacking-CQ3hTCO3.d.ts} +2 -2
- package/dist/{reward-hacking-D-QqXvg-.d.ts.map → reward-hacking-CQ3hTCO3.d.ts.map} +1 -1
- package/dist/{reward-hacking-a-kYs0-i.js → reward-hacking-GyN0kMd8.js} +3 -3
- package/dist/{reward-hacking-a-kYs0-i.js.map → reward-hacking-GyN0kMd8.js.map} +1 -1
- package/dist/rl.d.ts +6 -6
- package/dist/rl.js +9 -9
- package/dist/rollout/index.d.ts +1 -1
- package/dist/rollout/index.js +3 -3
- package/dist/{rollout-DLSUIWLu.js → rollout-DQFl0UXA.js} +2 -2
- package/dist/{rollout-DLSUIWLu.js.map → rollout-DQFl0UXA.js.map} +1 -1
- package/dist/{rubric-predictive-validity-BJf-8ejY.js → rubric-predictive-validity-BRR632r1.js} +2 -2
- package/dist/{rubric-predictive-validity-BJf-8ejY.js.map → rubric-predictive-validity-BRR632r1.js.map} +1 -1
- package/dist/{rubric-predictive-validity-C1dCLcvb.d.ts → rubric-predictive-validity-C4sztLR3.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-C1dCLcvb.d.ts.map → rubric-predictive-validity-C4sztLR3.d.ts.map} +1 -1
- package/dist/{run-evidence-DokQtX0-.d.ts → run-evidence-BDIircdA.d.ts} +3 -3
- package/dist/{run-evidence-DokQtX0-.d.ts.map → run-evidence-BDIircdA.d.ts.map} +1 -1
- package/dist/{run-record-DcObtIGh.d.ts → run-record-BPCa2rQ8.d.ts} +4 -4
- package/dist/{run-record-DcObtIGh.d.ts.map → run-record-BPCa2rQ8.d.ts.map} +1 -1
- package/dist/{run-record-BIwU2wdV.js → run-record-vRgqWmJw.js} +3 -3
- package/dist/{run-record-BIwU2wdV.js.map → run-record-vRgqWmJw.js.map} +1 -1
- package/dist/{semantic-concept-judge-Btozx3Vc.js → semantic-concept-judge-Bz64IckK.js} +4 -4
- package/dist/{semantic-concept-judge-Btozx3Vc.js.map → semantic-concept-judge-Bz64IckK.js.map} +1 -1
- package/dist/{server-Bz3WQJs6.js → server-KjXZZUDX.js} +3 -3
- package/dist/{server-Bz3WQJs6.js.map → server-KjXZZUDX.js.map} +1 -1
- package/dist/{skill-usage-BDQVPIG1.d.ts → skill-usage-CFDLLlhF.d.ts} +25 -47
- package/dist/skill-usage-CFDLLlhF.d.ts.map +1 -0
- package/dist/{skillopt-optimization-method-CwSYkv35.d.ts → skillopt-optimization-method-BpbnlvAZ.d.ts} +11 -12
- package/dist/skillopt-optimization-method-BpbnlvAZ.d.ts.map +1 -0
- package/dist/{skillopt-optimization-method-0UmPD6aP.js → skillopt-optimization-method-f4o9sUT4.js} +7 -7
- package/dist/{skillopt-optimization-method-0UmPD6aP.js.map → skillopt-optimization-method-f4o9sUT4.js.map} +1 -1
- package/dist/{statistics-CnGCLLqc.js → statistics-ByxzSiOM.js} +2 -2
- package/dist/{statistics-CnGCLLqc.js.map → statistics-ByxzSiOM.js.map} +1 -1
- package/dist/{statistics-CKOqre5S.d.ts → statistics-_7P642CN.d.ts} +2 -2
- package/dist/{statistics-CKOqre5S.d.ts.map → statistics-_7P642CN.d.ts.map} +1 -1
- package/dist/{summary-report-BEk8OFLs.js → summary-report-9A5y7EsK.js} +4 -4
- package/dist/{summary-report-BEk8OFLs.js.map → summary-report-9A5y7EsK.js.map} +1 -1
- package/dist/{summary-report-CPMINBqs.d.ts → summary-report-DHipz9Kx.d.ts} +3 -3
- package/dist/{summary-report-CPMINBqs.d.ts.map → summary-report-DHipz9Kx.d.ts.map} +1 -1
- package/dist/supervisor-run/index.d.ts +3 -2
- package/dist/supervisor-run/index.js +3 -2
- package/dist/{supervisor-run-B7lUGoyZ.js → supervisor-run-B2EWUmQY.js} +221 -519
- package/dist/supervisor-run-B2EWUmQY.js.map +1 -0
- package/dist/{test-graded-scenario-BsqWLmPt.js → test-graded-scenario-JHcKQNpq.js} +2 -2
- package/dist/{test-graded-scenario-BsqWLmPt.js.map → test-graded-scenario-JHcKQNpq.js.map} +1 -1
- package/dist/tools-DZk2Jn64.js +1876 -0
- package/dist/tools-DZk2Jn64.js.map +1 -0
- package/dist/traces.d.ts +6 -7
- package/dist/traces.js +5 -6
- package/dist/{types-DVjczBM9.d.ts → types-CKswbJGO.d.ts} +260 -6
- package/dist/types-CKswbJGO.d.ts.map +1 -0
- package/dist/{types-DiWLru6Z.d.ts → types-CTGbIm57.d.ts} +5 -5
- package/dist/{types-DiWLru6Z.d.ts.map → types-CTGbIm57.d.ts.map} +1 -1
- package/dist/types-CTvKfr5F.d.ts +804 -0
- package/dist/types-CTvKfr5F.d.ts.map +1 -0
- package/dist/{index-C61Wi7yg.d.ts → types-Dea6tiVI.d.ts} +47 -235
- package/dist/types-Dea6tiVI.d.ts.map +1 -0
- package/dist/wire/index.d.ts +3 -3
- package/dist/wire/index.js +1 -1
- package/docs/feedback-trajectories.md +100 -1
- package/docs/trace-analysis.md +374 -58
- package/package.json +7 -3
- package/dist/analyst-BkTS3C58.d.ts +0 -89
- package/dist/analyst-BkTS3C58.d.ts.map +0 -1
- package/dist/analyst-j5je5J7c.js +0 -152
- package/dist/analyst-j5je5J7c.js.map +0 -1
- package/dist/campaign-Dz8uQnhC.js.map +0 -1
- package/dist/client-DcvgkaZi.d.ts.map +0 -1
- package/dist/default-registry-Brxr728w.d.ts.map +0 -1
- package/dist/default-registry-CHmdy2An.js.map +0 -1
- package/dist/errors-8YnH8WlF.js.map +0 -1
- package/dist/errors-CEk209JS.d.ts.map +0 -1
- package/dist/feedback-trajectory-CVaeREXV.d.ts.map +0 -1
- package/dist/index-B4Fjfo5U.d.ts.map +0 -1
- package/dist/index-C61Wi7yg.d.ts.map +0 -1
- package/dist/llm-client-BiK4HW0u.d.ts +0 -290
- package/dist/llm-client-BiK4HW0u.d.ts.map +0 -1
- package/dist/raw-provider-sink-BU29Sh8h.d.ts +0 -134
- package/dist/raw-provider-sink-BU29Sh8h.d.ts.map +0 -1
- package/dist/replay-BRfMIs81.d.ts.map +0 -1
- package/dist/replay-C6wRg47C.js.map +0 -1
- package/dist/researcher-Doo95b50.d.ts.map +0 -1
- package/dist/skill-usage-BDQVPIG1.d.ts.map +0 -1
- package/dist/skillopt-optimization-method-CwSYkv35.d.ts.map +0 -1
- package/dist/store-CxJry_cs.d.ts +0 -229
- package/dist/store-CxJry_cs.d.ts.map +0 -1
- package/dist/supervisor-run-B7lUGoyZ.js.map +0 -1
- package/dist/tools-D8yTtNSN.js +0 -1190
- package/dist/tools-D8yTtNSN.js.map +0 -1
- package/dist/types-Cc3qbqzj.d.ts +0 -387
- package/dist/types-Cc3qbqzj.d.ts.map +0 -1
- package/dist/types-DVjczBM9.d.ts.map +0 -1
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { f as isRolloutLine, i as ROLLOUT_SCHEMA } from "./schema-C6DW4ZHR.js";
|
|
1
|
+
import { a as parseJsonl, i as parseJson, o as parseSupervisorTree, r as asRecord, s as workerSourceKey } from "./integrity-CCXTftiL.js";
|
|
3
2
|
import { d as transcriptFromEntries, i as openOpencodeDb, l as parseClaudeEntries, r as findOpencodeSessionsByDirectory, t as DEFAULT_OPENCODE_DB } from "./opencode-sqlite-8r6WUfHc.js";
|
|
4
3
|
import { basename, dirname, join } from "node:path";
|
|
5
4
|
import { appendFile, mkdir, readFile, readdir, writeFile } from "node:fs/promises";
|
|
@@ -18,6 +17,8 @@ function showMeasured(v) {
|
|
|
18
17
|
}
|
|
19
18
|
/** A source that carries every fact the analyzer can use. */
|
|
20
19
|
const NO_SOURCE_LIMITS = {
|
|
20
|
+
managerTokens: null,
|
|
21
|
+
workerTokens: null,
|
|
21
22
|
spendUsd: null,
|
|
22
23
|
workerVerdicts: null,
|
|
23
24
|
deliverables: null
|
|
@@ -32,179 +33,7 @@ const SUPERVISOR_RUN_ROLLUP_SCHEMA = "tangle.supervisor-run-rollup@1";
|
|
|
32
33
|
* with no filesystem, no process, and no network. All I/O lives in a reader
|
|
33
34
|
* (`loops-reader.ts` is one).
|
|
34
35
|
*/
|
|
35
|
-
function asRecord(v) {
|
|
36
|
-
return typeof v === "object" && v !== null ? v : {};
|
|
37
|
-
}
|
|
38
|
-
function num(v) {
|
|
39
|
-
return typeof v === "number" && Number.isFinite(v) ? v : 0;
|
|
40
|
-
}
|
|
41
|
-
function readSpend(v) {
|
|
42
|
-
const rec = asRecord(v);
|
|
43
|
-
const tok = asRecord(rec.tokens);
|
|
44
|
-
const cacheRead = tok.cacheRead ?? tok.cache_read;
|
|
45
|
-
const cacheWrite = tok.cacheWrite ?? tok.cache_write;
|
|
46
|
-
return {
|
|
47
|
-
tokens: {
|
|
48
|
-
input: num(tok.input),
|
|
49
|
-
output: num(tok.output),
|
|
50
|
-
cacheRead: num(cacheRead),
|
|
51
|
-
cacheWrite: num(cacheWrite),
|
|
52
|
-
hasCache: cacheRead !== void 0 || cacheWrite !== void 0
|
|
53
|
-
},
|
|
54
|
-
usd: num(rec.usd)
|
|
55
|
-
};
|
|
56
|
-
}
|
|
57
36
|
const NO_CACHE_COUNTERS = "the journal carries no cache-token counters for this role";
|
|
58
|
-
function parseJsonl(text) {
|
|
59
|
-
if (text === null) return [];
|
|
60
|
-
const out = [];
|
|
61
|
-
for (const line of text.split("\n")) {
|
|
62
|
-
const trimmed = line.trim();
|
|
63
|
-
if (!trimmed) continue;
|
|
64
|
-
try {
|
|
65
|
-
const parsed = JSON.parse(trimmed);
|
|
66
|
-
if (typeof parsed === "object" && parsed !== null) out.push(parsed);
|
|
67
|
-
} catch {}
|
|
68
|
-
}
|
|
69
|
-
return out;
|
|
70
|
-
}
|
|
71
|
-
function parseJson(text) {
|
|
72
|
-
if (text === null) return null;
|
|
73
|
-
try {
|
|
74
|
-
const parsed = JSON.parse(text);
|
|
75
|
-
return typeof parsed === "object" && parsed !== null ? parsed : null;
|
|
76
|
-
} catch {
|
|
77
|
-
return null;
|
|
78
|
-
}
|
|
79
|
-
}
|
|
80
|
-
function ms(at) {
|
|
81
|
-
if (typeof at !== "string") return null;
|
|
82
|
-
const t = Date.parse(at);
|
|
83
|
-
return Number.isFinite(t) ? t : null;
|
|
84
|
-
}
|
|
85
|
-
function parseSupervisorTree(src) {
|
|
86
|
-
const events = parseJsonl(src.journal);
|
|
87
|
-
const state = parseJson(src.state);
|
|
88
|
-
const spawns = [];
|
|
89
|
-
const closes = [];
|
|
90
|
-
let brainIn = 0;
|
|
91
|
-
let brainOut = 0;
|
|
92
|
-
let brainCacheRead = 0;
|
|
93
|
-
let brainCacheWrite = 0;
|
|
94
|
-
let brainHasCache = false;
|
|
95
|
-
let brainUsd = 0;
|
|
96
|
-
let meteredCount = 0;
|
|
97
|
-
let rootId = null;
|
|
98
|
-
for (const ev of events) {
|
|
99
|
-
const kind = typeof ev.kind === "string" ? ev.kind : "";
|
|
100
|
-
const id = typeof ev.id === "string" ? ev.id : "";
|
|
101
|
-
if (kind === "spawned") {
|
|
102
|
-
const parent = typeof ev.parent === "string" ? ev.parent : null;
|
|
103
|
-
const label = typeof ev.label === "string" ? ev.label : "";
|
|
104
|
-
if (parent === null && rootId === null) rootId = id;
|
|
105
|
-
spawns.push({
|
|
106
|
-
id,
|
|
107
|
-
parent,
|
|
108
|
-
label,
|
|
109
|
-
at: ms(ev.at)
|
|
110
|
-
});
|
|
111
|
-
} else if (kind === "settled") closes.push({
|
|
112
|
-
id,
|
|
113
|
-
kind: "settled",
|
|
114
|
-
status: typeof ev.status === "string" ? ev.status : null,
|
|
115
|
-
verdict: typeof ev.verdict === "string" ? ev.verdict : null,
|
|
116
|
-
at: ms(ev.at),
|
|
117
|
-
spend: readSpend(ev.spent),
|
|
118
|
-
hasSpend: asRecord(ev.spent).tokens !== void 0
|
|
119
|
-
});
|
|
120
|
-
else if (kind === "cancelled") closes.push({
|
|
121
|
-
id,
|
|
122
|
-
kind: "cancelled",
|
|
123
|
-
status: "cancelled",
|
|
124
|
-
verdict: typeof ev.reason === "string" ? ev.reason : null,
|
|
125
|
-
at: ms(ev.at),
|
|
126
|
-
spend: {
|
|
127
|
-
tokens: {
|
|
128
|
-
input: 0,
|
|
129
|
-
output: 0,
|
|
130
|
-
cacheRead: 0,
|
|
131
|
-
cacheWrite: 0,
|
|
132
|
-
hasCache: false
|
|
133
|
-
},
|
|
134
|
-
usd: 0
|
|
135
|
-
},
|
|
136
|
-
hasSpend: false
|
|
137
|
-
});
|
|
138
|
-
else if (kind === "metered") {
|
|
139
|
-
const s = readSpend(ev.spend);
|
|
140
|
-
brainIn += s.tokens.input;
|
|
141
|
-
brainOut += s.tokens.output;
|
|
142
|
-
brainCacheRead += s.tokens.cacheRead;
|
|
143
|
-
brainCacheWrite += s.tokens.cacheWrite;
|
|
144
|
-
brainHasCache = brainHasCache || s.tokens.hasCache;
|
|
145
|
-
brainUsd += s.usd;
|
|
146
|
-
meteredCount += 1;
|
|
147
|
-
}
|
|
148
|
-
}
|
|
149
|
-
const workerSpawns = spawns.filter((s) => s.id !== rootId);
|
|
150
|
-
const workerIds = new Set(workerSpawns.map((s) => s.id));
|
|
151
|
-
const workerCloses = closes.filter((c) => workerIds.has(c.id));
|
|
152
|
-
const workerLogs = /* @__PURE__ */ new Map();
|
|
153
|
-
for (const w of src.workers ?? []) {
|
|
154
|
-
const facts = {
|
|
155
|
-
started: null,
|
|
156
|
-
finished: false,
|
|
157
|
-
finishedAt: null,
|
|
158
|
-
passed: null,
|
|
159
|
-
finishedPatchBytes: null,
|
|
160
|
-
evidenceBytes: 0,
|
|
161
|
-
steersQueued: 0,
|
|
162
|
-
steersDelivered: 0,
|
|
163
|
-
questions: 0
|
|
164
|
-
};
|
|
165
|
-
for (const req of parseJsonl(w.inbox)) if (typeof req.message === "string" && req.message.trim().length > 0) facts.steersQueued += 1;
|
|
166
|
-
for (const ev of parseJsonl(w.events)) {
|
|
167
|
-
const kind = ev.kind;
|
|
168
|
-
if (kind === "message") {
|
|
169
|
-
if (ev.direction === "up") {
|
|
170
|
-
facts.questions += 1;
|
|
171
|
-
continue;
|
|
172
|
-
}
|
|
173
|
-
if (typeof ev.requestId !== "string") facts.steersQueued += 1;
|
|
174
|
-
if (ev.delivered === true) facts.steersDelivered += 1;
|
|
175
|
-
} else if (kind === "started") facts.started = ms(ev.at);
|
|
176
|
-
else if (kind === "finished") {
|
|
177
|
-
facts.finished = true;
|
|
178
|
-
facts.finishedAt = ms(ev.at);
|
|
179
|
-
facts.passed = typeof ev.passed === "boolean" ? ev.passed : null;
|
|
180
|
-
facts.finishedPatchBytes = typeof ev.patchBytes === "number" ? ev.patchBytes : null;
|
|
181
|
-
facts.evidenceBytes = typeof ev.evidence === "string" ? ev.evidence.length : 0;
|
|
182
|
-
}
|
|
183
|
-
}
|
|
184
|
-
workerLogs.set(w.label, facts);
|
|
185
|
-
}
|
|
186
|
-
const startedAt = ms(state?.startedAt) ?? spawns[0]?.at ?? null;
|
|
187
|
-
const completedAt = ms(state?.completedAt) ?? [...spawns.map((s) => s.at), ...closes.map((c) => c.at)].reduce((acc, t) => t === null ? acc : acc === null ? t : Math.max(acc, t), null);
|
|
188
|
-
return {
|
|
189
|
-
rootId,
|
|
190
|
-
spawns,
|
|
191
|
-
closes,
|
|
192
|
-
workerSpawns,
|
|
193
|
-
workerCloses,
|
|
194
|
-
brain: {
|
|
195
|
-
tokensIn: brainIn,
|
|
196
|
-
tokensOut: brainOut,
|
|
197
|
-
cacheRead: brainCacheRead,
|
|
198
|
-
cacheWrite: brainCacheWrite,
|
|
199
|
-
hasCache: brainHasCache,
|
|
200
|
-
usd: brainUsd,
|
|
201
|
-
meteredCount
|
|
202
|
-
},
|
|
203
|
-
workerLogs,
|
|
204
|
-
startedAt,
|
|
205
|
-
completedAt
|
|
206
|
-
};
|
|
207
|
-
}
|
|
208
37
|
/**
|
|
209
38
|
* Analyze already-read supervisor-run bytes. Pure and synchronous: same bytes
|
|
210
39
|
* in, same report out (modulo `generatedAt`, which `now` pins in tests).
|
|
@@ -217,33 +46,57 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
|
|
|
217
46
|
};
|
|
218
47
|
const journalMissing = src.supRunDir === null ? "no supervisor run dir under <ws>/.loops/supervisor" : "journal.jsonl absent";
|
|
219
48
|
const haveJournal = src.journal !== null;
|
|
220
|
-
const
|
|
49
|
+
const tree = parseSupervisorTree(src);
|
|
50
|
+
const state = tree.state;
|
|
221
51
|
const result = parseJson(src.result);
|
|
222
52
|
const judge = parseJson(src.judge);
|
|
223
|
-
const tree = parseSupervisorTree(src);
|
|
224
53
|
const { rootId, workerSpawns, workerCloses, startedAt, completedAt } = tree;
|
|
54
|
+
const spawnById = new Map(workerSpawns.map((spawn) => [spawn.id, spawn]));
|
|
55
|
+
const spawnsByLabel = /* @__PURE__ */ new Map();
|
|
56
|
+
for (const spawn of workerSpawns) {
|
|
57
|
+
const matches = spawnsByLabel.get(spawn.label) ?? [];
|
|
58
|
+
matches.push(spawn);
|
|
59
|
+
spawnsByLabel.set(spawn.label, matches);
|
|
60
|
+
}
|
|
61
|
+
const spawnsForSource = (worker) => {
|
|
62
|
+
if (worker.workerId === void 0) return spawnsByLabel.get(worker.label) ?? [];
|
|
63
|
+
const spawn = spawnById.get(worker.workerId);
|
|
64
|
+
return spawn === void 0 ? [] : [spawn];
|
|
65
|
+
};
|
|
66
|
+
const spawnForSource = (worker) => {
|
|
67
|
+
const matches = spawnsForSource(worker);
|
|
68
|
+
return matches.length === 1 ? matches[0] ?? null : null;
|
|
69
|
+
};
|
|
225
70
|
const supervisorWallMs = startedAt !== null && completedAt !== null && completedAt >= startedAt ? completedAt - startedAt : !haveJournal ? gap("supervisorWallMs", journalMissing) : gap("supervisorWallMs", "no parseable start/complete timestamps in state.json or journal");
|
|
226
71
|
const steerRows = [];
|
|
227
72
|
let steerQueuedTotal = 0;
|
|
228
73
|
let steerDeliveredTotal = 0;
|
|
229
74
|
let upLegMessages = 0;
|
|
230
75
|
if (src.workers !== null) for (const w of src.workers) {
|
|
231
|
-
const facts = tree.workerLogs.get(w
|
|
232
|
-
const queued = facts?.steersQueued ??
|
|
233
|
-
const delivered = facts?.steersDelivered ??
|
|
76
|
+
const facts = tree.workerLogs.get(workerSourceKey(w));
|
|
77
|
+
const queued = facts?.steersQueued ?? null;
|
|
78
|
+
const delivered = facts?.steersDelivered ?? null;
|
|
234
79
|
upLegMessages += facts?.questions ?? 0;
|
|
235
|
-
steerRows.push({
|
|
80
|
+
if (queued !== null && delivered !== null) steerRows.push({
|
|
81
|
+
workerId: w.workerId ?? null,
|
|
236
82
|
worker: w.label,
|
|
237
83
|
queued,
|
|
238
84
|
delivered
|
|
239
85
|
});
|
|
240
|
-
steerQueuedTotal += queued;
|
|
241
|
-
steerDeliveredTotal += delivered;
|
|
86
|
+
if (queued !== null) steerQueuedTotal += queued;
|
|
87
|
+
if (delivered !== null) steerDeliveredTotal += delivered;
|
|
242
88
|
}
|
|
243
89
|
const workersGapReason = src.workersMissingReason ?? "workers/ directory absent";
|
|
244
|
-
const
|
|
245
|
-
|
|
246
|
-
|
|
90
|
+
const unavailableReasons = (pick) => {
|
|
91
|
+
const reasons = tree.workerLogRows.map((facts) => pick(facts)).filter((reason) => reason !== null);
|
|
92
|
+
return reasons.length === 0 ? null : `exact steer accounting unavailable for ${reasons.length} worker row(s): ${[...new Set(reasons)].join(" | ")}`;
|
|
93
|
+
};
|
|
94
|
+
const queuedGapReason = unavailableReasons((facts) => facts.steersQueuedUnavailable);
|
|
95
|
+
const deliveredGapReason = unavailableReasons((facts) => facts.steersDeliveredUnavailable);
|
|
96
|
+
const workerEventsGapReason = unavailableReasons((facts) => !facts.eventsCaptured ? "events absent" : facts.eventsInvalidRows > 0 ? "events contain malformed rows" : null);
|
|
97
|
+
const steers = src.workers === null ? gap("steers", workersGapReason) : queuedGapReason === null ? steerQueuedTotal : gap("steers", queuedGapReason);
|
|
98
|
+
const steersDelivered = src.workers === null ? unavailable(workersGapReason) : deliveredGapReason === null ? steerDeliveredTotal : unavailable(deliveredGapReason);
|
|
99
|
+
const steersByWorker = src.workers === null ? unavailable(workersGapReason) : queuedGapReason === null && deliveredGapReason === null ? steerRows : unavailable(queuedGapReason ?? deliveredGapReason ?? workersGapReason);
|
|
247
100
|
const driverSteerCalls = src.driverLog === null ? gap("driverSteerCalls", "driver.log absent") : Math.max(0, (src.driverLog.match(/supervisor_steer/g) ?? []).length - registrationMentions(src.driverLog));
|
|
248
101
|
const timeline = [];
|
|
249
102
|
for (const s of workerSpawns) if (s.at !== null) timeline.push({
|
|
@@ -287,11 +140,39 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
|
|
|
287
140
|
sumWorkerWallMs += span * live;
|
|
288
141
|
}
|
|
289
142
|
const firstWorkerSpawnAt = workerSpawns.reduce((acc, s) => s.at === null ? acc : acc === null ? s.at : Math.min(acc, s.at), null);
|
|
290
|
-
const
|
|
291
|
-
const
|
|
292
|
-
const
|
|
293
|
-
|
|
294
|
-
|
|
143
|
+
const closeById = new Map(workerCloses.map((close) => [close.id, close]));
|
|
144
|
+
const childSpawnsByParent = /* @__PURE__ */ new Map();
|
|
145
|
+
for (const spawn of workerSpawns) {
|
|
146
|
+
if (spawn.parent === null) continue;
|
|
147
|
+
const siblings = childSpawnsByParent.get(spawn.parent) ?? [];
|
|
148
|
+
siblings.push(spawn);
|
|
149
|
+
childSpawnsByParent.set(spawn.parent, siblings);
|
|
150
|
+
}
|
|
151
|
+
let respawns = 0;
|
|
152
|
+
let observeThenRespawn = 0;
|
|
153
|
+
let respawnWithoutEvidence = 0;
|
|
154
|
+
const repeatedLabelSet = /* @__PURE__ */ new Set();
|
|
155
|
+
for (const siblings of childSpawnsByParent.values()) {
|
|
156
|
+
const labelCounts = /* @__PURE__ */ new Map();
|
|
157
|
+
for (const spawn of siblings) labelCounts.set(spawn.label, (labelCounts.get(spawn.label) ?? 0) + 1);
|
|
158
|
+
for (const [label, count] of labelCounts) if (count > 1) repeatedLabelSet.add(label);
|
|
159
|
+
const orderedSpawns = siblings.map((spawn, index) => ({
|
|
160
|
+
spawn,
|
|
161
|
+
index
|
|
162
|
+
})).filter((row) => row.spawn.at !== null).sort((a, b) => a.spawn.at - b.spawn.at || a.index - b.index);
|
|
163
|
+
const directCloseTimes = siblings.map((spawn) => closeById.get(spawn.id)?.at ?? null).filter((at) => at !== null).sort((a, b) => a - b);
|
|
164
|
+
const firstDirectClose = directCloseTimes[0] ?? null;
|
|
165
|
+
for (let i = 1; i < orderedSpawns.length; i += 1) {
|
|
166
|
+
const previous = orderedSpawns[i - 1]?.spawn.at;
|
|
167
|
+
const current = orderedSpawns[i]?.spawn.at;
|
|
168
|
+
if (previous === void 0 || current === void 0) continue;
|
|
169
|
+
if (firstDirectClose === null || current <= firstDirectClose) continue;
|
|
170
|
+
respawns += 1;
|
|
171
|
+
if (hasNumberBetween(directCloseTimes, previous, current)) observeThenRespawn += 1;
|
|
172
|
+
else respawnWithoutEvidence += 1;
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
const repeatedLabels = [...repeatedLabelSet];
|
|
295
176
|
const parentOf = new Map(tree.spawns.map((s) => [s.id, s.parent]));
|
|
296
177
|
let delegationDepth = 0;
|
|
297
178
|
for (const s of workerSpawns) {
|
|
@@ -333,89 +214,98 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
|
|
|
333
214
|
if (c.verdict !== null) settledVerdicts[c.verdict] = (settledVerdicts[c.verdict] ?? 0) + 1;
|
|
334
215
|
}
|
|
335
216
|
const verdictLimit = src.limits.workerVerdicts;
|
|
217
|
+
const acceptedLimit = verdictLimit ?? src.limits.deliverables;
|
|
336
218
|
let accepted = 0;
|
|
337
|
-
let rejected = 0;
|
|
338
219
|
let emptyPass = 0;
|
|
339
220
|
let evidenceBytes = 0;
|
|
221
|
+
const sourceVerdicts = [];
|
|
340
222
|
for (const w of src.workers ?? []) {
|
|
341
|
-
const f = tree.workerLogs.get(w
|
|
342
|
-
if (f
|
|
343
|
-
|
|
344
|
-
|
|
223
|
+
const f = tree.workerLogs.get(workerSourceKey(w));
|
|
224
|
+
if (f?.finished) evidenceBytes += f.evidenceBytes;
|
|
225
|
+
const spawn = spawnForSource(w);
|
|
226
|
+
const passed = (spawn === null ? null : closeById.get(spawn.id) ?? null)?.valid ?? f?.passed ?? null;
|
|
227
|
+
if (passed !== null) sourceVerdicts.push(passed);
|
|
228
|
+
if (passed === true) if ((w.patchBytes ?? f?.finishedPatchBytes ?? 0) > 0) accepted += 1;
|
|
345
229
|
else emptyPass += 1;
|
|
346
|
-
else if (f.passed === false) rejected += 1;
|
|
347
|
-
}
|
|
348
|
-
let observeThenRespawn = 0;
|
|
349
|
-
let respawnWithoutEvidence = 0;
|
|
350
|
-
const spawnTimes = workerSpawns.map((s) => s.at).filter((t) => t !== null).sort((a, b) => a - b);
|
|
351
|
-
const closeTimes = workerCloses.map((c) => c.at).filter((t) => t !== null).sort((a, b) => a - b);
|
|
352
|
-
for (let i = 1; i < spawnTimes.length; i += 1) {
|
|
353
|
-
const prevSpawn = spawnTimes[i - 1];
|
|
354
|
-
const thisSpawn = spawnTimes[i];
|
|
355
|
-
const sawEvidence = closeTimes.some((t) => t >= prevSpawn && t <= thisSpawn);
|
|
356
|
-
if (firstSettleAt !== null && thisSpawn > firstSettleAt) if (sawEvidence) observeThenRespawn += 1;
|
|
357
|
-
else respawnWithoutEvidence += 1;
|
|
358
230
|
}
|
|
231
|
+
const settledCloses = workerCloses.filter((close) => close.kind === "settled");
|
|
232
|
+
const structuredVerdicts = settledCloses.map((close) => close.valid).filter((valid) => valid !== null);
|
|
233
|
+
const journalVerdictsComplete = settledCloses.length > 0 && structuredVerdicts.length === settledCloses.length;
|
|
234
|
+
const sourceVerdictsComplete = sourceVerdicts.length > 0 && sourceVerdicts.length >= settledCloses.length;
|
|
235
|
+
const rejected = journalVerdictsComplete ? structuredVerdicts.filter((valid) => !valid).length : sourceVerdictsComplete ? sourceVerdicts.filter((valid) => !valid).length : 0;
|
|
236
|
+
const rejectedLimit = journalVerdictsComplete || sourceVerdictsComplete ? null : settledCloses.length > 0 ? verdictLimit ?? "a settled journal verdict has no validity and no matched worker log" : verdictLimit !== null ? verdictLimit : !haveJournal && src.workers === null ? workersGapReason : null;
|
|
359
237
|
const decision = {
|
|
360
238
|
settledByStatus: haveJournal ? settledByStatus : gap("settledByStatus", journalMissing),
|
|
361
239
|
settledVerdicts: verdictLimit !== null ? unavailable(verdictLimit) : haveJournal ? settledVerdicts : unavailable(journalMissing),
|
|
362
|
-
accepted:
|
|
363
|
-
rejected:
|
|
364
|
-
emptyPass:
|
|
240
|
+
accepted: acceptedLimit !== null ? gap("accepted", acceptedLimit) : src.workers === null ? unavailable(workersGapReason) : accepted,
|
|
241
|
+
rejected: rejectedLimit === null ? rejected : unavailable(rejectedLimit),
|
|
242
|
+
emptyPass: acceptedLimit !== null ? gap("emptyPass", acceptedLimit) : src.workers === null ? unavailable(workersGapReason) : emptyPass,
|
|
365
243
|
observeThenRespawn: haveJournal ? observeThenRespawn : unavailable(journalMissing),
|
|
366
244
|
respawnWithoutEvidence: haveJournal ? respawnWithoutEvidence : unavailable(journalMissing),
|
|
367
|
-
reviewActions: src.workers === null ? unavailable(workersGapReason) : steerQueuedTotal + upLegMessages,
|
|
368
|
-
workerEvidenceBytes: src.workers === null ? unavailable(workersGapReason) : evidenceBytes
|
|
245
|
+
reviewActions: src.workers === null ? unavailable(workersGapReason) : queuedGapReason === null ? steerQueuedTotal + upLegMessages : unavailable(queuedGapReason),
|
|
246
|
+
workerEvidenceBytes: src.workers === null ? unavailable(workersGapReason) : workerEventsGapReason === null ? evidenceBytes : unavailable(workerEventsGapReason)
|
|
369
247
|
};
|
|
370
248
|
const journalWorkerIn = workerCloses.reduce((a, c) => a + c.spend.tokens.input, 0);
|
|
371
249
|
const journalWorkerOut = workerCloses.reduce((a, c) => a + c.spend.tokens.output, 0);
|
|
372
250
|
const journalWorkerUsd = workerCloses.reduce((a, c) => a + c.spend.usd, 0);
|
|
373
|
-
const
|
|
251
|
+
const workerUsdById = /* @__PURE__ */ new Map();
|
|
252
|
+
for (const c of workerCloses) workerUsdById.set(c.id, (workerUsdById.get(c.id) ?? 0) + c.spend.usd);
|
|
253
|
+
const labelById = new Map(workerSpawns.map((spawn) => [spawn.id, spawn.label]));
|
|
374
254
|
const workerUsdByLabel = /* @__PURE__ */ new Map();
|
|
375
|
-
for (const
|
|
376
|
-
const label = labelById.get(
|
|
255
|
+
for (const close of workerCloses) {
|
|
256
|
+
const label = labelById.get(close.id);
|
|
377
257
|
if (label === void 0) continue;
|
|
378
|
-
workerUsdByLabel.set(label, (workerUsdByLabel.get(label) ?? 0) +
|
|
258
|
+
workerUsdByLabel.set(label, (workerUsdByLabel.get(label) ?? 0) + close.spend.usd);
|
|
379
259
|
}
|
|
380
260
|
const sq = src.harnessWorkerTokens;
|
|
381
261
|
const harnessGapReason = src.harnessMissingReason ?? "harness session store unavailable and journal settled spend is 0";
|
|
382
|
-
const
|
|
383
|
-
const
|
|
262
|
+
const workerTokenLimit = src.limits.workerTokens;
|
|
263
|
+
const workerIn = workerTokenLimit !== null ? gap("workers.tokensIn", workerTokenLimit) : sq !== null ? journalWorkerIn + sq.input : haveJournal ? journalWorkerIn : gap("workers.tokensIn", harnessGapReason);
|
|
264
|
+
const workerOut = workerTokenLimit !== null ? unavailable(workerTokenLimit) : sq !== null ? journalWorkerOut + sq.output : haveJournal ? journalWorkerOut : unavailable(harnessGapReason);
|
|
384
265
|
const stateResult = asRecord(state?.result);
|
|
385
266
|
const stateUsd = typeof stateResult.spentUsd === "number" ? stateResult.spentUsd : null;
|
|
386
267
|
const usdLimit = src.limits.spendUsd;
|
|
387
268
|
const totalUsd = usdLimit !== null ? gap("totalUsd", usdLimit) : stateUsd !== null ? round(stateUsd, 6) : haveJournal ? round(tree.brain.usd + journalWorkerUsd, 6) : gap("totalUsd", journalMissing);
|
|
388
269
|
const perWorker = (src.workers ?? []).map((w) => {
|
|
389
|
-
const f = tree.workerLogs.get(w
|
|
270
|
+
const f = tree.workerLogs.get(workerSourceKey(w));
|
|
271
|
+
const matchingSpawns = spawnsForSource(w);
|
|
272
|
+
const spawn = spawnForSource(w);
|
|
273
|
+
const close = spawn === null ? null : closeById.get(spawn.id) ?? null;
|
|
274
|
+
const passed = close?.valid ?? f?.passed ?? null;
|
|
275
|
+
const matchingRoles = new Set(matchingSpawns.map((candidate) => candidate.role));
|
|
390
276
|
return {
|
|
277
|
+
workerId: w.workerId ?? null,
|
|
391
278
|
worker: w.label,
|
|
279
|
+
role: matchingRoles.size === 1 ? matchingSpawns[0]?.role ?? null : null,
|
|
392
280
|
wallMs: f?.started != null && f.finishedAt != null ? f.finishedAt - f.started : null,
|
|
393
281
|
tokensIn: w.tokensIn ?? null,
|
|
394
282
|
tokensOut: w.tokensOut ?? null,
|
|
395
|
-
usd: usdLimit !== null ? null : workerUsdByLabel.get(w.label) ?? null,
|
|
283
|
+
usd: usdLimit !== null ? null : w.workerId === void 0 ? workerUsdByLabel.get(w.label) ?? null : workerUsdById.get(w.workerId) ?? null,
|
|
396
284
|
patchBytes: w.patchBytes ?? f?.finishedPatchBytes ?? null,
|
|
397
|
-
passed
|
|
285
|
+
passed,
|
|
286
|
+
score: close?.score ?? f?.score ?? null
|
|
398
287
|
};
|
|
399
288
|
});
|
|
400
289
|
const walls = perWorker.map((w) => w.wallMs).filter((w) => w !== null).sort((a, b) => a - b);
|
|
401
290
|
const brainCalls = parseJsonl(src.brainLog);
|
|
291
|
+
const managerTokenLimit = src.limits.managerTokens;
|
|
402
292
|
const economics = {
|
|
403
293
|
brain: {
|
|
404
|
-
tokensIn: haveJournal ? tree.brain.tokensIn : gap("brain.tokensIn", journalMissing),
|
|
405
|
-
tokensOut: haveJournal ? tree.brain.tokensOut : unavailable(journalMissing),
|
|
294
|
+
tokensIn: managerTokenLimit !== null ? gap("brain.tokensIn", managerTokenLimit) : haveJournal ? tree.brain.tokensIn : gap("brain.tokensIn", journalMissing),
|
|
295
|
+
tokensOut: managerTokenLimit !== null ? unavailable(managerTokenLimit) : haveJournal ? tree.brain.tokensOut : unavailable(journalMissing),
|
|
406
296
|
usd: usdLimit !== null ? unavailable(usdLimit) : haveJournal ? round(tree.brain.usd, 6) : unavailable(journalMissing),
|
|
407
|
-
cacheRead: !haveJournal ? unavailable(journalMissing) : tree.brain.hasCache ? tree.brain.cacheRead : unavailable(NO_CACHE_COUNTERS),
|
|
408
|
-
cacheWrite: !haveJournal ? unavailable(journalMissing) : tree.brain.hasCache ? tree.brain.cacheWrite : unavailable(NO_CACHE_COUNTERS),
|
|
409
|
-
source: haveJournal ? `journal metered events (n=${tree.brain.meteredCount})` : journalMissing
|
|
297
|
+
cacheRead: managerTokenLimit !== null ? unavailable(managerTokenLimit) : !haveJournal ? unavailable(journalMissing) : tree.brain.hasCache ? tree.brain.cacheRead : unavailable(NO_CACHE_COUNTERS),
|
|
298
|
+
cacheWrite: managerTokenLimit !== null ? unavailable(managerTokenLimit) : !haveJournal ? unavailable(journalMissing) : tree.brain.hasCache ? tree.brain.cacheWrite : unavailable(NO_CACHE_COUNTERS),
|
|
299
|
+
source: managerTokenLimit ?? (haveJournal ? `journal metered events (n=${tree.brain.meteredCount})` : journalMissing)
|
|
410
300
|
},
|
|
411
301
|
brainTruncations: src.brainLog === null ? gap("brain.brainTruncations", src.supRunDir === null ? "no supervisor run dir under <ws>/.loops/supervisor" : "brain.jsonl absent — loops predates the brain-call tap, so truncation cannot be ruled out") : brainCalls.filter((c) => c.finish_reason === "length").length,
|
|
412
302
|
workers: {
|
|
413
303
|
tokensIn: workerIn,
|
|
414
304
|
tokensOut: workerOut,
|
|
415
|
-
cacheRead: sq?.cacheRead !== void 0 ? sq.cacheRead : unavailable(NO_CACHE_COUNTERS),
|
|
416
|
-
cacheWrite: sq?.cacheWrite !== void 0 ? sq.cacheWrite : unavailable(NO_CACHE_COUNTERS),
|
|
305
|
+
cacheRead: workerTokenLimit !== null ? unavailable(workerTokenLimit) : sq?.cacheRead !== void 0 ? sq.cacheRead : unavailable(NO_CACHE_COUNTERS),
|
|
306
|
+
cacheWrite: workerTokenLimit !== null ? unavailable(workerTokenLimit) : sq?.cacheWrite !== void 0 ? sq.cacheWrite : unavailable(NO_CACHE_COUNTERS),
|
|
417
307
|
usd: usdLimit !== null ? unavailable(usdLimit) : haveJournal ? round(journalWorkerUsd, 6) : unavailable(journalMissing),
|
|
418
|
-
source: sq !== null ? `journal settled spend + ${sq.store} sessions (n=${sq.sessions})` : `journal settled spend only — ${src.harnessMissingReason ?? "harness session store unavailable"}`
|
|
308
|
+
source: workerTokenLimit !== null ? workerTokenLimit : sq !== null ? `journal settled spend + ${sq.store} sessions (n=${sq.sessions})` : `journal settled spend only — ${src.harnessMissingReason ?? "harness session store unavailable"}`
|
|
419
309
|
},
|
|
420
310
|
totalUsd,
|
|
421
311
|
totalUsdSource: usdLimit !== null ? usdLimit : stateUsd !== null ? `state.json result.spentUsd${journalWorkerUsd === 0 ? " — brain-priced only; worker CLI inference is unpriced (see worker token counts)" : ""}` : haveJournal ? "journal metered + settled usd" : journalMissing,
|
|
@@ -459,6 +349,17 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
|
|
|
459
349
|
traceCommand: src.traceCommand ?? "npx --yes @tangle-network/traces@latest analyze --harness opencode --cwd <worker-clone-cwd>"
|
|
460
350
|
};
|
|
461
351
|
}
|
|
352
|
+
/** Whether sorted values contain one value in the inclusive interval. */
|
|
353
|
+
function hasNumberBetween(sorted, low, high) {
|
|
354
|
+
let left = 0;
|
|
355
|
+
let right = sorted.length;
|
|
356
|
+
while (left < right) {
|
|
357
|
+
const middle = left + Math.floor((right - left) / 2);
|
|
358
|
+
if (sorted[middle] < low) left = middle + 1;
|
|
359
|
+
else right = middle;
|
|
360
|
+
}
|
|
361
|
+
return left < sorted.length && sorted[left] <= high;
|
|
362
|
+
}
|
|
462
363
|
/** `[driver] registered tools: …supervisor_steer…` is a banner, not an invocation. */
|
|
463
364
|
function registrationMentions(driverLog) {
|
|
464
365
|
let n = 0;
|
|
@@ -604,7 +505,7 @@ function rollupSupervisorRuns(reports) {
|
|
|
604
505
|
* | observeThenRespawn / respawnWithoutEvidence | full | full | ordering of spawn vs settle instants |
|
|
605
506
|
* | workerEvidenceBytes | full | PARTIAL | the child's closing message; 0 for pruned transcripts |
|
|
606
507
|
* | brain tokens in/out + cache | full | full | main-thread `message.usage` |
|
|
607
|
-
* | worker tokens in/out + cache | via harness join |
|
|
508
|
+
* | worker tokens in/out + cache | via harness join | full or unavailable | totals are refused if any spawned transcript was pruned; retained per-worker rows remain available |
|
|
608
509
|
* | perWorker wall | full | full | spawn → settle instants |
|
|
609
510
|
* | accepted / rejected / emptyPass / settledVerdicts | full | NONE | no per-worker verify step exists |
|
|
610
511
|
* | brain/worker/total usd, costPerAcceptedPatch | full | NONE | transcripts carry no price |
|
|
@@ -774,6 +675,8 @@ async function readClaudeCodeSupervisorRun(opts) {
|
|
|
774
675
|
const sessionId = basename(opts.transcriptPath).replace(/\.jsonl$/, "");
|
|
775
676
|
const runRef = opts.runRef ?? opts.transcriptPath;
|
|
776
677
|
const limits = {
|
|
678
|
+
managerTokens: null,
|
|
679
|
+
workerTokens: null,
|
|
777
680
|
spendUsd: SPEND_UNPRICED,
|
|
778
681
|
workerVerdicts: NO_VERDICTS,
|
|
779
682
|
deliverables: null
|
|
@@ -799,6 +702,8 @@ async function readClaudeCodeSupervisorRun(opts) {
|
|
|
799
702
|
harnessMissingReason: `session transcript unreadable at ${opts.transcriptPath}`,
|
|
800
703
|
limits: {
|
|
801
704
|
...limits,
|
|
705
|
+
managerTokens: `session transcript unreadable at ${opts.transcriptPath}`,
|
|
706
|
+
workerTokens: `session transcript unreadable at ${opts.transcriptPath}`,
|
|
802
707
|
deliverables: NO_DELIVERABLES
|
|
803
708
|
},
|
|
804
709
|
traceCommand
|
|
@@ -846,6 +751,7 @@ async function readClaudeCodeSupervisorRun(opts) {
|
|
|
846
751
|
const delivered = structured?.success === true || str(structured?.resumedAgentId) === target;
|
|
847
752
|
const rows = steersByTarget.get(target) ?? [];
|
|
848
753
|
rows.push({
|
|
754
|
+
requestId: use.id,
|
|
849
755
|
at: use.at,
|
|
850
756
|
delivered
|
|
851
757
|
});
|
|
@@ -871,6 +777,7 @@ async function readClaudeCodeSupervisorRun(opts) {
|
|
|
871
777
|
id: sessionId,
|
|
872
778
|
parent: null,
|
|
873
779
|
label: `session:${sessionId}`,
|
|
780
|
+
role: "supervisor",
|
|
874
781
|
at: startedAt
|
|
875
782
|
})];
|
|
876
783
|
for (const s of spawns) journalLines.push(line({
|
|
@@ -878,6 +785,7 @@ async function readClaudeCodeSupervisorRun(opts) {
|
|
|
878
785
|
id: s.agentId,
|
|
879
786
|
parent: s.parentId,
|
|
880
787
|
label: s.label,
|
|
788
|
+
role: "worker",
|
|
881
789
|
at: s.at
|
|
882
790
|
}));
|
|
883
791
|
const lastNotification = /* @__PURE__ */ new Map();
|
|
@@ -919,73 +827,55 @@ async function readClaudeCodeSupervisorRun(opts) {
|
|
|
919
827
|
}));
|
|
920
828
|
const settleAtByAgent = new Map([...lastNotification.entries()].map(([id, n]) => [id, n.at]));
|
|
921
829
|
const spawnAtByAgent = new Map(spawns.map((s) => [s.agentId, s.at]));
|
|
922
|
-
const byLabel = /* @__PURE__ */ new Map();
|
|
923
|
-
for (const s of spawns) {
|
|
924
|
-
const ids = byLabel.get(s.label) ?? [];
|
|
925
|
-
ids.push(s.agentId);
|
|
926
|
-
byLabel.set(s.label, ids);
|
|
927
|
-
}
|
|
928
830
|
const workers = [];
|
|
929
|
-
for (const
|
|
831
|
+
for (const spawn of spawns) {
|
|
832
|
+
const { agentId, label } = spawn;
|
|
930
833
|
const events = [];
|
|
931
834
|
const inbox = [];
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
at:
|
|
945
|
-
|
|
835
|
+
const child = childByAgentId.get(agentId) ?? null;
|
|
836
|
+
const startAt = child?.firstAt ?? spawnAtByAgent.get(agentId) ?? null;
|
|
837
|
+
const endAt = settleAtByAgent.get(agentId) ?? child?.lastAt ?? null;
|
|
838
|
+
if (startAt !== null) events.push(line({
|
|
839
|
+
kind: "started",
|
|
840
|
+
label,
|
|
841
|
+
at: startAt,
|
|
842
|
+
agentId
|
|
843
|
+
}));
|
|
844
|
+
for (const steer of steersByTarget.get(agentId) ?? []) {
|
|
845
|
+
inbox.push(line({
|
|
846
|
+
id: steer.requestId,
|
|
847
|
+
at: steer.at,
|
|
848
|
+
worker: label,
|
|
849
|
+
message: "steer"
|
|
946
850
|
}));
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
id: `${agentId}:${steer.at}`,
|
|
950
|
-
at: steer.at,
|
|
951
|
-
worker: label,
|
|
952
|
-
message: "steer"
|
|
953
|
-
}));
|
|
954
|
-
events.push(line({
|
|
955
|
-
kind: "message",
|
|
956
|
-
label,
|
|
957
|
-
direction: "down",
|
|
958
|
-
at: steer.at,
|
|
959
|
-
requestId: `${agentId}:${steer.at}`,
|
|
960
|
-
delivered: steer.delivered
|
|
961
|
-
}));
|
|
962
|
-
}
|
|
963
|
-
if (endAt !== null) events.push(line({
|
|
964
|
-
kind: "finished",
|
|
851
|
+
events.push(line({
|
|
852
|
+
kind: "message",
|
|
965
853
|
label,
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
854
|
+
direction: "down",
|
|
855
|
+
at: steer.at,
|
|
856
|
+
requestId: steer.requestId,
|
|
857
|
+
delivered: steer.delivered
|
|
969
858
|
}));
|
|
970
|
-
if (child !== null) {
|
|
971
|
-
tokensIn = (tokensIn ?? 0) + child.tokensIn;
|
|
972
|
-
tokensOut = (tokensOut ?? 0) + child.tokensOut;
|
|
973
|
-
cacheRead = (cacheRead ?? 0) + child.cacheRead;
|
|
974
|
-
cacheWrite = (cacheWrite ?? 0) + child.cacheWrite;
|
|
975
|
-
transcriptRef = child.path;
|
|
976
|
-
}
|
|
977
859
|
}
|
|
860
|
+
if (endAt !== null) events.push(line({
|
|
861
|
+
kind: "finished",
|
|
862
|
+
label,
|
|
863
|
+
at: endAt,
|
|
864
|
+
agentId,
|
|
865
|
+
...child?.finalReport === null || child?.finalReport === void 0 ? {} : { evidence: child.finalReport }
|
|
866
|
+
}));
|
|
978
867
|
workers.push({
|
|
868
|
+
workerId: agentId,
|
|
979
869
|
label,
|
|
980
|
-
events: events.length === 0 ?
|
|
981
|
-
inbox: inbox.length === 0 ?
|
|
870
|
+
events: events.length === 0 ? "" : `${events.join("\n")}\n`,
|
|
871
|
+
inbox: inbox.length === 0 ? "" : `${inbox.join("\n")}\n`,
|
|
982
872
|
patchBytes: null,
|
|
983
|
-
transcriptRef,
|
|
873
|
+
transcriptRef: child?.path ?? null,
|
|
984
874
|
patchPath: null,
|
|
985
|
-
tokensIn,
|
|
986
|
-
tokensOut,
|
|
987
|
-
cacheRead,
|
|
988
|
-
cacheWrite
|
|
875
|
+
tokensIn: child?.tokensIn ?? null,
|
|
876
|
+
tokensOut: child?.tokensOut ?? null,
|
|
877
|
+
cacheRead: child?.cacheRead ?? null,
|
|
878
|
+
cacheWrite: child?.cacheWrite ?? null
|
|
989
879
|
});
|
|
990
880
|
}
|
|
991
881
|
const joined = children.length;
|
|
@@ -1027,6 +917,7 @@ async function readClaudeCodeSupervisorRun(opts) {
|
|
|
1027
917
|
harnessMissingReason,
|
|
1028
918
|
limits: {
|
|
1029
919
|
...limits,
|
|
920
|
+
workerTokens: spawns.length === 0 ? null : harnessMissingReason,
|
|
1030
921
|
deliverables: NO_DELIVERABLES
|
|
1031
922
|
},
|
|
1032
923
|
rootTranscriptRef: opts.transcriptPath,
|
|
@@ -1100,7 +991,7 @@ function renderSupervisorRunMarkdown(r) {
|
|
|
1100
991
|
out.push(`| Spawn waves | ${showMeasured(o.waves)} |`);
|
|
1101
992
|
out.push(`| Wave sizes | ${isUnavailable(o.waveSizes) ? showMeasured(o.waveSizes) : `[${o.waveSizes.join(", ")}]`} |`);
|
|
1102
993
|
out.push(`| Max concurrency | ${showMeasured(o.maxConcurrency)} |`);
|
|
1103
|
-
out.push(`| Respawns (
|
|
994
|
+
out.push(`| Respawns (after same parent's first settle) | ${showMeasured(o.respawns)} |`);
|
|
1104
995
|
out.push(`| Repeated labels | ${isUnavailable(o.repeatedLabels) ? showMeasured(o.repeatedLabels) : o.repeatedLabels.length === 0 ? "none" : o.repeatedLabels.join(", ")} |`);
|
|
1105
996
|
out.push(`| Delegation depth | ${showMeasured(o.delegationDepth)} |`);
|
|
1106
997
|
out.push(`| Time to first spawn | ${fmtMs(o.timeToFirstSpawnMs)} |`);
|
|
@@ -1111,9 +1002,9 @@ function renderSupervisorRunMarkdown(r) {
|
|
|
1111
1002
|
if (!isUnavailable(o.steersByWorker) && o.steersByWorker.length > 0) {
|
|
1112
1003
|
out.push("### Steers per worker");
|
|
1113
1004
|
out.push("");
|
|
1114
|
-
out.push("| Worker | Queued | Delivered |");
|
|
1115
|
-
out.push("
|
|
1116
|
-
for (const s of o.steersByWorker) out.push(`| \`${s.worker}\` | ${s.queued} | ${s.delivered} |`);
|
|
1005
|
+
out.push("| Worker id | Label | Queued | Delivered |");
|
|
1006
|
+
out.push("|---|---|---:|---:|");
|
|
1007
|
+
for (const s of o.steersByWorker) out.push(`| ${s.workerId === null ? "unavailable — legacy label join" : `\`${s.workerId}\``} | \`${s.worker}\` | ${s.queued} | ${s.delivered} |`);
|
|
1117
1008
|
out.push("");
|
|
1118
1009
|
} else if (isUnavailable(o.steersByWorker)) out.push(`### Steers per worker\n\nunavailable — ${o.steersByWorker.unavailable}\n`);
|
|
1119
1010
|
out.push("## Decision quality");
|
|
@@ -1126,7 +1017,7 @@ function renderSupervisorRunMarkdown(r) {
|
|
|
1126
1017
|
out.push(`| Rejected (verify red) | ${showMeasured(d.rejected)} |`);
|
|
1127
1018
|
out.push(`| Empty pass (green, no patch) | ${showMeasured(d.emptyPass)} |`);
|
|
1128
1019
|
out.push(`| Evidence → respawn sequences | ${showMeasured(d.observeThenRespawn)} |`);
|
|
1129
|
-
out.push(`| Respawn with no settled evidence in front | ${showMeasured(d.respawnWithoutEvidence)} |`);
|
|
1020
|
+
out.push(`| Respawn with no same-parent settled evidence in front | ${showMeasured(d.respawnWithoutEvidence)} |`);
|
|
1130
1021
|
out.push(`| Review actions (steers + worker questions) | ${showMeasured(d.reviewActions)} |`);
|
|
1131
1022
|
out.push(`| Worker evidence returned | ${isUnavailable(d.workerEvidenceBytes) ? showMeasured(d.workerEvidenceBytes) : `${d.workerEvidenceBytes} bytes`} |`);
|
|
1132
1023
|
out.push("");
|
|
@@ -1148,9 +1039,9 @@ function renderSupervisorRunMarkdown(r) {
|
|
|
1148
1039
|
}
|
|
1149
1040
|
out.push("");
|
|
1150
1041
|
if (!isUnavailable(e.perWorker) && e.perWorker.length > 0) {
|
|
1151
|
-
out.push("| Worker | Wall | Tokens in | Tokens out | Patch bytes | Verify passed |");
|
|
1152
|
-
out.push("
|
|
1153
|
-
for (const w of e.perWorker) out.push(`| \`${w.worker}\` | ${w.wallMs === null ? "unavailable — no start/finish pair" : fmtMs(w.wallMs)} | ${w.tokensIn ?? "unavailable — store does not attribute tokens per worker"} | ${w.tokensOut ?? "unavailable — store does not attribute tokens per worker"} | ${w.patchBytes ?? "unavailable — no worker patch file"} | ${w.passed === null ? "unavailable — no
|
|
1042
|
+
out.push("| Worker id | Label | Role | Wall | Tokens in | Tokens out | Patch bytes | Verify passed | Score |");
|
|
1043
|
+
out.push("|---|---|---|---:|---:|---:|---:|---|---:|");
|
|
1044
|
+
for (const w of e.perWorker) out.push(`| ${w.workerId === null ? "unavailable — legacy label join" : `\`${w.workerId}\``} | \`${w.worker}\` | ${w.role ?? "unavailable — no journal join"} | ${w.wallMs === null ? "unavailable — no start/finish pair" : fmtMs(w.wallMs)} | ${w.tokensIn ?? "unavailable — store does not attribute tokens per worker"} | ${w.tokensOut ?? "unavailable — store does not attribute tokens per worker"} | ${w.patchBytes ?? "unavailable — no worker patch file"} | ${w.passed === null ? "unavailable — no verdict" : String(w.passed)} | ${w.score ?? "unavailable — no numeric score"} |`);
|
|
1154
1045
|
out.push("");
|
|
1155
1046
|
}
|
|
1156
1047
|
out.push("## Outcome");
|
|
@@ -1237,9 +1128,12 @@ async function readLoopsSupervisorRun(runDir, opts = {}) {
|
|
|
1237
1128
|
const supRunDir = await findSupervisorRunDirIn(ws);
|
|
1238
1129
|
const result = await readMaybe(join(runDir, "result.json"));
|
|
1239
1130
|
const resultObj = parseJson(result);
|
|
1131
|
+
const journal = supRunDir === null ? null : await readMaybe(join(supRunDir, "journal.jsonl"));
|
|
1132
|
+
const journalWorkerSpawns = parseJsonl(journal).filter((event) => event.kind === "spawned" && typeof event.parent === "string" && event.role !== "supervisor").length;
|
|
1240
1133
|
let workers = null;
|
|
1241
1134
|
let workersMissingReason = null;
|
|
1242
1135
|
const workerCwds = [];
|
|
1136
|
+
let workerStarts = 0;
|
|
1243
1137
|
if (supRunDir === null) workersMissingReason = `no supervisor run dir under ${join(ws, ".loops", "supervisor")}`;
|
|
1244
1138
|
else {
|
|
1245
1139
|
const workersDir = join(supRunDir, "workers");
|
|
@@ -1252,18 +1146,25 @@ async function readLoopsSupervisorRun(runDir, opts = {}) {
|
|
|
1252
1146
|
const events = await readMaybe(join(workersDir, `${label}.ndjson`));
|
|
1253
1147
|
const inbox = await readMaybe(join(workersDir, `${label}.inbox.ndjson`));
|
|
1254
1148
|
const patch = await readMaybe(join(workersDir, `${label}.patch`));
|
|
1149
|
+
const startedRows = parseJsonl(events).filter((event) => event.kind === "started");
|
|
1150
|
+
workerStarts += startedRows.length;
|
|
1151
|
+
const startedIds = startedRows.map((event) => typeof event.workerId === "string" ? event.workerId : typeof event.agentId === "string" ? event.agentId : null).filter((id) => id !== null);
|
|
1152
|
+
const distinctStartedIds = new Set(startedIds);
|
|
1153
|
+
const workerId = startedRows.length > 0 && startedIds.length === startedRows.length && distinctStartedIds.size === 1 ? startedIds[0] : void 0;
|
|
1255
1154
|
workers.push({
|
|
1155
|
+
...workerId === void 0 ? {} : { workerId },
|
|
1256
1156
|
label,
|
|
1257
1157
|
events,
|
|
1258
1158
|
inbox,
|
|
1259
1159
|
patchBytes: patch === null ? null : Buffer.byteLength(patch)
|
|
1260
1160
|
});
|
|
1261
|
-
for (const ev of
|
|
1161
|
+
for (const ev of startedRows) if (ev.kind === "started" && typeof ev.cwd === "string") workerCwds.push(ev.cwd);
|
|
1262
1162
|
}
|
|
1263
1163
|
}
|
|
1264
1164
|
}
|
|
1265
1165
|
let harnessWorkerTokens = null;
|
|
1266
1166
|
let harnessMissingReason = null;
|
|
1167
|
+
let workerCwdsWithoutSessions = 0;
|
|
1267
1168
|
if (opts.opencodeDb === null) harnessMissingReason = "opencode join disabled";
|
|
1268
1169
|
else if (workerCwds.length === 0) harnessMissingReason = "no worker clone cwds in workers/*.ndjson (nothing to join)";
|
|
1269
1170
|
else {
|
|
@@ -1274,12 +1175,17 @@ async function readLoopsSupervisorRun(runDir, opts = {}) {
|
|
|
1274
1175
|
let sessions = 0;
|
|
1275
1176
|
let input = 0;
|
|
1276
1177
|
let output = 0;
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1178
|
+
const distinctWorkerCwds = new Set(workerCwds);
|
|
1179
|
+
for (const cwd of distinctWorkerCwds) {
|
|
1180
|
+
const rows = findOpencodeSessionsByDirectory(db, cwd);
|
|
1181
|
+
if (rows.length === 0) workerCwdsWithoutSessions += 1;
|
|
1182
|
+
for (const row of rows) {
|
|
1183
|
+
if (seen.has(row.id)) continue;
|
|
1184
|
+
seen.add(row.id);
|
|
1185
|
+
sessions += 1;
|
|
1186
|
+
input += row.tokensInput;
|
|
1187
|
+
output += row.tokensOutput + row.tokensReasoning;
|
|
1188
|
+
}
|
|
1283
1189
|
}
|
|
1284
1190
|
harnessWorkerTokens = {
|
|
1285
1191
|
store: "opencode",
|
|
@@ -1287,10 +1193,17 @@ async function readLoopsSupervisorRun(runDir, opts = {}) {
|
|
|
1287
1193
|
input,
|
|
1288
1194
|
output
|
|
1289
1195
|
};
|
|
1196
|
+
if (workerCwdsWithoutSessions > 0) harnessMissingReason = `${workerCwdsWithoutSessions}/${distinctWorkerCwds.size} worker clone cwds have no opencode session`;
|
|
1290
1197
|
} finally {
|
|
1291
1198
|
db.close();
|
|
1292
1199
|
}
|
|
1293
1200
|
}
|
|
1201
|
+
const workerInvocations = Math.max(journalWorkerSpawns, workerStarts);
|
|
1202
|
+
const workerTokenGaps = [];
|
|
1203
|
+
if (workerCwds.length < workerInvocations) workerTokenGaps.push(`${workerInvocations - workerCwds.length}/${workerInvocations} worker invocations have no clone cwd for the opencode token join`);
|
|
1204
|
+
if (workerInvocations > 0 && harnessWorkerTokens === null) workerTokenGaps.push(harnessMissingReason ?? "worker harness token join unavailable");
|
|
1205
|
+
else if (workerCwdsWithoutSessions > 0 && harnessMissingReason !== null) workerTokenGaps.push(harnessMissingReason);
|
|
1206
|
+
const workerTokenLimit = workerTokenGaps.length === 0 ? null : workerTokenGaps.join("; ");
|
|
1294
1207
|
const patchPath = opts.patchPath ?? (typeof resultObj?.patchPath === "string" ? resultObj.patchPath : null);
|
|
1295
1208
|
let judge = await readMaybe(join(runDir, "judge.json"));
|
|
1296
1209
|
let judgeSource = judge === null ? null : join(runDir, "judge.json");
|
|
@@ -1306,7 +1219,7 @@ async function readLoopsSupervisorRun(runDir, opts = {}) {
|
|
|
1306
1219
|
instanceId: typeof resultObj?.iid === "string" ? resultObj.iid : instanceIdFromPath(runDir),
|
|
1307
1220
|
arm: typeof resultObj?.arm === "string" ? resultObj.arm : basename(runDir),
|
|
1308
1221
|
supRunDir,
|
|
1309
|
-
journal
|
|
1222
|
+
journal,
|
|
1310
1223
|
brainLog: supRunDir === null ? null : await readMaybe(join(supRunDir, "brain.jsonl")),
|
|
1311
1224
|
state: supRunDir === null ? null : await readMaybe(join(supRunDir, "state.json")),
|
|
1312
1225
|
progress: supRunDir === null ? null : await readMaybe(join(supRunDir, "progress.ndjson")),
|
|
@@ -1319,7 +1232,10 @@ async function readLoopsSupervisorRun(runDir, opts = {}) {
|
|
|
1319
1232
|
driverLog: await readMaybe(join(runDir, "driver.log")),
|
|
1320
1233
|
harnessWorkerTokens,
|
|
1321
1234
|
harnessMissingReason,
|
|
1322
|
-
limits:
|
|
1235
|
+
limits: {
|
|
1236
|
+
...NO_SOURCE_LIMITS,
|
|
1237
|
+
workerTokens: workerTokenLimit
|
|
1238
|
+
},
|
|
1323
1239
|
traceCommand: null
|
|
1324
1240
|
};
|
|
1325
1241
|
}
|
|
@@ -1460,220 +1376,6 @@ async function findSupervisorRunDirs(root) {
|
|
|
1460
1376
|
return found.sort();
|
|
1461
1377
|
}
|
|
1462
1378
|
//#endregion
|
|
1463
|
-
|
|
1464
|
-
/**
|
|
1465
|
-
* The supervision tree as `tangle.rollout.v1` rows.
|
|
1466
|
-
*
|
|
1467
|
-
* A supervisor run IS a tree of rollouts, so its nodes are not a new shape:
|
|
1468
|
-
* the root becomes one `RolloutLine` with `role: 'supervisor'`, every spawned
|
|
1469
|
-
* worker becomes a `RolloutLine` with `role: 'worker'` and
|
|
1470
|
-
* `parent_rollout_id` pointing at its spawner. The rows append to the same
|
|
1471
|
-
* ledger as solo-agent rollouts and join to them with the same keys.
|
|
1472
|
-
*
|
|
1473
|
-
* What the journal CANNOT supply is the transcript: a worker's messages live
|
|
1474
|
-
* in its harness store (opencode sqlite, Claude Code jsonl), which the
|
|
1475
|
-
* `src/rollout/readers/*` intake readers own. Rows minted here are therefore
|
|
1476
|
-
* GAP lines (`messages: []`, `provenance.gap` set) carrying identity,
|
|
1477
|
-
* structure, outcome and cost; hydrating them with messages is the readers'
|
|
1478
|
-
* job, keyed on `artifacts.transcript_ref`.
|
|
1479
|
-
*
|
|
1480
|
-
* Timing lives in `outcome.metrics` (`spawned_at` / `settled_at` / `wall_ms`)
|
|
1481
|
-
* rather than a schema field: `tangle.rollout.v1` describes ONE invocation,
|
|
1482
|
-
* and the inter-invocation event timeline — which is what waves, concurrency,
|
|
1483
|
-
* idle and utilization are computed from — is a property of the journal, not
|
|
1484
|
-
* of any single row. The analyzer reads that timeline; these rows carry the
|
|
1485
|
-
* per-node facts.
|
|
1486
|
-
*/
|
|
1487
|
-
const EMPTY_COST = {
|
|
1488
|
-
usd: null,
|
|
1489
|
-
tokens_in: null,
|
|
1490
|
-
tokens_out: null,
|
|
1491
|
-
tokens_reasoning: null,
|
|
1492
|
-
cache_read: null,
|
|
1493
|
-
cache_write: null,
|
|
1494
|
-
wall_s: null
|
|
1495
|
-
};
|
|
1496
|
-
/**
|
|
1497
|
-
* Mint the supervision tree as rollout rows. Returns the rows plus the gaps
|
|
1498
|
-
* that made any of them incomplete — same unavailable-vs-zero discipline as
|
|
1499
|
-
* the report: a row with no transcript says WHY, it never pretends to be empty.
|
|
1500
|
-
*/
|
|
1501
|
-
function supervisorRunRolloutLines(src, opts = {}) {
|
|
1502
|
-
const gaps = [];
|
|
1503
|
-
const tree = parseSupervisorTree(src);
|
|
1504
|
-
if (src.journal === null) {
|
|
1505
|
-
gaps.push(`tree: ${src.supRunDir === null ? "no supervisor run dir" : "journal absent"} — no nodes recoverable`);
|
|
1506
|
-
return {
|
|
1507
|
-
rootId: null,
|
|
1508
|
-
nodes: [],
|
|
1509
|
-
gaps
|
|
1510
|
-
};
|
|
1511
|
-
}
|
|
1512
|
-
const state = parseJson(src.state);
|
|
1513
|
-
const result = parseJson(src.result);
|
|
1514
|
-
const judge = parseJson(src.judge);
|
|
1515
|
-
const stateResult = asRecord(state?.result);
|
|
1516
|
-
const rootId = tree.rootId;
|
|
1517
|
-
const runId = opts.runId ?? rootId ?? src.runRef;
|
|
1518
|
-
const suite = opts.suite ?? "supervisor-run";
|
|
1519
|
-
const instanceId = src.instanceId ?? src.runRef;
|
|
1520
|
-
const split = opts.split ?? "search";
|
|
1521
|
-
const capturedAt = opts.capturedAt ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
1522
|
-
const base = {
|
|
1523
|
-
schema: ROLLOUT_SCHEMA,
|
|
1524
|
-
run_id: runId,
|
|
1525
|
-
experiment_id: opts.experimentId ?? null,
|
|
1526
|
-
candidate_id: opts.candidateId ?? null,
|
|
1527
|
-
generation: opts.generation ?? null,
|
|
1528
|
-
candidate_index: opts.candidateIndex ?? null,
|
|
1529
|
-
task: {
|
|
1530
|
-
suite,
|
|
1531
|
-
instance_id: instanceId,
|
|
1532
|
-
split,
|
|
1533
|
-
seed: opts.seed ?? null,
|
|
1534
|
-
rep: opts.rep ?? 0
|
|
1535
|
-
},
|
|
1536
|
-
messages: [],
|
|
1537
|
-
tool_defs: []
|
|
1538
|
-
};
|
|
1539
|
-
const nodes = [];
|
|
1540
|
-
if (rootId !== null) {
|
|
1541
|
-
const judgeResolved = typeof judge?.resolved === "boolean" ? judge.resolved : null;
|
|
1542
|
-
const reward = (typeof judge?.score === "number" ? judge.score : null) ?? (judgeResolved === null ? null : judgeResolved ? 1 : 0);
|
|
1543
|
-
if (reward === null) gaps.push("root.outcome.reward: no judge verdict for this run");
|
|
1544
|
-
const wallMs = tree.startedAt !== null && tree.completedAt !== null && tree.completedAt >= tree.startedAt ? tree.completedAt - tree.startedAt : null;
|
|
1545
|
-
nodes.push({
|
|
1546
|
-
...base,
|
|
1547
|
-
rollout_id: rootId,
|
|
1548
|
-
parent_rollout_id: null,
|
|
1549
|
-
role: "supervisor",
|
|
1550
|
-
policy: {
|
|
1551
|
-
harness: opts.supervisorHarness ?? null,
|
|
1552
|
-
harness_version: null,
|
|
1553
|
-
model: opts.supervisorModel ?? null,
|
|
1554
|
-
provider: null,
|
|
1555
|
-
profile_commit: null,
|
|
1556
|
-
sampling: null
|
|
1557
|
-
},
|
|
1558
|
-
outcome: {
|
|
1559
|
-
...unscreenedRewardFields(reward),
|
|
1560
|
-
reward_source: src.judgeSource,
|
|
1561
|
-
verdict: judge ?? null,
|
|
1562
|
-
metrics: {
|
|
1563
|
-
arm: src.arm,
|
|
1564
|
-
sup_status: typeof state?.status === "string" ? state.status : null,
|
|
1565
|
-
sup_verdict: typeof state?.verdict === "string" ? state.verdict : null,
|
|
1566
|
-
delivered: typeof stateResult.delivered === "boolean" ? stateResult.delivered : null,
|
|
1567
|
-
verify_pass: typeof result?.verify_pass === "boolean" ? result.verify_pass : null,
|
|
1568
|
-
started_at: tree.startedAt,
|
|
1569
|
-
completed_at: tree.completedAt,
|
|
1570
|
-
workers_spawned: tree.workerSpawns.length,
|
|
1571
|
-
brain_metered_events: tree.brain.meteredCount
|
|
1572
|
-
},
|
|
1573
|
-
is_completed: state?.status === "completed",
|
|
1574
|
-
is_truncated: false,
|
|
1575
|
-
error: null,
|
|
1576
|
-
realness_gated: false
|
|
1577
|
-
},
|
|
1578
|
-
cost: {
|
|
1579
|
-
...EMPTY_COST,
|
|
1580
|
-
usd: src.limits.spendUsd !== null ? null : typeof stateResult.spentUsd === "number" ? stateResult.spentUsd : tree.brain.usd,
|
|
1581
|
-
tokens_in: tree.brain.tokensIn,
|
|
1582
|
-
tokens_out: tree.brain.tokensOut,
|
|
1583
|
-
cache_read: tree.brain.hasCache ? tree.brain.cacheRead : null,
|
|
1584
|
-
cache_write: tree.brain.hasCache ? tree.brain.cacheWrite : null,
|
|
1585
|
-
wall_s: wallMs === null ? null : wallMs / 1e3
|
|
1586
|
-
},
|
|
1587
|
-
artifacts: {
|
|
1588
|
-
patch_path: typeof result?.patchPath === "string" ? result.patchPath : null,
|
|
1589
|
-
run_dir: src.supRunDir,
|
|
1590
|
-
transcript_ref: src.rootTranscriptRef !== void 0 ? src.rootTranscriptRef : src.supRunDir === null ? null : `${src.supRunDir}/journal.jsonl`
|
|
1591
|
-
},
|
|
1592
|
-
provenance: {
|
|
1593
|
-
captured_at: capturedAt,
|
|
1594
|
-
capture: "backfill",
|
|
1595
|
-
gap: "supervision journal carries structure and spend, not the brain transcript"
|
|
1596
|
-
}
|
|
1597
|
-
});
|
|
1598
|
-
} else gaps.push("tree.root: no parentless `spawned` event in the journal");
|
|
1599
|
-
const closeById = new Map(tree.closes.map((c) => [c.id, c]));
|
|
1600
|
-
const sourceByLabel = new Map((src.workers ?? []).map((w) => [w.label, w]));
|
|
1601
|
-
for (const spawn of tree.workerSpawns) {
|
|
1602
|
-
const close = closeById.get(spawn.id) ?? null;
|
|
1603
|
-
const facts = tree.workerLogs.get(spawn.label) ?? null;
|
|
1604
|
-
const workerSource = sourceByLabel.get(spawn.label) ?? null;
|
|
1605
|
-
const wallMs = facts?.started != null && facts.finishedAt != null ? facts.finishedAt - facts.started : null;
|
|
1606
|
-
const reward = facts?.passed === null || facts === null ? null : facts.passed ? 1 : 0;
|
|
1607
|
-
if (reward === null) gaps.push(`worker ${spawn.label}: no verify verdict (worker logs absent or unfinished)`);
|
|
1608
|
-
nodes.push({
|
|
1609
|
-
...base,
|
|
1610
|
-
rollout_id: spawn.id,
|
|
1611
|
-
parent_rollout_id: spawn.parent,
|
|
1612
|
-
role: "worker",
|
|
1613
|
-
policy: {
|
|
1614
|
-
harness: opts.workerHarness ?? null,
|
|
1615
|
-
harness_version: null,
|
|
1616
|
-
model: opts.workerModel ?? null,
|
|
1617
|
-
provider: null,
|
|
1618
|
-
profile_commit: null,
|
|
1619
|
-
sampling: null
|
|
1620
|
-
},
|
|
1621
|
-
outcome: {
|
|
1622
|
-
...unscreenedRewardFields(reward),
|
|
1623
|
-
reward_source: reward === null ? null : "worker-self-verify",
|
|
1624
|
-
verdict: close === null ? null : {
|
|
1625
|
-
kind: close.kind,
|
|
1626
|
-
status: close.status,
|
|
1627
|
-
verdict: close.verdict
|
|
1628
|
-
},
|
|
1629
|
-
metrics: {
|
|
1630
|
-
label: spawn.label,
|
|
1631
|
-
spawned_at: spawn.at,
|
|
1632
|
-
settled_at: close?.at ?? null,
|
|
1633
|
-
started_at: facts?.started ?? null,
|
|
1634
|
-
finished_at: facts?.finishedAt ?? null,
|
|
1635
|
-
wall_ms: wallMs,
|
|
1636
|
-
patch_bytes: facts?.finishedPatchBytes ?? null,
|
|
1637
|
-
evidence_bytes: facts?.evidenceBytes ?? null,
|
|
1638
|
-
steers_queued: facts?.steersQueued ?? null,
|
|
1639
|
-
steers_delivered: facts?.steersDelivered ?? null,
|
|
1640
|
-
questions: facts?.questions ?? null
|
|
1641
|
-
},
|
|
1642
|
-
is_completed: close?.kind === "settled",
|
|
1643
|
-
is_truncated: close?.kind === "cancelled",
|
|
1644
|
-
error: close?.kind === "cancelled" ? close.verdict ?? "cancelled" : null,
|
|
1645
|
-
realness_gated: false
|
|
1646
|
-
},
|
|
1647
|
-
cost: {
|
|
1648
|
-
...EMPTY_COST,
|
|
1649
|
-
usd: src.limits.spendUsd !== null || !close?.hasSpend ? null : close.spend.usd,
|
|
1650
|
-
tokens_in: workerSource?.tokensIn ?? (close?.hasSpend ? close.spend.tokens.input : null),
|
|
1651
|
-
tokens_out: workerSource?.tokensOut ?? (close?.hasSpend ? close.spend.tokens.output : null),
|
|
1652
|
-
cache_read: workerSource?.cacheRead ?? (close?.hasSpend && close.spend.tokens.hasCache ? close.spend.tokens.cacheRead : null),
|
|
1653
|
-
cache_write: workerSource?.cacheWrite ?? (close?.hasSpend && close.spend.tokens.hasCache ? close.spend.tokens.cacheWrite : null),
|
|
1654
|
-
wall_s: wallMs === null ? null : wallMs / 1e3
|
|
1655
|
-
},
|
|
1656
|
-
artifacts: {
|
|
1657
|
-
patch_path: workerSource?.patchPath !== void 0 ? workerSource.patchPath : src.supRunDir === null ? null : `${src.supRunDir}/workers/${spawn.label}.patch`,
|
|
1658
|
-
run_dir: src.supRunDir,
|
|
1659
|
-
transcript_ref: workerSource?.transcriptRef !== void 0 ? workerSource.transcriptRef : src.supRunDir === null ? null : `${src.supRunDir}/workers/${spawn.label}.ndjson`
|
|
1660
|
-
},
|
|
1661
|
-
provenance: {
|
|
1662
|
-
captured_at: capturedAt,
|
|
1663
|
-
capture: "backfill",
|
|
1664
|
-
gap: workerSource?.transcriptRef == null ? "worker transcript lives in the harness session store — hydrate via src/rollout/readers" : "worker transcript recorded by the harness; messages not inlined into this row"
|
|
1665
|
-
}
|
|
1666
|
-
});
|
|
1667
|
-
}
|
|
1668
|
-
const invalid = nodes.filter((n) => !isRolloutLine(n));
|
|
1669
|
-
if (invalid.length > 0) gaps.push(`${invalid.length} node(s) failed tangle.rollout.v1 validation`);
|
|
1670
|
-
return {
|
|
1671
|
-
rootId,
|
|
1672
|
-
nodes,
|
|
1673
|
-
gaps
|
|
1674
|
-
};
|
|
1675
|
-
}
|
|
1676
|
-
//#endregion
|
|
1677
|
-
export { NO_SOURCE_LIMITS as C, showMeasured as D, isUnavailable as E, unavailable as O, rollupSupervisorRuns as S, SUPERVISOR_RUN_SCHEMA as T, claudeCodeSupervisorRunReader as _, loopsSupervisorRunReader as a, parsePatch as b, supervisorReportStem as c, renderSupervisorRollupMarkdown as d, renderSupervisorRunHeadline as f, DEFAULT_STEER_TOOLS as g, DEFAULT_SPAWN_TOOLS as h, findSupervisorRunDirs as i, writeSupervisorRunReport as l, DEFAULT_CANCEL_TOOLS as m, analyzeSupervisorRun as n, readLoopsSupervisorRun as o, renderSupervisorRunMarkdown as p, findSupervisorRunDirIn as r, reportSupervisorRound as s, supervisorRunRolloutLines as t, writeSupervisorRunReportSafe as u, readClaudeCodeSupervisorRun as v, SUPERVISOR_RUN_ROLLUP_SCHEMA as w, parseSupervisorTree as x, analyzeSupervisorRunSources as y };
|
|
1379
|
+
export { SUPERVISOR_RUN_SCHEMA as C, unavailable as E, SUPERVISOR_RUN_ROLLUP_SCHEMA as S, showMeasured as T, readClaudeCodeSupervisorRun as _, readLoopsSupervisorRun as a, rollupSupervisorRuns as b, writeSupervisorRunReport as c, renderSupervisorRunHeadline as d, renderSupervisorRunMarkdown as f, claudeCodeSupervisorRunReader as g, DEFAULT_STEER_TOOLS as h, loopsSupervisorRunReader as i, writeSupervisorRunReportSafe as l, DEFAULT_SPAWN_TOOLS as m, findSupervisorRunDirIn as n, reportSupervisorRound as o, DEFAULT_CANCEL_TOOLS as p, findSupervisorRunDirs as r, supervisorReportStem as s, analyzeSupervisorRun as t, renderSupervisorRollupMarkdown as u, analyzeSupervisorRunSources as v, isUnavailable as w, NO_SOURCE_LIMITS as x, parsePatch as y };
|
|
1678
1380
|
|
|
1679
|
-
//# sourceMappingURL=supervisor-run-
|
|
1381
|
+
//# sourceMappingURL=supervisor-run-B2EWUmQY.js.map
|