@tangle-network/agent-eval 0.135.3 → 0.136.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/dist/{index-C61Wi7yg.d.ts → index-CyC1BTmn.d.ts} +43 -9
- package/dist/index-CyC1BTmn.d.ts.map +1 -0
- package/dist/index.d.ts +2 -2
- package/dist/index.js +1 -1
- package/dist/openapi.json +1 -1
- package/dist/supervisor-run/index.d.ts +2 -2
- package/dist/supervisor-run/index.js +1 -1
- package/dist/{supervisor-run-B7lUGoyZ.js → supervisor-run-Dr5HnTup.js} +272 -144
- package/dist/supervisor-run-Dr5HnTup.js.map +1 -0
- package/package.json +3 -3
- package/dist/index-C61Wi7yg.d.ts.map +0 -1
- package/dist/supervisor-run-B7lUGoyZ.js.map +0 -1
|
@@ -18,6 +18,8 @@ function showMeasured(v) {
|
|
|
18
18
|
}
|
|
19
19
|
/** A source that carries every fact the analyzer can use. */
|
|
20
20
|
const NO_SOURCE_LIMITS = {
|
|
21
|
+
managerTokens: null,
|
|
22
|
+
workerTokens: null,
|
|
21
23
|
spendUsd: null,
|
|
22
24
|
workerVerdicts: null,
|
|
23
25
|
deliverables: null
|
|
@@ -82,6 +84,31 @@ function ms(at) {
|
|
|
82
84
|
const t = Date.parse(at);
|
|
83
85
|
return Number.isFinite(t) ? t : null;
|
|
84
86
|
}
|
|
87
|
+
function readVerdict(v) {
|
|
88
|
+
if (typeof v === "string") return {
|
|
89
|
+
label: v,
|
|
90
|
+
valid: null,
|
|
91
|
+
score: null,
|
|
92
|
+
raw: v
|
|
93
|
+
};
|
|
94
|
+
if (typeof v !== "object" || v === null) return {
|
|
95
|
+
label: null,
|
|
96
|
+
valid: null,
|
|
97
|
+
score: null,
|
|
98
|
+
raw: null
|
|
99
|
+
};
|
|
100
|
+
const rec = v;
|
|
101
|
+
const valid = typeof rec.valid === "boolean" ? rec.valid : null;
|
|
102
|
+
return {
|
|
103
|
+
label: valid === null ? null : valid ? "valid" : "invalid",
|
|
104
|
+
valid,
|
|
105
|
+
score: typeof rec.score === "number" && Number.isFinite(rec.score) ? rec.score : null,
|
|
106
|
+
raw: v
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
function workerSourceKey(worker) {
|
|
110
|
+
return worker.workerId ?? worker.label;
|
|
111
|
+
}
|
|
85
112
|
function parseSupervisorTree(src) {
|
|
86
113
|
const events = parseJsonl(src.journal);
|
|
87
114
|
const state = parseJson(src.state);
|
|
@@ -102,26 +129,36 @@ function parseSupervisorTree(src) {
|
|
|
102
129
|
const parent = typeof ev.parent === "string" ? ev.parent : null;
|
|
103
130
|
const label = typeof ev.label === "string" ? ev.label : "";
|
|
104
131
|
if (parent === null && rootId === null) rootId = id;
|
|
132
|
+
const role = parent === null || ev.role === "supervisor" ? "supervisor" : "worker";
|
|
105
133
|
spawns.push({
|
|
106
134
|
id,
|
|
107
135
|
parent,
|
|
108
136
|
label,
|
|
137
|
+
role,
|
|
109
138
|
at: ms(ev.at)
|
|
110
139
|
});
|
|
111
|
-
} else if (kind === "settled")
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
140
|
+
} else if (kind === "settled") {
|
|
141
|
+
const verdict = readVerdict(ev.verdict);
|
|
142
|
+
closes.push({
|
|
143
|
+
id,
|
|
144
|
+
kind: "settled",
|
|
145
|
+
status: typeof ev.status === "string" ? ev.status : null,
|
|
146
|
+
verdict: verdict.label,
|
|
147
|
+
valid: verdict.valid,
|
|
148
|
+
score: verdict.score,
|
|
149
|
+
rawVerdict: verdict.raw,
|
|
150
|
+
at: ms(ev.at),
|
|
151
|
+
spend: readSpend(ev.spent),
|
|
152
|
+
hasSpend: asRecord(ev.spent).tokens !== void 0
|
|
153
|
+
});
|
|
154
|
+
} else if (kind === "cancelled") closes.push({
|
|
121
155
|
id,
|
|
122
156
|
kind: "cancelled",
|
|
123
157
|
status: "cancelled",
|
|
124
158
|
verdict: typeof ev.reason === "string" ? ev.reason : null,
|
|
159
|
+
valid: null,
|
|
160
|
+
score: null,
|
|
161
|
+
rawVerdict: typeof ev.reason === "string" ? ev.reason : null,
|
|
125
162
|
at: ms(ev.at),
|
|
126
163
|
spend: {
|
|
127
164
|
tokens: {
|
|
@@ -156,6 +193,7 @@ function parseSupervisorTree(src) {
|
|
|
156
193
|
finished: false,
|
|
157
194
|
finishedAt: null,
|
|
158
195
|
passed: null,
|
|
196
|
+
score: null,
|
|
159
197
|
finishedPatchBytes: null,
|
|
160
198
|
evidenceBytes: 0,
|
|
161
199
|
steersQueued: 0,
|
|
@@ -177,11 +215,12 @@ function parseSupervisorTree(src) {
|
|
|
177
215
|
facts.finished = true;
|
|
178
216
|
facts.finishedAt = ms(ev.at);
|
|
179
217
|
facts.passed = typeof ev.passed === "boolean" ? ev.passed : null;
|
|
218
|
+
facts.score = typeof ev.score === "number" && Number.isFinite(ev.score) ? ev.score : null;
|
|
180
219
|
facts.finishedPatchBytes = typeof ev.patchBytes === "number" ? ev.patchBytes : null;
|
|
181
220
|
facts.evidenceBytes = typeof ev.evidence === "string" ? ev.evidence.length : 0;
|
|
182
221
|
}
|
|
183
222
|
}
|
|
184
|
-
workerLogs.set(w
|
|
223
|
+
workerLogs.set(workerSourceKey(w), facts);
|
|
185
224
|
}
|
|
186
225
|
const startedAt = ms(state?.startedAt) ?? spawns[0]?.at ?? null;
|
|
187
226
|
const completedAt = ms(state?.completedAt) ?? [...spawns.map((s) => s.at), ...closes.map((c) => c.at)].reduce((acc, t) => t === null ? acc : acc === null ? t : Math.max(acc, t), null);
|
|
@@ -222,17 +261,34 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
|
|
|
222
261
|
const judge = parseJson(src.judge);
|
|
223
262
|
const tree = parseSupervisorTree(src);
|
|
224
263
|
const { rootId, workerSpawns, workerCloses, startedAt, completedAt } = tree;
|
|
264
|
+
const spawnById = new Map(workerSpawns.map((spawn) => [spawn.id, spawn]));
|
|
265
|
+
const spawnsByLabel = /* @__PURE__ */ new Map();
|
|
266
|
+
for (const spawn of workerSpawns) {
|
|
267
|
+
const matches = spawnsByLabel.get(spawn.label) ?? [];
|
|
268
|
+
matches.push(spawn);
|
|
269
|
+
spawnsByLabel.set(spawn.label, matches);
|
|
270
|
+
}
|
|
271
|
+
const spawnsForSource = (worker) => {
|
|
272
|
+
if (worker.workerId === void 0) return spawnsByLabel.get(worker.label) ?? [];
|
|
273
|
+
const spawn = spawnById.get(worker.workerId);
|
|
274
|
+
return spawn === void 0 ? [] : [spawn];
|
|
275
|
+
};
|
|
276
|
+
const spawnForSource = (worker) => {
|
|
277
|
+
const matches = spawnsForSource(worker);
|
|
278
|
+
return matches.length === 1 ? matches[0] ?? null : null;
|
|
279
|
+
};
|
|
225
280
|
const supervisorWallMs = startedAt !== null && completedAt !== null && completedAt >= startedAt ? completedAt - startedAt : !haveJournal ? gap("supervisorWallMs", journalMissing) : gap("supervisorWallMs", "no parseable start/complete timestamps in state.json or journal");
|
|
226
281
|
const steerRows = [];
|
|
227
282
|
let steerQueuedTotal = 0;
|
|
228
283
|
let steerDeliveredTotal = 0;
|
|
229
284
|
let upLegMessages = 0;
|
|
230
285
|
if (src.workers !== null) for (const w of src.workers) {
|
|
231
|
-
const facts = tree.workerLogs.get(w
|
|
286
|
+
const facts = tree.workerLogs.get(workerSourceKey(w));
|
|
232
287
|
const queued = facts?.steersQueued ?? 0;
|
|
233
288
|
const delivered = facts?.steersDelivered ?? 0;
|
|
234
289
|
upLegMessages += facts?.questions ?? 0;
|
|
235
290
|
steerRows.push({
|
|
291
|
+
workerId: w.workerId ?? null,
|
|
236
292
|
worker: w.label,
|
|
237
293
|
queued,
|
|
238
294
|
delivered
|
|
@@ -287,11 +343,39 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
|
|
|
287
343
|
sumWorkerWallMs += span * live;
|
|
288
344
|
}
|
|
289
345
|
const firstWorkerSpawnAt = workerSpawns.reduce((acc, s) => s.at === null ? acc : acc === null ? s.at : Math.min(acc, s.at), null);
|
|
290
|
-
const
|
|
291
|
-
const
|
|
292
|
-
const
|
|
293
|
-
|
|
294
|
-
|
|
346
|
+
const closeById = new Map(workerCloses.map((close) => [close.id, close]));
|
|
347
|
+
const childSpawnsByParent = /* @__PURE__ */ new Map();
|
|
348
|
+
for (const spawn of workerSpawns) {
|
|
349
|
+
if (spawn.parent === null) continue;
|
|
350
|
+
const siblings = childSpawnsByParent.get(spawn.parent) ?? [];
|
|
351
|
+
siblings.push(spawn);
|
|
352
|
+
childSpawnsByParent.set(spawn.parent, siblings);
|
|
353
|
+
}
|
|
354
|
+
let respawns = 0;
|
|
355
|
+
let observeThenRespawn = 0;
|
|
356
|
+
let respawnWithoutEvidence = 0;
|
|
357
|
+
const repeatedLabelSet = /* @__PURE__ */ new Set();
|
|
358
|
+
for (const siblings of childSpawnsByParent.values()) {
|
|
359
|
+
const labelCounts = /* @__PURE__ */ new Map();
|
|
360
|
+
for (const spawn of siblings) labelCounts.set(spawn.label, (labelCounts.get(spawn.label) ?? 0) + 1);
|
|
361
|
+
for (const [label, count] of labelCounts) if (count > 1) repeatedLabelSet.add(label);
|
|
362
|
+
const orderedSpawns = siblings.map((spawn, index) => ({
|
|
363
|
+
spawn,
|
|
364
|
+
index
|
|
365
|
+
})).filter((row) => row.spawn.at !== null).sort((a, b) => a.spawn.at - b.spawn.at || a.index - b.index);
|
|
366
|
+
const directCloseTimes = siblings.map((spawn) => closeById.get(spawn.id)?.at ?? null).filter((at) => at !== null).sort((a, b) => a - b);
|
|
367
|
+
const firstDirectClose = directCloseTimes[0] ?? null;
|
|
368
|
+
for (let i = 1; i < orderedSpawns.length; i += 1) {
|
|
369
|
+
const previous = orderedSpawns[i - 1]?.spawn.at;
|
|
370
|
+
const current = orderedSpawns[i]?.spawn.at;
|
|
371
|
+
if (previous === void 0 || current === void 0) continue;
|
|
372
|
+
if (firstDirectClose === null || current <= firstDirectClose) continue;
|
|
373
|
+
respawns += 1;
|
|
374
|
+
if (hasNumberBetween(directCloseTimes, previous, current)) observeThenRespawn += 1;
|
|
375
|
+
else respawnWithoutEvidence += 1;
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
const repeatedLabels = [...repeatedLabelSet];
|
|
295
379
|
const parentOf = new Map(tree.spawns.map((s) => [s.id, s.parent]));
|
|
296
380
|
let delegationDepth = 0;
|
|
297
381
|
for (const s of workerSpawns) {
|
|
@@ -333,35 +417,32 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
|
|
|
333
417
|
if (c.verdict !== null) settledVerdicts[c.verdict] = (settledVerdicts[c.verdict] ?? 0) + 1;
|
|
334
418
|
}
|
|
335
419
|
const verdictLimit = src.limits.workerVerdicts;
|
|
420
|
+
const acceptedLimit = verdictLimit ?? src.limits.deliverables;
|
|
336
421
|
let accepted = 0;
|
|
337
|
-
let rejected = 0;
|
|
338
422
|
let emptyPass = 0;
|
|
339
423
|
let evidenceBytes = 0;
|
|
424
|
+
const sourceVerdicts = [];
|
|
340
425
|
for (const w of src.workers ?? []) {
|
|
341
|
-
const f = tree.workerLogs.get(w
|
|
342
|
-
if (f
|
|
343
|
-
|
|
344
|
-
|
|
426
|
+
const f = tree.workerLogs.get(workerSourceKey(w));
|
|
427
|
+
if (f?.finished) evidenceBytes += f.evidenceBytes;
|
|
428
|
+
const spawn = spawnForSource(w);
|
|
429
|
+
const passed = (spawn === null ? null : closeById.get(spawn.id) ?? null)?.valid ?? f?.passed ?? null;
|
|
430
|
+
if (passed !== null) sourceVerdicts.push(passed);
|
|
431
|
+
if (passed === true) if ((w.patchBytes ?? f?.finishedPatchBytes ?? 0) > 0) accepted += 1;
|
|
345
432
|
else emptyPass += 1;
|
|
346
|
-
else if (f.passed === false) rejected += 1;
|
|
347
|
-
}
|
|
348
|
-
let observeThenRespawn = 0;
|
|
349
|
-
let respawnWithoutEvidence = 0;
|
|
350
|
-
const spawnTimes = workerSpawns.map((s) => s.at).filter((t) => t !== null).sort((a, b) => a - b);
|
|
351
|
-
const closeTimes = workerCloses.map((c) => c.at).filter((t) => t !== null).sort((a, b) => a - b);
|
|
352
|
-
for (let i = 1; i < spawnTimes.length; i += 1) {
|
|
353
|
-
const prevSpawn = spawnTimes[i - 1];
|
|
354
|
-
const thisSpawn = spawnTimes[i];
|
|
355
|
-
const sawEvidence = closeTimes.some((t) => t >= prevSpawn && t <= thisSpawn);
|
|
356
|
-
if (firstSettleAt !== null && thisSpawn > firstSettleAt) if (sawEvidence) observeThenRespawn += 1;
|
|
357
|
-
else respawnWithoutEvidence += 1;
|
|
358
433
|
}
|
|
434
|
+
const settledCloses = workerCloses.filter((close) => close.kind === "settled");
|
|
435
|
+
const structuredVerdicts = settledCloses.map((close) => close.valid).filter((valid) => valid !== null);
|
|
436
|
+
const journalVerdictsComplete = settledCloses.length > 0 && structuredVerdicts.length === settledCloses.length;
|
|
437
|
+
const sourceVerdictsComplete = sourceVerdicts.length > 0 && sourceVerdicts.length >= settledCloses.length;
|
|
438
|
+
const rejected = journalVerdictsComplete ? structuredVerdicts.filter((valid) => !valid).length : sourceVerdictsComplete ? sourceVerdicts.filter((valid) => !valid).length : 0;
|
|
439
|
+
const rejectedLimit = journalVerdictsComplete || sourceVerdictsComplete ? null : settledCloses.length > 0 ? verdictLimit ?? "a settled journal verdict has no validity and no matched worker log" : verdictLimit !== null ? verdictLimit : !haveJournal && src.workers === null ? workersGapReason : null;
|
|
359
440
|
const decision = {
|
|
360
441
|
settledByStatus: haveJournal ? settledByStatus : gap("settledByStatus", journalMissing),
|
|
361
442
|
settledVerdicts: verdictLimit !== null ? unavailable(verdictLimit) : haveJournal ? settledVerdicts : unavailable(journalMissing),
|
|
362
|
-
accepted:
|
|
363
|
-
rejected:
|
|
364
|
-
emptyPass:
|
|
443
|
+
accepted: acceptedLimit !== null ? gap("accepted", acceptedLimit) : src.workers === null ? unavailable(workersGapReason) : accepted,
|
|
444
|
+
rejected: rejectedLimit === null ? rejected : unavailable(rejectedLimit),
|
|
445
|
+
emptyPass: acceptedLimit !== null ? gap("emptyPass", acceptedLimit) : src.workers === null ? unavailable(workersGapReason) : emptyPass,
|
|
365
446
|
observeThenRespawn: haveJournal ? observeThenRespawn : unavailable(journalMissing),
|
|
366
447
|
respawnWithoutEvidence: haveJournal ? respawnWithoutEvidence : unavailable(journalMissing),
|
|
367
448
|
reviewActions: src.workers === null ? unavailable(workersGapReason) : steerQueuedTotal + upLegMessages,
|
|
@@ -370,52 +451,64 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
|
|
|
370
451
|
const journalWorkerIn = workerCloses.reduce((a, c) => a + c.spend.tokens.input, 0);
|
|
371
452
|
const journalWorkerOut = workerCloses.reduce((a, c) => a + c.spend.tokens.output, 0);
|
|
372
453
|
const journalWorkerUsd = workerCloses.reduce((a, c) => a + c.spend.usd, 0);
|
|
373
|
-
const
|
|
454
|
+
const workerUsdById = /* @__PURE__ */ new Map();
|
|
455
|
+
for (const c of workerCloses) workerUsdById.set(c.id, (workerUsdById.get(c.id) ?? 0) + c.spend.usd);
|
|
456
|
+
const labelById = new Map(workerSpawns.map((spawn) => [spawn.id, spawn.label]));
|
|
374
457
|
const workerUsdByLabel = /* @__PURE__ */ new Map();
|
|
375
|
-
for (const
|
|
376
|
-
const label = labelById.get(
|
|
458
|
+
for (const close of workerCloses) {
|
|
459
|
+
const label = labelById.get(close.id);
|
|
377
460
|
if (label === void 0) continue;
|
|
378
|
-
workerUsdByLabel.set(label, (workerUsdByLabel.get(label) ?? 0) +
|
|
461
|
+
workerUsdByLabel.set(label, (workerUsdByLabel.get(label) ?? 0) + close.spend.usd);
|
|
379
462
|
}
|
|
380
463
|
const sq = src.harnessWorkerTokens;
|
|
381
464
|
const harnessGapReason = src.harnessMissingReason ?? "harness session store unavailable and journal settled spend is 0";
|
|
382
|
-
const
|
|
383
|
-
const
|
|
465
|
+
const workerTokenLimit = src.limits.workerTokens;
|
|
466
|
+
const workerIn = workerTokenLimit !== null ? gap("workers.tokensIn", workerTokenLimit) : sq !== null ? journalWorkerIn + sq.input : haveJournal ? journalWorkerIn : gap("workers.tokensIn", harnessGapReason);
|
|
467
|
+
const workerOut = workerTokenLimit !== null ? unavailable(workerTokenLimit) : sq !== null ? journalWorkerOut + sq.output : haveJournal ? journalWorkerOut : unavailable(harnessGapReason);
|
|
384
468
|
const stateResult = asRecord(state?.result);
|
|
385
469
|
const stateUsd = typeof stateResult.spentUsd === "number" ? stateResult.spentUsd : null;
|
|
386
470
|
const usdLimit = src.limits.spendUsd;
|
|
387
471
|
const totalUsd = usdLimit !== null ? gap("totalUsd", usdLimit) : stateUsd !== null ? round(stateUsd, 6) : haveJournal ? round(tree.brain.usd + journalWorkerUsd, 6) : gap("totalUsd", journalMissing);
|
|
388
472
|
const perWorker = (src.workers ?? []).map((w) => {
|
|
389
|
-
const f = tree.workerLogs.get(w
|
|
473
|
+
const f = tree.workerLogs.get(workerSourceKey(w));
|
|
474
|
+
const matchingSpawns = spawnsForSource(w);
|
|
475
|
+
const spawn = spawnForSource(w);
|
|
476
|
+
const close = spawn === null ? null : closeById.get(spawn.id) ?? null;
|
|
477
|
+
const passed = close?.valid ?? f?.passed ?? null;
|
|
478
|
+
const matchingRoles = new Set(matchingSpawns.map((candidate) => candidate.role));
|
|
390
479
|
return {
|
|
480
|
+
workerId: w.workerId ?? null,
|
|
391
481
|
worker: w.label,
|
|
482
|
+
role: matchingRoles.size === 1 ? matchingSpawns[0]?.role ?? null : null,
|
|
392
483
|
wallMs: f?.started != null && f.finishedAt != null ? f.finishedAt - f.started : null,
|
|
393
484
|
tokensIn: w.tokensIn ?? null,
|
|
394
485
|
tokensOut: w.tokensOut ?? null,
|
|
395
|
-
usd: usdLimit !== null ? null : workerUsdByLabel.get(w.label) ?? null,
|
|
486
|
+
usd: usdLimit !== null ? null : w.workerId === void 0 ? workerUsdByLabel.get(w.label) ?? null : workerUsdById.get(w.workerId) ?? null,
|
|
396
487
|
patchBytes: w.patchBytes ?? f?.finishedPatchBytes ?? null,
|
|
397
|
-
passed
|
|
488
|
+
passed,
|
|
489
|
+
score: close?.score ?? f?.score ?? null
|
|
398
490
|
};
|
|
399
491
|
});
|
|
400
492
|
const walls = perWorker.map((w) => w.wallMs).filter((w) => w !== null).sort((a, b) => a - b);
|
|
401
493
|
const brainCalls = parseJsonl(src.brainLog);
|
|
494
|
+
const managerTokenLimit = src.limits.managerTokens;
|
|
402
495
|
const economics = {
|
|
403
496
|
brain: {
|
|
404
|
-
tokensIn: haveJournal ? tree.brain.tokensIn : gap("brain.tokensIn", journalMissing),
|
|
405
|
-
tokensOut: haveJournal ? tree.brain.tokensOut : unavailable(journalMissing),
|
|
497
|
+
tokensIn: managerTokenLimit !== null ? gap("brain.tokensIn", managerTokenLimit) : haveJournal ? tree.brain.tokensIn : gap("brain.tokensIn", journalMissing),
|
|
498
|
+
tokensOut: managerTokenLimit !== null ? unavailable(managerTokenLimit) : haveJournal ? tree.brain.tokensOut : unavailable(journalMissing),
|
|
406
499
|
usd: usdLimit !== null ? unavailable(usdLimit) : haveJournal ? round(tree.brain.usd, 6) : unavailable(journalMissing),
|
|
407
|
-
cacheRead: !haveJournal ? unavailable(journalMissing) : tree.brain.hasCache ? tree.brain.cacheRead : unavailable(NO_CACHE_COUNTERS),
|
|
408
|
-
cacheWrite: !haveJournal ? unavailable(journalMissing) : tree.brain.hasCache ? tree.brain.cacheWrite : unavailable(NO_CACHE_COUNTERS),
|
|
409
|
-
source: haveJournal ? `journal metered events (n=${tree.brain.meteredCount})` : journalMissing
|
|
500
|
+
cacheRead: managerTokenLimit !== null ? unavailable(managerTokenLimit) : !haveJournal ? unavailable(journalMissing) : tree.brain.hasCache ? tree.brain.cacheRead : unavailable(NO_CACHE_COUNTERS),
|
|
501
|
+
cacheWrite: managerTokenLimit !== null ? unavailable(managerTokenLimit) : !haveJournal ? unavailable(journalMissing) : tree.brain.hasCache ? tree.brain.cacheWrite : unavailable(NO_CACHE_COUNTERS),
|
|
502
|
+
source: managerTokenLimit ?? (haveJournal ? `journal metered events (n=${tree.brain.meteredCount})` : journalMissing)
|
|
410
503
|
},
|
|
411
504
|
brainTruncations: src.brainLog === null ? gap("brain.brainTruncations", src.supRunDir === null ? "no supervisor run dir under <ws>/.loops/supervisor" : "brain.jsonl absent — loops predates the brain-call tap, so truncation cannot be ruled out") : brainCalls.filter((c) => c.finish_reason === "length").length,
|
|
412
505
|
workers: {
|
|
413
506
|
tokensIn: workerIn,
|
|
414
507
|
tokensOut: workerOut,
|
|
415
|
-
cacheRead: sq?.cacheRead !== void 0 ? sq.cacheRead : unavailable(NO_CACHE_COUNTERS),
|
|
416
|
-
cacheWrite: sq?.cacheWrite !== void 0 ? sq.cacheWrite : unavailable(NO_CACHE_COUNTERS),
|
|
508
|
+
cacheRead: workerTokenLimit !== null ? unavailable(workerTokenLimit) : sq?.cacheRead !== void 0 ? sq.cacheRead : unavailable(NO_CACHE_COUNTERS),
|
|
509
|
+
cacheWrite: workerTokenLimit !== null ? unavailable(workerTokenLimit) : sq?.cacheWrite !== void 0 ? sq.cacheWrite : unavailable(NO_CACHE_COUNTERS),
|
|
417
510
|
usd: usdLimit !== null ? unavailable(usdLimit) : haveJournal ? round(journalWorkerUsd, 6) : unavailable(journalMissing),
|
|
418
|
-
source: sq !== null ? `journal settled spend + ${sq.store} sessions (n=${sq.sessions})` : `journal settled spend only — ${src.harnessMissingReason ?? "harness session store unavailable"}`
|
|
511
|
+
source: workerTokenLimit !== null ? workerTokenLimit : sq !== null ? `journal settled spend + ${sq.store} sessions (n=${sq.sessions})` : `journal settled spend only — ${src.harnessMissingReason ?? "harness session store unavailable"}`
|
|
419
512
|
},
|
|
420
513
|
totalUsd,
|
|
421
514
|
totalUsdSource: usdLimit !== null ? usdLimit : stateUsd !== null ? `state.json result.spentUsd${journalWorkerUsd === 0 ? " — brain-priced only; worker CLI inference is unpriced (see worker token counts)" : ""}` : haveJournal ? "journal metered + settled usd" : journalMissing,
|
|
@@ -459,6 +552,17 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
|
|
|
459
552
|
traceCommand: src.traceCommand ?? "npx --yes @tangle-network/traces@latest analyze --harness opencode --cwd <worker-clone-cwd>"
|
|
460
553
|
};
|
|
461
554
|
}
|
|
555
|
+
/** Whether sorted values contain one value in the inclusive interval. */
|
|
556
|
+
function hasNumberBetween(sorted, low, high) {
|
|
557
|
+
let left = 0;
|
|
558
|
+
let right = sorted.length;
|
|
559
|
+
while (left < right) {
|
|
560
|
+
const middle = left + Math.floor((right - left) / 2);
|
|
561
|
+
if (sorted[middle] < low) left = middle + 1;
|
|
562
|
+
else right = middle;
|
|
563
|
+
}
|
|
564
|
+
return left < sorted.length && sorted[left] <= high;
|
|
565
|
+
}
|
|
462
566
|
/** `[driver] registered tools: …supervisor_steer…` is a banner, not an invocation. */
|
|
463
567
|
function registrationMentions(driverLog) {
|
|
464
568
|
let n = 0;
|
|
@@ -604,7 +708,7 @@ function rollupSupervisorRuns(reports) {
|
|
|
604
708
|
* | observeThenRespawn / respawnWithoutEvidence | full | full | ordering of spawn vs settle instants |
|
|
605
709
|
* | workerEvidenceBytes | full | PARTIAL | the child's closing message; 0 for pruned transcripts |
|
|
606
710
|
* | brain tokens in/out + cache | full | full | main-thread `message.usage` |
|
|
607
|
-
* | worker tokens in/out + cache | via harness join |
|
|
711
|
+
* | worker tokens in/out + cache | via harness join | full or unavailable | totals are refused if any spawned transcript was pruned; retained per-worker rows remain available |
|
|
608
712
|
* | perWorker wall | full | full | spawn → settle instants |
|
|
609
713
|
* | accepted / rejected / emptyPass / settledVerdicts | full | NONE | no per-worker verify step exists |
|
|
610
714
|
* | brain/worker/total usd, costPerAcceptedPatch | full | NONE | transcripts carry no price |
|
|
@@ -774,6 +878,8 @@ async function readClaudeCodeSupervisorRun(opts) {
|
|
|
774
878
|
const sessionId = basename(opts.transcriptPath).replace(/\.jsonl$/, "");
|
|
775
879
|
const runRef = opts.runRef ?? opts.transcriptPath;
|
|
776
880
|
const limits = {
|
|
881
|
+
managerTokens: null,
|
|
882
|
+
workerTokens: null,
|
|
777
883
|
spendUsd: SPEND_UNPRICED,
|
|
778
884
|
workerVerdicts: NO_VERDICTS,
|
|
779
885
|
deliverables: null
|
|
@@ -799,6 +905,8 @@ async function readClaudeCodeSupervisorRun(opts) {
|
|
|
799
905
|
harnessMissingReason: `session transcript unreadable at ${opts.transcriptPath}`,
|
|
800
906
|
limits: {
|
|
801
907
|
...limits,
|
|
908
|
+
managerTokens: `session transcript unreadable at ${opts.transcriptPath}`,
|
|
909
|
+
workerTokens: `session transcript unreadable at ${opts.transcriptPath}`,
|
|
802
910
|
deliverables: NO_DELIVERABLES
|
|
803
911
|
},
|
|
804
912
|
traceCommand
|
|
@@ -871,13 +979,16 @@ async function readClaudeCodeSupervisorRun(opts) {
|
|
|
871
979
|
id: sessionId,
|
|
872
980
|
parent: null,
|
|
873
981
|
label: `session:${sessionId}`,
|
|
982
|
+
role: "supervisor",
|
|
874
983
|
at: startedAt
|
|
875
984
|
})];
|
|
985
|
+
const supervisorIds = new Set(spawns.map((spawn) => spawn.parentId));
|
|
876
986
|
for (const s of spawns) journalLines.push(line({
|
|
877
987
|
kind: "spawned",
|
|
878
988
|
id: s.agentId,
|
|
879
989
|
parent: s.parentId,
|
|
880
990
|
label: s.label,
|
|
991
|
+
role: supervisorIds.has(s.agentId) ? "supervisor" : "worker",
|
|
881
992
|
at: s.at
|
|
882
993
|
}));
|
|
883
994
|
const lastNotification = /* @__PURE__ */ new Map();
|
|
@@ -919,73 +1030,55 @@ async function readClaudeCodeSupervisorRun(opts) {
|
|
|
919
1030
|
}));
|
|
920
1031
|
const settleAtByAgent = new Map([...lastNotification.entries()].map(([id, n]) => [id, n.at]));
|
|
921
1032
|
const spawnAtByAgent = new Map(spawns.map((s) => [s.agentId, s.at]));
|
|
922
|
-
const byLabel = /* @__PURE__ */ new Map();
|
|
923
|
-
for (const s of spawns) {
|
|
924
|
-
const ids = byLabel.get(s.label) ?? [];
|
|
925
|
-
ids.push(s.agentId);
|
|
926
|
-
byLabel.set(s.label, ids);
|
|
927
|
-
}
|
|
928
1033
|
const workers = [];
|
|
929
|
-
for (const
|
|
1034
|
+
for (const spawn of spawns) {
|
|
1035
|
+
const { agentId, label } = spawn;
|
|
930
1036
|
const events = [];
|
|
931
1037
|
const inbox = [];
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
at:
|
|
945
|
-
|
|
1038
|
+
const child = childByAgentId.get(agentId) ?? null;
|
|
1039
|
+
const startAt = child?.firstAt ?? spawnAtByAgent.get(agentId) ?? null;
|
|
1040
|
+
const endAt = settleAtByAgent.get(agentId) ?? child?.lastAt ?? null;
|
|
1041
|
+
if (startAt !== null) events.push(line({
|
|
1042
|
+
kind: "started",
|
|
1043
|
+
label,
|
|
1044
|
+
at: startAt,
|
|
1045
|
+
agentId
|
|
1046
|
+
}));
|
|
1047
|
+
for (const steer of steersByTarget.get(agentId) ?? []) {
|
|
1048
|
+
inbox.push(line({
|
|
1049
|
+
id: `${agentId}:${steer.at}`,
|
|
1050
|
+
at: steer.at,
|
|
1051
|
+
worker: label,
|
|
1052
|
+
message: "steer"
|
|
946
1053
|
}));
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
id: `${agentId}:${steer.at}`,
|
|
950
|
-
at: steer.at,
|
|
951
|
-
worker: label,
|
|
952
|
-
message: "steer"
|
|
953
|
-
}));
|
|
954
|
-
events.push(line({
|
|
955
|
-
kind: "message",
|
|
956
|
-
label,
|
|
957
|
-
direction: "down",
|
|
958
|
-
at: steer.at,
|
|
959
|
-
requestId: `${agentId}:${steer.at}`,
|
|
960
|
-
delivered: steer.delivered
|
|
961
|
-
}));
|
|
962
|
-
}
|
|
963
|
-
if (endAt !== null) events.push(line({
|
|
964
|
-
kind: "finished",
|
|
1054
|
+
events.push(line({
|
|
1055
|
+
kind: "message",
|
|
965
1056
|
label,
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
1057
|
+
direction: "down",
|
|
1058
|
+
at: steer.at,
|
|
1059
|
+
requestId: `${agentId}:${steer.at}`,
|
|
1060
|
+
delivered: steer.delivered
|
|
969
1061
|
}));
|
|
970
|
-
if (child !== null) {
|
|
971
|
-
tokensIn = (tokensIn ?? 0) + child.tokensIn;
|
|
972
|
-
tokensOut = (tokensOut ?? 0) + child.tokensOut;
|
|
973
|
-
cacheRead = (cacheRead ?? 0) + child.cacheRead;
|
|
974
|
-
cacheWrite = (cacheWrite ?? 0) + child.cacheWrite;
|
|
975
|
-
transcriptRef = child.path;
|
|
976
|
-
}
|
|
977
1062
|
}
|
|
1063
|
+
if (endAt !== null) events.push(line({
|
|
1064
|
+
kind: "finished",
|
|
1065
|
+
label,
|
|
1066
|
+
at: endAt,
|
|
1067
|
+
agentId,
|
|
1068
|
+
...child?.finalReport === null || child?.finalReport === void 0 ? {} : { evidence: child.finalReport }
|
|
1069
|
+
}));
|
|
978
1070
|
workers.push({
|
|
1071
|
+
workerId: agentId,
|
|
979
1072
|
label,
|
|
980
1073
|
events: events.length === 0 ? null : `${events.join("\n")}\n`,
|
|
981
1074
|
inbox: inbox.length === 0 ? null : `${inbox.join("\n")}\n`,
|
|
982
1075
|
patchBytes: null,
|
|
983
|
-
transcriptRef,
|
|
1076
|
+
transcriptRef: child?.path ?? null,
|
|
984
1077
|
patchPath: null,
|
|
985
|
-
tokensIn,
|
|
986
|
-
tokensOut,
|
|
987
|
-
cacheRead,
|
|
988
|
-
cacheWrite
|
|
1078
|
+
tokensIn: child?.tokensIn ?? null,
|
|
1079
|
+
tokensOut: child?.tokensOut ?? null,
|
|
1080
|
+
cacheRead: child?.cacheRead ?? null,
|
|
1081
|
+
cacheWrite: child?.cacheWrite ?? null
|
|
989
1082
|
});
|
|
990
1083
|
}
|
|
991
1084
|
const joined = children.length;
|
|
@@ -1027,6 +1120,7 @@ async function readClaudeCodeSupervisorRun(opts) {
|
|
|
1027
1120
|
harnessMissingReason,
|
|
1028
1121
|
limits: {
|
|
1029
1122
|
...limits,
|
|
1123
|
+
workerTokens: spawns.length === 0 ? null : harnessMissingReason,
|
|
1030
1124
|
deliverables: NO_DELIVERABLES
|
|
1031
1125
|
},
|
|
1032
1126
|
rootTranscriptRef: opts.transcriptPath,
|
|
@@ -1100,7 +1194,7 @@ function renderSupervisorRunMarkdown(r) {
|
|
|
1100
1194
|
out.push(`| Spawn waves | ${showMeasured(o.waves)} |`);
|
|
1101
1195
|
out.push(`| Wave sizes | ${isUnavailable(o.waveSizes) ? showMeasured(o.waveSizes) : `[${o.waveSizes.join(", ")}]`} |`);
|
|
1102
1196
|
out.push(`| Max concurrency | ${showMeasured(o.maxConcurrency)} |`);
|
|
1103
|
-
out.push(`| Respawns (
|
|
1197
|
+
out.push(`| Respawns (after same parent's first settle) | ${showMeasured(o.respawns)} |`);
|
|
1104
1198
|
out.push(`| Repeated labels | ${isUnavailable(o.repeatedLabels) ? showMeasured(o.repeatedLabels) : o.repeatedLabels.length === 0 ? "none" : o.repeatedLabels.join(", ")} |`);
|
|
1105
1199
|
out.push(`| Delegation depth | ${showMeasured(o.delegationDepth)} |`);
|
|
1106
1200
|
out.push(`| Time to first spawn | ${fmtMs(o.timeToFirstSpawnMs)} |`);
|
|
@@ -1111,9 +1205,9 @@ function renderSupervisorRunMarkdown(r) {
|
|
|
1111
1205
|
if (!isUnavailable(o.steersByWorker) && o.steersByWorker.length > 0) {
|
|
1112
1206
|
out.push("### Steers per worker");
|
|
1113
1207
|
out.push("");
|
|
1114
|
-
out.push("| Worker | Queued | Delivered |");
|
|
1115
|
-
out.push("
|
|
1116
|
-
for (const s of o.steersByWorker) out.push(`| \`${s.worker}\` | ${s.queued} | ${s.delivered} |`);
|
|
1208
|
+
out.push("| Worker id | Label | Queued | Delivered |");
|
|
1209
|
+
out.push("|---|---|---:|---:|");
|
|
1210
|
+
for (const s of o.steersByWorker) out.push(`| ${s.workerId === null ? "unavailable — legacy label join" : `\`${s.workerId}\``} | \`${s.worker}\` | ${s.queued} | ${s.delivered} |`);
|
|
1117
1211
|
out.push("");
|
|
1118
1212
|
} else if (isUnavailable(o.steersByWorker)) out.push(`### Steers per worker\n\nunavailable — ${o.steersByWorker.unavailable}\n`);
|
|
1119
1213
|
out.push("## Decision quality");
|
|
@@ -1126,7 +1220,7 @@ function renderSupervisorRunMarkdown(r) {
|
|
|
1126
1220
|
out.push(`| Rejected (verify red) | ${showMeasured(d.rejected)} |`);
|
|
1127
1221
|
out.push(`| Empty pass (green, no patch) | ${showMeasured(d.emptyPass)} |`);
|
|
1128
1222
|
out.push(`| Evidence → respawn sequences | ${showMeasured(d.observeThenRespawn)} |`);
|
|
1129
|
-
out.push(`| Respawn with no settled evidence in front | ${showMeasured(d.respawnWithoutEvidence)} |`);
|
|
1223
|
+
out.push(`| Respawn with no same-parent settled evidence in front | ${showMeasured(d.respawnWithoutEvidence)} |`);
|
|
1130
1224
|
out.push(`| Review actions (steers + worker questions) | ${showMeasured(d.reviewActions)} |`);
|
|
1131
1225
|
out.push(`| Worker evidence returned | ${isUnavailable(d.workerEvidenceBytes) ? showMeasured(d.workerEvidenceBytes) : `${d.workerEvidenceBytes} bytes`} |`);
|
|
1132
1226
|
out.push("");
|
|
@@ -1148,9 +1242,9 @@ function renderSupervisorRunMarkdown(r) {
|
|
|
1148
1242
|
}
|
|
1149
1243
|
out.push("");
|
|
1150
1244
|
if (!isUnavailable(e.perWorker) && e.perWorker.length > 0) {
|
|
1151
|
-
out.push("| Worker | Wall | Tokens in | Tokens out | Patch bytes | Verify passed |");
|
|
1152
|
-
out.push("
|
|
1153
|
-
for (const w of e.perWorker) out.push(`| \`${w.worker}\` | ${w.wallMs === null ? "unavailable — no start/finish pair" : fmtMs(w.wallMs)} | ${w.tokensIn ?? "unavailable — store does not attribute tokens per worker"} | ${w.tokensOut ?? "unavailable — store does not attribute tokens per worker"} | ${w.patchBytes ?? "unavailable — no worker patch file"} | ${w.passed === null ? "unavailable — no
|
|
1245
|
+
out.push("| Worker id | Label | Role | Wall | Tokens in | Tokens out | Patch bytes | Verify passed | Score |");
|
|
1246
|
+
out.push("|---|---|---|---:|---:|---:|---:|---|---:|");
|
|
1247
|
+
for (const w of e.perWorker) out.push(`| ${w.workerId === null ? "unavailable — legacy label join" : `\`${w.workerId}\``} | \`${w.worker}\` | ${w.role ?? "unavailable — no journal join"} | ${w.wallMs === null ? "unavailable — no start/finish pair" : fmtMs(w.wallMs)} | ${w.tokensIn ?? "unavailable — store does not attribute tokens per worker"} | ${w.tokensOut ?? "unavailable — store does not attribute tokens per worker"} | ${w.patchBytes ?? "unavailable — no worker patch file"} | ${w.passed === null ? "unavailable — no verdict" : String(w.passed)} | ${w.score ?? "unavailable — no numeric score"} |`);
|
|
1154
1248
|
out.push("");
|
|
1155
1249
|
}
|
|
1156
1250
|
out.push("## Outcome");
|
|
@@ -1237,9 +1331,12 @@ async function readLoopsSupervisorRun(runDir, opts = {}) {
|
|
|
1237
1331
|
const supRunDir = await findSupervisorRunDirIn(ws);
|
|
1238
1332
|
const result = await readMaybe(join(runDir, "result.json"));
|
|
1239
1333
|
const resultObj = parseJson(result);
|
|
1334
|
+
const journal = supRunDir === null ? null : await readMaybe(join(supRunDir, "journal.jsonl"));
|
|
1335
|
+
const journalWorkerSpawns = parseJsonl(journal).filter((event) => event.kind === "spawned" && typeof event.parent === "string" && event.role !== "supervisor").length;
|
|
1240
1336
|
let workers = null;
|
|
1241
1337
|
let workersMissingReason = null;
|
|
1242
1338
|
const workerCwds = [];
|
|
1339
|
+
let workerStarts = 0;
|
|
1243
1340
|
if (supRunDir === null) workersMissingReason = `no supervisor run dir under ${join(ws, ".loops", "supervisor")}`;
|
|
1244
1341
|
else {
|
|
1245
1342
|
const workersDir = join(supRunDir, "workers");
|
|
@@ -1252,18 +1349,25 @@ async function readLoopsSupervisorRun(runDir, opts = {}) {
|
|
|
1252
1349
|
const events = await readMaybe(join(workersDir, `${label}.ndjson`));
|
|
1253
1350
|
const inbox = await readMaybe(join(workersDir, `${label}.inbox.ndjson`));
|
|
1254
1351
|
const patch = await readMaybe(join(workersDir, `${label}.patch`));
|
|
1352
|
+
const startedRows = parseJsonl(events).filter((event) => event.kind === "started");
|
|
1353
|
+
workerStarts += startedRows.length;
|
|
1354
|
+
const startedIds = startedRows.map((event) => typeof event.workerId === "string" ? event.workerId : typeof event.agentId === "string" ? event.agentId : null).filter((id) => id !== null);
|
|
1355
|
+
const distinctStartedIds = new Set(startedIds);
|
|
1356
|
+
const workerId = startedRows.length > 0 && startedIds.length === startedRows.length && distinctStartedIds.size === 1 ? startedIds[0] : void 0;
|
|
1255
1357
|
workers.push({
|
|
1358
|
+
...workerId === void 0 ? {} : { workerId },
|
|
1256
1359
|
label,
|
|
1257
1360
|
events,
|
|
1258
1361
|
inbox,
|
|
1259
1362
|
patchBytes: patch === null ? null : Buffer.byteLength(patch)
|
|
1260
1363
|
});
|
|
1261
|
-
for (const ev of
|
|
1364
|
+
for (const ev of startedRows) if (ev.kind === "started" && typeof ev.cwd === "string") workerCwds.push(ev.cwd);
|
|
1262
1365
|
}
|
|
1263
1366
|
}
|
|
1264
1367
|
}
|
|
1265
1368
|
let harnessWorkerTokens = null;
|
|
1266
1369
|
let harnessMissingReason = null;
|
|
1370
|
+
let workerCwdsWithoutSessions = 0;
|
|
1267
1371
|
if (opts.opencodeDb === null) harnessMissingReason = "opencode join disabled";
|
|
1268
1372
|
else if (workerCwds.length === 0) harnessMissingReason = "no worker clone cwds in workers/*.ndjson (nothing to join)";
|
|
1269
1373
|
else {
|
|
@@ -1274,12 +1378,17 @@ async function readLoopsSupervisorRun(runDir, opts = {}) {
|
|
|
1274
1378
|
let sessions = 0;
|
|
1275
1379
|
let input = 0;
|
|
1276
1380
|
let output = 0;
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1381
|
+
const distinctWorkerCwds = new Set(workerCwds);
|
|
1382
|
+
for (const cwd of distinctWorkerCwds) {
|
|
1383
|
+
const rows = findOpencodeSessionsByDirectory(db, cwd);
|
|
1384
|
+
if (rows.length === 0) workerCwdsWithoutSessions += 1;
|
|
1385
|
+
for (const row of rows) {
|
|
1386
|
+
if (seen.has(row.id)) continue;
|
|
1387
|
+
seen.add(row.id);
|
|
1388
|
+
sessions += 1;
|
|
1389
|
+
input += row.tokensInput;
|
|
1390
|
+
output += row.tokensOutput + row.tokensReasoning;
|
|
1391
|
+
}
|
|
1283
1392
|
}
|
|
1284
1393
|
harnessWorkerTokens = {
|
|
1285
1394
|
store: "opencode",
|
|
@@ -1287,10 +1396,17 @@ async function readLoopsSupervisorRun(runDir, opts = {}) {
|
|
|
1287
1396
|
input,
|
|
1288
1397
|
output
|
|
1289
1398
|
};
|
|
1399
|
+
if (workerCwdsWithoutSessions > 0) harnessMissingReason = `${workerCwdsWithoutSessions}/${distinctWorkerCwds.size} worker clone cwds have no opencode session`;
|
|
1290
1400
|
} finally {
|
|
1291
1401
|
db.close();
|
|
1292
1402
|
}
|
|
1293
1403
|
}
|
|
1404
|
+
const workerInvocations = Math.max(journalWorkerSpawns, workerStarts);
|
|
1405
|
+
const workerTokenGaps = [];
|
|
1406
|
+
if (workerCwds.length < workerInvocations) workerTokenGaps.push(`${workerInvocations - workerCwds.length}/${workerInvocations} worker invocations have no clone cwd for the opencode token join`);
|
|
1407
|
+
if (workerInvocations > 0 && harnessWorkerTokens === null) workerTokenGaps.push(harnessMissingReason ?? "worker harness token join unavailable");
|
|
1408
|
+
else if (workerCwdsWithoutSessions > 0 && harnessMissingReason !== null) workerTokenGaps.push(harnessMissingReason);
|
|
1409
|
+
const workerTokenLimit = workerTokenGaps.length === 0 ? null : workerTokenGaps.join("; ");
|
|
1294
1410
|
const patchPath = opts.patchPath ?? (typeof resultObj?.patchPath === "string" ? resultObj.patchPath : null);
|
|
1295
1411
|
let judge = await readMaybe(join(runDir, "judge.json"));
|
|
1296
1412
|
let judgeSource = judge === null ? null : join(runDir, "judge.json");
|
|
@@ -1306,7 +1422,7 @@ async function readLoopsSupervisorRun(runDir, opts = {}) {
|
|
|
1306
1422
|
instanceId: typeof resultObj?.iid === "string" ? resultObj.iid : instanceIdFromPath(runDir),
|
|
1307
1423
|
arm: typeof resultObj?.arm === "string" ? resultObj.arm : basename(runDir),
|
|
1308
1424
|
supRunDir,
|
|
1309
|
-
journal
|
|
1425
|
+
journal,
|
|
1310
1426
|
brainLog: supRunDir === null ? null : await readMaybe(join(supRunDir, "brain.jsonl")),
|
|
1311
1427
|
state: supRunDir === null ? null : await readMaybe(join(supRunDir, "state.json")),
|
|
1312
1428
|
progress: supRunDir === null ? null : await readMaybe(join(supRunDir, "progress.ndjson")),
|
|
@@ -1319,7 +1435,10 @@ async function readLoopsSupervisorRun(runDir, opts = {}) {
|
|
|
1319
1435
|
driverLog: await readMaybe(join(runDir, "driver.log")),
|
|
1320
1436
|
harnessWorkerTokens,
|
|
1321
1437
|
harnessMissingReason,
|
|
1322
|
-
limits:
|
|
1438
|
+
limits: {
|
|
1439
|
+
...NO_SOURCE_LIMITS,
|
|
1440
|
+
workerTokens: workerTokenLimit
|
|
1441
|
+
},
|
|
1323
1442
|
traceCommand: null
|
|
1324
1443
|
};
|
|
1325
1444
|
}
|
|
@@ -1465,8 +1584,8 @@ async function findSupervisorRunDirs(root) {
|
|
|
1465
1584
|
* The supervision tree as `tangle.rollout.v1` rows.
|
|
1466
1585
|
*
|
|
1467
1586
|
* A supervisor run IS a tree of rollouts, so its nodes are not a new shape:
|
|
1468
|
-
* the root becomes one `RolloutLine` with `role: 'supervisor'`, every
|
|
1469
|
-
*
|
|
1587
|
+
* the root becomes one `RolloutLine` with `role: 'supervisor'`, and every
|
|
1588
|
+
* spawned invocation keeps its explicit supervisor/worker role with
|
|
1470
1589
|
* `parent_rollout_id` pointing at its spawner. The rows append to the same
|
|
1471
1590
|
* ledger as solo-agent rollouts and join to them with the same keys.
|
|
1472
1591
|
*
|
|
@@ -1493,6 +1612,9 @@ const EMPTY_COST = {
|
|
|
1493
1612
|
cache_write: null,
|
|
1494
1613
|
wall_s: null
|
|
1495
1614
|
};
|
|
1615
|
+
function hasWorkerId(worker) {
|
|
1616
|
+
return worker.workerId !== void 0;
|
|
1617
|
+
}
|
|
1496
1618
|
/**
|
|
1497
1619
|
* Mint the supervision tree as rollout rows. Returns the rows plus the gaps
|
|
1498
1620
|
* that made any of them incomplete — same unavailable-vs-zero discipline as
|
|
@@ -1578,10 +1700,10 @@ function supervisorRunRolloutLines(src, opts = {}) {
|
|
|
1578
1700
|
cost: {
|
|
1579
1701
|
...EMPTY_COST,
|
|
1580
1702
|
usd: src.limits.spendUsd !== null ? null : typeof stateResult.spentUsd === "number" ? stateResult.spentUsd : tree.brain.usd,
|
|
1581
|
-
tokens_in: tree.brain.tokensIn,
|
|
1582
|
-
tokens_out: tree.brain.tokensOut,
|
|
1583
|
-
cache_read: tree.brain.hasCache ? tree.brain.cacheRead : null,
|
|
1584
|
-
cache_write: tree.brain.hasCache ? tree.brain.cacheWrite : null,
|
|
1703
|
+
tokens_in: src.limits.managerTokens === null ? tree.brain.tokensIn : null,
|
|
1704
|
+
tokens_out: src.limits.managerTokens === null ? tree.brain.tokensOut : null,
|
|
1705
|
+
cache_read: src.limits.managerTokens === null && tree.brain.hasCache ? tree.brain.cacheRead : null,
|
|
1706
|
+
cache_write: src.limits.managerTokens === null && tree.brain.hasCache ? tree.brain.cacheWrite : null,
|
|
1585
1707
|
wall_s: wallMs === null ? null : wallMs / 1e3
|
|
1586
1708
|
},
|
|
1587
1709
|
artifacts: {
|
|
@@ -1597,34 +1719,40 @@ function supervisorRunRolloutLines(src, opts = {}) {
|
|
|
1597
1719
|
});
|
|
1598
1720
|
} else gaps.push("tree.root: no parentless `spawned` event in the journal");
|
|
1599
1721
|
const closeById = new Map(tree.closes.map((c) => [c.id, c]));
|
|
1600
|
-
const
|
|
1722
|
+
const sourceById = new Map((src.workers ?? []).filter(hasWorkerId).map((worker) => [worker.workerId, worker]));
|
|
1723
|
+
const fallbackSourceByLabel = new Map((src.workers ?? []).filter((worker) => worker.workerId === void 0).map((worker) => [worker.label, worker]));
|
|
1601
1724
|
for (const spawn of tree.workerSpawns) {
|
|
1602
1725
|
const close = closeById.get(spawn.id) ?? null;
|
|
1603
|
-
const
|
|
1604
|
-
const
|
|
1726
|
+
const workerSource = sourceById.get(spawn.id) ?? fallbackSourceByLabel.get(spawn.label) ?? null;
|
|
1727
|
+
const facts = tree.workerLogs.get(workerSource?.workerId ?? workerSource?.label ?? spawn.label) ?? null;
|
|
1605
1728
|
const wallMs = facts?.started != null && facts.finishedAt != null ? facts.finishedAt - facts.started : null;
|
|
1606
|
-
const
|
|
1607
|
-
|
|
1729
|
+
const score = close?.score ?? facts?.score ?? null;
|
|
1730
|
+
const passed = close?.valid ?? facts?.passed ?? null;
|
|
1731
|
+
const reward = score ?? (passed === null ? null : passed ? 1 : 0);
|
|
1732
|
+
if (reward === null) gaps.push(`child ${spawn.label}: no verify verdict (child logs absent or unfinished)`);
|
|
1733
|
+
const isSupervisor = spawn.role === "supervisor";
|
|
1608
1734
|
nodes.push({
|
|
1609
1735
|
...base,
|
|
1610
1736
|
rollout_id: spawn.id,
|
|
1611
1737
|
parent_rollout_id: spawn.parent,
|
|
1612
|
-
role:
|
|
1738
|
+
role: spawn.role,
|
|
1613
1739
|
policy: {
|
|
1614
|
-
harness: opts.workerHarness ?? null,
|
|
1740
|
+
harness: isSupervisor ? opts.supervisorHarness ?? null : opts.workerHarness ?? null,
|
|
1615
1741
|
harness_version: null,
|
|
1616
|
-
model: opts.workerModel ?? null,
|
|
1742
|
+
model: isSupervisor ? opts.supervisorModel ?? null : opts.workerModel ?? null,
|
|
1617
1743
|
provider: null,
|
|
1618
1744
|
profile_commit: null,
|
|
1619
1745
|
sampling: null
|
|
1620
1746
|
},
|
|
1621
1747
|
outcome: {
|
|
1622
1748
|
...unscreenedRewardFields(reward),
|
|
1623
|
-
reward_source: reward === null ? null : "worker-self-verify",
|
|
1749
|
+
reward_source: reward === null ? null : score === null ? "worker-self-verify" : "worker-verdict-score",
|
|
1624
1750
|
verdict: close === null ? null : {
|
|
1625
1751
|
kind: close.kind,
|
|
1626
1752
|
status: close.status,
|
|
1627
|
-
verdict: close.
|
|
1753
|
+
verdict: close.rawVerdict,
|
|
1754
|
+
valid: close.valid,
|
|
1755
|
+
score: close.score
|
|
1628
1756
|
},
|
|
1629
1757
|
metrics: {
|
|
1630
1758
|
label: spawn.label,
|
|
@@ -1633,7 +1761,7 @@ function supervisorRunRolloutLines(src, opts = {}) {
|
|
|
1633
1761
|
started_at: facts?.started ?? null,
|
|
1634
1762
|
finished_at: facts?.finishedAt ?? null,
|
|
1635
1763
|
wall_ms: wallMs,
|
|
1636
|
-
patch_bytes: facts?.finishedPatchBytes ?? null,
|
|
1764
|
+
patch_bytes: workerSource?.patchBytes ?? facts?.finishedPatchBytes ?? null,
|
|
1637
1765
|
evidence_bytes: facts?.evidenceBytes ?? null,
|
|
1638
1766
|
steers_queued: facts?.steersQueued ?? null,
|
|
1639
1767
|
steers_delivered: facts?.steersDelivered ?? null,
|
|
@@ -1676,4 +1804,4 @@ function supervisorRunRolloutLines(src, opts = {}) {
|
|
|
1676
1804
|
//#endregion
|
|
1677
1805
|
export { NO_SOURCE_LIMITS as C, showMeasured as D, isUnavailable as E, unavailable as O, rollupSupervisorRuns as S, SUPERVISOR_RUN_SCHEMA as T, claudeCodeSupervisorRunReader as _, loopsSupervisorRunReader as a, parsePatch as b, supervisorReportStem as c, renderSupervisorRollupMarkdown as d, renderSupervisorRunHeadline as f, DEFAULT_STEER_TOOLS as g, DEFAULT_SPAWN_TOOLS as h, findSupervisorRunDirs as i, writeSupervisorRunReport as l, DEFAULT_CANCEL_TOOLS as m, analyzeSupervisorRun as n, readLoopsSupervisorRun as o, renderSupervisorRunMarkdown as p, findSupervisorRunDirIn as r, reportSupervisorRound as s, supervisorRunRolloutLines as t, writeSupervisorRunReportSafe as u, readClaudeCodeSupervisorRun as v, SUPERVISOR_RUN_ROLLUP_SCHEMA as w, parseSupervisorTree as x, analyzeSupervisorRunSources as y };
|
|
1678
1806
|
|
|
1679
|
-
//# sourceMappingURL=supervisor-run-
|
|
1807
|
+
//# sourceMappingURL=supervisor-run-Dr5HnTup.js.map
|