@tangle-network/agent-eval 0.124.0 → 0.126.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +60 -35
- package/README.md +270 -189
- package/dist/analyst/index.d.ts +15 -145
- package/dist/analyst/index.js +33 -47
- package/dist/analyst/index.js.map +1 -1
- package/dist/benchmarks/index.d.ts +45 -162
- package/dist/benchmarks/index.js +8 -9
- package/dist/campaign/index.d.ts +3655 -5365
- package/dist/campaign/index.js +21 -95
- package/dist/{chunk-R226UZOI.js → chunk-474LBSOX.js} +2 -2
- package/dist/{chunk-W5B3ZGP3.js → chunk-4B7ZZHPX.js} +8 -6
- package/dist/{chunk-W5B3ZGP3.js.map → chunk-4B7ZZHPX.js.map} +1 -1
- package/dist/{chunk-DT7OXY3C.js → chunk-CM4OILD2.js} +535 -846
- package/dist/chunk-CM4OILD2.js.map +1 -0
- package/dist/{chunk-HM6V7F3M.js → chunk-FO7HEH76.js} +3 -3
- package/dist/chunk-IILEIWGW.js +635 -0
- package/dist/chunk-IILEIWGW.js.map +1 -0
- package/dist/{chunk-EQUK3RFS.js → chunk-J5SQWP6Y.js} +8 -5
- package/dist/chunk-J5SQWP6Y.js.map +1 -0
- package/dist/chunk-KO2PZOGP.js +4637 -0
- package/dist/chunk-KO2PZOGP.js.map +1 -0
- package/dist/{chunk-4Y7AAATF.js → chunk-LKKT3IVV.js} +574 -81
- package/dist/chunk-LKKT3IVV.js.map +1 -0
- package/dist/chunk-M7AH34KV.js +155 -0
- package/dist/chunk-M7AH34KV.js.map +1 -0
- package/dist/chunk-NTOV7RU5.js +7152 -0
- package/dist/chunk-NTOV7RU5.js.map +1 -0
- package/dist/{chunk-QFQZ3U3X.js → chunk-OCFJACJU.js} +2 -2
- package/dist/{chunk-GID26AN4.js → chunk-P22LJ3Y2.js} +4 -6
- package/dist/{chunk-GID26AN4.js.map → chunk-P22LJ3Y2.js.map} +1 -1
- package/dist/{chunk-SJT4OBVL.js → chunk-SDPM6554.js} +3 -3
- package/dist/{chunk-D5JZ7UDZ.js → chunk-UCLVDLCH.js} +136 -50
- package/dist/chunk-UCLVDLCH.js.map +1 -0
- package/dist/chunk-UI4YMIN2.js +105 -0
- package/dist/chunk-UI4YMIN2.js.map +1 -0
- package/dist/chunk-VBQ3CRKH.js +291 -0
- package/dist/chunk-VBQ3CRKH.js.map +1 -0
- package/dist/{chunk-JKDNAOF5.js → chunk-W4L6C2XT.js} +2 -2
- package/dist/{chunk-GRCDRKII.js → chunk-WS3NZZQQ.js} +58 -20
- package/dist/chunk-WS3NZZQQ.js.map +1 -0
- package/dist/cli.js +3 -3
- package/dist/contract/index.d.ts +3221 -3094
- package/dist/contract/index.js +173 -42
- package/dist/contract/index.js.map +1 -1
- package/dist/control.js +2 -3
- package/dist/fuzz.d.ts +14 -1
- package/dist/fuzz.js +1 -1
- package/dist/hosted/index.d.ts +8 -1
- package/dist/index.d.ts +208 -690
- package/dist/index.js +185 -500
- package/dist/index.js.map +1 -1
- package/dist/openapi.json +1 -1
- package/dist/rl.d.ts +5 -100
- package/dist/rl.js +4 -5
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +9 -1
- package/dist/rollout/index.js +6 -6
- package/dist/{run-campaign-I3JXKVAK.js → run-campaign-LVFKZCEU.js} +3 -3
- package/dist/supervisor-run/index.d.ts +156 -4
- package/dist/supervisor-run/index.js +14 -2
- package/dist/traces.js +2 -3
- package/dist/wire/index.d.ts +14 -1
- package/dist/wire/index.js +3 -3
- package/docs/campaign-proposers.md +363 -168
- package/docs/design/loop-taxonomy.md +142 -190
- package/docs/design.md +1 -1
- package/docs/distributed-driver.md +8 -11
- package/docs/feature-guide.md +20 -19
- package/docs/knowledge-readiness.md +2 -5
- package/docs/multi-shot-optimization.md +35 -27
- package/docs/rollout.md +5 -5
- package/package.json +4 -4
- package/dist/chunk-4Y7AAATF.js.map +0 -1
- package/dist/chunk-5PVZVCZB.js +0 -9190
- package/dist/chunk-5PVZVCZB.js.map +0 -1
- package/dist/chunk-A6GT67HT.js +0 -550
- package/dist/chunk-A6GT67HT.js.map +0 -1
- package/dist/chunk-D5JZ7UDZ.js.map +0 -1
- package/dist/chunk-DT7OXY3C.js.map +0 -1
- package/dist/chunk-EQUK3RFS.js.map +0 -1
- package/dist/chunk-GC4ATIKK.js +0 -317
- package/dist/chunk-GC4ATIKK.js.map +0 -1
- package/dist/chunk-GRCDRKII.js.map +0 -1
- package/dist/chunk-LOW3U7JZ.js +0 -328
- package/dist/chunk-LOW3U7JZ.js.map +0 -1
- package/dist/chunk-MGGFVCJ7.js +0 -288
- package/dist/chunk-MGGFVCJ7.js.map +0 -1
- package/dist/chunk-PMITBABE.js +0 -3841
- package/dist/chunk-PMITBABE.js.map +0 -1
- package/dist/chunk-R7ZRE2KV.js +0 -138
- package/dist/chunk-R7ZRE2KV.js.map +0 -1
- /package/dist/{chunk-R226UZOI.js.map → chunk-474LBSOX.js.map} +0 -0
- /package/dist/{chunk-HM6V7F3M.js.map → chunk-FO7HEH76.js.map} +0 -0
- /package/dist/{chunk-QFQZ3U3X.js.map → chunk-OCFJACJU.js.map} +0 -0
- /package/dist/{chunk-SJT4OBVL.js.map → chunk-SDPM6554.js.map} +0 -0
- /package/dist/{chunk-JKDNAOF5.js.map → chunk-W4L6C2XT.js.map} +0 -0
- /package/dist/{run-campaign-I3JXKVAK.js.map → run-campaign-LVFKZCEU.js.map} +0 -0
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import {
|
|
2
2
|
DEFAULT_OPENCODE_DB,
|
|
3
3
|
findOpencodeSessionsByDirectory,
|
|
4
|
-
openOpencodeDb
|
|
5
|
-
|
|
4
|
+
openOpencodeDb,
|
|
5
|
+
parseClaudeEntries,
|
|
6
|
+
transcriptFromEntries
|
|
7
|
+
} from "./chunk-VBQ3CRKH.js";
|
|
6
8
|
import {
|
|
7
9
|
ROLLOUT_SCHEMA,
|
|
8
10
|
isRolloutLine
|
|
@@ -20,6 +22,11 @@ function showMeasured(v) {
|
|
|
20
22
|
if (v === null) return "null";
|
|
21
23
|
return String(v);
|
|
22
24
|
}
|
|
25
|
+
var NO_SOURCE_LIMITS = {
|
|
26
|
+
spendUsd: null,
|
|
27
|
+
workerVerdicts: null,
|
|
28
|
+
deliverables: null
|
|
29
|
+
};
|
|
23
30
|
var SUPERVISOR_RUN_SCHEMA = "tangle.supervisor-run@1";
|
|
24
31
|
var SUPERVISOR_RUN_ROLLUP_SCHEMA = "tangle.supervisor-run-rollup@1";
|
|
25
32
|
|
|
@@ -33,13 +40,25 @@ function num(v) {
|
|
|
33
40
|
function readSpend(v) {
|
|
34
41
|
const rec = asRecord(v);
|
|
35
42
|
const tok = asRecord(rec.tokens);
|
|
36
|
-
|
|
43
|
+
const cacheRead = tok.cacheRead ?? tok.cache_read;
|
|
44
|
+
const cacheWrite = tok.cacheWrite ?? tok.cache_write;
|
|
45
|
+
return {
|
|
46
|
+
tokens: {
|
|
47
|
+
input: num(tok.input),
|
|
48
|
+
output: num(tok.output),
|
|
49
|
+
cacheRead: num(cacheRead),
|
|
50
|
+
cacheWrite: num(cacheWrite),
|
|
51
|
+
hasCache: cacheRead !== void 0 || cacheWrite !== void 0
|
|
52
|
+
},
|
|
53
|
+
usd: num(rec.usd)
|
|
54
|
+
};
|
|
37
55
|
}
|
|
56
|
+
var NO_CACHE_COUNTERS = "the journal carries no cache-token counters for this role";
|
|
38
57
|
function parseJsonl(text) {
|
|
39
58
|
if (text === null) return [];
|
|
40
59
|
const out = [];
|
|
41
|
-
for (const
|
|
42
|
-
const trimmed =
|
|
60
|
+
for (const line2 of text.split("\n")) {
|
|
61
|
+
const trimmed = line2.trim();
|
|
43
62
|
if (!trimmed) continue;
|
|
44
63
|
try {
|
|
45
64
|
const parsed = JSON.parse(trimmed);
|
|
@@ -70,6 +89,9 @@ function parseSupervisorTree(src) {
|
|
|
70
89
|
const closes = [];
|
|
71
90
|
let brainIn = 0;
|
|
72
91
|
let brainOut = 0;
|
|
92
|
+
let brainCacheRead = 0;
|
|
93
|
+
let brainCacheWrite = 0;
|
|
94
|
+
let brainHasCache = false;
|
|
73
95
|
let brainUsd = 0;
|
|
74
96
|
let meteredCount = 0;
|
|
75
97
|
let rootId = null;
|
|
@@ -88,7 +110,8 @@ function parseSupervisorTree(src) {
|
|
|
88
110
|
status: typeof ev.status === "string" ? ev.status : null,
|
|
89
111
|
verdict: typeof ev.verdict === "string" ? ev.verdict : null,
|
|
90
112
|
at: ms(ev.at),
|
|
91
|
-
spend: readSpend(ev.spent)
|
|
113
|
+
spend: readSpend(ev.spent),
|
|
114
|
+
hasSpend: asRecord(ev.spent).tokens !== void 0
|
|
92
115
|
});
|
|
93
116
|
} else if (kind === "cancelled") {
|
|
94
117
|
closes.push({
|
|
@@ -97,12 +120,19 @@ function parseSupervisorTree(src) {
|
|
|
97
120
|
status: "cancelled",
|
|
98
121
|
verdict: typeof ev.reason === "string" ? ev.reason : null,
|
|
99
122
|
at: ms(ev.at),
|
|
100
|
-
spend: {
|
|
123
|
+
spend: {
|
|
124
|
+
tokens: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, hasCache: false },
|
|
125
|
+
usd: 0
|
|
126
|
+
},
|
|
127
|
+
hasSpend: false
|
|
101
128
|
});
|
|
102
129
|
} else if (kind === "metered") {
|
|
103
130
|
const s = readSpend(ev.spend);
|
|
104
131
|
brainIn += s.tokens.input;
|
|
105
132
|
brainOut += s.tokens.output;
|
|
133
|
+
brainCacheRead += s.tokens.cacheRead;
|
|
134
|
+
brainCacheWrite += s.tokens.cacheWrite;
|
|
135
|
+
brainHasCache = brainHasCache || s.tokens.hasCache;
|
|
106
136
|
brainUsd += s.usd;
|
|
107
137
|
meteredCount += 1;
|
|
108
138
|
}
|
|
@@ -158,7 +188,15 @@ function parseSupervisorTree(src) {
|
|
|
158
188
|
closes,
|
|
159
189
|
workerSpawns,
|
|
160
190
|
workerCloses,
|
|
161
|
-
brain: {
|
|
191
|
+
brain: {
|
|
192
|
+
tokensIn: brainIn,
|
|
193
|
+
tokensOut: brainOut,
|
|
194
|
+
cacheRead: brainCacheRead,
|
|
195
|
+
cacheWrite: brainCacheWrite,
|
|
196
|
+
hasCache: brainHasCache,
|
|
197
|
+
usd: brainUsd,
|
|
198
|
+
meteredCount
|
|
199
|
+
},
|
|
162
200
|
workerLogs,
|
|
163
201
|
startedAt,
|
|
164
202
|
completedAt
|
|
@@ -292,6 +330,7 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
|
|
|
292
330
|
settledByStatus[key] = (settledByStatus[key] ?? 0) + 1;
|
|
293
331
|
if (c.verdict !== null) settledVerdicts[c.verdict] = (settledVerdicts[c.verdict] ?? 0) + 1;
|
|
294
332
|
}
|
|
333
|
+
const verdictLimit = src.limits.workerVerdicts;
|
|
295
334
|
let accepted = 0;
|
|
296
335
|
let rejected = 0;
|
|
297
336
|
let emptyPass = 0;
|
|
@@ -320,10 +359,10 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
|
|
|
320
359
|
}
|
|
321
360
|
const decision = {
|
|
322
361
|
settledByStatus: haveJournal ? settledByStatus : gap("settledByStatus", journalMissing),
|
|
323
|
-
settledVerdicts: haveJournal ? settledVerdicts : unavailable(journalMissing),
|
|
324
|
-
accepted: src.workers === null ? unavailable(workersGapReason) : accepted,
|
|
325
|
-
rejected: src.workers === null ? unavailable(workersGapReason) : rejected,
|
|
326
|
-
emptyPass: src.workers === null ? unavailable(workersGapReason) : emptyPass,
|
|
362
|
+
settledVerdicts: verdictLimit !== null ? unavailable(verdictLimit) : haveJournal ? settledVerdicts : unavailable(journalMissing),
|
|
363
|
+
accepted: verdictLimit !== null ? gap("accepted", verdictLimit) : src.workers === null ? unavailable(workersGapReason) : accepted,
|
|
364
|
+
rejected: verdictLimit !== null ? unavailable(verdictLimit) : src.workers === null ? unavailable(workersGapReason) : rejected,
|
|
365
|
+
emptyPass: verdictLimit !== null || src.limits.deliverables !== null ? unavailable(verdictLimit ?? src.limits.deliverables) : src.workers === null ? unavailable(workersGapReason) : emptyPass,
|
|
327
366
|
observeThenRespawn: haveJournal ? observeThenRespawn : unavailable(journalMissing),
|
|
328
367
|
respawnWithoutEvidence: haveJournal ? respawnWithoutEvidence : unavailable(journalMissing),
|
|
329
368
|
reviewActions: src.workers === null ? unavailable(workersGapReason) : steerQueuedTotal + upLegMessages,
|
|
@@ -332,21 +371,29 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
|
|
|
332
371
|
const journalWorkerIn = workerCloses.reduce((a, c) => a + c.spend.tokens.input, 0);
|
|
333
372
|
const journalWorkerOut = workerCloses.reduce((a, c) => a + c.spend.tokens.output, 0);
|
|
334
373
|
const journalWorkerUsd = workerCloses.reduce((a, c) => a + c.spend.usd, 0);
|
|
374
|
+
const labelById = new Map(tree.workerSpawns.map((s) => [s.id, s.label]));
|
|
375
|
+
const workerUsdByLabel = /* @__PURE__ */ new Map();
|
|
376
|
+
for (const c of workerCloses) {
|
|
377
|
+
const label = labelById.get(c.id);
|
|
378
|
+
if (label === void 0) continue;
|
|
379
|
+
workerUsdByLabel.set(label, (workerUsdByLabel.get(label) ?? 0) + c.spend.usd);
|
|
380
|
+
}
|
|
335
381
|
const sq = src.harnessWorkerTokens;
|
|
336
382
|
const harnessGapReason = src.harnessMissingReason ?? "harness session store unavailable and journal settled spend is 0";
|
|
337
383
|
const workerIn = sq !== null ? journalWorkerIn + sq.input : journalWorkerIn > 0 ? journalWorkerIn : gap("workers.tokensIn", harnessGapReason);
|
|
338
384
|
const workerOut = sq !== null ? journalWorkerOut + sq.output : journalWorkerOut > 0 ? journalWorkerOut : unavailable(harnessGapReason);
|
|
339
385
|
const stateResult = asRecord(state?.result);
|
|
340
386
|
const stateUsd = typeof stateResult.spentUsd === "number" ? stateResult.spentUsd : null;
|
|
341
|
-
const
|
|
387
|
+
const usdLimit = src.limits.spendUsd;
|
|
388
|
+
const totalUsd = usdLimit !== null ? gap("totalUsd", usdLimit) : stateUsd !== null ? round(stateUsd, 6) : haveJournal ? round(tree.brain.usd + journalWorkerUsd, 6) : gap("totalUsd", journalMissing);
|
|
342
389
|
const perWorker = (src.workers ?? []).map((w) => {
|
|
343
390
|
const f = tree.workerLogs.get(w.label);
|
|
344
391
|
return {
|
|
345
392
|
worker: w.label,
|
|
346
393
|
wallMs: f?.started != null && f.finishedAt != null ? f.finishedAt - f.started : null,
|
|
347
|
-
tokensIn:
|
|
348
|
-
tokensOut:
|
|
349
|
-
usd:
|
|
394
|
+
tokensIn: w.tokensIn ?? null,
|
|
395
|
+
tokensOut: w.tokensOut ?? null,
|
|
396
|
+
usd: usdLimit !== null ? null : workerUsdByLabel.get(w.label) ?? null,
|
|
350
397
|
patchBytes: w.patchBytes ?? f?.finishedPatchBytes ?? null,
|
|
351
398
|
passed: f?.passed ?? null
|
|
352
399
|
};
|
|
@@ -357,7 +404,9 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
|
|
|
357
404
|
brain: {
|
|
358
405
|
tokensIn: haveJournal ? tree.brain.tokensIn : gap("brain.tokensIn", journalMissing),
|
|
359
406
|
tokensOut: haveJournal ? tree.brain.tokensOut : unavailable(journalMissing),
|
|
360
|
-
usd: haveJournal ? round(tree.brain.usd, 6) : unavailable(journalMissing),
|
|
407
|
+
usd: usdLimit !== null ? unavailable(usdLimit) : haveJournal ? round(tree.brain.usd, 6) : unavailable(journalMissing),
|
|
408
|
+
cacheRead: !haveJournal ? unavailable(journalMissing) : tree.brain.hasCache ? tree.brain.cacheRead : unavailable(NO_CACHE_COUNTERS),
|
|
409
|
+
cacheWrite: !haveJournal ? unavailable(journalMissing) : tree.brain.hasCache ? tree.brain.cacheWrite : unavailable(NO_CACHE_COUNTERS),
|
|
361
410
|
source: haveJournal ? `journal metered events (n=${tree.brain.meteredCount})` : journalMissing
|
|
362
411
|
},
|
|
363
412
|
brainTruncations: src.brainLog === null ? gap(
|
|
@@ -367,11 +416,13 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
|
|
|
367
416
|
workers: {
|
|
368
417
|
tokensIn: workerIn,
|
|
369
418
|
tokensOut: workerOut,
|
|
370
|
-
|
|
419
|
+
cacheRead: sq?.cacheRead !== void 0 ? sq.cacheRead : unavailable(NO_CACHE_COUNTERS),
|
|
420
|
+
cacheWrite: sq?.cacheWrite !== void 0 ? sq.cacheWrite : unavailable(NO_CACHE_COUNTERS),
|
|
421
|
+
usd: usdLimit !== null ? unavailable(usdLimit) : haveJournal ? round(journalWorkerUsd, 6) : unavailable(journalMissing),
|
|
371
422
|
source: sq !== null ? `journal settled spend + ${sq.store} sessions (n=${sq.sessions})` : `journal settled spend only \u2014 ${src.harnessMissingReason ?? "harness session store unavailable"}`
|
|
372
423
|
},
|
|
373
424
|
totalUsd,
|
|
374
|
-
totalUsdSource: stateUsd !== null ? `state.json result.spentUsd${journalWorkerUsd === 0 ? " \u2014 brain-priced only; worker CLI inference is unpriced (see worker token counts)" : ""}` : haveJournal ? "journal metered + settled usd" : journalMissing,
|
|
425
|
+
totalUsdSource: usdLimit !== null ? usdLimit : stateUsd !== null ? `state.json result.spentUsd${journalWorkerUsd === 0 ? " \u2014 brain-priced only; worker CLI inference is unpriced (see worker token counts)" : ""}` : haveJournal ? "journal metered + settled usd" : journalMissing,
|
|
375
426
|
costPerAcceptedPatchUsd: isUnavailable(totalUsd) ? unavailable(totalUsd.unavailable) : isUnavailable(decision.accepted) ? unavailable(decision.accepted.unavailable) : decision.accepted === 0 ? unavailable("no accepted worker patch (cost has no denominator)") : round(totalUsd / decision.accepted, 6),
|
|
376
427
|
workerWallMsDistribution: walls.length === 0 ? unavailable(
|
|
377
428
|
src.workers === null ? workersGapReason : "no worker start/finish pairs captured"
|
|
@@ -385,7 +436,7 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
|
|
|
385
436
|
},
|
|
386
437
|
perWorker: src.workers === null ? unavailable(workersGapReason) : perWorker
|
|
387
438
|
};
|
|
388
|
-
const patchStats = src.patch === null ? gap("patch", "delivered patch file absent") : parsePatch(src.patch);
|
|
439
|
+
const patchStats = src.patch === null ? gap("patch", src.limits.deliverables ?? "delivered patch file absent") : parsePatch(src.patch);
|
|
389
440
|
const outcome = {
|
|
390
441
|
supStatus: pickString(state, "status") ?? pickString(result, "sup_status") ?? gap("supStatus", "no state.json / result.json status"),
|
|
391
442
|
supVerdict: pickString(state, "verdict") ?? pickString(result, "sup_verdict") ?? unavailable("no state.json / result.json verdict"),
|
|
@@ -411,13 +462,13 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
|
|
|
411
462
|
economics,
|
|
412
463
|
outcome,
|
|
413
464
|
gaps,
|
|
414
|
-
traceCommand: "npx --yes @tangle-network/traces@latest analyze --harness opencode --cwd <worker-clone-cwd>"
|
|
465
|
+
traceCommand: src.traceCommand ?? "npx --yes @tangle-network/traces@latest analyze --harness opencode --cwd <worker-clone-cwd>"
|
|
415
466
|
};
|
|
416
467
|
}
|
|
417
468
|
function registrationMentions(driverLog) {
|
|
418
469
|
let n = 0;
|
|
419
|
-
for (const
|
|
420
|
-
if (
|
|
470
|
+
for (const line2 of driverLog.split("\n")) {
|
|
471
|
+
if (line2.includes("registered tools:") && line2.includes("supervisor_steer")) n += 1;
|
|
421
472
|
}
|
|
422
473
|
return n;
|
|
423
474
|
}
|
|
@@ -439,20 +490,20 @@ function parsePatch(text) {
|
|
|
439
490
|
const testFiles = /* @__PURE__ */ new Set();
|
|
440
491
|
let added = 0;
|
|
441
492
|
let removed = 0;
|
|
442
|
-
for (const
|
|
443
|
-
if (
|
|
444
|
-
const p =
|
|
493
|
+
for (const line2 of text.split("\n")) {
|
|
494
|
+
if (line2.startsWith("+++ ")) {
|
|
495
|
+
const p = line2.slice(4).trim().replace(/^b\//, "");
|
|
445
496
|
if (p !== "/dev/null") {
|
|
446
497
|
files.add(p);
|
|
447
498
|
if (isTestPath(p)) testFiles.add(p);
|
|
448
499
|
}
|
|
449
500
|
continue;
|
|
450
501
|
}
|
|
451
|
-
if (
|
|
502
|
+
if (line2.startsWith("--- ") || line2.startsWith("diff --git") || line2.startsWith("index ")) {
|
|
452
503
|
continue;
|
|
453
504
|
}
|
|
454
|
-
if (
|
|
455
|
-
else if (
|
|
505
|
+
if (line2.startsWith("+")) added += 1;
|
|
506
|
+
else if (line2.startsWith("-")) removed += 1;
|
|
456
507
|
}
|
|
457
508
|
return {
|
|
458
509
|
files: files.size,
|
|
@@ -506,9 +557,426 @@ function rollupSupervisorRuns(reports) {
|
|
|
506
557
|
};
|
|
507
558
|
}
|
|
508
559
|
|
|
560
|
+
// src/supervisor-run/claude-code-reader.ts
|
|
561
|
+
import { readdir, readFile } from "fs/promises";
|
|
562
|
+
import { basename, dirname, join } from "path";
|
|
563
|
+
var DEFAULT_SPAWN_TOOLS = ["Agent", "Task"];
|
|
564
|
+
var DEFAULT_STEER_TOOLS = ["SendMessage"];
|
|
565
|
+
var DEFAULT_CANCEL_TOOLS = ["TaskStop", "KillAgent"];
|
|
566
|
+
var SPEND_UNPRICED = "Claude Code transcripts record token usage but never a price \u2014 usd is not in the store";
|
|
567
|
+
var NO_VERDICTS = "Claude Code runs no per-worker verify step \u2014 a subagent reports prose, not pass/fail";
|
|
568
|
+
var NO_DELIVERABLES = "Claude Code retains no per-worker patch \u2014 subagents commit to git, they do not hand back a diff";
|
|
569
|
+
var isRecord = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
|
|
570
|
+
var str = (v) => typeof v === "string" ? v : null;
|
|
571
|
+
function blockText(content) {
|
|
572
|
+
if (typeof content === "string") return content;
|
|
573
|
+
if (!Array.isArray(content)) return "";
|
|
574
|
+
const parts = [];
|
|
575
|
+
for (const b of content) {
|
|
576
|
+
if (isRecord(b) && b.type === "text" && typeof b.text === "string") parts.push(b.text);
|
|
577
|
+
}
|
|
578
|
+
return parts.join("\n");
|
|
579
|
+
}
|
|
580
|
+
var tag = (xml, name) => {
|
|
581
|
+
const m = xml.match(new RegExp(`<${name}>([\\s\\S]*?)</${name}>`));
|
|
582
|
+
return m === null ? null : m[1];
|
|
583
|
+
};
|
|
584
|
+
function parseNotifications(text, at) {
|
|
585
|
+
const out = [];
|
|
586
|
+
for (const m of text.matchAll(/<task-notification>[\s\S]*?<\/task-notification>/g)) {
|
|
587
|
+
const xml = m[0];
|
|
588
|
+
const taskId = tag(xml, "task-id");
|
|
589
|
+
if (taskId === null) continue;
|
|
590
|
+
out.push({
|
|
591
|
+
taskId: taskId.trim(),
|
|
592
|
+
toolUseId: tag(xml, "tool-use-id")?.trim() ?? null,
|
|
593
|
+
status: tag(xml, "status")?.trim() ?? "unknown",
|
|
594
|
+
summary: tag(xml, "summary")?.trim() ?? null,
|
|
595
|
+
at
|
|
596
|
+
});
|
|
597
|
+
}
|
|
598
|
+
return out;
|
|
599
|
+
}
|
|
600
|
+
function threadCalls(entries) {
|
|
601
|
+
const uses = [];
|
|
602
|
+
const results = /* @__PURE__ */ new Map();
|
|
603
|
+
const notifications = [];
|
|
604
|
+
let firstAt = null;
|
|
605
|
+
let lastAt = null;
|
|
606
|
+
for (const entry of entries) {
|
|
607
|
+
if (entry.timestamp !== null) {
|
|
608
|
+
if (firstAt === null) firstAt = entry.timestamp;
|
|
609
|
+
lastAt = entry.timestamp;
|
|
610
|
+
}
|
|
611
|
+
const content = entry.message.content;
|
|
612
|
+
if (entry.type === "assistant") {
|
|
613
|
+
if (!Array.isArray(content)) continue;
|
|
614
|
+
for (const block of content) {
|
|
615
|
+
if (!isRecord(block) || block.type !== "tool_use") continue;
|
|
616
|
+
const id = str(block.id);
|
|
617
|
+
const name = str(block.name);
|
|
618
|
+
if (id === null || name === null) continue;
|
|
619
|
+
uses.push({
|
|
620
|
+
id,
|
|
621
|
+
name,
|
|
622
|
+
input: isRecord(block.input) ? block.input : {},
|
|
623
|
+
at: entry.timestamp
|
|
624
|
+
});
|
|
625
|
+
}
|
|
626
|
+
continue;
|
|
627
|
+
}
|
|
628
|
+
if (typeof content === "string") {
|
|
629
|
+
notifications.push(...parseNotifications(content, entry.timestamp));
|
|
630
|
+
continue;
|
|
631
|
+
}
|
|
632
|
+
if (!Array.isArray(content)) continue;
|
|
633
|
+
for (const block of content) {
|
|
634
|
+
if (!isRecord(block)) continue;
|
|
635
|
+
if (block.type === "tool_result" && typeof block.tool_use_id === "string") {
|
|
636
|
+
results.set(block.tool_use_id, {
|
|
637
|
+
id: block.tool_use_id,
|
|
638
|
+
at: entry.timestamp,
|
|
639
|
+
structured: entry.toolUseResult,
|
|
640
|
+
text: blockText(block.content)
|
|
641
|
+
});
|
|
642
|
+
} else if (block.type === "text" && typeof block.text === "string") {
|
|
643
|
+
notifications.push(...parseNotifications(block.text, entry.timestamp));
|
|
644
|
+
}
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
return { uses, results, notifications, firstAt, lastAt };
|
|
648
|
+
}
|
|
649
|
+
function spawnedAgentId(result) {
|
|
650
|
+
if (result === void 0) return null;
|
|
651
|
+
if (isRecord(result.structured)) {
|
|
652
|
+
const id = str(result.structured.agentId);
|
|
653
|
+
if (id !== null) return id;
|
|
654
|
+
}
|
|
655
|
+
return result.text.match(/agentId:\s*([A-Za-z0-9_-]+)/)?.[1] ?? null;
|
|
656
|
+
}
|
|
657
|
+
async function readChildren(dir) {
|
|
658
|
+
const names = await readdir(dir).catch(() => null);
|
|
659
|
+
if (names === null) return [];
|
|
660
|
+
const out = [];
|
|
661
|
+
for (const name of names.filter((n) => n.endsWith(".jsonl")).sort()) {
|
|
662
|
+
const path = join(dir, name);
|
|
663
|
+
const raw = await readFile(path, "utf8").catch(() => null);
|
|
664
|
+
if (raw === null) continue;
|
|
665
|
+
const entries = parseClaudeEntries(raw);
|
|
666
|
+
const projected = transcriptFromEntries(entries, { includeSidechain: true });
|
|
667
|
+
const metaRaw = await readFile(path.replace(/\.jsonl$/, ".meta.json"), "utf8").catch(() => null);
|
|
668
|
+
let meta = {};
|
|
669
|
+
if (metaRaw !== null) {
|
|
670
|
+
try {
|
|
671
|
+
const parsed = JSON.parse(metaRaw);
|
|
672
|
+
if (isRecord(parsed)) meta = parsed;
|
|
673
|
+
} catch {
|
|
674
|
+
meta = {};
|
|
675
|
+
}
|
|
676
|
+
}
|
|
677
|
+
const agentId = entries.find((e) => e.agentId !== null)?.agentId ?? name.replace(/^agent-/, "").replace(/\.jsonl$/, "");
|
|
678
|
+
out.push({
|
|
679
|
+
agentId,
|
|
680
|
+
path,
|
|
681
|
+
description: str(meta.description),
|
|
682
|
+
spawnToolUseId: str(meta.toolUseId),
|
|
683
|
+
spawnDepth: typeof meta.spawnDepth === "number" ? meta.spawnDepth : null,
|
|
684
|
+
entries,
|
|
685
|
+
tokensIn: projected.usage.tokensIn,
|
|
686
|
+
tokensOut: projected.usage.tokensOut,
|
|
687
|
+
cacheRead: projected.usage.cacheRead,
|
|
688
|
+
cacheWrite: projected.usage.cacheWrite,
|
|
689
|
+
firstAt: projected.startedAt,
|
|
690
|
+
lastAt: projected.endedAt,
|
|
691
|
+
model: projected.model,
|
|
692
|
+
finalReport: [...projected.messages].reverse().find((m) => m.role === "assistant" && typeof m.content === "string")?.content ?? null
|
|
693
|
+
});
|
|
694
|
+
}
|
|
695
|
+
return out;
|
|
696
|
+
}
|
|
697
|
+
var line = (obj) => JSON.stringify(obj);
|
|
698
|
+
async function readClaudeCodeSupervisorRun(opts) {
|
|
699
|
+
const spawnTools = new Set(opts.spawnTools ?? DEFAULT_SPAWN_TOOLS);
|
|
700
|
+
const steerTools = new Set(opts.steerTools ?? DEFAULT_STEER_TOOLS);
|
|
701
|
+
const cancelTools = new Set(opts.cancelTools ?? DEFAULT_CANCEL_TOOLS);
|
|
702
|
+
const raw = await readFile(opts.transcriptPath, "utf8").catch(() => null);
|
|
703
|
+
const sessionId = basename(opts.transcriptPath).replace(/\.jsonl$/, "");
|
|
704
|
+
const runRef = opts.runRef ?? opts.transcriptPath;
|
|
705
|
+
const limits = { spendUsd: SPEND_UNPRICED, workerVerdicts: NO_VERDICTS, deliverables: null };
|
|
706
|
+
const traceCommand = `npx --yes @tangle-network/traces@latest analyze --harness claude-code --session ${sessionId}`;
|
|
707
|
+
if (raw === null) {
|
|
708
|
+
return {
|
|
709
|
+
runRef,
|
|
710
|
+
instanceId: opts.instanceId ?? sessionId,
|
|
711
|
+
arm: opts.arm ?? null,
|
|
712
|
+
supRunDir: null,
|
|
713
|
+
journal: null,
|
|
714
|
+
brainLog: null,
|
|
715
|
+
state: null,
|
|
716
|
+
progress: null,
|
|
717
|
+
workers: null,
|
|
718
|
+
workersMissingReason: `session transcript unreadable at ${opts.transcriptPath}`,
|
|
719
|
+
result: null,
|
|
720
|
+
judge: null,
|
|
721
|
+
judgeSource: null,
|
|
722
|
+
patch: null,
|
|
723
|
+
driverLog: null,
|
|
724
|
+
harnessWorkerTokens: null,
|
|
725
|
+
harnessMissingReason: `session transcript unreadable at ${opts.transcriptPath}`,
|
|
726
|
+
limits: { ...limits, deliverables: NO_DELIVERABLES },
|
|
727
|
+
traceCommand
|
|
728
|
+
};
|
|
729
|
+
}
|
|
730
|
+
const allEntries = parseClaudeEntries(raw);
|
|
731
|
+
const main = threadCalls(allEntries.filter((e) => !e.isSidechain));
|
|
732
|
+
const mainTranscript = transcriptFromEntries(allEntries);
|
|
733
|
+
const subagentsDir = opts.subagentsDir === void 0 ? join(dirname(opts.transcriptPath), sessionId, "subagents") : opts.subagentsDir;
|
|
734
|
+
const children = subagentsDir === null ? [] : await readChildren(subagentsDir);
|
|
735
|
+
const childByAgentId = new Map(children.map((c) => [c.agentId, c]));
|
|
736
|
+
const childBySpawnToolUseId = new Map(
|
|
737
|
+
children.filter((c) => c.spawnToolUseId !== null).map((c) => [c.spawnToolUseId, c])
|
|
738
|
+
);
|
|
739
|
+
const threads = [{ id: sessionId, calls: main }];
|
|
740
|
+
for (const child of children) {
|
|
741
|
+
threads.push({ id: child.agentId, calls: threadCalls(child.entries) });
|
|
742
|
+
}
|
|
743
|
+
const spawns = [];
|
|
744
|
+
const steersByTarget = /* @__PURE__ */ new Map();
|
|
745
|
+
const cancels = [];
|
|
746
|
+
const settles = [];
|
|
747
|
+
for (const thread of threads) {
|
|
748
|
+
for (const use of thread.calls.uses) {
|
|
749
|
+
if (spawnTools.has(use.name)) {
|
|
750
|
+
const result = thread.calls.results.get(use.id);
|
|
751
|
+
const agentId = spawnedAgentId(result) ?? childBySpawnToolUseId.get(use.id)?.agentId ?? null;
|
|
752
|
+
if (agentId === null) continue;
|
|
753
|
+
const label = str(use.input.description) ?? childByAgentId.get(agentId)?.description ?? agentId;
|
|
754
|
+
spawns.push({
|
|
755
|
+
agentId,
|
|
756
|
+
parentId: thread.id,
|
|
757
|
+
label,
|
|
758
|
+
// The spawn is complete when the launch call is answered; the tool_use
|
|
759
|
+
// instant is the fallback for a call that never got a result line.
|
|
760
|
+
at: result?.at ?? use.at,
|
|
761
|
+
model: isRecord(result?.structured) ? str(result.structured.resolvedModel) : null
|
|
762
|
+
});
|
|
763
|
+
continue;
|
|
764
|
+
}
|
|
765
|
+
if (steerTools.has(use.name)) {
|
|
766
|
+
const target = str(use.input.to) ?? str(use.input.recipient);
|
|
767
|
+
if (target === null) continue;
|
|
768
|
+
const result = thread.calls.results.get(use.id);
|
|
769
|
+
const structured = isRecord(result?.structured) ? result.structured : null;
|
|
770
|
+
const delivered = structured?.success === true || str(structured?.resumedAgentId) === target;
|
|
771
|
+
const rows = steersByTarget.get(target) ?? [];
|
|
772
|
+
rows.push({ at: use.at, delivered });
|
|
773
|
+
steersByTarget.set(target, rows);
|
|
774
|
+
continue;
|
|
775
|
+
}
|
|
776
|
+
if (cancelTools.has(use.name)) {
|
|
777
|
+
const target = str(use.input.agentId) ?? str(use.input.to) ?? str(use.input.taskId);
|
|
778
|
+
if (target !== null) cancels.push({ agentId: target, at: use.at });
|
|
779
|
+
}
|
|
780
|
+
}
|
|
781
|
+
settles.push(...thread.calls.notifications);
|
|
782
|
+
}
|
|
783
|
+
const spawnedIds = new Set(spawns.map((s) => s.agentId));
|
|
784
|
+
const cancelledIds = new Set(
|
|
785
|
+
cancels.filter((c) => spawnedIds.has(c.agentId)).map((c) => c.agentId)
|
|
786
|
+
);
|
|
787
|
+
const startedAt = main.firstAt;
|
|
788
|
+
const completedAt = main.lastAt;
|
|
789
|
+
const journalLines = [
|
|
790
|
+
line({
|
|
791
|
+
kind: "spawned",
|
|
792
|
+
id: sessionId,
|
|
793
|
+
parent: null,
|
|
794
|
+
label: `session:${sessionId}`,
|
|
795
|
+
at: startedAt
|
|
796
|
+
})
|
|
797
|
+
];
|
|
798
|
+
for (const s of spawns) {
|
|
799
|
+
journalLines.push(
|
|
800
|
+
line({ kind: "spawned", id: s.agentId, parent: s.parentId, label: s.label, at: s.at })
|
|
801
|
+
);
|
|
802
|
+
}
|
|
803
|
+
const lastNotification = /* @__PURE__ */ new Map();
|
|
804
|
+
for (const n of settles) {
|
|
805
|
+
if (!spawnedIds.has(n.taskId)) continue;
|
|
806
|
+
lastNotification.set(n.taskId, n);
|
|
807
|
+
}
|
|
808
|
+
for (const [agentId, n] of lastNotification) {
|
|
809
|
+
if (cancelledIds.has(agentId)) continue;
|
|
810
|
+
journalLines.push(
|
|
811
|
+
line({
|
|
812
|
+
kind: "settled",
|
|
813
|
+
id: agentId,
|
|
814
|
+
status: n.status,
|
|
815
|
+
verdict: n.summary,
|
|
816
|
+
at: n.at
|
|
817
|
+
// No `spent` key: Claude Code prices nothing, and a zeroed spend object
|
|
818
|
+
// would read as a $0 worker. `limits.spendUsd` carries the reason.
|
|
819
|
+
})
|
|
820
|
+
);
|
|
821
|
+
}
|
|
822
|
+
const lastCancel = /* @__PURE__ */ new Map();
|
|
823
|
+
for (const c of cancels) {
|
|
824
|
+
if (!spawnedIds.has(c.agentId)) continue;
|
|
825
|
+
lastCancel.set(c.agentId, c);
|
|
826
|
+
}
|
|
827
|
+
for (const c of lastCancel.values()) {
|
|
828
|
+
journalLines.push(
|
|
829
|
+
line({ kind: "cancelled", id: c.agentId, reason: "stopped by supervisor", at: c.at })
|
|
830
|
+
);
|
|
831
|
+
}
|
|
832
|
+
journalLines.push(
|
|
833
|
+
line({
|
|
834
|
+
kind: "metered",
|
|
835
|
+
id: sessionId,
|
|
836
|
+
spend: {
|
|
837
|
+
tokens: {
|
|
838
|
+
input: mainTranscript.usage.tokensIn,
|
|
839
|
+
output: mainTranscript.usage.tokensOut,
|
|
840
|
+
cacheRead: mainTranscript.usage.cacheRead,
|
|
841
|
+
cacheWrite: mainTranscript.usage.cacheWrite
|
|
842
|
+
}
|
|
843
|
+
},
|
|
844
|
+
at: completedAt
|
|
845
|
+
})
|
|
846
|
+
);
|
|
847
|
+
const settleAtByAgent = new Map(
|
|
848
|
+
[...lastNotification.entries()].map(([id, n]) => [id, n.at])
|
|
849
|
+
);
|
|
850
|
+
const spawnAtByAgent = new Map(spawns.map((s) => [s.agentId, s.at]));
|
|
851
|
+
const byLabel = /* @__PURE__ */ new Map();
|
|
852
|
+
for (const s of spawns) {
|
|
853
|
+
const ids = byLabel.get(s.label) ?? [];
|
|
854
|
+
ids.push(s.agentId);
|
|
855
|
+
byLabel.set(s.label, ids);
|
|
856
|
+
}
|
|
857
|
+
const workers = [];
|
|
858
|
+
for (const [label, agentIds] of byLabel) {
|
|
859
|
+
const events = [];
|
|
860
|
+
const inbox = [];
|
|
861
|
+
let tokensIn = null;
|
|
862
|
+
let tokensOut = null;
|
|
863
|
+
let cacheRead = null;
|
|
864
|
+
let cacheWrite = null;
|
|
865
|
+
let transcriptRef = null;
|
|
866
|
+
for (const agentId of agentIds) {
|
|
867
|
+
const child = childByAgentId.get(agentId) ?? null;
|
|
868
|
+
const startAt = child?.firstAt ?? spawnAtByAgent.get(agentId) ?? null;
|
|
869
|
+
const endAt = settleAtByAgent.get(agentId) ?? child?.lastAt ?? null;
|
|
870
|
+
if (startAt !== null) events.push(line({ kind: "started", label, at: startAt, agentId }));
|
|
871
|
+
for (const steer of steersByTarget.get(agentId) ?? []) {
|
|
872
|
+
inbox.push(
|
|
873
|
+
line({ id: `${agentId}:${steer.at}`, at: steer.at, worker: label, message: "steer" })
|
|
874
|
+
);
|
|
875
|
+
events.push(
|
|
876
|
+
line({
|
|
877
|
+
kind: "message",
|
|
878
|
+
label,
|
|
879
|
+
direction: "down",
|
|
880
|
+
at: steer.at,
|
|
881
|
+
requestId: `${agentId}:${steer.at}`,
|
|
882
|
+
delivered: steer.delivered
|
|
883
|
+
})
|
|
884
|
+
);
|
|
885
|
+
}
|
|
886
|
+
if (endAt !== null) {
|
|
887
|
+
events.push(
|
|
888
|
+
line({
|
|
889
|
+
kind: "finished",
|
|
890
|
+
label,
|
|
891
|
+
at: endAt,
|
|
892
|
+
agentId,
|
|
893
|
+
// No `passed` / `patchBytes`: this harness has neither. Emitting
|
|
894
|
+
// `passed: false` here would invent a failed worker.
|
|
895
|
+
// `evidence` is the child's closing message — what it handed back.
|
|
896
|
+
...child?.finalReport === null || child?.finalReport === void 0 ? {} : { evidence: child.finalReport }
|
|
897
|
+
})
|
|
898
|
+
);
|
|
899
|
+
}
|
|
900
|
+
if (child !== null) {
|
|
901
|
+
tokensIn = (tokensIn ?? 0) + child.tokensIn;
|
|
902
|
+
tokensOut = (tokensOut ?? 0) + child.tokensOut;
|
|
903
|
+
cacheRead = (cacheRead ?? 0) + child.cacheRead;
|
|
904
|
+
cacheWrite = (cacheWrite ?? 0) + child.cacheWrite;
|
|
905
|
+
transcriptRef = child.path;
|
|
906
|
+
}
|
|
907
|
+
}
|
|
908
|
+
workers.push({
|
|
909
|
+
label,
|
|
910
|
+
events: events.length === 0 ? null : `${events.join("\n")}
|
|
911
|
+
`,
|
|
912
|
+
inbox: inbox.length === 0 ? null : `${inbox.join("\n")}
|
|
913
|
+
`,
|
|
914
|
+
patchBytes: null,
|
|
915
|
+
transcriptRef,
|
|
916
|
+
patchPath: null,
|
|
917
|
+
tokensIn,
|
|
918
|
+
tokensOut,
|
|
919
|
+
cacheRead,
|
|
920
|
+
cacheWrite
|
|
921
|
+
});
|
|
922
|
+
}
|
|
923
|
+
const joined = children.length;
|
|
924
|
+
const harnessWorkerTokens = joined === 0 ? null : {
|
|
925
|
+
store: "claude-code subagent transcripts",
|
|
926
|
+
sessions: joined,
|
|
927
|
+
input: children.reduce((a, c) => a + c.tokensIn, 0),
|
|
928
|
+
output: children.reduce((a, c) => a + c.tokensOut, 0),
|
|
929
|
+
cacheRead: children.reduce((a, c) => a + c.cacheRead, 0),
|
|
930
|
+
cacheWrite: children.reduce((a, c) => a + c.cacheWrite, 0)
|
|
931
|
+
};
|
|
932
|
+
const missingChildren = spawns.filter((s) => !childByAgentId.has(s.agentId)).length;
|
|
933
|
+
const harnessMissingReason = subagentsDir === null ? "subagent transcript join disabled" : joined === 0 ? `no subagent transcripts under ${subagentsDir}` : missingChildren === 0 ? null : `${missingChildren}/${spawns.length} spawned agents have no retained transcript under ${subagentsDir} (Claude Code prunes them; their tokens are unrecoverable)`;
|
|
934
|
+
const liveWorkers = spawns.filter(
|
|
935
|
+
(s) => !lastNotification.has(s.agentId) && !cancelledIds.has(s.agentId)
|
|
936
|
+
).length;
|
|
937
|
+
const state = JSON.stringify({
|
|
938
|
+
id: sessionId,
|
|
939
|
+
// Derived, not asserted: a spawned agent with no notification and no stop
|
|
940
|
+
// is still running, which is exactly what a live transcript looks like.
|
|
941
|
+
status: liveWorkers > 0 ? "running" : "idle",
|
|
942
|
+
startedAt,
|
|
943
|
+
completedAt,
|
|
944
|
+
result: { delivered: null }
|
|
945
|
+
});
|
|
946
|
+
return {
|
|
947
|
+
runRef,
|
|
948
|
+
instanceId: opts.instanceId ?? sessionId,
|
|
949
|
+
arm: opts.arm ?? null,
|
|
950
|
+
supRunDir: subagentsDir,
|
|
951
|
+
journal: `${journalLines.join("\n")}
|
|
952
|
+
`,
|
|
953
|
+
brainLog: null,
|
|
954
|
+
state,
|
|
955
|
+
progress: null,
|
|
956
|
+
workers,
|
|
957
|
+
workersMissingReason: null,
|
|
958
|
+
result: null,
|
|
959
|
+
judge: null,
|
|
960
|
+
judgeSource: null,
|
|
961
|
+
patch: null,
|
|
962
|
+
driverLog: null,
|
|
963
|
+
harnessWorkerTokens,
|
|
964
|
+
harnessMissingReason,
|
|
965
|
+
limits: { ...limits, deliverables: NO_DELIVERABLES },
|
|
966
|
+
rootTranscriptRef: opts.transcriptPath,
|
|
967
|
+
traceCommand
|
|
968
|
+
};
|
|
969
|
+
}
|
|
970
|
+
function claudeCodeSupervisorRunReader(opts) {
|
|
971
|
+
return {
|
|
972
|
+
runRef: opts.runRef ?? opts.transcriptPath,
|
|
973
|
+
read: () => readClaudeCodeSupervisorRun(opts)
|
|
974
|
+
};
|
|
975
|
+
}
|
|
976
|
+
|
|
509
977
|
// src/supervisor-run/loops-reader.ts
|
|
510
|
-
import { appendFile, mkdir, readdir, readFile, writeFile } from "fs/promises";
|
|
511
|
-
import { basename, join } from "path";
|
|
978
|
+
import { appendFile, mkdir, readdir as readdir2, readFile as readFile2, writeFile } from "fs/promises";
|
|
979
|
+
import { basename as basename2, join as join2 } from "path";
|
|
512
980
|
|
|
513
981
|
// src/supervisor-run/render.ts
|
|
514
982
|
function renderSupervisorRunHeadline(r) {
|
|
@@ -607,13 +1075,13 @@ unavailable \u2014 ${o.steersByWorker.unavailable}
|
|
|
607
1075
|
out.push("");
|
|
608
1076
|
out.push("## Economics");
|
|
609
1077
|
out.push("");
|
|
610
|
-
out.push("| Role | Tokens in | Tokens out | USD | Source |");
|
|
611
|
-
out.push("
|
|
1078
|
+
out.push("| Role | Tokens in | Tokens out | Cache read | Cache write | USD | Source |");
|
|
1079
|
+
out.push("|---|---:|---:|---:|---:|---:|---|");
|
|
612
1080
|
out.push(
|
|
613
|
-
`| brain | ${showMeasured(e.brain.tokensIn)} | ${showMeasured(e.brain.tokensOut)} | ${showMeasured(e.brain.usd)} | ${e.brain.source} |`
|
|
1081
|
+
`| brain | ${showMeasured(e.brain.tokensIn)} | ${showMeasured(e.brain.tokensOut)} | ${showMeasured(e.brain.cacheRead)} | ${showMeasured(e.brain.cacheWrite)} | ${showMeasured(e.brain.usd)} | ${e.brain.source} |`
|
|
614
1082
|
);
|
|
615
1083
|
out.push(
|
|
616
|
-
`| workers | ${showMeasured(e.workers.tokensIn)} | ${showMeasured(e.workers.tokensOut)} | ${showMeasured(e.workers.usd)} | ${e.workers.source} |`
|
|
1084
|
+
`| workers | ${showMeasured(e.workers.tokensIn)} | ${showMeasured(e.workers.tokensOut)} | ${showMeasured(e.workers.cacheRead)} | ${showMeasured(e.workers.cacheWrite)} | ${showMeasured(e.workers.usd)} | ${e.workers.source} |`
|
|
617
1085
|
);
|
|
618
1086
|
out.push("");
|
|
619
1087
|
if (!isUnavailable(e.brainTruncations) && e.brainTruncations > 0) {
|
|
@@ -637,11 +1105,11 @@ unavailable \u2014 ${o.steersByWorker.unavailable}
|
|
|
637
1105
|
}
|
|
638
1106
|
out.push("");
|
|
639
1107
|
if (!isUnavailable(e.perWorker) && e.perWorker.length > 0) {
|
|
640
|
-
out.push("| Worker | Wall | Patch bytes | Verify passed |");
|
|
641
|
-
out.push("
|
|
1108
|
+
out.push("| Worker | Wall | Tokens in | Tokens out | Patch bytes | Verify passed |");
|
|
1109
|
+
out.push("|---|---:|---:|---:|---:|---|");
|
|
642
1110
|
for (const w of e.perWorker) {
|
|
643
1111
|
out.push(
|
|
644
|
-
`| \`${w.worker}\` | ${w.wallMs === null ? "unavailable \u2014 no start/finish pair" : fmtMs(w.wallMs)} | ${w.patchBytes ?? "unavailable \u2014 no worker patch file"} | ${w.passed === null ? "unavailable \u2014 no finished event" : String(w.passed)} |`
|
|
1112
|
+
`| \`${w.worker}\` | ${w.wallMs === null ? "unavailable \u2014 no start/finish pair" : fmtMs(w.wallMs)} | ${w.tokensIn ?? "unavailable \u2014 store does not attribute tokens per worker"} | ${w.tokensOut ?? "unavailable \u2014 store does not attribute tokens per worker"} | ${w.patchBytes ?? "unavailable \u2014 no worker patch file"} | ${w.passed === null ? "unavailable \u2014 no finished event" : String(w.passed)} |`
|
|
645
1113
|
);
|
|
646
1114
|
}
|
|
647
1115
|
out.push("");
|
|
@@ -721,27 +1189,27 @@ function renderSupervisorRollupMarkdown(rollup, title = "Round rollup") {
|
|
|
721
1189
|
|
|
722
1190
|
// src/supervisor-run/loops-reader.ts
|
|
723
1191
|
async function readMaybe(path) {
|
|
724
|
-
return
|
|
1192
|
+
return readFile2(path, "utf8").catch(() => null);
|
|
725
1193
|
}
|
|
726
1194
|
async function findSupervisorRunDirIn(ws) {
|
|
727
|
-
const root =
|
|
728
|
-
const entries = await
|
|
729
|
-
const dirs = entries.filter((e) => e.isDirectory()).map((e) =>
|
|
1195
|
+
const root = join2(ws, ".loops", "supervisor");
|
|
1196
|
+
const entries = await readdir2(root, { withFileTypes: true }).catch(() => []);
|
|
1197
|
+
const dirs = entries.filter((e) => e.isDirectory()).map((e) => join2(root, e.name));
|
|
730
1198
|
return dirs[0] ?? null;
|
|
731
1199
|
}
|
|
732
1200
|
async function readLoopsSupervisorRun(runDir, opts = {}) {
|
|
733
|
-
const ws = opts.ws ??
|
|
1201
|
+
const ws = opts.ws ?? join2(runDir, "ws");
|
|
734
1202
|
const supRunDir = await findSupervisorRunDirIn(ws);
|
|
735
|
-
const result = await readMaybe(
|
|
1203
|
+
const result = await readMaybe(join2(runDir, "result.json"));
|
|
736
1204
|
const resultObj = parseJson(result);
|
|
737
1205
|
let workers = null;
|
|
738
1206
|
let workersMissingReason = null;
|
|
739
1207
|
const workerCwds = [];
|
|
740
1208
|
if (supRunDir === null) {
|
|
741
|
-
workersMissingReason = `no supervisor run dir under ${
|
|
1209
|
+
workersMissingReason = `no supervisor run dir under ${join2(ws, ".loops", "supervisor")}`;
|
|
742
1210
|
} else {
|
|
743
|
-
const workersDir =
|
|
744
|
-
const entries = await
|
|
1211
|
+
const workersDir = join2(supRunDir, "workers");
|
|
1212
|
+
const entries = await readdir2(workersDir).catch(() => null);
|
|
745
1213
|
if (entries === null) {
|
|
746
1214
|
workersMissingReason = `workers/ directory absent under ${supRunDir}`;
|
|
747
1215
|
} else {
|
|
@@ -752,9 +1220,9 @@ async function readLoopsSupervisorRun(runDir, opts = {}) {
|
|
|
752
1220
|
].sort();
|
|
753
1221
|
workers = [];
|
|
754
1222
|
for (const label of labels) {
|
|
755
|
-
const events = await readMaybe(
|
|
756
|
-
const inbox = await readMaybe(
|
|
757
|
-
const patch = await readMaybe(
|
|
1223
|
+
const events = await readMaybe(join2(workersDir, `${label}.ndjson`));
|
|
1224
|
+
const inbox = await readMaybe(join2(workersDir, `${label}.inbox.ndjson`));
|
|
1225
|
+
const patch = await readMaybe(join2(workersDir, `${label}.patch`));
|
|
758
1226
|
workers.push({
|
|
759
1227
|
label,
|
|
760
1228
|
events,
|
|
@@ -799,8 +1267,8 @@ async function readLoopsSupervisorRun(runDir, opts = {}) {
|
|
|
799
1267
|
}
|
|
800
1268
|
}
|
|
801
1269
|
const patchPath = opts.patchPath ?? (typeof resultObj?.patchPath === "string" ? resultObj.patchPath : null);
|
|
802
|
-
let judge = await readMaybe(
|
|
803
|
-
let judgeSource = judge === null ? null :
|
|
1270
|
+
let judge = await readMaybe(join2(runDir, "judge.json"));
|
|
1271
|
+
let judgeSource = judge === null ? null : join2(runDir, "judge.json");
|
|
804
1272
|
if (judge === null && opts.ledgerPath !== void 0) {
|
|
805
1273
|
const row = await findLedgerRow(opts.ledgerPath, runDir);
|
|
806
1274
|
if (row !== null) {
|
|
@@ -811,21 +1279,25 @@ async function readLoopsSupervisorRun(runDir, opts = {}) {
|
|
|
811
1279
|
return {
|
|
812
1280
|
runRef: runDir,
|
|
813
1281
|
instanceId: typeof resultObj?.iid === "string" ? resultObj.iid : instanceIdFromPath(runDir),
|
|
814
|
-
arm: typeof resultObj?.arm === "string" ? resultObj.arm :
|
|
1282
|
+
arm: typeof resultObj?.arm === "string" ? resultObj.arm : basename2(runDir),
|
|
815
1283
|
supRunDir,
|
|
816
|
-
journal: supRunDir === null ? null : await readMaybe(
|
|
817
|
-
brainLog: supRunDir === null ? null : await readMaybe(
|
|
818
|
-
state: supRunDir === null ? null : await readMaybe(
|
|
819
|
-
progress: supRunDir === null ? null : await readMaybe(
|
|
1284
|
+
journal: supRunDir === null ? null : await readMaybe(join2(supRunDir, "journal.jsonl")),
|
|
1285
|
+
brainLog: supRunDir === null ? null : await readMaybe(join2(supRunDir, "brain.jsonl")),
|
|
1286
|
+
state: supRunDir === null ? null : await readMaybe(join2(supRunDir, "state.json")),
|
|
1287
|
+
progress: supRunDir === null ? null : await readMaybe(join2(supRunDir, "progress.ndjson")),
|
|
820
1288
|
workers,
|
|
821
1289
|
workersMissingReason,
|
|
822
1290
|
result,
|
|
823
1291
|
judge,
|
|
824
1292
|
judgeSource,
|
|
825
1293
|
patch: patchPath === null ? null : await readMaybe(patchPath),
|
|
826
|
-
driverLog: await readMaybe(
|
|
1294
|
+
driverLog: await readMaybe(join2(runDir, "driver.log")),
|
|
827
1295
|
harnessWorkerTokens,
|
|
828
|
-
harnessMissingReason
|
|
1296
|
+
harnessMissingReason,
|
|
1297
|
+
// loops prices its own inference, runs a verify per worker, and keeps each
|
|
1298
|
+
// worker's patch — every fact the analyzer can use is expressible here.
|
|
1299
|
+
limits: NO_SOURCE_LIMITS,
|
|
1300
|
+
traceCommand: null
|
|
829
1301
|
};
|
|
830
1302
|
}
|
|
831
1303
|
function loopsSupervisorRunReader(runDir, opts = {}) {
|
|
@@ -836,7 +1308,7 @@ async function findLedgerRow(ledgerPath, runDir) {
|
|
|
836
1308
|
const exact = rows.find((r) => r.runDir === runDir);
|
|
837
1309
|
if (exact !== void 0) return exact;
|
|
838
1310
|
const iid = instanceIdFromPath(runDir);
|
|
839
|
-
const arm =
|
|
1311
|
+
const arm = basename2(runDir);
|
|
840
1312
|
return rows.find((r) => r.iid === iid && r.arm === arm) ?? null;
|
|
841
1313
|
}
|
|
842
1314
|
function instanceIdFromPath(runDir) {
|
|
@@ -863,9 +1335,9 @@ async function writeSupervisorRunReport(runDir, opts = {}) {
|
|
|
863
1335
|
const stem = opts.reportDir === void 0 ? "run-report" : supervisorReportStem(runDir);
|
|
864
1336
|
if (opts.reportDir !== void 0) await mkdir(opts.reportDir, { recursive: true }).catch(() => {
|
|
865
1337
|
});
|
|
866
|
-
await writeFile(
|
|
1338
|
+
await writeFile(join2(dest, `${stem}.json`), JSON.stringify(report, null, 1)).catch(() => {
|
|
867
1339
|
});
|
|
868
|
-
await writeFile(
|
|
1340
|
+
await writeFile(join2(dest, `${stem}.md`), md).catch(() => {
|
|
869
1341
|
});
|
|
870
1342
|
const headline = renderSupervisorRunHeadline(report);
|
|
871
1343
|
if (opts.appendHeadlineTo !== void 0) {
|
|
@@ -882,8 +1354,8 @@ function supervisorReportStem(runDir) {
|
|
|
882
1354
|
const iid = parts[parts.length - 2] ?? "instance";
|
|
883
1355
|
const rep = parts.find((p) => /^rep-\d+$/.test(p));
|
|
884
1356
|
const armRunsIdx = parts.indexOf("arm-runs");
|
|
885
|
-
const
|
|
886
|
-
return [
|
|
1357
|
+
const tag2 = armRunsIdx >= 0 ? parts[armRunsIdx + 1] : void 0;
|
|
1358
|
+
return [tag2, rep, iid, arm].filter((s) => s !== void 0 && s !== "runs").join(".").replace(/[^A-Za-z0-9._-]/g, "_");
|
|
887
1359
|
}
|
|
888
1360
|
async function writeSupervisorRunReportSafe(runDir, opts = {}) {
|
|
889
1361
|
try {
|
|
@@ -905,16 +1377,16 @@ async function reportSupervisorRound(outDir, opts = {}) {
|
|
|
905
1377
|
const rollup = rollupSupervisorRuns(reports);
|
|
906
1378
|
const md = renderSupervisorRollupMarkdown(
|
|
907
1379
|
rollup,
|
|
908
|
-
opts.title ?? `Round rollup \u2014 ${
|
|
1380
|
+
opts.title ?? `Round rollup \u2014 ${basename2(outDir)}`
|
|
909
1381
|
);
|
|
910
1382
|
const dest = opts.reportDir ?? outDir;
|
|
911
1383
|
if (opts.reportDir !== void 0) await mkdir(opts.reportDir, { recursive: true }).catch(() => {
|
|
912
1384
|
});
|
|
913
|
-
await writeFile(
|
|
1385
|
+
await writeFile(join2(dest, "run-report-round.json"), JSON.stringify(rollup, null, 1)).catch(
|
|
914
1386
|
() => {
|
|
915
1387
|
}
|
|
916
1388
|
);
|
|
917
|
-
await writeFile(
|
|
1389
|
+
await writeFile(join2(dest, "run-report-round.md"), md).catch(() => {
|
|
918
1390
|
});
|
|
919
1391
|
if (opts.appendHeadlineTo !== void 0) {
|
|
920
1392
|
await appendFile(opts.appendHeadlineTo, `${md}
|
|
@@ -928,17 +1400,17 @@ async function findSupervisorRunDirs(root) {
|
|
|
928
1400
|
const found = [];
|
|
929
1401
|
const walk = async (dir, depth) => {
|
|
930
1402
|
if (depth > 8) return;
|
|
931
|
-
const entries = await
|
|
1403
|
+
const entries = await readdir2(dir, { withFileTypes: true }).catch(() => []);
|
|
932
1404
|
for (const e of entries) {
|
|
933
1405
|
if (!e.isDirectory()) continue;
|
|
934
1406
|
if (e.name === "node_modules" || e.name === ".git") continue;
|
|
935
|
-
const full =
|
|
1407
|
+
const full = join2(dir, e.name);
|
|
936
1408
|
if (e.name === "runs") {
|
|
937
|
-
const iids = await
|
|
1409
|
+
const iids = await readdir2(full, { withFileTypes: true }).catch(() => []);
|
|
938
1410
|
for (const iid of iids) {
|
|
939
1411
|
if (!iid.isDirectory()) continue;
|
|
940
|
-
const arms = await
|
|
941
|
-
for (const arm of arms) if (arm.isDirectory()) found.push(
|
|
1412
|
+
const arms = await readdir2(join2(full, iid.name), { withFileTypes: true }).catch(() => []);
|
|
1413
|
+
for (const arm of arms) if (arm.isDirectory()) found.push(join2(full, iid.name, arm.name));
|
|
942
1414
|
}
|
|
943
1415
|
continue;
|
|
944
1416
|
}
|
|
@@ -1036,15 +1508,17 @@ function supervisorRunRolloutLines(src, opts = {}) {
|
|
|
1036
1508
|
},
|
|
1037
1509
|
cost: {
|
|
1038
1510
|
...EMPTY_COST,
|
|
1039
|
-
usd: typeof stateResult.spentUsd === "number" ? stateResult.spentUsd : tree.brain.usd,
|
|
1511
|
+
usd: src.limits.spendUsd !== null ? null : typeof stateResult.spentUsd === "number" ? stateResult.spentUsd : tree.brain.usd,
|
|
1040
1512
|
tokens_in: tree.brain.tokensIn,
|
|
1041
1513
|
tokens_out: tree.brain.tokensOut,
|
|
1514
|
+
cache_read: tree.brain.hasCache ? tree.brain.cacheRead : null,
|
|
1515
|
+
cache_write: tree.brain.hasCache ? tree.brain.cacheWrite : null,
|
|
1042
1516
|
wall_s: wallMs === null ? null : wallMs / 1e3
|
|
1043
1517
|
},
|
|
1044
1518
|
artifacts: {
|
|
1045
1519
|
patch_path: typeof result?.patchPath === "string" ? result.patchPath : null,
|
|
1046
1520
|
run_dir: src.supRunDir,
|
|
1047
|
-
transcript_ref: src.supRunDir === null ? null : `${src.supRunDir}/journal.jsonl`
|
|
1521
|
+
transcript_ref: src.rootTranscriptRef !== void 0 ? src.rootTranscriptRef : src.supRunDir === null ? null : `${src.supRunDir}/journal.jsonl`
|
|
1048
1522
|
},
|
|
1049
1523
|
provenance: {
|
|
1050
1524
|
captured_at: capturedAt,
|
|
@@ -1056,9 +1530,11 @@ function supervisorRunRolloutLines(src, opts = {}) {
|
|
|
1056
1530
|
gaps.push("tree.root: no parentless `spawned` event in the journal");
|
|
1057
1531
|
}
|
|
1058
1532
|
const closeById = new Map(tree.closes.map((c) => [c.id, c]));
|
|
1533
|
+
const sourceByLabel = new Map((src.workers ?? []).map((w) => [w.label, w]));
|
|
1059
1534
|
for (const spawn of tree.workerSpawns) {
|
|
1060
1535
|
const close = closeById.get(spawn.id) ?? null;
|
|
1061
1536
|
const facts = tree.workerLogs.get(spawn.label) ?? null;
|
|
1537
|
+
const workerSource = sourceByLabel.get(spawn.label) ?? null;
|
|
1062
1538
|
const wallMs = facts?.started != null && facts.finishedAt != null ? facts.finishedAt - facts.started : null;
|
|
1063
1539
|
const reward = facts?.passed === null || facts === null ? null : facts.passed ? 1 : 0;
|
|
1064
1540
|
if (reward === null) {
|
|
@@ -1098,22 +1574,33 @@ function supervisorRunRolloutLines(src, opts = {}) {
|
|
|
1098
1574
|
is_truncated: close?.kind === "cancelled",
|
|
1099
1575
|
error: close?.kind === "cancelled" ? close.verdict ?? "cancelled" : null
|
|
1100
1576
|
},
|
|
1577
|
+
// `close.spend` is only a measurement when the close event carried one;
|
|
1578
|
+
// a store that never priced this worker must leave null, not 0.
|
|
1101
1579
|
cost: {
|
|
1102
1580
|
...EMPTY_COST,
|
|
1103
|
-
usd: close?.spend.usd
|
|
1104
|
-
tokens_in: close?.spend.tokens.input
|
|
1105
|
-
tokens_out: close?.spend.tokens.output
|
|
1581
|
+
usd: src.limits.spendUsd !== null || !close?.hasSpend ? null : close.spend.usd,
|
|
1582
|
+
tokens_in: workerSource?.tokensIn ?? (close?.hasSpend ? close.spend.tokens.input : null),
|
|
1583
|
+
tokens_out: workerSource?.tokensOut ?? (close?.hasSpend ? close.spend.tokens.output : null),
|
|
1584
|
+
// Mirror the tokens_in/out journal fallback so a loops-shaped store whose
|
|
1585
|
+
// `settled` spend carries cache counters is not reported as null cache
|
|
1586
|
+
// beside real tokens. Gate on `hasCache` — a spend object without cache
|
|
1587
|
+
// counters must stay null, not a fabricated 0.
|
|
1588
|
+
cache_read: workerSource?.cacheRead ?? (close?.hasSpend && close.spend.tokens.hasCache ? close.spend.tokens.cacheRead : null),
|
|
1589
|
+
cache_write: workerSource?.cacheWrite ?? (close?.hasSpend && close.spend.tokens.hasCache ? close.spend.tokens.cacheWrite : null),
|
|
1106
1590
|
wall_s: wallMs === null ? null : wallMs / 1e3
|
|
1107
1591
|
},
|
|
1592
|
+
// A reader that knows where its worker artifacts live says so; only the
|
|
1593
|
+
// loops layout is derivable from `supRunDir`, so guessing it for another
|
|
1594
|
+
// store would mint rows pointing at paths that never existed.
|
|
1108
1595
|
artifacts: {
|
|
1109
|
-
patch_path: src.supRunDir === null ? null : `${src.supRunDir}/workers/${spawn.label}.patch`,
|
|
1596
|
+
patch_path: workerSource?.patchPath !== void 0 ? workerSource.patchPath : src.supRunDir === null ? null : `${src.supRunDir}/workers/${spawn.label}.patch`,
|
|
1110
1597
|
run_dir: src.supRunDir,
|
|
1111
|
-
transcript_ref: src.supRunDir === null ? null : `${src.supRunDir}/workers/${spawn.label}.ndjson`
|
|
1598
|
+
transcript_ref: workerSource?.transcriptRef !== void 0 ? workerSource.transcriptRef : src.supRunDir === null ? null : `${src.supRunDir}/workers/${spawn.label}.ndjson`
|
|
1112
1599
|
},
|
|
1113
1600
|
provenance: {
|
|
1114
1601
|
captured_at: capturedAt,
|
|
1115
1602
|
capture: "backfill",
|
|
1116
|
-
gap: "worker transcript lives in the harness session store \u2014 hydrate via src/rollout/readers"
|
|
1603
|
+
gap: workerSource?.transcriptRef == null ? "worker transcript lives in the harness session store \u2014 hydrate via src/rollout/readers" : "worker transcript recorded by the harness; messages not inlined into this row"
|
|
1117
1604
|
}
|
|
1118
1605
|
});
|
|
1119
1606
|
}
|
|
@@ -1128,12 +1615,18 @@ export {
|
|
|
1128
1615
|
unavailable,
|
|
1129
1616
|
isUnavailable,
|
|
1130
1617
|
showMeasured,
|
|
1618
|
+
NO_SOURCE_LIMITS,
|
|
1131
1619
|
SUPERVISOR_RUN_SCHEMA,
|
|
1132
1620
|
SUPERVISOR_RUN_ROLLUP_SCHEMA,
|
|
1133
1621
|
parseSupervisorTree,
|
|
1134
1622
|
analyzeSupervisorRunSources,
|
|
1135
1623
|
parsePatch,
|
|
1136
1624
|
rollupSupervisorRuns,
|
|
1625
|
+
DEFAULT_SPAWN_TOOLS,
|
|
1626
|
+
DEFAULT_STEER_TOOLS,
|
|
1627
|
+
DEFAULT_CANCEL_TOOLS,
|
|
1628
|
+
readClaudeCodeSupervisorRun,
|
|
1629
|
+
claudeCodeSupervisorRunReader,
|
|
1137
1630
|
renderSupervisorRunHeadline,
|
|
1138
1631
|
renderSupervisorRunMarkdown,
|
|
1139
1632
|
renderSupervisorRollupMarkdown,
|
|
@@ -1148,4 +1641,4 @@ export {
|
|
|
1148
1641
|
findSupervisorRunDirs,
|
|
1149
1642
|
supervisorRunRolloutLines
|
|
1150
1643
|
};
|
|
1151
|
-
//# sourceMappingURL=chunk-
|
|
1644
|
+
//# sourceMappingURL=chunk-LKKT3IVV.js.map
|