@tangle-network/agent-eval 0.181.0 → 0.182.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/dist/analyst/index.d.ts +3 -3
- package/dist/analyst/index.js +2 -2
- package/dist/{benchmark-command-B57n9vjz.js → benchmark-command-7X7HPNHh.js} +3 -3
- package/dist/{benchmark-command-B57n9vjz.js.map → benchmark-command-7X7HPNHh.js.map} +1 -1
- package/dist/benchmarks/index.js +2 -2
- package/dist/campaign/index.js +6 -6
- package/dist/{campaign-4_ppJW5X.js → campaign-B2U7WcFz.js} +8 -8
- package/dist/{campaign-4_ppJW5X.js.map → campaign-B2U7WcFz.js.map} +1 -1
- package/dist/{campaign-evidence-B8oF9xQ6.js → campaign-evidence-PYPUftRy.js} +2 -2
- package/dist/{campaign-evidence-B8oF9xQ6.js.map → campaign-evidence-PYPUftRy.js.map} +1 -1
- package/dist/cli.js +2 -2
- package/dist/contract/index.js +7 -7
- package/dist/{default-registry-aL7xUrUz.js → default-registry-DHWwIwa8.js} +2 -2
- package/dist/{default-registry-aL7xUrUz.js.map → default-registry-DHWwIwa8.js.map} +1 -1
- package/dist/{define-agent-eval-Ddu33JH9.js → define-agent-eval-DSIPB4hi.js} +7 -7
- package/dist/{define-agent-eval-Ddu33JH9.js.map → define-agent-eval-DSIPB4hi.js.map} +1 -1
- package/dist/{eval-campaign-aYdtjtJR.js → eval-campaign-CCEkwzoH.js} +2 -2
- package/dist/{eval-campaign-aYdtjtJR.js.map → eval-campaign-CCEkwzoH.js.map} +1 -1
- package/dist/experiment/index.js +5 -5
- package/dist/{index-DNntP4ch.d.ts → index-DbG8QbH9.d.ts} +2 -2
- package/dist/{index-DNntP4ch.d.ts.map → index-DbG8QbH9.d.ts.map} +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +11 -11
- package/dist/{integrity-DH5ng72x.js → integrity-CgQce0FV.js} +2 -2
- package/dist/{integrity-DH5ng72x.js.map → integrity-CgQce0FV.js.map} +1 -1
- package/dist/{llm-judge-DEFZeSiu.js → llm-judge-BPFR12VJ.js} +3 -3
- package/dist/{llm-judge-DEFZeSiu.js.map → llm-judge-BPFR12VJ.js.map} +1 -1
- package/dist/openapi.json +1 -1
- package/dist/{paired-arms-D4aeIHUy.js → paired-arms-JSDcs2zx.js} +16 -3
- package/dist/paired-arms-JSDcs2zx.js.map +1 -0
- package/dist/{power-and-mde-B8F2RdcD.js → power-and-mde-BJOpt6pq.js} +2 -2
- package/dist/{power-and-mde-B8F2RdcD.js.map → power-and-mde-BJOpt6pq.js.map} +1 -1
- package/dist/{produced-state-BHboMaab.js → produced-state-CkoG43gI.js} +3 -3
- package/dist/{produced-state-BHboMaab.js.map → produced-state-CkoG43gI.js.map} +1 -1
- package/dist/{promotion-policy-CDMMxzb6.js → promotion-policy-BKFrl7CC.js} +3 -3
- package/dist/{promotion-policy-CDMMxzb6.js.map → promotion-policy-BKFrl7CC.js.map} +1 -1
- package/dist/release-confidence-BcqeQTHW.d.ts.map +1 -1
- package/dist/{report-command-V1ecVgAv.js → report-command-BvqaxlVG.js} +32 -201
- package/dist/report-command-BvqaxlVG.js.map +1 -0
- package/dist/reporting.js +3 -3
- package/dist/rl.js +3 -3
- package/dist/rollout/index.js +1 -2
- package/dist/{rollout-B-UF5R6w.js → rollout-BsN6pOU7.js} +153 -12
- package/dist/rollout-BsN6pOU7.js.map +1 -0
- package/dist/{run-record-Br-Yzt_k.js → run-record-DPa2PeHx.js} +2 -2
- package/dist/{run-record-Br-Yzt_k.js.map → run-record-DPa2PeHx.js.map} +1 -1
- package/dist/{sequential-DAsyV2T9.js → sequential-D_xHrBZe.js} +2 -2
- package/dist/{sequential-DAsyV2T9.js.map → sequential-D_xHrBZe.js.map} +1 -1
- package/dist/{skillopt-optimization-method-CL_0aArC.js → skillopt-optimization-method-B2IEaCeF.js} +3 -3
- package/dist/{skillopt-optimization-method-CL_0aArC.js.map → skillopt-optimization-method-B2IEaCeF.js.map} +1 -1
- package/dist/statistical-heldout-CpVd6FmY.d.ts.map +1 -1
- package/dist/{summary-report-e-MaOAHV.js → summary-report-IGAOtgPd.js} +3 -3
- package/dist/{summary-report-e-MaOAHV.js.map → summary-report-IGAOtgPd.js.map} +1 -1
- package/dist/supervisor-run/index.d.ts +20 -44
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +8 -8
- package/dist/supervisor-run/index.js.map +1 -1
- package/dist/{terminal-record-BtPwKTSr.js → terminal-record-BHocwFw1.js} +5 -22
- package/dist/terminal-record-BHocwFw1.js.map +1 -0
- package/dist/{types-lPkDQNqJ.d.ts → types-BlIJ5URG.d.ts} +17 -26
- package/dist/types-BlIJ5URG.d.ts.map +1 -0
- package/docs/public-api.md +5 -8
- package/package.json +1 -1
- package/dist/opencode-sqlite-CNw3vubS.js +0 -145
- package/dist/opencode-sqlite-CNw3vubS.js.map +0 -1
- package/dist/paired-arms-D4aeIHUy.js.map +0 -1
- package/dist/report-command-V1ecVgAv.js.map +0 -1
- package/dist/rollout-B-UF5R6w.js.map +0 -1
- package/dist/terminal-record-BtPwKTSr.js.map +0 -1
- package/dist/types-lPkDQNqJ.d.ts.map +0 -1
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { o as summarizeNumberSeries } from "./descriptive-1V17A-qa.js";
|
|
2
|
-
import {
|
|
3
|
-
import { a as parseJson, c as workerSourceKey, d as SUPERVISOR_RUN_SCHEMA, f as isUnavailable, i as asRecord, l as NO_SOURCE_LIMITS, m as unavailable, o as parseJsonl, p as showMeasured, r as readTerminalRecord, s as parseSupervisorTree, u as SUPERVISOR_RUN_ROLLUP_SCHEMA } from "./terminal-record-BtPwKTSr.js";
|
|
2
|
+
import { a as parseJson, c as workerSourceKey, d as SUPERVISOR_RUN_SCHEMA, f as isUnavailable, i as asRecord, l as NO_SOURCE_LIMITS, m as unavailable, o as parseJsonl, p as showMeasured, r as readTerminalRecord, s as parseSupervisorTree, u as SUPERVISOR_RUN_ROLLUP_SCHEMA } from "./terminal-record-BHocwFw1.js";
|
|
4
3
|
import { basename, join, resolve } from "node:path";
|
|
5
4
|
import { appendFile, mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises";
|
|
6
5
|
//#region src/supervisor-run/analyze.ts
|
|
@@ -8,7 +7,7 @@ import { appendFile, mkdir, readFile, readdir, stat, writeFile } from "node:fs/p
|
|
|
8
7
|
* The pure analyzer. Takes already-read bytes (`SupervisorRunSources`) and
|
|
9
8
|
* returns the report — every metric derivable from a synthetic journal string
|
|
10
9
|
* with no filesystem, no process, and no network. All I/O lives in a reader
|
|
11
|
-
* (`
|
|
10
|
+
* (`reader.ts` is one).
|
|
12
11
|
*/
|
|
13
12
|
const NO_CACHE_COUNTERS = "the journal carries no cache-token counters for this role";
|
|
14
13
|
const NO_CACHE_BREAKDOWN = "Runtime recorded cacheBreakdownKnown:false — the provider reported a total without splitting cache reads from writes";
|
|
@@ -22,7 +21,7 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
|
|
|
22
21
|
gaps.push(`${what}: ${reason}`);
|
|
23
22
|
return unavailable(reason);
|
|
24
23
|
};
|
|
25
|
-
const journalMissing = src.journalMissingReason ?? (src.supRunDir === null ? "no supervisor run
|
|
24
|
+
const journalMissing = src.journalMissingReason ?? (src.supRunDir === null ? "no Runtime supervisor run directory" : "journal.jsonl absent");
|
|
26
25
|
const haveJournal = src.journal !== null;
|
|
27
26
|
const tree = parseSupervisorTree(src);
|
|
28
27
|
const state = tree.state;
|
|
@@ -362,7 +361,7 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
|
|
|
362
361
|
cacheWrite: managerTokenLimit !== null ? unavailable(managerTokenLimit) : !haveJournal ? unavailable(journalMissing) : brainTokensUnreported !== null ? unavailable(brainTokensUnreported) : !tree.brain.hasCache ? unavailable(NO_CACHE_COUNTERS) : tree.brain.cacheBreakdownKnown ? tree.brain.cacheWrite : unavailable(NO_CACHE_BREAKDOWN),
|
|
363
362
|
source: managerTokenLimit ?? (haveJournal ? `journal metered events (n=${tree.brain.meteredCount})${tree.brain.usdKnownCount > 0 && tree.brain.usdUnknownCount > 0 ? ` — ${tree.brain.usdUnknownCount} unpriced` : ""}` : journalMissing)
|
|
364
363
|
},
|
|
365
|
-
brainTruncations: src.brainLog === null ? gap("brain.brainTruncations", src.brainLogMissingReason ?? (src.supRunDir === null ? "no supervisor run
|
|
364
|
+
brainTruncations: src.brainLog === null ? gap("brain.brainTruncations", src.brainLogMissingReason ?? (src.supRunDir === null ? "no Runtime supervisor run directory" : "brain.jsonl absent — this source has no brain-call tap, so truncation cannot be ruled out")) : brainCalls.filter((c) => c.finish_reason === "length").length,
|
|
366
365
|
workers: {
|
|
367
366
|
tokensIn: workerIn,
|
|
368
367
|
tokensOut: workerOut,
|
|
@@ -632,7 +631,7 @@ function renderSupervisorRunMarkdown(r) {
|
|
|
632
631
|
out.push("");
|
|
633
632
|
out.push("| Worker id | Label | Queued | Delivered |");
|
|
634
633
|
out.push("|---|---|---:|---:|");
|
|
635
|
-
for (const s of o.steersByWorker) out.push(`| ${s.workerId === null ? "unavailable —
|
|
634
|
+
for (const s of o.steersByWorker) out.push(`| ${s.workerId === null ? "unavailable — worker id absent; label join used" : `\`${s.workerId}\``} | \`${s.worker}\` | ${s.queued} | ${s.delivered} |`);
|
|
636
635
|
out.push("");
|
|
637
636
|
} else if (isUnavailable(o.steersByWorker)) out.push(`### Steers per worker\n\nunavailable — ${o.steersByWorker.unavailable}\n`);
|
|
638
637
|
out.push("## Decision quality");
|
|
@@ -670,7 +669,7 @@ function renderSupervisorRunMarkdown(r) {
|
|
|
670
669
|
if (!isUnavailable(e.perWorker) && e.perWorker.length > 0) {
|
|
671
670
|
out.push("| Worker id | Label | Role | Runtime | Profile digest | Status | Failure | Infra | Wall | Tokens in | Tokens out | Patch bytes | Verify passed | Score |");
|
|
672
671
|
out.push("|---|---|---|---|---|---|---|---|---:|---:|---:|---:|---|---:|");
|
|
673
|
-
for (const w of e.perWorker) out.push(`| ${w.workerId === null ? "unavailable —
|
|
672
|
+
for (const w of e.perWorker) out.push(`| ${w.workerId === null ? "unavailable — worker id absent; label join used" : `\`${w.workerId}\``} | \`${w.worker}\` | ${w.role ?? "unavailable — source recorded no role"} | ${w.runtime ?? "unavailable — source recorded no runtime"} | ${w.profileDigest === null ? "unavailable — source recorded no profile digest" : `\`${w.profileDigest}\``} | ${w.status ?? "unavailable — no terminal event"} | ${w.failure ?? "none recorded"} | ${w.infra ?? "unavailable"} | ${w.wallMs === null ? "unavailable — no spawn/finish pair" : fmtMs(w.wallMs)} | ${w.tokensIn ?? "unavailable — store does not attribute tokens per worker"} | ${w.tokensOut ?? "unavailable — store does not attribute tokens per worker"} | ${w.patchBytes ?? "unavailable — no worker patch file"} | ${w.passed === null ? "unavailable — no verdict" : String(w.passed)} | ${w.score ?? "unavailable — no numeric score"} |`);
|
|
674
673
|
out.push("");
|
|
675
674
|
}
|
|
676
675
|
if (e.resourceRecords !== void 0) out.push("## Named resource receipts", "", ...resourceRows(e.resourceRecords));
|
|
@@ -752,9 +751,8 @@ function renderSupervisorRollupMarkdown(rollup, title = "Round rollup") {
|
|
|
752
751
|
* record Runtime writes when `supervise()` threw before a result landed. Both
|
|
753
752
|
* pass through as bytes; `terminal-record.ts` reads them. The reader does not
|
|
754
753
|
* decide which kinds count: a kind it refuses is a run that recorded its
|
|
755
|
-
* outcome and got reported as having none.
|
|
756
|
-
*
|
|
757
|
-
* a status came from, and a synthetic legacy document would mislabel it.
|
|
754
|
+
* outcome and got reported as having none. The reader does not synthesize status fields. The analyzer names the Runtime
|
|
755
|
+
* record that supplied a status, so status provenance cannot be mislabelled.
|
|
758
756
|
*
|
|
759
757
|
* `usdKnown: false` / `tokensKnown: false` on ONE record is not a limit of this
|
|
760
758
|
* store. The store recorded every other record completely, so the flags travel
|
|
@@ -775,7 +773,7 @@ const FAILURE_FILE = "failure.json";
|
|
|
775
773
|
* attempt threw on a caller input error 16 minutes before the corrected attempt
|
|
776
774
|
* opened the spawn journal. At that instant the directory held `observer.jsonl`
|
|
777
775
|
* (2 records) and the failure record and no journal, so a journal-only test
|
|
778
|
-
* routed it to
|
|
776
|
+
* routed it to an unrelated reader, which reads neither file, and the recorded
|
|
779
777
|
* throw was reported as no run at all.
|
|
780
778
|
*/
|
|
781
779
|
const RUNTIME_RUN_DIR_MARKERS = [
|
|
@@ -783,7 +781,7 @@ const RUNTIME_RUN_DIR_MARKERS = [
|
|
|
783
781
|
OBSERVER_FILE,
|
|
784
782
|
FAILURE_FILE
|
|
785
783
|
];
|
|
786
|
-
async function readMaybe
|
|
784
|
+
async function readMaybe(path) {
|
|
787
785
|
return readFile(path, "utf8").catch((error) => {
|
|
788
786
|
if (typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT") return null;
|
|
789
787
|
throw error;
|
|
@@ -1013,7 +1011,7 @@ function assertFailureRecord(failure, failurePath) {
|
|
|
1013
1011
|
* The journal's begin stamp in the analyzer's state-document shape. It carries
|
|
1014
1012
|
* the run identity and start instant only; the terminal status lives in
|
|
1015
1013
|
* Runtime's own `result.json` / `failure.json`, which the analyzer reads by
|
|
1016
|
-
* name, so no
|
|
1014
|
+
* name, so no synthetic status field is fabricated here.
|
|
1017
1015
|
*/
|
|
1018
1016
|
function runtimeBeginState(root, startedAt) {
|
|
1019
1017
|
return JSON.stringify({
|
|
@@ -1023,7 +1021,7 @@ function runtimeBeginState(root, startedAt) {
|
|
|
1023
1021
|
}
|
|
1024
1022
|
/**
|
|
1025
1023
|
* Sources for a run dir whose spawn journal does not exist. Mirrors the
|
|
1026
|
-
* absent shape
|
|
1024
|
+
* absent shape the reader returns for a missing store: every
|
|
1027
1025
|
* journal-dependent metric downstream reads `unavailable`, never 0.
|
|
1028
1026
|
*/
|
|
1029
1027
|
/**
|
|
@@ -1067,7 +1065,7 @@ function absentRuntimeSupervisorRun(runDir, resultText, failureText, instanceId)
|
|
|
1067
1065
|
* Read one agent-runtime `createFileRunContext(dir)` directory.
|
|
1068
1066
|
*
|
|
1069
1067
|
* A run dir without `spawn-journal.jsonl` returns the same absent-shaped
|
|
1070
|
-
* sources
|
|
1068
|
+
* sources the reader returns for a missing store: `journal` and
|
|
1071
1069
|
* `workers` null, each with its reason, so every dependent metric reads
|
|
1072
1070
|
* `unavailable` — never 0 and never a throw. Its `result.json` and
|
|
1073
1071
|
* `failure.json` are still read, so a run that died before its first spawn
|
|
@@ -1082,11 +1080,11 @@ function absentRuntimeSupervisorRun(runDir, resultText, failureText, instanceId)
|
|
|
1082
1080
|
*/
|
|
1083
1081
|
async function readRuntimeSupervisorRun(runDir, opts = {}) {
|
|
1084
1082
|
const journalPath = join(runDir, JOURNAL_FILE);
|
|
1085
|
-
const rawJournal = opts.strict === true ? await readFile(journalPath, "utf8") : await readMaybe
|
|
1083
|
+
const rawJournal = opts.strict === true ? await readFile(journalPath, "utf8") : await readMaybe(journalPath);
|
|
1086
1084
|
const resultPath = join(runDir, RESULT_FILE);
|
|
1087
1085
|
const failurePath = join(runDir, FAILURE_FILE);
|
|
1088
|
-
const resultText = await readMaybe
|
|
1089
|
-
const failureText = await readMaybe
|
|
1086
|
+
const resultText = await readMaybe(resultPath);
|
|
1087
|
+
const failureText = await readMaybe(failurePath);
|
|
1090
1088
|
const failure = parseOptionalRecord(failureText, failurePath);
|
|
1091
1089
|
assertFailureRecord(failure, failurePath);
|
|
1092
1090
|
const result = parseOptionalRecord(resultText, resultPath);
|
|
@@ -1152,188 +1150,21 @@ async function isFile(path) {
|
|
|
1152
1150
|
});
|
|
1153
1151
|
}
|
|
1154
1152
|
//#endregion
|
|
1155
|
-
//#region src/supervisor-run/
|
|
1153
|
+
//#region src/supervisor-run/reader.ts
|
|
1156
1154
|
/**
|
|
1157
|
-
*
|
|
1158
|
-
* supervisor writes — `<runDir>/ws/.agent/supervisor/<id>/{journal.jsonl,
|
|
1159
|
-
* state.json, progress.ndjson, workers/*.ndjson}` alongside the run's
|
|
1160
|
-
* `result.json` / `judge.json` / `driver.log` / delivered patch. Runs written
|
|
1161
|
-
* before the `.agent` rename live under `<ws>/.loops/supervisor/<id>` and are
|
|
1162
|
-
* still found via fallback.
|
|
1163
|
-
*
|
|
1164
|
-
* Nothing in `analyze.ts` knows this layout exists. A different store (an
|
|
1165
|
-
* archive, an object bucket, a database) implements the same interface and
|
|
1166
|
-
* gets the same report.
|
|
1155
|
+
* Runtime supervisor-run reader and report writers.
|
|
1167
1156
|
*
|
|
1168
|
-
*
|
|
1169
|
-
*
|
|
1170
|
-
|
|
1171
|
-
async function readMaybe(path) {
|
|
1172
|
-
return readFile(path, "utf8").catch(() => null);
|
|
1173
|
-
}
|
|
1174
|
-
/**
|
|
1175
|
-
* Locate the (single) supervisor run dir under `<ws>/.agent/supervisor`, falling back to the
|
|
1176
|
-
* pre-rename `<ws>/.loops/supervisor` so runs written by older supervisors stay analyzable.
|
|
1177
|
-
*/
|
|
1178
|
-
async function findSupervisorRunDirIn(ws) {
|
|
1179
|
-
for (const stateDir of [".agent", ".loops"]) {
|
|
1180
|
-
const root = join(ws, stateDir, "supervisor");
|
|
1181
|
-
const dirs = (await readdir(root, { withFileTypes: true }).catch(() => [])).filter((e) => e.isDirectory()).map((e) => join(root, e.name));
|
|
1182
|
-
if (dirs[0] !== void 0) return dirs[0];
|
|
1183
|
-
}
|
|
1184
|
-
return null;
|
|
1185
|
-
}
|
|
1186
|
-
/**
|
|
1187
|
-
* Read a loops supervisor run directory into source bytes. Never throws on a
|
|
1188
|
-
* missing artifact — an absent file becomes a `null` field, which is what makes
|
|
1189
|
-
* the dependent metric `unavailable` instead of 0.
|
|
1157
|
+
* Runtime's file-backed supervision context is the only live on-disk contract.
|
|
1158
|
+
* Other stores implement `SupervisorRunReader` and feed the same pure analyzer;
|
|
1159
|
+
* this module owns path discovery and report output for Runtime runs.
|
|
1190
1160
|
*/
|
|
1191
|
-
async function readLoopsSupervisorRun(runDir, opts = {}) {
|
|
1192
|
-
const ws = opts.ws ?? join(runDir, "ws");
|
|
1193
|
-
const supRunDir = await findSupervisorRunDirIn(ws);
|
|
1194
|
-
const result = await readMaybe(join(runDir, "result.json"));
|
|
1195
|
-
const resultObj = parseJson(result);
|
|
1196
|
-
const journal = supRunDir === null ? null : await readMaybe(join(supRunDir, "journal.jsonl"));
|
|
1197
|
-
const journalWorkerSpawns = parseJsonl(journal).filter((event) => event.kind === "spawned" && typeof event.parent === "string" && event.role !== "supervisor").length;
|
|
1198
|
-
let workers = null;
|
|
1199
|
-
let workersMissingReason = null;
|
|
1200
|
-
const workerCwds = [];
|
|
1201
|
-
let workerStarts = 0;
|
|
1202
|
-
if (supRunDir === null) workersMissingReason = `no supervisor run dir under ${join(ws, ".agent", "supervisor")} (or legacy ${join(ws, ".loops", "supervisor")})`;
|
|
1203
|
-
else {
|
|
1204
|
-
const workersDir = join(supRunDir, "workers");
|
|
1205
|
-
const entries = await readdir(workersDir).catch(() => null);
|
|
1206
|
-
if (entries === null) workersMissingReason = `workers/ directory absent under ${supRunDir}`;
|
|
1207
|
-
else {
|
|
1208
|
-
const labels = [...new Set(entries.filter((f) => f.endsWith(".ndjson")).map((f) => f.replace(/\.inbox\.ndjson$/, "").replace(/\.ndjson$/, "")))].sort();
|
|
1209
|
-
workers = [];
|
|
1210
|
-
for (const label of labels) {
|
|
1211
|
-
const events = await readMaybe(join(workersDir, `${label}.ndjson`));
|
|
1212
|
-
const inbox = await readMaybe(join(workersDir, `${label}.inbox.ndjson`));
|
|
1213
|
-
const patch = await readMaybe(join(workersDir, `${label}.patch`));
|
|
1214
|
-
const startedRows = parseJsonl(events).filter((event) => event.kind === "started");
|
|
1215
|
-
workerStarts += startedRows.length;
|
|
1216
|
-
const startedIds = startedRows.map((event) => typeof event.workerId === "string" ? event.workerId : typeof event.agentId === "string" ? event.agentId : null).filter((id) => id !== null);
|
|
1217
|
-
const distinctStartedIds = new Set(startedIds);
|
|
1218
|
-
const workerId = startedRows.length > 0 && startedIds.length === startedRows.length && distinctStartedIds.size === 1 ? startedIds[0] : void 0;
|
|
1219
|
-
workers.push({
|
|
1220
|
-
...workerId === void 0 ? {} : { workerId },
|
|
1221
|
-
label,
|
|
1222
|
-
events,
|
|
1223
|
-
inbox,
|
|
1224
|
-
patchBytes: patch === null ? null : Buffer.byteLength(patch)
|
|
1225
|
-
});
|
|
1226
|
-
for (const ev of startedRows) if (ev.kind === "started" && typeof ev.cwd === "string") workerCwds.push(ev.cwd);
|
|
1227
|
-
}
|
|
1228
|
-
}
|
|
1229
|
-
}
|
|
1230
|
-
let harnessWorkerTokens = null;
|
|
1231
|
-
let harnessMissingReason = null;
|
|
1232
|
-
let workerCwdsWithoutSessions = 0;
|
|
1233
|
-
if (opts.opencodeDb === null) harnessMissingReason = "opencode join disabled";
|
|
1234
|
-
else if (workerCwds.length === 0) harnessMissingReason = "no worker clone cwds in workers/*.ndjson (nothing to join)";
|
|
1235
|
-
else {
|
|
1236
|
-
const db = await openOpencodeDb(opts.opencodeDb ?? DEFAULT_OPENCODE_DB);
|
|
1237
|
-
if (db === null) harnessMissingReason = `opencode session store unreadable at ${opts.opencodeDb ?? DEFAULT_OPENCODE_DB}`;
|
|
1238
|
-
else try {
|
|
1239
|
-
const seen = /* @__PURE__ */ new Set();
|
|
1240
|
-
let sessions = 0;
|
|
1241
|
-
let input = 0;
|
|
1242
|
-
let output = 0;
|
|
1243
|
-
const distinctWorkerCwds = new Set(workerCwds);
|
|
1244
|
-
for (const cwd of distinctWorkerCwds) {
|
|
1245
|
-
const rows = findOpencodeSessionsByDirectory(db, cwd);
|
|
1246
|
-
if (rows.length === 0) workerCwdsWithoutSessions += 1;
|
|
1247
|
-
for (const row of rows) {
|
|
1248
|
-
if (seen.has(row.id)) continue;
|
|
1249
|
-
seen.add(row.id);
|
|
1250
|
-
sessions += 1;
|
|
1251
|
-
input += row.tokensInput;
|
|
1252
|
-
output += row.tokensOutput + row.tokensReasoning;
|
|
1253
|
-
}
|
|
1254
|
-
}
|
|
1255
|
-
harnessWorkerTokens = {
|
|
1256
|
-
store: "opencode",
|
|
1257
|
-
sessions,
|
|
1258
|
-
input,
|
|
1259
|
-
output
|
|
1260
|
-
};
|
|
1261
|
-
if (workerCwdsWithoutSessions > 0) harnessMissingReason = `${workerCwdsWithoutSessions}/${distinctWorkerCwds.size} worker clone cwds have no opencode session`;
|
|
1262
|
-
} finally {
|
|
1263
|
-
db.close();
|
|
1264
|
-
}
|
|
1265
|
-
}
|
|
1266
|
-
const workerInvocations = Math.max(journalWorkerSpawns, workerStarts);
|
|
1267
|
-
const workerTokenGaps = [];
|
|
1268
|
-
if (workerCwds.length < workerInvocations) workerTokenGaps.push(`${workerInvocations - workerCwds.length}/${workerInvocations} worker invocations have no clone cwd for the opencode token join`);
|
|
1269
|
-
if (workerInvocations > 0 && harnessWorkerTokens === null) workerTokenGaps.push(harnessMissingReason ?? "worker harness token join unavailable");
|
|
1270
|
-
else if (workerCwdsWithoutSessions > 0 && harnessMissingReason !== null) workerTokenGaps.push(harnessMissingReason);
|
|
1271
|
-
const workerTokenLimit = workerTokenGaps.length === 0 ? null : workerTokenGaps.join("; ");
|
|
1272
|
-
const patchPath = opts.patchPath ?? (typeof resultObj?.patchPath === "string" ? resultObj.patchPath : null);
|
|
1273
|
-
let judge = await readMaybe(join(runDir, "judge.json"));
|
|
1274
|
-
let judgeSource = judge === null ? null : join(runDir, "judge.json");
|
|
1275
|
-
if (judge === null && opts.ledgerPath !== void 0) {
|
|
1276
|
-
const row = await findLedgerRow(opts.ledgerPath, runDir);
|
|
1277
|
-
if (row !== null) {
|
|
1278
|
-
judge = JSON.stringify(row);
|
|
1279
|
-
judgeSource = `${opts.ledgerPath} (ledger row)`;
|
|
1280
|
-
}
|
|
1281
|
-
}
|
|
1282
|
-
return {
|
|
1283
|
-
runRef: runDir,
|
|
1284
|
-
instanceId: typeof resultObj?.iid === "string" ? resultObj.iid : instanceIdFromPath(runDir),
|
|
1285
|
-
arm: typeof resultObj?.arm === "string" ? resultObj.arm : basename(runDir),
|
|
1286
|
-
supRunDir,
|
|
1287
|
-
journal,
|
|
1288
|
-
brainLog: supRunDir === null ? null : await readMaybe(join(supRunDir, "brain.jsonl")),
|
|
1289
|
-
state: supRunDir === null ? null : await readMaybe(join(supRunDir, "state.json")),
|
|
1290
|
-
progress: supRunDir === null ? null : await readMaybe(join(supRunDir, "progress.ndjson")),
|
|
1291
|
-
workers,
|
|
1292
|
-
workersMissingReason,
|
|
1293
|
-
result,
|
|
1294
|
-
judge,
|
|
1295
|
-
judgeSource,
|
|
1296
|
-
patch: patchPath === null ? null : await readMaybe(patchPath),
|
|
1297
|
-
driverLog: await readMaybe(join(runDir, "driver.log")),
|
|
1298
|
-
harnessWorkerTokens,
|
|
1299
|
-
harnessMissingReason,
|
|
1300
|
-
limits: {
|
|
1301
|
-
...NO_SOURCE_LIMITS,
|
|
1302
|
-
workerTokens: workerTokenLimit
|
|
1303
|
-
},
|
|
1304
|
-
traceCommand: null
|
|
1305
|
-
};
|
|
1306
|
-
}
|
|
1307
|
-
/** The loops on-disk layout, as a `SupervisorRunReader`. */
|
|
1308
|
-
function loopsSupervisorRunReader(runDir, opts = {}) {
|
|
1309
|
-
return {
|
|
1310
|
-
runRef: runDir,
|
|
1311
|
-
read: () => readLoopsSupervisorRun(runDir, opts)
|
|
1312
|
-
};
|
|
1313
|
-
}
|
|
1314
|
-
/** The ledger row whose `runDir` is this run (falling back to iid + arm match). */
|
|
1315
|
-
async function findLedgerRow(ledgerPath, runDir) {
|
|
1316
|
-
const rows = parseJsonl(await readMaybe(ledgerPath));
|
|
1317
|
-
const exact = rows.find((r) => r.runDir === runDir);
|
|
1318
|
-
if (exact !== void 0) return exact;
|
|
1319
|
-
const iid = instanceIdFromPath(runDir);
|
|
1320
|
-
const arm = basename(runDir);
|
|
1321
|
-
return rows.find((r) => r.iid === iid && r.arm === arm) ?? null;
|
|
1322
|
-
}
|
|
1323
|
-
/** `<outDir>/runs/<iid>/<arm>` → `<iid>`. */
|
|
1324
|
-
function instanceIdFromPath(runDir) {
|
|
1325
|
-
const parts = runDir.split("/").filter(Boolean);
|
|
1326
|
-
const armIdx = parts.length - 1;
|
|
1327
|
-
const iid = parts[armIdx - 1];
|
|
1328
|
-
return parts[armIdx - 2] === "runs" && iid !== void 0 ? iid : null;
|
|
1329
|
-
}
|
|
1330
1161
|
/**
|
|
1331
|
-
* Analyze a supervisor run. Accepts a run directory
|
|
1332
|
-
*
|
|
1162
|
+
* Analyze a supervisor run. Accepts a Runtime run directory, any
|
|
1163
|
+
* `SupervisorRunReader`, or already-read source bytes — so a
|
|
1333
1164
|
* caller with its own store never has to touch the filesystem layout.
|
|
1334
1165
|
*/
|
|
1335
|
-
async function analyzeSupervisorRun(input
|
|
1336
|
-
if (typeof input === "string") return analyzeSupervisorRunSources(await
|
|
1166
|
+
async function analyzeSupervisorRun(input) {
|
|
1167
|
+
if (typeof input === "string") return analyzeSupervisorRunSources(await readRuntimeSupervisorRun(input));
|
|
1337
1168
|
if (isReader(input)) return analyzeSupervisorRunSources(await input.read());
|
|
1338
1169
|
return analyzeSupervisorRunSources(input);
|
|
1339
1170
|
}
|
|
@@ -1347,7 +1178,7 @@ function isReader(input) {
|
|
|
1347
1178
|
* every metric says why.
|
|
1348
1179
|
*/
|
|
1349
1180
|
async function writeSupervisorRunReport(runDir, opts = {}) {
|
|
1350
|
-
const report = analyzeSupervisorRunSources(await
|
|
1181
|
+
const report = analyzeSupervisorRunSources(await readRuntimeSupervisorRun(runDir));
|
|
1351
1182
|
const md = renderSupervisorRunMarkdown(report);
|
|
1352
1183
|
const dest = opts.reportDir ?? runDir;
|
|
1353
1184
|
const stem = opts.reportDir === void 0 ? "run-report" : supervisorReportStem(runDir);
|
|
@@ -1416,13 +1247,13 @@ async function reportSupervisorRound(outDir, opts = {}) {
|
|
|
1416
1247
|
return rollup;
|
|
1417
1248
|
}
|
|
1418
1249
|
/**
|
|
1419
|
-
* Every
|
|
1250
|
+
* Every Runtime supervisor run below `root`.
|
|
1420
1251
|
*
|
|
1421
1252
|
* When `root` itself is one run, return no children so callers can distinguish
|
|
1422
1253
|
* a single report from a parent-directory rollup.
|
|
1423
1254
|
*/
|
|
1424
1255
|
async function findSupervisorRunDirs(root) {
|
|
1425
|
-
if (await isRuntimeSupervisorRunDir(root)
|
|
1256
|
+
if (await isRuntimeSupervisorRunDir(root)) return [];
|
|
1426
1257
|
const found = [];
|
|
1427
1258
|
const walk = async (dir, depth) => {
|
|
1428
1259
|
if (depth > 8) return;
|
|
@@ -1431,7 +1262,7 @@ async function findSupervisorRunDirs(root) {
|
|
|
1431
1262
|
if (!e.isDirectory()) continue;
|
|
1432
1263
|
if (e.name === "node_modules" || e.name === ".git") continue;
|
|
1433
1264
|
const full = join(dir, e.name);
|
|
1434
|
-
if (await isRuntimeSupervisorRunDir(full)
|
|
1265
|
+
if (await isRuntimeSupervisorRunDir(full)) {
|
|
1435
1266
|
found.push(full);
|
|
1436
1267
|
continue;
|
|
1437
1268
|
}
|
|
@@ -1446,7 +1277,7 @@ async function findSupervisorRunDirs(root) {
|
|
|
1446
1277
|
/**
|
|
1447
1278
|
* `agent-eval supervisor-run report <runDir>` takes one run directory and
|
|
1448
1279
|
* prints the report the module already renders. The command reads through
|
|
1449
|
-
* `analyzeSupervisorRun`, so a Runtime run
|
|
1280
|
+
* `analyzeSupervisorRun`, so a Runtime run directory takes the
|
|
1450
1281
|
* same path and the status comes from the record `terminal-record.ts` names.
|
|
1451
1282
|
*
|
|
1452
1283
|
* Exit codes follow the other self-parsing subcommands: 2 for a usage error,
|
|
@@ -1547,6 +1378,6 @@ async function runSupervisorRunCommand(argv, io = PROCESS_IO) {
|
|
|
1547
1378
|
}
|
|
1548
1379
|
}
|
|
1549
1380
|
//#endregion
|
|
1550
|
-
export {
|
|
1381
|
+
export { writeSupervisorRunReport as a, readRuntimeSupervisorRun as c, renderSupervisorRunHeadline as d, renderSupervisorRunMarkdown as f, rollupSupervisorRuns as h, reportSupervisorRound as i, runtimeSupervisorRunReader as l, parsePatch as m, analyzeSupervisorRun as n, writeSupervisorRunReportSafe as o, analyzeSupervisorRunSources as p, findSupervisorRunDirs as r, isRuntimeSupervisorRunDir as s, runSupervisorRunCommand as t, renderSupervisorRollupMarkdown as u };
|
|
1551
1382
|
|
|
1552
|
-
//# sourceMappingURL=report-command-
|
|
1383
|
+
//# sourceMappingURL=report-command-BvqaxlVG.js.map
|