@tangle-network/agent-eval 0.150.1 → 0.150.2

package/dist/supervisor-run/index.js

@@ -1,5 +1,6 @@
+import { o as summarizeNumberSeries } from "../descriptive-jDOuI6mz.js";
 import { d as transcriptFromEntries, i as openOpencodeDb, l as parseClaudeEntries, r as findOpencodeSessionsByDirectory, t as DEFAULT_OPENCODE_DB } from "../opencode-sqlite-DJWAXLms.js";
-import { a as parseJsonl, c as SUPERVISOR_RUN_INTEGRITY_SCHEMA, i as parseJson, n as supervisorRunRolloutLines, o as parseSupervisorTree, r as asRecord, s as workerSourceKey, t as analyzeSupervisorRunIntegrity } from "../integrity-DysDBWDu.js";
+import { a as parseJsonl, c as SUPERVISOR_RUN_INTEGRITY_SCHEMA, i as parseJson, n as supervisorRunRolloutLines, o as parseSupervisorTree, r as asRecord, s as workerSourceKey, t as analyzeSupervisorRunIntegrity } from "../integrity-DL91tucI.js";
 import { basename, dirname, join } from "node:path";
 import { appendFile, mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises";
 //#region src/supervisor-run/types.ts
@@ -44,7 +45,7 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
 		gaps.push(`${what}: ${reason}`);
 		return unavailable(reason);
 	};
-	const journalMissing = src.supRunDir === null ? "no supervisor run dir under <ws>/.agent/supervisor (or legacy <ws>/.loops/supervisor)" : "journal.jsonl absent";
+	const journalMissing = src.journalMissingReason ?? (src.supRunDir === null ? "no supervisor run dir under <ws>/.agent/supervisor (or legacy <ws>/.loops/supervisor)" : "journal.jsonl absent");
 	const haveJournal = src.journal !== null;
 	const tree = parseSupervisorTree(src);
 	const state = tree.state;
@@ -68,7 +69,22 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
 		const matches = spawnsForSource(worker);
 		return matches.length === 1 ? matches[0] ?? null : null;
 	};
-	const supervisorWallMs = startedAt !== null && completedAt !== null && completedAt >= startedAt ? completedAt - startedAt : !haveJournal ? gap("supervisorWallMs", journalMissing) : gap("supervisorWallMs", "no parseable start/complete timestamps in state.json or journal");
+	const wallSpanStart = startedAt ?? tree.firstEventAt;
+	const wallSpanEnd = tree.lastEventAt;
+	let supervisorWallMs;
+	let supervisorWallSource;
+	if (startedAt !== null && completedAt !== null && completedAt >= startedAt) {
+		supervisorWallMs = completedAt - startedAt;
+		supervisorWallSource = "stamps";
+	} else if (completedAt === null && wallSpanStart !== null && wallSpanEnd !== null && wallSpanEnd >= wallSpanStart) {
+		supervisorWallMs = wallSpanEnd - wallSpanStart;
+		supervisorWallSource = "journal-span";
+	} else {
+		const reason = !haveJournal ? journalMissing : "no parseable start/complete timestamps in state.json or journal";
+		supervisorWallMs = gap("supervisorWallMs", reason);
+		supervisorWallSource = unavailable(reason);
+	}
+	const wallEndAt = completedAt ?? (supervisorWallSource === "journal-span" ? wallSpanEnd : null);
 	const steerRows = [];
 	let steerQueuedTotal = 0;
 	let steerDeliveredTotal = 0;
@@ -135,8 +151,8 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
 		if (live > maxConcurrency) maxConcurrency = live;
 		prev = step.at;
 	}
-	if (prev !== null && completedAt !== null && completedAt >= prev) {
-		const span = completedAt - prev;
+	if (prev !== null && wallEndAt !== null && wallEndAt >= prev) {
+		const span = wallEndAt - prev;
 		if (live === 0) idleMs += span;
 		sumWorkerWallMs += span * live;
 	}
@@ -203,6 +219,7 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
 		delegationDepth: haveJournal ? delegationDepth : unavailable(journalMissing),
 		timeToFirstSpawnMs: startedAt !== null && firstWorkerSpawnAt !== null ? firstWorkerSpawnAt - startedAt : haveJournal ? unavailable("no worker spawn timestamps") : unavailable(journalMissing),
 		supervisorWallMs,
+		supervisorWallSource,
 		idleMs: isUnavailable(supervisorWallMs) ? unavailable(supervisorWallMs.unavailable) : idleMs,
 		idlePct: isUnavailable(supervisorWallMs) || supervisorWallMs === 0 ? isUnavailable(supervisorWallMs) ? unavailable(supervisorWallMs.unavailable) : unavailable("supervisor wall is 0ms") : round(idleMs / supervisorWallMs * 100, 1),
 		workerUtilization: isUnavailable(supervisorWallMs) || supervisorWallMs === 0 ? isUnavailable(supervisorWallMs) ? unavailable(supervisorWallMs.unavailable) : unavailable("supervisor wall is 0ms") : round(sumWorkerWallMs / supervisorWallMs, 3)
@@ -267,8 +284,24 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
 	const workerOut = workerTokenLimit !== null ? unavailable(workerTokenLimit) : sq !== null ? journalWorkerOut + sq.output : haveJournal ? journalWorkerOut : unavailable(harnessGapReason);
 	const stateResult = asRecord(state?.result);
 	const stateUsd = typeof stateResult.spentUsd === "number" ? stateResult.spentUsd : null;
+	const resultSpentTotal = asRecord(result?.spentTotal);
+	const resultCloseUsd = typeof resultSpentTotal.usd === "number" && Number.isFinite(resultSpentTotal.usd) ? resultSpentTotal.usd : null;
+	const closeUsd = stateUsd ?? resultCloseUsd;
 	const usdLimit = src.limits.spendUsd;
 	const totalUsd = usdLimit !== null ? gap("totalUsd", usdLimit) : stateUsd !== null ? round(stateUsd, 6) : haveJournal ? round(tree.brain.usd + journalWorkerUsd, 6) : gap("totalUsd", journalMissing);
+	const journalSpendRecords = tree.brain.meteredCount + rootChildCloses.filter((close) => close.hasSpend).length;
+	const journalDerivedAvailable = usdLimit === null && haveJournal;
+	const closeRecordAvailable = usdLimit === null && closeUsd !== null;
+	const spend = {
+		journalDerived: {
+			usd: journalDerivedAvailable ? round(tree.brain.usd + journalWorkerUsd, 6) : unavailable(usdLimit ?? journalMissing),
+			records: journalDerivedAvailable ? journalSpendRecords : 0
+		},
+		closeRecord: {
+			usd: closeRecordAvailable ? round(closeUsd, 6) : unavailable(usdLimit ?? "no close record: neither state.json result.spentUsd nor result.json spentTotal.usd is present"),
+			records: closeRecordAvailable ? 1 : 0
+		}
+	};
 	const perWorker = (src.workers ?? []).map((w) => {
 		const f = tree.workerLogs.get(workerSourceKey(w));
 		const matchingSpawns = spawnsForSource(w);
@@ -297,7 +330,7 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
 			score: close?.score ?? f?.score ?? null
 		};
 	});
-	const walls = perWorker.map((w) => w.wallMs).filter((w) => w !== null).sort((a, b) => a - b);
+	const wallDistribution = summarizeNumberSeries(perWorker.map((w) => w.wallMs).filter((w) => w !== null));
 	const brainCalls = parseJsonl(src.brainLog);
 	const managerTokenLimit = src.limits.managerTokens;
 	const economics = {
@@ -318,17 +351,11 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
 			usd: usdLimit !== null ? unavailable(usdLimit) : haveJournal ? round(journalWorkerUsd, 6) : unavailable(journalMissing),
 			source: workerTokenLimit !== null ? workerTokenLimit : sq !== null ? `journal settled spend + ${sq.store} sessions (n=${sq.sessions})` : `journal settled spend only — ${src.harnessMissingReason ?? "harness session store unavailable"}`
 		},
+		spend,
 		totalUsd,
 		totalUsdSource: usdLimit !== null ? usdLimit : stateUsd !== null ? `state.json result.spentUsd${rootChildCloses.length > 0 && journalWorkerUsd === 0 ? " — brain-priced only; worker CLI inference is unpriced (see worker token counts)" : ""}` : haveJournal ? "journal metered + settled usd" : journalMissing,
 		costPerAcceptedPatchUsd: isUnavailable(totalUsd) ? unavailable(totalUsd.unavailable) : isUnavailable(decision.accepted) ? unavailable(decision.accepted.unavailable) : decision.accepted === 0 ? unavailable("no accepted worker patch (cost has no denominator)") : round(totalUsd / decision.accepted, 6),
-		workerWallMsDistribution: walls.length === 0 ? unavailable(src.workers === null ? workersGapReason : "no worker start/finish pairs captured") : {
-			n: walls.length,
-			min: walls[0],
-			p50: quantile(walls, .5),
-			p90: quantile(walls, .9),
-			max: walls[walls.length - 1],
-			sum: walls.reduce((a, b) => a + b, 0)
-		},
+		workerWallMsDistribution: wallDistribution === null ? unavailable(src.workers === null ? workersGapReason : "no worker start/finish pairs captured") : wallDistribution,
 		perWorker: src.workers === null ? unavailable(workersGapReason) : perWorker
 	};
 	const patchStats = src.patch === null ? gap("patch", src.limits.deliverables ?? "delivered patch file absent") : parsePatch(src.patch);
@@ -386,10 +413,6 @@ function round(v, digits) {
 	const f = 10 ** digits;
 	return Math.round(v * f) / f;
 }
-function quantile(sorted, q) {
-	if (sorted.length === 0) return 0;
-	return sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil(q * sorted.length) - 1))];
-}
 /** Unified-diff stats. Counts `+++ b/<path>` targets, body +/- lines, and test-file touches. */
 function parsePatch(text) {
 	const files = /* @__PURE__ */ new Set();
@@ -435,6 +458,8 @@ function rollupSupervisorRuns(reports) {
 	const spawnVals = known(reports.map((r) => r.orchestration.workersSpawned));
 	const acceptVals = known(reports.map((r) => r.decision.accepted));
 	const usdVals = known(reports.map((r) => r.economics.totalUsd));
+	const journalSpendVals = known(reports.map((r) => r.economics.spend.journalDerived.usd));
+	const closeSpendVals = known(reports.map((r) => r.economics.spend.closeRecord.usd));
 	const resolvedVals = known(reports.map((r) => r.outcome.judgeResolved));
 	const sum = (xs) => xs.reduce((a, b) => a + b, 0);
 	const mean = (xs) => xs.length === 0 ? unavailable("no cell reported this metric") : round(sum(xs) / xs.length, 3);
@@ -461,6 +486,16 @@ function rollupSupervisorRuns(reports) {
 		workersSpawnedTotal: spawnVals.length === 0 ? unavailable("no cell reported spawns") : sum(spawnVals),
 		acceptedTotal: acceptVals.length === 0 ? unavailable("no cell reported acceptance") : sum(acceptVals),
 		usdTotal: usdVals.length === 0 ? unavailable("no cell reported spend") : round(sum(usdVals), 6),
+		spendUsd: {
+			journalDerived: {
+				value: journalSpendVals.length === 0 ? unavailable("no cell measured journal-derived spend") : round(sum(journalSpendVals), 6),
+				runs: journalSpendVals.length
+			},
+			closeRecord: {
+				value: closeSpendVals.length === 0 ? unavailable("no cell carried a close record") : round(sum(closeSpendVals), 6),
+				runs: closeSpendVals.length
+			}
+		},
 		resolvedCount: resolvedVals.length === 0 ? unavailable("no cell reported a judge verdict") : resolvedVals.filter((v) => v === true).length,
 		perCell
 	};
@@ -961,7 +996,7 @@ function renderSupervisorRunHeadline(r) {
 		`RUN-REPORT ${r.instanceId ?? "?"} [${r.arm ?? "?"}]`,
 		`  steers=${steerNote}`,
 		`  waves=${showMeasured(o.waves)} sizes=${isUnavailable(o.waveSizes) ? `unavailable — ${o.waveSizes.unavailable}` : `[${o.waveSizes.join(",")}]`} workers=${showMeasured(o.workersSpawned)} settled=${showMeasured(o.workersSettled)} cancelled=${showMeasured(o.workersCancelled)}`,
-		`  concurrency max=${showMeasured(o.maxConcurrency)} utilization=${showMeasured(o.workerUtilization)} idle=${fmtMs(o.idleMs)} (${showMeasured(o.idlePct)}%) wall=${fmtMs(o.supervisorWallMs)}`,
+		`  concurrency max=${showMeasured(o.maxConcurrency)} utilization=${showMeasured(o.workerUtilization)} idle=${fmtMs(o.idleMs)} (${showMeasured(o.idlePct)}%) wall=${fmtMs(o.supervisorWallMs)}${o.supervisorWallSource === "journal-span" ? " (journal-span lower bound)" : ""}`,
 		`  respawns=${showMeasured(o.respawns)} evidence→respawn=${showMeasured(r.decision.observeThenRespawn)} blind-respawn=${showMeasured(r.decision.respawnWithoutEvidence)} depth=${showMeasured(o.delegationDepth)}`,
 		`  accepted=${showMeasured(r.decision.accepted)} rejected=${showMeasured(r.decision.rejected)} empty-pass=${showMeasured(r.decision.emptyPass)}`,
 		`  brain=$${showMeasured(r.economics.brain.usd)} total=$${showMeasured(r.economics.totalUsd)} judge.resolved=${showMeasured(r.outcome.judgeResolved)} score=${showMeasured(r.outcome.judgeScore)} verify=${showMeasured(r.outcome.verifyPass)}`,
@@ -1008,7 +1043,7 @@ function renderSupervisorRunMarkdown(r) {
 	out.push(`| Repeated labels | ${isUnavailable(o.repeatedLabels) ? showMeasured(o.repeatedLabels) : o.repeatedLabels.length === 0 ? "none" : o.repeatedLabels.join(", ")} |`);
 	out.push(`| Delegation depth | ${showMeasured(o.delegationDepth)} |`);
 	out.push(`| Time to first spawn | ${fmtMs(o.timeToFirstSpawnMs)} |`);
-	out.push(`| Supervisor wall | ${fmtMs(o.supervisorWallMs)} |`);
+	out.push(`| Supervisor wall | ${fmtMs(o.supervisorWallMs)}${isUnavailable(o.supervisorWallSource) ? "" : ` (source: ${o.supervisorWallSource}${o.supervisorWallSource === "journal-span" ? ", lower bound" : ""})`} |`);
 	out.push(`| Idle (zero live workers) | ${fmtMs(o.idleMs)} (${showMeasured(o.idlePct)}%) |`);
 	out.push(`| Worker utilization (Σ worker wall ÷ supervisor wall) | ${showMeasured(o.workerUtilization)} |`);
 	out.push("");
@@ -1044,6 +1079,7 @@ function renderSupervisorRunMarkdown(r) {
 	if (!isUnavailable(e.brainTruncations) && e.brainTruncations > 0) out.push(`- **BRAIN OUTPUT TRUNCATED: ${e.brainTruncations} completion(s) hit \`finish_reason: "length"\`** — the supervisor acted on a half-written plan. Its output ceiling is too low; see \`brain.jsonl\` for the per-call \`req_max_tokens\`.`);
 	else out.push(`- Brain completions truncated (finish_reason=length): ${showMeasured(e.brainTruncations)}`);
 	out.push(`- Total USD: ${showMeasured(e.totalUsd)} (source: ${e.totalUsdSource})`);
+	out.push(`- Spend measured two ways: journal-derived $${showMeasured(e.spend.journalDerived.usd)} over ${e.spend.journalDerived.records} journal record(s) (execution accounting) · close-record $${showMeasured(e.spend.closeRecord.usd)} over ${e.spend.closeRecord.records} close record(s) (billing-shaped). Divergence is a signal, not an error.`);
 	out.push(`- Cost per accepted patch: ${showMeasured(e.costPerAcceptedPatchUsd)}`);
 	if (isUnavailable(e.workerWallMsDistribution)) out.push(`- Worker wall distribution: unavailable — ${e.workerWallMsDistribution.unavailable}`);
 	else {
@@ -1101,6 +1137,7 @@ function renderSupervisorRollupMarkdown(rollup, title = "Round rollup") {
 	out.push(`- Idle share (mean): ${showMeasured(rollup.idlePctMean)}%`);
 	out.push(`- Workers spawned: ${showMeasured(rollup.workersSpawnedTotal)} · accepted: ${showMeasured(rollup.acceptedTotal)}`);
 	out.push(`- Spend: $${showMeasured(rollup.usdTotal)} · judged resolved: ${showMeasured(rollup.resolvedCount)}/${rollup.cells}`);
+	out.push(`- Spend measured two ways: journal-derived $${showMeasured(rollup.spendUsd.journalDerived.value)} over ${rollup.spendUsd.journalDerived.runs}/${rollup.cells} runs (execution accounting) · close-record $${showMeasured(rollup.spendUsd.closeRecord.value)} over ${rollup.spendUsd.closeRecord.runs}/${rollup.cells} runs (billing-shaped)`);
 	out.push("");
 	out.push("| Instance | Arm | Steers | Waves | Utilization | Idle % | Resolved | USD |");
 	out.push("|---|---|---:|---:|---:|---:|---|---:|");
@@ -1330,15 +1367,56 @@ function runtimeState(root, startedAt, result, trajectory, resultPath, trajector
 	});
 }
 /**
+* Sources for a run dir whose spawn journal does not exist. Mirrors the
+* absent shape `readLoopsSupervisorRun` returns for a missing store: every
+* journal-dependent metric downstream reads `unavailable`, never 0.
+*/
+function absentRuntimeSupervisorRun(runDir, resultText) {
+	const reason = `no Runtime spawn journal (${JOURNAL_FILE}) under ${runDir}`;
+	return {
+		runRef: runDir,
+		instanceId: null,
+		arm: null,
+		supRunDir: null,
+		journal: null,
+		journalMissingReason: reason,
+		brainLog: null,
+		brainLogMissingReason: "Runtime FileRunContext records spend but not model completion finish reasons",
+		state: null,
+		progress: null,
+		workers: null,
+		workersMissingReason: reason,
+		result: resultText,
+		judge: null,
+		judgeSource: null,
+		patch: null,
+		driverLog: null,
+		harnessWorkerTokens: null,
+		harnessMissingReason: "Runtime FileRunContext has no external worker-token join",
+		limits: NO_SOURCE_LIMITS,
+		rootTranscriptRef: null,
+		traceCommand: "unavailable — Runtime FileRunContext records no provider-session trace identity"
+	};
+}
+/**
 * Read one agent-runtime `createFileRunContext(dir)` directory.
 *
+* A run dir without `spawn-journal.jsonl` returns the same absent-shaped
+* sources `readLoopsSupervisorRun` returns for a missing store: `journal` and
+* `workers` null, each with its reason, so every dependent metric reads
+* `unavailable` — never 0 and never a throw. Pass `strict: true` to throw on
+* the missing journal instead. A journal that exists but cannot be parsed
+* always throws: a corrupt journal is a defect, not an absence.
+*
 * The reader translates storage envelopes only. It does not assign research
 * roles, interpret artifacts, or turn process completion into a quality
 * verdict.
 */
-async function readRuntimeSupervisorRun(runDir) {
+async function readRuntimeSupervisorRun(runDir, opts = {}) {
 	const journalPath = join(runDir, JOURNAL_FILE);
-	const normalized = parseEnvelopeJournal(await readFile(journalPath, "utf8"), journalPath);
+	const rawJournal = opts.strict === true ? await readFile(journalPath, "utf8") : await readMaybe$1(journalPath);
+	if (rawJournal === null) return absentRuntimeSupervisorRun(runDir, await readMaybe$1(join(runDir, RESULT_FILE)));
+	const normalized = parseEnvelopeJournal(rawJournal, journalPath);
 	const resultText = await readMaybe$1(join(runDir, RESULT_FILE));
 	const trajectoryText = await readMaybe$1(join(runDir, TRAJECTORY_FILE));
 	const result = parseOptionalRecord(resultText, join(runDir, RESULT_FILE));
@@ -1381,10 +1459,10 @@ async function readRuntimeSupervisorRun(runDir) {
 	};
 }
 /** The agent-runtime file-backed layout as a `SupervisorRunReader`. */
-function runtimeSupervisorRunReader(runDir) {
+function runtimeSupervisorRunReader(runDir, opts = {}) {
 	return {
 		runRef: runDir,
-		read: () => readRuntimeSupervisorRun(runDir)
+		read: () => readRuntimeSupervisorRun(runDir, opts)
 	};
 }
 /** True when a directory contains Runtime's canonical file-backed journal. */