@tangle-network/agent-eval 1.0.15 → 1.0.17

package/dist/{render-CsHLoMUl.js → runtime-rollouts-kGn1K5ut.js}

@@ -5,10 +5,8 @@ import { t as roundDecimal } from "./round-decimal-ZmjAQPo1.js";
 import { S as licenseClassOfName } from "./schema-ChIZfnqY.js";
 import { _ as showMeasured, a as readTerminalRecord, c as parseJsonRecord, d as recordOrNull, f as workerSourceKey, g as isUnavailable, h as SUPERVISOR_RUN_SCHEMA, l as parseJsonl, n as supervisorRunRolloutLinesFromFacts, o as asRecord, p as NO_SOURCE_LIMITS, s as nonEmptyString, u as parseSupervisorTree, v as unavailable, y as expandJournalEventGroup } from "./rollout-nodes-DhSPJ2X7.js";
 import { basename, join } from "node:path";
-import { execFile } from "node:child_process";
 import { readFile, readdir, stat } from "node:fs/promises";
 import { sandboxEventsSession, toChatMessages, toOtlpSpans } from "@tangle-network/harness-sessions";
-import { promisify } from "node:util";
 // src/supervisor-run/analyze-context.ts
 /** The parsed sources and shared lookups every metric group reads. `gap` records an unavailable measurement on the report. */
 function createAnalysis(src) {
@@ -1371,286 +1369,6 @@ async function isFile(path) {
 		throw error;
 	}
 }
-// src/supervisor-run/discovery-rollouts.ts
-/**
-* A Discovery run directory as `tangle.rollout.v1` rows: the supervision tree
-* `supervisorRunRolloutLines` mints, plus what only a Discovery run records.
-*
-*   - Producer: each node's `materialized` receipt names the model it ran on
-*     (`opencode/zai-coding-plan/glm-5.3` → harness, provider, model), and the
-*     root's `root-segment` names the root's. The license class follows.
-*   - Episode: each execution's input (`execution-input.taskRef`) and result
-*     (`execution-result.outRef`, a `{content, events}` blob), read as Sandbox
-*     sessions in order, so a node steered twice keeps both turns and every tool
-*     call. A node with no event stream stays a lossy projection.
-*   - A node respawned under the same id (a reopened run) is one row.
-*   - Reward: the run's newest known play-check verdict on the root row
-*     (`solved` → 1, a staged `SCORE` → stages passed / stages), and each
-*     agent's own commits on its `workers/<label>` branch of `deliverables.git`.
-*   - Screen: the calibration frozen in the run's receipt, where a fake fixture
-*     failed every stage of the check, and the goal audit, whose `contradiction`
-*     gates the reward. A check nothing calibrated stays unscreened.
-*
-* Every read is of files the run already wrote; nothing in the run dir is changed.
-*/
-const run = promisify(execFile);
-/** A run's pursuit: its directory name without the `-YYYYMMDD<letter>` date suffix. */
-function discoveryPursuitOf(runDir) {
-	return basename(runDir).replace(/-20\d{6}[a-z0-9]*$/u, "");
-}
-/** `opencode/zai-coding-plan/glm-5.3` → harness, provider, model. */
-function producerOf(id) {
-	const parts = id.split("/");
-	if (parts.length >= 3) return {
-		harness: parts[0],
-		provider: parts[1],
-		model: parts.slice(2).join("/")
-	};
-	if (parts.length === 2) return {
-		harness: null,
-		provider: parts[0],
-		model: parts[1]
-	};
-	return {
-		harness: null,
-		provider: null,
-		model: id
-	};
-}
-/** A row holding the agent's whole stream is no longer lossy; a workspace capture gap still stands. */
-function wholeEpisode({ lossy_projection: _, gap, ...kept }) {
-	const workspace = gap?.match(/workspace capture evidence \w+/u)?.[0];
-	return workspace === void 0 ? kept : {
-		...kept,
-		gap: workspace
-	};
-}
-const text = (v) => typeof v === "string" && v.length > 0 ? v : void 0;
-async function journalFacts(runDir) {
-	const facts = /* @__PURE__ */ new Map();
-	const raw = await readFile(join(runDir, "spawn-journal.jsonl"), "utf8").catch(() => "");
-	const node = (id) => {
-		const fact = facts.get(id) ?? {};
-		facts.set(id, fact);
-		return fact;
-	};
-	for (const row of parseJsonl(raw)) {
-		const expanded = expandJournalEventGroup(row);
-		if (!expanded.succeeded) continue;
-		for (const entry of expanded.value) {
-			const event = recordOrNull(entry.event);
-			const id = text(event?.id);
-			if (event === null || id === void 0) continue;
-			if (event.kind === "spawned") node(id).taskRef ??= text(recordOrNull(event.identity)?.taskDigest);
-			else if (event.kind === "materialized") {
-				const model = recordOrNull(recordOrNull(event.receipt)?.model);
-				if (model?.status === "known" && text(model.id)) node(id).model = text(model.id);
-			} else if (event.kind === "root-segment") {
-				const to = recordOrNull(event.to);
-				const model = text(to?.model);
-				if (model) node(id).model = [
-					text(to?.harness),
-					text(to?.provider),
-					model
-				].join("/");
-			} else if (event.kind === "settled" && text(event.outRef)) node(id).outRef = text(event.outRef);
-			else if (event.kind === "execution-input") {
-				const fact = node(id);
-				fact.inputRef = text(event.taskRef);
-				fact.unansweredSince ??= text(event.at);
-			} else if (event.kind === "execution-result" && text(event.outRef)) {
-				const fact = node(id);
-				fact.unansweredSince = void 0;
-				const taskRef = fact.inputRef ?? fact.taskRef;
-				(fact.executions ??= []).push({
-					taskRef,
-					outRef: event.outRef
-				});
-			}
-		}
-	}
-	return facts;
-}
-const readJson = (path) => readFile(path, "utf8").then(JSON.parse).catch(() => void 0);
-async function blobValue(runDir, ref) {
-	if (ref === void 0 || !/^sha256:[0-9a-f]{64}$/u.test(ref)) return void 0;
-	return readJson(join(runDir, "blobs", `${ref.replace(":", "-")}.json`));
-}
-const jsonlRows = async (path) => parseJsonl(await readFile(path, "utf8").catch(() => null));
-/** One Sandbox session per execution, oldest first; older runs keep only the settled result. */
-async function nodeSessions(runDir, harness, id, fact) {
-	const executions = fact.executions ?? [{
-		taskRef: fact.taskRef,
-		outRef: fact.outRef
-	}];
-	const sessions = [];
-	for (const [index, { taskRef, outRef }] of executions.entries()) {
-		const [input, result] = await Promise.all([blobValue(runDir, taskRef), blobValue(runDir, outRef)]);
-		const task = text(input) ?? text(recordOrNull(input)?.prompt) ?? text(recordOrNull(input)?.instruction);
-		const answer = text(result) ?? text(recordOrNull(result)?.content);
-		const stream = recordOrNull(result)?.events;
-		const events = Array.isArray(stream) ? stream : [];
-		if (task === void 0 || answer === void 0 && events.length === 0) continue;
-		const nativeSessionId = `${id}:${index}`;
-		sessions.push(sandboxEventsSession({
-			harness,
-			nativeSessionId,
-			task,
-			answer: answer ?? "",
-			events
-		}));
-	}
-	return sessions;
-}
-/**
-* The newest play-check row whose verdict is known. A staged check
-* (`SCORE {"reached": r, "of": n}`) that is not solved failed at stage r, so it
-* passed r − 1 of n: Discovery's own `stagesPassed` (runner/calibration.mjs).
-* Only the SCORE line itself is parsed; the check prints more lines after it.
-*/
-async function discoveryPlayCheckReward(runDir) {
-	const rows = await jsonlRows(join(runDir, "play-checks.jsonl"));
-	const single = rows.length === 0 ? recordOrNull(await readJson(join(runDir, "play-check.json"))) : null;
-	if (single) rows.push(single);
-	const last = rows.filter((row) => row.known !== false && typeof row.solved === "boolean").sort((a, b) => String(a.at ?? "").localeCompare(String(b.at ?? ""))).at(-1);
-	if (last === void 0) return null;
-	const detail = typeof last.detail === "string" ? last.detail : "";
-	let reward = last.solved === true ? 1 : 0;
-	const score = detail.split("\n").find((line) => line.startsWith("SCORE {"));
-	if (reward === 0 && score !== void 0) try {
-		const { reached, of } = JSON.parse(score.slice(6));
-		if (Number.isSafeInteger(reached) && Number.isSafeInteger(of) && 1 <= reached && reached <= of) reward = (reached - 1) / of;
-	} catch {}
-	const key = last.contentHash ?? last.commit ?? last.at;
-	const ruling = (await jsonlRows(join(runDir, "goal-audits.jsonl"))).filter((a) => a.key === key && a.verdict !== "unavailable").at(-1);
-	const receipt = recordOrNull(await readJson(join(runDir, "RECEIPT.json")));
-	const calibration = recordOrNull(recordOrNull(receipt?.check)?.calibration);
-	const stages = Object.values(recordOrNull(calibration?.stages) ?? {});
-	return {
-		reward,
-		calibrated: stages.length > 0 && stages.every((state) => state === "calibrated"),
-		contradicted: (ruling?.verdict ?? recordOrNull(last.goalAudit)?.verdict) === "contradiction",
-		at: String(last.at ?? ""),
-		taggedAt: text(receipt?.taggedAt),
-		commit: typeof last.commit === "string" ? last.commit : null,
-		detail: detail.slice(0, 500)
-	};
-}
-/** A solved run is stopped, so it reads `cancelled`; it is whole if no turn was open at the tag. */
-const holdsTheSolve = (check, unansweredSince) => check.reward === 1 && !check.contradicted && (!unansweredSince || Date.parse(check.taggedAt ?? "") <= Date.parse(unansweredSince));
-/** Commits each `workers/<label>` branch holds beyond the commit every worker started from. */
-async function deliverableCommits(runDir) {
-	const git = (args) => run("git", [
-		"-C",
-		join(runDir, "deliverables.git"),
-		...args
-	], { timeout: 3e4 });
-	let refs;
-	try {
-		const { stdout } = await git([
-			"for-each-ref",
-			"--format=%(refname:short)",
-			"refs/heads/workers"
-		]);
-		refs = stdout.split("\n").filter(Boolean);
-	} catch {
-		return null;
-	}
-	if (refs.length === 0) return /* @__PURE__ */ new Map();
-	const base = await git([
-		"merge-base",
-		"--octopus",
-		...refs
-	]).then(({ stdout }) => stdout.trim()).catch(() => "");
-	const counts = /* @__PURE__ */ new Map();
-	for (const ref of refs) {
-		const counted = await git([
-			"rev-list",
-			"--count",
-			ref,
-			...base ? [`^${base}`] : []
-		]).catch(() => null);
-		if (counted === null) return null;
-		counts.set(ref.slice(8), Number(counted.stdout.trim()));
-	}
-	return counts;
-}
-/** Read one Discovery run directory into rollout rows. */
-async function discoveryRunRolloutLines(runDir, options = {}) {
-	const pursuit = discoveryPursuitOf(runDir);
-	const src = await readRuntimeSupervisorRun(runDir);
-	const facts0 = parseSupervisorTree(src);
-	const seen = /* @__PURE__ */ new Set();
-	const workerSpawns = facts0.workerSpawns.filter((spawn) => !seen.has(spawn.id) && seen.add(spawn.id));
-	const tree = supervisorRunRolloutLinesFromFacts(src, {
-		...facts0,
-		workerSpawns
-	}, {
-		suite: `discovery/${pursuit}`,
-		runId: basename(runDir),
-		...options.capturedAt === void 0 ? {} : { capturedAt: options.capturedAt }
-	});
-	const facts = await journalFacts(runDir);
-	const check = await discoveryPlayCheckReward(runDir);
-	const commits = await deliverableCommits(runDir);
-	const lines = [];
-	const otlp = /* @__PURE__ */ new Map();
-	for (const node of tree.nodes) {
-		const fact = facts.get(node.rollout_id) ?? {};
-		const producer = fact.model === void 0 ? null : producerOf(fact.model);
-		const license = licenseClassOfName(producer?.model) ?? licenseClassOfName(producer?.harness);
-		const sessions = await nodeSessions(runDir, producer?.harness ?? node.policy.harness ?? "sandbox", node.rollout_id, fact);
-		const messages = sessions.flatMap((session) => toChatMessages(session));
-		const label = typeof node.outcome.metrics.label === "string" ? node.outcome.metrics.label : null;
-		const isRoot = node.parent_rollout_id === null;
-		const outcome = isRoot && check !== null ? {
-			...node.outcome,
-			...check.calibrated || check.contradicted ? screenedRewardFields(check.reward, check.contradicted) : unscreenedRewardFields(check.reward),
-			...holdsTheSolve(check, fact.unansweredSince) ? {
-				is_completed: true,
-				error: null
-			} : {},
-			reward_source: "discovery/play-check",
-			verdict: {
-				at: check.at,
-				commit: check.commit,
-				detail: check.detail
-			}
-		} : node.outcome;
-		const spans = sessions.map((session) => toOtlpSpans(session)).filter((s) => s.length > 1).flat();
-		otlp.set(node.rollout_id, spans.map((span) => `${JSON.stringify(span)}\n`).join(""));
-		lines.push({
-			...node,
-			policy: {
-				...node.policy,
-				...producer === null ? {} : {
-					harness: producer.harness ?? node.policy.harness,
-					provider: producer.provider,
-					model: producer.model
-				},
-				...license === null ? {} : { license_class: license }
-			},
-			messages: sessions.length === 0 ? node.messages : messages,
-			provenance: messages.some((m) => m.role === "tool") ? wholeEpisode(node.provenance) : node.provenance,
-			outcome: {
-				...outcome,
-				metrics: {
-					...outcome.metrics,
-					deliverable_commits: commits === null ? null : commits.get(isRoot ? "root" : label ?? "") ?? 0
-				}
-			}
-		});
-	}
-	const notes = commits === null ? ["deliverables.git unreadable: deliverable_commits is null"] : [];
-	return {
-		runDir,
-		pursuit,
-		lines,
-		gaps: tree.gaps.filter((gap) => check === null || gap.code !== "root-reward-unavailable"),
-		notes,
-		otlp
-	};
-}
 // src/supervisor-run/reader.ts
 /**
 * Runtime supervisor-run reader and report writers.
@@ -1903,6 +1621,200 @@ function renderSupervisorRollupMarkdown(rollup, title = "Round rollup") {
 	for (const cell of rollup.perCell) if (cell.resourceRecords !== void 0) out.push(`## Resource receipts — ${cell.instanceId ?? "?"} [${cell.arm ?? "?"}]`, "", ...resourceRows(cell.resourceRecords));
 	return out.join("\n");
 }
-export { findSupervisorRunDirs as a, discoveryRunRolloutLines as c, analyzeSupervisorRunSources as d, parsePatch as f, analyzeSupervisorRun as i, isRuntimeSupervisorRunDir as l, renderSupervisorRunHeadline as n, discoveryPlayCheckReward as o, renderSupervisorRunMarkdown as r, discoveryPursuitOf as s, renderSupervisorRollupMarkdown as t, readRuntimeSupervisorRun as u };
+// src/supervisor-run/runtime-rollouts.ts
+/**
+* A Runtime run directory as `tangle.rollout.v1` rows: the supervision tree
+* `supervisorRunRolloutLines` mints, with each node's whole episode.
+*
+*   - Producer: each node's `materialized` receipt names the model it ran on
+*     (`opencode/zai-coding-plan/glm-5.3` → harness, provider, model), and the
+*     root's `root-segment` names the root's. The license class follows.
+*   - Episode: each execution's input (`execution-input.taskRef`) and result
+*     (`execution-result.outRef`, a `{content, events}` blob), read as Sandbox
+*     sessions in order, so a node steered twice keeps both turns and every tool
+*     call. A node with no event stream stays a lossy projection.
+*   - A node respawned under the same id (a reopened run) is one row.
+*   - Reward: the caller's `RunVerdict` on the root row. The caller owns what
+*     its check means; this reader owns how a reward is screened and when a
+*     stopped episode is whole.
+*
+* Every read is of files Runtime already wrote; nothing in the run dir is changed.
+*/
+/** `opencode/zai-coding-plan/glm-5.3` → harness, provider, model. */
+function producerOf(id) {
+	const parts = id.split("/");
+	if (parts.length >= 3) return {
+		harness: parts[0],
+		provider: parts[1],
+		model: parts.slice(2).join("/")
+	};
+	if (parts.length === 2) return {
+		harness: null,
+		provider: parts[0],
+		model: parts[1]
+	};
+	return {
+		harness: null,
+		provider: null,
+		model: id
+	};
+}
+/** A row holding the agent's whole stream is no longer lossy; a workspace capture gap still stands. */
+function wholeEpisode({ lossy_projection: _, gap, ...kept }) {
+	const workspace = gap?.match(/workspace capture evidence \w+/u)?.[0];
+	return workspace === void 0 ? kept : {
+		...kept,
+		gap: workspace
+	};
+}
+const text = (v) => typeof v === "string" && v.length > 0 ? v : void 0;
+async function journalFacts(runDir) {
+	const facts = /* @__PURE__ */ new Map();
+	const raw = await readFile(join(runDir, "spawn-journal.jsonl"), "utf8").catch(() => "");
+	const node = (id) => {
+		const fact = facts.get(id) ?? {};
+		facts.set(id, fact);
+		return fact;
+	};
+	for (const row of parseJsonl(raw)) {
+		const expanded = expandJournalEventGroup(row);
+		if (!expanded.succeeded) continue;
+		for (const entry of expanded.value) {
+			const event = recordOrNull(entry.event);
+			const id = text(event?.id);
+			if (event === null || id === void 0) continue;
+			if (event.kind === "spawned") node(id).taskRef ??= text(recordOrNull(event.identity)?.taskDigest);
+			else if (event.kind === "materialized") {
+				const model = recordOrNull(recordOrNull(event.receipt)?.model);
+				if (model?.status === "known" && text(model.id)) node(id).model = text(model.id);
+			} else if (event.kind === "root-segment") {
+				const to = recordOrNull(event.to);
+				const model = text(to?.model);
+				if (model) node(id).model = [
+					text(to?.harness),
+					text(to?.provider),
+					model
+				].join("/");
+			} else if (event.kind === "settled" && text(event.outRef)) node(id).outRef = text(event.outRef);
+			else if (event.kind === "execution-input") {
+				const fact = node(id);
+				fact.inputRef = text(event.taskRef);
+				fact.unansweredSince ??= text(event.at);
+			} else if (event.kind === "execution-result" && text(event.outRef)) {
+				const fact = node(id);
+				fact.unansweredSince = void 0;
+				const taskRef = fact.inputRef ?? fact.taskRef;
+				(fact.executions ??= []).push({
+					taskRef,
+					outRef: event.outRef
+				});
+			}
+		}
+	}
+	return facts;
+}
+const readJson = (path) => readFile(path, "utf8").then(JSON.parse).catch(() => void 0);
+async function blobValue(runDir, ref) {
+	if (ref === void 0 || !/^sha256:[0-9a-f]{64}$/u.test(ref)) return void 0;
+	return readJson(join(runDir, "blobs", `${ref.replace(":", "-")}.json`));
+}
+/** One Sandbox session per execution, oldest first; older runs keep only the settled result. */
+async function nodeSessions(runDir, harness, id, fact) {
+	const executions = fact.executions ?? [{
+		taskRef: fact.taskRef,
+		outRef: fact.outRef
+	}];
+	const sessions = [];
+	for (const [index, { taskRef, outRef }] of executions.entries()) {
+		const [input, result] = await Promise.all([blobValue(runDir, taskRef), blobValue(runDir, outRef)]);
+		const task = text(input) ?? text(recordOrNull(input)?.prompt) ?? text(recordOrNull(input)?.instruction);
+		const answer = text(result) ?? text(recordOrNull(result)?.content);
+		const stream = recordOrNull(result)?.events;
+		const events = Array.isArray(stream) ? stream : [];
+		if (task === void 0 || answer === void 0 && events.length === 0) continue;
+		const nativeSessionId = `${id}:${index}`;
+		sessions.push(sandboxEventsSession({
+			harness,
+			nativeSessionId,
+			task,
+			answer: answer ?? "",
+			events
+		}));
+	}
+	return sessions;
+}
+/**
+* The root's outcome under the caller's verdict. A run stopped on its success reads `cancelled`;
+* its episode is whole if no input was pending when it was stopped.
+*/
+function judged(node, verdict, pending) {
+	if (verdict === null || node.parent_rollout_id !== null) return node.outcome;
+	const { reward, screened, gated = false, source, stoppedAt = "", commit = null } = verdict;
+	const ended = reward === 1 && !gated && (!pending || Date.parse(stoppedAt) <= Date.parse(pending));
+	return {
+		...node.outcome,
+		...screened || gated ? screenedRewardFields(reward, gated) : unscreenedRewardFields(reward),
+		...ended ? {
+			is_completed: true,
+			error: null
+		} : {},
+		reward_source: source,
+		verdict: {
+			at: verdict.at ?? "",
+			commit,
+			detail: (verdict.detail ?? "").slice(0, 500)
+		}
+	};
+}
+/** Read one Runtime run directory into rollout rows; `suite` defaults to `runtime/<run>`. */
+async function runtimeRunRolloutLines(runDir, options = {}) {
+	const verdict = options.verdict ?? null;
+	const src = await readRuntimeSupervisorRun(runDir);
+	const facts0 = parseSupervisorTree(src);
+	const seen = /* @__PURE__ */ new Set();
+	const workerSpawns = facts0.workerSpawns.filter((spawn) => !seen.has(spawn.id) && seen.add(spawn.id));
+	const tree = supervisorRunRolloutLinesFromFacts(src, {
+		...facts0,
+		workerSpawns
+	}, {
+		suite: options.suite ?? `runtime/${basename(runDir)}`,
+		runId: basename(runDir),
+		...options.capturedAt === void 0 ? {} : { capturedAt: options.capturedAt }
+	});
+	const facts = await journalFacts(runDir);
+	const lines = [];
+	const otlp = /* @__PURE__ */ new Map();
+	for (const node of tree.nodes) {
+		const fact = facts.get(node.rollout_id) ?? {};
+		const producer = fact.model === void 0 ? null : producerOf(fact.model);
+		const license = licenseClassOfName(producer?.model) ?? licenseClassOfName(producer?.harness);
+		const sessions = await nodeSessions(runDir, producer?.harness ?? node.policy.harness ?? "sandbox", node.rollout_id, fact);
+		const messages = sessions.flatMap((session) => toChatMessages(session));
+		const spans = sessions.map((session) => toOtlpSpans(session)).filter((s) => s.length > 1).flat();
+		otlp.set(node.rollout_id, spans.map((span) => `${JSON.stringify(span)}\n`).join(""));
+		lines.push({
+			...node,
+			policy: {
+				...node.policy,
+				...producer === null ? {} : {
+					harness: producer.harness ?? node.policy.harness,
+					provider: producer.provider,
+					model: producer.model
+				},
+				...license === null ? {} : { license_class: license }
+			},
+			messages: sessions.length === 0 ? node.messages : messages,
+			provenance: messages.some((m) => m.role === "tool") ? wholeEpisode(node.provenance) : node.provenance,
+			outcome: judged(node, verdict, fact.unansweredSince)
+		});
+	}
+	return {
+		runDir,
+		lines,
+		gaps: tree.gaps.filter((gap) => verdict === null || gap.code !== "root-reward-unavailable"),
+		otlp
+	};
+}
+export { analyzeSupervisorRun as a, readRuntimeSupervisorRun as c, renderSupervisorRunMarkdown as i, analyzeSupervisorRunSources as l, renderSupervisorRollupMarkdown as n, findSupervisorRunDirs as o, renderSupervisorRunHeadline as r, isRuntimeSupervisorRunDir as s, runtimeRunRolloutLines as t, parsePatch as u };
 
-//# sourceMappingURL=render-CsHLoMUl.js.map
+//# sourceMappingURL=runtime-rollouts-kGn1K5ut.js.map