@iowarp/clio-coder 0.3.8 → 0.4.0

package/src/domains/eval/metrics/tracked.ts

@@ -0,0 +1,413 @@
+import { readFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import type { RunReceipt } from "../../dispatch/types.js";
+import { listSessionLedgerRefs, parseSessionEntries } from "../../session/archive-readers.js";
+import { extractReasoningTokens } from "../../session/context-accounting.js";
+import type { SessionEntry } from "../../session/entries.js";
+import { evalHarnessMetricsFromReceipt } from "../harness-metrics.js";
+import type {
+	EvalMetricSource,
+	EvalSourcedNullableNumber,
+	EvalSourcedNumber,
+	EvalTrackedMetricsV1,
+} from "../schema/verdict.js";
+
+export interface EvalLedgerSnapshot {
+	entries: SessionEntry[];
+	compiledPromptHashes: string[];
+	promptManifests: EvalPromptManifestObservation[];
+	contextSnapshots: EvalContextSnapshotObservation[];
+}
+
+export interface EvalPromptManifestObservation {
+	systemPromptHash: string;
+	thinkingLevel: string | null;
+	projectPreload: {
+		mode: "full" | "synopsis" | "none";
+		chars: number;
+		lines: number;
+		reason: string | null;
+		nearLimit: boolean;
+		label: string;
+	} | null;
+	fragments: Array<{ id: string; contentHash: string }>;
+}
+
+export interface EvalContextSnapshotObservation {
+	runtimeId: string | null;
+	modelId: string | null;
+	promptHash: string | null;
+	toolSignature: string | null;
+}
+
+export interface BuildEvalTrackedMetricsInput {
+	ledgerEntries: ReadonlyArray<SessionEntry>;
+	receipt: RunReceipt | null;
+	fallbackWallClockMs: number;
+}
+
+interface AssistantCall {
+	payload: Record<string, unknown>;
+	promptCache: Record<string, unknown> | null;
+	backend: Record<string, unknown> | null;
+	timing: Record<string, unknown> | null;
+	usage: Record<string, unknown> | null;
+}
+
+interface NumericReading {
+	value: number;
+	source: EvalMetricSource;
+}
+
+/** Read every isolated session ledger and its prompt manifest before cleanup. */
+export async function readEvalLedgerSnapshot(stateDir: string): Promise<EvalLedgerSnapshot> {
+	const refs = await listSessionLedgerRefs(stateDir);
+	const entries: SessionEntry[] = [];
+	const compiledPromptHashes: string[] = [];
+	const promptManifests: EvalPromptManifestObservation[] = [];
+	const contextSnapshots: EvalContextSnapshotObservation[] = [];
+	for (const ref of refs) {
+		try {
+			const raw = await readFile(ref.path, "utf8");
+			entries.push(...parseSessionEntries(raw, ref.path).entries);
+		} catch {
+			// An unreadable ledger contributes no invented measurements.
+		}
+		try {
+			const manifest = await readFile(join(dirname(ref.path), "prompt-manifest.jsonl"), "utf8");
+			for (const line of manifest.split(/\r?\n/u)) {
+				const record = parseJsonRecord(line);
+				if (record === null) continue;
+				const hash = record?.systemPromptHash;
+				if (typeof hash !== "string" || !/^[a-f0-9]{64}$/u.test(hash)) continue;
+				compiledPromptHashes.push(hash);
+				const observation = promptManifestObservation(record);
+				if (observation !== null) promptManifests.push(observation);
+			}
+		} catch {
+			// A missing manifest leaves compiled prompt provenance unresolved.
+		}
+		try {
+			const snapshots = await readFile(join(dirname(ref.path), "context-snapshots.jsonl"), "utf8");
+			for (const line of snapshots.split(/\r?\n/u)) {
+				const record = parseJsonRecord(line);
+				if (record === null) continue;
+				contextSnapshots.push({
+					runtimeId: nullableString(record.runtimeId),
+					modelId: nullableString(record.modelId),
+					promptHash: nullableDigest(record.promptHash),
+					toolSignature: nullableDigest(record.toolSignature),
+				});
+			}
+		} catch {
+			// A missing context snapshot leaves runtime and tool-surface provenance unresolved.
+		}
+	}
+	return { entries, compiledPromptHashes: [...new Set(compiledPromptHashes)], promptManifests, contextSnapshots };
+}
+
+/** Fold structured ledger calls and a sealed receipt into verdict metrics. */
+export function buildEvalTrackedMetrics(input: BuildEvalTrackedMetricsInput): EvalTrackedMetricsV1 {
+	const calls = assistantCalls(input.ledgerEntries);
+	const compactionEntries = input.ledgerEntries.filter((entry) => entry.kind === "compactionSummary");
+	const compactionUsage = compactionEntries.flatMap((entry) => {
+		if (entry.kind !== "compactionSummary" || !isRecord(entry.usage)) return [];
+		return [entry.usage];
+	});
+
+	const modelCallReadings: NumericReading[] = calls.map(() => ledgerReading(1));
+	for (const entry of compactionEntries) {
+		if (entry.kind !== "compactionSummary") continue;
+		const apiCalls = isRecord(entry.usage) ? nonNegativeNumber(entry.usage.apiCalls) : null;
+		modelCallReadings.push(apiCalls === null ? estimatedReading(1) : ledgerReading(apiCalls));
+	}
+
+	const uncachedReadings = calls.map(uncachedPrefillForCall);
+	const cacheReadings = calls.map(cacheReadForCall);
+	const generatedReadings = calls.map(generatedForCall);
+	for (const usage of compactionUsage) {
+		uncachedReadings.push(readingFromUsage(usage, "input"));
+		cacheReadings.push(readingFromUsage(usage, "cacheRead"));
+		generatedReadings.push(readingFromUsage(usage, "output"));
+	}
+
+	const reasoning = reasoningMetric(input.receipt, calls, compactionUsage);
+	const receiptToolMetrics = input.receipt === null ? null : evalHarnessMetricsFromReceipt(input.receipt);
+	const ledgerToolCalls = input.ledgerEntries.filter(
+		(entry) => entry.kind === "message" && entry.role === "tool_call",
+	).length;
+	const ledgerToolErrors = input.ledgerEntries.filter((entry) => {
+		if (entry.kind !== "message" || entry.role !== "tool_result" || !isRecord(entry.payload)) return false;
+		return entry.payload.isError === true || entry.payload.outcome === "error";
+	}).length;
+	const receiptToolErrors = input.receipt?.toolStats.reduce((sum, stat) => sum + finiteNonNegative(stat.errors), 0);
+	const expectedColdReasons = expectedColdReasonMetrics(calls);
+
+	return {
+		modelCalls: sumReadings(modelCallReadings, "ledger"),
+		uncachedPrefillTokens: sumReadings(uncachedReadings, "estimated"),
+		cacheReadTokens: sumReadings(cacheReadings, "estimated"),
+		generatedTokens: sumReadings(generatedReadings, "estimated"),
+		reasoningTokens: reasoning,
+		toolCalls:
+			receiptToolMetrics === null
+				? { value: ledgerToolCalls, source: "ledger" }
+				: { value: receiptToolMetrics.toolCalls, source: "receipt" },
+		toolErrors:
+			receiptToolErrors === undefined
+				? { value: ledgerToolErrors, source: "ledger" }
+				: { value: receiptToolErrors, source: "receipt" },
+		ttftMsFirstCall: firstCallTtft(calls),
+		wallClockMs: wallClockMetric(input.receipt, input.fallbackWallClockMs),
+		contextTokensAtEnd: contextTokensAtEnd(calls, compactionEntries),
+		compactions: { value: compactionEntries.length, source: "ledger" },
+		expectedColdReasons,
+	};
+}
+
+export function emptyEvalTrackedMetrics(source: EvalMetricSource = "estimated"): EvalTrackedMetricsV1 {
+	const zero = (): EvalSourcedNumber => ({ value: 0, source });
+	return {
+		modelCalls: zero(),
+		uncachedPrefillTokens: zero(),
+		cacheReadTokens: zero(),
+		generatedTokens: zero(),
+		reasoningTokens: { value: null, source },
+		toolCalls: zero(),
+		toolErrors: zero(),
+		ttftMsFirstCall: zero(),
+		wallClockMs: zero(),
+		contextTokensAtEnd: zero(),
+		compactions: zero(),
+		expectedColdReasons: {},
+	};
+}
+
+function assistantCalls(entries: ReadonlyArray<SessionEntry>): AssistantCall[] {
+	return entries.flatMap((entry) => {
+		if (entry.kind !== "message" || entry.role !== "assistant" || !isRecord(entry.payload)) return [];
+		const promptCache = recordField(entry.payload, "promptCache");
+		const timing = recordField(entry.payload, "timing");
+		const usage = recordField(entry.payload, "usage");
+		if (promptCache === null && timing === null && usage === null) return [];
+		return [
+			{
+				payload: entry.payload,
+				promptCache,
+				backend: promptCache === null ? null : recordField(promptCache, "backend"),
+				timing,
+				usage,
+			},
+		];
+	});
+}
+
+function uncachedPrefillForCall(call: AssistantCall): NumericReading {
+	const promptTokens = call.backend === null ? null : nonNegativeNumber(call.backend.promptTokens);
+	const cachedTokens = call.backend === null ? null : nonNegativeNumber(call.backend.cachedTokens);
+	if (promptTokens !== null && cachedTokens !== null && cachedTokens <= promptTokens) {
+		return ledgerReading(promptTokens - cachedTokens);
+	}
+	const piInput = call.promptCache === null ? null : nonNegativeNumber(call.promptCache.input);
+	if (piInput !== null) return ledgerReading(piInput);
+	const legacyInput = call.usage === null ? null : nonNegativeNumber(call.usage.input);
+	return legacyInput === null ? estimatedReading(0) : estimatedReading(legacyInput);
+}
+
+function cacheReadForCall(call: AssistantCall): NumericReading {
+	const cachedTokens = call.backend === null ? null : nonNegativeNumber(call.backend.cachedTokens);
+	if (cachedTokens !== null) return ledgerReading(cachedTokens);
+	const piCacheRead = call.promptCache === null ? null : nonNegativeNumber(call.promptCache.cacheRead);
+	if (piCacheRead !== null) return ledgerReading(piCacheRead);
+	const legacyCacheRead = call.usage === null ? null : nonNegativeNumber(call.usage.cacheRead);
+	return legacyCacheRead === null ? estimatedReading(0) : estimatedReading(legacyCacheRead);
+}
+
+function generatedForCall(call: AssistantCall): NumericReading {
+	const predictedTokens = call.backend === null ? null : nonNegativeNumber(call.backend.predictedTokens);
+	if (predictedTokens !== null) return ledgerReading(predictedTokens);
+	const output = call.usage === null ? null : nonNegativeNumber(call.usage.output);
+	return output === null ? estimatedReading(0) : ledgerReading(output);
+}
+
+function readingFromUsage(usage: Record<string, unknown>, field: string): NumericReading {
+	const value = nonNegativeNumber(usage[field]);
+	return value === null ? estimatedReading(0) : ledgerReading(value);
+}
+
+function reasoningMetric(
+	receipt: RunReceipt | null,
+	calls: ReadonlyArray<AssistantCall>,
+	compactionUsage: ReadonlyArray<Record<string, unknown>>,
+): EvalSourcedNullableNumber {
+	if (receipt !== null && typeof receipt.reasoningTokenCount === "number") {
+		return { value: finiteNonNegative(receipt.reasoningTokenCount), source: "receipt" };
+	}
+	let total = 0;
+	let measured = false;
+	for (const call of calls) {
+		const value = extractReasoningTokens(call.usage);
+		if (value === null) continue;
+		measured = true;
+		total += finiteNonNegative(value);
+	}
+	for (const usage of compactionUsage) {
+		const value = nonNegativeNumber(usage.reasoning);
+		if (value === null) continue;
+		measured = true;
+		total += value;
+	}
+	return measured ? { value: total, source: "ledger" } : { value: null, source: "estimated" };
+}
+
+function firstCallTtft(calls: ReadonlyArray<AssistantCall>): EvalSourcedNumber {
+	const first = calls[0];
+	const value = first?.timing === null || first?.timing === undefined ? null : nonNegativeNumber(first.timing.ttftMs);
+	return value === null ? { value: 0, source: "estimated" } : { value, source: "ledger" };
+}
+
+function wallClockMetric(receipt: RunReceipt | null, fallback: number): EvalSourcedNumber {
+	if (receipt !== null) {
+		const started = Date.parse(receipt.startedAt);
+		const ended = Date.parse(receipt.endedAt);
+		if (Number.isFinite(started) && Number.isFinite(ended) && ended >= started) {
+			return { value: ended - started, source: "receipt" };
+		}
+	}
+	return { value: finiteNonNegative(fallback), source: "estimated" };
+}
+
+function contextTokensAtEnd(
+	calls: ReadonlyArray<AssistantCall>,
+	compactions: ReadonlyArray<SessionEntry>,
+): EvalSourcedNumber {
+	const lastCall = calls.at(-1);
+	if (lastCall !== undefined) {
+		const lastReading = contextForCall(lastCall);
+		if (lastReading !== null && lastReading > 0) return { value: lastReading, source: "ledger" };
+		if (lastReading === 0) {
+			for (const call of [...calls.slice(0, -1)].reverse()) {
+				const reading = contextForCall(call);
+				if (reading !== null && reading > 0) return { value: reading, source: "ledger" };
+			}
+		}
+	}
+	const lastCompaction = compactions.at(-1);
+	if (lastCompaction?.kind === "compactionSummary") {
+		const tokensAfter = nonNegativeNumber(lastCompaction.tokensAfter);
+		if (tokensAfter !== null) return { value: tokensAfter, source: "ledger" };
+	}
+	return { value: 0, source: "estimated" };
+}
+
+function contextForCall(call: AssistantCall): number | null {
+	const promptTokens = call.backend === null ? null : nonNegativeNumber(call.backend.promptTokens);
+	const predictedTokens = call.backend === null ? null : nonNegativeNumber(call.backend.predictedTokens);
+	if (promptTokens !== null && predictedTokens !== null) return promptTokens + predictedTokens;
+	const input = call.promptCache === null ? null : nonNegativeNumber(call.promptCache.input);
+	const cacheRead = call.promptCache === null ? null : nonNegativeNumber(call.promptCache.cacheRead);
+	const output = call.usage === null ? null : nonNegativeNumber(call.usage.output);
+	return input === null || cacheRead === null || output === null ? null : input + cacheRead + output;
+}
+
+function expectedColdReasonMetrics(calls: ReadonlyArray<AssistantCall>): Record<string, EvalSourcedNumber> {
+	const counts = new Map<string, number>();
+	for (const call of calls) {
+		const reasons = call.promptCache?.expectedColdReasons;
+		if (!Array.isArray(reasons)) continue;
+		const unique = new Set(reasons.filter((reason): reason is string => typeof reason === "string" && reason.length > 0));
+		for (const reason of unique) counts.set(reason, (counts.get(reason) ?? 0) + 1);
+	}
+	return Object.fromEntries(
+		[...counts.entries()]
+			.sort(([left], [right]) => left.localeCompare(right))
+			.map(([reason, value]) => [reason, { value, source: "ledger" as const }]),
+	);
+}
+
+function sumReadings(readings: ReadonlyArray<NumericReading>, emptySource: EvalMetricSource): EvalSourcedNumber {
+	if (readings.length === 0) return { value: 0, source: emptySource };
+	return {
+		value: readings.reduce((sum, reading) => sum + reading.value, 0),
+		source: readings.some((reading) => reading.source === "estimated") ? "estimated" : "ledger",
+	};
+}
+
+function ledgerReading(value: number): NumericReading {
+	return { value: finiteNonNegative(value), source: "ledger" };
+}
+
+function estimatedReading(value: number): NumericReading {
+	return { value: finiteNonNegative(value), source: "estimated" };
+}
+
+function finiteNonNegative(value: number): number {
+	return Number.isFinite(value) && value >= 0 ? value : 0;
+}
+
+function nonNegativeNumber(value: unknown): number | null {
+	return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : null;
+}
+
+function recordField(record: Record<string, unknown>, field: string): Record<string, unknown> | null {
+	return isRecord(record[field]) ? record[field] : null;
+}
+
+function parseJsonRecord(line: string): Record<string, unknown> | null {
+	if (line.trim().length === 0) return null;
+	try {
+		const parsed: unknown = JSON.parse(line);
+		return isRecord(parsed) ? parsed : null;
+	} catch {
+		return null;
+	}
+}
+
+function promptManifestObservation(record: Record<string, unknown>): EvalPromptManifestObservation | null {
+	const systemPromptHash = nullableDigest(record.systemPromptHash);
+	if (systemPromptHash === null || !Array.isArray(record.fragments)) return null;
+	const fragments = record.fragments.flatMap((entry) => {
+		if (!isRecord(entry) || typeof entry.id !== "string") return [];
+		const contentHash = nullableDigest(entry.contentHash);
+		return contentHash === null ? [] : [{ id: entry.id, contentHash }];
+	});
+	const preload = record.projectPreload;
+	const projectPreload: EvalPromptManifestObservation["projectPreload"] =
+		preload === null
+			? null
+			: isRecord(preload) &&
+					(preload.mode === "full" || preload.mode === "synopsis" || preload.mode === "none") &&
+					typeof preload.chars === "number" &&
+					Number.isInteger(preload.chars) &&
+					typeof preload.lines === "number" &&
+					Number.isInteger(preload.lines) &&
+					typeof preload.nearLimit === "boolean" &&
+					typeof preload.label === "string"
+				? {
+						mode: preload.mode as "full" | "synopsis" | "none",
+						chars: preload.chars,
+						lines: preload.lines,
+						reason: nullableString(preload.reason),
+						nearLimit: preload.nearLimit,
+						label: preload.label,
+					}
+				: null;
+	return {
+		systemPromptHash,
+		thinkingLevel: nullableString(record.thinkingLevel),
+		projectPreload,
+		fragments,
+	};
+}
+
+function nullableString(value: unknown): string | null {
+	return typeof value === "string" && value.length > 0 ? value : null;
+}
+
+function nullableDigest(value: unknown): string | null {
+	return typeof value === "string" && /^[a-f0-9]{64}$/u.test(value) ? value : null;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}

package/src/domains/eval/provenance.ts

@@ -1,5 +1,9 @@
 import { spawnSync } from "node:child_process";
+import { readSettings } from "../../core/config.js";
 import { readClioVersion, resolvePackageRoot } from "../../core/package-root.js";
+import type { RunReceipt } from "../dispatch/types.js";
+import type { EvalServingConfigurationV1 } from "./schema/serving.js";
+import type { EvalSuiteTargetV2 } from "./schema/suite.js";
 import type { EvalClioProvenance, EvalEnvironmentProvenance } from "./types.js";
 
 export interface EvalProvenanceOptions {
@@ -7,6 +11,14 @@ export interface EvalProvenanceOptions {
 	commit?: string | null;
 }
 
+export interface EvalServingObservation {
+	targetId: string;
+	runtimeId: string | null;
+	modelId: string | null;
+	thinkingLevel: string | null;
+	compiledPromptHash: string | null;
+}
+
 export function evalClioProvenance(options: EvalProvenanceOptions = {}): EvalClioProvenance {
 	return {
 		version: readClioVersion(),
@@ -22,6 +34,45 @@ export function evalEnvironmentProvenance(): EvalEnvironmentProvenance {
 	};
 }
 
+/** Capture serving identity after the matrix has run while a local server is awake. */
+export async function evalServingConfiguration(
+	targets: ReadonlyArray<EvalSuiteTargetV2>,
+	observations: ReadonlyArray<EvalServingObservation>,
+): Promise<EvalServingConfigurationV1> {
+	const targetId = targets.length === 1 ? (targets[0]?.id ?? "unknown") : "multiple";
+	const configured = configuredTarget(targetId);
+	const target = targets.length === 1 ? targets[0] : undefined;
+	const runtimeId = consensus(observations.map((entry) => entry.runtimeId)) ?? configured?.runtime ?? null;
+	const modelId =
+		consensus(observations.map((entry) => entry.modelId)) ?? target?.model ?? configured?.defaultModel ?? null;
+	const props = configured?.url === undefined ? null : await readServingProps(configured.url, modelId);
+	return {
+		targetId,
+		runtimeId,
+		modelId,
+		serverBuild: props?.serverBuild ?? null,
+		total_slots: props?.totalSlots ?? null,
+		thinkingLevel: consensus(observations.map((entry) => entry.thinkingLevel)) ?? target?.thinking ?? null,
+		compiledPromptHash: consensus(observations.map((entry) => entry.compiledPromptHash)),
+	};
+}
+
+export function evalServingObservationFrom(
+	target: EvalSuiteTargetV2,
+	receipt: RunReceipt | null,
+	compiledPromptHashes: ReadonlyArray<string>,
+): EvalServingObservation {
+	const compiledPromptHash =
+		receipt?.staticCompositionHash ?? receipt?.compiledPromptHash ?? consensus(compiledPromptHashes);
+	return {
+		targetId: receipt?.targetId ?? target.id,
+		runtimeId: receipt?.runtimeId ?? null,
+		modelId: receipt?.wireModelId ?? target.model ?? null,
+		thinkingLevel: receipt?.runtimeResolution?.effectiveThinkingLevel ?? target.thinking ?? null,
+		compiledPromptHash,
+	};
+}
+
 function currentClioCommit(): string | null {
 	const result = spawnSync("git", ["rev-parse", "HEAD"], {
 		cwd: resolvePackageRoot(),
@@ -33,3 +84,69 @@ function currentClioCommit(): string | null {
 	const value = result.stdout.trim();
 	return value.length > 0 ? value : null;
 }
+
+interface ConfiguredServingTarget {
+	runtime: string;
+	url?: string;
+	defaultModel?: string;
+}
+
+function configuredTarget(targetId: string): ConfiguredServingTarget | null {
+	try {
+		const target = readSettings().targets.find((entry) => entry.id === targetId);
+		if (target === undefined) return null;
+		return {
+			runtime: target.runtime,
+			...(target.url === undefined ? {} : { url: target.url }),
+			...(target.defaultModel === undefined ? {} : { defaultModel: target.defaultModel }),
+		};
+	} catch {
+		return null;
+	}
+}
+
+async function readServingProps(
+	targetUrl: string,
+	modelId: string | null,
+): Promise<{ serverBuild: string | null; totalSlots: number | null } | null> {
+	const root = targetUrl.replace(/\/+$/u, "").replace(/\/v1$/u, "");
+	try {
+		const response = await fetch(`${root}/props`, { signal: AbortSignal.timeout(5_000) });
+		if (!response.ok) return null;
+		const value: unknown = await response.json();
+		if (!isRecord(value)) return null;
+		const totalSlots = positiveInteger(value.total_slots) ?? (await readServingSlots(root, modelId));
+		return {
+			serverBuild: typeof value.build_info === "string" && value.build_info.length > 0 ? value.build_info : null,
+			totalSlots,
+		};
+	} catch {
+		return null;
+	}
+}
+
+async function readServingSlots(root: string, modelId: string | null): Promise<number | null> {
+	if (modelId === null) return null;
+	try {
+		const query = new URLSearchParams({ model: modelId });
+		const response = await fetch(`${root}/slots?${query}`, { signal: AbortSignal.timeout(5_000) });
+		if (!response.ok) return null;
+		const value: unknown = await response.json();
+		return Array.isArray(value) && value.length > 0 ? value.length : null;
+	} catch {
+		return null;
+	}
+}
+
+function consensus(values: ReadonlyArray<string | null>): string | null {
+	const known = [...new Set(values.filter((value): value is string => typeof value === "string" && value.length > 0))];
+	return known.length === 1 ? (known[0] ?? null) : null;
+}
+
+function positiveInteger(value: unknown): number | null {
+	return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : null;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}

package/src/domains/eval/reports/comparison.ts

@@ -0,0 +1,128 @@
+import type { EvalCompareV4Summary } from "../compare/compare.js";
+import { renderEvalComparisonV4 } from "../compare/compare.js";
+
+export type EvalComparisonReportFormat = "text" | "json" | "md" | "junit";
+
+export function renderEvalComparisonReportV1(
+	summary: EvalCompareV4Summary,
+	format: EvalComparisonReportFormat,
+): string {
+	if (format === "json") return `${JSON.stringify(summary, null, 2)}\n`;
+	if (format === "md") return renderMarkdown(summary);
+	if (format === "junit") return renderJunit(summary);
+	return renderEvalComparisonV4(summary);
+}
+
+function renderMarkdown(summary: EvalCompareV4Summary): string {
+	const rows = summary.behavioralMetrics.map(
+		(row) =>
+			`| ${cell(row.scenarioId)} | ${cell(row.role)} | ${cell(`${row.target.id}/${row.target.model ?? "none"}`)} | ${row.family} | ${row.metric} | ${format(row.baseline.mean)} | ${format(row.baseline.variance)} | ${row.baseline.measured}/${row.baseline.observations} | ${format(row.candidate.mean)} | ${format(row.candidate.variance)} | ${row.candidate.measured}/${row.candidate.observations} | ${row.change} | ${row.varianceChange} | ${row.comparability.comparable ? "comparable" : cell(row.comparability.mismatchedFields.join(", "))} | ${row.hardGate ? "hard" : "informational"} |`,
+	);
+	const scenarioRows = summary.scenarioReports.map(
+		(report) => `| ${cell(report.id)} | ${changeCounts(report.metrics)} | ${changeCounts(report.variance)} |`,
+	);
+	const roleRows = summary.roleReports.map(
+		(report) => `| ${cell(report.id)} | ${changeCounts(report.metrics)} | ${changeCounts(report.variance)} |`,
+	);
+	return [
+		`# Eval comparison ${summary.baselineEvalId} → ${summary.candidateEvalId}`,
+		"",
+		`Behavioral hard gate: **${summary.hardGate.pass ? "pass" : "fail"}**`,
+		...summary.hardGate.failures.map(
+			(failure) =>
+				`- Hard failure: ${failure.scenarioId} / ${failure.role} / ${failure.target.id}/${failure.target.model ?? "none"} / ${failure.metric}: ${failure.change}`,
+		),
+		...summary.envelopeMismatches.map(
+			(mismatch) =>
+				`- Incomparable envelope: ${mismatch.scenarioId} / ${mismatch.role} / ${mismatch.target.id}/${mismatch.target.model ?? "none"}: ${mismatch.fields.join(", ")}`,
+		),
+		...summary.affectedCorpusResults.map(
+			(result) => `- Affected corpus result: ${result.scenarioId} / ${result.role}: ${result.changedFields.join(", ")}`,
+		),
+		`Pass-rate delta: ${(summary.passRateDelta * 100).toFixed(2)}%`,
+		`Token delta: ${summary.tokenDelta === null ? "unmeasured" : summary.tokenDelta}`,
+		`Wall-time delta ms: ${summary.wallTimeDelta}`,
+		"",
+		"| Scenario | Role | Target/model | Family | Metric | Baseline mean | Baseline variance | Baseline measured | Candidate mean | Candidate variance | Candidate measured | Change | Variance | Comparability | Gate |",
+		"|---|---|---|---|---|---:|---:|---:|---:|---:|---:|---|---|---|---|",
+		...rows,
+		"",
+		"## Per-scenario baseline/candidate report",
+		"",
+		"| Scenario | Metric changes | Variance changes |",
+		"|---|---|---|",
+		...scenarioRows,
+		"",
+		"## Per-role baseline/candidate report",
+		"",
+		"| Role | Metric changes | Variance changes |",
+		"|---|---|---|",
+		...roleRows,
+		"",
+	].join("\n");
+}
+
+function renderJunit(summary: EvalCompareV4Summary): string {
+	const failures = new Set(
+		summary.hardGate.failures.map((failure) =>
+			JSON.stringify([failure.scenarioId, failure.role, failure.target.id, failure.target.model, failure.metric]),
+		),
+	);
+	const represented = new Set<string>();
+	const cases = summary.behavioralMetrics.map((row) => {
+		const name = `${row.scenarioId}[${row.role}:${row.target.id}:${row.target.model ?? "none"}].${row.metric}`;
+		const key = JSON.stringify([row.scenarioId, row.role, row.target.id, row.target.model, row.metric]);
+		represented.add(key);
+		const detail = `change=${row.change} variance=${row.varianceChange} baseline=${format(row.baseline.mean)} candidate=${format(row.candidate.mean)}`;
+		return failures.has(key)
+			? `  <testcase classname="eval.behavior.${escapeXml(row.family)}" name="${escapeXml(name)}"><failure message="${escapeXml(row.change)}">${escapeXml(detail)}</failure></testcase>`
+			: `  <testcase classname="eval.behavior.${escapeXml(row.family)}" name="${escapeXml(name)}"><system-out>${escapeXml(detail)}</system-out></testcase>`;
+	});
+	for (const failure of summary.hardGate.failures) {
+		const key = JSON.stringify([
+			failure.scenarioId,
+			failure.role,
+			failure.target.id,
+			failure.target.model,
+			failure.metric,
+		]);
+		if (represented.has(key)) continue;
+		const name = `${failure.scenarioId}[${failure.role}:${failure.target.id}:${failure.target.model ?? "none"}].${failure.metric}`;
+		cases.push(
+			`  <testcase classname="eval.behavior.hard" name="${escapeXml(name)}"><failure message="${escapeXml(failure.change)}">hard behavioral gate</failure></testcase>`,
+		);
+	}
+	for (const failure of summary.hardGate.envelopeFailures) {
+		const name = `${failure.scenarioId}[${failure.role}:${failure.target.id}:${failure.target.model ?? "none"}].execution-envelope`;
+		cases.push(
+			`  <testcase classname="eval.behavior.envelope" name="${escapeXml(name)}"><failure message="incomparable">${escapeXml(failure.fields.join(", "))}</failure></testcase>`,
+		);
+	}
+	return [
+		`<testsuite name="eval-comparison" tests="${cases.length}" failures="${summary.hardGate.failures.length + summary.hardGate.envelopeFailures.length}">`,
+		...cases,
+		"</testsuite>",
+		"",
+	].join("\n");
+}
+
+function changeCounts(counts: {
+	improved: number;
+	regressed: number;
+	unchanged: number;
+	incomparable: number;
+}): string {
+	return `improved ${counts.improved}, regressed ${counts.regressed}, unchanged ${counts.unchanged}, incomparable ${counts.incomparable}`;
+}
+
+function format(value: number | null): string {
+	return value === null ? "null" : Number.isInteger(value) ? String(value) : value.toFixed(4);
+}
+
+function cell(value: string): string {
+	return value.replaceAll("|", "\\|");
+}
+
+function escapeXml(value: string): string {
+	return value.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;").replaceAll('"', "&quot;");
+}