@iowarp/clio-coder 0.3.8 → 0.3.9

package/src/domains/eval/metrics/call-ledger-stream.ts

@@ -0,0 +1,112 @@
+import { performance } from "node:perf_hooks";
+import type { SessionEntry } from "../../session/entries.js";
+
+export interface EvalCallLedgerFold {
+	push(chunk: string): void;
+	entries(): SessionEntry[];
+}
+
+/** Fold structured assistant completions into the eval's per-call ledger. */
+export function createEvalCallLedgerFold(now: () => number = () => performance.now()): EvalCallLedgerFold {
+	const entries: SessionEntry[] = [];
+	let pending = "";
+	let activeStartedAt: number | null = null;
+	let activeFirstOutputAt: number | null = null;
+
+	const consume = (line: string): void => {
+		const event = parseRecord(line);
+		if (event === null) return;
+		if (event.type === "message_start" && isAssistantMessage(event.message)) {
+			activeStartedAt = now();
+			activeFirstOutputAt = null;
+			return;
+		}
+		if (event.type === "message_update" && activeStartedAt !== null && activeFirstOutputAt === null) {
+			activeFirstOutputAt = now();
+			return;
+		}
+		if (event.type !== "message_end" || !isAssistantMessage(event.message)) return;
+		const message = event.message;
+		const usage = isRecord(message.usage) ? message.usage : null;
+		if (usage === null) {
+			activeStartedAt = null;
+			activeFirstOutputAt = null;
+			return;
+		}
+		const endedAt = now();
+		const promptCache: Record<string, unknown> = {
+			input: nonNegativeNumber(usage.input) ?? 0,
+			cacheRead: nonNegativeNumber(usage.cacheRead) ?? 0,
+			cacheWrite: nonNegativeNumber(usage.cacheWrite) ?? 0,
+			backendVerdict: "unknown",
+		};
+		if (isRecord(message.backendTimings)) promptCache.backend = structuredClone(message.backendTimings);
+		const previous = entries.at(-1);
+		entries.push({
+			kind: "message",
+			role: "assistant",
+			turnId: `eval-call-${entries.length + 1}`,
+			parentTurnId: previous?.turnId ?? null,
+			timestamp: messageTimestamp(message.timestamp),
+			payload: {
+				promptCache,
+				timing: {
+					ttftMs:
+						activeStartedAt === null ? null : Math.round(Math.max(0, (activeFirstOutputAt ?? endedAt) - activeStartedAt)),
+					apiMs: activeStartedAt === null ? 0 : Math.round(Math.max(0, endedAt - activeStartedAt)),
+				},
+				usage: structuredClone(usage),
+			},
+		});
+		activeStartedAt = null;
+		activeFirstOutputAt = null;
+	};
+
+	return {
+		push(chunk: string): void {
+			pending += chunk;
+			for (;;) {
+				const newline = pending.indexOf("\n");
+				if (newline === -1) break;
+				consume(pending.slice(0, newline).replace(/\r$/u, ""));
+				pending = pending.slice(newline + 1);
+			}
+		},
+		entries(): SessionEntry[] {
+			if (pending.length > 0) {
+				consume(pending.replace(/\r$/u, ""));
+				pending = "";
+			}
+			return structuredClone(entries);
+		},
+	};
+}
+
+function isAssistantMessage(value: unknown): value is Record<string, unknown> & { role: "assistant" } {
+	return isRecord(value) && value.role === "assistant";
+}
+
+function messageTimestamp(value: unknown): string {
+	const milliseconds = nonNegativeNumber(value);
+	if (milliseconds === null) return new Date(0).toISOString();
+	const date = new Date(milliseconds);
+	return Number.isFinite(date.getTime()) ? date.toISOString() : new Date(0).toISOString();
+}
+
+function nonNegativeNumber(value: unknown): number | null {
+	return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : null;
+}
+
+function parseRecord(line: string): Record<string, unknown> | null {
+	if (line.trim().length === 0) return null;
+	try {
+		const value: unknown = JSON.parse(line);
+		return isRecord(value) ? value : null;
+	} catch {
+		return null;
+	}
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}

package/src/domains/eval/metrics/tracked.ts

@@ -0,0 +1,413 @@
+import { readFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import type { RunReceipt } from "../../dispatch/types.js";
+import { listSessionLedgerRefs, parseSessionEntries } from "../../session/archive-readers.js";
+import { extractReasoningTokens } from "../../session/context-accounting.js";
+import type { SessionEntry } from "../../session/entries.js";
+import { evalHarnessMetricsFromReceipt } from "../harness-metrics.js";
+import type {
+	EvalMetricSource,
+	EvalSourcedNullableNumber,
+	EvalSourcedNumber,
+	EvalTrackedMetricsV1,
+} from "../schema/verdict.js";
+
+export interface EvalLedgerSnapshot {
+	entries: SessionEntry[];
+	compiledPromptHashes: string[];
+	promptManifests: EvalPromptManifestObservation[];
+	contextSnapshots: EvalContextSnapshotObservation[];
+}
+
+export interface EvalPromptManifestObservation {
+	systemPromptHash: string;
+	thinkingLevel: string | null;
+	projectPreload: {
+		mode: "full" | "synopsis" | "none";
+		chars: number;
+		lines: number;
+		reason: string | null;
+		nearLimit: boolean;
+		label: string;
+	} | null;
+	fragments: Array<{ id: string; contentHash: string }>;
+}
+
+export interface EvalContextSnapshotObservation {
+	runtimeId: string | null;
+	modelId: string | null;
+	promptHash: string | null;
+	toolSignature: string | null;
+}
+
+export interface BuildEvalTrackedMetricsInput {
+	ledgerEntries: ReadonlyArray<SessionEntry>;
+	receipt: RunReceipt | null;
+	fallbackWallClockMs: number;
+}
+
+interface AssistantCall {
+	payload: Record<string, unknown>;
+	promptCache: Record<string, unknown> | null;
+	backend: Record<string, unknown> | null;
+	timing: Record<string, unknown> | null;
+	usage: Record<string, unknown> | null;
+}
+
+interface NumericReading {
+	value: number;
+	source: EvalMetricSource;
+}
+
+/** Read every isolated session ledger and its prompt manifest before cleanup. */
+export async function readEvalLedgerSnapshot(stateDir: string): Promise<EvalLedgerSnapshot> {
+	const refs = await listSessionLedgerRefs(stateDir);
+	const entries: SessionEntry[] = [];
+	const compiledPromptHashes: string[] = [];
+	const promptManifests: EvalPromptManifestObservation[] = [];
+	const contextSnapshots: EvalContextSnapshotObservation[] = [];
+	for (const ref of refs) {
+		try {
+			const raw = await readFile(ref.path, "utf8");
+			entries.push(...parseSessionEntries(raw, ref.path).entries);
+		} catch {
+			// An unreadable ledger contributes no invented measurements.
+		}
+		try {
+			const manifest = await readFile(join(dirname(ref.path), "prompt-manifest.jsonl"), "utf8");
+			for (const line of manifest.split(/\r?\n/u)) {
+				const record = parseJsonRecord(line);
+				if (record === null) continue;
+				const hash = record?.systemPromptHash;
+				if (typeof hash !== "string" || !/^[a-f0-9]{64}$/u.test(hash)) continue;
+				compiledPromptHashes.push(hash);
+				const observation = promptManifestObservation(record);
+				if (observation !== null) promptManifests.push(observation);
+			}
+		} catch {
+			// A missing manifest leaves compiled prompt provenance unresolved.
+		}
+		try {
+			const snapshots = await readFile(join(dirname(ref.path), "context-snapshots.jsonl"), "utf8");
+			for (const line of snapshots.split(/\r?\n/u)) {
+				const record = parseJsonRecord(line);
+				if (record === null) continue;
+				contextSnapshots.push({
+					runtimeId: nullableString(record.runtimeId),
+					modelId: nullableString(record.modelId),
+					promptHash: nullableDigest(record.promptHash),
+					toolSignature: nullableDigest(record.toolSignature),
+				});
+			}
+		} catch {
+			// A missing context snapshot leaves runtime and tool-surface provenance unresolved.
+		}
+	}
+	return { entries, compiledPromptHashes: [...new Set(compiledPromptHashes)], promptManifests, contextSnapshots };
+}
+
+/** Fold structured ledger calls and a sealed receipt into verdict metrics. */
+export function buildEvalTrackedMetrics(input: BuildEvalTrackedMetricsInput): EvalTrackedMetricsV1 {
+	const calls = assistantCalls(input.ledgerEntries);
+	const compactionEntries = input.ledgerEntries.filter((entry) => entry.kind === "compactionSummary");
+	const compactionUsage = compactionEntries.flatMap((entry) => {
+		if (entry.kind !== "compactionSummary" || !isRecord(entry.usage)) return [];
+		return [entry.usage];
+	});
+
+	const modelCallReadings: NumericReading[] = calls.map(() => ledgerReading(1));
+	for (const entry of compactionEntries) {
+		if (entry.kind !== "compactionSummary") continue;
+		const apiCalls = isRecord(entry.usage) ? nonNegativeNumber(entry.usage.apiCalls) : null;
+		modelCallReadings.push(apiCalls === null ? estimatedReading(1) : ledgerReading(apiCalls));
+	}
+
+	const uncachedReadings = calls.map(uncachedPrefillForCall);
+	const cacheReadings = calls.map(cacheReadForCall);
+	const generatedReadings = calls.map(generatedForCall);
+	for (const usage of compactionUsage) {
+		uncachedReadings.push(readingFromUsage(usage, "input"));
+		cacheReadings.push(readingFromUsage(usage, "cacheRead"));
+		generatedReadings.push(readingFromUsage(usage, "output"));
+	}
+
+	const reasoning = reasoningMetric(input.receipt, calls, compactionUsage);
+	const receiptToolMetrics = input.receipt === null ? null : evalHarnessMetricsFromReceipt(input.receipt);
+	const ledgerToolCalls = input.ledgerEntries.filter(
+		(entry) => entry.kind === "message" && entry.role === "tool_call",
+	).length;
+	const ledgerToolErrors = input.ledgerEntries.filter((entry) => {
+		if (entry.kind !== "message" || entry.role !== "tool_result" || !isRecord(entry.payload)) return false;
+		return entry.payload.isError === true || entry.payload.outcome === "error";
+	}).length;
+	const receiptToolErrors = input.receipt?.toolStats.reduce((sum, stat) => sum + finiteNonNegative(stat.errors), 0);
+	const expectedColdReasons = expectedColdReasonMetrics(calls);
+
+	return {
+		modelCalls: sumReadings(modelCallReadings, "ledger"),
+		uncachedPrefillTokens: sumReadings(uncachedReadings, "estimated"),
+		cacheReadTokens: sumReadings(cacheReadings, "estimated"),
+		generatedTokens: sumReadings(generatedReadings, "estimated"),
+		reasoningTokens: reasoning,
+		toolCalls:
+			receiptToolMetrics === null
+				? { value: ledgerToolCalls, source: "ledger" }
+				: { value: receiptToolMetrics.toolCalls, source: "receipt" },
+		toolErrors:
+			receiptToolErrors === undefined
+				? { value: ledgerToolErrors, source: "ledger" }
+				: { value: receiptToolErrors, source: "receipt" },
+		ttftMsFirstCall: firstCallTtft(calls),
+		wallClockMs: wallClockMetric(input.receipt, input.fallbackWallClockMs),
+		contextTokensAtEnd: contextTokensAtEnd(calls, compactionEntries),
+		compactions: { value: compactionEntries.length, source: "ledger" },
+		expectedColdReasons,
+	};
+}
+
+export function emptyEvalTrackedMetrics(source: EvalMetricSource = "estimated"): EvalTrackedMetricsV1 {
+	const zero = (): EvalSourcedNumber => ({ value: 0, source });
+	return {
+		modelCalls: zero(),
+		uncachedPrefillTokens: zero(),
+		cacheReadTokens: zero(),
+		generatedTokens: zero(),
+		reasoningTokens: { value: null, source },
+		toolCalls: zero(),
+		toolErrors: zero(),
+		ttftMsFirstCall: zero(),
+		wallClockMs: zero(),
+		contextTokensAtEnd: zero(),
+		compactions: zero(),
+		expectedColdReasons: {},
+	};
+}
+
+function assistantCalls(entries: ReadonlyArray<SessionEntry>): AssistantCall[] {
+	return entries.flatMap((entry) => {
+		if (entry.kind !== "message" || entry.role !== "assistant" || !isRecord(entry.payload)) return [];
+		const promptCache = recordField(entry.payload, "promptCache");
+		const timing = recordField(entry.payload, "timing");
+		const usage = recordField(entry.payload, "usage");
+		if (promptCache === null && timing === null && usage === null) return [];
+		return [
+			{
+				payload: entry.payload,
+				promptCache,
+				backend: promptCache === null ? null : recordField(promptCache, "backend"),
+				timing,
+				usage,
+			},
+		];
+	});
+}
+
+function uncachedPrefillForCall(call: AssistantCall): NumericReading {
+	const promptTokens = call.backend === null ? null : nonNegativeNumber(call.backend.promptTokens);
+	const cachedTokens = call.backend === null ? null : nonNegativeNumber(call.backend.cachedTokens);
+	if (promptTokens !== null && cachedTokens !== null && cachedTokens <= promptTokens) {
+		return ledgerReading(promptTokens - cachedTokens);
+	}
+	const piInput = call.promptCache === null ? null : nonNegativeNumber(call.promptCache.input);
+	if (piInput !== null) return ledgerReading(piInput);
+	const legacyInput = call.usage === null ? null : nonNegativeNumber(call.usage.input);
+	return legacyInput === null ? estimatedReading(0) : estimatedReading(legacyInput);
+}
+
+function cacheReadForCall(call: AssistantCall): NumericReading {
+	const cachedTokens = call.backend === null ? null : nonNegativeNumber(call.backend.cachedTokens);
+	if (cachedTokens !== null) return ledgerReading(cachedTokens);
+	const piCacheRead = call.promptCache === null ? null : nonNegativeNumber(call.promptCache.cacheRead);
+	if (piCacheRead !== null) return ledgerReading(piCacheRead);
+	const legacyCacheRead = call.usage === null ? null : nonNegativeNumber(call.usage.cacheRead);
+	return legacyCacheRead === null ? estimatedReading(0) : estimatedReading(legacyCacheRead);
+}
+
+function generatedForCall(call: AssistantCall): NumericReading {
+	const predictedTokens = call.backend === null ? null : nonNegativeNumber(call.backend.predictedTokens);
+	if (predictedTokens !== null) return ledgerReading(predictedTokens);
+	const output = call.usage === null ? null : nonNegativeNumber(call.usage.output);
+	return output === null ? estimatedReading(0) : ledgerReading(output);
+}
+
+function readingFromUsage(usage: Record<string, unknown>, field: string): NumericReading {
+	const value = nonNegativeNumber(usage[field]);
+	return value === null ? estimatedReading(0) : ledgerReading(value);
+}
+
+function reasoningMetric(
+	receipt: RunReceipt | null,
+	calls: ReadonlyArray<AssistantCall>,
+	compactionUsage: ReadonlyArray<Record<string, unknown>>,
+): EvalSourcedNullableNumber {
+	if (receipt !== null && typeof receipt.reasoningTokenCount === "number") {
+		return { value: finiteNonNegative(receipt.reasoningTokenCount), source: "receipt" };
+	}
+	let total = 0;
+	let measured = false;
+	for (const call of calls) {
+		const value = extractReasoningTokens(call.usage);
+		if (value === null) continue;
+		measured = true;
+		total += finiteNonNegative(value);
+	}
+	for (const usage of compactionUsage) {
+		const value = nonNegativeNumber(usage.reasoning);
+		if (value === null) continue;
+		measured = true;
+		total += value;
+	}
+	return measured ? { value: total, source: "ledger" } : { value: null, source: "estimated" };
+}
+
+function firstCallTtft(calls: ReadonlyArray<AssistantCall>): EvalSourcedNumber {
+	const first = calls[0];
+	const value = first?.timing === null || first?.timing === undefined ? null : nonNegativeNumber(first.timing.ttftMs);
+	return value === null ? { value: 0, source: "estimated" } : { value, source: "ledger" };
+}
+
+function wallClockMetric(receipt: RunReceipt | null, fallback: number): EvalSourcedNumber {
+	if (receipt !== null) {
+		const started = Date.parse(receipt.startedAt);
+		const ended = Date.parse(receipt.endedAt);
+		if (Number.isFinite(started) && Number.isFinite(ended) && ended >= started) {
+			return { value: ended - started, source: "receipt" };
+		}
+	}
+	return { value: finiteNonNegative(fallback), source: "estimated" };
+}
+
+function contextTokensAtEnd(
+	calls: ReadonlyArray<AssistantCall>,
+	compactions: ReadonlyArray<SessionEntry>,
+): EvalSourcedNumber {
+	const lastCall = calls.at(-1);
+	if (lastCall !== undefined) {
+		const lastReading = contextForCall(lastCall);
+		if (lastReading !== null && lastReading > 0) return { value: lastReading, source: "ledger" };
+		if (lastReading === 0) {
+			for (const call of [...calls.slice(0, -1)].reverse()) {
+				const reading = contextForCall(call);
+				if (reading !== null && reading > 0) return { value: reading, source: "ledger" };
+			}
+		}
+	}
+	const lastCompaction = compactions.at(-1);
+	if (lastCompaction?.kind === "compactionSummary") {
+		const tokensAfter = nonNegativeNumber(lastCompaction.tokensAfter);
+		if (tokensAfter !== null) return { value: tokensAfter, source: "ledger" };
+	}
+	return { value: 0, source: "estimated" };
+}
+
+function contextForCall(call: AssistantCall): number | null {
+	const promptTokens = call.backend === null ? null : nonNegativeNumber(call.backend.promptTokens);
+	const predictedTokens = call.backend === null ? null : nonNegativeNumber(call.backend.predictedTokens);
+	if (promptTokens !== null && predictedTokens !== null) return promptTokens + predictedTokens;
+	const input = call.promptCache === null ? null : nonNegativeNumber(call.promptCache.input);
+	const cacheRead = call.promptCache === null ? null : nonNegativeNumber(call.promptCache.cacheRead);
+	const output = call.usage === null ? null : nonNegativeNumber(call.usage.output);
+	return input === null || cacheRead === null || output === null ? null : input + cacheRead + output;
+}
+
+function expectedColdReasonMetrics(calls: ReadonlyArray<AssistantCall>): Record<string, EvalSourcedNumber> {
+	const counts = new Map<string, number>();
+	for (const call of calls) {
+		const reasons = call.promptCache?.expectedColdReasons;
+		if (!Array.isArray(reasons)) continue;
+		const unique = new Set(reasons.filter((reason): reason is string => typeof reason === "string" && reason.length > 0));
+		for (const reason of unique) counts.set(reason, (counts.get(reason) ?? 0) + 1);
+	}
+	return Object.fromEntries(
+		[...counts.entries()]
+			.sort(([left], [right]) => left.localeCompare(right))
+			.map(([reason, value]) => [reason, { value, source: "ledger" as const }]),
+	);
+}
+
+function sumReadings(readings: ReadonlyArray<NumericReading>, emptySource: EvalMetricSource): EvalSourcedNumber {
+	if (readings.length === 0) return { value: 0, source: emptySource };
+	return {
+		value: readings.reduce((sum, reading) => sum + reading.value, 0),
+		source: readings.some((reading) => reading.source === "estimated") ? "estimated" : "ledger",
+	};
+}
+
+function ledgerReading(value: number): NumericReading {
+	return { value: finiteNonNegative(value), source: "ledger" };
+}
+
+function estimatedReading(value: number): NumericReading {
+	return { value: finiteNonNegative(value), source: "estimated" };
+}
+
+function finiteNonNegative(value: number): number {
+	return Number.isFinite(value) && value >= 0 ? value : 0;
+}
+
+function nonNegativeNumber(value: unknown): number | null {
+	return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : null;
+}
+
+function recordField(record: Record<string, unknown>, field: string): Record<string, unknown> | null {
+	return isRecord(record[field]) ? record[field] : null;
+}
+
+function parseJsonRecord(line: string): Record<string, unknown> | null {
+	if (line.trim().length === 0) return null;
+	try {
+		const parsed: unknown = JSON.parse(line);
+		return isRecord(parsed) ? parsed : null;
+	} catch {
+		return null;
+	}
+}
+
+function promptManifestObservation(record: Record<string, unknown>): EvalPromptManifestObservation | null {
+	const systemPromptHash = nullableDigest(record.systemPromptHash);
+	if (systemPromptHash === null || !Array.isArray(record.fragments)) return null;
+	const fragments = record.fragments.flatMap((entry) => {
+		if (!isRecord(entry) || typeof entry.id !== "string") return [];
+		const contentHash = nullableDigest(entry.contentHash);
+		return contentHash === null ? [] : [{ id: entry.id, contentHash }];
+	});
+	const preload = record.projectPreload;
+	const projectPreload: EvalPromptManifestObservation["projectPreload"] =
+		preload === null
+			? null
+			: isRecord(preload) &&
+					(preload.mode === "full" || preload.mode === "synopsis" || preload.mode === "none") &&
+					typeof preload.chars === "number" &&
+					Number.isInteger(preload.chars) &&
+					typeof preload.lines === "number" &&
+					Number.isInteger(preload.lines) &&
+					typeof preload.nearLimit === "boolean" &&
+					typeof preload.label === "string"
+				? {
+						mode: preload.mode as "full" | "synopsis" | "none",
+						chars: preload.chars,
+						lines: preload.lines,
+						reason: nullableString(preload.reason),
+						nearLimit: preload.nearLimit,
+						label: preload.label,
+					}
+				: null;
+	return {
+		systemPromptHash,
+		thinkingLevel: nullableString(record.thinkingLevel),
+		projectPreload,
+		fragments,
+	};
+}
+
+function nullableString(value: unknown): string | null {
+	return typeof value === "string" && value.length > 0 ? value : null;
+}
+
+function nullableDigest(value: unknown): string | null {
+	return typeof value === "string" && /^[a-f0-9]{64}$/u.test(value) ? value : null;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}

package/src/domains/eval/provenance.ts

@@ -1,5 +1,9 @@
 import { spawnSync } from "node:child_process";
+import { readSettings } from "../../core/config.js";
 import { readClioVersion, resolvePackageRoot } from "../../core/package-root.js";
+import type { RunReceipt } from "../dispatch/types.js";
+import type { EvalServingConfigurationV1 } from "./schema/serving.js";
+import type { EvalSuiteTargetV2 } from "./schema/suite.js";
 import type { EvalClioProvenance, EvalEnvironmentProvenance } from "./types.js";
 
 export interface EvalProvenanceOptions {
@@ -7,6 +11,14 @@ export interface EvalProvenanceOptions {
 	commit?: string | null;
 }
 
+export interface EvalServingObservation {
+	targetId: string;
+	runtimeId: string | null;
+	modelId: string | null;
+	thinkingLevel: string | null;
+	compiledPromptHash: string | null;
+}
+
 export function evalClioProvenance(options: EvalProvenanceOptions = {}): EvalClioProvenance {
 	return {
 		version: readClioVersion(),
@@ -22,6 +34,45 @@ export function evalEnvironmentProvenance(): EvalEnvironmentProvenance {
 	};
 }
 
+/** Capture serving identity after the matrix has run while a local server is awake. */
+export async function evalServingConfiguration(
+	targets: ReadonlyArray<EvalSuiteTargetV2>,
+	observations: ReadonlyArray<EvalServingObservation>,
+): Promise<EvalServingConfigurationV1> {
+	const targetId = targets.length === 1 ? (targets[0]?.id ?? "unknown") : "multiple";
+	const configured = configuredTarget(targetId);
+	const target = targets.length === 1 ? targets[0] : undefined;
+	const runtimeId = consensus(observations.map((entry) => entry.runtimeId)) ?? configured?.runtime ?? null;
+	const modelId =
+		consensus(observations.map((entry) => entry.modelId)) ?? target?.model ?? configured?.defaultModel ?? null;
+	const props = configured?.url === undefined ? null : await readServingProps(configured.url, modelId);
+	return {
+		targetId,
+		runtimeId,
+		modelId,
+		serverBuild: props?.serverBuild ?? null,
+		total_slots: props?.totalSlots ?? null,
+		thinkingLevel: consensus(observations.map((entry) => entry.thinkingLevel)) ?? target?.thinking ?? null,
+		compiledPromptHash: consensus(observations.map((entry) => entry.compiledPromptHash)),
+	};
+}
+
+export function evalServingObservationFrom(
+	target: EvalSuiteTargetV2,
+	receipt: RunReceipt | null,
+	compiledPromptHashes: ReadonlyArray<string>,
+): EvalServingObservation {
+	const compiledPromptHash =
+		receipt?.staticCompositionHash ?? receipt?.compiledPromptHash ?? consensus(compiledPromptHashes);
+	return {
+		targetId: receipt?.targetId ?? target.id,
+		runtimeId: receipt?.runtimeId ?? null,
+		modelId: receipt?.wireModelId ?? target.model ?? null,
+		thinkingLevel: receipt?.runtimeResolution?.effectiveThinkingLevel ?? target.thinking ?? null,
+		compiledPromptHash,
+	};
+}
+
 function currentClioCommit(): string | null {
 	const result = spawnSync("git", ["rev-parse", "HEAD"], {
 		cwd: resolvePackageRoot(),
@@ -33,3 +84,69 @@ function currentClioCommit(): string | null {
 	const value = result.stdout.trim();
 	return value.length > 0 ? value : null;
 }
+
+interface ConfiguredServingTarget {
+	runtime: string;
+	url?: string;
+	defaultModel?: string;
+}
+
+function configuredTarget(targetId: string): ConfiguredServingTarget | null {
+	try {
+		const target = readSettings().targets.find((entry) => entry.id === targetId);
+		if (target === undefined) return null;
+		return {
+			runtime: target.runtime,
+			...(target.url === undefined ? {} : { url: target.url }),
+			...(target.defaultModel === undefined ? {} : { defaultModel: target.defaultModel }),
+		};
+	} catch {
+		return null;
+	}
+}
+
+async function readServingProps(
+	targetUrl: string,
+	modelId: string | null,
+): Promise<{ serverBuild: string | null; totalSlots: number | null } | null> {
+	const root = targetUrl.replace(/\/+$/u, "").replace(/\/v1$/u, "");
+	try {
+		const response = await fetch(`${root}/props`, { signal: AbortSignal.timeout(5_000) });
+		if (!response.ok) return null;
+		const value: unknown = await response.json();
+		if (!isRecord(value)) return null;
+		const totalSlots = positiveInteger(value.total_slots) ?? (await readServingSlots(root, modelId));
+		return {
+			serverBuild: typeof value.build_info === "string" && value.build_info.length > 0 ? value.build_info : null,
+			totalSlots,
+		};
+	} catch {
+		return null;
+	}
+}
+
+async function readServingSlots(root: string, modelId: string | null): Promise<number | null> {
+	if (modelId === null) return null;
+	try {
+		const query = new URLSearchParams({ model: modelId });
+		const response = await fetch(`${root}/slots?${query}`, { signal: AbortSignal.timeout(5_000) });
+		if (!response.ok) return null;
+		const value: unknown = await response.json();
+		return Array.isArray(value) && value.length > 0 ? value.length : null;
+	} catch {
+		return null;
+	}
+}
+
+function consensus(values: ReadonlyArray<string | null>): string | null {
+	const known = [...new Set(values.filter((value): value is string => typeof value === "string" && value.length > 0))];
+	return known.length === 1 ? (known[0] ?? null) : null;
+}
+
+function positiveInteger(value: unknown): number | null {
+	return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : null;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}