@iowarp/clio-coder 0.3.8 → 0.3.9

package/src/domains/eval/compare/envelope.ts

@@ -0,0 +1,128 @@
+import type { EvalArtifactResultV4 } from "../schema/artifact.js";
+import type { EvalExecutionEnvelopeV1, EvalExecutionMatrixDimensionV1 } from "../schema/execution-envelope.js";
+
+export interface EvalEnvelopeMismatchV1 {
+	scenarioId: string;
+	role: string;
+	target: { id: string; model: string | null };
+	fields: string[];
+}
+
+export interface EvalEnvelopeComparabilityV1 {
+	comparable: boolean;
+	mismatchedFields: string[];
+}
+
+export function compareEvalExecutionEnvelopesV1(
+	identity: { scenarioId: string; role: string; target: { id: string; model: string | null } },
+	baseline: ReadonlyArray<EvalArtifactResultV4>,
+	candidate: ReadonlyArray<EvalArtifactResultV4>,
+	baselineDimensions: ReadonlyArray<EvalExecutionMatrixDimensionV1>,
+	candidateDimensions: ReadonlyArray<EvalExecutionMatrixDimensionV1>,
+): EvalEnvelopeMismatchV1 | null {
+	const leftDimensions = [...baselineDimensions].sort();
+	const rightDimensions = [...candidateDimensions].sort();
+	if (stableJson(leftDimensions) !== stableJson(rightDimensions)) {
+		return { ...identity, fields: ["matrix.dimensions"] };
+	}
+	if (
+		(baseline.some((result) => result.executionEnvelope !== undefined) &&
+			baseline.some((result) => result.executionEnvelope === undefined)) ||
+		(candidate.some((result) => result.executionEnvelope !== undefined) &&
+			candidate.some((result) => result.executionEnvelope === undefined))
+	) {
+		return { ...identity, fields: ["executionEnvelope.missingTrial"] };
+	}
+	const ignored = new Set(leftDimensions);
+	const baselineEnvelopes = uniqueEnvelopes(baseline, ignored);
+	const candidateEnvelopes = uniqueEnvelopes(candidate, ignored);
+	if (baselineEnvelopes.length === 0 && candidateEnvelopes.length === 0) return null;
+	if (baselineEnvelopes.length === 0 || candidateEnvelopes.length === 0) {
+		return { ...identity, fields: ["executionEnvelope"] };
+	}
+	if (baselineEnvelopes.length > 1 || candidateEnvelopes.length > 1) {
+		return { ...identity, fields: ["executionEnvelope.withinRunVariance"] };
+	}
+	const left = baselineEnvelopes[0];
+	const right = candidateEnvelopes[0];
+	if (left === undefined || right === undefined || stableJson(left) === stableJson(right)) return null;
+	return { ...identity, fields: differingFields(left, right, ignored) };
+}
+
+function uniqueEnvelopes(
+	results: ReadonlyArray<EvalArtifactResultV4>,
+	ignored: ReadonlySet<EvalExecutionMatrixDimensionV1>,
+): EvalExecutionEnvelopeV1[] {
+	const byIdentity = new Map<string, EvalExecutionEnvelopeV1>();
+	for (const result of results) {
+		if (result.executionEnvelope === undefined) continue;
+		const normalized = normalizedEnvelope(result.executionEnvelope, ignored);
+		byIdentity.set(stableJson(normalized), normalized);
+	}
+	return [...byIdentity.values()];
+}
+
+function normalizedEnvelope(
+	envelope: EvalExecutionEnvelopeV1,
+	ignored: ReadonlySet<EvalExecutionMatrixDimensionV1>,
+): EvalExecutionEnvelopeV1 {
+	return {
+		...envelope,
+		prompt: ignored.has("prompt") ? { fragments: [], compositionHash: null } : envelope.prompt,
+		recipe: ignored.has("recipe") ? null : envelope.recipe,
+		target: ignored.has("target") ? "<matrix>" : envelope.target,
+		wireModel: ignored.has("wireModel") ? null : envelope.wireModel,
+		runtime: ignored.has("runtime") ? null : envelope.runtime,
+		thinkingLevel: ignored.has("thinkingLevel") ? null : envelope.thinkingLevel,
+		toolSignature: ignored.has("toolSignature") ? null : envelope.toolSignature,
+		autonomy: ignored.has("autonomy") ? null : envelope.autonomy,
+		policyHashes: ignored.has("policy") ? { rulePack: null, project: null } : envelope.policyHashes,
+		projectContext: ignored.has("projectContext")
+			? {
+					kind: "none",
+					tier: null,
+					contentHash: null,
+					chars: null,
+					sections: [],
+					rulesApplied: [],
+					operatorProfileApplied: null,
+				}
+			: envelope.projectContext,
+		corpus: ignored.has("corpus") ? { id: "<matrix>", version: "<matrix>" } : envelope.corpus,
+	};
+}
+
+function differingFields(
+	left: EvalExecutionEnvelopeV1,
+	right: EvalExecutionEnvelopeV1,
+	ignored: ReadonlySet<EvalExecutionMatrixDimensionV1>,
+): string[] {
+	const fields: Array<[EvalExecutionMatrixDimensionV1, string, unknown, unknown]> = [
+		["prompt", "prompt", left.prompt, right.prompt],
+		["recipe", "recipe", left.recipe, right.recipe],
+		["target", "target", left.target, right.target],
+		["wireModel", "wireModel", left.wireModel, right.wireModel],
+		["runtime", "runtime", left.runtime, right.runtime],
+		["thinkingLevel", "thinkingLevel", left.thinkingLevel, right.thinkingLevel],
+		["toolSignature", "toolSignature", left.toolSignature, right.toolSignature],
+		["autonomy", "autonomy", left.autonomy, right.autonomy],
+		["policy", "policyHashes", left.policyHashes, right.policyHashes],
+		["projectContext", "projectContext", left.projectContext, right.projectContext],
+		["corpus", "corpus", left.corpus, right.corpus],
+	];
+	return fields.flatMap(([dimension, field, baseline, candidate]) =>
+		ignored.has(dimension) || stableJson(baseline) === stableJson(candidate) ? [] : [field],
+	);
+}
+
+function stableJson(value: unknown): string {
+	if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`;
+	if (typeof value === "object" && value !== null) {
+		return `{${Object.entries(value as Record<string, unknown>)
+			.filter(([, entry]) => entry !== undefined)
+			.sort(([left], [right]) => left.localeCompare(right))
+			.map(([key, entry]) => `${JSON.stringify(key)}:${stableJson(entry)}`)
+			.join(",")}}`;
+	}
+	return JSON.stringify(value);
+}

package/src/domains/eval/compare/gates.ts

@@ -15,6 +15,7 @@ export interface EvalGateFailure {
 export interface EvalGateResult {
 	pass: boolean;
 	failures: EvalGateFailure[];
+	informational: EvalGateFailure[];
 }
 
 /**
@@ -30,21 +31,30 @@ export interface EvalGateResult {
  * collapsing the matrix into a single aggregate nobody can act on.
  */
 export function evaluateGate(artifact: EvalArtifactV4, thresholds: EvalSuiteThresholdsV2): EvalGateResult {
-	const failures: EvalGateFailure[] = [];
-	for (const assertion of thresholds.fail) {
+	const failures = evaluateAssertions(artifact, thresholds.fail);
+	const informational = evaluateAssertions(artifact, thresholds.informational ?? []);
+	return { pass: failures.length === 0, failures, informational };
+}
+
+function evaluateAssertions(
+	artifact: EvalArtifactV4,
+	assertions: ReadonlyArray<EvalMetricAssertion>,
+): EvalGateFailure[] {
+	const findings: EvalGateFailure[] = [];
+	for (const assertion of assertions) {
 		const whole = resolveMetricAssertion(assertion, {}, artifact);
 		if (!whole.unresolved) {
-			if (whole.holds) failures.push({ assertion, actual: whole.actual, unresolved: false });
+			if (whole.holds) findings.push({ assertion, actual: whole.actual, unresolved: false });
 			continue;
 		}
 		if (artifact.results.length === 0) {
-			failures.push({ assertion, actual: null, unresolved: true });
+			findings.push({ assertion, actual: null, unresolved: true });
 			continue;
 		}
 		for (const result of artifact.results) {
 			const perRun = resolveMetricAssertion(assertion, result.metrics);
 			if (!perRun.unresolved && !perRun.holds) continue;
-			failures.push({
+			findings.push({
 				assertion,
 				actual: perRun.actual,
 				unresolved: perRun.unresolved,
@@ -53,7 +63,7 @@ export function evaluateGate(artifact: EvalArtifactV4, thresholds: EvalSuiteThre
 			});
 		}
 	}
-	return { pass: failures.length === 0, failures };
+	return findings;
 }
 
 /** One operator-facing line per gate failure, naming the run when there is one. */
@@ -64,3 +74,11 @@ export function renderGateFailure(failure: EvalGateFailure): string {
 		? `  ${metric}${run}: unresolved metric (fail closed)\n`
 		: `  ${metric} ${op} ${JSON.stringify(value)}${run}: actual ${JSON.stringify(failure.actual)}\n`;
 }
+
+export function renderInformationalBudget(finding: EvalGateFailure): string {
+	const run = finding.taskId === undefined ? "" : ` [${finding.taskId}#${finding.repeatIndex ?? 0}]`;
+	const { metric, op, value } = finding.assertion;
+	return finding.unresolved
+		? `  ${metric}${run}: unmeasured informational budget\n`
+		: `  ${metric} ${op} ${JSON.stringify(value)}${run}: actual ${JSON.stringify(finding.actual)}\n`;
+}

package/src/domains/eval/compare/thresholds.ts

@@ -5,13 +5,40 @@ import type { EvalMetricAssertion, EvalSuiteThresholdsV2 } from "../schema/suite
 
 export function loadThresholds(path: string): EvalSuiteThresholdsV2 {
 	const parsed = parseYaml(readFileSync(path, "utf8")) as unknown;
-	if (isRecord(parsed) && Array.isArray(parsed.fail)) return { fail: parsed.fail as EvalMetricAssertion[] };
-	if (isRecord(parsed) && isRecord(parsed.thresholds) && Array.isArray(parsed.thresholds.fail)) {
-		return { fail: parsed.thresholds.fail as EvalMetricAssertion[] };
+	const root = isRecord(parsed) && isRecord(parsed.thresholds) ? parsed.thresholds : parsed;
+	if (isRecord(root) && (Array.isArray(root.fail) || Array.isArray(root.informational))) {
+		return {
+			fail: parseAssertions(root.fail, `${path}.fail`),
+			informational: parseAssertions(root.informational, `${path}.informational`),
+		};
 	}
 	throw new Error(`invalid thresholds file: ${path}`);
 }
 
+function parseAssertions(value: unknown, source: string): EvalMetricAssertion[] {
+	if (value === undefined) return [];
+	if (!Array.isArray(value)) throw new Error(`${source}: expected array`);
+	return value.map((entry, index) => {
+		if (!isRecord(entry)) throw new Error(`${source}[${index}]: expected object`);
+		if (typeof entry.metric !== "string" || entry.metric.length === 0) {
+			throw new Error(`${source}[${index}].metric: expected non-empty string`);
+		}
+		if (!isOp(entry.op)) throw new Error(`${source}[${index}].op: expected lt, lte, gt, gte, eq, or neq`);
+		if (!isScalar(entry.value)) throw new Error(`${source}[${index}].value: expected scalar`);
+		return { metric: entry.metric, op: entry.op, value: entry.value };
+	});
+}
+
+function isOp(value: unknown): value is EvalMetricAssertion["op"] {
+	return value === "lt" || value === "lte" || value === "gt" || value === "gte" || value === "eq" || value === "neq";
+}
+
+function isScalar(value: unknown): value is number | string | boolean {
+	return (
+		(typeof value === "number" && Number.isFinite(value)) || typeof value === "string" || typeof value === "boolean"
+	);
+}
+
 export interface EvalAssertionResolution {
 	actual: number | string | boolean | null;
 	/** True when neither the metric map nor the artifact carries a value for this metric. */

package/src/domains/eval/execution-provenance.ts

@@ -0,0 +1,240 @@
+import { createHash } from "node:crypto";
+import { discoverAgentRecipes } from "../agents/registry.js";
+import { agentSpecFingerprint, normalizeAgentSpec } from "../agents/spec.js";
+import type { RunReceipt } from "../dispatch/types.js";
+import { loadFragments } from "../prompts/fragment-loader.js";
+import { createSafetyPolicyEngine } from "../safety/policy-engine.js";
+import type { EvalLedgerSnapshot, EvalPromptManifestObservation } from "./metrics/tracked.js";
+import type { EvalExecutionEnvelopeV1, EvalPromptFragmentIdentityV1 } from "./schema/execution-envelope.js";
+import { EVAL_EXECUTION_ENVELOPE_SCHEMA_V1 } from "./schema/execution-envelope.js";
+import type { EvalSuiteTargetV2, EvalSuiteTaskV2 } from "./schema/suite.js";
+
+/** Bounded execution facts a deterministic grader can surface from an in-process dispatch. */
+export interface EvalExecutionObservationV1 {
+	compositionHash: string | null;
+	target: string | null;
+	wireModel: string | null;
+	runtime: string | null;
+	thinkingLevel: string | null;
+	toolSignature: string | null;
+	autonomy: string | null;
+	policyHashes: { rulePack: string | null; project: string | null };
+	projectContext: {
+		tier: string | null;
+		contentHash: string | null;
+		chars: number | null;
+		sections: string[];
+		rulesApplied: string[];
+		operatorProfileApplied: boolean | null;
+	} | null;
+}
+
+export interface BuildEvalExecutionEnvelopeInput {
+	task: EvalSuiteTaskV2;
+	target: EvalSuiteTargetV2;
+	cwd: string | null;
+	receipt: RunReceipt | null;
+	ledger: EvalLedgerSnapshot;
+	observation?: EvalExecutionObservationV1;
+}
+
+/**
+ * Bind one behavioral result to the exact model-facing and policy-facing
+ * identity available at the runner boundary. Nulls mean the declared
+ * machinery-only mode has no such model concept; they never stand in for a
+ * guessed value.
+ */
+export function buildEvalExecutionEnvelopeV1(input: BuildEvalExecutionEnvelopeInput): EvalExecutionEnvelopeV1 {
+	const scenario = input.task.behavioral;
+	if (scenario === undefined) throw new Error(`behavioral task ${input.task.id} has no behavioral scenario`);
+	const manifest = input.ledger.promptManifests.at(-1) ?? null;
+	const contextSnapshot = input.ledger.contextSnapshots.at(-1) ?? null;
+	const recipe = recipeIdentity(
+		input,
+		scenario.execution.subject.kind === "worker" ? scenario.execution.subject.role : null,
+	);
+	const policy = policyIdentity(input.cwd, input.receipt, input.observation);
+	const autonomy =
+		input.receipt?.autonomyEnforcement?.autonomy ?? input.observation?.autonomy ?? input.task.runner.autonomy ?? null;
+	const promptFragments = promptFragmentIdentities(manifest, recipe, autonomy);
+	const compositionHash =
+		input.receipt?.staticCompositionHash ??
+		input.observation?.compositionHash ??
+		manifest?.systemPromptHash ??
+		contextSnapshot?.promptHash ??
+		null;
+	const projectContext = projectContextIdentity(input, manifest, promptFragments);
+	return {
+		schema: EVAL_EXECUTION_ENVELOPE_SCHEMA_V1,
+		prompt: { fragments: promptFragments, compositionHash },
+		recipe: recipe === null ? null : { id: recipe.id, version: recipe.version, contentHash: recipe.contentHash },
+		target: input.receipt?.targetId ?? input.observation?.target ?? input.target.id,
+		wireModel:
+			input.receipt?.wireModelId ?? input.observation?.wireModel ?? contextSnapshot?.modelId ?? input.target.model ?? null,
+		runtime: input.receipt?.runtimeId ?? input.observation?.runtime ?? contextSnapshot?.runtimeId ?? null,
+		thinkingLevel:
+			input.receipt?.runtimeResolution?.effectiveThinkingLevel ??
+			input.observation?.thinkingLevel ??
+			manifest?.thinkingLevel ??
+			input.target.thinking ??
+			null,
+		toolSignature:
+			input.receipt?.toolSignature ?? input.observation?.toolSignature ?? contextSnapshot?.toolSignature ?? null,
+		autonomy,
+		policyHashes: policy,
+		projectContext,
+		corpus: { ...scenario.corpus },
+	};
+}
+
+function recipeIdentity(
+	input: BuildEvalExecutionEnvelopeInput,
+	role: string | null,
+): { id: string; version: number; contentHash: string; personaHash: string } | null {
+	const id = input.receipt?.agentId ?? input.task.runner.agent ?? role;
+	if (id === null || input.cwd === null) return null;
+	try {
+		const recipe = discoverAgentRecipes(input.cwd).find((entry) => entry.id === id);
+		if (recipe === undefined) return null;
+		return {
+			id: recipe.id,
+			version: recipe.version,
+			contentHash: agentSpecFingerprint(normalizeAgentSpec(recipe)),
+			personaHash: sha256(recipe.body),
+		};
+	} catch {
+		return null;
+	}
+}
+
+function promptFragmentIdentities(
+	manifest: EvalPromptManifestObservation | null,
+	recipe: { id: string; version: number; personaHash: string } | null,
+	autonomy: string | null,
+): EvalPromptFragmentIdentityV1[] {
+	let versions = new Map<string, number>();
+	try {
+		versions = new Map([...loadFragments().byId.values()].map((fragment) => [fragment.id, fragment.version]));
+	} catch {
+		// The manifest still carries exact hashes; only authored versions become unavailable.
+	}
+	if (manifest !== null) {
+		return manifest.fragments
+			.map((fragment) => ({
+				id: fragment.id,
+				version: versions.get(fragment.id) ?? ("unversioned" as const),
+				contentHash: fragment.contentHash,
+			}))
+			.sort((left, right) => left.id.localeCompare(right.id));
+	}
+	if (recipe === null) return [];
+	const selected = ["identity.clio-worker", "operating.contract", "operating.worker"];
+	if (autonomy !== null) selected.push(`safety.${autonomy}`);
+	const fragments: EvalPromptFragmentIdentityV1[] = [];
+	try {
+		const table = loadFragments();
+		for (const id of selected) {
+			const fragment = table.byId.get(id);
+			if (fragment !== undefined) {
+				fragments.push({ id, version: fragment.version, contentHash: fragment.contentHash });
+			}
+		}
+	} catch {
+		// The composition hash remains authoritative when resource discovery fails.
+	}
+	fragments.push({ id: `persona.${recipe.id}`, version: recipe.version, contentHash: recipe.personaHash });
+	return fragments.sort((left, right) => left.id.localeCompare(right.id));
+}
+
+function policyIdentity(
+	cwd: string | null,
+	receipt: RunReceipt | null,
+	observation: EvalExecutionObservationV1 | undefined,
+): EvalExecutionEnvelopeV1["policyHashes"] {
+	const sealed = receipt?.reproducibility?.safetyPolicy;
+	if (sealed !== undefined) return { rulePack: sealed.rulePackHash, project: sealed.projectPolicyHash };
+	if (observation !== undefined) return { ...observation.policyHashes };
+	if (cwd === null) return { rulePack: null, project: null };
+	try {
+		const metadata = createSafetyPolicyEngine({ cwd }).metadata();
+		return { rulePack: metadata.rulePackHash, project: metadata.projectPolicyHash };
+	} catch {
+		return { rulePack: null, project: null };
+	}
+}
+
+function projectContextIdentity(
+	input: BuildEvalExecutionEnvelopeInput,
+	manifest: EvalPromptManifestObservation | null,
+	fragments: ReadonlyArray<EvalPromptFragmentIdentityV1>,
+): EvalExecutionEnvelopeV1["projectContext"] {
+	const receipt = input.receipt;
+	if (receipt?.projectContext !== undefined) {
+		const sections = [...(receipt.projectContext.sections ?? [])].sort();
+		const hasContentBearingContext = sections.some((section) => section !== "workspace-root");
+		return {
+			kind: "worker",
+			tier: receipt.projectContext.tier,
+			contentHash: hasContentBearingContext ? (receipt.projectContext.contentHash ?? null) : null,
+			chars: hasContentBearingContext ? (receipt.projectContext.chars ?? null) : null,
+			sections,
+			rulesApplied: [...(receipt.rulesApplied ?? [])].sort(),
+			operatorProfileApplied: receipt.operatorProfileApplied ?? null,
+		};
+	}
+	const observed = input.observation?.projectContext;
+	if (observed !== undefined && observed !== null) {
+		const sections = [...observed.sections].sort();
+		const hasContentBearingContext = sections.some((section) => section !== "workspace-root");
+		return {
+			kind: "worker",
+			tier: observed.tier,
+			contentHash: hasContentBearingContext ? observed.contentHash : null,
+			chars: hasContentBearingContext ? observed.chars : null,
+			sections,
+			rulesApplied: [...observed.rulesApplied].sort(),
+			operatorProfileApplied: observed.operatorProfileApplied,
+		};
+	}
+	if (manifest !== null) {
+		const contextFragments = fragments.filter((fragment) => fragment.id.startsWith("context."));
+		const preload = manifest.projectPreload;
+		const identity = {
+			preload,
+			fragments: contextFragments.map((fragment) => [fragment.id, fragment.contentHash]),
+		};
+		return {
+			kind: "session",
+			tier: preload?.mode ?? null,
+			contentHash: sha256(stableJson(identity)),
+			chars: preload?.chars ?? null,
+			sections: contextFragments.map((fragment) => fragment.id).sort(),
+			rulesApplied: [],
+			operatorProfileApplied: contextFragments.some((fragment) => fragment.id === "context.operator-profile"),
+		};
+	}
+	return {
+		kind: "none",
+		tier: null,
+		contentHash: null,
+		chars: null,
+		sections: [],
+		rulesApplied: [],
+		operatorProfileApplied: null,
+	};
+}
+
+function stableJson(value: unknown): string {
+	if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`;
+	if (typeof value === "object" && value !== null) {
+		return `{${Object.entries(value as Record<string, unknown>)
+			.sort(([left], [right]) => left.localeCompare(right))
+			.map(([key, entry]) => `${JSON.stringify(key)}:${stableJson(entry)}`)
+			.join(",")}}`;
+	}
+	return JSON.stringify(value);
+}
+
+function sha256(value: string): string {
+	return createHash("sha256").update(value, "utf8").digest("hex");
+}

package/src/domains/eval/metrics/aggregate.ts

@@ -0,0 +1,136 @@
+import {
+	EVAL_TRACKED_METRIC_NAMES,
+	type EvalMetricSource,
+	type EvalTrackedMetricName,
+	type EvalVerdictEnvelopeV1,
+} from "../schema/verdict.js";
+
+export interface EvalMetricDistributionV1 {
+	observations: number;
+	measured: number;
+	unmeasured: number;
+	mean: number | null;
+	min: number | null;
+	max: number | null;
+	p90: number | null;
+	variance: number | null;
+	standardDeviation: number | null;
+	sources: EvalMetricSource[];
+}
+
+export type EvalTrackedMetricDistributionsV1 = Record<EvalTrackedMetricName, EvalMetricDistributionV1> & {
+	expectedColdReasons: Record<string, EvalMetricDistributionV1>;
+};
+
+export interface EvalScenarioAggregateV1 {
+	scenarioId: string;
+	trials: number;
+	k: number;
+	passed: number;
+	failed: number;
+	unmeasured: number;
+	machineryFailures: number;
+	passAtK: number;
+	passPowK: number;
+	trackedMetrics: EvalTrackedMetricDistributionsV1;
+}
+
+/** Aggregate each scenario independently so trial outcomes never cross tasks. */
+export function aggregateEvalVerdicts(verdicts: ReadonlyArray<EvalVerdictEnvelopeV1>): EvalScenarioAggregateV1[] {
+	const byScenario = new Map<string, EvalVerdictEnvelopeV1[]>();
+	for (const verdict of verdicts) {
+		const group = byScenario.get(verdict.scenarioId) ?? [];
+		group.push(verdict);
+		byScenario.set(verdict.scenarioId, group);
+	}
+	return [...byScenario.entries()]
+		.sort(([left], [right]) => left.localeCompare(right))
+		.map(([scenarioId, group]) => aggregateScenario(scenarioId, group));
+}
+
+function aggregateScenario(
+	scenarioId: string,
+	verdicts: ReadonlyArray<EvalVerdictEnvelopeV1>,
+): EvalScenarioAggregateV1 {
+	const ordered = [...verdicts].sort((left, right) => left.trialIndex - right.trialIndex);
+	const passed = ordered.filter((verdict) => verdict.outcome === "pass").length;
+	const failed = ordered.filter((verdict) => verdict.outcome === "fail").length;
+	const unmeasured = ordered.filter((verdict) => verdict.outcome === "unmeasured").length;
+	const machineryFailures = ordered.filter((verdict) => verdict.machinery === "infrastructure_failure").length;
+	const fixed = Object.fromEntries(
+		EVAL_TRACKED_METRIC_NAMES.map((name) => [name, distribution(ordered.map((verdict) => verdict.trackedMetrics[name]))]),
+	) as Record<EvalTrackedMetricName, EvalMetricDistributionV1>;
+	const reasons = new Set(ordered.flatMap((verdict) => Object.keys(verdict.trackedMetrics.expectedColdReasons)));
+	const expectedColdReasons = Object.fromEntries(
+		[...reasons]
+			.sort((left, right) => left.localeCompare(right))
+			.map((reason) => [
+				reason,
+				distribution(
+					ordered.map(
+						(verdict) => verdict.trackedMetrics.expectedColdReasons[reason] ?? { value: 0, source: "ledger" as const },
+					),
+				),
+			]),
+	);
+	const k = ordered.length;
+	return {
+		scenarioId,
+		trials: k,
+		k,
+		passed,
+		failed,
+		unmeasured,
+		machineryFailures,
+		passAtK: k > 0 && passed > 0 ? 1 : 0,
+		passPowK: k > 0 && passed === k ? 1 : 0,
+		trackedMetrics: { ...fixed, expectedColdReasons },
+	};
+}
+
+function distribution(
+	metrics: ReadonlyArray<{ value: number | null; source: EvalMetricSource }>,
+): EvalMetricDistributionV1 {
+	const values = metrics.flatMap((metric) => (metric.value === null ? [] : [metric.value]));
+	const sources = [...new Set(metrics.map((metric) => metric.source))].sort(compareSources);
+	if (values.length === 0) {
+		return {
+			observations: metrics.length,
+			measured: 0,
+			unmeasured: metrics.length,
+			mean: null,
+			min: null,
+			max: null,
+			p90: null,
+			variance: null,
+			standardDeviation: null,
+			sources,
+		};
+	}
+	const ordered = [...values].sort((left, right) => left - right);
+	const p90Index = Math.max(0, Math.ceil(ordered.length * 0.9) - 1);
+	const mean = ordered.reduce((sum, value) => sum + value, 0) / ordered.length;
+	const variance = ordered.reduce((sum, value) => sum + (value - mean) ** 2, 0) / ordered.length;
+	return {
+		observations: metrics.length,
+		measured: values.length,
+		unmeasured: metrics.length - values.length,
+		mean,
+		min: ordered[0] ?? null,
+		max: ordered.at(-1) ?? null,
+		p90: ordered[p90Index] ?? null,
+		variance,
+		standardDeviation: Math.sqrt(variance),
+		sources,
+	};
+}
+
+function compareSources(left: EvalMetricSource, right: EvalMetricSource): number {
+	return sourceOrder(left) - sourceOrder(right);
+}
+
+function sourceOrder(source: EvalMetricSource): number {
+	if (source === "ledger") return 0;
+	if (source === "receipt") return 1;
+	return 2;
+}