@iowarp/clio-coder 0.3.7 → 0.3.9

package/src/domains/eval/schema/adapter.ts

@@ -0,0 +1,111 @@
+import { createHash } from "node:crypto";
+import type { EvalArtifactResultV4 } from "./artifact.js";
+import {
+	assertEvalBehaviorReferencesVerdictV1,
+	type EvalBehaviorFactSourceV1,
+	type EvalBehaviorJudgeFactV1,
+	type EvalBehaviorScenarioV1,
+	type EvalBehaviorVerdictV1,
+	judgeEvalBehaviorV1,
+} from "./behavioral.js";
+import {
+	EVAL_VERDICT_SCHEMA_V1,
+	type EvalTrackedMetricsV1,
+	type EvalVerdictEnvelopeV1,
+	parseEvalVerdictEnvelopeV1,
+} from "./verdict.js";
+
+type AdaptableSuiteResult = Pick<
+	EvalArtifactResultV4,
+	"assignmentId" | "terminalReceiptDigest" | "taskId" | "repeatIndex" | "pass" | "failureClass" | "metrics"
+>;
+
+/** Adapt one Suite v2 matrix result into the versioned verdict carried by Artifact v4. */
+export function adaptSuiteV2ResultToVerdictV1(
+	result: AdaptableSuiteResult,
+	trackedMetrics: EvalTrackedMetricsV1,
+): EvalVerdictEnvelopeV1 {
+	// The suite runner owns the one final pass decision. In particular, a
+	// declared grader failure changes `pass` without pretending the runner or
+	// its invariants broke; every other failed result is a machinery failure.
+	const machinery = result.pass || result.failureClass === "grader_failed" ? "ok" : "infrastructure_failure";
+	const outcome = result.pass ? "pass" : "fail";
+	const graderExitCode = result.metrics["task.exitCode"];
+	return parseEvalVerdictEnvelopeV1({
+		schema: EVAL_VERDICT_SCHEMA_V1,
+		scenarioId: result.taskId,
+		trialIndex: result.repeatIndex,
+		outcome,
+		machinery,
+		reason: result.pass ? null : (result.failureClass ?? "result_failed"),
+		trackedMetrics,
+		behavioral: null,
+		evidence: {
+			assignmentId: result.assignmentId,
+			terminalReceiptDigest: result.terminalReceiptDigest,
+			graderExitCode: typeof graderExitCode === "number" && Number.isInteger(graderExitCode) ? graderExitCode : null,
+		},
+	});
+}
+
+/** Adapt Suite v2 metrics into deterministic, bounded facts for a behavioral scenario. */
+export function adaptSuiteV2ResultToBehaviorV1(
+	result: AdaptableSuiteResult,
+	verdict: EvalVerdictEnvelopeV1,
+	scenario: EvalBehaviorScenarioV1,
+): EvalBehaviorVerdictV1 {
+	const requestedFacts = new Set(
+		[...scenario.expectedBehavior, ...scenario.forbiddenBehavior].map(
+			(rule) => `${rule.fact.source}\u0000${rule.fact.key}`,
+		),
+	);
+	const observedSources = new Set<EvalBehaviorFactSourceV1>();
+	const facts = Object.entries(result.metrics).flatMap(([key, value]) => {
+		if (value === null) return [];
+		const source = metricFactSource(key);
+		observedSources.add(source);
+		if (!requestedFacts.has(`${source}\u0000${key}`)) return [];
+		const serialized = JSON.stringify({ source, key, value });
+		const digest = createHash("sha256").update(serialized, "utf8").digest("hex");
+		const fact: EvalBehaviorJudgeFactV1 = {
+			id: `metric-${digest.slice(0, 16)}`,
+			source,
+			key,
+			value,
+			evidence: { locator: `artifact.metrics.${key}`, digest, excerpt: serialized.slice(0, 1_000) },
+		};
+		return [fact];
+	});
+	const allSources: EvalBehaviorFactSourceV1[] = ["transcript", "tool", "receipt", "grader"];
+	const unavailableSources = allSources.filter(
+		(source) => !observedSources.has(source) || (source === "tool" && scenario.execution.toolTarget === "none"),
+	);
+	const behavior = judgeEvalBehaviorV1(scenario, verdict, {
+		facts,
+		unavailableSources,
+		infrastructureFailure: verdict.machinery === "infrastructure_failure",
+	});
+	assertEvalBehaviorReferencesVerdictV1(behavior, verdict);
+	return behavior;
+}
+
+function metricFactSource(key: string): EvalBehaviorFactSourceV1 {
+	if (key.startsWith("tools.")) return "tool";
+	if (
+		key.startsWith("task.") ||
+		key.startsWith("claims.") ||
+		key.startsWith("completion.") ||
+		key === "result.pass" ||
+		key === "verifier.exitCode"
+	)
+		return "grader";
+	if (
+		key.startsWith("receipt.") ||
+		key.startsWith("evidence.") ||
+		key.startsWith("boundary.") ||
+		key.startsWith("loop.") ||
+		key.startsWith("cost.")
+	)
+		return "receipt";
+	return "transcript";
+}

package/src/domains/eval/schema/artifact.ts

@@ -1,4 +1,10 @@
+import type { EvalScenarioAggregateV1 } from "../metrics/aggregate.js";
 import type { EvalClioProvenance, EvalEnvironmentProvenance } from "../types.js";
+import type { EvalBehaviorVerdictV1 } from "./behavioral.js";
+import type { EvalBehaviorMetricsV1 } from "./behavioral-metrics.js";
+import type { EvalExecutionEnvelopeV1, EvalExecutionMatrixDimensionV1 } from "./execution-envelope.js";
+import type { EvalServingConfigurationV1 } from "./serving.js";
+import type { EvalVerdictEnvelopeV1 } from "./verdict.js";
 
 export interface EvalTokenMetricsV4 {
 	input: number;
@@ -45,6 +51,14 @@ export interface EvalArtifactResultV4 extends EvalArtifactAssignmentReference {
 	failureClass: string | null;
 	metrics: Record<string, number | string | boolean | null>;
 	artifacts: Record<string, string | string[] | null>;
+	/** Additive Suite v2 adapter output. Current runners always populate it. */
+	verdict?: EvalVerdictEnvelopeV1;
+	/** Optional sibling document that references the unchanged verdict v1 identity. */
+	behavioral?: EvalBehaviorVerdictV1;
+	/** Additive, typed multi-metric projection for behavioral comparison. */
+	behavioralMetrics?: EvalBehaviorMetricsV1;
+	/** Exact prompt, route, policy, and project-context identity for a behavioral result. */
+	executionEnvelope?: EvalExecutionEnvelopeV1;
 }
 
 /** The only current eval artifact format. Routing accepts this version only. */
@@ -61,8 +75,14 @@ export interface EvalArtifactV4 {
 		target: string;
 		model: string | null;
 		thinking: string | null;
+		/** Envelope fields intentionally varied by this matrix. */
+		dimensions?: EvalExecutionMatrixDimensionV1[];
 	};
+	/** Exact serving facts used to decide whether two eval runs are comparable. */
+	servingConfiguration?: EvalServingConfigurationV1;
 	summary: EvalArtifactSummaryV4;
+	/** Scenario reductions over the versioned per-trial verdicts. */
+	aggregates?: EvalScenarioAggregateV1[];
 	results: EvalArtifactResultV4[];
 }
 

package/src/domains/eval/schema/behavioral-metrics.ts

@@ -0,0 +1,204 @@
+import type { EvalArtifactResultV4 } from "./artifact.js";
+import type { EvalBehaviorCategoryV1, EvalBehaviorLabelV1 } from "./behavioral.js";
+
+export const EVAL_BEHAVIOR_METRICS_SCHEMA_V1 = "clio.eval.behavior.metrics.v1" as const;
+
+export const EVAL_BEHAVIOR_METRIC_DEFINITIONS_V1 = [
+	{
+		name: "correctness.taskSolved",
+		family: "correctness",
+		direction: "higher" as const,
+		hardGate: true,
+		source: "grader" as const,
+	},
+	{
+		name: "safety.violations",
+		family: "safety",
+		direction: "lower" as const,
+		hardGate: true,
+		source: "behavioral-label" as const,
+	},
+	{
+		name: "behavior.labelViolations",
+		family: "behavior",
+		direction: "lower" as const,
+		hardGate: false,
+		source: "behavioral-label" as const,
+	},
+	{
+		name: "efficiency.toolCalls",
+		family: "efficiency",
+		direction: "lower" as const,
+		hardGate: false,
+		source: "tool-event" as const,
+	},
+	{
+		name: "exploration.unnecessaryReads",
+		family: "exploration",
+		direction: "lower" as const,
+		hardGate: false,
+		source: "tool-event" as const,
+	},
+	{
+		name: "delegation.quality",
+		family: "delegation",
+		direction: "higher" as const,
+		hardGate: false,
+		source: "behavioral-label" as const,
+	},
+	{
+		name: "claims.unsupported",
+		family: "claims",
+		direction: "lower" as const,
+		hardGate: false,
+		source: "grader" as const,
+	},
+	{
+		name: "tokens.total",
+		family: "tokens",
+		direction: "lower" as const,
+		hardGate: false,
+		source: "runner" as const,
+	},
+	{
+		name: "latency.wallMs",
+		family: "latency",
+		direction: "lower" as const,
+		hardGate: false,
+		source: "runner" as const,
+	},
+	{
+		name: "cost.usd",
+		family: "cost",
+		direction: "lower" as const,
+		hardGate: false,
+		source: "receipt" as const,
+	},
+] as const;
+
+export type EvalBehaviorMetricDefinitionV1 = (typeof EVAL_BEHAVIOR_METRIC_DEFINITIONS_V1)[number];
+export type EvalBehaviorMetricNameV1 = EvalBehaviorMetricDefinitionV1["name"];
+export type EvalBehaviorMetricFamilyV1 = EvalBehaviorMetricDefinitionV1["family"];
+export type EvalBehaviorMetricDirectionV1 = "higher" | "lower";
+export type EvalBehaviorMetricSourceV1 = "grader" | "behavioral-label" | "tool-event" | "runner" | "receipt";
+
+export interface EvalBehaviorMetricObservationV1 {
+	value: number | null;
+	source: EvalBehaviorMetricSourceV1;
+}
+
+export interface EvalBehaviorMetricsV1 {
+	schema: typeof EVAL_BEHAVIOR_METRICS_SCHEMA_V1;
+	scenarioId: string;
+	role: string;
+	target: { id: string; model: string | null };
+	metrics: Record<EvalBehaviorMetricNameV1, EvalBehaviorMetricObservationV1>;
+}
+
+export function buildEvalBehaviorMetricsV1(
+	result: Pick<EvalArtifactResultV4, "taskId" | "target" | "metrics" | "behavioral">,
+	role: string,
+): EvalBehaviorMetricsV1 {
+	const label = (category: EvalBehaviorCategoryV1): EvalBehaviorLabelV1 | null =>
+		result.behavioral?.labels.find((entry) => entry.category === category)?.label ?? null;
+	const observedLabels = result.behavioral?.labels.filter(
+		(entry) => entry.label === "satisfied" || entry.label === "violated",
+	);
+	const values: Record<EvalBehaviorMetricNameV1, number | null> = {
+		"correctness.taskSolved": booleanMetric(result.metrics["task.solved"]),
+		"safety.violations": violationMetric(label("safety_comprehension")),
+		"behavior.labelViolations":
+			observedLabels === undefined || observedLabels.length !== result.behavioral?.labels.length
+				? null
+				: observedLabels.filter((entry) => entry.label === "violated").length,
+		"efficiency.toolCalls": numberMetric(result.metrics["tools.totalCalls"]),
+		"exploration.unnecessaryReads": numberMetric(result.metrics["tools.read.outsideAllowed"]),
+		"delegation.quality": qualityMetric(label("delegation")),
+		"claims.unsupported": numberMetric(result.metrics["claims.unsupported"]),
+		"tokens.total": numberMetric(result.metrics["tokens.total"]),
+		"latency.wallMs": numberMetric(result.metrics["latency.wallMs"]),
+		"cost.usd": numberMetric(result.metrics["cost.usd"]),
+	};
+	const metrics = Object.fromEntries(
+		EVAL_BEHAVIOR_METRIC_DEFINITIONS_V1.map((definition) => [
+			definition.name,
+			{ value: values[definition.name], source: definition.source },
+		]),
+	) as Record<EvalBehaviorMetricNameV1, EvalBehaviorMetricObservationV1>;
+	return {
+		schema: EVAL_BEHAVIOR_METRICS_SCHEMA_V1,
+		scenarioId: result.taskId,
+		role,
+		target: { id: result.target.id, model: result.target.model },
+		metrics,
+	};
+}
+
+export function parseEvalBehaviorMetricsV1(value: unknown, source = "behavioral metrics"): EvalBehaviorMetricsV1 {
+	const record = asRecord(value, source);
+	if (record.schema !== EVAL_BEHAVIOR_METRICS_SCHEMA_V1) {
+		throw new Error(`${source}.schema: expected ${EVAL_BEHAVIOR_METRICS_SCHEMA_V1}`);
+	}
+	const target = asRecord(record.target, `${source}.target`);
+	const rawMetrics = asRecord(record.metrics, `${source}.metrics`);
+	const expectedMetricNames = new Set(EVAL_BEHAVIOR_METRIC_DEFINITIONS_V1.map((definition) => definition.name));
+	for (const metricName of Object.keys(rawMetrics)) {
+		if (!expectedMetricNames.has(metricName as EvalBehaviorMetricNameV1)) {
+			throw new Error(`${source}.metrics.${metricName}: unknown behavioral metric`);
+		}
+	}
+	const metrics = Object.fromEntries(
+		EVAL_BEHAVIOR_METRIC_DEFINITIONS_V1.map((definition) => {
+			const observation = asRecord(rawMetrics[definition.name], `${source}.metrics.${definition.name}`);
+			const metricSource = observation.source;
+			if (metricSource !== definition.source) {
+				throw new Error(`${source}.metrics.${definition.name}.source: expected ${definition.source}`);
+			}
+			const metricValue = observation.value;
+			if (metricValue !== null && (typeof metricValue !== "number" || !Number.isFinite(metricValue))) {
+				throw new Error(`${source}.metrics.${definition.name}.value: expected finite number or null`);
+			}
+			return [definition.name, { value: metricValue, source: metricSource }];
+		}),
+	) as Record<EvalBehaviorMetricNameV1, EvalBehaviorMetricObservationV1>;
+	return {
+		schema: EVAL_BEHAVIOR_METRICS_SCHEMA_V1,
+		scenarioId: readString(record.scenarioId, `${source}.scenarioId`),
+		role: readString(record.role, `${source}.role`),
+		target: {
+			id: readString(target.id, `${source}.target.id`),
+			model: target.model === null ? null : readString(target.model, `${source}.target.model`),
+		},
+		metrics,
+	};
+}
+
+function numberMetric(value: unknown): number | null {
+	return typeof value === "number" && Number.isFinite(value) ? value : null;
+}
+
+function booleanMetric(value: unknown): number | null {
+	return typeof value === "boolean" ? (value ? 1 : 0) : null;
+}
+
+function violationMetric(label: EvalBehaviorLabelV1 | null): number | null {
+	if (label === "violated") return 1;
+	if (label === "satisfied") return 0;
+	return null;
+}
+
+function qualityMetric(label: EvalBehaviorLabelV1 | null): number | null {
+	if (label === "satisfied") return 1;
+	if (label === "violated") return 0;
+	return null;
+}
+
+function asRecord(value: unknown, source: string): Record<string, unknown> {
+	if (typeof value === "object" && value !== null && !Array.isArray(value)) return value as Record<string, unknown>;
+	throw new Error(`${source}: expected object`);
+}
+
+function readString(value: unknown, source: string): string {
+	if (typeof value === "string" && value.length > 0) return value;
+	throw new Error(`${source}: expected non-empty string`);
+}