@iowarp/clio-coder 0.3.8 → 0.4.0

package/src/domains/eval/schema/execution-envelope.ts

@@ -0,0 +1,194 @@
+export const EVAL_EXECUTION_ENVELOPE_SCHEMA_V1 = "clio.eval.execution-envelope.v1" as const;
+
+export const EVAL_EXECUTION_MATRIX_DIMENSIONS_V1 = [
+	"prompt",
+	"recipe",
+	"target",
+	"wireModel",
+	"runtime",
+	"thinkingLevel",
+	"toolSignature",
+	"autonomy",
+	"policy",
+	"projectContext",
+	"corpus",
+] as const;
+
+export type EvalExecutionMatrixDimensionV1 = (typeof EVAL_EXECUTION_MATRIX_DIMENSIONS_V1)[number];
+export type EvalPromptFragmentVersionV1 = number | "unversioned";
+
+export interface EvalPromptFragmentIdentityV1 {
+	id: string;
+	version: EvalPromptFragmentVersionV1;
+	contentHash: string;
+}
+
+export interface EvalExecutionEnvelopeV1 {
+	schema: typeof EVAL_EXECUTION_ENVELOPE_SCHEMA_V1;
+	prompt: {
+		fragments: EvalPromptFragmentIdentityV1[];
+		/** Null means the machinery-only scenario made no model request. */
+		compositionHash: string | null;
+	};
+	/** Null for a main-agent or machinery runner that does not execute an agent recipe. */
+	recipe: { id: string; version: number; contentHash: string } | null;
+	target: string;
+	wireModel: string | null;
+	runtime: string | null;
+	thinkingLevel: string | null;
+	toolSignature: string | null;
+	autonomy: string | null;
+	policyHashes: {
+		rulePack: string | null;
+		project: string | null;
+	};
+	projectContext: {
+		kind: "none" | "session" | "worker";
+		tier: string | null;
+		contentHash: string | null;
+		chars: number | null;
+		sections: string[];
+		rulesApplied: string[];
+		operatorProfileApplied: boolean | null;
+	};
+	corpus: { id: string; version: string };
+}
+
+export function parseEvalExecutionEnvelopeV1(value: unknown, source = "execution envelope"): EvalExecutionEnvelopeV1 {
+	const record = asRecord(value, source);
+	if (record.schema !== EVAL_EXECUTION_ENVELOPE_SCHEMA_V1) {
+		throw new Error(`${source}.schema: expected ${EVAL_EXECUTION_ENVELOPE_SCHEMA_V1}`);
+	}
+	const prompt = asRecord(record.prompt, `${source}.prompt`);
+	if (!Array.isArray(prompt.fragments)) throw new Error(`${source}.prompt.fragments: expected array`);
+	const fragments = prompt.fragments.map((entry, index) => {
+		const fragment = asRecord(entry, `${source}.prompt.fragments[${index}]`);
+		const version = fragment.version;
+		if (version !== "unversioned" && (!Number.isInteger(version) || typeof version !== "number" || version <= 0)) {
+			throw new Error(`${source}.prompt.fragments[${index}].version: expected positive integer or unversioned`);
+		}
+		return {
+			id: readString(fragment.id, `${source}.prompt.fragments[${index}].id`),
+			version: version as EvalPromptFragmentVersionV1,
+			contentHash: readDigest(fragment.contentHash, `${source}.prompt.fragments[${index}].contentHash`),
+		};
+	});
+	const fragmentIds = fragments.map((fragment) => fragment.id);
+	if (new Set(fragmentIds).size !== fragmentIds.length) {
+		throw new Error(`${source}.prompt.fragments: duplicate fragment id`);
+	}
+	const recipe =
+		record.recipe === null
+			? null
+			: (() => {
+					const value = asRecord(record.recipe, `${source}.recipe`);
+					const version = value.version;
+					if (!Number.isInteger(version) || typeof version !== "number" || version <= 0) {
+						throw new Error(`${source}.recipe.version: expected positive integer`);
+					}
+					return {
+						id: readString(value.id, `${source}.recipe.id`),
+						version,
+						contentHash: readDigest(value.contentHash, `${source}.recipe.contentHash`),
+					};
+				})();
+	const policyHashes = asRecord(record.policyHashes, `${source}.policyHashes`);
+	const projectContext = asRecord(record.projectContext, `${source}.projectContext`);
+	const projectKind = projectContext.kind;
+	if (projectKind !== "none" && projectKind !== "session" && projectKind !== "worker") {
+		throw new Error(`${source}.projectContext.kind: expected none, session, or worker`);
+	}
+	const corpus = asRecord(record.corpus, `${source}.corpus`);
+	return {
+		schema: EVAL_EXECUTION_ENVELOPE_SCHEMA_V1,
+		prompt: {
+			fragments,
+			compositionHash: readNullableDigest(prompt.compositionHash, `${source}.prompt.compositionHash`),
+		},
+		recipe,
+		target: readString(record.target, `${source}.target`),
+		wireModel: readNullableString(record.wireModel, `${source}.wireModel`),
+		runtime: readNullableString(record.runtime, `${source}.runtime`),
+		thinkingLevel: readNullableString(record.thinkingLevel, `${source}.thinkingLevel`),
+		toolSignature: readNullableDigest(record.toolSignature, `${source}.toolSignature`),
+		autonomy: readNullableString(record.autonomy, `${source}.autonomy`),
+		policyHashes: {
+			rulePack: readNullableDigest(policyHashes.rulePack, `${source}.policyHashes.rulePack`),
+			project: readNullableDigest(policyHashes.project, `${source}.policyHashes.project`),
+		},
+		projectContext: {
+			kind: projectKind,
+			tier: readNullableString(projectContext.tier, `${source}.projectContext.tier`),
+			contentHash: readNullableDigest(projectContext.contentHash, `${source}.projectContext.contentHash`),
+			chars: readNullableNonNegativeInteger(projectContext.chars, `${source}.projectContext.chars`),
+			sections: readStringArray(projectContext.sections, `${source}.projectContext.sections`),
+			rulesApplied: readStringArray(projectContext.rulesApplied, `${source}.projectContext.rulesApplied`),
+			operatorProfileApplied: readNullableBoolean(
+				projectContext.operatorProfileApplied,
+				`${source}.projectContext.operatorProfileApplied`,
+			),
+		},
+		corpus: {
+			id: readString(corpus.id, `${source}.corpus.id`),
+			version: readString(corpus.version, `${source}.corpus.version`),
+		},
+	};
+}
+
+export function parseEvalExecutionMatrixDimensionsV1(value: unknown, source: string): EvalExecutionMatrixDimensionV1[] {
+	if (!Array.isArray(value)) throw new Error(`${source}: expected array`);
+	const allowed = new Set<string>(EVAL_EXECUTION_MATRIX_DIMENSIONS_V1);
+	const dimensions = value.map((entry, index) => {
+		if (typeof entry !== "string" || !allowed.has(entry)) {
+			throw new Error(`${source}[${index}]: expected a declared execution-envelope dimension`);
+		}
+		return entry as EvalExecutionMatrixDimensionV1;
+	});
+	if (new Set(dimensions).size !== dimensions.length) throw new Error(`${source}: duplicate matrix dimension`);
+	return dimensions;
+}
+
+function readString(value: unknown, source: string): string {
+	if (typeof value !== "string" || value.length === 0) throw new Error(`${source}: expected non-empty string`);
+	return value;
+}
+
+function readNullableString(value: unknown, source: string): string | null {
+	if (value === null) return null;
+	return readString(value, source);
+}
+
+function readDigest(value: unknown, source: string): string {
+	const digest = readString(value, source);
+	if (!/^[a-f0-9]{64}$/u.test(digest)) throw new Error(`${source}: expected sha256 digest`);
+	return digest;
+}
+
+function readNullableDigest(value: unknown, source: string): string | null {
+	return value === null ? null : readDigest(value, source);
+}
+
+function readNullableNonNegativeInteger(value: unknown, source: string): number | null {
+	if (value === null) return null;
+	if (typeof value !== "number" || !Number.isInteger(value) || value < 0) {
+		throw new Error(`${source}: expected non-negative integer or null`);
+	}
+	return value;
+}
+
+function readNullableBoolean(value: unknown, source: string): boolean | null {
+	if (value === null || typeof value === "boolean") return value;
+	throw new Error(`${source}: expected boolean or null`);
+}
+
+function readStringArray(value: unknown, source: string): string[] {
+	if (!Array.isArray(value)) throw new Error(`${source}: expected array`);
+	const entries = value.map((entry, index) => readString(entry, `${source}[${index}]`));
+	if (new Set(entries).size !== entries.length) throw new Error(`${source}: duplicate value`);
+	return entries;
+}
+
+function asRecord(value: unknown, source: string): Record<string, unknown> {
+	if (typeof value === "object" && value !== null && !Array.isArray(value)) return value as Record<string, unknown>;
+	throw new Error(`${source}: expected object`);
+}

package/src/domains/eval/schema/serving.ts

@@ -0,0 +1,105 @@
+export interface EvalServingConfigurationV1 {
+	targetId: string;
+	runtimeId: string | null;
+	modelId: string | null;
+	serverBuild: string | null;
+	total_slots: number | null;
+	thinkingLevel: string | null;
+	compiledPromptHash: string | null;
+}
+
+export function parseEvalServingConfigurationV1(value: unknown, source: string): EvalServingConfigurationV1 {
+	const record = asRecord(value, source);
+	const targetId = readString(record.targetId, `${source}.targetId`);
+	const totalSlots = record.total_slots;
+	if (totalSlots !== null && (typeof totalSlots !== "number" || !Number.isInteger(totalSlots) || totalSlots <= 0)) {
+		throw new Error(`${source}.total_slots: expected positive integer or null`);
+	}
+	const compiledPromptHash = readNullableString(record.compiledPromptHash, `${source}.compiledPromptHash`);
+	if (compiledPromptHash !== null && !/^[a-f0-9]{64}$/u.test(compiledPromptHash)) {
+		throw new Error(`${source}.compiledPromptHash: expected sha256 digest or null`);
+	}
+	return {
+		targetId,
+		runtimeId: readNullableString(record.runtimeId, `${source}.runtimeId`),
+		modelId: readNullableString(record.modelId, `${source}.modelId`),
+		serverBuild: readNullableString(record.serverBuild, `${source}.serverBuild`),
+		total_slots: totalSlots as number | null,
+		thinkingLevel: readNullableString(record.thinkingLevel, `${source}.thinkingLevel`),
+		compiledPromptHash,
+	};
+}
+
+/**
+ * The serving facts an artifact was produced under.
+ *
+ * An artifact written before the observation existed has no record of its own,
+ * so the matrix it declared is the best available reading and the missing
+ * fields stay null rather than being invented. This lives beside
+ * `sameEvalServingConfiguration` because the two are one rule: what a
+ * comparison reads, and when it decides two readings agree. Anything asking
+ * whether two eval runs are comparable must go through both, or it is answering
+ * a different question from the one `eval compare` answers.
+ *
+ * The parameter is structural rather than `EvalArtifactV4` so this module stays
+ * free of an import cycle through `schema/artifact.ts`.
+ */
+export function evalServingConfigurationOf(artifact: {
+	readonly servingConfiguration?: EvalServingConfigurationV1;
+	readonly matrix: { readonly target: string; readonly model: string | null; readonly thinking: string | null };
+}): EvalServingConfigurationV1 {
+	return (
+		artifact.servingConfiguration ?? {
+			targetId: artifact.matrix.target,
+			runtimeId: null,
+			modelId: artifact.matrix.model,
+			serverBuild: null,
+			total_slots: null,
+			thinkingLevel: artifact.matrix.thinking,
+			compiledPromptHash: null,
+		}
+	);
+}
+
+export function sameEvalServingConfiguration(
+	left: EvalServingConfigurationV1,
+	right: EvalServingConfigurationV1,
+): boolean {
+	return (
+		left.targetId === right.targetId &&
+		left.runtimeId === right.runtimeId &&
+		left.modelId === right.modelId &&
+		left.serverBuild === right.serverBuild &&
+		left.total_slots === right.total_slots &&
+		left.thinkingLevel === right.thinkingLevel &&
+		left.compiledPromptHash === right.compiledPromptHash
+	);
+}
+
+export function renderEvalServingConfiguration(config: EvalServingConfigurationV1): string {
+	return [
+		`target=${config.targetId}`,
+		`runtime=${config.runtimeId ?? "unknown"}`,
+		`model=${config.modelId ?? "unknown"}`,
+		`server_build=${config.serverBuild ?? "unknown"}`,
+		`total_slots=${config.total_slots ?? "unknown"}`,
+		`thinking=${config.thinkingLevel ?? "unknown"}`,
+		`compiled_prompt_hash=${config.compiledPromptHash ?? "unknown"}`,
+	].join(" ");
+}
+
+function readString(value: unknown, source: string): string {
+	if (typeof value !== "string" || value.length === 0) throw new Error(`${source}: expected string`);
+	return value;
+}
+
+function readNullableString(value: unknown, source: string): string | null {
+	if (value === null) return null;
+	if (typeof value !== "string" || value.length === 0) throw new Error(`${source}: expected string or null`);
+	return value;
+}
+
+function asRecord(value: unknown, source: string): Record<string, unknown> {
+	if (typeof value === "object" && value !== null && !Array.isArray(value)) return value as Record<string, unknown>;
+	throw new Error(`${source}: expected object`);
+}

package/src/domains/eval/schema/suite.ts

@@ -1,3 +1,6 @@
+import type { EvalBehaviorScenarioV1 } from "./behavioral.js";
+import type { EvalExecutionMatrixDimensionV1 } from "./execution-envelope.js";
+
 export const EVAL_SUITE_V2_VERSION = 2;
 
 export const CANONICAL_METRICS = [
@@ -15,6 +18,13 @@ export const CANONICAL_METRICS = [
 	"tools.totalCalls",
 	"tools.failed",
 	"tools.blocked",
+	"tools.read.distinctPaths",
+	"tools.read.outsideAllowed",
+	"tools.read.decoyHits",
+	"tools.calls.dispatch",
+	"tools.blocked.bash",
+	"claims.unsupported",
+	"completion.reported",
 	"context.indexedFiles",
 	"context.coverage",
 	"context.structuralHash",
@@ -103,13 +113,20 @@ export const CANONICAL_METRICS = [
 	"stream.messageUpdateCount",
 	"stream.usageDoubleCounted",
 	"stream.segmentUsageMatchesMessages",
-	// The task outcome, measured and never gated: whether the model solved the
-	// workload. Reported beside the invariants above so a report can say "the
-	// model failed and Clio behaved" instead of one number for both.
+	// The task outcome: whether the model solved the workload. Its exit code is
+	// recorded beside the machinery invariants so a failed final result can still
+	// say "the grader failed and Clio behaved" rather than conflating the two.
 	"task.exitCode",
 	"task.solved",
 ] as const;
 
+/**
+ * Per-tool call and block counters use a dynamic suffix because extension tool
+ * ids are not closed at build time. They otherwise have the same scalar,
+ * observable-event semantics as the fixed canonical metrics above.
+ */
+export const CANONICAL_METRIC_PREFIXES = ["tools.calls.", "tools.blocked."] as const;
+
 export type EvalMetricName = (typeof CANONICAL_METRICS)[number];
 export type EvalAssertionOp = "lt" | "lte" | "gt" | "gte" | "eq" | "neq";
 export type EvalRunnerKind = "clio-run" | "context-index" | "context-init" | "external-command";
@@ -138,6 +155,8 @@ export interface EvalSuiteTargetV2 {
 export interface EvalSuiteMatrixV2 {
 	targets: EvalSuiteTargetV2[];
 	repeats: number;
+	/** Execution-envelope fields this suite intentionally varies. All other fields must match for comparison. */
+	dimensions?: EvalExecutionMatrixDimensionV1[];
 	/**
 	 * Cumulative receipt-cost ceiling in USD for the whole matrix. Once the
 	 * summed `cost.usd` of finished items exceeds it, remaining items fail
@@ -167,6 +186,8 @@ export interface EvalWorkspaceV2 {
 export interface EvalRunnerV2 {
 	kind: EvalRunnerKind;
 	prompt?: string;
+	/** One-run headless authority used to exercise allow and deny recovery. */
+	autonomy?: "read-only" | "suggest" | "auto-edit" | "full-auto";
 	/**
 	 * Fleet agent recipe id for the clio-run runner. When set the runner
 	 * invokes `clio-coder run --agent <id> --json`, whose stream ends with the full
@@ -183,11 +204,10 @@ export interface EvalRunnerV2 {
 export interface EvalVerifyV2 {
 	commands?: string[];
 	/**
-	 * Commands that measure the task outcome without gating on it. They run in
-	 * the workspace before the gating verifiers and record `task.exitCode` and
-	 * `task.solved`; a nonzero exit is data, never a failure. This is what keeps
-	 * "the model did not solve it" from being reported as "Clio broke", which
-	 * `commands` above cannot express because a failing verifier fails the item.
+	 * Commands that grade the task outcome. They run in the workspace before the
+	 * machinery verifiers and record `task.exitCode` and `task.solved`; a nonzero
+	 * exit fails the final result while the verdict keeps machinery `ok`. This is
+	 * what keeps "the model did not solve it" distinct from "Clio broke".
 	 */
 	measure?: string[];
 	assertions?: EvalMetricAssertion[];
@@ -196,11 +216,19 @@ export interface EvalVerifyV2 {
 
 export interface EvalMetricsSpecV2 {
 	collect: string[];
+	/**
+	 * Public fixture paths used to reduce read events to bounded counters. Raw
+	 * paths stay out of behavioral fact values and evidence excerpts; the metric
+	 * map records only distinct, outside-allowlist, and declared-decoy counts.
+	 */
+	readObservation?: { allowedPaths: string[]; decoyPaths: string[] };
 }
 
 export interface EvalSuiteTaskV2 {
 	id: string;
 	tags: string[];
+	/** Additive behavioral contract evaluated against observable run facts. */
+	behavioral?: EvalBehaviorScenarioV1;
 	workspace: EvalWorkspaceV2;
 	runner: EvalRunnerV2;
 	verify: EvalVerifyV2;
@@ -210,6 +238,8 @@ export interface EvalSuiteTaskV2 {
 
 export interface EvalSuiteThresholdsV2 {
 	fail: EvalMetricAssertion[];
+	/** Budget notices are reported but never change the command exit status. */
+	informational?: EvalMetricAssertion[];
 }
 
 export interface EvalSuiteV2 {

package/src/domains/eval/schema/validate.ts

@@ -1,3 +1,5 @@
+import { parseEvalBehaviorScenarioV1 } from "./behavioral.js";
+import { parseEvalExecutionMatrixDimensionsV1 } from "./execution-envelope.js";
 import {
 	EVAL_SUITE_V2_VERSION,
 	type EvalAssertionOp,
@@ -87,12 +89,26 @@ function readMatrix(value: unknown, path: string, issues: EvalValidationIssue[])
 		];
 	});
 	if (repeats === null || targets.length === 0) return null;
+	let dimensions: EvalSuiteV2["matrix"]["dimensions"];
+	if (value.dimensions !== undefined) {
+		try {
+			dimensions = parseEvalExecutionMatrixDimensionsV1(value.dimensions, `${path}.dimensions`);
+		} catch (error) {
+			issues.push({ path: `${path}.dimensions`, message: error instanceof Error ? error.message : String(error) });
+			return null;
+		}
+	}
 	const maxCostUsd = value.maxCostUsd;
 	if (maxCostUsd !== undefined && (typeof maxCostUsd !== "number" || !Number.isFinite(maxCostUsd) || maxCostUsd < 0)) {
 		issues.push({ path: `${path}.maxCostUsd`, message: "expected non-negative number" });
 		return null;
 	}
-	return { targets, repeats, ...(maxCostUsd === undefined ? {} : { maxCostUsd }) };
+	return {
+		targets,
+		repeats,
+		...(dimensions === undefined ? {} : { dimensions }),
+		...(maxCostUsd === undefined ? {} : { maxCostUsd }),
+	};
 }
 
 function readTasks(value: unknown, path: string, issues: EvalValidationIssue[]): EvalSuiteTaskV2[] | null {
@@ -121,10 +137,12 @@ function readTask(value: unknown, path: string, issues: EvalValidationIssue[]):
 	const workspace = readWorkspace(value.workspace, `${path}.workspace`, issues);
 	const runner = readRunner(value.runner, `${path}.runner`, issues);
 	const timeoutMs = readPositiveInteger(value, path, "timeoutMs", issues);
+	const behavioral = readBehavioral(value.behavioral, `${path}.behavioral`, issues);
 	if (id === null || workspace === null || runner === null || timeoutMs === null) return null;
 	return {
 		id,
 		tags: readOptionalStringArray(value, "tags", `${path}.tags`, issues),
+		...(behavioral === undefined ? {} : { behavioral }),
 		workspace,
 		runner,
 		verify: readVerify(value.verify, `${path}.verify`, issues),
@@ -133,6 +151,20 @@ function readTask(value: unknown, path: string, issues: EvalValidationIssue[]):
 	};
 }
 
+function readBehavioral(
+	value: unknown,
+	path: string,
+	issues: EvalValidationIssue[],
+): EvalSuiteTaskV2["behavioral"] | undefined {
+	if (value === undefined) return undefined;
+	try {
+		return parseEvalBehaviorScenarioV1(value, path);
+	} catch (error) {
+		issues.push({ path, message: error instanceof Error ? error.message : String(error) });
+		return undefined;
+	}
+}
+
 function readWorkspace(
 	value: unknown,
 	path: string,
@@ -178,6 +210,10 @@ function readRunner(
 	}
 	const prompt = optionalString(value, "prompt");
 	const agent = optionalString(value, "agent");
+	const autonomy = optionalString(value, "autonomy");
+	if (autonomy !== undefined && !["read-only", "suggest", "auto-edit", "full-auto"].includes(autonomy)) {
+		issues.push({ path: `${path}.autonomy`, message: "expected read-only, suggest, auto-edit, or full-auto" });
+	}
 	if (agent !== undefined && kind !== "clio-run") {
 		issues.push({ path: `${path}.agent`, message: "agent is only valid on the clio-run runner" });
 	}
@@ -185,6 +221,7 @@ function readRunner(
 	return {
 		kind: kind as EvalRunnerKind,
 		...(prompt === undefined ? {} : { prompt }),
+		...(autonomy === undefined ? {} : { autonomy: autonomy as NonNullable<EvalSuiteTaskV2["runner"]["autonomy"]> }),
 		...(agent === undefined ? {} : { agent }),
 		...(command === undefined ? {} : { command }),
 		commands: readOptionalStringArray(value, "commands", `${path}.commands`, issues),
@@ -221,7 +258,22 @@ function readMetrics(
 		issues.push({ path, message: "expected object" });
 		return { collect: [] };
 	}
-	return { collect: readOptionalStringArray(value, "collect", `${path}.collect`, issues) };
+	const observation = value.readObservation;
+	let readObservation: EvalSuiteV2["tasks"][number]["metrics"]["readObservation"];
+	if (observation !== undefined) {
+		if (!isRecord(observation)) {
+			issues.push({ path: `${path}.readObservation`, message: "expected object" });
+		} else {
+			readObservation = {
+				allowedPaths: readOptionalStringArray(observation, "allowedPaths", `${path}.readObservation.allowedPaths`, issues),
+				decoyPaths: readOptionalStringArray(observation, "decoyPaths", `${path}.readObservation.decoyPaths`, issues),
+			};
+		}
+	}
+	return {
+		collect: readOptionalStringArray(value, "collect", `${path}.collect`, issues),
+		...(readObservation === undefined ? {} : { readObservation }),
+	};
 }
 
 function readThresholds(
@@ -234,7 +286,10 @@ function readThresholds(
 		issues.push({ path, message: "expected object" });
 		return undefined;
 	}
-	return { fail: readAssertions(value.fail, `${path}.fail`, issues) };
+	return {
+		fail: readAssertions(value.fail, `${path}.fail`, issues),
+		informational: readAssertions(value.informational, `${path}.informational`, issues),
+	};
 }
 
 function readAssertions(value: unknown, path: string, issues: EvalValidationIssue[]): EvalMetricAssertion[] {