@iowarp/clio-coder 0.3.7 → 0.3.9

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@iowarp/clio-coder",
-	"version": "0.3.7",
+	"version": "0.3.9",
 	"description": "Coding agent for HPC and scientific-software developers, part of IOWarp's CLIO ecosystem of agentic science.",
 	"keywords": [
 		"ai",
@@ -92,6 +92,7 @@
 		"//": "below here: a real model target, chosen with --target <id>; costs money and/or GPU time, never run in CI",
 		"live:smoke": "node --import tsx benchmarks/internal/live-smoke.ts",
 		"live:fleet-dispatch": "node --import tsx benchmarks/internal/live-fleet-dispatch.ts",
+		"live:release-residue": "node --import tsx tests/live/release-residue.ts",
 		"live:tui": "node --import tsx benchmarks/internal/pty-drive.ts",
 		"live:home": "node --import tsx benchmarks/internal/live-home.ts"
 	},
@@ -107,7 +108,6 @@
 	},
 	"overrides": {
 		"@anthropic-ai/sdk": "0.105.0",
-		"ip-address": "10.2.0",
 		"esbuild": "0.28.1"
 	},
 	"devDependencies": {

package/src/cli/agents.ts

@@ -9,7 +9,7 @@ import { printError } from "./shared.js";
 
 const HELP = `clio-coder agents [--json] [--all]
 
-List user-facing agent specs from built-in, user, and project recipes.
+List user-facing agent specs from built-in, extension, user, and project recipes.
 
 Flags:
   --json   emit specs as JSON instead of the formatted table

package/src/cli/argv.ts

@@ -15,6 +15,11 @@ export function printError(message: string, detail?: string): void {
 	if (detail) process.stderr.write(`  ${detail}\n`);
 }
 
+/** An expected condition worth a line on stderr that is neither an error nor a success. */
+export function printNote(message: string): void {
+	process.stderr.write(`${chalk.yellow("note:")} ${message}\n`);
+}
+
 export function printOk(message: string): void {
 	process.stdout.write(`${chalk.green("ok:")} ${message}\n`);
 }

package/src/cli/config-inspect.ts

@@ -1,7 +1,7 @@
 /**
  * The effective-customization graph behind `clio-coder config inspect`. This is the
  * "why is Clio behaving this way" surface: it answers what settings, context
- * files, rules, skills, prompts, agents, extensions, safety, memory, hooks, and
+ * files, rules, skills, prompts, agents, fleets, extensions, safety, memory, hooks, and
  * the operator profile loaded, from where, with what precedence, and what each
  * costs in context.
  *
@@ -12,8 +12,9 @@
  */
 
 import { existsSync, readFileSync } from "node:fs";
-import { basename } from "node:path";
+import { basename, join } from "node:path";
 import type { ClioSettings } from "../core/config.js";
+import { resolvePackageRoot } from "../core/package-root.js";
 import { readLayeredSettings, type SettingsOrigin, settingsSourceFor } from "../core/settings-layers.js";
 import { clioDataDir } from "../core/xdg.js";
 import {
@@ -39,7 +40,8 @@ export type CustomizationCategory =
 	| "extension"
 	| "skill-root"
 	| "prompt-root"
-	| "agents"
+	| "agent-root"
+	| "fleet-root"
 	| "safety"
 	| "memory";
 
@@ -265,18 +267,43 @@ function inspectResourceRoots(cwd: string, graph: CustomizationGraph): void {
 	for (const [kind, category] of [
 		["skills", "skill-root"],
 		["prompts", "prompt-root"],
+		["agents", "agent-root"],
+		["fleets", "fleet-root"],
 	] as const) {
 		try {
+			if (kind === "agents" || kind === "fleets") {
+				const builtinPath = join(
+					resolvePackageRoot(),
+					"src",
+					"domains",
+					"agents",
+					kind === "agents" ? "builtins" : "fleets",
+				);
+				graph.entries.push({
+					category,
+					id: `${kind}:builtin`,
+					scope: "builtin",
+					sourcePath: builtinPath,
+					trust: "trusted",
+					precedence: "layer",
+					reloadClass: "next-turn",
+					detail: { present: existsSync(builtinPath), source: "builtin", resourcePrecedence: 0 },
+				});
+			}
 			for (const root of defaultScopedResourceRoots(kind, cwd)) {
 				graph.entries.push({
 					category,
 					id: `${kind}:${root.scope}`,
-					scope: root.scope,
+					scope: root.scope === "package" ? "extension" : root.scope,
 					sourcePath: root.path,
 					trust: root.scope === "package" ? "untrusted" : "trusted",
-					precedence: "single",
+					precedence: "layer",
 					reloadClass: "next-turn",
-					detail: { present: existsSync(root.path), source: root.source },
+					detail: {
+						present: existsSync(root.path),
+						source: root.source,
+						...(root.precedence === undefined ? {} : { resourcePrecedence: root.precedence }),
+					},
 				});
 			}
 		} catch (err) {

package/src/cli/config.ts

@@ -4,7 +4,7 @@ import { printError } from "./shared.js";
 const HELP = `clio-coder config inspect [--json]
 
 Print the effective-customization graph: what settings, context files, rules,
-skills, prompts, agents, extensions, safety, memory, hooks, and the operator
+skills, prompts, agents, fleets, extensions, safety, memory, hooks, and the operator
 profile loaded, from where, with what precedence, trust, reload class, and
 context cost. Read-only; nothing is created.
 

package/src/cli/configure.ts

@@ -27,7 +27,12 @@ import { fingerprintNativeRuntime } from "../domains/providers/probe/fingerprint
 import { getRuntimeRegistry } from "../domains/providers/registry.js";
 import { registerBuiltinRuntimes } from "../domains/providers/runtimes/builtins.js";
 import { greetLmStudio } from "../domains/providers/runtimes/common/lmstudio-http.js";
-import type { ProbeContext, ProbeResult, RuntimeDescriptor } from "../domains/providers/types/runtime-descriptor.js";
+import type {
+	ProbeContext,
+	ProbeModelStatus,
+	ProbeResult,
+	RuntimeDescriptor,
+} from "../domains/providers/types/runtime-descriptor.js";
 import type { TargetDescriptor } from "../domains/providers/types/target-descriptor.js";
 import { registerClioOAuthProviders } from "../engine/oauth.js";
 import { ask, askYesNo } from "./ask.js";
@@ -44,7 +49,7 @@ import { createDelayedManualCodeInput } from "./oauth-manual-input.js";
 import { promptOAuthSelection } from "./oauth-select.js";
 import { credentialWriteFailed, printError, printOk, printPlaintextCredentialWarning } from "./shared.js";
 import { terminalColumns, wrapPlain } from "./text-layout.js";
-import { validateModelChoice } from "./validate-model.js";
+import { type LiveModelInventory, validateModelChoice } from "./validate-model.js";
 
 const HELP = `clio-coder configure
 
@@ -70,7 +75,8 @@ Non-interactive flags:
   --agent-profile-model <id>       model to use for --agent-profile
   --api-key-env <VAR>              read API key from this env var at call time
   --api-key <literal>              store API key in credentials.yaml
-  --force                          allow a model outside the runtime catalog
+  --force                          save a model outside the runtime catalog, or one the
+                                   target does not advertise, without refusing
   --gateway                        mark the target as a gateway
   --lifecycle <user-managed|clio-managed>
                                   resident model lifecycle policy
@@ -378,19 +384,35 @@ function catalogContextWindowFor(runtime: RuntimeDescriptor, modelId: string): n
 	return null;
 }
 
-function validateResolvedModel(runtimeId: string, modelId: string | undefined, force: boolean): boolean {
+/**
+ * Check the resolved model against the catalog where there is one and against
+ * the target's own list where there is not. A runtime with no catalog used to
+ * pass any string here while the same command wrote the live list that ruled
+ * it out into `wireModels`, so the target saved as `ok` and failed on its first
+ * turn. Without a catalog or a live list there is nothing to check against, and
+ * the note says so rather than implying the id was verified.
+ */
+function validateResolvedModel(
+	runtime: RuntimeDescriptor,
+	target: Pick<TargetDescriptor, "id" | "url">,
+	modelId: string | undefined,
+	force: boolean,
+	inventory: WireModelInventory,
+): boolean {
 	if (!modelId) return true;
-	const validation = validateModelChoice({
-		runtimeId,
-		modelId,
-		knownModels: runtimeKnownModelsFor(runtimeId),
-		force,
-	});
+	const knownModels = runtimeKnownModelsFor(runtime.id);
+	const live = inventory.source === "probe" ? liveInventoryFor(target, inventory) : undefined;
+	const validation = validateModelChoice({ runtimeId: runtime.id, modelId, knownModels, live, force });
 	if (!validation.ok) {
 		process.stderr.write(`error: ${validation.reason}\n`);
 		return false;
 	}
 	if (validation.warning) process.stderr.write(`warning: ${validation.warning}\n`);
+	if (knownModels.length === 0 && live === undefined && runtime.kind === "http") {
+		process.stderr.write(
+			`warning: could not verify model '${modelId}': ${target.url ?? `target '${target.id}'`} returned no model list; it is saved as written\n`,
+		);
+	}
 	return true;
 }
 
@@ -412,10 +434,15 @@ function validateContextWindowOverride(
 	return true;
 }
 
-async function runtimeProbe(runtime: RuntimeDescriptor, target: TargetDescriptor): Promise<ProbeResult | null> {
+async function runtimeProbe(
+	runtime: RuntimeDescriptor,
+	target: TargetDescriptor,
+	authToken?: string,
+): Promise<ProbeResult | null> {
 	if (typeof runtime.probe !== "function") return null;
 	try {
-		return await runtime.probe(target, await buildProbeContext(runtime, target));
+		const context = await buildProbeContext(runtime, target);
+		return await runtime.probe(target, authToken === undefined ? context : { ...context, authToken });
 	} catch (err) {
 		return { ok: false, error: err instanceof Error ? err.message : String(err) };
 	}
@@ -679,17 +706,69 @@ async function loginOAuthRuntime(rl: ReturnType<typeof createInterface>, runtime
 	}
 }
 
+/**
+ * The model ids a target can be configured with, and where they came from.
+ * `probe` is the only source that says anything about the server in front of
+ * us right now; `catalog` is static provider knowledge, and `existing` is the
+ * list a previous configure recorded, which the server may no longer match.
+ */
+interface WireModelInventory {
+	models: string[];
+	source: "catalog" | "probe" | "existing" | "none";
+	/** Per-model load state when the probe reported it. */
+	modelStates?: Readonly<Record<string, ProbeModelStatus>> | undefined;
+}
+
+/**
+ * Every id the runtime would resolve. LM Studio lists a loaded model under its
+ * instance id and keeps the model key only in the state map, and the request
+ * path accepts either, so both count as advertised.
+ */
+function advertisedModelIds(inventory: Pick<WireModelInventory, "models" | "modelStates">): string[] {
+	const ids = [...inventory.models];
+	for (const id of Object.keys(inventory.modelStates ?? {})) if (!ids.includes(id)) ids.push(id);
+	return ids;
+}
+
+function residentModelIds(modelStates: WireModelInventory["modelStates"]): string[] {
+	return Object.entries(modelStates ?? {})
+		.filter(([, status]) => status.state === "loaded" || status.state === "loading")
+		.map(([id]) => id);
+}
+
+function liveInventoryFor(
+	target: Pick<TargetDescriptor, "id" | "url">,
+	inventory: WireModelInventory,
+): LiveModelInventory {
+	return {
+		targetId: target.id,
+		url: target.url,
+		models: advertisedModelIds(inventory),
+		resident: residentModelIds(inventory.modelStates),
+	};
+}
+
 async function resolveSupportedWireModels(
 	runtime: RuntimeDescriptor,
 	target: TargetDescriptor,
 	existing?: TargetDescriptor,
 	authToken?: string,
-): Promise<string[]> {
+): Promise<WireModelInventory> {
 	const known = listKnownModelsForRuntime(runtime.id);
-	if (known.length > 0) return known;
-	const discovered = runtime.kind === "http" ? await runtimeProbeModels(runtime, target, authToken) : [];
-	if (discovered.length > 0) return discovered;
-	return existing?.wireModels ? [...existing.wireModels] : [];
+	if (known.length > 0) return { models: known, source: "catalog" };
+	if (runtime.kind === "http") {
+		// The full probe carries load state alongside the ids; a runtime that
+		// lists models only through probeModels still gets its ids checked.
+		const probe = await runtimeProbe(runtime, target, authToken);
+		if (probe?.ok && probe.models && probe.models.length > 0) {
+			return { models: [...probe.models], source: "probe", modelStates: probe.modelStates };
+		}
+		const discovered = await runtimeProbeModels(runtime, target, authToken);
+		if (discovered.length > 0) return { models: discovered, source: "probe" };
+	}
+	if (existing?.wireModels && existing.wireModels.length > 0)
+		return { models: [...existing.wireModels], source: "existing" };
+	return { models: [], source: "none" };
 }
 
 function resolveModelChoice(
@@ -810,7 +889,8 @@ async function runNonInteractive(runtime: RuntimeDescriptor, args: ParsedArgs):
 		...(args.reasoning !== undefined ? { reasoning: args.reasoning } : {}),
 		...(existing?.lmstudio ? { lmstudio: existing.lmstudio } : {}),
 	});
-	const wireModels = await resolveSupportedWireModels(runtime, seed, existing, args.apiKey);
+	const inventory = await resolveSupportedWireModels(runtime, seed, existing, args.apiKey);
+	const wireModels = inventory.models;
 	const model =
 		args.model ??
 		existing?.defaultModel ??
@@ -820,7 +900,7 @@ async function runNonInteractive(runtime: RuntimeDescriptor, args: ParsedArgs):
 		refuseCatalogSeededModel(runtime, support);
 		return 2;
 	}
-	if (!validateResolvedModel(runtime.id, model, args.force)) return 2;
+	if (!validateResolvedModel(runtime, seed, model, args.force, inventory)) return 2;
 	if (!validateContextWindowOverride(runtime, model, args.contextWindow, args.force)) return 2;
 	const descriptor = buildDescriptor(runtime, args.id, {
 		...(url !== undefined ? { url } : {}),
@@ -1192,7 +1272,10 @@ async function runInteractive(
 	}
 
 	let model: string | undefined = defaults.model;
-	let wireModels: string[] = existing?.wireModels ? [...existing.wireModels] : [];
+	let inventory: WireModelInventory =
+		existing?.wireModels && existing.wireModels.length > 0
+			? { models: [...existing.wireModels], source: "existing" }
+			: { models: [], source: "none" };
 	const tentative = buildDescriptor(runtime, targetId, {
 		...(url !== undefined ? { url } : {}),
 		...(apiKeyEnv !== undefined ? { apiKeyEnv } : {}),
@@ -1211,8 +1294,9 @@ async function runInteractive(
 	});
 
 	if (runtime.kind === "http") {
-		wireModels = await resolveSupportedWireModels(runtime, tentative, existing ?? undefined);
+		inventory = await resolveSupportedWireModels(runtime, tentative, existing ?? undefined);
 	}
+	const wireModels = inventory.models;
 	// The wizard shows the whole list, so a catalog-ordered runtime does not need
 	// --model here the way the non-interactive path does. It does need to stop
 	// offering the alphabetically first id as though it were the recommended one.
@@ -1234,11 +1318,11 @@ async function runInteractive(
 				process.stdout.write("  a model is required: enter a number from the list or a model id.\n");
 				continue;
 			}
-			if (validateResolvedModel(runtime.id, model, defaults.force)) break;
+			if (validateResolvedModel(runtime, tentative, model, defaults.force, inventory)) break;
 		}
 	} else {
 		for (;;) {
-			if (model && validateResolvedModel(runtime.id, model, defaults.force)) break;
+			if (model && validateResolvedModel(runtime, tentative, model, defaults.force, inventory)) break;
 			const manual = await ask(rl, "Default model id (blank to leave empty)", model ?? "");
 			if (manual === null) return 0;
 			model = manual.length > 0 ? manual : undefined;

package/src/cli/doctor-state-size.ts

@@ -0,0 +1,82 @@
+import { lstatSync, readdirSync } from "node:fs";
+import { join } from "node:path";
+import { resolveClioDirs } from "../core/xdg.js";
+import type { DoctorFinding } from "../domains/lifecycle/doctor.js";
+
+interface MeasuredEntry {
+	bytes: number;
+	error: string | null;
+}
+
+/** Report aggregate state usage and the largest top-level contributor. */
+export function stateStorageFinding(stateDir = resolveClioDirs().state): DoctorFinding {
+	let names: string[];
+	try {
+		names = readdirSync(stateDir);
+	} catch (error) {
+		return {
+			ok: false,
+			name: "state storage",
+			detail: `${stateDir} could not be measured: ${error instanceof Error ? error.message : String(error)}`,
+		};
+	}
+
+	let total = 0;
+	let largest: { name: string; bytes: number } | null = null;
+	for (const name of names.sort()) {
+		const measured = measureEntry(join(stateDir, name));
+		if (measured.error !== null) {
+			return { ok: false, name: "state storage", detail: measured.error };
+		}
+		total += measured.bytes;
+		if (largest === null || measured.bytes > largest.bytes) largest = { name, bytes: measured.bytes };
+	}
+
+	return {
+		ok: true,
+		name: "state storage",
+		detail:
+			largest === null
+				? `${formatBytes(total)} (${total.toLocaleString("en-US")} bytes); largest contributor none`
+				: `${formatBytes(total)} (${total.toLocaleString("en-US")} bytes); largest contributor ${largest.name} at ${formatBytes(largest.bytes)} (${largest.bytes.toLocaleString("en-US")} bytes)`,
+	};
+}
+
+function measureEntry(path: string): MeasuredEntry {
+	let stats: ReturnType<typeof lstatSync>;
+	try {
+		stats = lstatSync(path);
+	} catch (error) {
+		return {
+			bytes: 0,
+			error: `${path} could not be inspected: ${error instanceof Error ? error.message : String(error)}`,
+		};
+	}
+	if (!stats.isDirectory() || stats.isSymbolicLink()) return { bytes: stats.size, error: null };
+
+	let names: string[];
+	try {
+		names = readdirSync(path);
+	} catch (error) {
+		return { bytes: 0, error: `${path} could not be listed: ${error instanceof Error ? error.message : String(error)}` };
+	}
+	let bytes = 0;
+	for (const name of names) {
+		const child = measureEntry(join(path, name));
+		if (child.error !== null) return child;
+		bytes += child.bytes;
+	}
+	return { bytes, error: null };
+}
+
+function formatBytes(bytes: number): string {
+	if (bytes < 1024) return `${bytes} B`;
+	const units = ["KiB", "MiB", "GiB", "TiB"];
+	let value = bytes;
+	let unit = -1;
+	do {
+		value /= 1024;
+		unit += 1;
+	} while (value >= 1024 && unit < units.length - 1);
+	return `${value.toFixed(value >= 10 ? 1 : 2)} ${units[unit]}`;
+}

package/src/cli/doctor.ts

@@ -4,8 +4,10 @@ import {
 	runDoctor,
 	runDoctorFleetChecks,
 	runDoctorInteropChecks,
+	runDoctorModelChecks,
 	runDoctorRuntimeChecks,
 } from "../domains/lifecycle/doctor.js";
+import { stateStorageFinding } from "./doctor-state-size.js";
 import { printError } from "./shared.js";
 
 const HELP = `clio-coder doctor [--fix] [--json]
@@ -34,7 +36,11 @@ export async function runDoctorCommand(args: ReadonlyArray<string> = []): Promis
 	// gets every check on the home it just built.
 	const untouched = !fix && isUninitializedHome();
 	const findings = runDoctor({ fix });
+	const storageChecks = untouched ? [] : [stateStorageFinding()];
 	const runtimeChecks = await runDoctorRuntimeChecks();
+	// Every model pointer is checked against what its target advertises, so a
+	// placeholder id saved by configure is reported here and not on the first turn.
+	const modelChecks = await runDoctorModelChecks();
 	// The interop and fleet sweeps read the state and config roots through the
 	// ensuring accessors, which create them, and there is no fleet or interop
 	// state to inspect before Clio has ever written anything. On a home Clio has
@@ -43,7 +49,7 @@ export async function runDoctorCommand(args: ReadonlyArray<string> = []): Promis
 	// Fleet preflight probes each configured node over SSH and persists the
 	// per-node eligibility verdicts dispatch placement enforces.
 	const fleetChecks = untouched ? [] : await runDoctorFleetChecks();
-	const all = [...findings, ...runtimeChecks, ...interopChecks, ...fleetChecks];
+	const all = [...findings, ...storageChecks, ...runtimeChecks, ...modelChecks, ...interopChecks, ...fleetChecks];
 	const ok = all.every((f) => f.ok);
 	if (json) {
 		process.stdout.write(`${JSON.stringify({ ok, fix, findings: all }, null, 2)}\n`);

package/src/cli/eval.ts

@@ -2,10 +2,10 @@ import { resolve } from "node:path";
 import { InvalidIdError } from "../core/safe-id.js";
 import { clioDataDir } from "../core/xdg.js";
 import { loadEvalArtifactV4, writeEvalArtifactV4 } from "../domains/eval/artifacts/store.js";
-import { compareEvalArtifactsV4, renderEvalComparisonV4 } from "../domains/eval/compare/compare.js";
-import { evaluateGate, renderGateFailure } from "../domains/eval/compare/gates.js";
+import { compareEvalArtifactsV4 } from "../domains/eval/compare/compare.js";
+import { evaluateGate, renderGateFailure, renderInformationalBudget } from "../domains/eval/compare/gates.js";
 import { loadThresholds } from "../domains/eval/compare/thresholds.js";
-
+import { type EvalComparisonReportFormat, renderEvalComparisonReportV1 } from "../domains/eval/reports/comparison.js";
 import { renderEvalJsonReportV4 } from "../domains/eval/reports/json.js";
 import { renderEvalJunitReportV4 } from "../domains/eval/reports/junit.js";
 import { renderEvalMarkdownReportV4 } from "../domains/eval/reports/markdown.js";
@@ -22,10 +22,10 @@ const HELP = `clio-coder eval <command>
 
 Commands:
   clio-coder eval validate --suite <suite.yaml>
-  clio-coder eval run --suite <suite.yaml> [--target <id>] [--model <id>] [--out <path>] [--clio-coder-entry <path>]
-  clio-coder eval run --task-file <tasks.yaml> [--repeat <n>] [--out <path>] [--clio-coder-entry <path>]
+	  clio-coder eval run --suite <suite.yaml> [--trials <n>] [--target <id>] [--model <id>] [--out <path>] [--clio-coder-entry <path>]
+	  clio-coder eval run --task-file <tasks.yaml> [--repeat <n>] [--out <path>] [--clio-coder-entry <path>]
   clio-coder eval report <evalId> --format text|json|md|swe-jsonl|junit
-  clio-coder eval compare <baselineEvalId> <candidateEvalId>
+	  clio-coder eval compare <baselineEvalId> <candidateEvalId> [--metric <name>] [--format text|json|md|junit] [--allow-config-drift]
   clio-coder eval gate <candidateEvalId> --baseline <baselineEvalId> [--thresholds <file>]
 `;
 
@@ -37,6 +37,7 @@ interface ParsedEvalArgs {
 	suite?: string;
 	taskFile?: string;
 	repeat: number;
+	trials?: number;
 	target?: string;
 	model?: string;
 	out?: string;
@@ -44,13 +45,21 @@ interface ParsedEvalArgs {
 	evalId?: string;
 	format: EvalReportFormat;
 	compareIds: string[];
+	metric?: string;
+	allowConfigDrift: boolean;
 	baseline?: string;
 	thresholds?: string;
 	help: boolean;
 }
 
 function parseEvalArgs(args: ReadonlyArray<string>): ParsedEvalArgs {
-	const parsed: ParsedEvalArgs = { repeat: 1, compareIds: [], format: "text", help: false };
+	const parsed: ParsedEvalArgs = {
+		repeat: 1,
+		compareIds: [],
+		format: "text",
+		allowConfigDrift: false,
+		help: false,
+	};
 	for (let index = 0; index < args.length; index += 1) {
 		const arg = args[index];
 		if (arg === undefined) continue;
@@ -110,6 +119,11 @@ function parseEvalArgs(args: ReadonlyArray<string>): ParsedEvalArgs {
 				index += 1;
 				continue;
 			}
+			if (arg === "--trials") {
+				parsed.trials = positiveInteger(requiredValue(args, index, "--trials"), "--trials");
+				index += 1;
+				continue;
+			}
 			throw new Error(`unknown eval run argument: ${arg}`);
 		}
 		if (parsed.command === "report") {
@@ -125,6 +139,20 @@ function parseEvalArgs(args: ReadonlyArray<string>): ParsedEvalArgs {
 			throw new Error(`unexpected eval report argument: ${arg}`);
 		}
 		if (parsed.command === "compare") {
+			if (arg === "--format") {
+				parsed.format = comparisonFormat(requiredValue(args, index, "--format"));
+				index += 1;
+				continue;
+			}
+			if (arg === "--metric") {
+				parsed.metric = requiredValue(args, index, "--metric");
+				index += 1;
+				continue;
+			}
+			if (arg === "--allow-config-drift") {
+				parsed.allowConfigDrift = true;
+				continue;
+			}
 			if (!arg.startsWith("-")) {
 				parsed.compareIds.push(arg);
 				continue;
@@ -202,15 +230,21 @@ async function runEvalRun(parsed: ParsedEvalArgs): Promise<number> {
 		const loaded =
 			parsed.suite !== undefined
 				? await loadEvalSuiteFile(parsed.suite)
-				: await loadV1TaskFileAsSuite(parsed.taskFile ?? "", parsed.repeat);
+				: await loadV1TaskFileAsSuite(parsed.taskFile ?? "", parsed.trials ?? parsed.repeat);
 		const resolveOptions: { target?: string; model?: string } = {};
 		if (parsed.target !== undefined) resolveOptions.target = parsed.target;
 		if (parsed.model !== undefined) resolveOptions.model = parsed.model;
-		const suite = resolveSuiteForRun(loaded.suite, resolveOptions);
+		const suite = resolveSuiteForRun(loaded.suite, {
+			...resolveOptions,
+			...(parsed.trials ? { trials: parsed.trials } : {}),
+		});
 		// Runner commands execute with cwd inside prepared eval workspaces, so a
 		// relative --clio-coder-entry must be pinned to the invoking directory here.
 		const clioEntry = resolve(parsed.clioEntry ?? process.argv[1] ?? "dist/cli/index.js");
-		const artifact = await runEvalSuiteV2({ ...loaded, suite }, { clioEntry });
+		const artifact = await runEvalSuiteV2(
+			{ ...loaded, suite },
+			{ clioEntry, freshWorkspaces: parsed.trials !== undefined },
+		);
 		const artifactPath = await writeEvalArtifactV4(clioDataDir(), artifact, parsed.out);
 		process.stdout.write(`${renderEvalTextReportV4(artifact)}artifact: ${artifactPath}\n`);
 		// A suite that declares thresholds is gated by them here, on the artifact
@@ -221,6 +255,10 @@ async function runEvalRun(parsed: ParsedEvalArgs): Promise<number> {
 			process.stdout.write(`gate: fail (${gate.failures.length} threshold failure)\n`);
 			for (const failure of gate.failures) process.stdout.write(renderGateFailure(failure));
 		}
+		if (gate !== null && gate.informational.length > 0) {
+			process.stdout.write(`informational budgets: ${gate.informational.length} notice\n`);
+			for (const finding of gate.informational) process.stdout.write(renderInformationalBudget(finding));
+		}
 		return artifact.summary.failed === 0 && (gate === null || gate.pass) ? 0 : 1;
 	} catch (error) {
 		return handleEvalLoadError(error, 1);
@@ -246,8 +284,12 @@ async function runEvalCompareCommand(parsed: ParsedEvalArgs): Promise<number> {
 		const dataDir = clioDataDir();
 		const baseline = await loadEvalArtifactV4(dataDir, baselineEvalId);
 		const candidate = await loadEvalArtifactV4(dataDir, candidateEvalId);
-		process.stdout.write(renderEvalComparisonV4(compareEvalArtifactsV4(baseline, candidate)));
-		return 0;
+		const summary = compareEvalArtifactsV4(baseline, candidate, {
+			allowConfigDrift: parsed.allowConfigDrift,
+			...(parsed.metric === undefined ? {} : { metric: parsed.metric }),
+		});
+		process.stdout.write(renderEvalComparisonReportV1(summary, parsed.format as EvalComparisonReportFormat));
+		return summary.hardGate.pass ? 0 : 1;
 	} catch (error) {
 		printError(error instanceof Error ? error.message : String(error));
 		return error instanceof InvalidIdError ? 2 : 1;
@@ -258,18 +300,35 @@ async function runEvalGateCommand(parsed: ParsedEvalArgs): Promise<number> {
 	try {
 		const dataDir = clioDataDir();
 		const candidate = await loadEvalArtifactV4(dataDir, parsed.evalId ?? "");
-		await loadEvalArtifactV4(dataDir, parsed.baseline ?? "");
+		const baseline = await loadEvalArtifactV4(dataDir, parsed.baseline ?? "");
 		const thresholds =
 			parsed.thresholds === undefined
-				? { fail: [{ metric: "result.pass", op: "eq" as const, value: false }] }
+				? { fail: [{ metric: "result.pass", op: "eq" as const, value: false }], informational: [] }
 				: loadThresholds(parsed.thresholds);
 		const gate = evaluateGate(candidate, thresholds);
-		if (gate.pass) {
+		const comparison = compareEvalArtifactsV4(baseline, candidate);
+		if (gate.informational.length > 0) {
+			process.stdout.write(`informational budgets: ${gate.informational.length} notice\n`);
+			for (const finding of gate.informational) process.stdout.write(renderInformationalBudget(finding));
+		}
+		if (gate.pass && comparison.hardGate.pass) {
 			process.stdout.write("gate: pass\n");
 			return 0;
 		}
-		process.stdout.write(`gate: fail (${gate.failures.length} threshold failure)\n`);
+		const failureCount =
+			gate.failures.length + comparison.hardGate.failures.length + comparison.hardGate.envelopeFailures.length;
+		process.stdout.write(`gate: fail (${failureCount} hard failure)\n`);
 		for (const failure of gate.failures) process.stdout.write(renderGateFailure(failure));
+		for (const failure of comparison.hardGate.failures) {
+			process.stdout.write(
+				`  ${failure.metric} [${failure.scenarioId}:${failure.role}:${failure.target.id}/${failure.target.model ?? "none"}]: ${failure.change} (hard behavioral gate)\n`,
+			);
+		}
+		for (const failure of comparison.hardGate.envelopeFailures) {
+			process.stdout.write(
+				`  execution envelope [${failure.scenarioId}:${failure.role}:${failure.target.id}/${failure.target.model ?? "none"}]: incomparable fields ${failure.fields.join(", ")}\n`,
+			);
+		}
 		return 1;
 	} catch (error) {
 		printError(error instanceof Error ? error.message : String(error));
@@ -313,3 +372,8 @@ function reportFormat(value: string): EvalReportFormat {
 	if (value === "text" || value === "json" || value === "md" || value === "swe-jsonl" || value === "junit") return value;
 	throw new Error("--format must be text, json, md, swe-jsonl, or junit");
 }
+
+function comparisonFormat(value: string): EvalComparisonReportFormat {
+	if (value === "text" || value === "json" || value === "md" || value === "junit") return value;
+	throw new Error("eval compare --format must be text, json, md, or junit");
+}