@iowarp/clio-coder 0.4.0 → 0.4.2

package/src/domains/eval/schema/suite.ts

@@ -129,7 +129,7 @@ export const CANONICAL_METRIC_PREFIXES = ["tools.calls.", "tools.blocked."] as c
 
 export type EvalMetricName = (typeof CANONICAL_METRICS)[number];
 export type EvalAssertionOp = "lt" | "lte" | "gt" | "gte" | "eq" | "neq";
-export type EvalRunnerKind = "clio-run" | "context-index" | "context-init" | "external-command";
+export type EvalRunnerKind = "clio-coder-run" | "context-index" | "context-init" | "external-command";
 export type EvalWorkspaceKind = "local" | "git" | "temp-copy";
 
 export interface EvalMetricAssertion {
@@ -189,7 +189,7 @@ export interface EvalRunnerV2 {
 	/** One-run headless authority used to exercise allow and deny recovery. */
 	autonomy?: "read-only" | "suggest" | "auto-edit" | "full-auto";
 	/**
-	 * Fleet agent recipe id for the clio-run runner. When set the runner
+	 * Fleet agent recipe id for the clio-coder-run runner. When set the runner
 	 * invokes `clio-coder run --agent <id> --json`, whose stream ends with the full
 	 * sealed RunReceipt, so receipt-derived evidence metrics resolve. Without
 	 * it the main-agent headless path runs and emits no receipt.

package/src/domains/eval/schema/validate.ts

@@ -1,5 +1,6 @@
 import { parseEvalBehaviorScenarioV1 } from "./behavioral.js";
 import { parseEvalExecutionMatrixDimensionsV1 } from "./execution-envelope.js";
+import { normalizeEvalRunnerKind } from "./naming.js";
 import {
 	EVAL_SUITE_V2_VERSION,
 	type EvalAssertionOp,
@@ -10,7 +11,7 @@ import {
 	type EvalWorkspaceKind,
 } from "./suite.js";
 
-const RUNNER_KINDS = new Set<EvalRunnerKind>(["clio-run", "context-index", "context-init", "external-command"]);
+const RUNNER_KINDS = new Set<EvalRunnerKind>(["clio-coder-run", "context-index", "context-init", "external-command"]);
 const WORKSPACE_KINDS = new Set<EvalWorkspaceKind>(["local", "git", "temp-copy"]);
 const OPS = new Set<EvalAssertionOp>(["lt", "lte", "gt", "gte", "eq", "neq"]);
 
@@ -203,9 +204,12 @@ function readRunner(
 		issues.push({ path, message: "expected object" });
 		return null;
 	}
-	const kind = value.kind;
+	const kind = normalizeEvalRunnerKind(value.kind);
 	if (typeof kind !== "string" || !RUNNER_KINDS.has(kind as EvalRunnerKind)) {
-		issues.push({ path: `${path}.kind`, message: "expected clio-run, context-index, context-init, or external-command" });
+		issues.push({
+			path: `${path}.kind`,
+			message: "expected clio-coder-run, context-index, context-init, or external-command",
+		});
 		return null;
 	}
 	const prompt = optionalString(value, "prompt");
@@ -214,8 +218,8 @@ function readRunner(
 	if (autonomy !== undefined && !["read-only", "suggest", "auto-edit", "full-auto"].includes(autonomy)) {
 		issues.push({ path: `${path}.autonomy`, message: "expected read-only, suggest, auto-edit, or full-auto" });
 	}
-	if (agent !== undefined && kind !== "clio-run") {
-		issues.push({ path: `${path}.agent`, message: "agent is only valid on the clio-run runner" });
+	if (agent !== undefined && kind !== "clio-coder-run") {
+		issues.push({ path: `${path}.agent`, message: "agent is only valid on the clio-coder-run runner" });
 	}
 	const command = optionalString(value, "command");
 	return {

package/src/domains/eval/schema/verdict.ts

@@ -1,4 +1,6 @@
-export const EVAL_VERDICT_SCHEMA_V1 = "clio.eval.verdict.v1" as const;
+import { normalizeEvalSchemaId } from "./naming.js";
+
+export const EVAL_VERDICT_SCHEMA_V1 = "clio-coder.eval.verdict.v1" as const;
 
 export const EVAL_TRACKED_METRIC_NAMES = [
 	"modelCalls",
@@ -68,7 +70,7 @@ export type EvalVerdictParseResult = { ok: true; verdict: EvalVerdictEnvelopeV1
 /** Parse the v1 verdict and reject contradictions that could publish a false pass. */
 export function parseEvalVerdictEnvelopeV1(value: unknown, source = "verdict"): EvalVerdictEnvelopeV1 {
 	const record = asRecord(value, source);
-	if (record.schema !== EVAL_VERDICT_SCHEMA_V1) {
+	if (normalizeEvalSchemaId(record.schema) !== EVAL_VERDICT_SCHEMA_V1) {
 		throw new Error(`${source}.schema: expected ${EVAL_VERDICT_SCHEMA_V1}`);
 	}
 	const scenarioId = readNonEmptyString(record, source, "scenarioId");

package/src/domains/eval/store.ts

@@ -60,9 +60,16 @@ export function evalRoot(dataDir: string): string {
 	return join(dataDir, "evals");
 }
 
+/**
+ * `skills-eval` is the only writer of this legacy version-1 shape; it nests
+ * under its own subdirectory so a legacy id that collides with a `clio-coder
+ * eval` run's version-4 id (evalArtifactPathV4, artifacts/store.ts) cannot
+ * clobber that artifact, and so `eval inventory`'s top-level readdir of
+ * evalRoot() never trips over a shape it does not parse.
+ */
 export function evalArtifactPath(dataDir: string, evalId: string): string {
 	assertSafeId(evalId, "eval");
-	return join(evalRoot(dataDir), `${evalId}.json`);
+	return join(evalRoot(dataDir), "skills-eval", `${evalId}.json`);
 }
 
 /**
@@ -118,7 +125,7 @@ function parseArtifact(value: unknown, source: string): EvalRunArtifact {
 		evalId: readString(value, source, "evalId"),
 		taskFile: readString(value, source, "taskFile"),
 		taskFileHash: readString(value, source, "taskFileHash"),
-		clio: parseClioProvenance(readRequiredArtifactField(value, source, "clio"), `${source}.clio`),
+		clioCoder: parseClioProvenance(readEvalProvenance(value, source), `${source}.clioCoder`),
 		environment: parseEnvironmentProvenance(
 			readRequiredArtifactField(value, source, "environment"),
 			`${source}.environment`,
@@ -235,7 +242,7 @@ function parseRunPaths(value: unknown, source: string): EvalRunPaths {
 function readRequiredArtifactField(
 	record: Record<string, unknown>,
 	source: string,
-	field: "clio" | "environment" | "paths",
+	field: "environment" | "paths",
 ): unknown {
 	const value = record[field];
 	if (value === undefined) {
@@ -246,6 +253,16 @@ function readRequiredArtifactField(
 	return value;
 }
 
+function readEvalProvenance(record: Record<string, unknown>, source: string): unknown {
+	const value = record.clioCoder ?? record.clio;
+	if (value === undefined) {
+		throw new Error(
+			`eval artifact has an invalid schema: missing required field 'clioCoder' in ${source}. Re-run the evaluation suite to generate a complete artifact.`,
+		);
+	}
+	return value;
+}
+
 function parseHarnessMetrics(value: unknown, source: string): EvalHarnessMetrics {
 	if (!isRecord(value)) throw new Error(`${source}: expected object`);
 	return {

package/src/domains/eval/suites/run.ts

@@ -1,6 +1,7 @@
 import { mkdtemp, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { resolve } from "node:path";
+import { namingCompatibilityEnvironment } from "../../../core/naming-compat.js";
 import { resolveMetricAssertion } from "../compare/thresholds.js";
 import { buildEvalExecutionEnvelopeV1, type EvalExecutionObservationV1 } from "../execution-provenance.js";
 import { aggregateEvalVerdicts } from "../metrics/aggregate.js";
@@ -32,6 +33,7 @@ import { type EvalRunnerOutput, runExternalCommandRunner } from "../runners/exte
 import { adaptSuiteV2ResultToBehaviorV1, adaptSuiteV2ResultToVerdictV1 } from "../schema/adapter.js";
 import type { EvalArtifactResultV4, EvalArtifactV4 } from "../schema/artifact.js";
 import { buildEvalBehaviorMetricsV1 } from "../schema/behavioral-metrics.js";
+import { normalizeEvalSchemaId } from "../schema/naming.js";
 import type { EvalServingConfigurationV1 } from "../schema/serving.js";
 import type { EvalMetricAssertion, EvalSuiteTargetV2, LoadedEvalSuiteV2 } from "../schema/suite.js";
 import { createEvalId } from "../store.js";
@@ -102,7 +104,7 @@ export async function runEvalSuiteV2(
 }
 
 /** Known receipt cost of one finished matrix item; unpriced runs count zero. */
-export function resultCostUsd(result: Pick<EvalArtifactResultV4, "metrics">): number {
+function resultCostUsd(result: Pick<EvalArtifactResultV4, "metrics">): number {
 	const value = result.metrics["cost.usd"];
 	return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : 0;
 }
@@ -156,7 +158,7 @@ async function runMatrixItem(
 	// state directory would mix sibling processes' runs and yesterday's sessions
 	// into the reading; pinning it here also means an item leaves nothing behind
 	// in the operator's own state.
-	const stateDir = await mkdtemp(resolve(tempCopy?.tempRoot ?? tmpdir(), "clio-eval-state-"));
+	const stateDir = await mkdtemp(resolve(tempCopy?.tempRoot ?? tmpdir(), "clio-coder-eval-state-"));
 	try {
 		workspace = await prepareWorkspace(loaded.baseDir, task, freshWorkspace, tempCopy);
 		const setup = await runCommandVerifiers(task.workspace.setup ?? [], workspace.dir, task.timeoutMs);
@@ -178,7 +180,11 @@ async function runMatrixItem(
 		// the workers it attested are still running.
 		const journalMetrics = invariantMetrics(stateDir, receiptExitCode);
 		const measurement = await measureTaskOutcome(task, workspace.dir, {
-			CLIO_EVAL_RUNNER_STDOUT_FILE: runnerStdoutFile,
+			...namingCompatibilityEnvironment(
+				"CLIO_CODER_EVAL_RUNNER_STDOUT_FILE",
+				"CLIO_EVAL_RUNNER_STDOUT_FILE",
+				runnerStdoutFile,
+			),
 		});
 		executionObservation = measurement.executionObservation;
 		const metrics: Record<string, number | string | boolean | null> = {
@@ -407,13 +413,13 @@ function graderBehaviorMeasurement(stdout: string): {
 			continue;
 		}
 		if (!isRecord(value)) continue;
-		if (value.schema === "clio.eval.measure.v1" && isRecord(value.metrics)) {
+		if (normalizeEvalSchemaId(value.schema) === "clio-coder.eval.measure.v1" && isRecord(value.metrics)) {
 			for (const [key, metric] of Object.entries(value.metrics)) {
 				if (key !== "claims.unsupported" && key !== "completion.reported") continue;
 				if (typeof metric === "boolean" || (typeof metric === "number" && Number.isFinite(metric))) metrics[key] = metric;
 			}
 		}
-		if (value.schema === "clio.eval.execution-observation.v1") {
+		if (normalizeEvalSchemaId(value.schema) === "clio-coder.eval.execution-observation.v1") {
 			executionObservation = parseExecutionObservation(value);
 		}
 	}
@@ -503,7 +509,7 @@ function buildArtifact(
 		version: 4,
 		evalId,
 		suite: { id: loaded.suite.suite.id, hash: loaded.hash },
-		clio: evalClioProvenance({ entry: clioEntry }),
+		clioCoder: evalClioProvenance({ entry: clioEntry }),
 		environment: evalEnvironmentProvenance(),
 		matrix: {
 			...artifactMatrixIdentity(loaded.suite.matrix.targets),

package/src/domains/eval/types.ts

@@ -126,7 +126,7 @@ export interface EvalRunArtifact {
 	evalId: string;
 	taskFile: string;
 	taskFileHash: string;
-	clio: EvalClioProvenance;
+	clioCoder: EvalClioProvenance;
 	environment: EvalEnvironmentProvenance;
 	target: string | null;
 	model: string | null;

package/src/domains/eval/workspaces/git.ts

@@ -7,7 +7,7 @@ import type { PreparedEvalWorkspace } from "./local.js";
 
 export async function prepareGitWorkspace(workspace: EvalWorkspaceV2): Promise<PreparedEvalWorkspace> {
 	if (workspace.url === undefined) throw new Error("git workspace requires url");
-	const dest = await mkdtemp(resolve(tmpdir(), "clio-eval-git-"));
+	const dest = await mkdtemp(resolve(tmpdir(), "clio-coder-eval-git-"));
 	try {
 		await runGit(["clone", "--quiet", workspace.url, dest], process.cwd());
 		const ref = workspace.checkout ?? workspace.commit;

package/src/domains/eval/workspaces/temp-copy.ts

@@ -33,7 +33,7 @@ export async function prepareTempCopyWorkspace(
 	options: PrepareTempCopyWorkspaceOptions = {},
 ): Promise<PreparedEvalWorkspace> {
 	const source = resolve(baseDir, workspace.path ?? ".");
-	const dest = await mkdtemp(resolve(options.tempRoot ?? tmpdir(), "clio-eval-workspace-"));
+	const dest = await mkdtemp(resolve(options.tempRoot ?? tmpdir(), "clio-coder-eval-workspace-"));
 	try {
 		const selection = await gitCopySelection(source);
 		const excludes = workspace.excludes ?? [];

package/src/domains/evidence/build.ts

@@ -56,6 +56,7 @@ const MAX_TASK_CHARS = 500;
 const TRANSCRIPT_TEXT_MAX_CHARS = 500;
 const TOOL_PREVIEW_MAX_CHARS = 240;
 const TASK_LEDGER_ROW_LIMIT = 200;
+const MODEL_PRIVATE_MARKER = "[not sent to model]";
 
 export interface BuildEvidenceOptions {
 	/** Evidence bundles are written under <dataDir>/evidence/. */
@@ -892,6 +893,7 @@ function sessionToolEvents(entries: ReadonlyArray<LinkedSessionEntry>): Evidence
 	for (const linked of entries) {
 		const entry = linked.entry;
 		if (entry.kind === "bashExecution") {
+			const contextMarker = entry.excludeFromContext === true ? `${MODEL_PRIVATE_MARKER} ` : "";
 			events.push({
 				source: "session-entry",
 				runId: linked.link.runId,
@@ -906,8 +908,8 @@ function sessionToolEvents(entries: ReadonlyArray<LinkedSessionEntry>): Evidence
 				timestamp: entry.timestamp,
 				linkKind: "session-bash-execution",
 				confidence: "exact",
-				argsPreview: truncateText(entry.command, TOOL_PREVIEW_MAX_CHARS),
-				resultPreview: truncateText(entry.output, TOOL_PREVIEW_MAX_CHARS),
+				argsPreview: truncateText(`${contextMarker}${entry.command}`, TOOL_PREVIEW_MAX_CHARS),
+				resultPreview: truncateText(`${contextMarker}${entry.output}`, TOOL_PREVIEW_MAX_CHARS),
 			});
 			continue;
 		}
@@ -1354,9 +1356,10 @@ function renderSessionTranscriptEntry(linked: LinkedSessionEntry): string[] {
 	}
 	if (entry.kind === "bashExecution") {
 		const status = entry.cancelled ? "cancelled" : entry.exitCode === null ? "exit=?" : `exit=${entry.exitCode}`;
+		const contextMarker = entry.excludeFromContext === true ? ` ${MODEL_PRIVATE_MARKER}` : "";
 		return [
-			`${prefix} bash ${status}: $ ${truncateText(entry.command, TRANSCRIPT_TEXT_MAX_CHARS)}`,
-			`  output: ${previewUnknown(entry.output)}`,
+			`${prefix} bash ${status}${contextMarker}: $ ${truncateText(entry.command, TRANSCRIPT_TEXT_MAX_CHARS)}`,
+			`  output${contextMarker}: ${previewUnknown(entry.output)}`,
 		];
 	}
 	if (entry.kind === "branchSummary")
@@ -1625,7 +1628,7 @@ function parseRunReceipt(value: unknown, source: string): RunReceipt {
 		costUsd: readNumber(value, source, "costUsd"),
 		compiledPromptHash: readNullableString(value, source, "compiledPromptHash"),
 		staticCompositionHash: readNullableString(value, source, "staticCompositionHash"),
-		clioVersion: readString(value, source, "clioVersion"),
+		clioCoderVersion: readReceiptVersion(value, source),
 		piMonoVersion: readString(value, source, "piMonoVersion"),
 		platform: readString(value, source, "platform"),
 		nodeVersion: readString(value, source, "nodeVersion"),
@@ -1640,6 +1643,11 @@ function parseRunReceipt(value: unknown, source: string): RunReceipt {
 	};
 }
 
+function readReceiptVersion(value: Record<string, unknown>, source: string): string {
+	if (typeof value.clioCoderVersion === "string") return value.clioCoderVersion;
+	return readString(value, source, "clioVersion");
+}
+
 function readToolStats(record: Record<string, unknown>, source: string, field: string): ToolCallStat[] {
 	const value = record[field];
 	if (!Array.isArray(value)) throw new Error(`${source}.${field}: expected array`);

package/src/domains/evidence/trust-status.ts

@@ -619,6 +619,15 @@ export function adaptRunReceiptValidationStatus(
 	if (receipt.hostVerification?.status === "verified") {
 		return attributed("validated", receiptSource(receipt), { kind: "validator", id: "host-verification" }, artifacts);
 	}
+	// A batch member the failure was charged away from carries a check that
+	// exited non-zero, so the validator has no verdict for this run: neither the
+	// "failed" the exit code alone would suggest nor the "validated" a passing
+	// check earns. It stops here rather than falling through to receipt-quality,
+	// which cannot see that a declared host check failed on the tree this run
+	// shipped in.
+	if (receipt.hostVerification?.status === "not_implicated") {
+		return attributed("unknown", receiptSource(receipt), { kind: "validator", id: "host-verification" }, artifacts);
+	}
 	if (qualityHasFailure(receipt.quality)) {
 		return attributed("failed", receiptSource(receipt), { kind: "validator", id: "receipt-quality" }, artifacts);
 	}
@@ -939,7 +948,7 @@ export interface GroundedEvidenceValidationInput {
  * Project validation that the evidence linker grounded in an executed
  * artifact. Grounding requires at least one independently observed artifact:
  * a self-report of the run under inspection cannot ground its own validation
- * (docs/evidence-and-memory.md, "composition rules"), and an empty reference
+ * (docs/architecture/evidence-and-memory.md, "composition rules"), and an empty reference
  * list grounds
  * nothing at all.
  */

package/src/domains/extensions/contract.ts

@@ -5,8 +5,10 @@ import type {
 	ExtensionInstallResult,
 	ExtensionListOptions,
 	ExtensionMutationResult,
+	ExtensionReloadPrepareResult,
 	ExtensionResourceKind,
 	ExtensionResourceRoot,
+	ExtensionSnapshot,
 	InstalledExtension,
 } from "./manager.js";
 
@@ -17,6 +19,18 @@ export interface ExtensionsContract extends DomainContract {
 	enable(id: string, options?: ExtensionListOptions): ExtensionMutationResult;
 	disable(id: string, options?: ExtensionListOptions): ExtensionMutationResult;
 	remove(id: string, options?: ExtensionListOptions): ExtensionMutationResult;
+	/** Committed projection for the bundle's cwd; ephemeral generation-0 build for any other cwd. */
 	resourceRoots(kind: ExtensionResourceKind, cwd?: string): ExtensionResourceRoot[];
-	reload(): Promise<void>;
+	snapshot(): ExtensionSnapshot | null;
+	generation(): number;
+	/**
+	 * Build and validate the next generation from disk without publishing it.
+	 * Never throws: a build failure or an in-flight candidate is reported as a
+	 * rejection and the committed generation is untouched. A rejected build
+	 * still burns its reserved generation number. The candidate publishes
+	 * itself with one assignment-only call after the caller has confirmed
+	 * `current()` on the same stack. Nothing is published at start(); the
+	 * composition root publishes the boot generation together with its hooks.
+	 */
+	prepareReload(): ExtensionReloadPrepareResult;
 }