@iowarp/clio-coder 0.3.6 → 0.3.8

package/src/domains/eval/metrics/evidence.ts

@@ -6,7 +6,35 @@
  * than silently pass.
  */
 
-import type { RunReceipt } from "../../dispatch/types.js";
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import type { RunEnvelope, RunReceipt } from "../../dispatch/types.js";
+import { formatTrustSummary, summarizeTrustStatus } from "../../evidence/trust-projection.js";
+import {
+	adaptRunReceiptTrustStatus,
+	inspectRunReceiptTrustStatus,
+	TRUST_STATUS_AXES,
+} from "../../evidence/trust-status.js";
+
+function dispatchScopeMetrics(receipt: RunReceipt): Record<string, string | boolean | number> {
+	const scope = receipt.pathScope;
+	if (scope === undefined) return {};
+	const entries = [...scope.workingContextPaths, ...scope.writeBoundaries];
+	const evidence = entries.flatMap((entry) => entry.evidence);
+	const count = (source: string): number => evidence.filter((entry) => entry.source === source).length;
+	return {
+		"dispatch.scope.mode": scope.mode,
+		"dispatch.scope.inferredPathCount": entries.filter((entry) =>
+			entry.evidence.some((item) => item.provenance === "inferred"),
+		).length,
+		"dispatch.scope.derivedPathCount": entries.filter((entry) =>
+			entry.evidence.some((item) => item.provenance === "derived"),
+		).length,
+		"dispatch.scope.source.task": count("task"),
+		"dispatch.scope.source.briefing": count("briefing"),
+		"dispatch.scope.source.writeRoots": count("writeRoots"),
+	};
+}
 
 /**
  * Extract the sealed RunReceipt that `clio-coder run --agent … --json` prints after
@@ -53,9 +81,14 @@ function isReceiptShaped(value: unknown): value is RunReceipt {
  * mirrors the durable findings summary and is omitted when that summary is
  * absent, so a gate on the metric fails closed instead of inventing a value.
  */
-export function evidenceMetricsFromReceipt(receipt: RunReceipt): Record<string, string | boolean | number> {
+export function evidenceMetricsFromReceipt(
+	receipt: RunReceipt,
+	options: { envelope?: RunEnvelope | null } = {},
+): Record<string, string | boolean | number> {
 	return {
 		"evidence.verification": receipt.verification.state,
+		...dispatchScopeMetrics(receipt),
+		...evidenceTrustMetrics(receipt, options.envelope ?? null),
 		...(receipt.findingsSummary === undefined
 			? {}
 			: { "evidence.firstPassSuccess": receipt.findingsSummary.firstPassSuccess === true }),
@@ -64,3 +97,47 @@ export function evidenceMetricsFromReceipt(receipt: RunReceipt): Record<string,
 		...(typeof receipt.costUsd === "number" && Number.isFinite(receipt.costUsd) ? { "cost.usd": receipt.costUsd } : {}),
 	};
 }
+
+/**
+ * The canonical trust status as metrics, so a benchmark table reads the same
+ * verdict as every operator surface instead of the raw receipt marker. The
+ * receipt is authenticated against its ledger row when the caller can supply
+ * one; without a row the canonical model reports the seal unchecked and the
+ * receipt-owned axes unobserved, which is the honest answer for a receipt
+ * that only ever arrived through stdout.
+ */
+function evidenceTrustMetrics(
+	receipt: RunReceipt,
+	envelope: RunEnvelope | null,
+): Record<string, string | boolean | number> {
+	// A missing row means the seal was never checked, not that it failed:
+	// only a row that exists and disagrees can break a seal.
+	const status =
+		envelope === null ? adaptRunReceiptTrustStatus(receipt) : inspectRunReceiptTrustStatus(receipt, envelope).status;
+	const summary = summarizeTrustStatus(status);
+	return {
+		"evidence.trust.version": summary.version,
+		"evidence.trust.verdict": summary.verdict,
+		"evidence.trust.summary": formatTrustSummary(status),
+		...Object.fromEntries(TRUST_STATUS_AXES.map((axis) => [`evidence.trust.${axis}`, status[axis].state])),
+	};
+}
+
+/**
+ * The ledger row a stdout receipt was sealed from, read from the state dir
+ * the run wrote to. Null when the ledger or the row cannot be read; the
+ * caller then reports the seal unchecked rather than guessing.
+ */
+export function readRunEnvelopeForReceipt(receipt: RunReceipt, stateDir: string): RunEnvelope | null {
+	try {
+		const parsed: unknown = JSON.parse(readFileSync(join(stateDir, "runs.json"), "utf8"));
+		if (!Array.isArray(parsed)) return null;
+		const row = parsed.find(
+			(entry): entry is RunEnvelope =>
+				typeof entry === "object" && entry !== null && (entry as { id?: unknown }).id === receipt.runId,
+		);
+		return row ?? null;
+	} catch {
+		return null;
+	}
+}

package/src/domains/eval/runners/clio-run.ts

@@ -1,5 +1,10 @@
 import { shellQuote } from "../../../core/shell-quote.js";
-import { evidenceMetricsFromReceipt, receiptFromRunJsonStdout } from "../metrics/evidence.js";
+import { clioStateDir } from "../../../core/xdg.js";
+import {
+	evidenceMetricsFromReceipt,
+	readRunEnvelopeForReceipt,
+	receiptFromRunJsonStdout,
+} from "../metrics/evidence.js";
 import { streamInvariantMetrics } from "../metrics/invariants.js";
 import { tokenMetricEntries } from "../metrics/token-stream.js";
 import type { EvalRunnerV2, EvalSuiteTargetV2 } from "../schema/suite.js";
@@ -35,6 +40,11 @@ export async function runClioRunRunner(
 	// prints; a runner without a receipt leaves them absent so any gate on
 	// them fails closed instead of reading prose labels.
 	const receipt = receiptFromRunJsonStdout(result.stdout);
+	// The canonical trust metrics authenticate the receipt against the ledger
+	// row the child wrote; a stdout receipt on its own only ever reads as an
+	// unchecked seal.
+	const envelope =
+		receipt === null ? null : readRunEnvelopeForReceipt(receipt, env?.CLIO_CODER_STATE_DIR ?? clioStateDir());
 	return {
 		assignmentId: receipt === null ? null : (receipt.lineage?.rootRunId ?? receipt.runId),
 		terminalReceiptDigest: receipt?.integrity.digest ?? null,
@@ -52,7 +62,7 @@ export async function runClioRunRunner(
 			"tools.failed": tools.failed,
 			"tools.blocked": tools.blocked,
 			"verifier.exitCode": result.exitCode,
-			...(receipt === null ? {} : evidenceMetricsFromReceipt(receipt)),
+			...(receipt === null ? {} : evidenceMetricsFromReceipt(receipt, { envelope })),
 			...(receipt === null
 				? {}
 				: { "evidence.qualityLabel": receipt.quality.typedValidations.length > 0 ? "measured" : "unmeasured" }),

package/src/domains/evidence/build.ts

@@ -29,8 +29,8 @@ import { extractRunProvenance, provenanceTranscriptLines } from "./provenance.js
 import { createRedactionTally, redactSecretsDeep, redactSecretsText } from "./redact.js";
 import { buildEvidenceTrustStatusFile } from "./run-trust.js";
 import { EVIDENCE_FILES, evidenceDirectory, findingsFile } from "./store.js";
-import type { TrustArtifactReference } from "./trust-status.js";
-import { inspectRunReceiptTrustStatus } from "./trust-status.js";
+import type { CanonicalTrustStatus, TrustArtifactReference } from "./trust-status.js";
+import { inspectRunReceiptTrustStatus, retiredIntegrityVersionOf } from "./trust-status.js";
 import {
 	EVIDENCE_VERSION,
 	type EvidenceAuditLinkedRow,
@@ -45,6 +45,7 @@ import {
 	type EvidenceReceiptFile,
 	type EvidenceRunLink,
 	type EvidenceRunSource,
+	type EvidenceSeverity,
 	type EvidenceTag,
 	type EvidenceToolEvent,
 	type EvidenceTrustStatusFile,
@@ -195,7 +196,10 @@ export async function buildEvidence(options: BuildEvidenceOptions): Promise<Evid
 		redactedToolEvents,
 		protectedArtifacts,
 	);
-	const transcript = redactSecretsText(renderTranscript(overview, redactedRunSources, sessionLinks), tally);
+	const transcript = redactSecretsText(
+		renderTranscript(overview, redactedRunSources, sessionLinks, trustStatusRaw),
+		tally,
+	);
 	const finalOverview: EvidenceOverview = { ...overview, redactionCount: tally.count };
 	await writeEvidenceFiles(
 		directory,
@@ -285,8 +289,14 @@ function buildFindings(
 		const status = trustByRun.get(source.envelope.id);
 		if (status === undefined) throw new Error(`canonical trust status missing for run ${source.envelope.id}`);
 		if (source.receiptError !== null) {
-			const tag = source.receiptIntegrityFailed ? "receipt-integrity" : "unknown";
-			findings.push(finding(findings.length, "warn", tag, source.envelope.id, source.receiptError));
+			// The canonical axis, not the raw reason string, says whether the seal
+			// was retired: `receipt-retired` is the expected state of every run
+			// that predates an integrity bump and is an info row, while
+			// `receipt-integrity` stays the warning for a seal that was checked
+			// and rejected.
+			const retired = retiredIntegrityVersionOf(status.artifactIntegrity) !== null;
+			const tag = retired ? "receipt-retired" : source.receiptIntegrityFailed ? "receipt-integrity" : "unknown";
+			findings.push(finding(findings.length, retired ? "info" : "warn", tag, source.envelope.id, source.receiptError));
 		}
 		if (
 			isSuccessfulRun(source) &&
@@ -359,6 +369,13 @@ function buildFindings(
 				),
 			);
 		}
+		// The remaining axes were never read here, so a run could finish with
+		// no independent review, a contradictory context record, or a mutation
+		// nobody validated, and findings.md said nothing. Each finding is
+		// selected from the canonical state the way the validation and
+		// autonomy findings above are, with the detailed record left in
+		// trust-status.json.
+		findings.push(...axisFindings(findings.length, source, status));
 		if (source.envelope.status === "stale" || source.envelope.status === "dead") {
 			findings.push(
 				finding(findings.length, "warn", "timeout", source.envelope.id, `run status ${source.envelope.status}`),
@@ -601,7 +618,7 @@ async function writeEvidenceFiles(
 	await writeJson(join(directory, "overview.json"), overview);
 	await writeFile(
 		join(directory, "transcript.md"),
-		transcript ?? renderTranscript(overview, runSources, sessionLinks),
+		transcript ?? renderTranscript(overview, runSources, sessionLinks, trustStatus),
 		"utf8",
 	);
 	await writeJsonl(join(directory, "trace.raw.jsonl"), rawTraceRows(runSources));
@@ -636,9 +653,11 @@ function cleanedTraceRows(
 	const rows: EvidenceCleanTraceRow[] = [];
 	for (const source of runSources) {
 		const envelope = source.envelope;
-		// Clean trace rows project only provenance carried by the sealed receipt.
+		// Clean trace rows project only provenance carried by an authenticated
+		// receipt: a rejected or retired seal contributes none of its fields.
 		// Runs without receipt provenance keep the standard row shape.
-		const provenance = source.receipt === null ? {} : extractRunProvenance(source.receipt);
+		const receipt = authenticatedReceipt(source);
+		const provenance = receipt === null ? {} : extractRunProvenance(receipt);
 		rows.push({
 			kind: "run",
 			runId: envelope.id,
@@ -1258,7 +1277,9 @@ function renderTranscript(
 	overview: EvidenceOverview,
 	runSources: ReadonlyArray<EvidenceRunSource>,
 	sessionLinks: SessionLinkResult,
+	trustStatus: EvidenceTrustStatusFile,
 ): string {
+	const trustByRun = new Map(trustStatus.runs.map((entry) => [entry.runId, entry.status]));
 	const lines = [
 		`# Evidence ${overview.evidenceId}`,
 		"",
@@ -1274,10 +1295,11 @@ function renderTranscript(
 			`- ${envelope.id} status=${envelope.status} exit=${exitCode ?? "?"} agent=${envelope.agentId} target=${envelope.targetId}`,
 		);
 		lines.push(`  task: ${truncateText(envelope.task, MAX_TASK_CHARS)}`);
-		// Run section rendering reads provenance from the sealed receipt and adds
-		// sentences only for fields that are present.
+		// Run section rendering adds a sentence per provenance field the
+		// canonical projection admits; a rejected or retired seal adds none.
 		if (source.receipt !== null) {
-			for (const line of provenanceTranscriptLines(extractRunProvenance(source.receipt))) {
+			const status = trustByRun.get(envelope.id);
+			for (const line of provenanceTranscriptLines(extractRunProvenance(source.receipt), status)) {
 				lines.push(`  ${line}`);
 			}
 		}
@@ -1438,6 +1460,39 @@ function renderSessionTranscriptEntry(linked: LinkedSessionEntry): string[] {
 	return [`${prefix} ${String(_exhaustive)}`];
 }
 
+/** Independent review, context provenance, and completion evidence, read off the canonical axes. */
+function axisFindings(offset: number, source: EvidenceRunSource, status: CanonicalTrustStatus): EvidenceFinding[] {
+	const out: EvidenceFinding[] = [];
+	const runId = source.envelope.id;
+	const next = (severity: EvidenceSeverity, tag: EvidenceTag, message: string): void => {
+		out.push(finding(offset + out.length, severity, tag, runId, message));
+	};
+	const review = status.independentReview.state;
+	if (review === "failed") {
+		next("warn", "independent-review", "independent review failed for this run");
+	} else if (review === "not_independent") {
+		next("warn", "independent-review", "review was not independent of the run it judged");
+	} else if (review === "inconclusive") {
+		next("info", "independent-review", "independent review was inconclusive");
+	} else if (isSuccessfulRun(source) && (review === "absent" || review === "unknown")) {
+		next("info", "independent-review", "run was not independently reviewed; its result rests on its own receipt");
+	}
+	if (status.contextProvenance.state === "invalid") {
+		next("warn", "context-provenance", "context provenance record is contradictory; see trust-status.json");
+	}
+	const completion = status.completionEvidence.state;
+	if (completion === "incomplete") {
+		next(
+			"warn",
+			"completion-evidence",
+			"run mutated the workspace and finished without validation evidence at the completion boundary",
+		);
+	} else if (completion === "limited") {
+		next("info", "completion-evidence", "run finished with an explicit limitation instead of validation evidence");
+	}
+	return out;
+}
+
 function renderFindings(findings: ReadonlyArray<EvidenceFinding>): string {
 	if (findings.length === 0) return "# Findings\n\nNo findings.\n";
 	const lines = ["# Findings", ""];
@@ -1507,6 +1562,9 @@ async function readReceipt(
 	}
 	const integrity = inspectRunReceiptTrustStatus(receipt, envelope).integrity;
 	if (!integrity.ok) {
+		// A retired seal is set aside unread, the way a missing receipt is; it
+		// is not a failed one, and its reason already names both versions.
+		if (integrity.retired !== undefined) return { receipt, error: integrity.reason, integrityFailed: false };
 		return { receipt, error: `receipt integrity: ${integrity.reason}`, integrityFailed: true };
 	}
 	return { receipt, error: null, integrityFailed: false };

package/src/domains/evidence/index.ts

@@ -5,6 +5,7 @@ export { buildEvalEvidence, evalEvidenceId } from "./eval.js";
 export { FINISH_CONTRACT_EVIDENCE_TAGS, finishContractEvidenceTags } from "./finish-contract-map.js";
 export type { RunEscalationCounts, RunProvenanceView } from "./provenance.js";
 export {
+	admitRunProvenance,
 	extractRunProvenance,
 	hasRunProvenance,
 	PERSONA_HASH_PREFIX_CHARS,
@@ -22,6 +23,20 @@ export {
 	loadEvidenceRunProvenance,
 	loadEvidenceTrustStatus,
 } from "./store.js";
+export type { TrustSummaryProjection, TrustVerdict } from "./trust-projection.js";
+export {
+	formatTrustAxes,
+	formatTrustSummary,
+	formatTrustSummaryLine,
+	summarizeTrustStatus,
+	TRUST_STATE_WORDS,
+	TRUST_SUMMARY_MAX_REFS,
+	TRUST_SUMMARY_VERSION,
+	TRUST_VERDICTS,
+	trustStateWord,
+	trustSummaryReferences,
+	trustVerdict,
+} from "./trust-projection.js";
 export type {
 	AbsentTrustStatus,
 	AdaptFinishContractTrustOptions,
@@ -35,6 +50,8 @@ export type {
 	GroundedEvidenceValidationInput,
 	IndependentReviewStatus,
 	PersistedRunReceiptTrustFacts,
+	ReceiptIntegrityOutcome,
+	RetiredReceiptIntegrity,
 	RunReceiptTrustInspection,
 	TrustAbsenceReason,
 	TrustArtifactKind,
@@ -63,12 +80,16 @@ export {
 	inspectRunReceiptTrustStatus,
 	normalizeTrustStatus,
 	projectTrustStatus,
+	retiredIntegrityVersionOf,
+	retiredReceiptIntegrity,
+	retiredReceiptIntegrityReason,
 	TRUST_STATUS_AXES,
 	TRUST_STATUS_MAX_ARTIFACT_REFERENCES,
 	TRUST_STATUS_MAX_IDENTIFIER_LENGTH,
 	TRUST_STATUS_STATES,
 	TRUST_STATUS_VERSION,
 	validateTrustStatus,
+	verifyReceiptIntegrityOutcome,
 } from "./trust-status.js";
 export type {
 	EvidenceAuditLinkedRow,

package/src/domains/evidence/provenance.ts

@@ -4,6 +4,7 @@ import type {
 	RunReceipt,
 	RunReceiptAutonomyEnforcement,
 } from "../dispatch/types.js";
+import type { CanonicalTrustStatus } from "./trust-status.js";
 
 /**
  * Worker permission-escalation counters projected from sealed receipt
@@ -50,6 +51,33 @@ export function extractRunProvenance(receipt: ProvenanceReceipt): RunProvenanceV
 	return view;
 }
 
+/**
+ * The slice of a provenance view the canonical projection admits. The
+ * projection is the only thing that decides whether an axis is reported, so
+ * the autonomy detail (`autonomy=`, `mode=`, `dangerousBypass=`) is kept only
+ * when the projection reports the autonomy axis in a recorded state, and the
+ * non-axis provenance (pipeline, persona override, escalations) only when the
+ * seal that carries it verified. With no projection at hand no axis detail is
+ * admitted at all, whatever the receipt says; the non-axis provenance passes
+ * through for a caller that prints it beside its own integrity banner.
+ */
+export function admitRunProvenance(view: RunProvenanceView, status?: CanonicalTrustStatus): RunProvenanceView {
+	const admitted: RunProvenanceView = {};
+	if (status === undefined || status.artifactIntegrity.state === "verified") {
+		if (view.pipeline !== undefined) admitted.pipeline = view.pipeline;
+		if (view.personaOverride !== undefined) admitted.personaOverride = view.personaOverride;
+		if (view.escalation !== undefined) admitted.escalation = view.escalation;
+	}
+	const autonomy = status?.autonomyEnforcement.state;
+	if (
+		view.autonomyEnforcement !== undefined &&
+		(autonomy === "enforced" || autonomy === "approximated" || autonomy === "bypassed")
+	) {
+		admitted.autonomyEnforcement = view.autonomyEnforcement;
+	}
+	return admitted;
+}
+
 /** True when at least one provenance field set is present. */
 export function hasRunProvenance(view: RunProvenanceView): boolean {
 	return (
@@ -144,10 +172,15 @@ function formatPersonaHashPrefix(promptHash: string): string {
 }
 
 /**
- * Human transcript sentences for each present provenance field set.
- * An empty view adds no lines to the transcript run section.
+ * Human transcript sentences for each admitted provenance field set, gated
+ * through {@link admitRunProvenance}. The autonomy line carries only the
+ * detail the canonical projection does not: the policy name, the external
+ * mode, and the bypass flag. The axis itself (mediated, approximated,
+ * bypassed) is printed by the trust summary and nowhere else, so one
+ * autonomy fact never appears in two vocabularies on one screen.
  */
-export function provenanceTranscriptLines(view: RunProvenanceView): string[] {
+export function provenanceTranscriptLines(source: RunProvenanceView, status?: CanonicalTrustStatus): string[] {
+	const view = admitRunProvenance(source, status);
 	const lines: string[] = [];
 	if (view.pipeline !== undefined) {
 		const { fromRunId, position, inputBytes, inputTruncated } = view.pipeline;
@@ -163,20 +196,22 @@ export function provenanceTranscriptLines(view: RunProvenanceView): string[] {
 		lines.push(`escalations: ${requested} requested, ${approved} approved, ${denied} denied, ${timedOut} timed out`);
 	}
 	if (view.autonomyEnforcement !== undefined) {
-		const { grade, autonomy, externalMode, dangerousBypass } = view.autonomyEnforcement;
+		const { autonomy, externalMode, dangerousBypass } = view.autonomyEnforcement;
 		const mode = externalMode !== undefined ? ` mode=${externalMode}` : "";
 		const bypass = dangerousBypass === true ? " dangerousBypass=true" : "";
-		lines.push(`autonomy enforcement: ${grade} autonomy=${autonomy}${mode}${bypass}`);
+		lines.push(`autonomy: ${autonomy}${mode}${bypass}`);
 	}
 	return lines;
 }
 
 /**
- * Compact ` key=value` suffix for the dispatch tool's per-run and CLI lines.
- * The leading space lets callers append it directly. An empty provenance view
- * returns an empty string.
+ * Compact ` key=value` suffix for the dispatch tool's per-run and CLI lines,
+ * gated through {@link admitRunProvenance} on the same terms as the
+ * transcript lines. The leading space lets callers append it directly. An
+ * empty admitted view returns an empty string.
  */
-export function provenanceCompactSuffix(view: RunProvenanceView): string {
+export function provenanceCompactSuffix(source: RunProvenanceView, status?: CanonicalTrustStatus): string {
+	const view = admitRunProvenance(source, status);
 	const parts: string[] = [];
 	if (view.pipeline !== undefined) {
 		const { fromRunId, position, inputBytes, inputTruncated } = view.pipeline;
@@ -191,10 +226,10 @@ export function provenanceCompactSuffix(view: RunProvenanceView): string {
 		parts.push(`escalations=${requested}req/${approved}appr/${denied}deny/${timedOut}timeout`);
 	}
 	if (view.autonomyEnforcement !== undefined) {
-		const { grade, autonomy, externalMode, dangerousBypass } = view.autonomyEnforcement;
+		const { autonomy, externalMode, dangerousBypass } = view.autonomyEnforcement;
 		const mode = externalMode !== undefined ? `/${externalMode}` : "";
 		const bypass = dangerousBypass === true ? "/bypass" : "";
-		parts.push(`enforcement=${grade}:${autonomy}${mode}${bypass}`);
+		parts.push(`autonomy=${autonomy}${mode}${bypass}`);
 	}
 	return parts.length === 0 ? "" : ` ${parts.join(" ")}`;
 }