@iowarp/clio-coder 0.5.0 → 0.5.2

package/src/cli/skills-eval.ts

@@ -1,9 +1,9 @@
 import { spawn } from "node:child_process";
 import { createHash, randomBytes } from "node:crypto";
-import { existsSync } from "node:fs";
-import { cp, mkdir, mkdtemp, readdir, readFile, rm, stat, writeFile } from "node:fs/promises";
+import { existsSync, realpathSync } from "node:fs";
+import { cp, mkdir, mkdtemp, readdir, readFile, readlink, realpath, rm, stat, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
-import { join, resolve } from "node:path";
+import { isAbsolute, join, relative as relativePath, resolve } from "node:path";
 import { performance } from "node:perf_hooks";
 import { combineBashOutput, runBashCommand } from "../core/bash-exec.js";
 import { HEADLESS_PERMISSION_DENIED_MARKER } from "../core/headless-permission.js";
@@ -20,6 +20,7 @@ import {
 } from "../domains/eval/index.js";
 import { buildEvalEvidence } from "../domains/evidence/index.js";
 import {
+	checkSkillDrift,
 	discoverMarketplaceSkills,
 	loadSkills,
 	parseSkillEvals,
@@ -44,6 +45,22 @@ import { formatColumns, printError } from "./shared.js";
  * command lists, receipt-backed token/cost totals when headless main-agent
  * receipts are present, and per-bullet detail in a `skill-eval.json` sidecar
  * registered in the bundle's `overview.json` files list.
+ *
+ * Two things the sidecar records beyond the rubric result.
+ *
+ * `subject` names the exact artifact the run measured: the resolved base
+ * directory, how it was resolved, both hashes of the SKILL.md, and whether that
+ * content still matches whatever hash was recorded for it. A bundle that names
+ * only a skill and an evals.md cannot be compared against another bundle for
+ * the same skill, because nothing in either says whether the skill changed.
+ *
+ * `attribution` answers "did the skill change anything?" by scoring the
+ * baseline arm too. That arm was always executed and always thrown away: the
+ * treatment judge is told the baseline exists only for context. A second,
+ * isolated judge scores it against the same bullets, and the two verdict sets
+ * are paired per bullet. It is advisory in the strict sense: `pass`,
+ * `exitCode` and `failureClass` keep their treatment-only meaning, and no
+ * attribution value reaches the exit code. `--no-attribution` skips it.
  */
 
 const DEFAULT_RUN_TIMEOUT_MS = 600_000;
@@ -82,6 +99,14 @@ export interface SkillsEvalOptions {
 	 * fetch the open web is measuring something else as well.
 	 */
 	allowNetwork: boolean;
+	/**
+	 * Skip the baseline judge, and with it the paired attribution.
+	 *
+	 * Attribution costs one extra judge run per scenario. Turning it off is a
+	 * cost decision, not a result: a skipped comparison is recorded
+	 * `not-attempted` with the reason, never `no-change`.
+	 */
+	noAttribution: boolean;
 	scenario?: string;
 	target?: string;
 	timeoutSeconds?: number;
@@ -104,6 +129,18 @@ export interface ScoredBullet {
 	reason: string;
 }
 
+/**
+ * One skill activation this run actually performed, as the activation contract
+ * reported it. `hash` is the sha256 of the bytes the child read.
+ *
+ * @internal Exported for contract tests.
+ */
+export interface ObservedActivation {
+	name: string;
+	hash: string;
+	path: string;
+}
+
 /** Exported for contracts tests. */
 export interface CapturedRun {
 	sessionId: string | null;
@@ -113,6 +150,8 @@ export interface CapturedRun {
 	timedOut: boolean;
 	wallTimeMs: number;
 	stderr: string;
+	/** Skill loads this run completed successfully; empty when none did. */
+	activations: ObservedActivation[];
 }
 
 interface ScenarioUsage {
@@ -121,12 +160,141 @@ interface ScenarioUsage {
 	harness: EvalHarnessMetrics;
 }
 
+/**
+ * The exact artifact a run measured.
+ *
+ * A bundle used to name only its `evals.md` by hash, so two runs of the same
+ * skill could not be told apart when the SKILL.md between them had changed.
+ * Everything here is already computed elsewhere in the run: the loader hashes
+ * the file, `resolveSkillBaseDir` resolves which copy activation would pick,
+ * and `checkSkillDrift` compares it against whatever recorded hash speaks for
+ * it. Discarding all of it was the whole gap.
+ *
+ * @internal Exported for contract tests.
+ */
+export interface SkillEvalSubject {
+	name: string;
+	baseDir: string;
+	/** How the copy was resolved: `path`, `<source>/<scope>`, or `catalog`. */
+	origin: string;
+	/** sha256 of the SKILL.md exactly as read. */
+	sha256: string;
+	/** sha256 with install-lifecycle provenance stripped; this is what pins compare. */
+	normalizedHash: string;
+	/**
+	 * Digest over every file under `baseDir`, not just SKILL.md.
+	 *
+	 * A skill is a directory: the body can point at `references/` and scripts
+	 * the run will read. Naming the artifact by its SKILL.md hash alone would
+	 * claim an identity for content that hash does not cover, and would miss an
+	 * edit to a reference file entirely. Null when the tree could not be read.
+	 */
+	treeSha256: string | null;
+	/**
+	 * Whether the artifact could be snapshotted for the run. False when the body
+	 * carries package references, which resolve against an owning package root
+	 * above the skill directory: a copy of the directory alone would deliver
+	 * different instructions from the live path.
+	 */
+	pinnable: boolean;
+	evalsPath: string;
+	evalsSha256: string;
+	/** Null when nothing on this machine recorded a hash for the skill. */
+	drift: { verdict: "match" | "mismatch"; authority: string; expected: string } | null;
+}
+
+/**
+ * Whether the artifact on disk still matched {@link SkillEvalSubject} across
+ * one scenario's treatment arm.
+ *
+ * `verified` states exactly one thing: the tree digest taken immediately before
+ * the treatment arm and again immediately after both equalled the run-level
+ * subject. It is not proof of what the child activated. The treatment child
+ * loads the skill from the live source directory by path, so an edit landing
+ * between resolution and the arm, or between two scenarios, would otherwise be
+ * measured under the previous artifact's recorded identity. This check closes
+ * that window; it cannot see a change reverted entirely inside the arm, and the
+ * sidecar says so rather than implying activation was witnessed.
+ */
+export type SubjectVerification =
+	| "verified"
+	| "mismatch"
+	| "unreadable"
+	| "not-pinned"
+	| "not-activated"
+	| "activation-mismatch"
+	| "not-checked";
+
+/**
+ * Whether the skill changed the outcome for one bullet, relative to the same
+ * bullet in the baseline arm.
+ *
+ * `unmeasured` is not a comparison that came out even. It is the absence of a
+ * comparison, and it is returned whenever either arm failed to produce a
+ * verdict for that bullet.
+ *
+ * @internal Exported for contract tests.
+ */
+export type BulletAttribution = "helped" | "regressed" | "no-change-pass" | "no-change-fail" | "unmeasured";
+
+/**
+ * The scenario-level rollup.
+ *
+ * `not-attempted` and `unmeasured` are deliberately separate. The first means
+ * the comparison was never run, the second means it was impossible. Collapsing
+ * either into `no-change` would report "the skill made no difference" about a
+ * measurement that never happened.
+ *
+ * @internal Exported for contract tests.
+ */
+export type ScenarioAttributionVerdict =
+	| "helped"
+	| "regressed"
+	| "mixed"
+	| "no-change"
+	| "unmeasured"
+	| "not-attempted";
+
+/** @internal Exported for contract tests. */
+export interface AttributedBullet {
+	index: number;
+	text: string;
+	baseline: BulletVerdict;
+	treatment: BulletVerdict;
+	attribution: BulletAttribution;
+	/** Why this bullet is unmeasured; absent once both arms supplied evidence. */
+	reason?: string;
+}
+
+/** @internal Exported for contract tests. */
+export interface ScenarioAttribution {
+	verdict: ScenarioAttributionVerdict;
+	/** Why the verdict is `unmeasured` or `not-attempted`; null once a real comparison ran. */
+	reason: string | null;
+	bullets: AttributedBullet[];
+	counts: {
+		helped: number;
+		regressed: number;
+		noChangePass: number;
+		noChangeFail: number;
+		unmeasured: number;
+	};
+}
+
 interface ScenarioOutcome {
 	scenario: SkillEvalScenario;
 	bullets: ScoredBullet[];
 	baseline: CapturedRun | null;
 	treatment: CapturedRun | null;
 	judge: CapturedRun | null;
+	/** The isolated judge run that scored the baseline arm; null when none ran. */
+	baselineJudge: CapturedRun | null;
+	/** Whether the artifact on disk still matched the subject across this scenario. */
+	subjectVerification: SubjectVerification;
+	/** Tree digest observed after this scenario's treatment arm; null when unread. */
+	observedTreeSha256: string | null;
+	/** Advisory paired comparison. Never a gate, never folded into `pass`. */
+	attribution: ScenarioAttribution;
 	/** Seed workspace cloned for both arms; removed after the run, kept as a record. */
 	workspace: string;
 	wallTimeMs: number;
@@ -134,6 +302,211 @@ interface ScenarioOutcome {
 	usage: ScenarioUsage;
 }
 
+/**
+ * Did the skill change this bullet's outcome?
+ *
+ * Either arm failing to produce a verdict makes the pair unmeasurable, and that
+ * check comes first: a bullet the baseline judge never scored carries no
+ * information about the treatment, whatever the treatment did.
+ *
+ * @internal Exported for contract tests.
+ */
+export function deriveBulletAttribution(baseline: BulletVerdict, treatment: BulletVerdict): BulletAttribution {
+	if (baseline === "unmeasured" || baseline === "error") return "unmeasured";
+	if (treatment === "unmeasured" || treatment === "error") return "unmeasured";
+	if (baseline === "fail" && treatment === "pass") return "helped";
+	if (baseline === "pass" && treatment === "fail") return "regressed";
+	return baseline === "pass" ? "no-change-pass" : "no-change-fail";
+}
+
+/**
+ * Roll per-bullet attributions into one scenario verdict.
+ *
+ * Order matters and is deliberate. An unmeasured bullet poisons the scenario,
+ * because a rollup that ignored it would report a comparison over a subset
+ * while naming the whole. `mixed` outranks both single-direction verdicts, so a
+ * skill that fixed three bullets and broke one is never reported as simply
+ * having helped.
+ *
+ * @internal Exported for contract tests.
+ */
+export function deriveScenarioAttributionVerdict(
+	bullets: ReadonlyArray<AttributedBullet>,
+): Exclude<ScenarioAttributionVerdict, "not-attempted"> {
+	if (bullets.length === 0) return "unmeasured";
+	if (bullets.some((bullet) => bullet.attribution === "unmeasured")) return "unmeasured";
+	const helped = bullets.some((bullet) => bullet.attribution === "helped");
+	const regressed = bullets.some((bullet) => bullet.attribution === "regressed");
+	if (helped && regressed) return "mixed";
+	if (regressed) return "regressed";
+	if (helped) return "helped";
+	return "no-change";
+}
+
+function attributionCounts(bullets: ReadonlyArray<AttributedBullet>): ScenarioAttribution["counts"] {
+	const counts = { helped: 0, regressed: 0, noChangePass: 0, noChangeFail: 0, unmeasured: 0 };
+	for (const bullet of bullets) {
+		if (bullet.attribution === "helped") counts.helped += 1;
+		else if (bullet.attribution === "regressed") counts.regressed += 1;
+		else if (bullet.attribution === "no-change-pass") counts.noChangePass += 1;
+		else if (bullet.attribution === "no-change-fail") counts.noChangeFail += 1;
+		else counts.unmeasured += 1;
+	}
+	return counts;
+}
+
+/**
+ * Pair two arms' strict verdicts into a per-bullet comparison.
+ *
+ * Both sides have to have supplied unambiguous evidence for the same bullet.
+ * When either did not, the bullet is `unmeasured` and carries the reason it was
+ * refused, so a reader can tell "the judge omitted it" from "the judge answered
+ * with a string" from "the judge contradicted itself". The reason used to be
+ * dropped during pairing, which left an unmeasured scenario with nothing saying
+ * why.
+ *
+ * @internal Exported for contract tests.
+ */
+export function attributionFromStrict(
+	scenario: SkillEvalScenario,
+	baseline: StrictJudgeVerdicts,
+	treatment: StrictJudgeVerdicts,
+): ScenarioAttribution {
+	const verdictOf = (pass: boolean | undefined): BulletVerdict =>
+		pass === undefined ? "unmeasured" : pass ? "pass" : "fail";
+	const refusal = (side: string, strict: StrictJudgeVerdicts, index: number): string | null => {
+		if (strict.verdicts.has(index)) return null;
+		if (strict.absent !== null) return `${side} judge: ${strict.absent}`;
+		return `${side} judge: ${strict.rejected.get(index) ?? `no verdict for bullet ${index}`}`;
+	};
+	const bullets: AttributedBullet[] = scenario.expected.map((text, i) => {
+		const index = i + 1;
+		const baselinePass = baseline.verdicts.get(index);
+		const treatmentPass = treatment.verdicts.get(index);
+		const baselineVerdict = verdictOf(baselinePass);
+		const treatmentVerdict = verdictOf(treatmentPass);
+		const reason = refusal("baseline", baseline, index) ?? refusal("treatment", treatment, index);
+		return {
+			index,
+			text,
+			baseline: baselineVerdict,
+			treatment: treatmentVerdict,
+			attribution: deriveBulletAttribution(baselineVerdict, treatmentVerdict),
+			...(reason !== null ? { reason } : {}),
+		};
+	});
+	const verdict = deriveScenarioAttributionVerdict(bullets);
+	const firstReason = bullets.find((bullet) => bullet.reason !== undefined)?.reason ?? null;
+	return {
+		verdict,
+		reason: verdict === "unmeasured" ? firstReason : null,
+		bullets,
+		counts: attributionCounts(bullets),
+	};
+}
+
+/**
+ * A digest of the whole skill directory, path-sensitive and order-independent.
+ *
+ * Sorted relative paths are hashed alongside their contents, so adding, moving
+ * or removing a reference file changes the digest as surely as editing one
+ * does. Null on any read failure rather than a partial digest, because a digest
+ * over some of a tree would compare unequal for a reason that is not a change.
+ *
+ * @internal Exported for contract tests.
+ */
+export async function skillTreeDigest(baseDir: string): Promise<string | null> {
+	interface Entry {
+		relative: string;
+		kind: "file" | "symlink";
+		target?: string;
+	}
+	const root = resolve(baseDir);
+	const entries: Entry[] = [];
+	const seenDirs = new Set<string>();
+	const walk = async (dir: string): Promise<void> => {
+		// A directory symlink pointing at an ancestor would otherwise walk forever.
+		const real = await realpath(dir);
+		if (seenDirs.has(real)) throw new Error("cycle");
+		seenDirs.add(real);
+		const found = await readdir(dir, { withFileTypes: true });
+		for (const entry of found.sort((a, b) => a.name.localeCompare(b.name))) {
+			const full = join(dir, entry.name);
+			if (entry.isSymbolicLink()) {
+				// A link is part of the artifact's identity: retargeting one changes
+				// what the skill reads without changing any regular file. The link's
+				// own target string is hashed, and a link leaving the tree makes the
+				// artifact unverifiable rather than silently half-covered.
+				const target = await readlink(full);
+				const resolved = resolve(dir, target);
+				const relative = relativePath(root, resolved);
+				if (relative.startsWith("..") || isAbsolute(relative)) throw new Error("escaping symlink");
+				entries.push({ relative: full.slice(root.length), kind: "symlink", target });
+				continue;
+			}
+			if (entry.isDirectory()) await walk(full);
+			else if (entry.isFile()) entries.push({ relative: full.slice(root.length), kind: "file" });
+		}
+	};
+	try {
+		await walk(root);
+		const hash = createHash("sha256");
+		for (const entry of entries.sort((a, b) => a.relative.localeCompare(b.relative))) {
+			hash.update(entry.relative, "utf8");
+			hash.update("\0");
+			hash.update(entry.kind, "utf8");
+			hash.update("\0");
+			if (entry.kind === "symlink") hash.update(entry.target ?? "", "utf8");
+			else hash.update(await readFile(join(root, entry.relative)));
+			hash.update("\0");
+		}
+		return hash.digest("hex");
+	} catch {
+		return null;
+	}
+}
+
+/**
+ * Whether the artifact can be snapshotted for the run.
+ *
+ * The treatment arm runs against a private copy rather than the live source
+ * directory. Hashing the source before and after the arm catches an edit that
+ * persists and nothing else: an A to B to A change inside the arm leaves both
+ * observations equal while the child read B. A copy under a per-run temp root
+ * is not reachable from the source path, so an ordinary edit there cannot
+ * affect the run.
+ *
+ * What that is, precisely: an isolated snapshot, not an immutable pin. The
+ * copy sits in a writable temp directory, and the arm runs at full-auto, so a
+ * model that chose to edit the copy, read it, and restore it would not be
+ * detected. The harness has no mechanism that would make the copy read-only to
+ * its own child, and building one is out of scope here. These results are
+ * evidence about a cooperative model, which is the same caveat
+ * `materializeSkillEvalWorkspaces` already records about arm isolation.
+ *
+ * Returns false for the one case a copy would misrepresent: a body carrying
+ * package references. Those resolve against the owning package root, which
+ * sits above the skill directory and is not part of the copy, so a copied
+ * skill would deliver different instructions from the ones the live path
+ * delivers. A tree whose digest cannot be taken is a separate failure and is
+ * reported by `skillTreeDigest` returning null.
+ *
+ * @internal Exported for contract tests.
+ */
+export function artifactIsPinnable(skillBody: string): boolean {
+	return !skillBody.includes("${pluginRoot}") && !skillBody.includes("${component:");
+}
+
+/** No comparison was attempted. Records why, and never reads as `no-change`. */
+function notAttemptedAttribution(reason: string): ScenarioAttribution {
+	return { verdict: "not-attempted", reason, bullets: [], counts: attributionCounts([]) };
+}
+
+/** A comparison was impossible. Distinct from one that was never started. */
+function unmeasurableAttribution(reason: string): ScenarioAttribution {
+	return { verdict: "unmeasured", reason, bullets: [], counts: attributionCounts([]) };
+}
+
 async function runSkillsEvalCommand(nameOrPath: string, options: SkillsEvalOptions): Promise<number> {
 	const resolved = resolveSkillBaseDir(nameOrPath, process.cwd());
 	if (resolved.baseDir === null) {
@@ -164,6 +537,42 @@ async function runSkillsEvalCommand(nameOrPath: string, options: SkillsEvalOptio
 	for (const diagnostic of parsed.diagnostics) {
 		process.stderr.write(`clio-coder eval skill: ${diagnostic}\n`);
 	}
+	// Coherence check on the subject itself. `skill` was loaded, and its hashes
+	// taken, before the tree digest; an edit landing in that gap would produce a
+	// subject naming one SKILL.md revision and a tree containing another. The
+	// SKILL.md is re-read here and compared, so the recorded identity describes
+	// one state of the directory or admits it could not.
+	const subjectTree = await skillTreeDigest(resolved.baseDir);
+	const reread = await readFile(join(resolved.baseDir, "SKILL.md"), "utf8").catch(() => null);
+	const coherent = reread !== null && createHash("sha256").update(reread, "utf8").digest("hex") === skill.hash;
+	const pinnable = reread !== null && artifactIsPinnable(reread);
+	const driftReport = checkSkillDrift(skill, process.cwd());
+	const subject: SkillEvalSubject = {
+		name: skill.name,
+		baseDir: resolved.baseDir,
+		origin: resolved.origin ?? "unknown",
+		sha256: skill.hash,
+		normalizedHash: skill.normalizedHash,
+		treeSha256: coherent ? subjectTree : null,
+		pinnable,
+		evalsPath,
+		evalsSha256: createHash("sha256").update(evalsRaw, "utf8").digest("hex"),
+		drift:
+			driftReport === null
+				? null
+				: { verdict: driftReport.verdict, authority: driftReport.authority, expected: driftReport.expected },
+	};
+	// Said once, before any arm runs. A bundle measured against content that no
+	// longer matches its recorded form is evidence about a different artifact
+	// than the one it names, and the reader has to know that up front. It does
+	// not block: drift never gates activation either.
+	if (subject.drift?.verdict === "mismatch") {
+		process.stderr.write(
+			`clio-coder eval skill: WARNING skill_drift: ${skill.name} content (sha256 ${skill.normalizedHash.slice(0, 12)}…) ` +
+				`does not match the hash recorded for it by the ${subject.drift.authority} (expected ${subject.drift.expected.slice(0, 12)}…); ` +
+				"this run measures the copy on disk, not the recorded one\n",
+		);
+	}
 	const matcher = options.scenario === undefined ? null : scenarioMatcher(options.scenario);
 	if (options.scenario !== undefined && matcher === null) {
 		printError(`invalid --scenario "${options.scenario}": use a scenario id like S1 or a bare number`);
@@ -189,24 +598,39 @@ async function runSkillsEvalCommand(nameOrPath: string, options: SkillsEvalOptio
 	}
 	const startedAt = new Date().toISOString();
 	const outcomes: ScenarioOutcome[] = [];
+	const attributionEnabled = !options.noAttribution;
 	for (const scenario of scenarios) {
-		process.stderr.write(`clio-coder eval skill: ${skill.name} ${scenario.id} baseline/treatment/judge...\n`);
+		process.stderr.write(
+			`clio-coder eval skill: ${skill.name} ${scenario.id} baseline/treatment/judge${attributionEnabled ? "/baseline-judge" : ""}...\n`,
+		);
 		outcomes.push(
-			await runScenario(
-				skill.name,
-				resolved.baseDir,
+			await runScenario({
+				skillName: skill.name,
+				skillBaseDir: resolved.baseDir,
 				scenario,
-				options.target,
+				target: options.target,
 				timeoutMs,
 				workspaceOverride,
-				options.trustFixtures,
+				trustFixtures: options.trustFixtures,
 				childEnv,
-			),
+				attributionEnabled,
+				subjectTreeSha256: subject.treeSha256,
+				subjectSha256: subject.sha256,
+				pinnable: subject.pinnable,
+			}),
+		);
+	}
+	const unverified = outcomes.filter((outcome) => outcome.subjectVerification === "mismatch");
+	if (unverified.length > 0) {
+		process.stderr.write(
+			`clio-coder eval skill: WARNING subject_changed: the skill tree at ${resolved.baseDir} changed during ${unverified
+				.map((outcome) => outcome.scenario.id)
+				.join(", ")}; those scenarios' rubric results stand but their baseline comparison is unmeasured\n`,
 		);
 	}
 	const endedAt = new Date().toISOString();
 
-	const artifact = synthesizeArtifact(skill.name, evalsPath, evalsRaw, startedAt, endedAt, outcomes, options.target);
+	const artifact = synthesizeArtifact(subject, evalsPath, evalsRaw, startedAt, endedAt, outcomes, options.target);
 	let evidenceId: string | null = null;
 	let evidenceDirectory: string | null = null;
 	const evidenceErrors: string[] = [];
@@ -222,7 +646,7 @@ async function runSkillsEvalCommand(nameOrPath: string, options: SkillsEvalOptio
 		evidenceDirectory = built.directory;
 		await writeFile(
 			join(built.directory, SKILL_EVAL_SIDECAR),
-			`${JSON.stringify(sidecar(skill.name, artifact.evalId, outcomes, options.allowNetwork), null, 2)}\n`,
+			`${JSON.stringify(sidecar(subject, artifact.evalId, outcomes, options.allowNetwork, attributionEnabled), null, 2)}\n`,
 			"utf8",
 		);
 	} catch (error) {
@@ -234,23 +658,35 @@ async function runSkillsEvalCommand(nameOrPath: string, options: SkillsEvalOptio
 
 	if (options.json) {
 		for (const outcome of outcomes) {
+			const attributed = new Map(outcome.attribution.bullets.map((item) => [item.index, item]));
 			for (const bullet of outcome.bullets) {
+				const pair = attributed.get(bullet.index);
 				process.stdout.write(
 					`${JSON.stringify({
 						schema: "experimental",
 						kind: "skill-eval-bullet",
 						skill: skill.name,
+						skillSha256: subject.normalizedHash,
+						skillOrigin: subject.origin,
+						skillDrift: subject.drift?.verdict ?? null,
 						scenario: outcome.scenario.id,
 						title: outcome.scenario.title,
 						bullet: bullet.index,
 						expected: bullet.text,
 						verdict: bullet.verdict,
 						reason: bullet.reason,
+						// Advisory, never a gate: `verdict` above is the rubric result and
+						// is unchanged by anything here.
+						baselineVerdict: pair?.baseline ?? null,
+						attribution: pair?.attribution ?? null,
+						scenarioAttribution: outcome.attribution.verdict,
+						attributionReason: outcome.attribution.reason,
 						network: networkPolicyLabel(options.allowNetwork),
 						autonomy: ARM_AUTONOMY,
 						baselineSessionId: outcome.baseline?.sessionId ?? null,
 						treatmentSessionId: outcome.treatment?.sessionId ?? null,
 						judgeSessionId: outcome.judge?.sessionId ?? null,
+						baselineJudgeSessionId: outcome.baselineJudge?.sessionId ?? null,
 						evalId: artifact.evalId,
 						evidenceId,
 					})}\n`,
@@ -258,7 +694,7 @@ async function runSkillsEvalCommand(nameOrPath: string, options: SkillsEvalOptio
 			}
 		}
 	} else {
-		printHumanReport(skill.name, outcomes, artifact.evalId, evidenceId, evidenceDirectory, options.allowNetwork);
+		printHumanReport(subject, outcomes, artifact.evalId, evidenceId, evidenceDirectory, options.allowNetwork);
 	}
 	const anyFailure = outcomes.some((outcome) =>
 		outcome.bullets.some((bullet) => bullet.verdict === "fail" || bullet.verdict === "error"),
@@ -300,7 +736,16 @@ function describeArmPolicyOutcome(allowNetwork: boolean): string {
 	return `policy: the baseline and treatment arms ran at autonomy ${ARM_AUTONOMY}; ${network}`;
 }
 
-type EvalArm = "baseline" | "treatment" | "judge";
+type EvalArm = "baseline" | "treatment" | "judge" | "baseline-judge";
+
+/**
+ * A judge arm scores text and is told to call no tools, so granting it
+ * unattended write and exec buys nothing. Both judge arms are excluded from
+ * `--autonomy full-auto` for that reason.
+ */
+function isJudgeArm(arm: EvalArm): boolean {
+	return arm === "judge" || arm === "baseline-judge";
+}
 
 /**
  * The argv for one arm's child `clio-coder run`. Every arm streams the full JSON
@@ -318,7 +763,7 @@ function armRunArgs(
 	options: { target?: string | undefined; skillBaseDir?: string | undefined } = {},
 ): string[] {
 	const args = ["run", "--json", "--json-events", "full", "--no-skills"];
-	if (arm !== "judge") args.push("--autonomy", ARM_AUTONOMY);
+	if (!isJudgeArm(arm)) args.push("--autonomy", ARM_AUTONOMY);
 	if (options.skillBaseDir !== undefined) args.push("--skill", options.skillBaseDir);
 	if (options.target !== undefined) args.push("--target", options.target);
 	args.push(prompt);
@@ -432,22 +877,81 @@ async function resolveWorkspaceOverride(workspace: string | undefined): Promise<
 	}
 }
 
-async function runScenario(
-	skillName: string,
-	skillBaseDir: string,
-	scenario: SkillEvalScenario,
-	target: string | undefined,
+/**
+ * One arm's child process, behind a seam.
+ *
+ * The default is {@link captureHeadlessRun}, which spawns a real
+ * `clio-coder run`. Contract tests substitute a scripted runner so the arm
+ * sequencing, the skip gates and the recorded reasons are exercised through the
+ * production orchestration rather than by constructing already-correct typed
+ * data, and without paying for an inference.
+ *
+ * @internal
+ */
+export type SkillEvalArmRunner = (
+	args: ReadonlyArray<string>,
+	cwd: string,
 	timeoutMs: number,
-	workspaceOverride: string | null,
-	trustFixtures: boolean,
-	childEnv: NodeJS.ProcessEnv,
-): Promise<ScenarioOutcome> {
+	env: NodeJS.ProcessEnv,
+) => Promise<CapturedRun>;
+
+/** @internal Exported for contract tests. */
+export interface RunScenarioInput {
+	skillName: string;
+	skillBaseDir: string;
+	scenario: SkillEvalScenario;
+	target: string | undefined;
+	timeoutMs: number;
+	workspaceOverride: string | null;
+	trustFixtures: boolean;
+	childEnv: NodeJS.ProcessEnv;
+	attributionEnabled: boolean;
+	/** Tree digest recorded at resolution; the scenario re-checks against it. */
+	subjectTreeSha256: string | null;
+	/** sha256 of the subject's SKILL.md, compared against the activation receipt. */
+	subjectSha256: string;
+	/** False when the artifact cannot be snapshotted faithfully; see artifactIsPinnable. */
+	pinnable: boolean;
+	runner?: SkillEvalArmRunner;
+}
+
+/** @internal Exported for contract tests. */
+export async function runScenario(input: RunScenarioInput): Promise<ScenarioOutcome> {
+	const {
+		skillName,
+		skillBaseDir,
+		scenario,
+		target,
+		timeoutMs,
+		workspaceOverride,
+		trustFixtures,
+		childEnv,
+		attributionEnabled,
+		subjectTreeSha256,
+		subjectSha256,
+		pinnable,
+	} = input;
+	const runArm = input.runner ?? captureHeadlessRun;
 	// Published per-scenario figure: monotonic so a clock correction during a
 	// long sweep cannot land in one row's wall time.
 	const scenarioStart = performance.now();
 	const workspace = await mkdtemp(join(tmpdir(), "clio-coder-skill-eval-seed-"));
 	let runWorkspaces: MaterializedSkillEvalWorkspaces | null = null;
+	let subjectVerification: SubjectVerification = "not-checked";
+	let observedTreeSha256: string | null = null;
 	try {
+		// A scenario that ended before the comparison could run records why. When
+		// attribution is off the operator's own choice is the operative reason,
+		// because the baseline judge would have been skipped either way.
+		const skipAttribution = (reason: string): ScenarioAttribution =>
+			attributionEnabled ? unmeasurableAttribution(reason) : notAttemptedAttribution(ATTRIBUTION_DISABLED_REASON);
+		const outcomeBase = () => ({
+			workspace,
+			subjectVerification,
+			observedTreeSha256,
+			wallTimeMs: Math.round(performance.now() - scenarioStart),
+		});
+
 		if (workspaceOverride !== null) await copyWorkspace(workspaceOverride, workspace);
 		const fixtureError = await runFixtureCommands(scenario, workspace, timeoutMs, trustFixtures);
 		if (fixtureError !== null) {
@@ -457,24 +961,43 @@ async function runScenario(
 				baseline: null,
 				treatment: null,
 				judge: null,
-				workspace,
-				wallTimeMs: Math.round(performance.now() - scenarioStart),
+				baselineJudge: null,
+				attribution: skipAttribution(fixtureError),
+				...outcomeBase(),
 				infraError: fixtureError,
 			});
 		}
-		runWorkspaces = await materializeSkillEvalWorkspaces(workspace);
-		const baseline = await captureHeadlessRun(
+		runWorkspaces = await materializeSkillEvalWorkspaces(workspace, pinnable ? skillBaseDir : null);
+		const baseline = await runArm(
 			armRunArgs("baseline", scenario.setup, { target }),
 			runWorkspaces.baseline,
 			timeoutMs,
 			childEnv,
 		);
-		const treatment = await captureHeadlessRun(
-			armRunArgs("treatment", `/skill ${skillName} ${scenario.setup}`, { target, skillBaseDir }),
+		// The arm runs against a private copy, so an edit to the source tree
+		// cannot reach the child mid-run. The copy is still digested either side
+		// of the arm, so the claim rests on a measurement rather than on the
+		// assumption that nothing touched it.
+		const pinned = pinnable ? runWorkspaces.pin : null;
+		const armSkillDir = pinned ?? skillBaseDir;
+		const before = await skillTreeDigest(armSkillDir);
+		const treatment = await runArm(
+			armRunArgs("treatment", `/skill ${skillName} ${scenario.setup}`, { target, skillBaseDir: armSkillDir }),
 			runWorkspaces.treatment,
 			timeoutMs,
 			childEnv,
 		);
+		const after = await skillTreeDigest(armSkillDir);
+		observedTreeSha256 = after;
+		// Three conditions, in the order they can fail. The artifact must be
+		// pinnable, the pin must have held, and the child must have actually
+		// activated it. Disk equality alone is satisfied by a run that activated
+		// nothing, so it is necessary and not sufficient.
+		subjectVerification = !pinnable
+			? "not-pinned"
+			: verifySubjectTree(subjectTreeSha256, before, after) === "verified"
+				? verifyObservedActivation(skillName, subjectSha256, join(armSkillDir, "SKILL.md"), treatment.activations)
+				: verifySubjectTree(subjectTreeSha256, before, after);
 		const infra = runInfraError("baseline", baseline) ?? runInfraError("treatment", treatment);
 		if (infra !== null) {
 			return await completeScenarioOutcome({
@@ -483,8 +1006,9 @@ async function runScenario(
 				baseline,
 				treatment,
 				judge: null,
-				workspace,
-				wallTimeMs: Math.round(performance.now() - scenarioStart),
+				baselineJudge: null,
+				attribution: skipAttribution(infra),
+				...outcomeBase(),
 				infraError: infra,
 			});
 		}
@@ -499,8 +1023,9 @@ async function runScenario(
 				baseline,
 				treatment,
 				judge: null,
-				workspace,
-				wallTimeMs: Math.round(performance.now() - scenarioStart),
+				baselineJudge: null,
+				attribution: skipAttribution(wall),
+				...outcomeBase(),
 				infraError: wall,
 			});
 		}
@@ -508,7 +1033,7 @@ async function runScenario(
 		// terminating tool (artifact plan/review/report) prints only that tool's
 		// result line in text mode, while the event stream carries the artifact
 		// content the verdict may live in.
-		const judge = await captureHeadlessRun(
+		const judge = await runArm(
 			armRunArgs("judge", judgePrompt(scenario, baseline.transcript, treatment.transcript), { target }),
 			runWorkspaces.judge,
 			timeoutMs,
@@ -522,20 +1047,38 @@ async function runScenario(
 				baseline,
 				treatment,
 				judge,
-				workspace,
-				wallTimeMs: Math.round(performance.now() - scenarioStart),
+				baselineJudge: null,
+				attribution: skipAttribution(judgeInfra),
+				...outcomeBase(),
 				infraError: judgeInfra,
 			});
 		}
 		const bullets = parseJudgeVerdicts(scenario, judge);
+		// The rubric result above stands whatever the comparison decides. The
+		// comparison, unlike the rubric, is refused when the artifact measured is
+		// not demonstrably the artifact the subject names.
+		const attributed = await attributeScenario({
+			scenario,
+			bullets,
+			baseline,
+			judge,
+			target,
+			timeoutMs,
+			childEnv,
+			attributionEnabled,
+			subjectVerification,
+			workspace: runWorkspaces.baselineJudge,
+			runner: runArm,
+		});
 		return await completeScenarioOutcome({
 			scenario,
 			bullets,
 			baseline,
 			treatment,
 			judge,
-			workspace,
-			wallTimeMs: Math.round(performance.now() - scenarioStart),
+			baselineJudge: attributed.baselineJudge,
+			attribution: attributed.attribution,
+			...outcomeBase(),
 			infraError: null,
 		});
 	} finally {
@@ -547,10 +1090,189 @@ async function runScenario(
 	}
 }
 
+/** The exact reason recorded when the operator turned the comparison off. */
+const ATTRIBUTION_DISABLED_REASON = "disabled by --no-attribution";
+
+/**
+ * Compare the tree digests taken either side of the treatment arm against the
+ * one the subject was resolved from.
+ *
+ * All three must agree. `verified` therefore means "the artifact on disk was
+ * the subject's artifact before the arm and still was after it", which is a
+ * narrower claim than "the child activated the subject", and the sidecar states
+ * that distinction rather than letting the word imply more.
+ *
+ * @internal Exported for contract tests.
+ */
+export function verifySubjectTree(
+	subject: string | null,
+	before: string | null,
+	after: string | null,
+): SubjectVerification {
+	if (subject === null || before === null || after === null) return "unreadable";
+	return subject === before && before === after ? "verified" : "mismatch";
+}
+
+/**
+ * Did the treatment child actually activate the artifact the subject names?
+ *
+ * Disk equality answers a different question. It says the directory looked the
+ * same either side of the arm, which is true of a run that activated nothing at
+ * all, and true of a run that loaded a revision written and reverted inside the
+ * arm. Neither is a measurement of the subject, and reporting `helped` about
+ * either would attribute a difference to an artifact that was never read.
+ *
+ * So the positive evidence is the activation receipt: the child reports the
+ * sha256 of the bytes it read, and that has to equal the pinned SKILL.md's.
+ * Several activations of the same name are ambiguous and are refused rather
+ * than resolved by picking one.
+ *
+ * @internal Exported for contract tests.
+ */
+export function verifyObservedActivation(
+	skillName: string,
+	expectedSha256: string,
+	expectedPath: string,
+	activations: ReadonlyArray<ObservedActivation>,
+): SubjectVerification {
+	const matching = activations.filter((entry) => entry.name === skillName);
+	if (matching.length === 0) return "not-activated";
+	if (matching.length > 1 && new Set(matching.map((entry) => `${entry.hash}|${entry.path}`)).size > 1) {
+		return "activation-mismatch";
+	}
+	const observed = matching[0];
+	if (observed === undefined || observed.hash !== expectedSha256) return "activation-mismatch";
+	// The hash says which bytes; the path says which copy. Two directories can
+	// hold the same SKILL.md and different reference files beside it, and the
+	// body's own pointers resolve against the directory it was loaded from, so a
+	// matching hash from somewhere else is not the artifact under test.
+	return samePath(observed.path, expectedPath) ? "verified" : "activation-mismatch";
+}
+
+/** Compare two paths as the filesystem resolves them, falling back to lexical. */
+function samePath(left: string, right: string): boolean {
+	if (left.length === 0 || right.length === 0) return false;
+	try {
+		return realpathSync.native(left) === realpathSync.native(right);
+	} catch {
+		return resolve(left) === resolve(right);
+	}
+}
+
+/** Why the comparison was refused, named after what was not established. */
+const SUBJECT_REFUSAL: Record<Exclude<SubjectVerification, "verified">, string> = {
+	mismatch:
+		"the measured skill tree changed on disk during this scenario, so its two arms did not run against one artifact",
+	unreadable: "the measured skill tree could not be re-read, so the artifact under test is unverified",
+	"not-pinned":
+		"the skill body carries package references, which resolve against an owning package root outside the skill directory, so the artifact could not be copied faithfully for this run",
+	"not-activated":
+		"the treatment arm recorded no successful activation of this skill, so nothing establishes that the measured artifact ran",
+	"activation-mismatch":
+		"the treatment arm activated content whose hash is not the pinned artifact's, so the comparison would name the wrong revision",
+	"not-checked": "the artifact identity was never checked for this scenario",
+};
+
+interface AttributeScenarioInput {
+	scenario: SkillEvalScenario;
+	/** Treatment verdicts from the legacy rubric parser, used only as a skip gate. */
+	bullets: ReadonlyArray<ScoredBullet>;
+	baseline: CapturedRun;
+	/** The treatment judge run, re-read under strict rules for the comparison. */
+	judge: CapturedRun;
+	target: string | undefined;
+	timeoutMs: number;
+	childEnv: NodeJS.ProcessEnv;
+	attributionEnabled: boolean;
+	subjectVerification: SubjectVerification;
+	workspace: string;
+	runner: SkillEvalArmRunner;
+}
+
+/**
+ * Score the baseline arm and pair it with the treatment, or say why not.
+ *
+ * The baseline run is already paid for by the time this is reached; only the
+ * judge that reads it is new. Five states end the comparison before it produces
+ * a verdict, and each is recorded with its own reason rather than collapsed
+ * into one: the operator disabled it, the artifact measured was not
+ * demonstrably the artifact the subject names, the treatment produced nothing
+ * to compare against, the baseline judge failed to run, or the baseline judge
+ * hit the harness's own permission wall.
+ *
+ * The verdicts it pairs come from {@link strictJudgeVerdicts}, not from the
+ * rubric parser, so a judge that answered with a missing, null or string `pass`
+ * contributes nothing instead of contributing an invented `fail`.
+ */
+async function attributeScenario(
+	input: AttributeScenarioInput,
+): Promise<{ attribution: ScenarioAttribution; baselineJudge: CapturedRun | null }> {
+	if (!input.attributionEnabled) {
+		return { attribution: notAttemptedAttribution(ATTRIBUTION_DISABLED_REASON), baselineJudge: null };
+	}
+	// Nothing establishes which artifact ran, so nothing can be attributed to
+	// one. Each refusal names what was not established rather than collapsing
+	// into one verdict.
+	if (input.subjectVerification !== "verified") {
+		return { attribution: unmeasurableAttribution(SUBJECT_REFUSAL[input.subjectVerification]), baselineJudge: null };
+	}
+	// The same comparative eligibility the baseline judge gets. A treatment judge
+	// that collected the harness's own denial and then answered was scoring under
+	// a constraint the other arm did not have. The rubric result it produced
+	// stands; only the comparison is refused.
+	const treatmentWall = permissionWallReason("judge", input.judge);
+	if (treatmentWall !== null) return { attribution: unmeasurableAttribution(treatmentWall), baselineJudge: null };
+	// Nothing on the treatment side was scored, so no pair can be formed. Running
+	// the baseline judge anyway would spend an inference to learn nothing.
+	if (!input.bullets.some((bullet) => bullet.verdict === "pass" || bullet.verdict === "fail")) {
+		return {
+			attribution: unmeasurableAttribution(
+				"the treatment arm produced no scored bullet, so there is nothing to compare a baseline against",
+			),
+			baselineJudge: null,
+		};
+	}
+	const baselineJudge = await input.runner(
+		armRunArgs("baseline-judge", baselineJudgePrompt(input.scenario, input.baseline.transcript), {
+			target: input.target,
+		}),
+		input.workspace,
+		input.timeoutMs,
+		input.childEnv,
+	);
+	const infra = runInfraError("baseline-judge", baselineJudge);
+	if (infra !== null) {
+		// The treatment verdicts stand: this failure is about the comparison, not
+		// about the skill, so it never touches `bullets` or the exit code.
+		return { attribution: unmeasurableAttribution(infra), baselineJudge };
+	}
+	// The prompt tells the judge to use no tools, which is an instruction and not
+	// enforcement. A judge that tried anyway and collected the harness's denial
+	// was scoring under a constraint the treatment judge did not have.
+	const wall = permissionWallReason("baseline-judge", baselineJudge);
+	if (wall !== null) return { attribution: unmeasurableAttribution(wall), baselineJudge };
+	return {
+		attribution: attributionFromStrict(
+			input.scenario,
+			strictJudgeVerdicts(input.scenario, baselineJudge),
+			strictJudgeVerdicts(input.scenario, input.judge),
+		),
+		baselineJudge,
+	};
+}
+
 export interface MaterializedSkillEvalWorkspaces {
 	baseline: string;
 	treatment: string;
 	judge: string;
+	/** The baseline judge gets its own root for the same reason the other arms do. */
+	baselineJudge: string;
+	/**
+	 * Private per-run copy of the artifact the treatment arm loads; null when the
+	 * artifact cannot be copied faithfully. Isolated from the source directory,
+	 * not read-only to the arm itself.
+	 */
+	pin: string | null;
 	cleanup(): Promise<void>;
 }
 
@@ -581,17 +1303,33 @@ async function armWorkspace(created: string[]): Promise<string> {
 }
 
 /** @internal Exported for contract tests. */
-async function materializeSkillEvalWorkspaces(seedWorkspace: string): Promise<MaterializedSkillEvalWorkspaces> {
+async function materializeSkillEvalWorkspaces(
+	seedWorkspace: string,
+	pinSource: string | null,
+): Promise<MaterializedSkillEvalWorkspaces> {
 	const created: string[] = [];
 	try {
 		const baseline = await armWorkspace(created);
 		const treatment = await armWorkspace(created);
 		const judge = await armWorkspace(created);
+		const baselineJudge = await armWorkspace(created);
+		let pin: string | null = null;
+		if (pinSource !== null) {
+			const root = await mkdtemp(join(tmpdir(), "clio-coder-skill-eval-pin-"));
+			created.push(root);
+			pin = join(root, "skill");
+			await cp(pinSource, pin, { recursive: true, preserveTimestamps: true, verbatimSymlinks: true });
+		}
+		// Only the acting arms get the fixture. A judge scores text and is told to
+		// call no tools, so seeding its workspace would only give it the artifacts
+		// it is supposed to read about.
 		await Promise.all([copyWorkspace(seedWorkspace, baseline), copyWorkspace(seedWorkspace, treatment)]);
 		return {
 			baseline,
 			treatment,
 			judge,
+			baselineJudge,
+			pin,
 			cleanup: async () => {
 				await Promise.all(created.map((path) => rm(path, { recursive: true, force: true })));
 			},
@@ -613,7 +1351,7 @@ async function copyWorkspace(source: string, destination: string): Promise<void>
 async function completeScenarioOutcome(outcome: Omit<ScenarioOutcome, "usage">): Promise<ScenarioOutcome> {
 	return {
 		...outcome,
-		usage: await usageForCapturedRuns([outcome.baseline, outcome.treatment, outcome.judge]),
+		usage: await usageForCapturedRuns([outcome.baseline, outcome.treatment, outcome.judge, outcome.baselineJudge]),
 	};
 }
 
@@ -816,12 +1554,36 @@ function loadsSkillBody(tool: string, args: unknown): boolean {
 	return args.scope === "skills" && typeof args.name === "string" && args.name.trim().length > 0;
 }
 
+/**
+ * The activation contract inside a successful skill load's result details.
+ *
+ * `runSkillsScope` records `name`, `filePath` and `hash` on every activation,
+ * and the agent loop puts the tool result's details on the execution-end event.
+ * Anything missing either field is not an activation receipt and is ignored
+ * rather than half-read.
+ */
+function activationFromResult(result: unknown): ObservedActivation | null {
+	if (!isRecord(result)) return null;
+	const details = isRecord(result.details) ? result.details : null;
+	if (details === null) return null;
+	const name = readString(details.name);
+	const hash = readString(details.hash);
+	if (name === null || hash === null) return null;
+	return { name, hash, path: readString(details.filePath) ?? readString(details.path) ?? "" };
+}
+
 /** @internal Exported for contract tests. */
-function parseRunStdout(stdout: string): { sessionId: string | null; transcript: string; finalText: string } {
+export function parseRunStdout(stdout: string): {
+	sessionId: string | null;
+	transcript: string;
+	finalText: string;
+	activations: ObservedActivation[];
+} {
 	let sessionId: string | null = null;
 	const lines: string[] = [];
 	let finalText = "";
 	let sawJson = false;
+	const activations: ObservedActivation[] = [];
 	const streamedText = new Map<number, string>();
 	// Tool calls whose result is the skill's own SKILL.md. Correlated by
 	// toolCallId, which both the start and end events carry.
@@ -867,6 +1629,15 @@ function parseRunStdout(stdout: string): { sessionId: string | null; transcript:
 			const tool = readString(event.toolName) ?? readString(event.tool) ?? "tool";
 			const status = event.isError === true ? "error" : "ok";
 			const callId = readString(event.toolCallId);
+			// A successful skill load carries the activation contract in its result
+			// details: the name, the file and the sha256 of the bytes the child
+			// actually read. That is the only evidence in this stream about which
+			// artifact ran, and it is collected here and kept out of the transcript
+			// so the judge still never sees the body or its identity.
+			if (event.isError !== true && callId !== null && skillBodyCallIds.has(callId)) {
+				const activation = activationFromResult(event.result);
+				if (activation !== null) activations.push(activation);
+			}
 			// The skill body is the instructions, not the behavior. Left in the
 			// transcript it is the easiest thing in the run for a judge to quote,
 			// and a 30B judge scored bullets as passing from SKILL.md prose that
@@ -903,9 +1674,9 @@ function parseRunStdout(stdout: string): { sessionId: string | null; transcript:
 	}
 	if (!sawJson) {
 		const text = stdout.trim();
-		return { sessionId: null, transcript: text, finalText: text };
+		return { sessionId: null, transcript: text, finalText: text, activations: [] };
 	}
-	return { sessionId, transcript: elide(lines.join("\n")), finalText };
+	return { sessionId, transcript: elide(lines.join("\n")), finalText, activations };
 }
 
 function judgePrompt(scenario: SkillEvalScenario, baselineTranscript: string, treatmentTranscript: string): string {
@@ -932,6 +1703,41 @@ function judgePrompt(scenario: SkillEvalScenario, baselineTranscript: string, tr
 	].join("\n");
 }
 
+/**
+ * Score the baseline arm alone, against the same bullets.
+ *
+ * Isolated on purpose. The treatment judge above sees both transcripts and is
+ * told to score only the treatment; giving the baseline judge the treatment
+ * transcript as well would let a strong treatment run colour the baseline's
+ * verdicts, and the comparison those verdicts feed would then be measuring the
+ * judge. The bullet contract, the strict-JSON shape and the no-tools rule are
+ * identical, so `parseJudgeVerdicts` reads both without branching.
+ *
+ * The treatment prompt is deliberately left byte-identical to what it was
+ * before attribution existed, so treatment verdicts stay comparable with every
+ * bundle already on disk.
+ */
+function baselineJudgePrompt(scenario: SkillEvalScenario, baselineTranscript: string): string {
+	const bullets = scenario.expected.map((text, index) => `${index + 1}. ${text}`).join("\n");
+	return [
+		"You are scoring an agent run. One transcript follows: it ran WITHOUT any skill loaded.",
+		"Score each EXPECTED bullet strictly against the TRANSCRIPT.",
+		"A bullet passes only if the transcript observably satisfies it; anything unverifiable from the transcript fails.",
+		'Reply with STRICT JSON only, no prose and no code fences, exactly: {"bullets":[{"index":1,"pass":true,"reason":"<= 25 words"}]}',
+		`Include one entry per bullet, indexes 1 through ${scenario.expected.length} in order.`,
+		"Do not use any tools. Respond with the JSON verdict directly.",
+		"",
+		`SCENARIO ${scenario.id} - ${scenario.title}`,
+		`SETUP: ${scenario.setup}`,
+		"",
+		"EXPECTED BULLETS:",
+		bullets,
+		"",
+		"TRANSCRIPT:",
+		baselineTranscript.length > 0 ? baselineTranscript : "(empty)",
+	].join("\n");
+}
+
 /**
  * Why a judge response carried no verdict, named after the thing that failed.
  * A response that opened a bullets object and never closed it is the observed
@@ -984,6 +1790,85 @@ function parseJudgeVerdicts(scenario: SkillEvalScenario, judge: CapturedRun): Sc
 	});
 }
 
+/**
+ * Verdicts strict enough to compare two arms against each other.
+ *
+ * {@link parseJudgeVerdicts} above is the legacy rubric gate and its coercions
+ * are load-bearing for `pass` / `exitCode` / `failureClass`, so it is left
+ * exactly as it is. It is, however, forgiving in ways that are fine for a
+ * one-armed verdict and wrong for a comparison: `pass: item.pass === true`
+ * turns a missing, null or string field into a genuine `fail`, and
+ * `Number.parseInt(String(item.index))` reads `"1oops"` as bullet 1. Pairing a
+ * real treatment `pass` against an invented baseline `fail` reports `helped`
+ * about a measurement the judge never made.
+ *
+ * So comparison reads the raw judge output again under stricter rules, and
+ * anything that does not clear them stays out of the comparison rather than
+ * entering it as a verdict. Rejecting evidence here cannot change the rubric
+ * result: the two parsers have separate callers on purpose.
+ *
+ * @internal Exported for contract tests.
+ */
+export interface StrictJudgeVerdicts {
+	/** Bullet index to its unambiguous boolean verdict. */
+	verdicts: Map<number, boolean>;
+	/** Bullet index to why its evidence was refused. */
+	rejected: Map<number, string>;
+	/** Set when the run produced no parseable verdict object at all. */
+	absent: string | null;
+}
+
+/** @internal Exported for contract tests. */
+export function strictJudgeVerdicts(scenario: SkillEvalScenario, judge: CapturedRun): StrictJudgeVerdicts {
+	const verdicts = new Map<number, boolean>();
+	const rejected = new Map<number, string>();
+	const parsed = extractBulletsObject(judge.finalText) ?? extractBulletsObject(judge.transcript);
+	if (parsed === null) return { verdicts, rejected, absent: judgeVerdictAbsenceReason(judge) };
+	if (!Array.isArray(parsed.bullets)) {
+		return { verdicts, rejected, absent: "judge verdict object carried no bullets array" };
+	}
+	const count = scenario.expected.length;
+	const seen = new Map<number, boolean>();
+	for (const item of parsed.bullets) {
+		if (!isRecord(item)) continue;
+		// A numeric index and nothing else. A string that happens to start with
+		// digits is not an index the judge chose; it is a parse accident.
+		const index = item.index;
+		if (typeof index !== "number" || !Number.isInteger(index) || index < 1 || index > count) {
+			continue;
+		}
+		// A rejection for an index is final and order-independent. An entry that
+		// arrives after a valid one still poisons it: the judge emitted two
+		// answers for one bullet and only one of them is usable, so which one it
+		// "meant" is a guess, and a guess is not comparison evidence.
+		if (typeof item.pass !== "boolean") {
+			rejected.set(index, `judge gave no boolean pass for bullet ${index}: comparison evidence refused`);
+			verdicts.delete(index);
+			continue;
+		}
+		if (rejected.has(index)) {
+			verdicts.delete(index);
+			continue;
+		}
+		const previous = seen.get(index);
+		if (previous !== undefined && previous !== item.pass) {
+			// Two contradictory verdicts for one bullet. Taking either would pick a
+			// winner the judge never picked.
+			rejected.set(index, `judge gave contradictory verdicts for bullet ${index}: comparison evidence refused`);
+			verdicts.delete(index);
+			continue;
+		}
+		seen.set(index, item.pass);
+		verdicts.set(index, item.pass);
+	}
+	for (let index = 1; index <= count; index += 1) {
+		if (!verdicts.has(index) && !rejected.has(index)) {
+			rejected.set(index, `judge omitted bullet ${index}: comparison evidence absent`);
+		}
+	}
+	return { verdicts, rejected, absent: null };
+}
+
 /**
  * Find the JSON object carrying the judge's `bullets` array anywhere in the
  * text: models wrap verdicts in prose, code fences, or terminating tool
@@ -1043,7 +1928,7 @@ function balancedJsonSlice(text: string, start: number): string | null {
 }
 
 function synthesizeArtifact(
-	skillName: string,
+	subject: SkillEvalSubject,
 	evalsPath: string,
 	evalsRaw: string,
 	startedAt: string,
@@ -1051,6 +1936,7 @@ function synthesizeArtifact(
 	outcomes: ReadonlyArray<ScenarioOutcome>,
 	target: string | undefined,
 ): EvalRunArtifact {
+	const skillName = subject.name;
 	const contentHash = createHash("sha256").update(evalsRaw, "utf8").digest("hex");
 	const stamp = startedAt.replace(/[-:.]/g, "");
 	// Random suffix for the same reason createEvalId carries one: the stamp plus
@@ -1070,6 +1956,14 @@ function synthesizeArtifact(
 			tags: [
 				"skill-eval",
 				`skill:${skillName}`,
+				// The artifact identity, on every record. Two bundles for one skill
+				// name are only comparable when both say which content they ran.
+				`skill-sha:${subject.normalizedHash.slice(0, 12)}`,
+				`skill-origin:${subject.origin}`,
+				...(subject.drift !== null ? [`skill-drift:${subject.drift.verdict}`] : []),
+				// Advisory. `pass` and `exitCode` below keep their treatment-only
+				// meaning; this tag is read by nothing that gates.
+				`attribution:${outcome.attribution.verdict}`,
 				...(unmeasured ? ["scenario:unmeasured"] : []),
 				...outcome.bullets.map((bullet) => `bullet-${bullet.index}:${bullet.verdict}`),
 			],
@@ -1107,25 +2001,46 @@ function synthesizeArtifact(
 	};
 }
 
-function sidecar(
-	skillName: string,
+/** @internal Exported for contract tests. */
+export function sidecar(
+	subject: SkillEvalSubject,
 	evalId: string,
 	outcomes: ReadonlyArray<ScenarioOutcome>,
 	allowNetwork: boolean,
+	attributionEnabled: boolean,
 ): unknown {
 	return {
-		version: 1,
+		version: 2,
 		schema: "experimental",
 		kind: "skill-eval",
-		skill: skillName,
+		skill: subject.name,
+		// What this run measured, by identity rather than by name. `version` moved
+		// to 2 for this block; readers of version 1 keep working, they just never
+		// learn which copy produced their numbers.
+		subject,
 		evalId,
 		network: networkPolicyLabel(allowNetwork),
 		autonomy: ARM_AUTONOMY,
+		attributionEnabled,
+		attributionSummary: attributionSummary(outcomes),
 		deltas: [
 			"bullet verdicts are judge-scored from run transcripts, not command exit codes; the evals-domain artifact carries scenario-level records with empty command lists",
 			"a bullet the judge never scored is recorded unmeasured, not failed: its scenario record is pass:false with exitCode 3 and no failureClass",
 			"an arm whose transcript carries the headless permission wall is recorded unmeasured with an infraError: the harness's own gate is not a verdict about the skill",
 			"tokens and cost are rolled up from headless main-agent receipts when those receipts are present",
+			"attribution is advisory and gates nothing: pass, exitCode and failureClass keep their treatment-only meaning",
+			"attribution unmeasured means the comparison was impossible; not-attempted means it was never run. Neither is no-change",
+			"the baseline judge scores the baseline transcript alone; the treatment judge prompt is unchanged from version 1 bundles",
+			"comparison reads both judges' raw output under strict rules (boolean pass, integer in-range index, no contradictory duplicate); refused evidence stays unmeasured and never becomes a verdict",
+			"the rubric bullets above keep the legacy parser's forgiving coercions: tightening comparison eligibility does not move the pass/exitCode gate",
+			"the treatment arm runs against a private per-run copy of the skill directory, so an edit to the source during the arm cannot reach the child; subject.pinnable is false when the body carries package references, which resolve above the skill directory and cannot be snapshotted faithfully",
+			"that copy is an isolated snapshot, not an immutable pin: it lives in a writable temp directory and the arm runs at full-auto, so a model that edited the copy, read it and restored it would not be detected. These are results about a cooperative model, the same caveat the arm workspaces already carry",
+			"subjectVerification is verified only when the snapshot held (tree digest equal before and after the arm) AND the treatment arm reported a successful activation whose sha256 equals subject.sha256 and whose file is the snapshot's own SKILL.md by canonical path. not-activated, activation-mismatch, not-pinned, mismatch and unreadable each name what was not established",
+			"a scenario whose subjectVerification is not verified keeps its rubric result and records its comparison unmeasured",
+			"treeSha256 covers regular files and in-tree symlink targets; a symlink leaving the directory makes the tree unverifiable rather than partially hashed",
+			"both judges are checked for the headless permission wall before their verdicts are eligible for comparison; the legacy rubric result is unaffected either way",
+			"subject.drift records whether the measured content still matches its recorded hash; a mismatch never blocks the run",
+			"subject hashes cover SKILL.md (sha256, normalizedHash) and the whole skill directory (treeSha256); neither covers resources the skill reads from outside its own directory",
 			"this sidecar is additive and is registered in overview.json files[]",
 		],
 		scenarios: outcomes.map((outcome) => ({
@@ -1136,13 +2051,31 @@ function sidecar(
 			infraError: outcome.infraError,
 			usage: outcome.usage,
 			bullets: outcome.bullets,
+			subjectVerification: outcome.subjectVerification,
+			observedTreeSha256: outcome.observedTreeSha256,
+			attribution: outcome.attribution,
 			baseline: sidecarRun(outcome.baseline),
 			treatment: sidecarRun(outcome.treatment),
 			judge: sidecarRun(outcome.judge),
+			baselineJudge: sidecarRun(outcome.baselineJudge),
 		})),
 	};
 }
 
+/** Scenario counts per attribution verdict. A rollup, never collapsed to one score. */
+function attributionSummary(outcomes: ReadonlyArray<ScenarioOutcome>): Record<ScenarioAttributionVerdict, number> {
+	const summary: Record<ScenarioAttributionVerdict, number> = {
+		helped: 0,
+		regressed: 0,
+		mixed: 0,
+		"no-change": 0,
+		unmeasured: 0,
+		"not-attempted": 0,
+	};
+	for (const outcome of outcomes) summary[outcome.attribution.verdict] += 1;
+	return summary;
+}
+
 function sidecarRun(run: CapturedRun | null): unknown {
 	if (run === null) return null;
 	return {
@@ -1156,17 +2089,27 @@ function sidecarRun(run: CapturedRun | null): unknown {
 }
 
 function printHumanReport(
-	skillName: string,
+	subject: SkillEvalSubject,
 	outcomes: ReadonlyArray<ScenarioOutcome>,
 	evalId: string,
 	evidenceId: string | null,
 	evidenceDirectory: string | null,
 	allowNetwork: boolean,
 ): void {
-	const rows: string[][] = [["scenario", "bullet", "verdict", "expected"]];
+	const skillName = subject.name;
+	// `vs base` is the same bullet in the arm that ran without the skill. Kept in
+	// its own column so a reader never mistakes it for part of the rubric verdict.
+	const rows: string[][] = [["scenario", "bullet", "verdict", "vs base", "expected"]];
 	for (const outcome of outcomes) {
+		const attributed = new Map(outcome.attribution.bullets.map((item) => [item.index, item]));
 		for (const bullet of outcome.bullets) {
-			rows.push([outcome.scenario.id, String(bullet.index), bullet.verdict, truncate(bullet.text, 76)]);
+			rows.push([
+				outcome.scenario.id,
+				String(bullet.index),
+				bullet.verdict,
+				attributed.get(bullet.index)?.attribution ?? "-",
+				truncate(bullet.text, 64),
+			]);
 		}
 	}
 	process.stdout.write(formatColumns(rows));
@@ -1204,13 +2147,78 @@ function printHumanReport(
 			);
 		}
 	}
+	printAttributionReport(outcomes);
 	process.stdout.write(`${describeArmPolicyOutcome(allowNetwork)}\n`);
+	process.stdout.write(
+		`subject: ${skillName} from ${subject.origin} at ${subject.baseDir} (sha256 ${subject.normalizedHash.slice(0, 12)}…)\n`,
+	);
+	if (subject.drift !== null) {
+		process.stdout.write(
+			subject.drift.verdict === "mismatch"
+				? `subject drift: MISMATCH against the ${subject.drift.authority} (expected ${subject.drift.expected.slice(0, 12)}…); this run measured the copy on disk\n`
+				: `subject drift: matches the ${subject.drift.authority}\n`,
+		);
+	}
+	const unverified = outcomes.filter((outcome) => outcome.subjectVerification !== "verified");
+	if (unverified.length > 0) {
+		process.stdout.write(
+			`subject verification: ${unverified.map((o) => `${o.scenario.id}=${o.subjectVerification}`).join(", ")}; ` +
+				"those scenarios kept their rubric result and recorded no baseline comparison\n",
+		);
+	}
 	process.stdout.write(`eval artifact: ${evalId}\n`);
 	if (evidenceId !== null && evidenceDirectory !== null) {
 		process.stdout.write(`evidence: ${evidenceId} at ${evidenceDirectory} (per-bullet detail in skill-eval.json)\n`);
 	}
 }
 
+/**
+ * The paired comparison, stated as scenario counts and never as one score.
+ *
+ * Deliberately separated from the rubric block above it. A reader who takes
+ * "3/4 bullets passed" and "1 scenario regressed" as the same measurement will
+ * draw the wrong conclusion from both: the first says whether the skill met its
+ * rubric, the second says whether it changed anything relative to no skill at
+ * all. Nothing here influences the exit code.
+ */
+function printAttributionReport(outcomes: ReadonlyArray<ScenarioOutcome>): void {
+	if (outcomes.length === 0) return;
+	const summary = attributionSummary(outcomes);
+	const compared = summary.helped + summary.regressed + summary.mixed + summary["no-change"];
+	if (compared === 0) {
+		const reason = outcomes.find((outcome) => outcome.attribution.reason !== null)?.attribution.reason ?? null;
+		process.stdout.write(
+			`attribution: no scenario was compared against its baseline${reason !== null ? ` (${reason})` : ""}\n`,
+		);
+		return;
+	}
+	const parts = [
+		`${summary.helped} helped`,
+		`${summary.regressed} regressed`,
+		`${summary.mixed} mixed`,
+		`${summary["no-change"]} no-change`,
+	];
+	if (summary.unmeasured > 0) parts.push(`${summary.unmeasured} unmeasured`);
+	if (summary["not-attempted"] > 0) parts.push(`${summary["not-attempted"]} not-attempted`);
+	process.stdout.write(
+		`attribution (advisory, vs the no-skill baseline; gates nothing): ${parts.join(", ")} of ${outcomes.length} scenario${outcomes.length === 1 ? "" : "s"}\n`,
+	);
+	for (const outcome of outcomes) {
+		if (outcome.attribution.verdict === "regressed" || outcome.attribution.verdict === "mixed") {
+			const regressed = outcome.attribution.bullets
+				.filter((bullet) => bullet.attribution === "regressed")
+				.map((bullet) => String(bullet.index));
+			process.stdout.write(
+				`${outcome.scenario.id}: ${outcome.attribution.verdict}; the baseline passed bullet${regressed.length === 1 ? "" : "s"} ${regressed.join(", ")} and the treatment did not\n`,
+			);
+		} else if (outcome.attribution.reason !== null) {
+			process.stdout.write(
+				`${outcome.scenario.id}: attribution ${outcome.attribution.verdict}: ${outcome.attribution.reason}\n`,
+			);
+		}
+	}
+}
+
 function preview(value: unknown): string {
 	if (value === undefined) return "";
 	const text = typeof value === "string" ? value : safeStringify(value);
@@ -1249,7 +2257,7 @@ function isRecord(value: unknown): value is Record<string, unknown> {
 
 /** The experimental evals.md lane stays under eval, alongside package suite evals. */
 export async function runSkillEvalCli(args: ReadonlyArray<string>): Promise<number> {
-	const options: SkillsEvalOptions = { json: false, trustFixtures: false, allowNetwork: false };
+	const options: SkillsEvalOptions = { json: false, trustFixtures: false, allowNetwork: false, noAttribution: false };
 	let source: string | undefined;
 	try {
 		for (let i = 0; i < args.length; i++) {
@@ -1258,6 +2266,7 @@ export async function runSkillEvalCli(args: ReadonlyArray<string>): Promise<numb
 			if (arg === "--json") options.json = true;
 			else if (arg === "--trust-fixtures") options.trustFixtures = true;
 			else if (arg === "--allow-network") options.allowNetwork = true;
+			else if (arg === "--no-attribution") options.noAttribution = true;
 			else if (["--scenario", "--target", "--workspace", "--timeout"].includes(arg)) {
 				const value = args[++i];
 				if (!value || value.startsWith("-")) throw new Error(`${arg} requires a value`);
@@ -1270,7 +2279,8 @@ export async function runSkillEvalCli(args: ReadonlyArray<string>): Promise<numb
 				else options.workspace = value;
 			} else if (arg === "--help" || arg === "-h") {
 				process.stdout.write(
-					"clio-coder eval skill <name|path> [--scenario <id>] [--target <id>] [--workspace <path>] [--timeout <seconds>] [--trust-fixtures] [--allow-network] [--json]\n",
+					"clio-coder eval skill <name|path> [--scenario <id>] [--target <id>] [--workspace <path>] [--timeout <seconds>] [--trust-fixtures] [--allow-network] [--no-attribution] [--json]\n" +
+						"  --no-attribution  skip the baseline judge and the paired comparison; saves one judge run per scenario\n",
 				);
 				return 0;
 			} else if (arg.startsWith("-") || source) throw new Error(`unexpected skill eval argument: ${arg}`);