@uinaf/skillcheck 1.2.0 → 1.4.0

package/README.md

@@ -45,7 +45,7 @@ current directory.
 <root>/skills/<skill>/SKILL.md                       linted
 <root>/skills/<skill>/evals/<scenario>/task.md       the problem + input files
 <root>/skills/<skill>/evals/<scenario>/criteria.json the weighted checklist
-<root>/.skillcheck/                                  results, scratch, scorecards
+<root>/.skillcheck/                                  results, scorecards
 ```
 
 `cli/*/skills/<skill>/` is scanned too, for repos that keep a skill next to the

package/dist/cli.js

@@ -2,7 +2,9 @@
 import { lintSkills } from "./lint.js";
 import { encodeRunNamePart, generateRun, requiredEvalPackages, resolvePackageDir, runNameFor } from "./scenario.js";
 import { execFileSync, spawnSync } from "node:child_process";
+import { createHash } from "node:crypto";
 import fs from "node:fs";
+import os from "node:os";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
 //#region src/cli.ts
@@ -48,7 +50,7 @@ function parseArgs(argv) {
 			const v = argv[++i];
 			if (v === void 0 || v.startsWith("--")) throw new Error(`${a} needs a value`);
 			flags.set(a, v);
-		} else if (a === "--all" || a === "--allow-mixed") flags.set(a, true);
+		} else if (a === "--all" || a === "--allow-mixed" || a === "--control") flags.set(a, true);
 		else throw new Error(`unknown flag: ${a}`);
 	}
 	return {
@@ -61,9 +63,10 @@ function resolveRoot(flags) {
 }
 function stateDirs(root) {
 	const base = path.join(root, ".skillcheck");
+	const id = createHash("sha256").update(path.resolve(root)).digest("hex").slice(0, 12);
 	return {
 		results: path.join(base, "results"),
-		scratch: path.join(base, "scratch"),
+		scratch: path.join(fs.realpathSync(os.tmpdir()), `skillcheck-${id}`),
 		scorecards: path.join(base, "scorecards")
 	};
 }
@@ -95,6 +98,7 @@ function runOptions(flags) {
 		judgeModel,
 		judgeEffort,
 		maxTurns: flags.has("--max-turns") ? parsePositiveInt("--max-turns", flags.get("--max-turns")) : void 0,
+		control: flags.get("--control") === true,
 		trials: flags.has("--trials") ? parsePositiveInt("--trials", flags.get("--trials")) : 1
 	};
 }
@@ -104,7 +108,8 @@ function runConfigOf(opts) {
 		agent_effort: opts.agentEffort ?? null,
 		judge_model: opts.judgeModel,
 		judge_effort: opts.judgeEffort ?? null,
-		trials: opts.trials ?? 1
+		trials: opts.trials ?? 1,
+		agent_access: "online"
 	};
 }
 function configKey(c) {
@@ -113,12 +118,13 @@ function configKey(c) {
 		c.agent_effort ?? null,
 		c.judge_model,
 		c.judge_effort ?? null,
-		c.trials ?? 1
+		c.trials ?? 1,
+		c.agent_access ?? "offline"
 	]);
 }
 function describeConfig(c) {
 	const effort = (e) => e ? `@${e}` : "";
-	return `agent ${c.agent_model}${effort(c.agent_effort)}, judge ${c.judge_model}${effort(c.judge_effort)}, trials ${c.trials ?? 1}`;
+	return `agent ${c.agent_model}${effort(c.agent_effort)}, judge ${c.judge_model}${effort(c.judge_effort)}, trials ${c.trials ?? 1}, ${c.agent_access ?? "offline"}`;
 }
 function assertUniformConfig(entries, allowMixed) {
 	if (allowMixed) return;
@@ -180,12 +186,12 @@ function classifyRow(raw, stats) {
 	if (typeof res.score !== "number" || typeof res.success !== "boolean") return { error: "promptfoo result carried no usable score" };
 	const components = Array.isArray(res.gradingResult?.componentResults) ? res.gradingResult.componentResults : [];
 	const checklist = components.find((c) => c?.metadata?.assertionSet?.type === "assert-set");
-	const skillUsed = components.find((c) => c?.assertion?.type === "skill-used");
-	if (typeof checklist?.score !== "number" || typeof skillUsed?.pass !== "boolean") return { error: "promptfoo result carried no checklist or skill-used verdict" };
+	const skillUsed = components.find((c) => c?.assertion?.type === "skill-used" || c?.assertion?.metric === "skill-used");
+	if (typeof checklist?.score !== "number" || typeof skillUsed?.score !== "number") return { error: "promptfoo result carried no checklist or skill-used verdict" };
 	return {
 		score: checklist.score,
 		pass: res.success,
-		skillUsed: skillUsed.pass
+		skillUsed: skillUsed.score >= 1
 	};
 }
 const NOISY_SPREAD = .2;
@@ -235,19 +241,32 @@ function metaPath(resultPath) {
 function attemptPath(resultPath) {
 	return `${resultPath}.attempt`;
 }
+function ensurePrivateDir(dir) {
+	fs.mkdirSync(dir, {
+		recursive: true,
+		mode: 448
+	});
+	const st = fs.lstatSync(dir);
+	const uid = process.getuid?.();
+	if (!st.isDirectory() || uid !== void 0 && st.uid !== uid) throw new Error(`scratch dir ${dir} is not a directory owned by this user`);
+	if ((st.mode & 63) !== 0) fs.chmodSync(dir, 448);
+}
 function runScenario(scenarioDir, opts, root) {
 	const dirs = stateDirs(root);
+	ensurePrivateDir(dirs.scratch);
 	const { name, configPath, skill, scenario } = generateRun(path.resolve(scenarioDir), opts, {
 		scratchDir: dirs.scratch,
 		transformPath: path.join(here, `transform${selfExt}`),
-		grokProviderPath: path.join(here, `grok-provider${selfExt}`)
+		grokProviderPath: path.join(here, `grok-provider${selfExt}`),
+		skillEvidencePath: path.join(here, `skill-evidence${selfExt}`)
 	});
 	fs.mkdirSync(dirs.results, { recursive: true });
 	const resultPath = path.join(dirs.results, `${name}.json`);
 	const identity = {
 		skill,
 		scenario,
-		harness: opts.harness
+		harness: opts.harness,
+		variant: opts.control ? "control" : "skill"
 	};
 	fs.writeFileSync(attemptPath(resultPath), JSON.stringify(identity) + "\n");
 	fs.rmSync(resultPath, { force: true });
@@ -314,7 +333,8 @@ function resultRunConfig(raw, meta, harness) {
 		agent_effort: m.agent_effort ?? null,
 		judge_model: m.judge_model,
 		judge_effort: m.judge_effort ?? null,
-		trials: m.trials ?? 1
+		trials: m.trials ?? 1,
+		agent_access: m.agent_access === "online" ? "online" : "offline"
 	};
 	const r = raw;
 	const agent = r?.config?.providers?.[0]?.config;
@@ -326,7 +346,8 @@ function resultRunConfig(raw, meta, harness) {
 		agent_effort: typeof agent?.effort === "string" ? agent.effort : null,
 		judge_model: judgeName(judge),
 		judge_effort: typeof judgeEffort === "string" ? judgeEffort : null,
-		trials: r?.results?.results?.length ?? 1
+		trials: r?.results?.results?.length ?? 1,
+		agent_access: "offline"
 	};
 }
 function readJson(file) {
@@ -356,7 +377,7 @@ function discoverScenarios(root) {
 	}
 	return found.sort();
 }
-const RUN_FLAGS = "[--root DIR] [--harness claude|codex|grok] [--agent MODEL] [--agent-effort EFFORT] [--judge MODEL] [--judge-effort EFFORT] [--trials K] [--max-turns N]";
+const RUN_FLAGS = "[--root DIR] [--harness claude|codex|grok] [--agent MODEL] [--agent-effort EFFORT] [--judge MODEL] [--judge-effort EFFORT] [--trials K] [--max-turns N] [--control]";
 function cmdRun(argv) {
 	const { positional, flags } = parseArgs(argv);
 	if (positional.length !== 1) fail(`usage: skillcheck run <scenario-dir> ${RUN_FLAGS}`);
@@ -381,7 +402,7 @@ function cmdSweep(argv) {
 	const wanted = configKey(runConfigOf(opts));
 	let passed = 0, failed = 0, errored = 0, skipped = 0;
 	for (const dir of discoverScenarios(root)) {
-		const name = runNameFor(dir, opts.harness);
+		const name = runNameFor(dir, opts.harness, opts.control);
 		const resultPath = path.join(resultsDir, `${name}.json`);
 		if (!all && fs.existsSync(resultPath) && !fs.existsSync(attemptPath(resultPath))) {
 			const raw = readJson(resultPath);
@@ -416,7 +437,8 @@ function resultIdentity(file, dir) {
 		if (identity !== null && typeof identity === "object" && "skill" in identity && typeof identity.skill === "string" && "scenario" in identity && typeof identity.scenario === "string" && "harness" in identity && (identity.harness === "claude" || identity.harness === "codex" || identity.harness === "grok")) return {
 			skill: identity.skill,
 			scenario: identity.scenario,
-			harness: identity.harness
+			harness: identity.harness,
+			variant: "variant" in identity && identity.variant === "control" ? "control" : "skill"
 		};
 	} catch {}
 	if (/^~v3~[0-9a-f]{64}$/.test(base)) throw new Error(`missing identity metadata for ${file}`);
@@ -429,13 +451,17 @@ function resultIdentity(file, dir) {
 			return part;
 		}
 	};
-	const suffix = base.match(/--(codex|grok|cursor)$/);
+	const unsuffixed = base.replace(/--control$/, "");
+	const variant = unsuffixed !== base && unsuffixed.includes("--") ? "control" : "skill";
+	const stem = variant === "control" ? unsuffixed : base;
+	const suffix = stem.match(/--(codex|grok|cursor)$/);
 	const harness = suffix === null ? "claude" : suffix[1];
-	const [skill, ...rest] = base.replace(/--(codex|grok|cursor)$/, "").split("--");
+	const [skill, ...rest] = stem.replace(/--(codex|grok|cursor)$/, "").split("--");
 	return {
 		skill: decode(skill),
 		scenario: decode(rest.join("--")),
-		harness
+		harness,
+		variant
 	};
 }
 function reduceResults(dir, allowMixed) {
@@ -460,7 +486,7 @@ function reduceResults(dir, allowMixed) {
 		const raw = readJson(path.join(dir, f));
 		const base = f.replace(/\.json$/, "");
 		const meta = readJson(path.join(dir, `${base}.meta.json`));
-		const { skill, scenario, harness } = resultIdentity(f, dir);
+		const { skill, scenario, harness, variant } = resultIdentity(f, dir);
 		const config = resultRunConfig(raw, meta, harness);
 		const verdict = classifyResult(raw, config.trials);
 		if ("error" in verdict) {
@@ -472,7 +498,8 @@ function reduceResults(dir, allowMixed) {
 		const key = entryKey({
 			skill,
 			scenario,
-			harness
+			harness,
+			variant
 		});
 		gradedAt.set(key, Math.max(gradedAt.get(key) ?? 0, fs.statSync(path.join(dir, f)).mtimeMs));
 		const sha = typeof meta?.skills_tree_sha === "string" ? meta.skills_tree_sha : "unattested";
@@ -482,6 +509,7 @@ function reduceResults(dir, allowMixed) {
 			skill,
 			scenario,
 			harness,
+			variant,
 			skills_tree_sha: sha,
 			...stats,
 			...config,
@@ -501,7 +529,8 @@ function entryKey(e) {
 	return [
 		e.skill,
 		e.scenario,
-		e.harness
+		e.harness,
+		e.variant ?? "skill"
 	].join("\0");
 }
 function mergeScorecard(existing, fresh) {
@@ -558,27 +587,47 @@ function cmdSummarize(argv) {
 		scenarios: merged.entries
 	};
 	fs.writeFileSync(out, JSON.stringify(scorecard, null, 2) + "\n");
-	console.log(`${out}: ${merged.entries.length} scenario(s), ${merged.entries.filter((e) => e.pass).length} passing, ${skipped.length} skipped file(s)`);
+	const scenarios = merged.entries.filter((e) => e.variant !== "control");
+	console.log(`${out}: ${scenarios.length} scenario(s), ${scenarios.filter((e) => e.pass).length} passing, ${merged.entries.length - scenarios.length} control(s), ${skipped.length} skipped file(s)`);
 	if (existing.length > 0) console.log(`merged into today's scorecard: ${entries.length} from this run, ${merged.carried} carried over`);
 	console.log(`\n${formatSkillTable(summarizeSkills(merged.entries))}`);
-	for (const e of merged.entries.filter((x) => x.noisy)) console.log(`NOISY ${e.skill}/${e.scenario} (${e.harness}) ${formatStats(e)}`);
+	for (const e of merged.entries.filter((x) => x.noisy)) {
+		const tag = e.variant === "control" ? ", control" : "";
+		console.log(`NOISY ${e.skill}/${e.scenario} (${e.harness}${tag}) ${formatStats(e)}`);
+	}
+	for (const s of summarizeSkills(merged.entries)) for (const scenario of s.no_lift) console.log(`NO LIFT ${s.skill}/${scenario} (${s.harness}): passes without the skill`);
 }
 function summarizeSkills(entries) {
 	const groups = /* @__PURE__ */ new Map();
+	const controls = /* @__PURE__ */ new Map();
 	for (const e of entries) {
 		const key = `${e.skill}\0${e.harness}`;
-		groups.set(key, [...groups.get(key) ?? [], e]);
+		if (e.variant === "control") controls.set(`${key}\0${e.scenario}`, e);
+		else groups.set(key, [...groups.get(key) ?? [], e]);
 	}
 	const mean = (xs) => round4(xs.reduce((a, b) => a + b, 0) / xs.length);
-	return [...groups.values()].map((rows) => ({
-		skill: rows[0].skill,
-		harness: rows[0].harness,
-		scenarios: rows.length,
-		pass_all: rows.filter((r) => r.pass).length,
-		pass_rate: mean(rows.map((r) => r.pass_rate ?? (r.pass ? 1 : 0))),
-		score: mean(rows.map((r) => r.score)),
-		noisy: rows.filter((r) => r.noisy).map((r) => r.scenario)
-	}));
+	return [...groups.entries()].map(([key, rows]) => {
+		const paired = rows.flatMap((r) => {
+			const c = controls.get(`${key}\0${r.scenario}`);
+			return c === void 0 ? [] : [{
+				skill: r,
+				control: c
+			}];
+		});
+		const controlScore = paired.length === 0 ? null : mean(paired.map((p) => p.control.score));
+		return {
+			skill: rows[0].skill,
+			harness: rows[0].harness,
+			scenarios: rows.length,
+			pass_all: rows.filter((r) => r.pass).length,
+			pass_rate: mean(rows.map((r) => r.pass_rate ?? (r.pass ? 1 : 0))),
+			score: mean(rows.map((r) => r.score)),
+			noisy: rows.filter((r) => r.noisy).map((r) => r.scenario),
+			control_score: controlScore,
+			lift: controlScore === null ? null : round4(mean(paired.map((p) => p.skill.score)) - controlScore),
+			no_lift: paired.filter((p) => p.control.pass).map((p) => p.skill.scenario)
+		};
+	});
 }
 function formatSkillTable(rows) {
 	const table = [[
@@ -588,7 +637,9 @@ function formatSkillTable(rows) {
 		"pass^k",
 		"pass rate",
 		"score",
-		"noisy"
+		"noisy",
+		"control",
+		"lift"
 	], ...rows.map((r) => [
 		r.skill,
 		r.harness,
@@ -596,7 +647,9 @@ function formatSkillTable(rows) {
 		`${r.pass_all}/${r.scenarios}`,
 		r.pass_rate.toFixed(2),
 		r.score.toFixed(2),
-		String(r.noisy.length)
+		String(r.noisy.length),
+		r.control_score === null ? "-" : r.control_score.toFixed(2),
+		r.lift === null ? "-" : `${r.lift >= 0 ? "+" : ""}${r.lift.toFixed(2)}`
 	])];
 	const widths = table[0].map((_, i) => Math.max(...table.map((row) => row[i].length)));
 	return table.map((row) => row.map((c, i) => c.padEnd(widths[i])).join("  ").trimEnd()).join("\n");
@@ -635,4 +688,4 @@ if (isMainModule()) {
 	}
 }
 //#endregion
-export { NOISY_SPREAD, aggregateTrials, assertUniformConfig, classifyResult, formatStats, mergeScorecard, parseArgs, parsePositiveInt, reduceResults, resolveRoot, resultRunConfig, runConfigOf, runOptions, stateDirs, summarizeSkills, toolVersion, treeShaOf };
+export { NOISY_SPREAD, aggregateTrials, assertUniformConfig, classifyResult, ensurePrivateDir, formatStats, mergeScorecard, parseArgs, parsePositiveInt, reduceResults, resolveRoot, resultRunConfig, runConfigOf, runOptions, stateDirs, summarizeSkills, toolVersion, treeShaOf };

package/dist/grok-provider.js

@@ -27,7 +27,6 @@ var GrokProvider = class {
 			"streaming-json",
 			"--permission-mode",
 			"acceptEdits",
-			"--disable-web-search",
 			"--no-subagents",
 			...model ? ["--model", model] : [],
 			"-p",

package/dist/scenario.js

@@ -1,8 +1,8 @@
 import { createRequire } from "node:module";
-import fs from "node:fs";
-import path from "node:path";
 import { createHash } from "node:crypto";
+import fs from "node:fs";
 import os from "node:os";
+import path from "node:path";
 //#region src/scenario.ts
 const DEFAULT_CLAUDE_AGENT = "claude-opus-5";
 function loadScenario(scenarioDir) {
@@ -12,15 +12,17 @@ function loadScenario(scenarioDir) {
 	const taskMd = fs.readFileSync(path.join(scenarioDir, "task.md"), "utf8");
 	const criteria = JSON.parse(fs.readFileSync(path.join(scenarioDir, "criteria.json"), "utf8"));
 	if (criteria.type !== "weighted_checklist" || !Array.isArray(criteria.checklist) || criteria.checklist.length === 0) throw new Error(`unsupported or empty criteria in ${scenarioDir}`);
+	if (criteria.skill_use !== void 0 && criteria.skill_use !== "required" && criteria.skill_use !== "optional") throw new Error(`skill_use must be "required" or "optional" in ${scenarioDir}`);
 	for (const item of criteria.checklist) if (!(typeof item?.name === "string" && item.name.trim() !== "" && typeof item?.description === "string" && item.description.trim() !== "" && Number.isFinite(item?.max_score) && item.max_score > 0)) throw new Error(`invalid checklist item in ${scenarioDir}: ${JSON.stringify(item)}`);
 	const files = [];
-	let prompt = taskMd.replace(/^=+ FILE: (.+?) =+\n([\s\S]*?)\n=+ END FILE =+$/gm, (_, name, content) => {
+	const task = taskMd.replace(/^=+ FILE: (.+?) =+\n([\s\S]*?)\n=+ END FILE =+$/gm, (_, name, content) => {
 		files.push({
 			name: name.trim(),
 			content: content + "\n"
 		});
 		return `(Input file \`${name.trim()}\` is available in your working directory.)`;
 	});
+	let prompt = task;
 	const skillDir = path.resolve(scenarioDir, "../..");
 	if (isHiddenSkill(fs.readFileSync(path.join(skillDir, "SKILL.md"), "utf8"))) prompt = `Use the ${skill} skill for this task.\n\n${prompt}`;
 	return {
@@ -29,6 +31,7 @@ function loadScenario(scenarioDir) {
 		name: `${skill}--${scenario}`,
 		skillDir,
 		prompt,
+		task,
 		files,
 		criteria
 	};
@@ -37,13 +40,18 @@ function encodeRunNamePart(part) {
 	if (!part.includes("--") && !part.startsWith("-") && !part.endsWith("-") && !part.startsWith("~v2~")) return part;
 	return `~v2~${encodeURIComponent(part).replaceAll("-", "%2D").replaceAll("~", "%7E")}`;
 }
-function runNameFor(scenarioDir, harness) {
+function runNameFor(scenarioDir, harness, control = false) {
 	const m = path.resolve(scenarioDir).match(/skills\/([^/]+)\/evals\/([^/]+)$/);
 	if (!m) throw new Error(`not a scenario dir (want .../skills/<skill>/evals/<scenario>): ${scenarioDir}`);
 	const name = `${encodeRunNamePart(m[1])}--${encodeRunNamePart(m[2])}`;
-	const full = harness === "claude" ? name : `${name}--${harness}`;
+	const full = `${harness === "claude" ? name : `${name}--${harness}`}${control ? "--control" : ""}`;
 	if (Buffer.byteLength(`${full}.json.attempt`) <= 255) return full;
-	return `~v3~${createHash("sha256").update(JSON.stringify([
+	return `~v3~${createHash("sha256").update(JSON.stringify(control ? [
+		m[1],
+		m[2],
+		harness,
+		"control"
+	] : [
 		m[1],
 		m[2],
 		harness
@@ -71,7 +79,7 @@ const RESERVED = /* @__PURE__ */ new Set([
 	".grok",
 	"node_modules"
 ]);
-function materialize(s, runDir, harness) {
+function materialize(s, runDir, harness, control = false) {
 	const workdir = path.join(runDir, "workdir");
 	fs.rmSync(runDir, {
 		recursive: true,
@@ -95,7 +103,7 @@ function materialize(s, runDir, harness) {
 		fs.mkdirSync(path.dirname(dest), { recursive: true });
 		fs.writeFileSync(dest, content);
 	}
-	const roots = harness === "codex" ? [".claude", ".agents"] : harness === "grok" ? [".grok"] : [".claude"];
+	const roots = control ? [] : harness === "codex" ? [".claude", ".agents"] : harness === "grok" ? [".grok"] : [".claude"];
 	for (const root of roots) fs.cpSync(s.skillDir, path.join(workdir, root, "skills", s.skill), {
 		recursive: true,
 		filter: (src) => path.basename(src) !== "evals"
@@ -140,7 +148,9 @@ function agentProvider(opts, workdir, skill, paths) {
 			skip_git_repo_check: true,
 			enable_streaming: true,
 			sandbox_mode: "workspace-write",
-			cli_env: { CODEX_HOME: process.env.CODEX_HOME ?? path.join(os.homedir(), ".codex") }
+			network_access_enabled: true,
+			web_search_enabled: true,
+			cli_env: { CODEX_HOME: path.join(workdir, "..", "..", "codex-home") }
 		}
 	};
 	return {
@@ -151,14 +161,17 @@ function agentProvider(opts, workdir, skill, paths) {
 			apiKeyRequired: false,
 			working_dir: workdir,
 			setting_sources: ["project"],
-			skills: [skill],
+			...opts.control ? {} : { skills: [skill] },
 			permission_mode: "acceptEdits",
 			append_allowed_tools: [
 				"Read",
 				"Write",
 				"Edit",
 				"Glob",
-				"Grep"
+				"Grep",
+				"Bash",
+				"WebFetch",
+				"WebSearch"
 			],
 			max_turns: opts.maxTurns ?? 50
 		}
@@ -218,7 +231,7 @@ function buildConfig(s, trials, opts, paths) {
 			description: s.criteria.context,
 			providers: [trialLabel(i)],
 			vars: {
-				task: s.prompt,
+				task: opts.control ? s.task : s.prompt,
 				workdir: t.workdir,
 				manifest: t.manifestPath
 			},
@@ -231,8 +244,13 @@ function buildConfig(s, trials, opts, paths) {
 					weight: item.max_score
 				}))
 			}, {
-				type: "skill-used",
-				value: s.skill
+				type: "javascript",
+				value: `file://${paths.skillEvidencePath}`,
+				metric: "skill-used",
+				config: {
+					skill: s.skill,
+					required: !opts.control && s.criteria.skill_use !== "optional"
+				}
 			}]
 		}))
 	};
@@ -259,15 +277,31 @@ function requiredEvalPackages(opts, hasAnthropicKey) {
 	if (opts.harness === "codex") pkgs.push("@openai/codex-sdk");
 	return pkgs;
 }
+function privateCodexHome(dir) {
+	const source = process.env.CODEX_HOME ?? path.join(os.homedir(), ".codex");
+	fs.rmSync(dir, {
+		recursive: true,
+		force: true
+	});
+	fs.mkdirSync(dir, {
+		recursive: true,
+		mode: 448
+	});
+	for (const file of ["config.toml", "auth.json"]) {
+		const from = path.join(source, file);
+		if (fs.existsSync(from)) fs.symlinkSync(from, path.join(dir, file));
+	}
+}
 function generateRun(scenarioDir, opts, paths) {
 	const s = loadScenario(scenarioDir);
-	const name = runNameFor(scenarioDir, opts.harness);
+	const name = runNameFor(scenarioDir, opts.harness, opts.control);
 	const runDir = path.join(paths.scratchDir, name);
 	fs.rmSync(runDir, {
 		recursive: true,
 		force: true
 	});
-	const trials = Array.from({ length: opts.trials ?? 1 }, (_, i) => materialize(s, path.join(runDir, trialLabel(i)), opts.harness));
+	const trials = Array.from({ length: opts.trials ?? 1 }, (_, i) => materialize(s, path.join(runDir, trialLabel(i)), opts.harness, opts.control));
+	if (opts.harness === "codex") privateCodexHome(path.join(runDir, "codex-home"));
 	const sdkDir = sdkNodeModulesDir();
 	if (sdkDir !== void 0) {
 		const link = path.join(runDir, "node_modules");
@@ -288,4 +322,4 @@ function generateRun(scenarioDir, opts, paths) {
 	};
 }
 //#endregion
-export { DEFAULT_CLAUDE_AGENT, buildConfig, encodeRunNamePart, generateRun, isHiddenSkill, loadScenario, materialize, requiredEvalPackages, resolvePackageDir, runNameFor, sdkNodeModulesDir, stripHiddenFlag, trialLabel };
+export { DEFAULT_CLAUDE_AGENT, buildConfig, encodeRunNamePart, generateRun, isHiddenSkill, loadScenario, materialize, privateCodexHome, requiredEvalPackages, resolvePackageDir, runNameFor, sdkNodeModulesDir, stripHiddenFlag, trialLabel };

package/dist/skill-evidence.js

@@ -0,0 +1,49 @@
+import fs from "node:fs";
+import path from "node:path";
+//#region src/skill-evidence.ts
+const SKILL_ROOTS = [
+	".claude",
+	".agents",
+	".grok"
+];
+function real(file) {
+	try {
+		return fs.realpathSync(file);
+	} catch {
+		return;
+	}
+}
+function skillEvidence(context) {
+	const skill = context.config?.skill;
+	const workdir = context.vars.workdir;
+	if (typeof skill !== "string" || typeof workdir !== "string") return void 0;
+	const calls = (value) => Array.isArray(value) ? value.filter((c) => c !== null && typeof c === "object") : [];
+	const metadata = context.metadata ?? context.providerResponse?.metadata;
+	if (calls(metadata?.skillCalls).find((c) => c.name === skill && c.is_error !== true)) return `skill call ${skill}`;
+	const installed = new Set(SKILL_ROOTS.map((root) => real(path.join(workdir, root, "skills", skill, "SKILL.md"))).filter((p) => p !== void 0));
+	const root = real(workdir) ?? workdir;
+	if (calls(metadata?.toolCalls).find((c) => {
+		if (c.name !== "Read" || c.is_error !== false || typeof c.output !== "string") return false;
+		const file = c.input?.file_path;
+		if (typeof file !== "string") return false;
+		const named = path.resolve(workdir, file);
+		const inside = [workdir, root].some((w) => {
+			const rel = path.relative(w, named);
+			return rel !== ".." && !rel.startsWith(`..${path.sep}`) && !path.isAbsolute(rel);
+		});
+		const target = real(named);
+		return inside && target !== void 0 && installed.has(target);
+	})) return `read of the installed ${skill}/SKILL.md`;
+}
+function assertSkillUsed(_output, context) {
+	const evidence = skillEvidence(context);
+	const required = context.config?.required !== false;
+	const used = evidence !== void 0;
+	return {
+		pass: used || !required,
+		score: used ? 1 : 0,
+		reason: used ? `skill used: ${evidence}` : `skill ${String(context.config?.skill)} not loaded${required ? "" : " (optional for this scenario)"}`
+	};
+}
+//#endregion
+export { assertSkillUsed as default, skillEvidence };

package/dist/transform.js

@@ -1,9 +1,9 @@
+import { createHash } from "node:crypto";
 import fs from "node:fs";
 import path from "node:path";
-import { createHash } from "node:crypto";
 //#region src/transform.ts
-const PER_FILE_CAP = 4e3;
-const TOTAL_CAP = 24e3;
+const PER_FILE_CAP = 16e3;
+const TOTAL_CAP = 64e3;
 function safeSlice(text, end) {
 	let cut = text.slice(0, end);
 	const last = cut.charCodeAt(cut.length - 1);

package/docs/adoption.md

@@ -63,7 +63,6 @@ Commit `.skillcheck/scorecards/`. Gitignore the rest:
 
 ```gitignore
 .skillcheck/results/
-.skillcheck/scratch/
 ```
 
 A scorecard is only comparable against the tree it graded and the configuration

package/docs/scenarios.md

@@ -54,7 +54,16 @@ item needs a non-empty `name` and `description` and a positive `max_score`.
 Each item becomes one `llm-rubric` assertion weighted by `max_score`, inside an
 assert-set with threshold 0.7. A separate `skill-used` assertion sits outside
 that aggregate, so a run that produces good output without ever loading the
-skill still fails. There is no test-level threshold: both must pass. The
+skill still fails. The skill counts as loaded when the harness reports a skill
+call for it, or when the agent completed a read of the installed
+`<config-root>/skills/<skill>/SKILL.md` in its workdir. Claude Code often reads
+the file directly instead of calling its Skill tool, and either way the same
+instructions reach its context.
+
+An out-of-lane scenario, where the right answer is to decline the skill, sets
+`"skill_use": "optional"` in `criteria.json`. The assertion then always passes,
+and the result still records whether the skill was loaded. Omitted, it is
+`"required"`. There is no test-level threshold: both must pass. The
 reported score is the assert-set's weighted score; skill-used is reported
 separately as a rate across trials.
 
@@ -62,14 +71,28 @@ Write descriptions a judge can check against the deliverable: an observable
 property, not a feeling. Weight the items that would make a reviewer reject the
 work.
 
+The agent runs online, like a real session: it has a shell and web access on
+every harness (Claude: Bash, WebFetch, WebSearch; Codex: network and web search
+in its `workspace-write` sandbox; Grok: its CLI defaults). It runs on the
+operator's machine with the operator's logins (Codex gets a per-run
+`CODEX_HOME` carrying only its config and login, so the operator's own skills
+and global guidance stay out), so a task must never ask for a
+live mutation such as posting a comment, pushing, publishing, or writing to a
+shared workspace. Put that state in fixture files and grade the plan. Checks
+the agent can run for itself (install, build, test, fetch a public page) are
+fair to require.
+
+Name a specific tool or version only when the skill teaches it. Otherwise grade
+the property the tool provides, so an equivalent approach passes.
+
 ## What the judge sees
 
 The agent's final message, plus every file in the workdir that differs from the
 pre-run manifest. Unchanged inputs are omitted; deleted inputs, unreadable
 files, and non-regular files are named rather than read.
 
-Sections are sorted by path, each file is capped at 4,000 characters and the
-appended total at 24,000, with truncation stated inline. Very large outputs make
+Sections are sorted by path, each file is capped at 16,000 characters and the
+appended total at 64,000, with truncation stated inline. Very large outputs make
 rubric judges return nothing at all, which is why the caps exist. Keep fixtures
 small enough that the deliverable fits.
 
@@ -84,8 +107,10 @@ frontmatter block; body text mentioning the key does not count.
 
 ## The workdir
 
-Per run, under `<root>/.skillcheck/scratch/<name>/`, rebuilt from scratch each
-time. The skill under test is installed where the harness discovers skills:
+Per run, under a scratch directory in the system temp dir
+(`skillcheck-<hash of the root>/<name>/trial-<n>/`), rebuilt from scratch each
+time. It stays outside the root so the agent cannot reach the skill's source,
+its evals, or the repository's own agent guidance. The skill under test is installed where the harness discovers skills:
 `.claude/skills/<skill>/`, plus `.agents/skills/<skill>/` on codex, or
 `.grok/skills/<skill>/` on Grok, with its `evals/` directory excluded, so
 criteria never leak into the agent's context.

package/docs/usage.md

@@ -38,7 +38,7 @@ skillcheck run <scenario-dir> --harness grok
 skillcheck run <scenario-dir> --trials 3 --agent-effort medium
 ```
 
-Materializes the scenario into `<root>/.skillcheck/scratch/<name>/trial-<n>/workdir`,
+Materializes the scenario into a temp-dir workdir per trial ([workdir](scenarios.md#the-workdir)),
 installs the skill under test into that workdir, drives the agent, and grades
 the files it wrote. Exit 0 means pass, 1 means graded fail, and 2 means error.
 Exit 2 covers missing usable promptfoo output or optional eval peers. The
@@ -82,6 +82,18 @@ So is a result with fewer rows than trials, or a row without its checklist and
 With `--trials` above 1, `run` and `sweep` print min, spread, pass count,
 skill-used count, and `NOISY`.
 
+### Control
+
+`--control` runs a scenario without the skill: nothing is installed in the
+workdir, a hidden skill's explicit invocation is dropped from the task, and the
+`skill-used` assertion never fails. Its result sits beside the skill run
+(`<name>--control.json`) with `variant: "control"` in the sidecar, and `sweep
+--control` covers every scenario. `summarize` pairs each scenario with its
+control and adds two columns per skill: the mean control score, and the lift
+(skill score minus control score over the paired scenarios). A scenario whose
+control passes every trial prints as `NO LIFT`: it passes without the skill, so
+it does not test the skill.
+
 ### Agent effort
 
 `--agent-effort low|medium|high|xhigh|max` sets the Claude agent's effort.
@@ -96,8 +108,8 @@ Omitting it leaves Claude Code's default. Only `--harness claude` takes it;
 workdir with the skill under `.grok/skills/`. It uses native streaming events
 to count a completed read of that skill's `SKILL.md` as `skill-used` evidence.
 Grok must be logged in locally or have its supported credentials configured.
-The run disables web search and subagents and grants edit permission in the
-workdir. `--agent` selects a Grok model ID.
+The run disables subagents and grants edit permission in the workdir; web
+search stays on. `--agent` selects a Grok model ID.
 
 `--judge` takes either a bare Claude model (graded through the Anthropic
 selection in [auth](#auth)) or a provider-qualified promptfoo id, passed
@@ -200,6 +212,7 @@ Each successful run writes a `<name>.meta.json` sidecar next to its result:
   "judge_model": "claude-opus-5",
   "judge_effort": null,
   "trials": 3,
+  "agent_access": "online",
   "aggregate": { "pass": false, "pass_rate": 0.6667, "score": 0.81, "...": "..." },
   "ran_at": "<ISO timestamp>",
   "tool_version": "<skillcheck version>"
@@ -207,7 +220,8 @@ Each successful run writes a `<name>.meta.json` sidecar next to its result:
 ```
 
 `summarize` reads those sidecars and refuses to mix skills-tree revisions or
-run configurations (agent model and effort, judge model and effort, trials) in
+run configurations (agent model and effort, judge model and effort, trials, and
+agent access) in
 one scorecard, including retained rows from partial reruns, unless
 `--allow-mixed`. Configurations are compared within a harness, since harnesses
 differ by design. Sidecars written before run configurations were recorded fall
@@ -218,9 +232,10 @@ becomes `mixed` and per-entry shas remain. A result with no sidecar reduces as
 
 ## State
 
-`<root>/.skillcheck/` holds `scratch/` and `results/`, both disposable and safe
-to gitignore, and `scorecards/`, which is meant to be committed. Nothing is ever
-written inside the installed package.
+`<root>/.skillcheck/` holds `results/`, disposable and safe to gitignore, and
+`scorecards/`, which is meant to be committed. Scratch workdirs live under the
+system temp dir, outside the root. Nothing is ever written inside the installed
+package.
 
 ## Auth
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uinaf/skillcheck",
-  "version": "1.2.0",
+  "version": "1.4.0",
   "description": "Lint and eval harness for agent skills",
   "homepage": "https://github.com/uinaf/skillcheck#readme",
   "bugs": {