@uinaf/skillcheck 1.3.0 → 1.4.1

package/dist/cli.js

@@ -50,7 +50,7 @@ function parseArgs(argv) {
 			const v = argv[++i];
 			if (v === void 0 || v.startsWith("--")) throw new Error(`${a} needs a value`);
 			flags.set(a, v);
-		} else if (a === "--all" || a === "--allow-mixed") flags.set(a, true);
+		} else if (a === "--all" || a === "--allow-mixed" || a === "--control") flags.set(a, true);
 		else throw new Error(`unknown flag: ${a}`);
 	}
 	return {
@@ -98,6 +98,7 @@ function runOptions(flags) {
 		judgeModel,
 		judgeEffort,
 		maxTurns: flags.has("--max-turns") ? parsePositiveInt("--max-turns", flags.get("--max-turns")) : void 0,
+		control: flags.get("--control") === true,
 		trials: flags.has("--trials") ? parsePositiveInt("--trials", flags.get("--trials")) : 1
 	};
 }
@@ -107,7 +108,8 @@ function runConfigOf(opts) {
 		agent_effort: opts.agentEffort ?? null,
 		judge_model: opts.judgeModel,
 		judge_effort: opts.judgeEffort ?? null,
-		trials: opts.trials ?? 1
+		trials: opts.trials ?? 1,
+		agent_access: "online"
 	};
 }
 function configKey(c) {
@@ -116,12 +118,13 @@ function configKey(c) {
 		c.agent_effort ?? null,
 		c.judge_model,
 		c.judge_effort ?? null,
-		c.trials ?? 1
+		c.trials ?? 1,
+		c.agent_access ?? "offline"
 	]);
 }
 function describeConfig(c) {
 	const effort = (e) => e ? `@${e}` : "";
-	return `agent ${c.agent_model}${effort(c.agent_effort)}, judge ${c.judge_model}${effort(c.judge_effort)}, trials ${c.trials ?? 1}`;
+	return `agent ${c.agent_model}${effort(c.agent_effort)}, judge ${c.judge_model}${effort(c.judge_effort)}, trials ${c.trials ?? 1}, ${c.agent_access ?? "offline"}`;
 }
 function assertUniformConfig(entries, allowMixed) {
 	if (allowMixed) return;
@@ -262,7 +265,8 @@ function runScenario(scenarioDir, opts, root) {
 	const identity = {
 		skill,
 		scenario,
-		harness: opts.harness
+		harness: opts.harness,
+		variant: opts.control ? "control" : "skill"
 	};
 	fs.writeFileSync(attemptPath(resultPath), JSON.stringify(identity) + "\n");
 	fs.rmSync(resultPath, { force: true });
@@ -329,7 +333,8 @@ function resultRunConfig(raw, meta, harness) {
 		agent_effort: m.agent_effort ?? null,
 		judge_model: m.judge_model,
 		judge_effort: m.judge_effort ?? null,
-		trials: m.trials ?? 1
+		trials: m.trials ?? 1,
+		agent_access: m.agent_access === "online" ? "online" : "offline"
 	};
 	const r = raw;
 	const agent = r?.config?.providers?.[0]?.config;
@@ -341,7 +346,8 @@ function resultRunConfig(raw, meta, harness) {
 		agent_effort: typeof agent?.effort === "string" ? agent.effort : null,
 		judge_model: judgeName(judge),
 		judge_effort: typeof judgeEffort === "string" ? judgeEffort : null,
-		trials: r?.results?.results?.length ?? 1
+		trials: r?.results?.results?.length ?? 1,
+		agent_access: "offline"
 	};
 }
 function readJson(file) {
@@ -351,6 +357,17 @@ function readJson(file) {
 		return;
 	}
 }
+function liveScenarios(root) {
+	const cli = path.join(root, "cli");
+	if (!(fs.existsSync(path.join(root, "skills")) || fs.existsSync(cli) && fs.readdirSync(cli).some((d) => fs.existsSync(path.join(cli, d, "skills"))))) return void 0;
+	return new Set(discoverScenarios(root).map((dir) => {
+		const parts = dir.split(path.sep);
+		return `${parts.at(-3)}\0${parts.at(-1)}`;
+	}));
+}
+function isLive(live, e) {
+	return live === void 0 || live.has(`${e.skill}\0${e.scenario}`);
+}
 function discoverScenarios(root) {
 	const roots = [path.join(root, "skills")];
 	const cliDir = path.join(root, "cli");
@@ -371,7 +388,7 @@ function discoverScenarios(root) {
 	}
 	return found.sort();
 }
-const RUN_FLAGS = "[--root DIR] [--harness claude|codex|grok] [--agent MODEL] [--agent-effort EFFORT] [--judge MODEL] [--judge-effort EFFORT] [--trials K] [--max-turns N]";
+const RUN_FLAGS = "[--root DIR] [--harness claude|codex|grok] [--agent MODEL] [--agent-effort EFFORT] [--judge MODEL] [--judge-effort EFFORT] [--trials K] [--max-turns N] [--control]";
 function cmdRun(argv) {
 	const { positional, flags } = parseArgs(argv);
 	if (positional.length !== 1) fail(`usage: skillcheck run <scenario-dir> ${RUN_FLAGS}`);
@@ -396,7 +413,7 @@ function cmdSweep(argv) {
 	const wanted = configKey(runConfigOf(opts));
 	let passed = 0, failed = 0, errored = 0, skipped = 0;
 	for (const dir of discoverScenarios(root)) {
-		const name = runNameFor(dir, opts.harness);
+		const name = runNameFor(dir, opts.harness, opts.control);
 		const resultPath = path.join(resultsDir, `${name}.json`);
 		if (!all && fs.existsSync(resultPath) && !fs.existsSync(attemptPath(resultPath))) {
 			const raw = readJson(resultPath);
@@ -431,7 +448,8 @@ function resultIdentity(file, dir) {
 		if (identity !== null && typeof identity === "object" && "skill" in identity && typeof identity.skill === "string" && "scenario" in identity && typeof identity.scenario === "string" && "harness" in identity && (identity.harness === "claude" || identity.harness === "codex" || identity.harness === "grok")) return {
 			skill: identity.skill,
 			scenario: identity.scenario,
-			harness: identity.harness
+			harness: identity.harness,
+			variant: "variant" in identity && identity.variant === "control" ? "control" : "skill"
 		};
 	} catch {}
 	if (/^~v3~[0-9a-f]{64}$/.test(base)) throw new Error(`missing identity metadata for ${file}`);
@@ -444,24 +462,39 @@ function resultIdentity(file, dir) {
 			return part;
 		}
 	};
-	const suffix = base.match(/--(codex|grok|cursor)$/);
+	const unsuffixed = base.replace(/--control$/, "");
+	const variant = unsuffixed !== base && unsuffixed.includes("--") ? "control" : "skill";
+	const stem = variant === "control" ? unsuffixed : base;
+	const suffix = stem.match(/--(codex|grok|cursor)$/);
 	const harness = suffix === null ? "claude" : suffix[1];
-	const [skill, ...rest] = base.replace(/--(codex|grok|cursor)$/, "").split("--");
+	const [skill, ...rest] = stem.replace(/--(codex|grok|cursor)$/, "").split("--");
 	return {
 		skill: decode(skill),
 		scenario: decode(rest.join("--")),
-		harness
+		harness,
+		variant
 	};
 }
-function reduceResults(dir, allowMixed) {
+function reduceResults(dir, allowMixed, live) {
 	const entries = [];
 	const skipped = [];
+	const retired = /* @__PURE__ */ new Set();
 	const gradedAt = /* @__PURE__ */ new Map();
 	const shas = /* @__PURE__ */ new Set();
 	const files = fs.readdirSync(dir);
 	const incomplete = new Set(files.filter((f) => f.endsWith(".json.attempt")).map((f) => f.replace(/\.attempt$/, "")));
 	const results = files.filter((f) => f.endsWith(".json") && !f.endsWith(".meta.json"));
 	for (const f of [.../* @__PURE__ */ new Set([...results, ...incomplete])].sort()) {
+		let identity;
+		try {
+			identity = resultIdentity(f, dir);
+		} catch {
+			identity = void 0;
+		}
+		if (identity !== void 0 && identity.scenario !== "" && !isLive(live, identity)) {
+			retired.add(entryKey(identity));
+			continue;
+		}
 		if (f.endsWith("--cursor.json")) {
 			console.error(`skipping ${f}: Cursor harness is retired`);
 			skipped.push(f);
@@ -475,7 +508,7 @@ function reduceResults(dir, allowMixed) {
 		const raw = readJson(path.join(dir, f));
 		const base = f.replace(/\.json$/, "");
 		const meta = readJson(path.join(dir, `${base}.meta.json`));
-		const { skill, scenario, harness } = resultIdentity(f, dir);
+		const { skill, scenario, harness, variant } = resultIdentity(f, dir);
 		const config = resultRunConfig(raw, meta, harness);
 		const verdict = classifyResult(raw, config.trials);
 		if ("error" in verdict) {
@@ -487,7 +520,8 @@ function reduceResults(dir, allowMixed) {
 		const key = entryKey({
 			skill,
 			scenario,
-			harness
+			harness,
+			variant
 		});
 		gradedAt.set(key, Math.max(gradedAt.get(key) ?? 0, fs.statSync(path.join(dir, f)).mtimeMs));
 		const sha = typeof meta?.skills_tree_sha === "string" ? meta.skills_tree_sha : "unattested";
@@ -497,6 +531,7 @@ function reduceResults(dir, allowMixed) {
 			skill,
 			scenario,
 			harness,
+			variant,
 			skills_tree_sha: sha,
 			...stats,
 			...config,
@@ -509,14 +544,16 @@ function reduceResults(dir, allowMixed) {
 		treeSha: shas.size === 1 ? [...shas][0] : shas.size === 0 ? "none" : "mixed",
 		entries,
 		skipped,
-		gradedAt
+		gradedAt,
+		retired
 	};
 }
 function entryKey(e) {
 	return [
 		e.skill,
 		e.scenario,
-		e.harness
+		e.harness,
+		e.variant ?? "skill"
 	].join("\0");
 }
 function mergeScorecard(existing, fresh) {
@@ -551,12 +588,17 @@ function readExistingScorecard(out) {
 function cmdSummarize(argv) {
 	const { positional, flags } = parseArgs(argv);
 	if (positional.length > 0) fail("usage: skillcheck summarize [--root DIR] [--allow-mixed]");
-	const dirs = stateDirs(resolveRoot(flags));
+	const root = resolveRoot(flags);
+	const dirs = stateDirs(root);
 	if (!fs.existsSync(dirs.results)) fail(`no results directory at ${dirs.results}; run some evals first`);
-	const { entries, skipped, gradedAt } = reduceResults(dirs.results, flags.get("--allow-mixed") === true);
+	const live = liveScenarios(root);
+	const { entries, skipped, gradedAt, retired: retiredResults } = reduceResults(dirs.results, flags.get("--allow-mixed") === true, live);
 	fs.mkdirSync(dirs.scorecards, { recursive: true });
 	const out = path.join(dirs.scorecards, `${(/* @__PURE__ */ new Date()).toISOString().slice(0, 10)}.json`);
-	const existing = readExistingScorecard(out);
+	const previous = readExistingScorecard(out);
+	const existing = previous.filter((e) => isLive(live, e));
+	for (const e of previous) if (!isLive(live, e)) retiredResults.add(entryKey(e));
+	const retired = retiredResults.size;
 	const skippedKeys = new Set(skipped.map((file) => ({
 		key: entryKey(resultIdentity(file, dirs.results)),
 		modifiedAt: fs.statSync(fs.existsSync(path.join(dirs.results, `${file}.attempt`)) ? path.join(dirs.results, `${file}.attempt`) : path.join(dirs.results, file)).mtimeMs
@@ -573,27 +615,48 @@ function cmdSummarize(argv) {
 		scenarios: merged.entries
 	};
 	fs.writeFileSync(out, JSON.stringify(scorecard, null, 2) + "\n");
-	console.log(`${out}: ${merged.entries.length} scenario(s), ${merged.entries.filter((e) => e.pass).length} passing, ${skipped.length} skipped file(s)`);
+	if (retired > 0) console.log(`dropped ${retired} row(s) for scenarios no longer in the tree`);
+	const scenarios = merged.entries.filter((e) => e.variant !== "control");
+	console.log(`${out}: ${scenarios.length} scenario(s), ${scenarios.filter((e) => e.pass).length} passing, ${merged.entries.length - scenarios.length} control(s), ${skipped.length} skipped file(s)`);
 	if (existing.length > 0) console.log(`merged into today's scorecard: ${entries.length} from this run, ${merged.carried} carried over`);
 	console.log(`\n${formatSkillTable(summarizeSkills(merged.entries))}`);
-	for (const e of merged.entries.filter((x) => x.noisy)) console.log(`NOISY ${e.skill}/${e.scenario} (${e.harness}) ${formatStats(e)}`);
+	for (const e of merged.entries.filter((x) => x.noisy)) {
+		const tag = e.variant === "control" ? ", control" : "";
+		console.log(`NOISY ${e.skill}/${e.scenario} (${e.harness}${tag}) ${formatStats(e)}`);
+	}
+	for (const s of summarizeSkills(merged.entries)) for (const scenario of s.no_lift) console.log(`NO LIFT ${s.skill}/${scenario} (${s.harness}): passes without the skill`);
 }
 function summarizeSkills(entries) {
 	const groups = /* @__PURE__ */ new Map();
+	const controls = /* @__PURE__ */ new Map();
 	for (const e of entries) {
 		const key = `${e.skill}\0${e.harness}`;
-		groups.set(key, [...groups.get(key) ?? [], e]);
+		if (e.variant === "control") controls.set(`${key}\0${e.scenario}`, e);
+		else groups.set(key, [...groups.get(key) ?? [], e]);
 	}
 	const mean = (xs) => round4(xs.reduce((a, b) => a + b, 0) / xs.length);
-	return [...groups.values()].map((rows) => ({
-		skill: rows[0].skill,
-		harness: rows[0].harness,
-		scenarios: rows.length,
-		pass_all: rows.filter((r) => r.pass).length,
-		pass_rate: mean(rows.map((r) => r.pass_rate ?? (r.pass ? 1 : 0))),
-		score: mean(rows.map((r) => r.score)),
-		noisy: rows.filter((r) => r.noisy).map((r) => r.scenario)
-	}));
+	return [...groups.entries()].map(([key, rows]) => {
+		const paired = rows.flatMap((r) => {
+			const c = controls.get(`${key}\0${r.scenario}`);
+			return c === void 0 ? [] : [{
+				skill: r,
+				control: c
+			}];
+		});
+		const controlScore = paired.length === 0 ? null : mean(paired.map((p) => p.control.score));
+		return {
+			skill: rows[0].skill,
+			harness: rows[0].harness,
+			scenarios: rows.length,
+			pass_all: rows.filter((r) => r.pass).length,
+			pass_rate: mean(rows.map((r) => r.pass_rate ?? (r.pass ? 1 : 0))),
+			score: mean(rows.map((r) => r.score)),
+			noisy: rows.filter((r) => r.noisy).map((r) => r.scenario),
+			control_score: controlScore,
+			lift: controlScore === null ? null : round4(mean(paired.map((p) => p.skill.score)) - controlScore),
+			no_lift: paired.filter((p) => p.control.pass).map((p) => p.skill.scenario)
+		};
+	});
 }
 function formatSkillTable(rows) {
 	const table = [[
@@ -603,7 +666,9 @@ function formatSkillTable(rows) {
 		"pass^k",
 		"pass rate",
 		"score",
-		"noisy"
+		"noisy",
+		"control",
+		"lift"
 	], ...rows.map((r) => [
 		r.skill,
 		r.harness,
@@ -611,7 +676,9 @@ function formatSkillTable(rows) {
 		`${r.pass_all}/${r.scenarios}`,
 		r.pass_rate.toFixed(2),
 		r.score.toFixed(2),
-		String(r.noisy.length)
+		String(r.noisy.length),
+		r.control_score === null ? "-" : r.control_score.toFixed(2),
+		r.lift === null ? "-" : `${r.lift >= 0 ? "+" : ""}${r.lift.toFixed(2)}`
 	])];
 	const widths = table[0].map((_, i) => Math.max(...table.map((row) => row[i].length)));
 	return table.map((row) => row.map((c, i) => c.padEnd(widths[i])).join("  ").trimEnd()).join("\n");
@@ -650,4 +717,4 @@ if (isMainModule()) {
 	}
 }
 //#endregion
-export { NOISY_SPREAD, aggregateTrials, assertUniformConfig, classifyResult, ensurePrivateDir, formatStats, mergeScorecard, parseArgs, parsePositiveInt, reduceResults, resolveRoot, resultRunConfig, runConfigOf, runOptions, stateDirs, summarizeSkills, toolVersion, treeShaOf };
+export { NOISY_SPREAD, aggregateTrials, assertUniformConfig, classifyResult, ensurePrivateDir, formatStats, liveScenarios, mergeScorecard, parseArgs, parsePositiveInt, reduceResults, resolveRoot, resultRunConfig, runConfigOf, runOptions, stateDirs, summarizeSkills, toolVersion, treeShaOf };

package/dist/grok-provider.js

@@ -27,7 +27,6 @@ var GrokProvider = class {
 			"streaming-json",
 			"--permission-mode",
 			"acceptEdits",
-			"--disable-web-search",
 			"--no-subagents",
 			...model ? ["--model", model] : [],
 			"-p",

package/dist/scenario.js

@@ -15,13 +15,14 @@ function loadScenario(scenarioDir) {
 	if (criteria.skill_use !== void 0 && criteria.skill_use !== "required" && criteria.skill_use !== "optional") throw new Error(`skill_use must be "required" or "optional" in ${scenarioDir}`);
 	for (const item of criteria.checklist) if (!(typeof item?.name === "string" && item.name.trim() !== "" && typeof item?.description === "string" && item.description.trim() !== "" && Number.isFinite(item?.max_score) && item.max_score > 0)) throw new Error(`invalid checklist item in ${scenarioDir}: ${JSON.stringify(item)}`);
 	const files = [];
-	let prompt = taskMd.replace(/^=+ FILE: (.+?) =+\n([\s\S]*?)\n=+ END FILE =+$/gm, (_, name, content) => {
+	const task = taskMd.replace(/^=+ FILE: (.+?) =+\n([\s\S]*?)\n=+ END FILE =+$/gm, (_, name, content) => {
 		files.push({
 			name: name.trim(),
 			content: content + "\n"
 		});
 		return `(Input file \`${name.trim()}\` is available in your working directory.)`;
 	});
+	let prompt = task;
 	const skillDir = path.resolve(scenarioDir, "../..");
 	if (isHiddenSkill(fs.readFileSync(path.join(skillDir, "SKILL.md"), "utf8"))) prompt = `Use the ${skill} skill for this task.\n\n${prompt}`;
 	return {
@@ -30,6 +31,7 @@ function loadScenario(scenarioDir) {
 		name: `${skill}--${scenario}`,
 		skillDir,
 		prompt,
+		task,
 		files,
 		criteria
 	};
@@ -38,13 +40,18 @@ function encodeRunNamePart(part) {
 	if (!part.includes("--") && !part.startsWith("-") && !part.endsWith("-") && !part.startsWith("~v2~")) return part;
 	return `~v2~${encodeURIComponent(part).replaceAll("-", "%2D").replaceAll("~", "%7E")}`;
 }
-function runNameFor(scenarioDir, harness) {
+function runNameFor(scenarioDir, harness, control = false) {
 	const m = path.resolve(scenarioDir).match(/skills\/([^/]+)\/evals\/([^/]+)$/);
 	if (!m) throw new Error(`not a scenario dir (want .../skills/<skill>/evals/<scenario>): ${scenarioDir}`);
 	const name = `${encodeRunNamePart(m[1])}--${encodeRunNamePart(m[2])}`;
-	const full = harness === "claude" ? name : `${name}--${harness}`;
+	const full = `${harness === "claude" ? name : `${name}--${harness}`}${control ? "--control" : ""}`;
 	if (Buffer.byteLength(`${full}.json.attempt`) <= 255) return full;
-	return `~v3~${createHash("sha256").update(JSON.stringify([
+	return `~v3~${createHash("sha256").update(JSON.stringify(control ? [
+		m[1],
+		m[2],
+		harness,
+		"control"
+	] : [
 		m[1],
 		m[2],
 		harness
@@ -72,7 +79,7 @@ const RESERVED = /* @__PURE__ */ new Set([
 	".grok",
 	"node_modules"
 ]);
-function materialize(s, runDir, harness) {
+function materialize(s, runDir, harness, control = false) {
 	const workdir = path.join(runDir, "workdir");
 	fs.rmSync(runDir, {
 		recursive: true,
@@ -96,7 +103,7 @@ function materialize(s, runDir, harness) {
 		fs.mkdirSync(path.dirname(dest), { recursive: true });
 		fs.writeFileSync(dest, content);
 	}
-	const roots = harness === "codex" ? [".claude", ".agents"] : harness === "grok" ? [".grok"] : [".claude"];
+	const roots = control ? [] : harness === "codex" ? [".claude", ".agents"] : harness === "grok" ? [".grok"] : [".claude"];
 	for (const root of roots) fs.cpSync(s.skillDir, path.join(workdir, root, "skills", s.skill), {
 		recursive: true,
 		filter: (src) => path.basename(src) !== "evals"
@@ -141,7 +148,9 @@ function agentProvider(opts, workdir, skill, paths) {
 			skip_git_repo_check: true,
 			enable_streaming: true,
 			sandbox_mode: "workspace-write",
-			cli_env: { CODEX_HOME: process.env.CODEX_HOME ?? path.join(os.homedir(), ".codex") }
+			network_access_enabled: true,
+			web_search_enabled: true,
+			cli_env: { CODEX_HOME: path.join(workdir, "..", "..", "codex-home") }
 		}
 	};
 	return {
@@ -152,14 +161,17 @@ function agentProvider(opts, workdir, skill, paths) {
 			apiKeyRequired: false,
 			working_dir: workdir,
 			setting_sources: ["project"],
-			skills: [skill],
+			...opts.control ? {} : { skills: [skill] },
 			permission_mode: "acceptEdits",
 			append_allowed_tools: [
 				"Read",
 				"Write",
 				"Edit",
 				"Glob",
-				"Grep"
+				"Grep",
+				"Bash",
+				"WebFetch",
+				"WebSearch"
 			],
 			max_turns: opts.maxTurns ?? 50
 		}
@@ -219,7 +231,7 @@ function buildConfig(s, trials, opts, paths) {
 			description: s.criteria.context,
 			providers: [trialLabel(i)],
 			vars: {
-				task: s.prompt,
+				task: opts.control ? s.task : s.prompt,
 				workdir: t.workdir,
 				manifest: t.manifestPath
 			},
@@ -237,7 +249,7 @@ function buildConfig(s, trials, opts, paths) {
 				metric: "skill-used",
 				config: {
 					skill: s.skill,
-					required: s.criteria.skill_use !== "optional"
+					required: !opts.control && s.criteria.skill_use !== "optional"
 				}
 			}]
 		}))
@@ -265,15 +277,31 @@ function requiredEvalPackages(opts, hasAnthropicKey) {
 	if (opts.harness === "codex") pkgs.push("@openai/codex-sdk");
 	return pkgs;
 }
+function privateCodexHome(dir) {
+	const source = process.env.CODEX_HOME ?? path.join(os.homedir(), ".codex");
+	fs.rmSync(dir, {
+		recursive: true,
+		force: true
+	});
+	fs.mkdirSync(dir, {
+		recursive: true,
+		mode: 448
+	});
+	for (const file of ["config.toml", "auth.json"]) {
+		const from = path.join(source, file);
+		if (fs.existsSync(from)) fs.symlinkSync(from, path.join(dir, file));
+	}
+}
 function generateRun(scenarioDir, opts, paths) {
 	const s = loadScenario(scenarioDir);
-	const name = runNameFor(scenarioDir, opts.harness);
+	const name = runNameFor(scenarioDir, opts.harness, opts.control);
 	const runDir = path.join(paths.scratchDir, name);
 	fs.rmSync(runDir, {
 		recursive: true,
 		force: true
 	});
-	const trials = Array.from({ length: opts.trials ?? 1 }, (_, i) => materialize(s, path.join(runDir, trialLabel(i)), opts.harness));
+	const trials = Array.from({ length: opts.trials ?? 1 }, (_, i) => materialize(s, path.join(runDir, trialLabel(i)), opts.harness, opts.control));
+	if (opts.harness === "codex") privateCodexHome(path.join(runDir, "codex-home"));
 	const sdkDir = sdkNodeModulesDir();
 	if (sdkDir !== void 0) {
 		const link = path.join(runDir, "node_modules");
@@ -294,4 +322,4 @@ function generateRun(scenarioDir, opts, paths) {
 	};
 }
 //#endregion
-export { DEFAULT_CLAUDE_AGENT, buildConfig, encodeRunNamePart, generateRun, isHiddenSkill, loadScenario, materialize, requiredEvalPackages, resolvePackageDir, runNameFor, sdkNodeModulesDir, stripHiddenFlag, trialLabel };
+export { DEFAULT_CLAUDE_AGENT, buildConfig, encodeRunNamePart, generateRun, isHiddenSkill, loadScenario, materialize, privateCodexHome, requiredEvalPackages, resolvePackageDir, runNameFor, sdkNodeModulesDir, stripHiddenFlag, trialLabel };

package/docs/scenarios.md

@@ -71,15 +71,16 @@ Write descriptions a judge can check against the deliverable: an observable
 property, not a feeling. Weight the items that would make a reviewer reject the
 work.
 
-On the Claude harness the agent can read, search, and write files in its
-workdir, but has no shell, so it cannot install, build, test, or reach the
-network. The shell stays off because the agent runs on the operator's machine
-with the operator's credentials. Codex runs shell commands inside its
-`workspace-write` sandbox, and Grok follows its own CLI permission mode. Write criteria a shell-less agent can meet, so one scenario
-scores comparably across harnesses: a checklist item that requires live proof,
-such as a frozen lockfile or a verified release, fails on Claude. Grade whether
-the deliverable names the checks it could not run and hands them over
-precisely.
+The agent runs online, like a real session: it has a shell and web access on
+every harness (Claude: Bash, WebFetch, WebSearch; Codex: network and web search
+in its `workspace-write` sandbox; Grok: its CLI defaults). It runs on the
+operator's machine with the operator's logins (Codex gets a per-run
+`CODEX_HOME` carrying only its config and login, so the operator's own skills
+and global guidance stay out), so a task must never ask for a
+live mutation such as posting a comment, pushing, publishing, or writing to a
+shared workspace. Put that state in fixture files and grade the plan. Checks
+the agent can run for itself (install, build, test, fetch a public page) are
+fair to require.
 
 Name a specific tool or version only when the skill teaches it. Otherwise grade
 the property the tool provides, so an equivalent approach passes.

package/docs/usage.md

@@ -82,6 +82,18 @@ So is a result with fewer rows than trials, or a row without its checklist and
 With `--trials` above 1, `run` and `sweep` print min, spread, pass count,
 skill-used count, and `NOISY`.
 
+### Control
+
+`--control` runs a scenario without the skill: nothing is installed in the
+workdir, a hidden skill's explicit invocation is dropped from the task, and the
+`skill-used` assertion never fails. Its result sits beside the skill run
+(`<name>--control.json`) with `variant: "control"` in the sidecar, and `sweep
+--control` covers every scenario. `summarize` pairs each scenario with its
+control and adds two columns per skill: the mean control score, and the lift
+(skill score minus control score over the paired scenarios). A scenario whose
+control passes every trial prints as `NO LIFT`: it passes without the skill, so
+it does not test the skill.
+
 ### Agent effort
 
 `--agent-effort low|medium|high|xhigh|max` sets the Claude agent's effort.
@@ -96,8 +108,8 @@ Omitting it leaves Claude Code's default. Only `--harness claude` takes it;
 workdir with the skill under `.grok/skills/`. It uses native streaming events
 to count a completed read of that skill's `SKILL.md` as `skill-used` evidence.
 Grok must be logged in locally or have its supported credentials configured.
-The run disables web search and subagents and grants edit permission in the
-workdir. `--agent` selects a Grok model ID.
+The run disables subagents and grants edit permission in the workdir; web
+search stays on. `--agent` selects a Grok model ID.
 
 `--judge` takes either a bare Claude model (graded through the Anthropic
 selection in [auth](#auth)) or a provider-qualified promptfoo id, passed
@@ -167,6 +179,11 @@ six of twenty-nine scenarios therefore leaves twenty-nine rows in the file, not
 six. A same-date file that cannot be parsed stops the write instead of being
 overwritten.
 
+Rows for scenarios that no longer exist in the tree, whether carried from
+today's scorecard or left in `results/`, are dropped and counted on stdout.
+A root with no skills tree (`skills/` or `cli/*/skills/`) holds results only
+and keeps every row.
+
 Files that are not promptfoo results and ungraded transport errors are skipped
 with a warning rather than failing the reduction. Graded assertion failures
 remain scored results. If a skipped file matches an existing scorecard row,
@@ -200,6 +217,7 @@ Each successful run writes a `<name>.meta.json` sidecar next to its result:
   "judge_model": "claude-opus-5",
   "judge_effort": null,
   "trials": 3,
+  "agent_access": "online",
   "aggregate": { "pass": false, "pass_rate": 0.6667, "score": 0.81, "...": "..." },
   "ran_at": "<ISO timestamp>",
   "tool_version": "<skillcheck version>"
@@ -207,7 +225,8 @@ Each successful run writes a `<name>.meta.json` sidecar next to its result:
 ```
 
 `summarize` reads those sidecars and refuses to mix skills-tree revisions or
-run configurations (agent model and effort, judge model and effort, trials) in
+run configurations (agent model and effort, judge model and effort, trials, and
+agent access) in
 one scorecard, including retained rows from partial reruns, unless
 `--allow-mixed`. Configurations are compared within a harness, since harnesses
 differ by design. Sidecars written before run configurations were recorded fall

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uinaf/skillcheck",
-  "version": "1.3.0",
+  "version": "1.4.1",
   "description": "Lint and eval harness for agent skills",
   "homepage": "https://github.com/uinaf/skillcheck#readme",
   "bugs": {