@uinaf/skillcheck 1.0.2 → 1.2.0

package/README.md

@@ -28,7 +28,8 @@ pre-npm git tags are covered there too.
 ```sh
 skillcheck lint                              # structural lint, no credentials
 skillcheck run skills/wat/evals/basic        # one scenario, graded end to end
-skillcheck sweep && skillcheck summarize     # every scenario, then a scorecard
+skillcheck sweep --trials 3                  # every scenario, three trials each
+skillcheck summarize                         # per-scenario and per-skill scorecard
 ```
 
 `lint` needs nothing. `run` and `sweep` need model auth, which is why sweeps

package/dist/cli.js

@@ -12,11 +12,18 @@ const selfExt = path.extname(fileURLToPath(import.meta.url));
 function toolVersion() {
 	return JSON.parse(fs.readFileSync(path.join(packageDir, "package.json"), "utf8")).version;
 }
-function parseMaxTurns(raw) {
+function parsePositiveInt(flag, raw) {
 	const n = Number(raw);
-	if (!Number.isInteger(n) || n <= 0) throw new Error(`--max-turns must be a positive integer, got ${JSON.stringify(raw)}`);
+	if (!Number.isInteger(n) || n <= 0) throw new Error(`${flag} must be a positive integer, got ${JSON.stringify(raw)}`);
 	return n;
 }
+const AGENT_EFFORTS = [
+	"low",
+	"medium",
+	"high",
+	"xhigh",
+	"max"
+];
 function fail(msg) {
 	console.error(msg);
 	process.exit(1);
@@ -29,8 +36,10 @@ function parseArgs(argv) {
 		"--agent",
 		"--judge",
 		"--judge-effort",
+		"--agent-effort",
 		"--harness",
-		"--max-turns"
+		"--max-turns",
+		"--trials"
 	]);
 	for (let i = 0; i < argv.length; i++) {
 		const a = argv[i];
@@ -60,28 +69,67 @@ function stateDirs(root) {
 }
 function runOptions(flags) {
 	const harness = flags.get("--harness") ?? "claude";
-	if (harness !== "claude" && harness !== "codex" && harness !== "grok") fail(`--harness must be claude, codex, or grok, got ${harness}`);
-	if (flags.has("--max-turns") && harness !== "claude") fail("--max-turns is only supported with --harness claude");
+	if (harness !== "claude" && harness !== "codex" && harness !== "grok") throw new Error(`--harness must be claude, codex, or grok, got ${harness}`);
+	if (flags.has("--max-turns") && harness !== "claude") throw new Error("--max-turns is only supported with --harness claude");
+	const agentEffort = flags.get("--agent-effort");
+	if (agentEffort !== void 0) {
+		if (harness !== "claude") throw new Error(`--agent-effort is only supported with --harness claude; ${harness} has no effort setting wired`);
+		if (!AGENT_EFFORTS.includes(agentEffort)) throw new Error(`--agent-effort must be ${AGENT_EFFORTS.join(", ")}, got ${agentEffort}`);
+	}
 	const agent = flags.get("--agent");
 	const judgeModel = flags.get("--judge") ?? "claude-opus-5";
 	const judgeEffort = flags.get("--judge-effort");
 	if (judgeEffort !== void 0) {
-		if (![
+		const levels = judgeModel.includes(":") ? [
 			"minimal",
 			"low",
 			"medium",
 			"high"
-		].includes(judgeEffort)) fail(`--judge-effort must be minimal, low, medium, or high, got ${judgeEffort}`);
-		if (!judgeModel.includes(":")) fail("--judge-effort needs a provider-qualified --judge (e.g. openai:chat:gpt-5.6-sol); the Anthropic judge does not take a reasoning effort");
+		] : AGENT_EFFORTS;
+		if (!levels.includes(judgeEffort)) throw new Error(`--judge-effort for ${judgeModel} must be ${levels.join(", ")}, got ${judgeEffort}`);
 	}
 	return {
 		harness,
 		agentModel: agent,
+		agentEffort,
 		judgeModel,
 		judgeEffort,
-		maxTurns: flags.has("--max-turns") ? parseMaxTurns(flags.get("--max-turns")) : void 0
+		maxTurns: flags.has("--max-turns") ? parsePositiveInt("--max-turns", flags.get("--max-turns")) : void 0,
+		trials: flags.has("--trials") ? parsePositiveInt("--trials", flags.get("--trials")) : 1
+	};
+}
+function runConfigOf(opts) {
+	return {
+		agent_model: opts.agentModel ?? (opts.harness === "claude" ? "claude-opus-5" : `${opts.harness}-default`),
+		agent_effort: opts.agentEffort ?? null,
+		judge_model: opts.judgeModel,
+		judge_effort: opts.judgeEffort ?? null,
+		trials: opts.trials ?? 1
 	};
 }
+function configKey(c) {
+	return JSON.stringify([
+		c.agent_model,
+		c.agent_effort ?? null,
+		c.judge_model,
+		c.judge_effort ?? null,
+		c.trials ?? 1
+	]);
+}
+function describeConfig(c) {
+	const effort = (e) => e ? `@${e}` : "";
+	return `agent ${c.agent_model}${effort(c.agent_effort)}, judge ${c.judge_model}${effort(c.judge_effort)}, trials ${c.trials ?? 1}`;
+}
+function assertUniformConfig(entries, allowMixed) {
+	if (allowMixed) return;
+	const byHarness = /* @__PURE__ */ new Map();
+	for (const e of entries) {
+		const configs = byHarness.get(e.harness) ?? /* @__PURE__ */ new Map();
+		configs.set(configKey(e), e);
+		byHarness.set(e.harness, configs);
+	}
+	for (const [harness, configs] of byHarness) if (configs.size > 1) throw new Error(`${harness} results span multiple run configurations (${[...configs.values()].map(describeConfig).join("; ")}); rerun them to match or pass --allow-mixed`);
+}
 function ensureEvalPackages(opts) {
 	const missing = requiredEvalPackages(opts, process.env.ANTHROPIC_API_KEY !== void 0).filter((pkg) => resolvePackageDir(pkg) === void 0);
 	if (missing.length === 0) return;
@@ -105,21 +153,76 @@ function promptfooEntry() {
 	if (typeof rel !== "string") throw new Error(`promptfoo at ${dir} declares no bin`);
 	return path.join(dir, rel);
 }
-function classifyResult(raw) {
+const GRADED_REASONS = /* @__PURE__ */ new Set([0, 1]);
+const ERRORED_REASON = 2;
+function classifyResult(raw, expectedTrials) {
 	const root = raw;
-	const res = root?.results?.results?.[0];
+	const rows = root?.results?.results;
+	if (!Array.isArray(rows) || rows.length === 0) return { error: "promptfoo output carried no result" };
+	if (expectedTrials !== void 0 && rows.length !== expectedTrials) return { error: `promptfoo returned ${rows.length} of ${expectedTrials} trials` };
+	const stats = rows.length === 1 ? root?.results?.stats ?? void 0 : void 0;
+	const trials = [];
+	for (const [i, row] of rows.entries()) {
+		const verdict = classifyRow(row, stats);
+		if ("error" in verdict) return { error: rows.length === 1 ? verdict.error : `trial ${i + 1}: ${verdict.error}` };
+		trials.push(verdict);
+	}
+	return { trials };
+}
+function classifyRow(raw, stats) {
+	const res = raw;
 	if (res === void 0 || res === null) return { error: "promptfoo output carried no result" };
 	const message = typeof res.error === "string" ? res.error.trim() : "";
-	const stats = root?.results?.stats ?? void 0;
-	const gradedByStats = stats !== void 0 && (stats.errors ?? 0) === 0 && ((stats.failures ?? 0) > 0 || (stats.successes ?? 0) > 0);
-	if (message !== "" && !gradedByStats) return { error: message };
-	if (stats !== void 0 && (stats.errors ?? 0) > 0 && (stats.successes ?? 0) === 0 && (stats.failures ?? 0) === 0) return { error: "promptfoo reported an errored test with nothing graded" };
+	if (res.failureReason === ERRORED_REASON) return { error: message || "promptfoo reported an errored test" };
+	const graded = GRADED_REASONS.has(res.failureReason) || stats !== void 0 && (stats.errors ?? 0) === 0 && ((stats.failures ?? 0) > 0 || (stats.successes ?? 0) > 0);
+	if (message !== "" && !graded) return { error: message };
+	if (!graded && stats !== void 0 && (stats.errors ?? 0) > 0 && (stats.successes ?? 0) === 0 && (stats.failures ?? 0) === 0) return { error: "promptfoo reported an errored test with nothing graded" };
 	if (typeof res.score !== "number" || typeof res.success !== "boolean") return { error: "promptfoo result carried no usable score" };
+	const components = Array.isArray(res.gradingResult?.componentResults) ? res.gradingResult.componentResults : [];
+	const checklist = components.find((c) => c?.metadata?.assertionSet?.type === "assert-set");
+	const skillUsed = components.find((c) => c?.assertion?.type === "skill-used");
+	if (typeof checklist?.score !== "number" || typeof skillUsed?.pass !== "boolean") return { error: "promptfoo result carried no checklist or skill-used verdict" };
 	return {
-		score: res.score,
-		pass: res.success
+		score: checklist.score,
+		pass: res.success,
+		skillUsed: skillUsed.pass
 	};
 }
+const NOISY_SPREAD = .2;
+const round4 = (n) => Math.round(n * 1e4) / 1e4;
+function aggregateTrials(trials) {
+	if (trials.length === 0) throw new Error("cannot aggregate zero trials");
+	const scores = trials.map((t) => t.score);
+	const passes = trials.filter((t) => t.pass).length;
+	const min = Math.min(...scores);
+	const spread = Math.max(...scores) - min;
+	const skillUsed = trials.filter((t) => t.skillUsed).length;
+	return {
+		trials: trials.length,
+		pass: passes === trials.length,
+		passes,
+		pass_rate: round4(passes / trials.length),
+		score: round4(scores.reduce((a, b) => a + b, 0) / trials.length),
+		score_min: round4(min),
+		score_spread: round4(spread),
+		skill_used: skillUsed,
+		skill_used_rate: round4(skillUsed / trials.length),
+		noisy: passes > 0 && passes < trials.length || spread >= .199999999
+	};
+}
+function formatStats(s) {
+	const line = `score=${s.score.toFixed(4)}`;
+	if (s.trials === 1) return line;
+	return [
+		line,
+		`min=${s.score_min.toFixed(4)}`,
+		`spread=${s.score_spread.toFixed(4)}`,
+		`pass^${s.trials}=${s.pass ? "yes" : "no"}`,
+		`passes=${s.passes}/${s.trials}`,
+		`skill-used=${s.skill_used}/${s.trials}`,
+		...s.noisy ? ["NOISY"] : []
+	].join(" ");
+}
 function gitHead(root) {
 	return execFileSync("git", ["rev-parse", "HEAD"], {
 		cwd: root,
@@ -177,24 +280,62 @@ function runScenario(scenarioDir, opts, root) {
 	if (rc !== 0) return outcome;
 	let verdict;
 	try {
-		verdict = classifyResult(JSON.parse(fs.readFileSync(resultPath, "utf8")));
+		verdict = classifyResult(JSON.parse(fs.readFileSync(resultPath, "utf8")), opts.trials ?? 1);
 	} catch {
 		verdict = { error: "promptfoo produced no parseable result file" };
 	}
-	outcome.score = verdict.score;
-	outcome.pass = verdict.pass;
-	outcome.error = verdict.error;
-	if (verdict.score !== void 0) {
-		fs.writeFileSync(metaPath(resultPath), JSON.stringify({
-			skills_tree_sha: sha,
-			...identity,
-			ran_at: (/* @__PURE__ */ new Date()).toISOString(),
-			tool_version: toolVersion()
-		}, null, 2) + "\n");
-		fs.rmSync(attemptPath(resultPath), { force: true });
-	}
+	if ("error" in verdict) return {
+		...outcome,
+		error: verdict.error
+	};
+	outcome.stats = aggregateTrials(verdict.trials);
+	fs.writeFileSync(metaPath(resultPath), JSON.stringify({
+		skills_tree_sha: sha,
+		...identity,
+		...runConfigOf(opts),
+		aggregate: outcome.stats,
+		ran_at: (/* @__PURE__ */ new Date()).toISOString(),
+		tool_version: toolVersion()
+	}, null, 2) + "\n");
+	fs.rmSync(attemptPath(resultPath), { force: true });
 	return outcome;
 }
+function judgeName(judge) {
+	if (typeof judge === "string") return judge.replace(/^anthropic:messages:/, "");
+	const j = judge;
+	const name = j?.config?.model ?? j?.id;
+	if (typeof name !== "string") return "unknown";
+	return j?.config?.effort === void 0 ? name : name.replace(/^anthropic:messages:/, "");
+}
+function resultRunConfig(raw, meta, harness) {
+	const m = meta;
+	if (typeof m?.agent_model === "string" && typeof m.judge_model === "string") return {
+		agent_model: m.agent_model,
+		agent_effort: m.agent_effort ?? null,
+		judge_model: m.judge_model,
+		judge_effort: m.judge_effort ?? null,
+		trials: m.trials ?? 1
+	};
+	const r = raw;
+	const agent = r?.config?.providers?.[0]?.config;
+	const judge = r?.config?.defaultTest?.options?.provider;
+	const judgeConfig = judge?.config;
+	const judgeEffort = judgeConfig?.reasoning_effort ?? judgeConfig?.effort;
+	return {
+		agent_model: typeof agent?.model === "string" ? agent.model : `${harness}-default`,
+		agent_effort: typeof agent?.effort === "string" ? agent.effort : null,
+		judge_model: judgeName(judge),
+		judge_effort: typeof judgeEffort === "string" ? judgeEffort : null,
+		trials: r?.results?.results?.length ?? 1
+	};
+}
+function readJson(file) {
+	try {
+		return JSON.parse(fs.readFileSync(file, "utf8"));
+	} catch {
+		return;
+	}
+}
 function discoverScenarios(root) {
 	const roots = [path.join(root, "skills")];
 	const cliDir = path.join(root, "cli");
@@ -215,48 +356,54 @@ function discoverScenarios(root) {
 	}
 	return found.sort();
 }
+const RUN_FLAGS = "[--root DIR] [--harness claude|codex|grok] [--agent MODEL] [--agent-effort EFFORT] [--judge MODEL] [--judge-effort EFFORT] [--trials K] [--max-turns N]";
 function cmdRun(argv) {
 	const { positional, flags } = parseArgs(argv);
-	if (positional.length !== 1) fail("usage: skillcheck run <scenario-dir> [--root DIR] [--agent MODEL] [--judge MODEL] [--judge-effort EFFORT] [--harness claude|codex|grok]");
+	if (positional.length !== 1) fail(`usage: skillcheck run <scenario-dir> ${RUN_FLAGS}`);
 	const opts = runOptions(flags);
 	ensureEvalPackages(opts);
 	const o = runScenario(positional[0], opts, resolveRoot(flags));
-	if (o.score === void 0) {
+	if (o.stats === void 0) {
 		console.error(`ERROR ${o.name}: ${o.error ?? "no usable result"} (promptfoo rc=${o.rc})`);
 		process.exit(2);
 	}
-	console.log(`${o.pass ? "PASS" : "FAIL"} ${o.name} score=${o.score.toFixed(4)} (results: ${o.resultPath})`);
-	process.exit(o.pass ? 0 : 1);
+	console.log(`${o.stats.pass ? "PASS" : "FAIL"} ${o.name} ${formatStats(o.stats)} (results: ${o.resultPath})`);
+	process.exit(o.stats.pass ? 0 : 1);
 }
 function cmdSweep(argv) {
 	const { positional, flags } = parseArgs(argv);
-	if (positional.length > 0) fail("usage: skillcheck sweep [--root DIR] [--all]");
+	if (positional.length > 0) fail(`usage: skillcheck sweep ${RUN_FLAGS} [--all]`);
 	const root = resolveRoot(flags);
 	const opts = runOptions(flags);
 	ensureEvalPackages(opts);
 	const all = flags.get("--all") === true;
 	const resultsDir = stateDirs(root).results;
+	const wanted = configKey(runConfigOf(opts));
 	let passed = 0, failed = 0, errored = 0, skipped = 0;
 	for (const dir of discoverScenarios(root)) {
 		const name = runNameFor(dir, opts.harness);
 		const resultPath = path.join(resultsDir, `${name}.json`);
-		if (!all && fs.existsSync(resultPath) && !fs.existsSync(attemptPath(resultPath))) try {
-			if (classifyResult(JSON.parse(fs.readFileSync(resultPath, "utf8"))).score !== void 0) {
+		if (!all && fs.existsSync(resultPath) && !fs.existsSync(attemptPath(resultPath))) {
+			const raw = readJson(resultPath);
+			const config = resultRunConfig(raw, readJson(metaPath(resultPath)), opts.harness);
+			const graded = !("error" in classifyResult(raw, config.trials));
+			if (graded && configKey(config) === wanted) {
 				skipped++;
 				console.log(`SKIP  ${name} (results exist; use --all to rerun)`);
 				continue;
 			}
-		} catch {}
+			if (graded) console.log(`RERUN ${name} (results used ${describeConfig(config)})`);
+		}
 		const o = runScenario(dir, opts, root);
-		if (o.score === void 0) {
+		if (o.stats === void 0) {
 			errored++;
 			console.log(`ERROR ${o.name} ${o.error ?? "no usable result"} (promptfoo rc=${o.rc})`);
-		} else if (o.pass) {
+		} else if (o.stats.pass) {
 			passed++;
-			console.log(`PASS  ${o.name} score=${o.score.toFixed(4)}`);
+			console.log(`PASS  ${o.name} ${formatStats(o.stats)}`);
 		} else {
 			failed++;
-			console.log(`FAIL  ${o.name} score=${o.score.toFixed(4)}`);
+			console.log(`FAIL  ${o.name} ${formatStats(o.stats)}`);
 		}
 	}
 	console.log(`\nsweep: ${passed} passed, ${failed} failed, ${errored} errored, ${skipped} skipped`);
@@ -310,45 +457,36 @@ function reduceResults(dir, allowMixed) {
 			skipped.push(f);
 			continue;
 		}
-		let raw;
-		try {
-			raw = JSON.parse(fs.readFileSync(path.join(dir, f), "utf8"));
-		} catch {
-			raw = void 0;
-		}
-		const res = raw?.results?.results?.[0];
-		const verdict = classifyResult(raw);
-		if (verdict.score === void 0 || verdict.pass === void 0) {
+		const raw = readJson(path.join(dir, f));
+		const base = f.replace(/\.json$/, "");
+		const meta = readJson(path.join(dir, `${base}.meta.json`));
+		const { skill, scenario, harness } = resultIdentity(f, dir);
+		const config = resultRunConfig(raw, meta, harness);
+		const verdict = classifyResult(raw, config.trials);
+		if ("error" in verdict) {
 			console.error(`skipping ${f}: ${verdict.error}`);
 			skipped.push(f);
 			continue;
 		}
-		const provider = raw.config?.providers?.[0];
-		const judge = raw.config?.defaultTest?.options?.provider;
-		const base = f.replace(/\.json$/, "");
-		const { skill, scenario, harness } = resultIdentity(f, dir);
+		const rows = raw.results.results;
 		const key = entryKey({
 			skill,
 			scenario,
 			harness
 		});
 		gradedAt.set(key, Math.max(gradedAt.get(key) ?? 0, fs.statSync(path.join(dir, f)).mtimeMs));
-		let sha = "unattested";
-		try {
-			sha = JSON.parse(fs.readFileSync(path.join(dir, `${base}.meta.json`), "utf8")).skills_tree_sha ?? "unattested";
-		} catch {}
+		const sha = typeof meta?.skills_tree_sha === "string" ? meta.skills_tree_sha : "unattested";
 		shas.add(sha);
+		const stats = aggregateTrials(verdict.trials);
 		entries.push({
 			skill,
 			scenario,
 			harness,
 			skills_tree_sha: sha,
-			score: verdict.score,
-			pass: verdict.pass,
-			agent_model: provider?.config?.model ?? `${harness}-default`,
-			judge_model: typeof judge === "string" ? judge.replace(/^anthropic:messages:/, "") : judge?.config?.model ?? judge?.id ?? "unknown",
-			latency_ms: res.latencyMs,
-			tokens: (res.tokenUsage?.total ?? 0) + (res.tokenUsage?.assertions?.total ?? 0)
+			...stats,
+			...config,
+			latency_ms: Math.round(rows.reduce((a, r) => a + (r.latencyMs ?? 0), 0) / rows.length),
+			tokens: rows.reduce((a, r) => a + (r.tokenUsage?.total ?? 0) + (r.tokenUsage?.assertions?.total ?? 0), 0)
 		});
 	}
 	if (shas.size > 1 && !allowMixed) throw new Error(`results span multiple skills-tree revisions (${[...shas].join(", ")}); rerun stale ones or pass --allow-mixed`);
@@ -411,7 +549,9 @@ function cmdSummarize(argv) {
 	if (existing.some((entry) => skippedKeys.has(entryKey(entry)))) throw new Error("skipped rerun matches an existing score; refusing to carry it or overwrite the scorecard");
 	const merged = mergeScorecard(existing, entries);
 	const treeSha = treeShaOf(merged.entries);
-	if (treeSha === "mixed" && flags.get("--allow-mixed") !== true) throw new Error("scorecard spans multiple skills-tree revisions; rerun stale ones or pass --allow-mixed");
+	const allowMixed = flags.get("--allow-mixed") === true;
+	if (treeSha === "mixed" && !allowMixed) throw new Error("scorecard spans multiple skills-tree revisions; rerun stale ones or pass --allow-mixed");
+	assertUniformConfig(merged.entries, allowMixed);
 	const scorecard = {
 		ran_at: (/* @__PURE__ */ new Date()).toISOString(),
 		skills_tree_sha: treeSha,
@@ -420,6 +560,46 @@ function cmdSummarize(argv) {
 	fs.writeFileSync(out, JSON.stringify(scorecard, null, 2) + "\n");
 	console.log(`${out}: ${merged.entries.length} scenario(s), ${merged.entries.filter((e) => e.pass).length} passing, ${skipped.length} skipped file(s)`);
 	if (existing.length > 0) console.log(`merged into today's scorecard: ${entries.length} from this run, ${merged.carried} carried over`);
+	console.log(`\n${formatSkillTable(summarizeSkills(merged.entries))}`);
+	for (const e of merged.entries.filter((x) => x.noisy)) console.log(`NOISY ${e.skill}/${e.scenario} (${e.harness}) ${formatStats(e)}`);
+}
+function summarizeSkills(entries) {
+	const groups = /* @__PURE__ */ new Map();
+	for (const e of entries) {
+		const key = `${e.skill}\0${e.harness}`;
+		groups.set(key, [...groups.get(key) ?? [], e]);
+	}
+	const mean = (xs) => round4(xs.reduce((a, b) => a + b, 0) / xs.length);
+	return [...groups.values()].map((rows) => ({
+		skill: rows[0].skill,
+		harness: rows[0].harness,
+		scenarios: rows.length,
+		pass_all: rows.filter((r) => r.pass).length,
+		pass_rate: mean(rows.map((r) => r.pass_rate ?? (r.pass ? 1 : 0))),
+		score: mean(rows.map((r) => r.score)),
+		noisy: rows.filter((r) => r.noisy).map((r) => r.scenario)
+	}));
+}
+function formatSkillTable(rows) {
+	const table = [[
+		"skill",
+		"harness",
+		"scenarios",
+		"pass^k",
+		"pass rate",
+		"score",
+		"noisy"
+	], ...rows.map((r) => [
+		r.skill,
+		r.harness,
+		String(r.scenarios),
+		`${r.pass_all}/${r.scenarios}`,
+		r.pass_rate.toFixed(2),
+		r.score.toFixed(2),
+		String(r.noisy.length)
+	])];
+	const widths = table[0].map((_, i) => Math.max(...table.map((row) => row[i].length)));
+	return table.map((row) => row.map((c, i) => c.padEnd(widths[i])).join("  ").trimEnd()).join("\n");
 }
 function cmdLint(argv) {
 	const { positional, flags } = parseArgs(argv);
@@ -455,4 +635,4 @@ if (isMainModule()) {
 	}
 }
 //#endregion
-export { classifyResult, mergeScorecard, parseArgs, parseMaxTurns, reduceResults, resolveRoot, stateDirs, toolVersion, treeShaOf };
+export { NOISY_SPREAD, aggregateTrials, assertUniformConfig, classifyResult, formatStats, mergeScorecard, parseArgs, parsePositiveInt, reduceResults, resolveRoot, resultRunConfig, runConfigOf, runOptions, stateDirs, summarizeSkills, toolVersion, treeShaOf };

package/dist/scenario.js

@@ -4,6 +4,7 @@ import path from "node:path";
 import { createHash } from "node:crypto";
 import os from "node:os";
 //#region src/scenario.ts
+const DEFAULT_CLAUDE_AGENT = "claude-opus-5";
 function loadScenario(scenarioDir) {
 	const match = scenarioDir.match(/skills\/([^/]+)\/evals\/([^/]+)$/);
 	if (!match) throw new Error(`not a scenario dir (want .../skills/<skill>/evals/<scenario>): ${scenarioDir}`);
@@ -146,6 +147,7 @@ function agentProvider(opts, workdir, skill, paths) {
 		id: "anthropic:claude-agent-sdk",
 		config: {
 			model: opts.agentModel ?? "claude-opus-5",
+			...opts.agentEffort ? { effort: opts.agentEffort } : {},
 			apiKeyRequired: false,
 			working_dir: workdir,
 			setting_sources: ["project"],
@@ -162,19 +164,29 @@ function agentProvider(opts, workdir, skill, paths) {
 		}
 	};
 }
-function buildConfig(s, workdir, manifestPath, opts, paths) {
+function trialLabel(index) {
+	return `trial-${index + 1}`;
+}
+function buildConfig(s, trials, opts, paths) {
 	return {
 		description: `${s.skill}/${s.scenario}`,
 		prompts: ["{{task}}"],
-		providers: [agentProvider(opts, workdir, s.skill, paths)],
+		providers: trials.map((t, i) => ({
+			...agentProvider(opts, t.workdir, s.skill, paths),
+			label: trialLabel(i)
+		})),
 		defaultTest: { options: {
 			provider: opts.judgeModel.includes(":") ? opts.judgeEffort === void 0 ? opts.judgeModel : {
 				id: opts.judgeModel,
 				config: { reasoning_effort: opts.judgeEffort }
-			} : process.env.ANTHROPIC_API_KEY ? `anthropic:messages:${opts.judgeModel}` : {
+			} : process.env.ANTHROPIC_API_KEY ? opts.judgeEffort === void 0 ? `anthropic:messages:${opts.judgeModel}` : {
+				id: `anthropic:messages:${opts.judgeModel}`,
+				config: { effort: opts.judgeEffort }
+			} : {
 				id: "anthropic:claude-agent-sdk",
 				config: {
 					model: opts.judgeModel,
+					...opts.judgeEffort ? { effort: opts.judgeEffort } : {},
 					apiKeyRequired: false,
 					max_turns: 3,
 					output_format: {
@@ -202,12 +214,13 @@ function buildConfig(s, workdir, manifestPath, opts, paths) {
 			},
 			transform: `file://${paths.transformPath}`
 		} },
-		tests: [{
+		tests: trials.map((t, i) => ({
 			description: s.criteria.context,
+			providers: [trialLabel(i)],
 			vars: {
 				task: s.prompt,
-				workdir,
-				manifest: manifestPath
+				workdir: t.workdir,
+				manifest: t.manifestPath
 			},
 			assert: [{
 				type: "assert-set",
@@ -221,7 +234,7 @@ function buildConfig(s, workdir, manifestPath, opts, paths) {
 				type: "skill-used",
 				value: s.skill
 			}]
-		}]
+		}))
 	};
 }
 const SDK_PACKAGES = ["@anthropic-ai/claude-agent-sdk", "@openai/codex-sdk"];
@@ -250,7 +263,11 @@ function generateRun(scenarioDir, opts, paths) {
 	const s = loadScenario(scenarioDir);
 	const name = runNameFor(scenarioDir, opts.harness);
 	const runDir = path.join(paths.scratchDir, name);
-	const { workdir, manifestPath } = materialize(s, runDir, opts.harness);
+	fs.rmSync(runDir, {
+		recursive: true,
+		force: true
+	});
+	const trials = Array.from({ length: opts.trials ?? 1 }, (_, i) => materialize(s, path.join(runDir, trialLabel(i)), opts.harness));
 	const sdkDir = sdkNodeModulesDir();
 	if (sdkDir !== void 0) {
 		const link = path.join(runDir, "node_modules");
@@ -260,7 +277,7 @@ function generateRun(scenarioDir, opts, paths) {
 		});
 		fs.symlinkSync(sdkDir, link, "dir");
 	}
-	const config = buildConfig(s, workdir, manifestPath, opts, paths);
+	const config = buildConfig(s, trials, opts, paths);
 	const configPath = path.join(runDir, "promptfooconfig.json");
 	fs.writeFileSync(configPath, JSON.stringify(config, null, 2));
 	return {
@@ -271,4 +288,4 @@ function generateRun(scenarioDir, opts, paths) {
 	};
 }
 //#endregion
-export { buildConfig, encodeRunNamePart, generateRun, isHiddenSkill, loadScenario, materialize, requiredEvalPackages, resolvePackageDir, runNameFor, sdkNodeModulesDir, stripHiddenFlag };
+export { DEFAULT_CLAUDE_AGENT, buildConfig, encodeRunNamePart, generateRun, isHiddenSkill, loadScenario, materialize, requiredEvalPackages, resolvePackageDir, runNameFor, sdkNodeModulesDir, stripHiddenFlag, trialLabel };

package/docs/adoption.md

@@ -66,9 +66,11 @@ Commit `.skillcheck/scorecards/`. Gitignore the rest:
 .skillcheck/scratch/
 ```
 
-A scorecard is only comparable against the tree it graded, which is why every
-result carries the root repo's HEAD and `summarize` refuses to mix revisions
-without `--allow-mixed`.
+A scorecard is only comparable against the tree it graded and the configuration
+it ran with, which is why every result carries the root repo's HEAD, the agent
+and judge models and efforts, and the trial count, and why `summarize` refuses
+to mix them without `--allow-mixed`. Use `--trials 3` or more before calling a
+skill change better or worse; one trial cannot tell a regression from noise.
 
 ## Upgrading
 

package/docs/scenarios.md

@@ -54,7 +54,9 @@ item needs a non-empty `name` and `description` and a positive `max_score`.
 Each item becomes one `llm-rubric` assertion weighted by `max_score`, inside an
 assert-set with threshold 0.7. A separate `skill-used` assertion sits outside
 that aggregate, so a run that produces good output without ever loading the
-skill still fails. There is no test-level threshold: both must pass.
+skill still fails. There is no test-level threshold: both must pass. The
+reported score is the assert-set's weighted score; skill-used is reported
+separately as a rate across trials.
 
 Write descriptions a judge can check against the deliverable: an observable
 property, not a feeling. Weight the items that would make a reviewer reject the

package/docs/usage.md

@@ -35,9 +35,10 @@ skillcheck run skills/<skill>/evals/<scenario>
 skillcheck run <scenario-dir> --agent MODEL --judge MODEL --harness codex
 skillcheck run <scenario-dir> --harness claude --max-turns 80
 skillcheck run <scenario-dir> --harness grok
+skillcheck run <scenario-dir> --trials 3 --agent-effort medium
 ```
 
-Materializes the scenario into `<root>/.skillcheck/scratch/<name>/workdir`,
+Materializes the scenario into `<root>/.skillcheck/scratch/<name>/trial-<n>/workdir`,
 installs the skill under test into that workdir, drives the agent, and grades
 the files it wrote. Exit 0 means pass, 1 means graded fail, and 2 means error.
 Exit 2 covers missing usable promptfoo output or optional eval peers. The
@@ -53,6 +54,44 @@ and a Claude agent limit of 50 turns. `--max-turns` changes that limit only for
 Claude; passing it with `codex` or `grok` fails before the eval starts. On
 those harnesses, omitting `--agent` leaves the model to that CLI's own default.
 
+### Trials
+
+One trial is one sample of a noisy process: the same scenario and skill can
+score 0.49 and then 0.99. `--trials <k>` (default 1) runs the agent k times,
+each in its own workdir with its own manifest, all graded in one promptfoo eval.
+promptfoo's `--repeat` is not used because it reuses one set of vars and one
+`working_dir`, so concurrent trials would write into the same tree and each
+would be graded on all of their deliverables.
+
+A scenario's result aggregates its trials:
+
+| Field                           | Meaning                                                             |
+| ------------------------------- | ------------------------------------------------------------------- |
+| `pass`                          | pass^k: every trial passed. Exit 0 needs this                       |
+| `passes`, `pass_rate`           | Trials that passed, as a count and a fraction                       |
+| `score`                         | Mean weighted checklist score (the assert-set, without skill-used)  |
+| `score_min`                     | Lowest trial score                                                  |
+| `score_spread`                  | Highest minus lowest trial score                                    |
+| `skill_used`, `skill_used_rate` | Trials whose `skill-used` assertion passed, as a count and fraction |
+| `noisy`                         | Trials both passed and failed, or `score_spread` is at least 0.2    |
+
+A trial that errored was never graded, so one errored trial makes the whole
+scenario an ERROR: pass^k over fewer than k trials is not the requested number.
+So is a result with fewer rows than trials, or a row without its checklist and
+`skill-used` components.
+With `--trials` above 1, `run` and `sweep` print min, spread, pass count,
+skill-used count, and `NOISY`.
+
+### Agent effort
+
+`--agent-effort low|medium|high|xhigh|max` sets the Claude agent's effort.
+promptfoo passes it to the Agent SDK, which starts Claude Code with `--effort`.
+Omitting it leaves Claude Code's default. Only `--harness claude` takes it;
+`codex` and `grok` fail before the eval starts. It is separate from
+`--judge-effort`.
+
+### Harnesses and judges
+
 `--harness grok` runs the locally installed Grok Build CLI in the disposable
 workdir with the skill under `.grok/skills/`. It uses native streaming events
 to count a completed read of that skill's `SKILL.md` as `skill-used` evidence.
@@ -70,9 +109,16 @@ skillcheck run <scenario-dir> --judge openai:chat:gpt-5.6-sol --judge-effort hig
 
 A provider-qualified judge authenticates through that provider's own env
 (`OPENAI_API_KEY`, plus `OPENAI_BASE_URL` for a gateway) and is recorded
-verbatim in the scorecard's `judge_model` column. `--judge-effort`
-(minimal|low|medium|high) sets `reasoning_effort` and requires a
-provider-qualified judge; the Anthropic judge does not take one.
+verbatim in the scorecard's `judge_model` column. For it, `--judge-effort`
+(minimal|low|medium|high) sets `reasoning_effort`. For a bare Claude judge,
+`--judge-effort` takes Claude's levels (low|medium|high|xhigh|max) and is passed
+as `effort` on either Anthropic path; the SDK judge starts Claude Code with
+`--effort`:
+
+```sh
+skillcheck run <scenario-dir> --agent claude-opus-5-5 --agent-effort medium \
+  --judge claude-opus-5-5 --judge-effort high --trials 3
+```
 
 ## Sweep
 
@@ -86,7 +132,14 @@ order, sequentially. A scenario needs both `task.md` and `criteria.json` to be
 discovered. Exit 2 if anything errored, 1 if anything failed, else 0.
 
 `EVALS_CONCURRENCY` is passed to promptfoo as `-j` (default 4). It parallelizes
-within one scenario, not across them.
+the trials of one scenario, not separate scenarios. To spread scenarios, start
+several `skillcheck run` processes; each scenario has its own result, attempt
+marker, and scratch directory.
+
+A scenario is skipped only when its completed result was graded with the same
+run configuration: agent model, agent effort, judge model, judge effort, and
+trial count. A result from another configuration is rerun and reported as
+`RERUN`.
 
 One known failure mode: judge calls through a gateway can drop at the transport
 layer ([uinaf/zebroid-infra#44](https://github.com/uinaf/zebroid-infra/issues/44)).
@@ -102,7 +155,10 @@ skillcheck summarize [--allow-mixed]
 
 Reduces `<root>/.skillcheck/results/*.json` into
 `<root>/.skillcheck/scorecards/<UTC-date>.json`: one entry per scenario with
-skill, scenario, harness, tree sha, score, pass, both models, latency, tokens.
+skill, scenario, harness, tree sha, the trial aggregate from [trials](#trials),
+the run configuration, mean latency per trial, and tokens summed over trials.
+It then prints one row per skill and harness: scenarios, pass^k count, mean pass
+rate, mean score, and noisy count. Each noisy scenario follows on its own line.
 
 If a scorecard for today already exists, the two are merged on
 `(skill, scenario, harness)`: entries from this run win, entries it did not
@@ -139,13 +195,23 @@ Each successful run writes a `<name>.meta.json` sidecar next to its result:
   "skill": "<skill directory name>",
   "scenario": "<scenario directory name>",
   "harness": "claude",
+  "agent_model": "claude-opus-5",
+  "agent_effort": "medium",
+  "judge_model": "claude-opus-5",
+  "judge_effort": null,
+  "trials": 3,
+  "aggregate": { "pass": false, "pass_rate": 0.6667, "score": 0.81, "...": "..." },
   "ran_at": "<ISO timestamp>",
   "tool_version": "<skillcheck version>"
 }
 ```
 
-`summarize` reads those sidecars and refuses to mix skills-tree revisions in one
-scorecard, including retained rows from partial reruns, unless `--allow-mixed`.
+`summarize` reads those sidecars and refuses to mix skills-tree revisions or
+run configurations (agent model and effort, judge model and effort, trials) in
+one scorecard, including retained rows from partial reruns, unless
+`--allow-mixed`. Configurations are compared within a harness, since harnesses
+differ by design. Sidecars written before run configurations were recorded fall
+back to the promptfoo config stored in the result.
 Rejection leaves the existing scorecard unchanged. With the override, the top-level `skills_tree_sha`
 becomes `mixed` and per-entry shas remain. A result with no sidecar reduces as
 `unattested`.
@@ -169,3 +235,10 @@ written inside the installed package.
 
 A bare `--judge` model stays on the Anthropic selection regardless of the
 agent harness; a provider-qualified `--judge` uses that provider's env instead.
+
+The Claude agent loads project settings only, so an `apiKeyHelper` or `env`
+block in the operator's `~/.claude/settings.json` never reaches it; the run
+fails with `Not logged in`. Export the gateway variables instead:
+`ANTHROPIC_BASE_URL` and `ANTHROPIC_AUTH_TOKEN` set to the helper's output.
+Running from inside a Claude Code session also leaks that session's
+`CLAUDECODE` and `CLAUDE_CODE_*` variables into the agent; run from a plain shell.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uinaf/skillcheck",
-  "version": "1.0.2",
+  "version": "1.2.0",
   "description": "Lint and eval harness for agent skills",
   "homepage": "https://github.com/uinaf/skillcheck#readme",
   "bugs": {
@@ -33,17 +33,17 @@
     "prepublishOnly": "pnpm run verify:full"
   },
   "devDependencies": {
-    "@anthropic-ai/claude-agent-sdk": "^0.3.233",
-    "@openai/codex-sdk": "^0.147.0",
-    "@types/node": "^26.2.0",
-    "promptfoo": "^0.122.0",
+    "@anthropic-ai/claude-agent-sdk": "^0.3.281",
+    "@openai/codex-sdk": "^0.156.1",
+    "@types/node": "^26.6.2",
+    "promptfoo": "^0.123.1",
     "vite": "catalog:",
     "vite-plus": "catalog:"
   },
   "peerDependencies": {
-    "@anthropic-ai/claude-agent-sdk": "^0.3.233",
-    "@openai/codex-sdk": "^0.147.0",
-    "promptfoo": "^0.122.0"
+    "@anthropic-ai/claude-agent-sdk": "^0.3.281",
+    "@openai/codex-sdk": "^0.156.1",
+    "promptfoo": "^0.123.1"
   },
   "peerDependenciesMeta": {
     "@anthropic-ai/claude-agent-sdk": {
@@ -56,8 +56,15 @@
       "optional": true
     }
   },
+  "devEngines": {
+    "runtime": {
+      "name": "node",
+      "version": "^24.11.0 || >=26.0.0",
+      "onFail": "error"
+    }
+  },
   "engines": {
     "node": ">=24"
   },
-  "packageManager": "pnpm@12.0.0"
+  "packageManager": "pnpm@12.4.2"
 }