@uinaf/skillcheck 1.1.0 → 1.3.0

package/README.md

@@ -45,7 +45,7 @@ current directory.
 <root>/skills/<skill>/SKILL.md                       linted
 <root>/skills/<skill>/evals/<scenario>/task.md       the problem + input files
 <root>/skills/<skill>/evals/<scenario>/criteria.json the weighted checklist
-<root>/.skillcheck/                                  results, scratch, scorecards
+<root>/.skillcheck/                                  results, scorecards
 ```
 
 `cli/*/skills/<skill>/` is scanned too, for repos that keep a skill next to the

package/dist/cli.js

@@ -2,7 +2,9 @@
 import { lintSkills } from "./lint.js";
 import { encodeRunNamePart, generateRun, requiredEvalPackages, resolvePackageDir, runNameFor } from "./scenario.js";
 import { execFileSync, spawnSync } from "node:child_process";
+import { createHash } from "node:crypto";
 import fs from "node:fs";
+import os from "node:os";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
 //#region src/cli.ts
@@ -61,9 +63,10 @@ function resolveRoot(flags) {
 }
 function stateDirs(root) {
 	const base = path.join(root, ".skillcheck");
+	const id = createHash("sha256").update(path.resolve(root)).digest("hex").slice(0, 12);
 	return {
 		results: path.join(base, "results"),
-		scratch: path.join(base, "scratch"),
+		scratch: path.join(fs.realpathSync(os.tmpdir()), `skillcheck-${id}`),
 		scorecards: path.join(base, "scorecards")
 	};
 }
@@ -80,13 +83,13 @@ function runOptions(flags) {
 	const judgeModel = flags.get("--judge") ?? "claude-opus-5";
 	const judgeEffort = flags.get("--judge-effort");
 	if (judgeEffort !== void 0) {
-		if (![
+		const levels = judgeModel.includes(":") ? [
 			"minimal",
 			"low",
 			"medium",
 			"high"
-		].includes(judgeEffort)) throw new Error(`--judge-effort must be minimal, low, medium, or high, got ${judgeEffort}`);
-		if (!judgeModel.includes(":")) throw new Error("--judge-effort needs a provider-qualified --judge (e.g. openai:chat:gpt-5.6-sol); the Anthropic judge does not take a reasoning effort");
+		] : AGENT_EFFORTS;
+		if (!levels.includes(judgeEffort)) throw new Error(`--judge-effort for ${judgeModel} must be ${levels.join(", ")}, got ${judgeEffort}`);
 	}
 	return {
 		harness,
@@ -180,12 +183,12 @@ function classifyRow(raw, stats) {
 	if (typeof res.score !== "number" || typeof res.success !== "boolean") return { error: "promptfoo result carried no usable score" };
 	const components = Array.isArray(res.gradingResult?.componentResults) ? res.gradingResult.componentResults : [];
 	const checklist = components.find((c) => c?.metadata?.assertionSet?.type === "assert-set");
-	const skillUsed = components.find((c) => c?.assertion?.type === "skill-used");
-	if (typeof checklist?.score !== "number" || typeof skillUsed?.pass !== "boolean") return { error: "promptfoo result carried no checklist or skill-used verdict" };
+	const skillUsed = components.find((c) => c?.assertion?.type === "skill-used" || c?.assertion?.metric === "skill-used");
+	if (typeof checklist?.score !== "number" || typeof skillUsed?.score !== "number") return { error: "promptfoo result carried no checklist or skill-used verdict" };
 	return {
 		score: checklist.score,
 		pass: res.success,
-		skillUsed: skillUsed.pass
+		skillUsed: skillUsed.score >= 1
 	};
 }
 const NOISY_SPREAD = .2;
@@ -235,12 +238,24 @@ function metaPath(resultPath) {
 function attemptPath(resultPath) {
 	return `${resultPath}.attempt`;
 }
+function ensurePrivateDir(dir) {
+	fs.mkdirSync(dir, {
+		recursive: true,
+		mode: 448
+	});
+	const st = fs.lstatSync(dir);
+	const uid = process.getuid?.();
+	if (!st.isDirectory() || uid !== void 0 && st.uid !== uid) throw new Error(`scratch dir ${dir} is not a directory owned by this user`);
+	if ((st.mode & 63) !== 0) fs.chmodSync(dir, 448);
+}
 function runScenario(scenarioDir, opts, root) {
 	const dirs = stateDirs(root);
+	ensurePrivateDir(dirs.scratch);
 	const { name, configPath, skill, scenario } = generateRun(path.resolve(scenarioDir), opts, {
 		scratchDir: dirs.scratch,
 		transformPath: path.join(here, `transform${selfExt}`),
-		grokProviderPath: path.join(here, `grok-provider${selfExt}`)
+		grokProviderPath: path.join(here, `grok-provider${selfExt}`),
+		skillEvidencePath: path.join(here, `skill-evidence${selfExt}`)
 	});
 	fs.mkdirSync(dirs.results, { recursive: true });
 	const resultPath = path.join(dirs.results, `${name}.json`);
@@ -304,7 +319,8 @@ function judgeName(judge) {
 	if (typeof judge === "string") return judge.replace(/^anthropic:messages:/, "");
 	const j = judge;
 	const name = j?.config?.model ?? j?.id;
-	return typeof name === "string" ? name : "unknown";
+	if (typeof name !== "string") return "unknown";
+	return j?.config?.effort === void 0 ? name : name.replace(/^anthropic:messages:/, "");
 }
 function resultRunConfig(raw, meta, harness) {
 	const m = meta;
@@ -318,7 +334,8 @@ function resultRunConfig(raw, meta, harness) {
 	const r = raw;
 	const agent = r?.config?.providers?.[0]?.config;
 	const judge = r?.config?.defaultTest?.options?.provider;
-	const judgeEffort = judge?.config?.reasoning_effort;
+	const judgeConfig = judge?.config;
+	const judgeEffort = judgeConfig?.reasoning_effort ?? judgeConfig?.effort;
 	return {
 		agent_model: typeof agent?.model === "string" ? agent.model : `${harness}-default`,
 		agent_effort: typeof agent?.effort === "string" ? agent.effort : null,
@@ -633,4 +650,4 @@ if (isMainModule()) {
 	}
 }
 //#endregion
-export { NOISY_SPREAD, aggregateTrials, assertUniformConfig, classifyResult, formatStats, mergeScorecard, parseArgs, parsePositiveInt, reduceResults, resolveRoot, resultRunConfig, runConfigOf, runOptions, stateDirs, summarizeSkills, toolVersion, treeShaOf };
+export { NOISY_SPREAD, aggregateTrials, assertUniformConfig, classifyResult, ensurePrivateDir, formatStats, mergeScorecard, parseArgs, parsePositiveInt, reduceResults, resolveRoot, resultRunConfig, runConfigOf, runOptions, stateDirs, summarizeSkills, toolVersion, treeShaOf };

package/dist/scenario.js

@@ -1,8 +1,8 @@
 import { createRequire } from "node:module";
-import fs from "node:fs";
-import path from "node:path";
 import { createHash } from "node:crypto";
+import fs from "node:fs";
 import os from "node:os";
+import path from "node:path";
 //#region src/scenario.ts
 const DEFAULT_CLAUDE_AGENT = "claude-opus-5";
 function loadScenario(scenarioDir) {
@@ -12,6 +12,7 @@ function loadScenario(scenarioDir) {
 	const taskMd = fs.readFileSync(path.join(scenarioDir, "task.md"), "utf8");
 	const criteria = JSON.parse(fs.readFileSync(path.join(scenarioDir, "criteria.json"), "utf8"));
 	if (criteria.type !== "weighted_checklist" || !Array.isArray(criteria.checklist) || criteria.checklist.length === 0) throw new Error(`unsupported or empty criteria in ${scenarioDir}`);
+	if (criteria.skill_use !== void 0 && criteria.skill_use !== "required" && criteria.skill_use !== "optional") throw new Error(`skill_use must be "required" or "optional" in ${scenarioDir}`);
 	for (const item of criteria.checklist) if (!(typeof item?.name === "string" && item.name.trim() !== "" && typeof item?.description === "string" && item.description.trim() !== "" && Number.isFinite(item?.max_score) && item.max_score > 0)) throw new Error(`invalid checklist item in ${scenarioDir}: ${JSON.stringify(item)}`);
 	const files = [];
 	let prompt = taskMd.replace(/^=+ FILE: (.+?) =+\n([\s\S]*?)\n=+ END FILE =+$/gm, (_, name, content) => {
@@ -179,10 +180,14 @@ function buildConfig(s, trials, opts, paths) {
 			provider: opts.judgeModel.includes(":") ? opts.judgeEffort === void 0 ? opts.judgeModel : {
 				id: opts.judgeModel,
 				config: { reasoning_effort: opts.judgeEffort }
-			} : process.env.ANTHROPIC_API_KEY ? `anthropic:messages:${opts.judgeModel}` : {
+			} : process.env.ANTHROPIC_API_KEY ? opts.judgeEffort === void 0 ? `anthropic:messages:${opts.judgeModel}` : {
+				id: `anthropic:messages:${opts.judgeModel}`,
+				config: { effort: opts.judgeEffort }
+			} : {
 				id: "anthropic:claude-agent-sdk",
 				config: {
 					model: opts.judgeModel,
+					...opts.judgeEffort ? { effort: opts.judgeEffort } : {},
 					apiKeyRequired: false,
 					max_turns: 3,
 					output_format: {
@@ -227,8 +232,13 @@ function buildConfig(s, trials, opts, paths) {
 					weight: item.max_score
 				}))
 			}, {
-				type: "skill-used",
-				value: s.skill
+				type: "javascript",
+				value: `file://${paths.skillEvidencePath}`,
+				metric: "skill-used",
+				config: {
+					skill: s.skill,
+					required: s.criteria.skill_use !== "optional"
+				}
 			}]
 		}))
 	};

package/dist/skill-evidence.js

@@ -0,0 +1,49 @@
+import fs from "node:fs";
+import path from "node:path";
+//#region src/skill-evidence.ts
+const SKILL_ROOTS = [
+	".claude",
+	".agents",
+	".grok"
+];
+function real(file) {
+	try {
+		return fs.realpathSync(file);
+	} catch {
+		return;
+	}
+}
+function skillEvidence(context) {
+	const skill = context.config?.skill;
+	const workdir = context.vars.workdir;
+	if (typeof skill !== "string" || typeof workdir !== "string") return void 0;
+	const calls = (value) => Array.isArray(value) ? value.filter((c) => c !== null && typeof c === "object") : [];
+	const metadata = context.metadata ?? context.providerResponse?.metadata;
+	if (calls(metadata?.skillCalls).find((c) => c.name === skill && c.is_error !== true)) return `skill call ${skill}`;
+	const installed = new Set(SKILL_ROOTS.map((root) => real(path.join(workdir, root, "skills", skill, "SKILL.md"))).filter((p) => p !== void 0));
+	const root = real(workdir) ?? workdir;
+	if (calls(metadata?.toolCalls).find((c) => {
+		if (c.name !== "Read" || c.is_error !== false || typeof c.output !== "string") return false;
+		const file = c.input?.file_path;
+		if (typeof file !== "string") return false;
+		const named = path.resolve(workdir, file);
+		const inside = [workdir, root].some((w) => {
+			const rel = path.relative(w, named);
+			return rel !== ".." && !rel.startsWith(`..${path.sep}`) && !path.isAbsolute(rel);
+		});
+		const target = real(named);
+		return inside && target !== void 0 && installed.has(target);
+	})) return `read of the installed ${skill}/SKILL.md`;
+}
+function assertSkillUsed(_output, context) {
+	const evidence = skillEvidence(context);
+	const required = context.config?.required !== false;
+	const used = evidence !== void 0;
+	return {
+		pass: used || !required,
+		score: used ? 1 : 0,
+		reason: used ? `skill used: ${evidence}` : `skill ${String(context.config?.skill)} not loaded${required ? "" : " (optional for this scenario)"}`
+	};
+}
+//#endregion
+export { assertSkillUsed as default, skillEvidence };

package/dist/transform.js

@@ -1,9 +1,9 @@
+import { createHash } from "node:crypto";
 import fs from "node:fs";
 import path from "node:path";
-import { createHash } from "node:crypto";
 //#region src/transform.ts
-const PER_FILE_CAP = 4e3;
-const TOTAL_CAP = 24e3;
+const PER_FILE_CAP = 16e3;
+const TOTAL_CAP = 64e3;
 function safeSlice(text, end) {
 	let cut = text.slice(0, end);
 	const last = cut.charCodeAt(cut.length - 1);

package/docs/adoption.md

@@ -63,7 +63,6 @@ Commit `.skillcheck/scorecards/`. Gitignore the rest:
 
 ```gitignore
 .skillcheck/results/
-.skillcheck/scratch/
 ```
 
 A scorecard is only comparable against the tree it graded and the configuration

package/docs/scenarios.md

@@ -54,7 +54,16 @@ item needs a non-empty `name` and `description` and a positive `max_score`.
 Each item becomes one `llm-rubric` assertion weighted by `max_score`, inside an
 assert-set with threshold 0.7. A separate `skill-used` assertion sits outside
 that aggregate, so a run that produces good output without ever loading the
-skill still fails. There is no test-level threshold: both must pass. The
+skill still fails. The skill counts as loaded when the harness reports a skill
+call for it, or when the agent completed a read of the installed
+`<config-root>/skills/<skill>/SKILL.md` in its workdir. Claude Code often reads
+the file directly instead of calling its Skill tool, and either way the same
+instructions reach its context.
+
+An out-of-lane scenario, where the right answer is to decline the skill, sets
+`"skill_use": "optional"` in `criteria.json`. The assertion then always passes,
+and the result still records whether the skill was loaded. Omitted, it is
+`"required"`. There is no test-level threshold: both must pass. The
 reported score is the assert-set's weighted score; skill-used is reported
 separately as a rate across trials.
 
@@ -62,14 +71,27 @@ Write descriptions a judge can check against the deliverable: an observable
 property, not a feeling. Weight the items that would make a reviewer reject the
 work.
 
+On the Claude harness the agent can read, search, and write files in its
+workdir, but has no shell, so it cannot install, build, test, or reach the
+network. The shell stays off because the agent runs on the operator's machine
+with the operator's credentials. Codex runs shell commands inside its
+`workspace-write` sandbox, and Grok follows its own CLI permission mode. Write criteria a shell-less agent can meet, so one scenario
+scores comparably across harnesses: a checklist item that requires live proof,
+such as a frozen lockfile or a verified release, fails on Claude. Grade whether
+the deliverable names the checks it could not run and hands them over
+precisely.
+
+Name a specific tool or version only when the skill teaches it. Otherwise grade
+the property the tool provides, so an equivalent approach passes.
+
 ## What the judge sees
 
 The agent's final message, plus every file in the workdir that differs from the
 pre-run manifest. Unchanged inputs are omitted; deleted inputs, unreadable
 files, and non-regular files are named rather than read.
 
-Sections are sorted by path, each file is capped at 4,000 characters and the
-appended total at 24,000, with truncation stated inline. Very large outputs make
+Sections are sorted by path, each file is capped at 16,000 characters and the
+appended total at 64,000, with truncation stated inline. Very large outputs make
 rubric judges return nothing at all, which is why the caps exist. Keep fixtures
 small enough that the deliverable fits.
 
@@ -84,8 +106,10 @@ frontmatter block; body text mentioning the key does not count.
 
 ## The workdir
 
-Per run, under `<root>/.skillcheck/scratch/<name>/`, rebuilt from scratch each
-time. The skill under test is installed where the harness discovers skills:
+Per run, under a scratch directory in the system temp dir
+(`skillcheck-<hash of the root>/<name>/trial-<n>/`), rebuilt from scratch each
+time. It stays outside the root so the agent cannot reach the skill's source,
+its evals, or the repository's own agent guidance. The skill under test is installed where the harness discovers skills:
 `.claude/skills/<skill>/`, plus `.agents/skills/<skill>/` on codex, or
 `.grok/skills/<skill>/` on Grok, with its `evals/` directory excluded, so
 criteria never leak into the agent's context.

package/docs/usage.md

@@ -38,7 +38,7 @@ skillcheck run <scenario-dir> --harness grok
 skillcheck run <scenario-dir> --trials 3 --agent-effort medium
 ```
 
-Materializes the scenario into `<root>/.skillcheck/scratch/<name>/trial-<n>/workdir`,
+Materializes the scenario into a temp-dir workdir per trial ([workdir](scenarios.md#the-workdir)),
 installs the skill under test into that workdir, drives the agent, and grades
 the files it wrote. Exit 0 means pass, 1 means graded fail, and 2 means error.
 Exit 2 covers missing usable promptfoo output or optional eval peers. The
@@ -109,9 +109,16 @@ skillcheck run <scenario-dir> --judge openai:chat:gpt-5.6-sol --judge-effort hig
 
 A provider-qualified judge authenticates through that provider's own env
 (`OPENAI_API_KEY`, plus `OPENAI_BASE_URL` for a gateway) and is recorded
-verbatim in the scorecard's `judge_model` column. `--judge-effort`
-(minimal|low|medium|high) sets `reasoning_effort` and requires a
-provider-qualified judge; the Anthropic judge does not take one.
+verbatim in the scorecard's `judge_model` column. For it, `--judge-effort`
+(minimal|low|medium|high) sets `reasoning_effort`. For a bare Claude judge,
+`--judge-effort` takes Claude's levels (low|medium|high|xhigh|max) and is passed
+as `effort` on either Anthropic path; the SDK judge starts Claude Code with
+`--effort`:
+
+```sh
+skillcheck run <scenario-dir> --agent claude-opus-5-5 --agent-effort medium \
+  --judge claude-opus-5-5 --judge-effort high --trials 3
+```
 
 ## Sweep
 
@@ -211,9 +218,10 @@ becomes `mixed` and per-entry shas remain. A result with no sidecar reduces as
 
 ## State
 
-`<root>/.skillcheck/` holds `scratch/` and `results/`, both disposable and safe
-to gitignore, and `scorecards/`, which is meant to be committed. Nothing is ever
-written inside the installed package.
+`<root>/.skillcheck/` holds `results/`, disposable and safe to gitignore, and
+`scorecards/`, which is meant to be committed. Scratch workdirs live under the
+system temp dir, outside the root. Nothing is ever written inside the installed
+package.
 
 ## Auth
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uinaf/skillcheck",
-  "version": "1.1.0",
+  "version": "1.3.0",
   "description": "Lint and eval harness for agent skills",
   "homepage": "https://github.com/uinaf/skillcheck#readme",
   "bugs": {