@uinaf/skillcheck 1.6.1 → 1.7.0

package/dist/scenario.js

@@ -14,7 +14,11 @@ function loadScenario(scenarioDir) {
 	const taskMd = fs.readFileSync(path.join(scenarioDir, "task.md"), "utf8");
 	const criteria = JSON.parse(fs.readFileSync(path.join(scenarioDir, "criteria.json"), "utf8"));
 	if (criteria.type !== "weighted_checklist" || !Array.isArray(criteria.checklist) || criteria.checklist.length === 0) throw new Error(`unsupported or empty criteria in ${scenarioDir}`);
-	if (criteria.skill_use !== void 0 && criteria.skill_use !== "required" && criteria.skill_use !== "optional") throw new Error(`skill_use must be "required" or "optional" in ${scenarioDir}`);
+	if (criteria.skill_use !== void 0 && ![
+		"required",
+		"optional",
+		"forbidden"
+	].includes(criteria.skill_use)) throw new Error(`skill_use must be "required", "optional", or "forbidden" in ${scenarioDir}`);
 	for (const item of criteria.checklist) if (!(typeof item?.name === "string" && item.name.trim() !== "" && typeof item?.description === "string" && item.description.trim() !== "" && Number.isFinite(item?.max_score) && item.max_score > 0)) throw new Error(`invalid checklist item in ${scenarioDir}: ${JSON.stringify(item)}`);
 	const files = [];
 	const task = taskMd.replace(FILE_BLOCK, (_, name, content) => {
@@ -26,18 +30,41 @@ function loadScenario(scenarioDir) {
 	});
 	let prompt = task;
 	const skillDir = path.resolve(scenarioDir, "../..");
-	if (isHiddenSkill(fs.readFileSync(path.join(skillDir, "SKILL.md"), "utf8"))) prompt = `Use the ${skill} skill for this task.\n\n${prompt}`;
+	if (isHiddenSkill(fs.readFileSync(path.join(skillDir, "SKILL.md"), "utf8"))) {
+		if (criteria.skill_use === "forbidden" || criteria.install !== void 0) throw new Error(`a hidden skill is always invoked, so it has no routing to test: ${scenarioDir}`);
+		prompt = `Use the ${skill} skill for this task.\n\n${prompt}`;
+	}
+	const alternatives = resolveInstall(skillDir, criteria.install, scenarioDir);
 	return {
 		skill,
 		scenario,
 		name: `${skill}--${scenario}`,
 		skillDir,
+		alternatives,
 		prompt,
 		task,
 		files,
 		criteria
 	};
 }
+function resolveInstall(skillDir, install, scenarioDir) {
+	if (install === void 0) return [];
+	const skillsRoot = path.dirname(skillDir);
+	const self = path.basename(skillDir);
+	const invocable = (name) => {
+		const dir = path.join(skillsRoot, name);
+		const md = path.join(dir, "SKILL.md");
+		return name !== self && !name.startsWith(".") && fs.lstatSync(dir, { throwIfNoEntry: false })?.isDirectory() === true && fs.lstatSync(md, { throwIfNoEntry: false })?.isFile() === true && !isHiddenSkill(fs.readFileSync(md, "utf8"));
+	};
+	if (install === "all") {
+		const all = fs.readdirSync(skillsRoot).filter(invocable).sort();
+		if (all.length === 0) throw new Error(`install "all" finds no other model-invocable skill for ${scenarioDir}`);
+		return all;
+	}
+	if (!Array.isArray(install) || install.length === 0) throw new Error(`install must be "all" or a non-empty list of skills in ${scenarioDir}`);
+	for (const name of install) if (typeof name !== "string" || name.includes("/") || !invocable(name)) throw new Error(`install names ${JSON.stringify(name)}, not another model-invocable skill under the root, in ${scenarioDir}`);
+	return [...new Set(install)].sort();
+}
 function encodeRunNamePart(part) {
 	if (!part.includes("--") && !part.startsWith("-") && !part.endsWith("-") && !part.startsWith("~v2~")) return part;
 	return `~v2~${encodeURIComponent(part).replaceAll("-", "%2D").replaceAll("~", "%7E")}`;
@@ -83,6 +110,7 @@ const MARKDOWN_LINK = /\]\(\s*<([^>]+)>|\]\(\s*([^)\s]+)|^ {0,3}\[[^\]]+\]:\s*<?
 function linkedSiblings(skillDir) {
 	const skillsRoot = path.dirname(skillDir);
 	const realRoot = fs.realpathSync(skillsRoot);
+	const realSelf = fs.realpathSync(skillDir);
 	const self = path.basename(skillDir);
 	const found = /* @__PURE__ */ new Set([self]);
 	const pending = [skillDir];
@@ -94,7 +122,8 @@ function linkedSiblings(skillDir) {
 		const name = rel.split(path.sep)[0];
 		if (name === "" || name === ".." || path.isAbsolute(rel) || found.has(name)) continue;
 		if (!fs.existsSync(path.join(skillsRoot, name, "SKILL.md"))) continue;
-		if (path.dirname(fs.realpathSync(path.join(skillsRoot, name))) !== realRoot) continue;
+		const real = fs.realpathSync(path.join(skillsRoot, name));
+		if (path.dirname(real) !== realRoot || real === realSelf) continue;
 		found.add(name);
 		pending.push(path.join(skillsRoot, name));
 	}
@@ -146,9 +175,13 @@ function materialize(s, runDir, harness, control = false) {
 	}
 	const roots = control ? [] : harness === "codex" ? [".claude", ".agents"] : harness === "grok" ? [".grok"] : [".claude"];
 	const skillsRoot = path.dirname(s.skillDir);
-	const installed = control ? [] : [s.skill, ...linkedSiblings(s.skillDir)];
+	const installed = control ? [] : [.../* @__PURE__ */ new Set([
+		s.skill,
+		...linkedSiblings(s.skillDir),
+		...s.alternatives.flatMap((a) => [a, ...linkedSiblings(path.join(skillsRoot, a))])
+	])];
 	for (const root of roots) for (const name of installed) {
-		const from = path.join(skillsRoot, name);
+		const from = fs.realpathSync(path.join(skillsRoot, name));
 		fs.cpSync(from, path.join(workdir, root, "skills", name), {
 			recursive: true,
 			filter: (src) => src !== path.join(from, "evals")
@@ -177,7 +210,8 @@ function materialize(s, runDir, harness, control = false) {
 		manifestPath
 	};
 }
-function agentProvider(opts, workdir, skill, paths) {
+function agentProvider(opts, workdir, skills, paths) {
+	const [skill] = skills;
 	if (opts.harness === "grok") return {
 		id: `file://${paths.grokProviderPath}`,
 		config: {
@@ -211,7 +245,7 @@ function agentProvider(opts, workdir, skill, paths) {
 			apiKeyRequired: false,
 			working_dir: workdir,
 			setting_sources: ["project"],
-			...opts.control ? {} : { skills: [skill] },
+			...opts.control ? {} : { skills },
 			permission_mode: "acceptEdits",
 			append_allowed_tools: [
 				"Read",
@@ -235,7 +269,7 @@ function buildConfig(s, trials, opts, paths) {
 		description: `${s.skill}/${s.scenario}`,
 		prompts: ["{{task}}"],
 		providers: trials.map((t, i) => ({
-			...agentProvider(opts, t.workdir, s.skill, paths),
+			...agentProvider(opts, t.workdir, [s.skill, ...s.alternatives], paths),
 			label: trialLabel(i)
 		})),
 		defaultTest: { options: {
@@ -299,7 +333,8 @@ function buildConfig(s, trials, opts, paths) {
 				metric: "skill-used",
 				config: {
 					skill: s.skill,
-					required: !opts.control && s.criteria.skill_use !== "optional"
+					required: !opts.control && (s.criteria.skill_use ?? "required") === "required",
+					...!opts.control && s.criteria.skill_use === "forbidden" ? { forbidden: true } : {}
 				}
 			}]
 		}))
@@ -382,6 +417,7 @@ function handOverToAgent(runDir, trials) {
 }
 function generateRun(scenarioDir, opts, paths) {
 	const s = loadScenario(scenarioDir);
+	if (opts.harness === "grok" && s.criteria.skill_use === "forbidden") throw new Error(`grok cannot evidence a skill load reliably enough for skill_use "forbidden": ${scenarioDir}`);
 	const name = runNameFor(scenarioDir, opts.harness, opts.control);
 	const runDir = path.join(paths.scratchDir, name);
 	fs.rmSync(runDir, {

package/dist/skill-evidence.js

@@ -34,17 +34,19 @@ function skillEvidence(context) {
 		const target = real(named);
 		return inside && target !== void 0 && installed.has(target);
 	})) return `read of the installed ${skill}/SKILL.md`;
-	const nameLine = new RegExp(`^name:\\s*["']?${skill.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}["']?\\s*$`, "m");
-	if (calls(metadata?.toolCalls).find((c) => c.name === "Bash" && c.is_error === false && typeof c.input?.command === "string" && c.input.command.includes("SKILL.md") && typeof c.output === "string" && nameLine.test(c.output))) return `shell read of the installed ${skill}/SKILL.md`;
+	const opening = [...installed].map((p) => fs.readFileSync(p, "utf8").slice(0, 300).trim()).filter((t) => t.length >= 40);
+	if (calls(metadata?.toolCalls).find((c) => c.name === "Bash" && c.is_error === false && typeof c.output === "string" && opening.some((t) => c.output.includes(t)))) return `shell read of the installed ${skill}/SKILL.md`;
 }
 function assertSkillUsed(_output, context) {
 	const evidence = skillEvidence(context);
-	const required = context.config?.required !== false;
+	const forbidden = context.config?.forbidden === true;
+	const required = !forbidden && context.config?.required !== false;
 	const used = evidence !== void 0;
+	const skill = String(context.config?.skill);
 	return {
-		pass: used || !required,
+		pass: forbidden ? !used : used || !required,
 		score: used ? 1 : 0,
-		reason: used ? `skill used: ${evidence}` : `skill ${String(context.config?.skill)} not loaded${required ? "" : " (optional for this scenario)"}`
+		reason: used ? `skill used: ${evidence}${forbidden ? " (forbidden for this scenario)" : ""}` : `skill ${skill} not loaded${required || forbidden ? "" : " (optional for this scenario)"}`
 	};
 }
 //#endregion

package/docs/adoption.md

@@ -63,7 +63,9 @@ skillcheck sweep            # resumes: only scenarios without results
 skillcheck summarize        # writes .skillcheck/scorecards/<UTC-date>.json
 ```
 
-Commit `.skillcheck/scorecards/`. Gitignore the rest:
+Keep `.skillcheck/scorecards/` as the history, either committed here or in a
+separate evals repository that mounts it as the state directory
+([isolated runs](usage.md#isolated-runs)). Gitignore the rest:
 
 ```gitignore
 .skillcheck/results/

package/docs/scenarios.md

@@ -58,7 +58,7 @@ that aggregate, so a run that produces good output without ever loading the
 skill still fails. The skill counts as loaded when the harness reports a skill
 call for it, or when the agent completed a read of the installed
 `<config-root>/skills/<skill>/SKILL.md` in its workdir, or a successful shell
-command that names `SKILL.md` and printed this skill's frontmatter `name`.
+command whose output carries the installed file's opening text.
 Claude Code often reads the file directly instead of calling its Skill tool,
 sometimes from a shell, and either way the same instructions reach its context.
 
@@ -69,6 +69,26 @@ and the result still records whether the skill was loaded. Omitted, it is
 reported score is the assert-set's weighted score; skill-used is reported
 separately as a rate across trials.
 
+## Trigger scenarios
+
+A trigger scenario measures routing: whether a natural prompt loads this skill
+when adjacent skills compete for it. `"install"` names other skills under the
+same root to install beside it, or `"all"` installs every model-invocable one,
+as a plugin does. Alternatives must be real directories, not symlinks. A hidden
+skill is never an alternative and cannot be the skill under test: production
+loads it only on an explicit invocation, so there is no routing to measure.
+
+- Positive: `"skill_use": "required"` (the default) with the alternatives
+  installed. It fails when the agent routes elsewhere or loads nothing.
+- Near miss: `"skill_use": "forbidden"` on a prompt that belongs to a
+  neighbor or to no skill. It fails when this skill loads, and the skill-used
+  rate then counts the misfires. Not on Grok, whose load evidence is too thin
+  to tell an unloaded skill from an unseen read.
+
+Keep the checklist on the outcome the right lane should produce, so a near
+miss that loads nothing but does the work badly still fails. A control run of
+either installs nothing and requires nothing.
+
 Write descriptions a judge can check against the deliverable: an observable
 property, not a feeling. Weight the items that would make a reviewer reject the
 work.

package/docs/usage.md

@@ -107,15 +107,15 @@ would be graded on all of their deliverables.
 
 A scenario's result aggregates its trials:
 
-| Field                           | Meaning                                                             |
-| ------------------------------- | ------------------------------------------------------------------- |
-| `pass`                          | pass^k: every trial passed. Exit 0 needs this                       |
-| `passes`, `pass_rate`           | Trials that passed, as a count and a fraction                       |
-| `score`                         | Mean weighted checklist score (the assert-set, without skill-used)  |
-| `score_min`                     | Lowest trial score                                                  |
-| `score_spread`                  | Highest minus lowest trial score                                    |
-| `skill_used`, `skill_used_rate` | Trials whose `skill-used` assertion passed, as a count and fraction |
-| `noisy`                         | Trials both passed and failed, or `score_spread` is at least 0.2    |
+| Field                           | Meaning                                                                                           |
+| ------------------------------- | ------------------------------------------------------------------------------------------------- |
+| `pass`                          | pass^k: every trial passed. Exit 0 needs this                                                     |
+| `passes`, `pass_rate`           | Trials that passed, as a count and a fraction                                                     |
+| `score`                         | Mean weighted checklist score (the assert-set, without skill-used)                                |
+| `score_min`                     | Lowest trial score                                                                                |
+| `score_spread`                  | Highest minus lowest trial score                                                                  |
+| `skill_used`, `skill_used_rate` | Trials that loaded the skill, as a count and fraction; on a near-miss scenario these are misfires |
+| `noisy`                         | Trials both passed and failed, or `score_spread` is at least 0.2                                  |
 
 A trial that errored was never graded, so one errored trial makes the whole
 scenario an ERROR: pass^k over fewer than k trials is not the requested number.
@@ -281,7 +281,10 @@ becomes `mixed` and per-entry shas remain. A result with no sidecar reduces as
 ## State
 
 `<root>/.skillcheck/` holds `results/`, disposable and safe to gitignore, and
-`scorecards/`, which is meant to be committed. Scratch workdirs live under the
+`scorecards/`, the history to keep. In the isolated image it is a separate
+writable mount, so it can live outside the repository under test: a separate
+evals repository can pin the skill repository to a commit, keep its scorecards,
+and leave the skill repository with only scenarios and `skillcheck lint`. Scratch workdirs live under the
 system temp dir, outside the root. Nothing is ever written inside the installed
 package.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uinaf/skillcheck",
-  "version": "1.6.1",
+  "version": "1.7.0",
   "description": "Lint and eval harness for agent skills",
   "homepage": "https://github.com/uinaf/skillcheck#readme",
   "bugs": {