@uinaf/skillcheck 1.4.1 → 1.5.0

package/dist/cli.js

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
-import { lintSkills } from "./lint.js";
 import { encodeRunNamePart, generateRun, requiredEvalPackages, resolvePackageDir, runNameFor } from "./scenario.js";
+import { lintSkills } from "./lint.js";
 import { execFileSync, spawnSync } from "node:child_process";
 import { createHash } from "node:crypto";
 import fs from "node:fs";

package/dist/lint.js

@@ -1,3 +1,4 @@
+import { FILE_BLOCK } from "./scenario.js";
 import fs from "node:fs";
 import path from "node:path";
 //#region src/lint.ts
@@ -61,19 +62,36 @@ function lintSkill(dir, root, errors) {
 function lintSkills(root) {
 	const roots = ["skills", ...fs.globSync("cli/*/skills", { cwd: root })].map((r) => path.join(root, r));
 	const errors = [];
-	let count = 0;
+	const skills = [];
 	for (const dir of roots) {
 		if (!fs.existsSync(dir)) continue;
 		for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
 			if (!entry.isDirectory() || entry.name.startsWith(".")) continue;
 			lintSkill(path.join(dir, entry.name), root, errors);
-			count++;
+			skills.push(path.join(dir, entry.name));
 		}
 	}
+	lintTasks(skills, root, errors);
 	return {
 		errors,
-		count
+		count: skills.length
 	};
 }
+function lintTasks(skills, root, errors) {
+	const names = skills.map((dir) => path.basename(dir));
+	for (const dir of skills) {
+		const evals = path.join(dir, "evals");
+		if (!fs.existsSync(evals)) continue;
+		for (const scenario of fs.readdirSync(evals, { withFileTypes: true })) {
+			const file = path.join(evals, scenario.name, "task.md");
+			if (!scenario.isDirectory() || !fs.existsSync(file)) continue;
+			const text = fs.readFileSync(file, "utf8").replace(FILE_BLOCK, "");
+			for (const name of names) {
+				const escaped = name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+				if (new RegExp(`(?<![\\w-])${escaped}(?![\\w-])`, "i").test(text)) errors.push(`${path.relative(root, file)}: task names skill ${name}`);
+			}
+		}
+	}
+}
 //#endregion
 export { lintSkills };

package/dist/scenario.js

@@ -4,6 +4,7 @@ import fs from "node:fs";
 import os from "node:os";
 import path from "node:path";
 //#region src/scenario.ts
+const FILE_BLOCK = /^=+ FILE: (.+?) =+\n([\s\S]*?)\n=+ END FILE =+$/gm;
 const DEFAULT_CLAUDE_AGENT = "claude-opus-5";
 function loadScenario(scenarioDir) {
 	const match = scenarioDir.match(/skills\/([^/]+)\/evals\/([^/]+)$/);
@@ -15,7 +16,7 @@ function loadScenario(scenarioDir) {
 	if (criteria.skill_use !== void 0 && criteria.skill_use !== "required" && criteria.skill_use !== "optional") throw new Error(`skill_use must be "required" or "optional" in ${scenarioDir}`);
 	for (const item of criteria.checklist) if (!(typeof item?.name === "string" && item.name.trim() !== "" && typeof item?.description === "string" && item.description.trim() !== "" && Number.isFinite(item?.max_score) && item.max_score > 0)) throw new Error(`invalid checklist item in ${scenarioDir}: ${JSON.stringify(item)}`);
 	const files = [];
-	const task = taskMd.replace(/^=+ FILE: (.+?) =+\n([\s\S]*?)\n=+ END FILE =+$/gm, (_, name, content) => {
+	const task = taskMd.replace(FILE_BLOCK, (_, name, content) => {
 		files.push({
 			name: name.trim(),
 			content: content + "\n"
@@ -322,4 +323,4 @@ function generateRun(scenarioDir, opts, paths) {
 	};
 }
 //#endregion
-export { DEFAULT_CLAUDE_AGENT, buildConfig, encodeRunNamePart, generateRun, isHiddenSkill, loadScenario, materialize, privateCodexHome, requiredEvalPackages, resolvePackageDir, runNameFor, sdkNodeModulesDir, stripHiddenFlag, trialLabel };
+export { DEFAULT_CLAUDE_AGENT, FILE_BLOCK, buildConfig, encodeRunNamePart, generateRun, isHiddenSkill, loadScenario, materialize, privateCodexHome, requiredEvalPackages, resolvePackageDir, runNameFor, sdkNodeModulesDir, stripHiddenFlag, trialLabel };

package/docs/scenarios.md

@@ -35,6 +35,7 @@ own examiner or embed generated dependencies.
 
 Write the task the way a user would write it. Do not name the skill, describe
 its steps, or hint at the checklist: routing is part of what is being measured.
+`skillcheck lint` rejects a task whose prose names any skill in the root.
 
 ## criteria.json
 

package/docs/usage.md

@@ -20,6 +20,9 @@ Checks each `<root>/skills/<skill>/`:
 - `disable-model-invocation`, when present, is the bare YAML boolean `true`.
   A quoted `"true"` is an error
 - Relative links in the body resolve on disk
+- No eval `task.md` names a skill from the root, outside its inline input
+  files. A named skill gives routing away, and a no-skill control can read
+  the operator's installed copy
 
 Code spans and fenced blocks are stripped before links are checked, so example
 links never fail. External schemes and `#anchors` pass. Dot-directories under

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uinaf/skillcheck",
-  "version": "1.4.1",
+  "version": "1.5.0",
   "description": "Lint and eval harness for agent skills",
   "homepage": "https://github.com/uinaf/skillcheck#readme",
   "bugs": {