@uinaf/skillcheck 1.4.0 → 1.5.0

package/dist/cli.js

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
-import { lintSkills } from "./lint.js";
 import { encodeRunNamePart, generateRun, requiredEvalPackages, resolvePackageDir, runNameFor } from "./scenario.js";
+import { lintSkills } from "./lint.js";
 import { execFileSync, spawnSync } from "node:child_process";
 import { createHash } from "node:crypto";
 import fs from "node:fs";
@@ -357,6 +357,17 @@ function readJson(file) {
 		return;
 	}
 }
+function liveScenarios(root) {
+	const cli = path.join(root, "cli");
+	if (!(fs.existsSync(path.join(root, "skills")) || fs.existsSync(cli) && fs.readdirSync(cli).some((d) => fs.existsSync(path.join(cli, d, "skills"))))) return void 0;
+	return new Set(discoverScenarios(root).map((dir) => {
+		const parts = dir.split(path.sep);
+		return `${parts.at(-3)}\0${parts.at(-1)}`;
+	}));
+}
+function isLive(live, e) {
+	return live === void 0 || live.has(`${e.skill}\0${e.scenario}`);
+}
 function discoverScenarios(root) {
 	const roots = [path.join(root, "skills")];
 	const cliDir = path.join(root, "cli");
@@ -464,15 +475,26 @@ function resultIdentity(file, dir) {
 		variant
 	};
 }
-function reduceResults(dir, allowMixed) {
+function reduceResults(dir, allowMixed, live) {
 	const entries = [];
 	const skipped = [];
+	const retired = /* @__PURE__ */ new Set();
 	const gradedAt = /* @__PURE__ */ new Map();
 	const shas = /* @__PURE__ */ new Set();
 	const files = fs.readdirSync(dir);
 	const incomplete = new Set(files.filter((f) => f.endsWith(".json.attempt")).map((f) => f.replace(/\.attempt$/, "")));
 	const results = files.filter((f) => f.endsWith(".json") && !f.endsWith(".meta.json"));
 	for (const f of [.../* @__PURE__ */ new Set([...results, ...incomplete])].sort()) {
+		let identity;
+		try {
+			identity = resultIdentity(f, dir);
+		} catch {
+			identity = void 0;
+		}
+		if (identity !== void 0 && identity.scenario !== "" && !isLive(live, identity)) {
+			retired.add(entryKey(identity));
+			continue;
+		}
 		if (f.endsWith("--cursor.json")) {
 			console.error(`skipping ${f}: Cursor harness is retired`);
 			skipped.push(f);
@@ -522,7 +544,8 @@ function reduceResults(dir, allowMixed) {
 		treeSha: shas.size === 1 ? [...shas][0] : shas.size === 0 ? "none" : "mixed",
 		entries,
 		skipped,
-		gradedAt
+		gradedAt,
+		retired
 	};
 }
 function entryKey(e) {
@@ -565,12 +588,17 @@ function readExistingScorecard(out) {
 function cmdSummarize(argv) {
 	const { positional, flags } = parseArgs(argv);
 	if (positional.length > 0) fail("usage: skillcheck summarize [--root DIR] [--allow-mixed]");
-	const dirs = stateDirs(resolveRoot(flags));
+	const root = resolveRoot(flags);
+	const dirs = stateDirs(root);
 	if (!fs.existsSync(dirs.results)) fail(`no results directory at ${dirs.results}; run some evals first`);
-	const { entries, skipped, gradedAt } = reduceResults(dirs.results, flags.get("--allow-mixed") === true);
+	const live = liveScenarios(root);
+	const { entries, skipped, gradedAt, retired: retiredResults } = reduceResults(dirs.results, flags.get("--allow-mixed") === true, live);
 	fs.mkdirSync(dirs.scorecards, { recursive: true });
 	const out = path.join(dirs.scorecards, `${(/* @__PURE__ */ new Date()).toISOString().slice(0, 10)}.json`);
-	const existing = readExistingScorecard(out);
+	const previous = readExistingScorecard(out);
+	const existing = previous.filter((e) => isLive(live, e));
+	for (const e of previous) if (!isLive(live, e)) retiredResults.add(entryKey(e));
+	const retired = retiredResults.size;
 	const skippedKeys = new Set(skipped.map((file) => ({
 		key: entryKey(resultIdentity(file, dirs.results)),
 		modifiedAt: fs.statSync(fs.existsSync(path.join(dirs.results, `${file}.attempt`)) ? path.join(dirs.results, `${file}.attempt`) : path.join(dirs.results, file)).mtimeMs
@@ -587,6 +615,7 @@ function cmdSummarize(argv) {
 		scenarios: merged.entries
 	};
 	fs.writeFileSync(out, JSON.stringify(scorecard, null, 2) + "\n");
+	if (retired > 0) console.log(`dropped ${retired} row(s) for scenarios no longer in the tree`);
 	const scenarios = merged.entries.filter((e) => e.variant !== "control");
 	console.log(`${out}: ${scenarios.length} scenario(s), ${scenarios.filter((e) => e.pass).length} passing, ${merged.entries.length - scenarios.length} control(s), ${skipped.length} skipped file(s)`);
 	if (existing.length > 0) console.log(`merged into today's scorecard: ${entries.length} from this run, ${merged.carried} carried over`);
@@ -688,4 +717,4 @@ if (isMainModule()) {
 	}
 }
 //#endregion
-export { NOISY_SPREAD, aggregateTrials, assertUniformConfig, classifyResult, ensurePrivateDir, formatStats, mergeScorecard, parseArgs, parsePositiveInt, reduceResults, resolveRoot, resultRunConfig, runConfigOf, runOptions, stateDirs, summarizeSkills, toolVersion, treeShaOf };
+export { NOISY_SPREAD, aggregateTrials, assertUniformConfig, classifyResult, ensurePrivateDir, formatStats, liveScenarios, mergeScorecard, parseArgs, parsePositiveInt, reduceResults, resolveRoot, resultRunConfig, runConfigOf, runOptions, stateDirs, summarizeSkills, toolVersion, treeShaOf };

package/dist/lint.js

@@ -1,3 +1,4 @@
+import { FILE_BLOCK } from "./scenario.js";
 import fs from "node:fs";
 import path from "node:path";
 //#region src/lint.ts
@@ -61,19 +62,36 @@ function lintSkill(dir, root, errors) {
 function lintSkills(root) {
 	const roots = ["skills", ...fs.globSync("cli/*/skills", { cwd: root })].map((r) => path.join(root, r));
 	const errors = [];
-	let count = 0;
+	const skills = [];
 	for (const dir of roots) {
 		if (!fs.existsSync(dir)) continue;
 		for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
 			if (!entry.isDirectory() || entry.name.startsWith(".")) continue;
 			lintSkill(path.join(dir, entry.name), root, errors);
-			count++;
+			skills.push(path.join(dir, entry.name));
 		}
 	}
+	lintTasks(skills, root, errors);
 	return {
 		errors,
-		count
+		count: skills.length
 	};
 }
+function lintTasks(skills, root, errors) {
+	const names = skills.map((dir) => path.basename(dir));
+	for (const dir of skills) {
+		const evals = path.join(dir, "evals");
+		if (!fs.existsSync(evals)) continue;
+		for (const scenario of fs.readdirSync(evals, { withFileTypes: true })) {
+			const file = path.join(evals, scenario.name, "task.md");
+			if (!scenario.isDirectory() || !fs.existsSync(file)) continue;
+			const text = fs.readFileSync(file, "utf8").replace(FILE_BLOCK, "");
+			for (const name of names) {
+				const escaped = name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+				if (new RegExp(`(?<![\\w-])${escaped}(?![\\w-])`, "i").test(text)) errors.push(`${path.relative(root, file)}: task names skill ${name}`);
+			}
+		}
+	}
+}
 //#endregion
 export { lintSkills };

package/dist/scenario.js

@@ -4,6 +4,7 @@ import fs from "node:fs";
 import os from "node:os";
 import path from "node:path";
 //#region src/scenario.ts
+const FILE_BLOCK = /^=+ FILE: (.+?) =+\n([\s\S]*?)\n=+ END FILE =+$/gm;
 const DEFAULT_CLAUDE_AGENT = "claude-opus-5";
 function loadScenario(scenarioDir) {
 	const match = scenarioDir.match(/skills\/([^/]+)\/evals\/([^/]+)$/);
@@ -15,7 +16,7 @@ function loadScenario(scenarioDir) {
 	if (criteria.skill_use !== void 0 && criteria.skill_use !== "required" && criteria.skill_use !== "optional") throw new Error(`skill_use must be "required" or "optional" in ${scenarioDir}`);
 	for (const item of criteria.checklist) if (!(typeof item?.name === "string" && item.name.trim() !== "" && typeof item?.description === "string" && item.description.trim() !== "" && Number.isFinite(item?.max_score) && item.max_score > 0)) throw new Error(`invalid checklist item in ${scenarioDir}: ${JSON.stringify(item)}`);
 	const files = [];
-	const task = taskMd.replace(/^=+ FILE: (.+?) =+\n([\s\S]*?)\n=+ END FILE =+$/gm, (_, name, content) => {
+	const task = taskMd.replace(FILE_BLOCK, (_, name, content) => {
 		files.push({
 			name: name.trim(),
 			content: content + "\n"
@@ -322,4 +323,4 @@ function generateRun(scenarioDir, opts, paths) {
 	};
 }
 //#endregion
-export { DEFAULT_CLAUDE_AGENT, buildConfig, encodeRunNamePart, generateRun, isHiddenSkill, loadScenario, materialize, privateCodexHome, requiredEvalPackages, resolvePackageDir, runNameFor, sdkNodeModulesDir, stripHiddenFlag, trialLabel };
+export { DEFAULT_CLAUDE_AGENT, FILE_BLOCK, buildConfig, encodeRunNamePart, generateRun, isHiddenSkill, loadScenario, materialize, privateCodexHome, requiredEvalPackages, resolvePackageDir, runNameFor, sdkNodeModulesDir, stripHiddenFlag, trialLabel };

package/docs/scenarios.md

@@ -35,6 +35,7 @@ own examiner or embed generated dependencies.
 
 Write the task the way a user would write it. Do not name the skill, describe
 its steps, or hint at the checklist: routing is part of what is being measured.
+`skillcheck lint` rejects a task whose prose names any skill in the root.
 
 ## criteria.json
 

package/docs/usage.md

@@ -20,6 +20,9 @@ Checks each `<root>/skills/<skill>/`:
 - `disable-model-invocation`, when present, is the bare YAML boolean `true`.
   A quoted `"true"` is an error
 - Relative links in the body resolve on disk
+- No eval `task.md` names a skill from the root, outside its inline input
+  files. A named skill gives routing away, and a no-skill control can read
+  the operator's installed copy
 
 Code spans and fenced blocks are stripped before links are checked, so example
 links never fail. External schemes and `#anchors` pass. Dot-directories under
@@ -179,6 +182,11 @@ six of twenty-nine scenarios therefore leaves twenty-nine rows in the file, not
 six. A same-date file that cannot be parsed stops the write instead of being
 overwritten.
 
+Rows for scenarios that no longer exist in the tree, whether carried from
+today's scorecard or left in `results/`, are dropped and counted on stdout.
+A root with no skills tree (`skills/` or `cli/*/skills/`) holds results only
+and keeps every row.
+
 Files that are not promptfoo results and ungraded transport errors are skipped
 with a warning rather than failing the reduction. Graded assertion failures
 remain scored results. If a skipped file matches an existing scorecard row,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uinaf/skillcheck",
-  "version": "1.4.0",
+  "version": "1.5.0",
   "description": "Lint and eval harness for agent skills",
   "homepage": "https://github.com/uinaf/skillcheck#readme",
   "bugs": {