@uinaf/skillcheck 1.4.0 → 1.4.1

package/dist/cli.js

@@ -357,6 +357,17 @@ function readJson(file) {
 		return;
 	}
 }
+function liveScenarios(root) {
+	const cli = path.join(root, "cli");
+	if (!(fs.existsSync(path.join(root, "skills")) || fs.existsSync(cli) && fs.readdirSync(cli).some((d) => fs.existsSync(path.join(cli, d, "skills"))))) return void 0;
+	return new Set(discoverScenarios(root).map((dir) => {
+		const parts = dir.split(path.sep);
+		return `${parts.at(-3)}\0${parts.at(-1)}`;
+	}));
+}
+function isLive(live, e) {
+	return live === void 0 || live.has(`${e.skill}\0${e.scenario}`);
+}
 function discoverScenarios(root) {
 	const roots = [path.join(root, "skills")];
 	const cliDir = path.join(root, "cli");
@@ -464,15 +475,26 @@ function resultIdentity(file, dir) {
 		variant
 	};
 }
-function reduceResults(dir, allowMixed) {
+function reduceResults(dir, allowMixed, live) {
 	const entries = [];
 	const skipped = [];
+	const retired = /* @__PURE__ */ new Set();
 	const gradedAt = /* @__PURE__ */ new Map();
 	const shas = /* @__PURE__ */ new Set();
 	const files = fs.readdirSync(dir);
 	const incomplete = new Set(files.filter((f) => f.endsWith(".json.attempt")).map((f) => f.replace(/\.attempt$/, "")));
 	const results = files.filter((f) => f.endsWith(".json") && !f.endsWith(".meta.json"));
 	for (const f of [.../* @__PURE__ */ new Set([...results, ...incomplete])].sort()) {
+		let identity;
+		try {
+			identity = resultIdentity(f, dir);
+		} catch {
+			identity = void 0;
+		}
+		if (identity !== void 0 && identity.scenario !== "" && !isLive(live, identity)) {
+			retired.add(entryKey(identity));
+			continue;
+		}
 		if (f.endsWith("--cursor.json")) {
 			console.error(`skipping ${f}: Cursor harness is retired`);
 			skipped.push(f);
@@ -522,7 +544,8 @@ function reduceResults(dir, allowMixed) {
 		treeSha: shas.size === 1 ? [...shas][0] : shas.size === 0 ? "none" : "mixed",
 		entries,
 		skipped,
-		gradedAt
+		gradedAt,
+		retired
 	};
 }
 function entryKey(e) {
@@ -565,12 +588,17 @@ function readExistingScorecard(out) {
 function cmdSummarize(argv) {
 	const { positional, flags } = parseArgs(argv);
 	if (positional.length > 0) fail("usage: skillcheck summarize [--root DIR] [--allow-mixed]");
-	const dirs = stateDirs(resolveRoot(flags));
+	const root = resolveRoot(flags);
+	const dirs = stateDirs(root);
 	if (!fs.existsSync(dirs.results)) fail(`no results directory at ${dirs.results}; run some evals first`);
-	const { entries, skipped, gradedAt } = reduceResults(dirs.results, flags.get("--allow-mixed") === true);
+	const live = liveScenarios(root);
+	const { entries, skipped, gradedAt, retired: retiredResults } = reduceResults(dirs.results, flags.get("--allow-mixed") === true, live);
 	fs.mkdirSync(dirs.scorecards, { recursive: true });
 	const out = path.join(dirs.scorecards, `${(/* @__PURE__ */ new Date()).toISOString().slice(0, 10)}.json`);
-	const existing = readExistingScorecard(out);
+	const previous = readExistingScorecard(out);
+	const existing = previous.filter((e) => isLive(live, e));
+	for (const e of previous) if (!isLive(live, e)) retiredResults.add(entryKey(e));
+	const retired = retiredResults.size;
 	const skippedKeys = new Set(skipped.map((file) => ({
 		key: entryKey(resultIdentity(file, dirs.results)),
 		modifiedAt: fs.statSync(fs.existsSync(path.join(dirs.results, `${file}.attempt`)) ? path.join(dirs.results, `${file}.attempt`) : path.join(dirs.results, file)).mtimeMs
@@ -587,6 +615,7 @@ function cmdSummarize(argv) {
 		scenarios: merged.entries
 	};
 	fs.writeFileSync(out, JSON.stringify(scorecard, null, 2) + "\n");
+	if (retired > 0) console.log(`dropped ${retired} row(s) for scenarios no longer in the tree`);
 	const scenarios = merged.entries.filter((e) => e.variant !== "control");
 	console.log(`${out}: ${scenarios.length} scenario(s), ${scenarios.filter((e) => e.pass).length} passing, ${merged.entries.length - scenarios.length} control(s), ${skipped.length} skipped file(s)`);
 	if (existing.length > 0) console.log(`merged into today's scorecard: ${entries.length} from this run, ${merged.carried} carried over`);
@@ -688,4 +717,4 @@ if (isMainModule()) {
 	}
 }
 //#endregion
-export { NOISY_SPREAD, aggregateTrials, assertUniformConfig, classifyResult, ensurePrivateDir, formatStats, mergeScorecard, parseArgs, parsePositiveInt, reduceResults, resolveRoot, resultRunConfig, runConfigOf, runOptions, stateDirs, summarizeSkills, toolVersion, treeShaOf };
+export { NOISY_SPREAD, aggregateTrials, assertUniformConfig, classifyResult, ensurePrivateDir, formatStats, liveScenarios, mergeScorecard, parseArgs, parsePositiveInt, reduceResults, resolveRoot, resultRunConfig, runConfigOf, runOptions, stateDirs, summarizeSkills, toolVersion, treeShaOf };

package/docs/usage.md

@@ -179,6 +179,11 @@ six of twenty-nine scenarios therefore leaves twenty-nine rows in the file, not
 six. A same-date file that cannot be parsed stops the write instead of being
 overwritten.
 
+Rows for scenarios that no longer exist in the tree, whether carried from
+today's scorecard or left in `results/`, are dropped and counted on stdout.
+A root with no skills tree (`skills/` or `cli/*/skills/`) holds results only
+and keeps every row.
+
 Files that are not promptfoo results and ungraded transport errors are skipped
 with a warning rather than failing the reduction. Graded assertion failures
 remain scored results. If a skipped file matches an existing scorecard row,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uinaf/skillcheck",
-  "version": "1.4.0",
+  "version": "1.4.1",
   "description": "Lint and eval harness for agent skills",
   "homepage": "https://github.com/uinaf/skillcheck#readme",
   "bugs": {