@uinaf/skillcheck 1.0.1 → 1.0.2

package/dist/cli.js

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 import { lintSkills } from "./lint.js";
-import { generateRun, requiredEvalPackages, resolvePackageDir, runNameFor } from "./scenario.js";
+import { encodeRunNamePart, generateRun, requiredEvalPackages, resolvePackageDir, runNameFor } from "./scenario.js";
 import { execFileSync, spawnSync } from "node:child_process";
 import fs from "node:fs";
 import path from "node:path";
@@ -134,14 +134,19 @@ function attemptPath(resultPath) {
 }
 function runScenario(scenarioDir, opts, root) {
 	const dirs = stateDirs(root);
-	const { name, configPath } = generateRun(path.resolve(scenarioDir), opts, {
+	const { name, configPath, skill, scenario } = generateRun(path.resolve(scenarioDir), opts, {
 		scratchDir: dirs.scratch,
 		transformPath: path.join(here, `transform${selfExt}`),
 		grokProviderPath: path.join(here, `grok-provider${selfExt}`)
 	});
 	fs.mkdirSync(dirs.results, { recursive: true });
 	const resultPath = path.join(dirs.results, `${name}.json`);
-	fs.writeFileSync(attemptPath(resultPath), "{}\n");
+	const identity = {
+		skill,
+		scenario,
+		harness: opts.harness
+	};
+	fs.writeFileSync(attemptPath(resultPath), JSON.stringify(identity) + "\n");
 	fs.rmSync(resultPath, { force: true });
 	fs.rmSync(metaPath(resultPath), { force: true });
 	const sha = gitHead(root);
@@ -182,7 +187,7 @@ function runScenario(scenarioDir, opts, root) {
 	if (verdict.score !== void 0) {
 		fs.writeFileSync(metaPath(resultPath), JSON.stringify({
 			skills_tree_sha: sha,
-			harness: opts.harness,
+			...identity,
 			ran_at: (/* @__PURE__ */ new Date()).toISOString(),
 			tool_version: toolVersion()
 		}, null, 2) + "\n");
@@ -235,11 +240,13 @@ function cmdSweep(argv) {
 	for (const dir of discoverScenarios(root)) {
 		const name = runNameFor(dir, opts.harness);
 		const resultPath = path.join(resultsDir, `${name}.json`);
-		if (!all && fs.existsSync(resultPath) && !fs.existsSync(attemptPath(resultPath))) {
-			skipped++;
-			console.log(`SKIP  ${name} (results exist; use --all to rerun)`);
-			continue;
-		}
+		if (!all && fs.existsSync(resultPath) && !fs.existsSync(attemptPath(resultPath))) try {
+			if (classifyResult(JSON.parse(fs.readFileSync(resultPath, "utf8"))).score !== void 0) {
+				skipped++;
+				console.log(`SKIP  ${name} (results exist; use --all to rerun)`);
+				continue;
+			}
+		} catch {}
 		const o = runScenario(dir, opts, root);
 		if (o.score === void 0) {
 			errored++;
@@ -255,16 +262,26 @@ function cmdSweep(argv) {
 	console.log(`\nsweep: ${passed} passed, ${failed} failed, ${errored} errored, ${skipped} skipped`);
 	process.exit(errored > 0 ? 2 : failed > 0 ? 1 : 0);
 }
-function resultIdentity(file) {
+function resultIdentity(file, dir) {
+	const base = file.replace(/\.json$/, "");
+	for (const sidecar of [`${base}.meta.json`, `${file}.attempt`]) try {
+		const identity = JSON.parse(fs.readFileSync(path.join(dir, sidecar), "utf8"));
+		if (identity !== null && typeof identity === "object" && "skill" in identity && typeof identity.skill === "string" && "scenario" in identity && typeof identity.scenario === "string" && "harness" in identity && (identity.harness === "claude" || identity.harness === "codex" || identity.harness === "grok")) return {
+			skill: identity.skill,
+			scenario: identity.scenario,
+			harness: identity.harness
+		};
+	} catch {}
+	if (/^~v3~[0-9a-f]{64}$/.test(base)) throw new Error(`missing identity metadata for ${file}`);
 	const decode = (part) => {
 		if (!part.startsWith("~v2~")) return part;
 		try {
-			return decodeURIComponent(part.slice(4));
+			const decoded = decodeURIComponent(part.slice(4));
+			return encodeRunNamePart(decoded) === part ? decoded : part;
 		} catch {
 			return part;
 		}
 	};
-	const base = file.replace(/\.json$/, "");
 	const suffix = base.match(/--(codex|grok|cursor)$/);
 	const harness = suffix === null ? "claude" : suffix[1];
 	const [skill, ...rest] = base.replace(/--(codex|grok|cursor)$/, "").split("--");
@@ -277,6 +294,7 @@ function resultIdentity(file) {
 function reduceResults(dir, allowMixed) {
 	const entries = [];
 	const skipped = [];
+	const gradedAt = /* @__PURE__ */ new Map();
 	const shas = /* @__PURE__ */ new Set();
 	const files = fs.readdirSync(dir);
 	const incomplete = new Set(files.filter((f) => f.endsWith(".json.attempt")).map((f) => f.replace(/\.attempt$/, "")));
@@ -308,7 +326,13 @@ function reduceResults(dir, allowMixed) {
 		const provider = raw.config?.providers?.[0];
 		const judge = raw.config?.defaultTest?.options?.provider;
 		const base = f.replace(/\.json$/, "");
-		const { skill, scenario, harness } = resultIdentity(f);
+		const { skill, scenario, harness } = resultIdentity(f, dir);
+		const key = entryKey({
+			skill,
+			scenario,
+			harness
+		});
+		gradedAt.set(key, Math.max(gradedAt.get(key) ?? 0, fs.statSync(path.join(dir, f)).mtimeMs));
 		let sha = "unattested";
 		try {
 			sha = JSON.parse(fs.readFileSync(path.join(dir, `${base}.meta.json`), "utf8")).skills_tree_sha ?? "unattested";
@@ -331,7 +355,8 @@ function reduceResults(dir, allowMixed) {
 	return {
 		treeSha: shas.size === 1 ? [...shas][0] : shas.size === 0 ? "none" : "mixed",
 		entries,
-		skipped
+		skipped,
+		gradedAt
 	};
 }
 function entryKey(e) {
@@ -375,11 +400,14 @@ function cmdSummarize(argv) {
 	if (positional.length > 0) fail("usage: skillcheck summarize [--root DIR] [--allow-mixed]");
 	const dirs = stateDirs(resolveRoot(flags));
 	if (!fs.existsSync(dirs.results)) fail(`no results directory at ${dirs.results}; run some evals first`);
-	const { entries, skipped } = reduceResults(dirs.results, flags.get("--allow-mixed") === true);
+	const { entries, skipped, gradedAt } = reduceResults(dirs.results, flags.get("--allow-mixed") === true);
 	fs.mkdirSync(dirs.scorecards, { recursive: true });
 	const out = path.join(dirs.scorecards, `${(/* @__PURE__ */ new Date()).toISOString().slice(0, 10)}.json`);
 	const existing = readExistingScorecard(out);
-	const skippedKeys = new Set(skipped.map((file) => entryKey(resultIdentity(file))));
+	const skippedKeys = new Set(skipped.map((file) => ({
+		key: entryKey(resultIdentity(file, dirs.results)),
+		modifiedAt: fs.statSync(fs.existsSync(path.join(dirs.results, `${file}.attempt`)) ? path.join(dirs.results, `${file}.attempt`) : path.join(dirs.results, file)).mtimeMs
+	})).filter(({ key, modifiedAt }) => (gradedAt.get(key) ?? 0) <= modifiedAt).map(({ key }) => key));
 	if (existing.some((entry) => skippedKeys.has(entryKey(entry)))) throw new Error("skipped rerun matches an existing score; refusing to carry it or overwrite the scorecard");
 	const merged = mergeScorecard(existing, entries);
 	const treeSha = treeShaOf(merged.entries);

package/dist/scenario.js

@@ -40,7 +40,13 @@ function runNameFor(scenarioDir, harness) {
 	const m = path.resolve(scenarioDir).match(/skills\/([^/]+)\/evals\/([^/]+)$/);
 	if (!m) throw new Error(`not a scenario dir (want .../skills/<skill>/evals/<scenario>): ${scenarioDir}`);
 	const name = `${encodeRunNamePart(m[1])}--${encodeRunNamePart(m[2])}`;
-	return harness === "claude" ? name : `${name}--${harness}`;
+	const full = harness === "claude" ? name : `${name}--${harness}`;
+	if (Buffer.byteLength(`${full}.json.attempt`) <= 255) return full;
+	return `~v3~${createHash("sha256").update(JSON.stringify([
+		m[1],
+		m[2],
+		harness
+	])).digest("hex")}`;
 }
 function frontmatterRange(text) {
 	const lines = text.split("\n");
@@ -259,7 +265,9 @@ function generateRun(scenarioDir, opts, paths) {
 	fs.writeFileSync(configPath, JSON.stringify(config, null, 2));
 	return {
 		name,
-		configPath
+		configPath,
+		skill: s.skill,
+		scenario: s.scenario
 	};
 }
 //#endregion

package/docs/usage.md

@@ -115,14 +115,19 @@ Files that are not promptfoo results and ungraded transport errors are skipped
 with a warning rather than failing the reduction. Graded assertion failures
 remain scored results. If a skipped file matches an existing scorecard row,
 summary generation fails and leaves the scorecard unchanged, so an errored rerun
-cannot carry forward its old score. This also applies with `--allow-mixed`.
+cannot carry forward its old score. A graded result for the same identity
+supersedes a skipped attempt only when the result file is newer. This also
+applies with `--allow-mixed`.
 Results from the retired Cursor harness are skipped with their original identity,
 so they cannot become Claude scores or silently carry an old Cursor row.
 Runs keep a `<name>.json.attempt` marker until a graded result and its provenance
-are written. An outstanding marker makes `summarize` skip that identity even
+are written. The marker records the original skill, scenario, and harness. An outstanding marker makes `summarize` skip that identity even
 when the child produced no result file or left partial output. The marker does
 not count as a result for the sweep's existence check, so no-output failures
 remain eligible for retry.
+Long escaped names use a short hashed filename; the original identity is kept
+in the attempt marker and result sidecar. `sweep` retries existing results that
+contain no grade, including results written by older versions without a marker.
 
 ## Provenance
 
@@ -131,6 +136,8 @@ Each successful run writes a `<name>.meta.json` sidecar next to its result:
 ```json
 {
   "skills_tree_sha": "<root repo HEAD at run time>",
+  "skill": "<skill directory name>",
+  "scenario": "<scenario directory name>",
   "harness": "claude",
   "ran_at": "<ISO timestamp>",
   "tool_version": "<skillcheck version>"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uinaf/skillcheck",
-  "version": "1.0.1",
+  "version": "1.0.2",
   "description": "Lint and eval harness for agent skills",
   "homepage": "https://github.com/uinaf/skillcheck#readme",
   "bugs": {