@uinaf/skillcheck 1.0.0 → 1.0.2

package/dist/cli.js

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 import { lintSkills } from "./lint.js";
-import { generateRun, requiredEvalPackages, resolvePackageDir, runNameFor } from "./scenario.js";
+import { encodeRunNamePart, generateRun, requiredEvalPackages, resolvePackageDir, runNameFor } from "./scenario.js";
 import { execFileSync, spawnSync } from "node:child_process";
 import fs from "node:fs";
 import path from "node:path";
@@ -134,14 +134,19 @@ function attemptPath(resultPath) {
 }
 function runScenario(scenarioDir, opts, root) {
 	const dirs = stateDirs(root);
-	const { name, configPath } = generateRun(path.resolve(scenarioDir), opts, {
+	const { name, configPath, skill, scenario } = generateRun(path.resolve(scenarioDir), opts, {
 		scratchDir: dirs.scratch,
 		transformPath: path.join(here, `transform${selfExt}`),
 		grokProviderPath: path.join(here, `grok-provider${selfExt}`)
 	});
 	fs.mkdirSync(dirs.results, { recursive: true });
 	const resultPath = path.join(dirs.results, `${name}.json`);
-	fs.writeFileSync(attemptPath(resultPath), "{}\n");
+	const identity = {
+		skill,
+		scenario,
+		harness: opts.harness
+	};
+	fs.writeFileSync(attemptPath(resultPath), JSON.stringify(identity) + "\n");
 	fs.rmSync(resultPath, { force: true });
 	fs.rmSync(metaPath(resultPath), { force: true });
 	const sha = gitHead(root);
@@ -182,7 +187,7 @@ function runScenario(scenarioDir, opts, root) {
 	if (verdict.score !== void 0) {
 		fs.writeFileSync(metaPath(resultPath), JSON.stringify({
 			skills_tree_sha: sha,
-			harness: opts.harness,
+			...identity,
 			ran_at: (/* @__PURE__ */ new Date()).toISOString(),
 			tool_version: toolVersion()
 		}, null, 2) + "\n");
@@ -235,11 +240,13 @@ function cmdSweep(argv) {
 	for (const dir of discoverScenarios(root)) {
 		const name = runNameFor(dir, opts.harness);
 		const resultPath = path.join(resultsDir, `${name}.json`);
-		if (!all && fs.existsSync(resultPath)) {
-			skipped++;
-			console.log(`SKIP  ${name} (results exist; use --all to rerun)`);
-			continue;
-		}
+		if (!all && fs.existsSync(resultPath) && !fs.existsSync(attemptPath(resultPath))) try {
+			if (classifyResult(JSON.parse(fs.readFileSync(resultPath, "utf8"))).score !== void 0) {
+				skipped++;
+				console.log(`SKIP  ${name} (results exist; use --all to rerun)`);
+				continue;
+			}
+		} catch {}
 		const o = runScenario(dir, opts, root);
 		if (o.score === void 0) {
 			errored++;
@@ -255,20 +262,39 @@ function cmdSweep(argv) {
 	console.log(`\nsweep: ${passed} passed, ${failed} failed, ${errored} errored, ${skipped} skipped`);
 	process.exit(errored > 0 ? 2 : failed > 0 ? 1 : 0);
 }
-function resultIdentity(file) {
+function resultIdentity(file, dir) {
 	const base = file.replace(/\.json$/, "");
+	for (const sidecar of [`${base}.meta.json`, `${file}.attempt`]) try {
+		const identity = JSON.parse(fs.readFileSync(path.join(dir, sidecar), "utf8"));
+		if (identity !== null && typeof identity === "object" && "skill" in identity && typeof identity.skill === "string" && "scenario" in identity && typeof identity.scenario === "string" && "harness" in identity && (identity.harness === "claude" || identity.harness === "codex" || identity.harness === "grok")) return {
+			skill: identity.skill,
+			scenario: identity.scenario,
+			harness: identity.harness
+		};
+	} catch {}
+	if (/^~v3~[0-9a-f]{64}$/.test(base)) throw new Error(`missing identity metadata for ${file}`);
+	const decode = (part) => {
+		if (!part.startsWith("~v2~")) return part;
+		try {
+			const decoded = decodeURIComponent(part.slice(4));
+			return encodeRunNamePart(decoded) === part ? decoded : part;
+		} catch {
+			return part;
+		}
+	};
 	const suffix = base.match(/--(codex|grok|cursor)$/);
 	const harness = suffix === null ? "claude" : suffix[1];
 	const [skill, ...rest] = base.replace(/--(codex|grok|cursor)$/, "").split("--");
 	return {
-		skill,
-		scenario: rest.join("--"),
+		skill: decode(skill),
+		scenario: decode(rest.join("--")),
 		harness
 	};
 }
 function reduceResults(dir, allowMixed) {
 	const entries = [];
 	const skipped = [];
+	const gradedAt = /* @__PURE__ */ new Map();
 	const shas = /* @__PURE__ */ new Set();
 	const files = fs.readdirSync(dir);
 	const incomplete = new Set(files.filter((f) => f.endsWith(".json.attempt")).map((f) => f.replace(/\.attempt$/, "")));
@@ -300,7 +326,13 @@ function reduceResults(dir, allowMixed) {
 		const provider = raw.config?.providers?.[0];
 		const judge = raw.config?.defaultTest?.options?.provider;
 		const base = f.replace(/\.json$/, "");
-		const { skill, scenario, harness } = resultIdentity(f);
+		const { skill, scenario, harness } = resultIdentity(f, dir);
+		const key = entryKey({
+			skill,
+			scenario,
+			harness
+		});
+		gradedAt.set(key, Math.max(gradedAt.get(key) ?? 0, fs.statSync(path.join(dir, f)).mtimeMs));
 		let sha = "unattested";
 		try {
 			sha = JSON.parse(fs.readFileSync(path.join(dir, `${base}.meta.json`), "utf8")).skills_tree_sha ?? "unattested";
@@ -323,7 +355,8 @@ function reduceResults(dir, allowMixed) {
 	return {
 		treeSha: shas.size === 1 ? [...shas][0] : shas.size === 0 ? "none" : "mixed",
 		entries,
-		skipped
+		skipped,
+		gradedAt
 	};
 }
 function entryKey(e) {
@@ -367,11 +400,14 @@ function cmdSummarize(argv) {
 	if (positional.length > 0) fail("usage: skillcheck summarize [--root DIR] [--allow-mixed]");
 	const dirs = stateDirs(resolveRoot(flags));
 	if (!fs.existsSync(dirs.results)) fail(`no results directory at ${dirs.results}; run some evals first`);
-	const { entries, skipped } = reduceResults(dirs.results, flags.get("--allow-mixed") === true);
+	const { entries, skipped, gradedAt } = reduceResults(dirs.results, flags.get("--allow-mixed") === true);
 	fs.mkdirSync(dirs.scorecards, { recursive: true });
 	const out = path.join(dirs.scorecards, `${(/* @__PURE__ */ new Date()).toISOString().slice(0, 10)}.json`);
 	const existing = readExistingScorecard(out);
-	const skippedKeys = new Set(skipped.map((file) => entryKey(resultIdentity(file))));
+	const skippedKeys = new Set(skipped.map((file) => ({
+		key: entryKey(resultIdentity(file, dirs.results)),
+		modifiedAt: fs.statSync(fs.existsSync(path.join(dirs.results, `${file}.attempt`)) ? path.join(dirs.results, `${file}.attempt`) : path.join(dirs.results, file)).mtimeMs
+	})).filter(({ key, modifiedAt }) => (gradedAt.get(key) ?? 0) <= modifiedAt).map(({ key }) => key));
 	if (existing.some((entry) => skippedKeys.has(entryKey(entry)))) throw new Error("skipped rerun matches an existing score; refusing to carry it or overwrite the scorecard");
 	const merged = mergeScorecard(existing, entries);
 	const treeSha = treeShaOf(merged.entries);

package/dist/grok-provider.js

@@ -113,6 +113,11 @@ var GrokProvider = class {
 			const abort = () => stop("grok run aborted");
 			callOptions?.abortSignal?.addEventListener("abort", abort, { once: true });
 			if (callOptions?.abortSignal?.aborted) abort();
+			child.on("exit", () => {
+				if (process.platform !== "win32" && child.pid !== void 0) try {
+					process.kill(-child.pid, "SIGKILL");
+				} catch {}
+			});
 			child.on("close", (code, signal) => {
 				clearTimeout(timer);
 				callOptions?.abortSignal?.removeEventListener("abort", abort);

package/dist/scenario.js

@@ -32,10 +32,21 @@ function loadScenario(scenarioDir) {
 		criteria
 	};
 }
+function encodeRunNamePart(part) {
+	if (!part.includes("--") && !part.startsWith("-") && !part.endsWith("-") && !part.startsWith("~v2~")) return part;
+	return `~v2~${encodeURIComponent(part).replaceAll("-", "%2D").replaceAll("~", "%7E")}`;
+}
 function runNameFor(scenarioDir, harness) {
 	const m = path.resolve(scenarioDir).match(/skills\/([^/]+)\/evals\/([^/]+)$/);
 	if (!m) throw new Error(`not a scenario dir (want .../skills/<skill>/evals/<scenario>): ${scenarioDir}`);
-	return harness === "claude" ? `${m[1]}--${m[2]}` : `${m[1]}--${m[2]}--${harness}`;
+	const name = `${encodeRunNamePart(m[1])}--${encodeRunNamePart(m[2])}`;
+	const full = harness === "claude" ? name : `${name}--${harness}`;
+	if (Buffer.byteLength(`${full}.json.attempt`) <= 255) return full;
+	return `~v3~${createHash("sha256").update(JSON.stringify([
+		m[1],
+		m[2],
+		harness
+	])).digest("hex")}`;
 }
 function frontmatterRange(text) {
 	const lines = text.split("\n");
@@ -254,8 +265,10 @@ function generateRun(scenarioDir, opts, paths) {
 	fs.writeFileSync(configPath, JSON.stringify(config, null, 2));
 	return {
 		name,
-		configPath
+		configPath,
+		skill: s.skill,
+		scenario: s.scenario
 	};
 }
 //#endregion
-export { buildConfig, generateRun, isHiddenSkill, loadScenario, materialize, requiredEvalPackages, resolvePackageDir, runNameFor, sdkNodeModulesDir, stripHiddenFlag };
+export { buildConfig, encodeRunNamePart, generateRun, isHiddenSkill, loadScenario, materialize, requiredEvalPackages, resolvePackageDir, runNameFor, sdkNodeModulesDir, stripHiddenFlag };

package/docs/scenarios.md

@@ -9,7 +9,9 @@ A scenario is two files in a frozen location:
 
 The path is the identity: `<skill>--<scenario>` names the run, the result file,
 and the scorecard entry. Codex and Grok results gain `--codex` and `--grok`
-suffixes, so harnesses can hold results side by side.
+suffixes, so harnesses can hold results side by side. Double hyphens and
+leading or trailing hyphens in a skill or scenario name are escaped in the run
+and result filename; the scorecard keeps the original name.
 A directory missing either file is not discovered.
 
 ## task.md

package/docs/usage.md

@@ -77,7 +77,7 @@ provider-qualified judge; the Anthropic judge does not take one.
 ## Sweep
 
 ```sh
-skillcheck sweep           # only scenarios without results
+skillcheck sweep           # scenarios without completed results
 skillcheck sweep --all     # rerun everything
 ```
 
@@ -91,8 +91,8 @@ within one scenario, not across them.
 One known failure mode: judge calls through a gateway can drop at the transport
 layer ([uinaf/zebroid-infra#44](https://github.com/uinaf/zebroid-infra/issues/44)).
 That surfaces as an ERROR with no usable result, not as a graded FAIL, and the
-mitigation is a rerun. `sweep` without `--all` resumes, so a rerun only picks up
-what is missing.
+mitigation is a rerun. `sweep` without `--all` resumes, so a rerun picks up
+missing or ungraded results.
 
 ## Summarize
 
@@ -115,14 +115,19 @@ Files that are not promptfoo results and ungraded transport errors are skipped
 with a warning rather than failing the reduction. Graded assertion failures
 remain scored results. If a skipped file matches an existing scorecard row,
 summary generation fails and leaves the scorecard unchanged, so an errored rerun
-cannot carry forward its old score. This also applies with `--allow-mixed`.
+cannot carry forward its old score. A graded result for the same identity
+supersedes a skipped attempt only when the result file is newer. This also
+applies with `--allow-mixed`.
 Results from the retired Cursor harness are skipped with their original identity,
 so they cannot become Claude scores or silently carry an old Cursor row.
 Runs keep a `<name>.json.attempt` marker until a graded result and its provenance
-are written. An outstanding marker makes `summarize` skip that identity even
+are written. The marker records the original skill, scenario, and harness. An outstanding marker makes `summarize` skip that identity even
 when the child produced no result file or left partial output. The marker does
 not count as a result for the sweep's existence check, so no-output failures
 remain eligible for retry.
+Long escaped names use a short hashed filename; the original identity is kept
+in the attempt marker and result sidecar. `sweep` retries existing results that
+contain no grade, including results written by older versions without a marker.
 
 ## Provenance
 
@@ -131,6 +136,8 @@ Each successful run writes a `<name>.meta.json` sidecar next to its result:
 ```json
 {
   "skills_tree_sha": "<root repo HEAD at run time>",
+  "skill": "<skill directory name>",
+  "scenario": "<scenario directory name>",
   "harness": "claude",
   "ran_at": "<ISO timestamp>",
   "tool_version": "<skillcheck version>"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uinaf/skillcheck",
-  "version": "1.0.0",
+  "version": "1.0.2",
   "description": "Lint and eval harness for agent skills",
   "homepage": "https://github.com/uinaf/skillcheck#readme",
   "bugs": {