@uinaf/skillcheck 0.5.0 → 0.5.2

package/dist/cli.js

@@ -107,9 +107,9 @@ function promptfooEntry() {
 function classifyResult(raw) {
 	const root = raw;
 	const res = root?.results?.results?.[0];
-	if (res === void 0) return { error: "promptfoo output carried no result" };
+	if (res === void 0 || res === null) return { error: "promptfoo output carried no result" };
 	const message = typeof res.error === "string" ? res.error.trim() : "";
-	const stats = root?.results?.stats;
+	const stats = root?.results?.stats ?? void 0;
 	const gradedByStats = stats !== void 0 && (stats.errors ?? 0) === 0 && ((stats.failures ?? 0) > 0 || (stats.successes ?? 0) > 0);
 	if (message !== "" && !gradedByStats) return { error: message };
 	if (stats !== void 0 && (stats.errors ?? 0) > 0 && (stats.successes ?? 0) === 0 && (stats.failures ?? 0) === 0) return { error: "promptfoo reported an errored test with nothing graded" };
@@ -128,6 +128,9 @@ function gitHead(root) {
 function metaPath(resultPath) {
 	return resultPath.replace(/\.json$/, ".meta.json");
 }
+function attemptPath(resultPath) {
+	return `${resultPath}.attempt`;
+}
 function runScenario(scenarioDir, opts, root) {
 	const dirs = stateDirs(root);
 	const { name, configPath } = generateRun(path.resolve(scenarioDir), opts, {
@@ -137,6 +140,7 @@ function runScenario(scenarioDir, opts, root) {
 	});
 	fs.mkdirSync(dirs.results, { recursive: true });
 	const resultPath = path.join(dirs.results, `${name}.json`);
+	fs.writeFileSync(attemptPath(resultPath), "{}\n");
 	fs.rmSync(resultPath, { force: true });
 	fs.rmSync(metaPath(resultPath), { force: true });
 	const sha = gitHead(root);
@@ -174,12 +178,15 @@ function runScenario(scenarioDir, opts, root) {
 	outcome.score = verdict.score;
 	outcome.pass = verdict.pass;
 	outcome.error = verdict.error;
-	if (verdict.score !== void 0) fs.writeFileSync(metaPath(resultPath), JSON.stringify({
-		skills_tree_sha: sha,
-		harness: opts.harness,
-		ran_at: (/* @__PURE__ */ new Date()).toISOString(),
-		tool_version: toolVersion()
-	}, null, 2) + "\n");
+	if (verdict.score !== void 0) {
+		fs.writeFileSync(metaPath(resultPath), JSON.stringify({
+			skills_tree_sha: sha,
+			harness: opts.harness,
+			ran_at: (/* @__PURE__ */ new Date()).toISOString(),
+			tool_version: toolVersion()
+		}, null, 2) + "\n");
+		fs.rmSync(attemptPath(resultPath), { force: true });
+	}
 	return outcome;
 }
 function discoverScenarios(root) {
@@ -247,11 +254,30 @@ function cmdSweep(argv) {
 	console.log(`\nsweep: ${passed} passed, ${failed} failed, ${errored} errored, ${skipped} skipped`);
 	process.exit(errored > 0 ? 2 : failed > 0 ? 1 : 0);
 }
+function resultIdentity(file) {
+	const base = file.replace(/\.json$/, "");
+	const suffix = base.match(/--(codex|cursor)$/);
+	const harness = suffix === null ? "claude" : suffix[1];
+	const [skill, ...rest] = base.replace(/--(codex|cursor)$/, "").split("--");
+	return {
+		skill,
+		scenario: rest.join("--"),
+		harness
+	};
+}
 function reduceResults(dir, allowMixed) {
 	const entries = [];
 	const skipped = [];
 	const shas = /* @__PURE__ */ new Set();
-	for (const f of fs.readdirSync(dir).filter((f) => f.endsWith(".json") && !f.endsWith(".meta.json")).sort()) {
+	const files = fs.readdirSync(dir);
+	const incomplete = new Set(files.filter((f) => f.endsWith(".json.attempt")).map((f) => f.replace(/\.attempt$/, "")));
+	const results = files.filter((f) => f.endsWith(".json") && !f.endsWith(".meta.json"));
+	for (const f of [.../* @__PURE__ */ new Set([...results, ...incomplete])].sort()) {
+		if (incomplete.has(f)) {
+			console.error(`skipping ${f}: attempt did not complete with a graded result`);
+			skipped.push(f);
+			continue;
+		}
 		let raw;
 		try {
 			raw = JSON.parse(fs.readFileSync(path.join(dir, f), "utf8"));
@@ -259,17 +285,16 @@ function reduceResults(dir, allowMixed) {
 			raw = void 0;
 		}
 		const res = raw?.results?.results?.[0];
-		if (typeof res?.score !== "number" || typeof res?.success !== "boolean") {
-			console.error(`skipping ${f}: not a promptfoo result`);
+		const verdict = classifyResult(raw);
+		if (verdict.score === void 0 || verdict.pass === void 0) {
+			console.error(`skipping ${f}: ${verdict.error}`);
 			skipped.push(f);
 			continue;
 		}
 		const provider = raw.config?.providers?.[0];
 		const judge = raw.config?.defaultTest?.options?.provider;
 		const base = f.replace(/\.json$/, "");
-		const suffix = base.match(/--(codex|cursor)$/);
-		const harness = suffix === null ? "claude" : suffix[1];
-		const [skill, ...rest] = base.replace(/--(codex|cursor)$/, "").split("--");
+		const { skill, scenario, harness } = resultIdentity(f);
 		let sha = "unattested";
 		try {
 			sha = JSON.parse(fs.readFileSync(path.join(dir, `${base}.meta.json`), "utf8")).skills_tree_sha ?? "unattested";
@@ -277,11 +302,11 @@ function reduceResults(dir, allowMixed) {
 		shas.add(sha);
 		entries.push({
 			skill,
-			scenario: rest.join("--"),
+			scenario,
 			harness,
 			skills_tree_sha: sha,
-			score: res.score,
-			pass: res.success,
+			score: verdict.score,
+			pass: verdict.pass,
 			agent_model: provider?.config?.model ?? `${harness}-default`,
 			judge_model: typeof judge === "string" ? judge.replace(/^anthropic:messages:/, "") : judge?.config?.model ?? judge?.id ?? "unknown",
 			latency_ms: res.latencyMs,
@@ -335,15 +360,19 @@ function cmdSummarize(argv) {
 	const { positional, flags } = parseArgs(argv);
 	if (positional.length > 0) fail("usage: skillcheck summarize [--root DIR] [--allow-mixed]");
 	const dirs = stateDirs(resolveRoot(flags));
-	if (!fs.existsSync(dirs.results)) fail(`no results directory at ${dirs.results} — run some evals first`);
+	if (!fs.existsSync(dirs.results)) fail(`no results directory at ${dirs.results}; run some evals first`);
 	const { entries, skipped } = reduceResults(dirs.results, flags.get("--allow-mixed") === true);
 	fs.mkdirSync(dirs.scorecards, { recursive: true });
 	const out = path.join(dirs.scorecards, `${(/* @__PURE__ */ new Date()).toISOString().slice(0, 10)}.json`);
 	const existing = readExistingScorecard(out);
+	const skippedKeys = new Set(skipped.map((file) => entryKey(resultIdentity(file))));
+	if (existing.some((entry) => skippedKeys.has(entryKey(entry)))) throw new Error("skipped rerun matches an existing score; refusing to carry it or overwrite the scorecard");
 	const merged = mergeScorecard(existing, entries);
+	const treeSha = treeShaOf(merged.entries);
+	if (treeSha === "mixed" && flags.get("--allow-mixed") !== true) throw new Error("scorecard spans multiple skills-tree revisions; rerun stale ones or pass --allow-mixed");
 	const scorecard = {
 		ran_at: (/* @__PURE__ */ new Date()).toISOString(),
-		skills_tree_sha: treeShaOf(merged.entries),
+		skills_tree_sha: treeSha,
 		scenarios: merged.entries
 	};
 	fs.writeFileSync(out, JSON.stringify(scorecard, null, 2) + "\n");

package/dist/cursor-process.js

@@ -0,0 +1,71 @@
+import { spawn } from "node:child_process";
+//#region src/cursor-process.ts
+let child;
+let terminal = false;
+let cleaning = false;
+let watchdog = setTimeout(cleanup, 5e3);
+function cleanup() {
+	if (cleaning) return;
+	cleaning = true;
+	clearTimeout(watchdog);
+	if (process.platform !== "win32") process.kill(-process.pid, "SIGKILL");
+	child?.kill("SIGKILL");
+	process.exit(1);
+}
+function fail(error) {
+	if (terminal) return;
+	terminal = true;
+	process.send?.({
+		type: "failure",
+		error
+	}, cleanup);
+	clearTimeout(watchdog);
+	watchdog = setTimeout(cleanup, 1e3);
+}
+if (!process.send) process.exit(1);
+process.on("disconnect", cleanup);
+process.on("message", (message) => {
+	if (message === "stop") return cleanup();
+	if (message === "cleanup" && terminal) {
+		process.send?.({ type: "cleanup" }, cleanup);
+		return;
+	}
+	if (child !== void 0 || terminal) return;
+	if (typeof message !== "object" || message === null || !("command" in message) || typeof message.command !== "string" || !("cwd" in message) || typeof message.cwd !== "string" || !("args" in message) || !Array.isArray(message.args) || !message.args.every((arg) => typeof arg === "string") || !("timeoutMs" in message) || typeof message.timeoutMs !== "number" || !Number.isFinite(message.timeoutMs) || message.timeoutMs <= 0) return fail("cursor-agent supervisor received invalid configuration");
+	clearTimeout(watchdog);
+	watchdog = setTimeout(() => fail("cursor-agent supervisor watchdog timed out"), message.timeoutMs + 1e3);
+	const command = message.command;
+	try {
+		child = spawn(command, message.args, {
+			cwd: message.cwd,
+			stdio: [
+				"pipe",
+				"inherit",
+				"inherit"
+			]
+		});
+	} catch (error) {
+		return fail(`failed to spawn ${command}: ${error instanceof Error ? error.message : String(error)}`);
+	}
+	child.on("error", (error) => fail(`failed to spawn ${command}: ${error.message}`));
+	child.stdin?.on("error", (error) => fail(`cursor-agent stdin failed: ${error.message}`));
+	process.stdin.on("error", (error) => fail(`cursor-agent prompt stream failed: ${error.message}`));
+	if (child.stdin) process.stdin.pipe(child.stdin);
+	child.on("exit", (code, signal) => {
+		if (terminal) return;
+		if (!child?.stdin?.writableFinished) return fail("cursor-agent stdin closed before prompt delivery");
+		terminal = true;
+		process.stdin.unpipe();
+		clearTimeout(watchdog);
+		watchdog = setTimeout(cleanup, 1e3);
+		process.send?.({
+			type: "terminal",
+			code,
+			signal
+		}, (error) => {
+			if (error) cleanup();
+		});
+	});
+});
+//#endregion
+export {};

package/dist/cursor-provider.js

@@ -1,4 +1,5 @@
-import { spawn } from "node:child_process";
+import { fork } from "node:child_process";
+import path from "node:path";
 //#region src/cursor-provider.ts
 function newStreamState() {
 	return {
@@ -65,53 +66,114 @@ var CursorAgentProvider = class {
 		const stderr = [];
 		let stdoutBuf = "";
 		return new Promise((resolve) => {
-			const child = spawn(command, args, {
-				cwd: this.config.working_dir,
-				env: process.env,
+			const timeoutMs = this.config.timeout_ms ?? 9e5;
+			const supervisor = fork(new URL(`./cursor-process${path.extname(import.meta.url)}`, import.meta.url), {
+				execArgv: [],
+				detached: process.platform !== "win32",
 				stdio: [
 					"pipe",
 					"pipe",
-					"pipe"
+					"pipe",
+					"ipc"
 				]
 			});
-			const timeoutMs = this.config.timeout_ms ?? 9e5;
-			const timer = setTimeout(() => {
-				child.kill("SIGKILL");
-				resolve({ error: `cursor-agent timed out after ${timeoutMs}ms` });
-			}, timeoutMs);
-			child.on("error", (err) => {
-				clearTimeout(timer);
-				resolve({ error: `failed to spawn ${command}: ${err.message}` });
-			});
-			child.stdout.on("data", (chunk) => {
-				stdoutBuf += chunk.toString("utf8");
-				const lines = stdoutBuf.split("\n");
-				stdoutBuf = lines.pop() ?? "";
-				for (const line of lines) foldLine(state, line);
-			});
-			child.stderr.on("data", (chunk) => {
-				stderr.push(chunk.toString("utf8"));
-			});
-			child.on("close", (code) => {
+			let failure;
+			let terminal;
+			let cleanupConfirmed = false;
+			let settled = false;
+			let cleanupTimer;
+			const finish = (closed, signal = null) => {
+				if (settled) return;
+				settled = true;
 				clearTimeout(timer);
+				clearTimeout(cleanupTimer);
+				supervisor.stdin?.destroy();
+				supervisor.stdout?.destroy();
+				supervisor.stderr?.destroy();
+				if (!closed) {
+					supervisor.kill("SIGKILL");
+					supervisor.unref();
+					if (supervisor.connected) supervisor.disconnect();
+				}
 				if (stdoutBuf !== "") foldLine(state, stdoutBuf);
-				if (state.result === void 0) {
+				const metadata = { skillCalls: state.skillCalls };
+				const fail = (error) => resolve({
+					error,
+					tokenUsage: state.tokenUsage,
+					metadata
+				});
+				if (failure !== void 0) {
 					const tail = stderr.join("").trim().slice(-2e3);
-					resolve({ error: `cursor-agent exited ${code ?? "by signal"} without a result event${tail === "" ? "" : `: ${tail}`}` });
-					return;
+					const missingResult = terminal !== void 0 && state.result === void 0 ? " without a result event" : "";
+					return fail(`${failure}${missingResult}${tail === "" ? "" : `: ${tail}`}`);
 				}
-				if (state.isError) {
-					resolve({ error: state.result || "cursor-agent reported an error result" });
-					return;
+				if (!closed || !cleanupConfirmed || terminal === void 0 || process.platform !== "win32" && signal !== "SIGKILL") return fail("cursor-agent supervisor ended without confirmed cleanup");
+				if (terminal.code !== 0 || terminal.signal !== null || state.result === void 0) {
+					const tail = stderr.join("").trim().slice(-2e3);
+					return fail(`cursor-agent exited ${terminal.signal ?? terminal.code ?? "without status"}${state.result === void 0 ? " without a result event" : ""}${tail === "" ? "" : `: ${tail}`}`);
 				}
+				if (state.isError) return fail(state.result || "cursor-agent reported an error result");
 				resolve({
 					output: state.result,
 					tokenUsage: state.tokenUsage,
-					metadata: { skillCalls: state.skillCalls }
+					metadata
 				});
+			};
+			const boundCleanup = () => {
+				cleanupTimer ??= setTimeout(() => {
+					failure = failure === void 0 ? "cursor-agent cleanup or pipe draining timed out" : `${failure}; cleanup or pipe draining timed out`;
+					finish(false);
+				}, 2e3);
+			};
+			const stop = (error) => {
+				if (settled) return;
+				failure ??= error;
+				boundCleanup();
+				if (supervisor.connected) supervisor.send("stop", () => {});
+			};
+			const timer = setTimeout(() => stop(`cursor-agent timed out after ${timeoutMs}ms`), timeoutMs);
+			supervisor.on("error", (err) => stop(`cursor-agent supervisor failed: ${err.message}`));
+			supervisor.on("disconnect", () => {
+				if (!cleanupConfirmed && !settled) stop("cursor-agent supervisor disconnected before cleanup");
+			});
+			supervisor.on("message", (message) => {
+				if (settled) return;
+				if (typeof message !== "object" || message === null) return;
+				if (!("type" in message)) return;
+				if (message.type === "terminal" && "code" in message && "signal" in message && (message.code === null || typeof message.code === "number") && (message.signal === null || typeof message.signal === "string")) {
+					terminal = {
+						code: message.code,
+						signal: message.signal
+					};
+					clearTimeout(timer);
+					if (terminal.code !== 0 || terminal.signal !== null) failure ??= `cursor-agent exited ${terminal.signal ?? terminal.code ?? "without status"}`;
+					boundCleanup();
+					supervisor.send("cleanup", (err) => {
+						if (err) stop(`cursor-agent cleanup request failed: ${err.message}`);
+					});
+				} else if (message.type === "failure" && "error" in message && typeof message.error === "string") stop(message.error);
+				else if (message.type === "cleanup" && terminal !== void 0) cleanupConfirmed = true;
+			});
+			supervisor.stdout?.on("data", (chunk) => {
+				stdoutBuf += chunk.toString("utf8");
+				const lines = stdoutBuf.split("\n");
+				stdoutBuf = lines.pop() ?? "";
+				for (const line of lines) foldLine(state, line);
+			});
+			supervisor.stderr?.on("data", (chunk) => stderr.push(chunk.toString("utf8")));
+			supervisor.stdout?.on("error", (err) => stop(`cursor-agent stdout failed: ${err.message}`));
+			supervisor.stderr?.on("error", (err) => stop(`cursor-agent stderr failed: ${err.message}`));
+			supervisor.stdin?.on("error", (err) => stop(`cursor-agent stdin failed: ${err.message}`));
+			supervisor.on("close", (_code, signal) => finish(true, signal));
+			supervisor.send({
+				command,
+				args,
+				cwd: this.config.working_dir,
+				timeoutMs
+			}, (err) => {
+				if (err) stop(`cursor-agent supervisor initialization failed: ${err.message}`);
 			});
-			child.stdin.write(prompt);
-			child.stdin.end();
+			supervisor.stdin?.end(prompt);
 		});
 	}
 };

package/docs/adoption.md

@@ -44,7 +44,7 @@ the registry.
 Sweeps need model credentials, so they stay off consumer CI and run from an
 operator machine or a job that already holds gateway auth. They also need the
 eval engine, which is an optional peer precisely so the lint-only install
-above stays small — install it next to the package on the operator machine:
+above stays small. Install it next to the package on the operator machine:
 
 ```sh
 pnpm add -D promptfoo @anthropic-ai/claude-agent-sdk @openai/codex-sdk

package/docs/usage.md

@@ -37,10 +37,10 @@ skillcheck run <scenario-dir> --agent MODEL --judge MODEL --harness codex --max-
 
 Materializes the scenario into `<root>/.skillcheck/scratch/<name>/workdir`,
 installs the skill under test into that workdir, drives the agent, and grades
-the files it wrote. Exit 0 pass, 1 graded fail, 2 error (promptfoo produced no
-usable result, or the optional eval peers are not installed — the message
-carries the exact `pnpm add` command; see
-[adoption](adoption.md#evals)).
+the files it wrote. Exit 0 means pass, 1 means graded fail, and 2 means error.
+Exit 2 covers missing usable promptfoo output or optional eval peers. The
+message carries the exact `pnpm add` command; see
+[adoption](adoption.md#evals).
 
 A test that errored was never graded, so it exits 2, prints the provider's
 message, and writes no provenance sidecar. It is never reported as
@@ -56,7 +56,13 @@ the model to that CLI's own default.
 cursor provider, so the run uses this package's own provider module, which
 replays the CLI's `stream-json` output: the `result` event becomes the graded
 output and `SKILL.md` reads under `.cursor/skills/` become the `skill-used`
-evidence. The judge leg is unchanged.
+evidence. Grading requires both a successful result event and harness exit code
+zero. On macOS and Linux, a supervisor owns the process group and kills remaining
+helpers when the harness exits, times out, or the provider disconnects. Output
+pipes have a separate two-second cleanup/drain deadline; incomplete cleanup is
+an error. Helpers that detach into another process group are outside this cleanup
+boundary; retained output pipes still cause a bounded error. Windows retains
+only direct-child cleanup. The judge leg is unchanged.
 
 `--judge` takes either a bare Claude model (graded through the Anthropic
 selection in [auth](#auth)) or a provider-qualified promptfoo id, passed
@@ -87,7 +93,7 @@ discovered. Exit 2 if anything errored, 1 if anything failed, else 0.
 within one scenario, not across them.
 
 One known failure mode: judge calls through a gateway can drop at the transport
-layer ([uinaf/agent-platform#28](https://github.com/uinaf/agent-platform/issues/28)).
+layer ([uinaf/zebroid-infra#44](https://github.com/uinaf/zebroid-infra/issues/44)).
 That surfaces as an ERROR with no usable result, not as a graded FAIL, and the
 mitigation is a rerun. `sweep` without `--all` resumes, so a rerun only picks up
 what is missing.
@@ -109,8 +115,16 @@ six of twenty-nine scenarios therefore leaves twenty-nine rows in the file, not
 six. A same-date file that cannot be parsed stops the write instead of being
 overwritten.
 
-Files that are not promptfoo results are skipped with a warning rather than
-failing the reduction.
+Files that are not promptfoo results and ungraded transport errors are skipped
+with a warning rather than failing the reduction. Graded assertion failures
+remain scored results. If a skipped file matches an existing scorecard row,
+summary generation fails and leaves the scorecard unchanged, so an errored rerun
+cannot carry forward its old score. This also applies with `--allow-mixed`.
+Runs keep a `<name>.json.attempt` marker until a graded result and its provenance
+are written. An outstanding marker makes `summarize` skip that identity even
+when the child produced no result file or left partial output. The marker does
+not count as a result for the sweep's existence check, so no-output failures
+remain eligible for retry.
 
 ## Provenance
 
@@ -126,7 +140,8 @@ Each successful run writes a `<name>.meta.json` sidecar next to its result:
 ```
 
 `summarize` reads those sidecars and refuses to mix skills-tree revisions in one
-scorecard unless `--allow-mixed`, in which case the top-level `skills_tree_sha`
+scorecard, including retained rows from partial reruns, unless `--allow-mixed`.
+Rejection leaves the existing scorecard unchanged. With the override, the top-level `skills_tree_sha`
 becomes `mixed` and per-entry shas remain. A result with no sidecar reduces as
 `unattested`.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uinaf/skillcheck",
-  "version": "0.5.0",
+  "version": "0.5.2",
   "description": "Lint and eval harness for agent skills",
   "homepage": "https://github.com/uinaf/skillcheck#readme",
   "bugs": {
@@ -27,9 +27,10 @@
     "registry": "https://registry.npmjs.org/"
   },
   "scripts": {
-    "verify": "vp check && vp pack && vp test run",
+    "verify": "vp run ready",
+    "verify:full": "vp run --no-cache ready",
     "prepare": "vp config --no-agent",
-    "prepublishOnly": "pnpm run verify"
+    "prepublishOnly": "pnpm run verify:full"
   },
   "devDependencies": {
     "@anthropic-ai/claude-agent-sdk": "^0.3.233",
@@ -58,5 +59,5 @@
   "engines": {
     "node": ">=24"
   },
-  "packageManager": "pnpm@11.22.0"
+  "packageManager": "pnpm@12.0.0"
 }