@uinaf/skillcheck 0.5.0 → 0.5.1

package/dist/cli.js

@@ -107,9 +107,9 @@ function promptfooEntry() {
 function classifyResult(raw) {
 	const root = raw;
 	const res = root?.results?.results?.[0];
-	if (res === void 0) return { error: "promptfoo output carried no result" };
+	if (res === void 0 || res === null) return { error: "promptfoo output carried no result" };
 	const message = typeof res.error === "string" ? res.error.trim() : "";
-	const stats = root?.results?.stats;
+	const stats = root?.results?.stats ?? void 0;
 	const gradedByStats = stats !== void 0 && (stats.errors ?? 0) === 0 && ((stats.failures ?? 0) > 0 || (stats.successes ?? 0) > 0);
 	if (message !== "" && !gradedByStats) return { error: message };
 	if (stats !== void 0 && (stats.errors ?? 0) > 0 && (stats.successes ?? 0) === 0 && (stats.failures ?? 0) === 0) return { error: "promptfoo reported an errored test with nothing graded" };
@@ -128,6 +128,9 @@ function gitHead(root) {
 function metaPath(resultPath) {
 	return resultPath.replace(/\.json$/, ".meta.json");
 }
+function attemptPath(resultPath) {
+	return `${resultPath}.attempt`;
+}
 function runScenario(scenarioDir, opts, root) {
 	const dirs = stateDirs(root);
 	const { name, configPath } = generateRun(path.resolve(scenarioDir), opts, {
@@ -137,6 +140,7 @@ function runScenario(scenarioDir, opts, root) {
 	});
 	fs.mkdirSync(dirs.results, { recursive: true });
 	const resultPath = path.join(dirs.results, `${name}.json`);
+	fs.writeFileSync(attemptPath(resultPath), "{}\n");
 	fs.rmSync(resultPath, { force: true });
 	fs.rmSync(metaPath(resultPath), { force: true });
 	const sha = gitHead(root);
@@ -174,12 +178,15 @@ function runScenario(scenarioDir, opts, root) {
 	outcome.score = verdict.score;
 	outcome.pass = verdict.pass;
 	outcome.error = verdict.error;
-	if (verdict.score !== void 0) fs.writeFileSync(metaPath(resultPath), JSON.stringify({
-		skills_tree_sha: sha,
-		harness: opts.harness,
-		ran_at: (/* @__PURE__ */ new Date()).toISOString(),
-		tool_version: toolVersion()
-	}, null, 2) + "\n");
+	if (verdict.score !== void 0) {
+		fs.writeFileSync(metaPath(resultPath), JSON.stringify({
+			skills_tree_sha: sha,
+			harness: opts.harness,
+			ran_at: (/* @__PURE__ */ new Date()).toISOString(),
+			tool_version: toolVersion()
+		}, null, 2) + "\n");
+		fs.rmSync(attemptPath(resultPath), { force: true });
+	}
 	return outcome;
 }
 function discoverScenarios(root) {
@@ -247,11 +254,30 @@ function cmdSweep(argv) {
 	console.log(`\nsweep: ${passed} passed, ${failed} failed, ${errored} errored, ${skipped} skipped`);
 	process.exit(errored > 0 ? 2 : failed > 0 ? 1 : 0);
 }
+function resultIdentity(file) {
+	const base = file.replace(/\.json$/, "");
+	const suffix = base.match(/--(codex|cursor)$/);
+	const harness = suffix === null ? "claude" : suffix[1];
+	const [skill, ...rest] = base.replace(/--(codex|cursor)$/, "").split("--");
+	return {
+		skill,
+		scenario: rest.join("--"),
+		harness
+	};
+}
 function reduceResults(dir, allowMixed) {
 	const entries = [];
 	const skipped = [];
 	const shas = /* @__PURE__ */ new Set();
-	for (const f of fs.readdirSync(dir).filter((f) => f.endsWith(".json") && !f.endsWith(".meta.json")).sort()) {
+	const files = fs.readdirSync(dir);
+	const incomplete = new Set(files.filter((f) => f.endsWith(".json.attempt")).map((f) => f.replace(/\.attempt$/, "")));
+	const results = files.filter((f) => f.endsWith(".json") && !f.endsWith(".meta.json"));
+	for (const f of [.../* @__PURE__ */ new Set([...results, ...incomplete])].sort()) {
+		if (incomplete.has(f)) {
+			console.error(`skipping ${f}: attempt did not complete with a graded result`);
+			skipped.push(f);
+			continue;
+		}
 		let raw;
 		try {
 			raw = JSON.parse(fs.readFileSync(path.join(dir, f), "utf8"));
@@ -259,17 +285,16 @@ function reduceResults(dir, allowMixed) {
 			raw = void 0;
 		}
 		const res = raw?.results?.results?.[0];
-		if (typeof res?.score !== "number" || typeof res?.success !== "boolean") {
-			console.error(`skipping ${f}: not a promptfoo result`);
+		const verdict = classifyResult(raw);
+		if (verdict.score === void 0 || verdict.pass === void 0) {
+			console.error(`skipping ${f}: ${verdict.error}`);
 			skipped.push(f);
 			continue;
 		}
 		const provider = raw.config?.providers?.[0];
 		const judge = raw.config?.defaultTest?.options?.provider;
 		const base = f.replace(/\.json$/, "");
-		const suffix = base.match(/--(codex|cursor)$/);
-		const harness = suffix === null ? "claude" : suffix[1];
-		const [skill, ...rest] = base.replace(/--(codex|cursor)$/, "").split("--");
+		const { skill, scenario, harness } = resultIdentity(f);
 		let sha = "unattested";
 		try {
 			sha = JSON.parse(fs.readFileSync(path.join(dir, `${base}.meta.json`), "utf8")).skills_tree_sha ?? "unattested";
@@ -277,11 +302,11 @@ function reduceResults(dir, allowMixed) {
 		shas.add(sha);
 		entries.push({
 			skill,
-			scenario: rest.join("--"),
+			scenario,
 			harness,
 			skills_tree_sha: sha,
-			score: res.score,
-			pass: res.success,
+			score: verdict.score,
+			pass: verdict.pass,
 			agent_model: provider?.config?.model ?? `${harness}-default`,
 			judge_model: typeof judge === "string" ? judge.replace(/^anthropic:messages:/, "") : judge?.config?.model ?? judge?.id ?? "unknown",
 			latency_ms: res.latencyMs,
@@ -335,15 +360,19 @@ function cmdSummarize(argv) {
 	const { positional, flags } = parseArgs(argv);
 	if (positional.length > 0) fail("usage: skillcheck summarize [--root DIR] [--allow-mixed]");
 	const dirs = stateDirs(resolveRoot(flags));
-	if (!fs.existsSync(dirs.results)) fail(`no results directory at ${dirs.results} — run some evals first`);
+	if (!fs.existsSync(dirs.results)) fail(`no results directory at ${dirs.results}; run some evals first`);
 	const { entries, skipped } = reduceResults(dirs.results, flags.get("--allow-mixed") === true);
 	fs.mkdirSync(dirs.scorecards, { recursive: true });
 	const out = path.join(dirs.scorecards, `${(/* @__PURE__ */ new Date()).toISOString().slice(0, 10)}.json`);
 	const existing = readExistingScorecard(out);
+	const skippedKeys = new Set(skipped.map((file) => entryKey(resultIdentity(file))));
+	if (existing.some((entry) => skippedKeys.has(entryKey(entry)))) throw new Error("skipped rerun matches an existing score; refusing to carry it or overwrite the scorecard");
 	const merged = mergeScorecard(existing, entries);
+	const treeSha = treeShaOf(merged.entries);
+	if (treeSha === "mixed" && flags.get("--allow-mixed") !== true) throw new Error("scorecard spans multiple skills-tree revisions; rerun stale ones or pass --allow-mixed");
 	const scorecard = {
 		ran_at: (/* @__PURE__ */ new Date()).toISOString(),
-		skills_tree_sha: treeShaOf(merged.entries),
+		skills_tree_sha: treeSha,
 		scenarios: merged.entries
 	};
 	fs.writeFileSync(out, JSON.stringify(scorecard, null, 2) + "\n");

package/docs/adoption.md

@@ -44,7 +44,7 @@ the registry.
 Sweeps need model credentials, so they stay off consumer CI and run from an
 operator machine or a job that already holds gateway auth. They also need the
 eval engine, which is an optional peer precisely so the lint-only install
-above stays small — install it next to the package on the operator machine:
+above stays small. Install it next to the package on the operator machine:
 
 ```sh
 pnpm add -D promptfoo @anthropic-ai/claude-agent-sdk @openai/codex-sdk

package/docs/usage.md

@@ -37,10 +37,10 @@ skillcheck run <scenario-dir> --agent MODEL --judge MODEL --harness codex --max-
 
 Materializes the scenario into `<root>/.skillcheck/scratch/<name>/workdir`,
 installs the skill under test into that workdir, drives the agent, and grades
-the files it wrote. Exit 0 pass, 1 graded fail, 2 error (promptfoo produced no
-usable result, or the optional eval peers are not installed — the message
-carries the exact `pnpm add` command; see
-[adoption](adoption.md#evals)).
+the files it wrote. Exit 0 means pass, 1 means graded fail, and 2 means error.
+Exit 2 covers missing usable promptfoo output or optional eval peers. The
+message carries the exact `pnpm add` command; see
+[adoption](adoption.md#evals).
 
 A test that errored was never graded, so it exits 2, prints the provider's
 message, and writes no provenance sidecar. It is never reported as
@@ -87,7 +87,7 @@ discovered. Exit 2 if anything errored, 1 if anything failed, else 0.
 within one scenario, not across them.
 
 One known failure mode: judge calls through a gateway can drop at the transport
-layer ([uinaf/agent-platform#28](https://github.com/uinaf/agent-platform/issues/28)).
+layer ([uinaf/zebroid-infra#44](https://github.com/uinaf/zebroid-infra/issues/44)).
 That surfaces as an ERROR with no usable result, not as a graded FAIL, and the
 mitigation is a rerun. `sweep` without `--all` resumes, so a rerun only picks up
 what is missing.
@@ -109,8 +109,16 @@ six of twenty-nine scenarios therefore leaves twenty-nine rows in the file, not
 six. A same-date file that cannot be parsed stops the write instead of being
 overwritten.
 
-Files that are not promptfoo results are skipped with a warning rather than
-failing the reduction.
+Files that are not promptfoo results and ungraded transport errors are skipped
+with a warning rather than failing the reduction. Graded assertion failures
+remain scored results. If a skipped file matches an existing scorecard row,
+summary generation fails and leaves the scorecard unchanged, so an errored rerun
+cannot carry forward its old score. This also applies with `--allow-mixed`.
+Runs keep a `<name>.json.attempt` marker until a graded result and its provenance
+are written. An outstanding marker makes `summarize` skip that identity even
+when the child produced no result file or left partial output. The marker does
+not count as a result for the sweep's existence check, so no-output failures
+remain eligible for retry.
 
 ## Provenance
 
@@ -126,7 +134,8 @@ Each successful run writes a `<name>.meta.json` sidecar next to its result:
 ```
 
 `summarize` reads those sidecars and refuses to mix skills-tree revisions in one
-scorecard unless `--allow-mixed`, in which case the top-level `skills_tree_sha`
+scorecard, including retained rows from partial reruns, unless `--allow-mixed`.
+Rejection leaves the existing scorecard unchanged. With the override, the top-level `skills_tree_sha`
 becomes `mixed` and per-entry shas remain. A result with no sidecar reduces as
 `unattested`.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uinaf/skillcheck",
-  "version": "0.5.0",
+  "version": "0.5.1",
   "description": "Lint and eval harness for agent skills",
   "homepage": "https://github.com/uinaf/skillcheck#readme",
   "bugs": {
@@ -27,9 +27,10 @@
     "registry": "https://registry.npmjs.org/"
   },
   "scripts": {
-    "verify": "vp check && vp pack && vp test run",
+    "verify": "vp run ready",
+    "verify:full": "vp run --no-cache ready",
     "prepare": "vp config --no-agent",
-    "prepublishOnly": "pnpm run verify"
+    "prepublishOnly": "pnpm run verify:full"
   },
   "devDependencies": {
     "@anthropic-ai/claude-agent-sdk": "^0.3.233",
@@ -58,5 +59,5 @@
   "engines": {
     "node": ">=24"
   },
-  "packageManager": "pnpm@11.22.0"
+  "packageManager": "pnpm@12.0.0"
 }