@uinaf/skillcheck 1.7.0 → 1.7.2

package/dist/cli.js

@@ -191,6 +191,11 @@ function classifyRow(raw, stats) {
 	const checklist = components.find((c) => c?.metadata?.assertionSet?.type === "assert-set");
 	const skillUsed = components.find((c) => c?.assertion?.type === "skill-used" || c?.assertion?.metric === "skill-used");
 	if (typeof checklist?.score !== "number" || typeof skillUsed?.score !== "number") return { error: "promptfoo result carried no checklist or skill-used verdict" };
+	const judgeError = (Array.isArray(checklist.componentResults) ? checklist.componentResults : []).find((c) => c?.metadata?.graderError === true);
+	if (judgeError) {
+		const reason = typeof judgeError.reason === "string" ? judgeError.reason.trim() : "";
+		return { error: `judge call failed${reason ? `: ${reason.slice(0, 300)}` : ""}` };
+	}
 	return {
 		score: checklist.score,
 		pass: res.success,

package/dist/skill-evidence.js

@@ -36,6 +36,18 @@ function skillEvidence(context) {
 	})) return `read of the installed ${skill}/SKILL.md`;
 	const opening = [...installed].map((p) => fs.readFileSync(p, "utf8").slice(0, 300).trim()).filter((t) => t.length >= 40);
 	if (calls(metadata?.toolCalls).find((c) => c.name === "Bash" && c.is_error === false && typeof c.output === "string" && opening.some((t) => c.output.includes(t)))) return `shell read of the installed ${skill}/SKILL.md`;
+	if (codexCommandOutputs(context.providerResponse?.raw).some((out) => opening.some((t) => out.includes(t)))) return `shell read of the installed ${skill}/SKILL.md`;
+}
+function codexCommandOutputs(raw) {
+	let parsed = raw;
+	if (typeof raw === "string") try {
+		parsed = JSON.parse(raw);
+	} catch {
+		return [];
+	}
+	const items = parsed?.items;
+	if (!Array.isArray(items)) return [];
+	return items.flatMap((item) => item?.type === "command_execution" && typeof item.aggregated_output === "string" ? [item.aggregated_output] : []);
 }
 function assertSkillUsed(_output, context) {
 	const evidence = skillEvidence(context);

package/docs/usage.md

@@ -228,7 +228,8 @@ and keeps every row.
 
 Files that are not promptfoo results and ungraded transport errors are skipped
 with a warning rather than failing the reduction. Graded assertion failures
-remain scored results. If a skipped file matches an existing scorecard row,
+remain scored results. A rubric item whose judge call failed (promptfoo tags it
+`graderError`) errors the trial instead, since it judged nothing. If a skipped file matches an existing scorecard row,
 summary generation fails and leaves the scorecard unchanged, so an errored rerun
 cannot carry forward its old score. A graded result for the same identity
 supersedes a skipped attempt only when the result file is newer. This also

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uinaf/skillcheck",
-  "version": "1.7.0",
+  "version": "1.7.2",
   "description": "Lint and eval harness for agent skills",
   "homepage": "https://github.com/uinaf/skillcheck#readme",
   "bugs": {