@uinaf/skillcheck 0.3.0 → 0.4.0

package/dist/cli.js

@@ -28,6 +28,7 @@ function parseArgs(argv) {
 		"--root",
 		"--agent",
 		"--judge",
+		"--judge-effort",
 		"--harness",
 		"--max-turns"
 	]);
@@ -60,10 +61,23 @@ function stateDirs(root) {
 function runOptions(flags) {
 	const harness = flags.get("--harness") ?? "claude";
 	if (harness !== "claude" && harness !== "codex" && harness !== "cursor") fail(`--harness must be claude, codex, or cursor, got ${harness}`);
+	const agent = flags.get("--agent");
+	const judgeModel = flags.get("--judge") ?? "claude-opus-5";
+	const judgeEffort = flags.get("--judge-effort");
+	if (judgeEffort !== void 0) {
+		if (![
+			"minimal",
+			"low",
+			"medium",
+			"high"
+		].includes(judgeEffort)) fail(`--judge-effort must be minimal, low, medium, or high, got ${judgeEffort}`);
+		if (!judgeModel.includes(":")) fail("--judge-effort needs a provider-qualified --judge (e.g. openai:chat:gpt-5.6-sol); the Anthropic judge does not take a reasoning effort");
+	}
 	return {
 		harness,
-		agentModel: flags.get("--agent"),
-		judgeModel: flags.get("--judge") ?? "claude-opus-5",
+		agentModel: agent,
+		judgeModel,
+		judgeEffort,
 		maxTurns: flags.has("--max-turns") ? parseMaxTurns(flags.get("--max-turns")) : void 0
 	};
 }
@@ -72,8 +86,9 @@ function classifyResult(raw) {
 	const res = root?.results?.results?.[0];
 	if (res === void 0) return { error: "promptfoo output carried no result" };
 	const message = typeof res.error === "string" ? res.error.trim() : "";
-	if (message !== "") return { error: message };
 	const stats = root?.results?.stats;
+	const gradedByStats = stats !== void 0 && (stats.errors ?? 0) === 0 && ((stats.failures ?? 0) > 0 || (stats.successes ?? 0) > 0);
+	if (message !== "" && !gradedByStats) return { error: message };
 	if (stats !== void 0 && (stats.errors ?? 0) > 0 && (stats.successes ?? 0) === 0 && (stats.failures ?? 0) === 0) return { error: "promptfoo reported an errored test with nothing graded" };
 	if (typeof res.score !== "number" || typeof res.success !== "boolean") return { error: "promptfoo result carried no usable score" };
 	return {
@@ -166,7 +181,7 @@ function discoverScenarios(root) {
 }
 function cmdRun(argv) {
 	const { positional, flags } = parseArgs(argv);
-	if (positional.length !== 1) fail("usage: skillcheck run <scenario-dir> [--root DIR] [--agent MODEL] [--judge MODEL] [--harness claude|codex|cursor]");
+	if (positional.length !== 1) fail("usage: skillcheck run <scenario-dir> [--root DIR] [--agent MODEL] [--judge MODEL] [--judge-effort EFFORT] [--harness claude|codex|cursor]");
 	const o = runScenario(positional[0], runOptions(flags), resolveRoot(flags));
 	if (o.score === void 0) {
 		console.error(`ERROR ${o.name}: ${o.error ?? "no usable result"} (promptfoo rc=${o.rc})`);
@@ -242,7 +257,7 @@ function reduceResults(dir, allowMixed) {
 			score: res.score,
 			pass: res.success,
 			agent_model: provider?.config?.model ?? `${harness}-default`,
-			judge_model: typeof judge === "string" ? judge.replace(/^anthropic:messages:/, "") : judge?.config?.model ?? "unknown",
+			judge_model: typeof judge === "string" ? judge.replace(/^anthropic:messages:/, "") : judge?.config?.model ?? judge?.id ?? "unknown",
 			latency_ms: res.latencyMs,
 			tokens: (res.tokenUsage?.total ?? 0) + (res.tokenUsage?.assertions?.total ?? 0)
 		});

package/dist/scenario.js

@@ -155,7 +155,10 @@ function buildConfig(s, workdir, manifestPath, opts, paths) {
 		prompts: ["{{task}}"],
 		providers: [agentProvider(opts, workdir, s.skill, paths)],
 		defaultTest: { options: {
-			provider: process.env.ANTHROPIC_API_KEY ? `anthropic:messages:${opts.judgeModel}` : {
+			provider: opts.judgeModel.includes(":") ? opts.judgeEffort === void 0 ? opts.judgeModel : {
+				id: opts.judgeModel,
+				config: { reasoning_effort: opts.judgeEffort }
+			} : process.env.ANTHROPIC_API_KEY ? `anthropic:messages:${opts.judgeModel}` : {
 				id: "anthropic:claude-agent-sdk",
 				config: {
 					model: opts.judgeModel,

package/docs/usage.md

@@ -56,6 +56,20 @@ replays the CLI's `stream-json` output: the `result` event becomes the graded
 output and `SKILL.md` reads under `.cursor/skills/` become the `skill-used`
 evidence. the judge leg is unchanged.
 
+`--judge` takes either a bare Claude model (graded through the Anthropic
+selection in [auth](#auth)) or a provider-qualified promptfoo id, passed
+through verbatim:
+
+```sh
+skillcheck run <scenario-dir> --judge openai:chat:gpt-5.6-sol --judge-effort high
+```
+
+a provider-qualified judge authenticates through that provider's own env
+(`OPENAI_API_KEY`, plus `OPENAI_BASE_URL` for a gateway) and is recorded
+verbatim in the scorecard's `judge_model` column. `--judge-effort`
+(minimal|low|medium|high) sets `reasoning_effort` and requires a
+provider-qualified judge; the Anthropic judge does not take one.
+
 ## sweep
 
 ```sh
@@ -130,5 +144,7 @@ written inside the installed package.
 | `CODEX_HOME` (default `~/.codex`)             | where the codex harness finds the local `codex` CLI login         |
 | `OPENAI_API_KEY`                              | codex agent auth when there is no local login                     |
 | `CURSOR_API_KEY`                              | cursor agent auth; a logged-in `cursor-agent` also works          |
+| `OPENAI_API_KEY` + `OPENAI_BASE_URL`          | a provider-qualified `--judge openai:…`, optionally via a gateway |
 
-the judge stays on the Anthropic selection regardless of the agent harness.
+a bare `--judge` model stays on the Anthropic selection regardless of the
+agent harness; a provider-qualified `--judge` uses that provider's env instead.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uinaf/skillcheck",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "Lint and eval harness for agent skills",
   "homepage": "https://github.com/uinaf/skillcheck#readme",
   "bugs": {