@uinaf/skillcheck 0.2.1 → 0.3.1

package/README.md

@@ -1,6 +1,6 @@
 ![skillcheck — lint and eval harness for agent skills.](https://uinaf.dev/og/banner/skillcheck.png)
 
-# @uinaf/skillcheck
+# uinaf/skillcheck
 
 lint and eval harness for agent skills. one CLI with two halves: a keyless
 structural lint any repo can run in CI, and a promptfoo-driven eval loop that

package/dist/cli.js

@@ -59,7 +59,7 @@ function stateDirs(root) {
 }
 function runOptions(flags) {
 	const harness = flags.get("--harness") ?? "claude";
-	if (harness !== "claude" && harness !== "codex") fail(`--harness must be claude or codex, got ${harness}`);
+	if (harness !== "claude" && harness !== "codex" && harness !== "cursor") fail(`--harness must be claude, codex, or cursor, got ${harness}`);
 	return {
 		harness,
 		agentModel: flags.get("--agent"),
@@ -72,8 +72,9 @@ function classifyResult(raw) {
 	const res = root?.results?.results?.[0];
 	if (res === void 0) return { error: "promptfoo output carried no result" };
 	const message = typeof res.error === "string" ? res.error.trim() : "";
-	if (message !== "") return { error: message };
 	const stats = root?.results?.stats;
+	const gradedByStats = stats !== void 0 && (stats.errors ?? 0) === 0 && ((stats.failures ?? 0) > 0 || (stats.successes ?? 0) > 0);
+	if (message !== "" && !gradedByStats) return { error: message };
 	if (stats !== void 0 && (stats.errors ?? 0) > 0 && (stats.successes ?? 0) === 0 && (stats.failures ?? 0) === 0) return { error: "promptfoo reported an errored test with nothing graded" };
 	if (typeof res.score !== "number" || typeof res.success !== "boolean") return { error: "promptfoo result carried no usable score" };
 	return {
@@ -94,7 +95,8 @@ function runScenario(scenarioDir, opts, root) {
 	const dirs = stateDirs(root);
 	const { name, configPath } = generateRun(path.resolve(scenarioDir), opts, {
 		scratchDir: dirs.scratch,
-		transformPath: path.join(here, `transform${selfExt}`)
+		transformPath: path.join(here, `transform${selfExt}`),
+		cursorProviderPath: path.join(here, `cursor-provider${selfExt}`)
 	});
 	fs.mkdirSync(dirs.results, { recursive: true });
 	const resultPath = path.join(dirs.results, `${name}.json`);
@@ -165,7 +167,7 @@ function discoverScenarios(root) {
 }
 function cmdRun(argv) {
 	const { positional, flags } = parseArgs(argv);
-	if (positional.length !== 1) fail("usage: skillcheck run <scenario-dir> [--root DIR] [--agent MODEL] [--judge MODEL] [--harness claude|codex]");
+	if (positional.length !== 1) fail("usage: skillcheck run <scenario-dir> [--root DIR] [--agent MODEL] [--judge MODEL] [--harness claude|codex|cursor]");
 	const o = runScenario(positional[0], runOptions(flags), resolveRoot(flags));
 	if (o.score === void 0) {
 		console.error(`ERROR ${o.name}: ${o.error ?? "no usable result"} (promptfoo rc=${o.rc})`);
@@ -225,8 +227,9 @@ function reduceResults(dir, allowMixed) {
 		const provider = raw.config?.providers?.[0];
 		const judge = raw.config?.defaultTest?.options?.provider;
 		const base = f.replace(/\.json$/, "");
-		const harness = base.endsWith("--codex") ? "codex" : "claude";
-		const [skill, ...rest] = base.replace(/--codex$/, "").split("--");
+		const suffix = base.match(/--(codex|cursor)$/);
+		const harness = suffix === null ? "claude" : suffix[1];
+		const [skill, ...rest] = base.replace(/--(codex|cursor)$/, "").split("--");
 		let sha = "unattested";
 		try {
 			sha = JSON.parse(fs.readFileSync(path.join(dir, `${base}.meta.json`), "utf8")).skills_tree_sha ?? "unattested";
@@ -239,7 +242,7 @@ function reduceResults(dir, allowMixed) {
 			skills_tree_sha: sha,
 			score: res.score,
 			pass: res.success,
-			agent_model: provider?.config?.model ?? "codex-default",
+			agent_model: provider?.config?.model ?? `${harness}-default`,
 			judge_model: typeof judge === "string" ? judge.replace(/^anthropic:messages:/, "") : judge?.config?.model ?? "unknown",
 			latency_ms: res.latencyMs,
 			tokens: (res.tokenUsage?.total ?? 0) + (res.tokenUsage?.assertions?.total ?? 0)

package/dist/cursor-provider.js

@@ -0,0 +1,119 @@
+import { spawn } from "node:child_process";
+//#region src/cursor-provider.ts
+function newStreamState() {
+	return {
+		isError: false,
+		skillCalls: []
+	};
+}
+const SKILL_MD = /(?:^|\/)\.cursor\/skills\/([^/]+)\/SKILL\.md$/;
+function foldLine(state, line) {
+	let event;
+	try {
+		event = JSON.parse(line);
+	} catch {
+		return;
+	}
+	if (typeof event !== "object" || event === null) return;
+	const e = event;
+	if (e.type === "tool_call" && e.subtype === "started") {
+		const p = e.tool_call?.readToolCall?.args?.path;
+		const m = typeof p === "string" ? SKILL_MD.exec(p) : null;
+		if (m) state.skillCalls.push({
+			name: m[1],
+			source: "project",
+			path: p
+		});
+		return;
+	}
+	if (e.type === "result") {
+		state.isError = e.is_error === true || e.subtype !== "success";
+		state.result = typeof e.result === "string" ? e.result : "";
+		if (e.usage) {
+			const prompt = e.usage.inputTokens ?? 0;
+			const completion = e.usage.outputTokens ?? 0;
+			state.tokenUsage = {
+				prompt,
+				completion,
+				cached: e.usage.cacheReadTokens ?? 0,
+				total: prompt + completion
+			};
+		}
+	}
+}
+var CursorAgentProvider = class {
+	providerId;
+	config;
+	constructor(options) {
+		this.providerId = options.id ?? "cursor-agent";
+		if (options.config?.working_dir === void 0) throw new Error("cursor-agent provider requires config.working_dir");
+		this.config = options.config;
+	}
+	id() {
+		return this.providerId;
+	}
+	async callApi(prompt) {
+		const command = this.config.command ?? "cursor-agent";
+		const args = [
+			"-p",
+			"--trust",
+			"--output-format",
+			"stream-json"
+		];
+		if (this.config.model !== void 0) args.push("--model", this.config.model);
+		const state = newStreamState();
+		const stderr = [];
+		let stdoutBuf = "";
+		return new Promise((resolve) => {
+			const child = spawn(command, args, {
+				cwd: this.config.working_dir,
+				env: process.env,
+				stdio: [
+					"pipe",
+					"pipe",
+					"pipe"
+				]
+			});
+			const timeoutMs = this.config.timeout_ms ?? 9e5;
+			const timer = setTimeout(() => {
+				child.kill("SIGKILL");
+				resolve({ error: `cursor-agent timed out after ${timeoutMs}ms` });
+			}, timeoutMs);
+			child.on("error", (err) => {
+				clearTimeout(timer);
+				resolve({ error: `failed to spawn ${command}: ${err.message}` });
+			});
+			child.stdout.on("data", (chunk) => {
+				stdoutBuf += chunk.toString("utf8");
+				const lines = stdoutBuf.split("\n");
+				stdoutBuf = lines.pop() ?? "";
+				for (const line of lines) foldLine(state, line);
+			});
+			child.stderr.on("data", (chunk) => {
+				stderr.push(chunk.toString("utf8"));
+			});
+			child.on("close", (code) => {
+				clearTimeout(timer);
+				if (stdoutBuf !== "") foldLine(state, stdoutBuf);
+				if (state.result === void 0) {
+					const tail = stderr.join("").trim().slice(-2e3);
+					resolve({ error: `cursor-agent exited ${code ?? "by signal"} without a result event${tail === "" ? "" : `: ${tail}`}` });
+					return;
+				}
+				if (state.isError) {
+					resolve({ error: state.result || "cursor-agent reported an error result" });
+					return;
+				}
+				resolve({
+					output: state.result,
+					tokenUsage: state.tokenUsage,
+					metadata: { skillCalls: state.skillCalls }
+				});
+			});
+			child.stdin.write(prompt);
+			child.stdin.end();
+		});
+	}
+};
+//#endregion
+export { CursorAgentProvider as default, foldLine, newStreamState };

package/dist/scenario.js

@@ -35,7 +35,7 @@ function loadScenario(scenarioDir) {
 function runNameFor(scenarioDir, harness) {
 	const m = path.resolve(scenarioDir).match(/skills\/([^/]+)\/evals\/([^/]+)$/);
 	if (!m) throw new Error(`not a scenario dir (want .../skills/<skill>/evals/<scenario>): ${scenarioDir}`);
-	return harness === "codex" ? `${m[1]}--${m[2]}--codex` : `${m[1]}--${m[2]}`;
+	return harness === "claude" ? `${m[1]}--${m[2]}` : `${m[1]}--${m[2]}--${harness}`;
 }
 function frontmatterRange(text) {
 	const lines = text.split("\n");
@@ -53,7 +53,11 @@ function stripHiddenFlag(skillMd) {
 	if (!range) return skillMd;
 	return skillMd.split("\n").filter((l, i) => !(i >= range[0] && i < range[1] && l.startsWith("disable-model-invocation:"))).join("\n");
 }
-const RESERVED = /* @__PURE__ */ new Set([".claude", ".agents"]);
+const RESERVED = /* @__PURE__ */ new Set([
+	".claude",
+	".agents",
+	".cursor"
+]);
 function materialize(s, runDir, harness) {
 	const workdir = path.join(runDir, "workdir");
 	fs.rmSync(runDir, {
@@ -78,7 +82,7 @@ function materialize(s, runDir, harness) {
 		fs.mkdirSync(path.dirname(dest), { recursive: true });
 		fs.writeFileSync(dest, content);
 	}
-	const roots = harness === "codex" ? [".claude", ".agents"] : [".claude"];
+	const roots = harness === "codex" ? [".claude", ".agents"] : harness === "cursor" ? [".cursor"] : [".claude"];
 	for (const root of roots) fs.cpSync(s.skillDir, path.join(workdir, root, "skills", s.skill), {
 		recursive: true,
 		filter: (src) => path.basename(src) !== "evals"
@@ -106,7 +110,14 @@ function materialize(s, runDir, harness) {
 		manifestPath
 	};
 }
-function agentProvider(opts, workdir, skill) {
+function agentProvider(opts, workdir, skill, paths) {
+	if (opts.harness === "cursor") return {
+		id: `file://${paths.cursorProviderPath}`,
+		config: {
+			...opts.agentModel ? { model: opts.agentModel } : {},
+			working_dir: workdir
+		}
+	};
 	if (opts.harness === "codex") return {
 		id: "openai:codex-sdk",
 		config: {
@@ -138,11 +149,11 @@ function agentProvider(opts, workdir, skill) {
 		}
 	};
 }
-function buildConfig(s, workdir, manifestPath, opts, transformPath) {
+function buildConfig(s, workdir, manifestPath, opts, paths) {
 	return {
 		description: `${s.skill}/${s.scenario}`,
 		prompts: ["{{task}}"],
-		providers: [agentProvider(opts, workdir, s.skill)],
+		providers: [agentProvider(opts, workdir, s.skill, paths)],
 		defaultTest: { options: {
 			provider: process.env.ANTHROPIC_API_KEY ? `anthropic:messages:${opts.judgeModel}` : {
 				id: "anthropic:claude-agent-sdk",
@@ -173,7 +184,7 @@ function buildConfig(s, workdir, manifestPath, opts, transformPath) {
 					}
 				}
 			},
-			transform: `file://${transformPath}`
+			transform: `file://${paths.transformPath}`
 		} },
 		tests: [{
 			description: s.criteria.context,
@@ -216,7 +227,7 @@ function generateRun(scenarioDir, opts, paths) {
 		});
 		fs.symlinkSync(sdkDir, link, "dir");
 	}
-	const config = buildConfig(s, workdir, manifestPath, opts, paths.transformPath);
+	const config = buildConfig(s, workdir, manifestPath, opts, paths);
 	const configPath = path.join(runDir, "promptfooconfig.json");
 	fs.writeFileSync(configPath, JSON.stringify(config, null, 2));
 	return {

package/docs/releasing.md

@@ -7,12 +7,12 @@ a push to `main` runs one workflow, `.github/workflows/release.yml`:
 ```text
 verify ──┐
          ├──> release   npm publish, OIDC + uinaf-releaser  (release environment)
-secrets ─┘
+scan ────┘
 ```
 
-`verify` and `secrets` are the shared gate, called from `verify.yml` and
-`secrets.yml`, so one definition serves pull requests, the merge queue, and this
-push. keep it that way: a second copy of the gate on a push-to-`main` workflow
+`verify` and `scan` are the shared gate: `verify` is called from `verify.yml`,
+and `scan` calls the shared scan in `uinaf/.github`, the same one `scan.yml`
+runs for pull requests. keep it that way: a second copy of the gate on a push-to-`main` workflow
 races this one over the same commit.
 
 the file name `release.yml` is load-bearing. see below.

package/docs/scenarios.md

@@ -8,9 +8,9 @@ a scenario is two files in a frozen location:
 ```
 
 the path is the identity: `<skill>--<scenario>` names the run, the result file,
-and the scorecard entry. on the codex harness the name gains a `--codex` suffix,
-so both harnesses can hold results side by side. a directory missing either file
-is not discovered.
+and the scorecard entry. on the codex and cursor harnesses the name gains a
+`--codex` or `--cursor` suffix, so every harness can hold results side by side.
+a directory missing either file is not discovered.
 
 ## task.md
 
@@ -27,9 +27,9 @@ Fix the failing check in the config below.
 
 each block is replaced in the prompt with a pointer ("Input file `config.json`
 is available in your working directory.") and written to disk. destinations
-must stay under the workdir, must not collide, and must not target `.claude/` or
-`.agents/`, since a fixture that writes agent config would be configuring its
-own examiner.
+must stay under the workdir, must not collide, and must not target `.claude/`,
+`.agents/`, or `.cursor/`, since a fixture that writes agent config would be
+configuring its own examiner.
 
 write the task the way a user would write it. do not name the skill, describe
 its steps, or hint at the checklist: routing is part of what is being measured.
@@ -81,8 +81,9 @@ frontmatter block; body text mentioning the key does not count.
 ## the workdir
 
 per run, under `<root>/.skillcheck/scratch/<name>/`, rebuilt from scratch each
-time. the skill under test is installed at `.claude/skills/<skill>/` (and also
-`.agents/skills/<skill>/` on the codex harness) with its `evals/` directory
+time. the skill under test is installed where the harness discovers skills —
+`.claude/skills/<skill>/`, plus `.agents/skills/<skill>/` on codex, or
+`.cursor/skills/<skill>/` alone on cursor — with its `evals/` directory
 excluded, so criteria never leak into the agent's context.
 
 scenario quality is behavioral proof; [authoring](authoring.md) covers the

package/docs/usage.md

@@ -45,8 +45,16 @@ message, and writes no provenance sidecar. it is never reported as
 `FAIL score=0.0000`; only a real judged verdict can fail a run.
 
 defaults: `--harness claude`, agent `claude-opus-5`, judge `claude-opus-5`,
-`--max-turns 50`. on the codex harness, omitting `--agent` leaves the model to
-the Codex CLI's own default.
+`--max-turns 50`. on the codex and cursor harnesses, omitting `--agent` leaves
+the model to that CLI's own default.
+
+`--harness cursor` drives the scenario through the Cursor Agent CLI
+(`cursor-agent` on PATH) with the skill installed under `.cursor/skills/`;
+`--agent` names a Cursor model id, e.g. `composer-2.5`. there is no promptfoo
+cursor provider, so the run uses this package's own provider module, which
+replays the CLI's `stream-json` output: the `result` event becomes the graded
+output and `SKILL.md` reads under `.cursor/skills/` become the `skill-used`
+evidence. the judge leg is unchanged.
 
 ## sweep
 
@@ -121,5 +129,6 @@ written inside the installed package.
 | `ANTHROPIC_API_KEY`                           | judge grades over `anthropic:messages:<model>` instead of the SDK |
 | `CODEX_HOME` (default `~/.codex`)             | where the codex harness finds the local `codex` CLI login         |
 | `OPENAI_API_KEY`                              | codex agent auth when there is no local login                     |
+| `CURSOR_API_KEY`                              | cursor agent auth; a logged-in `cursor-agent` also works          |
 
 the judge stays on the Anthropic selection regardless of the agent harness.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uinaf/skillcheck",
-  "version": "0.2.1",
+  "version": "0.3.1",
   "description": "Lint and eval harness for agent skills",
   "homepage": "https://github.com/uinaf/skillcheck#readme",
   "bugs": {