@icntswm/skillcheck 1.1.0 → 1.2.0

package/README.md

@@ -120,6 +120,10 @@ skillcheck run --only 2,5    # confirm what the batch flagged
 `skillcheck list` prints the skills and slash commands Claude Code sees. It
 stops Claude Code before the first model call, so it costs nothing.
 
+Already tuned a description with Anthropic's skill-creator? `skillcheck import
+eval_set.json --skill <name>` turns its trigger eval set into cases (see
+[Writing cases](docs/writing-cases.md#from-skill-creator)).
+
 ## Documentation
 
 | | |

package/dist/agents/claude.js

@@ -327,7 +327,8 @@ export class ClaudeAdapter {
                 },
             });
             const { text, costUsd, structuredOutput } = out.stream.result;
-            return { structured: structuredOutput, text, costUsd, error: withLoginHint(out.error, opts.configDir), durationMs: out.durationMs };
+            const error = withLoginHint(out.error, opts.configDir);
+            return { structured: structuredOutput, text, costUsd, usage: out.stream.usage, error, durationMs: out.durationMs };
         }
         finally {
             await rm(workdir, { recursive: true, force: true });
@@ -408,13 +409,15 @@ export class ClaudeAdapter {
                     clearTimeout(killTimer);
                 if (timeoutTimer)
                     clearTimeout(timeoutTimer);
-                // A non-zero exit is not a failure by itself: error_max_turns is normal.
-                // Only a stdout with nothing in it means the run really did not happen.
+                // A non-zero exit is not a failure by itself: error_max_turns is normal
+                // and ends with a result event. Without one, the run ended before routing
+                // was over, unless we stopped it ourselves.
                 if (!error)
                     error = stream.error;
-                if (!error && !stdoutSeen) {
+                if (!error && !stopping && !stream.finished) {
                     const detail = stderrText.trim().slice(0, 120);
-                    error = `claude exited with code ${code ?? signal}${detail ? `: ${detail}` : ""}`;
+                    const why = stdoutSeen ? `claude exited with code ${code ?? signal} before its result` : `claude exited with code ${code ?? signal}`;
+                    error = `${why}${detail ? `: ${detail}` : ""}`;
                 }
                 resolve({ stream, error, stoppedEarly, durationMs: Date.now() - startedAt });
             };

package/dist/cli.js

@@ -11,6 +11,7 @@ import { BUILTIN_SKILLS, ConfigError, findDefaultCasesFile, knownSkillNames, loa
 import { confusion } from "./confusion.js";
 import { loadSkillDocs } from "./describe.js";
 import { aggregate, judge } from "./judge.js";
+import { evalSetToSuite } from "./import.js";
 import { toJunit } from "./junit.js";
 import { lint, LINT_DEFAULTS } from "./lint.js";
 import { toMarkdown } from "./markdown.js";
@@ -26,6 +27,8 @@ Usage:
   skillcheck lint [file] [options]    static checks on descriptions and cases
   skillcheck list [options]           print the skills and commands the agent can load
   skillcheck init [file] [options]    write a starter cases file with those names
+  skillcheck import <file> --skill <name>
+                                      turn a skill-creator trigger eval set into cases
 
 run options:
   -a, --agent <name>     agent to route with (default: suite or claude)
@@ -59,6 +62,9 @@ list/init options:
                          CLAUDE_CONFIG_DIR; run, list and init also set it for the agent
 init options:
       --force            overwrite an existing file
+import options:
+      --skill <name>     the skill the eval set is about
+  -o, --out <file>       write the cases there instead of stdout (--force overwrites)
 common:
   -h, --help             show this help
       --version          show version
@@ -69,6 +75,7 @@ nothing reaches the model and nothing is billed, even when not logged in.
 
 Exit codes: 0 all passed, 1 some case failed or was skipped, 2 config or environment error.
 `;
+const COMMANDS = ["run", "check", "lint", "list", "init", "import"];
 const OPTIONS = {
     help: { type: "boolean", short: "h", default: false },
     version: { type: "boolean", default: false },
@@ -94,6 +101,7 @@ const OPTIONS = {
     strict: { type: "boolean", default: false },
     "config-dir": { type: "string" },
     force: { type: "boolean", default: false },
+    out: { type: "string", short: "o" },
 };
 /** Config/environment problems: message to stderr, exit 2. */
 class UsageError extends Error {
@@ -129,6 +137,7 @@ function parseFlags(values, cwd) {
         strict: values.strict,
         configDir: values["config-dir"] !== undefined ? resolveConfigDir(values["config-dir"], cwd) : undefined,
         force: values.force,
+        out: values.out,
     };
 }
 /** --config-dir: absolute, must be an existing directory. */
@@ -203,7 +212,7 @@ export async function main(argv, io = { stdout: process.stdout, stderr: process.
             io.stderr.write(USAGE);
             return 2;
         }
-        if (command !== "run" && command !== "check" && command !== "lint" && command !== "list" && command !== "init") {
+        if (!COMMANDS.includes(command)) {
             io.stderr.write(`skillcheck: unknown command "${command}"\n\n${USAGE}`);
             return 2;
         }
@@ -214,6 +223,8 @@ export async function main(argv, io = { stdout: process.stdout, stderr: process.
             return await listCommand(flags, io, deps);
         if (command === "init")
             return await initCommand(positionals[1], flags, io, deps);
+        if (command === "import")
+            return importCommand(positionals[1], flags, io);
         const file = casesFile(positionals[1], io);
         if (command === "run")
             return await runCommand(file, flags, io, deps);
@@ -376,6 +387,47 @@ async function initCommand(positional, flags, io, deps) {
     io.stdout.write(`wrote ${file} (${skills.length} skills listed)\n`);
     return 0;
 }
+function importCommand(positional, flags, io) {
+    if (positional === undefined) {
+        throw new UsageError("import needs the eval set file: skillcheck import <eval_set.json> --skill <name>");
+    }
+    const skill = flags.skill?.trim();
+    if (!skill || skill.includes(","))
+        throw new UsageError("import needs --skill <name>: the skill the eval set is about");
+    const source = path.resolve(io.cwd, positional);
+    let data;
+    try {
+        data = JSON.parse(fs.readFileSync(source, "utf8"));
+    }
+    catch (e) {
+        throw new UsageError(`cannot read ${positional}: ${e.message}`);
+    }
+    let suite;
+    try {
+        suite = evalSetToSuite(data, skill, source);
+    }
+    catch (e) {
+        if (e instanceof ConfigError)
+            throw new UsageError(`${positional}: ${e.errors.join("; ")}`);
+        throw e;
+    }
+    if (flags.out === undefined) {
+        io.stdout.write(suite.text);
+        return 0;
+    }
+    const out = path.resolve(io.cwd, flags.out);
+    if (fs.existsSync(out) && !flags.force)
+        throw new UsageError(`${flags.out} exists, use --force to overwrite`);
+    try {
+        fs.writeFileSync(out, suite.text);
+    }
+    catch (e) {
+        throw new UsageError(`cannot write ${flags.out}: ${e.message}`);
+    }
+    const total = suite.positive + suite.negative;
+    io.stdout.write(`wrote ${flags.out} (${total} cases: ${suite.positive} should load ${skill}, ${suite.negative} should not)\n`);
+    return 0;
+}
 const UNCOVERED_WIDTH = 78;
 function printLint(out, report, docs, suite) {
     const casesPart = suite ? `${report.cases} cases` : "no cases file";
@@ -477,8 +529,8 @@ async function runCommand(file, flags, io, deps) {
     const startedAtMs = Date.now();
     const fatal = new FatalStop();
     const outcome = flags.batch
-        ? await runBatched(adapter.runBatch.bind(adapter), items, flags, model, repeat, reporter, budget, fatal)
-        : await runIndividually(adapter, items, flags, model, directive, repeat, reporter, budget, fatal);
+        ? await runBatched(adapter.runBatch.bind(adapter), items, flags, model, reporter, budget, fatal)
+        : await runIndividually(adapter, items, flags, model, directive, reporter, budget, fatal);
     if (fatal.message !== null) {
         // an environment problem, not a routing result: no summary, no reports
         io.stderr.write(`skillcheck: stopped, the agent cannot run: ${fatal.message}\n`);
@@ -489,6 +541,7 @@ async function runCommand(file, flags, io, deps) {
     for (const item of skipped)
         reporter.caseSkipped(item.c);
     const notStartedRuns = skipped.reduce((n, item) => n + item.runs.length - item.done, 0);
+    const skippedRunCosts = skipped.flatMap((item) => item.runs.flatMap((v) => (v ? [v.costUsd] : [])));
     const ordered = outcome.results.filter((r) => r !== undefined);
     const expected = expectedNames(selected);
     const unavailable = outcome.sawAvailability ? expected.filter((name) => !outcome.available.has(name)) : [];
@@ -499,6 +552,7 @@ async function runCommand(file, flags, io, deps) {
         skipped: skipped.length,
         budget: skipped.length > 0 ? { limitUsd: flags.budget, spent: budget.spent, notStartedRuns } : undefined,
         estimatedUsd: budget.spent,
+        skippedRunCosts,
     });
     if (flags.batch) {
         // cases that only errored have no answer to confirm
@@ -519,6 +573,7 @@ async function runCommand(file, flags, io, deps) {
         confusion: pairs,
         batch: flags.batch,
         estimatedCostUsd: budget.spent,
+        skippedRunCosts,
         budgetUsd: flags.budget ?? null,
         budgetReached: skipped.length > 0,
     });
@@ -544,9 +599,9 @@ function writeReportFile(path, text, flag) {
     }
 }
 /** One agent call per case: the normal mode. */
-async function runIndividually(adapter, items, flags, model, directive, repeat, reporter, budget, fatal) {
+async function runIndividually(adapter, items, flags, model, directive, reporter, budget, fatal) {
     const totalRuns = items.reduce((n, item) => n + item.runs.length, 0);
-    reporter.header(items.length, repeat, 1, totalRuns);
+    reporter.header(items.length, repeatLabel(items), 1, totalRuns);
     const jobs = items.flatMap((item, jobIndex) => item.runs.map((_, runIndex) => ({ item, runIndex, jobIndex })));
     // keep cases in file order even though runs of different cases interleave
     const results = new Array(items.length);
@@ -582,17 +637,19 @@ async function runIndividually(adapter, items, flags, model, directive, repeat,
 }
 /** One agent call per chunk of cases; the model states its choice per request
  * instead of loading skills. Verdicts flow through the same judge. */
-async function runBatched(runBatch, items, flags, model, repeat, reporter, budget, fatal) {
+async function runBatched(runBatch, items, flags, model, reporter, budget, fatal) {
     const chunks = [];
     for (let i = 0; i < items.length; i += flags.batchSize)
         chunks.push(items.slice(i, i + flags.batchSize));
     const rounds = Math.max(...items.map((item) => item.runs.length));
     const jobs = [];
     for (let round = 0; round < rounds; round++) {
+        // a round past every repeat in the chunk makes no call
         for (const chunk of chunks)
-            jobs.push({ chunk, round });
+            if (chunk.some((item) => round < item.runs.length))
+                jobs.push({ chunk, round });
     }
-    reporter.batchHeader(items.length, repeat, chunks.length * rounds);
+    reporter.batchHeader(items.length, repeatLabel(items), jobs.length);
     const results = new Array(items.length);
     await runPool(jobs, flags.jobs, async (job) => {
         // a case with a smaller repeat only takes its first rounds
@@ -604,6 +661,8 @@ async function runBatched(runBatch, items, flags, model, repeat, reporter, budge
         const parsed = parseBatchAnswer(br.structured, br.text);
         const chunkError = br.error ?? parsed.error; // a chunk error is every case's error
         fatal.note(br.error);
+        // one call, one budget entry: its usage prices it when the cost never came
+        budget.add(br.costUsd, br.usage ?? null);
         active.forEach((item, i) => {
             const answer = parsed.answers.get(i + 1);
             const r = {
@@ -616,7 +675,6 @@ async function runBatched(runBatch, items, flags, model, repeat, reporter, budge
                 durationMs: br.durationMs,
             };
             const verdict = judge(item.c, r);
-            budget.add(verdict.costUsd);
             item.runs[job.round] = verdict;
             item.done++;
             if (item.done === item.runs.length) {
@@ -628,6 +686,13 @@ async function runBatched(runBatch, items, flags, model, repeat, reporter, budge
     }, { shouldStart: () => fatal.canStart(budget) });
     return { results, available: new Set(), sawAvailability: false };
 }
+/** "3", or "1–3" when cases override the repeat. */
+function repeatLabel(items) {
+    const counts = items.map((item) => item.runs.length);
+    const min = Math.min(...counts);
+    const max = Math.max(...counts);
+    return min === max ? String(min) : `${min}–${max}`;
+}
 function selectCases(suite, agent, only) {
     const forAgent = suite.cases.filter((c) => !c.agents || c.agents.includes(agent));
     if (!only)

package/dist/import.js

@@ -0,0 +1,50 @@
+import * as path from "node:path";
+import { parse as parseYaml } from "yaml";
+import { ConfigError, parseSuite } from "./cases.js";
+const PLAIN_NAME = /^[A-Za-z0-9._:/-]+$/;
+/**
+ * A skill-creator trigger eval set ([{query, should_trigger}], one skill) as a
+ * cases file: should_trigger true becomes expect, false becomes forbid.
+ */
+export function evalSetToSuite(data, skill, source) {
+    const errors = [];
+    if (!Array.isArray(data) || data.length === 0) {
+        throw new ConfigError(["expected a JSON array of {query, should_trigger}"]);
+    }
+    const items = [];
+    data.forEach((item, i) => {
+        const n = i + 1;
+        if (typeof item !== "object" || item === null || Array.isArray(item)) {
+            errors.push(`item ${n}: expected an object with query and should_trigger`);
+            return;
+        }
+        const { query, should_trigger: trigger } = item;
+        if (typeof query !== "string" || query.trim() === "")
+            errors.push(`item ${n}: query must be a non-empty string`);
+        if (typeof trigger !== "boolean")
+            errors.push(`item ${n}: should_trigger must be true or false`);
+        if (typeof query === "string" && typeof trigger === "boolean")
+            items.push({ query, trigger });
+    });
+    if (errors.length > 0)
+        throw new ConfigError(errors);
+    // plain only when YAML reads it back as the same string: not true, null, 123
+    const name = PLAIN_NAME.test(skill) && parseYaml(skill) === skill ? skill : JSON.stringify(skill);
+    const lines = [
+        `# skillcheck cases imported from ${path.basename(source)} (skill-creator trigger eval set).`,
+        "# should_trigger: true became expect, false became forbid: another skill may",
+        `# still load there, only ${skill} must not.`,
+        "agent: claude",
+        "repeat: 1",
+        "threshold: 1.0",
+        "cases:",
+    ];
+    for (const { query, trigger } of items) {
+        // a JSON string is a valid YAML double-quoted scalar, newlines and quotes included
+        lines.push(`  - query: ${JSON.stringify(query)}`, `    ${trigger ? "expect" : "forbid"}: [${name}]`);
+    }
+    const text = lines.join("\n") + "\n";
+    parseSuite(parseYaml(text));
+    const positive = items.filter((i) => i.trigger).length;
+    return { text, positive, negative: items.length - positive };
+}

package/dist/markdown.js

@@ -96,7 +96,7 @@ function failMark(c) {
     return bad.length > 0 && bad.every((r) => r.error !== null) ? "⚠️" : "❌";
 }
 /** The first non-ok run's reason, the pass share prefixed over several runs,
- * a diagnosis appended in italics. */
+ * a diagnosis appended in italics, then the case note. */
 function reasonCell(c) {
     const bad = c.runs.filter((r) => !r.ok);
     const parts = c.runs.length > 1 ? [`${c.passed}/${c.runs.length}`, bad[0]?.reason ?? ""] : [bad[0]?.reason ?? ""];
@@ -104,6 +104,8 @@ function reasonCell(c) {
     const diagnosis = bad.find((r) => r.diagnosis !== null)?.diagnosis;
     if (diagnosis)
         text += ` — _${diagnosis}_`;
+    if (c.note !== null)
+        text += ` · note: ${c.note}`;
     return text;
 }
 function passedRow(c) {

package/dist/report.js

@@ -11,7 +11,7 @@ export class Reporter {
         this.useColor = out.isTTY === true && !process.env.NO_COLOR;
     }
     header(nCases, repeat, nAgents, totalRuns) {
-        const runs = totalRuns ?? nCases * repeat * nAgents;
+        const runs = totalRuns ?? nCases * Number(repeat) * nAgents;
         const agents = nAgents === 1 ? "agent" : "agents";
         this.write(`${nCases} cases × ${repeat} repeat × ${nAgents} ${agents} = ${runs} runs\n`);
     }
@@ -36,6 +36,8 @@ export class Reporter {
         if (diagnosed?.diagnosis) {
             this.write(`      ${this.paint("33", `diagnosis ${label}: ${diagnosed.diagnosis}`)}\n`);
         }
+        if (!res.ok && res.case.note)
+            this.write(`      ${this.paint("2", `note ${label}: ${oneLine(res.case.note)}`)}\n`);
     }
     /** A case the budget never let start; printed after all real runs settle. */
     caseSkipped(c) {
@@ -68,7 +70,8 @@ export class Reporter {
         const failed = results.filter((r) => !r.ok).length;
         const skipped = extra?.skipped ?? 0;
         const skippedPart = skipped > 0 ? `, ${skipped} skipped` : "";
-        this.write(`${failed} failed${skippedPart} of ${results.length + skipped} · runs ${verdicts.length} · ${costLine(verdicts.map((v) => v.costUsd), extra?.estimatedUsd)}\n`);
+        const costs = [...verdicts.map((v) => v.costUsd), ...(extra?.skippedRunCosts ?? [])];
+        this.write(`${failed} failed${skippedPart} of ${results.length + skipped} · runs ${costs.length} · ${costLine(costs, extra?.estimatedUsd)}\n`);
     }
     paint(code, text) {
         return this.useColor ? `\u001b[${code}m${text}${RESET}` : text;

package/dist/results.js

@@ -2,7 +2,7 @@ import { readVersion } from "./version.js";
 /** The single source of truth for --json and --junit; computed once per run. */
 export function buildReport(input) {
     const verdicts = input.cases.flatMap((e) => e.result?.runs ?? []);
-    const costs = verdicts.map((v) => v.costUsd);
+    const costs = [...verdicts.map((v) => v.costUsd), ...(input.skippedRunCosts ?? [])];
     const known = costs.filter((c) => c !== null);
     return {
         tool: "skillcheck",
@@ -17,7 +17,7 @@ export function buildReport(input) {
             cases: input.cases.length,
             failed: input.cases.filter((e) => e.result && !e.result.ok).length,
             skipped: input.cases.filter((e) => !e.result).length,
-            runs: verdicts.length,
+            runs: costs.length,
             costUsd: known.reduce((a, b) => a + b, 0),
             unknownCostRuns: costs.length - known.length,
             estimatedCostUsd: input.estimatedCostUsd,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@icntswm/skillcheck",
-  "version": "1.1.0",
+  "version": "1.2.0",
   "description": "Regression tests for Claude Code skills: check that every request still loads the right skill",
   "type": "module",
   "bin": {