@icntswm/skillcheck 1.0.4 → 1.2.0

package/README.md

@@ -36,9 +36,10 @@ requests.
   from a broken one instead of letting you guess.
 - 🏷️ **Catches renames and typos.** Case names are checked against the skills
   the agent really has, so a renamed skill can't pass silently.
-- ⚙️ **CI-ready.** A GitHub Action (`uses: icntswm/skillcheck@v1`), JUnit
-  and JSON reports, clear exit codes, a spending cap (`--budget`), and
-  `--config-dir` to test only the skills in your repository.
+- ⚙️ **CI-ready.** A GitHub Action (`uses: icntswm/skillcheck@v1`) that
+  comments the report on the pull request, JUnit, JSON and Markdown reports,
+  clear exit codes, a spending cap (`--budget`), and `--config-dir` to test
+  only the skills in your repository.
 - 📄 **Plain YAML, one dependency.** Cases are readable by anyone on the team
   and live next to the skills they test.
 
@@ -119,13 +120,17 @@ skillcheck run --only 2,5    # confirm what the batch flagged
 `skillcheck list` prints the skills and slash commands Claude Code sees. It
 stops Claude Code before the first model call, so it costs nothing.
 
+Already tuned a description with Anthropic's skill-creator? `skillcheck import
+eval_set.json --skill <name>` turns its trigger eval set into cases (see
+[Writing cases](docs/writing-cases.md#from-skill-creator)).
+
 ## Documentation
 
 | | |
 |---|---|
 | [Writing cases](docs/writing-cases.md) | the file format and what makes a case catch regressions |
 | [Cost](docs/cost.md) | what a run costs and how to spend less |
-| [Reports and CI](docs/ci.md) | confusion block, JSON, JUnit, GitHub Actions, exit codes |
+| [Reports and CI](docs/ci.md) | confusion block, JSON, JUnit, Markdown, GitHub Actions, exit codes |
 | [How it works](docs/how-it-works.md) | what happens inside a run, and the limits of each level |
 
 `skillcheck --help` lists every command and option.

package/dist/agents/claude.js

@@ -327,7 +327,8 @@ export class ClaudeAdapter {
                 },
             });
             const { text, costUsd, structuredOutput } = out.stream.result;
-            return { structured: structuredOutput, text, costUsd, error: withLoginHint(out.error, opts.configDir), durationMs: out.durationMs };
+            const error = withLoginHint(out.error, opts.configDir);
+            return { structured: structuredOutput, text, costUsd, usage: out.stream.usage, error, durationMs: out.durationMs };
         }
         finally {
             await rm(workdir, { recursive: true, force: true });
@@ -408,13 +409,15 @@ export class ClaudeAdapter {
                     clearTimeout(killTimer);
                 if (timeoutTimer)
                     clearTimeout(timeoutTimer);
-                // A non-zero exit is not a failure by itself: error_max_turns is normal.
-                // Only a stdout with nothing in it means the run really did not happen.
+                // A non-zero exit is not a failure by itself: error_max_turns is normal
+                // and ends with a result event. Without one, the run ended before routing
+                // was over, unless we stopped it ourselves.
                 if (!error)
                     error = stream.error;
-                if (!error && !stdoutSeen) {
+                if (!error && !stopping && !stream.finished) {
                     const detail = stderrText.trim().slice(0, 120);
-                    error = `claude exited with code ${code ?? signal}${detail ? `: ${detail}` : ""}`;
+                    const why = stdoutSeen ? `claude exited with code ${code ?? signal} before its result` : `claude exited with code ${code ?? signal}`;
+                    error = `${why}${detail ? `: ${detail}` : ""}`;
                 }
                 resolve({ stream, error, stoppedEarly, durationMs: Date.now() - startedAt });
             };

package/dist/cli.js

@@ -11,8 +11,10 @@ import { BUILTIN_SKILLS, ConfigError, findDefaultCasesFile, knownSkillNames, loa
 import { confusion } from "./confusion.js";
 import { loadSkillDocs } from "./describe.js";
 import { aggregate, judge } from "./judge.js";
+import { evalSetToSuite } from "./import.js";
 import { toJunit } from "./junit.js";
 import { lint, LINT_DEFAULTS } from "./lint.js";
+import { toMarkdown } from "./markdown.js";
 import { runPool } from "./pool.js";
 import { oneLine, Reporter } from "./report.js";
 import { buildReport } from "./results.js";
@@ -25,6 +27,8 @@ Usage:
   skillcheck lint [file] [options]    static checks on descriptions and cases
   skillcheck list [options]           print the skills and commands the agent can load
   skillcheck init [file] [options]    write a starter cases file with those names
+  skillcheck import <file> --skill <name>
+                                      turn a skill-creator trigger eval set into cases
 
 run options:
   -a, --agent <name>     agent to route with (default: suite or claude)
@@ -42,6 +46,7 @@ run options:
       --json <path>      machine-readable report to path ("-" writes it to stdout
                          and the terminal report to stderr)
       --junit <path>     JUnit XML report (GitLab/GitHub test reporters) to path
+      --markdown <path>  Markdown summary (PR comments, GitHub job summary) to path
 run/check options:
       --skill <a,b>      keep only cases that mention these skills
 check options:
@@ -57,6 +62,9 @@ list/init options:
                          CLAUDE_CONFIG_DIR; run, list and init also set it for the agent
 init options:
       --force            overwrite an existing file
+import options:
+      --skill <name>     the skill the eval set is about
+  -o, --out <file>       write the cases there instead of stdout (--force overwrites)
 common:
   -h, --help             show this help
       --version          show version
@@ -67,6 +75,7 @@ nothing reaches the model and nothing is billed, even when not logged in.
 
 Exit codes: 0 all passed, 1 some case failed or was skipped, 2 config or environment error.
 `;
+const COMMANDS = ["run", "check", "lint", "list", "init", "import"];
 const OPTIONS = {
     help: { type: "boolean", short: "h", default: false },
     version: { type: "boolean", default: false },
@@ -86,11 +95,13 @@ const OPTIONS = {
     budget: { type: "string" },
     json: { type: "string" },
     junit: { type: "string" },
+    markdown: { type: "string" },
     top: { type: "string" },
     overlap: { type: "string" },
     strict: { type: "boolean", default: false },
     "config-dir": { type: "string" },
     force: { type: "boolean", default: false },
+    out: { type: "string", short: "o" },
 };
 /** Config/environment problems: message to stderr, exit 2. */
 class UsageError extends Error {
@@ -120,11 +131,13 @@ function parseFlags(values, cwd) {
         budget: numberFlag(values.budget),
         json: values.json,
         junit: values.junit,
+        markdown: values.markdown,
         top: intFlag(values.top, "top", 1) ?? 5,
         overlap: ratioFlag(values.overlap) ?? 0.3,
         strict: values.strict,
         configDir: values["config-dir"] !== undefined ? resolveConfigDir(values["config-dir"], cwd) : undefined,
         force: values.force,
+        out: values.out,
     };
 }
 /** --config-dir: absolute, must be an existing directory. */
@@ -199,7 +212,7 @@ export async function main(argv, io = { stdout: process.stdout, stderr: process.
             io.stderr.write(USAGE);
             return 2;
         }
-        if (command !== "run" && command !== "check" && command !== "lint" && command !== "list" && command !== "init") {
+        if (!COMMANDS.includes(command)) {
             io.stderr.write(`skillcheck: unknown command "${command}"\n\n${USAGE}`);
             return 2;
         }
@@ -210,6 +223,8 @@ export async function main(argv, io = { stdout: process.stdout, stderr: process.
             return await listCommand(flags, io, deps);
         if (command === "init")
             return await initCommand(positionals[1], flags, io, deps);
+        if (command === "import")
+            return importCommand(positionals[1], flags, io);
         const file = casesFile(positionals[1], io);
         if (command === "run")
             return await runCommand(file, flags, io, deps);
@@ -372,6 +387,47 @@ async function initCommand(positional, flags, io, deps) {
     io.stdout.write(`wrote ${file} (${skills.length} skills listed)\n`);
     return 0;
 }
+function importCommand(positional, flags, io) {
+    if (positional === undefined) {
+        throw new UsageError("import needs the eval set file: skillcheck import <eval_set.json> --skill <name>");
+    }
+    const skill = flags.skill?.trim();
+    if (!skill || skill.includes(","))
+        throw new UsageError("import needs --skill <name>: the skill the eval set is about");
+    const source = path.resolve(io.cwd, positional);
+    let data;
+    try {
+        data = JSON.parse(fs.readFileSync(source, "utf8"));
+    }
+    catch (e) {
+        throw new UsageError(`cannot read ${positional}: ${e.message}`);
+    }
+    let suite;
+    try {
+        suite = evalSetToSuite(data, skill, source);
+    }
+    catch (e) {
+        if (e instanceof ConfigError)
+            throw new UsageError(`${positional}: ${e.errors.join("; ")}`);
+        throw e;
+    }
+    if (flags.out === undefined) {
+        io.stdout.write(suite.text);
+        return 0;
+    }
+    const out = path.resolve(io.cwd, flags.out);
+    if (fs.existsSync(out) && !flags.force)
+        throw new UsageError(`${flags.out} exists, use --force to overwrite`);
+    try {
+        fs.writeFileSync(out, suite.text);
+    }
+    catch (e) {
+        throw new UsageError(`cannot write ${flags.out}: ${e.message}`);
+    }
+    const total = suite.positive + suite.negative;
+    io.stdout.write(`wrote ${flags.out} (${total} cases: ${suite.positive} should load ${skill}, ${suite.negative} should not)\n`);
+    return 0;
+}
 const UNCOVERED_WIDTH = 78;
 function printLint(out, report, docs, suite) {
     const casesPart = suite ? `${report.cases} cases` : "no cases file";
@@ -473,8 +529,8 @@ async function runCommand(file, flags, io, deps) {
     const startedAtMs = Date.now();
     const fatal = new FatalStop();
     const outcome = flags.batch
-        ? await runBatched(adapter.runBatch.bind(adapter), items, flags, model, repeat, reporter, budget, fatal)
-        : await runIndividually(adapter, items, flags, model, directive, repeat, reporter, budget, fatal);
+        ? await runBatched(adapter.runBatch.bind(adapter), items, flags, model, reporter, budget, fatal)
+        : await runIndividually(adapter, items, flags, model, directive, reporter, budget, fatal);
     if (fatal.message !== null) {
         // an environment problem, not a routing result: no summary, no reports
         io.stderr.write(`skillcheck: stopped, the agent cannot run: ${fatal.message}\n`);
@@ -485,6 +541,7 @@ async function runCommand(file, flags, io, deps) {
     for (const item of skipped)
         reporter.caseSkipped(item.c);
     const notStartedRuns = skipped.reduce((n, item) => n + item.runs.length - item.done, 0);
+    const skippedRunCosts = skipped.flatMap((item) => item.runs.flatMap((v) => (v ? [v.costUsd] : [])));
     const ordered = outcome.results.filter((r) => r !== undefined);
     const expected = expectedNames(selected);
     const unavailable = outcome.sawAvailability ? expected.filter((name) => !outcome.available.has(name)) : [];
@@ -495,6 +552,7 @@ async function runCommand(file, flags, io, deps) {
         skipped: skipped.length,
         budget: skipped.length > 0 ? { limitUsd: flags.budget, spent: budget.spent, notStartedRuns } : undefined,
         estimatedUsd: budget.spent,
+        skippedRunCosts,
     });
     if (flags.batch) {
         // cases that only errored have no answer to confirm
@@ -513,7 +571,9 @@ async function runCommand(file, flags, io, deps) {
         cases: items.map((item) => ({ c: item.c, threshold: item.threshold, result: outcome.results[item.jobIndex] ?? null })),
         unavailable,
         confusion: pairs,
+        batch: flags.batch,
         estimatedCostUsd: budget.spent,
+        skippedRunCosts,
         budgetUsd: flags.budget ?? null,
         budgetReached: skipped.length > 0,
     });
@@ -526,6 +586,8 @@ async function runCommand(file, flags, io, deps) {
     }
     if (flags.junit !== undefined)
         writeReportFile(flags.junit, toJunit(report), "--junit");
+    if (flags.markdown !== undefined)
+        writeReportFile(flags.markdown, toMarkdown(report), "--markdown");
     return ordered.some((r) => !r.ok) || skipped.length > 0 ? 1 : 0;
 }
 function writeReportFile(path, text, flag) {
@@ -537,9 +599,9 @@ function writeReportFile(path, text, flag) {
     }
 }
 /** One agent call per case: the normal mode. */
-async function runIndividually(adapter, items, flags, model, directive, repeat, reporter, budget, fatal) {
+async function runIndividually(adapter, items, flags, model, directive, reporter, budget, fatal) {
     const totalRuns = items.reduce((n, item) => n + item.runs.length, 0);
-    reporter.header(items.length, repeat, 1, totalRuns);
+    reporter.header(items.length, repeatLabel(items), 1, totalRuns);
     const jobs = items.flatMap((item, jobIndex) => item.runs.map((_, runIndex) => ({ item, runIndex, jobIndex })));
     // keep cases in file order even though runs of different cases interleave
     const results = new Array(items.length);
@@ -575,17 +637,19 @@ async function runIndividually(adapter, items, flags, model, directive, repeat,
 }
 /** One agent call per chunk of cases; the model states its choice per request
  * instead of loading skills. Verdicts flow through the same judge. */
-async function runBatched(runBatch, items, flags, model, repeat, reporter, budget, fatal) {
+async function runBatched(runBatch, items, flags, model, reporter, budget, fatal) {
     const chunks = [];
     for (let i = 0; i < items.length; i += flags.batchSize)
         chunks.push(items.slice(i, i + flags.batchSize));
     const rounds = Math.max(...items.map((item) => item.runs.length));
     const jobs = [];
     for (let round = 0; round < rounds; round++) {
+        // a round past every repeat in the chunk makes no call
         for (const chunk of chunks)
-            jobs.push({ chunk, round });
+            if (chunk.some((item) => round < item.runs.length))
+                jobs.push({ chunk, round });
     }
-    reporter.batchHeader(items.length, repeat, chunks.length * rounds);
+    reporter.batchHeader(items.length, repeatLabel(items), jobs.length);
     const results = new Array(items.length);
     await runPool(jobs, flags.jobs, async (job) => {
         // a case with a smaller repeat only takes its first rounds
@@ -597,6 +661,8 @@ async function runBatched(runBatch, items, flags, model, repeat, reporter, budge
         const parsed = parseBatchAnswer(br.structured, br.text);
         const chunkError = br.error ?? parsed.error; // a chunk error is every case's error
         fatal.note(br.error);
+        // one call, one budget entry: its usage prices it when the cost never came
+        budget.add(br.costUsd, br.usage ?? null);
         active.forEach((item, i) => {
             const answer = parsed.answers.get(i + 1);
             const r = {
@@ -609,7 +675,6 @@ async function runBatched(runBatch, items, flags, model, repeat, reporter, budge
                 durationMs: br.durationMs,
             };
             const verdict = judge(item.c, r);
-            budget.add(verdict.costUsd);
             item.runs[job.round] = verdict;
             item.done++;
             if (item.done === item.runs.length) {
@@ -621,6 +686,13 @@ async function runBatched(runBatch, items, flags, model, repeat, reporter, budge
     }, { shouldStart: () => fatal.canStart(budget) });
     return { results, available: new Set(), sawAvailability: false };
 }
+/** "3", or "1–3" when cases override the repeat. */
+function repeatLabel(items) {
+    const counts = items.map((item) => item.runs.length);
+    const min = Math.min(...counts);
+    const max = Math.max(...counts);
+    return min === max ? String(min) : `${min}–${max}`;
+}
 function selectCases(suite, agent, only) {
     const forAgent = suite.cases.filter((c) => !c.agents || c.agents.includes(agent));
     if (!only)

package/dist/import.js

@@ -0,0 +1,50 @@
+import * as path from "node:path";
+import { parse as parseYaml } from "yaml";
+import { ConfigError, parseSuite } from "./cases.js";
+const PLAIN_NAME = /^[A-Za-z0-9._:/-]+$/;
+/**
+ * A skill-creator trigger eval set ([{query, should_trigger}], one skill) as a
+ * cases file: should_trigger true becomes expect, false becomes forbid.
+ */
+export function evalSetToSuite(data, skill, source) {
+    const errors = [];
+    if (!Array.isArray(data) || data.length === 0) {
+        throw new ConfigError(["expected a JSON array of {query, should_trigger}"]);
+    }
+    const items = [];
+    data.forEach((item, i) => {
+        const n = i + 1;
+        if (typeof item !== "object" || item === null || Array.isArray(item)) {
+            errors.push(`item ${n}: expected an object with query and should_trigger`);
+            return;
+        }
+        const { query, should_trigger: trigger } = item;
+        if (typeof query !== "string" || query.trim() === "")
+            errors.push(`item ${n}: query must be a non-empty string`);
+        if (typeof trigger !== "boolean")
+            errors.push(`item ${n}: should_trigger must be true or false`);
+        if (typeof query === "string" && typeof trigger === "boolean")
+            items.push({ query, trigger });
+    });
+    if (errors.length > 0)
+        throw new ConfigError(errors);
+    // plain only when YAML reads it back as the same string: not true, null, 123
+    const name = PLAIN_NAME.test(skill) && parseYaml(skill) === skill ? skill : JSON.stringify(skill);
+    const lines = [
+        `# skillcheck cases imported from ${path.basename(source)} (skill-creator trigger eval set).`,
+        "# should_trigger: true became expect, false became forbid: another skill may",
+        `# still load there, only ${skill} must not.`,
+        "agent: claude",
+        "repeat: 1",
+        "threshold: 1.0",
+        "cases:",
+    ];
+    for (const { query, trigger } of items) {
+        // a JSON string is a valid YAML double-quoted scalar, newlines and quotes included
+        lines.push(`  - query: ${JSON.stringify(query)}`, `    ${trigger ? "expect" : "forbid"}: [${name}]`);
+    }
+    const text = lines.join("\n") + "\n";
+    parseSuite(parseYaml(text));
+    const positive = items.filter((i) => i.trigger).length;
+    return { text, positive, negative: items.length - positive };
+}

package/dist/markdown.js

@@ -0,0 +1,135 @@
+/** An HTML comment so GitHub renders the report body clean under it. */
+export const MARKDOWN_MARKER = "<!-- skillcheck -->";
+/** Failure-focused Markdown summary for PR comments and GitHub job summaries. */
+export function toMarkdown(report) {
+    const s = report.summary;
+    const lines = [MARKDOWN_MARKER, header(report)];
+    lines.push("");
+    lines.push(metaLine(report));
+    if (report.batch) {
+        lines.push("");
+        lines.push("> Batch mode: answers are the model's stated choice, not an actual Skill call. Confirm failures with a normal run.");
+    }
+    if (s.budgetReached && s.budgetUsd !== null) {
+        lines.push("");
+        lines.push(`> ⚠️ Budget $${s.budgetUsd.toFixed(2)} reached, remaining cases were skipped.`);
+    }
+    if (report.unavailable.length > 0) {
+        lines.push("");
+        lines.push(`> ⚠️ Expected skills not available to the agent: ${report.unavailable.join(", ")}`);
+    }
+    if (s.failed + s.skipped > 0) {
+        lines.push("");
+        lines.push("| | Case | Request | Loaded | Reason |");
+        lines.push("|---|---|---|---|---|");
+        for (const c of report.cases) {
+            if (c.status === "passed")
+                continue;
+            lines.push(failedRow(c));
+        }
+    }
+    const passed = report.cases.filter((c) => c.status === "passed");
+    if (passed.length > 0) {
+        lines.push("");
+        lines.push(`<details><summary>Passed (${passed.length})</summary>`);
+        lines.push("");
+        lines.push("| Case | Request | Loaded |");
+        lines.push("|---|---|---|");
+        for (const c of passed)
+            lines.push(passedRow(c));
+        lines.push("");
+        lines.push("</details>");
+    }
+    if (report.confusion.length > 0) {
+        lines.push("");
+        lines.push("**Confusion** — descriptions to rewrite:");
+        lines.push("");
+        for (const p of report.confusion)
+            lines.push(`- expected \`${p.expected}\` → got \`${p.got}\` (${p.count})`);
+    }
+    return lines.join("\n") + "\n";
+}
+/** ✅ with the pass count alone, ❌ with failed and skipped counts. */
+function header(report) {
+    const s = report.summary;
+    if (s.failed === 0 && s.skipped === 0) {
+        return `### ✅ skillcheck: ${s.cases} passed`;
+    }
+    const failedPart = s.failed > 0 ? `${s.failed} failed` : null;
+    const skippedPart = s.skipped > 0 ? `${s.skipped} skipped` : null;
+    const parts = [failedPart, skippedPart].filter((p) => p !== null).join(", ");
+    return `### ❌ skillcheck: ${parts} of ${s.cases}`;
+}
+/** Model, batch flag, runs, cost and duration, " · " separated. */
+function metaLine(report) {
+    const s = report.summary;
+    const parts = [];
+    if (report.model !== null)
+        parts.push(report.model);
+    if (report.batch)
+        parts.push("batch mode");
+    parts.push(`${s.runs} run${s.runs === 1 ? "" : "s"}`);
+    if (s.unknownCostRuns > 0 && s.estimatedCostUsd > s.costUsd)
+        parts.push(`cost ~$${s.estimatedCostUsd.toFixed(2)}`);
+    else
+        parts.push(`cost $${s.costUsd.toFixed(2)}`);
+    parts.push(duration(report.durationMs));
+    return parts.join(" · ");
+}
+/** Below a minute in seconds, then minutes and seconds, seconds rounded down. */
+function duration(ms) {
+    const total = Math.floor(ms / 1000);
+    if (total < 60)
+        return `${total}s`;
+    return `${Math.floor(total / 60)}m ${total % 60}s`;
+}
+/** ⏭️ for skipped, ⚠️ when every failed run errored, ❌ for the rest. */
+function failedRow(c) {
+    const cells = c.status === "skipped"
+        ? ["⏭️", caseLabel(c), oneLine(c.query), loaded(c), "budget reached"]
+        : [failMark(c), caseLabel(c), oneLine(c.query), loaded(c), reasonCell(c)];
+    return `| ${cells.map(esc).join(" | ")} |`;
+}
+/** ⚠️ when every non-ok run errored (the model never answered), ❌ otherwise. */
+function failMark(c) {
+    const bad = c.runs.filter((r) => !r.ok);
+    return bad.length > 0 && bad.every((r) => r.error !== null) ? "⚠️" : "❌";
+}
+/** The first non-ok run's reason, the pass share prefixed over several runs,
+ * a diagnosis appended in italics, then the case note. */
+function reasonCell(c) {
+    const bad = c.runs.filter((r) => !r.ok);
+    const parts = c.runs.length > 1 ? [`${c.passed}/${c.runs.length}`, bad[0]?.reason ?? ""] : [bad[0]?.reason ?? ""];
+    let text = parts.join(" · ");
+    const diagnosis = bad.find((r) => r.diagnosis !== null)?.diagnosis;
+    if (diagnosis)
+        text += ` — _${diagnosis}_`;
+    if (c.note !== null)
+        text += ` · note: ${c.note}`;
+    return text;
+}
+function passedRow(c) {
+    const score = c.runs.length > 1 ? ` (${c.passed}/${c.runs.length})` : "";
+    const cells = [`${caseLabel(c)}${score}`, oneLine(c.query), loaded(c)];
+    return `| ${cells.map(esc).join(" | ")} |`;
+}
+function caseLabel(c) {
+    return `#${c.id ?? c.index}`;
+}
+/** Unique loaded skills across runs, "—" when none. */
+function loaded(c) {
+    const names = [...new Set(c.runs.flatMap((r) => r.loaded))];
+    return names.length > 0 ? names.join(", ") : "—";
+}
+/** Collapse to one line, truncated to 80 chars with "…". */
+function oneLine(text) {
+    const flat = text.replace(/\s+/g, " ").trim();
+    return flat.length > 80 ? `${flat.slice(0, 80)}…` : flat;
+}
+/** Pipes so they cannot break the table, newlines as spaces, HTML inert. */
+function esc(text) {
+    return text
+        .replace(/\|/g, "\\|")
+        .replace(/[\r\n]+/g, " ")
+        .replace(/</g, "&lt;");
+}

package/dist/report.js

@@ -11,7 +11,7 @@ export class Reporter {
         this.useColor = out.isTTY === true && !process.env.NO_COLOR;
     }
     header(nCases, repeat, nAgents, totalRuns) {
-        const runs = totalRuns ?? nCases * repeat * nAgents;
+        const runs = totalRuns ?? nCases * Number(repeat) * nAgents;
         const agents = nAgents === 1 ? "agent" : "agents";
         this.write(`${nCases} cases × ${repeat} repeat × ${nAgents} ${agents} = ${runs} runs\n`);
     }
@@ -36,6 +36,8 @@ export class Reporter {
         if (diagnosed?.diagnosis) {
             this.write(`      ${this.paint("33", `diagnosis ${label}: ${diagnosed.diagnosis}`)}\n`);
         }
+        if (!res.ok && res.case.note)
+            this.write(`      ${this.paint("2", `note ${label}: ${oneLine(res.case.note)}`)}\n`);
     }
     /** A case the budget never let start; printed after all real runs settle. */
     caseSkipped(c) {
@@ -68,7 +70,8 @@ export class Reporter {
         const failed = results.filter((r) => !r.ok).length;
         const skipped = extra?.skipped ?? 0;
         const skippedPart = skipped > 0 ? `, ${skipped} skipped` : "";
-        this.write(`${failed} failed${skippedPart} of ${results.length + skipped} · runs ${verdicts.length} · ${costLine(verdicts.map((v) => v.costUsd), extra?.estimatedUsd)}\n`);
+        const costs = [...verdicts.map((v) => v.costUsd), ...(extra?.skippedRunCosts ?? [])];
+        this.write(`${failed} failed${skippedPart} of ${results.length + skipped} · runs ${costs.length} · ${costLine(costs, extra?.estimatedUsd)}\n`);
     }
     paint(code, text) {
         return this.useColor ? `\u001b[${code}m${text}${RESET}` : text;

package/dist/results.js

@@ -2,7 +2,7 @@ import { readVersion } from "./version.js";
 /** The single source of truth for --json and --junit; computed once per run. */
 export function buildReport(input) {
     const verdicts = input.cases.flatMap((e) => e.result?.runs ?? []);
-    const costs = verdicts.map((v) => v.costUsd);
+    const costs = [...verdicts.map((v) => v.costUsd), ...(input.skippedRunCosts ?? [])];
     const known = costs.filter((c) => c !== null);
     return {
         tool: "skillcheck",
@@ -10,13 +10,14 @@ export function buildReport(input) {
         file: input.file,
         agent: input.agent,
         model: input.model,
+        batch: input.batch ?? false,
         startedAt: new Date(input.startedAtMs).toISOString(),
         durationMs: input.durationMs,
         summary: {
             cases: input.cases.length,
             failed: input.cases.filter((e) => e.result && !e.result.ok).length,
             skipped: input.cases.filter((e) => !e.result).length,
-            runs: verdicts.length,
+            runs: costs.length,
             costUsd: known.reduce((a, b) => a + b, 0),
             unknownCostRuns: costs.length - known.length,
             estimatedCostUsd: input.estimatedCostUsd,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@icntswm/skillcheck",
-  "version": "1.0.4",
+  "version": "1.2.0",
   "description": "Regression tests for Claude Code skills: check that every request still loads the right skill",
   "type": "module",
   "bin": {