@icntswm/skillcheck 1.0.3 → 1.1.0

package/README.md

@@ -36,9 +36,10 @@ requests.
   from a broken one instead of letting you guess.
 - 🏷️ **Catches renames and typos.** Case names are checked against the skills
   the agent really has, so a renamed skill can't pass silently.
-- ⚙️ **CI-ready.** A GitHub Action (`uses: icntswm/skillcheck@v1`), JUnit
-  and JSON reports, clear exit codes, a spending cap (`--budget`), and
-  `--config-dir` to test only the skills in your repository.
+- ⚙️ **CI-ready.** A GitHub Action (`uses: icntswm/skillcheck@v1`) that
+  comments the report on the pull request, JUnit, JSON and Markdown reports,
+  clear exit codes, a spending cap (`--budget`), and `--config-dir` to test
+  only the skills in your repository.
 - 📄 **Plain YAML, one dependency.** Cases are readable by anyone on the team
   and live next to the skills they test.
 
@@ -125,7 +126,7 @@ stops Claude Code before the first model call, so it costs nothing.
 |---|---|
 | [Writing cases](docs/writing-cases.md) | the file format and what makes a case catch regressions |
 | [Cost](docs/cost.md) | what a run costs and how to spend less |
-| [Reports and CI](docs/ci.md) | confusion block, JSON, JUnit, GitHub Actions, exit codes |
+| [Reports and CI](docs/ci.md) | confusion block, JSON, JUnit, Markdown, GitHub Actions, exit codes |
 | [How it works](docs/how-it-works.md) | what happens inside a run, and the limits of each level |
 
 `skillcheck --help` lists every command and option.

package/dist/budget.js

@@ -18,8 +18,9 @@ function listInputPrice(model) {
     return 5e-6;
 }
 /**
- * Running spend estimate for --budget. Runs killed by early-stop never report
- * a cost, so they are priced from their token usage: at the rate seen in
+ * Running spend estimate for --budget. claude usually reports its cost even
+ * when early-stop kills it, but a run killed before the result event does
+ * not; such runs are priced from their token usage: at the rate seen in
  * finished runs, or at list price before any run finishes. Runs with neither
  * cost nor usage count at the average known cost.
  */

package/dist/cli.js

@@ -13,6 +13,7 @@ import { loadSkillDocs } from "./describe.js";
 import { aggregate, judge } from "./judge.js";
 import { toJunit } from "./junit.js";
 import { lint, LINT_DEFAULTS } from "./lint.js";
+import { toMarkdown } from "./markdown.js";
 import { runPool } from "./pool.js";
 import { oneLine, Reporter } from "./report.js";
 import { buildReport } from "./results.js";
@@ -42,6 +43,7 @@ run options:
       --json <path>      machine-readable report to path ("-" writes it to stdout
                          and the terminal report to stderr)
       --junit <path>     JUnit XML report (GitLab/GitHub test reporters) to path
+      --markdown <path>  Markdown summary (PR comments, GitHub job summary) to path
 run/check options:
       --skill <a,b>      keep only cases that mention these skills
 check options:
@@ -86,6 +88,7 @@ const OPTIONS = {
     budget: { type: "string" },
     json: { type: "string" },
     junit: { type: "string" },
+    markdown: { type: "string" },
     top: { type: "string" },
     overlap: { type: "string" },
     strict: { type: "boolean", default: false },
@@ -120,6 +123,7 @@ function parseFlags(values, cwd) {
         budget: numberFlag(values.budget),
         json: values.json,
         junit: values.junit,
+        markdown: values.markdown,
         top: intFlag(values.top, "top", 1) ?? 5,
         overlap: ratioFlag(values.overlap) ?? 0.3,
         strict: values.strict,
@@ -513,6 +517,7 @@ async function runCommand(file, flags, io, deps) {
         cases: items.map((item) => ({ c: item.c, threshold: item.threshold, result: outcome.results[item.jobIndex] ?? null })),
         unavailable,
         confusion: pairs,
+        batch: flags.batch,
         estimatedCostUsd: budget.spent,
         budgetUsd: flags.budget ?? null,
         budgetReached: skipped.length > 0,
@@ -526,6 +531,8 @@ async function runCommand(file, flags, io, deps) {
     }
     if (flags.junit !== undefined)
         writeReportFile(flags.junit, toJunit(report), "--junit");
+    if (flags.markdown !== undefined)
+        writeReportFile(flags.markdown, toMarkdown(report), "--markdown");
     return ordered.some((r) => !r.ok) || skipped.length > 0 ? 1 : 0;
 }
 function writeReportFile(path, text, flag) {

package/dist/junit.js

@@ -14,9 +14,10 @@ export function toJunit(report) {
     if (report.model !== null)
         properties.push(`      <property name="model" value="${esc(report.model)}"/>`);
     properties.push(`      <property name="costUsd" value="${report.summary.costUsd.toFixed(2)}"/>`);
-    // early-stopped runs report no cost, so costUsd alone understates the spend
+    // a run killed before its result event reports no cost, so costUsd alone understates the spend
     if (report.summary.unknownCostRuns > 0) {
         properties.push(`      <property name="unknownCostRuns" value="${report.summary.unknownCostRuns}"/>`);
+        properties.push(`      <property name="estimatedCostUsd" value="${report.summary.estimatedCostUsd.toFixed(2)}"/>`);
     }
     properties.push("    </properties>");
     const cases = rows.map(({ c, kind }) => testcase(c, kind, suiteName));

package/dist/markdown.js

@@ -0,0 +1,133 @@
+/** An HTML comment so GitHub renders the report body clean under it. */
+export const MARKDOWN_MARKER = "<!-- skillcheck -->";
+/** Failure-focused Markdown summary for PR comments and GitHub job summaries. */
+export function toMarkdown(report) {
+    const s = report.summary;
+    const lines = [MARKDOWN_MARKER, header(report)];
+    lines.push("");
+    lines.push(metaLine(report));
+    if (report.batch) {
+        lines.push("");
+        lines.push("> Batch mode: answers are the model's stated choice, not an actual Skill call. Confirm failures with a normal run.");
+    }
+    if (s.budgetReached && s.budgetUsd !== null) {
+        lines.push("");
+        lines.push(`> ⚠️ Budget $${s.budgetUsd.toFixed(2)} reached, remaining cases were skipped.`);
+    }
+    if (report.unavailable.length > 0) {
+        lines.push("");
+        lines.push(`> ⚠️ Expected skills not available to the agent: ${report.unavailable.join(", ")}`);
+    }
+    if (s.failed + s.skipped > 0) {
+        lines.push("");
+        lines.push("| | Case | Request | Loaded | Reason |");
+        lines.push("|---|---|---|---|---|");
+        for (const c of report.cases) {
+            if (c.status === "passed")
+                continue;
+            lines.push(failedRow(c));
+        }
+    }
+    const passed = report.cases.filter((c) => c.status === "passed");
+    if (passed.length > 0) {
+        lines.push("");
+        lines.push(`<details><summary>Passed (${passed.length})</summary>`);
+        lines.push("");
+        lines.push("| Case | Request | Loaded |");
+        lines.push("|---|---|---|");
+        for (const c of passed)
+            lines.push(passedRow(c));
+        lines.push("");
+        lines.push("</details>");
+    }
+    if (report.confusion.length > 0) {
+        lines.push("");
+        lines.push("**Confusion** — descriptions to rewrite:");
+        lines.push("");
+        for (const p of report.confusion)
+            lines.push(`- expected \`${p.expected}\` → got \`${p.got}\` (${p.count})`);
+    }
+    return lines.join("\n") + "\n";
+}
+/** ✅ with the pass count alone, ❌ with failed and skipped counts. */
+function header(report) {
+    const s = report.summary;
+    if (s.failed === 0 && s.skipped === 0) {
+        return `### ✅ skillcheck: ${s.cases} passed`;
+    }
+    const failedPart = s.failed > 0 ? `${s.failed} failed` : null;
+    const skippedPart = s.skipped > 0 ? `${s.skipped} skipped` : null;
+    const parts = [failedPart, skippedPart].filter((p) => p !== null).join(", ");
+    return `### ❌ skillcheck: ${parts} of ${s.cases}`;
+}
+/** Model, batch flag, runs, cost and duration, " · " separated. */
+function metaLine(report) {
+    const s = report.summary;
+    const parts = [];
+    if (report.model !== null)
+        parts.push(report.model);
+    if (report.batch)
+        parts.push("batch mode");
+    parts.push(`${s.runs} run${s.runs === 1 ? "" : "s"}`);
+    if (s.unknownCostRuns > 0 && s.estimatedCostUsd > s.costUsd)
+        parts.push(`cost ~$${s.estimatedCostUsd.toFixed(2)}`);
+    else
+        parts.push(`cost $${s.costUsd.toFixed(2)}`);
+    parts.push(duration(report.durationMs));
+    return parts.join(" · ");
+}
+/** Below a minute in seconds, then minutes and seconds, seconds rounded down. */
+function duration(ms) {
+    const total = Math.floor(ms / 1000);
+    if (total < 60)
+        return `${total}s`;
+    return `${Math.floor(total / 60)}m ${total % 60}s`;
+}
+/** ⏭️ for skipped, ⚠️ when every failed run errored, ❌ for the rest. */
+function failedRow(c) {
+    const cells = c.status === "skipped"
+        ? ["⏭️", caseLabel(c), oneLine(c.query), loaded(c), "budget reached"]
+        : [failMark(c), caseLabel(c), oneLine(c.query), loaded(c), reasonCell(c)];
+    return `| ${cells.map(esc).join(" | ")} |`;
+}
+/** ⚠️ when every non-ok run errored (the model never answered), ❌ otherwise. */
+function failMark(c) {
+    const bad = c.runs.filter((r) => !r.ok);
+    return bad.length > 0 && bad.every((r) => r.error !== null) ? "⚠️" : "❌";
+}
+/** The first non-ok run's reason, the pass share prefixed over several runs,
+ * a diagnosis appended in italics. */
+function reasonCell(c) {
+    const bad = c.runs.filter((r) => !r.ok);
+    const parts = c.runs.length > 1 ? [`${c.passed}/${c.runs.length}`, bad[0]?.reason ?? ""] : [bad[0]?.reason ?? ""];
+    let text = parts.join(" · ");
+    const diagnosis = bad.find((r) => r.diagnosis !== null)?.diagnosis;
+    if (diagnosis)
+        text += ` — _${diagnosis}_`;
+    return text;
+}
+function passedRow(c) {
+    const score = c.runs.length > 1 ? ` (${c.passed}/${c.runs.length})` : "";
+    const cells = [`${caseLabel(c)}${score}`, oneLine(c.query), loaded(c)];
+    return `| ${cells.map(esc).join(" | ")} |`;
+}
+function caseLabel(c) {
+    return `#${c.id ?? c.index}`;
+}
+/** Unique loaded skills across runs, "—" when none. */
+function loaded(c) {
+    const names = [...new Set(c.runs.flatMap((r) => r.loaded))];
+    return names.length > 0 ? names.join(", ") : "—";
+}
+/** Collapse to one line, truncated to 80 chars with "…". */
+function oneLine(text) {
+    const flat = text.replace(/\s+/g, " ").trim();
+    return flat.length > 80 ? `${flat.slice(0, 80)}…` : flat;
+}
+/** Pipes so they cannot break the table, newlines as spaces, HTML inert. */
+function esc(text) {
+    return text
+        .replace(/\|/g, "\\|")
+        .replace(/[\r\n]+/g, " ")
+        .replace(/</g, "&lt;");
+}

package/dist/results.js

@@ -10,6 +10,7 @@ export function buildReport(input) {
         file: input.file,
         agent: input.agent,
         model: input.model,
+        batch: input.batch ?? false,
         startedAt: new Date(input.startedAtMs).toISOString(),
         durationMs: input.durationMs,
         summary: {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@icntswm/skillcheck",
-  "version": "1.0.3",
+  "version": "1.1.0",
   "description": "Regression tests for Claude Code skills: check that every request still loads the right skill",
   "type": "module",
   "bin": {