@icntswm/skillcheck 0.1.0

package/dist/report.js

@@ -0,0 +1,93 @@
+import { DIAGNOSIS_LIMIT } from "./judge.js";
+const RESET = "\u001b[0m";
+const MARK_WIDTH = 6; // "ok" / "FAIL" padded, keeps the columns aligned
+const QUERY_WIDTH = 48;
+/** One line per finished case plus a summary; plain text when not a TTY. */
+export class Reporter {
+    out;
+    useColor;
+    constructor(out) {
+        this.out = out;
+        this.useColor = out.isTTY === true && !process.env.NO_COLOR;
+    }
+    header(nCases, repeat, nAgents, totalRuns) {
+        const runs = totalRuns ?? nCases * repeat * nAgents;
+        const agents = nAgents === 1 ? "agent" : "agents";
+        this.write(`${nCases} cases × ${repeat} repeat × ${nAgents} ${agents} = ${runs} runs\n`);
+    }
+    batchHeader(nCases, repeat, calls) {
+        this.write(`${nCases} cases × ${repeat} repeat, batch mode = ${calls} call${calls === 1 ? "" : "s"}\n`);
+    }
+    /** A dim advisory line under the summary. */
+    note(text) {
+        this.write(this.paint("2", text) + "\n");
+    }
+    caseDone(res) {
+        const label = `#${res.case.id ?? res.case.index}`;
+        const mark = res.ok ? this.paint("32", "ok".padEnd(MARK_WIDTH)) : this.paint("31", "FAIL".padEnd(MARK_WIDTH));
+        const runs = res.runs.length;
+        const score = runs > 1 ? ` ${res.passed}/${runs}` : "";
+        const loaded = [...new Set(res.runs.flatMap((r) => r.loaded))];
+        const failed = res.runs.find((r) => !r.ok);
+        const reason = !res.ok && failed ? ` · ${failed.reason}` : "";
+        const arrow = this.paint("2", "→");
+        this.write(`${mark}${label}  ${oneLine(res.case.query)}${score}  ${arrow} ${loaded.length > 0 ? loaded.join(", ") : this.paint("2", "—")}${reason}\n`);
+        const diagnosed = res.runs.find((r) => r.diagnosis);
+        if (diagnosed?.diagnosis) {
+            this.write(`      ${this.paint("33", `diagnosis ${label}: ${diagnosed.diagnosis}`)}\n`);
+        }
+    }
+    /** A case the budget never let start; printed after all real runs settle. */
+    caseSkipped(c) {
+        const label = `#${c.id ?? c.index}`;
+        const mark = this.paint("33", "skip".padEnd(MARK_WIDTH));
+        const arrow = this.paint("2", "→");
+        this.write(`${mark}${label}  ${oneLine(c.query)}  ${arrow} budget reached\n`);
+    }
+    summary(results, extra) {
+        const verdicts = results.flatMap((r) => r.runs);
+        this.write("\n");
+        const diagnoses = verdicts.filter((v) => v.diagnosis).length;
+        if (diagnoses >= 2)
+            this.write(this.paint("33", `warning: ${diagnoses} diagnoses. ${DIAGNOSIS_LIMIT}`) + "\n");
+        const unavailable = extra?.unavailable ?? [];
+        if (unavailable.length > 0) {
+            this.write(this.paint("33", `warning: expected skills not available to the agent: ${unavailable.join(", ")}`) + "\n");
+        }
+        const pairs = extra?.confusion ?? [];
+        if (pairs.length > 0) {
+            this.write("confusion:\n");
+            for (const p of pairs)
+                this.write(`  expected ${p.expected} → got ${p.got} (${p.count})\n`);
+        }
+        if (extra?.budget) {
+            const b = extra.budget;
+            const text = `budget $${b.limitUsd.toFixed(2)} reached (spent ~$${b.spent.toFixed(2)}), ${b.notStartedRuns} runs not started`;
+            this.write(this.paint("33", text) + "\n");
+        }
+        const failed = results.filter((r) => !r.ok).length;
+        const skipped = extra?.skipped ?? 0;
+        const skippedPart = skipped > 0 ? `, ${skipped} skipped` : "";
+        this.write(`${failed} failed${skippedPart} of ${results.length + skipped} · runs ${verdicts.length} · ${costLine(verdicts.map((v) => v.costUsd))}\n`);
+    }
+    paint(code, text) {
+        return this.useColor ? `\u001b[${code}m${text}${RESET}` : text;
+    }
+    write(text) {
+        this.out.write(text);
+    }
+}
+function costLine(costs) {
+    const known = costs.filter((c) => c !== null);
+    if (known.length === 0)
+        return "cost ?";
+    const sum = known.reduce((a, b) => a + b, 0);
+    const unknown = costs.length - known.length;
+    const tail = unknown > 0 ? ` (+? for ${unknown} runs)` : "";
+    return `cost $${sum.toFixed(2)}${tail}`;
+}
+/** Collapse to one line and truncate; long queries must not wrap the table. */
+export function oneLine(text) {
+    const flat = text.replace(/\s+/g, " ").trim();
+    return flat.length > QUERY_WIDTH ? `${flat.slice(0, QUERY_WIDTH)}…` : flat;
+}

package/dist/results.js

@@ -0,0 +1,48 @@
+import { readVersion } from "./version.js";
+/** The single source of truth for --json and --junit; computed once per run. */
+export function buildReport(input) {
+    const verdicts = input.cases.flatMap((e) => e.result?.runs ?? []);
+    const costs = verdicts.map((v) => v.costUsd);
+    const known = costs.filter((c) => c !== null);
+    return {
+        tool: "skillcheck",
+        version: readVersion(),
+        file: input.file,
+        agent: input.agent,
+        model: input.model,
+        startedAt: new Date(input.startedAtMs).toISOString(),
+        durationMs: input.durationMs,
+        summary: {
+            cases: input.cases.length,
+            failed: input.cases.filter((e) => e.result && !e.result.ok).length,
+            skipped: input.cases.filter((e) => !e.result).length,
+            runs: verdicts.length,
+            costUsd: known.reduce((a, b) => a + b, 0),
+            unknownCostRuns: costs.length - known.length,
+            diagnoses: verdicts.filter((v) => v.diagnosis).length,
+            budgetUsd: input.budgetUsd,
+            budgetReached: input.budgetReached,
+        },
+        unavailable: input.unavailable,
+        confusion: input.confusion,
+        cases: input.cases.map(toCaseReport),
+    };
+}
+function toCaseReport(e) {
+    const c = e.c;
+    return {
+        index: c.index,
+        id: c.id ?? null,
+        query: c.query,
+        note: c.note ?? null,
+        expect: c.expect,
+        expect_any: c.expect_any,
+        forbid: c.forbid,
+        first: c.first ?? null,
+        none: c.none,
+        status: e.result ? (e.result.ok ? "passed" : "failed") : "skipped",
+        passed: e.result?.passed ?? 0,
+        threshold: e.result?.threshold ?? e.threshold,
+        runs: e.result?.runs ?? [],
+    };
+}

package/dist/version.js

@@ -0,0 +1,7 @@
+import * as fs from "node:fs";
+/** Package version, read from the package.json next to dist/ or src/. */
+export function readVersion() {
+    const url = new URL("../package.json", import.meta.url);
+    const pkg = JSON.parse(fs.readFileSync(url, "utf8"));
+    return pkg.version ?? "0.0.0";
+}

package/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "@icntswm/skillcheck",
+  "version": "0.1.0",
+  "description": "Regression tests for Claude Code skills: check that every request still loads the right skill",
+  "type": "module",
+  "bin": {
+    "skillcheck": "dist/cli.js"
+  },
+  "files": [
+    "dist"
+  ],
+  "engines": {
+    "node": ">=20"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json",
+    "lint": "tsc --noEmit",
+    "test": "vitest run",
+    "prepublishOnly": "npm run lint && npm test && npm run build"
+  },
+  "keywords": [
+    "claude-code",
+    "skills",
+    "agents",
+    "routing",
+    "eval",
+    "regression-testing"
+  ],
+  "license": "MIT",
+  "dependencies": {
+    "yaml": "^2.9.1"
+  },
+  "devDependencies": {
+    "@types/node": "^26.6.3",
+    "tsx": "^4.23.15",
+    "typescript": "^7.0.2",
+    "vitest": "^5.0.2"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/icntswm/skillcheck.git"
+  },
+  "homepage": "https://github.com/icntswm/skillcheck#readme",
+  "publishConfig": {
+    "access": "public"
+  }
+}