@icntswm/skillcheck 0.1.0 → 1.0.1

package/README.md

@@ -3,7 +3,6 @@
 [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
 [![npm](https://img.shields.io/npm/v/@icntswm/skillcheck.svg)](https://www.npmjs.com/package/@icntswm/skillcheck)
 ![Node 20+](https://img.shields.io/badge/node-20%2B-339933.svg)
-![Claude Code](https://img.shields.io/badge/works%20with-Claude%20Code-d97757.svg)
 
 **Regression tests for Claude Code skills.**
 
@@ -12,22 +11,12 @@ some of your requests load the wrong skill. Nothing warns you: the agent still
 answers, just with the wrong instructions. skillcheck catches this before your
 users do.
 
-```
-$ skillcheck run examples/demo/skillcheck.yaml --skill test-guard,find-bug
-5 cases × 1 repeat × 1 agent = 5 runs
-ok    #bug-500  POST /orders returns 500 with 'nil pointer deref…  → find-bug
-FAIL  #flaky-ci-only  this test is red in CI roughly once a day but I …  → find-bug · not loaded test-guard
-FAIL  #flaky-retry  TestCartMerge fails sometimes and goes green w…  → find-bug · not loaded test-guard; forbidden find-bug
-ok    #bug-consistent  TestCheckoutTotal fails on every run since this …  → find-bug
-ok    #perf-latency  the /search endpoint went from 80ms to 900ms aft…  → perf-profile
-
-confusion:
-  expected test-guard → got find-bug (2)
-2 failed of 5 · runs 5 · cost $0.25
-```
+![skillcheck catching a routing regression](docs/assets/demo.gif)
 
-This is a real run from the [demo](examples/demo): one skill's description got
-wider, and it started taking requests that belong to its neighbour.
+A real run on the [demo](examples/demo): one skill's description got wider,
+and it started taking requests that belong to its neighbour. The free `lint`
+flags a suspicious description, and one batch call finds the two misrouted
+requests.
 
 ## What you get
 
@@ -46,8 +35,9 @@ wider, and it started taking requests that belong to its neighbour.
   from a broken one instead of letting you guess.
 - 🏷️ **Catches renames and typos.** Case names are checked against the skills
   the agent really has, so a renamed skill can't pass silently.
-- ⚙️ **CI-ready.** JUnit and JSON reports, clear exit codes, a spending cap
-  (`--budget`), and `--config-dir` to test only the skills in your repository.
+- ⚙️ **CI-ready.** A GitHub Action (`uses: icntswm/skillcheck@v1`), JUnit
+  and JSON reports, clear exit codes, a spending cap (`--budget`), and
+  `--config-dir` to test only the skills in your repository.
 - 📄 **Plain YAML, one dependency.** Cases are readable by anyone on the team
   and live next to the skills they test.
 
@@ -86,7 +76,7 @@ Start at the cheapest level and go up only when it finds nothing.
 | `skillcheck run --batch` | one per 25 cases | which skill the model *says* it would load |
 | `skillcheck run` | one per case | which skill the model *actually* loads |
 
-On the demo suite one batch call covered all 12 cases for $0.05 and agreed
+On the demo suite one batch call covered all 12 cases for $0.05–0.10 and agreed
 with the normal run in 34 checks out of 34. More in [docs/cost.md](docs/cost.md).
 
 ## Does it really catch regressions?
@@ -144,6 +134,11 @@ stops Claude Code before the first model call, so it costs nothing.
 Works with Claude Code. Agents sit behind a small adapter interface, so others
 that support skills can be added.
 
+Requests can be in any language: `run` asks the model itself. `lint` compares
+words, so it is tuned for English and Russian, works roughly for other
+languages that put spaces between words, and is of little use for Chinese,
+Japanese or Korean.
+
 ## License
 
 [MIT](LICENSE)

package/dist/agents/claude.js

@@ -1,4 +1,5 @@
 import { spawn } from "node:child_process";
+import { rmSync } from "node:fs";
 import { mkdtemp, rm } from "node:fs/promises";
 import * as os from "node:os";
 import * as path from "node:path";
@@ -41,6 +42,10 @@ export class ClaudeStream {
     get sawInit() {
         return this.sawInitEvent;
     }
+    /** True after the result event: the run is over, nothing more will come. */
+    get finished() {
+        return this.sawResult;
+    }
     /** Feed one stdout line; true means the run can be stopped early. */
     push(line) {
         if (!line.startsWith("{"))
@@ -195,6 +200,25 @@ function spawnErrorMessage(e, bin) {
         return `${bin} not found`;
     return `spawn failed: ${err.message}`;
 }
+/** Live runs by process group id, with their tmp dirs; see abortActiveRuns. */
+const active = new Map();
+/**
+ * Kill every running claude group and remove its tmp dir. Runs are detached
+ * process groups, so Ctrl+C on skillcheck does not reach them: without this
+ * they keep running (and billing) after skillcheck exits.
+ */
+export function abortActiveRuns() {
+    for (const [pid, workdir] of active) {
+        try {
+            process.kill(-pid, "SIGTERM");
+        }
+        catch {
+            // already exited
+        }
+        rmSync(workdir, { recursive: true, force: true });
+    }
+    active.clear();
+}
 const LOGIN_HINT = " — with --config-dir, auth comes from ANTHROPIC_API_KEY or CLAUDE_CODE_OAUTH_TOKEN (claude setup-token)";
 /** A fresh config dir has no credentials; point the user at the env vars. */
 function withLoginHint(error, configDir) {
@@ -286,11 +310,13 @@ export class ClaudeAdapter {
             let stderrText = "";
             let error = null;
             let stoppedEarly = false;
+            let stopping = false;
             let settled = false;
             let killTimer;
             let timeoutTimer;
             // detached => child.pid is the process group id; ESRCH means it is gone
             const killGroup = (pid) => {
+                clearTimeout(timeoutTimer); // the run is ending anyway, a late timeout must not overwrite its outcome
                 try {
                     process.kill(-pid, "SIGTERM");
                 }
@@ -311,6 +337,8 @@ export class ClaudeAdapter {
                 if (settled)
                     return;
                 settled = true;
+                if (child?.pid)
+                    active.delete(child.pid);
                 if (killTimer)
                     clearTimeout(killTimer);
                 if (timeoutTimer)
@@ -334,6 +362,8 @@ export class ClaudeAdapter {
                 return;
             }
             const proc = child;
+            if (proc.pid)
+                active.set(proc.pid, workdir);
             proc.on("error", (e) => {
                 if (!error)
                     error = spawnErrorMessage(e, bin);
@@ -342,11 +372,9 @@ export class ClaudeAdapter {
             proc.stdout?.on("data", (chunk) => {
                 if (!stdoutSeen && chunk.trim() !== "")
                     stdoutSeen = true;
-                tail += chunk;
-                let nl;
-                while ((nl = tail.indexOf("\n")) >= 0) {
-                    const line = tail.slice(0, nl);
-                    tail = tail.slice(nl + 1);
+                const lines = (tail + chunk).split("\n");
+                tail = lines.pop(); // the unfinished last line waits for the next chunk
+                for (const line of lines) {
                     const stop = stream.push(line); // always parse: stop and abandon only gate the kill
                     if (error)
                         continue; // first failure wins, the group is already being killed
@@ -356,8 +384,10 @@ export class ClaudeAdapter {
                         if (proc.pid)
                             killGroup(proc.pid);
                     }
-                    else if (!stoppedEarly && ((control.earlyStop && stop) || control.stop?.(stream) === true)) {
-                        stoppedEarly = true;
+                    else if (!stopping && ((control.earlyStop && stop) || control.stop?.(stream) === true)) {
+                        stopping = true;
+                        // after the result event the run is complete: the kill only reaps the process
+                        stoppedEarly = !stream.finished;
                         if (proc.pid)
                             killGroup(proc.pid);
                     }

package/dist/cli.js

@@ -4,7 +4,7 @@ import { readFile } from "node:fs/promises";
 import * as path from "node:path";
 import { pathToFileURL } from "node:url";
 import { parseArgs } from "node:util";
-import { ClaudeAdapter, DEFAULT_DIRECTIVE } from "./agents/claude.js";
+import { abortActiveRuns, ClaudeAdapter, DEFAULT_DIRECTIVE } from "./agents/claude.js";
 import { BATCH_SCHEMA, buildBatchPrompt, parseBatchAnswer } from "./batch.js";
 import { Budget } from "./budget.js";
 import { BUILTIN_SKILLS, ConfigError, findDefaultCasesFile, knownSkillNames, loadSuite, unknownNames, } from "./cases.js";
@@ -667,6 +667,13 @@ function invokedDirectly() {
     }
 }
 if (invokedDirectly()) {
+    // agent runs are detached process groups that Ctrl+C does not reach
+    for (const [signal, code] of [["SIGINT", 130], ["SIGTERM", 143]]) {
+        process.once(signal, () => {
+            abortActiveRuns();
+            process.exit(code);
+        });
+    }
     main(process.argv.slice(2)).then((code) => {
         process.exitCode = code;
     });

package/dist/describe.js

@@ -194,7 +194,8 @@ function pluginDocs(configRoot, cwd) {
             const installPath = entry.installPath;
             if (typeof installPath !== "string" || installPath === "")
                 continue;
-            out.push(...pluginSkillDocs(plugin, installPath), ...pluginCommandDocs(plugin, installPath));
+            const manifest = pluginManifest(installPath);
+            out.push(...pluginSkillDocs(plugin, installPath, manifest), ...pluginCommandDocs(plugin, installPath, manifest));
         }
     }
     return out;
@@ -210,8 +211,8 @@ function disabledPlugins(configRoot) {
     }
     return out;
 }
-function pluginSkillDocs(plugin, installPath) {
-    const declared = pluginManifest(installPath)?.skills;
+function pluginSkillDocs(plugin, installPath, manifest) {
+    const declared = manifest?.skills;
     const dirs = Array.isArray(declared)
         ? declared.filter((s) => typeof s === "string").map((rel) => path.resolve(installPath, rel))
         : skillDirNames(path.join(installPath, "skills")).map((name) => path.join(installPath, "skills", name));
@@ -226,8 +227,8 @@ function pluginSkillDocs(plugin, installPath) {
     }
     return out;
 }
-function pluginCommandDocs(plugin, installPath) {
-    const declared = pluginManifest(installPath)?.commands;
+function pluginCommandDocs(plugin, installPath, manifest) {
+    const declared = manifest?.commands;
     let files;
     if (Array.isArray(declared)) {
         files = [];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@icntswm/skillcheck",
-  "version": "0.1.0",
+  "version": "1.0.1",
   "description": "Regression tests for Claude Code skills: check that every request still loads the right skill",
   "type": "module",
   "bin": {
@@ -32,7 +32,6 @@
   },
   "devDependencies": {
     "@types/node": "^26.6.3",
-    "tsx": "^4.23.15",
     "typescript": "^7.0.2",
     "vitest": "^5.0.2"
   },