@icntswm/skillcheck 1.0.1 → 1.0.3

package/README.md

@@ -1,5 +1,6 @@
 # skillcheck
 
+[![CI](https://github.com/icntswm/skillcheck/actions/workflows/ci.yml/badge.svg)](https://github.com/icntswm/skillcheck/actions/workflows/ci.yml)
 [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
 [![npm](https://img.shields.io/npm/v/@icntswm/skillcheck.svg)](https://www.npmjs.com/package/@icntswm/skillcheck)
 ![Node 20+](https://img.shields.io/badge/node-20%2B-339933.svg)

package/dist/agents/claude.js

@@ -12,6 +12,23 @@ export const DEFAULT_DIRECTIVE =
 function stringList(v) {
     return Array.isArray(v) ? v.filter((s) => typeof s === "string") : null;
 }
+function count(v) {
+    return typeof v === "number" && Number.isFinite(v) && v > 0 ? v : 0;
+}
+/** Sum API `usage` objects; cache writes without a 1h/5m split count as 5m. */
+function sumUsage(model, usages) {
+    const total = { model, input: 0, output: 0, cacheRead: 0, cacheWrite5m: 0, cacheWrite1h: 0 };
+    for (const u of usages) {
+        const split = u.cache_creation;
+        const write1h = count(split?.ephemeral_1h_input_tokens);
+        total.input += count(u.input_tokens);
+        total.output += count(u.output_tokens);
+        total.cacheRead += count(u.cache_read_input_tokens);
+        total.cacheWrite1h += write1h;
+        total.cacheWrite5m += Math.max(0, count(u.cache_creation_input_tokens) - write1h);
+    }
+    return total;
+}
 /**
  * Incremental parser of claude's stream-json stdout. One event per line.
  * Pushing lines also computes the early-stop signal, so the adapter can
@@ -23,6 +40,10 @@ export class ClaudeStream {
     resultText = "";
     sawResult = false;
     costUsd = null;
+    model = null;
+    /** latest usage per API message; a message streams as several events */
+    messageUsage = new Map();
+    resultUsage = null;
     availableSkills = null;
     initSkillList = null;
     initSlashList = null;
@@ -88,12 +109,20 @@ export class ClaudeStream {
             }
             if (typeof ev.total_cost_usd === "number")
                 this.costUsd = ev.total_cost_usd;
+            if (typeof ev.usage === "object" && ev.usage !== null)
+                this.resultUsage = ev.usage;
             return true;
         }
         return false;
     }
     handleAssistant(ev) {
         const message = ev.message;
+        if (typeof message?.model === "string")
+            this.model = message.model;
+        if (typeof message?.usage === "object" && message.usage !== null) {
+            const id = typeof message.id === "string" ? message.id : `#${this.messageUsage.size}`;
+            this.messageUsage.set(id, message.usage);
+        }
         const blocks = message?.content;
         if (!Array.isArray(blocks))
             return false;
@@ -118,6 +147,18 @@ export class ClaudeStream {
             this.sawSkill = true;
         return sawSkillHere;
     }
+    /**
+     * Billed tokens: the result event's total, or the sum over assistant
+     * messages when the run was killed first. Streamed output counts are
+     * partial, so the latter runs a little low.
+     */
+    get usage() {
+        if (this.resultUsage)
+            return sumUsage(this.model, [this.resultUsage]);
+        if (this.messageUsage.size === 0)
+            return null;
+        return sumUsage(this.model, [...this.messageUsage.values()]);
+    }
     /** Error reported by claude itself in the result event, null if none. */
     get error() {
         return this.failure;
@@ -202,22 +243,45 @@ function spawnErrorMessage(e, bin) {
 }
 /** Live runs by process group id, with their tmp dirs; see abortActiveRuns. */
 const active = new Map();
+function groupAlive(pid) {
+    try {
+        process.kill(-pid, 0);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
 /**
  * Kill every running claude group and remove its tmp dir. Runs are detached
  * process groups, so Ctrl+C on skillcheck does not reach them: without this
- * they keep running (and billing) after skillcheck exits.
+ * they keep running (and billing) after skillcheck exits. Groups still alive
+ * after graceMs get SIGKILL, so a process that ignores SIGTERM dies too.
  */
-export function abortActiveRuns() {
-    for (const [pid, workdir] of active) {
+export async function abortActiveRuns(graceMs = KILL_GRACE_MS) {
+    const runs = [...active];
+    active.clear();
+    for (const [pid] of runs) {
         try {
             process.kill(-pid, "SIGTERM");
         }
         catch {
             // already exited
         }
+    }
+    const deadline = Date.now() + graceMs;
+    while (runs.some(([pid]) => groupAlive(pid)) && Date.now() < deadline) {
+        await new Promise((r) => setTimeout(r, 50));
+    }
+    for (const [pid, workdir] of runs) {
+        try {
+            process.kill(-pid, "SIGKILL");
+        }
+        catch {
+            // already exited
+        }
         rmSync(workdir, { recursive: true, force: true });
     }
-    active.clear();
 }
 const LOGIN_HINT = " — with --config-dir, auth comes from ANTHROPIC_API_KEY or CLAUDE_CODE_OAUTH_TOKEN (claude setup-token)";
 /** A fresh config dir has no credentials; point the user at the env vars. */
@@ -242,7 +306,8 @@ export class ClaudeAdapter {
             });
             const { loaded, text, costUsd, availableSkills } = out.stream.result;
             const error = withLoginHint(out.error, opts.configDir);
-            return { loaded, text, costUsd, availableSkills, error, stoppedEarly: out.stoppedEarly, durationMs: out.durationMs };
+            const usage = out.stream.usage;
+            return { loaded, text, costUsd, usage, availableSkills, error, stoppedEarly: out.stoppedEarly, durationMs: out.durationMs };
         }
         finally {
             await rm(workdir, { recursive: true, force: true });

package/dist/budget.js

@@ -1,27 +1,63 @@
+/**
+ * Tokens in input-token equivalents. Anthropic prices every model the same
+ * way relative to its input price: output 5x, cache reads 0.1x, 5-minute
+ * cache writes 1.25x, 1-hour cache writes 2x.
+ */
+export function inputEquivalent(u) {
+    return u.input + 5 * u.output + 0.1 * u.cacheRead + 1.25 * u.cacheWrite5m + 2 * u.cacheWrite1h;
+}
+/**
+ * USD per input token until a finished run shows the real rate. Rounded up
+ * where unsure: the budget is a cap, so overestimating is the safe side.
+ */
+function listInputPrice(model) {
+    if (model?.includes("haiku"))
+        return 1e-6;
+    if (model?.includes("sonnet"))
+        return 3e-6;
+    return 5e-6;
+}
 /**
  * Running spend estimate for --budget. Runs killed by early-stop never report
- * a cost, so unknown costs are estimated at the average known cost (0 until
- * the first known cost arrives); without this the budget could never trigger.
+ * a cost, so they are priced from their token usage: at the rate seen in
+ * finished runs, or at list price before any run finishes. Runs with neither
+ * cost nor usage count at the average known cost.
  */
 export class Budget {
     limitUsd;
     knownSum = 0;
     knownCount = 0;
     unknownCount = 0;
+    rateCost = 0;
+    rateTokens = 0;
+    unpriced = [];
     constructor(limitUsd) {
         this.limitUsd = limitUsd;
     }
-    add(cost) {
-        if (cost === null)
-            this.unknownCount++;
-        else {
+    add(cost, usage = null) {
+        const tokens = usage ? inputEquivalent(usage) : 0;
+        if (cost !== null) {
             this.knownSum += cost;
             this.knownCount++;
+            if (tokens > 0) {
+                this.rateCost += cost;
+                this.rateTokens += tokens;
+            }
+        }
+        else if (usage && tokens > 0) {
+            this.unpriced.push(usage);
+        }
+        else {
+            this.unknownCount++;
         }
     }
     get spent() {
         const avg = this.knownCount > 0 ? this.knownSum / this.knownCount : 0;
-        return this.knownSum + this.unknownCount * avg;
+        const estimated = this.unpriced.reduce((sum, u) => {
+            const rate = this.rateTokens > 0 ? this.rateCost / this.rateTokens : listInputPrice(u.model);
+            return sum + inputEquivalent(u) * rate;
+        }, 0);
+        return this.knownSum + estimated + this.unknownCount * avg;
     }
     get exceeded() {
         return this.spent >= this.limitUsd;

package/dist/cli.js

@@ -440,7 +440,7 @@ class FatalStop {
             this.message = error;
     }
     canStart(budget) {
-        return this.message === null && (!budget || !budget.exceeded);
+        return this.message === null && !budget.exceeded;
     }
 }
 async function runCommand(file, flags, io, deps) {
@@ -466,7 +466,8 @@ async function runCommand(file, flags, io, deps) {
         const n = c.repeat ?? repeat;
         return { c, jobIndex: i, threshold: c.threshold ?? threshold, runs: Array.from({ length: n }), done: 0 };
     });
-    const budget = flags.budget !== undefined ? new Budget(flags.budget) : null;
+    // without --budget it only estimates the spend for the report
+    const budget = new Budget(flags.budget ?? Infinity);
     // "--json -" owns stdout, so the human report has to go to stderr
     const reporter = new Reporter(flags.json === "-" ? io.stderr : io.stdout);
     const startedAtMs = Date.now();
@@ -492,7 +493,8 @@ async function runCommand(file, flags, io, deps) {
         unavailable,
         confusion: pairs,
         skipped: skipped.length,
-        budget: skipped.length > 0 && budget ? { limitUsd: flags.budget, spent: budget.spent, notStartedRuns } : undefined,
+        budget: skipped.length > 0 ? { limitUsd: flags.budget, spent: budget.spent, notStartedRuns } : undefined,
+        estimatedUsd: budget.spent,
     });
     if (flags.batch) {
         // cases that only errored have no answer to confirm
@@ -511,6 +513,7 @@ async function runCommand(file, flags, io, deps) {
         cases: items.map((item) => ({ c: item.c, threshold: item.threshold, result: outcome.results[item.jobIndex] ?? null })),
         unavailable,
         confusion: pairs,
+        estimatedCostUsd: budget.spent,
         budgetUsd: flags.budget ?? null,
         budgetReached: skipped.length > 0,
     });
@@ -558,7 +561,7 @@ async function runIndividually(adapter, items, flags, model, directive, repeat,
                 available.add(name);
         }
         const verdict = judge(job.item.c, r);
-        budget?.add(verdict.costUsd);
+        budget.add(r.costUsd, r.usage);
         job.item.runs[job.runIndex] = verdict;
         job.item.done++;
         if (job.item.done === job.item.runs.length) {
@@ -606,7 +609,7 @@ async function runBatched(runBatch, items, flags, model, repeat, reporter, budge
                 durationMs: br.durationMs,
             };
             const verdict = judge(item.c, r);
-            budget?.add(verdict.costUsd);
+            budget.add(verdict.costUsd);
             item.runs[job.round] = verdict;
             item.done++;
             if (item.done === item.runs.length) {
@@ -670,8 +673,9 @@ if (invokedDirectly()) {
     // agent runs are detached process groups that Ctrl+C does not reach
     for (const [signal, code] of [["SIGINT", 130], ["SIGTERM", 143]]) {
         process.once(signal, () => {
-            abortActiveRuns();
-            process.exit(code);
+            // a second Ctrl+C during the grace period exits at once
+            process.once(signal, () => process.exit(code));
+            void abortActiveRuns().finally(() => process.exit(code));
         });
     }
     main(process.argv.slice(2)).then((code) => {

package/dist/report.js

@@ -68,7 +68,7 @@ export class Reporter {
         const failed = results.filter((r) => !r.ok).length;
         const skipped = extra?.skipped ?? 0;
         const skippedPart = skipped > 0 ? `, ${skipped} skipped` : "";
-        this.write(`${failed} failed${skippedPart} of ${results.length + skipped} · runs ${verdicts.length} · ${costLine(verdicts.map((v) => v.costUsd))}\n`);
+        this.write(`${failed} failed${skippedPart} of ${results.length + skipped} · runs ${verdicts.length} · ${costLine(verdicts.map((v) => v.costUsd), extra?.estimatedUsd)}\n`);
     }
     paint(code, text) {
         return this.useColor ? `\u001b[${code}m${text}${RESET}` : text;
@@ -77,12 +77,15 @@ export class Reporter {
         this.out.write(text);
     }
 }
-function costLine(costs) {
+function costLine(costs, estimated) {
     const known = costs.filter((c) => c !== null);
-    if (known.length === 0)
-        return "cost ?";
     const sum = known.reduce((a, b) => a + b, 0);
     const unknown = costs.length - known.length;
+    if (unknown > 0 && estimated !== undefined && estimated > sum) {
+        return `cost ~$${estimated.toFixed(2)} (${unknown} runs estimated from tokens)`;
+    }
+    if (known.length === 0)
+        return "cost ?";
     const tail = unknown > 0 ? ` (+? for ${unknown} runs)` : "";
     return `cost $${sum.toFixed(2)}${tail}`;
 }

package/dist/results.js

@@ -19,6 +19,7 @@ export function buildReport(input) {
             runs: verdicts.length,
             costUsd: known.reduce((a, b) => a + b, 0),
             unknownCostRuns: costs.length - known.length,
+            estimatedCostUsd: input.estimatedCostUsd,
             diagnoses: verdicts.filter((v) => v.diagnosis).length,
             budgetUsd: input.budgetUsd,
             budgetReached: input.budgetReached,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@icntswm/skillcheck",
-  "version": "1.0.1",
+  "version": "1.0.3",
   "description": "Regression tests for Claude Code skills: check that every request still loads the right skill",
   "type": "module",
   "bin": {