@icntswm/skillcheck 1.0.2 → 1.0.4

package/README.md

@@ -1,5 +1,6 @@
 # skillcheck
 
+[![CI](https://github.com/icntswm/skillcheck/actions/workflows/ci.yml/badge.svg)](https://github.com/icntswm/skillcheck/actions/workflows/ci.yml)
 [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
 [![npm](https://img.shields.io/npm/v/@icntswm/skillcheck.svg)](https://www.npmjs.com/package/@icntswm/skillcheck)
 ![Node 20+](https://img.shields.io/badge/node-20%2B-339933.svg)

package/dist/agents/claude.js

@@ -12,6 +12,23 @@ export const DEFAULT_DIRECTIVE =
 function stringList(v) {
     return Array.isArray(v) ? v.filter((s) => typeof s === "string") : null;
 }
+function count(v) {
+    return typeof v === "number" && Number.isFinite(v) && v > 0 ? v : 0;
+}
+/** Sum API `usage` objects; cache writes without a 1h/5m split count as 5m. */
+function sumUsage(model, usages) {
+    const total = { model, input: 0, output: 0, cacheRead: 0, cacheWrite5m: 0, cacheWrite1h: 0 };
+    for (const u of usages) {
+        const split = u.cache_creation;
+        const write1h = count(split?.ephemeral_1h_input_tokens);
+        total.input += count(u.input_tokens);
+        total.output += count(u.output_tokens);
+        total.cacheRead += count(u.cache_read_input_tokens);
+        total.cacheWrite1h += write1h;
+        total.cacheWrite5m += Math.max(0, count(u.cache_creation_input_tokens) - write1h);
+    }
+    return total;
+}
 /**
  * Incremental parser of claude's stream-json stdout. One event per line.
  * Pushing lines also computes the early-stop signal, so the adapter can
@@ -23,6 +40,10 @@ export class ClaudeStream {
     resultText = "";
     sawResult = false;
     costUsd = null;
+    model = null;
+    /** latest usage per API message; a message streams as several events */
+    messageUsage = new Map();
+    resultUsage = null;
     availableSkills = null;
     initSkillList = null;
     initSlashList = null;
@@ -88,12 +109,20 @@ export class ClaudeStream {
             }
             if (typeof ev.total_cost_usd === "number")
                 this.costUsd = ev.total_cost_usd;
+            if (typeof ev.usage === "object" && ev.usage !== null)
+                this.resultUsage = ev.usage;
             return true;
         }
         return false;
     }
     handleAssistant(ev) {
         const message = ev.message;
+        if (typeof message?.model === "string")
+            this.model = message.model;
+        if (typeof message?.usage === "object" && message.usage !== null) {
+            const id = typeof message.id === "string" ? message.id : `#${this.messageUsage.size}`;
+            this.messageUsage.set(id, message.usage);
+        }
         const blocks = message?.content;
         if (!Array.isArray(blocks))
             return false;
@@ -118,6 +147,18 @@ export class ClaudeStream {
             this.sawSkill = true;
         return sawSkillHere;
     }
+    /**
+     * Billed tokens: the result event's total, or the sum over assistant
+     * messages when the run was killed first. Streamed output counts are
+     * partial, so the latter runs a little low.
+     */
+    get usage() {
+        if (this.resultUsage)
+            return sumUsage(this.model, [this.resultUsage]);
+        if (this.messageUsage.size === 0)
+            return null;
+        return sumUsage(this.model, [...this.messageUsage.values()]);
+    }
     /** Error reported by claude itself in the result event, null if none. */
     get error() {
         return this.failure;
@@ -265,7 +306,8 @@ export class ClaudeAdapter {
             });
             const { loaded, text, costUsd, availableSkills } = out.stream.result;
             const error = withLoginHint(out.error, opts.configDir);
-            return { loaded, text, costUsd, availableSkills, error, stoppedEarly: out.stoppedEarly, durationMs: out.durationMs };
+            const usage = out.stream.usage;
+            return { loaded, text, costUsd, usage, availableSkills, error, stoppedEarly: out.stoppedEarly, durationMs: out.durationMs };
         }
         finally {
             await rm(workdir, { recursive: true, force: true });

package/dist/budget.js

@@ -1,27 +1,64 @@
 /**
- * Running spend estimate for --budget. Runs killed by early-stop never report
- * a cost, so unknown costs are estimated at the average known cost (0 until
- * the first known cost arrives); without this the budget could never trigger.
+ * Tokens in input-token equivalents. Anthropic prices every model the same
+ * way relative to its input price: output 5x, cache reads 0.1x, 5-minute
+ * cache writes 1.25x, 1-hour cache writes 2x.
+ */
+export function inputEquivalent(u) {
+    return u.input + 5 * u.output + 0.1 * u.cacheRead + 1.25 * u.cacheWrite5m + 2 * u.cacheWrite1h;
+}
+/**
+ * USD per input token until a finished run shows the real rate. Rounded up
+ * where unsure: the budget is a cap, so overestimating is the safe side.
+ */
+function listInputPrice(model) {
+    if (model?.includes("haiku"))
+        return 1e-6;
+    if (model?.includes("sonnet"))
+        return 3e-6;
+    return 5e-6;
+}
+/**
+ * Running spend estimate for --budget. claude usually reports its cost even
+ * when early-stop kills it, but a run killed before the result event does
+ * not; such runs are priced from their token usage: at the rate seen in
+ * finished runs, or at list price before any run finishes. Runs with neither
+ * cost nor usage count at the average known cost.
  */
 export class Budget {
     limitUsd;
     knownSum = 0;
     knownCount = 0;
     unknownCount = 0;
+    rateCost = 0;
+    rateTokens = 0;
+    unpriced = [];
     constructor(limitUsd) {
         this.limitUsd = limitUsd;
     }
-    add(cost) {
-        if (cost === null)
-            this.unknownCount++;
-        else {
+    add(cost, usage = null) {
+        const tokens = usage ? inputEquivalent(usage) : 0;
+        if (cost !== null) {
             this.knownSum += cost;
             this.knownCount++;
+            if (tokens > 0) {
+                this.rateCost += cost;
+                this.rateTokens += tokens;
+            }
+        }
+        else if (usage && tokens > 0) {
+            this.unpriced.push(usage);
+        }
+        else {
+            this.unknownCount++;
         }
     }
     get spent() {
         const avg = this.knownCount > 0 ? this.knownSum / this.knownCount : 0;
-        return this.knownSum + this.unknownCount * avg;
+        const estimated = this.unpriced.reduce((sum, u) => {
+            const rate = this.rateTokens > 0 ? this.rateCost / this.rateTokens : listInputPrice(u.model);
+            return sum + inputEquivalent(u) * rate;
+        }, 0);
+        return this.knownSum + estimated + this.unknownCount * avg;
     }
     get exceeded() {
         return this.spent >= this.limitUsd;

package/dist/cli.js

@@ -440,7 +440,7 @@ class FatalStop {
             this.message = error;
     }
     canStart(budget) {
-        return this.message === null && (!budget || !budget.exceeded);
+        return this.message === null && !budget.exceeded;
     }
 }
 async function runCommand(file, flags, io, deps) {
@@ -466,7 +466,8 @@ async function runCommand(file, flags, io, deps) {
         const n = c.repeat ?? repeat;
         return { c, jobIndex: i, threshold: c.threshold ?? threshold, runs: Array.from({ length: n }), done: 0 };
     });
-    const budget = flags.budget !== undefined ? new Budget(flags.budget) : null;
+    // without --budget it only estimates the spend for the report
+    const budget = new Budget(flags.budget ?? Infinity);
     // "--json -" owns stdout, so the human report has to go to stderr
     const reporter = new Reporter(flags.json === "-" ? io.stderr : io.stdout);
     const startedAtMs = Date.now();
@@ -492,7 +493,8 @@ async function runCommand(file, flags, io, deps) {
         unavailable,
         confusion: pairs,
         skipped: skipped.length,
-        budget: skipped.length > 0 && budget ? { limitUsd: flags.budget, spent: budget.spent, notStartedRuns } : undefined,
+        budget: skipped.length > 0 ? { limitUsd: flags.budget, spent: budget.spent, notStartedRuns } : undefined,
+        estimatedUsd: budget.spent,
     });
     if (flags.batch) {
         // cases that only errored have no answer to confirm
@@ -511,6 +513,7 @@ async function runCommand(file, flags, io, deps) {
         cases: items.map((item) => ({ c: item.c, threshold: item.threshold, result: outcome.results[item.jobIndex] ?? null })),
         unavailable,
         confusion: pairs,
+        estimatedCostUsd: budget.spent,
         budgetUsd: flags.budget ?? null,
         budgetReached: skipped.length > 0,
     });
@@ -558,7 +561,7 @@ async function runIndividually(adapter, items, flags, model, directive, repeat,
                 available.add(name);
         }
         const verdict = judge(job.item.c, r);
-        budget?.add(verdict.costUsd);
+        budget.add(r.costUsd, r.usage);
         job.item.runs[job.runIndex] = verdict;
         job.item.done++;
         if (job.item.done === job.item.runs.length) {
@@ -606,7 +609,7 @@ async function runBatched(runBatch, items, flags, model, repeat, reporter, budge
                 durationMs: br.durationMs,
             };
             const verdict = judge(item.c, r);
-            budget?.add(verdict.costUsd);
+            budget.add(verdict.costUsd);
             item.runs[job.round] = verdict;
             item.done++;
             if (item.done === item.runs.length) {

package/dist/junit.js

@@ -14,9 +14,10 @@ export function toJunit(report) {
     if (report.model !== null)
         properties.push(`      <property name="model" value="${esc(report.model)}"/>`);
     properties.push(`      <property name="costUsd" value="${report.summary.costUsd.toFixed(2)}"/>`);
-    // early-stopped runs report no cost, so costUsd alone understates the spend
+    // a run killed before its result event reports no cost, so costUsd alone understates the spend
     if (report.summary.unknownCostRuns > 0) {
         properties.push(`      <property name="unknownCostRuns" value="${report.summary.unknownCostRuns}"/>`);
+        properties.push(`      <property name="estimatedCostUsd" value="${report.summary.estimatedCostUsd.toFixed(2)}"/>`);
     }
     properties.push("    </properties>");
     const cases = rows.map(({ c, kind }) => testcase(c, kind, suiteName));

package/dist/report.js

@@ -68,7 +68,7 @@ export class Reporter {
         const failed = results.filter((r) => !r.ok).length;
         const skipped = extra?.skipped ?? 0;
         const skippedPart = skipped > 0 ? `, ${skipped} skipped` : "";
-        this.write(`${failed} failed${skippedPart} of ${results.length + skipped} · runs ${verdicts.length} · ${costLine(verdicts.map((v) => v.costUsd))}\n`);
+        this.write(`${failed} failed${skippedPart} of ${results.length + skipped} · runs ${verdicts.length} · ${costLine(verdicts.map((v) => v.costUsd), extra?.estimatedUsd)}\n`);
     }
     paint(code, text) {
         return this.useColor ? `\u001b[${code}m${text}${RESET}` : text;
@@ -77,12 +77,15 @@ export class Reporter {
         this.out.write(text);
     }
 }
-function costLine(costs) {
+function costLine(costs, estimated) {
     const known = costs.filter((c) => c !== null);
-    if (known.length === 0)
-        return "cost ?";
     const sum = known.reduce((a, b) => a + b, 0);
     const unknown = costs.length - known.length;
+    if (unknown > 0 && estimated !== undefined && estimated > sum) {
+        return `cost ~$${estimated.toFixed(2)} (${unknown} runs estimated from tokens)`;
+    }
+    if (known.length === 0)
+        return "cost ?";
     const tail = unknown > 0 ? ` (+? for ${unknown} runs)` : "";
     return `cost $${sum.toFixed(2)}${tail}`;
 }

package/dist/results.js

@@ -19,6 +19,7 @@ export function buildReport(input) {
             runs: verdicts.length,
             costUsd: known.reduce((a, b) => a + b, 0),
             unknownCostRuns: costs.length - known.length,
+            estimatedCostUsd: input.estimatedCostUsd,
             diagnoses: verdicts.filter((v) => v.diagnosis).length,
             budgetUsd: input.budgetUsd,
             budgetReached: input.budgetReached,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@icntswm/skillcheck",
-  "version": "1.0.2",
+  "version": "1.0.4",
   "description": "Regression tests for Claude Code skills: check that every request still loads the right skill",
   "type": "module",
   "bin": {