claude-slim 2.9.0 → 2.9.1

package/README.md

@@ -245,17 +245,14 @@ Token counts come from [js-tiktoken](https://github.com/nicolo-ribaudo/js-tiktok
 
 ---
 
-## v2.8.0 — What's new
+## v2.9.1 — What's new
 
-Accuracy release. Three reported numbers were wrong; the largest was wrong by an order of magnitude. **If your startup estimate drops sharply after upgrading, the old number was the inaccurate one.**
+Found while stress-testing the scanner against deliberately hostile `~/.claude` fixtures.
 
-- **Startup estimate no longer sums memory across every project on disk.** Claude Code loads `~/.claude/projects/<slug>/memory/` for the project you're in — not the other 40 project directories in your `~/.claude`. The old total scaled with how many projects you'd ever opened: on the dev machine it reported **116,259 tokens where the real per-session cost was 14,399**. Now scoped to the current project, with the cross-project total still shown and labelled as not a per-session cost.
-- **Skill listing cost is measured, not assumed.** Each skill adds a `- <name>: <description>` line to the system prompt. The flat 30-tokens-per-skill estimate stood in for all of them; measured across 68 installed skills the real spread is **30 → 509 tokens (mean 51)**. The per-plugin cost gradient can now tell five terse skills apart from five verbose ones.
-- **`~/.claude/agents/` and `~/.claude/commands/` are now scanned.** Previously invisible despite loading into every session — 12 agents worth ~2,254 tokens on the dev machine. Reported only; never moved or deleted, because there's no restore path for them yet.
-- **Fixed: plugin manifests were stuck at 2.7.0 for three releases**, so `claude plugin install` advertised a stale version. CI now fails on version drift.
-- **Fixed: the token cache grew without bound** — 355 of 776 entries (46%) pointed at deleted files. `flushCache()` now prunes them.
+- **Fixed: `scan` could hang forever on a single long line.** js-tiktoken's BPE is quadratic in the length of one whitespace-free run. Normal prose is fine (8,000 characters encodes in ~1ms), but 800 characters of Hangul cost ~450ms, 3,200 cost ~6.8s, and a 60,000-character run wedged `scan` past 20 seconds with **no output at all** — indistinguishable from a freeze. `SKILL.md` files hit this via base64 blobs, minified snippets, embedded JSON schemas, and CJK text. Long runs are now estimated by encoding a bounded prefix and scaling; **files without one produce byte-identical counts** (verified across 71 installed skills: 70 exact, one moved 0.01%). Hostile fixture: 20s+ → 1.78s, with no measurable cost on ordinary input.
+- **Fixed: a mistyped subcommand blamed the wrong thing.** `claude-slim scam` printed `error: too many arguments. Expected 0 arguments but got 1`. It now names the unknown command and lists the real ones.
 
-Tests: 206 → 241 (+35).
+Tests: 266 → 279 (+13).
 
 For older release notes, see [CHANGELOG.md](CHANGELOG.md).
 

package/dist/cli.js

@@ -244,7 +244,21 @@ program
     await flushCache();
 });
 // --- default (no subcommand) → run clean ---
+// Because the program itself carries an action, commander routes a mistyped
+// subcommand here and reports its own "too many arguments. Expected 0 arguments
+// but got 1" — which says nothing about what the user actually got wrong.
+// Accept the excess argument so we can name it instead.
+program.allowExcessArguments(true);
 program.action(async () => {
+    const stray = program.args;
+    if (stray.length > 0) {
+        const names = program.commands.map((c) => c.name()).join(', ');
+        console.error(`error: unknown command '${stray[0]}'`);
+        console.error(`available commands: ${names}`);
+        console.error(`run 'claude-slim --help' for usage`);
+        process.exitCode = 1;
+        return;
+    }
     await runCleanPipeline({ dryRun: false, auto: false, sessionsPerDay: 2, lookbackDays: 60 });
 });
 // --- shared clean pipeline ---

package/dist/tokenizer.js

@@ -39,11 +39,72 @@ export async function initTokenizer() {
 function hashContent(content) {
     return createHash('md5').update(content).digest('hex');
 }
+// js-tiktoken's BPE is quadratic in the length of a single whitespace-free run.
+// Ordinary prose of any length is fine — the pre-tokenizer splits on whitespace,
+// so 8,000 characters of normal text encodes in ~1ms. One long unbroken run does
+// not: measured on cl100k_base, 800 characters of Hangul costs ~450ms, 3,200
+// costs ~6.8s, and a 60,000-character run wedges the scan for minutes with no
+// output at all. Real SKILL.md files reach this with base64 blobs, minified
+// snippets, rule separators, and CJK text.
+//
+// 512 sits above anything a normal word, path, or URL produces and well below
+// where the curve turns painful.
+const MAX_ENCODE_RUN = 512;
+const LONG_RUN_PATTERN = /\S{513,}/;
+const FALLBACK_CHARS_PER_TOKEN = 4;
+/**
+ * Estimate an over-long run by encoding a bounded prefix and scaling.
+ *
+ * A fixed characters-per-token divisor cannot work here: measured over 1,000-char
+ * runs, cl100k_base yields 0.8 chars/token for Hangul but 8.0 for a repeated
+ * ASCII character — a 10× spread. Sampling the run's own prefix adapts to
+ * whatever it actually contains (base64, hex, minified JSON, CJK) at the cost of
+ * exactly one bounded encode.
+ */
+function estimateRun(segment, encode) {
+    const sample = segment.slice(0, MAX_ENCODE_RUN);
+    const sampleTokens = encode(sample).length;
+    if (sampleTokens === 0)
+        return 0;
+    return Math.ceil((sampleTokens * segment.length) / sample.length);
+}
+/**
+ * Encode `text`, estimating any whitespace-free run longer than
+ * {@link MAX_ENCODE_RUN} instead of feeding the whole run to the BPE.
+ *
+ * Text without such a run takes the fast path and is encoded whole, so counts
+ * for well-formed files are identical to encoding directly.
+ */
+function encodeBounded(text, encode) {
+    if (!LONG_RUN_PATTERN.test(text)) {
+        return encode(text).length;
+    }
+    let total = 0;
+    let buffered = '';
+    // Splitting on a captured group keeps the whitespace in the stream, so the
+    // buffered pieces still look to the encoder like the original text.
+    for (const segment of text.split(/(\s+)/)) {
+        if (segment.length > MAX_ENCODE_RUN) {
+            if (buffered) {
+                total += encode(buffered).length;
+                buffered = '';
+            }
+            total += estimateRun(segment, encode);
+            continue;
+        }
+        buffered += segment;
+    }
+    if (buffered) {
+        total += encode(buffered).length;
+    }
+    return total;
+}
 export function countTokens(text) {
     if (useFallback || !encoder) {
-        return Math.ceil(text.length / 4);
+        return Math.ceil(text.length / FALLBACK_CHARS_PER_TOKEN);
     }
-    return encoder.encode(text).length;
+    const enc = encoder;
+    return encodeBounded(text, (s) => enc.encode(s));
 }
 export function countTokensCached(text, filePath) {
     const hash = hashContent(text);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-slim",
-  "version": "2.9.0",
+  "version": "2.9.1",
   "description": "Audit and shrink your Claude Code startup context. Measures what every skill, plugin, agent, command, and memory file costs in the system prompt, then reversibly disables the dead weight. Non-destructive scan, tiered proposals, one-command restore — no proxy, no compression.",
   "type": "module",
   "bin": {