@smartmemory/compose 0.2.22-beta → 0.2.23-beta

package/.claude/skills/context-budget/SKILL.md

@@ -0,0 +1,81 @@
+---
+name: context-budget
+description: Use when the user wants to audit, measure, or trim the session-start context load — token cost of agents, skills, rules, MCP tool schemas, and the CLAUDE.md chain. Triggers on "context budget", "token audit", "what's loading my context", "trim my skills/agents", "how big is my context". Produces a ranked, classified cut list; never auto-applies cuts.
+---
+
+# Context Budget
+
+Audit the loaded surface a session pays for at startup, estimate its token cost, classify
+every component into **always / sometimes / rarely needed**, and produce a ranked cut list
+with estimated reclaim. Read-only — you surface recommendations; the user reviews and chooses.
+
+## When to use
+
+- "Run a context budget" / "audit my context" / "what's eating my context window"
+- After adding skills, agents, rules, or MCP servers — measure the new cost
+- Before trimming `~/.claude/skills/` or a project's `.claude/` surface
+
+## How it works
+
+The scan/classify/report logic lives in a tested module: `compose/lib/context-budget.js`
+(pure ESM, covered by `compose/test/context-budget.test.js`). This skill is a thin wrapper —
+run the module's CLI, then interpret the report for the user.
+
+### Step 1 — Gather live MCP tool counts
+
+`.mcp.json` lists servers but **not** how many tools each exposes. The session's own
+tool-list (the deferred-tools / connected-MCP reminders you can see) is the source of truth.
+Count the tools per server you actually have loaded, e.g. `compose=46,stratum=44`. Servers you
+don't pass a count for are still listed but flagged `tool-count-unknown` and excluded from the
+numeric total (the tool never fabricates a number).
+
+### Step 2 — Run the audit
+
+```bash
+node <compose-root>/lib/context-budget.js <project-root> \
+  --tool-counts=compose=46,stratum=44,playwright=25,filesystem=14,memory=9
+```
+
+- `<project-root>` defaults to `cwd` if omitted.
+- The CLI walks: `~/.claude/{agents,skills,rules,CLAUDE.md}` + the project's
+  `.claude/{agents,skills,rules}`, the `.mcp.json` servers, and the CLAUDE.md chain
+  (home → every `CLAUDE.md` from cwd up to the repo root).
+- Token estimate is a dependency-free ~4-chars-per-token heuristic — **relative budgeting,
+  not billing-accurate**. Use it to rank, not to bill.
+
+### Step 3 — Interpret the report
+
+The report prints three buckets and a TOP 5 RECLAIMS list. Walk the user through:
+
+- **ALWAYS** — referenced by name in the CLAUDE.md chain, or the CLAUDE.md chain itself. Keep.
+- **SOMETIMES** — domain-specific and not referenced in CLAUDE.md. Includes active MCP servers
+  the chain doesn't mention: they're load-bearing *while configured*, but if this project doesn't
+  use a server, disabling it in `.mcp.json` is often the single biggest reclaim (schemas are tens
+  of K tokens). Candidates for on-demand activation / disable-if-unused rather than always-loaded.
+- **RARELY** — duplicates across surfaces, overlapping rules, simple-CLI-wrapping MCP servers.
+  Recommend cutting.
+
+Flags worth calling out explicitly:
+- `duplicate` — same SKILL.md present in both `compose/.claude/skills/` and `~/.claude/skills/`
+  (the common real source of churn). Counted once; the copy is the cut.
+- `wraps-simple-cli` — an MCP server whose command is `git`/`gh`/`npm`/etc.; the Bash tool can
+  do that work without the schema overhead.
+- `over-N-lines` — an oversized agent (>200), skill (>400), or rule (>100) worth splitting.
+
+## Heuristics are guides, not rules
+
+Always surface the **reason** alongside each recommendation so the user can override. The buckets
+force a lazy-load decision per component; they don't make it for you.
+
+## Non-goals
+
+- **Never auto-apply cuts.** Present the list; the user decides and acts.
+- Not a replacement for manual review.
+- No cross-session token tracking (separate work).
+
+## Reference
+
+- Logic + contract: `compose/lib/context-budget.js`
+- Tests: `compose/test/context-budget.test.js`
+- Feature: `docs/features/COMP-CTXBUDGET-1/`
+- Source: ECC `affaan-m/everything-claude-code/skills/context-budget/SKILL.md`

package/lib/context-budget.js

@@ -0,0 +1,459 @@
+/**
+ * context-budget.js — Audit the session-start loaded surface and budget its token cost.
+ *
+ * Scans agents, skills, rules, MCP server tool schemas, and the CLAUDE.md chain;
+ * estimates per-component tokens; classifies each into always / sometimes / rarely
+ * needed; and renders a ranked cut list with estimated reclaim.
+ *
+ * Read-only. The human reviews and chooses cuts (auto-applying cuts is a non-goal).
+ * Backs the `/context-budget` skill — see compose/.claude/skills/context-budget/SKILL.md.
+ *
+ * Token estimates use a dependency-free ~4-chars-per-token heuristic. They are relative
+ * budgeting estimates, NOT billing-accurate; `estimateTokens` is pluggable so a real
+ * tokenizer can be swapped in later. COMP-CTXBUDGET-1.
+ */
+
+import { readFileSync, existsSync, readdirSync, statSync } from 'fs';
+import { join, basename, dirname } from 'path';
+import { createHash } from 'crypto';
+
+const TOKENS_PER_MCP_TOOL = 500;
+const SIMPLE_CLI_COMMANDS = new Set(['git', 'gh', 'npm', 'npx', 'cat', 'ls', 'grep', 'sed', 'awk']);
+
+// Flag thresholds (lines) from the plan.
+const FLAG_LINES = { agent: 200, skill: 400, rule: 100 };
+
+/**
+ * Estimate tokens for a chunk of text. Dependency-free ~4-chars-per-token heuristic.
+ * @param {string} text
+ * @returns {number}
+ */
+export function estimateTokens(text) {
+  if (!text) return 0;
+  return Math.ceil(text.length / 4);
+}
+
+function readTextSafe(path) {
+  try {
+    return readFileSync(path, 'utf-8');
+  } catch {
+    return null;
+  }
+}
+
+function lineCount(text) {
+  if (!text) return 0;
+  // Count lines without a trailing-newline off-by-one surprise.
+  const n = text.split('\n').length;
+  return text.endsWith('\n') ? n - 1 : n;
+}
+
+function contentHash(text) {
+  return createHash('sha1').update(text || '').digest('hex');
+}
+
+function makeComponent(kind, path, label, text, extraFlags = []) {
+  const lines = lineCount(text);
+  const flags = [...extraFlags];
+  const threshold = FLAG_LINES[kind];
+  if (threshold && lines > threshold) flags.push(`over-${threshold}-lines`);
+  return {
+    kind,
+    path,
+    label,
+    lines,
+    tokens: estimateTokens(text),
+    hash: contentHash(text),
+    flags,
+  };
+}
+
+// ---------- Surface scanners ----------
+
+function scanAgents(roots) {
+  const out = [];
+  for (const root of roots) {
+    for (const rel of ['.claude/agents', '.agents', 'agents']) {
+      const dir = join(root, rel);
+      if (!existsSync(dir)) continue;
+      let entries;
+      try {
+        entries = readdirSync(dir, { withFileTypes: true });
+      } catch {
+        continue;
+      }
+      for (const e of entries) {
+        if (!e.isFile() || !e.name.endsWith('.md')) continue;
+        const p = join(dir, e.name);
+        const text = readTextSafe(p);
+        if (text == null) continue;
+        out.push(makeComponent('agent', p, `agent:${basename(e.name, '.md')}`, text));
+      }
+    }
+  }
+  return out;
+}
+
+function scanSkills(roots) {
+  const out = [];
+  for (const root of roots) {
+    for (const rel of ['.claude/skills', 'skills']) {
+      const dir = join(root, rel);
+      if (!existsSync(dir)) continue;
+      let entries;
+      try {
+        entries = readdirSync(dir, { withFileTypes: true });
+      } catch {
+        continue;
+      }
+      for (const e of entries) {
+        if (!e.isDirectory()) continue;
+        const p = join(dir, e.name, 'SKILL.md');
+        const text = readTextSafe(p);
+        if (text == null) continue;
+        out.push(makeComponent('skill', p, `skill:${e.name}`, text));
+      }
+    }
+  }
+  return out;
+}
+
+function scanRules(roots) {
+  const out = [];
+  const seenPaths = new Set();
+  for (const root of roots) {
+    for (const rel of ['.claude/rules', 'rules']) {
+      const dir = join(root, rel);
+      if (!existsSync(dir)) continue;
+      walkMdFiles(dir).forEach((p) => {
+        if (seenPaths.has(p)) return;
+        seenPaths.add(p);
+        const text = readTextSafe(p);
+        if (text == null) return;
+        out.push(makeComponent('rule', p, `rule:${basename(p, '.md')}`, text));
+      });
+    }
+  }
+  // Mark content overlap: rules whose first heading line matches another rule's.
+  const byHeading = new Map();
+  for (const c of out) {
+    const head = (readTextSafe(c.path) || '').split('\n')[0].trim();
+    if (!head) continue;
+    if (byHeading.has(head)) {
+      c.flags.push('overlap');
+      byHeading.get(head).flags.push('overlap');
+    } else {
+      byHeading.set(head, c);
+    }
+  }
+  return out;
+}
+
+function walkMdFiles(dir) {
+  const out = [];
+  let entries;
+  try {
+    entries = readdirSync(dir, { withFileTypes: true });
+  } catch {
+    return out;
+  }
+  for (const e of entries) {
+    const p = join(dir, e.name);
+    if (e.isDirectory()) out.push(...walkMdFiles(p));
+    else if (e.isFile() && e.name.endsWith('.md')) out.push(p);
+  }
+  return out;
+}
+
+function scanMcpServers(mcpConfigPath, toolCounts = {}) {
+  const out = [];
+  if (!mcpConfigPath || !existsSync(mcpConfigPath)) return out;
+  let cfg;
+  try {
+    cfg = JSON.parse(readFileSync(mcpConfigPath, 'utf-8'));
+  } catch {
+    return out;
+  }
+  const servers = cfg.mcpServers || {};
+  for (const [name, spec] of Object.entries(servers)) {
+    const flags = [];
+    const cmd = basename(spec.command || '');
+    const firstArg = Array.isArray(spec.args) ? spec.args[0] || '' : '';
+    if (SIMPLE_CLI_COMMANDS.has(cmd) || SIMPLE_CLI_COMMANDS.has(basename(firstArg))) {
+      flags.push('wraps-simple-cli');
+    }
+    const count = toolCounts[name];
+    const hasCount = Number.isFinite(count) && count >= 0;
+    let tokens = 0;
+    if (hasCount) {
+      tokens = TOKENS_PER_MCP_TOOL * count;
+    } else {
+      flags.push('tool-count-unknown');
+    }
+    out.push({
+      kind: 'mcp-server',
+      path: mcpConfigPath,
+      label: `mcp-server:${name}`,
+      lines: 0,
+      tokens,
+      hash: contentHash(`mcp:${name}`),
+      flags,
+      toolCount: hasCount ? count : null,
+    });
+  }
+  return out;
+}
+
+/**
+ * Resolve the CLAUDE.md chain: home global + every CLAUDE.md from cwd upward to a repo
+ * boundary (a dir containing .git) or filesystem root.
+ */
+function claudeMdChain({ home, cwd }) {
+  const paths = [];
+  if (home) paths.push(join(home, '.claude', 'CLAUDE.md'));
+  let dir = cwd;
+  const seen = new Set();
+  while (dir && !seen.has(dir)) {
+    seen.add(dir);
+    paths.push(join(dir, 'CLAUDE.md'));
+    if (existsSync(join(dir, '.git'))) break;
+    const parent = dirname(dir);
+    if (parent === dir) break;
+    dir = parent;
+  }
+  const out = [];
+  const usedPaths = new Set();
+  for (const p of paths) {
+    if (usedPaths.has(p)) continue;
+    usedPaths.add(p);
+    const text = readTextSafe(p);
+    if (text == null) continue;
+    out.push(makeComponent('claude-md', p, `claude-md:${p}`, text));
+  }
+  return out;
+}
+
+/**
+ * Walk every surface into a flat inventory of components.
+ * @param {{cwd:string, home?:string, mcpConfigPath?:string, toolCounts?:object}} opts
+ * @returns {Array} components
+ */
+export function scanSurface({ cwd, home, mcpConfigPath, toolCounts = {} }) {
+  const roots = [home, cwd].filter(Boolean);
+  return [
+    ...scanAgents(roots),
+    ...scanSkills(roots),
+    ...scanRules(roots),
+    ...scanMcpServers(mcpConfigPath, toolCounts),
+    ...claudeMdChain({ home, cwd }),
+  ];
+}
+
+/**
+ * Collapse identical skill copies across surfaces (the real source of churn between
+ * compose/.claude/skills and ~/.claude/skills). Keeps the first occurrence; later
+ * identical copies are retained but marked `duplicateOf` and zeroed so totals don't
+ * double-count. Non-identical same-named skills are both kept.
+ */
+export function dedupeSkills(components) {
+  const seen = new Map(); // key: skill label + content hash -> kept component
+  return components.map((c) => {
+    if (c.kind !== 'skill') return c;
+    const key = `${c.label}::${c.hash}`;
+    if (seen.has(key)) {
+      return { ...c, duplicateOf: seen.get(key).path, tokens: 0, flags: [...c.flags, 'duplicate'] };
+    }
+    seen.set(key, c);
+    return c;
+  });
+}
+
+/**
+ * Classify a component into a budget bucket with an explaining reason.
+ * @param {object} component
+ * @param {{claudeMdText?:string, projectType?:string}} ctx
+ * @returns {{bucket:string, reason:string}}
+ */
+export function classifyComponent(component, ctx = {}) {
+  const { claudeMdText = '', projectType = '' } = ctx;
+  const flags = component.flags || [];
+  const name = component.label.split(':').slice(1).join(':');
+
+  // Duplicates are always a recommended cut.
+  if (component.duplicateOf) {
+    return { bucket: 'rarely', reason: `duplicate of ${component.duplicateOf}` };
+  }
+
+  // CLAUDE.md itself and MCP servers backing the project are load-bearing.
+  if (component.kind === 'claude-md') {
+    return { bucket: 'always', reason: 'CLAUDE.md chain is always loaded' };
+  }
+
+  if (nameReferenced(name, claudeMdText)) {
+    return { bucket: 'always', reason: `referenced by name in the CLAUDE.md chain` };
+  }
+
+  // Overlapping rules / unknown-or-CLI-wrapping MCP servers / over-size domain content.
+  if (flags.includes('overlap')) {
+    return { bucket: 'rarely', reason: 'content overlaps a sibling in the same module' };
+  }
+  if (component.kind === 'mcp-server' && flags.includes('wraps-simple-cli')) {
+    return { bucket: 'rarely', reason: 'wraps a simple CLI the Bash tool can call directly' };
+  }
+
+  if (component.kind === 'mcp-server') {
+    // A real server with tools, not referenced by name — load-bearing while configured,
+    // but a disable-if-unused candidate (schemas are the heaviest single line items).
+    return {
+      bucket: 'sometimes',
+      reason: 'active MCP server, not referenced in CLAUDE.md — disable in .mcp.json if unused',
+    };
+  }
+
+  // Default: domain-specific, not referenced => sometimes (consider lazy-load).
+  return { bucket: 'sometimes', reason: 'not referenced in CLAUDE.md — consider on-demand activation' };
+}
+
+/**
+ * Whole-word "referenced by name" check that tolerates hyphen/space variants.
+ * `compose` matches "the compose skill" but NOT "decompose"/"composed";
+ * `code-standards` matches "code standards", "code-standards", or "codestandards".
+ */
+export function nameReferenced(name, text) {
+  if (!name || !text) return false;
+  const n = name.toLowerCase();
+  const variants = new Set([n, n.replace(/-/g, ' '), n.replace(/-/g, '')]);
+  for (const v of variants) {
+    if (!v) continue;
+    const esc = v.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+    const re = new RegExp(`(^|[^a-z0-9])${esc}([^a-z0-9]|$)`, 'i');
+    if (re.test(text)) return true;
+  }
+  return false;
+}
+
+function formatTokens(n) {
+  if (n >= 1000) return `${(n / 1000).toFixed(1)}K`;
+  return String(n);
+}
+
+/**
+ * Build the structured report (and rendered text) from a classified inventory.
+ * Accepts raw components; classifies internally if a ctx is provided, else expects
+ * components already carrying a `bucket`.
+ */
+export function buildReport(components, ctx = {}) {
+  // Ensure each component is classified.
+  const classified = components.map((c) => {
+    if (c.bucket) return c;
+    const { bucket, reason } = classifyComponent(c, ctx);
+    return { ...c, bucket, reason };
+  });
+
+  const buckets = { always: [], sometimes: [], rarely: [] };
+  for (const c of classified) buckets[c.bucket].push(c);
+
+  const totalTokens = classified.reduce((a, c) => a + c.tokens, 0);
+
+  // Top reclaims: highest-token candidates among sometimes+rarely.
+  const topReclaims = [...buckets.sometimes, ...buckets.rarely]
+    .filter((c) => c.tokens > 0)
+    .sort((a, b) => b.tokens - a.tokens)
+    .slice(0, 5);
+
+  const text = renderReport({ buckets, totalTokens, topReclaims });
+  return { totalTokens, buckets, topReclaims, classified, text };
+}
+
+function renderBucketLines(list) {
+  return list
+    .slice()
+    .sort((a, b) => b.tokens - a.tokens)
+    .map((c) => {
+      const flagStr = c.flags && c.flags.length ? ` [${c.flags.join(', ')}]` : '';
+      return `  - ${c.label} (${c.lines} lines, ~${formatTokens(c.tokens)} tokens) — ${c.reason}${flagStr}`;
+    })
+    .join('\n');
+}
+
+function bucketTotal(list) {
+  return list.reduce((a, c) => a + c.tokens, 0);
+}
+
+function renderReport({ buckets, totalTokens, topReclaims }) {
+  const lines = [];
+  lines.push(`CONTEXT BUDGET — current load: ~${formatTokens(totalTokens)} tokens`);
+  lines.push('');
+  lines.push(`ALWAYS NEEDED (keep, total ~${formatTokens(bucketTotal(buckets.always))} tokens)`);
+  lines.push(renderBucketLines(buckets.always) || '  (none)');
+  lines.push('');
+  lines.push(
+    `SOMETIMES NEEDED (consider lazy-load, total ~${formatTokens(bucketTotal(buckets.sometimes))} tokens)`
+  );
+  lines.push(renderBucketLines(buckets.sometimes) || '  (none)');
+  lines.push('');
+  lines.push(`RARELY NEEDED (recommend cut, total ~${formatTokens(bucketTotal(buckets.rarely))} tokens)`);
+  lines.push(renderBucketLines(buckets.rarely) || '  (none)');
+  lines.push('');
+  lines.push('TOP 5 RECLAIMS:');
+  if (topReclaims.length === 0) {
+    lines.push('  (none)');
+  } else {
+    topReclaims.forEach((c, i) => {
+      lines.push(`  ${i + 1}. ${c.label} (~${formatTokens(c.tokens)} tokens) — ${c.reason}`);
+    });
+  }
+  const potential = bucketTotal(buckets.sometimes) + bucketTotal(buckets.rarely);
+  lines.push('');
+  lines.push(`Potential reclaim if all sometimes+rarely cut: ~${formatTokens(potential)} tokens`);
+  return lines.join('\n');
+}
+
+/**
+ * Top-level orchestrator: scan → dedupe → classify → report.
+ * @param {{cwd:string, home?:string, mcpConfigPath?:string, toolCounts?:object}} opts
+ */
+export function auditContextBudget({ cwd, home, mcpConfigPath, toolCounts = {} }) {
+  const resolvedMcp = mcpConfigPath || (cwd ? join(cwd, '.mcp.json') : undefined);
+  const raw = scanSurface({ cwd, home, mcpConfigPath: resolvedMcp, toolCounts });
+  const deduped = dedupeSkills(raw);
+
+  // Build the CLAUDE.md context text used for "referenced by name" classification.
+  const claudeMdText = deduped
+    .filter((c) => c.kind === 'claude-md')
+    .map((c) => readTextSafe(c.path) || '')
+    .join('\n');
+
+  return buildReport(deduped, { claudeMdText, projectType: 'node' });
+}
+
+// ---------- CLI guard ----------
+export function parseToolCounts(arg) {
+  const out = {};
+  if (!arg) return out;
+  for (const pair of arg.split(',')) {
+    const [name, n] = pair.split('=');
+    if (!name || n == null || n.trim() === '') continue;
+    const num = Number(n);
+    if (Number.isFinite(num) && num >= 0) out[name.trim()] = num;
+  }
+  return out;
+}
+
+function isMainModule() {
+  try {
+    return process.argv[1] && import.meta.url === `file://${process.argv[1]}`;
+  } catch {
+    return false;
+  }
+}
+
+if (isMainModule()) {
+  const args = process.argv.slice(2);
+  const cwd = args.find((a) => !a.startsWith('--')) || process.cwd();
+  const home = process.env.HOME;
+  const tcArg = args.find((a) => a.startsWith('--tool-counts='));
+  const toolCounts = parseToolCounts(tcArg ? tcArg.split('=').slice(1).join('=') : '');
+  const report = auditContextBudget({ cwd, home, toolCounts });
+  process.stdout.write(report.text + '\n');
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@smartmemory/compose",
-  "version": "0.2.22-beta",
+  "version": "0.2.23-beta",
   "description": "Structured AI dev pipeline — goal-to-product orchestration with gates, iteration loops, and feature lifecycle management.",
   "author": "SmartMemory",
   "license": "MIT",
@@ -19,11 +19,11 @@
     "dev:client": "vite",
     "build": "vite build",
     "preview": "vite preview",
-    "test": "node --import ./test/suppress-expected-drift.js --test test/*.test.js test/comp-obs-branch/*.test.js test/integration/*.test.js && npm run test:ui && npm run test:tracker",
+    "test": "node --import ./test/suppress-expected-drift.js --test --test-timeout=120000 test/*.test.js test/comp-obs-branch/*.test.js test/integration/*.test.js && npm run test:ui && npm run test:tracker",
     "test:ui": "vitest run",
     "test:tracker": "vitest run --config vitest.tracker.config.js",
-    "test:integration": "node --test test/integration/*.test.js",
-    "test:wave-6": "node --test test/wave-6-integration.test.js test/wave-6-contract-compliance.test.js",
+    "test:integration": "node --test --test-timeout=120000 test/integration/*.test.js",
+    "test:wave-6": "node --test --test-timeout=120000 test/wave-6-integration.test.js test/wave-6-contract-compliance.test.js",
     "prepublishOnly": "npm run build"
   },
   "keywords": [