mcp-context-cost 0.3.0 → 0.5.0

package/dist/audit/diff.js

@@ -0,0 +1,318 @@
+/** Parse and shape-check a stored report. A baseline that cannot be read is never "no change". */
+export function parseBaselineReport(text) {
+    let doc;
+    try {
+        doc = JSON.parse(text);
+    }
+    catch (e) {
+        return { report: null, problem: `baseline is not JSON: ${e.message}` };
+    }
+    if (!doc || typeof doc !== 'object' || Array.isArray(doc)) {
+        return { report: null, problem: 'baseline is not an audit report object' };
+    }
+    const r = doc;
+    if (!Array.isArray(r.configs)) {
+        return { report: null, problem: "baseline has no 'configs' array — is it the output of `audit --json`?" };
+    }
+    if (typeof r.methodologyVersion !== 'string' || typeof r.encoding !== 'string') {
+        return { report: null, problem: 'baseline is missing methodologyVersion/encoding — is it the output of `audit --json`?' };
+    }
+    for (const c of r.configs) {
+        if (!c || typeof c !== 'object' || typeof c.source !== 'string') {
+            return { report: null, problem: 'baseline has a config entry without a source path' };
+        }
+    }
+    return { report: doc };
+}
+function statesOf(cfg) {
+    const out = new Map();
+    for (const s of cfg.servers ?? [])
+        out.set(s.name, { present: true, tokens: typeof s.tokens === 'number' ? s.tokens : null });
+    // A skipped server IS in the config; it just has no number. Keeping it distinct
+    // from absent is the whole reason `removed` and `unmeasured-now` are separate kinds.
+    for (const s of cfg.skipped ?? [])
+        if (!out.has(s.name))
+            out.set(s.name, { present: true, tokens: null });
+    return out;
+}
+export function diffConfig(before, after, matchedBy) {
+    const afterShare = after.contextShare;
+    if (!before) {
+        return {
+            client: after.client,
+            source: after.source,
+            matchedBy: 'unmatched',
+            beforeTotal: null,
+            afterTotal: after.totalTokens,
+            delta: null,
+            beforeShare: null,
+            afterShare,
+            exact: false,
+            understatedBy: 0,
+            overstatedBy: 0,
+            servers: [],
+        };
+    }
+    const b = statesOf(before);
+    const a = statesOf(after);
+    const servers = [];
+    let understatedBy = 0;
+    let overstatedBy = 0;
+    let exact = true;
+    for (const name of new Set([...b.keys(), ...a.keys()])) {
+        const bs = b.get(name);
+        const as = a.get(name);
+        if (bs && !as) {
+            servers.push(bs.tokens === null
+                ? { name, kind: 'removed', before: null, after: null, delta: null, note: 'was in the config but never measured — removing it changed no measured cost' }
+                : { name, kind: 'removed', before: bs.tokens, after: null, delta: -bs.tokens });
+            continue;
+        }
+        if (!bs && as) {
+            servers.push(as.tokens === null
+                ? { name, kind: 'added', before: null, after: null, delta: null, note: 'added but not measurable — its cost is unknown, not zero' }
+                : { name, kind: 'added', before: null, after: as.tokens, delta: as.tokens });
+            continue;
+        }
+        if (!bs || !as)
+            continue;
+        if (bs.tokens !== null && as.tokens !== null) {
+            const delta = as.tokens - bs.tokens;
+            servers.push({ name, kind: delta === 0 ? 'unchanged' : 'changed', before: bs.tokens, after: as.tokens, delta });
+        }
+        else if (bs.tokens !== null && as.tokens === null) {
+            exact = false;
+            understatedBy += bs.tokens;
+            servers.push({
+                name,
+                kind: 'unmeasured-now',
+                before: bs.tokens,
+                after: null,
+                delta: null,
+                note: `measured ${bs.tokens.toLocaleString('en-US')} in the baseline and could not be measured now — its cost is missing from the total, not gone from your config`,
+            });
+        }
+        else if (bs.tokens === null && as.tokens !== null) {
+            exact = false;
+            overstatedBy += as.tokens;
+            servers.push({
+                name,
+                kind: 'unmeasured-before',
+                before: null,
+                after: as.tokens,
+                delta: null,
+                note: `could not be measured in the baseline and measures ${as.tokens.toLocaleString('en-US')} now — this cost is newly visible, not necessarily new`,
+            });
+        }
+        else {
+            servers.push({
+                name,
+                kind: 'unmeasured-both',
+                before: null,
+                after: null,
+                delta: null,
+                note: 'not measurable in either run — contributes 0 to both totals and hides an unknown cost',
+            });
+        }
+    }
+    // Biggest movers first; ties and non-deltas fall to the bottom in name order.
+    servers.sort((x, y) => Math.abs(y.delta ?? 0) - Math.abs(x.delta ?? 0) || x.name.localeCompare(y.name));
+    return {
+        client: after.client,
+        source: after.source,
+        matchedBy,
+        beforeTotal: before.totalTokens,
+        afterTotal: after.totalTokens,
+        delta: after.totalTokens - before.totalTokens,
+        beforeShare: before.contextShare ?? null,
+        afterShare,
+        exact,
+        understatedBy,
+        overstatedBy,
+        servers,
+    };
+}
+/**
+ * Pair current configs with baseline configs.
+ *
+ * Exact source path first. Then one deliberate fallback: if each side has
+ * exactly one config, they are the same config seen from two machines — the CI
+ * case, where a baseline recorded at /Users/… meets a checkout at /home/runner/….
+ * Anything looser would pair two unrelated clients and call the difference a
+ * change, so everything else stays unmatched and says so.
+ */
+export function pairConfigs(before, after) {
+    const unusedBefore = new Map(before.map((c) => [c.source, c]));
+    const pairs = [];
+    for (const cur of after) {
+        const hit = unusedBefore.get(cur.source);
+        if (hit) {
+            unusedBefore.delete(cur.source);
+            pairs.push({ before: hit, after: cur, matchedBy: 'source' });
+        }
+        else {
+            pairs.push({ before: null, after: cur, matchedBy: 'unmatched' });
+        }
+    }
+    if (before.length === 1 && after.length === 1 && pairs[0].before === null) {
+        pairs[0] = { before: before[0], after: after[0], matchedBy: 'sole-config' };
+        unusedBefore.delete(before[0].source);
+    }
+    return { pairs, dropped: [...unusedBefore.values()] };
+}
+export function buildDiff(baseline, current) {
+    const warnings = [];
+    let comparable = true;
+    if (baseline.methodologyVersion !== current.methodologyVersion) {
+        comparable = false;
+        warnings.push(`methodology changed (${baseline.methodologyVersion} → ${current.methodologyVersion}) — token counts from the two runs are not the same measurement`);
+    }
+    if (baseline.encoding !== current.encoding) {
+        comparable = false;
+        warnings.push(`encoding changed (${baseline.encoding} → ${current.encoding}) — the counts are in different units`);
+    }
+    if (baseline.contextWindow !== current.contextWindow) {
+        // Shares move, token counts do not. Worth saying, not worth invalidating.
+        warnings.push(`context window changed (${baseline.contextWindow.toLocaleString('en-US')} → ${current.contextWindow.toLocaleString('en-US')}) — shares are not comparable, token counts still are`);
+    }
+    const { pairs, dropped } = pairConfigs(baseline.configs, current.configs);
+    const configs = pairs.map((p) => diffConfig(p.before, p.after, p.matchedBy));
+    for (const c of configs) {
+        if (c.matchedBy === 'unmatched') {
+            warnings.push(`${c.source}: no matching config in the baseline — its ${c.afterTotal.toLocaleString('en-US')} tokens are shown as a total, not a change`);
+        }
+        if (c.matchedBy === 'sole-config' && c.source !== baseline.configs[0]?.source) {
+            warnings.push(`paired ${c.source} with the baseline's ${baseline.configs[0]?.source} — one config on each side, different paths`);
+        }
+    }
+    for (const d of dropped) {
+        warnings.push(`${d.source}: in the baseline (${d.totalTokens.toLocaleString('en-US')} tokens) and not found now — a config that disappeared is not a config that got cheaper`);
+    }
+    const increases = configs.filter((c) => typeof c.delta === 'number' && c.delta > 0);
+    increases.sort((a, b) => b.delta - a.delta);
+    return {
+        baselineGeneratedAt: baseline.generatedAt,
+        baselineMethodologyVersion: baseline.methodologyVersion,
+        comparable,
+        droppedConfigs: dropped.map((d) => ({ client: d.client, source: d.source, totalTokens: d.totalTokens })),
+        warnings,
+        configs,
+        worstIncrease: increases.length ? { source: increases[0].source, delta: increases[0].delta } : null,
+    };
+}
+const n = (x) => x.toLocaleString('en-US');
+const signed = (x) => `${x >= 0 ? '+' : '−'}${n(Math.abs(x))}`;
+const pct = (x) => `${(x * 100).toFixed(1)}%`;
+/** `--config <path>` records the client as 'explicit', which is a parser detail, not a name. */
+const clientLabel = (client) => (client === 'explicit' || !client ? 'this client' : client);
+export function formatDiff(diff, contextWindow) {
+    const lines = [];
+    lines.push('');
+    lines.push(`diff vs baseline measured ${diff.baselineGeneratedAt} (methodology ${diff.baselineMethodologyVersion})`);
+    for (const c of diff.configs) {
+        lines.push('');
+        if (c.matchedBy === 'unmatched' || c.delta === null || c.beforeTotal === null) {
+            lines.push(`  ${c.source}  ${n(c.afterTotal)} tokens — no baseline for this config, so nothing to compare`);
+            continue;
+        }
+        const rows = c.servers.filter((s) => s.kind !== 'unchanged');
+        lines.push(`  ${c.source}`);
+        lines.push(`    ${n(c.beforeTotal)}  →  ${n(c.afterTotal)}   ${signed(c.delta)}`);
+        if (rows.length) {
+            lines.push('');
+            const w = Math.max(...rows.map((r) => r.name.length), 6);
+            for (const r of rows) {
+                const from = r.before === null ? '—' : n(r.before);
+                const to = r.after === null ? '—' : n(r.after);
+                const d = r.delta === null ? '' : `  ${signed(r.delta)}`;
+                lines.push(`    ${r.kind.padEnd(17)} ${r.name.padEnd(w)}  ${from.padStart(9)} → ${to.padStart(9)}${d}`);
+            }
+        }
+        const unchanged = c.servers.length - rows.length;
+        if (unchanged)
+            lines.push(`    (${unchanged} server${unchanged === 1 ? '' : 's'} unchanged)`);
+        lines.push('');
+        if (!c.exact) {
+            // The headline sentence is where a skimmer stops, so it must not assert a change
+            // this run could not establish. A server that died takes its tokens out of the
+            // total exactly like a server you uninstalled — printing "removes 2,378 tokens"
+            // and correcting it two lines down is the flattering reading getting read.
+            lines.push(`    Not a clean comparison: a server changed measured-ness between the two runs.`);
+            lines.push(`    The measured total moved ${signed(c.delta)}, but that is not what your config did.`);
+            lines.push('');
+            for (const r of c.servers) {
+                if (r.kind === 'unmeasured-now' || r.kind === 'unmeasured-before')
+                    lines.push(`      ${r.name}: ${r.note}`);
+            }
+            if (c.understatedBy)
+                lines.push(`      → true cost is at least ${n(c.understatedBy)} higher than the ${n(c.afterTotal)} measured now.`);
+            if (c.overstatedBy)
+                lines.push(`      → up to ${n(c.overstatedBy)} of that movement was already being paid, just unmeasured.`);
+        }
+        else if (c.delta === 0) {
+            lines.push(`    No change: this config costs the same ${n(c.afterTotal)} tokens per request as the baseline.`);
+        }
+        else if (c.delta > 0) {
+            lines.push(`    This change adds ${n(c.delta)} tokens to every request in ${clientLabel(c.client)} — ` +
+                `${pct(c.beforeShare ?? 0)} → ${pct(c.afterShare)} of a ${n(contextWindow)}-token context window.`);
+        }
+        else {
+            lines.push(`    This change removes ${n(Math.abs(c.delta))} tokens from every request in ${clientLabel(c.client)} — ` +
+                `${pct(c.beforeShare ?? 0)} → ${pct(c.afterShare)} of a ${n(contextWindow)}-token context window.`);
+        }
+        const blind = c.servers.filter((r) => r.kind === 'unmeasured-both' || ((r.kind === 'added' || r.kind === 'removed') && r.delta === null));
+        if (blind.length) {
+            lines.push('');
+            for (const r of blind)
+                lines.push(`    ${r.name}: ${r.note}`);
+        }
+    }
+    if (diff.warnings.length) {
+        lines.push('');
+        lines.push('  diff warnings');
+        for (const w of diff.warnings)
+            lines.push(`    ${w}`);
+    }
+    if (!diff.comparable) {
+        lines.push('');
+        lines.push('  The two runs are not the same measurement, so the numbers above are not a change.');
+        lines.push('  Re-record the baseline with this version: mcp-context-cost audit --json > baseline.json');
+    }
+    return lines.join('\n');
+}
+/**
+ * `--max-increase N` — the CI gate. Fails on an increase over the limit, and
+ * equally on any reason the increase could not be established.
+ *
+ * That second half is the point. A gate that passes when a server failed to
+ * start, or when the baseline covered a config this run never found, is a green
+ * check on a question nobody asked. Everything this portfolio has learned says
+ * unchecked must not read as clean, so an inexact diff fails and names why.
+ */
+export function evaluateIncreaseGate(diff, limit) {
+    const reasons = [];
+    if (!diff.comparable)
+        reasons.push('the baseline is not the same measurement as this run — nothing was compared');
+    for (const c of diff.configs) {
+        if (c.matchedBy === 'unmatched') {
+            reasons.push(`${c.source}: no baseline to check its ${n(c.afterTotal)} tokens against`);
+        }
+        else if (!c.exact) {
+            reasons.push(`${c.source}: a server changed measured-ness, so the change could not be established exactly`);
+        }
+    }
+    for (const d of diff.droppedConfigs) {
+        reasons.push(`${d.source}: covered by the baseline and not found in this run`);
+    }
+    const increase = diff.worstIncrease?.delta ?? (diff.configs.some((c) => typeof c.delta === 'number') ? 0 : null);
+    if (reasons.length === 0 && increase !== null && increase > limit) {
+        reasons.push(`${diff.worstIncrease.source}: +${n(increase)} tokens per request, over the ${n(limit)} allowed`);
+    }
+    return { limit, pass: reasons.length === 0, increase, reasons };
+}
+export function formatGate(gate) {
+    if (gate.pass) {
+        return `increase ok: ${gate.increase === null ? 'no change to measure' : `${signed(gate.increase)} tokens`} ≤ ${n(gate.limit)} allowed`;
+    }
+    return ['INCREASE FAIL:', ...gate.reasons.map((r) => `  ${r}`)].join('\n');
+}

package/dist/audit/run.d.ts

@@ -1,6 +1,10 @@
 import type { Measurement } from '../core/types.js';
+import { type DivergenceRun } from '../core/divergence.js';
 import { type AuditReport } from './audit.js';
+import { type ToolSearchEnv, type ToolSearchSource } from './deferral.js';
 import { type LoadedConfig } from './config.js';
+/** Where the published `tools-delta/v1` run lives when `--claude` doesn't override it. */
+export declare const DEFAULT_DIVERGENCE_URL = "https://raw.githubusercontent.com/athakur3/mcp-context-cost/main/results/divergence.json";
 export interface AuditOptions {
     /** Explicit config path(s); when empty, every known client location is tried. */
     configPaths?: string[];
@@ -11,9 +15,39 @@ export interface AuditOptions {
     docker?: boolean;
     contextWindow?: number;
     budget?: number;
+    /** Join each measured server against the published Claude divergence run. */
+    claude?: boolean;
+    /** Override the divergence.json source — mainly for tests and self-hosted mirrors. */
+    divergenceUrl?: string;
+    /**
+     * The tool-search variables as this process's SHELL has them. Defaults to
+     * this process's environment. Overridable so a test can state a machine
+     * rather than inherit the one it runs on.
+     */
+    env?: ToolSearchEnv;
+    /**
+     * The same variables as Claude Code's own settings files set them — the other
+     * half of the answer, and the half a shell cannot show. Defaults to reading
+     * those files off the machine being audited (`discoverSettings`).
+     */
+    settings?: ToolSearchSource[];
     onProgress?: (name: string, done: number, total: number) => void;
 }
+/** Fetch and parse the published divergence run. Never throws: a failure is a report problem, not a crash. */
+export declare function fetchDivergence(url: string): Promise<{
+    run: DivergenceRun | null;
+    problem?: string;
+}>;
 export declare function discover(opts?: AuditOptions): LoadedConfig[];
+/**
+ * Read every settings file Claude Code would take its deferral setting from.
+ *
+ * A sibling of `discover` above and deliberately separate from it: that one
+ * finds which servers exist, this one finds how the client treats them. The
+ * setting does not live in a client config, and reading only the shell that
+ * launched this audit answers for the wrong machine.
+ */
+export declare function discoverSettings(opts?: AuditOptions): ToolSearchSource[];
 /** Measure every distinct stdio server across the given configs, once each. */
 export declare function measureAll(configs: LoadedConfig[], opts?: AuditOptions): Promise<Map<string, Measurement>>;
 export declare function runAudit(opts?: AuditOptions): Promise<AuditReport>;

package/dist/audit/run.js

@@ -6,8 +6,25 @@
  */
 import { homedir } from 'node:os';
 import { measureServer } from '../sweep/run.js';
+import { parseDivergence } from '../core/divergence.js';
 import { buildReport, serverKey } from './audit.js';
-import { configCandidates, loadConfigs } from './config.js';
+import { toolSearchEnv } from './deferral.js';
+import { configCandidates, loadConfigs, loadSettingsSources, settingsCandidates, } from './config.js';
+/** Where the published `tools-delta/v1` run lives when `--claude` doesn't override it. */
+export const DEFAULT_DIVERGENCE_URL = 'https://raw.githubusercontent.com/athakur3/mcp-context-cost/main/results/divergence.json';
+/** Fetch and parse the published divergence run. Never throws: a failure is a report problem, not a crash. */
+export async function fetchDivergence(url) {
+    try {
+        const res = await fetch(url, { signal: AbortSignal.timeout(15_000) });
+        if (!res.ok)
+            return { run: null, problem: `claude divergence: HTTP ${res.status} fetching ${url}` };
+        const run = parseDivergence(await res.text());
+        return run ? { run } : { run: null, problem: `claude divergence: malformed data at ${url}` };
+    }
+    catch (e) {
+        return { run: null, problem: `claude divergence: failed to fetch ${url}: ${e.message}` };
+    }
+}
 export function discover(opts = {}) {
     const cwd = opts.cwd ?? process.cwd();
     const home = opts.home ?? homedir();
@@ -16,6 +33,19 @@ export function discover(opts = {}) {
         : configCandidates({ home, cwd, platform: process.platform, appData: process.env.APPDATA });
     return loadConfigs(candidates, cwd);
 }
+/**
+ * Read every settings file Claude Code would take its deferral setting from.
+ *
+ * A sibling of `discover` above and deliberately separate from it: that one
+ * finds which servers exist, this one finds how the client treats them. The
+ * setting does not live in a client config, and reading only the shell that
+ * launched this audit answers for the wrong machine.
+ */
+export function discoverSettings(opts = {}) {
+    const cwd = opts.cwd ?? process.cwd();
+    const home = opts.home ?? homedir();
+    return loadSettingsSources(settingsCandidates({ home, cwd, platform: process.platform, programData: process.env.ProgramData }));
+}
 /** Measure every distinct stdio server across the given configs, once each. */
 export async function measureAll(configs, opts = {}) {
     const unique = new Map();
@@ -52,8 +82,21 @@ export async function measureAll(configs, opts = {}) {
 export async function runAudit(opts = {}) {
     const configs = discover(opts);
     const measured = await measureAll(configs, opts);
-    return buildReport(configs, measured, {
+    let divergence = null;
+    let divergenceProblem;
+    if (opts.claude) {
+        const fetched = await fetchDivergence(opts.divergenceUrl ?? DEFAULT_DIVERGENCE_URL);
+        divergence = fetched.run;
+        divergenceProblem = fetched.problem;
+    }
+    const report = buildReport(configs, measured, {
         contextWindow: opts.contextWindow,
         budget: opts.budget,
+        divergence,
+        env: opts.env ?? toolSearchEnv(process.env),
+        settings: opts.settings ?? discoverSettings(opts),
     });
+    if (divergenceProblem)
+        report.problems.push(divergenceProblem);
+    return report;
 }

package/dist/cli.d.ts

@@ -6,3 +6,24 @@ export declare function verifyMeasurement(m: Measurement): {
     rederivedSha: string | null;
     problems: string[];
 };
+/** Derives a servers.yaml-style slug from a remote URL's hostname, e.g. mcp.deepwiki.com -> deepwiki. */
+export declare function slugFromUrl(url: string): string;
+/** Installed version, for error messages that need to say which one you are running. */
+export declare function cliVersion(): string;
+/**
+ * Reject flags this build does not know.
+ *
+ * An older CLI used to ignore an unrecognised flag and carry on. That is the exact failure
+ * this project exists to catch, in our own tool: `audit --baseline base.json
+ * --max-increase 2000` on a build without those flags ran a plain audit and **exited 0** —
+ * a green CI check on a gate that never ran. The README documents flags before they are
+ * published, so the version skew is not hypothetical; it is the normal case for anyone
+ * running `npx -y mcp-context-cost`.
+ *
+ * So an unknown flag is a usage error, and the message names the running version, because
+ * the likeliest cause is that the reader's command is newer than their install.
+ */
+export declare function unknownFlags(argv: string[], spec: {
+    value: string[];
+    boolean: string[];
+}): string[];