mcp-context-cost 0.8.0 → 0.9.0

package/README.md

@@ -183,10 +183,9 @@ INCREASE FAIL:
 Add `--claude` to annotate each server with its Anthropic-request cost from the published
 [Claude divergence](docs/METHODOLOGY.md#claude-divergence) run — an exact number when the
 published capture hash matches what you have installed, `—` (silence, not a stale guess)
-when it doesn't. The run holds 20 rows — the top 20 measured servers by tokens; 19 of them
-still match the capture on disk here, which is why
-[results/leaderboard.md](results/leaderboard.md) prints a claude number for 19 and leaves the
-twentieth blank rather than stale. Most installs will show a mix:
+when it doesn't. The run holds 20 rows — the top 20 measured servers by tokens when it ran —
+and [results/leaderboard.md](results/leaderboard.md) prints a claude number for the 16 that
+still match today and silence for the rest. Most installs will show a mix:
 
 ```
   server               tools  tokens   share   claude
@@ -202,14 +201,14 @@ Flags: `--json` (full report on stdout, progress on stderr), `--budget N`,
 
 The number `audit` gives you is the same measurement, run across a curated set of public
 servers — which is how you can tell it is a measurement and not this tool's opinion. It also
-shows what you are choosing between: across the 69 servers measured, cost spans **1,700×**,
-from the 32-token `postgres` reference server to github's 54,422. The table below is a
+shows what you are choosing between: across the 81 servers measured, cost spans **1,700×**,
+from `postgres` at 32 tokens to `github` at 54,622. The table below is a
 sample of that range; the full range is in
 [results/leaderboard.md](results/leaderboard.md).
 
 | server | context cost | tools |
 |---|---:|---:|
-| github (official) | **54,422 tokens** | 44 |
+| github (official) | **54,622 tokens** | 44 |
 | xcodebuildmcp | 26,594 | 24 |
 | brave-search | 25,456 | 8 |
 | notion | 17,500 | 24 |
@@ -217,7 +216,7 @@ sample of that range; the full range is in
 | filesystem (reference) | 2,823 | 14 |
 | markitdown | 64 | 1 |
 
-*(69 of 82 popular servers measured, sweeps of 2026-08-18 and 2026-08-19 — full table in
+*(81 of 106 popular servers measured, each row dated by its own most recent sweep — full table in
 [results/leaderboard.md](results/leaderboard.md); every failure is listed with its reason.
 Each measured server also has a [detail page](https://athakur3.github.io/mcp-context-cost/servers/)
 showing which tools its tokens are in.)*
@@ -252,7 +251,7 @@ exact launch command. Disputes reduce to a byte-level diff:
 
 ```bash
 npx -y mcp-context-cost verify results/github/measurement.json
-# OK github-mcp-server: 54422 tokens (o200k_base, methodology 1.0) — capture, hash, and count all agree
+# OK github-mcp-server: 54622 tokens (o200k_base, methodology 1.0) — capture, hash, and count all agree
 
 # or point it at a published measurement.json directly, no clone required
 npx -y mcp-context-cost verify --remote https://raw.githubusercontent.com/athakur3/mcp-context-cost/main/results/github/measurement.json
@@ -267,6 +266,12 @@ failed, `2` usage error.
 to the one that was published — which is exactly what `--claude` uses to decide whether it is
 allowed to show you a number.
 
+The number is also cross-checked against the other CLI that measures this,
+[`sd2k/mcp-tokens`](https://github.com/sd2k/mcp-tokens): the leaderboard's **mcp-tokens**
+column publishes its count beside ours wherever both tools saw the same tool set — same
+o200k encoding, differences documented, the divergence published rather than left to be
+discovered. Method: [CLI cross-check](docs/METHODOLOGY.md#cli-cross-check).
+
 Full definition: [docs/METHODOLOGY.md](docs/METHODOLOGY.md) — what is counted, what the
 number is *not*, config policy, failure taxonomy, frozen color bands, known divergences.
 
@@ -324,7 +329,7 @@ color bands are frozen against the observed distribution of the first full sweep
 
 ## Status
 
-Active. 46 of the 69 numbers come from the sweep of 2026-08-19 and 23 from 2026-08-18. Two
+Active. Every row carries the date of its own most recent measurement. Two
 weekly jobs re-measure the set — the `memory` reference server on Mondays, and a rotating
 sixth of the list on Wednesdays, so every row comes round within six weeks. Read each row's
 date as the date it means, and don't take the cadence on trust — the build history is

package/dist/cli.js

@@ -159,18 +159,33 @@ if (cmd === 'audit') {
     }
     const { runAudit } = await import('./audit/run.js');
     const { formatReport } = await import('./audit/audit.js');
-    const report = await runAudit({
-        configPaths: all('config'),
-        budget,
-        contextWindow: numeric('context'),
-        timeoutMs: numeric('timeout'),
-        concurrency: numeric('concurrency'),
-        docker: rest.includes('--docker'),
-        claude: rest.includes('--claude'),
-        divergenceUrl: argOf('divergence-url'),
-        // Progress goes to stderr so `--json` stdout stays a single parseable object.
-        onProgress: json ? undefined : (name, done, total) => process.stderr.write(`  [${done}/${total}] ${name}\n`),
-    });
+    const { DockerHarnessFault } = await import('./sweep/docker.js');
+    let report;
+    try {
+        report = await runAudit({
+            configPaths: all('config'),
+            budget,
+            contextWindow: numeric('context'),
+            timeoutMs: numeric('timeout'),
+            concurrency: numeric('concurrency'),
+            docker: rest.includes('--docker'),
+            claude: rest.includes('--claude'),
+            divergenceUrl: argOf('divergence-url'),
+            // Progress goes to stderr so `--json` stdout stays a single parseable object.
+            onProgress: json ? undefined : (name, done, total) => process.stderr.write(`  [${done}/${total}] ${name}\n`),
+        });
+    }
+    catch (e) {
+        // Docker failing as docker means every measurement through it would be a
+        // statement about this machine, so the audit refuses whole rather than
+        // reporting each server as broken.
+        if (e instanceof DockerHarnessFault) {
+            console.error(`audit --docker cannot answer for this machine: ${e.message}`);
+            console.error('Fix Docker here, or run without --docker.');
+            process.exit(1);
+        }
+        throw e;
+    }
     if (report.configs.length === 0) {
         const where = report.problems.length ? `\n${report.problems.map((p) => `  ${p}`).join('\n')}` : '';
         const empty = report.emptyConfigs ?? [];

package/dist/core/cross-check.d.ts

@@ -0,0 +1,93 @@
+import type { Measurement } from './types.js';
+/** Method identifier, versioned independently of METHODOLOGY_VERSION. */
+export declare const CROSS_CHECK_METHOD = "cli-cross-check/v1";
+/** The CLI being cross-checked against, and the release this run pins. */
+export declare const CROSS_CHECK_CLI = "sd2k/mcp-tokens";
+export declare const CROSS_CHECK_CLI_VERSION = "v0.2.5";
+/**
+ * The exact analyze invocation, recorded in every run. `--model gpt-4o` is
+ * load-bearing: tiktoken-rs maps it to o200k_base, and without a model the CLI
+ * falls back to cl100k_base — a systematic difference that would swamp the one
+ * being measured (spec/upstream-notes.md, finding 1).
+ */
+export declare const CROSS_CHECK_CLI_ARGS: readonly ['analyze', '--provider', 'tiktoken', '--model', 'gpt-4o', '--format', 'json'];
+export interface CrossCheckRow {
+    /** o200k count of our fresh canonical capture from this run — the headline. */
+    ourTokens: number;
+    /**
+     * o200k count of the same capture's name/description/input_schema
+     * projection — what the CLI's structs actually model, and therefore the
+     * number its count is compared against.
+     */
+    ourMappedTokens: number;
+    /** The CLI's `.tools.total` for the same server, launched by the CLI itself. */
+    cliTokens: number;
+    ourToolCount: number;
+    cliToolCount: number;
+    /** Both tools saw the same tool names (order-insensitive) — the comparison is between like and like. */
+    toolSetMatches: boolean;
+    /**
+     * Our fresh measurement listed dynamically (tools/list differed between its
+     * own two captures). The CLI's launch is a third capture, so even with
+     * matching names its residual mixes content drift with counter disagreement
+     * — recorded, never printed.
+     */
+    dynamic: boolean;
+    /** canonicalSha256 of our capture in this run; null when our measurement failed. */
+    capturedSha256: string | null;
+    /** Set when either side could not produce a number; nothing is published from the row. */
+    error?: string;
+}
+export interface CrossCheckRun {
+    method: string;
+    cli: string;
+    cliVersion: string;
+    /** The analyze invocation, verbatim, so the encoding choice is auditable. */
+    cliArgs: string[];
+    /** UTC day the run was taken (YYYY-MM-DD). */
+    measuredAt: string;
+    isolation: string;
+    servers: Record<string, CrossCheckRow>;
+}
+export declare function parseCrossCheck(text: string): CrossCheckRun | null;
+/** What the CLI's JSON report contributes to a row. */
+export interface CliReport {
+    total: number;
+    count: number;
+    names: string[];
+}
+/**
+ * Read the CLI's `--format json` report. The report is that CLI's contract, not
+ * ours, so this is deliberately narrow: `.tools.total` (the number the CLI's
+ * own action publishes), `.tools.count`, and the per-tool names — and a report
+ * that does not carry a usable total is a named problem, never a zero.
+ */
+export declare function parseCliReport(text: string): {
+    report?: CliReport;
+    problem?: string;
+};
+/** Order-insensitive equality of the two tool-name lists, repeats included. */
+export declare function sameToolSet(ours: string[], theirs: string[]): boolean;
+/**
+ * Build a row from our fresh measurement and the CLI's report. The caller
+ * decides whether to run the CLI at all; a failed side arrives here as an
+ * `error`, and the row keeps whatever the other side established.
+ */
+export declare function toCrossCheckRow(m: Measurement, cli: {
+    report?: CliReport;
+    problem?: string;
+}): CrossCheckRow;
+/**
+ * A row is only printable while it compares like with like: no error on either
+ * side, the same tool names seen by both tools, and our capture still the one
+ * published — the exact staleness rule the claude column follows.
+ */
+export declare function isComparable(row: CrossCheckRow | undefined, canonicalSha256: string | null): row is CrossCheckRow;
+/**
+ * Signed divergence of the CLI's count from our count of the projection it
+ * models, in percent. Against the mapped number and not the headline: the gap
+ * to the headline is field selection, published separately per server, and
+ * folding it in here would bury the number this column exists to check —
+ * whether two independent counters agree on the fields both count.
+ */
+export declare function divergencePct(row: CrossCheckRow): number | null;

package/dist/core/cross-check.js

@@ -0,0 +1,176 @@
+/**
+ * CLI cross-check — the other tool's number, published beside ours.
+ *
+ * `sd2k/mcp-tokens` measures the same thing this project measures, differently
+ * in two documented ways (spec/upstream-notes.md): its tiktoken provider picks
+ * the encoding from a `--model` argument with a cl100k_base fallback, and it
+ * counts a `serde_json` re-serialization of deserialized tool structs rather
+ * than the wire bytes. Measured (2026-09-03, addendum in the notes), the
+ * second difference is nearly all field modeling: the CLI's count lands within
+ * a fraction of a percent of our o200k count of the three-field
+ * name/description/input_schema projection — the same projection the Claude
+ * divergence starts from — not of the full capture. So the published
+ * comparison is against `mappedTokens` of the same fresh capture: like counted
+ * against like, with the field-selection gap already published separately as
+ * each server's field-selection share. The honest move is the one the notes
+ * chose on day one: publish the divergence ourselves rather than leave it to
+ * be discovered by critics.
+ *
+ * Every row is filed under the `canonicalSha256` of a capture taken by OUR
+ * client in the same run, minutes from the CLI's own launch — the session-start
+ * discipline: a re-sweep that moves the published capture marks the row stale
+ * instead of leaving a comparison against bytes that no longer exist. And a
+ * comparison is only printed when both tools counted the same tool set: the CLI
+ * launches the server itself, so on a server that changed between the two
+ * launches (or lists dynamically) the two numbers describe different schemas,
+ * and publishing their difference as "divergence" would be a category error.
+ * Those rows keep their data in the run file and print silence.
+ *
+ * Versioned independently of the o200k methodology, like `tools-delta/v1` and
+ * `deferred-load/v1`: a new published number, no change to the definition of
+ * the first. No `totalTokens` and no canonical hash moves.
+ */
+import { mappedTokens } from './divergence.js';
+import { toolNames } from './session-start.js';
+/** Method identifier, versioned independently of METHODOLOGY_VERSION. */
+export const CROSS_CHECK_METHOD = 'cli-cross-check/v1';
+/** The CLI being cross-checked against, and the release this run pins. */
+export const CROSS_CHECK_CLI = 'sd2k/mcp-tokens';
+export const CROSS_CHECK_CLI_VERSION = 'v0.2.5';
+/**
+ * The exact analyze invocation, recorded in every run. `--model gpt-4o` is
+ * load-bearing: tiktoken-rs maps it to o200k_base, and without a model the CLI
+ * falls back to cl100k_base — a systematic difference that would swamp the one
+ * being measured (spec/upstream-notes.md, finding 1).
+ */
+export const CROSS_CHECK_CLI_ARGS = ['analyze', '--provider', 'tiktoken', '--model', 'gpt-4o', '--format', 'json'];
+export function parseCrossCheck(text) {
+    let run;
+    try {
+        run = JSON.parse(text);
+    }
+    catch {
+        return null;
+    }
+    const r = run;
+    if (!r || typeof r.cliVersion !== 'string' || typeof r.measuredAt !== 'string')
+        return null;
+    if (!r.servers || typeof r.servers !== 'object')
+        return null;
+    return {
+        method: typeof r.method === 'string' ? r.method : CROSS_CHECK_METHOD,
+        cli: typeof r.cli === 'string' ? r.cli : CROSS_CHECK_CLI,
+        cliVersion: r.cliVersion,
+        cliArgs: Array.isArray(r.cliArgs) ? r.cliArgs.map(String) : [...CROSS_CHECK_CLI_ARGS],
+        measuredAt: r.measuredAt,
+        isolation: typeof r.isolation === 'string' ? r.isolation : 'not recorded',
+        servers: r.servers,
+    };
+}
+/**
+ * Read the CLI's `--format json` report. The report is that CLI's contract, not
+ * ours, so this is deliberately narrow: `.tools.total` (the number the CLI's
+ * own action publishes), `.tools.count`, and the per-tool names — and a report
+ * that does not carry a usable total is a named problem, never a zero.
+ */
+export function parseCliReport(text) {
+    let parsed;
+    try {
+        parsed = JSON.parse(text);
+    }
+    catch {
+        // `--format json` puts the report on stdout, but a launcher inside the same
+        // pipe can precede it with noise; the report is the outermost JSON object.
+        const start = text.indexOf('{');
+        const end = text.lastIndexOf('}');
+        if (start < 0 || end <= start)
+            return { problem: 'CLI stdout carried no JSON report' };
+        try {
+            parsed = JSON.parse(text.slice(start, end + 1));
+        }
+        catch {
+            return { problem: 'CLI stdout carried no parseable JSON report' };
+        }
+    }
+    const tools = parsed.tools;
+    if (!tools || typeof tools !== 'object')
+        return { problem: 'CLI report has no .tools object' };
+    if (typeof tools.total !== 'number' || !Number.isFinite(tools.total) || tools.total < 0) {
+        return { problem: 'CLI report has no usable .tools.total' };
+    }
+    const items = Array.isArray(tools.items) ? tools.items : [];
+    const names = [];
+    for (const item of items) {
+        const name = (item ?? {}).name;
+        if (typeof name === 'string' && name !== '')
+            names.push(name);
+    }
+    return {
+        report: {
+            total: tools.total,
+            count: typeof tools.count === 'number' ? tools.count : items.length,
+            names,
+        },
+    };
+}
+/** Order-insensitive equality of the two tool-name lists, repeats included. */
+export function sameToolSet(ours, theirs) {
+    if (ours.length !== theirs.length)
+        return false;
+    const a = [...ours].sort();
+    const b = [...theirs].sort();
+    return a.every((name, i) => name === b[i]);
+}
+/**
+ * Build a row from our fresh measurement and the CLI's report. The caller
+ * decides whether to run the CLI at all; a failed side arrives here as an
+ * `error`, and the row keeps whatever the other side established.
+ */
+export function toCrossCheckRow(m, cli) {
+    const measured = m.status === 'measured' || m.status === 'dynamic';
+    const raw = measured && Array.isArray(m.rawToolsCapture) ? m.rawToolsCapture : [];
+    const ours = toolNames(raw);
+    const row = {
+        ourTokens: measured && typeof m.totalTokens === 'number' ? m.totalTokens : 0,
+        ourMappedTokens: raw.length > 0 ? mappedTokens(raw) : 0,
+        cliTokens: cli.report?.total ?? 0,
+        ourToolCount: measured && typeof m.toolCount === 'number' ? m.toolCount : 0,
+        cliToolCount: cli.report?.count ?? 0,
+        toolSetMatches: cli.report !== undefined && measured ? sameToolSet(ours, cli.report.names) : false,
+        dynamic: m.status === 'dynamic',
+        capturedSha256: measured ? m.canonicalSha256 : null,
+    };
+    if (!measured)
+        row.error = `our measurement: ${m.status}: ${(m.notes ?? '').slice(0, 200)}`;
+    else if (cli.problem)
+        row.error = `cli: ${cli.problem.slice(0, 200)}`;
+    return row;
+}
+/**
+ * A row is only printable while it compares like with like: no error on either
+ * side, the same tool names seen by both tools, and our capture still the one
+ * published — the exact staleness rule the claude column follows.
+ */
+export function isComparable(row, canonicalSha256) {
+    return (row !== undefined &&
+        row.error === undefined &&
+        row.toolSetMatches &&
+        !row.dynamic &&
+        row.capturedSha256 !== null &&
+        canonicalSha256 !== null &&
+        row.capturedSha256 === canonicalSha256 &&
+        row.ourTokens > 0 &&
+        row.ourMappedTokens > 0);
+}
+/**
+ * Signed divergence of the CLI's count from our count of the projection it
+ * models, in percent. Against the mapped number and not the headline: the gap
+ * to the headline is field selection, published separately per server, and
+ * folding it in here would bury the number this column exists to check —
+ * whether two independent counters agree on the fields both count.
+ */
+export function divergencePct(row) {
+    if (row.ourMappedTokens <= 0)
+        return null;
+    return ((row.cliTokens - row.ourMappedTokens) / row.ourMappedTokens) * 100;
+}

package/dist/sweep/cross-check.d.ts

@@ -0,0 +1,31 @@
+import type { ServerEntry } from './report.js';
+import { type CrossCheckRun } from '../core/cross-check.js';
+export declare function loadCrossCheck(root?: string): CrossCheckRun | null;
+export declare function writeCrossCheck(run: CrossCheckRun, root?: string): void;
+/** Release-asset triple for where the CLI will actually run. */
+export declare function cliTriple(docker: boolean, platform?: NodeJS.Platform, arch?: NodeJS.Architecture): string;
+/**
+ * Fetch the pinned CLI release for `triple` into a host cache, verifying the
+ * archive against the release's own `.sha256` before anything is extracted or
+ * executed. `MCP_TOKENS_BIN` overrides the whole dance — that is how the tests
+ * substitute a shim, and how an airgapped machine supplies its own copy.
+ */
+export declare function ensureCliBinary(triple: string): Promise<string>;
+interface CliOutcome {
+    code: number | null;
+    stdout: string;
+    stderr: string;
+    timedOut: boolean;
+}
+/**
+ * Run the CLI against one server, host or containerized. In docker mode the
+ * binary is bind-mounted read-only and the CLI launches the server inside the
+ * container — same image, limits, dummy env and shared package caches as the
+ * measurement that just ran, so the npx install it pays is already warm.
+ */
+export declare function runCli(binPath: string, entry: ServerEntry, opts: {
+    docker: boolean;
+    timeoutMs: number;
+    graceMs?: number;
+}): Promise<CliOutcome>;
+export {};