mcp-context-cost 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 mcp-context-cost contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,90 @@
+# mcp-context-cost
+
+**Reproducible context-cost badges for MCP servers.**
+
+Every MCP server you wire into an agent injects its tool schemas into the model's context
+before any work happens. That cost is invisible — and it varies by **1,700×** across
+popular servers:
+
+| server | context cost | tools |
+|---|---:|---:|
+| github (official) | **54,422 tokens** | 44 |
+| brave-search | 25,456 | 8 |
+| notion | 17,500 | 24 |
+| playwright *(4.8M installs/week)* | 4,024 | 24 |
+| filesystem (reference) | 2,823 | 14 |
+| markitdown | 64 | 1 |
+
+*(57 of 82 popular servers measured, 2026-08-16 sweep — full table in
+[results/leaderboard.md](results/leaderboard.md); every failure is listed with its reason.)*
+
+This project makes that cost **legible and disputable**:
+
+```
+[context cost | 12,430 tokens]   ← shields.io badge, linked to the methodology
+```
+
+## Why trust the number?
+
+Every badge is backed by a `measurement.json` containing the raw `tools/list` capture, the
+SHA-256 of its canonical bytes, the pinned tokenizer (`o200k_base`), and the exact launch
+command. Disputes reduce to a byte-level diff:
+
+```bash
+npx tsx src/cli.ts verify results/github/measurement.json
+# OK github-mcp-server: 54422 tokens (o200k_base, methodology 1.0) — capture, hash, and count all agree
+```
+
+Full definition: [docs/METHODOLOGY.md](docs/METHODOLOGY.md) — what is counted, what the
+number is *not*, config policy, failure taxonomy, frozen color bands, known divergences.
+
+## What's in the repo
+
+| path | what |
+|---|---|
+| `src/core/` | the measurement spec, executable — canonical form, tokenizer, bands, badge JSON |
+| `src/sweep/` | raw-wire MCP stdio client + Dockerized batch sweep + leaderboard/dashboard generators |
+| `src/cli.ts` | `verify` (re-derive any published number) and `measure` |
+| `spec/fixtures/` | golden vectors shared by the TypeScript and bash implementations |
+| `upstream/` | staged contribution to [sd2k/mcp-tokens-action](https://github.com/sd2k/mcp-tokens-action): `badge.sh` + action patch + tests |
+| `servers.yaml` | 82 curated candidates with live install metrics and provenance |
+| `results/` · `badges/` | measurements, leaderboard, shields endpoint JSONs |
+
+## Measure your own server
+
+```bash
+npm ci
+npm run sweep -- --name my-server --command "npx -y my-mcp-server"
+cat badges/my-server.json   # strict shields.io endpoint JSON
+```
+
+Then in your README:
+
+```markdown
+[![context cost](https://img.shields.io/endpoint?url=<raw URL of badges/my-server.json>)](<methodology URL>)
+```
+
+Or self-serve from CI via the (staged) mcp-tokens-action badge inputs — see
+[upstream/action-patch.md](upstream/action-patch.md).
+
+## Development
+
+```bash
+npm test                        # 24 TS tests incl. golden fixtures + dispute drills
+npx tsc --noEmit                # typecheck
+./upstream/tests/badge-test.sh  # 21 bash tests — byte-identical to the TS reference
+npm run sweep:all -- --docker   # full curated sweep (Docker isolation)
+```
+
+Notable engineering choices: the MCP client is a deliberate ~150-line raw-wire
+implementation (SDK schema-parsing can reorder keys, which would corrupt canonical bytes);
+sweep servers run in credential-free Docker containers with recorded isolation; the badge
+color bands are frozen against the observed distribution of the first full sweep.
+
+## Status
+
+Pre-launch. Measurement pipeline, leaderboard, methodology, and upstream contribution are
+complete; the badge-adoption campaign and launch posts are drafted in `campaign/` awaiting
+go-ahead. Build log: [PROGRESS.md](PROGRESS.md).
+
+MIT © 2026

package/dist/cli.d.ts

@@ -0,0 +1,8 @@
+#!/usr/bin/env node
+import type { Measurement } from './core/types.js';
+export declare function verifyMeasurement(m: Measurement): {
+    ok: boolean;
+    rederivedTokens: number | null;
+    rederivedSha: string | null;
+    problems: string[];
+};

package/dist/cli.js

@@ -0,0 +1,78 @@
+#!/usr/bin/env node
+/**
+ * mcp-context-cost CLI — the dispute drill as a command.
+ *
+ *   mcp-context-cost verify <measurement.json>   re-derive the number from the
+ *                                                published capture; exit 1 on mismatch
+ *   mcp-context-cost measure --name x --command "npx -y ..."   one-off measurement
+ */
+import { readFileSync } from 'node:fs';
+import { canonicalString, countTokens, sha256Hex } from './core/canonical.js';
+import { toBadge } from './core/badge.js';
+export function verifyMeasurement(m) {
+    const problems = [];
+    if (!m.rawToolsCapture) {
+        return { ok: false, rederivedTokens: null, rederivedSha: null, problems: ['no rawToolsCapture in measurement'] };
+    }
+    const canonical = canonicalString(m.rawToolsCapture);
+    const tokens = countTokens(canonical);
+    const sha = sha256Hex(canonical);
+    if (tokens !== m.totalTokens)
+        problems.push(`token mismatch: re-derived ${tokens}, stored ${m.totalTokens}`);
+    if (sha !== m.canonicalSha256)
+        problems.push(`sha mismatch: re-derived ${sha}, stored ${m.canonicalSha256}`);
+    if (m.toolCount !== m.rawToolsCapture.length)
+        problems.push(`toolCount mismatch: capture has ${m.rawToolsCapture.length}, stored ${m.toolCount}`);
+    return { ok: problems.length === 0, rederivedTokens: tokens, rederivedSha: sha, problems };
+}
+const [, , cmd, ...rest] = process.argv;
+if (cmd === 'verify') {
+    const path = rest.find((a) => !a.startsWith('--'));
+    if (!path) {
+        console.error('usage: mcp-context-cost verify <measurement.json>');
+        process.exit(2);
+    }
+    const m = JSON.parse(readFileSync(path, 'utf8'));
+    const r = verifyMeasurement(m);
+    if (r.ok) {
+        console.log(`OK ${m.serverName}: ${r.rederivedTokens} tokens (${m.encoding}, methodology ${m.methodologyVersion}) — capture, hash, and count all agree`);
+        console.log(`badge: ${JSON.stringify(toBadge(m))}`);
+        process.exit(0);
+    }
+    console.error(`FAIL ${m.serverName}:`);
+    for (const p of r.problems)
+        console.error(`  - ${p}`);
+    process.exit(1);
+}
+else if (cmd === 'measure') {
+    const argOf = (name) => {
+        const i = rest.indexOf(`--${name}`);
+        return i >= 0 ? rest[i + 1] : undefined;
+    };
+    const name = argOf('name');
+    const command = argOf('command');
+    if (!name || !command) {
+        console.error('usage: mcp-context-cost measure --name <slug> --command "npx -y <server>" [--timeout ms] [--docker]');
+        process.exit(2);
+    }
+    const { measureServer } = await import('./sweep/run.js');
+    const m = await measureServer(name, command, {
+        timeoutMs: Number(argOf('timeout') ?? 60_000),
+        docker: rest.includes('--docker'),
+        dockerImage: argOf('docker-image'),
+    });
+    const ok = m.status === 'measured' || m.status === 'dynamic';
+    console.log(ok
+        ? `${name}: ${m.totalTokens} tokens across ${m.toolCount} tools (${m.status}) — results/${name}/measurement.json, badges/${name}.json`
+        : `${name}: ${m.status} — ${m.notes ?? ''}`);
+    process.exit(ok ? 0 : 1);
+}
+else if (cmd !== undefined && cmd !== '--help' && cmd !== '-h') {
+    console.error(`unknown command: ${cmd}`);
+    process.exit(2);
+}
+else {
+    console.log('mcp-context-cost — reproducible context-cost measurement for MCP servers');
+    console.log('  verify <measurement.json>    re-derive tokens+sha from the published capture');
+    console.log('  measure --name x --command "npx -y <server>"   run a one-off measurement');
+}

package/dist/core/badge.d.ts

@@ -0,0 +1,10 @@
+import type { BadgeJson, Measurement } from './types.js';
+export declare const BADGE_LABEL = "context cost";
+export declare const BADGE_CACHE_SECONDS = 3600;
+export declare function formatTokens(totalTokens: number): string;
+/**
+ * Measurement -> strict shields.io endpoint JSON. Non-measured statuses render
+ * grey "unknown" (used only in sweep snapshots; maintainer CI publishes only
+ * on success, so their badges degrade to last-known-good instead).
+ */
+export declare function toBadge(m: Measurement): BadgeJson;

package/dist/core/badge.js

@@ -0,0 +1,21 @@
+import { bandColor, UNKNOWN_COLOR } from './bands.js';
+export const BADGE_LABEL = 'context cost';
+export const BADGE_CACHE_SECONDS = 3600;
+export function formatTokens(totalTokens) {
+    return `${totalTokens.toLocaleString('en-US')} tokens`;
+}
+/**
+ * Measurement -> strict shields.io endpoint JSON. Non-measured statuses render
+ * grey "unknown" (used only in sweep snapshots; maintainer CI publishes only
+ * on success, so their badges degrade to last-known-good instead).
+ */
+export function toBadge(m) {
+    const measurable = (m.status === 'measured' || m.status === 'dynamic') && m.totalTokens !== null;
+    return {
+        schemaVersion: 1,
+        label: BADGE_LABEL,
+        message: measurable ? formatTokens(m.totalTokens) : 'unknown',
+        color: measurable ? bandColor(m.totalTokens) : UNKNOWN_COLOR,
+        cacheSeconds: BADGE_CACHE_SECONDS,
+    };
+}

package/dist/core/bands.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Provisional color bands (v0) — to be frozen against observed sweep
+ * percentiles before launch (see METHODOLOGY.md). Any change bumps the
+ * methodology version.
+ */
+export declare function bandColor(totalTokens: number): string;
+export declare const UNKNOWN_COLOR = "lightgrey";

package/dist/core/bands.js

@@ -0,0 +1,17 @@
+/**
+ * Provisional color bands (v0) — to be frozen against observed sweep
+ * percentiles before launch (see METHODOLOGY.md). Any change bumps the
+ * methodology version.
+ */
+export function bandColor(totalTokens) {
+    if (totalTokens < 1_000)
+        return 'brightgreen';
+    if (totalTokens < 5_000)
+        return 'green';
+    if (totalTokens < 15_000)
+        return 'yellow';
+    if (totalTokens < 30_000)
+        return 'orange';
+    return 'red';
+}
+export const UNKNOWN_COLOR = 'lightgrey';

package/dist/core/canonical.d.ts

@@ -0,0 +1,30 @@
+import type { Measurement, MeasurementStatus } from './types.js';
+export declare const METHODOLOGY_VERSION = "1.0";
+/**
+ * Canonical form: JSON.stringify over the PARSED tools array (no whitespace,
+ * first-occurrence key order, JSON.parse semantics — duplicate keys last-wins,
+ * numbers/strings re-serialized per ECMA-404). This is deliberately defined on
+ * the parsed value, not raw wire bytes: every JSON implementation reproduces it
+ * from the published capture, which is what makes numbers re-derivable anywhere.
+ */
+export declare function canonicalString(tools: unknown[]): string;
+export declare function sha256Hex(text: string): string;
+export declare function countTokens(text: string): number;
+/**
+ * totalTokens is authoritative (tokens of the whole canonical array — array
+ * punctuation included); per-tool numbers are diagnostic and won't sum to it.
+ */
+export declare function measureTools(tools: unknown[], meta: {
+    serverName: string;
+    serverVersion?: string;
+    launchCommand?: string;
+    envVarNames?: string[];
+    measuredAt?: string;
+}): Measurement;
+export declare function failedMeasurement(status: Exclude<MeasurementStatus, 'measured' | 'dynamic'>, meta: {
+    serverName: string;
+    serverVersion?: string;
+    launchCommand?: string;
+    notes?: string;
+    measuredAt?: string;
+}): Measurement;

package/dist/core/canonical.js

@@ -0,0 +1,72 @@
+import { createHash } from 'node:crypto';
+import { getEncoding } from 'js-tiktoken';
+export const METHODOLOGY_VERSION = '1.0';
+const enc = getEncoding('o200k_base');
+/**
+ * Canonical form: JSON.stringify over the PARSED tools array (no whitespace,
+ * first-occurrence key order, JSON.parse semantics — duplicate keys last-wins,
+ * numbers/strings re-serialized per ECMA-404). This is deliberately defined on
+ * the parsed value, not raw wire bytes: every JSON implementation reproduces it
+ * from the published capture, which is what makes numbers re-derivable anywhere.
+ */
+export function canonicalString(tools) {
+    return JSON.stringify(tools);
+}
+export function sha256Hex(text) {
+    return createHash('sha256').update(text, 'utf8').digest('hex');
+}
+export function countTokens(text) {
+    // disallowedSpecial=[] — special-token strings (e.g. "<|endoftext|>") in tool
+    // descriptions are counted as ordinary text instead of throwing.
+    return enc.encode(text, undefined, []).length;
+}
+/**
+ * totalTokens is authoritative (tokens of the whole canonical array — array
+ * punctuation included); per-tool numbers are diagnostic and won't sum to it.
+ */
+export function measureTools(tools, meta) {
+    const canonical = canonicalString(tools);
+    const perTool = tools.map((t) => {
+        const tool = t;
+        return {
+            name: tool.name ?? '(unnamed)',
+            tokens: countTokens(JSON.stringify(t)),
+            descriptionTokens: tool.description ? countTokens(tool.description) : 0,
+            inputSchemaTokens: tool.inputSchema ? countTokens(JSON.stringify(tool.inputSchema)) : 0,
+        };
+    });
+    return {
+        methodologyVersion: METHODOLOGY_VERSION,
+        provider: 'tiktoken',
+        encoding: 'o200k_base',
+        status: 'measured',
+        totalTokens: countTokens(canonical),
+        toolCount: tools.length,
+        tools: perTool,
+        canonicalSha256: sha256Hex(canonical),
+        rawToolsCapture: tools,
+        measuredAt: meta.measuredAt ?? new Date().toISOString(),
+        serverName: meta.serverName,
+        serverVersion: meta.serverVersion,
+        launchCommand: meta.launchCommand,
+        envVarNames: meta.envVarNames,
+    };
+}
+export function failedMeasurement(status, meta) {
+    return {
+        methodologyVersion: METHODOLOGY_VERSION,
+        provider: 'tiktoken',
+        encoding: 'o200k_base',
+        status,
+        totalTokens: null,
+        toolCount: null,
+        tools: [],
+        canonicalSha256: null,
+        rawToolsCapture: null,
+        measuredAt: meta.measuredAt ?? new Date().toISOString(),
+        serverName: meta.serverName,
+        serverVersion: meta.serverVersion,
+        launchCommand: meta.launchCommand,
+        notes: meta.notes,
+    };
+}

package/dist/core/index.d.ts

@@ -0,0 +1,5 @@
+export * from './types.js';
+export * from './canonical.js';
+export * from './bands.js';
+export * from './badge.js';
+export * from './snippet.js';

package/dist/core/index.js

@@ -0,0 +1,5 @@
+export * from './types.js';
+export * from './canonical.js';
+export * from './bands.js';
+export * from './badge.js';
+export * from './snippet.js';

package/dist/core/snippet.d.ts

@@ -0,0 +1,6 @@
+/**
+ * README badge snippet. Shields' `link` param does not work inside GitHub
+ * READMEs, so the image is wrapped in a markdown link to the methodology page
+ * — that link is the whole point of the badge.
+ */
+export declare function readmeSnippet(badgeJsonRawUrl: string, methodologyUrl: string): string;

package/dist/core/snippet.js

@@ -0,0 +1,9 @@
+/**
+ * README badge snippet. Shields' `link` param does not work inside GitHub
+ * READMEs, so the image is wrapped in a markdown link to the methodology page
+ * — that link is the whole point of the badge.
+ */
+export function readmeSnippet(badgeJsonRawUrl, methodologyUrl) {
+    const img = `https://img.shields.io/endpoint?url=${encodeURIComponent(badgeJsonRawUrl)}`;
+    return `[![context cost](${img})](${methodologyUrl})`;
+}

package/dist/core/types.d.ts

@@ -0,0 +1,50 @@
+/** Status taxonomy — every swept server gets exactly one; no silent drops. */
+export type MeasurementStatus = 'measured' | 'auth-required' | 'startup-failure' | 'timeout' | 'dynamic' | 'remote-auth-wall';
+export interface ToolMeasurement {
+    name: string;
+    /** Tokens of the whole tool object, canonically serialized. */
+    tokens: number;
+    descriptionTokens: number;
+    inputSchemaTokens: number;
+}
+/**
+ * The full reproducibility record published next to every badge.
+ * Re-tokenizing `rawToolsCapture` must reproduce `totalTokens` exactly.
+ */
+export interface Measurement {
+    methodologyVersion: string;
+    provider: 'tiktoken';
+    encoding: 'o200k_base';
+    status: MeasurementStatus;
+    totalTokens: number | null;
+    toolCount: number | null;
+    tools: ToolMeasurement[];
+    /** SHA-256 hex of the canonical bytes (see canonical.ts). */
+    canonicalSha256: string | null;
+    /** The tools array exactly as returned by tools/list, all pages concatenated. */
+    rawToolsCapture: unknown[] | null;
+    measuredAt: string;
+    serverName: string;
+    serverVersion?: string;
+    /** Exact launch command; env var NAMES only, never values. */
+    launchCommand?: string;
+    envVarNames?: string[];
+    /** How the server was isolated during measurement (docker image, network). */
+    isolation?: {
+        docker: boolean;
+        image?: string;
+        network?: string;
+        note?: string;
+    };
+    /** Request timeout in force during this measurement. */
+    timeoutMs?: number;
+    notes?: string;
+}
+/** Strict shields.io endpoint-badge schema — nothing extra. */
+export interface BadgeJson {
+    schemaVersion: 1;
+    label: string;
+    message: string;
+    color: string;
+    cacheSeconds: number;
+}

package/dist/core/types.js

@@ -0,0 +1 @@
+export {};

package/dist/sweep/client.d.ts

@@ -0,0 +1,39 @@
+export interface WireCapture {
+    serverInfo?: {
+        name?: string;
+        version?: string;
+    };
+    protocolVersion?: string;
+    tools: unknown[];
+    /** true when a second tools/list returned a different tool set */
+    stderrTail: string;
+}
+export declare class McpStdioClient {
+    private child;
+    private buffer;
+    private nextId;
+    private pending;
+    private stderrChunks;
+    private exited;
+    constructor(command: string, args: string[], env: Record<string, string | undefined>);
+    private onData;
+    private send;
+    private deadReason;
+    request(method: string, params: unknown, timeoutMs: number): Promise<any>;
+    notify(method: string, params?: unknown): void;
+    get stderrTail(): string;
+    close(): Promise<void>;
+}
+/**
+ * Launch a stdio MCP server, run initialize + paginated tools/list, capture the
+ * wire-order tools array. Caller owns error handling/status mapping.
+ */
+export declare function captureTools(spec: string | {
+    command: string;
+    argv: string[];
+}, opts?: {
+    timeoutMs?: number;
+    env?: Record<string, string>;
+}): Promise<WireCapture>;
+/** Shell-free command splitting: honors single/double quotes, no expansion. */
+export declare function splitCommand(line: string): string[];