mcp-context-cost 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +93 -10
- package/dist/audit/audit.d.ts +66 -0
- package/dist/audit/audit.js +201 -0
- package/dist/audit/config.d.ts +53 -0
- package/dist/audit/config.js +179 -0
- package/dist/audit/run.d.ts +19 -0
- package/dist/audit/run.js +59 -0
- package/dist/cli.js +87 -7
- package/dist/core/bands.d.ts +5 -0
- package/dist/core/bands.js +8 -0
- package/dist/core/canonical.js +3 -1
- package/dist/core/divergence.d.ts +71 -0
- package/dist/core/divergence.js +103 -0
- package/dist/core/index.d.ts +1 -0
- package/dist/core/index.js +1 -0
- package/dist/sweep/dashboard.js +17 -14
- package/dist/sweep/history.d.ts +30 -0
- package/dist/sweep/history.js +120 -0
- package/dist/sweep/regen.js +8 -1
- package/dist/sweep/report.d.ts +5 -0
- package/dist/sweep/report.js +39 -6
- package/dist/sweep/run.d.ts +11 -0
- package/dist/sweep/run.js +17 -4
- package/dist/sweep/server-pages.d.ts +17 -0
- package/dist/sweep/server-pages.js +228 -0
- package/dist/sweep/sweep-all.js +4 -1
- package/package.json +10 -3
package/README.md
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
# mcp-context-cost
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
[](https://www.npmjs.com/package/mcp-context-cost)
|
|
4
|
+
[](https://github.com/athakur3/mcp-context-cost/actions/workflows/ci.yml)
|
|
5
|
+
|
|
6
|
+
**Reproducible context-cost measurement for MCP servers — for the one you publish, and for
|
|
7
|
+
the stack you actually run.**
|
|
4
8
|
|
|
5
9
|
Every MCP server you wire into an agent injects its tool schemas into the model's context
|
|
6
10
|
before any work happens. That cost is invisible — and it varies by **1,700×** across
|
|
@@ -16,7 +20,9 @@ popular servers:
|
|
|
16
20
|
| markitdown | 64 | 1 |
|
|
17
21
|
|
|
18
22
|
*(57 of 82 popular servers measured, 2026-08-16 sweep — full table in
|
|
19
|
-
[results/leaderboard.md](results/leaderboard.md); every failure is listed with its reason.
|
|
23
|
+
[results/leaderboard.md](results/leaderboard.md); every failure is listed with its reason.
|
|
24
|
+
Each measured server also has a [detail page](https://athakur3.github.io/mcp-context-cost/servers/)
|
|
25
|
+
showing which tools its tokens are in.)*
|
|
20
26
|
|
|
21
27
|
This project makes that cost **legible and disputable**:
|
|
22
28
|
|
|
@@ -24,6 +30,68 @@ This project makes that cost **legible and disputable**:
|
|
|
24
30
|
[context cost | 12,430 tokens] ← shields.io badge, linked to the methodology
|
|
25
31
|
```
|
|
26
32
|
|
|
33
|
+
## What does *your* setup cost?
|
|
34
|
+
|
|
35
|
+
The leaderboard measures one server at a time. You don't run one server — you run a stack.
|
|
36
|
+
Point `audit` at your own MCP config and it measures every server you actually have installed:
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
npx -y mcp-context-cost audit
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
```
|
|
43
|
+
claude-desktop ~/Library/Application Support/Claude/claude_desktop_config.json
|
|
44
|
+
server tools tokens share
|
|
45
|
+
filesystem 14 2,823 35.7%
|
|
46
|
+
memory 9 2,378 30.1%
|
|
47
|
+
everything 13 1,708 21.6%
|
|
48
|
+
sequential-thinking 1 992 12.6%
|
|
49
|
+
────────────────────────────────────────────
|
|
50
|
+
total 37 7,901
|
|
51
|
+
|
|
52
|
+
Every request in this client carries 7,901 tokens of tool schemas — 4.0% of a
|
|
53
|
+
200,000-token context window, before you type anything.
|
|
54
|
+
|
|
55
|
+
heaviest tools
|
|
56
|
+
sequential-thinking · sequentialthinking 990
|
|
57
|
+
memory · search_nodes 323
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
It finds configs for Claude Desktop, Claude Code (`~/.claude.json`, `.mcp.json`), Cursor,
|
|
61
|
+
VS Code (`.vscode/mcp.json`), and Windsurf — or pass `--config <path>`. Servers are measured
|
|
62
|
+
by the same path as the published leaderboard (dual `tools/list` capture, `o200k_base` over
|
|
63
|
+
canonical JSON), so a server in both places gets the same number. Nothing is written to your
|
|
64
|
+
project, and env var **values** are never read into the output — only their names.
|
|
65
|
+
|
|
66
|
+
Totals are reported per config file, never merged: a context window belongs to one client
|
|
67
|
+
session, so summing Cursor's servers into Claude Desktop's total would describe a session
|
|
68
|
+
nobody runs.
|
|
69
|
+
|
|
70
|
+
**In CI**, make it a gate — the bundlesize move for agents:
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
npx -y mcp-context-cost audit --config .mcp.json --budget 20000
|
|
74
|
+
# exits 1 when the stack exceeds the budget, so a PR adding a 25K-token server fails
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
Flags: `--json` (full report on stdout, progress on stderr), `--budget N`, `--context N`
|
|
78
|
+
(default 200,000), `--timeout ms`, `--concurrency N`, `--docker`.
|
|
79
|
+
|
|
80
|
+
## What it costs on Claude
|
|
81
|
+
|
|
82
|
+
The badge counts every byte a server returns. An Anthropic request carries only `name`,
|
|
83
|
+
`description`, and `input_schema` — and counts them with a denser tokenizer. Both effects are
|
|
84
|
+
now measured against a pinned model and published beside the badge, and they do not cancel:
|
|
85
|
+
|
|
86
|
+
| server | badge (o200k) | Claude (`claude-opus-5`) | |
|
|
87
|
+
|---|---:|---:|---|
|
|
88
|
+
| github | 54,422 | **18,406** | 81% of the capture is `annotations`/`outputSchema` metadata Claude never sees |
|
|
89
|
+
| notion | 17,500 | **33,560** | almost no metadata to drop, so the tokenizer difference dominates |
|
|
90
|
+
|
|
91
|
+
So the heaviest server on the badge is not the heaviest server on Claude. Per-server
|
|
92
|
+
breakdowns are on each [detail page](https://athakur3.github.io/mcp-context-cost/servers/);
|
|
93
|
+
the method is [Claude divergence](docs/METHODOLOGY.md#claude-divergence).
|
|
94
|
+
|
|
27
95
|
## Why trust the number?
|
|
28
96
|
|
|
29
97
|
Every badge is backed by a `measurement.json` containing the raw `tools/list` capture, the
|
|
@@ -31,10 +99,17 @@ SHA-256 of its canonical bytes, the pinned tokenizer (`o200k_base`), and the exa
|
|
|
31
99
|
command. Disputes reduce to a byte-level diff:
|
|
32
100
|
|
|
33
101
|
```bash
|
|
34
|
-
npx
|
|
102
|
+
npx -y mcp-context-cost verify results/github/measurement.json
|
|
35
103
|
# OK github-mcp-server: 54422 tokens (o200k_base, methodology 1.0) — capture, hash, and count all agree
|
|
104
|
+
|
|
105
|
+
# or point it at a published measurement.json directly, no clone required
|
|
106
|
+
npx -y mcp-context-cost verify --remote https://raw.githubusercontent.com/athakur3/mcp-context-cost/main/results/github/measurement.json
|
|
36
107
|
```
|
|
37
108
|
|
|
109
|
+
Add `--json` for scripting (`{ ok, serverName, rederivedTokens, rederivedSha, problems, badge }`
|
|
110
|
+
on stdout, `badge` omitted on failure). Exit codes: `0` ok, `1` verification/measurement
|
|
111
|
+
failed, `2` usage error.
|
|
112
|
+
|
|
38
113
|
Full definition: [docs/METHODOLOGY.md](docs/METHODOLOGY.md) — what is counted, what the
|
|
39
114
|
number is *not*, config policy, failure taxonomy, frozen color bands, known divergences.
|
|
40
115
|
|
|
@@ -44,11 +119,14 @@ number is *not*, config policy, failure taxonomy, frozen color bands, known dive
|
|
|
44
119
|
|---|---|
|
|
45
120
|
| `src/core/` | the measurement spec, executable — canonical form, tokenizer, bands, badge JSON |
|
|
46
121
|
| `src/sweep/` | raw-wire MCP stdio client + Dockerized batch sweep + leaderboard/dashboard generators |
|
|
47
|
-
| `src/
|
|
122
|
+
| `src/audit/` | client-config discovery (5 clients, JSONC-tolerant) + the per-stack report |
|
|
123
|
+
| `src/cli.ts` | `audit` (measure your own stack), `verify` (re-derive any published number), `measure` |
|
|
48
124
|
| `spec/fixtures/` | golden vectors shared by the TypeScript and bash implementations |
|
|
125
|
+
| `tools/` | the one script that calls a network API (Claude divergence); kept out of the package so the library stays offline |
|
|
49
126
|
| `upstream/` | staged contribution to [sd2k/mcp-tokens-action](https://github.com/sd2k/mcp-tokens-action): `badge.sh` + action patch + tests |
|
|
50
127
|
| `servers.yaml` | 82 curated candidates with live install metrics and provenance |
|
|
51
|
-
| `results/` · `badges/` | measurements, leaderboard, shields endpoint JSONs |
|
|
128
|
+
| `results/` · `badges/` | measurements, leaderboard, history series, shields endpoint JSONs |
|
|
129
|
+
| `docs/` | methodology, dashboard, and a generated page per measured server |
|
|
52
130
|
|
|
53
131
|
## Measure your own server
|
|
54
132
|
|
|
@@ -61,16 +139,20 @@ cat badges/my-server.json # strict shields.io endpoint JSON
|
|
|
61
139
|
Then in your README:
|
|
62
140
|
|
|
63
141
|
```markdown
|
|
64
|
-
[](<
|
|
142
|
+
[](<link target>)
|
|
65
143
|
```
|
|
66
144
|
|
|
145
|
+
Point the link at the measurement behind the number — for servers in this sweep that is
|
|
146
|
+
`https://athakur3.github.io/mcp-context-cost/servers/<name>.html`; otherwise the
|
|
147
|
+
methodology page. A badge nobody can audit is decoration.
|
|
148
|
+
|
|
67
149
|
Or self-serve from CI via the (staged) mcp-tokens-action badge inputs — see
|
|
68
150
|
[upstream/action-patch.md](upstream/action-patch.md).
|
|
69
151
|
|
|
70
152
|
## Development
|
|
71
153
|
|
|
72
154
|
```bash
|
|
73
|
-
npm test #
|
|
155
|
+
npm test # 105 TS tests incl. golden fixtures + dispute drills
|
|
74
156
|
npx tsc --noEmit # typecheck
|
|
75
157
|
./upstream/tests/badge-test.sh # 21 bash tests — byte-identical to the TS reference
|
|
76
158
|
npm run sweep:all -- --docker # full curated sweep (Docker isolation)
|
|
@@ -83,8 +165,9 @@ color bands are frozen against the observed distribution of the first full sweep
|
|
|
83
165
|
|
|
84
166
|
## Status
|
|
85
167
|
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
168
|
+
Active. The leaderboard refreshes weekly; badge PRs are open across the ecosystem and
|
|
169
|
+
[sd2k/mcp-tokens-action#5](https://github.com/sd2k/mcp-tokens-action/pull/5) proposes the
|
|
170
|
+
self-serve badge path upstream. See [ROADMAP.md](ROADMAP.md) for what's next —
|
|
171
|
+
contributions welcome, especially new `servers.yaml` entries.
|
|
89
172
|
|
|
90
173
|
MIT © 2026
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import type { Measurement, MeasurementStatus, ToolMeasurement } from '../core/types.js';
|
|
2
|
+
import type { ConfiguredServer, LoadedConfig } from './config.js';
|
|
3
|
+
export declare const DEFAULT_CONTEXT_WINDOW = 200000;
|
|
4
|
+
export type AuditStatus = MeasurementStatus | 'remote-not-measurable';
|
|
5
|
+
export interface AuditServerResult {
|
|
6
|
+
name: string;
|
|
7
|
+
transport: 'stdio' | 'remote';
|
|
8
|
+
status: AuditStatus;
|
|
9
|
+
tokens: number | null;
|
|
10
|
+
toolCount: number | null;
|
|
11
|
+
/** Share of this config's measured total, 0–1. */
|
|
12
|
+
share: number | null;
|
|
13
|
+
command?: string;
|
|
14
|
+
url?: string;
|
|
15
|
+
/** Names only — a server's env values never enter a report. */
|
|
16
|
+
envVarNames: string[];
|
|
17
|
+
canonicalSha256?: string | null;
|
|
18
|
+
notes?: string;
|
|
19
|
+
}
|
|
20
|
+
export interface HeaviestTool {
|
|
21
|
+
server: string;
|
|
22
|
+
tool: string;
|
|
23
|
+
tokens: number;
|
|
24
|
+
}
|
|
25
|
+
export interface AuditConfigResult {
|
|
26
|
+
client: string;
|
|
27
|
+
source: string;
|
|
28
|
+
totalTokens: number;
|
|
29
|
+
toolCount: number;
|
|
30
|
+
serverCount: number;
|
|
31
|
+
contextShare: number;
|
|
32
|
+
servers: AuditServerResult[];
|
|
33
|
+
skipped: AuditServerResult[];
|
|
34
|
+
heaviestTools: HeaviestTool[];
|
|
35
|
+
}
|
|
36
|
+
export interface AuditReport {
|
|
37
|
+
methodologyVersion: string;
|
|
38
|
+
encoding: 'o200k_base';
|
|
39
|
+
generatedAt: string;
|
|
40
|
+
contextWindow: number;
|
|
41
|
+
configs: AuditConfigResult[];
|
|
42
|
+
budget?: {
|
|
43
|
+
limit: number;
|
|
44
|
+
worstTotal: number;
|
|
45
|
+
worstSource: string;
|
|
46
|
+
over: boolean;
|
|
47
|
+
};
|
|
48
|
+
problems: string[];
|
|
49
|
+
}
|
|
50
|
+
/** Cache key for measurement reuse: the exact argv two configs would spawn. */
|
|
51
|
+
export declare function serverKey(s: ConfiguredServer): string;
|
|
52
|
+
/**
|
|
53
|
+
* Assemble the report from configs + measurements. Pure: `runAudit` does the
|
|
54
|
+
* spawning, this does the arithmetic, so totals and shares are testable without
|
|
55
|
+
* launching a single server.
|
|
56
|
+
*/
|
|
57
|
+
export declare function buildReport(configs: LoadedConfig[], measured: Map<string, Measurement>, opts?: {
|
|
58
|
+
contextWindow?: number;
|
|
59
|
+
budget?: number;
|
|
60
|
+
generatedAt?: string;
|
|
61
|
+
}): AuditReport;
|
|
62
|
+
/** Human output. JSON output is the report object itself. */
|
|
63
|
+
export declare function formatReport(report: AuditReport): string;
|
|
64
|
+
/** Top-level tool list across every config — used by nothing yet, handy for --json consumers. */
|
|
65
|
+
export declare function allHeaviestTools(report: AuditReport, limit?: number): HeaviestTool[];
|
|
66
|
+
export type { ToolMeasurement };
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `audit` — measure the MCP servers a person actually has installed.
|
|
3
|
+
*
|
|
4
|
+
* The leaderboard answers "what does server X cost?". This answers the question
|
|
5
|
+
* the person paying the bill has: "what do MY servers cost, together, before I
|
|
6
|
+
* type anything?" Same measurement path as the sweep (dual tools/list capture,
|
|
7
|
+
* o200k_base over canonical JSON, full status taxonomy), pointed at a client
|
|
8
|
+
* config instead of servers.yaml.
|
|
9
|
+
*
|
|
10
|
+
* Totals are per config file, never merged across clients: a context window
|
|
11
|
+
* belongs to one client session, so summing Cursor's servers into Claude
|
|
12
|
+
* Desktop's total would describe a session nobody is running. Identical launch
|
|
13
|
+
* commands shared by two configs are still only measured once.
|
|
14
|
+
*/
|
|
15
|
+
import { METHODOLOGY_VERSION } from '../core/canonical.js';
|
|
16
|
+
export const DEFAULT_CONTEXT_WINDOW = 200_000;
|
|
17
|
+
/** Cache key for measurement reuse: the exact argv two configs would spawn. */
|
|
18
|
+
export function serverKey(s) {
|
|
19
|
+
return JSON.stringify(s.argv ?? [s.url ?? s.name]);
|
|
20
|
+
}
|
|
21
|
+
function measuredOk(m) {
|
|
22
|
+
return (m.status === 'measured' || m.status === 'dynamic') && typeof m.totalTokens === 'number';
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Assemble the report from configs + measurements. Pure: `runAudit` does the
|
|
26
|
+
* spawning, this does the arithmetic, so totals and shares are testable without
|
|
27
|
+
* launching a single server.
|
|
28
|
+
*/
|
|
29
|
+
export function buildReport(configs, measured, opts = {}) {
|
|
30
|
+
const contextWindow = opts.contextWindow ?? DEFAULT_CONTEXT_WINDOW;
|
|
31
|
+
const problems = [];
|
|
32
|
+
const results = [];
|
|
33
|
+
for (const cfg of configs) {
|
|
34
|
+
if (cfg.error) {
|
|
35
|
+
problems.push(`${cfg.source}: ${cfg.error}`);
|
|
36
|
+
continue;
|
|
37
|
+
}
|
|
38
|
+
const ok = [];
|
|
39
|
+
const skipped = [];
|
|
40
|
+
const tools = [];
|
|
41
|
+
for (const s of cfg.servers) {
|
|
42
|
+
const base = {
|
|
43
|
+
name: s.name,
|
|
44
|
+
transport: s.transport,
|
|
45
|
+
command: s.command,
|
|
46
|
+
url: s.url,
|
|
47
|
+
envVarNames: s.envVarNames,
|
|
48
|
+
};
|
|
49
|
+
if (s.transport === 'remote') {
|
|
50
|
+
skipped.push({
|
|
51
|
+
...base,
|
|
52
|
+
status: 'remote-not-measurable',
|
|
53
|
+
tokens: null,
|
|
54
|
+
toolCount: null,
|
|
55
|
+
share: null,
|
|
56
|
+
notes: `remote endpoint (${s.url ?? 'url'}) — stdio measurement does not apply`,
|
|
57
|
+
});
|
|
58
|
+
continue;
|
|
59
|
+
}
|
|
60
|
+
const m = measured.get(serverKey(s));
|
|
61
|
+
if (!m) {
|
|
62
|
+
skipped.push({ ...base, status: 'startup-failure', tokens: null, toolCount: null, share: null, notes: 'not measured' });
|
|
63
|
+
continue;
|
|
64
|
+
}
|
|
65
|
+
if (!measuredOk(m)) {
|
|
66
|
+
skipped.push({
|
|
67
|
+
...base,
|
|
68
|
+
status: m.status,
|
|
69
|
+
tokens: null,
|
|
70
|
+
toolCount: null,
|
|
71
|
+
share: null,
|
|
72
|
+
notes: m.notes?.split('\n')[0]?.slice(0, 200),
|
|
73
|
+
});
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
ok.push({
|
|
77
|
+
...base,
|
|
78
|
+
status: m.status,
|
|
79
|
+
tokens: m.totalTokens,
|
|
80
|
+
toolCount: m.toolCount,
|
|
81
|
+
share: null, // filled once the total is known
|
|
82
|
+
canonicalSha256: m.canonicalSha256,
|
|
83
|
+
notes: m.status === 'dynamic' ? m.notes : undefined,
|
|
84
|
+
});
|
|
85
|
+
for (const t of m.tools)
|
|
86
|
+
tools.push({ server: s.name, tool: t.name, tokens: t.tokens });
|
|
87
|
+
}
|
|
88
|
+
const totalTokens = ok.reduce((a, s) => a + (s.tokens ?? 0), 0);
|
|
89
|
+
const toolCount = ok.reduce((a, s) => a + (s.toolCount ?? 0), 0);
|
|
90
|
+
ok.sort((a, b) => (b.tokens ?? 0) - (a.tokens ?? 0));
|
|
91
|
+
for (const s of ok)
|
|
92
|
+
s.share = totalTokens > 0 ? (s.tokens ?? 0) / totalTokens : 0;
|
|
93
|
+
tools.sort((a, b) => b.tokens - a.tokens);
|
|
94
|
+
results.push({
|
|
95
|
+
client: cfg.client,
|
|
96
|
+
source: cfg.source,
|
|
97
|
+
totalTokens,
|
|
98
|
+
toolCount,
|
|
99
|
+
serverCount: ok.length,
|
|
100
|
+
// Deliberately no band: the color bands were frozen against the per-server
|
|
101
|
+
// distribution (n=57). A config total is a different population, so calling
|
|
102
|
+
// a 7,901-token *stack* "moderate" would borrow a scale that doesn't mean
|
|
103
|
+
// that here. Share of the context window is the honest framing.
|
|
104
|
+
contextShare: totalTokens / contextWindow,
|
|
105
|
+
servers: ok,
|
|
106
|
+
skipped,
|
|
107
|
+
heaviestTools: tools.slice(0, 5),
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
results.sort((a, b) => b.totalTokens - a.totalTokens);
|
|
111
|
+
const report = {
|
|
112
|
+
methodologyVersion: METHODOLOGY_VERSION,
|
|
113
|
+
encoding: 'o200k_base',
|
|
114
|
+
generatedAt: opts.generatedAt ?? new Date().toISOString(),
|
|
115
|
+
contextWindow,
|
|
116
|
+
configs: results,
|
|
117
|
+
problems,
|
|
118
|
+
};
|
|
119
|
+
if (typeof opts.budget === 'number') {
|
|
120
|
+
// The worst config is the gate: passing because your *lightest* client fits
|
|
121
|
+
// would be a green check on a session you don't run.
|
|
122
|
+
const worst = results[0];
|
|
123
|
+
report.budget = {
|
|
124
|
+
limit: opts.budget,
|
|
125
|
+
worstTotal: worst?.totalTokens ?? 0,
|
|
126
|
+
worstSource: worst?.source ?? '(none)',
|
|
127
|
+
over: (worst?.totalTokens ?? 0) > opts.budget,
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
return report;
|
|
131
|
+
}
|
|
132
|
+
const n = (x) => x.toLocaleString('en-US');
|
|
133
|
+
const pct = (x) => `${(x * 100).toFixed(1)}%`;
|
|
134
|
+
/** Human output. JSON output is the report object itself. */
|
|
135
|
+
export function formatReport(report) {
|
|
136
|
+
const lines = [];
|
|
137
|
+
lines.push(`mcp-context-cost audit · methodology ${report.methodologyVersion} · ${report.encoding} · context window ${n(report.contextWindow)}`);
|
|
138
|
+
for (const cfg of report.configs) {
|
|
139
|
+
lines.push('');
|
|
140
|
+
lines.push(`${cfg.client} ${cfg.source}`);
|
|
141
|
+
const rows = cfg.servers.map((s) => ({
|
|
142
|
+
name: s.name,
|
|
143
|
+
tools: s.toolCount === null ? '—' : String(s.toolCount),
|
|
144
|
+
tokens: s.tokens === null ? '—' : n(s.tokens),
|
|
145
|
+
share: s.share === null ? '—' : pct(s.share),
|
|
146
|
+
}));
|
|
147
|
+
const w = {
|
|
148
|
+
name: Math.max(6, ...rows.map((r) => r.name.length), 'total'.length),
|
|
149
|
+
tools: Math.max(5, ...rows.map((r) => r.tools.length)),
|
|
150
|
+
tokens: Math.max(6, ...rows.map((r) => r.tokens.length), n(cfg.totalTokens).length),
|
|
151
|
+
};
|
|
152
|
+
const line = (name, tools, tokens, share) => ` ${name.padEnd(w.name)} ${tools.padStart(w.tools)} ${tokens.padStart(w.tokens)} ${share.padStart(6)}`;
|
|
153
|
+
lines.push(line('server', 'tools', 'tokens', 'share'));
|
|
154
|
+
for (const r of rows)
|
|
155
|
+
lines.push(line(r.name, r.tools, r.tokens, r.share));
|
|
156
|
+
lines.push(` ${'─'.repeat(w.name + w.tools + w.tokens + 14)}`);
|
|
157
|
+
lines.push(line('total', String(cfg.toolCount), n(cfg.totalTokens), ''));
|
|
158
|
+
lines.push('');
|
|
159
|
+
lines.push(` Every request in this client carries ${n(cfg.totalTokens)} tokens of tool schemas — ` +
|
|
160
|
+
`${pct(cfg.contextShare)} of a ${n(report.contextWindow)}-token context window, before you type anything.`);
|
|
161
|
+
if (cfg.heaviestTools.length) {
|
|
162
|
+
lines.push('');
|
|
163
|
+
lines.push(' heaviest tools');
|
|
164
|
+
const tw = Math.max(...cfg.heaviestTools.map((t) => `${t.server} · ${t.tool}`.length));
|
|
165
|
+
for (const t of cfg.heaviestTools) {
|
|
166
|
+
lines.push(` ${`${t.server} · ${t.tool}`.padEnd(tw)} ${n(t.tokens).padStart(7)}`);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
if (cfg.skipped.length) {
|
|
170
|
+
lines.push('');
|
|
171
|
+
lines.push(' not measured');
|
|
172
|
+
const sw = Math.max(...cfg.skipped.map((s) => s.name.length));
|
|
173
|
+
for (const s of cfg.skipped) {
|
|
174
|
+
lines.push(` ${s.name.padEnd(sw)} ${s.status}${s.notes ? ` — ${s.notes}` : ''}`);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
if (report.problems.length) {
|
|
179
|
+
lines.push('');
|
|
180
|
+
lines.push('problems');
|
|
181
|
+
for (const p of report.problems)
|
|
182
|
+
lines.push(` ${p}`);
|
|
183
|
+
}
|
|
184
|
+
if (report.budget) {
|
|
185
|
+
lines.push('');
|
|
186
|
+
lines.push(report.budget.over
|
|
187
|
+
? `BUDGET FAIL: ${n(report.budget.worstTotal)} > ${n(report.budget.limit)} (${report.budget.worstSource})`
|
|
188
|
+
: `budget ok: ${n(report.budget.worstTotal)} ≤ ${n(report.budget.limit)}`);
|
|
189
|
+
}
|
|
190
|
+
lines.push('');
|
|
191
|
+
lines.push('These are wire tokens — what the server puts on the wire, counted with o200k_base. What your model is billed');
|
|
192
|
+
lines.push('differs per provider: measured ratios run 0.34×–1.92× on Anthropic requests. See docs/METHODOLOGY.md §claude-divergence.');
|
|
193
|
+
return lines.map((l) => l.replace(/\s+$/, '')).join('\n');
|
|
194
|
+
}
|
|
195
|
+
/** Top-level tool list across every config — used by nothing yet, handy for --json consumers. */
|
|
196
|
+
export function allHeaviestTools(report, limit = 10) {
|
|
197
|
+
return report.configs
|
|
198
|
+
.flatMap((c) => c.heaviestTools)
|
|
199
|
+
.sort((a, b) => b.tokens - a.tokens)
|
|
200
|
+
.slice(0, limit);
|
|
201
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
export interface ConfiguredServer {
|
|
2
|
+
name: string;
|
|
3
|
+
/** Which client's config this came from ('claude-desktop', 'cursor', ...). */
|
|
4
|
+
client: string;
|
|
5
|
+
/** Absolute path of the config file. */
|
|
6
|
+
source: string;
|
|
7
|
+
transport: 'stdio' | 'remote';
|
|
8
|
+
/** Display form of the launch command (argv joined) — stdio only. */
|
|
9
|
+
command?: string;
|
|
10
|
+
/** Exact argv, so paths containing spaces survive round-tripping. */
|
|
11
|
+
argv?: string[];
|
|
12
|
+
envVarNames: string[];
|
|
13
|
+
/** Values, needed to spawn the server. NEVER serialize this. */
|
|
14
|
+
env?: Record<string, string>;
|
|
15
|
+
/** Remote endpoint — recorded so the report can say why it was skipped. */
|
|
16
|
+
url?: string;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* JSON with comments and trailing commas — VS Code's mcp.json allows both, and
|
|
20
|
+
* hand-edited Claude/Cursor configs often pick up a trailing comma too. String
|
|
21
|
+
* literals are tracked so a `//` or `,}` inside a tool description survives.
|
|
22
|
+
*/
|
|
23
|
+
export declare function parseJsonc(text: string): unknown;
|
|
24
|
+
/**
|
|
25
|
+
* Pull every server out of one parsed config document. `cwd` selects the
|
|
26
|
+
* project scope in Claude Code's `~/.claude.json`, which keys per-project
|
|
27
|
+
* servers by absolute directory.
|
|
28
|
+
*/
|
|
29
|
+
export declare function extractServers(doc: unknown, meta: {
|
|
30
|
+
client: string;
|
|
31
|
+
source: string;
|
|
32
|
+
cwd?: string;
|
|
33
|
+
}): ConfiguredServer[];
|
|
34
|
+
export interface ConfigCandidate {
|
|
35
|
+
client: string;
|
|
36
|
+
path: string;
|
|
37
|
+
}
|
|
38
|
+
/** Every place a client config is known to live, whether or not it exists. */
|
|
39
|
+
export declare function configCandidates(env: {
|
|
40
|
+
home: string;
|
|
41
|
+
cwd: string;
|
|
42
|
+
platform: NodeJS.Platform;
|
|
43
|
+
appData?: string;
|
|
44
|
+
}): ConfigCandidate[];
|
|
45
|
+
export interface LoadedConfig {
|
|
46
|
+
client: string;
|
|
47
|
+
source: string;
|
|
48
|
+
servers: ConfiguredServer[];
|
|
49
|
+
/** Set when the file exists but could not be read/parsed. */
|
|
50
|
+
error?: string;
|
|
51
|
+
}
|
|
52
|
+
/** Read + parse the candidates that exist. Unreadable files are reported, not thrown. */
|
|
53
|
+
export declare function loadConfigs(candidates: ConfigCandidate[], cwd: string): LoadedConfig[];
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MCP client config discovery + parsing.
|
|
3
|
+
*
|
|
4
|
+
* The leaderboard measures servers one at a time; `audit` measures the set a
|
|
5
|
+
* person actually has installed. That set lives in a client config file, and
|
|
6
|
+
* every client spells it slightly differently:
|
|
7
|
+
*
|
|
8
|
+
* Claude Desktop / Claude Code / Cursor / Windsurf { "mcpServers": { ... } }
|
|
9
|
+
* VS Code (.vscode/mcp.json) { "servers": { ... } }
|
|
10
|
+
* Claude Code (~/.claude.json) also { "projects": { "<dir>": { "mcpServers": ... } } }
|
|
11
|
+
*
|
|
12
|
+
* Everything here is pure (paths in, servers out) so the discovery rules are
|
|
13
|
+
* testable without touching a real home directory.
|
|
14
|
+
*
|
|
15
|
+
* Env var VALUES are read (a server usually needs its key to start) but are
|
|
16
|
+
* never written to a report: report builders pick fields explicitly and only
|
|
17
|
+
* `envVarNames` is ever serialized.
|
|
18
|
+
*/
|
|
19
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
20
|
+
import { join } from 'node:path';
|
|
21
|
+
/**
|
|
22
|
+
* JSON with comments and trailing commas — VS Code's mcp.json allows both, and
|
|
23
|
+
* hand-edited Claude/Cursor configs often pick up a trailing comma too. String
|
|
24
|
+
* literals are tracked so a `//` or `,}` inside a tool description survives.
|
|
25
|
+
*/
|
|
26
|
+
export function parseJsonc(text) {
|
|
27
|
+
let out = '';
|
|
28
|
+
let inString = false;
|
|
29
|
+
let escaped = false;
|
|
30
|
+
let pendingComma = false;
|
|
31
|
+
const flush = (next) => {
|
|
32
|
+
if (!pendingComma)
|
|
33
|
+
return;
|
|
34
|
+
// A comma is only trailing if the next real character closes the container.
|
|
35
|
+
if (next !== '}' && next !== ']')
|
|
36
|
+
out += ',';
|
|
37
|
+
pendingComma = false;
|
|
38
|
+
};
|
|
39
|
+
for (let i = 0; i < text.length; i++) {
|
|
40
|
+
const c = text[i];
|
|
41
|
+
const n = text[i + 1];
|
|
42
|
+
if (inString) {
|
|
43
|
+
out += c;
|
|
44
|
+
if (escaped)
|
|
45
|
+
escaped = false;
|
|
46
|
+
else if (c === '\\')
|
|
47
|
+
escaped = true;
|
|
48
|
+
else if (c === '"')
|
|
49
|
+
inString = false;
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
52
|
+
if (c === '/' && n === '/') {
|
|
53
|
+
while (i < text.length && text[i] !== '\n')
|
|
54
|
+
i++;
|
|
55
|
+
continue;
|
|
56
|
+
}
|
|
57
|
+
if (c === '/' && n === '*') {
|
|
58
|
+
i += 2;
|
|
59
|
+
while (i < text.length && !(text[i] === '*' && text[i + 1] === '/'))
|
|
60
|
+
i++;
|
|
61
|
+
i++;
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
if (c === ',') {
|
|
65
|
+
pendingComma = true;
|
|
66
|
+
continue;
|
|
67
|
+
}
|
|
68
|
+
if (/\s/.test(c)) {
|
|
69
|
+
if (!pendingComma)
|
|
70
|
+
out += c;
|
|
71
|
+
continue;
|
|
72
|
+
}
|
|
73
|
+
flush(c);
|
|
74
|
+
out += c;
|
|
75
|
+
if (c === '"')
|
|
76
|
+
inString = true;
|
|
77
|
+
}
|
|
78
|
+
return JSON.parse(out);
|
|
79
|
+
}
|
|
80
|
+
function toServer(name, raw, client, source) {
|
|
81
|
+
if (raw.disabled === true)
|
|
82
|
+
return null;
|
|
83
|
+
const env = {};
|
|
84
|
+
for (const [k, v] of Object.entries(raw.env ?? {})) {
|
|
85
|
+
if (typeof v === 'string')
|
|
86
|
+
env[k] = v;
|
|
87
|
+
}
|
|
88
|
+
const envVarNames = Object.keys(env).sort();
|
|
89
|
+
// Remote entries carry a url (and sometimes type http/sse) instead of a command.
|
|
90
|
+
if (!raw.command && typeof raw.url === 'string') {
|
|
91
|
+
return { name, client, source, transport: 'remote', url: raw.url, envVarNames };
|
|
92
|
+
}
|
|
93
|
+
if (typeof raw.command !== 'string' || raw.command.trim() === '')
|
|
94
|
+
return null;
|
|
95
|
+
const args = Array.isArray(raw.args) ? raw.args.filter((a) => typeof a === 'string') : [];
|
|
96
|
+
const argv = [raw.command, ...args];
|
|
97
|
+
return {
|
|
98
|
+
name,
|
|
99
|
+
client,
|
|
100
|
+
source,
|
|
101
|
+
transport: 'stdio',
|
|
102
|
+
// Quote only the args that need it, so the printed command stays copy-pasteable.
|
|
103
|
+
command: argv.map((a) => (/\s/.test(a) ? JSON.stringify(a) : a)).join(' '),
|
|
104
|
+
argv,
|
|
105
|
+
envVarNames,
|
|
106
|
+
env: envVarNames.length ? env : undefined,
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Pull every server out of one parsed config document. `cwd` selects the
|
|
111
|
+
* project scope in Claude Code's `~/.claude.json`, which keys per-project
|
|
112
|
+
* servers by absolute directory.
|
|
113
|
+
*/
|
|
114
|
+
export function extractServers(doc, meta) {
|
|
115
|
+
if (!doc || typeof doc !== 'object')
|
|
116
|
+
return [];
|
|
117
|
+
const d = doc;
|
|
118
|
+
const out = [];
|
|
119
|
+
const addBlock = (block) => {
|
|
120
|
+
if (!block || typeof block !== 'object')
|
|
121
|
+
return;
|
|
122
|
+
for (const [name, raw] of Object.entries(block)) {
|
|
123
|
+
if (!raw || typeof raw !== 'object')
|
|
124
|
+
continue;
|
|
125
|
+
const s = toServer(name, raw, meta.client, meta.source);
|
|
126
|
+
if (s)
|
|
127
|
+
out.push(s);
|
|
128
|
+
}
|
|
129
|
+
};
|
|
130
|
+
addBlock(d.mcpServers);
|
|
131
|
+
addBlock(d.servers); // VS Code
|
|
132
|
+
if (meta.cwd && d.projects && typeof d.projects === 'object') {
|
|
133
|
+
const project = d.projects[meta.cwd];
|
|
134
|
+
if (project && typeof project === 'object')
|
|
135
|
+
addBlock(project.mcpServers);
|
|
136
|
+
}
|
|
137
|
+
// A name can legitimately appear in both blocks of the same file; keep the first.
|
|
138
|
+
const seen = new Set();
|
|
139
|
+
return out.filter((s) => (seen.has(s.name) ? false : (seen.add(s.name), true)));
|
|
140
|
+
}
|
|
141
|
+
/** Every place a client config is known to live, whether or not it exists. */
|
|
142
|
+
export function configCandidates(env) {
|
|
143
|
+
const { home, cwd, platform } = env;
|
|
144
|
+
const desktop = platform === 'darwin'
|
|
145
|
+
? join(home, 'Library', 'Application Support', 'Claude', 'claude_desktop_config.json')
|
|
146
|
+
: platform === 'win32'
|
|
147
|
+
? join(env.appData ?? join(home, 'AppData', 'Roaming'), 'Claude', 'claude_desktop_config.json')
|
|
148
|
+
: join(home, '.config', 'Claude', 'claude_desktop_config.json');
|
|
149
|
+
return [
|
|
150
|
+
{ client: 'claude-desktop', path: desktop },
|
|
151
|
+
{ client: 'claude-code', path: join(home, '.claude.json') },
|
|
152
|
+
{ client: 'claude-code', path: join(cwd, '.mcp.json') },
|
|
153
|
+
{ client: 'cursor', path: join(home, '.cursor', 'mcp.json') },
|
|
154
|
+
{ client: 'cursor', path: join(cwd, '.cursor', 'mcp.json') },
|
|
155
|
+
{ client: 'vscode', path: join(cwd, '.vscode', 'mcp.json') },
|
|
156
|
+
{ client: 'windsurf', path: join(home, '.codeium', 'windsurf', 'mcp_config.json') },
|
|
157
|
+
];
|
|
158
|
+
}
|
|
159
|
+
/** Read + parse the candidates that exist. Unreadable files are reported, not thrown. */
|
|
160
|
+
export function loadConfigs(candidates, cwd) {
|
|
161
|
+
const out = [];
|
|
162
|
+
for (const c of candidates) {
|
|
163
|
+
if (!existsSync(c.path))
|
|
164
|
+
continue;
|
|
165
|
+
try {
|
|
166
|
+
const doc = parseJsonc(readFileSync(c.path, 'utf8'));
|
|
167
|
+
const servers = extractServers(doc, { client: c.client, source: c.path, cwd });
|
|
168
|
+
// A config with no MCP block at all (e.g. a ~/.claude.json holding only
|
|
169
|
+
// session history) is not worth a line in the report.
|
|
170
|
+
if (servers.length === 0)
|
|
171
|
+
continue;
|
|
172
|
+
out.push({ client: c.client, source: c.path, servers });
|
|
173
|
+
}
|
|
174
|
+
catch (e) {
|
|
175
|
+
out.push({ client: c.client, source: c.path, servers: [], error: e.message });
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
return out;
|
|
179
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { Measurement } from '../core/types.js';
|
|
2
|
+
import { type AuditReport } from './audit.js';
|
|
3
|
+
import { type LoadedConfig } from './config.js';
|
|
4
|
+
export interface AuditOptions {
|
|
5
|
+
/** Explicit config path(s); when empty, every known client location is tried. */
|
|
6
|
+
configPaths?: string[];
|
|
7
|
+
cwd?: string;
|
|
8
|
+
home?: string;
|
|
9
|
+
timeoutMs?: number;
|
|
10
|
+
concurrency?: number;
|
|
11
|
+
docker?: boolean;
|
|
12
|
+
contextWindow?: number;
|
|
13
|
+
budget?: number;
|
|
14
|
+
onProgress?: (name: string, done: number, total: number) => void;
|
|
15
|
+
}
|
|
16
|
+
export declare function discover(opts?: AuditOptions): LoadedConfig[];
|
|
17
|
+
/** Measure every distinct stdio server across the given configs, once each. */
|
|
18
|
+
export declare function measureAll(configs: LoadedConfig[], opts?: AuditOptions): Promise<Map<string, Measurement>>;
|
|
19
|
+
export declare function runAudit(opts?: AuditOptions): Promise<AuditReport>;
|