mcp-context-cost 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +154 -12
- package/dist/audit/audit.d.ts +129 -0
- package/dist/audit/audit.js +312 -0
- package/dist/audit/config.d.ts +53 -0
- package/dist/audit/config.js +179 -0
- package/dist/audit/diff.d.ts +124 -0
- package/dist/audit/diff.js +318 -0
- package/dist/audit/run.d.ts +31 -0
- package/dist/audit/run.js +86 -0
- package/dist/cli.d.ts +21 -0
- package/dist/cli.js +187 -6
- package/dist/sweep/dashboard.d.ts +8 -0
- package/dist/sweep/dashboard.js +53 -5
- package/dist/sweep/docker.d.ts +8 -0
- package/dist/sweep/docker.js +7 -2
- package/dist/sweep/report.d.ts +2 -0
- package/dist/sweep/run.d.ts +13 -0
- package/dist/sweep/run.js +19 -5
- package/dist/sweep/sweep-all.js +1 -0
- package/package.json +6 -2
package/README.md
CHANGED
|
@@ -3,11 +3,142 @@
|
|
|
3
3
|
[](https://www.npmjs.com/package/mcp-context-cost)
|
|
4
4
|
[](https://github.com/athakur3/mcp-context-cost/actions/workflows/ci.yml)
|
|
5
5
|
|
|
6
|
-
**
|
|
6
|
+
**What do the MCP servers in your config cost you before you type anything — and what did
|
|
7
|
+
that last config change add to every session you will ever run?**
|
|
7
8
|
|
|
8
|
-
Every MCP server you wire into an agent injects its tool schemas into the model's context
|
|
9
|
-
|
|
10
|
-
|
|
9
|
+
Every MCP server you wire into an agent injects its tool schemas into the model's context on
|
|
10
|
+
every single request. You pay that whether or not the agent ends up using the tools, and no
|
|
11
|
+
client shows you the number. Point `audit` at your own MCP config:
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
npx -y mcp-context-cost audit
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
```
|
|
18
|
+
claude-desktop ~/Library/Application Support/Claude/claude_desktop_config.json
|
|
19
|
+
server tools tokens share
|
|
20
|
+
filesystem 14 2,823 35.7%
|
|
21
|
+
memory 9 2,378 30.1%
|
|
22
|
+
everything 13 1,708 21.6%
|
|
23
|
+
sequential-thinking 1 992 12.6%
|
|
24
|
+
────────────────────────────────────────────
|
|
25
|
+
total 37 7,901
|
|
26
|
+
|
|
27
|
+
Every request in this client carries 7,901 tokens of tool schemas — 4.0% of a
|
|
28
|
+
200,000-token context window, before you type anything.
|
|
29
|
+
|
|
30
|
+
heaviest tools
|
|
31
|
+
sequential-thinking · sequentialthinking 990
|
|
32
|
+
memory · search_nodes 323
|
|
33
|
+
|
|
34
|
+
trim: disabling 3 tools (sequential-thinking·sequentialthinking, memory·search_nodes,
|
|
35
|
+
memory·open_nodes) would recover 1,635 tokens (20.7% of this config) — if your client
|
|
36
|
+
supports per-tool filtering.
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
It finds configs for Claude Desktop, Claude Code (`~/.claude.json`, `.mcp.json`), Cursor,
|
|
40
|
+
VS Code (`.vscode/mcp.json`), and Windsurf — or pass `--config <path>`. Servers are measured
|
|
41
|
+
by the same path as the published leaderboard (dual `tools/list` capture, `o200k_base` over
|
|
42
|
+
canonical JSON), so a server in both places gets the same number. Nothing is written to your
|
|
43
|
+
project, and env var **values** are never read into the output — only their names.
|
|
44
|
+
|
|
45
|
+
Totals are reported per config file, never merged: a context window belongs to one client
|
|
46
|
+
session, so summing Cursor's servers into Claude Desktop's total would describe a session
|
|
47
|
+
nobody runs.
|
|
48
|
+
|
|
49
|
+
One nuance: Claude Code's tool search (default-on in recent versions) defers full MCP
|
|
50
|
+
schemas until used, loading only tool names at session start. Audit totals are the weight
|
|
51
|
+
of the schema surface itself — what loads upfront in clients without deferral (Claude
|
|
52
|
+
Desktop, Cursor, VS Code, Windsurf today), and what Claude Code's documented fallback
|
|
53
|
+
modes still load. Deferral-aware reporting is on the roadmap.
|
|
54
|
+
|
|
55
|
+
**In CI**, make it a gate — the bundlesize move for agents:
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
npx -y mcp-context-cost audit --config .mcp.json --budget 20000
|
|
59
|
+
# exits 1 when the stack exceeds the budget, so a PR adding a 25K-token server fails
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
The budget is an absolute ceiling. What a reviewer actually wants to know is what *this pull
|
|
63
|
+
request* did, so record a baseline and diff against it:
|
|
64
|
+
|
|
65
|
+
```bash
|
|
66
|
+
npx -y mcp-context-cost audit --config .mcp.json --json > baseline.json # on main
|
|
67
|
+
npx -y mcp-context-cost audit --config .mcp.json --baseline baseline.json --max-increase 2000
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
```
|
|
71
|
+
diff vs baseline measured 2026-08-18T01:51:49.555Z (methodology 1.0)
|
|
72
|
+
|
|
73
|
+
.mcp.json
|
|
74
|
+
2,378 → 5,201 +2,823
|
|
75
|
+
|
|
76
|
+
added filesystem — → 2,823 +2,823
|
|
77
|
+
(1 server unchanged)
|
|
78
|
+
|
|
79
|
+
This change adds 2,823 tokens to every request in this client — 1.2% → 2.6% of a
|
|
80
|
+
200,000-token context window.
|
|
81
|
+
|
|
82
|
+
INCREASE FAIL:
|
|
83
|
+
.mcp.json: +2,823 tokens per request, over the 2,000 allowed
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
> **Version note.** `--baseline` and `--max-increase` are **not in the published 0.3.0** —
|
|
87
|
+
> they are on `main` and ship in the next release. This matters more than a normal
|
|
88
|
+
> unreleased-feature note: 0.3.0 ignores flags it does not recognise, so running the command
|
|
89
|
+
> above against it produces a plain audit and **exit 0** — a passing CI check on a gate that
|
|
90
|
+
> never ran. Builds after 0.3.0 reject unknown flags with exit 2 instead. Until the next
|
|
91
|
+
> release, pin the gate to a version that has it, or it is not gating anything.
|
|
92
|
+
|
|
93
|
+
A baseline is just a stored `audit --json` report, so any artifact store works. Without
|
|
94
|
+
`--max-increase` the diff is informational and the exit code is unchanged.
|
|
95
|
+
|
|
96
|
+
[`examples/github-actions.yml`](examples/github-actions.yml) is the whole thing as a workflow:
|
|
97
|
+
measure the base branch, measure the PR, fail on the difference.
|
|
98
|
+
|
|
99
|
+
`--max-increase` fails on more than the number — it also fails whenever the increase could
|
|
100
|
+
not be established. A server that measured yesterday and won't start today takes its tokens
|
|
101
|
+
out of the total in exactly the way uninstalling it would, and reporting that as a saving is
|
|
102
|
+
the one mistake this tool must not make. So a server that crossed the measured/unmeasured
|
|
103
|
+
line, a config with no baseline, or a baseline config this run never found each fail the
|
|
104
|
+
gate and name themselves:
|
|
105
|
+
|
|
106
|
+
```
|
|
107
|
+
Not a clean comparison: a server changed measured-ness between the two runs.
|
|
108
|
+
The measured total moved −2,378, but that is not what your config did.
|
|
109
|
+
|
|
110
|
+
memory: measured 2,378 in the baseline and could not be measured now — its cost is
|
|
111
|
+
missing from the total, not gone from your config
|
|
112
|
+
→ true cost is at least 2,378 higher than the 0 measured now.
|
|
113
|
+
|
|
114
|
+
INCREASE FAIL:
|
|
115
|
+
.mcp.json: a server changed measured-ness, so the change could not be established exactly
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
Add `--claude` to annotate each server with its Anthropic-request cost from the published
|
|
119
|
+
[Claude divergence](docs/METHODOLOGY.md#claude-divergence) run — an exact number when the
|
|
120
|
+
published capture hash matches what you have installed, `—` (silence, not a stale guess)
|
|
121
|
+
when it doesn't (today the run covers the top 15 measured servers, so most installs will
|
|
122
|
+
show a mix):
|
|
123
|
+
|
|
124
|
+
```
|
|
125
|
+
server tools tokens share claude
|
|
126
|
+
github 44 54,422 95.8% 18,406
|
|
127
|
+
memory 9 2,378 4.2% —
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
Flags: `--json` (full report on stdout, progress on stderr), `--budget N`,
|
|
131
|
+
`--baseline <report.json>`, `--max-increase N`, `--context N` (default 200,000),
|
|
132
|
+
`--timeout ms`, `--concurrency N`, `--docker`, `--claude`.
|
|
133
|
+
|
|
134
|
+
## Where the numbers come from
|
|
135
|
+
|
|
136
|
+
The number `audit` gives you is the same measurement, run across a curated set of public
|
|
137
|
+
servers — which is how you can tell it is a measurement and not this tool's opinion. It also
|
|
138
|
+
shows what you are choosing between: across the 65 servers measured, cost spans **1,700×**,
|
|
139
|
+
from the 32-token `postgres` reference server to github's 54,422. The table below starts at
|
|
140
|
+
markitdown's 64 tokens, an 850× spread; the full range is in
|
|
141
|
+
[results/leaderboard.md](results/leaderboard.md).
|
|
11
142
|
|
|
12
143
|
| server | context cost | tools |
|
|
13
144
|
|---|---:|---:|
|
|
@@ -18,12 +149,13 @@ popular servers:
|
|
|
18
149
|
| filesystem (reference) | 2,823 | 14 |
|
|
19
150
|
| markitdown | 64 | 1 |
|
|
20
151
|
|
|
21
|
-
*(
|
|
152
|
+
*(65 of 82 popular servers measured, 2026-08-18 sweep — full table in
|
|
22
153
|
[results/leaderboard.md](results/leaderboard.md); every failure is listed with its reason.
|
|
23
154
|
Each measured server also has a [detail page](https://athakur3.github.io/mcp-context-cost/servers/)
|
|
24
155
|
showing which tools its tokens are in.)*
|
|
25
156
|
|
|
26
|
-
|
|
157
|
+
If you publish a server, the same measurement is available as a badge, so your users can see
|
|
158
|
+
the cost before they install rather than after:
|
|
27
159
|
|
|
28
160
|
```
|
|
29
161
|
[context cost | 12,430 tokens] ← shields.io badge, linked to the methodology
|
|
@@ -46,9 +178,9 @@ the method is [Claude divergence](docs/METHODOLOGY.md#claude-divergence).
|
|
|
46
178
|
|
|
47
179
|
## Why trust the number?
|
|
48
180
|
|
|
49
|
-
Every
|
|
50
|
-
SHA-256 of its canonical bytes, the pinned tokenizer (`o200k_base`), and the
|
|
51
|
-
command. Disputes reduce to a byte-level diff:
|
|
181
|
+
Every published number is backed by a `measurement.json` containing the raw `tools/list`
|
|
182
|
+
capture, the SHA-256 of its canonical bytes, the pinned tokenizer (`o200k_base`), and the
|
|
183
|
+
exact launch command. Disputes reduce to a byte-level diff:
|
|
52
184
|
|
|
53
185
|
```bash
|
|
54
186
|
npx -y mcp-context-cost verify results/github/measurement.json
|
|
@@ -62,6 +194,11 @@ Add `--json` for scripting (`{ ok, serverName, rederivedTokens, rederivedSha, pr
|
|
|
62
194
|
on stdout, `badge` omitted on failure). Exit codes: `0` ok, `1` verification/measurement
|
|
63
195
|
failed, `2` usage error.
|
|
64
196
|
|
|
197
|
+
`audit` runs that same code path on your own machine and reports each server's
|
|
198
|
+
`canonicalSha256` in `--json`, so you can check that the version you installed is byte-identical
|
|
199
|
+
to the one that was published — which is exactly what `--claude` uses to decide whether it is
|
|
200
|
+
allowed to show you a number.
|
|
201
|
+
|
|
65
202
|
Full definition: [docs/METHODOLOGY.md](docs/METHODOLOGY.md) — what is counted, what the
|
|
66
203
|
number is *not*, config policy, failure taxonomy, frozen color bands, known divergences.
|
|
67
204
|
|
|
@@ -71,7 +208,8 @@ number is *not*, config policy, failure taxonomy, frozen color bands, known dive
|
|
|
71
208
|
|---|---|
|
|
72
209
|
| `src/core/` | the measurement spec, executable — canonical form, tokenizer, bands, badge JSON |
|
|
73
210
|
| `src/sweep/` | raw-wire MCP stdio client + Dockerized batch sweep + leaderboard/dashboard generators |
|
|
74
|
-
| `src/
|
|
211
|
+
| `src/audit/` | client-config discovery (5 clients, JSONC-tolerant), the per-stack report, and the baseline diff |
|
|
212
|
+
| `src/cli.ts` | `audit` (measure your own stack), `verify` (re-derive any published number), `measure` |
|
|
75
213
|
| `spec/fixtures/` | golden vectors shared by the TypeScript and bash implementations |
|
|
76
214
|
| `tools/` | the one script that calls a network API (Claude divergence); kept out of the package so the library stays offline |
|
|
77
215
|
| `upstream/` | staged contribution to [sd2k/mcp-tokens-action](https://github.com/sd2k/mcp-tokens-action): `badge.sh` + action patch + tests |
|
|
@@ -103,7 +241,7 @@ Or self-serve from CI via the (staged) mcp-tokens-action badge inputs — see
|
|
|
103
241
|
## Development
|
|
104
242
|
|
|
105
243
|
```bash
|
|
106
|
-
npm test #
|
|
244
|
+
npm test # 158 TS tests incl. golden fixtures + dispute drills
|
|
107
245
|
npx tsc --noEmit # typecheck
|
|
108
246
|
./upstream/tests/badge-test.sh # 21 bash tests — byte-identical to the TS reference
|
|
109
247
|
npm run sweep:all -- --docker # full curated sweep (Docker isolation)
|
|
@@ -116,7 +254,11 @@ color bands are frozen against the observed distribution of the first full sweep
|
|
|
116
254
|
|
|
117
255
|
## Status
|
|
118
256
|
|
|
119
|
-
Active.
|
|
257
|
+
Active. 56 of the 65 numbers come from a single sweep on 2026-08-16, 3 from 2026-08-17, and
|
|
258
|
+
6 from 2026-08-18 (an upstream `mcp` package bump broke the old low-level-`Server` API these
|
|
259
|
+
six relied on; pinning `mcp<2` in their launch commands fixed startup, not this project's code);
|
|
260
|
+
the weekly job currently re-measures one server (`memory`), so treat the leaderboard as a
|
|
261
|
+
dated snapshot rather than a live feed. Badge PRs are open across the ecosystem and
|
|
120
262
|
[sd2k/mcp-tokens-action#5](https://github.com/sd2k/mcp-tokens-action/pull/5) proposes the
|
|
121
263
|
self-serve badge path upstream. See [ROADMAP.md](ROADMAP.md) for what's next —
|
|
122
264
|
contributions welcome, especially new `servers.yaml` entries.
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import { type DivergenceRun } from '../core/divergence.js';
|
|
2
|
+
import type { Measurement, MeasurementStatus, ToolMeasurement } from '../core/types.js';
|
|
3
|
+
import type { ConfiguredServer, LoadedConfig } from './config.js';
|
|
4
|
+
import { type AuditDiff, type IncreaseGate } from './diff.js';
|
|
5
|
+
export declare const DEFAULT_CONTEXT_WINDOW = 200000;
|
|
6
|
+
export type AuditStatus = MeasurementStatus | 'remote-not-measurable';
|
|
7
|
+
export interface AuditServerResult {
|
|
8
|
+
name: string;
|
|
9
|
+
transport: 'stdio' | 'remote';
|
|
10
|
+
status: AuditStatus;
|
|
11
|
+
tokens: number | null;
|
|
12
|
+
toolCount: number | null;
|
|
13
|
+
/** Share of this config's measured total, 0–1. */
|
|
14
|
+
share: number | null;
|
|
15
|
+
command?: string;
|
|
16
|
+
url?: string;
|
|
17
|
+
/** Names only — a server's env values never enter a report. */
|
|
18
|
+
envVarNames: string[];
|
|
19
|
+
canonicalSha256?: string | null;
|
|
20
|
+
/**
|
|
21
|
+
* Anthropic-request cost from the published Claude divergence run, only when
|
|
22
|
+
* its captured hash matches this install (`--claude`). `null` means the
|
|
23
|
+
* install doesn't match what was published — silence, not a stale guess.
|
|
24
|
+
* `undefined` means `--claude` wasn't requested at all.
|
|
25
|
+
*/
|
|
26
|
+
claudeTokens?: number | null;
|
|
27
|
+
notes?: string;
|
|
28
|
+
}
|
|
29
|
+
export interface HeaviestTool {
|
|
30
|
+
server: string;
|
|
31
|
+
tool: string;
|
|
32
|
+
tokens: number;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* What turning off the heaviest few tools would recover, for clients that let
|
|
36
|
+
* you disable individual tools rather than whole servers (Claude Code's
|
|
37
|
+
* per-tool permission rules, Cursor's per-tool toggles). `null` when there's
|
|
38
|
+
* nothing worth trimming (one tool total, or no measured tokens).
|
|
39
|
+
*/
|
|
40
|
+
export interface TrimAdvice {
|
|
41
|
+
tools: HeaviestTool[];
|
|
42
|
+
recoverableTokens: number;
|
|
43
|
+
recoverableShare: number;
|
|
44
|
+
}
|
|
45
|
+
export interface AuditConfigResult {
|
|
46
|
+
client: string;
|
|
47
|
+
source: string;
|
|
48
|
+
totalTokens: number;
|
|
49
|
+
toolCount: number;
|
|
50
|
+
serverCount: number;
|
|
51
|
+
contextShare: number;
|
|
52
|
+
servers: AuditServerResult[];
|
|
53
|
+
skipped: AuditServerResult[];
|
|
54
|
+
heaviestTools: HeaviestTool[];
|
|
55
|
+
trimAdvice: TrimAdvice | null;
|
|
56
|
+
}
|
|
57
|
+
export interface BudgetFitStep {
|
|
58
|
+
name: string;
|
|
59
|
+
tokens: number;
|
|
60
|
+
/** What the config still costs after removing this one and everything above it. */
|
|
61
|
+
remaining: number;
|
|
62
|
+
}
|
|
63
|
+
export interface BudgetFit {
|
|
64
|
+
/** How far over the limit the worst config starts. */
|
|
65
|
+
overBy: number;
|
|
66
|
+
/** Heaviest-first removals until the remainder fits. Empty if nothing can be removed. */
|
|
67
|
+
drop: BudgetFitStep[];
|
|
68
|
+
keptCount: number;
|
|
69
|
+
keptTokens: number;
|
|
70
|
+
/** False when removing every measured server still would not fit — a limit set too low. */
|
|
71
|
+
feasible: boolean;
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* The smallest heaviest-first set of servers that gets a config under its budget.
|
|
75
|
+
*
|
|
76
|
+
* `--budget` used to print "BUDGET FAIL: 84,455 > 20,000" and stop, which tells a reader
|
|
77
|
+
* they have a problem and nothing about the shape of it. The whole point of the audit
|
|
78
|
+
* surface is that the person running it is the person paying the tokens, and "you are over"
|
|
79
|
+
* is a measurement where "these two are why" is a decision.
|
|
80
|
+
*
|
|
81
|
+
* Heaviest-first is ONE ordering, not a recommendation: this cannot know which servers you
|
|
82
|
+
* need, and dropping by weight will sometimes name the one you cannot live without. That
|
|
83
|
+
* caveat is printed with the result rather than left implied.
|
|
84
|
+
*/
|
|
85
|
+
export declare function planBudgetFit(config: AuditConfigResult, limit: number): BudgetFit;
|
|
86
|
+
export interface AuditReport {
|
|
87
|
+
methodologyVersion: string;
|
|
88
|
+
encoding: 'o200k_base';
|
|
89
|
+
generatedAt: string;
|
|
90
|
+
contextWindow: number;
|
|
91
|
+
configs: AuditConfigResult[];
|
|
92
|
+
budget?: {
|
|
93
|
+
limit: number;
|
|
94
|
+
worstTotal: number;
|
|
95
|
+
worstSource: string;
|
|
96
|
+
over: boolean;
|
|
97
|
+
/** Present only when over budget: the arithmetic of getting back under it. */
|
|
98
|
+
fit?: BudgetFit;
|
|
99
|
+
};
|
|
100
|
+
/** Present only when a divergence run was supplied (`--claude`). */
|
|
101
|
+
claudeDivergence?: {
|
|
102
|
+
model: string;
|
|
103
|
+
measuredAt: string;
|
|
104
|
+
};
|
|
105
|
+
/** Present only when a baseline report was supplied (`--baseline`). */
|
|
106
|
+
diff?: AuditDiff;
|
|
107
|
+
/** Present only when `--max-increase` was supplied alongside a baseline. */
|
|
108
|
+
increaseGate?: IncreaseGate;
|
|
109
|
+
problems: string[];
|
|
110
|
+
}
|
|
111
|
+
/** Cache key for measurement reuse: the exact argv two configs would spawn. */
|
|
112
|
+
export declare function serverKey(s: ConfiguredServer): string;
|
|
113
|
+
/**
|
|
114
|
+
* Assemble the report from configs + measurements. Pure: `runAudit` does the
|
|
115
|
+
* spawning, this does the arithmetic, so totals and shares are testable without
|
|
116
|
+
* launching a single server.
|
|
117
|
+
*/
|
|
118
|
+
export declare function buildReport(configs: LoadedConfig[], measured: Map<string, Measurement>, opts?: {
|
|
119
|
+
contextWindow?: number;
|
|
120
|
+
budget?: number;
|
|
121
|
+
generatedAt?: string;
|
|
122
|
+
/** Published `tools-delta/v1` run to join against (`--claude`); omit to skip the join. */
|
|
123
|
+
divergence?: DivergenceRun | null;
|
|
124
|
+
}): AuditReport;
|
|
125
|
+
/** Human output. JSON output is the report object itself. */
|
|
126
|
+
export declare function formatReport(report: AuditReport): string;
|
|
127
|
+
/** Top-level tool list across every config — used by nothing yet, handy for --json consumers. */
|
|
128
|
+
export declare function allHeaviestTools(report: AuditReport, limit?: number): HeaviestTool[];
|
|
129
|
+
export type { ToolMeasurement };
|
|
@@ -0,0 +1,312 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `audit` — measure the MCP servers a person actually has installed.
|
|
3
|
+
*
|
|
4
|
+
* The leaderboard answers "what does server X cost?". This answers the question
|
|
5
|
+
* the person paying the bill has: "what do MY servers cost, together, before I
|
|
6
|
+
* type anything?" Same measurement path as the sweep (dual tools/list capture,
|
|
7
|
+
* o200k_base over canonical JSON, full status taxonomy), pointed at a client
|
|
8
|
+
* config instead of servers.yaml.
|
|
9
|
+
*
|
|
10
|
+
* Totals are per config file, never merged across clients: a context window
|
|
11
|
+
* belongs to one client session, so summing Cursor's servers into Claude
|
|
12
|
+
* Desktop's total would describe a session nobody is running. Identical launch
|
|
13
|
+
* commands shared by two configs are still only measured once.
|
|
14
|
+
*/
|
|
15
|
+
import { METHODOLOGY_VERSION } from '../core/canonical.js';
|
|
16
|
+
import { isCurrent } from '../core/divergence.js';
|
|
17
|
+
import { formatDiff, formatGate } from './diff.js';
|
|
18
|
+
export const DEFAULT_CONTEXT_WINDOW = 200_000;
|
|
19
|
+
const TRIM_TOOL_COUNT = 3;
|
|
20
|
+
function buildTrimAdvice(sortedTools, totalTokens) {
|
|
21
|
+
if (totalTokens <= 0 || sortedTools.length < 2)
|
|
22
|
+
return null;
|
|
23
|
+
const trimmed = sortedTools.slice(0, TRIM_TOOL_COUNT);
|
|
24
|
+
const recoverableTokens = trimmed.reduce((a, t) => a + t.tokens, 0);
|
|
25
|
+
return { tools: trimmed, recoverableTokens, recoverableShare: recoverableTokens / totalTokens };
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* The smallest heaviest-first set of servers that gets a config under its budget.
|
|
29
|
+
*
|
|
30
|
+
* `--budget` used to print "BUDGET FAIL: 84,455 > 20,000" and stop, which tells a reader
|
|
31
|
+
* they have a problem and nothing about the shape of it. The whole point of the audit
|
|
32
|
+
* surface is that the person running it is the person paying the tokens, and "you are over"
|
|
33
|
+
* is a measurement where "these two are why" is a decision.
|
|
34
|
+
*
|
|
35
|
+
* Heaviest-first is ONE ordering, not a recommendation: this cannot know which servers you
|
|
36
|
+
* need, and dropping by weight will sometimes name the one you cannot live without. That
|
|
37
|
+
* caveat is printed with the result rather than left implied.
|
|
38
|
+
*/
|
|
39
|
+
export function planBudgetFit(config, limit) {
|
|
40
|
+
const measured = config.servers
|
|
41
|
+
.filter((srv) => typeof srv.tokens === 'number' && srv.tokens > 0)
|
|
42
|
+
.sort((a, b) => b.tokens - a.tokens);
|
|
43
|
+
const overBy = config.totalTokens - limit;
|
|
44
|
+
const drop = [];
|
|
45
|
+
let remaining = config.totalTokens;
|
|
46
|
+
for (const srv of measured) {
|
|
47
|
+
if (remaining <= limit)
|
|
48
|
+
break;
|
|
49
|
+
remaining -= srv.tokens;
|
|
50
|
+
drop.push({ name: srv.name, tokens: srv.tokens, remaining });
|
|
51
|
+
}
|
|
52
|
+
return {
|
|
53
|
+
overBy,
|
|
54
|
+
drop,
|
|
55
|
+
keptCount: measured.length - drop.length,
|
|
56
|
+
keptTokens: remaining,
|
|
57
|
+
// Removing everything measured still leaves unmeasured/base cost behind, so the
|
|
58
|
+
// honest test is whether the remainder actually landed under the limit.
|
|
59
|
+
feasible: remaining <= limit,
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
/** Cache key for measurement reuse: the exact argv two configs would spawn. */
|
|
63
|
+
export function serverKey(s) {
|
|
64
|
+
return JSON.stringify(s.argv ?? [s.url ?? s.name]);
|
|
65
|
+
}
|
|
66
|
+
function measuredOk(m) {
|
|
67
|
+
return (m.status === 'measured' || m.status === 'dynamic') && typeof m.totalTokens === 'number';
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Assemble the report from configs + measurements. Pure: `runAudit` does the
|
|
71
|
+
* spawning, this does the arithmetic, so totals and shares are testable without
|
|
72
|
+
* launching a single server.
|
|
73
|
+
*/
|
|
74
|
+
export function buildReport(configs, measured, opts = {}) {
|
|
75
|
+
const contextWindow = opts.contextWindow ?? DEFAULT_CONTEXT_WINDOW;
|
|
76
|
+
const problems = [];
|
|
77
|
+
const results = [];
|
|
78
|
+
for (const cfg of configs) {
|
|
79
|
+
if (cfg.error) {
|
|
80
|
+
problems.push(`${cfg.source}: ${cfg.error}`);
|
|
81
|
+
continue;
|
|
82
|
+
}
|
|
83
|
+
const ok = [];
|
|
84
|
+
const skipped = [];
|
|
85
|
+
const tools = [];
|
|
86
|
+
for (const s of cfg.servers) {
|
|
87
|
+
const base = {
|
|
88
|
+
name: s.name,
|
|
89
|
+
transport: s.transport,
|
|
90
|
+
command: s.command,
|
|
91
|
+
url: s.url,
|
|
92
|
+
envVarNames: s.envVarNames,
|
|
93
|
+
};
|
|
94
|
+
if (s.transport === 'remote') {
|
|
95
|
+
skipped.push({
|
|
96
|
+
...base,
|
|
97
|
+
status: 'remote-not-measurable',
|
|
98
|
+
tokens: null,
|
|
99
|
+
toolCount: null,
|
|
100
|
+
share: null,
|
|
101
|
+
notes: `remote endpoint (${s.url ?? 'url'}) — stdio measurement does not apply`,
|
|
102
|
+
});
|
|
103
|
+
continue;
|
|
104
|
+
}
|
|
105
|
+
const m = measured.get(serverKey(s));
|
|
106
|
+
if (!m) {
|
|
107
|
+
skipped.push({ ...base, status: 'startup-failure', tokens: null, toolCount: null, share: null, notes: 'not measured' });
|
|
108
|
+
continue;
|
|
109
|
+
}
|
|
110
|
+
if (!measuredOk(m)) {
|
|
111
|
+
skipped.push({
|
|
112
|
+
...base,
|
|
113
|
+
status: m.status,
|
|
114
|
+
tokens: null,
|
|
115
|
+
toolCount: null,
|
|
116
|
+
share: null,
|
|
117
|
+
notes: m.notes?.split('\n')[0]?.slice(0, 200),
|
|
118
|
+
});
|
|
119
|
+
continue;
|
|
120
|
+
}
|
|
121
|
+
const divRow = opts.divergence?.servers[s.name];
|
|
122
|
+
ok.push({
|
|
123
|
+
...base,
|
|
124
|
+
status: m.status,
|
|
125
|
+
tokens: m.totalTokens,
|
|
126
|
+
toolCount: m.toolCount,
|
|
127
|
+
share: null, // filled once the total is known
|
|
128
|
+
canonicalSha256: m.canonicalSha256,
|
|
129
|
+
claudeTokens: opts.divergence ? (isCurrent(divRow, m.canonicalSha256 ?? null) ? divRow.claudeDelta : null) : undefined,
|
|
130
|
+
notes: m.status === 'dynamic' ? m.notes : undefined,
|
|
131
|
+
});
|
|
132
|
+
for (const t of m.tools)
|
|
133
|
+
tools.push({ server: s.name, tool: t.name, tokens: t.tokens });
|
|
134
|
+
}
|
|
135
|
+
const totalTokens = ok.reduce((a, s) => a + (s.tokens ?? 0), 0);
|
|
136
|
+
const toolCount = ok.reduce((a, s) => a + (s.toolCount ?? 0), 0);
|
|
137
|
+
ok.sort((a, b) => (b.tokens ?? 0) - (a.tokens ?? 0));
|
|
138
|
+
for (const s of ok)
|
|
139
|
+
s.share = totalTokens > 0 ? (s.tokens ?? 0) / totalTokens : 0;
|
|
140
|
+
tools.sort((a, b) => b.tokens - a.tokens);
|
|
141
|
+
results.push({
|
|
142
|
+
client: cfg.client,
|
|
143
|
+
source: cfg.source,
|
|
144
|
+
totalTokens,
|
|
145
|
+
toolCount,
|
|
146
|
+
serverCount: ok.length,
|
|
147
|
+
// Deliberately no band: the color bands were frozen against the per-server
|
|
148
|
+
// distribution (n=57). A config total is a different population, so calling
|
|
149
|
+
// a 7,901-token *stack* "moderate" would borrow a scale that doesn't mean
|
|
150
|
+
// that here. Share of the context window is the honest framing.
|
|
151
|
+
contextShare: totalTokens / contextWindow,
|
|
152
|
+
servers: ok,
|
|
153
|
+
skipped,
|
|
154
|
+
heaviestTools: tools.slice(0, 5),
|
|
155
|
+
trimAdvice: buildTrimAdvice(tools, totalTokens),
|
|
156
|
+
});
|
|
157
|
+
}
|
|
158
|
+
results.sort((a, b) => b.totalTokens - a.totalTokens);
|
|
159
|
+
const report = {
|
|
160
|
+
methodologyVersion: METHODOLOGY_VERSION,
|
|
161
|
+
encoding: 'o200k_base',
|
|
162
|
+
generatedAt: opts.generatedAt ?? new Date().toISOString(),
|
|
163
|
+
contextWindow,
|
|
164
|
+
configs: results,
|
|
165
|
+
problems,
|
|
166
|
+
};
|
|
167
|
+
if (opts.divergence) {
|
|
168
|
+
report.claudeDivergence = { model: opts.divergence.model, measuredAt: opts.divergence.measuredAt };
|
|
169
|
+
}
|
|
170
|
+
if (typeof opts.budget === 'number') {
|
|
171
|
+
// The worst config is the gate: passing because your *lightest* client fits
|
|
172
|
+
// would be a green check on a session you don't run.
|
|
173
|
+
const worst = results[0];
|
|
174
|
+
const over = (worst?.totalTokens ?? 0) > opts.budget;
|
|
175
|
+
report.budget = {
|
|
176
|
+
limit: opts.budget,
|
|
177
|
+
worstTotal: worst?.totalTokens ?? 0,
|
|
178
|
+
worstSource: worst?.source ?? '(none)',
|
|
179
|
+
over,
|
|
180
|
+
fit: over && worst ? planBudgetFit(worst, opts.budget) : undefined,
|
|
181
|
+
};
|
|
182
|
+
}
|
|
183
|
+
return report;
|
|
184
|
+
}
|
|
185
|
+
const n = (x) => x.toLocaleString('en-US');
|
|
186
|
+
const pct = (x) => `${(x * 100).toFixed(1)}%`;
|
|
187
|
+
/** Human output. JSON output is the report object itself. */
|
|
188
|
+
export function formatReport(report) {
|
|
189
|
+
const lines = [];
|
|
190
|
+
lines.push(`mcp-context-cost audit · methodology ${report.methodologyVersion} · ${report.encoding} · context window ${n(report.contextWindow)}`);
|
|
191
|
+
const showClaude = !!report.claudeDivergence;
|
|
192
|
+
for (const cfg of report.configs) {
|
|
193
|
+
lines.push('');
|
|
194
|
+
lines.push(`${cfg.client} ${cfg.source}`);
|
|
195
|
+
const rows = cfg.servers.map((s) => ({
|
|
196
|
+
name: s.name,
|
|
197
|
+
tools: s.toolCount === null ? '—' : String(s.toolCount),
|
|
198
|
+
tokens: s.tokens === null ? '—' : n(s.tokens),
|
|
199
|
+
share: s.share === null ? '—' : pct(s.share),
|
|
200
|
+
claude: s.claudeTokens == null ? '—' : n(s.claudeTokens),
|
|
201
|
+
}));
|
|
202
|
+
const w = {
|
|
203
|
+
name: Math.max(6, ...rows.map((r) => r.name.length), 'total'.length),
|
|
204
|
+
tools: Math.max(5, ...rows.map((r) => r.tools.length)),
|
|
205
|
+
tokens: Math.max(6, ...rows.map((r) => r.tokens.length), n(cfg.totalTokens).length),
|
|
206
|
+
claude: Math.max(6, ...rows.map((r) => r.claude.length)),
|
|
207
|
+
};
|
|
208
|
+
const line = (name, tools, tokens, share, claude) => ` ${name.padEnd(w.name)} ${tools.padStart(w.tools)} ${tokens.padStart(w.tokens)} ${share.padStart(6)}` +
|
|
209
|
+
(showClaude ? ` ${claude.padStart(w.claude)}` : '');
|
|
210
|
+
lines.push(line('server', 'tools', 'tokens', 'share', 'claude'));
|
|
211
|
+
for (const r of rows)
|
|
212
|
+
lines.push(line(r.name, r.tools, r.tokens, r.share, r.claude));
|
|
213
|
+
lines.push(` ${'─'.repeat(w.name + w.tools + w.tokens + 14 + (showClaude ? w.claude + 2 : 0))}`);
|
|
214
|
+
lines.push(line('total', String(cfg.toolCount), n(cfg.totalTokens), '', ''));
|
|
215
|
+
lines.push('');
|
|
216
|
+
lines.push(` Every request in this client carries ${n(cfg.totalTokens)} tokens of tool schemas — ` +
|
|
217
|
+
`${pct(cfg.contextShare)} of a ${n(report.contextWindow)}-token context window, before you type anything.`);
|
|
218
|
+
if (cfg.heaviestTools.length) {
|
|
219
|
+
lines.push('');
|
|
220
|
+
lines.push(' heaviest tools');
|
|
221
|
+
const tw = Math.max(...cfg.heaviestTools.map((t) => `${t.server} · ${t.tool}`.length));
|
|
222
|
+
for (const t of cfg.heaviestTools) {
|
|
223
|
+
lines.push(` ${`${t.server} · ${t.tool}`.padEnd(tw)} ${n(t.tokens).padStart(7)}`);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
if (cfg.trimAdvice) {
|
|
227
|
+
const names = cfg.trimAdvice.tools.map((t) => `${t.server}·${t.tool}`).join(', ');
|
|
228
|
+
lines.push('');
|
|
229
|
+
lines.push(` trim: disabling ${cfg.trimAdvice.tools.length} tool${cfg.trimAdvice.tools.length === 1 ? '' : 's'} ` +
|
|
230
|
+
`(${names}) would recover ${n(cfg.trimAdvice.recoverableTokens)} tokens ` +
|
|
231
|
+
`(${pct(cfg.trimAdvice.recoverableShare)} of this config) — if your client supports per-tool filtering.`);
|
|
232
|
+
}
|
|
233
|
+
if (cfg.skipped.length) {
|
|
234
|
+
lines.push('');
|
|
235
|
+
lines.push(' not measured');
|
|
236
|
+
const sw = Math.max(...cfg.skipped.map((s) => s.name.length));
|
|
237
|
+
for (const s of cfg.skipped) {
|
|
238
|
+
lines.push(` ${s.name.padEnd(sw)} ${s.status}${s.notes ? ` — ${s.notes}` : ''}`);
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
if (showClaude) {
|
|
243
|
+
lines.push('');
|
|
244
|
+
lines.push(` claude = Anthropic-request cost from the ${report.claudeDivergence.measuredAt} ${report.claudeDivergence.model} ` +
|
|
245
|
+
`divergence run, shown only where the published capture hash matches this install; '—' means no current match.`);
|
|
246
|
+
}
|
|
247
|
+
if (report.problems.length) {
|
|
248
|
+
lines.push('');
|
|
249
|
+
lines.push('problems');
|
|
250
|
+
for (const p of report.problems)
|
|
251
|
+
lines.push(` ${p}`);
|
|
252
|
+
}
|
|
253
|
+
if (report.budget) {
|
|
254
|
+
const b = report.budget;
|
|
255
|
+
lines.push('');
|
|
256
|
+
if (!b.over) {
|
|
257
|
+
const headroom = b.limit - b.worstTotal;
|
|
258
|
+
lines.push(`budget ok: ${n(b.worstTotal)} ≤ ${n(b.limit)} — ${n(headroom)} to spare`);
|
|
259
|
+
}
|
|
260
|
+
else {
|
|
261
|
+
lines.push(`BUDGET FAIL: ${n(b.worstTotal)} > ${n(b.limit)} (${b.worstSource})`);
|
|
262
|
+
const fit = b.fit;
|
|
263
|
+
if (fit && fit.drop.length) {
|
|
264
|
+
lines.push('');
|
|
265
|
+
lines.push(` over by ${n(fit.overBy)}. Heaviest-first, this is what gets you under:`);
|
|
266
|
+
for (const step of fit.drop) {
|
|
267
|
+
const verdict = step.remaining <= b.limit ? 'fits' : 'still over';
|
|
268
|
+
lines.push(` drop ${step.name.padEnd(22)} ${n(step.tokens).padStart(9)} → ${n(step.remaining).padStart(9)} ${verdict}`);
|
|
269
|
+
}
|
|
270
|
+
lines.push('');
|
|
271
|
+
if (fit.feasible && fit.keptCount === 0) {
|
|
272
|
+
// Arithmetically it fits, and the answer is useless: the only way under this
|
|
273
|
+
// limit is to run no servers at all. Saying "fits" here would be true and
|
|
274
|
+
// misleading, which is the pair this whole tool exists to keep apart.
|
|
275
|
+
lines.push(` no subset fits: every measured server would have to go. The limit is below`);
|
|
276
|
+
lines.push(` what any one of these servers costs.`);
|
|
277
|
+
}
|
|
278
|
+
else if (fit.feasible) {
|
|
279
|
+
const share = b.limit > 0 ? ` (${pct(fit.keptTokens / b.limit)} of budget)` : '';
|
|
280
|
+
lines.push(` keeps ${fit.keptCount} server(s) at ${n(fit.keptTokens)} tokens${share}`);
|
|
281
|
+
}
|
|
282
|
+
else {
|
|
283
|
+
lines.push(` even removing every measured server leaves ${n(fit.keptTokens)} — the limit is below this config's floor`);
|
|
284
|
+
}
|
|
285
|
+
lines.push('');
|
|
286
|
+
lines.push(' This is arithmetic, not advice: it cannot know which servers you need, and by');
|
|
287
|
+
lines.push(' weight alone it will sometimes name the one you cannot work without. Use --json');
|
|
288
|
+
lines.push(' for the full per-server list and pick your own order.');
|
|
289
|
+
}
|
|
290
|
+
else if (fit) {
|
|
291
|
+
lines.push(' nothing measured could be removed to get under the limit.');
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
if (report.diff)
|
|
296
|
+
lines.push(formatDiff(report.diff, report.contextWindow));
|
|
297
|
+
if (report.increaseGate) {
|
|
298
|
+
lines.push('');
|
|
299
|
+
lines.push(formatGate(report.increaseGate));
|
|
300
|
+
}
|
|
301
|
+
lines.push('');
|
|
302
|
+
lines.push('These are wire tokens — what the server puts on the wire, counted with o200k_base. What your model is billed');
|
|
303
|
+
lines.push('differs per provider: measured ratios run 0.34×–1.92× on Anthropic requests. See docs/METHODOLOGY.md §claude-divergence.');
|
|
304
|
+
return lines.map((l) => l.replace(/\s+$/, '')).join('\n');
|
|
305
|
+
}
|
|
306
|
+
/** Top-level tool list across every config — used by nothing yet, handy for --json consumers. */
|
|
307
|
+
export function allHeaviestTools(report, limit = 10) {
|
|
308
|
+
return report.configs
|
|
309
|
+
.flatMap((c) => c.heaviestTools)
|
|
310
|
+
.sort((a, b) => b.tokens - a.tokens)
|
|
311
|
+
.slice(0, limit);
|
|
312
|
+
}
|