mcp-context-cost 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -11
- package/dist/audit/deferral.js +14 -1
- package/dist/core/regression.d.ts +87 -5
- package/dist/core/regression.js +63 -22
- package/dist/core/types.d.ts +4 -4
- package/dist/sweep/cross-check.js +3 -0
- package/dist/sweep/dashboard.js +17 -2
- package/dist/sweep/docker.d.ts +46 -0
- package/dist/sweep/docker.js +82 -4
- package/dist/sweep/history.d.ts +16 -1
- package/dist/sweep/history.js +18 -5
- package/dist/sweep/published-stats.js +48 -3
- package/dist/sweep/regressions.js +131 -18
- package/dist/sweep/report.d.ts +44 -0
- package/dist/sweep/report.js +44 -1
- package/dist/sweep/run.d.ts +46 -4
- package/dist/sweep/run.js +56 -10
- package/dist/sweep/server-pages.js +15 -4
- package/dist/sweep/servers-schema.d.ts +41 -0
- package/dist/sweep/servers-schema.js +245 -0
- package/dist/sweep/session-start.js +2 -0
- package/dist/sweep/sweep-all.js +1 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -2,6 +2,27 @@
|
|
|
2
2
|
|
|
3
3
|
[](https://www.npmjs.com/package/mcp-context-cost)
|
|
4
4
|
[](https://github.com/athakur3/mcp-context-cost/actions/workflows/ci.yml)
|
|
5
|
+
[](https://github.com/marketplace/actions/mcp-context-cost)
|
|
6
|
+
|
|
7
|
+
Two entry points: `audit`, for the config you run (below), and a GitHub Action, for the
|
|
8
|
+
server you publish — five lines in a workflow that fail a pull request adding more context
|
|
9
|
+
cost than you meant to ship:
|
|
10
|
+
|
|
11
|
+
```yaml
|
|
12
|
+
- uses: athakur3/mcp-context-cost@v1
|
|
13
|
+
with:
|
|
14
|
+
name: my-server
|
|
15
|
+
command: node dist/index.js
|
|
16
|
+
baseline: .context-cost/baseline.json
|
|
17
|
+
max-increase: 500
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
Inputs: `name`, `command` or `remote`, `baseline`, `max-increase`, `budget`, `timeout`,
|
|
21
|
+
`version`. Outputs: `tokens`, `tools`, `status`, `measurement`, `badge` — written whether the
|
|
22
|
+
gate passed or not, so a later step can comment the number on the PR or publish the badge.
|
|
23
|
+
[Full workflow](examples/server-author-ci.yml) ·
|
|
24
|
+
[how the gate decides](#defend-the-number-dont-just-display-it) ·
|
|
25
|
+
[what the number is](docs/METHODOLOGY.md).
|
|
5
26
|
|
|
6
27
|
**What do the MCP servers in your config cost you before you type anything — and what did
|
|
7
28
|
that last config change add to every session you will ever run?**
|
|
@@ -105,7 +126,7 @@ costs **more** than loading the definitions would.
|
|
|
105
126
|
|
|
106
127
|
Three things the report will not do: it will not convert between units silently (in
|
|
107
128
|
threshold mode the stack is compared as a range, because the audit counts wire bytes and the
|
|
108
|
-
threshold is counted in what the client sends to the API — measured at 0.20×–1.92× across
|
|
129
|
+
threshold is counted in what the client sends to the API — measured at 0.20×–1.92× across 23
|
|
109
130
|
servers); it will not claim a posture the machine did not state readably, which is four
|
|
110
131
|
refusals and not one — when two places set the same variable to different values, when a
|
|
111
132
|
settings file exists and cannot be read, when the place that would decide sets the variable
|
|
@@ -183,8 +204,8 @@ INCREASE FAIL:
|
|
|
183
204
|
Add `--claude` to annotate each server with its Anthropic-request cost from the published
|
|
184
205
|
[Claude divergence](docs/METHODOLOGY.md#claude-divergence) run — an exact number when the
|
|
185
206
|
published capture hash matches what you have installed, `—` (silence, not a stale guess)
|
|
186
|
-
when it doesn't. The run holds
|
|
187
|
-
and [results/leaderboard.md](results/leaderboard.md) prints a claude number for the
|
|
207
|
+
when it doesn't. The run holds 23 rows — the measured servers it covered when it last ran —
|
|
208
|
+
and [results/leaderboard.md](results/leaderboard.md) prints a claude number for the 15 that
|
|
188
209
|
still match today and silence for the rest. Most installs will show a mix:
|
|
189
210
|
|
|
190
211
|
```
|
|
@@ -235,7 +256,7 @@ Flags: `--json` (full report on stdout, progress on stderr), `--budget N`,
|
|
|
235
256
|
|
|
236
257
|
The number `audit` gives you is the same measurement, run across a curated set of public
|
|
237
258
|
servers — which is how you can tell it is a measurement and not this tool's opinion. It also
|
|
238
|
-
shows what you are choosing between: across the
|
|
259
|
+
shows what you are choosing between: across the 86 servers measured, cost spans **1,700×**,
|
|
239
260
|
from `postgres` at 32 tokens to `github` at 54,622. The table below is a
|
|
240
261
|
sample of that range; the full range is in
|
|
241
262
|
[results/leaderboard.md](results/leaderboard.md).
|
|
@@ -250,7 +271,7 @@ sample of that range; the full range is in
|
|
|
250
271
|
| filesystem (reference) | 2,823 | 14 |
|
|
251
272
|
| markitdown | 64 | 1 |
|
|
252
273
|
|
|
253
|
-
*(
|
|
274
|
+
*(86 of 106 popular servers measured, each row dated by its own most recent sweep — full table in
|
|
254
275
|
[results/leaderboard.md](results/leaderboard.md); every failure is listed with its reason.
|
|
255
276
|
Each measured server also has a [detail page](https://athakur3.github.io/mcp-context-cost/servers/)
|
|
256
277
|
showing which tools its tokens are in.)*
|
|
@@ -328,7 +349,7 @@ number is *not*, config policy, failure taxonomy, frozen color bands, known dive
|
|
|
328
349
|
| `spec/fixtures/` | golden vectors shared by the TypeScript and bash implementations |
|
|
329
350
|
| `tools/` | the one script that calls a network API (Claude divergence); kept out of the package so the library stays offline |
|
|
330
351
|
| `upstream/` | `badge.sh` + composite-action patch + bash tests — the self-serve badge recipe, carried here |
|
|
331
|
-
| `servers.yaml` |
|
|
352
|
+
| `servers.yaml` | 106 curated candidates with live install metrics and provenance |
|
|
332
353
|
| `results/` · `badges/` | measurements, leaderboard, history series, shields endpoint JSONs |
|
|
333
354
|
| `docs/` | methodology, dashboard, and a generated page per measured server |
|
|
334
355
|
|
|
@@ -350,11 +371,11 @@ Then in your README:
|
|
|
350
371
|
|
|
351
372
|
A badge says what your server costs today; it does nothing about the release
|
|
352
373
|
that adds 1,200 tokens to every user's context next month. Across the servers
|
|
353
|
-
measured here
|
|
354
|
-
[movement report](results/regressions.md) has
|
|
355
|
-
against
|
|
356
|
-
would have said so first. `measure` takes the same
|
|
357
|
-
your own CI can be that check:
|
|
374
|
+
measured here most costs hold steady from sweep to sweep, but when a cost does
|
|
375
|
+
move it usually moves up: the [movement report](results/regressions.md) has 11
|
|
376
|
+
servers ratcheting upward against 6 that got cheaper, and none of those
|
|
377
|
+
maintainers had a check that would have said so first. `measure` takes the same
|
|
378
|
+
gate flags `audit` does, so your own CI can be that check:
|
|
358
379
|
|
|
359
380
|
```bash
|
|
360
381
|
# on your default branch, once — commit the result
|
package/dist/audit/deferral.js
CHANGED
|
@@ -229,7 +229,20 @@ export function resolveToolSearchSources(sources) {
|
|
|
229
229
|
export const PUBLISHED_WIRE_TO_CLIENT_RATIO = {
|
|
230
230
|
low: 0.2,
|
|
231
231
|
high: 1.92,
|
|
232
|
-
|
|
232
|
+
// A snapshot of the run this package was cut against, which is what `source`
|
|
233
|
+
// below says it is — the installed package has no `results/` to read, so when
|
|
234
|
+
// a live run is supplied `wireToClientRatio` uses that instead and this is
|
|
235
|
+
// never consulted. It may therefore lag the run on trunk, and a test holds it
|
|
236
|
+
// to the two things that matter: the band must still be accurate to the
|
|
237
|
+
// precision it is published at, because a wrong band gives a wrong
|
|
238
|
+
// above/below verdict, and the count must never *exceed* the run, because
|
|
239
|
+
// that would be a claim about servers nobody measured.
|
|
240
|
+
//
|
|
241
|
+
// It read 20 from the day the run covered the top 20 until the run widened to
|
|
242
|
+
// every measured server on 2026-09-05, with nothing comparing the two. The
|
|
243
|
+
// band had not moved — the servers added sat inside it — which is exactly how
|
|
244
|
+
// a number like this goes wrong quietly.
|
|
245
|
+
servers: 23,
|
|
233
246
|
source: 'the published claude-opus-5 divergence run',
|
|
234
247
|
};
|
|
235
248
|
/** Derive the band from a supplied divergence run, falling back to the published one. */
|
|
@@ -42,6 +42,8 @@ export interface DatedMeasurement {
|
|
|
42
42
|
tokens: number;
|
|
43
43
|
toolCount: number;
|
|
44
44
|
status: string;
|
|
45
|
+
/** What the server called itself at `initialize`, or `''` when not recorded. */
|
|
46
|
+
version?: string;
|
|
45
47
|
}
|
|
46
48
|
/** Method identifier, versioned independently of METHODOLOGY_VERSION. */
|
|
47
49
|
export declare const REGRESSION_METHOD = "cost-regression/v1";
|
|
@@ -67,6 +69,14 @@ export interface ToolVectorEntry {
|
|
|
67
69
|
canonicalSha256: string;
|
|
68
70
|
totalTokens: number;
|
|
69
71
|
tools: ToolVector[];
|
|
72
|
+
/**
|
|
73
|
+
* The version the server reported at `initialize`, absent when it reported
|
|
74
|
+
* none or when the entry was stored before this field existed. Entries are
|
|
75
|
+
* deduped by capture and never rewritten, so an existing one does not gain a
|
|
76
|
+
* version retroactively — which is correct: identical bytes are the same
|
|
77
|
+
* definitions, and what upstream called them that day is not on record.
|
|
78
|
+
*/
|
|
79
|
+
version?: string;
|
|
70
80
|
}
|
|
71
81
|
export interface ToolVectorFile {
|
|
72
82
|
method: string;
|
|
@@ -150,6 +160,17 @@ export interface CostChange {
|
|
|
150
160
|
fromToolCount: number;
|
|
151
161
|
toToolCount: number;
|
|
152
162
|
deltaTools: number;
|
|
163
|
+
/**
|
|
164
|
+
* What the server called itself on each side, or `''` where the row does not
|
|
165
|
+
* say. Never inferred: a movement whose earlier row predates the version
|
|
166
|
+
* column names no release, because nothing on disk records which one it was.
|
|
167
|
+
*
|
|
168
|
+
* The two being *equal* is a reading in its own right, and a common one —
|
|
169
|
+
* the cost moved while the version did not, which means the change came from
|
|
170
|
+
* a dependency the server does not pin rather than from its own release.
|
|
171
|
+
*/
|
|
172
|
+
fromVersion: string;
|
|
173
|
+
toVersion: string;
|
|
153
174
|
mechanism: Mechanism;
|
|
154
175
|
/** True when the movement clears both thresholds. */
|
|
155
176
|
significant: boolean;
|
|
@@ -162,11 +183,44 @@ export interface CostChange {
|
|
|
162
183
|
/** Per-tool breakdown, or null when the two captures are not both on record. */
|
|
163
184
|
attribution: ToolAttribution | null;
|
|
164
185
|
}
|
|
186
|
+
/**
|
|
187
|
+
* A cost that has been measured more than once, under the same isolation, and
|
|
188
|
+
* has not moved.
|
|
189
|
+
*
|
|
190
|
+
* Published because it is a measured fact and not the absence of one. The
|
|
191
|
+
* report used to swallow it: `latestChange` returned null both for this and
|
|
192
|
+
* for "there is nothing to compare", and the page counted every null the same
|
|
193
|
+
* way — so a number four sweeps had confirmed read exactly like a number
|
|
194
|
+
* measured once.
|
|
195
|
+
*/
|
|
196
|
+
export interface UnchangedSeries {
|
|
197
|
+
server: string;
|
|
198
|
+
tokens: number;
|
|
199
|
+
toolCount: number;
|
|
200
|
+
/** The first date this cost was recorded — when it became this, not when it was last looked at. */
|
|
201
|
+
since: string;
|
|
202
|
+
/** The newest measurement on record. */
|
|
203
|
+
measuredThrough: string;
|
|
204
|
+
/** How many comparable measurements carry this same cost, `since` included. */
|
|
205
|
+
sweeps: number;
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* What one server's series supports. Three readings, and they are three
|
|
209
|
+
* different claims: the cost moved, the cost was confirmed and held, or there
|
|
210
|
+
* is not yet a second comparable measurement to say either.
|
|
211
|
+
*/
|
|
212
|
+
export type SeriesReading = {
|
|
213
|
+
kind: 'changed';
|
|
214
|
+
change: CostChange;
|
|
215
|
+
} | {
|
|
216
|
+
kind: 'unchanged';
|
|
217
|
+
held: UnchangedSeries;
|
|
218
|
+
} | {
|
|
219
|
+
kind: 'incomparable';
|
|
220
|
+
};
|
|
165
221
|
export declare function isSignificant(deltaTokens: number, deltaPct: number): boolean;
|
|
166
222
|
/**
|
|
167
|
-
*
|
|
168
|
-
* isn't one: fewer than two comparable measurements, or a series that has never
|
|
169
|
-
* changed.
|
|
223
|
+
* What one server's history supports, as one of three readings.
|
|
170
224
|
*
|
|
171
225
|
* Deliberately not "the newest pair". A server that grew 82% one week and held
|
|
172
226
|
* that cost since has a newest pair of zero, and reporting only that would hide
|
|
@@ -176,10 +230,23 @@ export declare function isSignificant(deltaTokens: number, deltaPct: number): bo
|
|
|
176
230
|
* actually happened — `2026-08-19 → 2026-08-26`, not a span up to today.
|
|
177
231
|
* `measuredThrough` then carries how long the new cost has held.
|
|
178
232
|
*
|
|
233
|
+
* Where that walk ends decides the reading. Reaching the start of the series
|
|
234
|
+
* means every comparable measurement agrees, which is `unchanged` — a fact
|
|
235
|
+
* about the server. Having fewer than two comparable rows to walk means there
|
|
236
|
+
* is nothing to compare, which is `incomparable` — an absence. The two used
|
|
237
|
+
* to share one `null` return, and the report counted them together.
|
|
238
|
+
*
|
|
179
239
|
* `rows` is that server's history, already narrowed to the run a trend may be
|
|
180
240
|
* drawn across (`plottableSeries`), so the isolation rule is applied once, in
|
|
181
241
|
* the place that owns it.
|
|
182
242
|
*/
|
|
243
|
+
export declare function readSeries(server: string, rows: DatedMeasurement[], vectors?: ToolVectorFile | null): SeriesReading;
|
|
244
|
+
/**
|
|
245
|
+
* The most recent comparable movement, or null when there was none. Kept as
|
|
246
|
+
* the narrow question `readSeries` answers in full, for the callers that only
|
|
247
|
+
* ever wanted a movement — but a caller that publishes a count must ask
|
|
248
|
+
* `readSeries`, because null here is two different facts.
|
|
249
|
+
*/
|
|
183
250
|
export declare function latestChange(server: string, rows: DatedMeasurement[], vectors?: ToolVectorFile | null): CostChange | null;
|
|
184
251
|
export interface RegressionSummary {
|
|
185
252
|
changes: CostChange[];
|
|
@@ -188,7 +255,22 @@ export interface RegressionSummary {
|
|
|
188
255
|
significant: number;
|
|
189
256
|
/** Net tokens the measured set gained (or lost) across every listed movement. */
|
|
190
257
|
netTokens: number;
|
|
191
|
-
/**
|
|
258
|
+
/**
|
|
259
|
+
* Servers measured more than once under the same isolation whose cost has
|
|
260
|
+
* not moved. Counted apart from `withoutComparison`, which they were folded
|
|
261
|
+
* into while both readings shared one `null`.
|
|
262
|
+
*/
|
|
263
|
+
unchanged: UnchangedSeries[];
|
|
264
|
+
/**
|
|
265
|
+
* Servers with no second comparable measurement to diff against — a first
|
|
266
|
+
* measurement, or every earlier run taken under different isolation. Not
|
|
267
|
+
* "has not moved": that is `unchanged`.
|
|
268
|
+
*/
|
|
192
269
|
withoutComparison: number;
|
|
193
270
|
}
|
|
194
|
-
|
|
271
|
+
/**
|
|
272
|
+
* `unchanged` is required rather than defaulted: every reading a caller
|
|
273
|
+
* classifies belongs to exactly one of the three, and a default would let a
|
|
274
|
+
* caller that forgot them publish a total that quietly does not sum.
|
|
275
|
+
*/
|
|
276
|
+
export declare function summarize(changes: CostChange[], withoutComparison: number, unchanged: UnchangedSeries[]): RegressionSummary;
|
package/dist/core/regression.js
CHANGED
|
@@ -38,6 +38,7 @@ export function parseToolVectorFile(text) {
|
|
|
38
38
|
date: e.date,
|
|
39
39
|
canonicalSha256: e.canonicalSha256,
|
|
40
40
|
totalTokens: e.totalTokens,
|
|
41
|
+
...(typeof e.version === 'string' && e.version ? { version: e.version } : {}),
|
|
41
42
|
tools: e.tools
|
|
42
43
|
.filter((t) => !!t && typeof t.name === 'string' && typeof t.tokens === 'number')
|
|
43
44
|
.map((t) => ({ name: t.name, tokens: t.tokens })),
|
|
@@ -62,6 +63,7 @@ export function vectorEntryOf(m) {
|
|
|
62
63
|
date,
|
|
63
64
|
canonicalSha256: m.canonicalSha256,
|
|
64
65
|
totalTokens: m.totalTokens,
|
|
66
|
+
...(m.serverVersion ? { version: m.serverVersion } : {}),
|
|
65
67
|
tools: m.tools.map((t) => ({ name: t.name, tokens: t.tokens })),
|
|
66
68
|
};
|
|
67
69
|
}
|
|
@@ -139,9 +141,7 @@ export function isSignificant(deltaTokens, deltaPct) {
|
|
|
139
141
|
return Math.abs(deltaPct) >= SIGNIFICANT_PCT && Math.abs(deltaTokens) >= SIGNIFICANT_TOKENS;
|
|
140
142
|
}
|
|
141
143
|
/**
|
|
142
|
-
*
|
|
143
|
-
* isn't one: fewer than two comparable measurements, or a series that has never
|
|
144
|
-
* changed.
|
|
144
|
+
* What one server's history supports, as one of three readings.
|
|
145
145
|
*
|
|
146
146
|
* Deliberately not "the newest pair". A server that grew 82% one week and held
|
|
147
147
|
* that cost since has a newest pair of zero, and reporting only that would hide
|
|
@@ -151,14 +151,20 @@ export function isSignificant(deltaTokens, deltaPct) {
|
|
|
151
151
|
* actually happened — `2026-08-19 → 2026-08-26`, not a span up to today.
|
|
152
152
|
* `measuredThrough` then carries how long the new cost has held.
|
|
153
153
|
*
|
|
154
|
+
* Where that walk ends decides the reading. Reaching the start of the series
|
|
155
|
+
* means every comparable measurement agrees, which is `unchanged` — a fact
|
|
156
|
+
* about the server. Having fewer than two comparable rows to walk means there
|
|
157
|
+
* is nothing to compare, which is `incomparable` — an absence. The two used
|
|
158
|
+
* to share one `null` return, and the report counted them together.
|
|
159
|
+
*
|
|
154
160
|
* `rows` is that server's history, already narrowed to the run a trend may be
|
|
155
161
|
* drawn across (`plottableSeries`), so the isolation rule is applied once, in
|
|
156
162
|
* the place that owns it.
|
|
157
163
|
*/
|
|
158
|
-
export function
|
|
164
|
+
export function readSeries(server, rows, vectors) {
|
|
159
165
|
const usable = rows.filter((r) => r.status === 'measured' || r.status === 'dynamic');
|
|
160
166
|
if (usable.length < 2)
|
|
161
|
-
return
|
|
167
|
+
return { kind: 'incomparable' };
|
|
162
168
|
const newest = usable[usable.length - 1];
|
|
163
169
|
// Walk back over measurements identical to the newest: the first of that run
|
|
164
170
|
// is when the current cost arrived.
|
|
@@ -169,14 +175,26 @@ export function latestChange(server, rows, vectors) {
|
|
|
169
175
|
break;
|
|
170
176
|
toIdx--;
|
|
171
177
|
}
|
|
178
|
+
/** The trailing run reaches back this far, and every row in it agrees. */
|
|
179
|
+
const held = () => ({
|
|
180
|
+
kind: 'unchanged',
|
|
181
|
+
held: {
|
|
182
|
+
server,
|
|
183
|
+
tokens: newest.tokens,
|
|
184
|
+
toolCount: newest.toolCount,
|
|
185
|
+
since: usable[toIdx].date,
|
|
186
|
+
measuredThrough: newest.date,
|
|
187
|
+
sweeps: usable.length - toIdx,
|
|
188
|
+
},
|
|
189
|
+
});
|
|
172
190
|
if (toIdx === 0)
|
|
173
|
-
return
|
|
191
|
+
return held(); // every comparable measurement agrees
|
|
174
192
|
const to = usable[toIdx];
|
|
175
193
|
const from = usable[toIdx - 1];
|
|
176
194
|
const deltaTokens = to.tokens - from.tokens;
|
|
177
195
|
const deltaTools = to.toolCount - from.toolCount;
|
|
178
196
|
if (deltaTokens === 0 && deltaTools === 0)
|
|
179
|
-
return
|
|
197
|
+
return held();
|
|
180
198
|
const deltaPct = from.tokens > 0 ? (deltaTokens / from.tokens) * 100 : 0;
|
|
181
199
|
// Attribution needs both sides on record. Matched by *cost as of that date*,
|
|
182
200
|
// not by date equality: vectors are deduped by capture and keep the first
|
|
@@ -209,23 +227,43 @@ export function latestChange(server, rows, vectors) {
|
|
|
209
227
|
attribution = attribute(fromVec, toVec, deltaTokens);
|
|
210
228
|
}
|
|
211
229
|
return {
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
230
|
+
kind: 'changed',
|
|
231
|
+
change: {
|
|
232
|
+
server,
|
|
233
|
+
fromDate: from.date,
|
|
234
|
+
toDate: to.date,
|
|
235
|
+
fromTokens: from.tokens,
|
|
236
|
+
toTokens: to.tokens,
|
|
237
|
+
deltaTokens,
|
|
238
|
+
deltaPct,
|
|
239
|
+
fromToolCount: from.toolCount,
|
|
240
|
+
toToolCount: to.toolCount,
|
|
241
|
+
deltaTools,
|
|
242
|
+
fromVersion: from.version ?? '',
|
|
243
|
+
toVersion: to.version ?? '',
|
|
244
|
+
mechanism: mechanismOf(deltaTokens, deltaTools),
|
|
245
|
+
significant: isSignificant(deltaTokens, deltaPct),
|
|
246
|
+
measuredThrough: newest.date,
|
|
247
|
+
attribution,
|
|
248
|
+
},
|
|
226
249
|
};
|
|
227
250
|
}
|
|
228
|
-
|
|
251
|
+
/**
|
|
252
|
+
* The most recent comparable movement, or null when there was none. Kept as
|
|
253
|
+
* the narrow question `readSeries` answers in full, for the callers that only
|
|
254
|
+
* ever wanted a movement — but a caller that publishes a count must ask
|
|
255
|
+
* `readSeries`, because null here is two different facts.
|
|
256
|
+
*/
|
|
257
|
+
export function latestChange(server, rows, vectors) {
|
|
258
|
+
const reading = readSeries(server, rows, vectors);
|
|
259
|
+
return reading.kind === 'changed' ? reading.change : null;
|
|
260
|
+
}
|
|
261
|
+
/**
|
|
262
|
+
* `unchanged` is required rather than defaulted: every reading a caller
|
|
263
|
+
* classifies belongs to exactly one of the three, and a default would let a
|
|
264
|
+
* caller that forgot them publish a total that quietly does not sum.
|
|
265
|
+
*/
|
|
266
|
+
export function summarize(changes, withoutComparison, unchanged) {
|
|
229
267
|
const sorted = [...changes].sort((a, b) => Math.abs(b.deltaPct) - Math.abs(a.deltaPct));
|
|
230
268
|
return {
|
|
231
269
|
changes: sorted,
|
|
@@ -233,6 +271,9 @@ export function summarize(changes, withoutComparison) {
|
|
|
233
271
|
shrank: sorted.filter((c) => c.deltaTokens < 0).length,
|
|
234
272
|
significant: sorted.filter((c) => c.significant).length,
|
|
235
273
|
netTokens: sorted.reduce((a, c) => a + c.deltaTokens, 0),
|
|
274
|
+
// Longest-held first: the sentence this section exists to publish is "this
|
|
275
|
+
// number has been the same since <date>", so the oldest `since` leads.
|
|
276
|
+
unchanged: [...unchanged].sort((a, b) => a.since.localeCompare(b.since) || b.tokens - a.tokens),
|
|
236
277
|
withoutComparison,
|
|
237
278
|
};
|
|
238
279
|
}
|
package/dist/core/types.d.ts
CHANGED
|
@@ -56,10 +56,10 @@ export interface Measurement {
|
|
|
56
56
|
/**
|
|
57
57
|
* The architecture the measurement ran on, as `<platform>/<arch>` (e.g.
|
|
58
58
|
* `linux/amd64`). Part of the isolation because some packages ship builds
|
|
59
|
-
* for only some of them: `local-mcp` was published as a startup failure
|
|
60
|
-
*
|
|
61
|
-
*
|
|
62
|
-
*
|
|
59
|
+
* for only some of them: `local-mcp` was published as a startup failure on
|
|
60
|
+
* the strength of a run whose real finding was "this laptop is arm64 and
|
|
61
|
+
* the package has no arm64 runtime" — a fact about the machine that the
|
|
62
|
+
* record gave no way to see.
|
|
63
63
|
*
|
|
64
64
|
* Absent on records written before this was captured, which is why it is
|
|
65
65
|
* optional; absence means unknown, never "the same as yours".
|
|
@@ -162,6 +162,7 @@ export function runCli(binPath, entry, opts) {
|
|
|
162
162
|
dummyEnv: entry.env ?? [],
|
|
163
163
|
dummyEnvValues: entry.envValues,
|
|
164
164
|
needsGit: entry.needsGit,
|
|
165
|
+
aptPackages: entry.aptPackages,
|
|
165
166
|
containerName,
|
|
166
167
|
binds: [`${binPath}:/opt/mcp-tokens:ro`],
|
|
167
168
|
});
|
|
@@ -287,6 +288,8 @@ if (isMain) {
|
|
|
287
288
|
dummyEnv: e.env ?? [],
|
|
288
289
|
dummyEnvValues: e.envValues,
|
|
289
290
|
needsGit: e.needsGit,
|
|
291
|
+
aptPackages: e.aptPackages,
|
|
292
|
+
notApplicable: e.notApplicable,
|
|
290
293
|
persist: false, // the measurements on disk are not this run's to rewrite
|
|
291
294
|
});
|
|
292
295
|
}
|
package/dist/sweep/dashboard.js
CHANGED
|
@@ -66,7 +66,22 @@ export function generateDashboard(root = process.cwd()) {
|
|
|
66
66
|
const median = totals.length ? totals.slice().sort((a, b) => a - b)[Math.floor(totals.length / 2)] : 0;
|
|
67
67
|
const max = totals.length ? Math.max(...totals) : 1;
|
|
68
68
|
const fmt = (n) => n.toLocaleString('en-US');
|
|
69
|
-
|
|
69
|
+
/**
|
|
70
|
+
* The stamp is the newest measurement on the page, not the moment the page
|
|
71
|
+
* was written.
|
|
72
|
+
*
|
|
73
|
+
* A wall-clock stamp made `docs/dashboard.html` differ on every regeneration,
|
|
74
|
+
* so the scheduled jobs' "no change to publish" guard could never fire: one
|
|
75
|
+
* file that always diffs is enough to publish a commit that says nothing
|
|
76
|
+
* happened. Dated to the data instead, the page changes when the data does —
|
|
77
|
+
* and the line now answers the question a reader actually has, which is how
|
|
78
|
+
* fresh these numbers are, not when the HTML was rendered.
|
|
79
|
+
*/
|
|
80
|
+
const dates = rows
|
|
81
|
+
.map((r) => String(r.m?.measuredAt ?? '').slice(0, 10))
|
|
82
|
+
.filter((d) => /^\d{4}-\d{2}-\d{2}$/.test(d))
|
|
83
|
+
.sort();
|
|
84
|
+
const newestMeasurement = dates[dates.length - 1] ?? null;
|
|
70
85
|
const barRows = measured
|
|
71
86
|
.map((r, i) => {
|
|
72
87
|
const m = r.m;
|
|
@@ -225,7 +240,7 @@ export function generateDashboard(root = process.cwd()) {
|
|
|
225
240
|
</style>
|
|
226
241
|
<div class="wrap">
|
|
227
242
|
<header>
|
|
228
|
-
<p class="eyebrow">methodology v1.0 · o200k_base ·
|
|
243
|
+
<p class="eyebrow">methodology v1.0 · o200k_base · newest measurement ${newestMeasurement ?? 'not yet taken'}</p>
|
|
229
244
|
<h1>mcp-context-cost</h1>
|
|
230
245
|
<p class="sub">What popular MCP servers cost in context tokens before the agent does any work — measured from raw <code>tools/list</code> captures, every number re-derivable from its published measurement file.</p>
|
|
231
246
|
</header>
|
package/dist/sweep/docker.d.ts
CHANGED
|
@@ -30,6 +30,23 @@ export interface DockerOptions {
|
|
|
30
30
|
* git already on PATH would actually run.
|
|
31
31
|
*/
|
|
32
32
|
needsGit?: boolean;
|
|
33
|
+
/**
|
|
34
|
+
* Debian packages to install before launch, for a server whose runtime needs
|
|
35
|
+
* a native library the slim base image does not carry.
|
|
36
|
+
*
|
|
37
|
+
* The same argument as `needsGit`, and the same shape: these are libraries a
|
|
38
|
+
* user's own machine already has, so installing them moves the container
|
|
39
|
+
* *towards* the conditions a plain `npx -y` run would find rather than away
|
|
40
|
+
* from them. `azure` is the case — a .NET server that fails on
|
|
41
|
+
* `node:22-slim` with "Couldn't find a valid ICU package", and again on
|
|
42
|
+
* libssl once ICU is satisfied. With both present it measures.
|
|
43
|
+
*
|
|
44
|
+
* Distinct from an env var that changes the server's own behaviour (see
|
|
45
|
+
* `elasticsearch` and `OTEL_SDK_DISABLED`), which is a different decision.
|
|
46
|
+
* The isolation record names what was installed either way, so a reader can
|
|
47
|
+
* see that the container was not the plain one.
|
|
48
|
+
*/
|
|
49
|
+
aptPackages?: string[];
|
|
33
50
|
/**
|
|
34
51
|
* Extra `-v` bind mounts, verbatim (`host:container:ro`). Used to hand a
|
|
35
52
|
* host-verified binary into the container (the cross-check CLI); mounts here
|
|
@@ -82,6 +99,7 @@ export interface EnsureImageOptions {
|
|
|
82
99
|
/** Run one docker invocation — injectable so tests never need a daemon. */
|
|
83
100
|
run?: (args: string[]) => Promise<{
|
|
84
101
|
code: number | null;
|
|
102
|
+
stdout?: string;
|
|
85
103
|
stderr: string;
|
|
86
104
|
}>;
|
|
87
105
|
/** Waits between pull attempts; attempts = delays + 1. */
|
|
@@ -101,6 +119,34 @@ export interface EnsureImageOptions {
|
|
|
101
119
|
* re-attempted by every remaining server in the sweep.
|
|
102
120
|
*/
|
|
103
121
|
export declare function ensureImage(image: string, opts?: EnsureImageOptions): Promise<void>;
|
|
122
|
+
/** Parse `uname -sm` output into `<platform>/<arch>`, or null if it is not that. */
|
|
123
|
+
export declare function platformFromUname(output: string): string | null;
|
|
124
|
+
/**
|
|
125
|
+
* The platform a container for this image actually runs as, asked of a
|
|
126
|
+
* container rather than inferred from the machine that starts one.
|
|
127
|
+
*
|
|
128
|
+
* `isolation.arch` exists to tell a broken server apart from one that ships no
|
|
129
|
+
* build for the architecture it was tried on, and it was being derived instead
|
|
130
|
+
* of observed: the platform half was assumed to be `linux` and the
|
|
131
|
+
* architecture half was the *host's* `process.arch`. Nothing here passes
|
|
132
|
+
* `--platform`, but `docker run` honours `DOCKER_DEFAULT_PLATFORM` and an
|
|
133
|
+
* image with no manifest for the host is emulated — so an amd64 container on
|
|
134
|
+
* an Apple Silicon laptop recorded `linux/arm64`. A field that can be wrong
|
|
135
|
+
* about the one thing it exists to establish is worse than no field.
|
|
136
|
+
*
|
|
137
|
+
* The obvious cheap answer does not work, which is why this starts a
|
|
138
|
+
* container. `docker image inspect` reports the variant the local store
|
|
139
|
+
* prefers, and on this machine 2026-09-05 it answered `linux/arm64` for a tag
|
|
140
|
+
* whose container, under `DOCKER_DEFAULT_PLATFORM=linux/amd64`, came up as
|
|
141
|
+
* `x86_64` — the same wrong answer as the host inference, reached a different
|
|
142
|
+
* way. Only the container knows.
|
|
143
|
+
*
|
|
144
|
+
* One `docker run` per image, memoized for the life of the process, so a sweep
|
|
145
|
+
* pays it about twice: once for the node base and once for the python one.
|
|
146
|
+
* Returns null when docker cannot say, leaving `arch` absent — which the
|
|
147
|
+
* record already documents as "unknown, never the same as yours".
|
|
148
|
+
*/
|
|
149
|
+
export declare function containerPlatform(image: string, opts?: EnsureImageOptions): Promise<string | null>;
|
|
104
150
|
export interface IsolationRecord {
|
|
105
151
|
docker: boolean;
|
|
106
152
|
image?: string;
|
package/dist/sweep/docker.js
CHANGED
|
@@ -33,12 +33,15 @@ export function isDockerRunFailure(message) {
|
|
|
33
33
|
}
|
|
34
34
|
function runDocker(args) {
|
|
35
35
|
return import('node:child_process').then(({ spawn }) => new Promise((resolve) => {
|
|
36
|
-
const child = spawn('docker', args, { stdio: ['ignore', '
|
|
36
|
+
const child = spawn('docker', args, { stdio: ['ignore', 'pipe', 'pipe'] });
|
|
37
|
+
let stdout = '';
|
|
37
38
|
let stderr = '';
|
|
39
|
+
child.stdout.setEncoding('utf8');
|
|
40
|
+
child.stdout.on('data', (chunk) => (stdout = (stdout + chunk).slice(-4000)));
|
|
38
41
|
child.stderr.setEncoding('utf8');
|
|
39
42
|
child.stderr.on('data', (chunk) => (stderr = (stderr + chunk).slice(-4000)));
|
|
40
|
-
child.on('error', (err) => resolve({ code: null, stderr: String(err.message) }));
|
|
41
|
-
child.on('exit', (code) => resolve({ code, stderr }));
|
|
43
|
+
child.on('error', (err) => resolve({ code: null, stdout, stderr: String(err.message) }));
|
|
44
|
+
child.on('exit', (code) => resolve({ code, stdout, stderr }));
|
|
42
45
|
}));
|
|
43
46
|
}
|
|
44
47
|
async function ensureImageOnce(image, opts) {
|
|
@@ -92,6 +95,71 @@ export function ensureImage(image, opts = {}) {
|
|
|
92
95
|
}
|
|
93
96
|
return p;
|
|
94
97
|
}
|
|
98
|
+
/**
|
|
99
|
+
* `uname` names, in Docker's vocabulary. Explicit rather than derived: an
|
|
100
|
+
* unrecognised machine name returns nothing, because a mapping guessed from a
|
|
101
|
+
* pattern would be the same kind of inference this function exists to remove.
|
|
102
|
+
*/
|
|
103
|
+
const UNAME_ARCH = {
|
|
104
|
+
x86_64: 'amd64',
|
|
105
|
+
amd64: 'amd64',
|
|
106
|
+
aarch64: 'arm64',
|
|
107
|
+
arm64: 'arm64',
|
|
108
|
+
armv7l: 'arm/v7',
|
|
109
|
+
armv6l: 'arm/v6',
|
|
110
|
+
i386: '386',
|
|
111
|
+
i686: '386',
|
|
112
|
+
ppc64le: 'ppc64le',
|
|
113
|
+
s390x: 's390x',
|
|
114
|
+
riscv64: 'riscv64',
|
|
115
|
+
};
|
|
116
|
+
/** Parse `uname -sm` output into `<platform>/<arch>`, or null if it is not that. */
|
|
117
|
+
export function platformFromUname(output) {
|
|
118
|
+
const [kernel, machine] = output.trim().split(/\s+/);
|
|
119
|
+
const arch = UNAME_ARCH[(machine ?? '').toLowerCase()];
|
|
120
|
+
if (!kernel || !arch)
|
|
121
|
+
return null;
|
|
122
|
+
return `${kernel.toLowerCase()}/${arch}`;
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* The platform a container for this image actually runs as, asked of a
|
|
126
|
+
* container rather than inferred from the machine that starts one.
|
|
127
|
+
*
|
|
128
|
+
* `isolation.arch` exists to tell a broken server apart from one that ships no
|
|
129
|
+
* build for the architecture it was tried on, and it was being derived instead
|
|
130
|
+
* of observed: the platform half was assumed to be `linux` and the
|
|
131
|
+
* architecture half was the *host's* `process.arch`. Nothing here passes
|
|
132
|
+
* `--platform`, but `docker run` honours `DOCKER_DEFAULT_PLATFORM` and an
|
|
133
|
+
* image with no manifest for the host is emulated — so an amd64 container on
|
|
134
|
+
* an Apple Silicon laptop recorded `linux/arm64`. A field that can be wrong
|
|
135
|
+
* about the one thing it exists to establish is worse than no field.
|
|
136
|
+
*
|
|
137
|
+
* The obvious cheap answer does not work, which is why this starts a
|
|
138
|
+
* container. `docker image inspect` reports the variant the local store
|
|
139
|
+
* prefers, and on this machine 2026-09-05 it answered `linux/arm64` for a tag
|
|
140
|
+
* whose container, under `DOCKER_DEFAULT_PLATFORM=linux/amd64`, came up as
|
|
141
|
+
* `x86_64` — the same wrong answer as the host inference, reached a different
|
|
142
|
+
* way. Only the container knows.
|
|
143
|
+
*
|
|
144
|
+
* One `docker run` per image, memoized for the life of the process, so a sweep
|
|
145
|
+
* pays it about twice: once for the node base and once for the python one.
|
|
146
|
+
* Returns null when docker cannot say, leaving `arch` absent — which the
|
|
147
|
+
* record already documents as "unknown, never the same as yours".
|
|
148
|
+
*/
|
|
149
|
+
export async function containerPlatform(image, opts = {}) {
|
|
150
|
+
const run = opts.run ?? runDocker;
|
|
151
|
+
if (!opts.run) {
|
|
152
|
+
const cached = platforms.get(image);
|
|
153
|
+
if (cached !== undefined)
|
|
154
|
+
return cached;
|
|
155
|
+
}
|
|
156
|
+
const r = await run(['run', '--rm', '--pull=missing', image, 'uname', '-sm']);
|
|
157
|
+
const platform = r.code === 0 ? platformFromUname(String(r.stdout ?? '')) : null;
|
|
158
|
+
if (!opts.run)
|
|
159
|
+
platforms.set(image, platform);
|
|
160
|
+
return platform;
|
|
161
|
+
}
|
|
162
|
+
const platforms = new Map();
|
|
95
163
|
/**
|
|
96
164
|
* Wrap a launch command line in `docker run`. The inner command is passed to
|
|
97
165
|
* `sh -lc` inside the container; quoting is preserved by argv (no host shell).
|
|
@@ -135,7 +203,14 @@ export function dockerize(commandLine, opts = {}) {
|
|
|
135
203
|
const gitPrefix = opts.needsGit
|
|
136
204
|
? 'command -v git >/dev/null 2>&1 || (apt-get update -qq && apt-get install -y -qq --no-install-recommends git >/dev/null 2>&1); '
|
|
137
205
|
: '';
|
|
138
|
-
|
|
206
|
+
// Package names are validated by the schema check before they reach here, and
|
|
207
|
+
// they are joined into a shell word list — so the schema's character class is
|
|
208
|
+
// what keeps this from being an injection point.
|
|
209
|
+
const packages = opts.aptPackages ?? [];
|
|
210
|
+
const aptPrefix = packages.length
|
|
211
|
+
? `apt-get update -qq >/dev/null 2>&1 && apt-get install -y -qq --no-install-recommends ${packages.join(' ')} >/dev/null 2>&1; `
|
|
212
|
+
: '';
|
|
213
|
+
argv.push(image, 'sh', '-lc', gitPrefix + aptPrefix + commandLine);
|
|
139
214
|
return {
|
|
140
215
|
command: 'docker',
|
|
141
216
|
argv,
|
|
@@ -145,6 +220,9 @@ export function dockerize(commandLine, opts = {}) {
|
|
|
145
220
|
network: 'bridge',
|
|
146
221
|
note: 'network enabled for package fetch; clean FS, no host credentials' +
|
|
147
222
|
(opts.needsGit ? ', git installed' : '') +
|
|
223
|
+
// Named rather than summarised: a record whose container carried extra
|
|
224
|
+
// libraries has to say which, or the number is not reproducible from it.
|
|
225
|
+
(packages.length ? `, installed ${packages.join(' ')}` : '') +
|
|
148
226
|
(opts.noSharedCache ? ', shared package cache bypassed' : ''),
|
|
149
227
|
},
|
|
150
228
|
};
|