mcp-context-cost 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +131 -38
- package/dist/audit/audit.d.ts +63 -0
- package/dist/audit/audit.js +120 -9
- package/dist/audit/diff.d.ts +124 -0
- package/dist/audit/diff.js +318 -0
- package/dist/audit/run.d.ts +12 -0
- package/dist/audit/run.js +28 -1
- package/dist/cli.d.ts +21 -0
- package/dist/cli.js +141 -7
- package/dist/sweep/dashboard.d.ts +8 -0
- package/dist/sweep/dashboard.js +53 -5
- package/dist/sweep/docker.d.ts +8 -0
- package/dist/sweep/docker.js +7 -2
- package/dist/sweep/report.d.ts +2 -0
- package/dist/sweep/run.d.ts +2 -0
- package/dist/sweep/run.js +6 -1
- package/dist/sweep/sweep-all.js +1 -0
- package/package.json +1 -1
|
@@ -0,0 +1,318 @@
|
|
|
1
|
+
/** Parse and shape-check a stored report. A baseline that cannot be read is never "no change". */
|
|
2
|
+
export function parseBaselineReport(text) {
|
|
3
|
+
let doc;
|
|
4
|
+
try {
|
|
5
|
+
doc = JSON.parse(text);
|
|
6
|
+
}
|
|
7
|
+
catch (e) {
|
|
8
|
+
return { report: null, problem: `baseline is not JSON: ${e.message}` };
|
|
9
|
+
}
|
|
10
|
+
if (!doc || typeof doc !== 'object' || Array.isArray(doc)) {
|
|
11
|
+
return { report: null, problem: 'baseline is not an audit report object' };
|
|
12
|
+
}
|
|
13
|
+
const r = doc;
|
|
14
|
+
if (!Array.isArray(r.configs)) {
|
|
15
|
+
return { report: null, problem: "baseline has no 'configs' array — is it the output of `audit --json`?" };
|
|
16
|
+
}
|
|
17
|
+
if (typeof r.methodologyVersion !== 'string' || typeof r.encoding !== 'string') {
|
|
18
|
+
return { report: null, problem: 'baseline is missing methodologyVersion/encoding — is it the output of `audit --json`?' };
|
|
19
|
+
}
|
|
20
|
+
for (const c of r.configs) {
|
|
21
|
+
if (!c || typeof c !== 'object' || typeof c.source !== 'string') {
|
|
22
|
+
return { report: null, problem: 'baseline has a config entry without a source path' };
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
return { report: doc };
|
|
26
|
+
}
|
|
27
|
+
function statesOf(cfg) {
|
|
28
|
+
const out = new Map();
|
|
29
|
+
for (const s of cfg.servers ?? [])
|
|
30
|
+
out.set(s.name, { present: true, tokens: typeof s.tokens === 'number' ? s.tokens : null });
|
|
31
|
+
// A skipped server IS in the config; it just has no number. Keeping it distinct
|
|
32
|
+
// from absent is the whole reason `removed` and `unmeasured-now` are separate kinds.
|
|
33
|
+
for (const s of cfg.skipped ?? [])
|
|
34
|
+
if (!out.has(s.name))
|
|
35
|
+
out.set(s.name, { present: true, tokens: null });
|
|
36
|
+
return out;
|
|
37
|
+
}
|
|
38
|
+
export function diffConfig(before, after, matchedBy) {
|
|
39
|
+
const afterShare = after.contextShare;
|
|
40
|
+
if (!before) {
|
|
41
|
+
return {
|
|
42
|
+
client: after.client,
|
|
43
|
+
source: after.source,
|
|
44
|
+
matchedBy: 'unmatched',
|
|
45
|
+
beforeTotal: null,
|
|
46
|
+
afterTotal: after.totalTokens,
|
|
47
|
+
delta: null,
|
|
48
|
+
beforeShare: null,
|
|
49
|
+
afterShare,
|
|
50
|
+
exact: false,
|
|
51
|
+
understatedBy: 0,
|
|
52
|
+
overstatedBy: 0,
|
|
53
|
+
servers: [],
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
const b = statesOf(before);
|
|
57
|
+
const a = statesOf(after);
|
|
58
|
+
const servers = [];
|
|
59
|
+
let understatedBy = 0;
|
|
60
|
+
let overstatedBy = 0;
|
|
61
|
+
let exact = true;
|
|
62
|
+
for (const name of new Set([...b.keys(), ...a.keys()])) {
|
|
63
|
+
const bs = b.get(name);
|
|
64
|
+
const as = a.get(name);
|
|
65
|
+
if (bs && !as) {
|
|
66
|
+
servers.push(bs.tokens === null
|
|
67
|
+
? { name, kind: 'removed', before: null, after: null, delta: null, note: 'was in the config but never measured — removing it changed no measured cost' }
|
|
68
|
+
: { name, kind: 'removed', before: bs.tokens, after: null, delta: -bs.tokens });
|
|
69
|
+
continue;
|
|
70
|
+
}
|
|
71
|
+
if (!bs && as) {
|
|
72
|
+
servers.push(as.tokens === null
|
|
73
|
+
? { name, kind: 'added', before: null, after: null, delta: null, note: 'added but not measurable — its cost is unknown, not zero' }
|
|
74
|
+
: { name, kind: 'added', before: null, after: as.tokens, delta: as.tokens });
|
|
75
|
+
continue;
|
|
76
|
+
}
|
|
77
|
+
if (!bs || !as)
|
|
78
|
+
continue;
|
|
79
|
+
if (bs.tokens !== null && as.tokens !== null) {
|
|
80
|
+
const delta = as.tokens - bs.tokens;
|
|
81
|
+
servers.push({ name, kind: delta === 0 ? 'unchanged' : 'changed', before: bs.tokens, after: as.tokens, delta });
|
|
82
|
+
}
|
|
83
|
+
else if (bs.tokens !== null && as.tokens === null) {
|
|
84
|
+
exact = false;
|
|
85
|
+
understatedBy += bs.tokens;
|
|
86
|
+
servers.push({
|
|
87
|
+
name,
|
|
88
|
+
kind: 'unmeasured-now',
|
|
89
|
+
before: bs.tokens,
|
|
90
|
+
after: null,
|
|
91
|
+
delta: null,
|
|
92
|
+
note: `measured ${bs.tokens.toLocaleString('en-US')} in the baseline and could not be measured now — its cost is missing from the total, not gone from your config`,
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
else if (bs.tokens === null && as.tokens !== null) {
|
|
96
|
+
exact = false;
|
|
97
|
+
overstatedBy += as.tokens;
|
|
98
|
+
servers.push({
|
|
99
|
+
name,
|
|
100
|
+
kind: 'unmeasured-before',
|
|
101
|
+
before: null,
|
|
102
|
+
after: as.tokens,
|
|
103
|
+
delta: null,
|
|
104
|
+
note: `could not be measured in the baseline and measures ${as.tokens.toLocaleString('en-US')} now — this cost is newly visible, not necessarily new`,
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
else {
|
|
108
|
+
servers.push({
|
|
109
|
+
name,
|
|
110
|
+
kind: 'unmeasured-both',
|
|
111
|
+
before: null,
|
|
112
|
+
after: null,
|
|
113
|
+
delta: null,
|
|
114
|
+
note: 'not measurable in either run — contributes 0 to both totals and hides an unknown cost',
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
// Biggest movers first; ties and non-deltas fall to the bottom in name order.
|
|
119
|
+
servers.sort((x, y) => Math.abs(y.delta ?? 0) - Math.abs(x.delta ?? 0) || x.name.localeCompare(y.name));
|
|
120
|
+
return {
|
|
121
|
+
client: after.client,
|
|
122
|
+
source: after.source,
|
|
123
|
+
matchedBy,
|
|
124
|
+
beforeTotal: before.totalTokens,
|
|
125
|
+
afterTotal: after.totalTokens,
|
|
126
|
+
delta: after.totalTokens - before.totalTokens,
|
|
127
|
+
beforeShare: before.contextShare ?? null,
|
|
128
|
+
afterShare,
|
|
129
|
+
exact,
|
|
130
|
+
understatedBy,
|
|
131
|
+
overstatedBy,
|
|
132
|
+
servers,
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Pair current configs with baseline configs.
|
|
137
|
+
*
|
|
138
|
+
* Exact source path first. Then one deliberate fallback: if each side has
|
|
139
|
+
* exactly one config, they are the same config seen from two machines — the CI
|
|
140
|
+
* case, where a baseline recorded at /Users/… meets a checkout at /home/runner/….
|
|
141
|
+
* Anything looser would pair two unrelated clients and call the difference a
|
|
142
|
+
* change, so everything else stays unmatched and says so.
|
|
143
|
+
*/
|
|
144
|
+
export function pairConfigs(before, after) {
|
|
145
|
+
const unusedBefore = new Map(before.map((c) => [c.source, c]));
|
|
146
|
+
const pairs = [];
|
|
147
|
+
for (const cur of after) {
|
|
148
|
+
const hit = unusedBefore.get(cur.source);
|
|
149
|
+
if (hit) {
|
|
150
|
+
unusedBefore.delete(cur.source);
|
|
151
|
+
pairs.push({ before: hit, after: cur, matchedBy: 'source' });
|
|
152
|
+
}
|
|
153
|
+
else {
|
|
154
|
+
pairs.push({ before: null, after: cur, matchedBy: 'unmatched' });
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
if (before.length === 1 && after.length === 1 && pairs[0].before === null) {
|
|
158
|
+
pairs[0] = { before: before[0], after: after[0], matchedBy: 'sole-config' };
|
|
159
|
+
unusedBefore.delete(before[0].source);
|
|
160
|
+
}
|
|
161
|
+
return { pairs, dropped: [...unusedBefore.values()] };
|
|
162
|
+
}
|
|
163
|
+
export function buildDiff(baseline, current) {
|
|
164
|
+
const warnings = [];
|
|
165
|
+
let comparable = true;
|
|
166
|
+
if (baseline.methodologyVersion !== current.methodologyVersion) {
|
|
167
|
+
comparable = false;
|
|
168
|
+
warnings.push(`methodology changed (${baseline.methodologyVersion} → ${current.methodologyVersion}) — token counts from the two runs are not the same measurement`);
|
|
169
|
+
}
|
|
170
|
+
if (baseline.encoding !== current.encoding) {
|
|
171
|
+
comparable = false;
|
|
172
|
+
warnings.push(`encoding changed (${baseline.encoding} → ${current.encoding}) — the counts are in different units`);
|
|
173
|
+
}
|
|
174
|
+
if (baseline.contextWindow !== current.contextWindow) {
|
|
175
|
+
// Shares move, token counts do not. Worth saying, not worth invalidating.
|
|
176
|
+
warnings.push(`context window changed (${baseline.contextWindow.toLocaleString('en-US')} → ${current.contextWindow.toLocaleString('en-US')}) — shares are not comparable, token counts still are`);
|
|
177
|
+
}
|
|
178
|
+
const { pairs, dropped } = pairConfigs(baseline.configs, current.configs);
|
|
179
|
+
const configs = pairs.map((p) => diffConfig(p.before, p.after, p.matchedBy));
|
|
180
|
+
for (const c of configs) {
|
|
181
|
+
if (c.matchedBy === 'unmatched') {
|
|
182
|
+
warnings.push(`${c.source}: no matching config in the baseline — its ${c.afterTotal.toLocaleString('en-US')} tokens are shown as a total, not a change`);
|
|
183
|
+
}
|
|
184
|
+
if (c.matchedBy === 'sole-config' && c.source !== baseline.configs[0]?.source) {
|
|
185
|
+
warnings.push(`paired ${c.source} with the baseline's ${baseline.configs[0]?.source} — one config on each side, different paths`);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
for (const d of dropped) {
|
|
189
|
+
warnings.push(`${d.source}: in the baseline (${d.totalTokens.toLocaleString('en-US')} tokens) and not found now — a config that disappeared is not a config that got cheaper`);
|
|
190
|
+
}
|
|
191
|
+
const increases = configs.filter((c) => typeof c.delta === 'number' && c.delta > 0);
|
|
192
|
+
increases.sort((a, b) => b.delta - a.delta);
|
|
193
|
+
return {
|
|
194
|
+
baselineGeneratedAt: baseline.generatedAt,
|
|
195
|
+
baselineMethodologyVersion: baseline.methodologyVersion,
|
|
196
|
+
comparable,
|
|
197
|
+
droppedConfigs: dropped.map((d) => ({ client: d.client, source: d.source, totalTokens: d.totalTokens })),
|
|
198
|
+
warnings,
|
|
199
|
+
configs,
|
|
200
|
+
worstIncrease: increases.length ? { source: increases[0].source, delta: increases[0].delta } : null,
|
|
201
|
+
};
|
|
202
|
+
}
|
|
203
|
+
const n = (x) => x.toLocaleString('en-US');
|
|
204
|
+
const signed = (x) => `${x >= 0 ? '+' : '−'}${n(Math.abs(x))}`;
|
|
205
|
+
const pct = (x) => `${(x * 100).toFixed(1)}%`;
|
|
206
|
+
/** `--config <path>` records the client as 'explicit', which is a parser detail, not a name. */
|
|
207
|
+
const clientLabel = (client) => (client === 'explicit' || !client ? 'this client' : client);
|
|
208
|
+
export function formatDiff(diff, contextWindow) {
|
|
209
|
+
const lines = [];
|
|
210
|
+
lines.push('');
|
|
211
|
+
lines.push(`diff vs baseline measured ${diff.baselineGeneratedAt} (methodology ${diff.baselineMethodologyVersion})`);
|
|
212
|
+
for (const c of diff.configs) {
|
|
213
|
+
lines.push('');
|
|
214
|
+
if (c.matchedBy === 'unmatched' || c.delta === null || c.beforeTotal === null) {
|
|
215
|
+
lines.push(` ${c.source} ${n(c.afterTotal)} tokens — no baseline for this config, so nothing to compare`);
|
|
216
|
+
continue;
|
|
217
|
+
}
|
|
218
|
+
const rows = c.servers.filter((s) => s.kind !== 'unchanged');
|
|
219
|
+
lines.push(` ${c.source}`);
|
|
220
|
+
lines.push(` ${n(c.beforeTotal)} → ${n(c.afterTotal)} ${signed(c.delta)}`);
|
|
221
|
+
if (rows.length) {
|
|
222
|
+
lines.push('');
|
|
223
|
+
const w = Math.max(...rows.map((r) => r.name.length), 6);
|
|
224
|
+
for (const r of rows) {
|
|
225
|
+
const from = r.before === null ? '—' : n(r.before);
|
|
226
|
+
const to = r.after === null ? '—' : n(r.after);
|
|
227
|
+
const d = r.delta === null ? '' : ` ${signed(r.delta)}`;
|
|
228
|
+
lines.push(` ${r.kind.padEnd(17)} ${r.name.padEnd(w)} ${from.padStart(9)} → ${to.padStart(9)}${d}`);
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
const unchanged = c.servers.length - rows.length;
|
|
232
|
+
if (unchanged)
|
|
233
|
+
lines.push(` (${unchanged} server${unchanged === 1 ? '' : 's'} unchanged)`);
|
|
234
|
+
lines.push('');
|
|
235
|
+
if (!c.exact) {
|
|
236
|
+
// The headline sentence is where a skimmer stops, so it must not assert a change
|
|
237
|
+
// this run could not establish. A server that died takes its tokens out of the
|
|
238
|
+
// total exactly like a server you uninstalled — printing "removes 2,378 tokens"
|
|
239
|
+
// and correcting it two lines down is the flattering reading getting read.
|
|
240
|
+
lines.push(` Not a clean comparison: a server changed measured-ness between the two runs.`);
|
|
241
|
+
lines.push(` The measured total moved ${signed(c.delta)}, but that is not what your config did.`);
|
|
242
|
+
lines.push('');
|
|
243
|
+
for (const r of c.servers) {
|
|
244
|
+
if (r.kind === 'unmeasured-now' || r.kind === 'unmeasured-before')
|
|
245
|
+
lines.push(` ${r.name}: ${r.note}`);
|
|
246
|
+
}
|
|
247
|
+
if (c.understatedBy)
|
|
248
|
+
lines.push(` → true cost is at least ${n(c.understatedBy)} higher than the ${n(c.afterTotal)} measured now.`);
|
|
249
|
+
if (c.overstatedBy)
|
|
250
|
+
lines.push(` → up to ${n(c.overstatedBy)} of that movement was already being paid, just unmeasured.`);
|
|
251
|
+
}
|
|
252
|
+
else if (c.delta === 0) {
|
|
253
|
+
lines.push(` No change: this config costs the same ${n(c.afterTotal)} tokens per request as the baseline.`);
|
|
254
|
+
}
|
|
255
|
+
else if (c.delta > 0) {
|
|
256
|
+
lines.push(` This change adds ${n(c.delta)} tokens to every request in ${clientLabel(c.client)} — ` +
|
|
257
|
+
`${pct(c.beforeShare ?? 0)} → ${pct(c.afterShare)} of a ${n(contextWindow)}-token context window.`);
|
|
258
|
+
}
|
|
259
|
+
else {
|
|
260
|
+
lines.push(` This change removes ${n(Math.abs(c.delta))} tokens from every request in ${clientLabel(c.client)} — ` +
|
|
261
|
+
`${pct(c.beforeShare ?? 0)} → ${pct(c.afterShare)} of a ${n(contextWindow)}-token context window.`);
|
|
262
|
+
}
|
|
263
|
+
const blind = c.servers.filter((r) => r.kind === 'unmeasured-both' || ((r.kind === 'added' || r.kind === 'removed') && r.delta === null));
|
|
264
|
+
if (blind.length) {
|
|
265
|
+
lines.push('');
|
|
266
|
+
for (const r of blind)
|
|
267
|
+
lines.push(` ${r.name}: ${r.note}`);
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
if (diff.warnings.length) {
|
|
271
|
+
lines.push('');
|
|
272
|
+
lines.push(' diff warnings');
|
|
273
|
+
for (const w of diff.warnings)
|
|
274
|
+
lines.push(` ${w}`);
|
|
275
|
+
}
|
|
276
|
+
if (!diff.comparable) {
|
|
277
|
+
lines.push('');
|
|
278
|
+
lines.push(' The two runs are not the same measurement, so the numbers above are not a change.');
|
|
279
|
+
lines.push(' Re-record the baseline with this version: mcp-context-cost audit --json > baseline.json');
|
|
280
|
+
}
|
|
281
|
+
return lines.join('\n');
|
|
282
|
+
}
|
|
283
|
+
/**
|
|
284
|
+
* `--max-increase N` — the CI gate. Fails on an increase over the limit, and
|
|
285
|
+
* equally on any reason the increase could not be established.
|
|
286
|
+
*
|
|
287
|
+
* That second half is the point. A gate that passes when a server failed to
|
|
288
|
+
* start, or when the baseline covered a config this run never found, is a green
|
|
289
|
+
* check on a question nobody asked. Everything this portfolio has learned says
|
|
290
|
+
* unchecked must not read as clean, so an inexact diff fails and names why.
|
|
291
|
+
*/
|
|
292
|
+
export function evaluateIncreaseGate(diff, limit) {
|
|
293
|
+
const reasons = [];
|
|
294
|
+
if (!diff.comparable)
|
|
295
|
+
reasons.push('the baseline is not the same measurement as this run — nothing was compared');
|
|
296
|
+
for (const c of diff.configs) {
|
|
297
|
+
if (c.matchedBy === 'unmatched') {
|
|
298
|
+
reasons.push(`${c.source}: no baseline to check its ${n(c.afterTotal)} tokens against`);
|
|
299
|
+
}
|
|
300
|
+
else if (!c.exact) {
|
|
301
|
+
reasons.push(`${c.source}: a server changed measured-ness, so the change could not be established exactly`);
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
for (const d of diff.droppedConfigs) {
|
|
305
|
+
reasons.push(`${d.source}: covered by the baseline and not found in this run`);
|
|
306
|
+
}
|
|
307
|
+
const increase = diff.worstIncrease?.delta ?? (diff.configs.some((c) => typeof c.delta === 'number') ? 0 : null);
|
|
308
|
+
if (reasons.length === 0 && increase !== null && increase > limit) {
|
|
309
|
+
reasons.push(`${diff.worstIncrease.source}: +${n(increase)} tokens per request, over the ${n(limit)} allowed`);
|
|
310
|
+
}
|
|
311
|
+
return { limit, pass: reasons.length === 0, increase, reasons };
|
|
312
|
+
}
|
|
313
|
+
export function formatGate(gate) {
|
|
314
|
+
if (gate.pass) {
|
|
315
|
+
return `increase ok: ${gate.increase === null ? 'no change to measure' : `${signed(gate.increase)} tokens`} ≤ ${n(gate.limit)} allowed`;
|
|
316
|
+
}
|
|
317
|
+
return ['INCREASE FAIL:', ...gate.reasons.map((r) => ` ${r}`)].join('\n');
|
|
318
|
+
}
|
package/dist/audit/run.d.ts
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import type { Measurement } from '../core/types.js';
|
|
2
|
+
import { type DivergenceRun } from '../core/divergence.js';
|
|
2
3
|
import { type AuditReport } from './audit.js';
|
|
3
4
|
import { type LoadedConfig } from './config.js';
|
|
5
|
+
/** Where the published `tools-delta/v1` run lives when `--claude` doesn't override it. */
|
|
6
|
+
export declare const DEFAULT_DIVERGENCE_URL = "https://raw.githubusercontent.com/athakur3/mcp-context-cost/main/results/divergence.json";
|
|
4
7
|
export interface AuditOptions {
|
|
5
8
|
/** Explicit config path(s); when empty, every known client location is tried. */
|
|
6
9
|
configPaths?: string[];
|
|
@@ -11,8 +14,17 @@ export interface AuditOptions {
|
|
|
11
14
|
docker?: boolean;
|
|
12
15
|
contextWindow?: number;
|
|
13
16
|
budget?: number;
|
|
17
|
+
/** Join each measured server against the published Claude divergence run. */
|
|
18
|
+
claude?: boolean;
|
|
19
|
+
/** Override the divergence.json source — mainly for tests and self-hosted mirrors. */
|
|
20
|
+
divergenceUrl?: string;
|
|
14
21
|
onProgress?: (name: string, done: number, total: number) => void;
|
|
15
22
|
}
|
|
23
|
+
/** Fetch and parse the published divergence run. Never throws: a failure is a report problem, not a crash. */
|
|
24
|
+
export declare function fetchDivergence(url: string): Promise<{
|
|
25
|
+
run: DivergenceRun | null;
|
|
26
|
+
problem?: string;
|
|
27
|
+
}>;
|
|
16
28
|
export declare function discover(opts?: AuditOptions): LoadedConfig[];
|
|
17
29
|
/** Measure every distinct stdio server across the given configs, once each. */
|
|
18
30
|
export declare function measureAll(configs: LoadedConfig[], opts?: AuditOptions): Promise<Map<string, Measurement>>;
|
package/dist/audit/run.js
CHANGED
|
@@ -6,8 +6,24 @@
|
|
|
6
6
|
*/
|
|
7
7
|
import { homedir } from 'node:os';
|
|
8
8
|
import { measureServer } from '../sweep/run.js';
|
|
9
|
+
import { parseDivergence } from '../core/divergence.js';
|
|
9
10
|
import { buildReport, serverKey } from './audit.js';
|
|
10
11
|
import { configCandidates, loadConfigs } from './config.js';
|
|
12
|
+
/** Where the published `tools-delta/v1` run lives when `--claude` doesn't override it. */
|
|
13
|
+
export const DEFAULT_DIVERGENCE_URL = 'https://raw.githubusercontent.com/athakur3/mcp-context-cost/main/results/divergence.json';
|
|
14
|
+
/** Fetch and parse the published divergence run. Never throws: a failure is a report problem, not a crash. */
|
|
15
|
+
export async function fetchDivergence(url) {
|
|
16
|
+
try {
|
|
17
|
+
const res = await fetch(url, { signal: AbortSignal.timeout(15_000) });
|
|
18
|
+
if (!res.ok)
|
|
19
|
+
return { run: null, problem: `claude divergence: HTTP ${res.status} fetching ${url}` };
|
|
20
|
+
const run = parseDivergence(await res.text());
|
|
21
|
+
return run ? { run } : { run: null, problem: `claude divergence: malformed data at ${url}` };
|
|
22
|
+
}
|
|
23
|
+
catch (e) {
|
|
24
|
+
return { run: null, problem: `claude divergence: failed to fetch ${url}: ${e.message}` };
|
|
25
|
+
}
|
|
26
|
+
}
|
|
11
27
|
export function discover(opts = {}) {
|
|
12
28
|
const cwd = opts.cwd ?? process.cwd();
|
|
13
29
|
const home = opts.home ?? homedir();
|
|
@@ -52,8 +68,19 @@ export async function measureAll(configs, opts = {}) {
|
|
|
52
68
|
export async function runAudit(opts = {}) {
|
|
53
69
|
const configs = discover(opts);
|
|
54
70
|
const measured = await measureAll(configs, opts);
|
|
55
|
-
|
|
71
|
+
let divergence = null;
|
|
72
|
+
let divergenceProblem;
|
|
73
|
+
if (opts.claude) {
|
|
74
|
+
const fetched = await fetchDivergence(opts.divergenceUrl ?? DEFAULT_DIVERGENCE_URL);
|
|
75
|
+
divergence = fetched.run;
|
|
76
|
+
divergenceProblem = fetched.problem;
|
|
77
|
+
}
|
|
78
|
+
const report = buildReport(configs, measured, {
|
|
56
79
|
contextWindow: opts.contextWindow,
|
|
57
80
|
budget: opts.budget,
|
|
81
|
+
divergence,
|
|
58
82
|
});
|
|
83
|
+
if (divergenceProblem)
|
|
84
|
+
report.problems.push(divergenceProblem);
|
|
85
|
+
return report;
|
|
59
86
|
}
|
package/dist/cli.d.ts
CHANGED
|
@@ -6,3 +6,24 @@ export declare function verifyMeasurement(m: Measurement): {
|
|
|
6
6
|
rederivedSha: string | null;
|
|
7
7
|
problems: string[];
|
|
8
8
|
};
|
|
9
|
+
/** Derives a servers.yaml-style slug from a remote URL's hostname, e.g. mcp.deepwiki.com -> deepwiki. */
|
|
10
|
+
export declare function slugFromUrl(url: string): string;
|
|
11
|
+
/** Installed version, for error messages that need to say which one you are running. */
|
|
12
|
+
export declare function cliVersion(): string;
|
|
13
|
+
/**
|
|
14
|
+
* Reject flags this build does not know.
|
|
15
|
+
*
|
|
16
|
+
* An older CLI used to ignore an unrecognised flag and carry on. That is the exact failure
|
|
17
|
+
* this project exists to catch, in our own tool: `audit --baseline base.json
|
|
18
|
+
* --max-increase 2000` on a build without those flags ran a plain audit and **exited 0** —
|
|
19
|
+
* a green CI check on a gate that never ran. The README documents flags before they are
|
|
20
|
+
* published, so the version skew is not hypothetical; it is the normal case for anyone
|
|
21
|
+
* running `npx -y mcp-context-cost`.
|
|
22
|
+
*
|
|
23
|
+
* So an unknown flag is a usage error, and the message names the running version, because
|
|
24
|
+
* the likeliest cause is that the reader's command is newer than their install.
|
|
25
|
+
*/
|
|
26
|
+
export declare function unknownFlags(argv: string[], spec: {
|
|
27
|
+
value: string[];
|
|
28
|
+
boolean: string[];
|
|
29
|
+
}): string[];
|
package/dist/cli.js
CHANGED
|
@@ -2,16 +2,26 @@
|
|
|
2
2
|
/**
|
|
3
3
|
* mcp-context-cost CLI — the dispute drill as a command.
|
|
4
4
|
*
|
|
5
|
-
* mcp-context-cost audit [--budget N] [--json]
|
|
6
|
-
* MCP config; exit 1 if over budget
|
|
5
|
+
* mcp-context-cost audit [--budget N] [--claude] [--json] measure the servers in your
|
|
6
|
+
* own MCP config; exit 1 if over budget.
|
|
7
|
+
* mcp-context-cost audit --baseline <report.json> [--max-increase N] diff against a
|
|
8
|
+
* stored earlier report; exit 1 if this
|
|
9
|
+
* config change adds more than N tokens
|
|
10
|
+
* to every request (or if it can't tell).
|
|
11
|
+
* --claude adds each server's Anthropic-
|
|
12
|
+
* request cost where the published capture
|
|
13
|
+
* hash matches what's installed.
|
|
7
14
|
* mcp-context-cost verify <measurement.json> [--json] re-derive the number from the
|
|
8
15
|
* published capture; exit 1 on mismatch
|
|
9
16
|
* mcp-context-cost verify --remote <url> [--json] same, fetched from a measurement URL
|
|
10
17
|
* mcp-context-cost measure --name x --command "npx -y ..." one-off measurement
|
|
18
|
+
* mcp-context-cost measure --remote <url> [--name x] same, via the mcp-remote bridge
|
|
19
|
+
* (name defaults to the URL's hostname)
|
|
11
20
|
*
|
|
12
21
|
* Exit codes: 0 ok, 1 verification/measurement/budget failed, 2 usage error.
|
|
13
22
|
*/
|
|
14
23
|
import { readFileSync } from 'node:fs';
|
|
24
|
+
import { createRequire } from 'node:module';
|
|
15
25
|
import { canonicalString, countTokens, sha256Hex } from './core/canonical.js';
|
|
16
26
|
import { toBadge } from './core/badge.js';
|
|
17
27
|
export function verifyMeasurement(m) {
|
|
@@ -30,8 +40,68 @@ export function verifyMeasurement(m) {
|
|
|
30
40
|
problems.push(`toolCount mismatch: capture has ${m.rawToolsCapture.length}, stored ${m.toolCount}`);
|
|
31
41
|
return { ok: problems.length === 0, rederivedTokens: tokens, rederivedSha: sha, problems };
|
|
32
42
|
}
|
|
43
|
+
/** Derives a servers.yaml-style slug from a remote URL's hostname, e.g. mcp.deepwiki.com -> deepwiki. */
|
|
44
|
+
export function slugFromUrl(url) {
|
|
45
|
+
const host = new URL(url).hostname.replace(/^(www|mcp)\./, '');
|
|
46
|
+
return host.replace(/[^a-z0-9]+/gi, '-').replace(/^-+|-+$/g, '').toLowerCase() || 'remote';
|
|
47
|
+
}
|
|
48
|
+
/** Installed version, for error messages that need to say which one you are running. */
|
|
49
|
+
export function cliVersion() {
|
|
50
|
+
try {
|
|
51
|
+
return createRequire(import.meta.url)('../package.json').version;
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
return 'unknown';
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Reject flags this build does not know.
|
|
59
|
+
*
|
|
60
|
+
* An older CLI used to ignore an unrecognised flag and carry on. That is the exact failure
|
|
61
|
+
* this project exists to catch, in our own tool: `audit --baseline base.json
|
|
62
|
+
* --max-increase 2000` on a build without those flags ran a plain audit and **exited 0** —
|
|
63
|
+
* a green CI check on a gate that never ran. The README documents flags before they are
|
|
64
|
+
* published, so the version skew is not hypothetical; it is the normal case for anyone
|
|
65
|
+
* running `npx -y mcp-context-cost`.
|
|
66
|
+
*
|
|
67
|
+
* So an unknown flag is a usage error, and the message names the running version, because
|
|
68
|
+
* the likeliest cause is that the reader's command is newer than their install.
|
|
69
|
+
*/
|
|
70
|
+
export function unknownFlags(argv, spec) {
|
|
71
|
+
const known = new Set([...spec.value, ...spec.boolean]);
|
|
72
|
+
const unknown = [];
|
|
73
|
+
for (let i = 0; i < argv.length; i++) {
|
|
74
|
+
const tok = argv[i];
|
|
75
|
+
if (!tok.startsWith('--'))
|
|
76
|
+
continue;
|
|
77
|
+
const name = tok.slice(2).split('=')[0];
|
|
78
|
+
if (!known.has(name)) {
|
|
79
|
+
unknown.push(tok.split('=')[0]);
|
|
80
|
+
continue;
|
|
81
|
+
}
|
|
82
|
+
// Skip a value-taking flag's value, so `--command "--weird"` is not read as a flag.
|
|
83
|
+
if (spec.value.includes(name) && !tok.includes('='))
|
|
84
|
+
i++;
|
|
85
|
+
}
|
|
86
|
+
return unknown;
|
|
87
|
+
}
|
|
88
|
+
function rejectUnknownFlags(cmd, argv, spec) {
|
|
89
|
+
const bad = unknownFlags(argv, spec);
|
|
90
|
+
if (!bad.length)
|
|
91
|
+
return;
|
|
92
|
+
const all = [...spec.value, ...spec.boolean].sort().map((f) => `--${f}`).join(' ');
|
|
93
|
+
console.error(`unknown flag for \`${cmd}\`: ${bad.join(', ')}`);
|
|
94
|
+
console.error(`this is mcp-context-cost ${cliVersion()} — if you copied the command from the README,`);
|
|
95
|
+
console.error(`your install may be older than the docs. Try: npx -y mcp-context-cost@latest ${cmd} ...`);
|
|
96
|
+
console.error(`known flags for ${cmd}: ${all}`);
|
|
97
|
+
process.exit(2);
|
|
98
|
+
}
|
|
33
99
|
const [, , cmd, ...rest] = process.argv;
|
|
34
100
|
if (cmd === 'audit') {
|
|
101
|
+
rejectUnknownFlags('audit', rest, {
|
|
102
|
+
value: ['config', 'budget', 'baseline', 'max-increase', 'context', 'timeout', 'concurrency', 'divergence-url'],
|
|
103
|
+
boolean: ['json', 'docker', 'claude'],
|
|
104
|
+
});
|
|
35
105
|
const argOf = (name) => {
|
|
36
106
|
const i = rest.indexOf(`--${name}`);
|
|
37
107
|
return i >= 0 ? rest[i + 1] : undefined;
|
|
@@ -49,7 +119,44 @@ if (cmd === 'audit') {
|
|
|
49
119
|
}
|
|
50
120
|
return v;
|
|
51
121
|
};
|
|
122
|
+
const nonNegative = (name) => {
|
|
123
|
+
const raw = argOf(name);
|
|
124
|
+
if (raw === undefined)
|
|
125
|
+
return undefined;
|
|
126
|
+
const v = Number(raw);
|
|
127
|
+
if (!Number.isFinite(v) || v < 0) {
|
|
128
|
+
console.error(`--${name} must be zero or a positive number, got '${raw}'`);
|
|
129
|
+
process.exit(2);
|
|
130
|
+
}
|
|
131
|
+
return v;
|
|
132
|
+
};
|
|
52
133
|
const budget = numeric('budget');
|
|
134
|
+
const baselinePath = argOf('baseline');
|
|
135
|
+
const maxIncrease = nonNegative('max-increase');
|
|
136
|
+
if (maxIncrease !== undefined && !baselinePath) {
|
|
137
|
+
console.error('--max-increase needs a --baseline to measure the increase against');
|
|
138
|
+
process.exit(2);
|
|
139
|
+
}
|
|
140
|
+
const { buildDiff, evaluateIncreaseGate, parseBaselineReport } = await import('./audit/diff.js');
|
|
141
|
+
// Read and shape-check the baseline BEFORE measuring anything: a typo in the path
|
|
142
|
+
// should cost a second, not a full server sweep that is then thrown away.
|
|
143
|
+
let baseline;
|
|
144
|
+
if (baselinePath) {
|
|
145
|
+
let raw;
|
|
146
|
+
try {
|
|
147
|
+
raw = readFileSync(baselinePath, 'utf8');
|
|
148
|
+
}
|
|
149
|
+
catch (e) {
|
|
150
|
+
console.error(`cannot read baseline ${baselinePath}: ${e.message}`);
|
|
151
|
+
process.exit(2);
|
|
152
|
+
}
|
|
153
|
+
const parsed = parseBaselineReport(raw);
|
|
154
|
+
if (!parsed.report) {
|
|
155
|
+
console.error(`${baselinePath}: ${parsed.problem}`);
|
|
156
|
+
process.exit(2);
|
|
157
|
+
}
|
|
158
|
+
baseline = parsed.report;
|
|
159
|
+
}
|
|
53
160
|
const { runAudit } = await import('./audit/run.js');
|
|
54
161
|
const { formatReport } = await import('./audit/audit.js');
|
|
55
162
|
const report = await runAudit({
|
|
@@ -59,6 +166,8 @@ if (cmd === 'audit') {
|
|
|
59
166
|
timeoutMs: numeric('timeout'),
|
|
60
167
|
concurrency: numeric('concurrency'),
|
|
61
168
|
docker: rest.includes('--docker'),
|
|
169
|
+
claude: rest.includes('--claude'),
|
|
170
|
+
divergenceUrl: argOf('divergence-url'),
|
|
62
171
|
// Progress goes to stderr so `--json` stdout stays a single parseable object.
|
|
63
172
|
onProgress: json ? undefined : (name, done, total) => process.stderr.write(` [${done}/${total}] ${name}\n`),
|
|
64
173
|
});
|
|
@@ -71,10 +180,16 @@ if (cmd === 'audit') {
|
|
|
71
180
|
`Point at one explicitly: mcp-context-cost audit --config <path/to/mcp.json>`);
|
|
72
181
|
process.exit(1);
|
|
73
182
|
}
|
|
183
|
+
if (baseline) {
|
|
184
|
+
report.diff = buildDiff(baseline, report);
|
|
185
|
+
if (maxIncrease !== undefined)
|
|
186
|
+
report.increaseGate = evaluateIncreaseGate(report.diff, maxIncrease);
|
|
187
|
+
}
|
|
74
188
|
console.log(json ? JSON.stringify(report) : formatReport(report));
|
|
75
|
-
process.exit(report.budget?.over ? 1 : 0);
|
|
189
|
+
process.exit(report.budget?.over || report.increaseGate?.pass === false ? 1 : 0);
|
|
76
190
|
}
|
|
77
191
|
else if (cmd === 'verify') {
|
|
192
|
+
rejectUnknownFlags('verify', rest, { value: ['remote'], boolean: ['json'] });
|
|
78
193
|
const json = rest.includes('--json');
|
|
79
194
|
const remoteIdx = rest.indexOf('--remote');
|
|
80
195
|
const remoteUrl = remoteIdx >= 0 ? rest[remoteIdx + 1] : undefined;
|
|
@@ -121,21 +236,36 @@ else if (cmd === 'verify') {
|
|
|
121
236
|
process.exit(1);
|
|
122
237
|
}
|
|
123
238
|
else if (cmd === 'measure') {
|
|
239
|
+
rejectUnknownFlags('measure', rest, {
|
|
240
|
+
value: ['name', 'command', 'remote', 'timeout', 'docker-image'],
|
|
241
|
+
boolean: ['docker'],
|
|
242
|
+
});
|
|
124
243
|
const argOf = (name) => {
|
|
125
244
|
const i = rest.indexOf(`--${name}`);
|
|
126
245
|
return i >= 0 ? rest[i + 1] : undefined;
|
|
127
246
|
};
|
|
128
|
-
const name = argOf('name');
|
|
129
247
|
const command = argOf('command');
|
|
130
|
-
|
|
248
|
+
const remoteUrl = argOf('remote');
|
|
249
|
+
if (remoteUrl && !/^https?:\/\//i.test(remoteUrl)) {
|
|
250
|
+
console.error(`--remote must be an http(s) URL, got '${remoteUrl}'`);
|
|
251
|
+
process.exit(2);
|
|
252
|
+
}
|
|
253
|
+
if (!command && !remoteUrl) {
|
|
254
|
+
console.error('usage: mcp-context-cost measure --name <slug> --command "npx -y <server>" [--timeout ms] [--docker]');
|
|
255
|
+
console.error(' mcp-context-cost measure --remote <url> [--name <slug>] [--timeout ms] [--docker]');
|
|
256
|
+
process.exit(2);
|
|
257
|
+
}
|
|
258
|
+
const name = argOf('name') ?? (remoteUrl ? slugFromUrl(remoteUrl) : undefined);
|
|
259
|
+
if (!name) {
|
|
131
260
|
console.error('usage: mcp-context-cost measure --name <slug> --command "npx -y <server>" [--timeout ms] [--docker]');
|
|
132
261
|
process.exit(2);
|
|
133
262
|
}
|
|
134
263
|
const { measureServer } = await import('./sweep/run.js');
|
|
135
|
-
const m = await measureServer(name, command, {
|
|
264
|
+
const m = await measureServer(name, remoteUrl ? `npx -y mcp-remote ${remoteUrl}` : command, {
|
|
136
265
|
timeoutMs: Number(argOf('timeout') ?? 60_000),
|
|
137
266
|
docker: rest.includes('--docker'),
|
|
138
267
|
dockerImage: argOf('docker-image'),
|
|
268
|
+
argv: remoteUrl ? ['npx', '-y', 'mcp-remote', remoteUrl] : undefined,
|
|
139
269
|
});
|
|
140
270
|
const ok = m.status === 'measured' || m.status === 'dynamic';
|
|
141
271
|
console.log(ok
|
|
@@ -149,10 +279,14 @@ else if (cmd !== undefined && cmd !== '--help' && cmd !== '-h') {
|
|
|
149
279
|
}
|
|
150
280
|
else {
|
|
151
281
|
console.log('mcp-context-cost — reproducible context-cost measurement for MCP servers');
|
|
152
|
-
console.log(' audit [--config <path>] [--budget N] measure the servers in your own MCP config');
|
|
282
|
+
console.log(' audit [--config <path>] [--budget N] [--claude] measure the servers in your own MCP config');
|
|
153
283
|
console.log(' [--json] [--context N] [--timeout ms] [--concurrency N] [--docker]');
|
|
284
|
+
console.log(' [--baseline <report.json>] [--max-increase N] diff against an earlier');
|
|
285
|
+
console.log(' audit --json report; --max-increase');
|
|
286
|
+
console.log(' fails when a change adds too much');
|
|
154
287
|
console.log(' verify <measurement.json> [--json] re-derive tokens+sha from the published capture');
|
|
155
288
|
console.log(' verify --remote <url> [--json] same, fetched from a measurement URL');
|
|
156
289
|
console.log(' measure --name x --command "npx -y <server>" run a one-off measurement');
|
|
290
|
+
console.log(' measure --remote <url> [--name x] measure a remote server via mcp-remote');
|
|
157
291
|
console.log('exit codes: 0 ok, 1 verification/measurement/budget failed, 2 usage error');
|
|
158
292
|
}
|
|
@@ -1 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A 12-point-max inline trend line, oldest to newest. Muted stroke (this is
|
|
3
|
+
* texture, not a headline number) with the current value picked out as an
|
|
4
|
+
* accent dot, per the sparkline spec: de-emphasis hue for the line, accent
|
|
5
|
+
* for "now". Flat series still draw a level line rather than faking a zero
|
|
6
|
+
* baseline. Returns '' when there's nothing to trend (0-1 points).
|
|
7
|
+
*/
|
|
8
|
+
export declare function renderSparkline(tokens: number[]): string;
|
|
1
9
|
export declare function generateDashboard(root?: string): string;
|