mcp-context-cost 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +78 -1
- package/dist/audit/audit.d.ts +22 -0
- package/dist/audit/audit.js +43 -0
- package/dist/audit/run.d.ts +12 -0
- package/dist/audit/run.js +26 -0
- package/dist/cli.js +72 -3
- package/dist/core/capture-index.d.ts +76 -0
- package/dist/core/capture-index.js +92 -0
- package/dist/core/regression.d.ts +181 -0
- package/dist/core/regression.js +199 -0
- package/dist/core/server-diff.d.ts +73 -0
- package/dist/core/server-diff.js +197 -0
- package/dist/sweep/published-stats.d.ts +5 -0
- package/dist/sweep/published-stats.js +9 -0
- package/dist/sweep/regen.js +12 -2
- package/dist/sweep/regressions.d.ts +26 -0
- package/dist/sweep/regressions.js +251 -0
- package/dist/sweep/report.d.ts +2 -1
- package/dist/sweep/report.js +14 -1
- package/dist/sweep/sweep-all.js +6 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -210,9 +210,26 @@ prose every request carries — and only descriptions at or above the 90th perce
|
|
|
210
210
|
A config where nothing is out of distribution is told that in those words, and a baseline
|
|
211
211
|
that cannot be fetched is a named problem, never a silently skipped check.
|
|
212
212
|
|
|
213
|
+
Add `--changed` to ask the other question — *did the servers I already have get heavier?*
|
|
214
|
+
Each installed server is identified against the published capture history by its canonical
|
|
215
|
+
hash, never by its name, because the name in your config is a label you chose and the bytes
|
|
216
|
+
are not:
|
|
217
|
+
|
|
218
|
+
```
|
|
219
|
+
changed — published versions of your servers that have moved since
|
|
220
|
+
(index 2026-09-04, 2 published captures; matched by canonical hash, never by name):
|
|
221
|
+
notes (published as obsidian) — you have the capture published 2026-08-19 at 1,132 tokens;
|
|
222
|
+
the current one is 2,062 (+930, 2026-08-26)
|
|
223
|
+
updating all 1 would add 930 tokens to every request in this client.
|
|
224
|
+
```
|
|
225
|
+
|
|
226
|
+
A server whose bytes match no published capture — a version never measured here, a fork, a
|
|
227
|
+
pin — is reported as unidentified with nothing claimed about it.
|
|
228
|
+
Method: [capture index](docs/METHODOLOGY.md#capture-index).
|
|
229
|
+
|
|
213
230
|
Flags: `--json` (full report on stdout, progress on stderr), `--budget N`,
|
|
214
231
|
`--baseline <report.json>`, `--max-increase N`, `--context N` (default 200,000),
|
|
215
|
-
`--timeout ms`, `--concurrency N`, `--docker`, `--claude`, `--suggest`.
|
|
232
|
+
`--timeout ms`, `--concurrency N`, `--docker`, `--claude`, `--suggest`, `--changed`.
|
|
216
233
|
|
|
217
234
|
## Where the numbers come from
|
|
218
235
|
|
|
@@ -238,6 +255,14 @@ sample of that range; the full range is in
|
|
|
238
255
|
Each measured server also has a [detail page](https://athakur3.github.io/mcp-context-cost/servers/)
|
|
239
256
|
showing which tools its tokens are in.)*
|
|
240
257
|
|
|
258
|
+
Because the set is re-measured on a rotation and most entries launch unpinned, the same data
|
|
259
|
+
answers a question no client asks: **what did this server cost last month?**
|
|
260
|
+
[results/regressions.md](results/regressions.md) reports each server's most recent movement —
|
|
261
|
+
dated to when it happened, separated into *shipped more tools* versus *same tools, rewritten*,
|
|
262
|
+
and compared only within one isolation. The ecosystem ratchets upward: of the servers whose
|
|
263
|
+
cost has moved at all, 9 moved up against 1 that moved down. Method:
|
|
264
|
+
[cost movement](docs/METHODOLOGY.md#cost-movement).
|
|
265
|
+
|
|
241
266
|
If you publish a server, the same measurement is available as a badge, so your users can see
|
|
242
267
|
the cost before they install rather than after:
|
|
243
268
|
|
|
@@ -321,6 +346,58 @@ Then in your README:
|
|
|
321
346
|
[](<link target>)
|
|
322
347
|
```
|
|
323
348
|
|
|
349
|
+
### Defend the number, don't just display it
|
|
350
|
+
|
|
351
|
+
A badge says what your server costs today; it does nothing about the release
|
|
352
|
+
that adds 1,200 tokens to every user's context next month. Across the servers
|
|
353
|
+
measured here, that release is the norm rather than the exception — the
|
|
354
|
+
[movement report](results/regressions.md) has nine servers ratcheting upward
|
|
355
|
+
against one that got cheaper, and none of those maintainers had a check that
|
|
356
|
+
would have said so first. `measure` takes the same gate flags `audit` does, so
|
|
357
|
+
your own CI can be that check:
|
|
358
|
+
|
|
359
|
+
```bash
|
|
360
|
+
# on your default branch, once — commit the result
|
|
361
|
+
npx -y mcp-context-cost measure --name my-server --command "node dist/index.js"
|
|
362
|
+
cp results/my-server/measurement.json .context-cost/baseline.json
|
|
363
|
+
|
|
364
|
+
# on every pull request
|
|
365
|
+
npx -y mcp-context-cost measure --name my-server --command "node dist/index.js" \
|
|
366
|
+
--baseline .context-cost/baseline.json --max-increase 500
|
|
367
|
+
```
|
|
368
|
+
|
|
369
|
+
```
|
|
370
|
+
diff vs baseline .context-cost/baseline.json
|
|
371
|
+
my-server: 61 → 182 +121 tokens (2 → 3 tools)
|
|
372
|
+
added: bulk_export (43)
|
|
373
|
+
grew: search 30 → 108 (+78)
|
|
374
|
+
|
|
375
|
+
INCREASE FAIL: +121 tokens, over the 100 allowed — this change adds that to every request of every install.
|
|
376
|
+
```
|
|
377
|
+
|
|
378
|
+
Both sides are single measurements carrying per-tool counts, so an established
|
|
379
|
+
change is attributed exactly: which tools arrived, which grew, and by how much.
|
|
380
|
+
And `--max-increase` fails on more than the number — a server that stops
|
|
381
|
+
starting on the branch makes the total go *down*, and reporting that as an
|
|
382
|
+
improvement is the one mistake a gate like this must not make, so a change that
|
|
383
|
+
could not be established fails too.
|
|
384
|
+
|
|
385
|
+
As a GitHub Action, that whole workflow is five lines
|
|
386
|
+
([full example](examples/server-author-ci.yml)):
|
|
387
|
+
|
|
388
|
+
```yaml
|
|
389
|
+
- uses: athakur3/mcp-context-cost@v1
|
|
390
|
+
with:
|
|
391
|
+
name: my-server
|
|
392
|
+
command: node dist/index.js
|
|
393
|
+
baseline: .context-cost/baseline.json
|
|
394
|
+
max-increase: 500
|
|
395
|
+
```
|
|
396
|
+
|
|
397
|
+
It exposes `tokens`, `tools`, `status`, `measurement` and `badge` as outputs —
|
|
398
|
+
available whether the gate passed or not — so a later step can comment the
|
|
399
|
+
number on the pull request or publish the badge.
|
|
400
|
+
|
|
324
401
|
Point the link at the measurement behind the number — for servers in this sweep that is
|
|
325
402
|
`https://athakur3.github.io/mcp-context-cost/servers/<name>.html`; otherwise the
|
|
326
403
|
methodology page. A badge nobody can audit is decoration.
|
package/dist/audit/audit.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type DivergenceRun } from '../core/divergence.js';
|
|
2
2
|
import { type ToolShapeBaseline, type ToolSuggestion } from '../core/tool-shape.js';
|
|
3
|
+
import { type CaptureIndex, type CaptureVerdict } from '../core/capture-index.js';
|
|
3
4
|
import type { Measurement, MeasurementStatus, ToolMeasurement } from '../core/types.js';
|
|
4
5
|
import type { ConfiguredServer, LoadedConfig } from './config.js';
|
|
5
6
|
import { type DeferralVerdict, type ToolSearchEnv, type ToolSearchSource } from './deferral.js';
|
|
@@ -57,6 +58,18 @@ export interface ConfigSuggestions {
|
|
|
57
58
|
outOfDistribution: ToolSuggestion[];
|
|
58
59
|
checkedTools: number;
|
|
59
60
|
}
|
|
61
|
+
/**
|
|
62
|
+
* `--changed`: one locally measured server placed against the published capture
|
|
63
|
+
* history, joined by canonical hash rather than by name. The local label is
|
|
64
|
+
* carried alongside the published server the bytes identify, because they need
|
|
65
|
+
* not agree — a config may call the official GitHub server anything at all, and
|
|
66
|
+
* a name that disagrees with the bytes is information rather than noise.
|
|
67
|
+
*/
|
|
68
|
+
export interface ServerCaptureVerdict {
|
|
69
|
+
/** The name this config gave the server. */
|
|
70
|
+
name: string;
|
|
71
|
+
verdict: CaptureVerdict;
|
|
72
|
+
}
|
|
60
73
|
export interface AuditConfigResult {
|
|
61
74
|
client: string;
|
|
62
75
|
source: string;
|
|
@@ -70,6 +83,8 @@ export interface AuditConfigResult {
|
|
|
70
83
|
trimAdvice: TrimAdvice | null;
|
|
71
84
|
/** Present only when `--suggest` ran with a usable baseline. */
|
|
72
85
|
suggestions?: ConfigSuggestions;
|
|
86
|
+
/** Present only when `--changed` ran with a usable capture index. */
|
|
87
|
+
captureVerdicts?: ServerCaptureVerdict[];
|
|
73
88
|
/**
|
|
74
89
|
* Whether this client loads the total up front or defers it, and — when the
|
|
75
90
|
* client decides that by a threshold — which side of it this stack is on.
|
|
@@ -150,6 +165,11 @@ export interface AuditReport {
|
|
|
150
165
|
toolCount: number;
|
|
151
166
|
serverCount: number;
|
|
152
167
|
};
|
|
168
|
+
/** Which published capture index `--changed` joined against. */
|
|
169
|
+
captureIndex?: {
|
|
170
|
+
generatedAt: string;
|
|
171
|
+
captureCount: number;
|
|
172
|
+
};
|
|
153
173
|
/** Present only when a baseline report was supplied (`--baseline`). */
|
|
154
174
|
diff?: AuditDiff;
|
|
155
175
|
/** Present only when `--max-increase` was supplied alongside a baseline. */
|
|
@@ -184,6 +204,8 @@ export declare function buildReport(configs: LoadedConfig[], measured: Map<strin
|
|
|
184
204
|
divergence?: DivergenceRun | null;
|
|
185
205
|
/** Published `tool-shape/v1` baseline (`--suggest`); omit to skip suggestions. */
|
|
186
206
|
toolShape?: ToolShapeBaseline | null;
|
|
207
|
+
/** Published `capture-index/v1` (`--changed`); omit to skip the version join. */
|
|
208
|
+
captureIndex?: CaptureIndex | null;
|
|
187
209
|
/**
|
|
188
210
|
* The audited machine's SHELL tool-search variables. Passed in rather than
|
|
189
211
|
* read here so this stays pure and a report is reproducible from its
|
package/dist/audit/audit.js
CHANGED
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
import { METHODOLOGY_VERSION } from '../core/canonical.js';
|
|
16
16
|
import { isCurrent } from '../core/divergence.js';
|
|
17
17
|
import { SUGGEST_DESCRIPTION_PERCENTILE, suggestFor, } from '../core/tool-shape.js';
|
|
18
|
+
import { identify } from '../core/capture-index.js';
|
|
18
19
|
import { evaluateDeferral, PUBLISHED_WIRE_TO_CLIENT_RATIO, SHELL_SOURCE, } from './deferral.js';
|
|
19
20
|
import { formatDiff, formatGate } from './diff.js';
|
|
20
21
|
export const DEFAULT_CONTEXT_WINDOW = 200_000;
|
|
@@ -280,6 +281,9 @@ export function buildReport(configs, measured, opts = {}) {
|
|
|
280
281
|
heaviestTools: tools.slice(0, 5),
|
|
281
282
|
trimAdvice: buildTrimAdvice(tools, totalTokens),
|
|
282
283
|
suggestions: opts.toolShape ? buildSuggestions(shapePool, opts.toolShape) : undefined,
|
|
284
|
+
captureVerdicts: opts.captureIndex
|
|
285
|
+
? ok.map((s) => ({ name: s.name, verdict: identify(s.canonicalSha256, opts.captureIndex) }))
|
|
286
|
+
: undefined,
|
|
283
287
|
};
|
|
284
288
|
built.push(result);
|
|
285
289
|
shared.set(result, sharedHere);
|
|
@@ -305,6 +309,12 @@ export function buildReport(configs, measured, opts = {}) {
|
|
|
305
309
|
serverCount: opts.toolShape.serverCount,
|
|
306
310
|
};
|
|
307
311
|
}
|
|
312
|
+
if (opts.captureIndex) {
|
|
313
|
+
report.captureIndex = {
|
|
314
|
+
generatedAt: opts.captureIndex.generatedAt,
|
|
315
|
+
captureCount: Object.keys(opts.captureIndex.captures).length,
|
|
316
|
+
};
|
|
317
|
+
}
|
|
308
318
|
if (typeof opts.budget === 'number') {
|
|
309
319
|
// The worst config is the gate: passing because your *lightest* client fits
|
|
310
320
|
// would be a green check on a session you don't run.
|
|
@@ -640,6 +650,39 @@ export function formatReport(report) {
|
|
|
640
650
|
`(${names}) would recover ${n(cfg.trimAdvice.recoverableTokens)} tokens ` +
|
|
641
651
|
`(${pct(cfg.trimAdvice.recoverableShare)} of this config) — if your client supports per-tool filtering.`);
|
|
642
652
|
}
|
|
653
|
+
if (cfg.captureVerdicts) {
|
|
654
|
+
const behind = cfg.captureVerdicts.filter((v) => v.verdict.kind === 'behind');
|
|
655
|
+
const current = cfg.captureVerdicts.filter((v) => v.verdict.kind === 'current');
|
|
656
|
+
const unknown = cfg.captureVerdicts.length - behind.length - current.length;
|
|
657
|
+
const idx = report.captureIndex
|
|
658
|
+
? `index ${report.captureIndex.generatedAt}, ${n(report.captureIndex.captureCount)} published captures`
|
|
659
|
+
: 'the published capture index';
|
|
660
|
+
lines.push('');
|
|
661
|
+
if (behind.length === 0) {
|
|
662
|
+
lines.push(` changed: no server here is running a published capture that has since moved ` +
|
|
663
|
+
`(${idx}; matched by canonical hash, never by name).`);
|
|
664
|
+
}
|
|
665
|
+
else {
|
|
666
|
+
const total = behind.reduce((a, v) => a + v.verdict.deltaTokens, 0);
|
|
667
|
+
lines.push(` changed — published versions of your servers that have moved since ` +
|
|
668
|
+
`(${idx}; matched by canonical hash, never by name):`);
|
|
669
|
+
for (const { name, verdict: v } of behind) {
|
|
670
|
+
// The local label and the published server are printed together: the
|
|
671
|
+
// bytes decide which server this is, and a name that disagrees is a
|
|
672
|
+
// fact worth seeing rather than one to smooth over.
|
|
673
|
+
const alias = name === v.server ? name : `${name} (published as ${v.server})`;
|
|
674
|
+
lines.push(` ${alias} — you have the capture published ${v.yourDate} at ${n(v.yourTokens)} tokens; ` +
|
|
675
|
+
`the current one is ${n(v.currentTokens)} (${v.deltaTokens >= 0 ? '+' : '−'}${n(Math.abs(v.deltaTokens))}, ${v.currentDate})`);
|
|
676
|
+
}
|
|
677
|
+
lines.push(` updating all ${behind.length} would ${total >= 0 ? 'add' : 'remove'} ${n(Math.abs(total))} tokens ` +
|
|
678
|
+
`${total >= 0 ? 'to' : 'from'} every request in this client.`);
|
|
679
|
+
}
|
|
680
|
+
if (unknown > 0) {
|
|
681
|
+
lines.push(` ${unknown} server${unknown === 1 ? '' : 's'} could not be identified: the installed bytes match no ` +
|
|
682
|
+
`published capture — a version never measured here, or one published before the index began. ` +
|
|
683
|
+
`Nothing is claimed about ${unknown === 1 ? 'it' : 'them'}.`);
|
|
684
|
+
}
|
|
685
|
+
}
|
|
643
686
|
if (cfg.suggestions) {
|
|
644
687
|
const sg = cfg.suggestions;
|
|
645
688
|
const base = report.toolShape
|
package/dist/audit/run.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { Measurement } from '../core/types.js';
|
|
2
2
|
import { type DivergenceRun } from '../core/divergence.js';
|
|
3
3
|
import { type ToolShapeBaseline } from '../core/tool-shape.js';
|
|
4
|
+
import { type CaptureIndex } from '../core/capture-index.js';
|
|
4
5
|
import { type AuditReport } from './audit.js';
|
|
5
6
|
import { type ToolSearchEnv, type ToolSearchSource } from './deferral.js';
|
|
6
7
|
import { type LoadedConfig } from './config.js';
|
|
@@ -8,6 +9,8 @@ import { type LoadedConfig } from './config.js';
|
|
|
8
9
|
export declare const DEFAULT_DIVERGENCE_URL = "https://raw.githubusercontent.com/athakur3/mcp-context-cost/main/results/divergence.json";
|
|
9
10
|
/** Where the published `tool-shape/v1` baseline lives when `--suggest` doesn't override it. */
|
|
10
11
|
export declare const DEFAULT_TOOL_SHAPE_URL = "https://raw.githubusercontent.com/athakur3/mcp-context-cost/main/results/tool-shape.json";
|
|
12
|
+
/** Where the published `capture-index/v1` lives when `--changed` doesn't override it. */
|
|
13
|
+
export declare const DEFAULT_CAPTURE_INDEX_URL = "https://raw.githubusercontent.com/athakur3/mcp-context-cost/main/results/capture-index.json";
|
|
11
14
|
export interface AuditOptions {
|
|
12
15
|
/** Explicit config path(s); when empty, every known client location is tried. */
|
|
13
16
|
configPaths?: string[];
|
|
@@ -26,6 +29,10 @@ export interface AuditOptions {
|
|
|
26
29
|
suggest?: boolean;
|
|
27
30
|
/** Override the tool-shape.json source — mainly for tests and self-hosted mirrors. */
|
|
28
31
|
toolShapeUrl?: string;
|
|
32
|
+
/** Identify each server against the published capture history, by hash, and report what has moved. */
|
|
33
|
+
changed?: boolean;
|
|
34
|
+
/** Override the capture-index.json source — mainly for tests and self-hosted mirrors. */
|
|
35
|
+
captureIndexUrl?: string;
|
|
29
36
|
/**
|
|
30
37
|
* The tool-search variables as this process's SHELL has them. Defaults to
|
|
31
38
|
* this process's environment. Overridable so a test can state a machine
|
|
@@ -40,6 +47,11 @@ export interface AuditOptions {
|
|
|
40
47
|
settings?: ToolSearchSource[];
|
|
41
48
|
onProgress?: (name: string, done: number, total: number) => void;
|
|
42
49
|
}
|
|
50
|
+
/** Fetch and parse the published capture index. Never throws: a failure is a report problem, not a crash. */
|
|
51
|
+
export declare function fetchCaptureIndex(url: string): Promise<{
|
|
52
|
+
index: CaptureIndex | null;
|
|
53
|
+
problem?: string;
|
|
54
|
+
}>;
|
|
43
55
|
/** Fetch and parse the published tool-shape baseline. Never throws: a failure is a report problem, not a crash. */
|
|
44
56
|
export declare function fetchToolShape(url: string): Promise<{
|
|
45
57
|
baseline: ToolShapeBaseline | null;
|
package/dist/audit/run.js
CHANGED
|
@@ -8,6 +8,7 @@ import { homedir } from 'node:os';
|
|
|
8
8
|
import { measureServer } from '../sweep/run.js';
|
|
9
9
|
import { parseDivergence } from '../core/divergence.js';
|
|
10
10
|
import { parseToolShapeBaseline } from '../core/tool-shape.js';
|
|
11
|
+
import { parseCaptureIndex } from '../core/capture-index.js';
|
|
11
12
|
import { buildReport, serverKey } from './audit.js';
|
|
12
13
|
import { toolSearchEnv } from './deferral.js';
|
|
13
14
|
import { configCandidates, loadConfigs, loadSettingsSources, settingsCandidates, } from './config.js';
|
|
@@ -15,6 +16,21 @@ import { configCandidates, loadConfigs, loadSettingsSources, settingsCandidates,
|
|
|
15
16
|
export const DEFAULT_DIVERGENCE_URL = 'https://raw.githubusercontent.com/athakur3/mcp-context-cost/main/results/divergence.json';
|
|
16
17
|
/** Where the published `tool-shape/v1` baseline lives when `--suggest` doesn't override it. */
|
|
17
18
|
export const DEFAULT_TOOL_SHAPE_URL = 'https://raw.githubusercontent.com/athakur3/mcp-context-cost/main/results/tool-shape.json';
|
|
19
|
+
/** Where the published `capture-index/v1` lives when `--changed` doesn't override it. */
|
|
20
|
+
export const DEFAULT_CAPTURE_INDEX_URL = 'https://raw.githubusercontent.com/athakur3/mcp-context-cost/main/results/capture-index.json';
|
|
21
|
+
/** Fetch and parse the published capture index. Never throws: a failure is a report problem, not a crash. */
|
|
22
|
+
export async function fetchCaptureIndex(url) {
|
|
23
|
+
try {
|
|
24
|
+
const res = await fetch(url, { signal: AbortSignal.timeout(15_000) });
|
|
25
|
+
if (!res.ok)
|
|
26
|
+
return { index: null, problem: `capture index: HTTP ${res.status} fetching ${url}` };
|
|
27
|
+
const index = parseCaptureIndex(await res.text());
|
|
28
|
+
return index ? { index } : { index: null, problem: `capture index: malformed data at ${url}` };
|
|
29
|
+
}
|
|
30
|
+
catch (e) {
|
|
31
|
+
return { index: null, problem: `capture index: failed to fetch ${url}: ${e.message}` };
|
|
32
|
+
}
|
|
33
|
+
}
|
|
18
34
|
/** Fetch and parse the published tool-shape baseline. Never throws: a failure is a report problem, not a crash. */
|
|
19
35
|
export async function fetchToolShape(url) {
|
|
20
36
|
try {
|
|
@@ -112,11 +128,19 @@ export async function runAudit(opts = {}) {
|
|
|
112
128
|
toolShape = fetched.baseline;
|
|
113
129
|
toolShapeProblem = fetched.problem;
|
|
114
130
|
}
|
|
131
|
+
let captureIndex = null;
|
|
132
|
+
let captureIndexProblem;
|
|
133
|
+
if (opts.changed) {
|
|
134
|
+
const fetched = await fetchCaptureIndex(opts.captureIndexUrl ?? DEFAULT_CAPTURE_INDEX_URL);
|
|
135
|
+
captureIndex = fetched.index;
|
|
136
|
+
captureIndexProblem = fetched.problem;
|
|
137
|
+
}
|
|
115
138
|
const report = buildReport(configs, measured, {
|
|
116
139
|
contextWindow: opts.contextWindow,
|
|
117
140
|
budget: opts.budget,
|
|
118
141
|
divergence,
|
|
119
142
|
toolShape,
|
|
143
|
+
captureIndex,
|
|
120
144
|
env: opts.env ?? toolSearchEnv(process.env),
|
|
121
145
|
settings: opts.settings ?? discoverSettings(opts),
|
|
122
146
|
});
|
|
@@ -124,5 +148,7 @@ export async function runAudit(opts = {}) {
|
|
|
124
148
|
report.problems.push(divergenceProblem);
|
|
125
149
|
if (toolShapeProblem)
|
|
126
150
|
report.problems.push(toolShapeProblem);
|
|
151
|
+
if (captureIndexProblem)
|
|
152
|
+
report.problems.push(captureIndexProblem);
|
|
127
153
|
return report;
|
|
128
154
|
}
|
package/dist/cli.js
CHANGED
|
@@ -99,8 +99,19 @@ function rejectUnknownFlags(cmd, argv, spec) {
|
|
|
99
99
|
const [, , cmd, ...rest] = process.argv;
|
|
100
100
|
if (cmd === 'audit') {
|
|
101
101
|
rejectUnknownFlags('audit', rest, {
|
|
102
|
-
value: [
|
|
103
|
-
|
|
102
|
+
value: [
|
|
103
|
+
'config',
|
|
104
|
+
'budget',
|
|
105
|
+
'baseline',
|
|
106
|
+
'max-increase',
|
|
107
|
+
'context',
|
|
108
|
+
'timeout',
|
|
109
|
+
'concurrency',
|
|
110
|
+
'divergence-url',
|
|
111
|
+
'tool-shape-url',
|
|
112
|
+
'capture-index-url',
|
|
113
|
+
],
|
|
114
|
+
boolean: ['json', 'docker', 'claude', 'suggest', 'changed'],
|
|
104
115
|
});
|
|
105
116
|
const argOf = (name) => {
|
|
106
117
|
const i = rest.indexOf(`--${name}`);
|
|
@@ -173,6 +184,8 @@ if (cmd === 'audit') {
|
|
|
173
184
|
divergenceUrl: argOf('divergence-url'),
|
|
174
185
|
suggest: rest.includes('--suggest'),
|
|
175
186
|
toolShapeUrl: argOf('tool-shape-url'),
|
|
187
|
+
changed: rest.includes('--changed'),
|
|
188
|
+
captureIndexUrl: argOf('capture-index-url'),
|
|
176
189
|
// Progress goes to stderr so `--json` stdout stays a single parseable object.
|
|
177
190
|
onProgress: json ? undefined : (name, done, total) => process.stderr.write(` [${done}/${total}] ${name}\n`),
|
|
178
191
|
});
|
|
@@ -282,7 +295,7 @@ else if (cmd === 'verify') {
|
|
|
282
295
|
}
|
|
283
296
|
else if (cmd === 'measure') {
|
|
284
297
|
rejectUnknownFlags('measure', rest, {
|
|
285
|
-
value: ['name', 'command', 'remote', 'timeout', 'docker-image'],
|
|
298
|
+
value: ['name', 'command', 'remote', 'timeout', 'docker-image', 'baseline', 'max-increase', 'budget'],
|
|
286
299
|
boolean: ['docker'],
|
|
287
300
|
});
|
|
288
301
|
const argOf = (name) => {
|
|
@@ -305,6 +318,45 @@ else if (cmd === 'measure') {
|
|
|
305
318
|
console.error('usage: mcp-context-cost measure --name <slug> --command "npx -y <server>" [--timeout ms] [--docker]');
|
|
306
319
|
process.exit(2);
|
|
307
320
|
}
|
|
321
|
+
const { diffServer, evaluateServerGate, formatServerDiff, parseBaselineMeasurement, } = await import('./core/server-diff.js');
|
|
322
|
+
// Gate limits are read before anything is measured: an unusable number is a
|
|
323
|
+
// usage error, and finding that out after a two-minute container launch is
|
|
324
|
+
// the wrong time to find it out.
|
|
325
|
+
const numericFlag = (flag) => {
|
|
326
|
+
const raw = argOf(flag);
|
|
327
|
+
if (raw === undefined)
|
|
328
|
+
return undefined;
|
|
329
|
+
const v = Number(raw);
|
|
330
|
+
if (!Number.isFinite(v) || v < 0) {
|
|
331
|
+
console.error(`--${flag} must be a non-negative number, got '${raw}'`);
|
|
332
|
+
process.exit(2);
|
|
333
|
+
}
|
|
334
|
+
return v;
|
|
335
|
+
};
|
|
336
|
+
const maxIncrease = numericFlag('max-increase');
|
|
337
|
+
const budget = numericFlag('budget');
|
|
338
|
+
const baselinePath = argOf('baseline');
|
|
339
|
+
if (maxIncrease !== undefined && !baselinePath) {
|
|
340
|
+
console.error('--max-increase needs --baseline <measurement.json> to compare against');
|
|
341
|
+
process.exit(2);
|
|
342
|
+
}
|
|
343
|
+
let baseline = null;
|
|
344
|
+
if (baselinePath) {
|
|
345
|
+
let raw;
|
|
346
|
+
try {
|
|
347
|
+
raw = readFileSync(baselinePath, 'utf8');
|
|
348
|
+
}
|
|
349
|
+
catch (e) {
|
|
350
|
+
console.error(`cannot read baseline ${baselinePath}: ${e.message}`);
|
|
351
|
+
process.exit(2);
|
|
352
|
+
}
|
|
353
|
+
const parsed = parseBaselineMeasurement(raw);
|
|
354
|
+
if (!parsed.measurement) {
|
|
355
|
+
console.error(`${baselinePath}: ${parsed.problem}`);
|
|
356
|
+
process.exit(2);
|
|
357
|
+
}
|
|
358
|
+
baseline = parsed.measurement;
|
|
359
|
+
}
|
|
308
360
|
const { measureServer } = await import('./sweep/run.js');
|
|
309
361
|
const m = await measureServer(name, remoteUrl ? `npx -y mcp-remote ${remoteUrl}` : command, {
|
|
310
362
|
timeoutMs: Number(argOf('timeout') ?? 60_000),
|
|
@@ -316,6 +368,20 @@ else if (cmd === 'measure') {
|
|
|
316
368
|
console.log(ok
|
|
317
369
|
? `${name}: ${m.totalTokens} tokens across ${m.toolCount} tools (${m.status}) — results/${name}/measurement.json, badges/${name}.json`
|
|
318
370
|
: `${name}: ${m.status} — ${m.notes ?? ''}`);
|
|
371
|
+
if (baseline || budget !== undefined || maxIncrease !== undefined) {
|
|
372
|
+
const diff = diffServer(name, baseline, m);
|
|
373
|
+
if (baseline) {
|
|
374
|
+
console.log('');
|
|
375
|
+
console.log(`diff vs baseline ${baselinePath}`);
|
|
376
|
+
console.log(formatServerDiff(diff));
|
|
377
|
+
}
|
|
378
|
+
const gate = evaluateServerGate(diff, { budget, maxIncrease });
|
|
379
|
+
if (!gate.pass) {
|
|
380
|
+
console.log('');
|
|
381
|
+
console.error(gate.failure);
|
|
382
|
+
process.exit(1);
|
|
383
|
+
}
|
|
384
|
+
}
|
|
319
385
|
process.exit(ok ? 0 : 1);
|
|
320
386
|
}
|
|
321
387
|
else if (cmd !== undefined && cmd !== '--help' && cmd !== '-h') {
|
|
@@ -333,5 +399,8 @@ else {
|
|
|
333
399
|
console.log(' verify --remote <url> [--json] same, fetched from a measurement URL');
|
|
334
400
|
console.log(' measure --name x --command "npx -y <server>" run a one-off measurement');
|
|
335
401
|
console.log(' measure --remote <url> [--name x] measure a remote server via mcp-remote');
|
|
402
|
+
console.log(' [--baseline <measurement.json>] [--max-increase N] [--budget N]');
|
|
403
|
+
console.log(' gate your own server in CI: fail the');
|
|
404
|
+
console.log(' build when a change adds too much');
|
|
336
405
|
console.log('exit codes: 0 ok, 1 verification/measurement/budget failed, 2 usage error');
|
|
337
406
|
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which published version is this, by its bytes?
|
|
3
|
+
*
|
|
4
|
+
* `audit --changed` answers "did the servers in my config get heavier?" — and
|
|
5
|
+
* the whole question turns on joining a machine's installed server to the
|
|
6
|
+
* published history. Matching by *name* would be a lie waiting to happen: a
|
|
7
|
+
* config's keys are arbitrary local labels, so a server a user calls `github`
|
|
8
|
+
* may be a fork, a pin, or something else entirely, and reporting the official
|
|
9
|
+
* server's movement against it would be a confident false statement.
|
|
10
|
+
*
|
|
11
|
+
* So the join is byte identity — the same discipline the Claude column already
|
|
12
|
+
* uses to decide whether it may print. `results/capture-index.json` maps the
|
|
13
|
+
* `canonicalSha256` of every capture the project has ever published to the
|
|
14
|
+
* server and date it belongs to. A local measurement either *is* one of those
|
|
15
|
+
* captures, exactly, or it is not in the published history at all, and there is
|
|
16
|
+
* no third state to be fuzzy about.
|
|
17
|
+
*
|
|
18
|
+
* The index is derivable from the per-server tool vectors, which is why it can
|
|
19
|
+
* only see as far back as those vectors go: a version published before the
|
|
20
|
+
* vectors existed is not in the index and reads as unknown, not as absent from
|
|
21
|
+
* history. Versioned independently, like every published artifact.
|
|
22
|
+
*/
|
|
23
|
+
/** Method identifier, versioned independently of METHODOLOGY_VERSION. */
|
|
24
|
+
export declare const CAPTURE_INDEX_METHOD = "capture-index/v1";
|
|
25
|
+
export interface IndexedCapture {
|
|
26
|
+
server: string;
|
|
27
|
+
/** The day this capture was first measured. */
|
|
28
|
+
date: string;
|
|
29
|
+
totalTokens: number;
|
|
30
|
+
toolCount: number;
|
|
31
|
+
}
|
|
32
|
+
export interface CaptureIndex {
|
|
33
|
+
method: string;
|
|
34
|
+
/** UTC day the index was derived (YYYY-MM-DD). */
|
|
35
|
+
generatedAt: string;
|
|
36
|
+
/** canonicalSha256 → the capture it identifies. */
|
|
37
|
+
captures: Record<string, IndexedCapture>;
|
|
38
|
+
/** server → the newest published capture's hash. */
|
|
39
|
+
current: Record<string, string>;
|
|
40
|
+
}
|
|
41
|
+
export declare function parseCaptureIndex(text: string): CaptureIndex | null;
|
|
42
|
+
/**
|
|
43
|
+
* What the index can say about one locally measured server.
|
|
44
|
+
*
|
|
45
|
+
* `unknown` is the honest and common outcome: users install versions this
|
|
46
|
+
* project has never measured, and versions published before the capture index
|
|
47
|
+
* existed are not in it either. It is reported, not hidden — an absence of a
|
|
48
|
+
* record about that version, not a statement that nothing changed.
|
|
49
|
+
*/
|
|
50
|
+
export type CaptureVerdict = {
|
|
51
|
+
kind: 'behind';
|
|
52
|
+
/** The published server this capture belongs to — established by bytes, not by name. */
|
|
53
|
+
server: string;
|
|
54
|
+
yourDate: string;
|
|
55
|
+
yourTokens: number;
|
|
56
|
+
currentDate: string;
|
|
57
|
+
currentTokens: number;
|
|
58
|
+
/** What moving to the current published version would add to every request. */
|
|
59
|
+
deltaTokens: number;
|
|
60
|
+
} | {
|
|
61
|
+
kind: 'current';
|
|
62
|
+
server: string;
|
|
63
|
+
date: string;
|
|
64
|
+
tokens: number;
|
|
65
|
+
} | {
|
|
66
|
+
kind: 'unknown';
|
|
67
|
+
};
|
|
68
|
+
/**
|
|
69
|
+
* Identify a local capture against the published index.
|
|
70
|
+
*
|
|
71
|
+
* A hash that matches the server's newest published capture is `current`; one
|
|
72
|
+
* that matches an older capture is `behind`, carrying the exact delta to
|
|
73
|
+
* current. Anything else — including a version newer than anything published —
|
|
74
|
+
* is `unknown`, because the index cannot describe what it has never measured.
|
|
75
|
+
*/
|
|
76
|
+
export declare function identify(canonicalSha256: string | null | undefined, index: CaptureIndex): CaptureVerdict;
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which published version is this, by its bytes?
|
|
3
|
+
*
|
|
4
|
+
* `audit --changed` answers "did the servers in my config get heavier?" — and
|
|
5
|
+
* the whole question turns on joining a machine's installed server to the
|
|
6
|
+
* published history. Matching by *name* would be a lie waiting to happen: a
|
|
7
|
+
* config's keys are arbitrary local labels, so a server a user calls `github`
|
|
8
|
+
* may be a fork, a pin, or something else entirely, and reporting the official
|
|
9
|
+
* server's movement against it would be a confident false statement.
|
|
10
|
+
*
|
|
11
|
+
* So the join is byte identity — the same discipline the Claude column already
|
|
12
|
+
* uses to decide whether it may print. `results/capture-index.json` maps the
|
|
13
|
+
* `canonicalSha256` of every capture the project has ever published to the
|
|
14
|
+
* server and date it belongs to. A local measurement either *is* one of those
|
|
15
|
+
* captures, exactly, or it is not in the published history at all, and there is
|
|
16
|
+
* no third state to be fuzzy about.
|
|
17
|
+
*
|
|
18
|
+
* The index is derivable from the per-server tool vectors, which is why it can
|
|
19
|
+
* only see as far back as those vectors go: a version published before the
|
|
20
|
+
* vectors existed is not in the index and reads as unknown, not as absent from
|
|
21
|
+
* history. Versioned independently, like every published artifact.
|
|
22
|
+
*/
|
|
23
|
+
/** Method identifier, versioned independently of METHODOLOGY_VERSION. */
|
|
24
|
+
export const CAPTURE_INDEX_METHOD = 'capture-index/v1';
|
|
25
|
+
export function parseCaptureIndex(text) {
|
|
26
|
+
let parsed;
|
|
27
|
+
try {
|
|
28
|
+
parsed = JSON.parse(text);
|
|
29
|
+
}
|
|
30
|
+
catch {
|
|
31
|
+
return null;
|
|
32
|
+
}
|
|
33
|
+
const i = parsed;
|
|
34
|
+
if (!i || typeof i.generatedAt !== 'string')
|
|
35
|
+
return null;
|
|
36
|
+
if (!i.captures || typeof i.captures !== 'object')
|
|
37
|
+
return null;
|
|
38
|
+
if (!i.current || typeof i.current !== 'object')
|
|
39
|
+
return null;
|
|
40
|
+
const captures = {};
|
|
41
|
+
for (const [sha, c] of Object.entries(i.captures)) {
|
|
42
|
+
const v = c;
|
|
43
|
+
if (typeof v?.server !== 'string' || typeof v.date !== 'string')
|
|
44
|
+
continue;
|
|
45
|
+
if (typeof v.totalTokens !== 'number' || typeof v.toolCount !== 'number')
|
|
46
|
+
continue;
|
|
47
|
+
captures[sha] = { server: v.server, date: v.date, totalTokens: v.totalTokens, toolCount: v.toolCount };
|
|
48
|
+
}
|
|
49
|
+
const current = {};
|
|
50
|
+
for (const [server, sha] of Object.entries(i.current))
|
|
51
|
+
if (typeof sha === 'string')
|
|
52
|
+
current[server] = sha;
|
|
53
|
+
return {
|
|
54
|
+
method: typeof i.method === 'string' ? i.method : CAPTURE_INDEX_METHOD,
|
|
55
|
+
generatedAt: i.generatedAt,
|
|
56
|
+
captures,
|
|
57
|
+
current,
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Identify a local capture against the published index.
|
|
62
|
+
*
|
|
63
|
+
* A hash that matches the server's newest published capture is `current`; one
|
|
64
|
+
* that matches an older capture is `behind`, carrying the exact delta to
|
|
65
|
+
* current. Anything else — including a version newer than anything published —
|
|
66
|
+
* is `unknown`, because the index cannot describe what it has never measured.
|
|
67
|
+
*/
|
|
68
|
+
export function identify(canonicalSha256, index) {
|
|
69
|
+
if (!canonicalSha256)
|
|
70
|
+
return { kind: 'unknown' };
|
|
71
|
+
const mine = index.captures[canonicalSha256];
|
|
72
|
+
if (!mine)
|
|
73
|
+
return { kind: 'unknown' };
|
|
74
|
+
const currentSha = index.current[mine.server];
|
|
75
|
+
if (!currentSha || currentSha === canonicalSha256) {
|
|
76
|
+
return { kind: 'current', server: mine.server, date: mine.date, tokens: mine.totalTokens };
|
|
77
|
+
}
|
|
78
|
+
const current = index.captures[currentSha];
|
|
79
|
+
// A `current` pointer with no capture behind it describes nothing; treat the
|
|
80
|
+
// version as identified but with nothing to compare it against.
|
|
81
|
+
if (!current)
|
|
82
|
+
return { kind: 'current', server: mine.server, date: mine.date, tokens: mine.totalTokens };
|
|
83
|
+
return {
|
|
84
|
+
kind: 'behind',
|
|
85
|
+
server: mine.server,
|
|
86
|
+
yourDate: mine.date,
|
|
87
|
+
yourTokens: mine.totalTokens,
|
|
88
|
+
currentDate: current.date,
|
|
89
|
+
currentTokens: current.totalTokens,
|
|
90
|
+
deltaTokens: current.totalTokens - mine.totalTokens,
|
|
91
|
+
};
|
|
92
|
+
}
|