mcp-context-cost 0.13.2 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -14
- package/dist/audit/audit.d.ts +21 -1
- package/dist/audit/audit.js +71 -17
- package/dist/audit/config.d.ts +19 -2
- package/dist/audit/config.js +140 -24
- package/dist/audit/deferral.d.ts +45 -4
- package/dist/audit/deferral.js +27 -4
- package/dist/audit/remote.d.ts +56 -0
- package/dist/audit/remote.js +126 -0
- package/dist/audit/run.d.ts +32 -2
- package/dist/audit/run.js +60 -5
- package/dist/cli.js +2 -1
- package/dist/sweep/cross-check.js +2 -2
- package/dist/sweep/pr-check.d.ts +62 -0
- package/dist/sweep/pr-check.js +296 -0
- package/dist/sweep/registry-scan.d.ts +355 -0
- package/dist/sweep/registry-scan.js +432 -0
- package/dist/sweep/run.d.ts +17 -0
- package/dist/sweep/run.js +49 -9
- package/dist/sweep/servers-schema.d.ts +7 -0
- package/dist/sweep/servers-schema.js +8 -1
- package/package.json +3 -1
package/dist/audit/deferral.d.ts
CHANGED
|
@@ -37,11 +37,34 @@
|
|
|
37
37
|
* generation; `CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS` "keeps tool search
|
|
38
38
|
* off. You can't override it by setting `ENABLE_TOOL_SEARCH` yourself."
|
|
39
39
|
* A server with `alwaysLoad: true` loads at session start regardless.
|
|
40
|
+
* - The same page, re-read 2026-09-06 (the roadmap's dated re-read). The
|
|
41
|
+
* value table stands as quoted, and four things moved around it:
|
|
42
|
+
* (1) on Google Cloud's Agent Platform, tool search is on by default for
|
|
43
|
+
* the Claude 4.5 generation and later "the same as on the Anthropic API"
|
|
44
|
+
* — "Before v2.1.221, Claude Code disabled tool search for all models on
|
|
45
|
+
* Google Cloud's Agent Platform unless you set ENABLE_TOOL_SEARCH=true";
|
|
46
|
+
* the exception below already names only the earlier models. (2) Under
|
|
47
|
+
* `CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS`, "your organization can keep
|
|
48
|
+
* tool search on through managed settings, on Claude Code v2.1.227 or
|
|
49
|
+
* later" — on a direct connection or a gateway, not on a cloud provider.
|
|
50
|
+
* This audit reads the managed settings file for the three variables and
|
|
51
|
+
* nothing else, so that override is not read here and the variable is
|
|
52
|
+
* still resolved as "off". (3) `alwaysLoad: true` is an entry field on
|
|
53
|
+
* every server type, and a tool can carry `"anthropic/alwaysLoad": true`
|
|
54
|
+
* in its `_meta`. The entry form is read from the config now
|
|
55
|
+
* (`DeferralServer.alwaysLoad`) and counted rather than listed; the
|
|
56
|
+
* per-tool form is still a listed condition. (4) "Claude Code truncates
|
|
57
|
+
* tool descriptions and server instructions at 2KB each", which bounds what
|
|
58
|
+
* a deferring session loads at start — noted beside the session-start
|
|
59
|
+
* metric in METHODOLOGY, not applied to any number here.
|
|
40
60
|
*
|
|
41
|
-
* No default deferral is on record here for the other
|
|
42
|
-
* discovers
|
|
43
|
-
*
|
|
44
|
-
*
|
|
61
|
+
* No default deferral is on record here for the other clients this tool
|
|
62
|
+
* discovers — Claude Desktop, Cursor, VS Code, Windsurf, and from 2026-09-06
|
|
63
|
+
* Codex CLI, Gemini CLI, Zed, Kiro and Goose, whose configuration pages were
|
|
64
|
+
* read that day and say nothing about deferring tool definitions. That is an
|
|
65
|
+
* absence of a record, not a measurement of those clients, and it is printed
|
|
66
|
+
* as such — the same rule the rest of this project follows for a value it has
|
|
67
|
+
* not observed.
|
|
45
68
|
*/
|
|
46
69
|
import type { DivergenceRun } from '../core/divergence.js';
|
|
47
70
|
/** Share of the context window at which deferral activates under `auto`. */
|
|
@@ -267,8 +290,17 @@ export declare const PUBLISHED_WIRE_TO_CLIENT_RATIO: WireToClientRatio;
|
|
|
267
290
|
export declare function wireToClientRatio(run?: DivergenceRun | null): WireToClientRatio;
|
|
268
291
|
/** One measured server, as the deferral arithmetic needs it. */
|
|
269
292
|
export interface DeferralServer {
|
|
293
|
+
/** For naming the servers a verdict singles out; the arithmetic never reads it. */
|
|
294
|
+
name?: string;
|
|
270
295
|
/** o200k tokens over the wire capture — the audit's own unit. */
|
|
271
296
|
tokens: number;
|
|
297
|
+
/**
|
|
298
|
+
* The entry is pinned `alwaysLoad: true`, so its tools load at session start
|
|
299
|
+
* whatever the tool-search setting says, and it does not count toward a
|
|
300
|
+
* threshold — the documented `auto` mode counts "the tools it would
|
|
301
|
+
* otherwise defer".
|
|
302
|
+
*/
|
|
303
|
+
alwaysLoad?: boolean;
|
|
272
304
|
/**
|
|
273
305
|
* Anthropic's own count for this server from a current divergence row, when
|
|
274
306
|
* `--claude` supplied one. `null` means no current match, `undefined` means
|
|
@@ -367,6 +399,15 @@ export interface DeferralVerdict {
|
|
|
367
399
|
sharedMeasurements: number;
|
|
368
400
|
/** Conditions this cannot read, under which a deferring client pays in full. */
|
|
369
401
|
exceptions: string[];
|
|
402
|
+
/**
|
|
403
|
+
* Servers in this scope pinned `alwaysLoad: true` in their entry, and their
|
|
404
|
+
* wire tokens. Read from the config, so it is stated rather than listed as a
|
|
405
|
+
* condition: whatever the mode, these load at session start.
|
|
406
|
+
*/
|
|
407
|
+
alwaysLoad: {
|
|
408
|
+
servers: string[];
|
|
409
|
+
tokens: number;
|
|
410
|
+
};
|
|
370
411
|
}
|
|
371
412
|
/**
|
|
372
413
|
* Read one session's deferral position. Pure arithmetic over a built scope — no
|
package/dist/audit/deferral.js
CHANGED
|
@@ -286,8 +286,23 @@ export function wireToClientRatio(run) {
|
|
|
286
286
|
return PUBLISHED_WIRE_TO_CLIENT_RATIO;
|
|
287
287
|
return { low, high, fixedOverhead, servers, source: `the ${run.measuredAt} ${run.model} divergence run` };
|
|
288
288
|
}
|
|
289
|
-
/**
|
|
290
|
-
|
|
289
|
+
/**
|
|
290
|
+
* Clients this tool discovers that have no default deferral on record. Each
|
|
291
|
+
* client's own MCP configuration page was read on the date config.ts gives
|
|
292
|
+
* and says nothing about deferring tool definitions — Windsurf's states a cap
|
|
293
|
+
* of 100 tools, which is a different thing and not a deferral.
|
|
294
|
+
*/
|
|
295
|
+
const NO_DEFERRAL_ON_RECORD = new Set([
|
|
296
|
+
'claude-desktop',
|
|
297
|
+
'cursor',
|
|
298
|
+
'vscode',
|
|
299
|
+
'windsurf',
|
|
300
|
+
'codex',
|
|
301
|
+
'gemini',
|
|
302
|
+
'zed',
|
|
303
|
+
'kiro',
|
|
304
|
+
'goose',
|
|
305
|
+
]);
|
|
291
306
|
/**
|
|
292
307
|
* Where deferral does not apply even when the machine's setting says it should.
|
|
293
308
|
* None of these can be read from the config or the environment, so they are
|
|
@@ -298,7 +313,7 @@ const EXCEPTIONS = [
|
|
|
298
313
|
'a Microsoft Foundry deployment hosted on Azure, which rejects tool search server-side',
|
|
299
314
|
"Google Cloud's Agent Platform on a model earlier than the Claude 4.5 generation",
|
|
300
315
|
'a model without support for tool_reference blocks (before Sonnet 4.5 / Haiku 4.5 / Opus 4.5)',
|
|
301
|
-
'a
|
|
316
|
+
'a tool whose _meta carries "anthropic/alwaysLoad": true, which this audit does not read from a capture',
|
|
302
317
|
];
|
|
303
318
|
function estimate(servers, ratio) {
|
|
304
319
|
let low = 0;
|
|
@@ -341,6 +356,11 @@ export function evaluateDeferral(scope, opts) {
|
|
|
341
356
|
const wireTokens = scope.servers.reduce((a, s) => a + s.tokens, 0);
|
|
342
357
|
const isFloor = scope.skippedCount > 0;
|
|
343
358
|
const sharedMeasurements = scope.sharedMeasurements;
|
|
359
|
+
const pinned = scope.servers.filter((s) => s.alwaysLoad === true);
|
|
360
|
+
const alwaysLoad = {
|
|
361
|
+
servers: pinned.map((s) => s.name ?? '(unnamed)'),
|
|
362
|
+
tokens: pinned.reduce((a, s) => a + s.tokens, 0),
|
|
363
|
+
};
|
|
344
364
|
// Every field a verdict carries, at its "nothing to say" value. Each mode
|
|
345
365
|
// below overrides only what it can actually answer.
|
|
346
366
|
const base = {
|
|
@@ -364,6 +384,7 @@ export function evaluateDeferral(scope, opts) {
|
|
|
364
384
|
distanceTokens: null,
|
|
365
385
|
crosses: null,
|
|
366
386
|
exceptions: [],
|
|
387
|
+
alwaysLoad,
|
|
367
388
|
};
|
|
368
389
|
if (scope.client !== 'claude-code') {
|
|
369
390
|
return {
|
|
@@ -422,7 +443,9 @@ export function evaluateDeferral(scope, opts) {
|
|
|
422
443
|
};
|
|
423
444
|
}
|
|
424
445
|
const ratio = wireToClientRatio(opts.divergence);
|
|
425
|
-
|
|
446
|
+
// Only what the client would otherwise defer is held against the threshold;
|
|
447
|
+
// a pinned server loads either way and is not part of the question.
|
|
448
|
+
const clientTokens = estimate(scope.servers.filter((s) => s.alwaysLoad !== true), ratio);
|
|
426
449
|
// At-or-above, on the documented "defers all of them once the definitions
|
|
427
450
|
// reach 10%". A range that is entirely over is over even if it is a floor:
|
|
428
451
|
// more unmeasured tokens cannot take it back under.
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What a remote MCP endpoint says to a request that carries no credential.
|
|
3
|
+
*
|
|
4
|
+
* `audit` used to skip every `url` entry as `remote-not-measurable`, which
|
|
5
|
+
* under-counted exactly the stacks the audit is for — vendors are moving to
|
|
6
|
+
* hosted endpoints. The obvious fix, handing every url to the `mcp-remote`
|
|
7
|
+
* bridge the sweep already measures open endpoints through, is wrong on a
|
|
8
|
+
* developer machine: against an OAuth-walled endpoint mcp-remote opens a
|
|
9
|
+
* browser window and waits for the callback (its README says so), which an
|
|
10
|
+
* audit has no business doing once per server in someone's config; and in a
|
|
11
|
+
* headless run it waits until this harness's timeout, so the row would read
|
|
12
|
+
* `timeout` — a word that blames the clock for a credential.
|
|
13
|
+
*
|
|
14
|
+
* So the endpoint is asked first, with the request every MCP session begins
|
|
15
|
+
* with: `initialize` over streamable HTTP, then a GET with an event-stream
|
|
16
|
+
* `accept` for an SSE endpoint that refuses POST. Probed 2026-09-06 from this
|
|
17
|
+
* repository: `mcp.linear.app/mcp`, `mcp.zapier.com/api/mcp/mcp` and
|
|
18
|
+
* `mcp.vercel.com` each answer `401` with a `WWW-Authenticate: Bearer …`
|
|
19
|
+
* header naming the OAuth resource; `mcp.deepwiki.com/mcp`, `learn.microsoft.com/api/mcp`,
|
|
20
|
+
* `docs.mcp.cloudflare.com/sse` and `huggingface.co/mcp` answer `200` with a
|
|
21
|
+
* session or a stream. Nothing is inferred from a hostname: the status line
|
|
22
|
+
* and the header are the server's own words, and the report quotes them.
|
|
23
|
+
*
|
|
24
|
+
* Only an endpoint that answered without a credential is handed to the
|
|
25
|
+
* bridge, so the bridge never has a reason to open a browser. Header VALUES a
|
|
26
|
+
* config carries (a static bearer token) are sent with the probe and the
|
|
27
|
+
* bridge, exactly as env values are spawned into a stdio server, and never
|
|
28
|
+
* written to a report — see `ConfiguredServer.headers` in config.ts.
|
|
29
|
+
*/
|
|
30
|
+
export interface RemoteProbe {
|
|
31
|
+
/**
|
|
32
|
+
* `open`: answered the unauthenticated request as an MCP endpoint does.
|
|
33
|
+
* `auth-walled`: answered 401 or 403 — it works, and it wants a credential
|
|
34
|
+
* this audit does not hold. `unreachable`: no MCP answer arrived — a
|
|
35
|
+
* connection failure, a timeout, or a status that is neither of the above.
|
|
36
|
+
*/
|
|
37
|
+
kind: 'open' | 'auth-walled' | 'unreachable';
|
|
38
|
+
/** The HTTP status that decided it, when a response arrived at all. */
|
|
39
|
+
status?: number;
|
|
40
|
+
/** The `WWW-Authenticate` header, verbatim (clipped), when the server sent one. */
|
|
41
|
+
wwwAuthenticate?: string;
|
|
42
|
+
/** One line a report can print: the server's own words, or the failure's. */
|
|
43
|
+
detail: string;
|
|
44
|
+
}
|
|
45
|
+
/** What a probe waits for an answer, unless the caller says otherwise. */
|
|
46
|
+
export declare const DEFAULT_PROBE_TIMEOUT_MS = 15000;
|
|
47
|
+
/**
|
|
48
|
+
* Ask one endpoint, once, and report what it said.
|
|
49
|
+
*
|
|
50
|
+
* Never throws: an endpoint this could not reach is a probe result, not a
|
|
51
|
+
* crash, because the audit goes on to the next server either way.
|
|
52
|
+
*/
|
|
53
|
+
export declare function probeRemote(url: string, opts?: {
|
|
54
|
+
headers?: Record<string, string>;
|
|
55
|
+
timeoutMs?: number;
|
|
56
|
+
}): Promise<RemoteProbe>;
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What a remote MCP endpoint says to a request that carries no credential.
|
|
3
|
+
*
|
|
4
|
+
* `audit` used to skip every `url` entry as `remote-not-measurable`, which
|
|
5
|
+
* under-counted exactly the stacks the audit is for — vendors are moving to
|
|
6
|
+
* hosted endpoints. The obvious fix, handing every url to the `mcp-remote`
|
|
7
|
+
* bridge the sweep already measures open endpoints through, is wrong on a
|
|
8
|
+
* developer machine: against an OAuth-walled endpoint mcp-remote opens a
|
|
9
|
+
* browser window and waits for the callback (its README says so), which an
|
|
10
|
+
* audit has no business doing once per server in someone's config; and in a
|
|
11
|
+
* headless run it waits until this harness's timeout, so the row would read
|
|
12
|
+
* `timeout` — a word that blames the clock for a credential.
|
|
13
|
+
*
|
|
14
|
+
* So the endpoint is asked first, with the request every MCP session begins
|
|
15
|
+
* with: `initialize` over streamable HTTP, then a GET with an event-stream
|
|
16
|
+
* `accept` for an SSE endpoint that refuses POST. Probed 2026-09-06 from this
|
|
17
|
+
* repository: `mcp.linear.app/mcp`, `mcp.zapier.com/api/mcp/mcp` and
|
|
18
|
+
* `mcp.vercel.com` each answer `401` with a `WWW-Authenticate: Bearer …`
|
|
19
|
+
* header naming the OAuth resource; `mcp.deepwiki.com/mcp`, `learn.microsoft.com/api/mcp`,
|
|
20
|
+
* `docs.mcp.cloudflare.com/sse` and `huggingface.co/mcp` answer `200` with a
|
|
21
|
+
* session or a stream. Nothing is inferred from a hostname: the status line
|
|
22
|
+
* and the header are the server's own words, and the report quotes them.
|
|
23
|
+
*
|
|
24
|
+
* Only an endpoint that answered without a credential is handed to the
|
|
25
|
+
* bridge, so the bridge never has a reason to open a browser. Header VALUES a
|
|
26
|
+
* config carries (a static bearer token) are sent with the probe and the
|
|
27
|
+
* bridge, exactly as env values are spawned into a stdio server, and never
|
|
28
|
+
* written to a report — see `ConfiguredServer.headers` in config.ts.
|
|
29
|
+
*/
|
|
30
|
+
/** The first request of every MCP session, which is all a probe needs to send. */
|
|
31
|
+
const INITIALIZE = JSON.stringify({
|
|
32
|
+
jsonrpc: '2.0',
|
|
33
|
+
id: 1,
|
|
34
|
+
method: 'initialize',
|
|
35
|
+
params: {
|
|
36
|
+
protocolVersion: '2025-06-18',
|
|
37
|
+
capabilities: {},
|
|
38
|
+
clientInfo: { name: 'mcp-context-cost', version: '0' },
|
|
39
|
+
},
|
|
40
|
+
});
|
|
41
|
+
/** Long enough to be quoted whole in a report, short enough not to be the report. */
|
|
42
|
+
const CLIP = 300;
|
|
43
|
+
const clip = (s) => {
|
|
44
|
+
const one = s.replace(/\s+/g, ' ').trim();
|
|
45
|
+
return one.length > CLIP ? `${one.slice(0, CLIP)}…` : one;
|
|
46
|
+
};
|
|
47
|
+
/** What a probe waits for an answer, unless the caller says otherwise. */
|
|
48
|
+
export const DEFAULT_PROBE_TIMEOUT_MS = 15_000;
|
|
49
|
+
/**
|
|
50
|
+
* Ask one endpoint, once, and report what it said.
|
|
51
|
+
*
|
|
52
|
+
* Never throws: an endpoint this could not reach is a probe result, not a
|
|
53
|
+
* crash, because the audit goes on to the next server either way.
|
|
54
|
+
*/
|
|
55
|
+
export async function probeRemote(url, opts = {}) {
|
|
56
|
+
const timeoutMs = opts.timeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS;
|
|
57
|
+
const configured = opts.headers ?? {};
|
|
58
|
+
const attempt = async (init) => {
|
|
59
|
+
const res = await fetch(url, { ...init, signal: AbortSignal.timeout(timeoutMs), redirect: 'follow' });
|
|
60
|
+
// The headers are the answer. The body is not read — an event stream
|
|
61
|
+
// stays open for the life of a session, and this is not a session.
|
|
62
|
+
try {
|
|
63
|
+
await res.body?.cancel();
|
|
64
|
+
}
|
|
65
|
+
catch {
|
|
66
|
+
/* a body that refuses to be cancelled changes nothing the headers said */
|
|
67
|
+
}
|
|
68
|
+
return res;
|
|
69
|
+
};
|
|
70
|
+
try {
|
|
71
|
+
let res = await attempt({
|
|
72
|
+
method: 'POST',
|
|
73
|
+
headers: {
|
|
74
|
+
...configured,
|
|
75
|
+
'content-type': 'application/json',
|
|
76
|
+
accept: 'application/json, text/event-stream',
|
|
77
|
+
'mcp-protocol-version': '2025-06-18',
|
|
78
|
+
},
|
|
79
|
+
body: INITIALIZE,
|
|
80
|
+
});
|
|
81
|
+
// The older SSE transport opens its stream on GET and may refuse the POST
|
|
82
|
+
// outright; an endpoint that does is asked the way it expects to be.
|
|
83
|
+
if (res.status === 404 || res.status === 405) {
|
|
84
|
+
res = await attempt({ method: 'GET', headers: { ...configured, accept: 'text/event-stream' } });
|
|
85
|
+
}
|
|
86
|
+
return classify(res);
|
|
87
|
+
}
|
|
88
|
+
catch (e) {
|
|
89
|
+
return { kind: 'unreachable', detail: describeFailure(e, timeoutMs) };
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
function classify(res) {
|
|
93
|
+
const www = res.headers.get('www-authenticate');
|
|
94
|
+
const type = (res.headers.get('content-type') ?? '').split(';')[0].trim().toLowerCase();
|
|
95
|
+
if (res.status === 401 || res.status === 403) {
|
|
96
|
+
return {
|
|
97
|
+
kind: 'auth-walled',
|
|
98
|
+
status: res.status,
|
|
99
|
+
...(www ? { wwwAuthenticate: clip(www) } : {}),
|
|
100
|
+
detail: `HTTP ${res.status}${www ? ` — WWW-Authenticate: ${clip(www)}` : ''}`,
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
if (res.ok) {
|
|
104
|
+
// A JSON-RPC answer or an event stream is what an MCP endpoint sends. A
|
|
105
|
+
// 200 carrying HTML is a login page or a docs site at that address.
|
|
106
|
+
if (type === 'application/json' || type === 'text/event-stream') {
|
|
107
|
+
return { kind: 'open', status: res.status, detail: `HTTP ${res.status} ${type}` };
|
|
108
|
+
}
|
|
109
|
+
return {
|
|
110
|
+
kind: 'unreachable',
|
|
111
|
+
status: res.status,
|
|
112
|
+
detail: `HTTP ${res.status} with ${type || 'no content-type'}, which is not an MCP response`,
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
return { kind: 'unreachable', status: res.status, detail: `HTTP ${res.status}` };
|
|
116
|
+
}
|
|
117
|
+
function describeFailure(e, timeoutMs) {
|
|
118
|
+
const err = e;
|
|
119
|
+
if (err?.name === 'TimeoutError' || err?.name === 'AbortError')
|
|
120
|
+
return `no answer within ${timeoutMs}ms`;
|
|
121
|
+
// Node's fetch wraps the socket error: "fetch failed" with the code underneath.
|
|
122
|
+
const code = err?.cause?.code;
|
|
123
|
+
if (code)
|
|
124
|
+
return code;
|
|
125
|
+
return err?.cause?.message ?? err?.message ?? String(e);
|
|
126
|
+
}
|
package/dist/audit/run.d.ts
CHANGED
|
@@ -4,7 +4,8 @@ import { type ToolShapeBaseline } from '../core/tool-shape.js';
|
|
|
4
4
|
import { type CaptureIndex } from '../core/capture-index.js';
|
|
5
5
|
import { type AuditReport } from './audit.js';
|
|
6
6
|
import { type ToolSearchEnv, type ToolSearchSource } from './deferral.js';
|
|
7
|
-
import { type
|
|
7
|
+
import { type RemoteProbe } from './remote.js';
|
|
8
|
+
import { type ConfiguredServer, type LoadedConfig } from './config.js';
|
|
8
9
|
/** Where the published `tools-delta/v1` run lives when `--claude` doesn't override it. */
|
|
9
10
|
export declare const DEFAULT_DIVERGENCE_URL = "https://raw.githubusercontent.com/athakur3/mcp-context-cost/main/results/divergence.json";
|
|
10
11
|
/** Where the published `tool-shape/v1` baseline lives when `--suggest` doesn't override it. */
|
|
@@ -45,6 +46,12 @@ export interface AuditOptions {
|
|
|
45
46
|
* those files off the machine being audited (`discoverSettings`).
|
|
46
47
|
*/
|
|
47
48
|
settings?: ToolSearchSource[];
|
|
49
|
+
/**
|
|
50
|
+
* What each remote endpoint said to an unauthenticated `initialize`, keyed
|
|
51
|
+
* like `measureAll`'s map. `runAudit` probes them (`probeRemotes`); a test can
|
|
52
|
+
* state the answers instead of reaching a network.
|
|
53
|
+
*/
|
|
54
|
+
remotes?: Map<string, RemoteProbe>;
|
|
48
55
|
onProgress?: (name: string, done: number, total: number) => void;
|
|
49
56
|
}
|
|
50
57
|
/** Fetch and parse the published capture index. Never throws: a failure is a report problem, not a crash. */
|
|
@@ -72,6 +79,29 @@ export declare function discover(opts?: AuditOptions): LoadedConfig[];
|
|
|
72
79
|
* launched this audit answers for the wrong machine.
|
|
73
80
|
*/
|
|
74
81
|
export declare function discoverSettings(opts?: AuditOptions): ToolSearchSource[];
|
|
75
|
-
/**
|
|
82
|
+
/**
|
|
83
|
+
* Ask every distinct remote endpoint what it says to an unauthenticated
|
|
84
|
+
* `initialize`, once each. This comes before any launch — remote.ts says why an
|
|
85
|
+
* endpoint is asked before the bridge is pointed at it.
|
|
86
|
+
*/
|
|
87
|
+
export declare function probeRemotes(configs: LoadedConfig[], opts?: AuditOptions): Promise<Map<string, RemoteProbe>>;
|
|
88
|
+
/**
|
|
89
|
+
* The bridge launch for an endpoint that answered without a credential.
|
|
90
|
+
*
|
|
91
|
+
* `argv` is exact, for the host path; `command` is the same launch as one
|
|
92
|
+
* shell line, for the docker path, which runs it through `sh -lc` (docker.ts).
|
|
93
|
+
* Both carry the entry's header values, as a stdio launch carries env values.
|
|
94
|
+
* `display` carries the names only, and is the form a report may print.
|
|
95
|
+
*/
|
|
96
|
+
export declare function bridgeLaunch(s: ConfiguredServer): {
|
|
97
|
+
argv: string[];
|
|
98
|
+
command: string;
|
|
99
|
+
display: string;
|
|
100
|
+
};
|
|
101
|
+
/**
|
|
102
|
+
* Measure every distinct server across the given configs, once each: each
|
|
103
|
+
* stdio server by its launch, and each remote endpoint the probe found open
|
|
104
|
+
* through the bridge. A walled or unreachable endpoint is not launched at all.
|
|
105
|
+
*/
|
|
76
106
|
export declare function measureAll(configs: LoadedConfig[], opts?: AuditOptions): Promise<Map<string, Measurement>>;
|
|
77
107
|
export declare function runAudit(opts?: AuditOptions): Promise<AuditReport>;
|
package/dist/audit/run.js
CHANGED
|
@@ -11,6 +11,7 @@ import { parseToolShapeBaseline } from '../core/tool-shape.js';
|
|
|
11
11
|
import { parseCaptureIndex } from '../core/capture-index.js';
|
|
12
12
|
import { buildReport, serverKey } from './audit.js';
|
|
13
13
|
import { toolSearchEnv } from './deferral.js';
|
|
14
|
+
import { DEFAULT_PROBE_TIMEOUT_MS, probeRemote } from './remote.js';
|
|
14
15
|
import { configCandidates, loadConfigs, loadSettingsSources, settingsCandidates, } from './config.js';
|
|
15
16
|
/** Where the published `tools-delta/v1` run lives when `--claude` doesn't override it. */
|
|
16
17
|
export const DEFAULT_DIVERGENCE_URL = 'https://raw.githubusercontent.com/athakur3/mcp-context-cost/main/results/divergence.json';
|
|
@@ -78,12 +79,63 @@ export function discoverSettings(opts = {}) {
|
|
|
78
79
|
const home = opts.home ?? homedir();
|
|
79
80
|
return loadSettingsSources(settingsCandidates({ home, cwd, platform: process.platform, programData: process.env.ProgramData }));
|
|
80
81
|
}
|
|
81
|
-
/**
|
|
82
|
+
/**
|
|
83
|
+
* Ask every distinct remote endpoint what it says to an unauthenticated
|
|
84
|
+
* `initialize`, once each. This comes before any launch — remote.ts says why an
|
|
85
|
+
* endpoint is asked before the bridge is pointed at it.
|
|
86
|
+
*/
|
|
87
|
+
export async function probeRemotes(configs, opts = {}) {
|
|
88
|
+
const unique = new Map();
|
|
89
|
+
for (const cfg of configs) {
|
|
90
|
+
for (const s of cfg.servers) {
|
|
91
|
+
if (s.transport !== 'remote' || !s.url)
|
|
92
|
+
continue;
|
|
93
|
+
if (!unique.has(serverKey(s)))
|
|
94
|
+
unique.set(serverKey(s), s);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
const out = new Map();
|
|
98
|
+
// One HTTP exchange each; the launch timeout is the wrong bound for it.
|
|
99
|
+
const timeoutMs = Math.min(opts.timeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS, DEFAULT_PROBE_TIMEOUT_MS);
|
|
100
|
+
await Promise.all([...unique].map(async ([key, s]) => {
|
|
101
|
+
out.set(key, await probeRemote(s.url, { headers: s.headers, timeoutMs }));
|
|
102
|
+
}));
|
|
103
|
+
return out;
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* The bridge launch for an endpoint that answered without a credential.
|
|
107
|
+
*
|
|
108
|
+
* `argv` is exact, for the host path; `command` is the same launch as one
|
|
109
|
+
* shell line, for the docker path, which runs it through `sh -lc` (docker.ts).
|
|
110
|
+
* Both carry the entry's header values, as a stdio launch carries env values.
|
|
111
|
+
* `display` carries the names only, and is the form a report may print.
|
|
112
|
+
*/
|
|
113
|
+
export function bridgeLaunch(s) {
|
|
114
|
+
const url = s.url ?? '';
|
|
115
|
+
const argv = ['npx', '-y', 'mcp-remote', url];
|
|
116
|
+
const display = [...argv];
|
|
117
|
+
// mcp-remote refuses a plain-http endpoint unless told it is on purpose.
|
|
118
|
+
if (/^http:/i.test(url)) {
|
|
119
|
+
argv.push('--allow-http');
|
|
120
|
+
display.push('--allow-http');
|
|
121
|
+
}
|
|
122
|
+
for (const [k, v] of Object.entries(s.headers ?? {})) {
|
|
123
|
+
argv.push('--header', `${k}: ${v}`);
|
|
124
|
+
display.push('--header', k);
|
|
125
|
+
}
|
|
126
|
+
const shellQuote = (a) => (/^[A-Za-z0-9_@%+=:,./-]+$/.test(a) ? a : `'${a.replace(/'/g, `'\\''`)}'`);
|
|
127
|
+
return { argv, command: argv.map(shellQuote).join(' '), display: display.join(' ') };
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* Measure every distinct server across the given configs, once each: each
|
|
131
|
+
* stdio server by its launch, and each remote endpoint the probe found open
|
|
132
|
+
* through the bridge. A walled or unreachable endpoint is not launched at all.
|
|
133
|
+
*/
|
|
82
134
|
export async function measureAll(configs, opts = {}) {
|
|
83
135
|
const unique = new Map();
|
|
84
136
|
for (const cfg of configs) {
|
|
85
137
|
for (const s of cfg.servers) {
|
|
86
|
-
if (s.transport !== '
|
|
138
|
+
if (s.transport === 'remote' && opts.remotes?.get(serverKey(s))?.kind !== 'open')
|
|
87
139
|
continue;
|
|
88
140
|
// Two clients pointing at the same argv are one measurement, not two.
|
|
89
141
|
if (!unique.has(serverKey(s)))
|
|
@@ -97,8 +149,9 @@ export async function measureAll(configs, opts = {}) {
|
|
|
97
149
|
const worker = async () => {
|
|
98
150
|
for (let next = queue.shift(); next; next = queue.shift()) {
|
|
99
151
|
const [key, s] = next;
|
|
100
|
-
const
|
|
101
|
-
|
|
152
|
+
const launch = s.transport === 'remote' ? bridgeLaunch(s) : null;
|
|
153
|
+
const m = await measureServer(s.name, launch ? launch.command : (s.command ?? ''), {
|
|
154
|
+
argv: launch ? launch.argv : s.argv,
|
|
102
155
|
env: s.env,
|
|
103
156
|
timeoutMs: opts.timeoutMs ?? 60_000,
|
|
104
157
|
docker: opts.docker,
|
|
@@ -113,7 +166,8 @@ export async function measureAll(configs, opts = {}) {
|
|
|
113
166
|
}
|
|
114
167
|
export async function runAudit(opts = {}) {
|
|
115
168
|
const configs = discover(opts);
|
|
116
|
-
const
|
|
169
|
+
const remotes = opts.remotes ?? (await probeRemotes(configs, opts));
|
|
170
|
+
const measured = await measureAll(configs, { ...opts, remotes });
|
|
117
171
|
let divergence = null;
|
|
118
172
|
let divergenceProblem;
|
|
119
173
|
if (opts.claude) {
|
|
@@ -143,6 +197,7 @@ export async function runAudit(opts = {}) {
|
|
|
143
197
|
captureIndex,
|
|
144
198
|
env: opts.env ?? toolSearchEnv(process.env),
|
|
145
199
|
settings: opts.settings ?? discoverSettings(opts),
|
|
200
|
+
remotes,
|
|
146
201
|
});
|
|
147
202
|
if (divergenceProblem)
|
|
148
203
|
report.problems.push(divergenceProblem);
|
package/dist/cli.js
CHANGED
|
@@ -342,7 +342,8 @@ if (cmd === 'audit') {
|
|
|
342
342
|
console.error(`no MCP config found at the path(s) given: ${all('config').join(', ')}. ` +
|
|
343
343
|
`Nothing else was searched, because --config was set.`);
|
|
344
344
|
else
|
|
345
|
-
console.error(`no MCP config found. Looked in the standard Claude Desktop / Claude Code / Cursor / VS Code / Windsurf
|
|
345
|
+
console.error(`no MCP config found. Looked in the standard Claude Desktop / Claude Code / Cursor / VS Code / Windsurf / ` +
|
|
346
|
+
`Codex CLI / Gemini CLI / Zed / Kiro / Goose locations.${where}\n` +
|
|
346
347
|
`Point at one explicitly: mcp-context-cost audit --config <path/to/mcp.json>`);
|
|
347
348
|
process.exit(1);
|
|
348
349
|
}
|
|
@@ -28,7 +28,7 @@ import { homedir, tmpdir } from 'node:os';
|
|
|
28
28
|
import { join, resolve } from 'node:path';
|
|
29
29
|
import { fileURLToPath } from 'node:url';
|
|
30
30
|
import { parse } from 'yaml';
|
|
31
|
-
import { measureServer } from './run.js';
|
|
31
|
+
import { isSelfContainerised, measureServer } from './run.js';
|
|
32
32
|
import { DockerHarnessFault, defaultImageFor, dockerize } from './docker.js';
|
|
33
33
|
import { splitCommand } from './client.js';
|
|
34
34
|
import { selectShard, shardIndexForDate } from './shard.js';
|
|
@@ -305,7 +305,7 @@ if (isMain) {
|
|
|
305
305
|
console.log(` ${e.name}: our measurement ${m.status} — recorded, nothing to compare`);
|
|
306
306
|
continue;
|
|
307
307
|
}
|
|
308
|
-
const selfDocker = e.command
|
|
308
|
+
const selfDocker = isSelfContainerised(e.command);
|
|
309
309
|
const out = await runCli(docker && selfDocker ? await hostBin() : binPath, e, {
|
|
310
310
|
docker: docker && !selfDocker,
|
|
311
311
|
timeoutMs,
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { type SchemaProblem } from './servers-schema.js';
|
|
2
|
+
import type { ServerEntry } from './report.js';
|
|
3
|
+
import type { Measurement } from '../core/types.js';
|
|
4
|
+
/**
|
|
5
|
+
* The fields that change what a sweep launches — the option object
|
|
6
|
+
* sweep-all.ts hands to `measureServer`, plus the timeout it is given. A change
|
|
7
|
+
* to `metric`, `metricSource`, `category`, `repo`, `package` or `deprecated`
|
|
8
|
+
* changes a row's text, not its process, and is not re-measured here.
|
|
9
|
+
*/
|
|
10
|
+
export declare const LAUNCH_FIELDS: readonly ["command", "dockerImage", "aptPackages", "needsGit", "env", "envValues", "timeoutSeconds", "notApplicable"];
|
|
11
|
+
/**
|
|
12
|
+
* How many entries one pull request may launch, at the default budget. The
|
|
13
|
+
* worst case per entry is a timeout retried on `TIMEOUT_RETRY_FACTOR` times
|
|
14
|
+
* its budget (run.ts:351-352), so at the workflow's `--default-timeout 240`
|
|
15
|
+
* one entry can hold the runner for 240 × (1 + 2) = 720s, and this many of
|
|
16
|
+
* them measured one at a time is what pr-check.yml's `timeout-minutes` is
|
|
17
|
+
* sized to (`launchBudgetSeconds`).
|
|
18
|
+
*
|
|
19
|
+
* The count alone does not hold that bound. An entry's own `timeoutSeconds`
|
|
20
|
+
* replaces the default (the rotation's rule, kept here), and the schema bounds
|
|
21
|
+
* it only below — "a whole number of seconds greater than zero"
|
|
22
|
+
* (servers-schema.ts:166-169) — so one added entry with `timeoutSeconds: 4000`
|
|
23
|
+
* would cost 12000s under the same count and be killed by the job limit with
|
|
24
|
+
* no line printed for it. The bound is held because the script also refuses,
|
|
25
|
+
* before any launch, a selection whose `worstCaseSeconds` exceeds the budget
|
|
26
|
+
* the count was sized to. A PR that trips either is asked to split or shorten,
|
|
27
|
+
* rather than have the runner cut it off with the last entries unmeasured and
|
|
28
|
+
* no line saying so.
|
|
29
|
+
*/
|
|
30
|
+
export declare const DEFAULT_MAX_ENTRIES = 4;
|
|
31
|
+
/** The launched entries among a selection: remote endpoints and self-containerised commands are listed, not spawned. */
|
|
32
|
+
export declare function launchedEntries(entries: ServerEntry[]): ServerEntry[];
|
|
33
|
+
/**
|
|
34
|
+
* The most seconds a selection can hold the runner: each launched entry's own
|
|
35
|
+
* budget (`timeoutSeconds`, or the default) once, then `TIMEOUT_RETRY_FACTOR`
|
|
36
|
+
* times more on the retry. Listed entries cost nothing and are not counted.
|
|
37
|
+
*/
|
|
38
|
+
export declare function worstCaseSeconds(entries: ServerEntry[], defaultTimeout: number): number;
|
|
39
|
+
/** What the job's `timeout-minutes` was sized to: `maxEntries` launches, each at the default budget and its retry. */
|
|
40
|
+
export declare function launchBudgetSeconds(maxEntries: number, defaultTimeout: number): number;
|
|
41
|
+
export interface ServersDiff {
|
|
42
|
+
/** Head fails the shape check; nothing is diffed from a malformed document. */
|
|
43
|
+
problems: SchemaProblem[];
|
|
44
|
+
/** Names present in head and absent from base. */
|
|
45
|
+
added: ServerEntry[];
|
|
46
|
+
/** Names in both whose `LAUNCH_FIELDS` projection differs. */
|
|
47
|
+
relaunched: ServerEntry[];
|
|
48
|
+
}
|
|
49
|
+
/** The part of an entry that decides what gets spawned, as one comparable string. */
|
|
50
|
+
export declare function launchSignature(entry: ServerEntry): string;
|
|
51
|
+
/**
|
|
52
|
+
* What a pull request adds or relaunches, given the parsed base and head
|
|
53
|
+
* documents. Head is shape-checked first and returns only problems when it has
|
|
54
|
+
* any. Base is the committed branch, which `npm test` already holds valid; an
|
|
55
|
+
* unreadable base is treated as empty, which makes every head entry "added"
|
|
56
|
+
* and lets the entry cap refuse the run rather than measuring the whole file.
|
|
57
|
+
*/
|
|
58
|
+
export declare function entriesToMeasure(base: unknown, head: unknown): ServersDiff;
|
|
59
|
+
/** The line printed for a launched entry — sweep-all's summary form, with the evidence behind a failure. */
|
|
60
|
+
export declare function summarise(name: string, m: Measurement, secs: number): string;
|
|
61
|
+
/** Whether an outcome fails the check. */
|
|
62
|
+
export declare function failsCheck(status: Measurement['status']): boolean;
|