mcp-context-cost 0.16.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -1
- package/dist/core/canonical.d.ts +16 -1
- package/dist/core/canonical.js +26 -9
- package/dist/core/types.d.ts +14 -0
- package/dist/sweep/client.d.ts +54 -1
- package/dist/sweep/client.js +31 -3
- package/dist/sweep/published-stats.d.ts +29 -0
- package/dist/sweep/published-stats.js +47 -3
- package/dist/sweep/report.d.ts +36 -0
- package/dist/sweep/report.js +30 -0
- package/dist/sweep/run.d.ts +9 -0
- package/dist/sweep/run.js +1 -1
- package/dist/sweep/server-pages.js +39 -5
- package/dist/sweep/servers-schema.d.ts +1 -0
- package/dist/sweep/servers-schema.js +30 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -130,6 +130,40 @@ On a machine where none of them is set, the same stack reads:
|
|
|
130
130
|
a tool whose _meta carries "anthropic/alwaysLoad": true, which this audit does not read from a capture
|
|
131
131
|
```
|
|
132
132
|
|
|
133
|
+
**Do not take that table on trust — your own client will tell you.** Everything above is read
|
|
134
|
+
from Anthropic's documentation, and documentation about someone else's product is exactly the
|
|
135
|
+
kind of claim this project refuses to leave unchecked elsewhere. Claude Code writes its own
|
|
136
|
+
decision to a debug log, before it sends anything, so you can check your machine rather than
|
|
137
|
+
believe this page. In a directory with an `.mcp.json`:
|
|
138
|
+
|
|
139
|
+
```bash
|
|
140
|
+
claude --debug-file /tmp/cc.txt -p "ok"
|
|
141
|
+
grep -E 'ToolSearch|Dynamic tool loading|Auto tool search' /tmp/cc.txt
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
Three line shapes answer three different questions:
|
|
145
|
+
|
|
146
|
+
| the line | what it tells you |
|
|
147
|
+
|---|---|
|
|
148
|
+
| `[ToolSearch:optimistic] mode=…, ENABLE_TOOL_SEARCH=…, result=…` | which mode it picked at startup, and the value it read. **Optimistic is its own word for a guess** — it can be revised below |
|
|
149
|
+
| `Dynamic tool loading: 0/N deferred tools included` | the one that settles it: how many of the `N` deferrable tools went into the request. `0/N` is deferral actually happening |
|
|
150
|
+
| `[ToolSearch:optimistic] disabled: ANTHROPIC_BASE_URL=… is not a first-party Anthropic host` | the fallback in the table above, firing, in the client's own words |
|
|
151
|
+
|
|
152
|
+
Read the *later* requests, not the first. A stdio server can finish connecting after the first
|
|
153
|
+
request has already gone, so an early low count is a race rather than a finding.
|
|
154
|
+
|
|
155
|
+
**The mode most likely to surprise you is `auto`.** It reads as the careful setting and it is the
|
|
156
|
+
one that loads everything up front at ordinary sizes: below the threshold it does not defer, by
|
|
157
|
+
design, and a line reading `Auto tool search disabled: … (threshold: …)` is that decision being
|
|
158
|
+
made. The threshold is a *percentage of the context window*, so a model with a larger window has
|
|
159
|
+
a proportionally larger one — pass `--context` to `audit` to compare against the window you
|
|
160
|
+
actually run.
|
|
161
|
+
|
|
162
|
+
None of this is free, and it is one trivial request. These are Claude Code's own debug lines
|
|
163
|
+
rather than a documented interface, so they can change; the table above is what this project
|
|
164
|
+
holds to a dated re-read. No other client discovered by `audit` writes anything comparable,
|
|
165
|
+
which is why their rows say an absence of a record rather than a measurement.
|
|
166
|
+
|
|
133
167
|
Set `ENABLE_TOOL_SEARCH=false` in that shell and the same config reports the opposite —
|
|
134
168
|
`loads every tool definition up front here`, naming the variable and the place it was read
|
|
135
169
|
from. Deferring is also not free: what a deferring client *does* load at session start —
|
|
@@ -312,7 +346,7 @@ now measured against a pinned model and published beside the badge, and they do
|
|
|
312
346
|
|
|
313
347
|
| server | badge (o200k) | Claude (`claude-opus-5`) | |
|
|
314
348
|
|---|---:|---:|---|
|
|
315
|
-
| github | 54,622 | **18,728** |
|
|
349
|
+
| github | 54,622 | **18,728** | 78% of the capture is `icons` metadata Claude never sees |
|
|
316
350
|
| notion | 17,500 | **33,560** | almost no metadata to drop, so the tokenizer difference dominates |
|
|
317
351
|
|
|
318
352
|
So the heaviest server on the badge is not the heaviest server on Claude. Per-server
|
package/dist/core/canonical.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { Measurement, MeasurementStatus } from './types.js';
|
|
1
|
+
import type { Measurement, MeasurementStatus, ToolMeasurement } from './types.js';
|
|
2
2
|
export declare const METHODOLOGY_VERSION = "1.0";
|
|
3
3
|
/**
|
|
4
4
|
* Canonical form: JSON.stringify over the PARSED tools array (no whitespace,
|
|
@@ -10,6 +10,21 @@ export declare const METHODOLOGY_VERSION = "1.0";
|
|
|
10
10
|
export declare function canonicalString(tools: unknown[]): string;
|
|
11
11
|
export declare function sha256Hex(text: string): string;
|
|
12
12
|
export declare function countTokens(text: string): number;
|
|
13
|
+
/**
|
|
14
|
+
* One tool's diagnostic breakdown, from the tool object alone.
|
|
15
|
+
*
|
|
16
|
+
* Exported and used by `measureTools` rather than inlined there, because two
|
|
17
|
+
* other callers have to agree with it exactly: the backfill that fills these
|
|
18
|
+
* fields in on records written before they existed, and the test that re-derives
|
|
19
|
+
* every published record's breakdown from that record's own capture. A second
|
|
20
|
+
* implementation of this arithmetic would be a second answer to the same
|
|
21
|
+
* question.
|
|
22
|
+
*
|
|
23
|
+
* A tool that ships no `outputSchema` or `annotations` records `0`. An absent
|
|
24
|
+
* key means something else — a record written before this existed — which is
|
|
25
|
+
* why `ToolMeasurement` makes them optional.
|
|
26
|
+
*/
|
|
27
|
+
export declare function measureTool(t: unknown): ToolMeasurement;
|
|
13
28
|
/**
|
|
14
29
|
* totalTokens is authoritative (tokens of the whole canonical array — array
|
|
15
30
|
* punctuation included); per-tool numbers are diagnostic and won't sum to it.
|
package/dist/core/canonical.js
CHANGED
|
@@ -20,21 +20,38 @@ export function countTokens(text) {
|
|
|
20
20
|
// descriptions are counted as ordinary text instead of throwing.
|
|
21
21
|
return enc.encode(text, undefined, []).length;
|
|
22
22
|
}
|
|
23
|
+
/**
|
|
24
|
+
* One tool's diagnostic breakdown, from the tool object alone.
|
|
25
|
+
*
|
|
26
|
+
* Exported and used by `measureTools` rather than inlined there, because two
|
|
27
|
+
* other callers have to agree with it exactly: the backfill that fills these
|
|
28
|
+
* fields in on records written before they existed, and the test that re-derives
|
|
29
|
+
* every published record's breakdown from that record's own capture. A second
|
|
30
|
+
* implementation of this arithmetic would be a second answer to the same
|
|
31
|
+
* question.
|
|
32
|
+
*
|
|
33
|
+
* A tool that ships no `outputSchema` or `annotations` records `0`. An absent
|
|
34
|
+
* key means something else — a record written before this existed — which is
|
|
35
|
+
* why `ToolMeasurement` makes them optional.
|
|
36
|
+
*/
|
|
37
|
+
export function measureTool(t) {
|
|
38
|
+
const tool = t;
|
|
39
|
+
return {
|
|
40
|
+
name: tool.name ?? '(unnamed)',
|
|
41
|
+
tokens: countTokens(JSON.stringify(t)),
|
|
42
|
+
descriptionTokens: tool.description ? countTokens(tool.description) : 0,
|
|
43
|
+
inputSchemaTokens: tool.inputSchema ? countTokens(JSON.stringify(tool.inputSchema)) : 0,
|
|
44
|
+
outputSchemaTokens: tool.outputSchema ? countTokens(JSON.stringify(tool.outputSchema)) : 0,
|
|
45
|
+
annotationsTokens: tool.annotations ? countTokens(JSON.stringify(tool.annotations)) : 0,
|
|
46
|
+
};
|
|
47
|
+
}
|
|
23
48
|
/**
|
|
24
49
|
* totalTokens is authoritative (tokens of the whole canonical array — array
|
|
25
50
|
* punctuation included); per-tool numbers are diagnostic and won't sum to it.
|
|
26
51
|
*/
|
|
27
52
|
export function measureTools(tools, meta) {
|
|
28
53
|
const canonical = canonicalString(tools);
|
|
29
|
-
const perTool = tools.map(
|
|
30
|
-
const tool = t;
|
|
31
|
-
return {
|
|
32
|
-
name: tool.name ?? '(unnamed)',
|
|
33
|
-
tokens: countTokens(JSON.stringify(t)),
|
|
34
|
-
descriptionTokens: tool.description ? countTokens(tool.description) : 0,
|
|
35
|
-
inputSchemaTokens: tool.inputSchema ? countTokens(JSON.stringify(tool.inputSchema)) : 0,
|
|
36
|
-
};
|
|
37
|
-
});
|
|
54
|
+
const perTool = tools.map(measureTool);
|
|
38
55
|
return {
|
|
39
56
|
methodologyVersion: METHODOLOGY_VERSION,
|
|
40
57
|
provider: 'tiktoken',
|
package/dist/core/types.d.ts
CHANGED
|
@@ -16,6 +16,20 @@ export interface ToolMeasurement {
|
|
|
16
16
|
tokens: number;
|
|
17
17
|
descriptionTokens: number;
|
|
18
18
|
inputSchemaTokens: number;
|
|
19
|
+
/**
|
|
20
|
+
* The two fields that used to sit inside `tokens` with nothing naming them.
|
|
21
|
+
* Across the measured set output schemas are about a sixth of every published
|
|
22
|
+
* token and annotations another thirtieth, so a reader could see that a tool
|
|
23
|
+
* was expensive without being able to see that its output schema was why.
|
|
24
|
+
*
|
|
25
|
+
* Optional because absent and zero are different claims, the same distinction
|
|
26
|
+
* `serverInstructions` draws below: `0` means the tool ships no such field,
|
|
27
|
+
* absent means the record predates the attribution. Every record written
|
|
28
|
+
* since carries both, and `tools/backfill-tool-attribution.ts` re-derives
|
|
29
|
+
* them for older ones out of the capture stored in the same file.
|
|
30
|
+
*/
|
|
31
|
+
outputSchemaTokens?: number;
|
|
32
|
+
annotationsTokens?: number;
|
|
19
33
|
}
|
|
20
34
|
/**
|
|
21
35
|
* The full reproducibility record published next to every badge.
|
package/dist/sweep/client.d.ts
CHANGED
|
@@ -58,9 +58,51 @@ export declare function evidenceTail(stderr: string, limit?: number, required?:
|
|
|
58
58
|
* one character of growth in somebody else's error message.
|
|
59
59
|
*/
|
|
60
60
|
export declare function clampNotes(text: string, limit: number, required?: string): string;
|
|
61
|
+
/**
|
|
62
|
+
* A client posture: what it declares at `initialize`, and how it answers the
|
|
63
|
+
* requests that declaration invites. The two live in one object because they
|
|
64
|
+
* are one decision — a client that declares `roots` and then returns
|
|
65
|
+
* "method not found" to `roots/list` has told the server something untrue, and
|
|
66
|
+
* a server is entitled to shape its tool list around the answer.
|
|
67
|
+
*
|
|
68
|
+
* This exists because the default posture, `{}`, is not neutral. A server may
|
|
69
|
+
* gate tools on what the client can do: measured 2026-09-06, the reference
|
|
70
|
+
* `everything` server exposes 13 tools to a client declaring nothing and 15 to
|
|
71
|
+
* one declaring roots and elicitation. So the published number for such a
|
|
72
|
+
* server is a floor, and which posture the sweep runs with is a measurement
|
|
73
|
+
* decision rather than a detail.
|
|
74
|
+
*
|
|
75
|
+
* `sampling` is deliberately absent. Declaring it says this client can ask a
|
|
76
|
+
* model for a completion, and it cannot; there is no honest minimal answer to
|
|
77
|
+
* `sampling/createMessage`, unlike an empty root list or a declined
|
|
78
|
+
* elicitation, both of which are ordinary states a real client can be in.
|
|
79
|
+
*/
|
|
80
|
+
export interface ClientPosture {
|
|
81
|
+
/** Sent verbatim as `capabilities` in `initialize`. */
|
|
82
|
+
capabilities: Record<string, unknown>;
|
|
83
|
+
/** Answers to server-initiated requests, by method. Every declared capability needs one. */
|
|
84
|
+
answers: Record<string, unknown>;
|
|
85
|
+
}
|
|
86
|
+
/** What the sweep has always declared: nothing, and so nothing to answer. */
|
|
87
|
+
export declare const MINIMAL_POSTURE: ClientPosture;
|
|
88
|
+
/**
|
|
89
|
+
* The two capabilities this harness can answer truthfully.
|
|
90
|
+
*
|
|
91
|
+
* `roots/list` returns an empty list: this client exposes no filesystem roots,
|
|
92
|
+
* which is a true statement about it rather than a refusal. `elicitation/create`
|
|
93
|
+
* declines: the protocol provides for a user declining to answer, and an
|
|
94
|
+
* unattended sweep has no user, so declining is the honest reply and not a
|
|
95
|
+
* failure to implement one.
|
|
96
|
+
*/
|
|
97
|
+
export declare const DECLARING_POSTURE: ClientPosture;
|
|
61
98
|
export declare class McpStdioClient {
|
|
62
99
|
/** Phrase this entry's declared status depends on — see `evidenceTail`. */
|
|
63
100
|
private keepEvidence?;
|
|
101
|
+
/**
|
|
102
|
+
* Answers to server-initiated requests, one per declared capability. Empty
|
|
103
|
+
* by default, which is correct only while `initialize` declares nothing.
|
|
104
|
+
*/
|
|
105
|
+
private answers;
|
|
64
106
|
private child;
|
|
65
107
|
private buffer;
|
|
66
108
|
private nextId;
|
|
@@ -69,7 +111,12 @@ export declare class McpStdioClient {
|
|
|
69
111
|
private exited;
|
|
70
112
|
constructor(command: string, args: string[], env: Record<string, string | undefined>,
|
|
71
113
|
/** Phrase this entry's declared status depends on — see `evidenceTail`. */
|
|
72
|
-
keepEvidence?: string | undefined
|
|
114
|
+
keepEvidence?: string | undefined,
|
|
115
|
+
/**
|
|
116
|
+
* Answers to server-initiated requests, one per declared capability. Empty
|
|
117
|
+
* by default, which is correct only while `initialize` declares nothing.
|
|
118
|
+
*/
|
|
119
|
+
answers?: Record<string, unknown>);
|
|
73
120
|
private onData;
|
|
74
121
|
private send;
|
|
75
122
|
private deadReason;
|
|
@@ -89,6 +136,12 @@ export declare function captureTools(spec: string | {
|
|
|
89
136
|
timeoutMs?: number;
|
|
90
137
|
env?: Record<string, string>;
|
|
91
138
|
keepEvidence?: string;
|
|
139
|
+
/**
|
|
140
|
+
* What to declare at `initialize`, and how to answer what that invites.
|
|
141
|
+
* Defaults to declaring nothing, which is what every published measurement
|
|
142
|
+
* was taken with.
|
|
143
|
+
*/
|
|
144
|
+
posture?: ClientPosture;
|
|
92
145
|
}): Promise<WireCapture>;
|
|
93
146
|
/** Shell-free command splitting: honors single/double quotes, no expansion. */
|
|
94
147
|
export declare function splitCommand(line: string): string[];
|
package/dist/sweep/client.js
CHANGED
|
@@ -212,8 +212,27 @@ function drop(text, isNoise, isProtected = () => false) {
|
|
|
212
212
|
.trim();
|
|
213
213
|
return kept || text.trim();
|
|
214
214
|
}
|
|
215
|
+
/** What the sweep has always declared: nothing, and so nothing to answer. */
|
|
216
|
+
export const MINIMAL_POSTURE = { capabilities: {}, answers: {} };
|
|
217
|
+
/**
|
|
218
|
+
* The two capabilities this harness can answer truthfully.
|
|
219
|
+
*
|
|
220
|
+
* `roots/list` returns an empty list: this client exposes no filesystem roots,
|
|
221
|
+
* which is a true statement about it rather than a refusal. `elicitation/create`
|
|
222
|
+
* declines: the protocol provides for a user declining to answer, and an
|
|
223
|
+
* unattended sweep has no user, so declining is the honest reply and not a
|
|
224
|
+
* failure to implement one.
|
|
225
|
+
*/
|
|
226
|
+
export const DECLARING_POSTURE = {
|
|
227
|
+
capabilities: { roots: { listChanged: false }, elicitation: {} },
|
|
228
|
+
answers: {
|
|
229
|
+
'roots/list': { roots: [] },
|
|
230
|
+
'elicitation/create': { action: 'decline' },
|
|
231
|
+
},
|
|
232
|
+
};
|
|
215
233
|
export class McpStdioClient {
|
|
216
234
|
keepEvidence;
|
|
235
|
+
answers;
|
|
217
236
|
child;
|
|
218
237
|
buffer = '';
|
|
219
238
|
nextId = 1;
|
|
@@ -222,8 +241,14 @@ export class McpStdioClient {
|
|
|
222
241
|
exited;
|
|
223
242
|
constructor(command, args, env,
|
|
224
243
|
/** Phrase this entry's declared status depends on — see `evidenceTail`. */
|
|
225
|
-
keepEvidence
|
|
244
|
+
keepEvidence,
|
|
245
|
+
/**
|
|
246
|
+
* Answers to server-initiated requests, one per declared capability. Empty
|
|
247
|
+
* by default, which is correct only while `initialize` declares nothing.
|
|
248
|
+
*/
|
|
249
|
+
answers = {}) {
|
|
226
250
|
this.keepEvidence = keepEvidence;
|
|
251
|
+
this.answers = answers;
|
|
227
252
|
this.child = spawn(command, args, {
|
|
228
253
|
env: { ...env },
|
|
229
254
|
stdio: ['pipe', 'pipe', 'pipe'],
|
|
@@ -279,6 +304,8 @@ export class McpStdioClient {
|
|
|
279
304
|
if (msg.id !== undefined && msg.id !== null) {
|
|
280
305
|
if (msg.method === 'ping')
|
|
281
306
|
this.send({ jsonrpc: '2.0', id: msg.id, result: {} });
|
|
307
|
+
else if (Object.hasOwn(this.answers, msg.method))
|
|
308
|
+
this.send({ jsonrpc: '2.0', id: msg.id, result: this.answers[msg.method] });
|
|
282
309
|
else
|
|
283
310
|
this.send({ jsonrpc: '2.0', id: msg.id, error: { code: -32601, message: 'method not found' } });
|
|
284
311
|
}
|
|
@@ -345,12 +372,13 @@ export class McpStdioClient {
|
|
|
345
372
|
*/
|
|
346
373
|
export async function captureTools(spec, opts = {}) {
|
|
347
374
|
const timeoutMs = opts.timeoutMs ?? 60_000;
|
|
375
|
+
const posture = opts.posture ?? MINIMAL_POSTURE;
|
|
348
376
|
const [cmd, ...args] = typeof spec === 'string' ? splitCommand(spec) : [spec.command, ...spec.argv];
|
|
349
|
-
const client = new McpStdioClient(cmd, args, { PATH: process.env.PATH, HOME: process.env.HOME, ...opts.env }, opts.keepEvidence);
|
|
377
|
+
const client = new McpStdioClient(cmd, args, { PATH: process.env.PATH, HOME: process.env.HOME, ...opts.env }, opts.keepEvidence, posture.answers);
|
|
350
378
|
try {
|
|
351
379
|
const init = await client.request('initialize', {
|
|
352
380
|
protocolVersion: PROTOCOL_VERSION,
|
|
353
|
-
capabilities:
|
|
381
|
+
capabilities: posture.capabilities,
|
|
354
382
|
clientInfo: { name: 'mcp-context-cost', version: '0.1.0' },
|
|
355
383
|
}, timeoutMs);
|
|
356
384
|
client.notify('notifications/initialized');
|
|
@@ -32,6 +32,17 @@ export interface PublishedStats {
|
|
|
32
32
|
github: {
|
|
33
33
|
badgeTokens: number;
|
|
34
34
|
claudeTokens: number | null;
|
|
35
|
+
/**
|
|
36
|
+
* The single heaviest field in github's capture that an Anthropic tools
|
|
37
|
+
* array has nowhere to put, and its share of the capture. Derived rather
|
|
38
|
+
* than described, because the sentence it feeds was wrong for as long as
|
|
39
|
+
* it was hand-written: it named `annotations`/`outputSchema` as "most of
|
|
40
|
+
* the capture" while github ships no `outputSchema` at all and 1.7% of
|
|
41
|
+
* annotations. The dropped weight was `icons`, at 78%. A claim about
|
|
42
|
+
* which bytes are dropped has to come from the bytes.
|
|
43
|
+
*/
|
|
44
|
+
dropField: string;
|
|
45
|
+
dropSharePct: number;
|
|
35
46
|
};
|
|
36
47
|
notion: {
|
|
37
48
|
badgeTokens: number;
|
|
@@ -77,6 +88,24 @@ export interface PublishedStats {
|
|
|
77
88
|
}
|
|
78
89
|
/** Named in README's sample table — the choice is editorial, the numbers are not. */
|
|
79
90
|
export declare const SAMPLE_SERVERS: readonly ['github', 'xcodebuildmcp', 'brave-search', 'notion', 'playwright', 'filesystem', 'markitdown'];
|
|
91
|
+
/**
|
|
92
|
+
* The heaviest field in a capture that an Anthropic request has nowhere to put,
|
|
93
|
+
* and what share of the capture it is.
|
|
94
|
+
*
|
|
95
|
+
* This exists because the README sentence it feeds was hand-written and wrong.
|
|
96
|
+
* It said most of github's capture was `annotations`/`outputSchema` metadata;
|
|
97
|
+
* github ships no `outputSchema` at all and 1.7% of annotations, and the weight
|
|
98
|
+
* being dropped was `icons` at 78%. The claim was plausible, unmaintained, and
|
|
99
|
+
* describing a different server's shape — so it is derived now, and it moves
|
|
100
|
+
* when the capture does.
|
|
101
|
+
*
|
|
102
|
+
* Ties break on the field name so the sentence does not flip between two
|
|
103
|
+
* equal-weight fields from one regeneration to the next.
|
|
104
|
+
*/
|
|
105
|
+
export declare function heaviestDroppedField(capture: unknown[] | null, totalTokens: number): {
|
|
106
|
+
dropField: string;
|
|
107
|
+
dropSharePct: number;
|
|
108
|
+
};
|
|
80
109
|
export declare function floorToTwoSignificant(n: number): number;
|
|
81
110
|
export declare function computePublishedStats(entries: ServerEntry[], root?: string): PublishedStats;
|
|
82
111
|
export type PageFile = 'README.md' | 'docs/index.md' | 'docs/METHODOLOGY.md';
|
|
@@ -29,6 +29,7 @@ import { existsSync, readFileSync, writeFileSync } from 'node:fs';
|
|
|
29
29
|
import { join } from 'node:path';
|
|
30
30
|
import { DEFAULT_CONTEXT_WINDOW } from '../audit/audit.js';
|
|
31
31
|
import { BAND_PRECISION, wireToClientRatio } from '../audit/deferral.js';
|
|
32
|
+
import { countTokens } from '../core/canonical.js';
|
|
32
33
|
import { fieldSelectionShare, isCurrent } from '../core/divergence.js';
|
|
33
34
|
import { sessionStartLoad } from '../core/session-start.js';
|
|
34
35
|
import { isGood } from './harness-guard.js';
|
|
@@ -44,6 +45,40 @@ export const SAMPLE_SERVERS = [
|
|
|
44
45
|
'filesystem',
|
|
45
46
|
'markitdown',
|
|
46
47
|
];
|
|
48
|
+
/**
|
|
49
|
+
* The three fields an Anthropic tool definition carries. Everything else a
|
|
50
|
+
* server ships in `tools/list` is dropped before the request — see
|
|
51
|
+
* docs/METHODOLOGY.md#claude-divergence.
|
|
52
|
+
*/
|
|
53
|
+
const ANTHROPIC_TOOL_FIELDS = new Set(['name', 'description', 'inputSchema']);
|
|
54
|
+
/**
|
|
55
|
+
* The heaviest field in a capture that an Anthropic request has nowhere to put,
|
|
56
|
+
* and what share of the capture it is.
|
|
57
|
+
*
|
|
58
|
+
* This exists because the README sentence it feeds was hand-written and wrong.
|
|
59
|
+
* It said most of github's capture was `annotations`/`outputSchema` metadata;
|
|
60
|
+
* github ships no `outputSchema` at all and 1.7% of annotations, and the weight
|
|
61
|
+
* being dropped was `icons` at 78%. The claim was plausible, unmaintained, and
|
|
62
|
+
* describing a different server's shape — so it is derived now, and it moves
|
|
63
|
+
* when the capture does.
|
|
64
|
+
*
|
|
65
|
+
* Ties break on the field name so the sentence does not flip between two
|
|
66
|
+
* equal-weight fields from one regeneration to the next.
|
|
67
|
+
*/
|
|
68
|
+
export function heaviestDroppedField(capture, totalTokens) {
|
|
69
|
+
const tally = new Map();
|
|
70
|
+
for (const tool of capture ?? []) {
|
|
71
|
+
for (const [k, v] of Object.entries((tool ?? {}))) {
|
|
72
|
+
if (ANTHROPIC_TOOL_FIELDS.has(k))
|
|
73
|
+
continue;
|
|
74
|
+
tally.set(k, (tally.get(k) ?? 0) + countTokens(JSON.stringify(v)));
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
const top = [...tally].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))[0];
|
|
78
|
+
if (!top || totalTokens <= 0)
|
|
79
|
+
return { dropField: 'none', dropSharePct: 0 };
|
|
80
|
+
return { dropField: top[0], dropSharePct: Math.round((100 * top[1]) / totalTokens) };
|
|
81
|
+
}
|
|
47
82
|
export function floorToTwoSignificant(n) {
|
|
48
83
|
const whole = Math.floor(n);
|
|
49
84
|
if (whole < 100)
|
|
@@ -153,7 +188,11 @@ export function computePublishedStats(entries, root = process.cwd()) {
|
|
|
153
188
|
runSize: Object.keys(div.servers).length,
|
|
154
189
|
currentCount: withClaude.length,
|
|
155
190
|
heaviestClaudeName: heaviest?.entry.name ?? null,
|
|
156
|
-
github: {
|
|
191
|
+
github: {
|
|
192
|
+
badgeTokens: badgeTokensOf('github'),
|
|
193
|
+
claudeTokens: currentDivRow('github')?.claudeDelta ?? null,
|
|
194
|
+
...heaviestDroppedField(githubRow.m.rawToolsCapture, githubRow.m.totalTokens),
|
|
195
|
+
},
|
|
157
196
|
notion: { badgeTokens: badgeTokensOf('notion'), claudeTokens: currentDivRow('notion')?.claudeDelta ?? null },
|
|
158
197
|
widest: { server: widest[0], full: widest[1].o200kFull, mapped: widest[1].o200kMapped },
|
|
159
198
|
shareMin: Math.min(...shares),
|
|
@@ -289,8 +328,13 @@ export const PAGE_CLAIMS = [
|
|
|
289
328
|
{
|
|
290
329
|
file: 'README.md',
|
|
291
330
|
id: 'claude-table:github',
|
|
292
|
-
template: '| github | {n} | **{q}** |
|
|
293
|
-
values: (s) => [
|
|
331
|
+
template: '| github | {n} | **{q}** | {d}% of the capture is `{w}` metadata Claude never sees |',
|
|
332
|
+
values: (s) => [
|
|
333
|
+
fmt(s.claude.github.badgeTokens),
|
|
334
|
+
q(s.claude.github.claudeTokens),
|
|
335
|
+
String(s.claude.github.dropSharePct),
|
|
336
|
+
s.claude.github.dropField,
|
|
337
|
+
],
|
|
294
338
|
},
|
|
295
339
|
{
|
|
296
340
|
file: 'README.md',
|
package/dist/sweep/report.d.ts
CHANGED
|
@@ -60,6 +60,40 @@ export interface ServerEntry {
|
|
|
60
60
|
* context cost nobody is watching.
|
|
61
61
|
*/
|
|
62
62
|
deprecated?: Deprecation;
|
|
63
|
+
/**
|
|
64
|
+
* A different project of the same name that this row does **not** measure.
|
|
65
|
+
*
|
|
66
|
+
* `name` is the only thing the leaderboard shows about an entry — the package
|
|
67
|
+
* id and the repo are on the server page, one click away — so a row whose
|
|
68
|
+
* name is shared with a better-known project reads as that project to anyone
|
|
69
|
+
* who does not click. That is not hypothetical: `octocode` here is the npm
|
|
70
|
+
* package `octocode-mcp` from `bgauryy/octocode`, and someone who works on
|
|
71
|
+
* `Muvon/octocode`, an unrelated Rust project with the same name, read the
|
|
72
|
+
* row as theirs and posted a public correction about how their server had
|
|
73
|
+
* been filed.
|
|
74
|
+
*
|
|
75
|
+
* Declared per entry rather than detected, because there is no way to detect
|
|
76
|
+
* it: two unrelated projects picking one word is a fact about the world, and
|
|
77
|
+
* the only honest source is someone noticing. Renaming the entry is the
|
|
78
|
+
* alternative and a much larger one — `name` keys `results/<name>/`,
|
|
79
|
+
* `badges/<name>.json`, the capture index and every history row, so it is a
|
|
80
|
+
* change to a published identifier rather than to a label.
|
|
81
|
+
*/
|
|
82
|
+
nameCollision?: NameCollision;
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* The other project, as a dated reading — the same shape a deprecation takes,
|
|
86
|
+
* for the same reason: it is a claim about something outside this repository,
|
|
87
|
+
* so it carries where it was read and when. A project can be renamed, archived
|
|
88
|
+
* or absorbed without anything here moving.
|
|
89
|
+
*/
|
|
90
|
+
export interface NameCollision {
|
|
91
|
+
/** The other project, as its own owner writes it — e.g. `Muvon/octocode`. */
|
|
92
|
+
project: string;
|
|
93
|
+
/** Where it was read. */
|
|
94
|
+
source: string;
|
|
95
|
+
/** The day it was read. */
|
|
96
|
+
readOn: string;
|
|
63
97
|
}
|
|
64
98
|
/**
|
|
65
99
|
* A deprecation as a dated reading, like every other reading here. `version`
|
|
@@ -89,6 +123,8 @@ export declare function mdCell(s: unknown): string;
|
|
|
89
123
|
* Returns '' for an entry with no deprecation, so a caller can splice it in
|
|
90
124
|
* without branching.
|
|
91
125
|
*/
|
|
126
|
+
/** `https://github.com/owner/repo` → `owner/repo`; anything else is left alone. */
|
|
127
|
+
export declare function shortRepo(url: string): string;
|
|
92
128
|
export declare function deprecationText(entry: ServerEntry): string;
|
|
93
129
|
export declare function loadRows(entries: ServerEntry[], root?: string): Row[];
|
|
94
130
|
/** results/divergence.json if a divergence run has been recorded, else null. */
|
package/dist/sweep/report.js
CHANGED
|
@@ -22,6 +22,11 @@ const mdLink = (url) => encodeURI(String(url ?? '')).replace(/\)/g, '%29');
|
|
|
22
22
|
* Returns '' for an entry with no deprecation, so a caller can splice it in
|
|
23
23
|
* without branching.
|
|
24
24
|
*/
|
|
25
|
+
/** `https://github.com/owner/repo` → `owner/repo`; anything else is left alone. */
|
|
26
|
+
export function shortRepo(url) {
|
|
27
|
+
const m = /^https?:\/\/(?:www\.)?github\.com\/([^/]+\/[^/#?]+)/.exec(url);
|
|
28
|
+
return m ? m[1].replace(/\.git$/, '') : url;
|
|
29
|
+
}
|
|
25
30
|
export function deprecationText(entry) {
|
|
26
31
|
const d = entry.deprecated;
|
|
27
32
|
if (!d)
|
|
@@ -240,6 +245,31 @@ export function writeLeaderboard(entries, root = process.cwd(), regressions) {
|
|
|
240
245
|
}
|
|
241
246
|
md.push('');
|
|
242
247
|
}
|
|
248
|
+
// Same rule as the deprecation section above: derived, and gone when no entry
|
|
249
|
+
// declares one. It is a section rather than a column because the fact belongs
|
|
250
|
+
// to a handful of rows and a column would put an empty cell on the other
|
|
251
|
+
// hundred — and because what a reader needs here is a sentence, not a cell.
|
|
252
|
+
const collided = rows.filter((r) => r.entry.nameCollision);
|
|
253
|
+
if (collided.length > 0) {
|
|
254
|
+
md.push('## Same name, different project');
|
|
255
|
+
md.push('');
|
|
256
|
+
md.push(`A row here is named for the package it launches, and ${collided.length === 1 ? 'one name is' : 'these names are'} ` +
|
|
257
|
+
`shared with an unrelated project. The table above shows only the name, so ` +
|
|
258
|
+
`${collided.length === 1 ? 'that row' : 'those rows'} can be read as the wrong software by anyone who does not ` +
|
|
259
|
+
`open the page — which has already happened once, publicly. What each row actually measures is its launch ` +
|
|
260
|
+
`command and its source repository, both on its detail page.`);
|
|
261
|
+
md.push('');
|
|
262
|
+
md.push('| server | measures | not to be confused with |');
|
|
263
|
+
md.push('|---|---|---|');
|
|
264
|
+
for (const r of collided) {
|
|
265
|
+
const c = r.entry.nameCollision;
|
|
266
|
+
const measures = r.entry.repo
|
|
267
|
+
? `\`${mdCell(r.entry.package ?? r.entry.command)}\` — [${mdCell(shortRepo(r.entry.repo))}](${mdLink(r.entry.repo)})`
|
|
268
|
+
: `\`${mdCell(r.entry.package ?? r.entry.command)}\``;
|
|
269
|
+
md.push(`| ${mdCell(r.entry.name)} | ${measures} | [${mdCell(c.project)}](${mdLink(c.source)}), read ${mdCell(c.readOn)} |`);
|
|
270
|
+
}
|
|
271
|
+
md.push('');
|
|
272
|
+
}
|
|
243
273
|
if (unmeasured.length > 0) {
|
|
244
274
|
md.push('## Not measured (and why)');
|
|
245
275
|
md.push('');
|
package/dist/sweep/run.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type ClientPosture } from './client.js';
|
|
1
2
|
import type { Measurement } from '../core/types.js';
|
|
2
3
|
/**
|
|
3
4
|
* Which kind of failure a dead server's own words describe.
|
|
@@ -105,6 +106,14 @@ export interface MeasureOptions {
|
|
|
105
106
|
* `audit` runs in the user's own directory and must not litter it.
|
|
106
107
|
*/
|
|
107
108
|
persist?: boolean;
|
|
109
|
+
/**
|
|
110
|
+
* What the client declares at `initialize`, and how it answers what that
|
|
111
|
+
* invites. Every published measurement was taken declaring nothing, which is
|
|
112
|
+
* the default; a probe passes a different posture to find out whether a
|
|
113
|
+
* server gates tools on it. Changing the default would change published
|
|
114
|
+
* numbers, so it is a measurement decision and not a flag to flip lightly.
|
|
115
|
+
*/
|
|
116
|
+
posture?: ClientPosture;
|
|
108
117
|
}
|
|
109
118
|
/**
|
|
110
119
|
* Whether a launch command is already its own `docker run`.
|
package/dist/sweep/run.js
CHANGED
|
@@ -231,7 +231,7 @@ export async function measureServer(name, command, opts = {}) {
|
|
|
231
231
|
// The declared evidence travels with the launch, because the truncation
|
|
232
232
|
// that could lose it happens inside the client, before anything here sees
|
|
233
233
|
// the message it will be classified from.
|
|
234
|
-
const captureOpts = { ...attemptOpts, keepEvidence: opts.notApplicable?.evidence };
|
|
234
|
+
const captureOpts = { ...attemptOpts, keepEvidence: opts.notApplicable?.evidence, posture: opts.posture };
|
|
235
235
|
const first = await captureTools(buildSpec(noSharedCache), captureOpts);
|
|
236
236
|
const second = await captureTools(buildSpec(noSharedCache), captureOpts);
|
|
237
237
|
r = measureTools(first.tools, {
|
|
@@ -27,13 +27,28 @@ const fmt = (n) => n.toLocaleString('en-US');
|
|
|
27
27
|
function isMeasured(m) {
|
|
28
28
|
return !!m && (m.status === 'measured' || m.status === 'dynamic') && typeof m.totalTokens === 'number';
|
|
29
29
|
}
|
|
30
|
+
/**
|
|
31
|
+
* The conditions a number was made under, including the machine.
|
|
32
|
+
*
|
|
33
|
+
* `isolation.arch` has been recorded since 0.12.0 and was rendered nowhere, so
|
|
34
|
+
* a reader could see the image and the network a measurement ran under but not
|
|
35
|
+
* the architecture — the one condition that decides whether a package could run
|
|
36
|
+
* at all. `local-mcp` is why the field exists: it was published as a broken
|
|
37
|
+
* server on the strength of a run whose real finding was the machine.
|
|
38
|
+
*
|
|
39
|
+
* Absent is printed as "architecture not on record" rather than omitted,
|
|
40
|
+
* because the record's own rule is that absent means unknown and never "the
|
|
41
|
+
* same as yours". Thirty-two of the published records predate the field and
|
|
42
|
+
* will say so until the rotation re-measures them.
|
|
43
|
+
*/
|
|
30
44
|
function isolationText(m) {
|
|
31
45
|
const iso = m.isolation;
|
|
32
46
|
if (!iso)
|
|
33
47
|
return 'not recorded';
|
|
48
|
+
const arch = iso.arch ?? 'architecture not on record';
|
|
34
49
|
if (!iso.docker)
|
|
35
|
-
return
|
|
36
|
-
return ['docker', iso.image, iso.network ? `network ${iso.network}` : '', iso.note]
|
|
50
|
+
return `host process (no container) · ${arch}`;
|
|
51
|
+
return ['docker', iso.image, iso.network ? `network ${iso.network}` : '', arch, iso.note]
|
|
37
52
|
.filter(Boolean)
|
|
38
53
|
.join(' · ');
|
|
39
54
|
}
|
|
@@ -99,6 +114,15 @@ export function renderServerPage(entry, m, history = [], divergence = null) {
|
|
|
99
114
|
md.push(`| category | ${mdCell(entry.category)} |`);
|
|
100
115
|
if (entry.repo)
|
|
101
116
|
md.push(`| source | ${mdCell(entry.repo)} |`);
|
|
117
|
+
// The page already names the package, the launch command and the repo, so a
|
|
118
|
+
// reader who reads it can tell which project this is. This row is for the
|
|
119
|
+
// reader who arrives from a badge, sees a familiar name in the heading, and
|
|
120
|
+
// has no reason to suspect there are two — which is the person the collision
|
|
121
|
+
// actually caught.
|
|
122
|
+
if (entry.nameCollision) {
|
|
123
|
+
md.push(`| not to be confused with | ${mdCell(entry.nameCollision.project)} ` +
|
|
124
|
+
`(${mdCell(entry.nameCollision.source)}, read ${mdCell(entry.nameCollision.readOn)}) — an unrelated project of the same name, not measured here |`);
|
|
125
|
+
}
|
|
102
126
|
md.push('');
|
|
103
127
|
if (m.status === 'dynamic') {
|
|
104
128
|
md.push('> This server\'s `tools/list` differed between two consecutive captures, so the number ' +
|
|
@@ -107,10 +131,20 @@ export function renderServerPage(entry, m, history = [], divergence = null) {
|
|
|
107
131
|
}
|
|
108
132
|
md.push('## Where the tokens are');
|
|
109
133
|
md.push('');
|
|
110
|
-
|
|
111
|
-
|
|
134
|
+
// The output-schema column appears only on pages that have one. It is the
|
|
135
|
+
// field most likely to explain an expensive tool — about a sixth of every
|
|
136
|
+
// published token across the set, and the largest thing the breakdown used to
|
|
137
|
+
// leave unnamed — but only a third of measured servers ship one, and a column
|
|
138
|
+
// of zeroes on the rest would cost every reader something to tell them
|
|
139
|
+
// nothing. `annotations` is recorded per tool for the same reason and
|
|
140
|
+
// deliberately gets no column: at roughly 3% of the set it almost never
|
|
141
|
+
// explains a row, and it is in the measurement file for anyone who looks.
|
|
142
|
+
const hasOutput = shown.some((t) => (t.outputSchemaTokens ?? 0) > 0);
|
|
143
|
+
md.push(`| tool | tokens | share | description | input schema |${hasOutput ? ' output schema |' : ''}`);
|
|
144
|
+
md.push(`|---|---:|---:|---:|---:|${hasOutput ? '---:|' : ''}`);
|
|
112
145
|
for (const t of shown) {
|
|
113
|
-
md.push(`| ${mdCell(t.name)} | ${fmt(t.tokens)} | ${pct(t.tokens)} | ${fmt(t.descriptionTokens)} | ${fmt(t.inputSchemaTokens)} |`
|
|
146
|
+
md.push(`| ${mdCell(t.name)} | ${fmt(t.tokens)} | ${pct(t.tokens)} | ${fmt(t.descriptionTokens)} | ${fmt(t.inputSchemaTokens)} |` +
|
|
147
|
+
(hasOutput ? ` ${fmt(t.outputSchemaTokens ?? 0)} |` : ''));
|
|
114
148
|
}
|
|
115
149
|
md.push('');
|
|
116
150
|
if (tools.length > shown.length) {
|
|
@@ -51,6 +51,7 @@ const FIELDS = {
|
|
|
51
51
|
envValues: 'optional',
|
|
52
52
|
notApplicable: 'optional',
|
|
53
53
|
deprecated: 'optional',
|
|
54
|
+
nameCollision: 'optional',
|
|
54
55
|
};
|
|
55
56
|
export const knownFields = Object.keys(FIELDS);
|
|
56
57
|
/**
|
|
@@ -218,6 +219,35 @@ export function validateEntry(raw, index) {
|
|
|
218
219
|
}
|
|
219
220
|
}
|
|
220
221
|
}
|
|
222
|
+
const coll = e.nameCollision;
|
|
223
|
+
if (coll !== undefined) {
|
|
224
|
+
if (!isPlainObject(coll))
|
|
225
|
+
bad('must be a mapping with project, source and readOn', 'nameCollision');
|
|
226
|
+
else {
|
|
227
|
+
for (const key of ['project', 'source', 'readOn']) {
|
|
228
|
+
const v = coll[key];
|
|
229
|
+
if (typeof v !== 'string' || v.trim() === '')
|
|
230
|
+
bad(`${key} is required and must be non-empty`, 'nameCollision');
|
|
231
|
+
}
|
|
232
|
+
// Same rule as a deprecation, and for the same reason: this is a claim
|
|
233
|
+
// about a project outside this repository, so it names where it was read.
|
|
234
|
+
if (typeof coll.source === 'string' && !/^https?:\/\//.test(coll.source)) {
|
|
235
|
+
bad('source must be a URL — a published claim carries its evidence', 'nameCollision');
|
|
236
|
+
}
|
|
237
|
+
if (typeof coll.readOn === 'string' && !/^\d{4}-\d{2}-\d{2}$/.test(coll.readOn)) {
|
|
238
|
+
bad('readOn must be YYYY-MM-DD', 'nameCollision');
|
|
239
|
+
}
|
|
240
|
+
// The whole point of the field is to name something this row is *not*.
|
|
241
|
+
// A declaration pointing at the entry's own repository says nothing.
|
|
242
|
+
if (typeof coll.source === 'string' && typeof e.repo === 'string' && coll.source === e.repo) {
|
|
243
|
+
bad('source is this entry\'s own repo — the field names a different project', 'nameCollision');
|
|
244
|
+
}
|
|
245
|
+
for (const key of Object.keys(coll)) {
|
|
246
|
+
if (!['project', 'source', 'readOn'].includes(key))
|
|
247
|
+
bad(`unknown key ${key}`, 'nameCollision');
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
}
|
|
221
251
|
return problems;
|
|
222
252
|
}
|
|
223
253
|
/** Shape-check the whole document, including the invariants that span entries. */
|
package/package.json
CHANGED