mcp-context-cost 0.13.0 → 0.13.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -8
- package/dist/audit/deferral.d.ts +32 -8
- package/dist/audit/deferral.js +39 -8
- package/dist/core/divergence.d.ts +20 -0
- package/dist/core/divergence.js +27 -0
- package/dist/sweep/client.d.ts +30 -2
- package/dist/sweep/client.js +137 -17
- package/dist/sweep/published-stats.d.ts +14 -0
- package/dist/sweep/published-stats.js +52 -9
- package/dist/sweep/regressions.js +11 -1
- package/dist/sweep/run.js +8 -4
- package/dist/sweep/tool-shape.js +14 -1
- package/package.json +4 -2
package/README.md
CHANGED
|
@@ -126,7 +126,7 @@ costs **more** than loading the definitions would.
|
|
|
126
126
|
|
|
127
127
|
Three things the report will not do: it will not convert between units silently (in
|
|
128
128
|
threshold mode the stack is compared as a range, because the audit counts wire bytes and the
|
|
129
|
-
threshold is counted in what the client sends to the API — measured at 0.
|
|
129
|
+
threshold is counted in what the client sends to the API — measured at 0.19×–1.93× across 86
|
|
130
130
|
servers); it will not claim a posture the machine did not state readably, which is four
|
|
131
131
|
refusals and not one — when two places set the same variable to different values, when a
|
|
132
132
|
settings file exists and cannot be read, when the place that would decide sets the variable
|
|
@@ -204,8 +204,8 @@ INCREASE FAIL:
|
|
|
204
204
|
Add `--claude` to annotate each server with its Anthropic-request cost from the published
|
|
205
205
|
[Claude divergence](docs/METHODOLOGY.md#claude-divergence) run — an exact number when the
|
|
206
206
|
published capture hash matches what you have installed, `—` (silence, not a stale guess)
|
|
207
|
-
when it doesn't. The run holds
|
|
208
|
-
and [results/leaderboard.md](results/leaderboard.md) prints a claude number for the
|
|
207
|
+
when it doesn't. The run holds 87 rows — the measured servers it covered when it last ran —
|
|
208
|
+
and [results/leaderboard.md](results/leaderboard.md) prints a claude number for the 86 that
|
|
209
209
|
still match today and silence for the rest. Most installs will show a mix:
|
|
210
210
|
|
|
211
211
|
```
|
|
@@ -218,11 +218,11 @@ Add `--suggest` to place each of your tools in the measured set's tool-shape dis
|
|
|
218
218
|
([method](docs/METHODOLOGY.md#tool-shape)) and get advice only where the data can point at
|
|
219
219
|
something. Only descriptions draw advice — schemas are functional surface; descriptions are
|
|
220
220
|
prose every request carries — and only descriptions at or above the 90th percentile of the
|
|
221
|
-
1,
|
|
221
|
+
1,430 measured tools:
|
|
222
222
|
|
|
223
223
|
```
|
|
224
224
|
suggest — descriptions at or above the 90th percentile of measured tools
|
|
225
|
-
(baseline 2026-09-
|
|
225
|
+
(baseline 2026-09-05: 1,430 tools across 87 measured servers):
|
|
226
226
|
stub · wordy — 345 tokens: description 321 (p92), schema 14
|
|
227
227
|
rewriting the description toward the measured median (27) would recover ≈294 tokens on every request
|
|
228
228
|
1 of 2 tools sit inside the distribution — no advice where nothing is measurably unusual.
|
|
@@ -256,7 +256,7 @@ Flags: `--json` (full report on stdout, progress on stderr), `--budget N`,
|
|
|
256
256
|
|
|
257
257
|
The number `audit` gives you is the same measurement, run across a curated set of public
|
|
258
258
|
servers — which is how you can tell it is a measurement and not this tool's opinion. It also
|
|
259
|
-
shows what you are choosing between: across the
|
|
259
|
+
shows what you are choosing between: across the 87 servers measured, cost spans **1,700×**,
|
|
260
260
|
from `postgres` at 32 tokens to `github` at 54,622. The table below is a
|
|
261
261
|
sample of that range; the full range is in
|
|
262
262
|
[results/leaderboard.md](results/leaderboard.md).
|
|
@@ -271,7 +271,7 @@ sample of that range; the full range is in
|
|
|
271
271
|
| filesystem (reference) | 2,823 | 14 |
|
|
272
272
|
| markitdown | 64 | 1 |
|
|
273
273
|
|
|
274
|
-
*(
|
|
274
|
+
*(87 of 106 popular servers measured, each row dated by its own most recent sweep — full table in
|
|
275
275
|
[results/leaderboard.md](results/leaderboard.md); every failure is listed with its reason.
|
|
276
276
|
Each measured server also has a [detail page](https://athakur3.github.io/mcp-context-cost/servers/)
|
|
277
277
|
showing which tools its tokens are in.)*
|
|
@@ -299,7 +299,7 @@ now measured against a pinned model and published beside the badge, and they do
|
|
|
299
299
|
|
|
300
300
|
| server | badge (o200k) | Claude (`claude-opus-5`) | |
|
|
301
301
|
|---|---:|---:|---|
|
|
302
|
-
| github | 54,622 |
|
|
302
|
+
| github | 54,622 | **18,728** | most of the capture is `annotations`/`outputSchema` metadata Claude never sees |
|
|
303
303
|
| notion | 17,500 | **33,560** | almost no metadata to drop, so the tokenizer difference dominates |
|
|
304
304
|
|
|
305
305
|
So the heaviest server on the badge is not the heaviest server on Claude. Per-server
|
package/dist/audit/deferral.d.ts
CHANGED
|
@@ -219,24 +219,48 @@ export declare function resolveToolSearchSources(sources: ToolSearchSource[]): R
|
|
|
219
219
|
* threshold is a share of the context window measured in what the client
|
|
220
220
|
* actually sends to the API — the name/description/input_schema projection,
|
|
221
221
|
* counted by Anthropic's tokenizer, plus the tool framework overhead. Those are
|
|
222
|
-
* not the same number and the gap is not small
|
|
223
|
-
*
|
|
224
|
-
*
|
|
225
|
-
*
|
|
226
|
-
*
|
|
222
|
+
* not the same number and the gap is not small; the fields below carry it, and
|
|
223
|
+
* the pages state it from the run rather than from here. Comparing the wire
|
|
224
|
+
* number directly against the threshold understates the deferrable side for
|
|
225
|
+
* schema-heavy servers and overstates it for metadata-heavy ones, in one
|
|
226
|
+
* direction each.
|
|
227
|
+
*
|
|
228
|
+
* The band is **marginal**: it converts a server's own bytes and excludes the
|
|
229
|
+
* tool framework overhead, which `fixedOverhead` carries separately because the
|
|
230
|
+
* API charges it once per request however many servers are attached. Keeping
|
|
231
|
+
* them together was wrong in a way that stayed invisible while the divergence
|
|
232
|
+
* run sampled the heavy end — on a 54,000-token server a fixed 328 is noise. On
|
|
233
|
+
* 2026-09-05 the run widened to every measured server and reached `postgres` at
|
|
234
|
+
* 32 tokens on the wire, where 328 of its 348 Claude tokens *are* the overhead:
|
|
235
|
+
* a per-server ratio of 10.88× that says nothing about converting bytes. Folded
|
|
236
|
+
* into the band it took the published upper bound from 1.92× to 10.88× and made
|
|
237
|
+
* the audit refuse threshold questions it had been answering correctly. Held
|
|
238
|
+
* apart, the band across the same 86 rows is 0.19×–1.93× — which is where it
|
|
239
|
+
* already was, from a sample a quarter the size.
|
|
227
240
|
*/
|
|
228
241
|
export interface WireToClientRatio {
|
|
229
242
|
low: number;
|
|
230
243
|
high: number;
|
|
244
|
+
/**
|
|
245
|
+
* Tokens the tool framework costs once per request, whatever is attached.
|
|
246
|
+
* Added to a stack a single time; never multiplied by anything.
|
|
247
|
+
*/
|
|
248
|
+
fixedOverhead: number;
|
|
231
249
|
/** How many servers the band was measured across, for the printed caveat. */
|
|
232
250
|
servers: number;
|
|
233
251
|
/** The run it came from, so a reader can date it. */
|
|
234
252
|
source: string;
|
|
235
253
|
}
|
|
236
254
|
/**
|
|
237
|
-
* The band as published in this repository's own `results/divergence.json
|
|
238
|
-
*
|
|
239
|
-
*
|
|
255
|
+
* The band as published in this repository's own `results/divergence.json`.
|
|
256
|
+
* Used when no divergence run was supplied; `--claude` recomputes it from the
|
|
257
|
+
* run it fetched.
|
|
258
|
+
*
|
|
259
|
+
* The model, the date and the count deliberately are not restated here. This
|
|
260
|
+
* docblock said `(claude-opus-5, 2026-08-19, 20 servers)` while the field below
|
|
261
|
+
* read 23 and the run on disk held 24 — prose beside a number, drifting from
|
|
262
|
+
* it, which is the same failure this constant's own guard exists to catch one
|
|
263
|
+
* level down. The fields are the record; `source` dates them.
|
|
240
264
|
*/
|
|
241
265
|
export declare const PUBLISHED_WIRE_TO_CLIENT_RATIO: WireToClientRatio;
|
|
242
266
|
/** Derive the band from a supplied divergence run, falling back to the published one. */
|
package/dist/audit/deferral.js
CHANGED
|
@@ -222,13 +222,20 @@ export function resolveToolSearchSources(sources) {
|
|
|
222
222
|
return { ...resolved, source: resolved.readFromMachine ? (base?.source ?? null) : null };
|
|
223
223
|
}
|
|
224
224
|
/**
|
|
225
|
-
* The band as published in this repository's own `results/divergence.json
|
|
226
|
-
*
|
|
227
|
-
*
|
|
225
|
+
* The band as published in this repository's own `results/divergence.json`.
|
|
226
|
+
* Used when no divergence run was supplied; `--claude` recomputes it from the
|
|
227
|
+
* run it fetched.
|
|
228
|
+
*
|
|
229
|
+
* The model, the date and the count deliberately are not restated here. This
|
|
230
|
+
* docblock said `(claude-opus-5, 2026-08-19, 20 servers)` while the field below
|
|
231
|
+
* read 23 and the run on disk held 24 — prose beside a number, drifting from
|
|
232
|
+
* it, which is the same failure this constant's own guard exists to catch one
|
|
233
|
+
* level down. The fields are the record; `source` dates them.
|
|
228
234
|
*/
|
|
229
235
|
export const PUBLISHED_WIRE_TO_CLIENT_RATIO = {
|
|
230
|
-
low: 0.
|
|
231
|
-
high: 1.
|
|
236
|
+
low: 0.19,
|
|
237
|
+
high: 1.93,
|
|
238
|
+
fixedOverhead: 328,
|
|
232
239
|
// A snapshot of the run this package was cut against, which is what `source`
|
|
233
240
|
// below says it is — the installed package has no `results/` to read, so when
|
|
234
241
|
// a live run is supplied `wireToClientRatio` uses that instead and this is
|
|
@@ -242,27 +249,42 @@ export const PUBLISHED_WIRE_TO_CLIENT_RATIO = {
|
|
|
242
249
|
// every measured server on 2026-09-05, with nothing comparing the two. The
|
|
243
250
|
// band had not moved — the servers added sat inside it — which is exactly how
|
|
244
251
|
// a number like this goes wrong quietly.
|
|
245
|
-
|
|
252
|
+
//
|
|
253
|
+
// The widening then moved it by a hair rather than by the fivefold the first
|
|
254
|
+
// reading suggested: 0.20×–1.92× over 23 servers, 0.19×–1.93× over 86. That a
|
|
255
|
+
// quarter of the set predicted the whole of it is the interesting part, and it
|
|
256
|
+
// is only true of the marginal band — see the interface above for what folding
|
|
257
|
+
// the fixed overhead in did to the same numbers.
|
|
258
|
+
servers: 86,
|
|
246
259
|
source: 'the published claude-opus-5 divergence run',
|
|
247
260
|
};
|
|
248
261
|
/** Derive the band from a supplied divergence run, falling back to the published one. */
|
|
249
262
|
export function wireToClientRatio(run) {
|
|
250
263
|
if (!run)
|
|
251
264
|
return PUBLISHED_WIRE_TO_CLIENT_RATIO;
|
|
265
|
+
// `probeDelta` is the run's own reading of the fixed overhead: one tiny tool
|
|
266
|
+
// attached, minus the same request with none. An upper bound, and the only
|
|
267
|
+
// measurement of it there is. A run that does not carry one converts as it
|
|
268
|
+
// always did rather than guessing at a correction.
|
|
269
|
+
const fixedOverhead = typeof run.probeDelta === 'number' && run.probeDelta > 0 ? run.probeDelta : 0;
|
|
252
270
|
let low = Infinity;
|
|
253
271
|
let high = -Infinity;
|
|
254
272
|
let servers = 0;
|
|
255
273
|
for (const row of Object.values(run.servers)) {
|
|
256
274
|
if (!row || row.error || typeof row.claudeDelta !== 'number' || !(row.o200kFull > 0))
|
|
257
275
|
continue;
|
|
258
|
-
const ratio = row.claudeDelta / row.o200kFull;
|
|
276
|
+
const ratio = (row.claudeDelta - fixedOverhead) / row.o200kFull;
|
|
277
|
+
// A server whose entire Claude cost is the overhead says nothing about
|
|
278
|
+
// converting bytes, and a negative ratio is not a conversion at all.
|
|
279
|
+
if (!(ratio > 0))
|
|
280
|
+
continue;
|
|
259
281
|
low = Math.min(low, ratio);
|
|
260
282
|
high = Math.max(high, ratio);
|
|
261
283
|
servers++;
|
|
262
284
|
}
|
|
263
285
|
if (servers === 0)
|
|
264
286
|
return PUBLISHED_WIRE_TO_CLIENT_RATIO;
|
|
265
|
-
return { low, high, servers, source: `the ${run.measuredAt} ${run.model} divergence run` };
|
|
287
|
+
return { low, high, fixedOverhead, servers, source: `the ${run.measuredAt} ${run.model} divergence run` };
|
|
266
288
|
}
|
|
267
289
|
/** Clients this tool discovers that have no default deferral on record. */
|
|
268
290
|
const NO_DEFERRAL_ON_RECORD = new Set(['claude-desktop', 'cursor', 'vscode', 'windsurf']);
|
|
@@ -299,6 +321,15 @@ function estimate(servers, ratio) {
|
|
|
299
321
|
estimated++;
|
|
300
322
|
}
|
|
301
323
|
}
|
|
324
|
+
// Once per request, not once per server — the band is marginal precisely so
|
|
325
|
+
// that this is added here and exactly one time. A published Anthropic count
|
|
326
|
+
// already carries a copy of it, which is why a stack holding one is left
|
|
327
|
+
// alone: adding another would be the same double count in the other
|
|
328
|
+
// direction.
|
|
329
|
+
if (exact === 0 && estimated > 0) {
|
|
330
|
+
low += ratio.fixedOverhead;
|
|
331
|
+
high += ratio.fixedOverhead;
|
|
332
|
+
}
|
|
302
333
|
return { low: Math.round(low), high: Math.round(high), exact, estimated };
|
|
303
334
|
}
|
|
304
335
|
/**
|
|
@@ -67,5 +67,25 @@ export declare function claudeRatio(row: DivergenceRow): number | null;
|
|
|
67
67
|
* caveat: a wrong number next to a fresh badge is worse than no number.
|
|
68
68
|
*/
|
|
69
69
|
export declare function isCurrent(row: DivergenceRow | undefined, canonicalSha256: string | null): row is DivergenceRow;
|
|
70
|
+
/**
|
|
71
|
+
* Drop rows that have stopped describing the capture they were computed from.
|
|
72
|
+
*
|
|
73
|
+
* The companion to `isCurrent`, one layer earlier. `isCurrent` stops a stale row
|
|
74
|
+
* reaching a page; this stops it being carried forward in the first place. A
|
|
75
|
+
* selection preserves every row it does not measure, and a re-sweep moves the
|
|
76
|
+
* capture of every server it re-measures — so between the two, a row outside the
|
|
77
|
+
* selection can go on being merged forward long after the capture it describes
|
|
78
|
+
* is gone. Eight of twenty-four rows had reached that state by 2026-09-05, and
|
|
79
|
+
* nothing said so: the cells were hidden, so the file looked complete while the
|
|
80
|
+
* count of rows had stopped meaning the count of usable rows.
|
|
81
|
+
*
|
|
82
|
+
* `captureSha` returns the canonical hash of what is on disk today, or undefined
|
|
83
|
+
* when there is no capture at all — which is also a reason to drop, because a
|
|
84
|
+
* row about a server that no longer measures is a row nothing can confirm.
|
|
85
|
+
*/
|
|
86
|
+
export declare function dropStaleRows(servers: Record<string, DivergenceRow>, captureSha: (name: string) => string | undefined): {
|
|
87
|
+
kept: Record<string, DivergenceRow>;
|
|
88
|
+
dropped: string[];
|
|
89
|
+
};
|
|
70
90
|
/** Parse results/divergence.json; anything malformed yields null, never throws. */
|
|
71
91
|
export declare function parseDivergence(text: string): DivergenceRun | null;
|
package/dist/core/divergence.js
CHANGED
|
@@ -86,6 +86,33 @@ export function isCurrent(row, canonicalSha256) {
|
|
|
86
86
|
return false;
|
|
87
87
|
return !!canonicalSha256 && row.capturedSha256 === canonicalSha256;
|
|
88
88
|
}
|
|
89
|
+
/**
|
|
90
|
+
* Drop rows that have stopped describing the capture they were computed from.
|
|
91
|
+
*
|
|
92
|
+
* The companion to `isCurrent`, one layer earlier. `isCurrent` stops a stale row
|
|
93
|
+
* reaching a page; this stops it being carried forward in the first place. A
|
|
94
|
+
* selection preserves every row it does not measure, and a re-sweep moves the
|
|
95
|
+
* capture of every server it re-measures — so between the two, a row outside the
|
|
96
|
+
* selection can go on being merged forward long after the capture it describes
|
|
97
|
+
* is gone. Eight of twenty-four rows had reached that state by 2026-09-05, and
|
|
98
|
+
* nothing said so: the cells were hidden, so the file looked complete while the
|
|
99
|
+
* count of rows had stopped meaning the count of usable rows.
|
|
100
|
+
*
|
|
101
|
+
* `captureSha` returns the canonical hash of what is on disk today, or undefined
|
|
102
|
+
* when there is no capture at all — which is also a reason to drop, because a
|
|
103
|
+
* row about a server that no longer measures is a row nothing can confirm.
|
|
104
|
+
*/
|
|
105
|
+
export function dropStaleRows(servers, captureSha) {
|
|
106
|
+
const kept = {};
|
|
107
|
+
const dropped = [];
|
|
108
|
+
for (const [name, row] of Object.entries(servers)) {
|
|
109
|
+
if (row.capturedSha256 && row.capturedSha256 === captureSha(name))
|
|
110
|
+
kept[name] = row;
|
|
111
|
+
else
|
|
112
|
+
dropped.push(name);
|
|
113
|
+
}
|
|
114
|
+
return { kept, dropped };
|
|
115
|
+
}
|
|
89
116
|
/** Parse results/divergence.json; anything malformed yields null, never throws. */
|
|
90
117
|
export function parseDivergence(text) {
|
|
91
118
|
let run;
|
package/dist/sweep/client.d.ts
CHANGED
|
@@ -33,16 +33,43 @@ export interface WireCapture {
|
|
|
33
33
|
* Noise is only dropped while something else survives. A package that fails
|
|
34
34
|
* *inside* npm (EBADPLATFORM, a failed postinstall) has npm's own lines as its
|
|
35
35
|
* only evidence, and a server whose whole output is a stack keeps the stack.
|
|
36
|
+
*
|
|
37
|
+
* `required` is the phrase a declared entry's published status depends on
|
|
38
|
+
* (`notApplicable.evidence` in servers.yaml). Everything else here is a budget
|
|
39
|
+
* decision — how much of a long failure is worth publishing — but this one is
|
|
40
|
+
* not. The entry says "this failure is the harness's, and here is the sentence
|
|
41
|
+
* that proves it", and `notApplicableReason` re-reads that sentence out of what
|
|
42
|
+
* survives: elide it and the declaration turns itself off, the row reverts to
|
|
43
|
+
* `startup-failure`, and this project publishes that someone else's working
|
|
44
|
+
* software is broken. The evidence is therefore kept whatever the budget, and
|
|
45
|
+
* everything else competes for what is left — the rule the old one lacked, and
|
|
46
|
+
* the reason it depended on how much the server happened to print.
|
|
47
|
+
*/
|
|
48
|
+
export declare function evidenceTail(stderr: string, limit?: number, required?: string): string;
|
|
49
|
+
/**
|
|
50
|
+
* Cut a record's notes to `limit` without cutting away the evidence its own
|
|
51
|
+
* status rests on.
|
|
52
|
+
*
|
|
53
|
+
* The same rule as `evidenceTail`, one layer up and for a different reason. The
|
|
54
|
+
* classifier reads the message *before* this cut, so this one cannot change a
|
|
55
|
+
* published status — it can only leave a record asserting `not-applicable` with
|
|
56
|
+
* the sentence that justifies it deleted, which is a claim published without its
|
|
57
|
+
* evidence. `windows-mcp` sits at exactly the cap today, so the margin here is
|
|
58
|
+
* one character of growth in somebody else's error message.
|
|
36
59
|
*/
|
|
37
|
-
export declare function
|
|
60
|
+
export declare function clampNotes(text: string, limit: number, required?: string): string;
|
|
38
61
|
export declare class McpStdioClient {
|
|
62
|
+
/** Phrase this entry's declared status depends on — see `evidenceTail`. */
|
|
63
|
+
private keepEvidence?;
|
|
39
64
|
private child;
|
|
40
65
|
private buffer;
|
|
41
66
|
private nextId;
|
|
42
67
|
private pending;
|
|
43
68
|
private stderrChunks;
|
|
44
69
|
private exited;
|
|
45
|
-
constructor(command: string, args: string[], env: Record<string, string | undefined
|
|
70
|
+
constructor(command: string, args: string[], env: Record<string, string | undefined>,
|
|
71
|
+
/** Phrase this entry's declared status depends on — see `evidenceTail`. */
|
|
72
|
+
keepEvidence?: string | undefined);
|
|
46
73
|
private onData;
|
|
47
74
|
private send;
|
|
48
75
|
private deadReason;
|
|
@@ -61,6 +88,7 @@ export declare function captureTools(spec: string | {
|
|
|
61
88
|
}, opts?: {
|
|
62
89
|
timeoutMs?: number;
|
|
63
90
|
env?: Record<string, string>;
|
|
91
|
+
keepEvidence?: string;
|
|
64
92
|
}): Promise<WireCapture>;
|
|
65
93
|
/** Shell-free command splitting: honors single/double quotes, no expansion. */
|
|
66
94
|
export declare function splitCommand(line: string): string[];
|
package/dist/sweep/client.js
CHANGED
|
@@ -6,6 +6,8 @@
|
|
|
6
6
|
*/
|
|
7
7
|
import { spawn } from 'node:child_process';
|
|
8
8
|
const PROTOCOL_VERSION = '2025-06-18';
|
|
9
|
+
/** How an elided middle is marked, in every layout here. */
|
|
10
|
+
const ELISION = ' […] ';
|
|
9
11
|
/**
|
|
10
12
|
* The part of a dead server's stderr worth keeping as evidence.
|
|
11
13
|
*
|
|
@@ -26,11 +28,24 @@ const PROTOCOL_VERSION = '2025-06-18';
|
|
|
26
28
|
* Noise is only dropped while something else survives. A package that fails
|
|
27
29
|
* *inside* npm (EBADPLATFORM, a failed postinstall) has npm's own lines as its
|
|
28
30
|
* only evidence, and a server whose whole output is a stack keeps the stack.
|
|
31
|
+
*
|
|
32
|
+
* `required` is the phrase a declared entry's published status depends on
|
|
33
|
+
* (`notApplicable.evidence` in servers.yaml). Everything else here is a budget
|
|
34
|
+
* decision — how much of a long failure is worth publishing — but this one is
|
|
35
|
+
* not. The entry says "this failure is the harness's, and here is the sentence
|
|
36
|
+
* that proves it", and `notApplicableReason` re-reads that sentence out of what
|
|
37
|
+
* survives: elide it and the declaration turns itself off, the row reverts to
|
|
38
|
+
* `startup-failure`, and this project publishes that someone else's working
|
|
39
|
+
* software is broken. The evidence is therefore kept whatever the budget, and
|
|
40
|
+
* everything else competes for what is left — the rule the old one lacked, and
|
|
41
|
+
* the reason it depended on how much the server happened to print.
|
|
29
42
|
*/
|
|
30
|
-
export function evidenceTail(stderr, limit = 600) {
|
|
31
|
-
const
|
|
32
|
-
const
|
|
33
|
-
|
|
43
|
+
export function evidenceTail(stderr, limit = 600, required) {
|
|
44
|
+
const needle = required ? required.toLowerCase() : undefined;
|
|
45
|
+
const protect = (l) => needle !== undefined && l.toLowerCase().includes(needle);
|
|
46
|
+
const withoutNoise = drop(stderr, (l) => /^npm (warn|notice)\b/.test(l), protect);
|
|
47
|
+
const withoutFrames = drop(withoutNoise, (l) => /^at\s/.test(l), protect);
|
|
48
|
+
return bothEnds(withoutFrames, limit, needle);
|
|
34
49
|
}
|
|
35
50
|
/**
|
|
36
51
|
* Keep the start and the end, eliding the middle.
|
|
@@ -47,10 +62,23 @@ export function evidenceTail(stderr, limit = 600) {
|
|
|
47
62
|
* and the middle is what goes. The split leans towards the head because a
|
|
48
63
|
* message that precedes its own noise is the more common shape here.
|
|
49
64
|
*/
|
|
50
|
-
function bothEnds(text, limit) {
|
|
65
|
+
function bothEnds(text, limit, needle) {
|
|
66
|
+
const kept = bothEndsPlain(text, limit);
|
|
67
|
+
if (needle === undefined)
|
|
68
|
+
return kept;
|
|
69
|
+
// Only pay for the anchored layout when the ordinary one lost the phrase and
|
|
70
|
+
// the raw text actually had it. An entry whose evidence never appeared is a
|
|
71
|
+
// declaration that does not hold, and must keep failing as one.
|
|
72
|
+
if (kept.toLowerCase().includes(needle))
|
|
73
|
+
return kept;
|
|
74
|
+
if (!text.toLowerCase().includes(needle))
|
|
75
|
+
return kept;
|
|
76
|
+
return aroundRequired(text, limit, needle);
|
|
77
|
+
}
|
|
78
|
+
function bothEndsPlain(text, limit) {
|
|
51
79
|
if (text.length <= limit)
|
|
52
80
|
return text;
|
|
53
|
-
const elision =
|
|
81
|
+
const elision = `\n${ELISION}\n`;
|
|
54
82
|
const budget = Math.max(0, limit - elision.length);
|
|
55
83
|
const lines = text.split('\n');
|
|
56
84
|
// Whole lines only: a boundary cut mid-word ("ool/prompt change") reads as
|
|
@@ -83,23 +111,119 @@ function bothEnds(text, limit) {
|
|
|
83
111
|
return text.slice(0, budget) + elision;
|
|
84
112
|
return `${headText}${elision}${tail.out.join('\n')}`;
|
|
85
113
|
}
|
|
86
|
-
/**
|
|
87
|
-
|
|
114
|
+
/**
|
|
115
|
+
* Keep the line the declared evidence is on, then spend what is left on context.
|
|
116
|
+
*
|
|
117
|
+
* Head before tail, the same lean `bothEndsPlain` takes and for the same reason:
|
|
118
|
+
* an explanation usually precedes its own aftermath. The elisions are marked, so
|
|
119
|
+
* a reader of the record can see that something was dropped around the sentence
|
|
120
|
+
* that was not.
|
|
121
|
+
*/
|
|
122
|
+
function aroundRequired(text, limit, needle) {
|
|
123
|
+
const lines = text.split('\n');
|
|
124
|
+
const k = lines.findIndex((l) => l.toLowerCase().includes(needle));
|
|
125
|
+
// Both elisions and the newlines joining at most five chunks, reserved before
|
|
126
|
+
// the anchor is sized: the budget is a published-record limit, and a layout
|
|
127
|
+
// that keeps the evidence by overrunning it has only moved the problem.
|
|
128
|
+
const reserve = 2 * ELISION.length + 4;
|
|
129
|
+
const anchor = windowAround(lines[k], needle, Math.max(needle.length, limit - reserve));
|
|
130
|
+
let budget = limit - anchor.length - reserve;
|
|
131
|
+
const head = [];
|
|
132
|
+
for (let i = 0; i < k; i++) {
|
|
133
|
+
const cost = lines[i].length + 1;
|
|
134
|
+
if (cost > budget)
|
|
135
|
+
break;
|
|
136
|
+
head.push(lines[i]);
|
|
137
|
+
budget -= cost;
|
|
138
|
+
}
|
|
139
|
+
const tail = [];
|
|
140
|
+
for (let i = lines.length - 1; i > k; i--) {
|
|
141
|
+
const cost = lines[i].length + 1;
|
|
142
|
+
if (cost > budget)
|
|
143
|
+
break;
|
|
144
|
+
tail.unshift(lines[i]);
|
|
145
|
+
budget -= cost;
|
|
146
|
+
}
|
|
147
|
+
const chunks = [];
|
|
148
|
+
if (head.length)
|
|
149
|
+
chunks.push(head.join('\n'));
|
|
150
|
+
if (head.length < k)
|
|
151
|
+
chunks.push(ELISION);
|
|
152
|
+
chunks.push(anchor);
|
|
153
|
+
if (tail.length < lines.length - 1 - k)
|
|
154
|
+
chunks.push(ELISION);
|
|
155
|
+
if (tail.length)
|
|
156
|
+
chunks.push(tail.join('\n'));
|
|
157
|
+
return chunks.join('\n');
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* A single line that alone overruns the budget, kept as a window around the
|
|
161
|
+
* match rather than from its start — a structured log puts the whole message on
|
|
162
|
+
* one line, and the phrase that matters can sit anywhere in it.
|
|
163
|
+
*/
|
|
164
|
+
function windowAround(line, needle, limit) {
|
|
165
|
+
if (line.length <= limit)
|
|
166
|
+
return line;
|
|
167
|
+
const at = line.toLowerCase().indexOf(needle);
|
|
168
|
+
const lead = Math.max(0, Math.floor((limit - needle.length) / 2));
|
|
169
|
+
const start = Math.max(0, Math.min(at - lead, line.length - limit));
|
|
170
|
+
return line.slice(start, start + limit);
|
|
171
|
+
}
|
|
172
|
+
/**
|
|
173
|
+
* Cut a record's notes to `limit` without cutting away the evidence its own
|
|
174
|
+
* status rests on.
|
|
175
|
+
*
|
|
176
|
+
* The same rule as `evidenceTail`, one layer up and for a different reason. The
|
|
177
|
+
* classifier reads the message *before* this cut, so this one cannot change a
|
|
178
|
+
* published status — it can only leave a record asserting `not-applicable` with
|
|
179
|
+
* the sentence that justifies it deleted, which is a claim published without its
|
|
180
|
+
* evidence. `windows-mcp` sits at exactly the cap today, so the margin here is
|
|
181
|
+
* one character of growth in somebody else's error message.
|
|
182
|
+
*/
|
|
183
|
+
export function clampNotes(text, limit, required) {
|
|
184
|
+
if (text.length <= limit)
|
|
185
|
+
return text;
|
|
186
|
+
const plain = text.slice(0, limit);
|
|
187
|
+
if (!required)
|
|
188
|
+
return plain;
|
|
189
|
+
const needle = required.toLowerCase();
|
|
190
|
+
if (plain.toLowerCase().includes(needle))
|
|
191
|
+
return plain;
|
|
192
|
+
const at = text.toLowerCase().indexOf(needle);
|
|
193
|
+
if (at < 0)
|
|
194
|
+
return plain;
|
|
195
|
+
const room = Math.max(0, limit - ELISION.length);
|
|
196
|
+
const width = Math.min(room, Math.max(needle.length, Math.floor(room / 2)));
|
|
197
|
+
const start = Math.max(0, Math.min(at - Math.floor((width - needle.length) / 2), text.length - width));
|
|
198
|
+
return text.slice(0, Math.max(0, room - width)) + ELISION + text.slice(start, start + width);
|
|
199
|
+
}
|
|
200
|
+
/**
|
|
201
|
+
* Drop matching lines, keeping the input whole if that would leave nothing.
|
|
202
|
+
*
|
|
203
|
+
* `isProtected` outranks `isNoise`: safari-mcp declares `EBADPLATFORM`, which npm
|
|
204
|
+
* prints on a line of its own, and a filter that reaches it first would delete
|
|
205
|
+
* the evidence before any budget was even applied.
|
|
206
|
+
*/
|
|
207
|
+
function drop(text, isNoise, isProtected = () => false) {
|
|
88
208
|
const kept = text
|
|
89
209
|
.split('\n')
|
|
90
|
-
.filter((l) => !isNoise(l.trim()))
|
|
210
|
+
.filter((l) => isProtected(l.trim()) || !isNoise(l.trim()))
|
|
91
211
|
.join('\n')
|
|
92
212
|
.trim();
|
|
93
213
|
return kept || text.trim();
|
|
94
214
|
}
|
|
95
215
|
export class McpStdioClient {
|
|
216
|
+
keepEvidence;
|
|
96
217
|
child;
|
|
97
218
|
buffer = '';
|
|
98
219
|
nextId = 1;
|
|
99
220
|
pending = new Map();
|
|
100
221
|
stderrChunks = [];
|
|
101
222
|
exited;
|
|
102
|
-
constructor(command, args, env
|
|
223
|
+
constructor(command, args, env,
|
|
224
|
+
/** Phrase this entry's declared status depends on — see `evidenceTail`. */
|
|
225
|
+
keepEvidence) {
|
|
226
|
+
this.keepEvidence = keepEvidence;
|
|
103
227
|
this.child = spawn(command, args, {
|
|
104
228
|
env: { ...env },
|
|
105
229
|
stdio: ['pipe', 'pipe', 'pipe'],
|
|
@@ -123,7 +247,7 @@ export class McpStdioClient {
|
|
|
123
247
|
resolve();
|
|
124
248
|
});
|
|
125
249
|
this.child.on('exit', (code) => {
|
|
126
|
-
const tail = evidenceTail(this.stderrTail);
|
|
250
|
+
const tail = evidenceTail(this.stderrTail, undefined, this.keepEvidence);
|
|
127
251
|
this.deadReason = `server exited (code ${code})${tail ? `; stderr tail: ${tail}` : ''}`;
|
|
128
252
|
for (const p of this.pending.values())
|
|
129
253
|
p.reject(new Error(this.deadReason));
|
|
@@ -185,7 +309,7 @@ export class McpStdioClient {
|
|
|
185
309
|
// without this a timed-out record carries no evidence at all — it says
|
|
186
310
|
// only that we waited. What the server managed to print before it
|
|
187
311
|
// stopped answering is usually the whole explanation.
|
|
188
|
-
const tail = evidenceTail(this.stderrTail);
|
|
312
|
+
const tail = evidenceTail(this.stderrTail, undefined, this.keepEvidence);
|
|
189
313
|
reject(new Error(`timeout after ${timeoutMs}ms waiting for ${method}${tail ? `; stderr tail: ${tail}` : ''}`));
|
|
190
314
|
}, timeoutMs);
|
|
191
315
|
this.pending.set(id, {
|
|
@@ -222,11 +346,7 @@ export class McpStdioClient {
|
|
|
222
346
|
export async function captureTools(spec, opts = {}) {
|
|
223
347
|
const timeoutMs = opts.timeoutMs ?? 60_000;
|
|
224
348
|
const [cmd, ...args] = typeof spec === 'string' ? splitCommand(spec) : [spec.command, ...spec.argv];
|
|
225
|
-
const client = new McpStdioClient(cmd, args, {
|
|
226
|
-
PATH: process.env.PATH,
|
|
227
|
-
HOME: process.env.HOME,
|
|
228
|
-
...opts.env,
|
|
229
|
-
});
|
|
349
|
+
const client = new McpStdioClient(cmd, args, { PATH: process.env.PATH, HOME: process.env.HOME, ...opts.env }, opts.keepEvidence);
|
|
230
350
|
try {
|
|
231
351
|
const init = await client.request('initialize', {
|
|
232
352
|
protocolVersion: PROTOCOL_VERSION,
|
|
@@ -49,6 +49,20 @@ export interface PublishedStats {
|
|
|
49
49
|
/** claudeDelta / o200kFull across the run's rows that carry a number. */
|
|
50
50
|
ratioMin: number;
|
|
51
51
|
ratioMax: number;
|
|
52
|
+
/** How many rows produced the band — see where it is set. */
|
|
53
|
+
ratioServers: number;
|
|
54
|
+
};
|
|
55
|
+
/**
|
|
56
|
+
* The published tool-shape baseline, which README quotes twice — once in
|
|
57
|
+
* prose and once inside the `--suggest` sample output. Both were hand-written
|
|
58
|
+
* and `STATIC_COUNTS` excused them as regen-maintained, which was not true of
|
|
59
|
+
* either: the file said 1,430 tools across 87 servers while the page said
|
|
60
|
+
* 1,150 across 81, and every test passed.
|
|
61
|
+
*/
|
|
62
|
+
toolShape: {
|
|
63
|
+
toolCount: number;
|
|
64
|
+
serverCount: number;
|
|
65
|
+
generatedAt: string;
|
|
52
66
|
};
|
|
53
67
|
deferralCostlierCount: number;
|
|
54
68
|
/** Servers whose most recent cost movement went up / down (cost-regression/v1). */
|
|
@@ -25,9 +25,10 @@
|
|
|
25
25
|
* never a silent skip), and only the slots are ever rewritten — spliced in
|
|
26
26
|
* place, so the page's own line wrapping survives.
|
|
27
27
|
*/
|
|
28
|
-
import { readFileSync, writeFileSync } from 'node:fs';
|
|
28
|
+
import { existsSync, readFileSync, writeFileSync } from 'node:fs';
|
|
29
29
|
import { join } from 'node:path';
|
|
30
30
|
import { DEFAULT_CONTEXT_WINDOW } from '../audit/audit.js';
|
|
31
|
+
import { wireToClientRatio } from '../audit/deferral.js';
|
|
31
32
|
import { fieldSelectionShare, isCurrent } from '../core/divergence.js';
|
|
32
33
|
import { sessionStartLoad } from '../core/session-start.js';
|
|
33
34
|
import { isGood } from './harness-guard.js';
|
|
@@ -106,10 +107,13 @@ export function computePublishedStats(entries, root = process.cwd()) {
|
|
|
106
107
|
const shares = currentRows
|
|
107
108
|
.map(([, r]) => fieldSelectionShare(r))
|
|
108
109
|
.filter((s) => s !== null && s >= 0);
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
110
|
+
// Asked of the same function the audit converts with, rather than recomputed
|
|
111
|
+
// here. Two derivations of one band is how a page ends up describing a
|
|
112
|
+
// conversion the tool does not perform — and these two differed twice over:
|
|
113
|
+
// this one worked from the rows whose capture is still current, and it divided
|
|
114
|
+
// by a delta that still had the fixed tool overhead in it.
|
|
115
|
+
const band = wireToClientRatio(div);
|
|
116
|
+
if (shares.length === 0 || band.servers === 0) {
|
|
113
117
|
throw new Error('no current divergence row — METHODOLOGY states ranges over them; run `npm run divergence`');
|
|
114
118
|
}
|
|
115
119
|
// The exemplar METHODOLOGY names for the field-selection effect is whichever
|
|
@@ -128,7 +132,15 @@ export function computePublishedStats(entries, root = process.cwd()) {
|
|
|
128
132
|
if (!githubRow?.m || typeof githubRow.m.totalTokens !== 'number') {
|
|
129
133
|
throw new Error('README quotes `verify` on results/github, which has no current measurement');
|
|
130
134
|
}
|
|
135
|
+
const shape = (() => {
|
|
136
|
+
const p = join(root, 'results', 'tool-shape.json');
|
|
137
|
+
if (!existsSync(p))
|
|
138
|
+
throw new Error('results/tool-shape.json is missing — README states its numbers');
|
|
139
|
+
const j = JSON.parse(readFileSync(p, 'utf8'));
|
|
140
|
+
return { toolCount: j.toolCount, serverCount: j.serverCount, generatedAt: j.generatedAt };
|
|
141
|
+
})();
|
|
131
142
|
return {
|
|
143
|
+
toolShape: shape,
|
|
132
144
|
candidateTotal: rows.length,
|
|
133
145
|
measuredCount: measured.length,
|
|
134
146
|
max,
|
|
@@ -146,8 +158,13 @@ export function computePublishedStats(entries, root = process.cwd()) {
|
|
|
146
158
|
widest: { server: widest[0], full: widest[1].o200kFull, mapped: widest[1].o200kMapped },
|
|
147
159
|
shareMin: Math.min(...shares),
|
|
148
160
|
shareMax: Math.max(...shares),
|
|
149
|
-
ratioMin:
|
|
150
|
-
ratioMax:
|
|
161
|
+
ratioMin: band.low,
|
|
162
|
+
ratioMax: band.high,
|
|
163
|
+
// What produced the band, never `runSize`: a row can sit in the run and
|
|
164
|
+
// contribute nothing to it — `gitlab` is there with an API error and a
|
|
165
|
+
// zero delta. "Across {runSize} servers" states the band was measured over
|
|
166
|
+
// one more server than measured it.
|
|
167
|
+
ratioServers: band.servers,
|
|
151
168
|
},
|
|
152
169
|
deferralCostlierCount: costlier.length,
|
|
153
170
|
movement: { grew: movement.grew, shrank: movement.shrank },
|
|
@@ -223,7 +240,21 @@ export const PAGE_CLAIMS = [
|
|
|
223
240
|
// page-number guard: three numbers written by hand beside the two
|
|
224
241
|
// sentences regen already kept true.
|
|
225
242
|
template: 'measured at {f}×–{f}× across {n} servers)',
|
|
226
|
-
values: (s) => [s.claude.ratioMin.toFixed(2), s.claude.ratioMax.toFixed(2), fmt(s.claude.
|
|
243
|
+
values: (s) => [s.claude.ratioMin.toFixed(2), s.claude.ratioMax.toFixed(2), fmt(s.claude.ratioServers)],
|
|
244
|
+
},
|
|
245
|
+
{
|
|
246
|
+
file: 'README.md',
|
|
247
|
+
id: 'tool-shape:prose',
|
|
248
|
+
template: 'only descriptions at or above the 90th percentile of the {n} measured tools:',
|
|
249
|
+
values: (s) => [fmt(s.toolShape.toolCount)],
|
|
250
|
+
},
|
|
251
|
+
{
|
|
252
|
+
file: 'README.md',
|
|
253
|
+
id: 'tool-shape:sample-header',
|
|
254
|
+
// Inside the `--suggest` sample block. A reader compares their own output
|
|
255
|
+
// against it, so a stale baseline line there is read as a current one.
|
|
256
|
+
template: '(baseline {w}: {n} tools across {n} measured servers):',
|
|
257
|
+
values: (s) => [s.toolShape.generatedAt, fmt(s.toolShape.toolCount), fmt(s.toolShape.serverCount)],
|
|
227
258
|
},
|
|
228
259
|
{
|
|
229
260
|
file: 'README.md',
|
|
@@ -317,11 +348,23 @@ export const PAGE_CLAIMS = [
|
|
|
317
348
|
fmt(s.claude.widest.mapped),
|
|
318
349
|
],
|
|
319
350
|
},
|
|
351
|
+
{
|
|
352
|
+
file: 'docs/METHODOLOGY.md',
|
|
353
|
+
id: 'divergence:band-parenthetical',
|
|
354
|
+
// The deferral section's own copy of the band. It was hand-written and held
|
|
355
|
+
// to `PUBLISHED_WIRE_TO_CLIENT_RATIO` by a test, which worked only while the
|
|
356
|
+
// constant tracked the run exactly — and the constant is a release-time
|
|
357
|
+
// snapshot that is deliberately allowed to lag. Two pages quoting two
|
|
358
|
+
// different sources for one number is the drift this file exists to end, so
|
|
359
|
+
// both pages state the run and the constant is guarded separately.
|
|
360
|
+
template: 'band ({f}×–{f}×\nacross {n} servers)',
|
|
361
|
+
values: (s) => [s.claude.ratioMin.toFixed(2), s.claude.ratioMax.toFixed(2), fmt(s.claude.ratioServers)],
|
|
362
|
+
},
|
|
320
363
|
{
|
|
321
364
|
file: 'docs/METHODOLOGY.md',
|
|
322
365
|
id: 'divergence:ratio-range',
|
|
323
366
|
template: 'it ranged from {f}× to {f}× across the {n} servers in the run,',
|
|
324
|
-
values: (s) => [s.claude.ratioMin.toFixed(2), s.claude.ratioMax.toFixed(2), fmt(s.claude.
|
|
367
|
+
values: (s) => [s.claude.ratioMin.toFixed(2), s.claude.ratioMax.toFixed(2), fmt(s.claude.ratioServers)],
|
|
325
368
|
},
|
|
326
369
|
{
|
|
327
370
|
file: 'docs/METHODOLOGY.md',
|
|
@@ -103,9 +103,19 @@ export function writeCaptureIndex(entries, root = process.cwd()) {
|
|
|
103
103
|
}
|
|
104
104
|
for (const sha of ambiguous)
|
|
105
105
|
delete captures[sha];
|
|
106
|
+
// Dated by the newest capture it indexes, not by the moment it was written —
|
|
107
|
+
// the same rule the dashboard was moved to. A clock stamp makes the file
|
|
108
|
+
// claim to be as fresh as the run that regenerated it, when what a reader
|
|
109
|
+
// needs to know is how far the data behind it reaches. It also means
|
|
110
|
+
// regenerating over unchanged vectors produces no diff, which is what lets a
|
|
111
|
+
// release gate ask "is anything derived here stale?" and get a real answer.
|
|
112
|
+
const newest = Object.values(captures)
|
|
113
|
+
.map((c) => c.date)
|
|
114
|
+
.sort()
|
|
115
|
+
.pop();
|
|
106
116
|
const index = {
|
|
107
117
|
method: CAPTURE_INDEX_METHOD,
|
|
108
|
-
generatedAt:
|
|
118
|
+
generatedAt: newest ?? '',
|
|
109
119
|
// Key order sorted so a re-run over unchanged vectors produces no diff noise.
|
|
110
120
|
captures: Object.fromEntries(Object.entries(captures).sort(([a], [b]) => a.localeCompare(b))),
|
|
111
121
|
current: Object.fromEntries(Object.entries(current).sort(([a], [b]) => a.localeCompare(b))),
|
package/dist/sweep/run.js
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
import { mkdirSync, writeFileSync } from 'node:fs';
|
|
8
8
|
import { join, resolve } from 'node:path';
|
|
9
9
|
import { fileURLToPath } from 'node:url';
|
|
10
|
-
import { captureTools } from './client.js';
|
|
10
|
+
import { captureTools, clampNotes } from './client.js';
|
|
11
11
|
import { DockerHarnessFault, defaultImageFor, dockerize, ensureImage, containerPlatform, isDockerRunFailure, } from './docker.js';
|
|
12
12
|
import { measureTools, failedMeasurement, canonicalString } from '../core/canonical.js';
|
|
13
13
|
import { toBadge } from '../core/badge.js';
|
|
@@ -206,8 +206,12 @@ export async function measureServer(name, command, opts = {}) {
|
|
|
206
206
|
: opts;
|
|
207
207
|
let r;
|
|
208
208
|
try {
|
|
209
|
-
|
|
210
|
-
|
|
209
|
+
// The declared evidence travels with the launch, because the truncation
|
|
210
|
+
// that could lose it happens inside the client, before anything here sees
|
|
211
|
+
// the message it will be classified from.
|
|
212
|
+
const captureOpts = { ...attemptOpts, keepEvidence: opts.notApplicable?.evidence };
|
|
213
|
+
const first = await captureTools(buildSpec(noSharedCache), captureOpts);
|
|
214
|
+
const second = await captureTools(buildSpec(noSharedCache), captureOpts);
|
|
211
215
|
r = measureTools(first.tools, {
|
|
212
216
|
serverName: first.serverInfo?.name ?? name,
|
|
213
217
|
serverVersion: first.serverInfo?.version,
|
|
@@ -236,7 +240,7 @@ export async function measureServer(name, command, opts = {}) {
|
|
|
236
240
|
launchCommand: command,
|
|
237
241
|
// The declared reason leads, but the raw failure stays behind it: the
|
|
238
242
|
// record has to remain checkable against the run that produced it.
|
|
239
|
-
notes: (declared ? `${declared} — ${msg}` : msg
|
|
243
|
+
notes: clampNotes(declared ? `${declared} — ${msg}` : msg, 700, opts.notApplicable?.evidence),
|
|
240
244
|
});
|
|
241
245
|
}
|
|
242
246
|
const iso = isolation ?? { docker: false };
|
package/dist/sweep/tool-shape.js
CHANGED
|
@@ -11,6 +11,7 @@ import { buildToolShapeBaseline } from '../core/tool-shape.js';
|
|
|
11
11
|
import { loadRows } from './report.js';
|
|
12
12
|
export function writeToolShapeBaseline(entries, root = process.cwd()) {
|
|
13
13
|
const tools = [];
|
|
14
|
+
const dates = [];
|
|
14
15
|
let serverCount = 0;
|
|
15
16
|
for (const r of loadRows(entries, root)) {
|
|
16
17
|
if (!r.m || (r.m.status !== 'measured' && r.m.status !== 'dynamic'))
|
|
@@ -19,8 +20,20 @@ export function writeToolShapeBaseline(entries, root = process.cwd()) {
|
|
|
19
20
|
continue;
|
|
20
21
|
serverCount++;
|
|
21
22
|
tools.push(...r.m.tools);
|
|
23
|
+
const day = String(r.m.measuredAt ?? '').slice(0, 10);
|
|
24
|
+
if (/^\d{4}-\d{2}-\d{2}$/.test(day))
|
|
25
|
+
dates.push(day);
|
|
22
26
|
}
|
|
23
|
-
|
|
27
|
+
// Dated by the newest measurement it summarises rather than by the clock, so
|
|
28
|
+
// the file says how far its data reaches and a re-derivation over unchanged
|
|
29
|
+
// measurements produces no diff. `buildToolShapeBaseline` still defaults to
|
|
30
|
+
// today when nobody supplies one, for callers with no measurement dates.
|
|
31
|
+
const generatedAt = dates.sort().pop();
|
|
32
|
+
const baseline = buildToolShapeBaseline(tools, {
|
|
33
|
+
serverCount,
|
|
34
|
+
methodologyVersion: METHODOLOGY_VERSION,
|
|
35
|
+
...(generatedAt ? { generatedAt } : {}),
|
|
36
|
+
});
|
|
24
37
|
writeFileSync(join(root, 'results', 'tool-shape.json'), JSON.stringify(baseline, null, 2) + '\n');
|
|
25
38
|
return baseline;
|
|
26
39
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mcp-context-cost",
|
|
3
|
-
"version": "0.13.
|
|
3
|
+
"version": "0.13.2",
|
|
4
4
|
"description": "Measure what your MCP servers cost in context tokens — audit your own config, or badge the server you publish",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -43,7 +43,9 @@
|
|
|
43
43
|
"session-start": "tsx src/sweep/session-start.ts",
|
|
44
44
|
"sweep": "tsx src/sweep/run.ts",
|
|
45
45
|
"sweep:all": "tsx src/sweep/sweep-all.ts",
|
|
46
|
-
"verify": "tsx src/cli.ts verify"
|
|
46
|
+
"verify": "tsx src/cli.ts verify",
|
|
47
|
+
"check:release": "tsx tools/release-readiness.ts",
|
|
48
|
+
"cut": "tsx tools/cut-changelog.ts"
|
|
47
49
|
},
|
|
48
50
|
"engines": {
|
|
49
51
|
"node": ">=20"
|