mcp-context-cost 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +101 -33
- package/dist/audit/audit.d.ts +48 -0
- package/dist/audit/audit.js +392 -7
- package/dist/audit/config.d.ts +46 -0
- package/dist/audit/config.js +86 -2
- package/dist/audit/deferral.d.ts +366 -0
- package/dist/audit/deferral.js +403 -0
- package/dist/audit/run.d.ts +22 -0
- package/dist/audit/run.js +17 -1
- package/dist/cli.js +11 -0
- package/dist/core/adoption.d.ts +226 -0
- package/dist/core/adoption.js +432 -0
- package/dist/core/canonical.d.ts +6 -0
- package/dist/core/canonical.js +3 -0
- package/dist/core/index.d.ts +1 -0
- package/dist/core/index.js +1 -0
- package/dist/core/session-start.d.ts +102 -0
- package/dist/core/session-start.js +186 -0
- package/dist/core/types.d.ts +8 -0
- package/dist/sweep/client.d.ts +6 -1
- package/dist/sweep/client.js +1 -0
- package/dist/sweep/dashboard.d.ts +10 -0
- package/dist/sweep/dashboard.js +32 -16
- package/dist/sweep/docker.d.ts +23 -0
- package/dist/sweep/docker.js +16 -12
- package/dist/sweep/harness-guard.d.ts +57 -0
- package/dist/sweep/harness-guard.js +144 -0
- package/dist/sweep/history.d.ts +33 -1
- package/dist/sweep/history.js +60 -5
- package/dist/sweep/regen.js +6 -1
- package/dist/sweep/report.d.ts +16 -0
- package/dist/sweep/report.js +79 -5
- package/dist/sweep/run.d.ts +41 -0
- package/dist/sweep/run.js +129 -36
- package/dist/sweep/server-pages.js +31 -6
- package/dist/sweep/session-start.d.ts +3 -0
- package/dist/sweep/session-start.js +103 -0
- package/dist/sweep/shard.d.ts +41 -0
- package/dist/sweep/shard.js +58 -0
- package/dist/sweep/sweep-all.js +56 -2
- package/package.json +3 -1
package/dist/core/canonical.js
CHANGED
|
@@ -52,6 +52,9 @@ export function measureTools(tools, meta) {
|
|
|
52
52
|
serverVersion: meta.serverVersion,
|
|
53
53
|
launchCommand: meta.launchCommand,
|
|
54
54
|
envVarNames: meta.envVarNames,
|
|
55
|
+
// Left undefined (and so absent from the JSON) when the caller had nothing
|
|
56
|
+
// to record, which is exactly how a pre-field measurement reads.
|
|
57
|
+
serverInstructions: meta.instructions,
|
|
55
58
|
};
|
|
56
59
|
}
|
|
57
60
|
export function failedMeasurement(status, meta) {
|
package/dist/core/index.d.ts
CHANGED
package/dist/core/index.js
CHANGED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import type { Measurement } from './types.js';
|
|
2
|
+
/** Method identifier, versioned independently of METHODOLOGY_VERSION. */
|
|
3
|
+
export declare const SESSION_START_METHOD = "deferred-load/v1";
|
|
4
|
+
/**
|
|
5
|
+
* Tool names in server-returned order. A tool without a usable name is dropped
|
|
6
|
+
* rather than given a placeholder — the same rule `toAnthropicTools` follows,
|
|
7
|
+
* for the same reason: an invented name would change the count being published.
|
|
8
|
+
*/
|
|
9
|
+
export declare function toolNames(raw: unknown[]): string[];
|
|
10
|
+
/**
|
|
11
|
+
* o200k count of the canonical tool-name list: `JSON.stringify` over the name
|
|
12
|
+
* array, the same serialization discipline as the headline number, so both
|
|
13
|
+
* re-derive from the same capture with the same five lines.
|
|
14
|
+
*/
|
|
15
|
+
export declare function toolNameTokens(raw: unknown[]): number;
|
|
16
|
+
/**
|
|
17
|
+
* The two halves are counted separately and added, rather than counted over one
|
|
18
|
+
* concatenated string. Concatenation would let boundary tokens merge across the
|
|
19
|
+
* seam, so the published total would not equal the sum of the parts printed
|
|
20
|
+
* beside it — a discrepancy of a token or two that no reader could account for.
|
|
21
|
+
*/
|
|
22
|
+
export declare function sessionStartTokens(raw: unknown[], instructions: string | null): number;
|
|
23
|
+
/** One server's instructions, captured beside — not inside — a measurement. */
|
|
24
|
+
export interface SessionStartRow {
|
|
25
|
+
/** Exactly what `initialize` returned; '' when the server returned none. */
|
|
26
|
+
instructions: string;
|
|
27
|
+
instructionsTokens: number;
|
|
28
|
+
/** SHA-256 of the instructions bytes — the dispute artifact, as ever. */
|
|
29
|
+
instructionsSha256: string;
|
|
30
|
+
/**
|
|
31
|
+
* `canonicalSha256` of the measurement this capture stood beside. A re-sweep
|
|
32
|
+
* moves that hash, which marks the row stale: instructions are a property of
|
|
33
|
+
* the same server build that produced the tools, so a changed tool set is
|
|
34
|
+
* reason enough to stop trusting the instructions captured with the old one.
|
|
35
|
+
*/
|
|
36
|
+
capturedSha256: string | null;
|
|
37
|
+
serverVersion?: string;
|
|
38
|
+
/** Set when the server could not be reached; no numbers are published. */
|
|
39
|
+
error?: string;
|
|
40
|
+
}
|
|
41
|
+
export interface SessionStartRun {
|
|
42
|
+
method: string;
|
|
43
|
+
/** UTC day the instructions were captured (YYYY-MM-DD). */
|
|
44
|
+
measuredAt: string;
|
|
45
|
+
/** How the servers were isolated during the capture. */
|
|
46
|
+
isolation?: string;
|
|
47
|
+
servers: Record<string, SessionStartRow>;
|
|
48
|
+
}
|
|
49
|
+
/** Where the instructions half of a figure came from — or that it is missing. */
|
|
50
|
+
export type InstructionsSource = 'measurement' | 'capture' | 'not-captured';
|
|
51
|
+
export interface SessionStartLoad {
|
|
52
|
+
toolCount: number;
|
|
53
|
+
toolNameTokens: number;
|
|
54
|
+
/** null when instructions have never been captured for this server. */
|
|
55
|
+
instructionsTokens: number | null;
|
|
56
|
+
/** Names plus instructions; names alone when instructions are unknown. */
|
|
57
|
+
totalTokens: number;
|
|
58
|
+
/** True when `totalTokens` is a lower bound, not a measurement. */
|
|
59
|
+
isFloor: boolean;
|
|
60
|
+
instructionsSource: InstructionsSource;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* The instructions recorded *inside* a measurement, distinguishing three states
|
|
64
|
+
* that JSON round-trips faithfully:
|
|
65
|
+
* - a string → captured, the server sent this
|
|
66
|
+
* - null → captured, the server sent none
|
|
67
|
+
* - undefined → never captured (every measurement predating the field)
|
|
68
|
+
*
|
|
69
|
+
* Absent-means-unknown rather than absent-means-zero, on the precedent set by
|
|
70
|
+
* history.csv's `isolation` column: an old row reads as unknown and is never
|
|
71
|
+
* back-filled from a value it did not record.
|
|
72
|
+
*/
|
|
73
|
+
export declare function measuredInstructions(m: Measurement): string | undefined;
|
|
74
|
+
/**
|
|
75
|
+
* Resolve one server's session-start load from its measurement and, if it has
|
|
76
|
+
* one, the instructions captured beside it.
|
|
77
|
+
*
|
|
78
|
+
* Precedence is measurement over capture and never the other way round: the
|
|
79
|
+
* measurement's instructions came off the same server process as its tools, in
|
|
80
|
+
* the same run, so it cannot be stale relative to itself. The side capture is
|
|
81
|
+
* the backfill for measurements taken before the field existed, and it is used
|
|
82
|
+
* only while it still points at the measurement on disk.
|
|
83
|
+
*
|
|
84
|
+
* Returns null for a measurement with no capture to read names from — a
|
|
85
|
+
* `startup-failure` has no session-start load because it has no session.
|
|
86
|
+
*/
|
|
87
|
+
export declare function sessionStartLoad(m: Measurement, row?: SessionStartRow): SessionStartLoad | null;
|
|
88
|
+
/**
|
|
89
|
+
* A side capture is usable only if it carries a number and still points at the
|
|
90
|
+
* measurement on disk. Unlike a stale divergence row — which is hidden, because
|
|
91
|
+
* there is nothing else to print — a stale row here degrades the figure to its
|
|
92
|
+
* names-only floor. The column never blanks; it only ever stops claiming to
|
|
93
|
+
* know the half it no longer knows.
|
|
94
|
+
*/
|
|
95
|
+
export declare function isCurrentInstructions(row: SessionStartRow | undefined, canonicalSha256: string | null): row is SessionStartRow;
|
|
96
|
+
/** Build a row from a freshly captured instructions string. */
|
|
97
|
+
export declare function toSessionStartRow(instructions: string | null, meta: {
|
|
98
|
+
capturedSha256: string | null;
|
|
99
|
+
serverVersion?: string;
|
|
100
|
+
}): SessionStartRow;
|
|
101
|
+
/** Parse results/session-start.json; anything malformed yields null, never throws. */
|
|
102
|
+
export declare function parseSessionStart(text: string): SessionStartRun | null;
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Session-start load — what a client actually puts in context when it defers
|
|
3
|
+
* tool definitions until they are used.
|
|
4
|
+
*
|
|
5
|
+
* The headline number counts every byte of `tools/list`, because that is what a
|
|
6
|
+
* client loading definitions eagerly pays. Clients increasingly do not: they put
|
|
7
|
+
* a *list of names* in context and fetch the definition only when the model
|
|
8
|
+
* reaches for the tool. That client pays a different bill at session start, and
|
|
9
|
+
* the headline number says nothing about it.
|
|
10
|
+
*
|
|
11
|
+
* Different, and usually far smaller — but *not* smaller by construction, and
|
|
12
|
+
* nothing here may claim it is. Only one of the two halves below is bounded by
|
|
13
|
+
* the definitions being deferred. Instructions are separate bytes that no part
|
|
14
|
+
* of the headline counts, and their length has nothing to do with the size of
|
|
15
|
+
* the tool set, so a server that re-lists its tools in its `instructions`
|
|
16
|
+
* charges a deferring client *more* than an eager one. That is not
|
|
17
|
+
* hypothetical: it is true of a server on the published leaderboard, which is
|
|
18
|
+
* the most useful thing this measurement has found, and every renderer of these
|
|
19
|
+
* numbers is expected to let the reader see it rather than smooth it away.
|
|
20
|
+
*
|
|
21
|
+
* Two quantities make up that bill, and only one of them is in the published
|
|
22
|
+
* capture:
|
|
23
|
+
*
|
|
24
|
+
* 1. **Tool names.** Derivable from `rawToolsCapture` — no re-measurement
|
|
25
|
+
* needed, and re-derivable by anyone from the same published bytes.
|
|
26
|
+
* 2. **Server instructions.** The `instructions` string a server returns from
|
|
27
|
+
* `initialize`. It is *not* part of `tools/list` and so is absent from every
|
|
28
|
+
* measurement recorded before this field existed.
|
|
29
|
+
*
|
|
30
|
+
* Which means a row can be in one of two honest states, and they are not the
|
|
31
|
+
* same claim: an exact figure (both halves known), or a **floor** (names
|
|
32
|
+
* measured, instructions never captured). A floor published as a figure is the
|
|
33
|
+
* failure mode this module exists to prevent — see `isFloor`, which every
|
|
34
|
+
* renderer must carry through to the reader.
|
|
35
|
+
*
|
|
36
|
+
* Versioned independently of the o200k methodology, on the same reasoning as
|
|
37
|
+
* `tools-delta/v1`: this adds a second published number and does not touch the
|
|
38
|
+
* definition of the first. No `totalTokens` and no canonical hash moves.
|
|
39
|
+
*/
|
|
40
|
+
import { countTokens, sha256Hex } from './canonical.js';
|
|
41
|
+
/** Method identifier, versioned independently of METHODOLOGY_VERSION. */
|
|
42
|
+
export const SESSION_START_METHOD = 'deferred-load/v1';
|
|
43
|
+
/**
|
|
44
|
+
* Tool names in server-returned order. A tool without a usable name is dropped
|
|
45
|
+
* rather than given a placeholder — the same rule `toAnthropicTools` follows,
|
|
46
|
+
* for the same reason: an invented name would change the count being published.
|
|
47
|
+
*/
|
|
48
|
+
export function toolNames(raw) {
|
|
49
|
+
const out = [];
|
|
50
|
+
for (const t of raw) {
|
|
51
|
+
const name = (t ?? {}).name;
|
|
52
|
+
if (typeof name === 'string' && name !== '')
|
|
53
|
+
out.push(name);
|
|
54
|
+
}
|
|
55
|
+
return out;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* o200k count of the canonical tool-name list: `JSON.stringify` over the name
|
|
59
|
+
* array, the same serialization discipline as the headline number, so both
|
|
60
|
+
* re-derive from the same capture with the same five lines.
|
|
61
|
+
*/
|
|
62
|
+
export function toolNameTokens(raw) {
|
|
63
|
+
return countTokens(JSON.stringify(toolNames(raw)));
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* The two halves are counted separately and added, rather than counted over one
|
|
67
|
+
* concatenated string. Concatenation would let boundary tokens merge across the
|
|
68
|
+
* seam, so the published total would not equal the sum of the parts printed
|
|
69
|
+
* beside it — a discrepancy of a token or two that no reader could account for.
|
|
70
|
+
*/
|
|
71
|
+
export function sessionStartTokens(raw, instructions) {
|
|
72
|
+
return toolNameTokens(raw) + (instructions ? countTokens(instructions) : 0);
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* The instructions recorded *inside* a measurement, distinguishing three states
|
|
76
|
+
* that JSON round-trips faithfully:
|
|
77
|
+
* - a string → captured, the server sent this
|
|
78
|
+
* - null → captured, the server sent none
|
|
79
|
+
* - undefined → never captured (every measurement predating the field)
|
|
80
|
+
*
|
|
81
|
+
* Absent-means-unknown rather than absent-means-zero, on the precedent set by
|
|
82
|
+
* history.csv's `isolation` column: an old row reads as unknown and is never
|
|
83
|
+
* back-filled from a value it did not record.
|
|
84
|
+
*/
|
|
85
|
+
export function measuredInstructions(m) {
|
|
86
|
+
if (typeof m.serverInstructions === 'string')
|
|
87
|
+
return m.serverInstructions;
|
|
88
|
+
if (m.serverInstructions === null)
|
|
89
|
+
return '';
|
|
90
|
+
return undefined;
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Resolve one server's session-start load from its measurement and, if it has
|
|
94
|
+
* one, the instructions captured beside it.
|
|
95
|
+
*
|
|
96
|
+
* Precedence is measurement over capture and never the other way round: the
|
|
97
|
+
* measurement's instructions came off the same server process as its tools, in
|
|
98
|
+
* the same run, so it cannot be stale relative to itself. The side capture is
|
|
99
|
+
* the backfill for measurements taken before the field existed, and it is used
|
|
100
|
+
* only while it still points at the measurement on disk.
|
|
101
|
+
*
|
|
102
|
+
* Returns null for a measurement with no capture to read names from — a
|
|
103
|
+
* `startup-failure` has no session-start load because it has no session.
|
|
104
|
+
*/
|
|
105
|
+
export function sessionStartLoad(m, row) {
|
|
106
|
+
if (!Array.isArray(m.rawToolsCapture))
|
|
107
|
+
return null;
|
|
108
|
+
const names = toolNameTokens(m.rawToolsCapture);
|
|
109
|
+
const toolCount = toolNames(m.rawToolsCapture).length;
|
|
110
|
+
const inMeasurement = measuredInstructions(m);
|
|
111
|
+
if (inMeasurement !== undefined) {
|
|
112
|
+
const t = inMeasurement ? countTokens(inMeasurement) : 0;
|
|
113
|
+
return {
|
|
114
|
+
toolCount,
|
|
115
|
+
toolNameTokens: names,
|
|
116
|
+
instructionsTokens: t,
|
|
117
|
+
totalTokens: names + t,
|
|
118
|
+
isFloor: false,
|
|
119
|
+
instructionsSource: 'measurement',
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
if (isCurrentInstructions(row, m.canonicalSha256)) {
|
|
123
|
+
return {
|
|
124
|
+
toolCount,
|
|
125
|
+
toolNameTokens: names,
|
|
126
|
+
instructionsTokens: row.instructionsTokens,
|
|
127
|
+
totalTokens: names + row.instructionsTokens,
|
|
128
|
+
isFloor: false,
|
|
129
|
+
instructionsSource: 'capture',
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
return {
|
|
133
|
+
toolCount,
|
|
134
|
+
toolNameTokens: names,
|
|
135
|
+
instructionsTokens: null,
|
|
136
|
+
totalTokens: names,
|
|
137
|
+
isFloor: true,
|
|
138
|
+
instructionsSource: 'not-captured',
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* A side capture is usable only if it carries a number and still points at the
|
|
143
|
+
* measurement on disk. Unlike a stale divergence row — which is hidden, because
|
|
144
|
+
* there is nothing else to print — a stale row here degrades the figure to its
|
|
145
|
+
* names-only floor. The column never blanks; it only ever stops claiming to
|
|
146
|
+
* know the half it no longer knows.
|
|
147
|
+
*/
|
|
148
|
+
export function isCurrentInstructions(row, canonicalSha256) {
|
|
149
|
+
if (!row || row.error)
|
|
150
|
+
return false;
|
|
151
|
+
if (typeof row.instructionsTokens !== 'number')
|
|
152
|
+
return false;
|
|
153
|
+
return !!canonicalSha256 && row.capturedSha256 === canonicalSha256;
|
|
154
|
+
}
|
|
155
|
+
/** Build a row from a freshly captured instructions string. */
|
|
156
|
+
export function toSessionStartRow(instructions, meta) {
|
|
157
|
+
const text = instructions ?? '';
|
|
158
|
+
return {
|
|
159
|
+
instructions: text,
|
|
160
|
+
instructionsTokens: text ? countTokens(text) : 0,
|
|
161
|
+
instructionsSha256: sha256Hex(text),
|
|
162
|
+
capturedSha256: meta.capturedSha256,
|
|
163
|
+
serverVersion: meta.serverVersion,
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
/** Parse results/session-start.json; anything malformed yields null, never throws. */
|
|
167
|
+
export function parseSessionStart(text) {
|
|
168
|
+
let run;
|
|
169
|
+
try {
|
|
170
|
+
run = JSON.parse(text);
|
|
171
|
+
}
|
|
172
|
+
catch {
|
|
173
|
+
return null;
|
|
174
|
+
}
|
|
175
|
+
const r = run;
|
|
176
|
+
if (!r || typeof r.measuredAt !== 'string')
|
|
177
|
+
return null;
|
|
178
|
+
if (!r.servers || typeof r.servers !== 'object')
|
|
179
|
+
return null;
|
|
180
|
+
return {
|
|
181
|
+
method: typeof r.method === 'string' ? r.method : SESSION_START_METHOD,
|
|
182
|
+
measuredAt: r.measuredAt,
|
|
183
|
+
isolation: typeof r.isolation === 'string' ? r.isolation : undefined,
|
|
184
|
+
servers: r.servers,
|
|
185
|
+
};
|
|
186
|
+
}
|
package/dist/core/types.d.ts
CHANGED
|
@@ -23,6 +23,14 @@ export interface Measurement {
|
|
|
23
23
|
canonicalSha256: string | null;
|
|
24
24
|
/** The tools array exactly as returned by tools/list, all pages concatenated. */
|
|
25
25
|
rawToolsCapture: unknown[] | null;
|
|
26
|
+
/**
|
|
27
|
+
* The `instructions` string returned by `initialize` — half of the
|
|
28
|
+
* session-start load (see core/session-start.ts). Three states, and they are
|
|
29
|
+
* different claims: a string (the server sent this), `null` (the server sent
|
|
30
|
+
* none), absent (never captured — every measurement predating this field).
|
|
31
|
+
* Absent is never read as zero.
|
|
32
|
+
*/
|
|
33
|
+
serverInstructions?: string | null;
|
|
26
34
|
measuredAt: string;
|
|
27
35
|
serverName: string;
|
|
28
36
|
serverVersion?: string;
|
package/dist/sweep/client.d.ts
CHANGED
|
@@ -5,7 +5,12 @@ export interface WireCapture {
|
|
|
5
5
|
};
|
|
6
6
|
protocolVersion?: string;
|
|
7
7
|
tools: unknown[];
|
|
8
|
-
/**
|
|
8
|
+
/**
|
|
9
|
+
* The `instructions` string from the initialize result — half of the
|
|
10
|
+
* session-start load. `null` means the server returned none, which is a
|
|
11
|
+
* different fact from never having asked, so the two are never conflated.
|
|
12
|
+
*/
|
|
13
|
+
instructions: string | null;
|
|
9
14
|
stderrTail: string;
|
|
10
15
|
}
|
|
11
16
|
export declare class McpStdioClient {
|
package/dist/sweep/client.js
CHANGED
|
@@ -164,6 +164,7 @@ export async function captureTools(spec, opts = {}) {
|
|
|
164
164
|
serverInfo: init?.serverInfo,
|
|
165
165
|
protocolVersion: init?.protocolVersion,
|
|
166
166
|
tools,
|
|
167
|
+
instructions: typeof init?.instructions === 'string' ? init.instructions : null,
|
|
167
168
|
stderrTail: client.stderrTail,
|
|
168
169
|
};
|
|
169
170
|
}
|
|
@@ -7,3 +7,13 @@
|
|
|
7
7
|
*/
|
|
8
8
|
export declare function renderSparkline(tokens: number[]): string;
|
|
9
9
|
export declare function generateDashboard(root?: string): string;
|
|
10
|
+
/**
|
|
11
|
+
* Render and write the dashboard. Exported so `regen.ts` refreshes it alongside
|
|
12
|
+
* the leaderboard and the server pages: this is the published Pages surface, and
|
|
13
|
+
* when only the CLI wrote it every sweep updated `results/` and `docs/servers/`
|
|
14
|
+
* while the page people actually open kept the previous run's numbers.
|
|
15
|
+
*/
|
|
16
|
+
export declare function writeDashboard(root?: string, out?: string): {
|
|
17
|
+
out: string;
|
|
18
|
+
bytes: number;
|
|
19
|
+
};
|
package/dist/sweep/dashboard.js
CHANGED
|
@@ -7,7 +7,7 @@ import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'node:fs';
|
|
|
7
7
|
import { dirname, join } from 'node:path';
|
|
8
8
|
import { parse } from 'yaml';
|
|
9
9
|
import { bandColor, BAND_META } from '../core/bands.js';
|
|
10
|
-
import { parseHistory } from './history.js';
|
|
10
|
+
import { parseHistory, plottableSeries } from './history.js';
|
|
11
11
|
/** Longest series a sparkline plots — a stat-tile trend, not a full chart. */
|
|
12
12
|
const SPARK_MAX_POINTS = 12;
|
|
13
13
|
/**
|
|
@@ -49,10 +49,9 @@ export function generateDashboard(root = process.cwd()) {
|
|
|
49
49
|
const dSrv = divergence.servers ?? {};
|
|
50
50
|
const historyPath = join(root, 'results', 'history.csv');
|
|
51
51
|
const history = existsSync(historyPath) ? parseHistory(readFileSync(historyPath, 'utf8')) : [];
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
.map((h) => h.tokens);
|
|
52
|
+
// Only the run of sweeps taken under the same isolation is plotted: a step
|
|
53
|
+
// across an isolation change is the harness moving, not the server.
|
|
54
|
+
const seriesFor = (name) => plottableSeries(history.filter((h) => h.server === name));
|
|
56
55
|
const rows = doc.servers.map((entry) => {
|
|
57
56
|
const p = join(root, 'results', entry.name, 'measurement.json');
|
|
58
57
|
return { entry, m: existsSync(p) ? JSON.parse(readFileSync(p, 'utf8')) : null };
|
|
@@ -78,8 +77,16 @@ export function generateDashboard(root = process.cwd()) {
|
|
|
78
77
|
const div = dSrv[r.entry.name];
|
|
79
78
|
const claudeTip = div ? ` · in a Claude request: ${fmt(div.claudeDelta)} tok` : '';
|
|
80
79
|
const series = seriesFor(r.entry.name);
|
|
81
|
-
const
|
|
82
|
-
const
|
|
80
|
+
const tokens = series.rows.map((h) => h.tokens);
|
|
81
|
+
const spark = renderSparkline(tokens);
|
|
82
|
+
const trendTip = tokens.length > 1
|
|
83
|
+
? ` · ${tokens.length}-sweep trend: ${tokens[0].toLocaleString('en-US')} → ${tokens[tokens.length - 1].toLocaleString('en-US')}` +
|
|
84
|
+
(series.dropped
|
|
85
|
+
? ` (${series.dropped} earlier sweep${series.dropped === 1 ? '' : 's'} measured under different isolation, not plotted)`
|
|
86
|
+
: '')
|
|
87
|
+
: series.dropped
|
|
88
|
+
? ` · earlier sweeps were measured under different isolation, so no trend is plotted`
|
|
89
|
+
: '';
|
|
83
90
|
// The whole row is the link to the server's detail page (docs/servers/).
|
|
84
91
|
return `<a class="row" href="servers/${encodeURIComponent(r.entry.name)}.html" data-tip="${esc(m.toolCount)} tools · largest: ${esc(largest?.name)} (${fmt(largest?.tokens ?? 0)} tok)${claudeTip}${trendTip} · ${esc(r.entry.category)} · ${esc(m.status)}${m.serverVersion ? ' · v' + esc(String(m.serverVersion).replace(/^v/, '')) : ''}">
|
|
85
92
|
<span class="rank">${i + 1}</span>
|
|
@@ -101,9 +108,9 @@ export function generateDashboard(root = process.cwd()) {
|
|
|
101
108
|
.map((r, i) => {
|
|
102
109
|
const m = r.m;
|
|
103
110
|
const div = dSrv[r.entry.name];
|
|
104
|
-
const
|
|
105
|
-
const trend =
|
|
106
|
-
? `${
|
|
111
|
+
const tokens = seriesFor(r.entry.name).rows.map((h) => h.tokens);
|
|
112
|
+
const trend = tokens.length > 1
|
|
113
|
+
? `${tokens[tokens.length - 1] - tokens[0] >= 0 ? '+' : ''}${fmt(tokens[tokens.length - 1] - tokens[0])} / ${tokens.length}d`
|
|
107
114
|
: '—';
|
|
108
115
|
return `<tr><td>${i + 1}</td><td>${esc(r.entry.name)}</td><td class="num">${fmt(m.totalTokens)}</td><td class="num">${div ? fmt(div.claudeDelta) : '—'}</td><td class="num">${esc(m.toolCount)}</td><td>${esc(BAND_META[bandColor(m.totalTokens)].label)}</td><td>${esc(r.entry.category)}</td><td class="num">${trend}</td></tr>`;
|
|
109
116
|
})
|
|
@@ -220,7 +227,7 @@ export function generateDashboard(root = process.cwd()) {
|
|
|
220
227
|
</div>
|
|
221
228
|
|
|
222
229
|
<h2>Leaderboard</h2>
|
|
223
|
-
<p class="h2sub">Tokens = o200k_base count of the canonical <code>tools/list</code> bytes — the wire payload. What a <em>Claude request</em> actually carries can differ sharply (github: 54,422 on the wire, ${dSrv['github'] ? fmt(dSrv['github'].claudeDelta) : '…'} in a request — 80% of its schema bytes are fields no Anthropic request sends). The trend line plots tokens across every sweep date on record (oldest→newest, dot = current); servers with one sweep so far show no line yet. Hover a row for exact numbers; <a href="METHODOLOGY.html#claude-divergence">method</a>.</p>
|
|
230
|
+
<p class="h2sub">Tokens = o200k_base count of the canonical <code>tools/list</code> bytes — the wire payload. What a <em>Claude request</em> actually carries can differ sharply (github: 54,422 on the wire, ${dSrv['github'] ? fmt(dSrv['github'].claudeDelta) : '…'} in a request — 80% of its schema bytes are fields no Anthropic request sends). The trend line plots tokens across every sweep date on record (oldest→newest, dot = current); servers with one sweep so far show no line yet, and a sweep taken under different isolation than the current one is left out rather than drawn as a change in the server. Hover a row for exact numbers; <a href="METHODOLOGY.html#claude-divergence">method</a>.</p>
|
|
224
231
|
<div class="board">
|
|
225
232
|
${barRows || '<p class="h2sub">Sweep in progress — first results land shortly.</p>'}
|
|
226
233
|
</div>
|
|
@@ -282,12 +289,21 @@ ${barRows || '<p class="h2sub">Sweep in progress — first results land shortly.
|
|
|
282
289
|
</script>
|
|
283
290
|
`;
|
|
284
291
|
}
|
|
292
|
+
/**
|
|
293
|
+
* Render and write the dashboard. Exported so `regen.ts` refreshes it alongside
|
|
294
|
+
* the leaderboard and the server pages: this is the published Pages surface, and
|
|
295
|
+
* when only the CLI wrote it every sweep updated `results/` and `docs/servers/`
|
|
296
|
+
* while the page people actually open kept the previous run's numbers.
|
|
297
|
+
*/
|
|
298
|
+
export function writeDashboard(root = process.cwd(), out = 'docs/dashboard.html') {
|
|
299
|
+
const html = generateDashboard(root);
|
|
300
|
+
mkdirSync(dirname(out), { recursive: true });
|
|
301
|
+
writeFileSync(out, html);
|
|
302
|
+
return { out, bytes: html.length };
|
|
303
|
+
}
|
|
285
304
|
const isMain = process.argv[1]?.endsWith('dashboard.ts') || process.argv[1]?.endsWith('dashboard.js');
|
|
286
305
|
if (isMain) {
|
|
287
306
|
const i = process.argv.indexOf('--out');
|
|
288
|
-
const
|
|
289
|
-
|
|
290
|
-
mkdirSync(dirname(out), { recursive: true });
|
|
291
|
-
writeFileSync(out, html);
|
|
292
|
-
console.log(`wrote ${out} (${(html.length / 1024).toFixed(0)}KB)`);
|
|
307
|
+
const w = writeDashboard(process.cwd(), i >= 0 ? process.argv[i + 1] : 'docs/dashboard.html');
|
|
308
|
+
console.log(`wrote ${w.out} (${(w.bytes / 1024).toFixed(0)}KB)`);
|
|
293
309
|
}
|
package/dist/sweep/docker.d.ts
CHANGED
|
@@ -10,6 +10,16 @@ export interface DockerOptions {
|
|
|
10
10
|
image?: string;
|
|
11
11
|
/** Extra env var NAMES to pass through with dummy values. */
|
|
12
12
|
dummyEnv?: string[];
|
|
13
|
+
/**
|
|
14
|
+
* Override the literal `dummy` value for specific `dummyEnv` names. Some
|
|
15
|
+
* servers parse an env var's shape before ever reaching tools/list (a URI
|
|
16
|
+
* scheme, a URL) and crash on a value that isn't real-looking rather than
|
|
17
|
+
* a value that isn't a real credential — e.g. `NEO4J_URI=dummy` fails
|
|
18
|
+
* `neo4j.exceptions.ConfigurationError` before the driver is even asked to
|
|
19
|
+
* connect. A locally-scoped placeholder (`bolt://localhost:7687`) clears
|
|
20
|
+
* that parse step without providing a working credential.
|
|
21
|
+
*/
|
|
22
|
+
dummyEnvValues?: Record<string, string>;
|
|
13
23
|
/** Container name, so a timed-out container can be force-removed. */
|
|
14
24
|
containerName?: string;
|
|
15
25
|
/**
|
|
@@ -20,6 +30,19 @@ export interface DockerOptions {
|
|
|
20
30
|
* git already on PATH would actually run.
|
|
21
31
|
*/
|
|
22
32
|
needsGit?: boolean;
|
|
33
|
+
/**
|
|
34
|
+
* Skip the shared npm/uv cache volumes, paying a cold install for a clean one.
|
|
35
|
+
*
|
|
36
|
+
* The caches are what make a re-sweep minutes instead of hours, but a cache
|
|
37
|
+
* entry can go bad and stay bad: npm remembers a failed postinstall, and every
|
|
38
|
+
* later install that resolves through it fails the same way. Observed
|
|
39
|
+
* 2026-08-19 — a cached `esbuild@0.25.12` whose postinstall exits 1 made
|
|
40
|
+
* `npx -y @shopify/dev-mcp@latest` exit 1 with no output, against a server that
|
|
41
|
+
* installs and answers fine from an empty cache. Nothing in the exit code tells
|
|
42
|
+
* that apart from a genuinely broken server, so the sweep retries here rather
|
|
43
|
+
* than publishing the downgrade.
|
|
44
|
+
*/
|
|
45
|
+
noSharedCache?: boolean;
|
|
23
46
|
}
|
|
24
47
|
export declare const DEFAULT_NODE_IMAGE = "public.ecr.aws/docker/library/node:22-slim";
|
|
25
48
|
export declare const DEFAULT_PYTHON_IMAGE = "ghcr.io/astral-sh/uv:python3.12-bookworm-slim";
|
package/dist/sweep/docker.js
CHANGED
|
@@ -23,17 +23,21 @@ export function dockerize(commandLine, opts = {}) {
|
|
|
23
23
|
'-w',
|
|
24
24
|
'/tmp',
|
|
25
25
|
// Shared package caches across containers — packages only, no credentials.
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
26
|
+
...(opts.noSharedCache
|
|
27
|
+
? []
|
|
28
|
+
: [
|
|
29
|
+
'-v',
|
|
30
|
+
'mcp-ctx-npm-cache:/tmp/.npm-cache',
|
|
31
|
+
'-e',
|
|
32
|
+
'npm_config_cache=/tmp/.npm-cache',
|
|
33
|
+
'-v',
|
|
34
|
+
'mcp-ctx-uv-cache:/tmp/.uv-cache',
|
|
35
|
+
'-e',
|
|
36
|
+
'UV_CACHE_DIR=/tmp/.uv-cache',
|
|
37
|
+
]),
|
|
34
38
|
];
|
|
35
39
|
for (const name of opts.dummyEnv ?? []) {
|
|
36
|
-
argv.push('-e', `${name}
|
|
40
|
+
argv.push('-e', `${name}=${opts.dummyEnvValues?.[name] ?? 'dummy'}`);
|
|
37
41
|
}
|
|
38
42
|
const gitPrefix = opts.needsGit
|
|
39
43
|
? 'command -v git >/dev/null 2>&1 || (apt-get update -qq && apt-get install -y -qq --no-install-recommends git >/dev/null 2>&1); '
|
|
@@ -46,9 +50,9 @@ export function dockerize(commandLine, opts = {}) {
|
|
|
46
50
|
docker: true,
|
|
47
51
|
image,
|
|
48
52
|
network: 'bridge',
|
|
49
|
-
note:
|
|
50
|
-
? '
|
|
51
|
-
|
|
53
|
+
note: 'network enabled for package fetch; clean FS, no host credentials' +
|
|
54
|
+
(opts.needsGit ? ', git installed' : '') +
|
|
55
|
+
(opts.noSharedCache ? ', shared package cache bypassed' : ''),
|
|
52
56
|
},
|
|
53
57
|
};
|
|
54
58
|
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import type { MeasurementStatus } from '../core/types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Below this many regressions the population signal doesn't exist yet, and a
|
|
4
|
+
* deliberately narrow sweep (`--only redis,serena`, two servers known to be
|
|
5
|
+
* shaky) must never be able to trip a fault by failing completely.
|
|
6
|
+
*/
|
|
7
|
+
export declare const MIN_REGRESSIONS = 5;
|
|
8
|
+
/**
|
|
9
|
+
* Share of the previously-good servers in this sweep that must regress before
|
|
10
|
+
* the harness is the likelier explanation. The largest genuine simultaneous
|
|
11
|
+
* upstream breakage on record here is a handful of PyPI servers that shared
|
|
12
|
+
* one unbounded dependency — single digits against ~65 measured, well under
|
|
13
|
+
* 10%. A harness fault, by contrast, takes everything it touches: the Docker
|
|
14
|
+
* outage above scored 100%. A majority sits far from the former and catches
|
|
15
|
+
* every instance of the latter this project has actually seen.
|
|
16
|
+
*/
|
|
17
|
+
export declare const FAULT_RATIO = 0.5;
|
|
18
|
+
/** Statuses that represent a real number on record — the thing worth protecting. */
|
|
19
|
+
export declare function isGood(status: MeasurementStatus): boolean;
|
|
20
|
+
/** The bytes of one server's published artifacts, exactly as they were pre-sweep. */
|
|
21
|
+
export interface Snapshot {
|
|
22
|
+
name: string;
|
|
23
|
+
status: MeasurementStatus | null;
|
|
24
|
+
/** Raw file contents, so a restore is byte-identical rather than re-serialized. */
|
|
25
|
+
measurementJson: string | null;
|
|
26
|
+
badgeJson: string | null;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Read what is currently published for each server. Called before the sweep
|
|
30
|
+
* starts; a server with nothing on record snapshots as `null` status, which no
|
|
31
|
+
* verdict counts either way.
|
|
32
|
+
*/
|
|
33
|
+
export declare function snapshot(names: string[], root?: string): Snapshot[];
|
|
34
|
+
export interface Verdict {
|
|
35
|
+
/** True when the sweep looks like a broken harness rather than broken servers. */
|
|
36
|
+
fault: boolean;
|
|
37
|
+
/** Servers that went from a real number to a failure in this sweep. */
|
|
38
|
+
regressed: string[];
|
|
39
|
+
/** How many servers had a real number on record and could therefore regress. */
|
|
40
|
+
comparable: number;
|
|
41
|
+
/** Human-readable reason, printed either way — a pass says why it passed. */
|
|
42
|
+
reason: string;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Compare pre-sweep snapshots against this sweep's outcomes.
|
|
46
|
+
*
|
|
47
|
+
* `current` maps server name to the status it just measured at. Servers absent
|
|
48
|
+
* from it were not swept and are ignored.
|
|
49
|
+
*/
|
|
50
|
+
export declare function verdict(prior: Snapshot[], current: Map<string, MeasurementStatus>): Verdict;
|
|
51
|
+
/**
|
|
52
|
+
* Put the snapshotted artifacts back, byte for byte. Only servers named in
|
|
53
|
+
* `names` are touched, and only where a prior file existed — a server whose
|
|
54
|
+
* first-ever measurement failed has nothing to restore and keeps its new
|
|
55
|
+
* (honest) failure record.
|
|
56
|
+
*/
|
|
57
|
+
export declare function restore(prior: Snapshot[], names: string[], root?: string): string[];
|