mcp-context-cost 0.4.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/README.md +101 -33
  2. package/dist/audit/audit.d.ts +48 -0
  3. package/dist/audit/audit.js +392 -7
  4. package/dist/audit/config.d.ts +46 -0
  5. package/dist/audit/config.js +86 -2
  6. package/dist/audit/deferral.d.ts +366 -0
  7. package/dist/audit/deferral.js +403 -0
  8. package/dist/audit/run.d.ts +22 -0
  9. package/dist/audit/run.js +17 -1
  10. package/dist/cli.js +11 -0
  11. package/dist/core/adoption.d.ts +226 -0
  12. package/dist/core/adoption.js +432 -0
  13. package/dist/core/canonical.d.ts +6 -0
  14. package/dist/core/canonical.js +3 -0
  15. package/dist/core/index.d.ts +1 -0
  16. package/dist/core/index.js +1 -0
  17. package/dist/core/session-start.d.ts +102 -0
  18. package/dist/core/session-start.js +186 -0
  19. package/dist/core/types.d.ts +8 -0
  20. package/dist/sweep/client.d.ts +6 -1
  21. package/dist/sweep/client.js +1 -0
  22. package/dist/sweep/dashboard.d.ts +10 -0
  23. package/dist/sweep/dashboard.js +32 -16
  24. package/dist/sweep/docker.d.ts +23 -0
  25. package/dist/sweep/docker.js +16 -12
  26. package/dist/sweep/harness-guard.d.ts +57 -0
  27. package/dist/sweep/harness-guard.js +144 -0
  28. package/dist/sweep/history.d.ts +33 -1
  29. package/dist/sweep/history.js +60 -5
  30. package/dist/sweep/regen.js +6 -1
  31. package/dist/sweep/report.d.ts +16 -0
  32. package/dist/sweep/report.js +79 -5
  33. package/dist/sweep/run.d.ts +41 -0
  34. package/dist/sweep/run.js +129 -36
  35. package/dist/sweep/server-pages.js +31 -6
  36. package/dist/sweep/session-start.d.ts +3 -0
  37. package/dist/sweep/session-start.js +103 -0
  38. package/dist/sweep/shard.d.ts +41 -0
  39. package/dist/sweep/shard.js +58 -0
  40. package/dist/sweep/sweep-all.js +56 -2
  41. package/package.json +3 -1
@@ -52,6 +52,9 @@ export function measureTools(tools, meta) {
52
52
  serverVersion: meta.serverVersion,
53
53
  launchCommand: meta.launchCommand,
54
54
  envVarNames: meta.envVarNames,
55
+ // Left undefined (and so absent from the JSON) when the caller had nothing
56
+ // to record, which is exactly how a pre-field measurement reads.
57
+ serverInstructions: meta.instructions,
55
58
  };
56
59
  }
57
60
  export function failedMeasurement(status, meta) {
@@ -1,6 +1,7 @@
1
1
  export * from './types.js';
2
2
  export * from './canonical.js';
3
3
  export * from './divergence.js';
4
+ export * from './session-start.js';
4
5
  export * from './bands.js';
5
6
  export * from './badge.js';
6
7
  export * from './snippet.js';
@@ -1,6 +1,7 @@
1
1
  export * from './types.js';
2
2
  export * from './canonical.js';
3
3
  export * from './divergence.js';
4
+ export * from './session-start.js';
4
5
  export * from './bands.js';
5
6
  export * from './badge.js';
6
7
  export * from './snippet.js';
@@ -0,0 +1,102 @@
1
+ import type { Measurement } from './types.js';
2
+ /** Method identifier, versioned independently of METHODOLOGY_VERSION. */
3
+ export declare const SESSION_START_METHOD = "deferred-load/v1";
4
+ /**
5
+ * Tool names in server-returned order. A tool without a usable name is dropped
6
+ * rather than given a placeholder — the same rule `toAnthropicTools` follows,
7
+ * for the same reason: an invented name would change the count being published.
8
+ */
9
+ export declare function toolNames(raw: unknown[]): string[];
10
+ /**
11
+ * o200k count of the canonical tool-name list: `JSON.stringify` over the name
12
+ * array, the same serialization discipline as the headline number, so both
13
+ * re-derive from the same capture with the same five lines.
14
+ */
15
+ export declare function toolNameTokens(raw: unknown[]): number;
16
+ /**
17
+ * The two halves are counted separately and added, rather than counted over one
18
+ * concatenated string. Concatenation would let boundary tokens merge across the
19
+ * seam, so the published total would not equal the sum of the parts printed
20
+ * beside it — a discrepancy of a token or two that no reader could account for.
21
+ */
22
+ export declare function sessionStartTokens(raw: unknown[], instructions: string | null): number;
23
+ /** One server's instructions, captured beside — not inside — a measurement. */
24
+ export interface SessionStartRow {
25
+ /** Exactly what `initialize` returned; '' when the server returned none. */
26
+ instructions: string;
27
+ instructionsTokens: number;
28
+ /** SHA-256 of the instructions bytes — the dispute artifact, as ever. */
29
+ instructionsSha256: string;
30
+ /**
31
+ * `canonicalSha256` of the measurement this capture stood beside. A re-sweep
32
+ * moves that hash, which marks the row stale: instructions are a property of
33
+ * the same server build that produced the tools, so a changed tool set is
34
+ * reason enough to stop trusting the instructions captured with the old one.
35
+ */
36
+ capturedSha256: string | null;
37
+ serverVersion?: string;
38
+ /** Set when the server could not be reached; no numbers are published. */
39
+ error?: string;
40
+ }
41
+ export interface SessionStartRun {
42
+ method: string;
43
+ /** UTC day the instructions were captured (YYYY-MM-DD). */
44
+ measuredAt: string;
45
+ /** How the servers were isolated during the capture. */
46
+ isolation?: string;
47
+ servers: Record<string, SessionStartRow>;
48
+ }
49
+ /** Where the instructions half of a figure came from — or that it is missing. */
50
+ export type InstructionsSource = 'measurement' | 'capture' | 'not-captured';
51
+ export interface SessionStartLoad {
52
+ toolCount: number;
53
+ toolNameTokens: number;
54
+ /** null when instructions have never been captured for this server. */
55
+ instructionsTokens: number | null;
56
+ /** Names plus instructions; names alone when instructions are unknown. */
57
+ totalTokens: number;
58
+ /** True when `totalTokens` is a lower bound, not a measurement. */
59
+ isFloor: boolean;
60
+ instructionsSource: InstructionsSource;
61
+ }
62
+ /**
63
+ * The instructions recorded *inside* a measurement, distinguishing three states
64
+ * that JSON round-trips faithfully:
65
+ * - a string → captured, the server sent this
66
+ * - null → captured, the server sent none
67
+ * - undefined → never captured (every measurement predating the field)
68
+ *
69
+ * Absent-means-unknown rather than absent-means-zero, on the precedent set by
70
+ * history.csv's `isolation` column: an old row reads as unknown and is never
71
+ * back-filled from a value it did not record.
72
+ */
73
+ export declare function measuredInstructions(m: Measurement): string | undefined;
74
+ /**
75
+ * Resolve one server's session-start load from its measurement and, if it has
76
+ * one, the instructions captured beside it.
77
+ *
78
+ * Precedence is measurement over capture and never the other way round: the
79
+ * measurement's instructions came off the same server process as its tools, in
80
+ * the same run, so it cannot be stale relative to itself. The side capture is
81
+ * the backfill for measurements taken before the field existed, and it is used
82
+ * only while it still points at the measurement on disk.
83
+ *
84
+ * Returns null for a measurement with no capture to read names from — a
85
+ * `startup-failure` has no session-start load because it has no session.
86
+ */
87
+ export declare function sessionStartLoad(m: Measurement, row?: SessionStartRow): SessionStartLoad | null;
88
+ /**
89
+ * A side capture is usable only if it carries a number and still points at the
90
+ * measurement on disk. Unlike a stale divergence row — which is hidden, because
91
+ * there is nothing else to print — a stale row here degrades the figure to its
92
+ * names-only floor. The column never blanks; it only ever stops claiming to
93
+ * know the half it no longer knows.
94
+ */
95
+ export declare function isCurrentInstructions(row: SessionStartRow | undefined, canonicalSha256: string | null): row is SessionStartRow;
96
+ /** Build a row from a freshly captured instructions string. */
97
+ export declare function toSessionStartRow(instructions: string | null, meta: {
98
+ capturedSha256: string | null;
99
+ serverVersion?: string;
100
+ }): SessionStartRow;
101
+ /** Parse results/session-start.json; anything malformed yields null, never throws. */
102
+ export declare function parseSessionStart(text: string): SessionStartRun | null;
@@ -0,0 +1,186 @@
1
+ /**
2
+ * Session-start load — what a client actually puts in context when it defers
3
+ * tool definitions until they are used.
4
+ *
5
+ * The headline number counts every byte of `tools/list`, because that is what a
6
+ * client loading definitions eagerly pays. Clients increasingly do not: they put
7
+ * a *list of names* in context and fetch the definition only when the model
8
+ * reaches for the tool. That client pays a different bill at session start, and
9
+ * the headline number says nothing about it.
10
+ *
11
+ * Different, and usually far smaller — but *not* smaller by construction, and
12
+ * nothing here may claim it is. Only one of the two halves below is bounded by
13
+ * the definitions being deferred. Instructions are separate bytes that no part
14
+ * of the headline counts, and their length has nothing to do with the size of
15
+ * the tool set, so a server that re-lists its tools in its `instructions`
16
+ * charges a deferring client *more* than an eager one. That is not
17
+ * hypothetical: it is true of a server on the published leaderboard, which is
18
+ * the most useful thing this measurement has found, and every renderer of these
19
+ * numbers is expected to let the reader see it rather than smooth it away.
20
+ *
21
+ * Two quantities make up that bill, and only one of them is in the published
22
+ * capture:
23
+ *
24
+ * 1. **Tool names.** Derivable from `rawToolsCapture` — no re-measurement
25
+ * needed, and re-derivable by anyone from the same published bytes.
26
+ * 2. **Server instructions.** The `instructions` string a server returns from
27
+ * `initialize`. It is *not* part of `tools/list` and so is absent from every
28
+ * measurement recorded before this field existed.
29
+ *
30
+ * Which means a row can be in one of two honest states, and they are not the
31
+ * same claim: an exact figure (both halves known), or a **floor** (names
32
+ * measured, instructions never captured). A floor published as a figure is the
33
+ * failure mode this module exists to prevent — see `isFloor`, which every
34
+ * renderer must carry through to the reader.
35
+ *
36
+ * Versioned independently of the o200k methodology, on the same reasoning as
37
+ * `tools-delta/v1`: this adds a second published number and does not touch the
38
+ * definition of the first. No `totalTokens` and no canonical hash moves.
39
+ */
40
+ import { countTokens, sha256Hex } from './canonical.js';
41
+ /** Method identifier, versioned independently of METHODOLOGY_VERSION. */
42
+ export const SESSION_START_METHOD = 'deferred-load/v1';
43
+ /**
44
+ * Tool names in server-returned order. A tool without a usable name is dropped
45
+ * rather than given a placeholder — the same rule `toAnthropicTools` follows,
46
+ * for the same reason: an invented name would change the count being published.
47
+ */
48
+ export function toolNames(raw) {
49
+ const out = [];
50
+ for (const t of raw) {
51
+ const name = (t ?? {}).name;
52
+ if (typeof name === 'string' && name !== '')
53
+ out.push(name);
54
+ }
55
+ return out;
56
+ }
57
+ /**
58
+ * o200k count of the canonical tool-name list: `JSON.stringify` over the name
59
+ * array, the same serialization discipline as the headline number, so both
60
+ * re-derive from the same capture with the same five lines.
61
+ */
62
+ export function toolNameTokens(raw) {
63
+ return countTokens(JSON.stringify(toolNames(raw)));
64
+ }
65
+ /**
66
+ * The two halves are counted separately and added, rather than counted over one
67
+ * concatenated string. Concatenation would let boundary tokens merge across the
68
+ * seam, so the published total would not equal the sum of the parts printed
69
+ * beside it — a discrepancy of a token or two that no reader could account for.
70
+ */
71
+ export function sessionStartTokens(raw, instructions) {
72
+ return toolNameTokens(raw) + (instructions ? countTokens(instructions) : 0);
73
+ }
74
+ /**
75
+ * The instructions recorded *inside* a measurement, distinguishing three states
76
+ * that JSON round-trips faithfully:
77
+ * - a string → captured, the server sent this
78
+ * - null → captured, the server sent none
79
+ * - undefined → never captured (every measurement predating the field)
80
+ *
81
+ * Absent-means-unknown rather than absent-means-zero, on the precedent set by
82
+ * history.csv's `isolation` column: an old row reads as unknown and is never
83
+ * back-filled from a value it did not record.
84
+ */
85
+ export function measuredInstructions(m) {
86
+ if (typeof m.serverInstructions === 'string')
87
+ return m.serverInstructions;
88
+ if (m.serverInstructions === null)
89
+ return '';
90
+ return undefined;
91
+ }
92
+ /**
93
+ * Resolve one server's session-start load from its measurement and, if it has
94
+ * one, the instructions captured beside it.
95
+ *
96
+ * Precedence is measurement over capture and never the other way round: the
97
+ * measurement's instructions came off the same server process as its tools, in
98
+ * the same run, so it cannot be stale relative to itself. The side capture is
99
+ * the backfill for measurements taken before the field existed, and it is used
100
+ * only while it still points at the measurement on disk.
101
+ *
102
+ * Returns null for a measurement with no capture to read names from — a
103
+ * `startup-failure` has no session-start load because it has no session.
104
+ */
105
+ export function sessionStartLoad(m, row) {
106
+ if (!Array.isArray(m.rawToolsCapture))
107
+ return null;
108
+ const names = toolNameTokens(m.rawToolsCapture);
109
+ const toolCount = toolNames(m.rawToolsCapture).length;
110
+ const inMeasurement = measuredInstructions(m);
111
+ if (inMeasurement !== undefined) {
112
+ const t = inMeasurement ? countTokens(inMeasurement) : 0;
113
+ return {
114
+ toolCount,
115
+ toolNameTokens: names,
116
+ instructionsTokens: t,
117
+ totalTokens: names + t,
118
+ isFloor: false,
119
+ instructionsSource: 'measurement',
120
+ };
121
+ }
122
+ if (isCurrentInstructions(row, m.canonicalSha256)) {
123
+ return {
124
+ toolCount,
125
+ toolNameTokens: names,
126
+ instructionsTokens: row.instructionsTokens,
127
+ totalTokens: names + row.instructionsTokens,
128
+ isFloor: false,
129
+ instructionsSource: 'capture',
130
+ };
131
+ }
132
+ return {
133
+ toolCount,
134
+ toolNameTokens: names,
135
+ instructionsTokens: null,
136
+ totalTokens: names,
137
+ isFloor: true,
138
+ instructionsSource: 'not-captured',
139
+ };
140
+ }
141
+ /**
142
+ * A side capture is usable only if it carries a number and still points at the
143
+ * measurement on disk. Unlike a stale divergence row — which is hidden, because
144
+ * there is nothing else to print — a stale row here degrades the figure to its
145
+ * names-only floor. The column never blanks; it only ever stops claiming to
146
+ * know the half it no longer knows.
147
+ */
148
+ export function isCurrentInstructions(row, canonicalSha256) {
149
+ if (!row || row.error)
150
+ return false;
151
+ if (typeof row.instructionsTokens !== 'number')
152
+ return false;
153
+ return !!canonicalSha256 && row.capturedSha256 === canonicalSha256;
154
+ }
155
+ /** Build a row from a freshly captured instructions string. */
156
+ export function toSessionStartRow(instructions, meta) {
157
+ const text = instructions ?? '';
158
+ return {
159
+ instructions: text,
160
+ instructionsTokens: text ? countTokens(text) : 0,
161
+ instructionsSha256: sha256Hex(text),
162
+ capturedSha256: meta.capturedSha256,
163
+ serverVersion: meta.serverVersion,
164
+ };
165
+ }
166
+ /** Parse results/session-start.json; anything malformed yields null, never throws. */
167
+ export function parseSessionStart(text) {
168
+ let run;
169
+ try {
170
+ run = JSON.parse(text);
171
+ }
172
+ catch {
173
+ return null;
174
+ }
175
+ const r = run;
176
+ if (!r || typeof r.measuredAt !== 'string')
177
+ return null;
178
+ if (!r.servers || typeof r.servers !== 'object')
179
+ return null;
180
+ return {
181
+ method: typeof r.method === 'string' ? r.method : SESSION_START_METHOD,
182
+ measuredAt: r.measuredAt,
183
+ isolation: typeof r.isolation === 'string' ? r.isolation : undefined,
184
+ servers: r.servers,
185
+ };
186
+ }
@@ -23,6 +23,14 @@ export interface Measurement {
23
23
  canonicalSha256: string | null;
24
24
  /** The tools array exactly as returned by tools/list, all pages concatenated. */
25
25
  rawToolsCapture: unknown[] | null;
26
+ /**
27
+ * The `instructions` string returned by `initialize` — half of the
28
+ * session-start load (see core/session-start.ts). Three states, and they are
29
+ * different claims: a string (the server sent this), `null` (the server sent
30
+ * none), absent (never captured — every measurement predating this field).
31
+ * Absent is never read as zero.
32
+ */
33
+ serverInstructions?: string | null;
26
34
  measuredAt: string;
27
35
  serverName: string;
28
36
  serverVersion?: string;
@@ -5,7 +5,12 @@ export interface WireCapture {
5
5
  };
6
6
  protocolVersion?: string;
7
7
  tools: unknown[];
8
- /** true when a second tools/list returned a different tool set */
8
+ /**
9
+ * The `instructions` string from the initialize result — half of the
10
+ * session-start load. `null` means the server returned none, which is a
11
+ * different fact from never having asked, so the two are never conflated.
12
+ */
13
+ instructions: string | null;
9
14
  stderrTail: string;
10
15
  }
11
16
  export declare class McpStdioClient {
@@ -164,6 +164,7 @@ export async function captureTools(spec, opts = {}) {
164
164
  serverInfo: init?.serverInfo,
165
165
  protocolVersion: init?.protocolVersion,
166
166
  tools,
167
+ instructions: typeof init?.instructions === 'string' ? init.instructions : null,
167
168
  stderrTail: client.stderrTail,
168
169
  };
169
170
  }
@@ -7,3 +7,13 @@
7
7
  */
8
8
  export declare function renderSparkline(tokens: number[]): string;
9
9
  export declare function generateDashboard(root?: string): string;
10
+ /**
11
+ * Render and write the dashboard. Exported so `regen.ts` refreshes it alongside
12
+ * the leaderboard and the server pages: this is the published Pages surface, and
13
+ * when only the CLI wrote it every sweep updated `results/` and `docs/servers/`
14
+ * while the page people actually open kept the previous run's numbers.
15
+ */
16
+ export declare function writeDashboard(root?: string, out?: string): {
17
+ out: string;
18
+ bytes: number;
19
+ };
@@ -7,7 +7,7 @@ import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'node:fs';
7
7
  import { dirname, join } from 'node:path';
8
8
  import { parse } from 'yaml';
9
9
  import { bandColor, BAND_META } from '../core/bands.js';
10
- import { parseHistory } from './history.js';
10
+ import { parseHistory, plottableSeries } from './history.js';
11
11
  /** Longest series a sparkline plots — a stat-tile trend, not a full chart. */
12
12
  const SPARK_MAX_POINTS = 12;
13
13
  /**
@@ -49,10 +49,9 @@ export function generateDashboard(root = process.cwd()) {
49
49
  const dSrv = divergence.servers ?? {};
50
50
  const historyPath = join(root, 'results', 'history.csv');
51
51
  const history = existsSync(historyPath) ? parseHistory(readFileSync(historyPath, 'utf8')) : [];
52
- const seriesFor = (name) => history
53
- .filter((h) => h.server === name)
54
- .sort((a, b) => a.date.localeCompare(b.date))
55
- .map((h) => h.tokens);
52
+ // Only the run of sweeps taken under the same isolation is plotted: a step
53
+ // across an isolation change is the harness moving, not the server.
54
+ const seriesFor = (name) => plottableSeries(history.filter((h) => h.server === name));
56
55
  const rows = doc.servers.map((entry) => {
57
56
  const p = join(root, 'results', entry.name, 'measurement.json');
58
57
  return { entry, m: existsSync(p) ? JSON.parse(readFileSync(p, 'utf8')) : null };
@@ -78,8 +77,16 @@ export function generateDashboard(root = process.cwd()) {
78
77
  const div = dSrv[r.entry.name];
79
78
  const claudeTip = div ? ` · in a Claude request: ${fmt(div.claudeDelta)} tok` : '';
80
79
  const series = seriesFor(r.entry.name);
81
- const spark = renderSparkline(series);
82
- const trendTip = series.length > 1 ? ` · ${series.length}-sweep trend: ${series[0].toLocaleString('en-US')} → ${series[series.length - 1].toLocaleString('en-US')}` : '';
80
+ const tokens = series.rows.map((h) => h.tokens);
81
+ const spark = renderSparkline(tokens);
82
+ const trendTip = tokens.length > 1
83
+ ? ` · ${tokens.length}-sweep trend: ${tokens[0].toLocaleString('en-US')} → ${tokens[tokens.length - 1].toLocaleString('en-US')}` +
84
+ (series.dropped
85
+ ? ` (${series.dropped} earlier sweep${series.dropped === 1 ? '' : 's'} measured under different isolation, not plotted)`
86
+ : '')
87
+ : series.dropped
88
+ ? ` · earlier sweeps were measured under different isolation, so no trend is plotted`
89
+ : '';
83
90
  // The whole row is the link to the server's detail page (docs/servers/).
84
91
  return `<a class="row" href="servers/${encodeURIComponent(r.entry.name)}.html" data-tip="${esc(m.toolCount)} tools · largest: ${esc(largest?.name)} (${fmt(largest?.tokens ?? 0)} tok)${claudeTip}${trendTip} · ${esc(r.entry.category)} · ${esc(m.status)}${m.serverVersion ? ' · v' + esc(String(m.serverVersion).replace(/^v/, '')) : ''}">
85
92
  <span class="rank">${i + 1}</span>
@@ -101,9 +108,9 @@ export function generateDashboard(root = process.cwd()) {
101
108
  .map((r, i) => {
102
109
  const m = r.m;
103
110
  const div = dSrv[r.entry.name];
104
- const series = seriesFor(r.entry.name);
105
- const trend = series.length > 1
106
- ? `${series[series.length - 1] - series[0] >= 0 ? '+' : ''}${fmt(series[series.length - 1] - series[0])} / ${series.length}d`
111
+ const tokens = seriesFor(r.entry.name).rows.map((h) => h.tokens);
112
+ const trend = tokens.length > 1
113
+ ? `${tokens[tokens.length - 1] - tokens[0] >= 0 ? '+' : ''}${fmt(tokens[tokens.length - 1] - tokens[0])} / ${tokens.length}d`
107
114
  : '—';
108
115
  return `<tr><td>${i + 1}</td><td>${esc(r.entry.name)}</td><td class="num">${fmt(m.totalTokens)}</td><td class="num">${div ? fmt(div.claudeDelta) : '—'}</td><td class="num">${esc(m.toolCount)}</td><td>${esc(BAND_META[bandColor(m.totalTokens)].label)}</td><td>${esc(r.entry.category)}</td><td class="num">${trend}</td></tr>`;
109
116
  })
@@ -220,7 +227,7 @@ export function generateDashboard(root = process.cwd()) {
220
227
  </div>
221
228
 
222
229
  <h2>Leaderboard</h2>
223
- <p class="h2sub">Tokens = o200k_base count of the canonical <code>tools/list</code> bytes — the wire payload. What a <em>Claude request</em> actually carries can differ sharply (github: 54,422 on the wire, ${dSrv['github'] ? fmt(dSrv['github'].claudeDelta) : '…'} in a request — 80% of its schema bytes are fields no Anthropic request sends). The trend line plots tokens across every sweep date on record (oldest→newest, dot = current); servers with one sweep so far show no line yet. Hover a row for exact numbers; <a href="METHODOLOGY.html#claude-divergence">method</a>.</p>
230
+ <p class="h2sub">Tokens = o200k_base count of the canonical <code>tools/list</code> bytes — the wire payload. What a <em>Claude request</em> actually carries can differ sharply (github: 54,422 on the wire, ${dSrv['github'] ? fmt(dSrv['github'].claudeDelta) : '…'} in a request — 80% of its schema bytes are fields no Anthropic request sends). The trend line plots tokens across every sweep date on record (oldest→newest, dot = current); servers with one sweep so far show no line yet, and a sweep taken under different isolation than the current one is left out rather than drawn as a change in the server. Hover a row for exact numbers; <a href="METHODOLOGY.html#claude-divergence">method</a>.</p>
224
231
  <div class="board">
225
232
  ${barRows || '<p class="h2sub">Sweep in progress — first results land shortly.</p>'}
226
233
  </div>
@@ -282,12 +289,21 @@ ${barRows || '<p class="h2sub">Sweep in progress — first results land shortly.
282
289
  </script>
283
290
  `;
284
291
  }
292
+ /**
293
+ * Render and write the dashboard. Exported so `regen.ts` refreshes it alongside
294
+ * the leaderboard and the server pages: this is the published Pages surface, and
295
+ * when only the CLI wrote it every sweep updated `results/` and `docs/servers/`
296
+ * while the page people actually open kept the previous run's numbers.
297
+ */
298
+ export function writeDashboard(root = process.cwd(), out = 'docs/dashboard.html') {
299
+ const html = generateDashboard(root);
300
+ mkdirSync(dirname(out), { recursive: true });
301
+ writeFileSync(out, html);
302
+ return { out, bytes: html.length };
303
+ }
285
304
  const isMain = process.argv[1]?.endsWith('dashboard.ts') || process.argv[1]?.endsWith('dashboard.js');
286
305
  if (isMain) {
287
306
  const i = process.argv.indexOf('--out');
288
- const out = i >= 0 ? process.argv[i + 1] : 'docs/dashboard.html';
289
- const html = generateDashboard();
290
- mkdirSync(dirname(out), { recursive: true });
291
- writeFileSync(out, html);
292
- console.log(`wrote ${out} (${(html.length / 1024).toFixed(0)}KB)`);
307
+ const w = writeDashboard(process.cwd(), i >= 0 ? process.argv[i + 1] : 'docs/dashboard.html');
308
+ console.log(`wrote ${w.out} (${(w.bytes / 1024).toFixed(0)}KB)`);
293
309
  }
@@ -10,6 +10,16 @@ export interface DockerOptions {
10
10
  image?: string;
11
11
  /** Extra env var NAMES to pass through with dummy values. */
12
12
  dummyEnv?: string[];
13
+ /**
14
+ * Override the literal `dummy` value for specific `dummyEnv` names. Some
15
+ * servers parse an env var's shape before ever reaching tools/list (a URI
16
+ * scheme, a URL) and crash on a value that isn't real-looking rather than
17
+ * a value that isn't a real credential — e.g. `NEO4J_URI=dummy` fails
18
+ * `neo4j.exceptions.ConfigurationError` before the driver is even asked to
19
+ * connect. A locally-scoped placeholder (`bolt://localhost:7687`) clears
20
+ * that parse step without providing a working credential.
21
+ */
22
+ dummyEnvValues?: Record<string, string>;
13
23
  /** Container name, so a timed-out container can be force-removed. */
14
24
  containerName?: string;
15
25
  /**
@@ -20,6 +30,19 @@ export interface DockerOptions {
20
30
  * git already on PATH would actually run.
21
31
  */
22
32
  needsGit?: boolean;
33
+ /**
34
+ * Skip the shared npm/uv cache volumes, paying a cold install for a clean one.
35
+ *
36
+ * The caches are what make a re-sweep minutes instead of hours, but a cache
37
+ * entry can go bad and stay bad: npm remembers a failed postinstall, and every
38
+ * later install that resolves through it fails the same way. Observed
39
+ * 2026-08-19 — a cached `esbuild@0.25.12` whose postinstall exits 1 made
40
+ * `npx -y @shopify/dev-mcp@latest` exit 1 with no output, against a server that
41
+ * installs and answers fine from an empty cache. Nothing in the exit code tells
42
+ * that apart from a genuinely broken server, so the sweep retries here rather
43
+ * than publishing the downgrade.
44
+ */
45
+ noSharedCache?: boolean;
23
46
  }
24
47
  export declare const DEFAULT_NODE_IMAGE = "public.ecr.aws/docker/library/node:22-slim";
25
48
  export declare const DEFAULT_PYTHON_IMAGE = "ghcr.io/astral-sh/uv:python3.12-bookworm-slim";
@@ -23,17 +23,21 @@ export function dockerize(commandLine, opts = {}) {
23
23
  '-w',
24
24
  '/tmp',
25
25
  // Shared package caches across containers — packages only, no credentials.
26
- '-v',
27
- 'mcp-ctx-npm-cache:/tmp/.npm-cache',
28
- '-e',
29
- 'npm_config_cache=/tmp/.npm-cache',
30
- '-v',
31
- 'mcp-ctx-uv-cache:/tmp/.uv-cache',
32
- '-e',
33
- 'UV_CACHE_DIR=/tmp/.uv-cache',
26
+ ...(opts.noSharedCache
27
+ ? []
28
+ : [
29
+ '-v',
30
+ 'mcp-ctx-npm-cache:/tmp/.npm-cache',
31
+ '-e',
32
+ 'npm_config_cache=/tmp/.npm-cache',
33
+ '-v',
34
+ 'mcp-ctx-uv-cache:/tmp/.uv-cache',
35
+ '-e',
36
+ 'UV_CACHE_DIR=/tmp/.uv-cache',
37
+ ]),
34
38
  ];
35
39
  for (const name of opts.dummyEnv ?? []) {
36
- argv.push('-e', `${name}=dummy`);
40
+ argv.push('-e', `${name}=${opts.dummyEnvValues?.[name] ?? 'dummy'}`);
37
41
  }
38
42
  const gitPrefix = opts.needsGit
39
43
  ? 'command -v git >/dev/null 2>&1 || (apt-get update -qq && apt-get install -y -qq --no-install-recommends git >/dev/null 2>&1); '
@@ -46,9 +50,9 @@ export function dockerize(commandLine, opts = {}) {
46
50
  docker: true,
47
51
  image,
48
52
  network: 'bridge',
49
- note: opts.needsGit
50
- ? 'network enabled for package fetch; clean FS, no host credentials, git installed'
51
- : 'network enabled for package fetch; clean FS, no host credentials',
53
+ note: 'network enabled for package fetch; clean FS, no host credentials' +
54
+ (opts.needsGit ? ', git installed' : '') +
55
+ (opts.noSharedCache ? ', shared package cache bypassed' : ''),
52
56
  },
53
57
  };
54
58
  }
@@ -0,0 +1,57 @@
1
+ import type { MeasurementStatus } from '../core/types.js';
2
+ /**
3
+ * Below this many regressions the population signal doesn't exist yet, and a
4
+ * deliberately narrow sweep (`--only redis,serena`, two servers known to be
5
+ * shaky) must never be able to trip a fault by failing completely.
6
+ */
7
+ export declare const MIN_REGRESSIONS = 5;
8
+ /**
9
+ * Share of the previously-good servers in this sweep that must regress before
10
+ * the harness is the likelier explanation. The largest genuine simultaneous
11
+ * upstream breakage on record here is a handful of PyPI servers that shared
12
+ * one unbounded dependency — single digits against ~65 measured, well under
13
+ * 10%. A harness fault, by contrast, takes everything it touches: the Docker
14
+ * outage above scored 100%. A majority sits far from the former and catches
15
+ * every instance of the latter this project has actually seen.
16
+ */
17
+ export declare const FAULT_RATIO = 0.5;
18
+ /** Statuses that represent a real number on record — the thing worth protecting. */
19
+ export declare function isGood(status: MeasurementStatus): boolean;
20
+ /** The bytes of one server's published artifacts, exactly as they were pre-sweep. */
21
+ export interface Snapshot {
22
+ name: string;
23
+ status: MeasurementStatus | null;
24
+ /** Raw file contents, so a restore is byte-identical rather than re-serialized. */
25
+ measurementJson: string | null;
26
+ badgeJson: string | null;
27
+ }
28
+ /**
29
+ * Read what is currently published for each server. Called before the sweep
30
+ * starts; a server with nothing on record snapshots as `null` status, which no
31
+ * verdict counts either way.
32
+ */
33
+ export declare function snapshot(names: string[], root?: string): Snapshot[];
34
+ export interface Verdict {
35
+ /** True when the sweep looks like a broken harness rather than broken servers. */
36
+ fault: boolean;
37
+ /** Servers that went from a real number to a failure in this sweep. */
38
+ regressed: string[];
39
+ /** How many servers had a real number on record and could therefore regress. */
40
+ comparable: number;
41
+ /** Human-readable reason, printed either way — a pass says why it passed. */
42
+ reason: string;
43
+ }
44
+ /**
45
+ * Compare pre-sweep snapshots against this sweep's outcomes.
46
+ *
47
+ * `current` maps server name to the status it just measured at. Servers absent
48
+ * from it were not swept and are ignored.
49
+ */
50
+ export declare function verdict(prior: Snapshot[], current: Map<string, MeasurementStatus>): Verdict;
51
+ /**
52
+ * Put the snapshotted artifacts back, byte for byte. Only servers named in
53
+ * `names` are touched, and only where a prior file existed — a server whose
54
+ * first-ever measurement failed has nothing to restore and keeps its new
55
+ * (honest) failure record.
56
+ */
57
+ export declare function restore(prior: Snapshot[], names: string[], root?: string): string[];