mcp-context-cost 0.11.1 → 0.11.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -278,7 +278,7 @@ now measured against a pinned model and published beside the badge, and they do
278
278
 
279
279
  | server | badge (o200k) | Claude (`claude-opus-5`) | |
280
280
  |---|---:|---:|---|
281
- | github | 54,422 | **18,406** | 81% of the capture is `annotations`/`outputSchema` metadata Claude never sees |
281
+ | github | 54,622 | **—** | most of the capture is `annotations`/`outputSchema` metadata Claude never sees |
282
282
  | notion | 17,500 | **33,560** | almost no metadata to drop, so the tokenizer difference dominates |
283
283
 
284
284
  So the heaviest server on the badge is not the heaviest server on Claude. Per-server
@@ -151,6 +151,13 @@ export interface AuditReport {
151
151
  worstTotal: number;
152
152
  worstSource: string;
153
153
  over: boolean;
154
+ /**
155
+ * Servers in the audited configs with no number, when there are any. Their
156
+ * cost is missing from `worstTotal`, so the budget could not be checked
157
+ * against the whole stack and the verdict fails rather than passing on a
158
+ * total that understates by an unknown amount.
159
+ */
160
+ unestablished?: string[];
154
161
  /** Present only when over budget: the arithmetic of getting back under it. */
155
162
  fit?: BudgetFit;
156
163
  };
@@ -179,6 +179,13 @@ export function buildReport(configs, measured, opts = {}) {
179
179
  const contextWindow = opts.contextWindow ?? DEFAULT_CONTEXT_WINDOW;
180
180
  const problems = [];
181
181
  const emptyConfigs = [];
182
+ // canonicalSha256 → the divergence row computed from it, so a server can be
183
+ // identified by its bytes whatever the local config calls it.
184
+ const divByHash = new Map();
185
+ for (const row of Object.values(opts.divergence?.servers ?? {})) {
186
+ if (row?.capturedSha256)
187
+ divByHash.set(row.capturedSha256, row);
188
+ }
182
189
  const built = [];
183
190
  // Across every config at once: a twin in one client's file is measured for
184
191
  // the other client's entry just the same.
@@ -319,12 +326,21 @@ export function buildReport(configs, measured, opts = {}) {
319
326
  // The worst config is the gate: passing because your *lightest* client fits
320
327
  // would be a green check on a session you don't run.
321
328
  const worst = results[0];
329
+ // A total is only a ceiling to compare against if it is the whole cost. A
330
+ // server that failed to start contributes 0, so the stack reads lighter
331
+ // than it is and the budget passes on a number that is missing a server —
332
+ // exactly the PR the README says this gate catches. The server-level gate
333
+ // (core/server-diff.ts) already refuses this; so does this one now.
334
+ const unestablished = results.flatMap((c) => c.skipped.filter((s) => s.status !== 'remote-not-measurable').map((s) => `${c.source}: ${s.name} (${s.status})`));
322
335
  const over = (worst?.totalTokens ?? 0) > opts.budget;
323
336
  report.budget = {
324
337
  limit: opts.budget,
325
338
  worstTotal: worst?.totalTokens ?? 0,
326
339
  worstSource: worst?.source ?? '(none)',
327
- over,
340
+ over: over || unestablished.length > 0,
341
+ // Named so the reader knows which way the number is wrong: the total
342
+ // understates by however much these cost, which nobody knows.
343
+ unestablished: unestablished.length ? unestablished : undefined,
328
344
  fit: over && worst ? planBudgetFit(worst, opts.budget) : undefined,
329
345
  };
330
346
  }
@@ -739,6 +755,15 @@ export function formatReport(report) {
739
755
  const headroom = b.limit - b.worstTotal;
740
756
  lines.push(`budget ok: ${n(b.worstTotal)} ≤ ${n(b.limit)} — ${n(headroom)} to spare`);
741
757
  }
758
+ else if (b.unestablished && b.worstTotal <= b.limit) {
759
+ // Under the line on what was measured, but not everything was: say which
760
+ // way the number is wrong rather than passing on it.
761
+ lines.push(`BUDGET FAIL: ${n(b.worstTotal)} ≤ ${n(b.limit)}, but the budget could not be checked against the ` +
762
+ `whole stack — ${b.unestablished.length} server(s) produced no number, so this total understates ` +
763
+ `by however much they cost:`);
764
+ for (const u of b.unestablished)
765
+ lines.push(` ${u}`);
766
+ }
742
767
  else {
743
768
  lines.push(`BUDGET FAIL: ${n(b.worstTotal)} > ${n(b.limit)} (${b.worstSource})`);
744
769
  const fit = b.fit;
@@ -182,9 +182,18 @@ export function configCandidates(env) {
182
182
  /** Read + parse the candidates that exist. Unreadable files are reported, not thrown. */
183
183
  export function loadConfigs(candidates, cwd) {
184
184
  const out = [];
185
+ // Running from your home directory nominates `~/.cursor/mcp.json` twice —
186
+ // once as the home candidate, once as the cwd one. Loaded twice it is
187
+ // reported twice, doubles that client's deferral scope, and under
188
+ // `--baseline` the second copy pairs with nothing and fails the gate. One
189
+ // path is one config however many ways it was nominated.
190
+ const seen = new Set();
185
191
  for (const c of candidates) {
186
192
  if (!existsSync(c.path))
187
193
  continue;
194
+ if (seen.has(c.path))
195
+ continue;
196
+ seen.add(c.path);
188
197
  try {
189
198
  const doc = parseJsonc(readFileSync(c.path, 'utf8'));
190
199
  const { servers, disabled } = extractDeclaration(doc, { client: c.client, source: c.path, cwd });
@@ -21,6 +21,17 @@ export function parseBaselineReport(text) {
21
21
  if (!c || typeof c !== 'object' || typeof c.source !== 'string') {
22
22
  return { report: null, problem: 'baseline has a config entry without a source path' };
23
23
  }
24
+ // The number the diff subtracts from. Unchecked, a hand-trimmed or
25
+ // jq-filtered baseline yields `after - undefined === NaN`, and `typeof NaN`
26
+ // is 'number' — so the gate reported an increase of 0 and passed. A
27
+ // baseline that cannot be read is never "no change".
28
+ const total = c.totalTokens;
29
+ if (typeof total !== 'number' || !Number.isFinite(total)) {
30
+ return {
31
+ report: null,
32
+ problem: `baseline config ${c.source} has no usable totalTokens — is it the output of \`audit --json\`?`,
33
+ };
34
+ }
24
35
  }
25
36
  return { report: doc };
26
37
  }
@@ -69,9 +80,25 @@ export function diffConfig(before, after, matchedBy) {
69
80
  continue;
70
81
  }
71
82
  if (!bs && as) {
72
- servers.push(as.tokens === null
73
- ? { name, kind: 'added', before: null, after: null, delta: null, note: 'added but not measurable — its cost is unknown, not zero' }
74
- : { name, kind: 'added', before: null, after: as.tokens, delta: as.tokens });
83
+ if (as.tokens === null) {
84
+ // Unknown, not zero so the total this diff subtracts from does not
85
+ // contain it and the increase understates. Marking the config inexact
86
+ // is what stops `--max-increase` from passing: a server added in a PR
87
+ // and unmeasurable in CI (no credential — the ordinary case) otherwise
88
+ // reads as +0 and clears even a zero-token allowance.
89
+ exact = false;
90
+ servers.push({
91
+ name,
92
+ kind: 'added',
93
+ before: null,
94
+ after: null,
95
+ delta: null,
96
+ note: 'added but not measurable — its cost is unknown, not zero',
97
+ });
98
+ }
99
+ else {
100
+ servers.push({ name, kind: 'added', before: null, after: as.tokens, delta: as.tokens });
101
+ }
75
102
  continue;
76
103
  }
77
104
  if (!bs || !as)
@@ -154,7 +181,15 @@ export function pairConfigs(before, after) {
154
181
  pairs.push({ before: null, after: cur, matchedBy: 'unmatched' });
155
182
  }
156
183
  }
157
- if (before.length === 1 && after.length === 1 && pairs[0].before === null) {
184
+ // One on each side reads as the same config seen from two machines, where the
185
+ // path legitimately differs (a laptop's baseline against a CI checkout). It
186
+ // does not survive the clients differing: a Claude Desktop baseline against a
187
+ // Claude Code run compares two unrelated stacks, and the gate then rests on
188
+ // that difference. The paths may differ; what they are configs *for* may not.
189
+ if (before.length === 1 &&
190
+ after.length === 1 &&
191
+ pairs[0].before === null &&
192
+ before[0].client === after[0].client) {
158
193
  pairs[0] = { before: before[0], after: after[0], matchedBy: 'sole-config' };
159
194
  unusedBefore.delete(before[0].source);
160
195
  }
@@ -298,13 +333,17 @@ export function evaluateIncreaseGate(diff, limit) {
298
333
  reasons.push(`${c.source}: no baseline to check its ${n(c.afterTotal)} tokens against`);
299
334
  }
300
335
  else if (!c.exact) {
301
- reasons.push(`${c.source}: a server changed measured-ness, so the change could not be established exactly`);
336
+ reasons.push(`${c.source}: a server's cost is not established on both sides, so the change could not be established exactly`);
302
337
  }
303
338
  }
304
339
  for (const d of diff.droppedConfigs) {
305
340
  reasons.push(`${d.source}: covered by the baseline and not found in this run`);
306
341
  }
307
- const increase = diff.worstIncrease?.delta ?? (diff.configs.some((c) => typeof c.delta === 'number') ? 0 : null);
342
+ // `Number.isFinite`, not `typeof === 'number'`: NaN passes the latter and
343
+ // then compares false against every limit, which is a silent pass.
344
+ const anyDelta = diff.configs.some((c) => Number.isFinite(c.delta));
345
+ const worst = diff.worstIncrease?.delta;
346
+ const increase = Number.isFinite(worst) ? worst : anyDelta ? 0 : null;
308
347
  if (reasons.length === 0 && increase !== null && increase > limit) {
309
348
  reasons.push(`${diff.worstIncrease.source}: +${n(increase)} tokens per request, over the ${n(limit)} allowed`);
310
349
  }
package/dist/cli.d.ts CHANGED
@@ -27,6 +27,11 @@ export declare function unknownFlags(argv: string[], spec: {
27
27
  value: string[];
28
28
  boolean: string[];
29
29
  }): string[];
30
+ /** Every flag name a command accepts — what tells a value apart from the next flag. */
31
+ export declare const knownFlagNames: (spec: {
32
+ value: string[];
33
+ boolean: string[];
34
+ }) => Set<string>;
30
35
  /**
31
36
  * Every value a value-taking flag was given, in either accepted spelling:
32
37
  * `--flag value` and `--flag=value`.
@@ -37,9 +42,9 @@ export declare function unknownFlags(argv: string[], spec: {
37
42
  * matched the bare token, so the gate it asked for silently did not run and the
38
43
  * command exited 0 — a green check on a check that never happened.
39
44
  */
40
- export declare function flagValues(argv: string[], name: string): string[];
45
+ export declare function flagValues(argv: string[], name: string, known?: Set<string>): string[];
41
46
  /** The last value given for a flag, or undefined when the flag is absent. */
42
- export declare function flagValue(argv: string[], name: string): string | undefined;
47
+ export declare function flagValue(argv: string[], name: string, known?: Set<string>): string | undefined;
43
48
  /**
44
49
  * Value-taking flags that appear with no usable value.
45
50
  *
package/dist/cli.js CHANGED
@@ -45,6 +45,19 @@ export function slugFromUrl(url) {
45
45
  const host = new URL(url).hostname.replace(/^(www|mcp)\./, '');
46
46
  return host.replace(/[^a-z0-9]+/gi, '-').replace(/^-+|-+$/g, '').toLowerCase() || 'remote';
47
47
  }
48
+ /**
49
+ * Report a `verify` failure and exit 1, in whichever shape the caller asked
50
+ * for. `--json` is documented as putting `{ ok, rederivedTokens, rederivedSha,
51
+ * problems }` on stdout; a script reading that gets nothing from a thrown
52
+ * exception, so every failure path goes through here.
53
+ */
54
+ function failVerify(json, problem) {
55
+ if (json)
56
+ console.log(JSON.stringify({ ok: false, rederivedTokens: null, rederivedSha: null, problems: [problem] }));
57
+ else
58
+ console.error(problem);
59
+ process.exit(1);
60
+ }
48
61
  /** Installed version, for error messages that need to say which one you are running. */
49
62
  export function cliVersion() {
50
63
  try {
@@ -85,6 +98,21 @@ export function unknownFlags(argv, spec) {
85
98
  }
86
99
  return unknown;
87
100
  }
101
+ /**
102
+ * Whether a token is another flag of *this* command, rather than a value that
103
+ * merely looks like one.
104
+ *
105
+ * The distinction is load-bearing: `--command "--weird"` is a legitimate launch
106
+ * command this CLI has always accepted, while `--max-increase --json` is a
107
+ * value slot swallowed by the next flag. Deciding on the `--` prefix alone
108
+ * cannot tell them apart; deciding against the command's own flag list can, and
109
+ * the list is already declared at every call site.
110
+ */
111
+ function isKnownFlagToken(tok, known) {
112
+ return tok.startsWith('--') && known.has(tok.slice(2).split('=')[0]);
113
+ }
114
+ /** Every flag name a command accepts — what tells a value apart from the next flag. */
115
+ export const knownFlagNames = (spec) => new Set([...spec.value, ...spec.boolean]);
88
116
  /**
89
117
  * Every value a value-taking flag was given, in either accepted spelling:
90
118
  * `--flag value` and `--flag=value`.
@@ -95,15 +123,15 @@ export function unknownFlags(argv, spec) {
95
123
  * matched the bare token, so the gate it asked for silently did not run and the
96
124
  * command exited 0 — a green check on a check that never happened.
97
125
  */
98
- export function flagValues(argv, name) {
126
+ export function flagValues(argv, name, known = new Set()) {
99
127
  const out = [];
100
128
  for (let i = 0; i < argv.length; i++) {
101
129
  const tok = argv[i];
102
130
  if (tok === `--${name}`) {
103
131
  const next = argv[i + 1];
104
- // A following flag is not this flag's value; that case is a usage error,
105
- // caught by `valuelessFlags`, and must not be read as a value here.
106
- if (next !== undefined && !next.startsWith('--'))
132
+ // Another flag of this command is not this flag's value; that case is a
133
+ // usage error, caught by `valuelessFlags`, and never read as a value.
134
+ if (next !== undefined && !isKnownFlagToken(next, known))
107
135
  out.push(next);
108
136
  continue;
109
137
  }
@@ -113,8 +141,8 @@ export function flagValues(argv, name) {
113
141
  return out;
114
142
  }
115
143
  /** The last value given for a flag, or undefined when the flag is absent. */
116
- export function flagValue(argv, name) {
117
- const values = flagValues(argv, name);
144
+ export function flagValue(argv, name, known = new Set()) {
145
+ const values = flagValues(argv, name, known);
118
146
  return values.length ? values[values.length - 1] : undefined;
119
147
  }
120
148
  /**
@@ -128,6 +156,7 @@ export function flagValue(argv, name) {
128
156
  * so it is refused in the same place and with the same severity.
129
157
  */
130
158
  export function valuelessFlags(argv, spec) {
159
+ const known = knownFlagNames(spec);
131
160
  const bad = [];
132
161
  for (let i = 0; i < argv.length; i++) {
133
162
  const tok = argv[i];
@@ -142,7 +171,7 @@ export function valuelessFlags(argv, spec) {
142
171
  continue;
143
172
  }
144
173
  const next = argv[i + 1];
145
- if (next === undefined || next.startsWith('--'))
174
+ if (next === undefined || isKnownFlagToken(next, known))
146
175
  bad.push(`--${name}`);
147
176
  else
148
177
  i++; // consume the value, so `--command "--weird"` is not re-read as a flag
@@ -169,7 +198,7 @@ function rejectUnknownFlags(cmd, argv, spec) {
169
198
  }
170
199
  const [, , cmd, ...rest] = process.argv;
171
200
  if (cmd === 'audit') {
172
- rejectUnknownFlags('audit', rest, {
201
+ const spec = {
173
202
  value: [
174
203
  'config',
175
204
  'budget',
@@ -183,9 +212,14 @@ if (cmd === 'audit') {
183
212
  'capture-index-url',
184
213
  ],
185
214
  boolean: ['json', 'docker', 'claude', 'suggest', 'changed'],
186
- });
187
- const argOf = (name) => flagValue(rest, name);
188
- const all = (name) => flagValues(rest, name);
215
+ };
216
+ rejectUnknownFlags('audit', rest, spec);
217
+ // The same flag list the rejection used, so a value that merely looks like a
218
+ // flag (`--command "--weird"`) is told apart from a value slot swallowed by
219
+ // the next flag.
220
+ const known = knownFlagNames(spec);
221
+ const argOf = (name) => flagValue(rest, name, known);
222
+ const all = (name) => flagValues(rest, name, known);
189
223
  const json = rest.includes('--json');
190
224
  const numeric = (name) => {
191
225
  const raw = argOf(name);
@@ -301,6 +335,12 @@ if (cmd === 'audit') {
301
335
  `${where}\n` +
302
336
  `${empty.length === 1 ? 'It was' : 'They were'} read and parsed; there is simply nothing declared to measure.\n` +
303
337
  `Declare a server in one of them, or point at a different config: mcp-context-cost audit --config <path/to/mcp.json>`);
338
+ else if (all('config').length)
339
+ // A path the user named is not a discovery miss. Saying "looked in the
340
+ // standard locations" describes something the command did not do, and
341
+ // then advises doing the thing they just did.
342
+ console.error(`no MCP config found at the path(s) given: ${all('config').join(', ')}. ` +
343
+ `Nothing else was searched, because --config was set.`);
304
344
  else
305
345
  console.error(`no MCP config found. Looked in the standard Claude Desktop / Claude Code / Cursor / VS Code / Windsurf locations.${where}\n` +
306
346
  `Point at one explicitly: mcp-context-cost audit --config <path/to/mcp.json>`);
@@ -315,9 +355,10 @@ if (cmd === 'audit') {
315
355
  process.exit(report.budget?.over || report.increaseGate?.pass === false ? 1 : 0);
316
356
  }
317
357
  else if (cmd === 'verify') {
318
- rejectUnknownFlags('verify', rest, { value: ['remote'], boolean: ['json'] });
358
+ const spec = { value: ['remote'], boolean: ['json'] };
359
+ rejectUnknownFlags('verify', rest, spec);
319
360
  const json = rest.includes('--json');
320
- const remoteUrl = flagValue(rest, 'remote');
361
+ const remoteUrl = flagValue(rest, 'remote', knownFlagNames(spec));
321
362
  const path = rest.find((a) => !a.startsWith('--') && a !== remoteUrl);
322
363
  if (!remoteUrl && !path) {
323
364
  console.error('usage: mcp-context-cost verify <measurement.json> [--json]');
@@ -333,18 +374,29 @@ else if (cmd === 'verify') {
333
374
  raw = await res.text();
334
375
  }
335
376
  catch (e) {
336
- const problem = `failed to fetch ${remoteUrl}: ${e.message}`;
337
- if (json)
338
- console.log(JSON.stringify({ ok: false, rederivedTokens: null, rederivedSha: null, problems: [problem] }));
339
- else
340
- console.error(problem);
341
- process.exit(1);
377
+ failVerify(json, `failed to fetch ${remoteUrl}: ${e.message}`);
342
378
  }
343
379
  }
344
380
  else {
345
- raw = readFileSync(path, 'utf8');
381
+ try {
382
+ raw = readFileSync(path, 'utf8');
383
+ }
384
+ catch (e) {
385
+ // The remote branch above reports a failed fetch in the documented shape;
386
+ // this one used to throw, so `--json` produced a stack trace on stderr and
387
+ // nothing at all on stdout — the contract a script parses.
388
+ failVerify(json, `cannot read ${path}: ${e.message}`);
389
+ }
390
+ }
391
+ let m;
392
+ try {
393
+ m = JSON.parse(raw);
394
+ }
395
+ catch (e) {
396
+ // Reachable remotely: a proxy, a captive portal or an HTML error page
397
+ // served with status 200 passes the `res.ok` check above and arrives here.
398
+ failVerify(json, `${remoteUrl ?? path} is not valid JSON: ${e.message}`);
346
399
  }
347
- const m = JSON.parse(raw);
348
400
  const r = verifyMeasurement(m);
349
401
  if (json) {
350
402
  console.log(JSON.stringify({ serverName: m.serverName, ...r, badge: r.ok ? toBadge(m) : undefined }));
@@ -361,11 +413,13 @@ else if (cmd === 'verify') {
361
413
  process.exit(1);
362
414
  }
363
415
  else if (cmd === 'measure') {
364
- rejectUnknownFlags('measure', rest, {
416
+ const spec = {
365
417
  value: ['name', 'command', 'remote', 'timeout', 'docker-image', 'baseline', 'max-increase', 'budget'],
366
418
  boolean: ['docker'],
367
- });
368
- const argOf = (name) => flagValue(rest, name);
419
+ };
420
+ rejectUnknownFlags('measure', rest, spec);
421
+ const known = knownFlagNames(spec);
422
+ const argOf = (name) => flagValue(rest, name, known);
369
423
  const command = argOf('command');
370
424
  const remoteUrl = argOf('remote');
371
425
  if (remoteUrl && !/^https?:\/\//i.test(remoteUrl)) {
@@ -143,6 +143,14 @@ export declare function decodeLoose(text: string): string;
143
143
  export interface EndpointBadge {
144
144
  url: string;
145
145
  linkTarget: string | null;
146
+ /**
147
+ * The badge's own label — markdown alt text or an `<img alt>`. What the badge
148
+ * calls itself is the only thing in a README that distinguishes *this*
149
+ * project's badge from any other shields endpoint badge, since the JSON can
150
+ * be hosted anywhere (the staged action publishes to a gist, with no path
151
+ * shape to recognise).
152
+ */
153
+ alt: string | null;
146
154
  }
147
155
  /**
148
156
  * Every shields endpoint badge in a file, decoded, each paired with its own
@@ -172,6 +180,21 @@ export declare function linksBackToProject(target: string, src?: BadgeSource): b
172
180
  * measurement here. See the header for why both count and why the link is
173
181
  * paired with the image rather than looked for anywhere in the file.
174
182
  */
183
+ /**
184
+ * Whether a badge calls itself this project's badge.
185
+ *
186
+ * The self-hosted branch below used to accept the link alone and look at
187
+ * nothing else, so *any* shields endpoint badge — a coverage badge, say —
188
+ * wrapped in a link to this repository counted as displaying ours. That
189
+ * inflates the one number this project keeps about itself, which is the last
190
+ * place it can afford a generous reading.
191
+ *
192
+ * The URL cannot decide it: self-hosted JSON lives wherever its author put it,
193
+ * and the staged action publishes to a gist. What every published snippet does
194
+ * carry is the label — `context cost` — so that is what is read, tolerantly
195
+ * enough to accept `Context-Cost` and strictly enough to reject `coverage`.
196
+ */
197
+ export declare function namesThisBadge(alt: string | null): boolean;
175
198
  export declare function displaysBadge(text: string, src?: BadgeSource): boolean;
176
199
  /**
177
200
  * Every shields endpoint `url` in a file whose JSON is served from this
@@ -137,7 +137,11 @@ export function endpointBadges(text) {
137
137
  if (last && !before.slice(last.index ?? 0).includes('</a>'))
138
138
  linkTarget = last[1];
139
139
  }
140
- out.push({ url: m[1], linkTarget });
140
+ // Markdown puts the alt before the image; HTML puts it in the same tag.
141
+ const beforeAlt = decoded.slice(Math.max(0, start - LINK_WINDOW), start);
142
+ const mdAlt = beforeAlt.match(/!\[([^\]]*)\]\(\s*[^\s)]*$/);
143
+ const tagAlt = after.match(/^[^<>]*?\balt\s*=\s*["']([^"']*)["']/i);
144
+ out.push({ url: m[1], linkTarget, alt: mdAlt ? mdAlt[1] : tagAlt ? tagAlt[1] : null });
141
145
  }
142
146
  return out;
143
147
  }
@@ -169,8 +173,28 @@ export function linksBackToProject(target, src = BADGE_SOURCE) {
169
173
  * measurement here. See the header for why both count and why the link is
170
174
  * paired with the image rather than looked for anywhere in the file.
171
175
  */
176
+ /**
177
+ * Whether a badge calls itself this project's badge.
178
+ *
179
+ * The self-hosted branch below used to accept the link alone and look at
180
+ * nothing else, so *any* shields endpoint badge — a coverage badge, say —
181
+ * wrapped in a link to this repository counted as displaying ours. That
182
+ * inflates the one number this project keeps about itself, which is the last
183
+ * place it can afford a generous reading.
184
+ *
185
+ * The URL cannot decide it: self-hosted JSON lives wherever its author put it,
186
+ * and the staged action publishes to a gist. What every published snippet does
187
+ * carry is the label — `context cost` — so that is what is read, tolerantly
188
+ * enough to accept `Context-Cost` and strictly enough to reject `coverage`.
189
+ */
190
+ export function namesThisBadge(alt) {
191
+ if (!alt)
192
+ return false;
193
+ return alt.toLowerCase().replace(/[^a-z]/g, '').includes('contextcost');
194
+ }
172
195
  export function displaysBadge(text, src = BADGE_SOURCE) {
173
- return endpointBadges(text).some((b) => hostedHere(b.url, src) || (b.linkTarget !== null && linksBackToProject(b.linkTarget, src)));
196
+ return endpointBadges(text).some((b) => hostedHere(b.url, src) ||
197
+ (namesThisBadge(b.alt) && b.linkTarget !== null && linksBackToProject(b.linkTarget, src)));
174
198
  }
175
199
  /**
176
200
  * Every shields endpoint `url` in a file whose JSON is served from this
@@ -72,14 +72,18 @@ export function identify(canonicalSha256, index) {
72
72
  if (!mine)
73
73
  return { kind: 'unknown' };
74
74
  const currentSha = index.current[mine.server];
75
- if (!currentSha || currentSha === canonicalSha256) {
75
+ if (currentSha === canonicalSha256) {
76
76
  return { kind: 'current', server: mine.server, date: mine.date, tokens: mine.totalTokens };
77
77
  }
78
- const current = index.captures[currentSha];
79
- // A `current` pointer with no capture behind it describes nothing; treat the
80
- // version as identified but with nothing to compare it against.
78
+ // The bytes are identified, but what is current for this server is not: the
79
+ // pointer is missing, or points at a capture the index dropped as ambiguous.
80
+ // `current` would be an affirmative claim that nothing has moved, which the
81
+ // audit prints as "no server here is running a published capture that has
82
+ // since moved" — told to someone who may be far behind. Unknown currency
83
+ // reads as unknown, which is the discipline the rest of this module keeps.
84
+ const current = currentSha ? index.captures[currentSha] : undefined;
81
85
  if (!current)
82
- return { kind: 'current', server: mine.server, date: mine.date, tokens: mine.totalTokens };
86
+ return { kind: 'unknown' };
83
87
  return {
84
88
  kind: 'behind',
85
89
  server: mine.server,
@@ -160,6 +160,9 @@ export function isComparable(row, canonicalSha256) {
160
160
  canonicalSha256 !== null &&
161
161
  row.capturedSha256 === canonicalSha256 &&
162
162
  row.ourTokens > 0 &&
163
+ // A report that parsed but carries `total: 0` is not a measurement of
164
+ // anything; published, it renders as a −100% divergence.
165
+ row.cliTokens > 0 &&
163
166
  row.ourMappedTokens > 0);
164
167
  }
165
168
  /**
@@ -55,6 +55,14 @@ export function mappedTokens(raw) {
55
55
  export function fieldSelectionShare(row) {
56
56
  if (row.o200kFull <= 0)
57
57
  return null;
58
+ // The projection can add bytes rather than remove them — `inputSchema`
59
+ // becomes the longer `input_schema`, and a tool with no description gains
60
+ // `description: ""` — so a server with almost no metadata to drop can map
61
+ // *larger* than it measured. There is no share of the payload removed in that
62
+ // case, and publishing a negative one reads as "−11.1% of the capture is
63
+ // MCP-only metadata", which is not a thing.
64
+ if (row.o200kMapped > row.o200kFull)
65
+ return null;
58
66
  return (row.o200kFull - row.o200kMapped) / row.o200kFull;
59
67
  }
60
68
  /**
@@ -124,7 +124,20 @@ export interface ToolAttribution {
124
124
  */
125
125
  unexplainedTokens: number;
126
126
  }
127
- export declare function attribute(from: ToolVectorEntry, to: ToolVectorEntry, deltaTokens: number): ToolAttribution;
127
+ /**
128
+ * Per-tool attribution, or null when the names cannot carry it.
129
+ *
130
+ * The breakdown matches tools by name, so a name that appears twice on either
131
+ * side makes the maps below lose one of them silently — and the lost tokens
132
+ * resurface as `unexplainedTokens`, which the report explains to the reader as
133
+ * canonical-array framing bytes. That is a confident false explanation. Two
134
+ * ways it happens: a server that ships duplicate or namespaced-collapsed tool
135
+ * names, and `measureTools` recording every nameless tool as the single key
136
+ * `(unnamed)` — an invented name, where `toolNames` and `toAnthropicTools`
137
+ * deliberately drop nameless tools rather than invent one. Where the names
138
+ * cannot identify the tools, there is no attribution to give.
139
+ */
140
+ export declare function attribute(from: ToolVectorEntry, to: ToolVectorEntry, deltaTokens: number): ToolAttribution | null;
128
141
  export interface CostChange {
129
142
  server: string;
130
143
  fromDate: string;
@@ -87,7 +87,23 @@ export function mechanismOf(deltaTokens, deltaTools) {
87
87
  // and rewritten, and the totals cannot separate the two.
88
88
  return 'mixed';
89
89
  }
90
+ /**
91
+ * Per-tool attribution, or null when the names cannot carry it.
92
+ *
93
+ * The breakdown matches tools by name, so a name that appears twice on either
94
+ * side makes the maps below lose one of them silently — and the lost tokens
95
+ * resurface as `unexplainedTokens`, which the report explains to the reader as
96
+ * canonical-array framing bytes. That is a confident false explanation. Two
97
+ * ways it happens: a server that ships duplicate or namespaced-collapsed tool
98
+ * names, and `measureTools` recording every nameless tool as the single key
99
+ * `(unnamed)` — an invented name, where `toolNames` and `toAnthropicTools`
100
+ * deliberately drop nameless tools rather than invent one. Where the names
101
+ * cannot identify the tools, there is no attribution to give.
102
+ */
90
103
  export function attribute(from, to, deltaTokens) {
104
+ const unique = (ts) => new Set(ts.map((t) => t.name)).size === ts.length;
105
+ if (!unique(from.tools) || !unique(to.tools))
106
+ return null;
91
107
  const before = new Map(from.tools.map((t) => [t.name, t.tokens]));
92
108
  const after = new Map(to.tools.map((t) => [t.name, t.tokens]));
93
109
  const added = [];
@@ -162,13 +178,36 @@ export function latestChange(server, rows, vectors) {
162
178
  if (deltaTokens === 0 && deltaTools === 0)
163
179
  return null;
164
180
  const deltaPct = from.tokens > 0 ? (deltaTokens / from.tokens) * 100 : 0;
165
- // Attribution needs both sides on record, matched by date. A vector file that
166
- // only holds the newer capture explains nothing about how the server got there.
181
+ // Attribution needs both sides on record. Matched by *cost as of that date*,
182
+ // not by date equality: vectors are deduped by capture and keep the first
183
+ // date a capture was seen, while `from` is the last row of the previous
184
+ // plateau — so the two dates coincide only when the previous cost was
185
+ // measured exactly once. With weekly sweeps and less frequent releases they
186
+ // almost never do, and a date-equality join therefore reported "only one of
187
+ // the two captures is on record" while holding both. The vector in force on a
188
+ // given day is the newest one recorded on or before it.
167
189
  let attribution = null;
168
- const fromVec = vectors?.entries.find((e) => e.date === from.date);
169
- const toVec = vectors?.entries.find((e) => e.date === to.date);
170
- if (fromVec && toVec)
190
+ const inForceOn = (date, tokensThatDay) => {
191
+ const upto = (vectors?.entries ?? []).filter((e) => e.date <= date);
192
+ // A same-day re-sweep replaces that day's history row but *appends* a
193
+ // capture, so one date can carry two. Only one of them is the one the row
194
+ // describes: prefer the capture whose total is the number history recorded.
195
+ const agreeing = upto.filter((e) => e.totalTokens === tokensThatDay);
196
+ const pool = agreeing.length > 0 ? agreeing : upto;
197
+ return pool.reduce((best, e) => (!best || e.date >= best.date ? e : best), undefined);
198
+ };
199
+ const fromVec = inForceOn(from.date, from.tokens);
200
+ const toVec = inForceOn(to.date, to.tokens);
201
+ // A vector only explains the row it agrees with. If the totals disagree the
202
+ // file does not cover this change — say so rather than attributing a delta
203
+ // to the wrong capture and publishing the mismatch as framing bytes.
204
+ if (fromVec &&
205
+ toVec &&
206
+ fromVec.canonicalSha256 !== toVec.canonicalSha256 &&
207
+ fromVec.totalTokens === from.tokens &&
208
+ toVec.totalTokens === to.tokens) {
171
209
  attribution = attribute(fromVec, toVec, deltaTokens);
210
+ }
172
211
  return {
173
212
  server,
174
213
  fromDate: from.date,
@@ -23,10 +23,18 @@ export function quantileTable(values) {
23
23
  * checkable by anyone holding the same JSON.
24
24
  */
25
25
  export function percentileOf(quantiles, value) {
26
+ // The LOWEST percentile whose quantile the value reaches, not the highest.
27
+ // Taking the highest reports a value tied with half the measured set as p100
28
+ // — "heavier than 100% of measured tools" about something exactly average for
29
+ // its tie — because every percentile across the tie carries the same
30
+ // quantile. The lowest names where the tie begins, which is what "heavier
31
+ // than P% of tools" means.
26
32
  let p = 0;
27
33
  for (let i = 0; i <= 100; i++) {
28
- if (quantiles[i] <= value)
29
- p = i;
34
+ if (quantiles[i] <= value) {
35
+ if (quantiles[i] < value || i === 0 || quantiles[i - 1] < quantiles[i])
36
+ p = i;
37
+ }
30
38
  else
31
39
  break;
32
40
  }
@@ -6,6 +6,8 @@
6
6
  import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'node:fs';
7
7
  import { dirname, join } from 'node:path';
8
8
  import { parse } from 'yaml';
9
+ import { isCurrent } from '../core/divergence.js';
10
+ import { loadRows } from './report.js';
9
11
  import { bandColor, BAND_META } from '../core/bands.js';
10
12
  import { parseHistory, plottableSeries } from './history.js';
11
13
  /** Longest series a sparkline plots — a stat-tile trend, not a full chart. */
@@ -52,10 +54,9 @@ export function generateDashboard(root = process.cwd()) {
52
54
  // Only the run of sweeps taken under the same isolation is plotted: a step
53
55
  // across an isolation change is the harness moving, not the server.
54
56
  const seriesFor = (name) => plottableSeries(history.filter((h) => h.server === name));
55
- const rows = doc.servers.map((entry) => {
56
- const p = join(root, 'results', entry.name, 'measurement.json');
57
- return { entry, m: existsSync(p) ? JSON.parse(readFileSync(p, 'utf8')) : null };
58
- });
57
+ // Shared with the leaderboard rather than re-read here: one definition of how
58
+ // a measurement is loaded, including its tolerance for a half-written file.
59
+ const rows = loadRows(doc.servers, root);
59
60
  const measured = rows
60
61
  .filter((r) => r.m && (r.m.status === 'measured' || r.m.status === 'dynamic') && r.m.totalTokens !== null)
61
62
  .sort((a, b) => (b.m.totalTokens ?? 0) - (a.m.totalTokens ?? 0));
@@ -74,7 +75,12 @@ export function generateDashboard(root = process.cwd()) {
74
75
  const meta = BAND_META[band];
75
76
  const largest = [...m.tools].sort((a, b) => b.tokens - a.tokens)[0];
76
77
  const pct = Math.max(1.2, (t / max) * 100);
77
- const div = dSrv[r.entry.name];
78
+ // Gated on the capture it was computed from, exactly as the leaderboard
79
+ // gates it: four rows here were publishing a Claude cost derived from
80
+ // bytes that no longer exist, while leaderboard.md printed `—` for the
81
+ // same four.
82
+ const dRaw = dSrv[r.entry.name];
83
+ const div = isCurrent(dRaw, m.canonicalSha256 ?? null) ? dRaw : undefined;
78
84
  const claudeTip = div ? ` · in a Claude request: ${fmt(div.claudeDelta)} tok` : '';
79
85
  const series = seriesFor(r.entry.name);
80
86
  const tokens = series.rows.map((h) => h.tokens);
@@ -107,10 +113,15 @@ export function generateDashboard(root = process.cwd()) {
107
113
  const tableRows = measured
108
114
  .map((r, i) => {
109
115
  const m = r.m;
110
- const div = dSrv[r.entry.name];
116
+ // Gated on the capture it was computed from, exactly as the leaderboard
117
+ // gates it: four rows here were publishing a Claude cost derived from
118
+ // bytes that no longer exist, while leaderboard.md printed `—` for the
119
+ // same four.
120
+ const dRaw = dSrv[r.entry.name];
121
+ const div = isCurrent(dRaw, m.canonicalSha256 ?? null) ? dRaw : undefined;
111
122
  const tokens = seriesFor(r.entry.name).rows.map((h) => h.tokens);
112
123
  const trend = tokens.length > 1
113
- ? `${tokens[tokens.length - 1] - tokens[0] >= 0 ? '+' : ''}${fmt(tokens[tokens.length - 1] - tokens[0])} / ${tokens.length}d`
124
+ ? `${tokens[tokens.length - 1] - tokens[0] >= 0 ? '+' : ''}${fmt(tokens[tokens.length - 1] - tokens[0])} over ${tokens.length} sweeps`
114
125
  : '—';
115
126
  return `<tr><td>${i + 1}</td><td>${esc(r.entry.name)}</td><td class="num">${fmt(m.totalTokens)}</td><td class="num">${div ? fmt(div.claudeDelta) : '—'}</td><td class="num">${esc(m.toolCount)}</td><td>${esc(BAND_META[bandColor(m.totalTokens)].label)}</td><td>${esc(r.entry.category)}</td><td class="num">${trend}</td></tr>`;
116
127
  })
@@ -227,7 +238,7 @@ export function generateDashboard(root = process.cwd()) {
227
238
  </div>
228
239
 
229
240
  <h2>Leaderboard</h2>
230
- <p class="h2sub">Tokens = o200k_base count of the canonical <code>tools/list</code> bytes — the wire payload. What a <em>Claude request</em> actually carries can differ sharply (github: 54,422 on the wire, ${dSrv['github'] ? fmt(dSrv['github'].claudeDelta) : '…'} in a request 80% of its schema bytes are fields no Anthropic request sends). The trend line plots tokens across every sweep date on record (oldest→newest, dot = current); servers with one sweep so far show no line yet, and a sweep taken under different isolation than the current one is left out rather than drawn as a change in the server. Hover a row for exact numbers; <a href="METHODOLOGY.html#claude-divergence">method</a>.</p>
241
+ <p class="h2sub">Tokens = o200k_base count of the canonical <code>tools/list</code> bytes — the wire payload. What a <em>Claude request</em> actually carries can differ sharply (a server can spend most of its schema bytes on fields no Anthropic request sends, and the denser tokenizer can push the rest back up). The trend line plots tokens across every sweep date on record (oldest→newest, dot = current); servers with one sweep so far show no line yet, and a sweep taken under different isolation than the current one is left out rather than drawn as a change in the server. Hover a row for exact numbers; <a href="METHODOLOGY.html#claude-divergence">method</a>.</p>
231
242
  <div class="board">
232
243
  ${barRows || '<p class="h2sub">Sweep in progress — first results land shortly.</p>'}
233
244
  </div>
@@ -47,7 +47,15 @@ export interface Verdict {
47
47
  * `current` maps server name to the status it just measured at. Servers absent
48
48
  * from it were not swept and are ignored.
49
49
  */
50
- export declare function verdict(prior: Snapshot[], current: Map<string, MeasurementStatus>): Verdict;
50
+ export declare function verdict(prior: Snapshot[], current: Map<string, MeasurementStatus>,
51
+ /**
52
+ * Servers this sweep could not measure because docker itself failed. They
53
+ * never reached a status, so they are absent from `current` and invisible to
54
+ * the comparison below — but they are the same fact it is looking for: a
55
+ * server that could have produced a number and did not. Counted here so the
56
+ * two symptoms share one threshold instead of being each other's blind spot.
57
+ */
58
+ dockerFaults?: number): Verdict;
51
59
  /**
52
60
  * Put the snapshotted artifacts back, byte for byte. Only servers named in
53
61
  * `names` are touched, and only where a prior file existed — a server whose
@@ -78,12 +78,21 @@ export function snapshot(names, root = process.cwd()) {
78
78
  * `current` maps server name to the status it just measured at. Servers absent
79
79
  * from it were not swept and are ignored.
80
80
  */
81
- export function verdict(prior, current) {
81
+ export function verdict(prior, current,
82
+ /**
83
+ * Servers this sweep could not measure because docker itself failed. They
84
+ * never reached a status, so they are absent from `current` and invisible to
85
+ * the comparison below — but they are the same fact it is looking for: a
86
+ * server that could have produced a number and did not. Counted here so the
87
+ * two symptoms share one threshold instead of being each other's blind spot.
88
+ */
89
+ dockerFaults = 0) {
82
90
  const comparableNames = prior
83
91
  .filter((s) => s.status !== null && isGood(s.status) && current.has(s.name))
84
92
  .map((s) => s.name);
85
93
  const regressed = comparableNames.filter((n) => !isGood(current.get(n)));
86
- const comparable = comparableNames.length;
94
+ const failed = regressed.length + dockerFaults;
95
+ const comparable = comparableNames.length + dockerFaults;
87
96
  if (comparable === 0) {
88
97
  return {
89
98
  fault: false,
@@ -94,15 +103,16 @@ export function verdict(prior, current) {
94
103
  reason: 'no prior measurement to compare against — harness check not performed',
95
104
  };
96
105
  }
97
- const ratio = regressed.length / comparable;
106
+ const ratio = failed / comparable;
98
107
  const pct = (ratio * 100).toFixed(0);
99
- if (regressed.length >= MIN_REGRESSIONS && ratio >= FAULT_RATIO) {
108
+ const how = dockerFaults > 0 ? ` (${regressed.length} regressed, ${dockerFaults} unmeasurable)` : '';
109
+ if (failed >= MIN_REGRESSIONS && ratio >= FAULT_RATIO) {
100
110
  return {
101
111
  fault: true,
102
112
  regressed,
103
113
  comparable,
104
- reason: `${regressed.length} of ${comparable} previously-measured servers (${pct}%) failed in ` +
105
- `this sweep — at or above the ${MIN_REGRESSIONS}-server, ` +
114
+ reason: `${failed} of ${comparable} previously-measured servers (${pct}%) produced no number in ` +
115
+ `this sweep${how} — at or above the ${MIN_REGRESSIONS}-server, ` +
106
116
  `${(FAULT_RATIO * 100).toFixed(0)}% threshold that reads as a broken harness ` +
107
117
  `rather than broken servers`,
108
118
  };
@@ -111,8 +121,8 @@ export function verdict(prior, current) {
111
121
  fault: false,
112
122
  regressed,
113
123
  comparable,
114
- reason: `${regressed.length} of ${comparable} previously-measured servers (${pct}%) failed in ` +
115
- `this sweep — below the harness-fault threshold, publishing normally`,
124
+ reason: `${failed} of ${comparable} previously-measured servers (${pct}%) produced no number in ` +
125
+ `this sweep${how} — below the harness-fault threshold, publishing normally`,
116
126
  };
117
127
  }
118
128
  /**
@@ -28,17 +28,22 @@ export interface PublishedStats {
28
28
  /** Rows the leaderboard prints a claude number for: measured AND capture-current. */
29
29
  currentCount: number;
30
30
  heaviestClaudeName: string | null;
31
+ /** `claudeTokens` is null when the published row no longer matches the capture on disk. */
31
32
  github: {
32
33
  badgeTokens: number;
33
- mappedTokens: number;
34
- claudeTokens: number;
35
- droppedPct: number;
34
+ claudeTokens: number | null;
36
35
  };
37
36
  notion: {
38
37
  badgeTokens: number;
39
- claudeTokens: number;
38
+ claudeTokens: number | null;
40
39
  };
41
- /** Field-selection share across the run's rows, as fractions of the payload. */
40
+ /** The current row showing the largest field-selection effect, and its two counts. */
41
+ widest: {
42
+ server: string;
43
+ full: number;
44
+ mapped: number;
45
+ };
46
+ /** Field-selection share across the run's *current* rows, as fractions of the payload. */
42
47
  shareMin: number;
43
48
  shareMax: number;
44
49
  /** claudeDelta / o200kFull across the run's rows that carry a number. */
@@ -75,25 +75,48 @@ export function computePublishedStats(entries, root = process.cwd()) {
75
75
  }
76
76
  if (!div)
77
77
  throw new Error('results/divergence.json is missing — README states its numbers');
78
- const divRow = (name) => {
78
+ /**
79
+ * A divergence row only where it still describes the capture on disk.
80
+ *
81
+ * The staleness gate is the whole discipline of this column, and skipping it
82
+ * here is how README came to print two different costs for github on one
83
+ * page: 54,422 from a row computed against bytes that no longer existed,
84
+ * beside 54,622 from the measurement. `withClaude` below already applied
85
+ * `isCurrent` to the very same run — the rule guarded one number and not its
86
+ * neighbour.
87
+ */
88
+ const currentDivRow = (name) => {
79
89
  const d = div.servers[name];
80
90
  if (!d)
81
91
  throw new Error(`README's Claude table names ${name}, which is not in the divergence run`);
82
- return d;
92
+ const onDisk = rows.find((r) => r.entry.name === name)?.m?.canonicalSha256 ?? null;
93
+ return isCurrent(d, onDisk) ? d : null;
83
94
  };
84
- const github = divRow('github');
85
- const notion = divRow('notion');
86
- const githubShare = fieldSelectionShare(github);
87
- if (githubShare === null)
88
- throw new Error('divergence run carries no o200k counts for github');
89
- const runRows = Object.values(div.servers);
90
- const shares = runRows.map((r) => fieldSelectionShare(r)).filter((s) => s !== null);
91
- const ratios = runRows
92
- .filter((r) => typeof r.claudeDelta === 'number' && r.claudeDelta > 0 && r.o200kFull > 0)
93
- .map((r) => r.claudeDelta / r.o200kFull);
95
+ /** The badge number comes from the measurement, never from a divergence row's copy of it. */
96
+ const badgeTokensOf = (name) => {
97
+ const r = measured.find((x) => x.entry.name === name);
98
+ if (!r)
99
+ throw new Error(`README's Claude table names ${name}, which has no current measurement`);
100
+ return r.m.totalTokens;
101
+ };
102
+ // Ranges are stated over the rows that are still current, for the same
103
+ // reason: a range whose endpoint comes from a superseded capture describes a
104
+ // set that no longer exists.
105
+ const currentRows = Object.entries(div.servers).filter(([name, r]) => isCurrent(r, rows.find((x) => x.entry.name === name)?.m?.canonicalSha256 ?? null));
106
+ const shares = currentRows
107
+ .map(([, r]) => fieldSelectionShare(r))
108
+ .filter((s) => s !== null && s >= 0);
109
+ const ratios = currentRows
110
+ .filter(([, r]) => typeof r.claudeDelta === 'number' && r.claudeDelta > 0 && r.o200kFull > 0)
111
+ .map(([, r]) => r.claudeDelta / r.o200kFull);
94
112
  if (shares.length === 0 || ratios.length === 0) {
95
- throw new Error('divergence run carries no usable rows — METHODOLOGY states its ranges');
113
+ throw new Error('no current divergence row — METHODOLOGY states ranges over them; run `npm run divergence`');
96
114
  }
115
+ // The exemplar METHODOLOGY names for the field-selection effect is whichever
116
+ // current row shows it most, rather than a server hardcoded into the prose.
117
+ const widest = currentRows
118
+ .filter(([, r]) => (fieldSelectionShare(r) ?? -1) >= 0)
119
+ .sort((a, b) => (fieldSelectionShare(b[1]) ?? 0) - (fieldSelectionShare(a[1]) ?? 0))[0];
97
120
  const withClaude = measured.filter((r) => isCurrent(div.servers[r.entry.name], r.m.canonicalSha256));
98
121
  const heaviest = [...withClaude].sort((a, b) => div.servers[b.entry.name].claudeDelta - div.servers[a.entry.name].claudeDelta)[0];
99
122
  const costlier = measured.filter((r) => {
@@ -118,13 +141,9 @@ export function computePublishedStats(entries, root = process.cwd()) {
118
141
  runSize: Object.keys(div.servers).length,
119
142
  currentCount: withClaude.length,
120
143
  heaviestClaudeName: heaviest?.entry.name ?? null,
121
- github: {
122
- badgeTokens: github.o200kFull,
123
- mappedTokens: github.o200kMapped,
124
- claudeTokens: github.claudeDelta,
125
- droppedPct: Math.round(githubShare * 100),
126
- },
127
- notion: { badgeTokens: notion.o200kFull, claudeTokens: notion.claudeDelta },
144
+ github: { badgeTokens: badgeTokensOf('github'), claudeTokens: currentDivRow('github')?.claudeDelta ?? null },
145
+ notion: { badgeTokens: badgeTokensOf('notion'), claudeTokens: currentDivRow('notion')?.claudeDelta ?? null },
146
+ widest: { server: widest[0], full: widest[1].o200kFull, mapped: widest[1].o200kMapped },
128
147
  shareMin: Math.min(...shares),
129
148
  shareMax: Math.max(...shares),
130
149
  ratioMin: Math.min(...ratios),
@@ -140,6 +159,8 @@ export function computePublishedStats(entries, root = process.cwd()) {
140
159
  }
141
160
  export const PAGE_FILES = ['README.md', 'docs/index.md', 'docs/METHODOLOGY.md'];
142
161
  const fmt = (n) => n.toLocaleString('en-US');
162
+ /** A number, or the em-dash that means "no current measurement" everywhere else here. */
163
+ const q = (n) => (n === null ? '—' : fmt(n));
143
164
  export const PAGE_CLAIMS = [
144
165
  {
145
166
  file: 'README.md',
@@ -210,14 +231,14 @@ export const PAGE_CLAIMS = [
210
231
  {
211
232
  file: 'README.md',
212
233
  id: 'claude-table:github',
213
- template: '| github | {n} | **{n}** | {d}% of the capture is `annotations`/`outputSchema` metadata Claude never sees |',
214
- values: (s) => [fmt(s.claude.github.badgeTokens), fmt(s.claude.github.claudeTokens), String(s.claude.github.droppedPct)],
234
+ template: '| github | {n} | **{q}** | most of the capture is `annotations`/`outputSchema` metadata Claude never sees |',
235
+ values: (s) => [fmt(s.claude.github.badgeTokens), q(s.claude.github.claudeTokens)],
215
236
  },
216
237
  {
217
238
  file: 'README.md',
218
239
  id: 'claude-table:notion',
219
- template: '| notion | {n} | **{n}** | almost no metadata to drop, so the tokenizer difference dominates |',
220
- values: (s) => [fmt(s.claude.notion.badgeTokens), fmt(s.claude.notion.claudeTokens)],
240
+ template: '| notion | {n} | **{q}** | almost no metadata to drop, so the tokenizer difference dominates |',
241
+ values: (s) => [fmt(s.claude.notion.badgeTokens), q(s.claude.notion.claudeTokens)],
221
242
  },
222
243
  {
223
244
  file: 'README.md',
@@ -252,12 +273,15 @@ export const PAGE_CLAIMS = [
252
273
  {
253
274
  file: 'docs/METHODOLOGY.md',
254
275
  id: 'divergence:share-range',
255
- template: 'this removes between {f}% and **{f}%** of the payload (github: {n} → {n} tokens).',
276
+ template: 'this removes between {f}% and **{f}%** of the payload ({w}: {n} → {n} tokens).',
277
+ // The exemplar is whichever current row shows the effect most, not a server
278
+ // named in the prose — a hardcoded name goes stale the week it is re-swept.
256
279
  values: (s) => [
257
280
  (s.claude.shareMin * 100).toFixed(1),
258
281
  (s.claude.shareMax * 100).toFixed(1),
259
- fmt(s.claude.github.badgeTokens),
260
- fmt(s.claude.github.mappedTokens),
282
+ s.claude.widest.server,
283
+ fmt(s.claude.widest.full),
284
+ fmt(s.claude.widest.mapped),
261
285
  ],
262
286
  },
263
287
  {
@@ -303,7 +327,7 @@ const escapeLiteral = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&').replace(/\
303
327
  export function compileTemplate(template) {
304
328
  let source = '';
305
329
  let last = 0;
306
- for (const slot of template.matchAll(/\{[ndwf]\}/g)) {
330
+ for (const slot of template.matchAll(/\{[ndwfq]\}/g)) {
307
331
  source += escapeLiteral(template.slice(last, slot.index));
308
332
  source +=
309
333
  slot[0] === '{n}'
@@ -312,7 +336,11 @@ export function compileTemplate(template) {
312
336
  ? '(\\d+)'
313
337
  : slot[0] === '{f}'
314
338
  ? '(\\d+\\.\\d+)'
315
- : '([A-Za-z0-9._-]+)';
339
+ : // `{q}` is a number that may not exist: the em-dash the leaderboard
340
+ // already prints for a row whose capture has moved on.
341
+ slot[0] === '{q}'
342
+ ? '([\\d,]+|—)'
343
+ : '([A-Za-z0-9._-]+)';
316
344
  last = slot.index + slot[0].length;
317
345
  }
318
346
  source += escapeLiteral(template.slice(last));
@@ -21,7 +21,20 @@ function csvCell(s) {
21
21
  export function loadRows(entries, root = process.cwd()) {
22
22
  return entries.map((entry) => {
23
23
  const p = join(root, 'results', entry.name, 'measurement.json');
24
- return { entry, m: existsSync(p) ? JSON.parse(readFileSync(p, 'utf8')) : null };
24
+ if (!existsSync(p))
25
+ return { entry, m: null };
26
+ try {
27
+ return { entry, m: JSON.parse(readFileSync(p, 'utf8')) };
28
+ }
29
+ catch {
30
+ // The same tolerance `appendHistory` states for the same file: a sweep
31
+ // killed mid-write leaves a truncated measurement, and every generator
32
+ // reads through here. Throwing meant one such file broke the leaderboard,
33
+ // the server pages, the dashboard, the tool-shape baseline and the
34
+ // published-stats check at once — weekly, with a SyntaxError that named
35
+ // no file. A server with no readable record reads as one with no record.
36
+ return { entry, m: null };
37
+ }
25
38
  });
26
39
  }
27
40
  /** results/divergence.json if a divergence run has been recorded, else null. */
@@ -25,6 +25,7 @@ import { join, resolve } from 'node:path';
25
25
  import { fileURLToPath } from 'node:url';
26
26
  import { parse } from 'yaml';
27
27
  import { measureServer } from './run.js';
28
+ import { DockerHarnessFault } from './docker.js';
28
29
  import { SESSION_START_METHOD, parseSessionStart, toSessionStartRow, } from '../core/session-start.js';
29
30
  export function loadSessionStart(root = process.cwd()) {
30
31
  const p = join(root, 'results', 'session-start.json');
@@ -63,15 +64,36 @@ if (isMain) {
63
64
  const queue = [...entries];
64
65
  async function worker() {
65
66
  for (let e = queue.shift(); e; e = queue.shift()) {
66
- const m = await measureServer(e.name, e.command, {
67
- timeoutMs: (e.timeoutSeconds ?? defaultTimeout) * 1000,
68
- docker,
69
- dockerImage: e.dockerImage,
70
- dummyEnv: e.env ?? [],
71
- dummyEnvValues: e.envValues,
72
- needsGit: e.needsGit,
73
- persist: false, // the measurements on disk are not this run's to rewrite
74
- });
67
+ let m;
68
+ try {
69
+ m = await measureServer(e.name, e.command, {
70
+ timeoutMs: (e.timeoutSeconds ?? defaultTimeout) * 1000,
71
+ docker,
72
+ dockerImage: e.dockerImage,
73
+ dummyEnv: e.env ?? [],
74
+ dummyEnvValues: e.envValues,
75
+ needsGit: e.needsGit,
76
+ persist: false, // the measurements on disk are not this run's to rewrite
77
+ });
78
+ }
79
+ catch (err) {
80
+ // One thrown error used to reject Promise.all and end the process
81
+ // before anything was written, discarding every capture the run had
82
+ // already completed — and skipping the `finally` that force-removes
83
+ // containers, orphaning the in-flight ones. A machine fault on one
84
+ // server is recorded against that server; the rest of the run stands.
85
+ if (!(err instanceof DockerHarnessFault))
86
+ throw err;
87
+ servers[e.name] = {
88
+ instructions: '',
89
+ instructionsTokens: 0,
90
+ instructionsSha256: '',
91
+ capturedSha256: null,
92
+ error: `docker harness fault: ${err.message.slice(0, 200)}`,
93
+ };
94
+ console.log(` ${e.name}: docker harness fault — recorded, run continues`);
95
+ continue;
96
+ }
75
97
  if (m.status !== 'measured' && m.status !== 'dynamic') {
76
98
  servers[e.name] = {
77
99
  instructions: '',
@@ -18,7 +18,7 @@ import { DockerHarnessFault } from './docker.js';
18
18
  import { writeLeaderboard } from './report.js';
19
19
  import { appendHistory } from './history.js';
20
20
  import { appendToolVectors, writeRegressions } from './regressions.js';
21
- import { FAULT_RATIO, MIN_REGRESSIONS, snapshot, verdict, restore } from './harness-guard.js';
21
+ import { snapshot, verdict, restore } from './harness-guard.js';
22
22
  import { selectShard, shardIndexForDate } from './shard.js';
23
23
  function arg(name) {
24
24
  const i = process.argv.indexOf(`--${name}`);
@@ -101,24 +101,25 @@ async function worker() {
101
101
  }
102
102
  }
103
103
  await Promise.all(Array.from({ length: Math.max(1, concurrency) }, () => worker()));
104
- // A docker fault on most of the slice is the harness-fault story with an
105
- // earlier symptom — the guard below can't see it (nothing regressed on disk;
106
- // the throws happened before anything was written), so it is judged here by
107
- // the guard's own thresholds. Below them, the sweep publishes what it did
108
- // measure and the faulted servers wait for the next cycle.
109
- if (dockerFaults.size >= MIN_REGRESSIONS && dockerFaults.size / entries.length >= FAULT_RATIO) {
110
- console.error(`\nHARNESS FAULT — docker could not run for ${dockerFaults.size} of ${entries.length} servers; ` +
111
- `refusing to publish this sweep.\n` +
112
- [...dockerFaults].map(([n, msg]) => ` ${n}: ${msg}`).join('\n') +
113
- `\n Nothing was overwritten — every previous record stands. ` +
114
- `Check the Docker daemon and registry path, then re-run.`);
115
- process.exit(1);
116
- }
117
104
  // Before publishing anything: is this sweep a statement about the servers, or
118
105
  // about the machine that measured them?
119
- const v = verdict(prior, statuses);
106
+ //
107
+ // The two symptoms are counted together, against one denominator. Judged apart
108
+ // they were each other's blind spot: a flaky daemon that throws for 6 of 14
109
+ // servers (6 ≥ 5, but 43% of the slice) and times out 4 more (4 < 5) trips
110
+ // neither threshold, and the sweep publishes with 10 of 14 servers producing no
111
+ // number and four good records overwritten with failures. A server that could
112
+ // have produced a number and didn't is one fact, however it failed.
113
+ const v = verdict(prior, statuses, dockerFaults.size);
120
114
  console.log(`harness check: ${v.reason}`);
115
+ if (dockerFaults.size > 0 && !v.fault) {
116
+ console.warn(` (${dockerFaults.size} docker fault(s) counted toward that check; those servers were not measured)`);
117
+ }
121
118
  if (v.fault) {
119
+ if (dockerFaults.size) {
120
+ console.error(`\ndocker could not run for ${dockerFaults.size} server(s):\n` +
121
+ [...dockerFaults].map(([n, msg]) => ` ${n}: ${msg}`).join('\n'));
122
+ }
122
123
  const restored = restore(prior, v.regressed);
123
124
  console.error(`\nHARNESS FAULT — refusing to publish this sweep.\n` +
124
125
  ` regressed: ${v.regressed.join(', ')}\n` +
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "mcp-context-cost",
3
- "version": "0.11.1",
3
+ "version": "0.11.3",
4
4
  "description": "Measure what your MCP servers cost in context tokens — audit your own config, or badge the server you publish",
5
5
  "type": "module",
6
6
  "license": "MIT",