mcp-context-cost 0.7.0 → 0.9.0

package/dist/sweep/published-stats.js

@@ -0,0 +1,383 @@
+/**
+ * The front pages' numbers, written by the same regeneration that writes the
+ * leaderboard — and checked against the same data in the suite.
+ *
+ * README and docs/index.md state numbers as prose: how many candidates, how
+ * many measured, the span, the sample tables, the Claude pair, the verify
+ * transcript. Those sentences were written by hand, so they were true on the
+ * day they were written and drifted with every scheduled re-sweep — by
+ * 2026-09-03 the leaderboard said 68 measured while both pages said 69, the
+ * exact front-page-contradicts-the-data failure repaired by hand once before
+ * (2026-08-20) and re-created by the first sweep after it.
+ *
+ * The deferral tables got the durable fix first: a test reads the page's own
+ * words against the resolver, so either side moving alone is a red check. That
+ * works there because the tables change only when code changes. These numbers
+ * change when *data* changes, on a schedule, with no human in the loop — a
+ * check alone would schedule its own red main. So the numbers get the
+ * leaderboard's treatment instead: regen patches them from results/, the
+ * scheduled jobs commit the pages beside the data, and the suite asserts the
+ * committed pages already agree — which fires only when someone changes data
+ * without running regen, or rewords a sentence regen maintains.
+ *
+ * Each claim is a template: fixed words with slots. The words have to be on
+ * the page as written (a missing anchor is a loud failure naming the claim,
+ * never a silent skip), and only the slots are ever rewritten — spliced in
+ * place, so the page's own line wrapping survives.
+ */
+import { readFileSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { DEFAULT_CONTEXT_WINDOW } from '../audit/audit.js';
+import { fieldSelectionShare, isCurrent } from '../core/divergence.js';
+import { sessionStartLoad } from '../core/session-start.js';
+import { isGood } from './harness-guard.js';
+import { loadDivergence, loadRows, loadSessionStartRun } from './report.js';
+/** Named in README's sample table — the choice is editorial, the numbers are not. */
+export const SAMPLE_SERVERS = [
+    'github',
+    'xcodebuildmcp',
+    'brave-search',
+    'notion',
+    'playwright',
+    'filesystem',
+    'markitdown',
+];
+export function floorToTwoSignificant(n) {
+    const whole = Math.floor(n);
+    if (whole < 100)
+        return whole;
+    const magnitude = 10 ** (Math.floor(Math.log10(whole)) - 1);
+    return Math.floor(whole / magnitude) * magnitude;
+}
+export function computePublishedStats(entries, root = process.cwd()) {
+    const rows = loadRows(entries, root);
+    const div = loadDivergence(root);
+    const ss = loadSessionStartRun(root);
+    const measured = rows
+        .filter((r) => r.m !== null && isGood(r.m.status))
+        .filter((r) => typeof r.m.totalTokens === 'number')
+        .sort((a, b) => b.m.totalTokens - a.m.totalTokens);
+    if (measured.length < 2)
+        throw new Error('fewer than two measured servers on disk — published stats cannot be computed');
+    const asPair = (r) => ({ name: r.entry.name, tokens: r.m.totalTokens });
+    const max = asPair(measured[0]);
+    const min = asPair(measured[measured.length - 1]);
+    if (min.tokens <= 0)
+        throw new Error(`cheapest measured server (${min.name}) has no positive token count`);
+    const sample = {};
+    for (const name of SAMPLE_SERVERS) {
+        const r = measured.find((x) => x.entry.name === name);
+        if (!r || typeof r.m.toolCount !== 'number') {
+            throw new Error(`README's sample table names ${name}, which has no current measurement`);
+        }
+        sample[name] = { tokens: r.m.totalTokens, tools: r.m.toolCount };
+    }
+    if (!div)
+        throw new Error('results/divergence.json is missing — README states its numbers');
+    const divRow = (name) => {
+        const d = div.servers[name];
+        if (!d)
+            throw new Error(`README's Claude table names ${name}, which is not in the divergence run`);
+        return d;
+    };
+    const github = divRow('github');
+    const notion = divRow('notion');
+    const githubShare = fieldSelectionShare(github);
+    if (githubShare === null)
+        throw new Error('divergence run carries no o200k counts for github');
+    const runRows = Object.values(div.servers);
+    const shares = runRows.map((r) => fieldSelectionShare(r)).filter((s) => s !== null);
+    const ratios = runRows
+        .filter((r) => typeof r.claudeDelta === 'number' && r.claudeDelta > 0 && r.o200kFull > 0)
+        .map((r) => r.claudeDelta / r.o200kFull);
+    if (shares.length === 0 || ratios.length === 0) {
+        throw new Error('divergence run carries no usable rows — METHODOLOGY states its ranges');
+    }
+    const withClaude = measured.filter((r) => isCurrent(div.servers[r.entry.name], r.m.canonicalSha256));
+    const heaviest = [...withClaude].sort((a, b) => div.servers[b.entry.name].claudeDelta - div.servers[a.entry.name].claudeDelta)[0];
+    const costlier = measured.filter((r) => {
+        const load = sessionStartLoad(r.m, ss?.servers[r.entry.name]);
+        return load !== null && load.totalTokens >= r.m.totalTokens;
+    });
+    const githubRow = rows.find((r) => r.entry.name === 'github');
+    if (!githubRow?.m || typeof githubRow.m.totalTokens !== 'number') {
+        throw new Error('README quotes `verify` on results/github, which has no current measurement');
+    }
+    return {
+        candidateTotal: rows.length,
+        measuredCount: measured.length,
+        max,
+        second: asPair(measured[1]),
+        min,
+        spanTimes: floorToTwoSignificant(max.tokens / min.tokens),
+        maxContextSharePct: Math.round((max.tokens / DEFAULT_CONTEXT_WINDOW) * 100),
+        sample,
+        claude: {
+            runSize: Object.keys(div.servers).length,
+            currentCount: withClaude.length,
+            heaviestClaudeName: heaviest?.entry.name ?? null,
+            github: {
+                badgeTokens: github.o200kFull,
+                mappedTokens: github.o200kMapped,
+                claudeTokens: github.claudeDelta,
+                droppedPct: Math.round(githubShare * 100),
+            },
+            notion: { badgeTokens: notion.o200kFull, claudeTokens: notion.claudeDelta },
+            shareMin: Math.min(...shares),
+            shareMax: Math.max(...shares),
+            ratioMin: Math.min(...ratios),
+            ratioMax: Math.max(...ratios),
+        },
+        deferralCostlierCount: costlier.length,
+        verify: {
+            serverName: githubRow.m.serverName ?? 'github',
+            tokens: githubRow.m.totalTokens,
+        },
+    };
+}
+export const PAGE_FILES = ['README.md', 'docs/index.md', 'docs/METHODOLOGY.md'];
+const fmt = (n) => n.toLocaleString('en-US');
+export const PAGE_CLAIMS = [
+    {
+        file: 'README.md',
+        id: 'span',
+        template: 'across the {n} servers measured, cost spans **{n}×**, from `{w}` at {n} tokens to `{w}` at {n}.',
+        values: (s) => [fmt(s.measuredCount), fmt(s.spanTimes), s.min.name, fmt(s.min.tokens), s.max.name, fmt(s.max.tokens)],
+    },
+    {
+        file: 'README.md',
+        id: 'sample:github',
+        template: '| github (official) | **{n} tokens** | {n} |',
+        values: (s) => [fmt(s.sample.github.tokens), fmt(s.sample.github.tools)],
+    },
+    {
+        file: 'README.md',
+        id: 'sample:xcodebuildmcp',
+        template: '| xcodebuildmcp | {n} | {n} |',
+        values: (s) => [fmt(s.sample.xcodebuildmcp.tokens), fmt(s.sample.xcodebuildmcp.tools)],
+    },
+    {
+        file: 'README.md',
+        id: 'sample:brave-search',
+        template: '| brave-search | {n} | {n} |',
+        values: (s) => [fmt(s.sample['brave-search'].tokens), fmt(s.sample['brave-search'].tools)],
+    },
+    {
+        file: 'README.md',
+        id: 'sample:notion',
+        template: '| notion | {n} | {n} |',
+        values: (s) => [fmt(s.sample.notion.tokens), fmt(s.sample.notion.tools)],
+    },
+    {
+        file: 'README.md',
+        id: 'sample:playwright',
+        template: '| playwright *(4.8M installs/week)* | {n} | {n} |',
+        values: (s) => [fmt(s.sample.playwright.tokens), fmt(s.sample.playwright.tools)],
+    },
+    {
+        file: 'README.md',
+        id: 'sample:filesystem',
+        template: '| filesystem (reference) | {n} | {n} |',
+        values: (s) => [fmt(s.sample.filesystem.tokens), fmt(s.sample.filesystem.tools)],
+    },
+    {
+        file: 'README.md',
+        id: 'sample:markitdown',
+        template: '| markitdown | {n} | {n} |',
+        values: (s) => [fmt(s.sample.markitdown.tokens), fmt(s.sample.markitdown.tools)],
+    },
+    {
+        file: 'README.md',
+        id: 'measured-of-candidates',
+        template: '*({n} of {n} popular servers measured, each row dated by its own most recent sweep — full table in',
+        values: (s) => [fmt(s.measuredCount), fmt(s.candidateTotal)],
+    },
+    {
+        file: 'README.md',
+        id: 'divergence-run-size',
+        template: 'The run holds {n} rows — the top {n} measured servers by tokens when it ran',
+        values: (s) => [fmt(s.claude.runSize), fmt(s.claude.runSize)],
+    },
+    {
+        file: 'README.md',
+        id: 'divergence-current-count',
+        template: 'prints a claude number for the {n} that still match today and silence for the rest',
+        values: (s) => [fmt(s.claude.currentCount)],
+    },
+    {
+        file: 'README.md',
+        id: 'claude-table:github',
+        template: '| github | {n} | **{n}** | {d}% of the capture is `annotations`/`outputSchema` metadata Claude never sees |',
+        values: (s) => [fmt(s.claude.github.badgeTokens), fmt(s.claude.github.claudeTokens), String(s.claude.github.droppedPct)],
+    },
+    {
+        file: 'README.md',
+        id: 'claude-table:notion',
+        template: '| notion | {n} | **{n}** | almost no metadata to drop, so the tokenizer difference dominates |',
+        values: (s) => [fmt(s.claude.notion.badgeTokens), fmt(s.claude.notion.claudeTokens)],
+    },
+    {
+        file: 'README.md',
+        id: 'verify-transcript',
+        template: '# OK {w}: {d} tokens (o200k_base, methodology 1.0) — capture, hash, and count all agree',
+        values: (s) => [s.verify.serverName, String(s.verify.tokens)],
+    },
+    {
+        file: 'docs/index.md',
+        id: 'index:counts',
+        template: 'We measure {n} popular MCP servers; {n} have a number today, and every failure is listed with its reason.',
+        values: (s) => [fmt(s.candidateTotal), fmt(s.measuredCount)],
+    },
+    {
+        file: 'docs/index.md',
+        id: 'index:span',
+        template: 'The spread is {n}×: from `{w}` at {n} tokens to `{w}` at **{n} tokens** — {d}% of a 200K context window, before the agent takes a single action.',
+        values: (s) => [fmt(s.spanTimes), s.min.name, fmt(s.min.tokens), s.max.name, fmt(s.max.tokens), String(s.maxContextSharePct)],
+    },
+    {
+        file: 'docs/index.md',
+        id: 'index:second-heaviest',
+        template: 'Second-heaviest is `{w}` at {n}.',
+        values: (s) => [s.second.name, fmt(s.second.tokens)],
+    },
+    {
+        file: 'docs/METHODOLOGY.md',
+        id: 'divergence:share-range',
+        template: 'this removes between {f}% and **{f}%** of the payload (github: {n} → {n} tokens).',
+        values: (s) => [
+            (s.claude.shareMin * 100).toFixed(1),
+            (s.claude.shareMax * 100).toFixed(1),
+            fmt(s.claude.github.badgeTokens),
+            fmt(s.claude.github.mappedTokens),
+        ],
+    },
+    {
+        file: 'docs/METHODOLOGY.md',
+        id: 'divergence:ratio-range',
+        template: 'it ranged from {f}× to {f}× across the top {n},',
+        values: (s) => [s.claude.ratioMin.toFixed(2), s.claude.ratioMax.toFixed(2), fmt(s.claude.runSize)],
+    },
+    {
+        file: 'docs/METHODOLOGY.md',
+        id: 'divergence:heaviest-pair',
+        template: '{w} is the heaviest server on o200k and {w} is the heaviest on Claude.',
+        values: (s) => {
+            if (!s.claude.heaviestClaudeName) {
+                throw new Error('no row has a current claude number — the heaviest-on-Claude sentence cannot be maintained');
+            }
+            return [s.max.name, s.claude.heaviestClaudeName];
+        },
+    },
+];
+export const CHECK_CLAIMS = [
+    {
+        file: 'README.md',
+        id: 'heaviest-differs-on-claude',
+        words: 'the heaviest server on the badge is not the heaviest server on Claude',
+        holds: (s) => s.claude.heaviestClaudeName === null
+            ? 'no row has a current claude number, so there is no heaviest server on Claude'
+            : s.claude.heaviestClaudeName === s.max.name
+                ? `the heaviest server on the badge (${s.max.name}) IS the heaviest on Claude now`
+                : null,
+    },
+    {
+        file: 'README.md',
+        id: 'deferring-costlier-somewhere',
+        words: 'for at least one server in the published set it costs **more** than loading the definitions would',
+        holds: (s) => s.deferralCostlierCount >= 1
+            ? null
+            : 'no measured server currently costs more at session start than its definitions',
+    },
+];
+const escapeLiteral = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&').replace(/\s+/g, '\\s+');
+/** Fixed words with `\s+` for every gap (prose wraps; a claim is its words, not its layout). */
+export function compileTemplate(template) {
+    let source = '';
+    let last = 0;
+    for (const slot of template.matchAll(/\{[ndwf]\}/g)) {
+        source += escapeLiteral(template.slice(last, slot.index));
+        source +=
+            slot[0] === '{n}'
+                ? '([\\d,]+)'
+                : slot[0] === '{d}'
+                    ? '(\\d+)'
+                    : slot[0] === '{f}'
+                        ? '(\\d+\\.\\d+)'
+                        : '([A-Za-z0-9._-]+)';
+        last = slot.index + slot[0].length;
+    }
+    source += escapeLiteral(template.slice(last));
+    return new RegExp(source, 'dg');
+}
+/** Apply one claim: find its anchor exactly once, splice the slots to `want`, touch nothing else. */
+export function applyClaim(text, claim, want) {
+    const matches = [...text.matchAll(compileTemplate(claim.template))];
+    if (matches.length !== 1) {
+        return {
+            text,
+            changed: false,
+            problem: matches.length === 0
+                ? `${claim.file}: claim '${claim.id}' not found — the sentence regen maintains is gone or reworded`
+                : `${claim.file}: claim '${claim.id}' matches ${matches.length} places — the anchor is ambiguous`,
+        };
+    }
+    const match = matches[0];
+    const got = match.slice(1);
+    if (got.length !== want.length) {
+        return {
+            text,
+            changed: false,
+            problem: `${claim.file}: claim '${claim.id}' has ${got.length} slots but ${want.length} values — the claim itself is broken`,
+        };
+    }
+    if (got.every((g, i) => g === want[i]))
+        return { text, problem: null, changed: false };
+    // Right-to-left so earlier slot offsets stay valid; the `d` flag supplies indices.
+    const indices = match.indices;
+    for (let g = want.length; g >= 1; g--) {
+        const [start, end] = indices[g];
+        text = text.slice(0, start) + want[g - 1] + text.slice(end);
+    }
+    return { text, problem: null, changed: true };
+}
+/** Apply every claim for one page to its text. Slots are spliced in place; anchors are never rewritten. */
+export function patchPageText(file, text, stats) {
+    const problems = [];
+    const updated = [];
+    for (const claim of PAGE_CLAIMS.filter((c) => c.file === file)) {
+        const applied = applyClaim(text, claim, claim.values(stats));
+        text = applied.text;
+        if (applied.problem)
+            problems.push(applied.problem);
+        else if (applied.changed)
+            updated.push(claim.id);
+    }
+    return { text, problems, updated };
+}
+/** Compute stats and report what regen would rewrite, without writing anything. */
+export function verifyPublishedPages(entries, root = process.cwd()) {
+    return applyTo(entries, root, false);
+}
+/** Compute stats and rewrite the pages in place. Returns what changed and any refusals. */
+export function applyPublishedStats(entries, root = process.cwd()) {
+    return applyTo(entries, root, true);
+}
+function applyTo(entries, root, write) {
+    const stats = computePublishedStats(entries, root);
+    const problems = [];
+    const updated = [];
+    const changedFiles = [];
+    for (const file of PAGE_FILES) {
+        const path = join(root, file);
+        const before = readFileSync(path, 'utf8');
+        const patch = patchPageText(file, before, stats);
+        problems.push(...patch.problems);
+        updated.push(...patch.updated);
+        if (patch.text !== before) {
+            changedFiles.push(file);
+            if (write)
+                writeFileSync(path, patch.text);
+        }
+    }
+    return { problems, updated, changedFiles };
+}

package/dist/sweep/regen.js

@@ -5,6 +5,7 @@ import { writeLeaderboard, percentiles } from './report.js';
 import { appendHistory } from './history.js';
 import { writeServerPages } from './server-pages.js';
 import { writeDashboard } from './dashboard.js';
+import { applyPublishedStats } from './published-stats.js';
 const doc = parse(readFileSync('servers.yaml', 'utf8'));
 writeLeaderboard(doc.servers);
 // History first: the server pages read history.csv for their over-time table.
@@ -13,7 +14,19 @@ const p = writeServerPages(doc.servers);
 // The dashboard reads the same results/ and history.csv as the pages do, so it
 // belongs in the same refresh — see writeDashboard's note on why it wasn't.
 const d = writeDashboard();
+// The front pages state numbers the sweep just changed; they are patched from
+// the same results/ the leaderboard was. A missing anchor is a page regen can
+// no longer maintain — refuse loudly rather than leave one number stale.
+const stats = applyPublishedStats(doc.servers);
+if (stats.problems.length > 0) {
+    for (const p of stats.problems)
+        console.error(`published stats: ${p}`);
+    process.exit(1);
+}
 console.log('leaderboard:', JSON.stringify(percentiles(doc.servers)));
 console.log(`history: ${h.rows} rows (${h.added >= 0 ? '+' : ''}${h.added})`);
 console.log(`server pages: ${p.pages}`);
 console.log(`dashboard: ${d.out} (${(d.bytes / 1024).toFixed(0)}KB)`);
+console.log(`published stats: ${stats.changedFiles.length > 0
+    ? `updated ${stats.updated.join(', ')} in ${stats.changedFiles.join(', ')}`
+    : 'pages already agree with the data'}`);

package/dist/sweep/report.d.ts

@@ -1,4 +1,6 @@
+import type { Measurement } from '../core/types.js';
 import { type DivergenceRun } from '../core/divergence.js';
+import { type CrossCheckRun } from '../core/cross-check.js';
 import { type SessionStartLoad, type SessionStartRun } from '../core/session-start.js';
 export interface ServerEntry {
     name: string;
@@ -21,12 +23,19 @@ export interface ServerEntry {
      */
     envValues?: Record<string, string>;
 }
+export interface Row {
+    entry: ServerEntry;
+    m: Measurement | null;
+}
 /** Neutralize markdown/table syntax in third-party strings (tool names, notes). */
 export declare function mdCell(s: unknown): string;
+export declare function loadRows(entries: ServerEntry[], root?: string): Row[];
 /** results/divergence.json if a divergence run has been recorded, else null. */
 export declare function loadDivergence(root?: string): DivergenceRun | null;
 /** results/session-start.json — the instructions backfill, if one exists. */
 export declare function loadSessionStartRun(root?: string): SessionStartRun | null;
+/** results/cross-check.json if a CLI cross-check run has been recorded, else null. */
+export declare function loadCrossCheckRun(root?: string): CrossCheckRun | null;
 /**
  * A session-start figure that is a floor reads `>= N`, never a bare `N`. The
  * marker is half the point of publishing the number: the names half is measured,

package/dist/sweep/report.js

@@ -5,6 +5,7 @@
 import { existsSync, readFileSync, writeFileSync } from 'node:fs';
 import { join } from 'node:path';
 import { isCurrent, parseDivergence } from '../core/divergence.js';
+import { divergencePct, isComparable, parseCrossCheck } from '../core/cross-check.js';
 import { SESSION_START_METHOD, parseSessionStart, sessionStartLoad, } from '../core/session-start.js';
 /** Neutralize markdown/table syntax in third-party strings (tool names, notes). */
 export function mdCell(s) {
@@ -17,7 +18,7 @@ function csvCell(s) {
     const v = String(s ?? '');
     return /[",\n]/.test(v) ? `"${v.replace(/"/g, '""')}"` : v;
 }
-function loadRows(entries, root = process.cwd()) {
+export function loadRows(entries, root = process.cwd()) {
     return entries.map((entry) => {
         const p = join(root, 'results', entry.name, 'measurement.json');
         return { entry, m: existsSync(p) ? JSON.parse(readFileSync(p, 'utf8')) : null };
@@ -33,6 +34,11 @@ export function loadSessionStartRun(root = process.cwd()) {
     const p = join(root, 'results', 'session-start.json');
     return existsSync(p) ? parseSessionStart(readFileSync(p, 'utf8')) : null;
 }
+/** results/cross-check.json if a CLI cross-check run has been recorded, else null. */
+export function loadCrossCheckRun(root = process.cwd()) {
+    const p = join(root, 'results', 'cross-check.json');
+    return existsSync(p) ? parseCrossCheck(readFileSync(p, 'utf8')) : null;
+}
 /**
  * A session-start figure that is a floor reads `>= N`, never a bare `N`. The
  * marker is half the point of publishing the number: the names half is measured,
@@ -48,6 +54,7 @@ export function writeLeaderboard(entries, root = process.cwd()) {
     const rows = loadRows(entries, root);
     const div = loadDivergence(root);
     const ss = loadSessionStartRun(root);
+    const xc = loadCrossCheckRun(root);
     /** Session-start load for a row, or null when there is no capture to read. */
     const session = (r) => r.m ? sessionStartLoad(r.m, ss?.servers[r.entry.name]) : null;
     /** Claude tokens for a row, or null when not measured / stale / errored. */
@@ -57,6 +64,14 @@ export function writeLeaderboard(entries, root = process.cwd()) {
         const d = div.servers[r.entry.name];
         return isCurrent(d, r.m.canonicalSha256) ? d.claudeDelta : null;
     };
+    /** The other CLI's row, or null when the comparison is not like-with-like. */
+    const crossCheck = (r) => {
+        if (!xc || !r.m)
+            return null;
+        const row = xc.servers[r.entry.name];
+        return isComparable(row, r.m.canonicalSha256) ? row : null;
+    };
+    const signedPct = (p) => `${p >= 0 ? '+' : '−'}${Math.abs(p).toFixed(1)}%`;
     const measured = rows
         .filter((r) => r.m && (r.m.status === 'measured' || r.m.status === 'dynamic'))
         .sort((a, b) => (b.m.totalTokens ?? 0) - (a.m.totalTokens ?? 0));
@@ -77,6 +92,30 @@ export function writeLeaderboard(entries, root = process.cwd()) {
             `See [Claude divergence](../docs/METHODOLOGY.md#claude-divergence).`);
         md.push('');
     }
+    if (xc) {
+        const pcts = measured
+            .map((r) => {
+            const row = crossCheck(r);
+            return row ? divergencePct(row) : null;
+        })
+            .filter((p) => p !== null);
+        md.push(`The **mcp-tokens** column is the other CLI's count of the same server — ` +
+            `\`${mdCell(xc.cli)}\` \`${mdCell(xc.cliVersion)}\` (${mdCell(xc.measuredAt)}, method \`${mdCell(xc.method)}\`), ` +
+            `invoked with \`--model gpt-4o\` so both columns count o200k tokens. Its structs model the three ` +
+            `request fields (name/description/input\\_schema), so its number sits below the tokens column ` +
+            `wherever a server ships metadata those fields do not carry — that gap is each server's ` +
+            `field-selection share, published on its page, not a disagreement of counters. The parenthesized ` +
+            `percentage is the disagreement of counters: the CLI's count against ours of the same three-field ` +
+            `projection` +
+            (pcts.length > 0
+                ? `, ${signedPct(Math.min(...pcts))} to ${signedPct(Math.max(...pcts))} across the ` +
+                    `${pcts.length} row${pcts.length === 1 ? '' : 's'} where both tools saw the same tool set.`
+                : `. No row currently compares like with like.`) +
+            ` A row prints only while the comparison is between like and like: the same tool names on both ` +
+            `sides, and our capture unchanged since the run. ` +
+            `See [CLI cross-check](../docs/METHODOLOGY.md#cli-cross-check).`);
+        md.push('');
+    }
     const floors = measured.filter((r) => session(r)?.isFloor).length;
     md.push(`The **session start** column is what a client puts in context when it *defers* tool definitions until they ` +
         `are used: the server's tool names plus the \`instructions\` string it returns from \`initialize\` ` +
@@ -119,16 +158,19 @@ export function writeLeaderboard(entries, root = process.cwd()) {
             `A client that defers definitions is better off on every other measured row and worse off on ${costlier.length === 1 ? 'this one' : 'these'}.`);
         md.push('');
     }
-    md.push(`| # | server | tokens | session start |${div ? ' claude |' : ''} tools | largest tool | status | category |`);
-    md.push(`|---:|---|---:|---:|${div ? '---:|' : ''}---:|---|---|---|`);
+    md.push(`| # | server | tokens | session start |${div ? ' claude |' : ''}${xc ? ' mcp-tokens |' : ''} tools | largest tool | status | category |`);
+    md.push(`|---:|---|---:|---:|${div ? '---:|' : ''}${xc ? '---:|' : ''}---:|---|---|---|`);
     measured.forEach((r, i) => {
         const m = r.m;
         const largest = [...m.tools].sort((a, b) => b.tokens - a.tokens)[0];
         const link = `[${mdCell(r.entry.name)}](../docs/servers/${encodeURIComponent(r.entry.name)}.md)`;
         const c = claude(r);
+        const x = crossCheck(r);
+        const xCell = x === null ? '—' : `${x.cliTokens.toLocaleString('en-US')} (${signedPct(divergencePct(x))})`;
         md.push(`| ${i + 1} | ${link} | ${m.totalTokens.toLocaleString('en-US')} |` +
             ` ${sessionStartCell(session(r))} |` +
             (div ? ` ${c === null ? '—' : c.toLocaleString('en-US')} |` : '') +
+            (xc ? ` ${xCell} |` : '') +
             ` ${m.toolCount} | ` +
             `${largest ? `${mdCell(largest.name)} (${largest.tokens.toLocaleString('en-US')})` : '—'} | ${m.status} | ${mdCell(r.entry.category)} |`);
     });
@@ -150,12 +192,14 @@ export function writeLeaderboard(entries, root = process.cwd()) {
     // every existing parser working.
     const csv = [
         'name,tokens,toolCount,status,category,metric,metricSource,claudeTokens,claudeModel,' +
-            'sessionStartTokens,sessionStartIsFloor,toolNameTokens,instructionsTokens',
+            'sessionStartTokens,sessionStartIsFloor,toolNameTokens,instructionsTokens,' +
+            'crossCheckTokens,crossCheckCliVersion',
     ];
     for (const r of rows) {
         const m = r.m;
         const c = claude(r);
         const ssl = session(r);
+        const x = crossCheck(r);
         csv.push([
             csvCell(r.entry.name),
             m?.totalTokens ?? '',
@@ -172,6 +216,8 @@ export function writeLeaderboard(entries, root = process.cwd()) {
             ssl ? String(ssl.isFloor) : '',
             ssl?.toolNameTokens ?? '',
             ssl?.instructionsTokens ?? '',
+            x?.cliTokens ?? '',
+            x === null ? '' : csvCell(xc.cliVersion),
         ].join(','));
     }
     writeFileSync(join(root, 'results', 'leaderboard.csv'), csv.join('\n') + '\n');

package/dist/sweep/run.js

@@ -8,7 +8,7 @@ import { mkdirSync, writeFileSync } from 'node:fs';
 import { join, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { captureTools } from './client.js';
-import { dockerize } from './docker.js';
+import { DockerHarnessFault, defaultImageFor, dockerize, ensureImage, isDockerRunFailure } from './docker.js';
 import { measureTools, failedMeasurement, canonicalString } from '../core/canonical.js';
 import { toBadge } from '../core/badge.js';
 function arg(name) {
@@ -66,6 +66,9 @@ export async function measureServer(name, command, opts = {}) {
         throw new Error(`invalid server name '${name}' — letters/digits/dot/dash/underscore only`);
     }
     const root = opts.root ?? process.cwd();
+    // True only when THIS code wraps the command in `docker run` — a command that
+    // is already its own `docker run` owns its exit codes and its image.
+    const dockerWrapped = opts.docker === true && !command.trimStart().startsWith('docker ');
     const hostSpec = opts.argv && opts.argv.length ? { command: opts.argv[0], argv: opts.argv.slice(1) } : command;
     let isolation = { docker: false };
     const containerNames = [];
@@ -123,7 +126,15 @@ export async function measureServer(name, command, opts = {}) {
             }
         }
         catch (err) {
+            if (err instanceof DockerHarnessFault)
+                throw err;
             const msg = err instanceof Error ? err.message : String(err);
+            // `docker run` failing as docker (exit 125, docker's own stderr) never
+            // launched the server — classifying it would publish a fact about this
+            // machine as a fact about the server, so it is thrown instead of returned.
+            if (dockerWrapped && isDockerRunFailure(msg)) {
+                throw new DockerHarnessFault(`docker could not run the container for ${name}: ${msg.slice(0, 400)}`);
+            }
             const status = msg.includes('timeout')
                 ? 'timeout'
                 : /auth|unauthorized|401|forbidden|credential|api.?key|token/i.test(msg)
@@ -135,6 +146,12 @@ export async function measureServer(name, command, opts = {}) {
         r.timeoutMs = attemptOpts.timeoutMs ?? 60_000;
         return r;
     }
+    // The base image is fetched before anything is measured: `--pull=missing`
+    // pulls lazily, so a registry hiccup would otherwise land mid-measurement and
+    // read as the server refusing to start. Throws DockerHarnessFault when the
+    // machine cannot produce the image at all — before any record is written.
+    if (dockerWrapped)
+        await ensureImage(opts.dockerImage ?? defaultImageFor(command));
     let m;
     try {
         m = await attempt(false);
@@ -189,11 +206,23 @@ if (isMain) {
         console.error('usage: npm run sweep -- --name <slug> --command "<launch command>"');
         process.exit(2);
     }
-    const m = await measureServer(name, command, {
-        timeoutMs: Number(arg('timeout') ?? 60_000),
-        docker: process.argv.includes('--docker'),
-        dockerImage: arg('docker-image'),
-    });
+    let m;
+    try {
+        m = await measureServer(name, command, {
+            timeoutMs: Number(arg('timeout') ?? 60_000),
+            docker: process.argv.includes('--docker'),
+            dockerImage: arg('docker-image'),
+        });
+    }
+    catch (err) {
+        if (err instanceof DockerHarnessFault) {
+            // Nothing was measured and nothing was written — exiting non-zero here is
+            // what keeps a scheduled job from reaching its commit step.
+            console.error(`HARNESS FAULT: ${err.message}`);
+            process.exit(1);
+        }
+        throw err;
+    }
     // CLI path only — measureServer itself stays history-free so concurrent
     // sweep-all workers never race on the same file.
     const { appendHistory } = await import('./history.js');