mcp-context-cost 0.4.0 → 0.5.0

package/dist/sweep/history.d.ts

@@ -7,8 +7,18 @@ export interface HistoryRow {
     toolCount: number;
     /** 'measured' or 'dynamic' — dynamic means the tool set moved between captures. */
     status: string;
+    /**
+     * How the measurement was taken: `docker` (isolated container), `host` (bare
+     * machine), or `''` when the row predates this column and the conditions are
+     * not on record. Two numbers taken under different isolation are not
+     * comparable — same server, different node, different resolution of an
+     * `@latest` tag, different ambient env — so a step between them is a property
+     * of the harness, not of the server. Recording it is what lets the trend line
+     * refuse to draw such a step; see `plottableSeries`.
+     */
+    isolation: string;
 }
-export declare const HISTORY_HEADER = "date,server,tokens,toolCount,status";
+export declare const HISTORY_HEADER = "date,server,tokens,toolCount,status,isolation";
 /** Parse history.csv text; malformed or non-numeric rows are dropped, not thrown. */
 export declare function parseHistory(text: string): HistoryRow[];
 export declare function formatHistory(rows: HistoryRow[]): string;
@@ -19,6 +29,7 @@ export declare function upsert(rows: HistoryRow[], row: HistoryRow): HistoryRow[
  * auth walls are recorded in the leaderboard's "not measured" section, and
  * writing them here as zeros would fabricate a drop to zero in the series.
  */
+export declare function isolationOf(m: Measurement): string;
 export declare function rowFor(server: string, m: Measurement): HistoryRow | null;
 /**
  * Fold every results/<server>/measurement.json into results/history.csv.
@@ -28,3 +39,24 @@ export declare function appendHistory(root?: string): {
     rows: number;
     added: number;
 };
+export interface PlottableSeries {
+    /** The rows a trend may be drawn across, oldest first. */
+    rows: HistoryRow[];
+    /** Older rows excluded because they were measured under a different isolation. */
+    dropped: number;
+    /** True when a plotted row's conditions are not on record (a pre-`isolation` write). */
+    conditionsUnknown: boolean;
+}
+/**
+ * The longest run of a server's history, ending at its newest row, that a trend
+ * line may honestly be drawn across.
+ *
+ * Walking back from the newest row, a row stops the run when its isolation is
+ * known, the newest row's isolation is known, and the two differ — a step across
+ * that boundary would say the server changed when what changed is how it was
+ * measured. An *unknown* isolation is not evidence either way, so it stays in
+ * the run and is reported through `conditionsUnknown` instead: the alternative,
+ * treating unknown as its own incompatible value, would silently blank every
+ * series recorded before this column existed.
+ */
+export declare function plottableSeries(rows: HistoryRow[]): PlottableSeries;

package/dist/sweep/history.js

@@ -8,7 +8,7 @@
  */
 import { existsSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
 import { join } from 'node:path';
-export const HISTORY_HEADER = 'date,server,tokens,toolCount,status';
+export const HISTORY_HEADER = 'date,server,tokens,toolCount,status,isolation';
 function csvCell(s) {
     const v = String(s ?? '');
     return /[",\n]/.test(v) ? `"${v.replace(/"/g, '""')}"` : v;
@@ -48,18 +48,27 @@ export function parseHistory(text) {
     for (const line of text.split('\n')) {
         if (!line.trim() || line.startsWith('date,'))
             continue;
-        const [date, server, tokens, toolCount, status] = splitCsvLine(line);
+        const [date, server, tokens, toolCount, status, isolation] = splitCsvLine(line);
         if (!/^\d{4}-\d{2}-\d{2}$/.test(date ?? '') || !server)
             continue;
         if (!/^\d+$/.test(tokens ?? '') || !/^\d+$/.test(toolCount ?? ''))
             continue;
-        rows.push({ date, server, tokens: Number(tokens), toolCount: Number(toolCount), status: status ?? '' });
+        // A 5-field row is a pre-`isolation` write: its conditions are unknown, and
+        // unknown is recorded as unknown rather than back-filled with a guess.
+        rows.push({
+            date,
+            server,
+            tokens: Number(tokens),
+            toolCount: Number(toolCount),
+            status: status ?? '',
+            isolation: isolation ?? '',
+        });
     }
     return rows;
 }
 export function formatHistory(rows) {
     const sorted = [...rows].sort((a, b) => a.date.localeCompare(b.date) || a.server.localeCompare(b.server));
-    const lines = sorted.map((r) => [csvCell(r.date), csvCell(r.server), r.tokens, r.toolCount, csvCell(r.status)].join(','));
+    const lines = sorted.map((r) => [csvCell(r.date), csvCell(r.server), r.tokens, r.toolCount, csvCell(r.status), csvCell(r.isolation)].join(','));
     return [HISTORY_HEADER, ...lines].join('\n') + '\n';
 }
 /** Upsert one row by (date, server) — the newest write for a day wins. */
@@ -76,6 +85,11 @@ export function upsert(rows, row) {
  * auth walls are recorded in the leaderboard's "not measured" section, and
  * writing them here as zeros would fabricate a drop to zero in the series.
  */
+export function isolationOf(m) {
+    if (!m.isolation)
+        return '';
+    return m.isolation.docker ? 'docker' : 'host';
+}
 export function rowFor(server, m) {
     if (m.status !== 'measured' && m.status !== 'dynamic')
         return null;
@@ -84,7 +98,14 @@ export function rowFor(server, m) {
     const date = String(m.measuredAt ?? '').slice(0, 10);
     if (!/^\d{4}-\d{2}-\d{2}$/.test(date))
         return null;
-    return { date, server, tokens: m.totalTokens, toolCount: m.toolCount, status: m.status };
+    return {
+        date,
+        server,
+        tokens: m.totalTokens,
+        toolCount: m.toolCount,
+        status: m.status,
+        isolation: isolationOf(m),
+    };
 }
 /**
  * Fold every results/<server>/measurement.json into results/history.csv.
@@ -118,3 +139,37 @@ export function appendHistory(root = process.cwd()) {
     writeFileSync(path, formatHistory(rows));
     return { rows: rows.length, added: rows.length - existing.length };
 }
+/**
+ * The longest run of a server's history, ending at its newest row, that a trend
+ * line may honestly be drawn across.
+ *
+ * Walking back from the newest row, a row stops the run when its isolation is
+ * known, the newest row's isolation is known, and the two differ — a step across
+ * that boundary would say the server changed when what changed is how it was
+ * measured. An *unknown* isolation is not evidence either way, so it stays in
+ * the run and is reported through `conditionsUnknown` instead: the alternative,
+ * treating unknown as its own incompatible value, would silently blank every
+ * series recorded before this column existed.
+ */
+export function plottableSeries(rows) {
+    const sorted = [...rows].sort((a, b) => a.date.localeCompare(b.date));
+    if (!sorted.length)
+        return { rows: [], dropped: 0, conditionsUnknown: false };
+    const current = sorted[sorted.length - 1].isolation;
+    let start = 0;
+    if (current) {
+        for (let i = sorted.length - 1; i >= 0; i--) {
+            const iso = sorted[i].isolation;
+            if (iso && iso !== current) {
+                start = i + 1;
+                break;
+            }
+        }
+    }
+    const kept = sorted.slice(start);
+    return {
+        rows: kept,
+        dropped: start,
+        conditionsUnknown: kept.some((r) => !r.isolation),
+    };
+}

package/dist/sweep/regen.js

@@ -1,14 +1,19 @@
-/** Regenerate leaderboard + history + server pages from results/: npx tsx src/sweep/regen.ts */
+/** Regenerate leaderboard + history + server pages + dashboard from results/: npx tsx src/sweep/regen.ts */
 import { readFileSync } from 'node:fs';
 import { parse } from 'yaml';
 import { writeLeaderboard, percentiles } from './report.js';
 import { appendHistory } from './history.js';
 import { writeServerPages } from './server-pages.js';
+import { writeDashboard } from './dashboard.js';
 const doc = parse(readFileSync('servers.yaml', 'utf8'));
 writeLeaderboard(doc.servers);
 // History first: the server pages read history.csv for their over-time table.
 const h = appendHistory();
 const p = writeServerPages(doc.servers);
+// The dashboard reads the same results/ and history.csv as the pages do, so it
+// belongs in the same refresh — see writeDashboard's note on why it wasn't.
+const d = writeDashboard();
 console.log('leaderboard:', JSON.stringify(percentiles(doc.servers)));
 console.log(`history: ${h.rows} rows (${h.added >= 0 ? '+' : ''}${h.added})`);
 console.log(`server pages: ${p.pages}`);
+console.log(`dashboard: ${d.out} (${(d.bytes / 1024).toFixed(0)}KB)`);

package/dist/sweep/report.d.ts

@@ -1,4 +1,5 @@
 import { type DivergenceRun } from '../core/divergence.js';
+import { type SessionStartLoad, type SessionStartRun } from '../core/session-start.js';
 export interface ServerEntry {
     name: string;
     command: string;
@@ -13,11 +14,26 @@ export interface ServerEntry {
     timeoutSeconds?: number;
     /** Launch needs the `git` binary (e.g. `uvx --from git+...`) — absent from the slim isolation images. */
     needsGit?: boolean;
+    /**
+     * Per-var overrides for the literal `dummy` placeholder docker mode injects
+     * for `env` names — for servers that parse a var's shape (URI scheme, URL)
+     * before ever reaching tools/list. See docker.ts `dummyEnvValues`.
+     */
+    envValues?: Record<string, string>;
 }
 /** Neutralize markdown/table syntax in third-party strings (tool names, notes). */
 export declare function mdCell(s: unknown): string;
 /** results/divergence.json if a divergence run has been recorded, else null. */
 export declare function loadDivergence(root?: string): DivergenceRun | null;
+/** results/session-start.json — the instructions backfill, if one exists. */
+export declare function loadSessionStartRun(root?: string): SessionStartRun | null;
+/**
+ * A session-start figure that is a floor reads `>= N`, never a bare `N`. The
+ * marker is half the point of publishing the number: the names half is measured,
+ * the instructions half has not been captured for this server, and a reader has
+ * to be able to tell that from a row where both halves are known.
+ */
+export declare function sessionStartCell(load: SessionStartLoad | null): string;
 export declare function writeLeaderboard(entries: ServerEntry[], root?: string): void;
 /** Percentile helper for freezing color bands against the observed distribution. */
 export declare function percentiles(entries: ServerEntry[], root?: string): Record<string, number>;

package/dist/sweep/report.js

@@ -5,6 +5,7 @@
 import { existsSync, readFileSync, writeFileSync } from 'node:fs';
 import { join } from 'node:path';
 import { isCurrent, parseDivergence } from '../core/divergence.js';
+import { SESSION_START_METHOD, parseSessionStart, sessionStartLoad, } from '../core/session-start.js';
 /** Neutralize markdown/table syntax in third-party strings (tool names, notes). */
 export function mdCell(s) {
     return String(s ?? '')
@@ -27,9 +28,28 @@ export function loadDivergence(root = process.cwd()) {
     const p = join(root, 'results', 'divergence.json');
     return existsSync(p) ? parseDivergence(readFileSync(p, 'utf8')) : null;
 }
+/** results/session-start.json — the instructions backfill, if one exists. */
+export function loadSessionStartRun(root = process.cwd()) {
+    const p = join(root, 'results', 'session-start.json');
+    return existsSync(p) ? parseSessionStart(readFileSync(p, 'utf8')) : null;
+}
+/**
+ * A session-start figure that is a floor reads `>= N`, never a bare `N`. The
+ * marker is half the point of publishing the number: the names half is measured,
+ * the instructions half has not been captured for this server, and a reader has
+ * to be able to tell that from a row where both halves are known.
+ */
+export function sessionStartCell(load) {
+    if (!load)
+        return '—';
+    return `${load.isFloor ? '≥' : ''}${load.totalTokens.toLocaleString('en-US')}`;
+}
 export function writeLeaderboard(entries, root = process.cwd()) {
     const rows = loadRows(entries, root);
     const div = loadDivergence(root);
+    const ss = loadSessionStartRun(root);
+    /** Session-start load for a row, or null when there is no capture to read. */
+    const session = (r) => r.m ? sessionStartLoad(r.m, ss?.servers[r.entry.name]) : null;
     /** Claude tokens for a row, or null when not measured / stale / errored. */
     const claude = (r) => {
         if (!div || !r.m)
@@ -57,14 +77,57 @@ export function writeLeaderboard(entries, root = process.cwd()) {
             `See [Claude divergence](../docs/METHODOLOGY.md#claude-divergence).`);
         md.push('');
     }
-    md.push(`| # | server | tokens |${div ? ' claude |' : ''} tools | largest tool | status | category |`);
-    md.push(`|---:|---|---:|${div ? '---:|' : ''}---:|---|---|---|`);
+    const floors = measured.filter((r) => session(r)?.isFloor).length;
+    md.push(`The **session start** column is what a client puts in context when it *defers* tool definitions until they ` +
+        `are used: the server's tool names plus the \`instructions\` string it returns from \`initialize\` ` +
+        `(method \`${SESSION_START_METHOD}\`). The tokens column is what a client that loads every definition up ` +
+        `front pays; this one is what the same server costs a client that does not. ` +
+        `See [session-start load](../docs/METHODOLOGY.md#session-start-load).`);
+    md.push('');
+    if (floors > 0) {
+        md.push(`**\`≥\` marks a floor, on ${floors} of ${measured.length} rows.** Tool names are counted exactly from ` +
+            `the published capture, but \`instructions\` is not part of \`tools/list\` and has not been captured for ` +
+            `these servers — so the figure is the names half alone and the true number is that or higher. A row stops ` +
+            `being a floor the first time the server is measured with its instructions.`);
+        md.push('');
+    }
+    // Deferring usually saves almost everything, but it is not guaranteed to save
+    // anything: `instructions` are bytes the headline never counted, and a server
+    // that re-lists its tools in prose can charge a deferring client more than an
+    // eager one. Those rows are the most useful thing this column finds, so they
+    // are named here rather than left for a reader to spot by comparing columns.
+    // Derived on every write — no row is listed by hand, and the paragraph
+    // disappears if the set ever empties, rather than asserting a stale count.
+    const costlier = measured.filter((r) => {
+        const load = session(r);
+        return load !== null && r.m.totalTokens !== null && load.totalTokens >= r.m.totalTokens;
+    });
+    if (costlier.length > 0) {
+        // Server names are backticked, not bolded: the lead sentence is already
+        // bold and a nested `**` would close it early, silently un-bolding the
+        // half of the sentence that carries the finding.
+        const named = costlier
+            .map((r) => {
+            const load = session(r);
+            return `\`${mdCell(r.entry.name)}\` pays ${load.isFloor ? '≥' : ''}${load.totalTokens.toLocaleString('en-US')} at session start against ${r.m.totalTokens.toLocaleString('en-US')} of definitions`;
+        })
+            .join('; ');
+        md.push(`**Deferring costs more than it saves on ${costlier.length} of ${measured.length} rows.** ${named}. ` +
+            `The names half is always a fraction of the headline, but \`instructions\` are bytes the tokens column ` +
+            `never counted and their length is independent of the tool set — so a server that re-lists its tools in ` +
+            `its instructions makes a deferring client pay for a prose copy of the schemas it just skipped. ` +
+            `A client that defers definitions is better off on every other measured row and worse off on ${costlier.length === 1 ? 'this one' : 'these'}.`);
+        md.push('');
+    }
+    md.push(`| # | server | tokens | session start |${div ? ' claude |' : ''} tools | largest tool | status | category |`);
+    md.push(`|---:|---|---:|---:|${div ? '---:|' : ''}---:|---|---|---|`);
     measured.forEach((r, i) => {
         const m = r.m;
         const largest = [...m.tools].sort((a, b) => b.tokens - a.tokens)[0];
         const link = `[${mdCell(r.entry.name)}](../docs/servers/${encodeURIComponent(r.entry.name)}.md)`;
         const c = claude(r);
         md.push(`| ${i + 1} | ${link} | ${m.totalTokens.toLocaleString('en-US')} |` +
+            ` ${sessionStartCell(session(r))} |` +
             (div ? ` ${c === null ? '—' : c.toLocaleString('en-US')} |` : '') +
             ` ${m.toolCount} | ` +
             `${largest ? `${mdCell(largest.name)} (${largest.tokens.toLocaleString('en-US')})` : '—'} | ${m.status} | ${mdCell(r.entry.category)} |`);
@@ -82,12 +145,17 @@ export function writeLeaderboard(entries, root = process.cwd()) {
         md.push('');
     }
     writeFileSync(join(root, 'results', 'leaderboard.md'), md.join('\n') + '\n');
-    // Columns are append-only: consumers key off the header, so adding the Claude
-    // pair at the end leaves every existing parser working.
-    const csv = ['name,tokens,toolCount,status,category,metric,metricSource,claudeTokens,claudeModel'];
+    // Columns are append-only: consumers key off the header, so each new group
+    // (the Claude pair, then the session-start four) goes on the end and leaves
+    // every existing parser working.
+    const csv = [
+        'name,tokens,toolCount,status,category,metric,metricSource,claudeTokens,claudeModel,' +
+            'sessionStartTokens,sessionStartIsFloor,toolNameTokens,instructionsTokens',
+    ];
     for (const r of rows) {
         const m = r.m;
         const c = claude(r);
+        const ssl = session(r);
         csv.push([
             csvCell(r.entry.name),
             m?.totalTokens ?? '',
@@ -98,6 +166,12 @@ export function writeLeaderboard(entries, root = process.cwd()) {
             csvCell(r.entry.metricSource),
             c ?? '',
             c === null ? '' : csvCell(div.model),
+            ssl?.totalTokens ?? '',
+            // Spelled out rather than left implicit: a consumer that ignores this
+            // column and sums the previous one is understating every floor row.
+            ssl ? String(ssl.isFloor) : '',
+            ssl?.toolNameTokens ?? '',
+            ssl?.instructionsTokens ?? '',
         ].join(','));
     }
     writeFileSync(join(root, 'results', 'leaderboard.csv'), csv.join('\n') + '\n');

package/dist/sweep/run.d.ts

@@ -7,6 +7,8 @@ export interface MeasureOptions {
     dockerImage?: string;
     /** env var NAMES to provide as dummy values (docker mode). */
     dummyEnv?: string[];
+    /** Override the literal `dummy` value for specific `dummyEnv` names — see docker.ts. */
+    dummyEnvValues?: Record<string, string>;
     /** Install `git` in the container before launch (docker mode) — see docker.ts. */
     needsGit?: boolean;
     /**
@@ -21,4 +23,43 @@ export interface MeasureOptions {
      */
     persist?: boolean;
 }
+/**
+ * Marks a startup-failure that was re-attempted from a cold package cache and
+ * failed the same way. Its absence on a startup-failure means the retry never
+ * ran (host mode, or a self-containerised command), not that it passed.
+ */
+export declare const RETRY_CONFIRMED_PREFIX = "reproduced with the shared package cache bypassed; ";
+/**
+ * Whether a failed measurement gets a second attempt with the shared package
+ * caches bypassed (see `DockerOptions.noSharedCache`).
+ *
+ * A poisoned cache entry and a genuinely broken server produce the same exit
+ * code, so a `startup-failure` is not published until it reproduces from a cold
+ * cache. Deliberately only `startup-failure`: a `timeout` is usually the slow
+ * install a cold retry would only make slower (it has its own retry — see
+ * `retriesWithLongerTimeout`), `auth-required` is a real answer about the
+ * server, and a command that is already its own `docker run` has no cache mount
+ * to bypass.
+ */
+export declare function retriesWithoutSharedCache(status: Measurement['status'], docker: boolean, command: string): boolean;
+/**
+ * Marks a `timeout` that was re-attempted on a larger budget and timed out
+ * again. Its absence on a timeout means the retry never ran, not that it passed.
+ */
+export declare const TIMEOUT_CONFIRMED_PREFIX = "reproduced on double the timeout budget; ";
+/** A timed-out measurement is re-attempted on this multiple of its budget. */
+export declare const TIMEOUT_RETRY_FACTOR = 2;
+/**
+ * Whether a timed-out measurement gets a second attempt on a larger budget.
+ *
+ * Sweeps and `audit` both run a worker pool, and a server that starts slowly
+ * under that contention is indistinguishable from one that never starts: both
+ * come back `timeout`. Two servers were published as failures for exactly this
+ * reason (`puppeteer` and `kubernetes`, 2026-08-19) and measured normally when
+ * re-run alone, so a timeout is not published until it survives a second, wider
+ * budget. Unlike the cold-cache retry this is isolation-independent — contention
+ * is not a property of the package cache, so host runs and commands that are
+ * their own `docker run` are retried too.
+ */
+export declare function retriesWithLongerTimeout(status: Measurement['status']): boolean;
 export declare function measureServer(name: string, command: string, opts?: MeasureOptions): Promise<Measurement>;

package/dist/sweep/run.js

@@ -15,6 +15,49 @@ function arg(name) {
     const i = process.argv.indexOf(`--${name}`);
     return i >= 0 ? process.argv[i + 1] : undefined;
 }
+/**
+ * Marks a startup-failure that was re-attempted from a cold package cache and
+ * failed the same way. Its absence on a startup-failure means the retry never
+ * ran (host mode, or a self-containerised command), not that it passed.
+ */
+export const RETRY_CONFIRMED_PREFIX = 'reproduced with the shared package cache bypassed; ';
+/**
+ * Whether a failed measurement gets a second attempt with the shared package
+ * caches bypassed (see `DockerOptions.noSharedCache`).
+ *
+ * A poisoned cache entry and a genuinely broken server produce the same exit
+ * code, so a `startup-failure` is not published until it reproduces from a cold
+ * cache. Deliberately only `startup-failure`: a `timeout` is usually the slow
+ * install a cold retry would only make slower (it has its own retry — see
+ * `retriesWithLongerTimeout`), `auth-required` is a real answer about the
+ * server, and a command that is already its own `docker run` has no cache mount
+ * to bypass.
+ */
+export function retriesWithoutSharedCache(status, docker, command) {
+    return status === 'startup-failure' && docker && !command.trimStart().startsWith('docker ');
+}
+/**
+ * Marks a `timeout` that was re-attempted on a larger budget and timed out
+ * again. Its absence on a timeout means the retry never ran, not that it passed.
+ */
+export const TIMEOUT_CONFIRMED_PREFIX = 'reproduced on double the timeout budget; ';
+/** A timed-out measurement is re-attempted on this multiple of its budget. */
+export const TIMEOUT_RETRY_FACTOR = 2;
+/**
+ * Whether a timed-out measurement gets a second attempt on a larger budget.
+ *
+ * Sweeps and `audit` both run a worker pool, and a server that starts slowly
+ * under that contention is indistinguishable from one that never starts: both
+ * come back `timeout`. Two servers were published as failures for exactly this
+ * reason (`puppeteer` and `kubernetes`, 2026-08-19) and measured normally when
+ * re-run alone, so a timeout is not published until it survives a second, wider
+ * budget. Unlike the cold-cache retry this is isolation-independent — contention
+ * is not a property of the package cache, so host runs and commands that are
+ * their own `docker run` are retried too.
+ */
+export function retriesWithLongerTimeout(status) {
+    return status === 'timeout';
+}
 export async function measureServer(name, command, opts = {}) {
     const persist = opts.persist !== false;
     // The name becomes a directory when persisting; that's the only reason it's
@@ -23,57 +66,107 @@ export async function measureServer(name, command, opts = {}) {
         throw new Error(`invalid server name '${name}' — letters/digits/dot/dash/underscore only`);
     }
     const root = opts.root ?? process.cwd();
-    let spec = opts.argv && opts.argv.length ? { command: opts.argv[0], argv: opts.argv.slice(1) } : command;
+    const hostSpec = opts.argv && opts.argv.length ? { command: opts.argv[0], argv: opts.argv.slice(1) } : command;
     let isolation = { docker: false };
-    let containerName;
-    if (opts.docker && command.trimStart().startsWith('docker ')) {
-        // Command is already a container — wrapping it again would need docker-in-docker.
-        isolation = { docker: true, note: 'command is itself a docker run (host-spawned container)' };
-    }
-    else if (opts.docker) {
-        containerName = `mcp-ctx-${name}-${process.pid}-${Math.floor(Math.random() * 1e6)}`;
+    const containerNames = [];
+    // A fresh container name per capture: some servers don't exit on stdin close
+    // (background timers keep the event loop alive), so `close()`'s SIGKILL only
+    // detaches the host-side `docker run` CLI — the container itself can outlive
+    // it. Reusing one name across both tools/list captures then races `--rm`'s
+    // async cleanup: the second `docker run --name X` collides with the first
+    // container mid-removal. Distinct names sidestep the race entirely; the
+    // `finally` below force-removes every name this call created either way.
+    function buildSpec(noSharedCache = false) {
+        if (opts.docker && command.trimStart().startsWith('docker ')) {
+            // Command is already a container — wrapping it again would need docker-in-docker.
+            isolation = { docker: true, note: 'command is itself a docker run (host-spawned container)' };
+            return hostSpec;
+        }
+        if (!opts.docker)
+            return hostSpec;
+        const containerName = `mcp-ctx-${name}-${process.pid}-${Math.floor(Math.random() * 1e6)}-${containerNames.length}`;
+        containerNames.push(containerName);
         const d = dockerize(command, {
             image: opts.dockerImage,
             dummyEnv: opts.dummyEnv,
+            dummyEnvValues: opts.dummyEnvValues,
             containerName,
             needsGit: opts.needsGit,
+            noSharedCache,
         });
-        spec = { command: d.command, argv: d.argv };
         isolation = d.isolation;
+        return { command: d.command, argv: d.argv };
+    }
+    async function attempt(noSharedCache, timeoutMsOverride) {
+        // A cold install has to fetch everything again, so the retry gets its own
+        // floor — otherwise the bypass would trade a cache-poisoned startup-failure
+        // for a timeout and report just as wrongly.
+        const attemptOpts = timeoutMsOverride !== undefined
+            ? { ...opts, timeoutMs: timeoutMsOverride }
+            : noSharedCache
+                ? { ...opts, timeoutMs: Math.max(opts.timeoutMs ?? 60_000, 240_000) }
+                : opts;
+        let r;
+        try {
+            const first = await captureTools(buildSpec(noSharedCache), attemptOpts);
+            const second = await captureTools(buildSpec(noSharedCache), attemptOpts);
+            r = measureTools(first.tools, {
+                serverName: first.serverInfo?.name ?? name,
+                serverVersion: first.serverInfo?.version,
+                launchCommand: command,
+                envVarNames: [...Object.keys(opts.env ?? {}), ...(opts.dummyEnv ?? [])],
+                instructions: first.instructions,
+            });
+            if (canonicalString(first.tools) !== canonicalString(second.tools)) {
+                r.status = 'dynamic';
+                r.notes = 'tools/list differed between two runs; value is for the first capture';
+            }
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            const status = msg.includes('timeout')
+                ? 'timeout'
+                : /auth|unauthorized|401|forbidden|credential|api.?key|token/i.test(msg)
+                    ? 'auth-required'
+                    : 'startup-failure';
+            r = failedMeasurement(status, { serverName: name, launchCommand: command, notes: msg.slice(0, 700) });
+        }
+        r.isolation = isolation;
+        r.timeoutMs = attemptOpts.timeoutMs ?? 60_000;
+        return r;
     }
     let m;
     try {
-        const first = await captureTools(spec, opts);
-        const second = await captureTools(spec, opts);
-        m = measureTools(first.tools, {
-            serverName: first.serverInfo?.name ?? name,
-            serverVersion: first.serverInfo?.version,
-            launchCommand: command,
-            envVarNames: [...Object.keys(opts.env ?? {}), ...(opts.dummyEnv ?? [])],
-        });
-        m.isolation = isolation;
-        m.timeoutMs = opts.timeoutMs ?? 60_000;
-        if (canonicalString(first.tools) !== canonicalString(second.tools)) {
-            m.status = 'dynamic';
-            m.notes = 'tools/list differed between two runs; value is for the first capture';
+        m = await attempt(false);
+        if (retriesWithoutSharedCache(m.status, opts.docker === true, command)) {
+            const retry = await attempt(true);
+            if (retry.status !== 'startup-failure') {
+                m = retry;
+            }
+            else {
+                // Keep the warm attempt — its stderr is the one worth reading — but say
+                // that the failure survived a cold cache, so a reader can tell a real
+                // breakage from a cache artifact without re-running the sweep.
+                m.notes = `${RETRY_CONFIRMED_PREFIX}${m.notes ?? ''}`.slice(0, 700);
+            }
+        }
+        else if (retriesWithLongerTimeout(m.status)) {
+            const retry = await attempt(false, (opts.timeoutMs ?? 60_000) * TIMEOUT_RETRY_FACTOR);
+            // The retry replaces the first attempt either way: it is the wider budget,
+            // so its `timeoutMs` is the number a reader needs to judge the verdict.
+            m = retry;
+            if (retry.status === 'timeout') {
+                m.notes = `${TIMEOUT_CONFIRMED_PREFIX}${m.notes ?? ''}`.slice(0, 700);
+            }
         }
-    }
-    catch (err) {
-        const msg = err instanceof Error ? err.message : String(err);
-        const status = msg.includes('timeout')
-            ? 'timeout'
-            : /auth|unauthorized|401|forbidden|credential|api.?key|token/i.test(msg)
-                ? 'auth-required'
-                : 'startup-failure';
-        m = failedMeasurement(status, { serverName: name, launchCommand: command, notes: msg.slice(0, 700) });
-        m.isolation = isolation;
-        m.timeoutMs = opts.timeoutMs ?? 60_000;
     }
     finally {
-        if (containerName) {
-            // Killing the docker CLI on timeout orphans the container — remove it.
+        if (containerNames.length) {
+            // Killing the docker CLI on timeout/non-exit orphans the container — remove it.
             const { spawn } = await import('node:child_process');
-            spawn('docker', ['rm', '-f', containerName], { stdio: 'ignore' }).on('error', () => { });
+            for (const n of containerNames) {
+                spawn('docker', ['rm', '-f', n], { stdio: 'ignore' }).on('error', () => { });
+            }
         }
     }
     if (!persist)

package/dist/sweep/server-pages.js

@@ -11,7 +11,7 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
 import { join } from 'node:path';
 import { bandColor, BAND_META } from '../core/bands.js';
 import { loadDivergence, mdCell } from './report.js';
-import { parseHistory } from './history.js';
+import { parseHistory, plottableSeries } from './history.js';
 import { claudeRatio, fieldSelectionShare, isCurrent } from '../core/divergence.js';
 /**
  * Pages are served from GitHub Pages (docs/), but results/ and badges/ are not
@@ -123,17 +123,42 @@ export function renderServerPage(entry, m, history = [], divergence = null) {
         md.push(...divergenceSection(divRow, divergence));
     }
     if (history.length > 1) {
+        // A change is only a change if both numbers were taken the same way, so the
+        // delta column is blank across an isolation boundary rather than reporting a
+        // difference the harness produced.
+        const plottable = plottableSeries(history);
+        const comparableFrom = plottable.rows[0]?.date;
         md.push('## Over time');
         md.push('');
-        md.push('| date | tokens | tools | change |');
-        md.push('|---|---:|---:|---:|');
+        md.push('| date | tokens | tools | measured in | change |');
+        md.push('|---|---:|---:|---|---:|');
         history.forEach((h, i) => {
             const prev = history[i - 1];
-            const delta = prev ? h.tokens - prev.tokens : null;
-            const change = delta === null ? '—' : delta === 0 ? 'no change' : `${delta > 0 ? '+' : ''}${fmt(delta)}`;
-            md.push(`| ${mdCell(h.date)} | ${fmt(h.tokens)} | ${h.toolCount} | ${change} |`);
+            const comparable = prev && (!prev.isolation || !h.isolation || prev.isolation === h.isolation);
+            const delta = prev && comparable ? h.tokens - prev.tokens : null;
+            const change = !prev
+                ? '—'
+                : !comparable
+                    ? 'not comparable'
+                    : delta === 0
+                        ? 'no change'
+                        : `${delta > 0 ? '+' : ''}${fmt(delta)}`;
+            md.push(`| ${mdCell(h.date)} | ${fmt(h.tokens)} | ${h.toolCount} | ${mdCell(h.isolation || 'not recorded')} | ${change} |`);
         });
         md.push('');
+        if (plottable.dropped) {
+            md.push(`> The first ${plottable.dropped} row${plottable.dropped === 1 ? ' was' : 's were'} measured under a ` +
+                `different isolation than the current measurement, so the published trend starts at ` +
+                `${mdCell(comparableFrom)}. Numbers taken on the host and inside a container are not ` +
+                `interchangeable — the package a \`@latest\` tag resolves to, the runtime version and the ` +
+                `ambient environment can all differ.`);
+            md.push('');
+        }
+        else if (plottable.conditionsUnknown) {
+            md.push('> Some of these sweeps predate the `isolation` column, so the conditions they were ' +
+                'measured under are not on record.');
+            md.push('');
+        }
         md.push(`Full series: [results/history.csv](${BLOB}/results/history.csv).`);
         md.push('');
     }