mcp-context-cost 0.11.3 → 0.12.0

package/README.md

@@ -184,7 +184,7 @@ Add `--claude` to annotate each server with its Anthropic-request cost from the
 [Claude divergence](docs/METHODOLOGY.md#claude-divergence) run — an exact number when the
 published capture hash matches what you have installed, `—` (silence, not a stale guess)
 when it doesn't. The run holds 20 rows — the top 20 measured servers by tokens when it ran —
-and [results/leaderboard.md](results/leaderboard.md) prints a claude number for the 16 that
+and [results/leaderboard.md](results/leaderboard.md) prints a claude number for the 12 that
 still match today and silence for the rest. Most installs will show a mix:
 
 ```
@@ -235,7 +235,7 @@ Flags: `--json` (full report on stdout, progress on stderr), `--budget N`,
 
 The number `audit` gives you is the same measurement, run across a curated set of public
 servers — which is how you can tell it is a measurement and not this tool's opinion. It also
-shows what you are choosing between: across the 81 servers measured, cost spans **1,700×**,
+shows what you are choosing between: across the 83 servers measured, cost spans **1,700×**,
 from `postgres` at 32 tokens to `github` at 54,622. The table below is a
 sample of that range; the full range is in
 [results/leaderboard.md](results/leaderboard.md).
@@ -244,13 +244,13 @@ sample of that range; the full range is in
 |---|---:|---:|
 | github (official) | **54,622 tokens** | 44 |
 | xcodebuildmcp | 26,594 | 24 |
-| brave-search | 25,456 | 8 |
+| brave-search | 25,487 | 8 |
 | notion | 17,500 | 24 |
 | playwright *(4.8M installs/week)* | 4,024 | 24 |
 | filesystem (reference) | 2,823 | 14 |
 | markitdown | 64 | 1 |
 
-*(81 of 106 popular servers measured, each row dated by its own most recent sweep — full table in
+*(83 of 106 popular servers measured, each row dated by its own most recent sweep — full table in
 [results/leaderboard.md](results/leaderboard.md); every failure is listed with its reason.
 Each measured server also has a [detail page](https://athakur3.github.io/mcp-context-cost/servers/)
 showing which tools its tokens are in.)*
@@ -260,7 +260,7 @@ answers a question no client asks: **what did this server cost last month?**
 [results/regressions.md](results/regressions.md) reports each server's most recent movement —
 dated to when it happened, separated into *shipped more tools* versus *same tools, rewritten*,
 and compared only within one isolation. The ecosystem ratchets upward: of the servers whose
-cost has moved at all, 9 moved up against 1 that moved down. Method:
+cost has moved at all, 11 moved up against 6 that moved down. Method:
 [cost movement](docs/METHODOLOGY.md#cost-movement).
 
 If you publish a server, the same measurement is available as a badge, so your users can see

package/dist/core/types.d.ts

@@ -1,5 +1,15 @@
 /** Status taxonomy — every swept server gets exactly one; no silent drops. */
-export type MeasurementStatus = 'measured' | 'auth-required' | 'startup-failure' | 'timeout' | 'dynamic' | 'remote-auth-wall';
+export type MeasurementStatus = 'measured' | 'auth-required' | 'startup-failure' | 'timeout'
+/**
+ * This harness cannot run the server, for a reason that is a property of the
+ * harness rather than of the software: an OS or architecture the package does
+ * not ship for, or a backing service the isolation deliberately does not
+ * provide. Distinct from `startup-failure`, which asserts the server did not
+ * come up — a claim about someone else's code that these entries do not
+ * support. Only ever set when the entry declares the reason AND the failure's
+ * own text corroborates it (see `notApplicable` in report.ts).
+ */
+ | 'not-applicable' | 'dynamic' | 'remote-auth-wall';
 export interface ToolMeasurement {
     name: string;
     /** Tokens of the whole tool object, canonically serialized. */
@@ -43,6 +53,18 @@ export interface Measurement {
         image?: string;
         network?: string;
         note?: string;
+        /**
+         * The architecture the measurement ran on, as `<platform>/<arch>` (e.g.
+         * `linux/amd64`). Part of the isolation because some packages ship builds
+         * for only some of them: `local-mcp` was published as a startup failure
+         * for weeks on the strength of a run whose real finding was "this laptop
+         * is arm64 and the package has no arm64 runtime" — a fact about the
+         * machine that the record gave no way to see.
+         *
+         * Absent on records written before this was captured, which is why it is
+         * optional; absence means unknown, never "the same as yours".
+         */
+        arch?: string;
     };
     /** Request timeout in force during this measurement. */
     timeoutMs?: number;

package/dist/sweep/client.d.ts

@@ -13,6 +13,28 @@ export interface WireCapture {
     instructions: string | null;
     stderrTail: string;
 }
+/**
+ * The part of a dead server's stderr worth keeping as evidence.
+ *
+ * A failure record is only useful if it contains the failure, and a plain tail
+ * reliably keeps the least useful part. `npx` prints a deprecation warning per
+ * transitive dependency and a version notice at the end, and a crashing process
+ * prints its message *before* the stack — so the last N characters of stderr
+ * are npm warnings and stack frames on exactly the servers whose failure needs
+ * explaining. Several published records ended up saying nothing about why the
+ * server did not start.
+ *
+ * This is not cosmetic. `run.ts` classifies a failure by reading these words:
+ * a record whose message was cut off is filed as `startup-failure` — the server
+ * is broken — when the surviving text would have said `auth-required`. Spending
+ * the budget on the message rather than the frames is what keeps the published
+ * taxonomy describing the server.
+ *
+ * Noise is only dropped while something else survives. A package that fails
+ * *inside* npm (EBADPLATFORM, a failed postinstall) has npm's own lines as its
+ * only evidence, and a server whose whole output is a stack keeps the stack.
+ */
+export declare function evidenceTail(stderr: string, limit?: number): string;
 export declare class McpStdioClient {
     private child;
     private buffer;

package/dist/sweep/client.js

@@ -6,6 +6,92 @@
  */
 import { spawn } from 'node:child_process';
 const PROTOCOL_VERSION = '2025-06-18';
+/**
+ * The part of a dead server's stderr worth keeping as evidence.
+ *
+ * A failure record is only useful if it contains the failure, and a plain tail
+ * reliably keeps the least useful part. `npx` prints a deprecation warning per
+ * transitive dependency and a version notice at the end, and a crashing process
+ * prints its message *before* the stack — so the last N characters of stderr
+ * are npm warnings and stack frames on exactly the servers whose failure needs
+ * explaining. Several published records ended up saying nothing about why the
+ * server did not start.
+ *
+ * This is not cosmetic. `run.ts` classifies a failure by reading these words:
+ * a record whose message was cut off is filed as `startup-failure` — the server
+ * is broken — when the surviving text would have said `auth-required`. Spending
+ * the budget on the message rather than the frames is what keeps the published
+ * taxonomy describing the server.
+ *
+ * Noise is only dropped while something else survives. A package that fails
+ * *inside* npm (EBADPLATFORM, a failed postinstall) has npm's own lines as its
+ * only evidence, and a server whose whole output is a stack keeps the stack.
+ */
+export function evidenceTail(stderr, limit = 600) {
+    const withoutNoise = drop(stderr, (l) => /^npm (warn|notice)\b/.test(l));
+    const withoutFrames = drop(withoutNoise, (l) => /^at\s/.test(l));
+    return bothEnds(withoutFrames, limit);
+}
+/**
+ * Keep the start and the end, eliding the middle.
+ *
+ * Dropping npm noise and stack frames is not enough on its own: a CLI that
+ * rejects its environment often prints one line saying why and then its entire
+ * usage screen, which is neither. kubernetes-mcp-server does exactly that, and
+ * a tail-only budget kept forty lines of flag documentation while discarding
+ * "no current-context is set and no contexts are defined in kubeconfig" — the
+ * only sentence in the output that explained anything.
+ *
+ * Failures put their explanation at one end or the other — a crash message
+ * above its aftermath, or an error at the end of a log — so both ends are kept
+ * and the middle is what goes. The split leans towards the head because a
+ * message that precedes its own noise is the more common shape here.
+ */
+function bothEnds(text, limit) {
+    if (text.length <= limit)
+        return text;
+    const elision = '\n […] \n';
+    const budget = Math.max(0, limit - elision.length);
+    const lines = text.split('\n');
+    // Whole lines only: a boundary cut mid-word ("ool/prompt change") reads as
+    // corruption and loses the token an evidence string would match on.
+    const take = (from, to, cap, fromEnd) => {
+        const out = [];
+        let used = 0;
+        for (let i = fromEnd ? to : from; fromEnd ? i >= from : i <= to; i += fromEnd ? -1 : 1) {
+            const cost = lines[i].length + 1;
+            if (used + cost > cap)
+                break;
+            fromEnd ? out.unshift(lines[i]) : out.push(lines[i]);
+            used += cost;
+        }
+        return { out, used };
+    };
+    const headCap = Math.ceil(budget * 0.6);
+    const head = take(0, lines.length - 1, headCap, false);
+    // Whole lines, except when the first line alone overruns the budget. A server
+    // that logs structured JSON puts its entire message on one line, so that line
+    // is both the most informative thing in the output and the only one that can
+    // never fit — slack-mcp-server's `{"level":"fatal","message":"Authentication
+    // required: ..."}` was dropped in full, and the record it left behind said a
+    // child process exited. Truncated evidence beats none.
+    const headText = head.out.length > 0 ? head.out.join('\n') : lines[0].slice(0, headCap);
+    const headUsed = head.out.length > 0 ? head.used : headText.length;
+    const tailFrom = head.out.length > 0 ? head.out.length : 1;
+    const tail = take(tailFrom, lines.length - 1, budget - headUsed, true);
+    if (headText === '' && tail.out.length === 0)
+        return text.slice(0, budget) + elision;
+    return `${headText}${elision}${tail.out.join('\n')}`;
+}
+/** Drop matching lines, keeping the input whole if that would leave nothing. */
+function drop(text, isNoise) {
+    const kept = text
+        .split('\n')
+        .filter((l) => !isNoise(l.trim()))
+        .join('\n')
+        .trim();
+    return kept || text.trim();
+}
 export class McpStdioClient {
     child;
     buffer = '';
@@ -37,7 +123,7 @@ export class McpStdioClient {
                 resolve();
             });
             this.child.on('exit', (code) => {
-                const tail = this.stderrTail.slice(-600);
+                const tail = evidenceTail(this.stderrTail);
                 this.deadReason = `server exited (code ${code})${tail ? `; stderr tail: ${tail}` : ''}`;
                 for (const p of this.pending.values())
                     p.reject(new Error(this.deadReason));
@@ -95,7 +181,12 @@ export class McpStdioClient {
         return new Promise((resolve, reject) => {
             const timer = setTimeout(() => {
                 this.pending.delete(id);
-                reject(new Error(`timeout after ${timeoutMs}ms waiting for ${method}`));
+                // A process that is killed mid-hang never reaches the exit handler, so
+                // without this a timed-out record carries no evidence at all — it says
+                // only that we waited. What the server managed to print before it
+                // stopped answering is usually the whole explanation.
+                const tail = evidenceTail(this.stderrTail);
+                reject(new Error(`timeout after ${timeoutMs}ms waiting for ${method}${tail ? `; stderr tail: ${tail}` : ''}`));
             }, timeoutMs);
             this.pending.set(id, {
                 resolve: (v) => {

package/dist/sweep/report.d.ts

@@ -23,6 +23,22 @@ export interface ServerEntry {
      * before ever reaching tools/list. See docker.ts `dummyEnvValues`.
      */
     envValues?: Record<string, string>;
+    /**
+     * Declares that a failure of this entry is this harness's limitation, not the
+     * server's — an OS or architecture the package does not ship for, or a
+     * backing service the isolation deliberately does not provide.
+     *
+     * `evidence` is what keeps the declaration honest. The status only becomes
+     * `not-applicable` when the failure's own text contains that substring, so an
+     * annotation left behind after upstream changes cannot quietly absorb a real
+     * breakage: the server fails a different way, the evidence stops matching,
+     * and it is published as the failure it actually is. The entry is still
+     * attempted every sweep, so the day it starts working it simply measures.
+     */
+    notApplicable?: {
+        reason: string;
+        evidence: string;
+    };
 }
 export interface Row {
     entry: ServerEntry;

package/dist/sweep/run.d.ts

@@ -1,4 +1,40 @@
 import type { Measurement } from '../core/types.js';
+/**
+ * Which kind of failure a dead server's own words describe.
+ *
+ * The distinction is the published one: `auth-required` says the server works
+ * and this harness has no credentials for it, `startup-failure` says the server
+ * did not come up. Only the text decides, so it matters that the text reaching
+ * here is the part that explains the failure rather than whatever happened to
+ * fall in the last few hundred bytes — see `evidenceTail` in client.ts, which
+ * exists because a truncated message was being filed as a broken server.
+ */
+export declare function classifyFailure(msg: string): 'timeout' | 'auth-required' | 'startup-failure';
+/**
+ * The architecture a measurement ran on, in Docker's vocabulary (`linux/amd64`).
+ *
+ * Worth recording because a package can ship builds for some architectures and
+ * not others, and then the *machine* decides the result. `local-mcp` sat
+ * published as a startup failure on the strength of a run whose actual finding
+ * was that the laptop was arm64 and the package ships no arm64 runtime — and
+ * the record gave a reader no way to notice.
+ *
+ * Containers are linux whatever the host is; with no explicit `--platform` they
+ * take the host's architecture, so that is the half worth reporting.
+ */
+export declare function measuringArch(docker: boolean): string;
+/**
+ * The declared reason, when this failure is the one the entry warned about.
+ *
+ * Corroboration is the whole point: an entry may declare that this harness
+ * cannot run it, but only the failure's own words can confirm that *this*
+ * failure is that one. A macOS-only package that starts failing for some new
+ * reason stops matching, and is published as the failure it actually is.
+ */
+export declare function notApplicableReason(declared: {
+    reason: string;
+    evidence: string;
+} | undefined, msg: string): string | null;
 export interface MeasureOptions {
     timeoutMs?: number;
     env?: Record<string, string>;
@@ -11,6 +47,11 @@ export interface MeasureOptions {
     dummyEnvValues?: Record<string, string>;
     /** Install `git` in the container before launch (docker mode) — see docker.ts. */
     needsGit?: boolean;
+    /** Declared harness limitation for this entry — see `notApplicable` in report.ts. */
+    notApplicable?: {
+        reason: string;
+        evidence: string;
+    };
     /**
      * Exact argv, when the caller already has it (client configs store command and
      * args separately). Avoids re-splitting a joined string on spaces, which would

package/dist/sweep/run.js

@@ -11,6 +11,56 @@ import { captureTools } from './client.js';
 import { DockerHarnessFault, defaultImageFor, dockerize, ensureImage, isDockerRunFailure } from './docker.js';
 import { measureTools, failedMeasurement, canonicalString } from '../core/canonical.js';
 import { toBadge } from '../core/badge.js';
+/**
+ * Which kind of failure a dead server's own words describe.
+ *
+ * The distinction is the published one: `auth-required` says the server works
+ * and this harness has no credentials for it, `startup-failure` says the server
+ * did not come up. Only the text decides, so it matters that the text reaching
+ * here is the part that explains the failure rather than whatever happened to
+ * fall in the last few hundred bytes — see `evidenceTail` in client.ts, which
+ * exists because a truncated message was being filed as a broken server.
+ */
+export function classifyFailure(msg) {
+    // Matched against this harness's own phrasing, not the bare word: these
+    // messages carry the server's stderr, and a server that prints "connection
+    // timeout" before dying did not time out — it exited, and saying otherwise
+    // blames the clock for a breakage.
+    if (/timeout after \d+ms waiting for/.test(msg))
+        return 'timeout';
+    return /auth|unauthorized|401|forbidden|credential|api.?key|token/i.test(msg)
+        ? 'auth-required'
+        : 'startup-failure';
+}
+/**
+ * The architecture a measurement ran on, in Docker's vocabulary (`linux/amd64`).
+ *
+ * Worth recording because a package can ship builds for some architectures and
+ * not others, and then the *machine* decides the result. `local-mcp` sat
+ * published as a startup failure on the strength of a run whose actual finding
+ * was that the laptop was arm64 and the package ships no arm64 runtime — and
+ * the record gave a reader no way to notice.
+ *
+ * Containers are linux whatever the host is; with no explicit `--platform` they
+ * take the host's architecture, so that is the half worth reporting.
+ */
+export function measuringArch(docker) {
+    const arch = process.arch === 'x64' ? 'amd64' : process.arch;
+    return `${docker ? 'linux' : process.platform}/${arch}`;
+}
+/**
+ * The declared reason, when this failure is the one the entry warned about.
+ *
+ * Corroboration is the whole point: an entry may declare that this harness
+ * cannot run it, but only the failure's own words can confirm that *this*
+ * failure is that one. A macOS-only package that starts failing for some new
+ * reason stops matching, and is published as the failure it actually is.
+ */
+export function notApplicableReason(declared, msg) {
+    if (!declared?.evidence)
+        return null;
+    return msg.toLowerCase().includes(declared.evidence.toLowerCase()) ? declared.reason : null;
+}
 function arg(name) {
     const i = process.argv.indexOf(`--${name}`);
     return i >= 0 ? process.argv[i + 1] : undefined;
@@ -135,14 +185,17 @@ export async function measureServer(name, command, opts = {}) {
             if (dockerWrapped && isDockerRunFailure(msg)) {
                 throw new DockerHarnessFault(`docker could not run the container for ${name}: ${msg.slice(0, 400)}`);
             }
-            const status = msg.includes('timeout')
-                ? 'timeout'
-                : /auth|unauthorized|401|forbidden|credential|api.?key|token/i.test(msg)
-                    ? 'auth-required'
-                    : 'startup-failure';
-            r = failedMeasurement(status, { serverName: name, launchCommand: command, notes: msg.slice(0, 700) });
+            const declared = notApplicableReason(opts.notApplicable, msg);
+            r = failedMeasurement(declared ? 'not-applicable' : classifyFailure(msg), {
+                serverName: name,
+                launchCommand: command,
+                // The declared reason leads, but the raw failure stays behind it: the
+                // record has to remain checkable against the run that produced it.
+                notes: (declared ? `${declared} — ${msg}` : msg).slice(0, 700),
+            });
         }
-        r.isolation = isolation;
+        const iso = isolation ?? { docker: false };
+        r.isolation = { ...iso, arch: measuringArch(iso.docker) };
         r.timeoutMs = attemptOpts.timeoutMs ?? 60_000;
         return r;
     }

package/dist/sweep/sweep-all.js

@@ -18,7 +18,7 @@ import { DockerHarnessFault } from './docker.js';
 import { writeLeaderboard } from './report.js';
 import { appendHistory } from './history.js';
 import { appendToolVectors, writeRegressions } from './regressions.js';
-import { snapshot, verdict, restore } from './harness-guard.js';
+import { MIN_REGRESSIONS, snapshot, verdict, restore } from './harness-guard.js';
 import { selectShard, shardIndexForDate } from './shard.js';
 function arg(name) {
     const i = process.argv.indexOf(`--${name}`);
@@ -55,6 +55,18 @@ if (shards !== undefined) {
     const index = shardIndexArg ?? shardIndexForDate(new Date(), shards);
     entries = selectShard(sweepable, shards, index);
     shardLabel = `, shard ${index + 1}/${shards}`;
+    // The harness guard needs MIN_REGRESSIONS previously-good servers to fail
+    // together before it will call a broken runner rather than broken servers.
+    // A slice smaller than that floor can never reach it, so a wedged Docker
+    // daemon would publish the whole slice as startup failures with nothing to
+    // trip. `--shards` is the one knob that can shrink a slice under the floor
+    // unattended, so it refuses here instead of measuring through it.
+    if (entries.length < MIN_REGRESSIONS) {
+        console.error(`--shards ${shards} cuts a ${entries.length}-server slice, below the ` +
+            `${MIN_REGRESSIONS}-server floor the harness guard needs to tell a broken runner ` +
+            `from broken servers. Use a smaller --shards, or --only to measure a handful by name.`);
+        process.exit(2);
+    }
     console.log(`shard ${index + 1}/${shards} of ${sweepable.length} sweepable: ${entries.map((e) => e.name).join(', ')}`);
 }
 console.log(`sweeping ${entries.length} servers (docker=${docker}, concurrency=${concurrency}${shardLabel})`);
@@ -81,6 +93,7 @@ async function worker() {
                 dummyEnv: e.env ?? [],
                 dummyEnvValues: e.envValues,
                 needsGit: e.needsGit,
+                notApplicable: e.notApplicable,
             });
         }
         catch (err) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mcp-context-cost",
-  "version": "0.11.3",
+  "version": "0.12.0",
   "description": "Measure what your MCP servers cost in context tokens — audit your own config, or badge the server you publish",
   "type": "module",
   "license": "MIT",