mcp-context-cost 0.4.0 → 0.5.0

package/dist/sweep/session-start.d.ts

@@ -0,0 +1,3 @@
+import { type SessionStartRun } from '../core/session-start.js';
+export declare function loadSessionStart(root?: string): SessionStartRun | null;
+export declare function writeSessionStart(run: SessionStartRun, root?: string): void;

package/dist/sweep/session-start.js

@@ -0,0 +1,103 @@
+/**
+ * Instructions capture — the half of the session-start load that `tools/list`
+ * does not carry:
+ *   npx tsx src/sweep/session-start.ts [--docker] [--only a,b] [--concurrency 3]
+ *
+ * Writes results/session-start.json. Deliberately writes nothing else: the
+ * measurements on disk keep their numbers, their hashes and their dates, and a
+ * capture taken today never gets filed under a sweep taken on another day.
+ *
+ * This is a backfill, and it is meant to shrink. Every sweep from now on records
+ * `serverInstructions` inside the measurement itself, which takes precedence
+ * over anything here (see core/session-start.ts), so a server drops out of
+ * needing this file the first time it is re-measured.
+ *
+ * A server is launched the same way a sweep launches it, through `measureServer`
+ * — same isolation, same dummy env, same retries. That costs a full tools/list
+ * to obtain a string from `initialize`, which is the price of the row being
+ * comparable to the published measurement rather than merely adjacent to it:
+ * the fresh capture's `canonicalSha256` is what the row is filed under, so a
+ * server whose tools have drifted since publication is marked stale instead of
+ * having today's instructions attached to yesterday's tools.
+ */
+import { existsSync, readFileSync, writeFileSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { parse } from 'yaml';
+import { measureServer } from './run.js';
+import { SESSION_START_METHOD, parseSessionStart, toSessionStartRow, } from '../core/session-start.js';
+export function loadSessionStart(root = process.cwd()) {
+    const p = join(root, 'results', 'session-start.json');
+    return existsSync(p) ? parseSessionStart(readFileSync(p, 'utf8')) : null;
+}
+export function writeSessionStart(run, root = process.cwd()) {
+    // Key order sorted so a re-run of the same servers produces no diff noise.
+    const servers = {};
+    for (const name of Object.keys(run.servers).sort())
+        servers[name] = run.servers[name];
+    writeFileSync(join(root, 'results', 'session-start.json'), JSON.stringify({ ...run, servers }, null, 2) + '\n');
+}
+function arg(name) {
+    const i = process.argv.indexOf(`--${name}`);
+    return i >= 0 ? process.argv[i + 1] : undefined;
+}
+// Exact path match, for the reason src/sweep/run.ts states: any other file whose
+// name merely ends the same way would otherwise run this block.
+const isMain = process.argv[1] !== undefined && resolve(process.argv[1]) === fileURLToPath(import.meta.url);
+if (isMain) {
+    const doc = parse(readFileSync('servers.yaml', 'utf8'));
+    const only = arg('only')?.split(',');
+    const docker = process.argv.includes('--docker');
+    const concurrency = Number(arg('concurrency') ?? 3);
+    const defaultTimeout = Number(arg('default-timeout') ?? 60);
+    const entries = doc.servers.filter((s) => {
+        if (only && !only.includes(s.name))
+            return false;
+        return !s.remote; // a remote server never reaches initialize without credentials
+    });
+    console.log(`capturing instructions for ${entries.length} servers (docker=${docker}, concurrency=${concurrency})`);
+    // Merged, not replaced: a run over `--only` must not delete the rows it did
+    // not visit. A row it did visit is overwritten, errors included.
+    const prior = loadSessionStart();
+    const servers = { ...(prior?.servers ?? {}) };
+    const queue = [...entries];
+    async function worker() {
+        for (let e = queue.shift(); e; e = queue.shift()) {
+            const m = await measureServer(e.name, e.command, {
+                timeoutMs: (e.timeoutSeconds ?? defaultTimeout) * 1000,
+                docker,
+                dockerImage: e.dockerImage,
+                dummyEnv: e.env ?? [],
+                dummyEnvValues: e.envValues,
+                needsGit: e.needsGit,
+                persist: false, // the measurements on disk are not this run's to rewrite
+            });
+            if (m.status !== 'measured' && m.status !== 'dynamic') {
+                servers[e.name] = {
+                    instructions: '',
+                    instructionsTokens: 0,
+                    instructionsSha256: '',
+                    capturedSha256: null,
+                    error: `${m.status}: ${(m.notes ?? '').slice(0, 200)}`,
+                };
+                console.log(`  ${e.name}: ${m.status}`);
+                continue;
+            }
+            const row = toSessionStartRow(m.serverInstructions ?? null, {
+                capturedSha256: m.canonicalSha256,
+                serverVersion: m.serverVersion,
+            });
+            servers[e.name] = row;
+            console.log(`  ${e.name}: ${row.instructionsTokens} instruction tokens`);
+        }
+    }
+    await Promise.all(Array.from({ length: Math.max(1, concurrency) }, () => worker()));
+    writeSessionStart({
+        method: SESSION_START_METHOD,
+        measuredAt: new Date().toISOString().slice(0, 10),
+        isolation: docker ? 'docker (same images and limits as a sweep)' : 'host process (no container)',
+        servers,
+    });
+    const ok = Object.values(servers).filter((r) => !r.error).length;
+    console.log(`done: ${ok}/${Object.keys(servers).length} rows carry instructions; results/session-start.json written`);
+}

package/dist/sweep/shard.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Which servers does *this week's* re-sweep measure?
+ *
+ * `history.csv` only grows for servers that get measured again, and the only
+ * scheduled measurement this project runs is the weekly self-badge job — one
+ * server, `memory`. Every other row's trend line therefore stops at whatever
+ * date a maintenance run last swept by hand. Re-measuring all 82 on a schedule
+ * is not the fix: a cold runner pays a full image pull and package install per
+ * server with no cache volume to carry over, so the whole set does not fit in
+ * one job's budget comfortably or cheaply.
+ *
+ * So the set rotates. Each week measures one deterministic slice, and after
+ * `shardCount` weeks every server has been re-measured exactly once. The
+ * schedule needs no state anywhere — the slice is a function of the date, so a
+ * missed week is simply a week's worth of servers measured a cycle later, not a
+ * cursor left pointing at the wrong place.
+ */
+/**
+ * Whole UTC weeks since the Unix epoch. Only differences matter — this is a
+ * counter that advances once every 7 days, not a calendar week number, so it
+ * has no year boundary to get wrong.
+ */
+export declare function weekIndex(date: Date): number;
+/** The shard this date falls in, always in `[0, shardCount)`. */
+export declare function shardIndexForDate(date: Date, shardCount: number): number;
+/**
+ * Deal `items` into `shardCount` shards round-robin and return shard `index`.
+ *
+ * Interleaving by position rather than slicing contiguously is deliberate:
+ * `servers.yaml` is written in rough cost order within its groups, so a
+ * contiguous slice would hand one week every heavyweight server and another
+ * week nothing but cheap ones. Round-robin spreads the expensive entries
+ * across shards, which is what keeps any single week inside a runner's budget.
+ *
+ * The cost of positional dealing is that inserting a server shifts the ones
+ * after it by one shard. That is harmless — a shifted server is measured one
+ * week early or late, once, and every server is still measured exactly once per
+ * cycle. It is worth naming, though: shard membership is stable only as long as
+ * `servers.yaml`'s order is.
+ */
+export declare function selectShard<T>(items: readonly T[], shardCount: number, index: number): T[];

package/dist/sweep/shard.js

@@ -0,0 +1,58 @@
+/**
+ * Which servers does *this week's* re-sweep measure?
+ *
+ * `history.csv` only grows for servers that get measured again, and the only
+ * scheduled measurement this project runs is the weekly self-badge job — one
+ * server, `memory`. Every other row's trend line therefore stops at whatever
+ * date a maintenance run last swept by hand. Re-measuring all 82 on a schedule
+ * is not the fix: a cold runner pays a full image pull and package install per
+ * server with no cache volume to carry over, so the whole set does not fit in
+ * one job's budget comfortably or cheaply.
+ *
+ * So the set rotates. Each week measures one deterministic slice, and after
+ * `shardCount` weeks every server has been re-measured exactly once. The
+ * schedule needs no state anywhere — the slice is a function of the date, so a
+ * missed week is simply a week's worth of servers measured a cycle later, not a
+ * cursor left pointing at the wrong place.
+ */
+const MS_PER_DAY = 86_400_000;
+/**
+ * Whole UTC weeks since the Unix epoch. Only differences matter — this is a
+ * counter that advances once every 7 days, not a calendar week number, so it
+ * has no year boundary to get wrong.
+ */
+export function weekIndex(date) {
+    const utcDay = Math.floor(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()) / MS_PER_DAY);
+    return Math.floor(utcDay / 7);
+}
+/** The shard this date falls in, always in `[0, shardCount)`. */
+export function shardIndexForDate(date, shardCount) {
+    if (!Number.isInteger(shardCount) || shardCount < 1) {
+        throw new Error(`shardCount must be a positive integer, got ${shardCount}`);
+    }
+    return weekIndex(date) % shardCount;
+}
+/**
+ * Deal `items` into `shardCount` shards round-robin and return shard `index`.
+ *
+ * Interleaving by position rather than slicing contiguously is deliberate:
+ * `servers.yaml` is written in rough cost order within its groups, so a
+ * contiguous slice would hand one week every heavyweight server and another
+ * week nothing but cheap ones. Round-robin spreads the expensive entries
+ * across shards, which is what keeps any single week inside a runner's budget.
+ *
+ * The cost of positional dealing is that inserting a server shifts the ones
+ * after it by one shard. That is harmless — a shifted server is measured one
+ * week early or late, once, and every server is still measured exactly once per
+ * cycle. It is worth naming, though: shard membership is stable only as long as
+ * `servers.yaml`'s order is.
+ */
+export function selectShard(items, shardCount, index) {
+    if (!Number.isInteger(shardCount) || shardCount < 1) {
+        throw new Error(`shardCount must be a positive integer, got ${shardCount}`);
+    }
+    if (!Number.isInteger(index) || index < 0 || index >= shardCount) {
+        throw new Error(`shard index must be in [0, ${shardCount}), got ${index}`);
+    }
+    return items.filter((_, i) => i % shardCount === index);
+}

package/dist/sweep/sweep-all.js

@@ -1,14 +1,23 @@
 /**
  * Batch sweep over servers.yaml:
  *   npx tsx src/sweep/sweep-all.ts [--docker] [--only name1,name2] [--concurrency 3]
+ *                                  [--shards N [--shard-index K]]
  * Writes results/<name>/measurement.json + badges/<name>.json per server, then
  * regenerates the leaderboard.
+ *
+ * `--shards N` measures only this week's rotating slice (see shard.ts), so a
+ * scheduled job can keep every server's history growing without sweeping all of
+ * servers.yaml in one runner. `--shard-index K` pins the slice instead of
+ * deriving it from the date — for reproducing a given week, not for scheduled
+ * use.
  */
 import { readFileSync } from 'node:fs';
 import { parse } from 'yaml';
 import { measureServer } from './run.js';
 import { writeLeaderboard } from './report.js';
 import { appendHistory } from './history.js';
+import { snapshot, verdict, restore } from './harness-guard.js';
+import { selectShard, shardIndexForDate } from './shard.js';
 function arg(name) {
     const i = process.argv.indexOf(`--${name}`);
     return i >= 0 ? process.argv[i + 1] : undefined;
@@ -18,16 +27,43 @@ const only = arg('only')?.split(',');
 const docker = process.argv.includes('--docker');
 const concurrency = Number(arg('concurrency') ?? 3);
 const defaultTimeout = Number(arg('default-timeout') ?? 60);
-const entries = doc.servers.filter((s) => {
+const shards = arg('shards') === undefined ? undefined : Number(arg('shards'));
+const shardIndexArg = arg('shard-index') === undefined ? undefined : Number(arg('shard-index'));
+if (shards !== undefined && only) {
+    // Both narrow the set, but the sharded one is meant to be a *complete*
+    // partition of servers.yaml. Silently intersecting them would produce a slice
+    // that belongs to no cycle and still looks like a normal week's sweep in the log.
+    console.error('--shards and --only both select servers; pass one or the other');
+    process.exit(2);
+}
+if (shardIndexArg !== undefined && shards === undefined) {
+    console.error('--shard-index needs --shards');
+    process.exit(2);
+}
+const sweepable = doc.servers.filter((s) => {
     if (only && !only.includes(s.name))
         return false;
     if (s.remote)
         return false; // remote servers handled separately (mcp-remote bridge)
     return true;
 });
-console.log(`sweeping ${entries.length} servers (docker=${docker}, concurrency=${concurrency})`);
+let entries = sweepable;
+let shardLabel = '';
+if (shards !== undefined) {
+    const index = shardIndexArg ?? shardIndexForDate(new Date(), shards);
+    entries = selectShard(sweepable, shards, index);
+    shardLabel = `, shard ${index + 1}/${shards}`;
+    console.log(`shard ${index + 1}/${shards} of ${sweepable.length} sweepable: ${entries.map((e) => e.name).join(', ')}`);
+}
+console.log(`sweeping ${entries.length} servers (docker=${docker}, concurrency=${concurrency}${shardLabel})`);
+// Taken before anything is measured: measureServer persists each result the
+// moment it has one, so this is the only chance to hold on to what was
+// published in case this sweep turns out to be measuring through a broken
+// harness (see harness-guard.ts).
+const prior = snapshot(entries.map((e) => e.name));
 const queue = [...entries];
 const summary = {};
+const statuses = new Map();
 async function worker() {
     for (let e = queue.shift(); e; e = queue.shift()) {
         const started = Date.now();
@@ -36,9 +72,11 @@ async function worker() {
             docker,
             dockerImage: e.dockerImage,
             dummyEnv: e.env ?? [],
+            dummyEnvValues: e.envValues,
             needsGit: e.needsGit,
         });
         const secs = ((Date.now() - started) / 1000).toFixed(0);
+        statuses.set(e.name, m.status);
         summary[e.name] =
             m.status === 'measured' || m.status === 'dynamic'
                 ? `${m.totalTokens} tokens / ${m.toolCount} tools (${m.status}, ${secs}s)`
@@ -47,7 +85,23 @@ async function worker() {
     }
 }
 await Promise.all(Array.from({ length: Math.max(1, concurrency) }, () => worker()));
+// Before publishing anything: is this sweep a statement about the servers, or
+// about the machine that measured them?
+const v = verdict(prior, statuses);
+console.log(`harness check: ${v.reason}`);
+if (v.fault) {
+    const restored = restore(prior, v.regressed);
+    console.error(`\nHARNESS FAULT — refusing to publish this sweep.\n` +
+        `  regressed: ${v.regressed.join(', ')}\n` +
+        `  restored ${restored.length} previous measurement(s) and badge(s); ` +
+        `leaderboard and history.csv are untouched.\n` +
+        `  Check the measurement environment (Docker daemon, network, disk) and re-run.`);
+    process.exit(1);
+}
 // Serial, after every worker has finished: history.csv is a read-modify-write.
+// Always the full set: the leaderboard is regenerated from what is on disk, so
+// a shard sweep republishes every server's most recent measurement rather than
+// dropping the servers this week did not touch.
 writeLeaderboard(doc.servers);
 const h = appendHistory();
 const measured = Object.values(summary).filter((s) => s.includes('tokens')).length;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mcp-context-cost",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "description": "Measure what your MCP servers cost in context tokens — audit your own config, or badge the server you publish",
   "type": "module",
   "license": "MIT",
@@ -37,7 +37,9 @@
     "test": "vitest run",
     "test:badge": "./upstream/tests/badge-test.sh",
     "typecheck": "tsc --noEmit && tsc -p tsconfig.tools.json",
+    "adoption": "tsx tools/measure-adoption.ts",
     "divergence": "tsx tools/measure-divergence.ts",
+    "session-start": "tsx src/sweep/session-start.ts",
     "sweep": "tsx src/sweep/run.ts",
     "sweep:all": "tsx src/sweep/sweep-all.ts",
     "verify": "tsx src/cli.ts verify"