mcp-context-cost 0.1.0

package/dist/sweep/run.js

@@ -0,0 +1,94 @@
+/**
+ * Single-server measurement runner:
+ *   npm run sweep -- --name memory --command "npx -y @modelcontextprotocol/server-memory"
+ * Writes results/<name>/measurement.json and badges/<name>.json.
+ * Runs tools/list capture TWICE; differing tool sets -> status "dynamic".
+ */
+import { mkdirSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { captureTools } from './client.js';
+import { dockerize } from './docker.js';
+import { measureTools, failedMeasurement, canonicalString } from '../core/canonical.js';
+import { toBadge } from '../core/badge.js';
+function arg(name) {
+    const i = process.argv.indexOf(`--${name}`);
+    return i >= 0 ? process.argv[i + 1] : undefined;
+}
+export async function measureServer(name, command, opts = {}) {
+    if (!/^[a-z0-9][a-z0-9._-]*$/i.test(name) || name.includes('..')) {
+        throw new Error(`invalid server name '${name}' — letters/digits/dot/dash/underscore only`);
+    }
+    const root = opts.root ?? process.cwd();
+    let spec = command;
+    let isolation = { docker: false };
+    let containerName;
+    if (opts.docker && command.trimStart().startsWith('docker ')) {
+        // Command is already a container — wrapping it again would need docker-in-docker.
+        isolation = { docker: true, note: 'command is itself a docker run (host-spawned container)' };
+    }
+    else if (opts.docker) {
+        containerName = `mcp-ctx-${name}-${process.pid}-${Math.floor(Math.random() * 1e6)}`;
+        const d = dockerize(command, { image: opts.dockerImage, dummyEnv: opts.dummyEnv, containerName });
+        spec = { command: d.command, argv: d.argv };
+        isolation = d.isolation;
+    }
+    let m;
+    try {
+        const first = await captureTools(spec, opts);
+        const second = await captureTools(spec, opts);
+        m = measureTools(first.tools, {
+            serverName: first.serverInfo?.name ?? name,
+            serverVersion: first.serverInfo?.version,
+            launchCommand: command,
+            envVarNames: [...Object.keys(opts.env ?? {}), ...(opts.dummyEnv ?? [])],
+        });
+        m.isolation = isolation;
+        m.timeoutMs = opts.timeoutMs ?? 60_000;
+        if (canonicalString(first.tools) !== canonicalString(second.tools)) {
+            m.status = 'dynamic';
+            m.notes = 'tools/list differed between two runs; value is for the first capture';
+        }
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        const status = msg.includes('timeout')
+            ? 'timeout'
+            : /auth|unauthorized|401|forbidden|credential|api.?key|token/i.test(msg)
+                ? 'auth-required'
+                : 'startup-failure';
+        m = failedMeasurement(status, { serverName: name, launchCommand: command, notes: msg.slice(0, 700) });
+        m.isolation = isolation;
+        m.timeoutMs = opts.timeoutMs ?? 60_000;
+    }
+    finally {
+        if (containerName) {
+            // Killing the docker CLI on timeout orphans the container — remove it.
+            const { spawn } = await import('node:child_process');
+            spawn('docker', ['rm', '-f', containerName], { stdio: 'ignore' }).on('error', () => { });
+        }
+    }
+    const resultDir = join(root, 'results', name);
+    mkdirSync(resultDir, { recursive: true });
+    writeFileSync(join(resultDir, 'measurement.json'), JSON.stringify(m, null, 2) + '\n');
+    mkdirSync(join(root, 'badges'), { recursive: true });
+    writeFileSync(join(root, 'badges', `${name}.json`), JSON.stringify(toBadge(m)) + '\n');
+    return m;
+}
+const isMain = process.argv[1]?.endsWith('run.ts') || process.argv[1]?.endsWith('run.js');
+if (isMain) {
+    const name = arg('name');
+    const command = arg('command');
+    if (!name || !command) {
+        console.error('usage: npm run sweep -- --name <slug> --command "<launch command>"');
+        process.exit(2);
+    }
+    const m = await measureServer(name, command, {
+        timeoutMs: Number(arg('timeout') ?? 60_000),
+        docker: process.argv.includes('--docker'),
+        dockerImage: arg('docker-image'),
+    });
+    console.log(m.status === 'measured' || m.status === 'dynamic'
+        ? `${name}: ${m.totalTokens} tokens across ${m.toolCount} tools (${m.status})`
+        : `${name}: ${m.status} — ${m.notes ?? ''}`);
+    process.exit(m.status === 'measured' || m.status === 'dynamic' ? 0 : 1);
+}

package/dist/sweep/sweep-all.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/sweep/sweep-all.js

@@ -0,0 +1,50 @@
+/**
+ * Batch sweep over servers.yaml:
+ *   npx tsx src/sweep/sweep-all.ts [--docker] [--only name1,name2] [--concurrency 3]
+ * Writes results/<name>/measurement.json + badges/<name>.json per server, then
+ * regenerates the leaderboard.
+ */
+import { readFileSync } from 'node:fs';
+import { parse } from 'yaml';
+import { measureServer } from './run.js';
+import { writeLeaderboard } from './report.js';
+function arg(name) {
+    const i = process.argv.indexOf(`--${name}`);
+    return i >= 0 ? process.argv[i + 1] : undefined;
+}
+const doc = parse(readFileSync('servers.yaml', 'utf8'));
+const only = arg('only')?.split(',');
+const docker = process.argv.includes('--docker');
+const concurrency = Number(arg('concurrency') ?? 3);
+const defaultTimeout = Number(arg('default-timeout') ?? 60);
+const entries = doc.servers.filter((s) => {
+    if (only && !only.includes(s.name))
+        return false;
+    if (s.remote)
+        return false; // remote servers handled separately (mcp-remote bridge)
+    return true;
+});
+console.log(`sweeping ${entries.length} servers (docker=${docker}, concurrency=${concurrency})`);
+const queue = [...entries];
+const summary = {};
+async function worker() {
+    for (let e = queue.shift(); e; e = queue.shift()) {
+        const started = Date.now();
+        const m = await measureServer(e.name, e.command, {
+            timeoutMs: (e.timeoutSeconds ?? defaultTimeout) * 1000,
+            docker,
+            dockerImage: e.dockerImage,
+            dummyEnv: e.env ?? [],
+        });
+        const secs = ((Date.now() - started) / 1000).toFixed(0);
+        summary[e.name] =
+            m.status === 'measured' || m.status === 'dynamic'
+                ? `${m.totalTokens} tokens / ${m.toolCount} tools (${m.status}, ${secs}s)`
+                : `${m.status} (${secs}s)`;
+        console.log(`  ${e.name}: ${summary[e.name]}`);
+    }
+}
+await Promise.all(Array.from({ length: Math.max(1, concurrency) }, () => worker()));
+writeLeaderboard(doc.servers);
+const measured = Object.values(summary).filter((s) => s.includes('tokens')).length;
+console.log(`done: ${measured}/${entries.length} measured; leaderboard regenerated`);

package/package.json

@@ -0,0 +1,52 @@
+{
+  "name": "mcp-context-cost",
+  "version": "0.1.0",
+  "description": "Reproducible context-cost badges for MCP servers — measure what a server's tool schemas cost before the agent does any work",
+  "type": "module",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/athakur3/mcp-context-cost.git"
+  },
+  "homepage": "https://athakur3.github.io/mcp-context-cost/",
+  "bugs": "https://github.com/athakur3/mcp-context-cost/issues",
+  "keywords": [
+    "mcp",
+    "model-context-protocol",
+    "tokens",
+    "context-window",
+    "badge",
+    "shields",
+    "developer-tools"
+  ],
+  "bin": {
+    "mcp-context-cost": "dist/cli.js"
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "test": "vitest run",
+    "test:badge": "./upstream/tests/badge-test.sh",
+    "typecheck": "tsc --noEmit",
+    "sweep": "tsx src/sweep/run.ts",
+    "sweep:all": "tsx src/sweep/sweep-all.ts",
+    "verify": "tsx src/cli.ts verify"
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "dependencies": {
+    "js-tiktoken": "^1.0.21",
+    "yaml": "^2.9.0"
+  },
+  "devDependencies": {
+    "@types/node": "^26.2.0",
+    "tsx": "^4.23.12",
+    "typescript": "^7.0.2",
+    "vitest": "^4.1.10"
+  }
+}