mcp-context-cost 0.11.2 → 0.11.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -179,6 +179,13 @@ export function buildReport(configs, measured, opts = {}) {
179
179
  const contextWindow = opts.contextWindow ?? DEFAULT_CONTEXT_WINDOW;
180
180
  const problems = [];
181
181
  const emptyConfigs = [];
182
+ // canonicalSha256 → the divergence row computed from it, so a server can be
183
+ // identified by its bytes whatever the local config calls it.
184
+ const divByHash = new Map();
185
+ for (const row of Object.values(opts.divergence?.servers ?? {})) {
186
+ if (row?.capturedSha256)
187
+ divByHash.set(row.capturedSha256, row);
188
+ }
182
189
  const built = [];
183
190
  // Across every config at once: a twin in one client's file is measured for
184
191
  // the other client's entry just the same.
@@ -182,9 +182,18 @@ export function configCandidates(env) {
182
182
  /** Read + parse the candidates that exist. Unreadable files are reported, not thrown. */
183
183
  export function loadConfigs(candidates, cwd) {
184
184
  const out = [];
185
+ // Running from your home directory nominates `~/.cursor/mcp.json` twice —
186
+ // once as the home candidate, once as the cwd one. Loaded twice it is
187
+ // reported twice, doubles that client's deferral scope, and under
188
+ // `--baseline` the second copy pairs with nothing and fails the gate. One
189
+ // path is one config however many ways it was nominated.
190
+ const seen = new Set();
185
191
  for (const c of candidates) {
186
192
  if (!existsSync(c.path))
187
193
  continue;
194
+ if (seen.has(c.path))
195
+ continue;
196
+ seen.add(c.path);
188
197
  try {
189
198
  const doc = parseJsonc(readFileSync(c.path, 'utf8'));
190
199
  const { servers, disabled } = extractDeclaration(doc, { client: c.client, source: c.path, cwd });
@@ -181,7 +181,15 @@ export function pairConfigs(before, after) {
181
181
  pairs.push({ before: null, after: cur, matchedBy: 'unmatched' });
182
182
  }
183
183
  }
184
- if (before.length === 1 && after.length === 1 && pairs[0].before === null) {
184
+ // One on each side reads as the same config seen from two machines, where the
185
+ // path legitimately differs (a laptop's baseline against a CI checkout). It
186
+ // does not survive the clients differing: a Claude Desktop baseline against a
187
+ // Claude Code run compares two unrelated stacks, and the gate then rests on
188
+ // that difference. The paths may differ; what they are configs *for* may not.
189
+ if (before.length === 1 &&
190
+ after.length === 1 &&
191
+ pairs[0].before === null &&
192
+ before[0].client === after[0].client) {
185
193
  pairs[0] = { before: before[0], after: after[0], matchedBy: 'sole-config' };
186
194
  unusedBefore.delete(before[0].source);
187
195
  }
package/dist/cli.js CHANGED
@@ -45,6 +45,19 @@ export function slugFromUrl(url) {
45
45
  const host = new URL(url).hostname.replace(/^(www|mcp)\./, '');
46
46
  return host.replace(/[^a-z0-9]+/gi, '-').replace(/^-+|-+$/g, '').toLowerCase() || 'remote';
47
47
  }
48
+ /**
49
+ * Report a `verify` failure and exit 1, in whichever shape the caller asked
50
+ * for. `--json` is documented as putting `{ ok, rederivedTokens, rederivedSha,
51
+ * problems }` on stdout; a script reading that gets nothing from a thrown
52
+ * exception, so every failure path goes through here.
53
+ */
54
+ function failVerify(json, problem) {
55
+ if (json)
56
+ console.log(JSON.stringify({ ok: false, rederivedTokens: null, rederivedSha: null, problems: [problem] }));
57
+ else
58
+ console.error(problem);
59
+ process.exit(1);
60
+ }
48
61
  /** Installed version, for error messages that need to say which one you are running. */
49
62
  export function cliVersion() {
50
63
  try {
@@ -322,6 +335,12 @@ if (cmd === 'audit') {
322
335
  `${where}\n` +
323
336
  `${empty.length === 1 ? 'It was' : 'They were'} read and parsed; there is simply nothing declared to measure.\n` +
324
337
  `Declare a server in one of them, or point at a different config: mcp-context-cost audit --config <path/to/mcp.json>`);
338
+ else if (all('config').length)
339
+ // A path the user named is not a discovery miss. Saying "looked in the
340
+ // standard locations" describes something the command did not do, and
341
+ // then advises doing the thing they just did.
342
+ console.error(`no MCP config found at the path(s) given: ${all('config').join(', ')}. ` +
343
+ `Nothing else was searched, because --config was set.`);
325
344
  else
326
345
  console.error(`no MCP config found. Looked in the standard Claude Desktop / Claude Code / Cursor / VS Code / Windsurf locations.${where}\n` +
327
346
  `Point at one explicitly: mcp-context-cost audit --config <path/to/mcp.json>`);
@@ -355,18 +374,29 @@ else if (cmd === 'verify') {
355
374
  raw = await res.text();
356
375
  }
357
376
  catch (e) {
358
- const problem = `failed to fetch ${remoteUrl}: ${e.message}`;
359
- if (json)
360
- console.log(JSON.stringify({ ok: false, rederivedTokens: null, rederivedSha: null, problems: [problem] }));
361
- else
362
- console.error(problem);
363
- process.exit(1);
377
+ failVerify(json, `failed to fetch ${remoteUrl}: ${e.message}`);
364
378
  }
365
379
  }
366
380
  else {
367
- raw = readFileSync(path, 'utf8');
381
+ try {
382
+ raw = readFileSync(path, 'utf8');
383
+ }
384
+ catch (e) {
385
+ // The remote branch above reports a failed fetch in the documented shape;
386
+ // this one used to throw, so `--json` produced a stack trace on stderr and
387
+ // nothing at all on stdout — the contract a script parses.
388
+ failVerify(json, `cannot read ${path}: ${e.message}`);
389
+ }
390
+ }
391
+ let m;
392
+ try {
393
+ m = JSON.parse(raw);
394
+ }
395
+ catch (e) {
396
+ // Reachable remotely: a proxy, a captive portal or an HTML error page
397
+ // served with status 200 passes the `res.ok` check above and arrives here.
398
+ failVerify(json, `${remoteUrl ?? path} is not valid JSON: ${e.message}`);
368
399
  }
369
- const m = JSON.parse(raw);
370
400
  const r = verifyMeasurement(m);
371
401
  if (json) {
372
402
  console.log(JSON.stringify({ serverName: m.serverName, ...r, badge: r.ok ? toBadge(m) : undefined }));
@@ -143,6 +143,14 @@ export declare function decodeLoose(text: string): string;
143
143
  export interface EndpointBadge {
144
144
  url: string;
145
145
  linkTarget: string | null;
146
+ /**
147
+ * The badge's own label — markdown alt text or an `<img alt>`. What the badge
148
+ * calls itself is the only thing in a README that distinguishes *this*
149
+ * project's badge from any other shields endpoint badge, since the JSON can
150
+ * be hosted anywhere (the staged action publishes to a gist, with no path
151
+ * shape to recognise).
152
+ */
153
+ alt: string | null;
146
154
  }
147
155
  /**
148
156
  * Every shields endpoint badge in a file, decoded, each paired with its own
@@ -172,6 +180,21 @@ export declare function linksBackToProject(target: string, src?: BadgeSource): b
172
180
  * measurement here. See the header for why both count and why the link is
173
181
  * paired with the image rather than looked for anywhere in the file.
174
182
  */
183
+ /**
184
+ * Whether a badge calls itself this project's badge.
185
+ *
186
+ * The self-hosted branch below used to accept the link alone and look at
187
+ * nothing else, so *any* shields endpoint badge — a coverage badge, say —
188
+ * wrapped in a link to this repository counted as displaying ours. That
189
+ * inflates the one number this project keeps about itself, which is the last
190
+ * place it can afford a generous reading.
191
+ *
192
+ * The URL cannot decide it: self-hosted JSON lives wherever its author put it,
193
+ * and the staged action publishes to a gist. What every published snippet does
194
+ * carry is the label — `context cost` — so that is what is read, tolerantly
195
+ * enough to accept `Context-Cost` and strictly enough to reject `coverage`.
196
+ */
197
+ export declare function namesThisBadge(alt: string | null): boolean;
175
198
  export declare function displaysBadge(text: string, src?: BadgeSource): boolean;
176
199
  /**
177
200
  * Every shields endpoint `url` in a file whose JSON is served from this
@@ -137,7 +137,11 @@ export function endpointBadges(text) {
137
137
  if (last && !before.slice(last.index ?? 0).includes('</a>'))
138
138
  linkTarget = last[1];
139
139
  }
140
- out.push({ url: m[1], linkTarget });
140
+ // Markdown puts the alt before the image; HTML puts it in the same tag.
141
+ const beforeAlt = decoded.slice(Math.max(0, start - LINK_WINDOW), start);
142
+ const mdAlt = beforeAlt.match(/!\[([^\]]*)\]\(\s*[^\s)]*$/);
143
+ const tagAlt = after.match(/^[^<>]*?\balt\s*=\s*["']([^"']*)["']/i);
144
+ out.push({ url: m[1], linkTarget, alt: mdAlt ? mdAlt[1] : tagAlt ? tagAlt[1] : null });
141
145
  }
142
146
  return out;
143
147
  }
@@ -169,8 +173,28 @@ export function linksBackToProject(target, src = BADGE_SOURCE) {
169
173
  * measurement here. See the header for why both count and why the link is
170
174
  * paired with the image rather than looked for anywhere in the file.
171
175
  */
176
+ /**
177
+ * Whether a badge calls itself this project's badge.
178
+ *
179
+ * The self-hosted branch below used to accept the link alone and look at
180
+ * nothing else, so *any* shields endpoint badge — a coverage badge, say —
181
+ * wrapped in a link to this repository counted as displaying ours. That
182
+ * inflates the one number this project keeps about itself, which is the last
183
+ * place it can afford a generous reading.
184
+ *
185
+ * The URL cannot decide it: self-hosted JSON lives wherever its author put it,
186
+ * and the staged action publishes to a gist. What every published snippet does
187
+ * carry is the label — `context cost` — so that is what is read, tolerantly
188
+ * enough to accept `Context-Cost` and strictly enough to reject `coverage`.
189
+ */
190
+ export function namesThisBadge(alt) {
191
+ if (!alt)
192
+ return false;
193
+ return alt.toLowerCase().replace(/[^a-z]/g, '').includes('contextcost');
194
+ }
172
195
  export function displaysBadge(text, src = BADGE_SOURCE) {
173
- return endpointBadges(text).some((b) => hostedHere(b.url, src) || (b.linkTarget !== null && linksBackToProject(b.linkTarget, src)));
196
+ return endpointBadges(text).some((b) => hostedHere(b.url, src) ||
197
+ (namesThisBadge(b.alt) && b.linkTarget !== null && linksBackToProject(b.linkTarget, src)));
174
198
  }
175
199
  /**
176
200
  * Every shields endpoint `url` in a file whose JSON is served from this
@@ -72,14 +72,18 @@ export function identify(canonicalSha256, index) {
72
72
  if (!mine)
73
73
  return { kind: 'unknown' };
74
74
  const currentSha = index.current[mine.server];
75
- if (!currentSha || currentSha === canonicalSha256) {
75
+ if (currentSha === canonicalSha256) {
76
76
  return { kind: 'current', server: mine.server, date: mine.date, tokens: mine.totalTokens };
77
77
  }
78
- const current = index.captures[currentSha];
79
- // A `current` pointer with no capture behind it describes nothing; treat the
80
- // version as identified but with nothing to compare it against.
78
+ // The bytes are identified, but what is current for this server is not: the
79
+ // pointer is missing, or points at a capture the index dropped as ambiguous.
80
+ // `current` would be an affirmative claim that nothing has moved, which the
81
+ // audit prints as "no server here is running a published capture that has
82
+ // since moved" — told to someone who may be far behind. Unknown currency
83
+ // reads as unknown, which is the discipline the rest of this module keeps.
84
+ const current = currentSha ? index.captures[currentSha] : undefined;
81
85
  if (!current)
82
- return { kind: 'current', server: mine.server, date: mine.date, tokens: mine.totalTokens };
86
+ return { kind: 'unknown' };
83
87
  return {
84
88
  kind: 'behind',
85
89
  server: mine.server,
@@ -160,6 +160,9 @@ export function isComparable(row, canonicalSha256) {
160
160
  canonicalSha256 !== null &&
161
161
  row.capturedSha256 === canonicalSha256 &&
162
162
  row.ourTokens > 0 &&
163
+ // A report that parsed but carries `total: 0` is not a measurement of
164
+ // anything; published, it renders as a −100% divergence.
165
+ row.cliTokens > 0 &&
163
166
  row.ourMappedTokens > 0);
164
167
  }
165
168
  /**
@@ -55,6 +55,14 @@ export function mappedTokens(raw) {
55
55
  export function fieldSelectionShare(row) {
56
56
  if (row.o200kFull <= 0)
57
57
  return null;
58
+ // The projection can add bytes rather than remove them — `inputSchema`
59
+ // becomes the longer `input_schema`, and a tool with no description gains
60
+ // `description: ""` — so a server with almost no metadata to drop can map
61
+ // *larger* than it measured. There is no share of the payload removed in that
62
+ // case, and publishing a negative one reads as "−11.1% of the capture is
63
+ // MCP-only metadata", which is not a thing.
64
+ if (row.o200kMapped > row.o200kFull)
65
+ return null;
58
66
  return (row.o200kFull - row.o200kMapped) / row.o200kFull;
59
67
  }
60
68
  /**
@@ -124,7 +124,20 @@ export interface ToolAttribution {
124
124
  */
125
125
  unexplainedTokens: number;
126
126
  }
127
- export declare function attribute(from: ToolVectorEntry, to: ToolVectorEntry, deltaTokens: number): ToolAttribution;
127
+ /**
128
+ * Per-tool attribution, or null when the names cannot carry it.
129
+ *
130
+ * The breakdown matches tools by name, so a name that appears twice on either
131
+ * side makes the maps below lose one of them silently — and the lost tokens
132
+ * resurface as `unexplainedTokens`, which the report explains to the reader as
133
+ * canonical-array framing bytes. That is a confident false explanation. Two
134
+ * ways it happens: a server that ships duplicate or namespaced-collapsed tool
135
+ * names, and `measureTools` recording every nameless tool as the single key
136
+ * `(unnamed)` — an invented name, where `toolNames` and `toAnthropicTools`
137
+ * deliberately drop nameless tools rather than invent one. Where the names
138
+ * cannot identify the tools, there is no attribution to give.
139
+ */
140
+ export declare function attribute(from: ToolVectorEntry, to: ToolVectorEntry, deltaTokens: number): ToolAttribution | null;
128
141
  export interface CostChange {
129
142
  server: string;
130
143
  fromDate: string;
@@ -87,7 +87,23 @@ export function mechanismOf(deltaTokens, deltaTools) {
87
87
  // and rewritten, and the totals cannot separate the two.
88
88
  return 'mixed';
89
89
  }
90
+ /**
91
+ * Per-tool attribution, or null when the names cannot carry it.
92
+ *
93
+ * The breakdown matches tools by name, so a name that appears twice on either
94
+ * side makes the maps below lose one of them silently — and the lost tokens
95
+ * resurface as `unexplainedTokens`, which the report explains to the reader as
96
+ * canonical-array framing bytes. That is a confident false explanation. Two
97
+ * ways it happens: a server that ships duplicate or namespaced-collapsed tool
98
+ * names, and `measureTools` recording every nameless tool as the single key
99
+ * `(unnamed)` — an invented name, where `toolNames` and `toAnthropicTools`
100
+ * deliberately drop nameless tools rather than invent one. Where the names
101
+ * cannot identify the tools, there is no attribution to give.
102
+ */
90
103
  export function attribute(from, to, deltaTokens) {
104
+ const unique = (ts) => new Set(ts.map((t) => t.name)).size === ts.length;
105
+ if (!unique(from.tools) || !unique(to.tools))
106
+ return null;
91
107
  const before = new Map(from.tools.map((t) => [t.name, t.tokens]));
92
108
  const after = new Map(to.tools.map((t) => [t.name, t.tokens]));
93
109
  const added = [];
@@ -23,10 +23,18 @@ export function quantileTable(values) {
23
23
  * checkable by anyone holding the same JSON.
24
24
  */
25
25
  export function percentileOf(quantiles, value) {
26
+ // The LOWEST percentile whose quantile the value reaches, not the highest.
27
+ // Taking the highest reports a value tied with half the measured set as p100
28
+ // — "heavier than 100% of measured tools" about something exactly average for
29
+ // its tie — because every percentile across the tie carries the same
30
+ // quantile. The lowest names where the tie begins, which is what "heavier
31
+ // than P% of tools" means.
26
32
  let p = 0;
27
33
  for (let i = 0; i <= 100; i++) {
28
- if (quantiles[i] <= value)
29
- p = i;
34
+ if (quantiles[i] <= value) {
35
+ if (quantiles[i] < value || i === 0 || quantiles[i - 1] < quantiles[i])
36
+ p = i;
37
+ }
30
38
  else
31
39
  break;
32
40
  }
@@ -7,6 +7,7 @@ import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'node:fs';
7
7
  import { dirname, join } from 'node:path';
8
8
  import { parse } from 'yaml';
9
9
  import { isCurrent } from '../core/divergence.js';
10
+ import { loadRows } from './report.js';
10
11
  import { bandColor, BAND_META } from '../core/bands.js';
11
12
  import { parseHistory, plottableSeries } from './history.js';
12
13
  /** Longest series a sparkline plots — a stat-tile trend, not a full chart. */
@@ -53,10 +54,9 @@ export function generateDashboard(root = process.cwd()) {
53
54
  // Only the run of sweeps taken under the same isolation is plotted: a step
54
55
  // across an isolation change is the harness moving, not the server.
55
56
  const seriesFor = (name) => plottableSeries(history.filter((h) => h.server === name));
56
- const rows = doc.servers.map((entry) => {
57
- const p = join(root, 'results', entry.name, 'measurement.json');
58
- return { entry, m: existsSync(p) ? JSON.parse(readFileSync(p, 'utf8')) : null };
59
- });
57
+ // Shared with the leaderboard rather than re-read here: one definition of how
58
+ // a measurement is loaded, including its tolerance for a half-written file.
59
+ const rows = loadRows(doc.servers, root);
60
60
  const measured = rows
61
61
  .filter((r) => r.m && (r.m.status === 'measured' || r.m.status === 'dynamic') && r.m.totalTokens !== null)
62
62
  .sort((a, b) => (b.m.totalTokens ?? 0) - (a.m.totalTokens ?? 0));
@@ -47,7 +47,15 @@ export interface Verdict {
47
47
  * `current` maps server name to the status it just measured at. Servers absent
48
48
  * from it were not swept and are ignored.
49
49
  */
50
- export declare function verdict(prior: Snapshot[], current: Map<string, MeasurementStatus>): Verdict;
50
+ export declare function verdict(prior: Snapshot[], current: Map<string, MeasurementStatus>,
51
+ /**
52
+ * Servers this sweep could not measure because docker itself failed. They
53
+ * never reached a status, so they are absent from `current` and invisible to
54
+ * the comparison below — but they are the same fact it is looking for: a
55
+ * server that could have produced a number and did not. Counted here so the
56
+ * two symptoms share one threshold instead of being each other's blind spot.
57
+ */
58
+ dockerFaults?: number): Verdict;
51
59
  /**
52
60
  * Put the snapshotted artifacts back, byte for byte. Only servers named in
53
61
  * `names` are touched, and only where a prior file existed — a server whose
@@ -78,12 +78,21 @@ export function snapshot(names, root = process.cwd()) {
78
78
  * `current` maps server name to the status it just measured at. Servers absent
79
79
  * from it were not swept and are ignored.
80
80
  */
81
- export function verdict(prior, current) {
81
+ export function verdict(prior, current,
82
+ /**
83
+ * Servers this sweep could not measure because docker itself failed. They
84
+ * never reached a status, so they are absent from `current` and invisible to
85
+ * the comparison below — but they are the same fact it is looking for: a
86
+ * server that could have produced a number and did not. Counted here so the
87
+ * two symptoms share one threshold instead of being each other's blind spot.
88
+ */
89
+ dockerFaults = 0) {
82
90
  const comparableNames = prior
83
91
  .filter((s) => s.status !== null && isGood(s.status) && current.has(s.name))
84
92
  .map((s) => s.name);
85
93
  const regressed = comparableNames.filter((n) => !isGood(current.get(n)));
86
- const comparable = comparableNames.length;
94
+ const failed = regressed.length + dockerFaults;
95
+ const comparable = comparableNames.length + dockerFaults;
87
96
  if (comparable === 0) {
88
97
  return {
89
98
  fault: false,
@@ -94,15 +103,16 @@ export function verdict(prior, current) {
94
103
  reason: 'no prior measurement to compare against — harness check not performed',
95
104
  };
96
105
  }
97
- const ratio = regressed.length / comparable;
106
+ const ratio = failed / comparable;
98
107
  const pct = (ratio * 100).toFixed(0);
99
- if (regressed.length >= MIN_REGRESSIONS && ratio >= FAULT_RATIO) {
108
+ const how = dockerFaults > 0 ? ` (${regressed.length} regressed, ${dockerFaults} unmeasurable)` : '';
109
+ if (failed >= MIN_REGRESSIONS && ratio >= FAULT_RATIO) {
100
110
  return {
101
111
  fault: true,
102
112
  regressed,
103
113
  comparable,
104
- reason: `${regressed.length} of ${comparable} previously-measured servers (${pct}%) failed in ` +
105
- `this sweep — at or above the ${MIN_REGRESSIONS}-server, ` +
114
+ reason: `${failed} of ${comparable} previously-measured servers (${pct}%) produced no number in ` +
115
+ `this sweep${how} — at or above the ${MIN_REGRESSIONS}-server, ` +
106
116
  `${(FAULT_RATIO * 100).toFixed(0)}% threshold that reads as a broken harness ` +
107
117
  `rather than broken servers`,
108
118
  };
@@ -111,8 +121,8 @@ export function verdict(prior, current) {
111
121
  fault: false,
112
122
  regressed,
113
123
  comparable,
114
- reason: `${regressed.length} of ${comparable} previously-measured servers (${pct}%) failed in ` +
115
- `this sweep — below the harness-fault threshold, publishing normally`,
124
+ reason: `${failed} of ${comparable} previously-measured servers (${pct}%) produced no number in ` +
125
+ `this sweep${how} — below the harness-fault threshold, publishing normally`,
116
126
  };
117
127
  }
118
128
  /**
@@ -21,7 +21,20 @@ function csvCell(s) {
21
21
  export function loadRows(entries, root = process.cwd()) {
22
22
  return entries.map((entry) => {
23
23
  const p = join(root, 'results', entry.name, 'measurement.json');
24
- return { entry, m: existsSync(p) ? JSON.parse(readFileSync(p, 'utf8')) : null };
24
+ if (!existsSync(p))
25
+ return { entry, m: null };
26
+ try {
27
+ return { entry, m: JSON.parse(readFileSync(p, 'utf8')) };
28
+ }
29
+ catch {
30
+ // The same tolerance `appendHistory` states for the same file: a sweep
31
+ // killed mid-write leaves a truncated measurement, and every generator
32
+ // reads through here. Throwing meant one such file broke the leaderboard,
33
+ // the server pages, the dashboard, the tool-shape baseline and the
34
+ // published-stats check at once — weekly, with a SyntaxError that named
35
+ // no file. A server with no readable record reads as one with no record.
36
+ return { entry, m: null };
37
+ }
25
38
  });
26
39
  }
27
40
  /** results/divergence.json if a divergence run has been recorded, else null. */
@@ -25,6 +25,7 @@ import { join, resolve } from 'node:path';
25
25
  import { fileURLToPath } from 'node:url';
26
26
  import { parse } from 'yaml';
27
27
  import { measureServer } from './run.js';
28
+ import { DockerHarnessFault } from './docker.js';
28
29
  import { SESSION_START_METHOD, parseSessionStart, toSessionStartRow, } from '../core/session-start.js';
29
30
  export function loadSessionStart(root = process.cwd()) {
30
31
  const p = join(root, 'results', 'session-start.json');
@@ -63,15 +64,36 @@ if (isMain) {
63
64
  const queue = [...entries];
64
65
  async function worker() {
65
66
  for (let e = queue.shift(); e; e = queue.shift()) {
66
- const m = await measureServer(e.name, e.command, {
67
- timeoutMs: (e.timeoutSeconds ?? defaultTimeout) * 1000,
68
- docker,
69
- dockerImage: e.dockerImage,
70
- dummyEnv: e.env ?? [],
71
- dummyEnvValues: e.envValues,
72
- needsGit: e.needsGit,
73
- persist: false, // the measurements on disk are not this run's to rewrite
74
- });
67
+ let m;
68
+ try {
69
+ m = await measureServer(e.name, e.command, {
70
+ timeoutMs: (e.timeoutSeconds ?? defaultTimeout) * 1000,
71
+ docker,
72
+ dockerImage: e.dockerImage,
73
+ dummyEnv: e.env ?? [],
74
+ dummyEnvValues: e.envValues,
75
+ needsGit: e.needsGit,
76
+ persist: false, // the measurements on disk are not this run's to rewrite
77
+ });
78
+ }
79
+ catch (err) {
80
+ // One thrown error used to reject Promise.all and end the process
81
+ // before anything was written, discarding every capture the run had
82
+ // already completed — and skipping the `finally` that force-removes
83
+ // containers, orphaning the in-flight ones. A machine fault on one
84
+ // server is recorded against that server; the rest of the run stands.
85
+ if (!(err instanceof DockerHarnessFault))
86
+ throw err;
87
+ servers[e.name] = {
88
+ instructions: '',
89
+ instructionsTokens: 0,
90
+ instructionsSha256: '',
91
+ capturedSha256: null,
92
+ error: `docker harness fault: ${err.message.slice(0, 200)}`,
93
+ };
94
+ console.log(` ${e.name}: docker harness fault — recorded, run continues`);
95
+ continue;
96
+ }
75
97
  if (m.status !== 'measured' && m.status !== 'dynamic') {
76
98
  servers[e.name] = {
77
99
  instructions: '',
@@ -18,7 +18,7 @@ import { DockerHarnessFault } from './docker.js';
18
18
  import { writeLeaderboard } from './report.js';
19
19
  import { appendHistory } from './history.js';
20
20
  import { appendToolVectors, writeRegressions } from './regressions.js';
21
- import { FAULT_RATIO, MIN_REGRESSIONS, snapshot, verdict, restore } from './harness-guard.js';
21
+ import { snapshot, verdict, restore } from './harness-guard.js';
22
22
  import { selectShard, shardIndexForDate } from './shard.js';
23
23
  function arg(name) {
24
24
  const i = process.argv.indexOf(`--${name}`);
@@ -101,24 +101,25 @@ async function worker() {
101
101
  }
102
102
  }
103
103
  await Promise.all(Array.from({ length: Math.max(1, concurrency) }, () => worker()));
104
- // A docker fault on most of the slice is the harness-fault story with an
105
- // earlier symptom — the guard below can't see it (nothing regressed on disk;
106
- // the throws happened before anything was written), so it is judged here by
107
- // the guard's own thresholds. Below them, the sweep publishes what it did
108
- // measure and the faulted servers wait for the next cycle.
109
- if (dockerFaults.size >= MIN_REGRESSIONS && dockerFaults.size / entries.length >= FAULT_RATIO) {
110
- console.error(`\nHARNESS FAULT — docker could not run for ${dockerFaults.size} of ${entries.length} servers; ` +
111
- `refusing to publish this sweep.\n` +
112
- [...dockerFaults].map(([n, msg]) => ` ${n}: ${msg}`).join('\n') +
113
- `\n Nothing was overwritten — every previous record stands. ` +
114
- `Check the Docker daemon and registry path, then re-run.`);
115
- process.exit(1);
116
- }
117
104
  // Before publishing anything: is this sweep a statement about the servers, or
118
105
  // about the machine that measured them?
119
- const v = verdict(prior, statuses);
106
+ //
107
+ // The two symptoms are counted together, against one denominator. Judged apart
108
+ // they were each other's blind spot: a flaky daemon that throws for 6 of 14
109
+ // servers (6 ≥ 5, but 43% of the slice) and times out 4 more (4 < 5) trips
110
+ // neither threshold, and the sweep publishes with 10 of 14 servers producing no
111
+ // number and four good records overwritten with failures. A server that could
112
+ // have produced a number and didn't is one fact, however it failed.
113
+ const v = verdict(prior, statuses, dockerFaults.size);
120
114
  console.log(`harness check: ${v.reason}`);
115
+ if (dockerFaults.size > 0 && !v.fault) {
116
+ console.warn(` (${dockerFaults.size} docker fault(s) counted toward that check; those servers were not measured)`);
117
+ }
121
118
  if (v.fault) {
119
+ if (dockerFaults.size) {
120
+ console.error(`\ndocker could not run for ${dockerFaults.size} server(s):\n` +
121
+ [...dockerFaults].map(([n, msg]) => ` ${n}: ${msg}`).join('\n'));
122
+ }
122
123
  const restored = restore(prior, v.regressed);
123
124
  console.error(`\nHARNESS FAULT — refusing to publish this sweep.\n` +
124
125
  ` regressed: ${v.regressed.join(', ')}\n` +
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "mcp-context-cost",
3
- "version": "0.11.2",
3
+ "version": "0.11.3",
4
4
  "description": "Measure what your MCP servers cost in context tokens — audit your own config, or badge the server you publish",
5
5
  "type": "module",
6
6
  "license": "MIT",