mcp-context-cost 0.11.2 → 0.11.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/audit/audit.js +7 -0
- package/dist/audit/config.js +9 -0
- package/dist/audit/diff.js +9 -1
- package/dist/cli.js +38 -8
- package/dist/core/adoption.d.ts +23 -0
- package/dist/core/adoption.js +26 -2
- package/dist/core/capture-index.js +9 -5
- package/dist/core/cross-check.js +3 -0
- package/dist/core/divergence.js +8 -0
- package/dist/core/regression.d.ts +14 -1
- package/dist/core/regression.js +16 -0
- package/dist/core/tool-shape.js +10 -2
- package/dist/sweep/dashboard.js +4 -4
- package/dist/sweep/harness-guard.d.ts +9 -1
- package/dist/sweep/harness-guard.js +18 -8
- package/dist/sweep/report.js +14 -1
- package/dist/sweep/session-start.js +31 -9
- package/dist/sweep/sweep-all.js +16 -15
- package/package.json +1 -1
package/dist/audit/audit.js
CHANGED
|
@@ -179,6 +179,13 @@ export function buildReport(configs, measured, opts = {}) {
|
|
|
179
179
|
const contextWindow = opts.contextWindow ?? DEFAULT_CONTEXT_WINDOW;
|
|
180
180
|
const problems = [];
|
|
181
181
|
const emptyConfigs = [];
|
|
182
|
+
// canonicalSha256 → the divergence row computed from it, so a server can be
|
|
183
|
+
// identified by its bytes whatever the local config calls it.
|
|
184
|
+
const divByHash = new Map();
|
|
185
|
+
for (const row of Object.values(opts.divergence?.servers ?? {})) {
|
|
186
|
+
if (row?.capturedSha256)
|
|
187
|
+
divByHash.set(row.capturedSha256, row);
|
|
188
|
+
}
|
|
182
189
|
const built = [];
|
|
183
190
|
// Across every config at once: a twin in one client's file is measured for
|
|
184
191
|
// the other client's entry just the same.
|
package/dist/audit/config.js
CHANGED
|
@@ -182,9 +182,18 @@ export function configCandidates(env) {
|
|
|
182
182
|
/** Read + parse the candidates that exist. Unreadable files are reported, not thrown. */
|
|
183
183
|
export function loadConfigs(candidates, cwd) {
|
|
184
184
|
const out = [];
|
|
185
|
+
// Running from your home directory nominates `~/.cursor/mcp.json` twice —
|
|
186
|
+
// once as the home candidate, once as the cwd one. Loaded twice it is
|
|
187
|
+
// reported twice, doubles that client's deferral scope, and under
|
|
188
|
+
// `--baseline` the second copy pairs with nothing and fails the gate. One
|
|
189
|
+
// path is one config however many ways it was nominated.
|
|
190
|
+
const seen = new Set();
|
|
185
191
|
for (const c of candidates) {
|
|
186
192
|
if (!existsSync(c.path))
|
|
187
193
|
continue;
|
|
194
|
+
if (seen.has(c.path))
|
|
195
|
+
continue;
|
|
196
|
+
seen.add(c.path);
|
|
188
197
|
try {
|
|
189
198
|
const doc = parseJsonc(readFileSync(c.path, 'utf8'));
|
|
190
199
|
const { servers, disabled } = extractDeclaration(doc, { client: c.client, source: c.path, cwd });
|
package/dist/audit/diff.js
CHANGED
|
@@ -181,7 +181,15 @@ export function pairConfigs(before, after) {
|
|
|
181
181
|
pairs.push({ before: null, after: cur, matchedBy: 'unmatched' });
|
|
182
182
|
}
|
|
183
183
|
}
|
|
184
|
-
|
|
184
|
+
// One on each side reads as the same config seen from two machines, where the
|
|
185
|
+
// path legitimately differs (a laptop's baseline against a CI checkout). It
|
|
186
|
+
// does not survive the clients differing: a Claude Desktop baseline against a
|
|
187
|
+
// Claude Code run compares two unrelated stacks, and the gate then rests on
|
|
188
|
+
// that difference. The paths may differ; what they are configs *for* may not.
|
|
189
|
+
if (before.length === 1 &&
|
|
190
|
+
after.length === 1 &&
|
|
191
|
+
pairs[0].before === null &&
|
|
192
|
+
before[0].client === after[0].client) {
|
|
185
193
|
pairs[0] = { before: before[0], after: after[0], matchedBy: 'sole-config' };
|
|
186
194
|
unusedBefore.delete(before[0].source);
|
|
187
195
|
}
|
package/dist/cli.js
CHANGED
|
@@ -45,6 +45,19 @@ export function slugFromUrl(url) {
|
|
|
45
45
|
const host = new URL(url).hostname.replace(/^(www|mcp)\./, '');
|
|
46
46
|
return host.replace(/[^a-z0-9]+/gi, '-').replace(/^-+|-+$/g, '').toLowerCase() || 'remote';
|
|
47
47
|
}
|
|
48
|
+
/**
|
|
49
|
+
* Report a `verify` failure and exit 1, in whichever shape the caller asked
|
|
50
|
+
* for. `--json` is documented as putting `{ ok, rederivedTokens, rederivedSha,
|
|
51
|
+
* problems }` on stdout; a script reading that gets nothing from a thrown
|
|
52
|
+
* exception, so every failure path goes through here.
|
|
53
|
+
*/
|
|
54
|
+
function failVerify(json, problem) {
|
|
55
|
+
if (json)
|
|
56
|
+
console.log(JSON.stringify({ ok: false, rederivedTokens: null, rederivedSha: null, problems: [problem] }));
|
|
57
|
+
else
|
|
58
|
+
console.error(problem);
|
|
59
|
+
process.exit(1);
|
|
60
|
+
}
|
|
48
61
|
/** Installed version, for error messages that need to say which one you are running. */
|
|
49
62
|
export function cliVersion() {
|
|
50
63
|
try {
|
|
@@ -322,6 +335,12 @@ if (cmd === 'audit') {
|
|
|
322
335
|
`${where}\n` +
|
|
323
336
|
`${empty.length === 1 ? 'It was' : 'They were'} read and parsed; there is simply nothing declared to measure.\n` +
|
|
324
337
|
`Declare a server in one of them, or point at a different config: mcp-context-cost audit --config <path/to/mcp.json>`);
|
|
338
|
+
else if (all('config').length)
|
|
339
|
+
// A path the user named is not a discovery miss. Saying "looked in the
|
|
340
|
+
// standard locations" describes something the command did not do, and
|
|
341
|
+
// then advises doing the thing they just did.
|
|
342
|
+
console.error(`no MCP config found at the path(s) given: ${all('config').join(', ')}. ` +
|
|
343
|
+
`Nothing else was searched, because --config was set.`);
|
|
325
344
|
else
|
|
326
345
|
console.error(`no MCP config found. Looked in the standard Claude Desktop / Claude Code / Cursor / VS Code / Windsurf locations.${where}\n` +
|
|
327
346
|
`Point at one explicitly: mcp-context-cost audit --config <path/to/mcp.json>`);
|
|
@@ -355,18 +374,29 @@ else if (cmd === 'verify') {
|
|
|
355
374
|
raw = await res.text();
|
|
356
375
|
}
|
|
357
376
|
catch (e) {
|
|
358
|
-
|
|
359
|
-
if (json)
|
|
360
|
-
console.log(JSON.stringify({ ok: false, rederivedTokens: null, rederivedSha: null, problems: [problem] }));
|
|
361
|
-
else
|
|
362
|
-
console.error(problem);
|
|
363
|
-
process.exit(1);
|
|
377
|
+
failVerify(json, `failed to fetch ${remoteUrl}: ${e.message}`);
|
|
364
378
|
}
|
|
365
379
|
}
|
|
366
380
|
else {
|
|
367
|
-
|
|
381
|
+
try {
|
|
382
|
+
raw = readFileSync(path, 'utf8');
|
|
383
|
+
}
|
|
384
|
+
catch (e) {
|
|
385
|
+
// The remote branch above reports a failed fetch in the documented shape;
|
|
386
|
+
// this one used to throw, so `--json` produced a stack trace on stderr and
|
|
387
|
+
// nothing at all on stdout — the contract a script parses.
|
|
388
|
+
failVerify(json, `cannot read ${path}: ${e.message}`);
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
let m;
|
|
392
|
+
try {
|
|
393
|
+
m = JSON.parse(raw);
|
|
394
|
+
}
|
|
395
|
+
catch (e) {
|
|
396
|
+
// Reachable remotely: a proxy, a captive portal or an HTML error page
|
|
397
|
+
// served with status 200 passes the `res.ok` check above and arrives here.
|
|
398
|
+
failVerify(json, `${remoteUrl ?? path} is not valid JSON: ${e.message}`);
|
|
368
399
|
}
|
|
369
|
-
const m = JSON.parse(raw);
|
|
370
400
|
const r = verifyMeasurement(m);
|
|
371
401
|
if (json) {
|
|
372
402
|
console.log(JSON.stringify({ serverName: m.serverName, ...r, badge: r.ok ? toBadge(m) : undefined }));
|
package/dist/core/adoption.d.ts
CHANGED
|
@@ -143,6 +143,14 @@ export declare function decodeLoose(text: string): string;
|
|
|
143
143
|
export interface EndpointBadge {
|
|
144
144
|
url: string;
|
|
145
145
|
linkTarget: string | null;
|
|
146
|
+
/**
|
|
147
|
+
* The badge's own label — markdown alt text or an `<img alt>`. What the badge
|
|
148
|
+
* calls itself is the only thing in a README that distinguishes *this*
|
|
149
|
+
* project's badge from any other shields endpoint badge, since the JSON can
|
|
150
|
+
* be hosted anywhere (the staged action publishes to a gist, with no path
|
|
151
|
+
* shape to recognise).
|
|
152
|
+
*/
|
|
153
|
+
alt: string | null;
|
|
146
154
|
}
|
|
147
155
|
/**
|
|
148
156
|
* Every shields endpoint badge in a file, decoded, each paired with its own
|
|
@@ -172,6 +180,21 @@ export declare function linksBackToProject(target: string, src?: BadgeSource): b
|
|
|
172
180
|
* measurement here. See the header for why both count and why the link is
|
|
173
181
|
* paired with the image rather than looked for anywhere in the file.
|
|
174
182
|
*/
|
|
183
|
+
/**
|
|
184
|
+
* Whether a badge calls itself this project's badge.
|
|
185
|
+
*
|
|
186
|
+
* The self-hosted branch below used to accept the link alone and look at
|
|
187
|
+
* nothing else, so *any* shields endpoint badge — a coverage badge, say —
|
|
188
|
+
* wrapped in a link to this repository counted as displaying ours. That
|
|
189
|
+
* inflates the one number this project keeps about itself, which is the last
|
|
190
|
+
* place it can afford a generous reading.
|
|
191
|
+
*
|
|
192
|
+
* The URL cannot decide it: self-hosted JSON lives wherever its author put it,
|
|
193
|
+
* and the staged action publishes to a gist. What every published snippet does
|
|
194
|
+
* carry is the label — `context cost` — so that is what is read, tolerantly
|
|
195
|
+
* enough to accept `Context-Cost` and strictly enough to reject `coverage`.
|
|
196
|
+
*/
|
|
197
|
+
export declare function namesThisBadge(alt: string | null): boolean;
|
|
175
198
|
export declare function displaysBadge(text: string, src?: BadgeSource): boolean;
|
|
176
199
|
/**
|
|
177
200
|
* Every shields endpoint `url` in a file whose JSON is served from this
|
package/dist/core/adoption.js
CHANGED
|
@@ -137,7 +137,11 @@ export function endpointBadges(text) {
|
|
|
137
137
|
if (last && !before.slice(last.index ?? 0).includes('</a>'))
|
|
138
138
|
linkTarget = last[1];
|
|
139
139
|
}
|
|
140
|
-
|
|
140
|
+
// Markdown puts the alt before the image; HTML puts it in the same tag.
|
|
141
|
+
const beforeAlt = decoded.slice(Math.max(0, start - LINK_WINDOW), start);
|
|
142
|
+
const mdAlt = beforeAlt.match(/!\[([^\]]*)\]\(\s*[^\s)]*$/);
|
|
143
|
+
const tagAlt = after.match(/^[^<>]*?\balt\s*=\s*["']([^"']*)["']/i);
|
|
144
|
+
out.push({ url: m[1], linkTarget, alt: mdAlt ? mdAlt[1] : tagAlt ? tagAlt[1] : null });
|
|
141
145
|
}
|
|
142
146
|
return out;
|
|
143
147
|
}
|
|
@@ -169,8 +173,28 @@ export function linksBackToProject(target, src = BADGE_SOURCE) {
|
|
|
169
173
|
* measurement here. See the header for why both count and why the link is
|
|
170
174
|
* paired with the image rather than looked for anywhere in the file.
|
|
171
175
|
*/
|
|
176
|
+
/**
|
|
177
|
+
* Whether a badge calls itself this project's badge.
|
|
178
|
+
*
|
|
179
|
+
* The self-hosted branch below used to accept the link alone and look at
|
|
180
|
+
* nothing else, so *any* shields endpoint badge — a coverage badge, say —
|
|
181
|
+
* wrapped in a link to this repository counted as displaying ours. That
|
|
182
|
+
* inflates the one number this project keeps about itself, which is the last
|
|
183
|
+
* place it can afford a generous reading.
|
|
184
|
+
*
|
|
185
|
+
* The URL cannot decide it: self-hosted JSON lives wherever its author put it,
|
|
186
|
+
* and the staged action publishes to a gist. What every published snippet does
|
|
187
|
+
* carry is the label — `context cost` — so that is what is read, tolerantly
|
|
188
|
+
* enough to accept `Context-Cost` and strictly enough to reject `coverage`.
|
|
189
|
+
*/
|
|
190
|
+
export function namesThisBadge(alt) {
|
|
191
|
+
if (!alt)
|
|
192
|
+
return false;
|
|
193
|
+
return alt.toLowerCase().replace(/[^a-z]/g, '').includes('contextcost');
|
|
194
|
+
}
|
|
172
195
|
export function displaysBadge(text, src = BADGE_SOURCE) {
|
|
173
|
-
return endpointBadges(text).some((b) => hostedHere(b.url, src) ||
|
|
196
|
+
return endpointBadges(text).some((b) => hostedHere(b.url, src) ||
|
|
197
|
+
(namesThisBadge(b.alt) && b.linkTarget !== null && linksBackToProject(b.linkTarget, src)));
|
|
174
198
|
}
|
|
175
199
|
/**
|
|
176
200
|
* Every shields endpoint `url` in a file whose JSON is served from this
|
|
@@ -72,14 +72,18 @@ export function identify(canonicalSha256, index) {
|
|
|
72
72
|
if (!mine)
|
|
73
73
|
return { kind: 'unknown' };
|
|
74
74
|
const currentSha = index.current[mine.server];
|
|
75
|
-
if (
|
|
75
|
+
if (currentSha === canonicalSha256) {
|
|
76
76
|
return { kind: 'current', server: mine.server, date: mine.date, tokens: mine.totalTokens };
|
|
77
77
|
}
|
|
78
|
-
|
|
79
|
-
//
|
|
80
|
-
//
|
|
78
|
+
// The bytes are identified, but what is current for this server is not: the
|
|
79
|
+
// pointer is missing, or points at a capture the index dropped as ambiguous.
|
|
80
|
+
// `current` would be an affirmative claim that nothing has moved, which the
|
|
81
|
+
// audit prints as "no server here is running a published capture that has
|
|
82
|
+
// since moved" — told to someone who may be far behind. Unknown currency
|
|
83
|
+
// reads as unknown, which is the discipline the rest of this module keeps.
|
|
84
|
+
const current = currentSha ? index.captures[currentSha] : undefined;
|
|
81
85
|
if (!current)
|
|
82
|
-
return { kind: '
|
|
86
|
+
return { kind: 'unknown' };
|
|
83
87
|
return {
|
|
84
88
|
kind: 'behind',
|
|
85
89
|
server: mine.server,
|
package/dist/core/cross-check.js
CHANGED
|
@@ -160,6 +160,9 @@ export function isComparable(row, canonicalSha256) {
|
|
|
160
160
|
canonicalSha256 !== null &&
|
|
161
161
|
row.capturedSha256 === canonicalSha256 &&
|
|
162
162
|
row.ourTokens > 0 &&
|
|
163
|
+
// A report that parsed but carries `total: 0` is not a measurement of
|
|
164
|
+
// anything; published, it renders as a −100% divergence.
|
|
165
|
+
row.cliTokens > 0 &&
|
|
163
166
|
row.ourMappedTokens > 0);
|
|
164
167
|
}
|
|
165
168
|
/**
|
package/dist/core/divergence.js
CHANGED
|
@@ -55,6 +55,14 @@ export function mappedTokens(raw) {
|
|
|
55
55
|
export function fieldSelectionShare(row) {
|
|
56
56
|
if (row.o200kFull <= 0)
|
|
57
57
|
return null;
|
|
58
|
+
// The projection can add bytes rather than remove them — `inputSchema`
|
|
59
|
+
// becomes the longer `input_schema`, and a tool with no description gains
|
|
60
|
+
// `description: ""` — so a server with almost no metadata to drop can map
|
|
61
|
+
// *larger* than it measured. There is no share of the payload removed in that
|
|
62
|
+
// case, and publishing a negative one reads as "−11.1% of the capture is
|
|
63
|
+
// MCP-only metadata", which is not a thing.
|
|
64
|
+
if (row.o200kMapped > row.o200kFull)
|
|
65
|
+
return null;
|
|
58
66
|
return (row.o200kFull - row.o200kMapped) / row.o200kFull;
|
|
59
67
|
}
|
|
60
68
|
/**
|
|
@@ -124,7 +124,20 @@ export interface ToolAttribution {
|
|
|
124
124
|
*/
|
|
125
125
|
unexplainedTokens: number;
|
|
126
126
|
}
|
|
127
|
-
|
|
127
|
+
/**
|
|
128
|
+
* Per-tool attribution, or null when the names cannot carry it.
|
|
129
|
+
*
|
|
130
|
+
* The breakdown matches tools by name, so a name that appears twice on either
|
|
131
|
+
* side makes the maps below lose one of them silently — and the lost tokens
|
|
132
|
+
* resurface as `unexplainedTokens`, which the report explains to the reader as
|
|
133
|
+
* canonical-array framing bytes. That is a confident false explanation. Two
|
|
134
|
+
* ways it happens: a server that ships duplicate or namespaced-collapsed tool
|
|
135
|
+
* names, and `measureTools` recording every nameless tool as the single key
|
|
136
|
+
* `(unnamed)` — an invented name, where `toolNames` and `toAnthropicTools`
|
|
137
|
+
* deliberately drop nameless tools rather than invent one. Where the names
|
|
138
|
+
* cannot identify the tools, there is no attribution to give.
|
|
139
|
+
*/
|
|
140
|
+
export declare function attribute(from: ToolVectorEntry, to: ToolVectorEntry, deltaTokens: number): ToolAttribution | null;
|
|
128
141
|
export interface CostChange {
|
|
129
142
|
server: string;
|
|
130
143
|
fromDate: string;
|
package/dist/core/regression.js
CHANGED
|
@@ -87,7 +87,23 @@ export function mechanismOf(deltaTokens, deltaTools) {
|
|
|
87
87
|
// and rewritten, and the totals cannot separate the two.
|
|
88
88
|
return 'mixed';
|
|
89
89
|
}
|
|
90
|
+
/**
|
|
91
|
+
* Per-tool attribution, or null when the names cannot carry it.
|
|
92
|
+
*
|
|
93
|
+
* The breakdown matches tools by name, so a name that appears twice on either
|
|
94
|
+
* side makes the maps below lose one of them silently — and the lost tokens
|
|
95
|
+
* resurface as `unexplainedTokens`, which the report explains to the reader as
|
|
96
|
+
* canonical-array framing bytes. That is a confident false explanation. Two
|
|
97
|
+
* ways it happens: a server that ships duplicate or namespaced-collapsed tool
|
|
98
|
+
* names, and `measureTools` recording every nameless tool as the single key
|
|
99
|
+
* `(unnamed)` — an invented name, where `toolNames` and `toAnthropicTools`
|
|
100
|
+
* deliberately drop nameless tools rather than invent one. Where the names
|
|
101
|
+
* cannot identify the tools, there is no attribution to give.
|
|
102
|
+
*/
|
|
90
103
|
export function attribute(from, to, deltaTokens) {
|
|
104
|
+
const unique = (ts) => new Set(ts.map((t) => t.name)).size === ts.length;
|
|
105
|
+
if (!unique(from.tools) || !unique(to.tools))
|
|
106
|
+
return null;
|
|
91
107
|
const before = new Map(from.tools.map((t) => [t.name, t.tokens]));
|
|
92
108
|
const after = new Map(to.tools.map((t) => [t.name, t.tokens]));
|
|
93
109
|
const added = [];
|
package/dist/core/tool-shape.js
CHANGED
|
@@ -23,10 +23,18 @@ export function quantileTable(values) {
|
|
|
23
23
|
* checkable by anyone holding the same JSON.
|
|
24
24
|
*/
|
|
25
25
|
export function percentileOf(quantiles, value) {
|
|
26
|
+
// The LOWEST percentile whose quantile the value reaches, not the highest.
|
|
27
|
+
// Taking the highest reports a value tied with half the measured set as p100
|
|
28
|
+
// — "heavier than 100% of measured tools" about something exactly average for
|
|
29
|
+
// its tie — because every percentile across the tie carries the same
|
|
30
|
+
// quantile. The lowest names where the tie begins, which is what "heavier
|
|
31
|
+
// than P% of tools" means.
|
|
26
32
|
let p = 0;
|
|
27
33
|
for (let i = 0; i <= 100; i++) {
|
|
28
|
-
if (quantiles[i] <= value)
|
|
29
|
-
|
|
34
|
+
if (quantiles[i] <= value) {
|
|
35
|
+
if (quantiles[i] < value || i === 0 || quantiles[i - 1] < quantiles[i])
|
|
36
|
+
p = i;
|
|
37
|
+
}
|
|
30
38
|
else
|
|
31
39
|
break;
|
|
32
40
|
}
|
package/dist/sweep/dashboard.js
CHANGED
|
@@ -7,6 +7,7 @@ import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'node:fs';
|
|
|
7
7
|
import { dirname, join } from 'node:path';
|
|
8
8
|
import { parse } from 'yaml';
|
|
9
9
|
import { isCurrent } from '../core/divergence.js';
|
|
10
|
+
import { loadRows } from './report.js';
|
|
10
11
|
import { bandColor, BAND_META } from '../core/bands.js';
|
|
11
12
|
import { parseHistory, plottableSeries } from './history.js';
|
|
12
13
|
/** Longest series a sparkline plots — a stat-tile trend, not a full chart. */
|
|
@@ -53,10 +54,9 @@ export function generateDashboard(root = process.cwd()) {
|
|
|
53
54
|
// Only the run of sweeps taken under the same isolation is plotted: a step
|
|
54
55
|
// across an isolation change is the harness moving, not the server.
|
|
55
56
|
const seriesFor = (name) => plottableSeries(history.filter((h) => h.server === name));
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
});
|
|
57
|
+
// Shared with the leaderboard rather than re-read here: one definition of how
|
|
58
|
+
// a measurement is loaded, including its tolerance for a half-written file.
|
|
59
|
+
const rows = loadRows(doc.servers, root);
|
|
60
60
|
const measured = rows
|
|
61
61
|
.filter((r) => r.m && (r.m.status === 'measured' || r.m.status === 'dynamic') && r.m.totalTokens !== null)
|
|
62
62
|
.sort((a, b) => (b.m.totalTokens ?? 0) - (a.m.totalTokens ?? 0));
|
|
@@ -47,7 +47,15 @@ export interface Verdict {
|
|
|
47
47
|
* `current` maps server name to the status it just measured at. Servers absent
|
|
48
48
|
* from it were not swept and are ignored.
|
|
49
49
|
*/
|
|
50
|
-
export declare function verdict(prior: Snapshot[], current: Map<string, MeasurementStatus
|
|
50
|
+
export declare function verdict(prior: Snapshot[], current: Map<string, MeasurementStatus>,
|
|
51
|
+
/**
|
|
52
|
+
* Servers this sweep could not measure because docker itself failed. They
|
|
53
|
+
* never reached a status, so they are absent from `current` and invisible to
|
|
54
|
+
* the comparison below — but they are the same fact it is looking for: a
|
|
55
|
+
* server that could have produced a number and did not. Counted here so the
|
|
56
|
+
* two symptoms share one threshold instead of being each other's blind spot.
|
|
57
|
+
*/
|
|
58
|
+
dockerFaults?: number): Verdict;
|
|
51
59
|
/**
|
|
52
60
|
* Put the snapshotted artifacts back, byte for byte. Only servers named in
|
|
53
61
|
* `names` are touched, and only where a prior file existed — a server whose
|
|
@@ -78,12 +78,21 @@ export function snapshot(names, root = process.cwd()) {
|
|
|
78
78
|
* `current` maps server name to the status it just measured at. Servers absent
|
|
79
79
|
* from it were not swept and are ignored.
|
|
80
80
|
*/
|
|
81
|
-
export function verdict(prior, current
|
|
81
|
+
export function verdict(prior, current,
|
|
82
|
+
/**
|
|
83
|
+
* Servers this sweep could not measure because docker itself failed. They
|
|
84
|
+
* never reached a status, so they are absent from `current` and invisible to
|
|
85
|
+
* the comparison below — but they are the same fact it is looking for: a
|
|
86
|
+
* server that could have produced a number and did not. Counted here so the
|
|
87
|
+
* two symptoms share one threshold instead of being each other's blind spot.
|
|
88
|
+
*/
|
|
89
|
+
dockerFaults = 0) {
|
|
82
90
|
const comparableNames = prior
|
|
83
91
|
.filter((s) => s.status !== null && isGood(s.status) && current.has(s.name))
|
|
84
92
|
.map((s) => s.name);
|
|
85
93
|
const regressed = comparableNames.filter((n) => !isGood(current.get(n)));
|
|
86
|
-
const
|
|
94
|
+
const failed = regressed.length + dockerFaults;
|
|
95
|
+
const comparable = comparableNames.length + dockerFaults;
|
|
87
96
|
if (comparable === 0) {
|
|
88
97
|
return {
|
|
89
98
|
fault: false,
|
|
@@ -94,15 +103,16 @@ export function verdict(prior, current) {
|
|
|
94
103
|
reason: 'no prior measurement to compare against — harness check not performed',
|
|
95
104
|
};
|
|
96
105
|
}
|
|
97
|
-
const ratio =
|
|
106
|
+
const ratio = failed / comparable;
|
|
98
107
|
const pct = (ratio * 100).toFixed(0);
|
|
99
|
-
|
|
108
|
+
const how = dockerFaults > 0 ? ` (${regressed.length} regressed, ${dockerFaults} unmeasurable)` : '';
|
|
109
|
+
if (failed >= MIN_REGRESSIONS && ratio >= FAULT_RATIO) {
|
|
100
110
|
return {
|
|
101
111
|
fault: true,
|
|
102
112
|
regressed,
|
|
103
113
|
comparable,
|
|
104
|
-
reason: `${
|
|
105
|
-
`this sweep — at or above the ${MIN_REGRESSIONS}-server, ` +
|
|
114
|
+
reason: `${failed} of ${comparable} previously-measured servers (${pct}%) produced no number in ` +
|
|
115
|
+
`this sweep${how} — at or above the ${MIN_REGRESSIONS}-server, ` +
|
|
106
116
|
`${(FAULT_RATIO * 100).toFixed(0)}% threshold that reads as a broken harness ` +
|
|
107
117
|
`rather than broken servers`,
|
|
108
118
|
};
|
|
@@ -111,8 +121,8 @@ export function verdict(prior, current) {
|
|
|
111
121
|
fault: false,
|
|
112
122
|
regressed,
|
|
113
123
|
comparable,
|
|
114
|
-
reason: `${
|
|
115
|
-
`this sweep — below the harness-fault threshold, publishing normally`,
|
|
124
|
+
reason: `${failed} of ${comparable} previously-measured servers (${pct}%) produced no number in ` +
|
|
125
|
+
`this sweep${how} — below the harness-fault threshold, publishing normally`,
|
|
116
126
|
};
|
|
117
127
|
}
|
|
118
128
|
/**
|
package/dist/sweep/report.js
CHANGED
|
@@ -21,7 +21,20 @@ function csvCell(s) {
|
|
|
21
21
|
export function loadRows(entries, root = process.cwd()) {
|
|
22
22
|
return entries.map((entry) => {
|
|
23
23
|
const p = join(root, 'results', entry.name, 'measurement.json');
|
|
24
|
-
|
|
24
|
+
if (!existsSync(p))
|
|
25
|
+
return { entry, m: null };
|
|
26
|
+
try {
|
|
27
|
+
return { entry, m: JSON.parse(readFileSync(p, 'utf8')) };
|
|
28
|
+
}
|
|
29
|
+
catch {
|
|
30
|
+
// The same tolerance `appendHistory` states for the same file: a sweep
|
|
31
|
+
// killed mid-write leaves a truncated measurement, and every generator
|
|
32
|
+
// reads through here. Throwing meant one such file broke the leaderboard,
|
|
33
|
+
// the server pages, the dashboard, the tool-shape baseline and the
|
|
34
|
+
// published-stats check at once — weekly, with a SyntaxError that named
|
|
35
|
+
// no file. A server with no readable record reads as one with no record.
|
|
36
|
+
return { entry, m: null };
|
|
37
|
+
}
|
|
25
38
|
});
|
|
26
39
|
}
|
|
27
40
|
/** results/divergence.json if a divergence run has been recorded, else null. */
|
|
@@ -25,6 +25,7 @@ import { join, resolve } from 'node:path';
|
|
|
25
25
|
import { fileURLToPath } from 'node:url';
|
|
26
26
|
import { parse } from 'yaml';
|
|
27
27
|
import { measureServer } from './run.js';
|
|
28
|
+
import { DockerHarnessFault } from './docker.js';
|
|
28
29
|
import { SESSION_START_METHOD, parseSessionStart, toSessionStartRow, } from '../core/session-start.js';
|
|
29
30
|
export function loadSessionStart(root = process.cwd()) {
|
|
30
31
|
const p = join(root, 'results', 'session-start.json');
|
|
@@ -63,15 +64,36 @@ if (isMain) {
|
|
|
63
64
|
const queue = [...entries];
|
|
64
65
|
async function worker() {
|
|
65
66
|
for (let e = queue.shift(); e; e = queue.shift()) {
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
67
|
+
let m;
|
|
68
|
+
try {
|
|
69
|
+
m = await measureServer(e.name, e.command, {
|
|
70
|
+
timeoutMs: (e.timeoutSeconds ?? defaultTimeout) * 1000,
|
|
71
|
+
docker,
|
|
72
|
+
dockerImage: e.dockerImage,
|
|
73
|
+
dummyEnv: e.env ?? [],
|
|
74
|
+
dummyEnvValues: e.envValues,
|
|
75
|
+
needsGit: e.needsGit,
|
|
76
|
+
persist: false, // the measurements on disk are not this run's to rewrite
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
catch (err) {
|
|
80
|
+
// One thrown error used to reject Promise.all and end the process
|
|
81
|
+
// before anything was written, discarding every capture the run had
|
|
82
|
+
// already completed — and skipping the `finally` that force-removes
|
|
83
|
+
// containers, orphaning the in-flight ones. A machine fault on one
|
|
84
|
+
// server is recorded against that server; the rest of the run stands.
|
|
85
|
+
if (!(err instanceof DockerHarnessFault))
|
|
86
|
+
throw err;
|
|
87
|
+
servers[e.name] = {
|
|
88
|
+
instructions: '',
|
|
89
|
+
instructionsTokens: 0,
|
|
90
|
+
instructionsSha256: '',
|
|
91
|
+
capturedSha256: null,
|
|
92
|
+
error: `docker harness fault: ${err.message.slice(0, 200)}`,
|
|
93
|
+
};
|
|
94
|
+
console.log(` ${e.name}: docker harness fault — recorded, run continues`);
|
|
95
|
+
continue;
|
|
96
|
+
}
|
|
75
97
|
if (m.status !== 'measured' && m.status !== 'dynamic') {
|
|
76
98
|
servers[e.name] = {
|
|
77
99
|
instructions: '',
|
package/dist/sweep/sweep-all.js
CHANGED
|
@@ -18,7 +18,7 @@ import { DockerHarnessFault } from './docker.js';
|
|
|
18
18
|
import { writeLeaderboard } from './report.js';
|
|
19
19
|
import { appendHistory } from './history.js';
|
|
20
20
|
import { appendToolVectors, writeRegressions } from './regressions.js';
|
|
21
|
-
import {
|
|
21
|
+
import { snapshot, verdict, restore } from './harness-guard.js';
|
|
22
22
|
import { selectShard, shardIndexForDate } from './shard.js';
|
|
23
23
|
function arg(name) {
|
|
24
24
|
const i = process.argv.indexOf(`--${name}`);
|
|
@@ -101,24 +101,25 @@ async function worker() {
|
|
|
101
101
|
}
|
|
102
102
|
}
|
|
103
103
|
await Promise.all(Array.from({ length: Math.max(1, concurrency) }, () => worker()));
|
|
104
|
-
// A docker fault on most of the slice is the harness-fault story with an
|
|
105
|
-
// earlier symptom — the guard below can't see it (nothing regressed on disk;
|
|
106
|
-
// the throws happened before anything was written), so it is judged here by
|
|
107
|
-
// the guard's own thresholds. Below them, the sweep publishes what it did
|
|
108
|
-
// measure and the faulted servers wait for the next cycle.
|
|
109
|
-
if (dockerFaults.size >= MIN_REGRESSIONS && dockerFaults.size / entries.length >= FAULT_RATIO) {
|
|
110
|
-
console.error(`\nHARNESS FAULT — docker could not run for ${dockerFaults.size} of ${entries.length} servers; ` +
|
|
111
|
-
`refusing to publish this sweep.\n` +
|
|
112
|
-
[...dockerFaults].map(([n, msg]) => ` ${n}: ${msg}`).join('\n') +
|
|
113
|
-
`\n Nothing was overwritten — every previous record stands. ` +
|
|
114
|
-
`Check the Docker daemon and registry path, then re-run.`);
|
|
115
|
-
process.exit(1);
|
|
116
|
-
}
|
|
117
104
|
// Before publishing anything: is this sweep a statement about the servers, or
|
|
118
105
|
// about the machine that measured them?
|
|
119
|
-
|
|
106
|
+
//
|
|
107
|
+
// The two symptoms are counted together, against one denominator. Judged apart
|
|
108
|
+
// they were each other's blind spot: a flaky daemon that throws for 6 of 14
|
|
109
|
+
// servers (6 ≥ 5, but 43% of the slice) and times out 4 more (4 < 5) trips
|
|
110
|
+
// neither threshold, and the sweep publishes with 10 of 14 servers producing no
|
|
111
|
+
// number and four good records overwritten with failures. A server that could
|
|
112
|
+
// have produced a number and didn't is one fact, however it failed.
|
|
113
|
+
const v = verdict(prior, statuses, dockerFaults.size);
|
|
120
114
|
console.log(`harness check: ${v.reason}`);
|
|
115
|
+
if (dockerFaults.size > 0 && !v.fault) {
|
|
116
|
+
console.warn(` (${dockerFaults.size} docker fault(s) counted toward that check; those servers were not measured)`);
|
|
117
|
+
}
|
|
121
118
|
if (v.fault) {
|
|
119
|
+
if (dockerFaults.size) {
|
|
120
|
+
console.error(`\ndocker could not run for ${dockerFaults.size} server(s):\n` +
|
|
121
|
+
[...dockerFaults].map(([n, msg]) => ` ${n}: ${msg}`).join('\n'));
|
|
122
|
+
}
|
|
122
123
|
const restored = restore(prior, v.regressed);
|
|
123
124
|
console.error(`\nHARNESS FAULT — refusing to publish this sweep.\n` +
|
|
124
125
|
` regressed: ${v.regressed.join(', ')}\n` +
|
package/package.json
CHANGED