mcp-context-cost 0.11.1 → 0.11.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/audit/audit.d.ts +7 -0
- package/dist/audit/audit.js +26 -1
- package/dist/audit/config.js +9 -0
- package/dist/audit/diff.js +45 -6
- package/dist/cli.d.ts +7 -2
- package/dist/cli.js +78 -24
- package/dist/core/adoption.d.ts +23 -0
- package/dist/core/adoption.js +26 -2
- package/dist/core/capture-index.js +9 -5
- package/dist/core/cross-check.js +3 -0
- package/dist/core/divergence.js +8 -0
- package/dist/core/regression.d.ts +14 -1
- package/dist/core/regression.js +44 -5
- package/dist/core/tool-shape.js +10 -2
- package/dist/sweep/dashboard.js +19 -8
- package/dist/sweep/harness-guard.d.ts +9 -1
- package/dist/sweep/harness-guard.js +18 -8
- package/dist/sweep/published-stats.d.ts +10 -5
- package/dist/sweep/published-stats.js +57 -29
- package/dist/sweep/report.js +14 -1
- package/dist/sweep/session-start.js +31 -9
- package/dist/sweep/sweep-all.js +16 -15
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -278,7 +278,7 @@ now measured against a pinned model and published beside the badge, and they do
|
|
|
278
278
|
|
|
279
279
|
| server | badge (o200k) | Claude (`claude-opus-5`) | |
|
|
280
280
|
|---|---:|---:|---|
|
|
281
|
-
| github | 54,
|
|
281
|
+
| github | 54,622 | **—** | most of the capture is `annotations`/`outputSchema` metadata Claude never sees |
|
|
282
282
|
| notion | 17,500 | **33,560** | almost no metadata to drop, so the tokenizer difference dominates |
|
|
283
283
|
|
|
284
284
|
So the heaviest server on the badge is not the heaviest server on Claude. Per-server
|
package/dist/audit/audit.d.ts
CHANGED
|
@@ -151,6 +151,13 @@ export interface AuditReport {
|
|
|
151
151
|
worstTotal: number;
|
|
152
152
|
worstSource: string;
|
|
153
153
|
over: boolean;
|
|
154
|
+
/**
|
|
155
|
+
* Servers in the audited configs with no number, when there are any. Their
|
|
156
|
+
* cost is missing from `worstTotal`, so the budget could not be checked
|
|
157
|
+
* against the whole stack and the verdict fails rather than passing on a
|
|
158
|
+
* total that understates by an unknown amount.
|
|
159
|
+
*/
|
|
160
|
+
unestablished?: string[];
|
|
154
161
|
/** Present only when over budget: the arithmetic of getting back under it. */
|
|
155
162
|
fit?: BudgetFit;
|
|
156
163
|
};
|
package/dist/audit/audit.js
CHANGED
|
@@ -179,6 +179,13 @@ export function buildReport(configs, measured, opts = {}) {
|
|
|
179
179
|
const contextWindow = opts.contextWindow ?? DEFAULT_CONTEXT_WINDOW;
|
|
180
180
|
const problems = [];
|
|
181
181
|
const emptyConfigs = [];
|
|
182
|
+
// canonicalSha256 → the divergence row computed from it, so a server can be
|
|
183
|
+
// identified by its bytes whatever the local config calls it.
|
|
184
|
+
const divByHash = new Map();
|
|
185
|
+
for (const row of Object.values(opts.divergence?.servers ?? {})) {
|
|
186
|
+
if (row?.capturedSha256)
|
|
187
|
+
divByHash.set(row.capturedSha256, row);
|
|
188
|
+
}
|
|
182
189
|
const built = [];
|
|
183
190
|
// Across every config at once: a twin in one client's file is measured for
|
|
184
191
|
// the other client's entry just the same.
|
|
@@ -319,12 +326,21 @@ export function buildReport(configs, measured, opts = {}) {
|
|
|
319
326
|
// The worst config is the gate: passing because your *lightest* client fits
|
|
320
327
|
// would be a green check on a session you don't run.
|
|
321
328
|
const worst = results[0];
|
|
329
|
+
// A total is only a ceiling to compare against if it is the whole cost. A
|
|
330
|
+
// server that failed to start contributes 0, so the stack reads lighter
|
|
331
|
+
// than it is and the budget passes on a number that is missing a server —
|
|
332
|
+
// exactly the PR the README says this gate catches. The server-level gate
|
|
333
|
+
// (core/server-diff.ts) already refuses this; so does this one now.
|
|
334
|
+
const unestablished = results.flatMap((c) => c.skipped.filter((s) => s.status !== 'remote-not-measurable').map((s) => `${c.source}: ${s.name} (${s.status})`));
|
|
322
335
|
const over = (worst?.totalTokens ?? 0) > opts.budget;
|
|
323
336
|
report.budget = {
|
|
324
337
|
limit: opts.budget,
|
|
325
338
|
worstTotal: worst?.totalTokens ?? 0,
|
|
326
339
|
worstSource: worst?.source ?? '(none)',
|
|
327
|
-
over,
|
|
340
|
+
over: over || unestablished.length > 0,
|
|
341
|
+
// Named so the reader knows which way the number is wrong: the total
|
|
342
|
+
// understates by however much these cost, which nobody knows.
|
|
343
|
+
unestablished: unestablished.length ? unestablished : undefined,
|
|
328
344
|
fit: over && worst ? planBudgetFit(worst, opts.budget) : undefined,
|
|
329
345
|
};
|
|
330
346
|
}
|
|
@@ -739,6 +755,15 @@ export function formatReport(report) {
|
|
|
739
755
|
const headroom = b.limit - b.worstTotal;
|
|
740
756
|
lines.push(`budget ok: ${n(b.worstTotal)} ≤ ${n(b.limit)} — ${n(headroom)} to spare`);
|
|
741
757
|
}
|
|
758
|
+
else if (b.unestablished && b.worstTotal <= b.limit) {
|
|
759
|
+
// Under the line on what was measured, but not everything was: say which
|
|
760
|
+
// way the number is wrong rather than passing on it.
|
|
761
|
+
lines.push(`BUDGET FAIL: ${n(b.worstTotal)} ≤ ${n(b.limit)}, but the budget could not be checked against the ` +
|
|
762
|
+
`whole stack — ${b.unestablished.length} server(s) produced no number, so this total understates ` +
|
|
763
|
+
`by however much they cost:`);
|
|
764
|
+
for (const u of b.unestablished)
|
|
765
|
+
lines.push(` ${u}`);
|
|
766
|
+
}
|
|
742
767
|
else {
|
|
743
768
|
lines.push(`BUDGET FAIL: ${n(b.worstTotal)} > ${n(b.limit)} (${b.worstSource})`);
|
|
744
769
|
const fit = b.fit;
|
package/dist/audit/config.js
CHANGED
|
@@ -182,9 +182,18 @@ export function configCandidates(env) {
|
|
|
182
182
|
/** Read + parse the candidates that exist. Unreadable files are reported, not thrown. */
|
|
183
183
|
export function loadConfigs(candidates, cwd) {
|
|
184
184
|
const out = [];
|
|
185
|
+
// Running from your home directory nominates `~/.cursor/mcp.json` twice —
|
|
186
|
+
// once as the home candidate, once as the cwd one. Loaded twice it is
|
|
187
|
+
// reported twice, doubles that client's deferral scope, and under
|
|
188
|
+
// `--baseline` the second copy pairs with nothing and fails the gate. One
|
|
189
|
+
// path is one config however many ways it was nominated.
|
|
190
|
+
const seen = new Set();
|
|
185
191
|
for (const c of candidates) {
|
|
186
192
|
if (!existsSync(c.path))
|
|
187
193
|
continue;
|
|
194
|
+
if (seen.has(c.path))
|
|
195
|
+
continue;
|
|
196
|
+
seen.add(c.path);
|
|
188
197
|
try {
|
|
189
198
|
const doc = parseJsonc(readFileSync(c.path, 'utf8'));
|
|
190
199
|
const { servers, disabled } = extractDeclaration(doc, { client: c.client, source: c.path, cwd });
|
package/dist/audit/diff.js
CHANGED
|
@@ -21,6 +21,17 @@ export function parseBaselineReport(text) {
|
|
|
21
21
|
if (!c || typeof c !== 'object' || typeof c.source !== 'string') {
|
|
22
22
|
return { report: null, problem: 'baseline has a config entry without a source path' };
|
|
23
23
|
}
|
|
24
|
+
// The number the diff subtracts from. Unchecked, a hand-trimmed or
|
|
25
|
+
// jq-filtered baseline yields `after - undefined === NaN`, and `typeof NaN`
|
|
26
|
+
// is 'number' — so the gate reported an increase of 0 and passed. A
|
|
27
|
+
// baseline that cannot be read is never "no change".
|
|
28
|
+
const total = c.totalTokens;
|
|
29
|
+
if (typeof total !== 'number' || !Number.isFinite(total)) {
|
|
30
|
+
return {
|
|
31
|
+
report: null,
|
|
32
|
+
problem: `baseline config ${c.source} has no usable totalTokens — is it the output of \`audit --json\`?`,
|
|
33
|
+
};
|
|
34
|
+
}
|
|
24
35
|
}
|
|
25
36
|
return { report: doc };
|
|
26
37
|
}
|
|
@@ -69,9 +80,25 @@ export function diffConfig(before, after, matchedBy) {
|
|
|
69
80
|
continue;
|
|
70
81
|
}
|
|
71
82
|
if (!bs && as) {
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
83
|
+
if (as.tokens === null) {
|
|
84
|
+
// Unknown, not zero — so the total this diff subtracts from does not
|
|
85
|
+
// contain it and the increase understates. Marking the config inexact
|
|
86
|
+
// is what stops `--max-increase` from passing: a server added in a PR
|
|
87
|
+
// and unmeasurable in CI (no credential — the ordinary case) otherwise
|
|
88
|
+
// reads as +0 and clears even a zero-token allowance.
|
|
89
|
+
exact = false;
|
|
90
|
+
servers.push({
|
|
91
|
+
name,
|
|
92
|
+
kind: 'added',
|
|
93
|
+
before: null,
|
|
94
|
+
after: null,
|
|
95
|
+
delta: null,
|
|
96
|
+
note: 'added but not measurable — its cost is unknown, not zero',
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
else {
|
|
100
|
+
servers.push({ name, kind: 'added', before: null, after: as.tokens, delta: as.tokens });
|
|
101
|
+
}
|
|
75
102
|
continue;
|
|
76
103
|
}
|
|
77
104
|
if (!bs || !as)
|
|
@@ -154,7 +181,15 @@ export function pairConfigs(before, after) {
|
|
|
154
181
|
pairs.push({ before: null, after: cur, matchedBy: 'unmatched' });
|
|
155
182
|
}
|
|
156
183
|
}
|
|
157
|
-
|
|
184
|
+
// One on each side reads as the same config seen from two machines, where the
|
|
185
|
+
// path legitimately differs (a laptop's baseline against a CI checkout). It
|
|
186
|
+
// does not survive the clients differing: a Claude Desktop baseline against a
|
|
187
|
+
// Claude Code run compares two unrelated stacks, and the gate then rests on
|
|
188
|
+
// that difference. The paths may differ; what they are configs *for* may not.
|
|
189
|
+
if (before.length === 1 &&
|
|
190
|
+
after.length === 1 &&
|
|
191
|
+
pairs[0].before === null &&
|
|
192
|
+
before[0].client === after[0].client) {
|
|
158
193
|
pairs[0] = { before: before[0], after: after[0], matchedBy: 'sole-config' };
|
|
159
194
|
unusedBefore.delete(before[0].source);
|
|
160
195
|
}
|
|
@@ -298,13 +333,17 @@ export function evaluateIncreaseGate(diff, limit) {
|
|
|
298
333
|
reasons.push(`${c.source}: no baseline to check its ${n(c.afterTotal)} tokens against`);
|
|
299
334
|
}
|
|
300
335
|
else if (!c.exact) {
|
|
301
|
-
reasons.push(`${c.source}: a server
|
|
336
|
+
reasons.push(`${c.source}: a server's cost is not established on both sides, so the change could not be established exactly`);
|
|
302
337
|
}
|
|
303
338
|
}
|
|
304
339
|
for (const d of diff.droppedConfigs) {
|
|
305
340
|
reasons.push(`${d.source}: covered by the baseline and not found in this run`);
|
|
306
341
|
}
|
|
307
|
-
|
|
342
|
+
// `Number.isFinite`, not `typeof === 'number'`: NaN passes the latter and
|
|
343
|
+
// then compares false against every limit, which is a silent pass.
|
|
344
|
+
const anyDelta = diff.configs.some((c) => Number.isFinite(c.delta));
|
|
345
|
+
const worst = diff.worstIncrease?.delta;
|
|
346
|
+
const increase = Number.isFinite(worst) ? worst : anyDelta ? 0 : null;
|
|
308
347
|
if (reasons.length === 0 && increase !== null && increase > limit) {
|
|
309
348
|
reasons.push(`${diff.worstIncrease.source}: +${n(increase)} tokens per request, over the ${n(limit)} allowed`);
|
|
310
349
|
}
|
package/dist/cli.d.ts
CHANGED
|
@@ -27,6 +27,11 @@ export declare function unknownFlags(argv: string[], spec: {
|
|
|
27
27
|
value: string[];
|
|
28
28
|
boolean: string[];
|
|
29
29
|
}): string[];
|
|
30
|
+
/** Every flag name a command accepts — what tells a value apart from the next flag. */
|
|
31
|
+
export declare const knownFlagNames: (spec: {
|
|
32
|
+
value: string[];
|
|
33
|
+
boolean: string[];
|
|
34
|
+
}) => Set<string>;
|
|
30
35
|
/**
|
|
31
36
|
* Every value a value-taking flag was given, in either accepted spelling:
|
|
32
37
|
* `--flag value` and `--flag=value`.
|
|
@@ -37,9 +42,9 @@ export declare function unknownFlags(argv: string[], spec: {
|
|
|
37
42
|
* matched the bare token, so the gate it asked for silently did not run and the
|
|
38
43
|
* command exited 0 — a green check on a check that never happened.
|
|
39
44
|
*/
|
|
40
|
-
export declare function flagValues(argv: string[], name: string): string[];
|
|
45
|
+
export declare function flagValues(argv: string[], name: string, known?: Set<string>): string[];
|
|
41
46
|
/** The last value given for a flag, or undefined when the flag is absent. */
|
|
42
|
-
export declare function flagValue(argv: string[], name: string): string | undefined;
|
|
47
|
+
export declare function flagValue(argv: string[], name: string, known?: Set<string>): string | undefined;
|
|
43
48
|
/**
|
|
44
49
|
* Value-taking flags that appear with no usable value.
|
|
45
50
|
*
|
package/dist/cli.js
CHANGED
|
@@ -45,6 +45,19 @@ export function slugFromUrl(url) {
|
|
|
45
45
|
const host = new URL(url).hostname.replace(/^(www|mcp)\./, '');
|
|
46
46
|
return host.replace(/[^a-z0-9]+/gi, '-').replace(/^-+|-+$/g, '').toLowerCase() || 'remote';
|
|
47
47
|
}
|
|
48
|
+
/**
|
|
49
|
+
* Report a `verify` failure and exit 1, in whichever shape the caller asked
|
|
50
|
+
* for. `--json` is documented as putting `{ ok, rederivedTokens, rederivedSha,
|
|
51
|
+
* problems }` on stdout; a script reading that gets nothing from a thrown
|
|
52
|
+
* exception, so every failure path goes through here.
|
|
53
|
+
*/
|
|
54
|
+
function failVerify(json, problem) {
|
|
55
|
+
if (json)
|
|
56
|
+
console.log(JSON.stringify({ ok: false, rederivedTokens: null, rederivedSha: null, problems: [problem] }));
|
|
57
|
+
else
|
|
58
|
+
console.error(problem);
|
|
59
|
+
process.exit(1);
|
|
60
|
+
}
|
|
48
61
|
/** Installed version, for error messages that need to say which one you are running. */
|
|
49
62
|
export function cliVersion() {
|
|
50
63
|
try {
|
|
@@ -85,6 +98,21 @@ export function unknownFlags(argv, spec) {
|
|
|
85
98
|
}
|
|
86
99
|
return unknown;
|
|
87
100
|
}
|
|
101
|
+
/**
|
|
102
|
+
* Whether a token is another flag of *this* command, rather than a value that
|
|
103
|
+
* merely looks like one.
|
|
104
|
+
*
|
|
105
|
+
* The distinction is load-bearing: `--command "--weird"` is a legitimate launch
|
|
106
|
+
* command this CLI has always accepted, while `--max-increase --json` is a
|
|
107
|
+
* value slot swallowed by the next flag. Deciding on the `--` prefix alone
|
|
108
|
+
* cannot tell them apart; deciding against the command's own flag list can, and
|
|
109
|
+
* the list is already declared at every call site.
|
|
110
|
+
*/
|
|
111
|
+
function isKnownFlagToken(tok, known) {
|
|
112
|
+
return tok.startsWith('--') && known.has(tok.slice(2).split('=')[0]);
|
|
113
|
+
}
|
|
114
|
+
/** Every flag name a command accepts — what tells a value apart from the next flag. */
|
|
115
|
+
export const knownFlagNames = (spec) => new Set([...spec.value, ...spec.boolean]);
|
|
88
116
|
/**
|
|
89
117
|
* Every value a value-taking flag was given, in either accepted spelling:
|
|
90
118
|
* `--flag value` and `--flag=value`.
|
|
@@ -95,15 +123,15 @@ export function unknownFlags(argv, spec) {
|
|
|
95
123
|
* matched the bare token, so the gate it asked for silently did not run and the
|
|
96
124
|
* command exited 0 — a green check on a check that never happened.
|
|
97
125
|
*/
|
|
98
|
-
export function flagValues(argv, name) {
|
|
126
|
+
export function flagValues(argv, name, known = new Set()) {
|
|
99
127
|
const out = [];
|
|
100
128
|
for (let i = 0; i < argv.length; i++) {
|
|
101
129
|
const tok = argv[i];
|
|
102
130
|
if (tok === `--${name}`) {
|
|
103
131
|
const next = argv[i + 1];
|
|
104
|
-
//
|
|
105
|
-
// caught by `valuelessFlags`, and
|
|
106
|
-
if (next !== undefined && !next
|
|
132
|
+
// Another flag of this command is not this flag's value; that case is a
|
|
133
|
+
// usage error, caught by `valuelessFlags`, and never read as a value.
|
|
134
|
+
if (next !== undefined && !isKnownFlagToken(next, known))
|
|
107
135
|
out.push(next);
|
|
108
136
|
continue;
|
|
109
137
|
}
|
|
@@ -113,8 +141,8 @@ export function flagValues(argv, name) {
|
|
|
113
141
|
return out;
|
|
114
142
|
}
|
|
115
143
|
/** The last value given for a flag, or undefined when the flag is absent. */
|
|
116
|
-
export function flagValue(argv, name) {
|
|
117
|
-
const values = flagValues(argv, name);
|
|
144
|
+
export function flagValue(argv, name, known = new Set()) {
|
|
145
|
+
const values = flagValues(argv, name, known);
|
|
118
146
|
return values.length ? values[values.length - 1] : undefined;
|
|
119
147
|
}
|
|
120
148
|
/**
|
|
@@ -128,6 +156,7 @@ export function flagValue(argv, name) {
|
|
|
128
156
|
* so it is refused in the same place and with the same severity.
|
|
129
157
|
*/
|
|
130
158
|
export function valuelessFlags(argv, spec) {
|
|
159
|
+
const known = knownFlagNames(spec);
|
|
131
160
|
const bad = [];
|
|
132
161
|
for (let i = 0; i < argv.length; i++) {
|
|
133
162
|
const tok = argv[i];
|
|
@@ -142,7 +171,7 @@ export function valuelessFlags(argv, spec) {
|
|
|
142
171
|
continue;
|
|
143
172
|
}
|
|
144
173
|
const next = argv[i + 1];
|
|
145
|
-
if (next === undefined || next
|
|
174
|
+
if (next === undefined || isKnownFlagToken(next, known))
|
|
146
175
|
bad.push(`--${name}`);
|
|
147
176
|
else
|
|
148
177
|
i++; // consume the value, so `--command "--weird"` is not re-read as a flag
|
|
@@ -169,7 +198,7 @@ function rejectUnknownFlags(cmd, argv, spec) {
|
|
|
169
198
|
}
|
|
170
199
|
const [, , cmd, ...rest] = process.argv;
|
|
171
200
|
if (cmd === 'audit') {
|
|
172
|
-
|
|
201
|
+
const spec = {
|
|
173
202
|
value: [
|
|
174
203
|
'config',
|
|
175
204
|
'budget',
|
|
@@ -183,9 +212,14 @@ if (cmd === 'audit') {
|
|
|
183
212
|
'capture-index-url',
|
|
184
213
|
],
|
|
185
214
|
boolean: ['json', 'docker', 'claude', 'suggest', 'changed'],
|
|
186
|
-
}
|
|
187
|
-
|
|
188
|
-
|
|
215
|
+
};
|
|
216
|
+
rejectUnknownFlags('audit', rest, spec);
|
|
217
|
+
// The same flag list the rejection used, so a value that merely looks like a
|
|
218
|
+
// flag (`--command "--weird"`) is told apart from a value slot swallowed by
|
|
219
|
+
// the next flag.
|
|
220
|
+
const known = knownFlagNames(spec);
|
|
221
|
+
const argOf = (name) => flagValue(rest, name, known);
|
|
222
|
+
const all = (name) => flagValues(rest, name, known);
|
|
189
223
|
const json = rest.includes('--json');
|
|
190
224
|
const numeric = (name) => {
|
|
191
225
|
const raw = argOf(name);
|
|
@@ -301,6 +335,12 @@ if (cmd === 'audit') {
|
|
|
301
335
|
`${where}\n` +
|
|
302
336
|
`${empty.length === 1 ? 'It was' : 'They were'} read and parsed; there is simply nothing declared to measure.\n` +
|
|
303
337
|
`Declare a server in one of them, or point at a different config: mcp-context-cost audit --config <path/to/mcp.json>`);
|
|
338
|
+
else if (all('config').length)
|
|
339
|
+
// A path the user named is not a discovery miss. Saying "looked in the
|
|
340
|
+
// standard locations" describes something the command did not do, and
|
|
341
|
+
// then advises doing the thing they just did.
|
|
342
|
+
console.error(`no MCP config found at the path(s) given: ${all('config').join(', ')}. ` +
|
|
343
|
+
`Nothing else was searched, because --config was set.`);
|
|
304
344
|
else
|
|
305
345
|
console.error(`no MCP config found. Looked in the standard Claude Desktop / Claude Code / Cursor / VS Code / Windsurf locations.${where}\n` +
|
|
306
346
|
`Point at one explicitly: mcp-context-cost audit --config <path/to/mcp.json>`);
|
|
@@ -315,9 +355,10 @@ if (cmd === 'audit') {
|
|
|
315
355
|
process.exit(report.budget?.over || report.increaseGate?.pass === false ? 1 : 0);
|
|
316
356
|
}
|
|
317
357
|
else if (cmd === 'verify') {
|
|
318
|
-
|
|
358
|
+
const spec = { value: ['remote'], boolean: ['json'] };
|
|
359
|
+
rejectUnknownFlags('verify', rest, spec);
|
|
319
360
|
const json = rest.includes('--json');
|
|
320
|
-
const remoteUrl = flagValue(rest, 'remote');
|
|
361
|
+
const remoteUrl = flagValue(rest, 'remote', knownFlagNames(spec));
|
|
321
362
|
const path = rest.find((a) => !a.startsWith('--') && a !== remoteUrl);
|
|
322
363
|
if (!remoteUrl && !path) {
|
|
323
364
|
console.error('usage: mcp-context-cost verify <measurement.json> [--json]');
|
|
@@ -333,18 +374,29 @@ else if (cmd === 'verify') {
|
|
|
333
374
|
raw = await res.text();
|
|
334
375
|
}
|
|
335
376
|
catch (e) {
|
|
336
|
-
|
|
337
|
-
if (json)
|
|
338
|
-
console.log(JSON.stringify({ ok: false, rederivedTokens: null, rederivedSha: null, problems: [problem] }));
|
|
339
|
-
else
|
|
340
|
-
console.error(problem);
|
|
341
|
-
process.exit(1);
|
|
377
|
+
failVerify(json, `failed to fetch ${remoteUrl}: ${e.message}`);
|
|
342
378
|
}
|
|
343
379
|
}
|
|
344
380
|
else {
|
|
345
|
-
|
|
381
|
+
try {
|
|
382
|
+
raw = readFileSync(path, 'utf8');
|
|
383
|
+
}
|
|
384
|
+
catch (e) {
|
|
385
|
+
// The remote branch above reports a failed fetch in the documented shape;
|
|
386
|
+
// this one used to throw, so `--json` produced a stack trace on stderr and
|
|
387
|
+
// nothing at all on stdout — the contract a script parses.
|
|
388
|
+
failVerify(json, `cannot read ${path}: ${e.message}`);
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
let m;
|
|
392
|
+
try {
|
|
393
|
+
m = JSON.parse(raw);
|
|
394
|
+
}
|
|
395
|
+
catch (e) {
|
|
396
|
+
// Reachable remotely: a proxy, a captive portal or an HTML error page
|
|
397
|
+
// served with status 200 passes the `res.ok` check above and arrives here.
|
|
398
|
+
failVerify(json, `${remoteUrl ?? path} is not valid JSON: ${e.message}`);
|
|
346
399
|
}
|
|
347
|
-
const m = JSON.parse(raw);
|
|
348
400
|
const r = verifyMeasurement(m);
|
|
349
401
|
if (json) {
|
|
350
402
|
console.log(JSON.stringify({ serverName: m.serverName, ...r, badge: r.ok ? toBadge(m) : undefined }));
|
|
@@ -361,11 +413,13 @@ else if (cmd === 'verify') {
|
|
|
361
413
|
process.exit(1);
|
|
362
414
|
}
|
|
363
415
|
else if (cmd === 'measure') {
|
|
364
|
-
|
|
416
|
+
const spec = {
|
|
365
417
|
value: ['name', 'command', 'remote', 'timeout', 'docker-image', 'baseline', 'max-increase', 'budget'],
|
|
366
418
|
boolean: ['docker'],
|
|
367
|
-
}
|
|
368
|
-
|
|
419
|
+
};
|
|
420
|
+
rejectUnknownFlags('measure', rest, spec);
|
|
421
|
+
const known = knownFlagNames(spec);
|
|
422
|
+
const argOf = (name) => flagValue(rest, name, known);
|
|
369
423
|
const command = argOf('command');
|
|
370
424
|
const remoteUrl = argOf('remote');
|
|
371
425
|
if (remoteUrl && !/^https?:\/\//i.test(remoteUrl)) {
|
package/dist/core/adoption.d.ts
CHANGED
|
@@ -143,6 +143,14 @@ export declare function decodeLoose(text: string): string;
|
|
|
143
143
|
export interface EndpointBadge {
|
|
144
144
|
url: string;
|
|
145
145
|
linkTarget: string | null;
|
|
146
|
+
/**
|
|
147
|
+
* The badge's own label — markdown alt text or an `<img alt>`. What the badge
|
|
148
|
+
* calls itself is the only thing in a README that distinguishes *this*
|
|
149
|
+
* project's badge from any other shields endpoint badge, since the JSON can
|
|
150
|
+
* be hosted anywhere (the staged action publishes to a gist, with no path
|
|
151
|
+
* shape to recognise).
|
|
152
|
+
*/
|
|
153
|
+
alt: string | null;
|
|
146
154
|
}
|
|
147
155
|
/**
|
|
148
156
|
* Every shields endpoint badge in a file, decoded, each paired with its own
|
|
@@ -172,6 +180,21 @@ export declare function linksBackToProject(target: string, src?: BadgeSource): b
|
|
|
172
180
|
* measurement here. See the header for why both count and why the link is
|
|
173
181
|
* paired with the image rather than looked for anywhere in the file.
|
|
174
182
|
*/
|
|
183
|
+
/**
|
|
184
|
+
* Whether a badge calls itself this project's badge.
|
|
185
|
+
*
|
|
186
|
+
* The self-hosted branch below used to accept the link alone and look at
|
|
187
|
+
* nothing else, so *any* shields endpoint badge — a coverage badge, say —
|
|
188
|
+
* wrapped in a link to this repository counted as displaying ours. That
|
|
189
|
+
* inflates the one number this project keeps about itself, which is the last
|
|
190
|
+
* place it can afford a generous reading.
|
|
191
|
+
*
|
|
192
|
+
* The URL cannot decide it: self-hosted JSON lives wherever its author put it,
|
|
193
|
+
* and the staged action publishes to a gist. What every published snippet does
|
|
194
|
+
* carry is the label — `context cost` — so that is what is read, tolerantly
|
|
195
|
+
* enough to accept `Context-Cost` and strictly enough to reject `coverage`.
|
|
196
|
+
*/
|
|
197
|
+
export declare function namesThisBadge(alt: string | null): boolean;
|
|
175
198
|
export declare function displaysBadge(text: string, src?: BadgeSource): boolean;
|
|
176
199
|
/**
|
|
177
200
|
* Every shields endpoint `url` in a file whose JSON is served from this
|
package/dist/core/adoption.js
CHANGED
|
@@ -137,7 +137,11 @@ export function endpointBadges(text) {
|
|
|
137
137
|
if (last && !before.slice(last.index ?? 0).includes('</a>'))
|
|
138
138
|
linkTarget = last[1];
|
|
139
139
|
}
|
|
140
|
-
|
|
140
|
+
// Markdown puts the alt before the image; HTML puts it in the same tag.
|
|
141
|
+
const beforeAlt = decoded.slice(Math.max(0, start - LINK_WINDOW), start);
|
|
142
|
+
const mdAlt = beforeAlt.match(/!\[([^\]]*)\]\(\s*[^\s)]*$/);
|
|
143
|
+
const tagAlt = after.match(/^[^<>]*?\balt\s*=\s*["']([^"']*)["']/i);
|
|
144
|
+
out.push({ url: m[1], linkTarget, alt: mdAlt ? mdAlt[1] : tagAlt ? tagAlt[1] : null });
|
|
141
145
|
}
|
|
142
146
|
return out;
|
|
143
147
|
}
|
|
@@ -169,8 +173,28 @@ export function linksBackToProject(target, src = BADGE_SOURCE) {
|
|
|
169
173
|
* measurement here. See the header for why both count and why the link is
|
|
170
174
|
* paired with the image rather than looked for anywhere in the file.
|
|
171
175
|
*/
|
|
176
|
+
/**
|
|
177
|
+
* Whether a badge calls itself this project's badge.
|
|
178
|
+
*
|
|
179
|
+
* The self-hosted branch below used to accept the link alone and look at
|
|
180
|
+
* nothing else, so *any* shields endpoint badge — a coverage badge, say —
|
|
181
|
+
* wrapped in a link to this repository counted as displaying ours. That
|
|
182
|
+
* inflates the one number this project keeps about itself, which is the last
|
|
183
|
+
* place it can afford a generous reading.
|
|
184
|
+
*
|
|
185
|
+
* The URL cannot decide it: self-hosted JSON lives wherever its author put it,
|
|
186
|
+
* and the staged action publishes to a gist. What every published snippet does
|
|
187
|
+
* carry is the label — `context cost` — so that is what is read, tolerantly
|
|
188
|
+
* enough to accept `Context-Cost` and strictly enough to reject `coverage`.
|
|
189
|
+
*/
|
|
190
|
+
export function namesThisBadge(alt) {
|
|
191
|
+
if (!alt)
|
|
192
|
+
return false;
|
|
193
|
+
return alt.toLowerCase().replace(/[^a-z]/g, '').includes('contextcost');
|
|
194
|
+
}
|
|
172
195
|
export function displaysBadge(text, src = BADGE_SOURCE) {
|
|
173
|
-
return endpointBadges(text).some((b) => hostedHere(b.url, src) ||
|
|
196
|
+
return endpointBadges(text).some((b) => hostedHere(b.url, src) ||
|
|
197
|
+
(namesThisBadge(b.alt) && b.linkTarget !== null && linksBackToProject(b.linkTarget, src)));
|
|
174
198
|
}
|
|
175
199
|
/**
|
|
176
200
|
* Every shields endpoint `url` in a file whose JSON is served from this
|
|
@@ -72,14 +72,18 @@ export function identify(canonicalSha256, index) {
|
|
|
72
72
|
if (!mine)
|
|
73
73
|
return { kind: 'unknown' };
|
|
74
74
|
const currentSha = index.current[mine.server];
|
|
75
|
-
if (
|
|
75
|
+
if (currentSha === canonicalSha256) {
|
|
76
76
|
return { kind: 'current', server: mine.server, date: mine.date, tokens: mine.totalTokens };
|
|
77
77
|
}
|
|
78
|
-
|
|
79
|
-
//
|
|
80
|
-
//
|
|
78
|
+
// The bytes are identified, but what is current for this server is not: the
|
|
79
|
+
// pointer is missing, or points at a capture the index dropped as ambiguous.
|
|
80
|
+
// `current` would be an affirmative claim that nothing has moved, which the
|
|
81
|
+
// audit prints as "no server here is running a published capture that has
|
|
82
|
+
// since moved" — told to someone who may be far behind. Unknown currency
|
|
83
|
+
// reads as unknown, which is the discipline the rest of this module keeps.
|
|
84
|
+
const current = currentSha ? index.captures[currentSha] : undefined;
|
|
81
85
|
if (!current)
|
|
82
|
-
return { kind: '
|
|
86
|
+
return { kind: 'unknown' };
|
|
83
87
|
return {
|
|
84
88
|
kind: 'behind',
|
|
85
89
|
server: mine.server,
|
package/dist/core/cross-check.js
CHANGED
|
@@ -160,6 +160,9 @@ export function isComparable(row, canonicalSha256) {
|
|
|
160
160
|
canonicalSha256 !== null &&
|
|
161
161
|
row.capturedSha256 === canonicalSha256 &&
|
|
162
162
|
row.ourTokens > 0 &&
|
|
163
|
+
// A report that parsed but carries `total: 0` is not a measurement of
|
|
164
|
+
// anything; published, it renders as a −100% divergence.
|
|
165
|
+
row.cliTokens > 0 &&
|
|
163
166
|
row.ourMappedTokens > 0);
|
|
164
167
|
}
|
|
165
168
|
/**
|
package/dist/core/divergence.js
CHANGED
|
@@ -55,6 +55,14 @@ export function mappedTokens(raw) {
|
|
|
55
55
|
export function fieldSelectionShare(row) {
|
|
56
56
|
if (row.o200kFull <= 0)
|
|
57
57
|
return null;
|
|
58
|
+
// The projection can add bytes rather than remove them — `inputSchema`
|
|
59
|
+
// becomes the longer `input_schema`, and a tool with no description gains
|
|
60
|
+
// `description: ""` — so a server with almost no metadata to drop can map
|
|
61
|
+
// *larger* than it measured. There is no share of the payload removed in that
|
|
62
|
+
// case, and publishing a negative one reads as "−11.1% of the capture is
|
|
63
|
+
// MCP-only metadata", which is not a thing.
|
|
64
|
+
if (row.o200kMapped > row.o200kFull)
|
|
65
|
+
return null;
|
|
58
66
|
return (row.o200kFull - row.o200kMapped) / row.o200kFull;
|
|
59
67
|
}
|
|
60
68
|
/**
|
|
@@ -124,7 +124,20 @@ export interface ToolAttribution {
|
|
|
124
124
|
*/
|
|
125
125
|
unexplainedTokens: number;
|
|
126
126
|
}
|
|
127
|
-
|
|
127
|
+
/**
|
|
128
|
+
* Per-tool attribution, or null when the names cannot carry it.
|
|
129
|
+
*
|
|
130
|
+
* The breakdown matches tools by name, so a name that appears twice on either
|
|
131
|
+
* side makes the maps below lose one of them silently — and the lost tokens
|
|
132
|
+
* resurface as `unexplainedTokens`, which the report explains to the reader as
|
|
133
|
+
* canonical-array framing bytes. That is a confident false explanation. Two
|
|
134
|
+
* ways it happens: a server that ships duplicate or namespaced-collapsed tool
|
|
135
|
+
* names, and `measureTools` recording every nameless tool as the single key
|
|
136
|
+
* `(unnamed)` — an invented name, where `toolNames` and `toAnthropicTools`
|
|
137
|
+
* deliberately drop nameless tools rather than invent one. Where the names
|
|
138
|
+
* cannot identify the tools, there is no attribution to give.
|
|
139
|
+
*/
|
|
140
|
+
export declare function attribute(from: ToolVectorEntry, to: ToolVectorEntry, deltaTokens: number): ToolAttribution | null;
|
|
128
141
|
export interface CostChange {
|
|
129
142
|
server: string;
|
|
130
143
|
fromDate: string;
|
package/dist/core/regression.js
CHANGED
|
@@ -87,7 +87,23 @@ export function mechanismOf(deltaTokens, deltaTools) {
|
|
|
87
87
|
// and rewritten, and the totals cannot separate the two.
|
|
88
88
|
return 'mixed';
|
|
89
89
|
}
|
|
90
|
+
/**
|
|
91
|
+
* Per-tool attribution, or null when the names cannot carry it.
|
|
92
|
+
*
|
|
93
|
+
* The breakdown matches tools by name, so a name that appears twice on either
|
|
94
|
+
* side makes the maps below lose one of them silently — and the lost tokens
|
|
95
|
+
* resurface as `unexplainedTokens`, which the report explains to the reader as
|
|
96
|
+
* canonical-array framing bytes. That is a confident false explanation. Two
|
|
97
|
+
* ways it happens: a server that ships duplicate or namespaced-collapsed tool
|
|
98
|
+
* names, and `measureTools` recording every nameless tool as the single key
|
|
99
|
+
* `(unnamed)` — an invented name, where `toolNames` and `toAnthropicTools`
|
|
100
|
+
* deliberately drop nameless tools rather than invent one. Where the names
|
|
101
|
+
* cannot identify the tools, there is no attribution to give.
|
|
102
|
+
*/
|
|
90
103
|
export function attribute(from, to, deltaTokens) {
|
|
104
|
+
const unique = (ts) => new Set(ts.map((t) => t.name)).size === ts.length;
|
|
105
|
+
if (!unique(from.tools) || !unique(to.tools))
|
|
106
|
+
return null;
|
|
91
107
|
const before = new Map(from.tools.map((t) => [t.name, t.tokens]));
|
|
92
108
|
const after = new Map(to.tools.map((t) => [t.name, t.tokens]));
|
|
93
109
|
const added = [];
|
|
@@ -162,13 +178,36 @@ export function latestChange(server, rows, vectors) {
|
|
|
162
178
|
if (deltaTokens === 0 && deltaTools === 0)
|
|
163
179
|
return null;
|
|
164
180
|
const deltaPct = from.tokens > 0 ? (deltaTokens / from.tokens) * 100 : 0;
|
|
165
|
-
// Attribution needs both sides on record
|
|
166
|
-
//
|
|
181
|
+
// Attribution needs both sides on record. Matched by *cost as of that date*,
|
|
182
|
+
// not by date equality: vectors are deduped by capture and keep the first
|
|
183
|
+
// date a capture was seen, while `from` is the last row of the previous
|
|
184
|
+
// plateau — so the two dates coincide only when the previous cost was
|
|
185
|
+
// measured exactly once. With weekly sweeps and less frequent releases they
|
|
186
|
+
// almost never do, and a date-equality join therefore reported "only one of
|
|
187
|
+
// the two captures is on record" while holding both. The vector in force on a
|
|
188
|
+
// given day is the newest one recorded on or before it.
|
|
167
189
|
let attribution = null;
|
|
168
|
-
const
|
|
169
|
-
|
|
170
|
-
|
|
190
|
+
const inForceOn = (date, tokensThatDay) => {
|
|
191
|
+
const upto = (vectors?.entries ?? []).filter((e) => e.date <= date);
|
|
192
|
+
// A same-day re-sweep replaces that day's history row but *appends* a
|
|
193
|
+
// capture, so one date can carry two. Only one of them is the one the row
|
|
194
|
+
// describes: prefer the capture whose total is the number history recorded.
|
|
195
|
+
const agreeing = upto.filter((e) => e.totalTokens === tokensThatDay);
|
|
196
|
+
const pool = agreeing.length > 0 ? agreeing : upto;
|
|
197
|
+
return pool.reduce((best, e) => (!best || e.date >= best.date ? e : best), undefined);
|
|
198
|
+
};
|
|
199
|
+
const fromVec = inForceOn(from.date, from.tokens);
|
|
200
|
+
const toVec = inForceOn(to.date, to.tokens);
|
|
201
|
+
// A vector only explains the row it agrees with. If the totals disagree the
|
|
202
|
+
// file does not cover this change — say so rather than attributing a delta
|
|
203
|
+
// to the wrong capture and publishing the mismatch as framing bytes.
|
|
204
|
+
if (fromVec &&
|
|
205
|
+
toVec &&
|
|
206
|
+
fromVec.canonicalSha256 !== toVec.canonicalSha256 &&
|
|
207
|
+
fromVec.totalTokens === from.tokens &&
|
|
208
|
+
toVec.totalTokens === to.tokens) {
|
|
171
209
|
attribution = attribute(fromVec, toVec, deltaTokens);
|
|
210
|
+
}
|
|
172
211
|
return {
|
|
173
212
|
server,
|
|
174
213
|
fromDate: from.date,
|
package/dist/core/tool-shape.js
CHANGED
|
@@ -23,10 +23,18 @@ export function quantileTable(values) {
|
|
|
23
23
|
* checkable by anyone holding the same JSON.
|
|
24
24
|
*/
|
|
25
25
|
export function percentileOf(quantiles, value) {
|
|
26
|
+
// The LOWEST percentile whose quantile the value reaches, not the highest.
|
|
27
|
+
// Taking the highest reports a value tied with half the measured set as p100
|
|
28
|
+
// — "heavier than 100% of measured tools" about something exactly average for
|
|
29
|
+
// its tie — because every percentile across the tie carries the same
|
|
30
|
+
// quantile. The lowest names where the tie begins, which is what "heavier
|
|
31
|
+
// than P% of tools" means.
|
|
26
32
|
let p = 0;
|
|
27
33
|
for (let i = 0; i <= 100; i++) {
|
|
28
|
-
if (quantiles[i] <= value)
|
|
29
|
-
|
|
34
|
+
if (quantiles[i] <= value) {
|
|
35
|
+
if (quantiles[i] < value || i === 0 || quantiles[i - 1] < quantiles[i])
|
|
36
|
+
p = i;
|
|
37
|
+
}
|
|
30
38
|
else
|
|
31
39
|
break;
|
|
32
40
|
}
|
package/dist/sweep/dashboard.js
CHANGED
|
@@ -6,6 +6,8 @@
|
|
|
6
6
|
import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'node:fs';
|
|
7
7
|
import { dirname, join } from 'node:path';
|
|
8
8
|
import { parse } from 'yaml';
|
|
9
|
+
import { isCurrent } from '../core/divergence.js';
|
|
10
|
+
import { loadRows } from './report.js';
|
|
9
11
|
import { bandColor, BAND_META } from '../core/bands.js';
|
|
10
12
|
import { parseHistory, plottableSeries } from './history.js';
|
|
11
13
|
/** Longest series a sparkline plots — a stat-tile trend, not a full chart. */
|
|
@@ -52,10 +54,9 @@ export function generateDashboard(root = process.cwd()) {
|
|
|
52
54
|
// Only the run of sweeps taken under the same isolation is plotted: a step
|
|
53
55
|
// across an isolation change is the harness moving, not the server.
|
|
54
56
|
const seriesFor = (name) => plottableSeries(history.filter((h) => h.server === name));
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
});
|
|
57
|
+
// Shared with the leaderboard rather than re-read here: one definition of how
|
|
58
|
+
// a measurement is loaded, including its tolerance for a half-written file.
|
|
59
|
+
const rows = loadRows(doc.servers, root);
|
|
59
60
|
const measured = rows
|
|
60
61
|
.filter((r) => r.m && (r.m.status === 'measured' || r.m.status === 'dynamic') && r.m.totalTokens !== null)
|
|
61
62
|
.sort((a, b) => (b.m.totalTokens ?? 0) - (a.m.totalTokens ?? 0));
|
|
@@ -74,7 +75,12 @@ export function generateDashboard(root = process.cwd()) {
|
|
|
74
75
|
const meta = BAND_META[band];
|
|
75
76
|
const largest = [...m.tools].sort((a, b) => b.tokens - a.tokens)[0];
|
|
76
77
|
const pct = Math.max(1.2, (t / max) * 100);
|
|
77
|
-
|
|
78
|
+
// Gated on the capture it was computed from, exactly as the leaderboard
|
|
79
|
+
// gates it: four rows here were publishing a Claude cost derived from
|
|
80
|
+
// bytes that no longer exist, while leaderboard.md printed `—` for the
|
|
81
|
+
// same four.
|
|
82
|
+
const dRaw = dSrv[r.entry.name];
|
|
83
|
+
const div = isCurrent(dRaw, m.canonicalSha256 ?? null) ? dRaw : undefined;
|
|
78
84
|
const claudeTip = div ? ` · in a Claude request: ${fmt(div.claudeDelta)} tok` : '';
|
|
79
85
|
const series = seriesFor(r.entry.name);
|
|
80
86
|
const tokens = series.rows.map((h) => h.tokens);
|
|
@@ -107,10 +113,15 @@ export function generateDashboard(root = process.cwd()) {
|
|
|
107
113
|
const tableRows = measured
|
|
108
114
|
.map((r, i) => {
|
|
109
115
|
const m = r.m;
|
|
110
|
-
|
|
116
|
+
// Gated on the capture it was computed from, exactly as the leaderboard
|
|
117
|
+
// gates it: four rows here were publishing a Claude cost derived from
|
|
118
|
+
// bytes that no longer exist, while leaderboard.md printed `—` for the
|
|
119
|
+
// same four.
|
|
120
|
+
const dRaw = dSrv[r.entry.name];
|
|
121
|
+
const div = isCurrent(dRaw, m.canonicalSha256 ?? null) ? dRaw : undefined;
|
|
111
122
|
const tokens = seriesFor(r.entry.name).rows.map((h) => h.tokens);
|
|
112
123
|
const trend = tokens.length > 1
|
|
113
|
-
? `${tokens[tokens.length - 1] - tokens[0] >= 0 ? '+' : ''}${fmt(tokens[tokens.length - 1] - tokens[0])}
|
|
124
|
+
? `${tokens[tokens.length - 1] - tokens[0] >= 0 ? '+' : ''}${fmt(tokens[tokens.length - 1] - tokens[0])} over ${tokens.length} sweeps`
|
|
114
125
|
: '—';
|
|
115
126
|
return `<tr><td>${i + 1}</td><td>${esc(r.entry.name)}</td><td class="num">${fmt(m.totalTokens)}</td><td class="num">${div ? fmt(div.claudeDelta) : '—'}</td><td class="num">${esc(m.toolCount)}</td><td>${esc(BAND_META[bandColor(m.totalTokens)].label)}</td><td>${esc(r.entry.category)}</td><td class="num">${trend}</td></tr>`;
|
|
116
127
|
})
|
|
@@ -227,7 +238,7 @@ export function generateDashboard(root = process.cwd()) {
|
|
|
227
238
|
</div>
|
|
228
239
|
|
|
229
240
|
<h2>Leaderboard</h2>
|
|
230
|
-
<p class="h2sub">Tokens = o200k_base count of the canonical <code>tools/list</code> bytes — the wire payload. What a <em>Claude request</em> actually carries can differ sharply (
|
|
241
|
+
<p class="h2sub">Tokens = o200k_base count of the canonical <code>tools/list</code> bytes — the wire payload. What a <em>Claude request</em> actually carries can differ sharply (a server can spend most of its schema bytes on fields no Anthropic request sends, and the denser tokenizer can push the rest back up). The trend line plots tokens across every sweep date on record (oldest→newest, dot = current); servers with one sweep so far show no line yet, and a sweep taken under different isolation than the current one is left out rather than drawn as a change in the server. Hover a row for exact numbers; <a href="METHODOLOGY.html#claude-divergence">method</a>.</p>
|
|
231
242
|
<div class="board">
|
|
232
243
|
${barRows || '<p class="h2sub">Sweep in progress — first results land shortly.</p>'}
|
|
233
244
|
</div>
|
|
@@ -47,7 +47,15 @@ export interface Verdict {
|
|
|
47
47
|
* `current` maps server name to the status it just measured at. Servers absent
|
|
48
48
|
* from it were not swept and are ignored.
|
|
49
49
|
*/
|
|
50
|
-
export declare function verdict(prior: Snapshot[], current: Map<string, MeasurementStatus
|
|
50
|
+
export declare function verdict(prior: Snapshot[], current: Map<string, MeasurementStatus>,
|
|
51
|
+
/**
|
|
52
|
+
* Servers this sweep could not measure because docker itself failed. They
|
|
53
|
+
* never reached a status, so they are absent from `current` and invisible to
|
|
54
|
+
* the comparison below — but they are the same fact it is looking for: a
|
|
55
|
+
* server that could have produced a number and did not. Counted here so the
|
|
56
|
+
* two symptoms share one threshold instead of being each other's blind spot.
|
|
57
|
+
*/
|
|
58
|
+
dockerFaults?: number): Verdict;
|
|
51
59
|
/**
|
|
52
60
|
* Put the snapshotted artifacts back, byte for byte. Only servers named in
|
|
53
61
|
* `names` are touched, and only where a prior file existed — a server whose
|
|
@@ -78,12 +78,21 @@ export function snapshot(names, root = process.cwd()) {
|
|
|
78
78
|
* `current` maps server name to the status it just measured at. Servers absent
|
|
79
79
|
* from it were not swept and are ignored.
|
|
80
80
|
*/
|
|
81
|
-
export function verdict(prior, current
|
|
81
|
+
export function verdict(prior, current,
|
|
82
|
+
/**
|
|
83
|
+
* Servers this sweep could not measure because docker itself failed. They
|
|
84
|
+
* never reached a status, so they are absent from `current` and invisible to
|
|
85
|
+
* the comparison below — but they are the same fact it is looking for: a
|
|
86
|
+
* server that could have produced a number and did not. Counted here so the
|
|
87
|
+
* two symptoms share one threshold instead of being each other's blind spot.
|
|
88
|
+
*/
|
|
89
|
+
dockerFaults = 0) {
|
|
82
90
|
const comparableNames = prior
|
|
83
91
|
.filter((s) => s.status !== null && isGood(s.status) && current.has(s.name))
|
|
84
92
|
.map((s) => s.name);
|
|
85
93
|
const regressed = comparableNames.filter((n) => !isGood(current.get(n)));
|
|
86
|
-
const
|
|
94
|
+
const failed = regressed.length + dockerFaults;
|
|
95
|
+
const comparable = comparableNames.length + dockerFaults;
|
|
87
96
|
if (comparable === 0) {
|
|
88
97
|
return {
|
|
89
98
|
fault: false,
|
|
@@ -94,15 +103,16 @@ export function verdict(prior, current) {
|
|
|
94
103
|
reason: 'no prior measurement to compare against — harness check not performed',
|
|
95
104
|
};
|
|
96
105
|
}
|
|
97
|
-
const ratio =
|
|
106
|
+
const ratio = failed / comparable;
|
|
98
107
|
const pct = (ratio * 100).toFixed(0);
|
|
99
|
-
|
|
108
|
+
const how = dockerFaults > 0 ? ` (${regressed.length} regressed, ${dockerFaults} unmeasurable)` : '';
|
|
109
|
+
if (failed >= MIN_REGRESSIONS && ratio >= FAULT_RATIO) {
|
|
100
110
|
return {
|
|
101
111
|
fault: true,
|
|
102
112
|
regressed,
|
|
103
113
|
comparable,
|
|
104
|
-
reason: `${
|
|
105
|
-
`this sweep — at or above the ${MIN_REGRESSIONS}-server, ` +
|
|
114
|
+
reason: `${failed} of ${comparable} previously-measured servers (${pct}%) produced no number in ` +
|
|
115
|
+
`this sweep${how} — at or above the ${MIN_REGRESSIONS}-server, ` +
|
|
106
116
|
`${(FAULT_RATIO * 100).toFixed(0)}% threshold that reads as a broken harness ` +
|
|
107
117
|
`rather than broken servers`,
|
|
108
118
|
};
|
|
@@ -111,8 +121,8 @@ export function verdict(prior, current) {
|
|
|
111
121
|
fault: false,
|
|
112
122
|
regressed,
|
|
113
123
|
comparable,
|
|
114
|
-
reason: `${
|
|
115
|
-
`this sweep — below the harness-fault threshold, publishing normally`,
|
|
124
|
+
reason: `${failed} of ${comparable} previously-measured servers (${pct}%) produced no number in ` +
|
|
125
|
+
`this sweep${how} — below the harness-fault threshold, publishing normally`,
|
|
116
126
|
};
|
|
117
127
|
}
|
|
118
128
|
/**
|
|
@@ -28,17 +28,22 @@ export interface PublishedStats {
|
|
|
28
28
|
/** Rows the leaderboard prints a claude number for: measured AND capture-current. */
|
|
29
29
|
currentCount: number;
|
|
30
30
|
heaviestClaudeName: string | null;
|
|
31
|
+
/** `claudeTokens` is null when the published row no longer matches the capture on disk. */
|
|
31
32
|
github: {
|
|
32
33
|
badgeTokens: number;
|
|
33
|
-
|
|
34
|
-
claudeTokens: number;
|
|
35
|
-
droppedPct: number;
|
|
34
|
+
claudeTokens: number | null;
|
|
36
35
|
};
|
|
37
36
|
notion: {
|
|
38
37
|
badgeTokens: number;
|
|
39
|
-
claudeTokens: number;
|
|
38
|
+
claudeTokens: number | null;
|
|
40
39
|
};
|
|
41
|
-
/**
|
|
40
|
+
/** The current row showing the largest field-selection effect, and its two counts. */
|
|
41
|
+
widest: {
|
|
42
|
+
server: string;
|
|
43
|
+
full: number;
|
|
44
|
+
mapped: number;
|
|
45
|
+
};
|
|
46
|
+
/** Field-selection share across the run's *current* rows, as fractions of the payload. */
|
|
42
47
|
shareMin: number;
|
|
43
48
|
shareMax: number;
|
|
44
49
|
/** claudeDelta / o200kFull across the run's rows that carry a number. */
|
|
@@ -75,25 +75,48 @@ export function computePublishedStats(entries, root = process.cwd()) {
|
|
|
75
75
|
}
|
|
76
76
|
if (!div)
|
|
77
77
|
throw new Error('results/divergence.json is missing — README states its numbers');
|
|
78
|
-
|
|
78
|
+
/**
|
|
79
|
+
* A divergence row only where it still describes the capture on disk.
|
|
80
|
+
*
|
|
81
|
+
* The staleness gate is the whole discipline of this column, and skipping it
|
|
82
|
+
* here is how README came to print two different costs for github on one
|
|
83
|
+
* page: 54,422 from a row computed against bytes that no longer existed,
|
|
84
|
+
* beside 54,622 from the measurement. `withClaude` below already applied
|
|
85
|
+
* `isCurrent` to the very same run — the rule guarded one number and not its
|
|
86
|
+
* neighbour.
|
|
87
|
+
*/
|
|
88
|
+
const currentDivRow = (name) => {
|
|
79
89
|
const d = div.servers[name];
|
|
80
90
|
if (!d)
|
|
81
91
|
throw new Error(`README's Claude table names ${name}, which is not in the divergence run`);
|
|
82
|
-
|
|
92
|
+
const onDisk = rows.find((r) => r.entry.name === name)?.m?.canonicalSha256 ?? null;
|
|
93
|
+
return isCurrent(d, onDisk) ? d : null;
|
|
83
94
|
};
|
|
84
|
-
|
|
85
|
-
const
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
95
|
+
/** The badge number comes from the measurement, never from a divergence row's copy of it. */
|
|
96
|
+
const badgeTokensOf = (name) => {
|
|
97
|
+
const r = measured.find((x) => x.entry.name === name);
|
|
98
|
+
if (!r)
|
|
99
|
+
throw new Error(`README's Claude table names ${name}, which has no current measurement`);
|
|
100
|
+
return r.m.totalTokens;
|
|
101
|
+
};
|
|
102
|
+
// Ranges are stated over the rows that are still current, for the same
|
|
103
|
+
// reason: a range whose endpoint comes from a superseded capture describes a
|
|
104
|
+
// set that no longer exists.
|
|
105
|
+
const currentRows = Object.entries(div.servers).filter(([name, r]) => isCurrent(r, rows.find((x) => x.entry.name === name)?.m?.canonicalSha256 ?? null));
|
|
106
|
+
const shares = currentRows
|
|
107
|
+
.map(([, r]) => fieldSelectionShare(r))
|
|
108
|
+
.filter((s) => s !== null && s >= 0);
|
|
109
|
+
const ratios = currentRows
|
|
110
|
+
.filter(([, r]) => typeof r.claudeDelta === 'number' && r.claudeDelta > 0 && r.o200kFull > 0)
|
|
111
|
+
.map(([, r]) => r.claudeDelta / r.o200kFull);
|
|
94
112
|
if (shares.length === 0 || ratios.length === 0) {
|
|
95
|
-
throw new Error('
|
|
113
|
+
throw new Error('no current divergence row — METHODOLOGY states ranges over them; run `npm run divergence`');
|
|
96
114
|
}
|
|
115
|
+
// The exemplar METHODOLOGY names for the field-selection effect is whichever
|
|
116
|
+
// current row shows it most, rather than a server hardcoded into the prose.
|
|
117
|
+
const widest = currentRows
|
|
118
|
+
.filter(([, r]) => (fieldSelectionShare(r) ?? -1) >= 0)
|
|
119
|
+
.sort((a, b) => (fieldSelectionShare(b[1]) ?? 0) - (fieldSelectionShare(a[1]) ?? 0))[0];
|
|
97
120
|
const withClaude = measured.filter((r) => isCurrent(div.servers[r.entry.name], r.m.canonicalSha256));
|
|
98
121
|
const heaviest = [...withClaude].sort((a, b) => div.servers[b.entry.name].claudeDelta - div.servers[a.entry.name].claudeDelta)[0];
|
|
99
122
|
const costlier = measured.filter((r) => {
|
|
@@ -118,13 +141,9 @@ export function computePublishedStats(entries, root = process.cwd()) {
|
|
|
118
141
|
runSize: Object.keys(div.servers).length,
|
|
119
142
|
currentCount: withClaude.length,
|
|
120
143
|
heaviestClaudeName: heaviest?.entry.name ?? null,
|
|
121
|
-
github: {
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
claudeTokens: github.claudeDelta,
|
|
125
|
-
droppedPct: Math.round(githubShare * 100),
|
|
126
|
-
},
|
|
127
|
-
notion: { badgeTokens: notion.o200kFull, claudeTokens: notion.claudeDelta },
|
|
144
|
+
github: { badgeTokens: badgeTokensOf('github'), claudeTokens: currentDivRow('github')?.claudeDelta ?? null },
|
|
145
|
+
notion: { badgeTokens: badgeTokensOf('notion'), claudeTokens: currentDivRow('notion')?.claudeDelta ?? null },
|
|
146
|
+
widest: { server: widest[0], full: widest[1].o200kFull, mapped: widest[1].o200kMapped },
|
|
128
147
|
shareMin: Math.min(...shares),
|
|
129
148
|
shareMax: Math.max(...shares),
|
|
130
149
|
ratioMin: Math.min(...ratios),
|
|
@@ -140,6 +159,8 @@ export function computePublishedStats(entries, root = process.cwd()) {
|
|
|
140
159
|
}
|
|
141
160
|
export const PAGE_FILES = ['README.md', 'docs/index.md', 'docs/METHODOLOGY.md'];
|
|
142
161
|
const fmt = (n) => n.toLocaleString('en-US');
|
|
162
|
+
/** A number, or the em-dash that means "no current measurement" everywhere else here. */
|
|
163
|
+
const q = (n) => (n === null ? '—' : fmt(n));
|
|
143
164
|
export const PAGE_CLAIMS = [
|
|
144
165
|
{
|
|
145
166
|
file: 'README.md',
|
|
@@ -210,14 +231,14 @@ export const PAGE_CLAIMS = [
|
|
|
210
231
|
{
|
|
211
232
|
file: 'README.md',
|
|
212
233
|
id: 'claude-table:github',
|
|
213
|
-
template: '| github | {n} | **{
|
|
214
|
-
values: (s) => [fmt(s.claude.github.badgeTokens),
|
|
234
|
+
template: '| github | {n} | **{q}** | most of the capture is `annotations`/`outputSchema` metadata Claude never sees |',
|
|
235
|
+
values: (s) => [fmt(s.claude.github.badgeTokens), q(s.claude.github.claudeTokens)],
|
|
215
236
|
},
|
|
216
237
|
{
|
|
217
238
|
file: 'README.md',
|
|
218
239
|
id: 'claude-table:notion',
|
|
219
|
-
template: '| notion | {n} | **{
|
|
220
|
-
values: (s) => [fmt(s.claude.notion.badgeTokens),
|
|
240
|
+
template: '| notion | {n} | **{q}** | almost no metadata to drop, so the tokenizer difference dominates |',
|
|
241
|
+
values: (s) => [fmt(s.claude.notion.badgeTokens), q(s.claude.notion.claudeTokens)],
|
|
221
242
|
},
|
|
222
243
|
{
|
|
223
244
|
file: 'README.md',
|
|
@@ -252,12 +273,15 @@ export const PAGE_CLAIMS = [
|
|
|
252
273
|
{
|
|
253
274
|
file: 'docs/METHODOLOGY.md',
|
|
254
275
|
id: 'divergence:share-range',
|
|
255
|
-
template: 'this removes between {f}% and **{f}%** of the payload (
|
|
276
|
+
template: 'this removes between {f}% and **{f}%** of the payload ({w}: {n} → {n} tokens).',
|
|
277
|
+
// The exemplar is whichever current row shows the effect most, not a server
|
|
278
|
+
// named in the prose — a hardcoded name goes stale the week it is re-swept.
|
|
256
279
|
values: (s) => [
|
|
257
280
|
(s.claude.shareMin * 100).toFixed(1),
|
|
258
281
|
(s.claude.shareMax * 100).toFixed(1),
|
|
259
|
-
|
|
260
|
-
fmt(s.claude.
|
|
282
|
+
s.claude.widest.server,
|
|
283
|
+
fmt(s.claude.widest.full),
|
|
284
|
+
fmt(s.claude.widest.mapped),
|
|
261
285
|
],
|
|
262
286
|
},
|
|
263
287
|
{
|
|
@@ -303,7 +327,7 @@ const escapeLiteral = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&').replace(/\
|
|
|
303
327
|
export function compileTemplate(template) {
|
|
304
328
|
let source = '';
|
|
305
329
|
let last = 0;
|
|
306
|
-
for (const slot of template.matchAll(/\{[
|
|
330
|
+
for (const slot of template.matchAll(/\{[ndwfq]\}/g)) {
|
|
307
331
|
source += escapeLiteral(template.slice(last, slot.index));
|
|
308
332
|
source +=
|
|
309
333
|
slot[0] === '{n}'
|
|
@@ -312,7 +336,11 @@ export function compileTemplate(template) {
|
|
|
312
336
|
? '(\\d+)'
|
|
313
337
|
: slot[0] === '{f}'
|
|
314
338
|
? '(\\d+\\.\\d+)'
|
|
315
|
-
:
|
|
339
|
+
: // `{q}` is a number that may not exist: the em-dash the leaderboard
|
|
340
|
+
// already prints for a row whose capture has moved on.
|
|
341
|
+
slot[0] === '{q}'
|
|
342
|
+
? '([\\d,]+|—)'
|
|
343
|
+
: '([A-Za-z0-9._-]+)';
|
|
316
344
|
last = slot.index + slot[0].length;
|
|
317
345
|
}
|
|
318
346
|
source += escapeLiteral(template.slice(last));
|
package/dist/sweep/report.js
CHANGED
|
@@ -21,7 +21,20 @@ function csvCell(s) {
|
|
|
21
21
|
export function loadRows(entries, root = process.cwd()) {
|
|
22
22
|
return entries.map((entry) => {
|
|
23
23
|
const p = join(root, 'results', entry.name, 'measurement.json');
|
|
24
|
-
|
|
24
|
+
if (!existsSync(p))
|
|
25
|
+
return { entry, m: null };
|
|
26
|
+
try {
|
|
27
|
+
return { entry, m: JSON.parse(readFileSync(p, 'utf8')) };
|
|
28
|
+
}
|
|
29
|
+
catch {
|
|
30
|
+
// The same tolerance `appendHistory` states for the same file: a sweep
|
|
31
|
+
// killed mid-write leaves a truncated measurement, and every generator
|
|
32
|
+
// reads through here. Throwing meant one such file broke the leaderboard,
|
|
33
|
+
// the server pages, the dashboard, the tool-shape baseline and the
|
|
34
|
+
// published-stats check at once — weekly, with a SyntaxError that named
|
|
35
|
+
// no file. A server with no readable record reads as one with no record.
|
|
36
|
+
return { entry, m: null };
|
|
37
|
+
}
|
|
25
38
|
});
|
|
26
39
|
}
|
|
27
40
|
/** results/divergence.json if a divergence run has been recorded, else null. */
|
|
@@ -25,6 +25,7 @@ import { join, resolve } from 'node:path';
|
|
|
25
25
|
import { fileURLToPath } from 'node:url';
|
|
26
26
|
import { parse } from 'yaml';
|
|
27
27
|
import { measureServer } from './run.js';
|
|
28
|
+
import { DockerHarnessFault } from './docker.js';
|
|
28
29
|
import { SESSION_START_METHOD, parseSessionStart, toSessionStartRow, } from '../core/session-start.js';
|
|
29
30
|
export function loadSessionStart(root = process.cwd()) {
|
|
30
31
|
const p = join(root, 'results', 'session-start.json');
|
|
@@ -63,15 +64,36 @@ if (isMain) {
|
|
|
63
64
|
const queue = [...entries];
|
|
64
65
|
async function worker() {
|
|
65
66
|
for (let e = queue.shift(); e; e = queue.shift()) {
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
67
|
+
let m;
|
|
68
|
+
try {
|
|
69
|
+
m = await measureServer(e.name, e.command, {
|
|
70
|
+
timeoutMs: (e.timeoutSeconds ?? defaultTimeout) * 1000,
|
|
71
|
+
docker,
|
|
72
|
+
dockerImage: e.dockerImage,
|
|
73
|
+
dummyEnv: e.env ?? [],
|
|
74
|
+
dummyEnvValues: e.envValues,
|
|
75
|
+
needsGit: e.needsGit,
|
|
76
|
+
persist: false, // the measurements on disk are not this run's to rewrite
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
catch (err) {
|
|
80
|
+
// One thrown error used to reject Promise.all and end the process
|
|
81
|
+
// before anything was written, discarding every capture the run had
|
|
82
|
+
// already completed — and skipping the `finally` that force-removes
|
|
83
|
+
// containers, orphaning the in-flight ones. A machine fault on one
|
|
84
|
+
// server is recorded against that server; the rest of the run stands.
|
|
85
|
+
if (!(err instanceof DockerHarnessFault))
|
|
86
|
+
throw err;
|
|
87
|
+
servers[e.name] = {
|
|
88
|
+
instructions: '',
|
|
89
|
+
instructionsTokens: 0,
|
|
90
|
+
instructionsSha256: '',
|
|
91
|
+
capturedSha256: null,
|
|
92
|
+
error: `docker harness fault: ${err.message.slice(0, 200)}`,
|
|
93
|
+
};
|
|
94
|
+
console.log(` ${e.name}: docker harness fault — recorded, run continues`);
|
|
95
|
+
continue;
|
|
96
|
+
}
|
|
75
97
|
if (m.status !== 'measured' && m.status !== 'dynamic') {
|
|
76
98
|
servers[e.name] = {
|
|
77
99
|
instructions: '',
|
package/dist/sweep/sweep-all.js
CHANGED
|
@@ -18,7 +18,7 @@ import { DockerHarnessFault } from './docker.js';
|
|
|
18
18
|
import { writeLeaderboard } from './report.js';
|
|
19
19
|
import { appendHistory } from './history.js';
|
|
20
20
|
import { appendToolVectors, writeRegressions } from './regressions.js';
|
|
21
|
-
import {
|
|
21
|
+
import { snapshot, verdict, restore } from './harness-guard.js';
|
|
22
22
|
import { selectShard, shardIndexForDate } from './shard.js';
|
|
23
23
|
function arg(name) {
|
|
24
24
|
const i = process.argv.indexOf(`--${name}`);
|
|
@@ -101,24 +101,25 @@ async function worker() {
|
|
|
101
101
|
}
|
|
102
102
|
}
|
|
103
103
|
await Promise.all(Array.from({ length: Math.max(1, concurrency) }, () => worker()));
|
|
104
|
-
// A docker fault on most of the slice is the harness-fault story with an
|
|
105
|
-
// earlier symptom — the guard below can't see it (nothing regressed on disk;
|
|
106
|
-
// the throws happened before anything was written), so it is judged here by
|
|
107
|
-
// the guard's own thresholds. Below them, the sweep publishes what it did
|
|
108
|
-
// measure and the faulted servers wait for the next cycle.
|
|
109
|
-
if (dockerFaults.size >= MIN_REGRESSIONS && dockerFaults.size / entries.length >= FAULT_RATIO) {
|
|
110
|
-
console.error(`\nHARNESS FAULT — docker could not run for ${dockerFaults.size} of ${entries.length} servers; ` +
|
|
111
|
-
`refusing to publish this sweep.\n` +
|
|
112
|
-
[...dockerFaults].map(([n, msg]) => ` ${n}: ${msg}`).join('\n') +
|
|
113
|
-
`\n Nothing was overwritten — every previous record stands. ` +
|
|
114
|
-
`Check the Docker daemon and registry path, then re-run.`);
|
|
115
|
-
process.exit(1);
|
|
116
|
-
}
|
|
117
104
|
// Before publishing anything: is this sweep a statement about the servers, or
|
|
118
105
|
// about the machine that measured them?
|
|
119
|
-
|
|
106
|
+
//
|
|
107
|
+
// The two symptoms are counted together, against one denominator. Judged apart
|
|
108
|
+
// they were each other's blind spot: a flaky daemon that throws for 6 of 14
|
|
109
|
+
// servers (6 ≥ 5, but 43% of the slice) and times out 4 more (4 < 5) trips
|
|
110
|
+
// neither threshold, and the sweep publishes with 10 of 14 servers producing no
|
|
111
|
+
// number and four good records overwritten with failures. A server that could
|
|
112
|
+
// have produced a number and didn't is one fact, however it failed.
|
|
113
|
+
const v = verdict(prior, statuses, dockerFaults.size);
|
|
120
114
|
console.log(`harness check: ${v.reason}`);
|
|
115
|
+
if (dockerFaults.size > 0 && !v.fault) {
|
|
116
|
+
console.warn(` (${dockerFaults.size} docker fault(s) counted toward that check; those servers were not measured)`);
|
|
117
|
+
}
|
|
121
118
|
if (v.fault) {
|
|
119
|
+
if (dockerFaults.size) {
|
|
120
|
+
console.error(`\ndocker could not run for ${dockerFaults.size} server(s):\n` +
|
|
121
|
+
[...dockerFaults].map(([n, msg]) => ` ${n}: ${msg}`).join('\n'));
|
|
122
|
+
}
|
|
122
123
|
const restored = restore(prior, v.regressed);
|
|
123
124
|
console.error(`\nHARNESS FAULT — refusing to publish this sweep.\n` +
|
|
124
125
|
` regressed: ${v.regressed.join(', ')}\n` +
|
package/package.json
CHANGED