mcp-context-cost 0.11.1 → 0.11.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/audit/audit.d.ts +7 -0
- package/dist/audit/audit.js +19 -1
- package/dist/audit/diff.js +36 -5
- package/dist/cli.d.ts +7 -2
- package/dist/cli.js +40 -16
- package/dist/core/regression.js +28 -5
- package/dist/sweep/dashboard.js +15 -4
- package/dist/sweep/published-stats.d.ts +10 -5
- package/dist/sweep/published-stats.js +57 -29
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -278,7 +278,7 @@ now measured against a pinned model and published beside the badge, and they do
|
|
|
278
278
|
|
|
279
279
|
| server | badge (o200k) | Claude (`claude-opus-5`) | |
|
|
280
280
|
|---|---:|---:|---|
|
|
281
|
-
| github | 54,
|
|
281
|
+
| github | 54,622 | **—** | most of the capture is `annotations`/`outputSchema` metadata Claude never sees |
|
|
282
282
|
| notion | 17,500 | **33,560** | almost no metadata to drop, so the tokenizer difference dominates |
|
|
283
283
|
|
|
284
284
|
So the heaviest server on the badge is not the heaviest server on Claude. Per-server
|
package/dist/audit/audit.d.ts
CHANGED
|
@@ -151,6 +151,13 @@ export interface AuditReport {
|
|
|
151
151
|
worstTotal: number;
|
|
152
152
|
worstSource: string;
|
|
153
153
|
over: boolean;
|
|
154
|
+
/**
|
|
155
|
+
* Servers in the audited configs with no number, when there are any. Their
|
|
156
|
+
* cost is missing from `worstTotal`, so the budget could not be checked
|
|
157
|
+
* against the whole stack and the verdict fails rather than passing on a
|
|
158
|
+
* total that understates by an unknown amount.
|
|
159
|
+
*/
|
|
160
|
+
unestablished?: string[];
|
|
154
161
|
/** Present only when over budget: the arithmetic of getting back under it. */
|
|
155
162
|
fit?: BudgetFit;
|
|
156
163
|
};
|
package/dist/audit/audit.js
CHANGED
|
@@ -319,12 +319,21 @@ export function buildReport(configs, measured, opts = {}) {
|
|
|
319
319
|
// The worst config is the gate: passing because your *lightest* client fits
|
|
320
320
|
// would be a green check on a session you don't run.
|
|
321
321
|
const worst = results[0];
|
|
322
|
+
// A total is only a ceiling to compare against if it is the whole cost. A
|
|
323
|
+
// server that failed to start contributes 0, so the stack reads lighter
|
|
324
|
+
// than it is and the budget passes on a number that is missing a server —
|
|
325
|
+
// exactly the PR the README says this gate catches. The server-level gate
|
|
326
|
+
// (core/server-diff.ts) already refuses this; so does this one now.
|
|
327
|
+
const unestablished = results.flatMap((c) => c.skipped.filter((s) => s.status !== 'remote-not-measurable').map((s) => `${c.source}: ${s.name} (${s.status})`));
|
|
322
328
|
const over = (worst?.totalTokens ?? 0) > opts.budget;
|
|
323
329
|
report.budget = {
|
|
324
330
|
limit: opts.budget,
|
|
325
331
|
worstTotal: worst?.totalTokens ?? 0,
|
|
326
332
|
worstSource: worst?.source ?? '(none)',
|
|
327
|
-
over,
|
|
333
|
+
over: over || unestablished.length > 0,
|
|
334
|
+
// Named so the reader knows which way the number is wrong: the total
|
|
335
|
+
// understates by however much these cost, which nobody knows.
|
|
336
|
+
unestablished: unestablished.length ? unestablished : undefined,
|
|
328
337
|
fit: over && worst ? planBudgetFit(worst, opts.budget) : undefined,
|
|
329
338
|
};
|
|
330
339
|
}
|
|
@@ -739,6 +748,15 @@ export function formatReport(report) {
|
|
|
739
748
|
const headroom = b.limit - b.worstTotal;
|
|
740
749
|
lines.push(`budget ok: ${n(b.worstTotal)} ≤ ${n(b.limit)} — ${n(headroom)} to spare`);
|
|
741
750
|
}
|
|
751
|
+
else if (b.unestablished && b.worstTotal <= b.limit) {
|
|
752
|
+
// Under the line on what was measured, but not everything was: say which
|
|
753
|
+
// way the number is wrong rather than passing on it.
|
|
754
|
+
lines.push(`BUDGET FAIL: ${n(b.worstTotal)} ≤ ${n(b.limit)}, but the budget could not be checked against the ` +
|
|
755
|
+
`whole stack — ${b.unestablished.length} server(s) produced no number, so this total understates ` +
|
|
756
|
+
`by however much they cost:`);
|
|
757
|
+
for (const u of b.unestablished)
|
|
758
|
+
lines.push(` ${u}`);
|
|
759
|
+
}
|
|
742
760
|
else {
|
|
743
761
|
lines.push(`BUDGET FAIL: ${n(b.worstTotal)} > ${n(b.limit)} (${b.worstSource})`);
|
|
744
762
|
const fit = b.fit;
|
package/dist/audit/diff.js
CHANGED
|
@@ -21,6 +21,17 @@ export function parseBaselineReport(text) {
|
|
|
21
21
|
if (!c || typeof c !== 'object' || typeof c.source !== 'string') {
|
|
22
22
|
return { report: null, problem: 'baseline has a config entry without a source path' };
|
|
23
23
|
}
|
|
24
|
+
// The number the diff subtracts from. Unchecked, a hand-trimmed or
|
|
25
|
+
// jq-filtered baseline yields `after - undefined === NaN`, and `typeof NaN`
|
|
26
|
+
// is 'number' — so the gate reported an increase of 0 and passed. A
|
|
27
|
+
// baseline that cannot be read is never "no change".
|
|
28
|
+
const total = c.totalTokens;
|
|
29
|
+
if (typeof total !== 'number' || !Number.isFinite(total)) {
|
|
30
|
+
return {
|
|
31
|
+
report: null,
|
|
32
|
+
problem: `baseline config ${c.source} has no usable totalTokens — is it the output of \`audit --json\`?`,
|
|
33
|
+
};
|
|
34
|
+
}
|
|
24
35
|
}
|
|
25
36
|
return { report: doc };
|
|
26
37
|
}
|
|
@@ -69,9 +80,25 @@ export function diffConfig(before, after, matchedBy) {
|
|
|
69
80
|
continue;
|
|
70
81
|
}
|
|
71
82
|
if (!bs && as) {
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
83
|
+
if (as.tokens === null) {
|
|
84
|
+
// Unknown, not zero — so the total this diff subtracts from does not
|
|
85
|
+
// contain it and the increase understates. Marking the config inexact
|
|
86
|
+
// is what stops `--max-increase` from passing: a server added in a PR
|
|
87
|
+
// and unmeasurable in CI (no credential — the ordinary case) otherwise
|
|
88
|
+
// reads as +0 and clears even a zero-token allowance.
|
|
89
|
+
exact = false;
|
|
90
|
+
servers.push({
|
|
91
|
+
name,
|
|
92
|
+
kind: 'added',
|
|
93
|
+
before: null,
|
|
94
|
+
after: null,
|
|
95
|
+
delta: null,
|
|
96
|
+
note: 'added but not measurable — its cost is unknown, not zero',
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
else {
|
|
100
|
+
servers.push({ name, kind: 'added', before: null, after: as.tokens, delta: as.tokens });
|
|
101
|
+
}
|
|
75
102
|
continue;
|
|
76
103
|
}
|
|
77
104
|
if (!bs || !as)
|
|
@@ -298,13 +325,17 @@ export function evaluateIncreaseGate(diff, limit) {
|
|
|
298
325
|
reasons.push(`${c.source}: no baseline to check its ${n(c.afterTotal)} tokens against`);
|
|
299
326
|
}
|
|
300
327
|
else if (!c.exact) {
|
|
301
|
-
reasons.push(`${c.source}: a server
|
|
328
|
+
reasons.push(`${c.source}: a server's cost is not established on both sides, so the change could not be established exactly`);
|
|
302
329
|
}
|
|
303
330
|
}
|
|
304
331
|
for (const d of diff.droppedConfigs) {
|
|
305
332
|
reasons.push(`${d.source}: covered by the baseline and not found in this run`);
|
|
306
333
|
}
|
|
307
|
-
|
|
334
|
+
// `Number.isFinite`, not `typeof === 'number'`: NaN passes the latter and
|
|
335
|
+
// then compares false against every limit, which is a silent pass.
|
|
336
|
+
const anyDelta = diff.configs.some((c) => Number.isFinite(c.delta));
|
|
337
|
+
const worst = diff.worstIncrease?.delta;
|
|
338
|
+
const increase = Number.isFinite(worst) ? worst : anyDelta ? 0 : null;
|
|
308
339
|
if (reasons.length === 0 && increase !== null && increase > limit) {
|
|
309
340
|
reasons.push(`${diff.worstIncrease.source}: +${n(increase)} tokens per request, over the ${n(limit)} allowed`);
|
|
310
341
|
}
|
package/dist/cli.d.ts
CHANGED
|
@@ -27,6 +27,11 @@ export declare function unknownFlags(argv: string[], spec: {
|
|
|
27
27
|
value: string[];
|
|
28
28
|
boolean: string[];
|
|
29
29
|
}): string[];
|
|
30
|
+
/** Every flag name a command accepts — what tells a value apart from the next flag. */
|
|
31
|
+
export declare const knownFlagNames: (spec: {
|
|
32
|
+
value: string[];
|
|
33
|
+
boolean: string[];
|
|
34
|
+
}) => Set<string>;
|
|
30
35
|
/**
|
|
31
36
|
* Every value a value-taking flag was given, in either accepted spelling:
|
|
32
37
|
* `--flag value` and `--flag=value`.
|
|
@@ -37,9 +42,9 @@ export declare function unknownFlags(argv: string[], spec: {
|
|
|
37
42
|
* matched the bare token, so the gate it asked for silently did not run and the
|
|
38
43
|
* command exited 0 — a green check on a check that never happened.
|
|
39
44
|
*/
|
|
40
|
-
export declare function flagValues(argv: string[], name: string): string[];
|
|
45
|
+
export declare function flagValues(argv: string[], name: string, known?: Set<string>): string[];
|
|
41
46
|
/** The last value given for a flag, or undefined when the flag is absent. */
|
|
42
|
-
export declare function flagValue(argv: string[], name: string): string | undefined;
|
|
47
|
+
export declare function flagValue(argv: string[], name: string, known?: Set<string>): string | undefined;
|
|
43
48
|
/**
|
|
44
49
|
* Value-taking flags that appear with no usable value.
|
|
45
50
|
*
|
package/dist/cli.js
CHANGED
|
@@ -85,6 +85,21 @@ export function unknownFlags(argv, spec) {
|
|
|
85
85
|
}
|
|
86
86
|
return unknown;
|
|
87
87
|
}
|
|
88
|
+
/**
|
|
89
|
+
* Whether a token is another flag of *this* command, rather than a value that
|
|
90
|
+
* merely looks like one.
|
|
91
|
+
*
|
|
92
|
+
* The distinction is load-bearing: `--command "--weird"` is a legitimate launch
|
|
93
|
+
* command this CLI has always accepted, while `--max-increase --json` is a
|
|
94
|
+
* value slot swallowed by the next flag. Deciding on the `--` prefix alone
|
|
95
|
+
* cannot tell them apart; deciding against the command's own flag list can, and
|
|
96
|
+
* the list is already declared at every call site.
|
|
97
|
+
*/
|
|
98
|
+
function isKnownFlagToken(tok, known) {
|
|
99
|
+
return tok.startsWith('--') && known.has(tok.slice(2).split('=')[0]);
|
|
100
|
+
}
|
|
101
|
+
/** Every flag name a command accepts — what tells a value apart from the next flag. */
|
|
102
|
+
export const knownFlagNames = (spec) => new Set([...spec.value, ...spec.boolean]);
|
|
88
103
|
/**
|
|
89
104
|
* Every value a value-taking flag was given, in either accepted spelling:
|
|
90
105
|
* `--flag value` and `--flag=value`.
|
|
@@ -95,15 +110,15 @@ export function unknownFlags(argv, spec) {
|
|
|
95
110
|
* matched the bare token, so the gate it asked for silently did not run and the
|
|
96
111
|
* command exited 0 — a green check on a check that never happened.
|
|
97
112
|
*/
|
|
98
|
-
export function flagValues(argv, name) {
|
|
113
|
+
export function flagValues(argv, name, known = new Set()) {
|
|
99
114
|
const out = [];
|
|
100
115
|
for (let i = 0; i < argv.length; i++) {
|
|
101
116
|
const tok = argv[i];
|
|
102
117
|
if (tok === `--${name}`) {
|
|
103
118
|
const next = argv[i + 1];
|
|
104
|
-
//
|
|
105
|
-
// caught by `valuelessFlags`, and
|
|
106
|
-
if (next !== undefined && !next
|
|
119
|
+
// Another flag of this command is not this flag's value; that case is a
|
|
120
|
+
// usage error, caught by `valuelessFlags`, and never read as a value.
|
|
121
|
+
if (next !== undefined && !isKnownFlagToken(next, known))
|
|
107
122
|
out.push(next);
|
|
108
123
|
continue;
|
|
109
124
|
}
|
|
@@ -113,8 +128,8 @@ export function flagValues(argv, name) {
|
|
|
113
128
|
return out;
|
|
114
129
|
}
|
|
115
130
|
/** The last value given for a flag, or undefined when the flag is absent. */
|
|
116
|
-
export function flagValue(argv, name) {
|
|
117
|
-
const values = flagValues(argv, name);
|
|
131
|
+
export function flagValue(argv, name, known = new Set()) {
|
|
132
|
+
const values = flagValues(argv, name, known);
|
|
118
133
|
return values.length ? values[values.length - 1] : undefined;
|
|
119
134
|
}
|
|
120
135
|
/**
|
|
@@ -128,6 +143,7 @@ export function flagValue(argv, name) {
|
|
|
128
143
|
* so it is refused in the same place and with the same severity.
|
|
129
144
|
*/
|
|
130
145
|
export function valuelessFlags(argv, spec) {
|
|
146
|
+
const known = knownFlagNames(spec);
|
|
131
147
|
const bad = [];
|
|
132
148
|
for (let i = 0; i < argv.length; i++) {
|
|
133
149
|
const tok = argv[i];
|
|
@@ -142,7 +158,7 @@ export function valuelessFlags(argv, spec) {
|
|
|
142
158
|
continue;
|
|
143
159
|
}
|
|
144
160
|
const next = argv[i + 1];
|
|
145
|
-
if (next === undefined || next
|
|
161
|
+
if (next === undefined || isKnownFlagToken(next, known))
|
|
146
162
|
bad.push(`--${name}`);
|
|
147
163
|
else
|
|
148
164
|
i++; // consume the value, so `--command "--weird"` is not re-read as a flag
|
|
@@ -169,7 +185,7 @@ function rejectUnknownFlags(cmd, argv, spec) {
|
|
|
169
185
|
}
|
|
170
186
|
const [, , cmd, ...rest] = process.argv;
|
|
171
187
|
if (cmd === 'audit') {
|
|
172
|
-
|
|
188
|
+
const spec = {
|
|
173
189
|
value: [
|
|
174
190
|
'config',
|
|
175
191
|
'budget',
|
|
@@ -183,9 +199,14 @@ if (cmd === 'audit') {
|
|
|
183
199
|
'capture-index-url',
|
|
184
200
|
],
|
|
185
201
|
boolean: ['json', 'docker', 'claude', 'suggest', 'changed'],
|
|
186
|
-
}
|
|
187
|
-
|
|
188
|
-
|
|
202
|
+
};
|
|
203
|
+
rejectUnknownFlags('audit', rest, spec);
|
|
204
|
+
// The same flag list the rejection used, so a value that merely looks like a
|
|
205
|
+
// flag (`--command "--weird"`) is told apart from a value slot swallowed by
|
|
206
|
+
// the next flag.
|
|
207
|
+
const known = knownFlagNames(spec);
|
|
208
|
+
const argOf = (name) => flagValue(rest, name, known);
|
|
209
|
+
const all = (name) => flagValues(rest, name, known);
|
|
189
210
|
const json = rest.includes('--json');
|
|
190
211
|
const numeric = (name) => {
|
|
191
212
|
const raw = argOf(name);
|
|
@@ -315,9 +336,10 @@ if (cmd === 'audit') {
|
|
|
315
336
|
process.exit(report.budget?.over || report.increaseGate?.pass === false ? 1 : 0);
|
|
316
337
|
}
|
|
317
338
|
else if (cmd === 'verify') {
|
|
318
|
-
|
|
339
|
+
const spec = { value: ['remote'], boolean: ['json'] };
|
|
340
|
+
rejectUnknownFlags('verify', rest, spec);
|
|
319
341
|
const json = rest.includes('--json');
|
|
320
|
-
const remoteUrl = flagValue(rest, 'remote');
|
|
342
|
+
const remoteUrl = flagValue(rest, 'remote', knownFlagNames(spec));
|
|
321
343
|
const path = rest.find((a) => !a.startsWith('--') && a !== remoteUrl);
|
|
322
344
|
if (!remoteUrl && !path) {
|
|
323
345
|
console.error('usage: mcp-context-cost verify <measurement.json> [--json]');
|
|
@@ -361,11 +383,13 @@ else if (cmd === 'verify') {
|
|
|
361
383
|
process.exit(1);
|
|
362
384
|
}
|
|
363
385
|
else if (cmd === 'measure') {
|
|
364
|
-
|
|
386
|
+
const spec = {
|
|
365
387
|
value: ['name', 'command', 'remote', 'timeout', 'docker-image', 'baseline', 'max-increase', 'budget'],
|
|
366
388
|
boolean: ['docker'],
|
|
367
|
-
}
|
|
368
|
-
|
|
389
|
+
};
|
|
390
|
+
rejectUnknownFlags('measure', rest, spec);
|
|
391
|
+
const known = knownFlagNames(spec);
|
|
392
|
+
const argOf = (name) => flagValue(rest, name, known);
|
|
369
393
|
const command = argOf('command');
|
|
370
394
|
const remoteUrl = argOf('remote');
|
|
371
395
|
if (remoteUrl && !/^https?:\/\//i.test(remoteUrl)) {
|
package/dist/core/regression.js
CHANGED
|
@@ -162,13 +162,36 @@ export function latestChange(server, rows, vectors) {
|
|
|
162
162
|
if (deltaTokens === 0 && deltaTools === 0)
|
|
163
163
|
return null;
|
|
164
164
|
const deltaPct = from.tokens > 0 ? (deltaTokens / from.tokens) * 100 : 0;
|
|
165
|
-
// Attribution needs both sides on record
|
|
166
|
-
//
|
|
165
|
+
// Attribution needs both sides on record. Matched by *cost as of that date*,
|
|
166
|
+
// not by date equality: vectors are deduped by capture and keep the first
|
|
167
|
+
// date a capture was seen, while `from` is the last row of the previous
|
|
168
|
+
// plateau — so the two dates coincide only when the previous cost was
|
|
169
|
+
// measured exactly once. With weekly sweeps and less frequent releases they
|
|
170
|
+
// almost never do, and a date-equality join therefore reported "only one of
|
|
171
|
+
// the two captures is on record" while holding both. The vector in force on a
|
|
172
|
+
// given day is the newest one recorded on or before it.
|
|
167
173
|
let attribution = null;
|
|
168
|
-
const
|
|
169
|
-
|
|
170
|
-
|
|
174
|
+
const inForceOn = (date, tokensThatDay) => {
|
|
175
|
+
const upto = (vectors?.entries ?? []).filter((e) => e.date <= date);
|
|
176
|
+
// A same-day re-sweep replaces that day's history row but *appends* a
|
|
177
|
+
// capture, so one date can carry two. Only one of them is the one the row
|
|
178
|
+
// describes: prefer the capture whose total is the number history recorded.
|
|
179
|
+
const agreeing = upto.filter((e) => e.totalTokens === tokensThatDay);
|
|
180
|
+
const pool = agreeing.length > 0 ? agreeing : upto;
|
|
181
|
+
return pool.reduce((best, e) => (!best || e.date >= best.date ? e : best), undefined);
|
|
182
|
+
};
|
|
183
|
+
const fromVec = inForceOn(from.date, from.tokens);
|
|
184
|
+
const toVec = inForceOn(to.date, to.tokens);
|
|
185
|
+
// A vector only explains the row it agrees with. If the totals disagree the
|
|
186
|
+
// file does not cover this change — say so rather than attributing a delta
|
|
187
|
+
// to the wrong capture and publishing the mismatch as framing bytes.
|
|
188
|
+
if (fromVec &&
|
|
189
|
+
toVec &&
|
|
190
|
+
fromVec.canonicalSha256 !== toVec.canonicalSha256 &&
|
|
191
|
+
fromVec.totalTokens === from.tokens &&
|
|
192
|
+
toVec.totalTokens === to.tokens) {
|
|
171
193
|
attribution = attribute(fromVec, toVec, deltaTokens);
|
|
194
|
+
}
|
|
172
195
|
return {
|
|
173
196
|
server,
|
|
174
197
|
fromDate: from.date,
|
package/dist/sweep/dashboard.js
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'node:fs';
|
|
7
7
|
import { dirname, join } from 'node:path';
|
|
8
8
|
import { parse } from 'yaml';
|
|
9
|
+
import { isCurrent } from '../core/divergence.js';
|
|
9
10
|
import { bandColor, BAND_META } from '../core/bands.js';
|
|
10
11
|
import { parseHistory, plottableSeries } from './history.js';
|
|
11
12
|
/** Longest series a sparkline plots — a stat-tile trend, not a full chart. */
|
|
@@ -74,7 +75,12 @@ export function generateDashboard(root = process.cwd()) {
|
|
|
74
75
|
const meta = BAND_META[band];
|
|
75
76
|
const largest = [...m.tools].sort((a, b) => b.tokens - a.tokens)[0];
|
|
76
77
|
const pct = Math.max(1.2, (t / max) * 100);
|
|
77
|
-
|
|
78
|
+
// Gated on the capture it was computed from, exactly as the leaderboard
|
|
79
|
+
// gates it: four rows here were publishing a Claude cost derived from
|
|
80
|
+
// bytes that no longer exist, while leaderboard.md printed `—` for the
|
|
81
|
+
// same four.
|
|
82
|
+
const dRaw = dSrv[r.entry.name];
|
|
83
|
+
const div = isCurrent(dRaw, m.canonicalSha256 ?? null) ? dRaw : undefined;
|
|
78
84
|
const claudeTip = div ? ` · in a Claude request: ${fmt(div.claudeDelta)} tok` : '';
|
|
79
85
|
const series = seriesFor(r.entry.name);
|
|
80
86
|
const tokens = series.rows.map((h) => h.tokens);
|
|
@@ -107,10 +113,15 @@ export function generateDashboard(root = process.cwd()) {
|
|
|
107
113
|
const tableRows = measured
|
|
108
114
|
.map((r, i) => {
|
|
109
115
|
const m = r.m;
|
|
110
|
-
|
|
116
|
+
// Gated on the capture it was computed from, exactly as the leaderboard
|
|
117
|
+
// gates it: four rows here were publishing a Claude cost derived from
|
|
118
|
+
// bytes that no longer exist, while leaderboard.md printed `—` for the
|
|
119
|
+
// same four.
|
|
120
|
+
const dRaw = dSrv[r.entry.name];
|
|
121
|
+
const div = isCurrent(dRaw, m.canonicalSha256 ?? null) ? dRaw : undefined;
|
|
111
122
|
const tokens = seriesFor(r.entry.name).rows.map((h) => h.tokens);
|
|
112
123
|
const trend = tokens.length > 1
|
|
113
|
-
? `${tokens[tokens.length - 1] - tokens[0] >= 0 ? '+' : ''}${fmt(tokens[tokens.length - 1] - tokens[0])}
|
|
124
|
+
? `${tokens[tokens.length - 1] - tokens[0] >= 0 ? '+' : ''}${fmt(tokens[tokens.length - 1] - tokens[0])} over ${tokens.length} sweeps`
|
|
114
125
|
: '—';
|
|
115
126
|
return `<tr><td>${i + 1}</td><td>${esc(r.entry.name)}</td><td class="num">${fmt(m.totalTokens)}</td><td class="num">${div ? fmt(div.claudeDelta) : '—'}</td><td class="num">${esc(m.toolCount)}</td><td>${esc(BAND_META[bandColor(m.totalTokens)].label)}</td><td>${esc(r.entry.category)}</td><td class="num">${trend}</td></tr>`;
|
|
116
127
|
})
|
|
@@ -227,7 +238,7 @@ export function generateDashboard(root = process.cwd()) {
|
|
|
227
238
|
</div>
|
|
228
239
|
|
|
229
240
|
<h2>Leaderboard</h2>
|
|
230
|
-
<p class="h2sub">Tokens = o200k_base count of the canonical <code>tools/list</code> bytes — the wire payload. What a <em>Claude request</em> actually carries can differ sharply (
|
|
241
|
+
<p class="h2sub">Tokens = o200k_base count of the canonical <code>tools/list</code> bytes — the wire payload. What a <em>Claude request</em> actually carries can differ sharply (a server can spend most of its schema bytes on fields no Anthropic request sends, and the denser tokenizer can push the rest back up). The trend line plots tokens across every sweep date on record (oldest→newest, dot = current); servers with one sweep so far show no line yet, and a sweep taken under different isolation than the current one is left out rather than drawn as a change in the server. Hover a row for exact numbers; <a href="METHODOLOGY.html#claude-divergence">method</a>.</p>
|
|
231
242
|
<div class="board">
|
|
232
243
|
${barRows || '<p class="h2sub">Sweep in progress — first results land shortly.</p>'}
|
|
233
244
|
</div>
|
|
@@ -28,17 +28,22 @@ export interface PublishedStats {
|
|
|
28
28
|
/** Rows the leaderboard prints a claude number for: measured AND capture-current. */
|
|
29
29
|
currentCount: number;
|
|
30
30
|
heaviestClaudeName: string | null;
|
|
31
|
+
/** `claudeTokens` is null when the published row no longer matches the capture on disk. */
|
|
31
32
|
github: {
|
|
32
33
|
badgeTokens: number;
|
|
33
|
-
|
|
34
|
-
claudeTokens: number;
|
|
35
|
-
droppedPct: number;
|
|
34
|
+
claudeTokens: number | null;
|
|
36
35
|
};
|
|
37
36
|
notion: {
|
|
38
37
|
badgeTokens: number;
|
|
39
|
-
claudeTokens: number;
|
|
38
|
+
claudeTokens: number | null;
|
|
40
39
|
};
|
|
41
|
-
/**
|
|
40
|
+
/** The current row showing the largest field-selection effect, and its two counts. */
|
|
41
|
+
widest: {
|
|
42
|
+
server: string;
|
|
43
|
+
full: number;
|
|
44
|
+
mapped: number;
|
|
45
|
+
};
|
|
46
|
+
/** Field-selection share across the run's *current* rows, as fractions of the payload. */
|
|
42
47
|
shareMin: number;
|
|
43
48
|
shareMax: number;
|
|
44
49
|
/** claudeDelta / o200kFull across the run's rows that carry a number. */
|
|
@@ -75,25 +75,48 @@ export function computePublishedStats(entries, root = process.cwd()) {
|
|
|
75
75
|
}
|
|
76
76
|
if (!div)
|
|
77
77
|
throw new Error('results/divergence.json is missing — README states its numbers');
|
|
78
|
-
|
|
78
|
+
/**
|
|
79
|
+
* A divergence row only where it still describes the capture on disk.
|
|
80
|
+
*
|
|
81
|
+
* The staleness gate is the whole discipline of this column, and skipping it
|
|
82
|
+
* here is how README came to print two different costs for github on one
|
|
83
|
+
* page: 54,422 from a row computed against bytes that no longer existed,
|
|
84
|
+
* beside 54,622 from the measurement. `withClaude` below already applied
|
|
85
|
+
* `isCurrent` to the very same run — the rule guarded one number and not its
|
|
86
|
+
* neighbour.
|
|
87
|
+
*/
|
|
88
|
+
const currentDivRow = (name) => {
|
|
79
89
|
const d = div.servers[name];
|
|
80
90
|
if (!d)
|
|
81
91
|
throw new Error(`README's Claude table names ${name}, which is not in the divergence run`);
|
|
82
|
-
|
|
92
|
+
const onDisk = rows.find((r) => r.entry.name === name)?.m?.canonicalSha256 ?? null;
|
|
93
|
+
return isCurrent(d, onDisk) ? d : null;
|
|
83
94
|
};
|
|
84
|
-
|
|
85
|
-
const
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
95
|
+
/** The badge number comes from the measurement, never from a divergence row's copy of it. */
|
|
96
|
+
const badgeTokensOf = (name) => {
|
|
97
|
+
const r = measured.find((x) => x.entry.name === name);
|
|
98
|
+
if (!r)
|
|
99
|
+
throw new Error(`README's Claude table names ${name}, which has no current measurement`);
|
|
100
|
+
return r.m.totalTokens;
|
|
101
|
+
};
|
|
102
|
+
// Ranges are stated over the rows that are still current, for the same
|
|
103
|
+
// reason: a range whose endpoint comes from a superseded capture describes a
|
|
104
|
+
// set that no longer exists.
|
|
105
|
+
const currentRows = Object.entries(div.servers).filter(([name, r]) => isCurrent(r, rows.find((x) => x.entry.name === name)?.m?.canonicalSha256 ?? null));
|
|
106
|
+
const shares = currentRows
|
|
107
|
+
.map(([, r]) => fieldSelectionShare(r))
|
|
108
|
+
.filter((s) => s !== null && s >= 0);
|
|
109
|
+
const ratios = currentRows
|
|
110
|
+
.filter(([, r]) => typeof r.claudeDelta === 'number' && r.claudeDelta > 0 && r.o200kFull > 0)
|
|
111
|
+
.map(([, r]) => r.claudeDelta / r.o200kFull);
|
|
94
112
|
if (shares.length === 0 || ratios.length === 0) {
|
|
95
|
-
throw new Error('
|
|
113
|
+
throw new Error('no current divergence row — METHODOLOGY states ranges over them; run `npm run divergence`');
|
|
96
114
|
}
|
|
115
|
+
// The exemplar METHODOLOGY names for the field-selection effect is whichever
|
|
116
|
+
// current row shows it most, rather than a server hardcoded into the prose.
|
|
117
|
+
const widest = currentRows
|
|
118
|
+
.filter(([, r]) => (fieldSelectionShare(r) ?? -1) >= 0)
|
|
119
|
+
.sort((a, b) => (fieldSelectionShare(b[1]) ?? 0) - (fieldSelectionShare(a[1]) ?? 0))[0];
|
|
97
120
|
const withClaude = measured.filter((r) => isCurrent(div.servers[r.entry.name], r.m.canonicalSha256));
|
|
98
121
|
const heaviest = [...withClaude].sort((a, b) => div.servers[b.entry.name].claudeDelta - div.servers[a.entry.name].claudeDelta)[0];
|
|
99
122
|
const costlier = measured.filter((r) => {
|
|
@@ -118,13 +141,9 @@ export function computePublishedStats(entries, root = process.cwd()) {
|
|
|
118
141
|
runSize: Object.keys(div.servers).length,
|
|
119
142
|
currentCount: withClaude.length,
|
|
120
143
|
heaviestClaudeName: heaviest?.entry.name ?? null,
|
|
121
|
-
github: {
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
claudeTokens: github.claudeDelta,
|
|
125
|
-
droppedPct: Math.round(githubShare * 100),
|
|
126
|
-
},
|
|
127
|
-
notion: { badgeTokens: notion.o200kFull, claudeTokens: notion.claudeDelta },
|
|
144
|
+
github: { badgeTokens: badgeTokensOf('github'), claudeTokens: currentDivRow('github')?.claudeDelta ?? null },
|
|
145
|
+
notion: { badgeTokens: badgeTokensOf('notion'), claudeTokens: currentDivRow('notion')?.claudeDelta ?? null },
|
|
146
|
+
widest: { server: widest[0], full: widest[1].o200kFull, mapped: widest[1].o200kMapped },
|
|
128
147
|
shareMin: Math.min(...shares),
|
|
129
148
|
shareMax: Math.max(...shares),
|
|
130
149
|
ratioMin: Math.min(...ratios),
|
|
@@ -140,6 +159,8 @@ export function computePublishedStats(entries, root = process.cwd()) {
|
|
|
140
159
|
}
|
|
141
160
|
export const PAGE_FILES = ['README.md', 'docs/index.md', 'docs/METHODOLOGY.md'];
|
|
142
161
|
const fmt = (n) => n.toLocaleString('en-US');
|
|
162
|
+
/** A number, or the em-dash that means "no current measurement" everywhere else here. */
|
|
163
|
+
const q = (n) => (n === null ? '—' : fmt(n));
|
|
143
164
|
export const PAGE_CLAIMS = [
|
|
144
165
|
{
|
|
145
166
|
file: 'README.md',
|
|
@@ -210,14 +231,14 @@ export const PAGE_CLAIMS = [
|
|
|
210
231
|
{
|
|
211
232
|
file: 'README.md',
|
|
212
233
|
id: 'claude-table:github',
|
|
213
|
-
template: '| github | {n} | **{
|
|
214
|
-
values: (s) => [fmt(s.claude.github.badgeTokens),
|
|
234
|
+
template: '| github | {n} | **{q}** | most of the capture is `annotations`/`outputSchema` metadata Claude never sees |',
|
|
235
|
+
values: (s) => [fmt(s.claude.github.badgeTokens), q(s.claude.github.claudeTokens)],
|
|
215
236
|
},
|
|
216
237
|
{
|
|
217
238
|
file: 'README.md',
|
|
218
239
|
id: 'claude-table:notion',
|
|
219
|
-
template: '| notion | {n} | **{
|
|
220
|
-
values: (s) => [fmt(s.claude.notion.badgeTokens),
|
|
240
|
+
template: '| notion | {n} | **{q}** | almost no metadata to drop, so the tokenizer difference dominates |',
|
|
241
|
+
values: (s) => [fmt(s.claude.notion.badgeTokens), q(s.claude.notion.claudeTokens)],
|
|
221
242
|
},
|
|
222
243
|
{
|
|
223
244
|
file: 'README.md',
|
|
@@ -252,12 +273,15 @@ export const PAGE_CLAIMS = [
|
|
|
252
273
|
{
|
|
253
274
|
file: 'docs/METHODOLOGY.md',
|
|
254
275
|
id: 'divergence:share-range',
|
|
255
|
-
template: 'this removes between {f}% and **{f}%** of the payload (
|
|
276
|
+
template: 'this removes between {f}% and **{f}%** of the payload ({w}: {n} → {n} tokens).',
|
|
277
|
+
// The exemplar is whichever current row shows the effect most, not a server
|
|
278
|
+
// named in the prose — a hardcoded name goes stale the week it is re-swept.
|
|
256
279
|
values: (s) => [
|
|
257
280
|
(s.claude.shareMin * 100).toFixed(1),
|
|
258
281
|
(s.claude.shareMax * 100).toFixed(1),
|
|
259
|
-
|
|
260
|
-
fmt(s.claude.
|
|
282
|
+
s.claude.widest.server,
|
|
283
|
+
fmt(s.claude.widest.full),
|
|
284
|
+
fmt(s.claude.widest.mapped),
|
|
261
285
|
],
|
|
262
286
|
},
|
|
263
287
|
{
|
|
@@ -303,7 +327,7 @@ const escapeLiteral = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&').replace(/\
|
|
|
303
327
|
export function compileTemplate(template) {
|
|
304
328
|
let source = '';
|
|
305
329
|
let last = 0;
|
|
306
|
-
for (const slot of template.matchAll(/\{[
|
|
330
|
+
for (const slot of template.matchAll(/\{[ndwfq]\}/g)) {
|
|
307
331
|
source += escapeLiteral(template.slice(last, slot.index));
|
|
308
332
|
source +=
|
|
309
333
|
slot[0] === '{n}'
|
|
@@ -312,7 +336,11 @@ export function compileTemplate(template) {
|
|
|
312
336
|
? '(\\d+)'
|
|
313
337
|
: slot[0] === '{f}'
|
|
314
338
|
? '(\\d+\\.\\d+)'
|
|
315
|
-
:
|
|
339
|
+
: // `{q}` is a number that may not exist: the em-dash the leaderboard
|
|
340
|
+
// already prints for a row whose capture has moved on.
|
|
341
|
+
slot[0] === '{q}'
|
|
342
|
+
? '([\\d,]+|—)'
|
|
343
|
+
: '([A-Za-z0-9._-]+)';
|
|
316
344
|
last = slot.index + slot[0].length;
|
|
317
345
|
}
|
|
318
346
|
source += escapeLiteral(template.slice(last));
|
package/package.json
CHANGED