mcp-context-cost 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -2
- package/dist/audit/audit.d.ts +24 -0
- package/dist/audit/audit.js +53 -1
- package/dist/audit/run.d.ts +12 -0
- package/dist/audit/run.js +26 -0
- package/dist/cli.js +4 -2
- package/dist/core/tool-shape.d.ts +94 -0
- package/dist/core/tool-shape.js +106 -0
- package/dist/sweep/regen.js +6 -0
- package/dist/sweep/tool-shape.d.ts +3 -0
- package/dist/sweep/tool-shape.js +26 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -193,9 +193,26 @@ still match today and silence for the rest. Most installs will show a mix:
|
|
|
193
193
|
memory 9 2,378 4.2% —
|
|
194
194
|
```
|
|
195
195
|
|
|
196
|
+
Add `--suggest` to place each of your tools in the measured set's tool-shape distribution
|
|
197
|
+
([method](docs/METHODOLOGY.md#tool-shape)) and get advice only where the data can point at
|
|
198
|
+
something. Only descriptions draw advice — schemas are functional surface; descriptions are
|
|
199
|
+
prose every request carries — and only descriptions at or above the 90th percentile of the
|
|
200
|
+
1,150 measured tools:
|
|
201
|
+
|
|
202
|
+
```
|
|
203
|
+
suggest — descriptions at or above the 90th percentile of measured tools
|
|
204
|
+
(baseline 2026-09-03: 1,150 tools across 81 measured servers):
|
|
205
|
+
stub · wordy — 345 tokens: description 321 (p92), schema 14
|
|
206
|
+
rewriting the description toward the measured median (27) would recover ≈294 tokens on every request
|
|
207
|
+
1 of 2 tools sit inside the distribution — no advice where nothing is measurably unusual.
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
A config where nothing is out of distribution is told that in those words, and a baseline
|
|
211
|
+
that cannot be fetched is a named problem, never a silently skipped check.
|
|
212
|
+
|
|
196
213
|
Flags: `--json` (full report on stdout, progress on stderr), `--budget N`,
|
|
197
214
|
`--baseline <report.json>`, `--max-increase N`, `--context N` (default 200,000),
|
|
198
|
-
`--timeout ms`, `--concurrency N`, `--docker`, `--claude`.
|
|
215
|
+
`--timeout ms`, `--concurrency N`, `--docker`, `--claude`, `--suggest`.
|
|
199
216
|
|
|
200
217
|
## Where the numbers come from
|
|
201
218
|
|
|
@@ -329,7 +346,10 @@ color bands are frozen against the observed distribution of the first full sweep
|
|
|
329
346
|
|
|
330
347
|
## Status
|
|
331
348
|
|
|
332
|
-
Active. Every row carries the date of its own most recent measurement
|
|
349
|
+
Active. Every row carries the date of its own most recent measurement, and what the data
|
|
350
|
+
says as a whole is written up, dated, in
|
|
351
|
+
[The State of MCP Context Cost](https://athakur3.github.io/mcp-context-cost/state-of-mcp-context-cost)
|
|
352
|
+
(September 2026). Two
|
|
333
353
|
weekly jobs re-measure the set — the `memory` reference server on Mondays, and a rotating
|
|
334
354
|
sixth of the list on Wednesdays, so every row comes round within six weeks. Read each row's
|
|
335
355
|
date as the date it means, and don't take the cadence on trust — the build history is
|
package/dist/audit/audit.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { type DivergenceRun } from '../core/divergence.js';
|
|
2
|
+
import { type ToolShapeBaseline, type ToolSuggestion } from '../core/tool-shape.js';
|
|
2
3
|
import type { Measurement, MeasurementStatus, ToolMeasurement } from '../core/types.js';
|
|
3
4
|
import type { ConfiguredServer, LoadedConfig } from './config.js';
|
|
4
5
|
import { type DeferralVerdict, type ToolSearchEnv, type ToolSearchSource } from './deferral.js';
|
|
@@ -43,6 +44,19 @@ export interface TrimAdvice {
|
|
|
43
44
|
recoverableTokens: number;
|
|
44
45
|
recoverableShare: number;
|
|
45
46
|
}
|
|
47
|
+
/**
|
|
48
|
+
* `--suggest`: this config's tools placed in the measured set's tool-shape
|
|
49
|
+
* distribution. Only descriptions draw advice (schemas are functional surface;
|
|
50
|
+
* descriptions are prose every request carries), and only descriptions the
|
|
51
|
+
* baseline puts at or above the threshold percentile — a config where nothing
|
|
52
|
+
* is measurably unusual gets that said in those words, not advice invented to
|
|
53
|
+
* have some.
|
|
54
|
+
*/
|
|
55
|
+
export interface ConfigSuggestions {
|
|
56
|
+
/** Heaviest-recovery first. */
|
|
57
|
+
outOfDistribution: ToolSuggestion[];
|
|
58
|
+
checkedTools: number;
|
|
59
|
+
}
|
|
46
60
|
export interface AuditConfigResult {
|
|
47
61
|
client: string;
|
|
48
62
|
source: string;
|
|
@@ -54,6 +68,8 @@ export interface AuditConfigResult {
|
|
|
54
68
|
skipped: AuditServerResult[];
|
|
55
69
|
heaviestTools: HeaviestTool[];
|
|
56
70
|
trimAdvice: TrimAdvice | null;
|
|
71
|
+
/** Present only when `--suggest` ran with a usable baseline. */
|
|
72
|
+
suggestions?: ConfigSuggestions;
|
|
57
73
|
/**
|
|
58
74
|
* Whether this client loads the total up front or defers it, and — when the
|
|
59
75
|
* client decides that by a threshold — which side of it this stack is on.
|
|
@@ -128,6 +144,12 @@ export interface AuditReport {
|
|
|
128
144
|
model: string;
|
|
129
145
|
measuredAt: string;
|
|
130
146
|
};
|
|
147
|
+
/** Which published tool-shape baseline `--suggest` read its percentiles from. */
|
|
148
|
+
toolShape?: {
|
|
149
|
+
generatedAt: string;
|
|
150
|
+
toolCount: number;
|
|
151
|
+
serverCount: number;
|
|
152
|
+
};
|
|
131
153
|
/** Present only when a baseline report was supplied (`--baseline`). */
|
|
132
154
|
diff?: AuditDiff;
|
|
133
155
|
/** Present only when `--max-increase` was supplied alongside a baseline. */
|
|
@@ -160,6 +182,8 @@ export declare function buildReport(configs: LoadedConfig[], measured: Map<strin
|
|
|
160
182
|
generatedAt?: string;
|
|
161
183
|
/** Published `tools-delta/v1` run to join against (`--claude`); omit to skip the join. */
|
|
162
184
|
divergence?: DivergenceRun | null;
|
|
185
|
+
/** Published `tool-shape/v1` baseline (`--suggest`); omit to skip suggestions. */
|
|
186
|
+
toolShape?: ToolShapeBaseline | null;
|
|
163
187
|
/**
|
|
164
188
|
* The audited machine's SHELL tool-search variables. Passed in rather than
|
|
165
189
|
* read here so this stays pure and a report is reproducible from its
|
package/dist/audit/audit.js
CHANGED
|
@@ -14,10 +14,21 @@
|
|
|
14
14
|
*/
|
|
15
15
|
import { METHODOLOGY_VERSION } from '../core/canonical.js';
|
|
16
16
|
import { isCurrent } from '../core/divergence.js';
|
|
17
|
+
import { SUGGEST_DESCRIPTION_PERCENTILE, suggestFor, } from '../core/tool-shape.js';
|
|
17
18
|
import { evaluateDeferral, PUBLISHED_WIRE_TO_CLIENT_RATIO, SHELL_SOURCE, } from './deferral.js';
|
|
18
19
|
import { formatDiff, formatGate } from './diff.js';
|
|
19
20
|
export const DEFAULT_CONTEXT_WINDOW = 200_000;
|
|
20
21
|
const TRIM_TOOL_COUNT = 3;
|
|
22
|
+
function buildSuggestions(pool, baseline) {
|
|
23
|
+
const outOfDistribution = [];
|
|
24
|
+
for (const { server, t } of pool) {
|
|
25
|
+
const s = suggestFor(server, t, baseline);
|
|
26
|
+
if (s)
|
|
27
|
+
outOfDistribution.push(s);
|
|
28
|
+
}
|
|
29
|
+
outOfDistribution.sort((a, b) => b.approxRecoverableTokens - a.approxRecoverableTokens);
|
|
30
|
+
return { outOfDistribution, checkedTools: pool.length };
|
|
31
|
+
}
|
|
21
32
|
function buildTrimAdvice(sortedTools, totalTokens) {
|
|
22
33
|
if (totalTokens <= 0 || sortedTools.length < 2)
|
|
23
34
|
return null;
|
|
@@ -189,6 +200,7 @@ export function buildReport(configs, measured, opts = {}) {
|
|
|
189
200
|
const ok = [];
|
|
190
201
|
const skipped = [];
|
|
191
202
|
const tools = [];
|
|
203
|
+
const shapePool = [];
|
|
192
204
|
// Counted only for servers that put a number into the total: a twin that
|
|
193
205
|
// failed to launch is already a floor, and adds nothing to a sum.
|
|
194
206
|
let sharedHere = 0;
|
|
@@ -240,8 +252,11 @@ export function buildReport(configs, measured, opts = {}) {
|
|
|
240
252
|
claudeTokens: opts.divergence ? (isCurrent(divRow, m.canonicalSha256 ?? null) ? divRow.claudeDelta : null) : undefined,
|
|
241
253
|
notes: m.status === 'dynamic' ? m.notes : undefined,
|
|
242
254
|
});
|
|
243
|
-
for (const t of m.tools)
|
|
255
|
+
for (const t of m.tools) {
|
|
244
256
|
tools.push({ server: s.name, tool: t.name, tokens: t.tokens });
|
|
257
|
+
if (opts.toolShape)
|
|
258
|
+
shapePool.push({ server: s.name, t });
|
|
259
|
+
}
|
|
245
260
|
}
|
|
246
261
|
const totalTokens = ok.reduce((a, s) => a + (s.tokens ?? 0), 0);
|
|
247
262
|
const toolCount = ok.reduce((a, s) => a + (s.toolCount ?? 0), 0);
|
|
@@ -264,6 +279,7 @@ export function buildReport(configs, measured, opts = {}) {
|
|
|
264
279
|
skipped,
|
|
265
280
|
heaviestTools: tools.slice(0, 5),
|
|
266
281
|
trimAdvice: buildTrimAdvice(tools, totalTokens),
|
|
282
|
+
suggestions: opts.toolShape ? buildSuggestions(shapePool, opts.toolShape) : undefined,
|
|
267
283
|
};
|
|
268
284
|
built.push(result);
|
|
269
285
|
shared.set(result, sharedHere);
|
|
@@ -282,6 +298,13 @@ export function buildReport(configs, measured, opts = {}) {
|
|
|
282
298
|
if (opts.divergence) {
|
|
283
299
|
report.claudeDivergence = { model: opts.divergence.model, measuredAt: opts.divergence.measuredAt };
|
|
284
300
|
}
|
|
301
|
+
if (opts.toolShape) {
|
|
302
|
+
report.toolShape = {
|
|
303
|
+
generatedAt: opts.toolShape.generatedAt,
|
|
304
|
+
toolCount: opts.toolShape.toolCount,
|
|
305
|
+
serverCount: opts.toolShape.serverCount,
|
|
306
|
+
};
|
|
307
|
+
}
|
|
285
308
|
if (typeof opts.budget === 'number') {
|
|
286
309
|
// The worst config is the gate: passing because your *lightest* client fits
|
|
287
310
|
// would be a green check on a session you don't run.
|
|
@@ -617,6 +640,35 @@ export function formatReport(report) {
|
|
|
617
640
|
`(${names}) would recover ${n(cfg.trimAdvice.recoverableTokens)} tokens ` +
|
|
618
641
|
`(${pct(cfg.trimAdvice.recoverableShare)} of this config) — if your client supports per-tool filtering.`);
|
|
619
642
|
}
|
|
643
|
+
if (cfg.suggestions) {
|
|
644
|
+
const sg = cfg.suggestions;
|
|
645
|
+
const base = report.toolShape
|
|
646
|
+
? `baseline ${report.toolShape.generatedAt}: ${n(report.toolShape.toolCount)} tools across ` +
|
|
647
|
+
`${report.toolShape.serverCount} measured servers`
|
|
648
|
+
: 'published baseline';
|
|
649
|
+
lines.push('');
|
|
650
|
+
if (sg.outOfDistribution.length === 0) {
|
|
651
|
+
lines.push(` suggest: every description in this config sits inside the measured distribution — ` +
|
|
652
|
+
`nothing the data can point at (${sg.checkedTools} tools against ${base}).`);
|
|
653
|
+
}
|
|
654
|
+
else {
|
|
655
|
+
lines.push(` suggest — descriptions at or above the ${SUGGEST_DESCRIPTION_PERCENTILE}th percentile of ` +
|
|
656
|
+
`measured tools (${base}):`);
|
|
657
|
+
const shown = sg.outOfDistribution.slice(0, 8);
|
|
658
|
+
for (const s of shown) {
|
|
659
|
+
lines.push(` ${s.server} · ${s.tool} — ${n(s.tokens)} tokens: description ${n(s.descriptionTokens)} ` +
|
|
660
|
+
`(p${s.descriptionPercentile}), schema ${n(s.inputSchemaTokens)}`);
|
|
661
|
+
lines.push(` rewriting the description toward the measured median (${n(s.medianDescriptionTokens)}) ` +
|
|
662
|
+
`would recover ≈${n(s.approxRecoverableTokens)} tokens on every request`);
|
|
663
|
+
}
|
|
664
|
+
if (sg.outOfDistribution.length > shown.length) {
|
|
665
|
+
lines.push(` …and ${sg.outOfDistribution.length - shown.length} more above the threshold.`);
|
|
666
|
+
}
|
|
667
|
+
const within = sg.checkedTools - sg.outOfDistribution.length;
|
|
668
|
+
lines.push(` ${within} of ${sg.checkedTools} tools sit inside the distribution — ` +
|
|
669
|
+
`no advice where nothing is measurably unusual.`);
|
|
670
|
+
}
|
|
671
|
+
}
|
|
620
672
|
if (cfg.skipped.length) {
|
|
621
673
|
lines.push('');
|
|
622
674
|
lines.push(' not measured');
|
package/dist/audit/run.d.ts
CHANGED
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
import type { Measurement } from '../core/types.js';
|
|
2
2
|
import { type DivergenceRun } from '../core/divergence.js';
|
|
3
|
+
import { type ToolShapeBaseline } from '../core/tool-shape.js';
|
|
3
4
|
import { type AuditReport } from './audit.js';
|
|
4
5
|
import { type ToolSearchEnv, type ToolSearchSource } from './deferral.js';
|
|
5
6
|
import { type LoadedConfig } from './config.js';
|
|
6
7
|
/** Where the published `tools-delta/v1` run lives when `--claude` doesn't override it. */
|
|
7
8
|
export declare const DEFAULT_DIVERGENCE_URL = "https://raw.githubusercontent.com/athakur3/mcp-context-cost/main/results/divergence.json";
|
|
9
|
+
/** Where the published `tool-shape/v1` baseline lives when `--suggest` doesn't override it. */
|
|
10
|
+
export declare const DEFAULT_TOOL_SHAPE_URL = "https://raw.githubusercontent.com/athakur3/mcp-context-cost/main/results/tool-shape.json";
|
|
8
11
|
export interface AuditOptions {
|
|
9
12
|
/** Explicit config path(s); when empty, every known client location is tried. */
|
|
10
13
|
configPaths?: string[];
|
|
@@ -19,6 +22,10 @@ export interface AuditOptions {
|
|
|
19
22
|
claude?: boolean;
|
|
20
23
|
/** Override the divergence.json source — mainly for tests and self-hosted mirrors. */
|
|
21
24
|
divergenceUrl?: string;
|
|
25
|
+
/** Place this config's tools in the published tool-shape distribution and advise where the data can. */
|
|
26
|
+
suggest?: boolean;
|
|
27
|
+
/** Override the tool-shape.json source — mainly for tests and self-hosted mirrors. */
|
|
28
|
+
toolShapeUrl?: string;
|
|
22
29
|
/**
|
|
23
30
|
* The tool-search variables as this process's SHELL has them. Defaults to
|
|
24
31
|
* this process's environment. Overridable so a test can state a machine
|
|
@@ -33,6 +40,11 @@ export interface AuditOptions {
|
|
|
33
40
|
settings?: ToolSearchSource[];
|
|
34
41
|
onProgress?: (name: string, done: number, total: number) => void;
|
|
35
42
|
}
|
|
43
|
+
/** Fetch and parse the published tool-shape baseline. Never throws: a failure is a report problem, not a crash. */
|
|
44
|
+
export declare function fetchToolShape(url: string): Promise<{
|
|
45
|
+
baseline: ToolShapeBaseline | null;
|
|
46
|
+
problem?: string;
|
|
47
|
+
}>;
|
|
36
48
|
/** Fetch and parse the published divergence run. Never throws: a failure is a report problem, not a crash. */
|
|
37
49
|
export declare function fetchDivergence(url: string): Promise<{
|
|
38
50
|
run: DivergenceRun | null;
|
package/dist/audit/run.js
CHANGED
|
@@ -7,11 +7,27 @@
|
|
|
7
7
|
import { homedir } from 'node:os';
|
|
8
8
|
import { measureServer } from '../sweep/run.js';
|
|
9
9
|
import { parseDivergence } from '../core/divergence.js';
|
|
10
|
+
import { parseToolShapeBaseline } from '../core/tool-shape.js';
|
|
10
11
|
import { buildReport, serverKey } from './audit.js';
|
|
11
12
|
import { toolSearchEnv } from './deferral.js';
|
|
12
13
|
import { configCandidates, loadConfigs, loadSettingsSources, settingsCandidates, } from './config.js';
|
|
13
14
|
/** Where the published `tools-delta/v1` run lives when `--claude` doesn't override it. */
|
|
14
15
|
export const DEFAULT_DIVERGENCE_URL = 'https://raw.githubusercontent.com/athakur3/mcp-context-cost/main/results/divergence.json';
|
|
16
|
+
/** Where the published `tool-shape/v1` baseline lives when `--suggest` doesn't override it. */
|
|
17
|
+
export const DEFAULT_TOOL_SHAPE_URL = 'https://raw.githubusercontent.com/athakur3/mcp-context-cost/main/results/tool-shape.json';
|
|
18
|
+
/** Fetch and parse the published tool-shape baseline. Never throws: a failure is a report problem, not a crash. */
|
|
19
|
+
export async function fetchToolShape(url) {
|
|
20
|
+
try {
|
|
21
|
+
const res = await fetch(url, { signal: AbortSignal.timeout(15_000) });
|
|
22
|
+
if (!res.ok)
|
|
23
|
+
return { baseline: null, problem: `tool shape: HTTP ${res.status} fetching ${url}` };
|
|
24
|
+
const baseline = parseToolShapeBaseline(await res.text());
|
|
25
|
+
return baseline ? { baseline } : { baseline: null, problem: `tool shape: malformed data at ${url}` };
|
|
26
|
+
}
|
|
27
|
+
catch (e) {
|
|
28
|
+
return { baseline: null, problem: `tool shape: failed to fetch ${url}: ${e.message}` };
|
|
29
|
+
}
|
|
30
|
+
}
|
|
15
31
|
/** Fetch and parse the published divergence run. Never throws: a failure is a report problem, not a crash. */
|
|
16
32
|
export async function fetchDivergence(url) {
|
|
17
33
|
try {
|
|
@@ -89,14 +105,24 @@ export async function runAudit(opts = {}) {
|
|
|
89
105
|
divergence = fetched.run;
|
|
90
106
|
divergenceProblem = fetched.problem;
|
|
91
107
|
}
|
|
108
|
+
let toolShape = null;
|
|
109
|
+
let toolShapeProblem;
|
|
110
|
+
if (opts.suggest) {
|
|
111
|
+
const fetched = await fetchToolShape(opts.toolShapeUrl ?? DEFAULT_TOOL_SHAPE_URL);
|
|
112
|
+
toolShape = fetched.baseline;
|
|
113
|
+
toolShapeProblem = fetched.problem;
|
|
114
|
+
}
|
|
92
115
|
const report = buildReport(configs, measured, {
|
|
93
116
|
contextWindow: opts.contextWindow,
|
|
94
117
|
budget: opts.budget,
|
|
95
118
|
divergence,
|
|
119
|
+
toolShape,
|
|
96
120
|
env: opts.env ?? toolSearchEnv(process.env),
|
|
97
121
|
settings: opts.settings ?? discoverSettings(opts),
|
|
98
122
|
});
|
|
99
123
|
if (divergenceProblem)
|
|
100
124
|
report.problems.push(divergenceProblem);
|
|
125
|
+
if (toolShapeProblem)
|
|
126
|
+
report.problems.push(toolShapeProblem);
|
|
101
127
|
return report;
|
|
102
128
|
}
|
package/dist/cli.js
CHANGED
|
@@ -99,8 +99,8 @@ function rejectUnknownFlags(cmd, argv, spec) {
|
|
|
99
99
|
const [, , cmd, ...rest] = process.argv;
|
|
100
100
|
if (cmd === 'audit') {
|
|
101
101
|
rejectUnknownFlags('audit', rest, {
|
|
102
|
-
value: ['config', 'budget', 'baseline', 'max-increase', 'context', 'timeout', 'concurrency', 'divergence-url'],
|
|
103
|
-
boolean: ['json', 'docker', 'claude'],
|
|
102
|
+
value: ['config', 'budget', 'baseline', 'max-increase', 'context', 'timeout', 'concurrency', 'divergence-url', 'tool-shape-url'],
|
|
103
|
+
boolean: ['json', 'docker', 'claude', 'suggest'],
|
|
104
104
|
});
|
|
105
105
|
const argOf = (name) => {
|
|
106
106
|
const i = rest.indexOf(`--${name}`);
|
|
@@ -171,6 +171,8 @@ if (cmd === 'audit') {
|
|
|
171
171
|
docker: rest.includes('--docker'),
|
|
172
172
|
claude: rest.includes('--claude'),
|
|
173
173
|
divergenceUrl: argOf('divergence-url'),
|
|
174
|
+
suggest: rest.includes('--suggest'),
|
|
175
|
+
toolShapeUrl: argOf('tool-shape-url'),
|
|
174
176
|
// Progress goes to stderr so `--json` stdout stays a single parseable object.
|
|
175
177
|
onProgress: json ? undefined : (name, done, total) => process.stderr.write(` [${done}/${total}] ${name}\n`),
|
|
176
178
|
});
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tool shape — where a tool's tokens actually are, and when that is worth
|
|
3
|
+
* saying anything about.
|
|
4
|
+
*
|
|
5
|
+
* Every published measurement already splits each tool into whole /
|
|
6
|
+
* description / input-schema token counts. This module turns the measured
|
|
7
|
+
* set's split into a baseline — a quantile table over every measured tool —
|
|
8
|
+
* so that `audit --suggest` can say something no lone number can: whether a
|
|
9
|
+
* tool's description is *unusually* heavy, against the population of tools
|
|
10
|
+
* people actually ship, and roughly what trimming it toward normal would
|
|
11
|
+
* recover.
|
|
12
|
+
*
|
|
13
|
+
* Two disciplines, both this project's usual ones. Advice is only ever about
|
|
14
|
+
* descriptions: a schema is functional surface and trimming it changes what
|
|
15
|
+
* the tool can do, while a description is prose about the tool and trimming
|
|
16
|
+
* it changes only how much every request pays to carry it. And advice is only
|
|
17
|
+
* given where the data can point at something: a tool inside the measured
|
|
18
|
+
* distribution gets silence, said in those words, not a suggestion invented
|
|
19
|
+
* to have one. The threshold is a named constant, the baseline is a dated,
|
|
20
|
+
* re-derivable artifact regenerated from the same published measurements as
|
|
21
|
+
* everything else, and the recovery figure is marked approximate because
|
|
22
|
+
* token boundaries make component counts sum only approximately.
|
|
23
|
+
*
|
|
24
|
+
* Versioned independently of the o200k methodology, like every column before
|
|
25
|
+
* it: `tool-shape/v1` adds an advisory reading and moves no `totalTokens` and
|
|
26
|
+
* no canonical hash.
|
|
27
|
+
*/
|
|
28
|
+
import type { ToolMeasurement } from './types.js';
|
|
29
|
+
/** Method identifier, versioned independently of METHODOLOGY_VERSION. */
|
|
30
|
+
export declare const TOOL_SHAPE_METHOD = "tool-shape/v1";
|
|
31
|
+
/**
|
|
32
|
+
* A description earns a suggestion only at or above this percentile of the
|
|
33
|
+
* measured distribution. 90 keeps advice rare and confident: nine of ten
|
|
34
|
+
* measured tools are "normal" by construction, and the advice names the exact
|
|
35
|
+
* percentile it fired at rather than hiding behind the threshold.
|
|
36
|
+
*/
|
|
37
|
+
export declare const SUGGEST_DESCRIPTION_PERCENTILE = 90;
|
|
38
|
+
export interface ToolShapeBaseline {
|
|
39
|
+
method: string;
|
|
40
|
+
methodologyVersion: string;
|
|
41
|
+
/** UTC day the baseline was derived (YYYY-MM-DD). */
|
|
42
|
+
generatedAt: string;
|
|
43
|
+
/** How many measured servers and tools the quantiles were derived from. */
|
|
44
|
+
serverCount: number;
|
|
45
|
+
toolCount: number;
|
|
46
|
+
/**
|
|
47
|
+
* Nearest-rank quantile tables, 101 values each (q[0] = min … q[100] = max),
|
|
48
|
+
* in o200k tokens. Published whole so a reader can re-derive any percentile
|
|
49
|
+
* claim from the same table the tool used.
|
|
50
|
+
*/
|
|
51
|
+
quantiles: {
|
|
52
|
+
tokens: number[];
|
|
53
|
+
descriptionTokens: number[];
|
|
54
|
+
inputSchemaTokens: number[];
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
/** Nearest-rank quantile table over `values` — same rank rule as the badge-band percentiles. */
|
|
58
|
+
export declare function quantileTable(values: number[]): number[];
|
|
59
|
+
/**
|
|
60
|
+
* The largest percentile whose quantile does not exceed `value` — read
|
|
61
|
+
* directly off the published table, so "heavier than P% of measured tools" is
|
|
62
|
+
* checkable by anyone holding the same JSON.
|
|
63
|
+
*/
|
|
64
|
+
export declare function percentileOf(quantiles: number[], value: number): number;
|
|
65
|
+
export declare function buildToolShapeBaseline(tools: ToolMeasurement[], meta: {
|
|
66
|
+
serverCount: number;
|
|
67
|
+
generatedAt?: string;
|
|
68
|
+
methodologyVersion: string;
|
|
69
|
+
}): ToolShapeBaseline;
|
|
70
|
+
export declare function parseToolShapeBaseline(text: string): ToolShapeBaseline | null;
|
|
71
|
+
export interface ToolSuggestion {
|
|
72
|
+
server: string;
|
|
73
|
+
tool: string;
|
|
74
|
+
tokens: number;
|
|
75
|
+
descriptionTokens: number;
|
|
76
|
+
inputSchemaTokens: number;
|
|
77
|
+
/** Where this description sits in the measured distribution (0–100). */
|
|
78
|
+
descriptionPercentile: number;
|
|
79
|
+
/** The measured set's median description, the "normal" being suggested toward. */
|
|
80
|
+
medianDescriptionTokens: number;
|
|
81
|
+
/**
|
|
82
|
+
* ≈ descriptionTokens − median. Approximate by construction: component
|
|
83
|
+
* counts do not sum exactly to the whole, and a rewritten description
|
|
84
|
+
* tokenizes as itself, not as an arithmetic difference.
|
|
85
|
+
*/
|
|
86
|
+
approxRecoverableTokens: number;
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* The suggestion for one tool, or null — null is the common and correct case.
|
|
90
|
+
* Null when the measurement predates component counts, when the description
|
|
91
|
+
* sits below the threshold percentile, or when trimming toward the median
|
|
92
|
+
* would recover nothing.
|
|
93
|
+
*/
|
|
94
|
+
export declare function suggestFor(server: string, t: ToolMeasurement, baseline: ToolShapeBaseline): ToolSuggestion | null;
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
/** Method identifier, versioned independently of METHODOLOGY_VERSION. */
|
|
2
|
+
export const TOOL_SHAPE_METHOD = 'tool-shape/v1';
|
|
3
|
+
/**
|
|
4
|
+
* A description earns a suggestion only at or above this percentile of the
|
|
5
|
+
* measured distribution. 90 keeps advice rare and confident: nine of ten
|
|
6
|
+
* measured tools are "normal" by construction, and the advice names the exact
|
|
7
|
+
* percentile it fired at rather than hiding behind the threshold.
|
|
8
|
+
*/
|
|
9
|
+
export const SUGGEST_DESCRIPTION_PERCENTILE = 90;
|
|
10
|
+
/** Nearest-rank quantile table over `values` — same rank rule as the badge-band percentiles. */
|
|
11
|
+
export function quantileTable(values) {
|
|
12
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
13
|
+
const n = sorted.length;
|
|
14
|
+
const q = [];
|
|
15
|
+
for (let p = 0; p <= 100; p++) {
|
|
16
|
+
q.push(p === 0 ? sorted[0] : sorted[Math.min(n - 1, Math.max(0, Math.ceil((p / 100) * n) - 1))]);
|
|
17
|
+
}
|
|
18
|
+
return q;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* The largest percentile whose quantile does not exceed `value` — read
|
|
22
|
+
* directly off the published table, so "heavier than P% of measured tools" is
|
|
23
|
+
* checkable by anyone holding the same JSON.
|
|
24
|
+
*/
|
|
25
|
+
export function percentileOf(quantiles, value) {
|
|
26
|
+
let p = 0;
|
|
27
|
+
for (let i = 0; i <= 100; i++) {
|
|
28
|
+
if (quantiles[i] <= value)
|
|
29
|
+
p = i;
|
|
30
|
+
else
|
|
31
|
+
break;
|
|
32
|
+
}
|
|
33
|
+
return p;
|
|
34
|
+
}
|
|
35
|
+
/** A tool whose measurement carries all three counts — the only kind a baseline may be built from. */
|
|
36
|
+
function complete(t) {
|
|
37
|
+
return (typeof t.tokens === 'number' && typeof t.descriptionTokens === 'number' && typeof t.inputSchemaTokens === 'number');
|
|
38
|
+
}
|
|
39
|
+
export function buildToolShapeBaseline(tools, meta) {
|
|
40
|
+
const usable = tools.filter(complete);
|
|
41
|
+
if (usable.length < 2)
|
|
42
|
+
throw new Error('fewer than two complete tool measurements — no distribution to derive');
|
|
43
|
+
return {
|
|
44
|
+
method: TOOL_SHAPE_METHOD,
|
|
45
|
+
methodologyVersion: meta.methodologyVersion,
|
|
46
|
+
generatedAt: meta.generatedAt ?? new Date().toISOString().slice(0, 10),
|
|
47
|
+
serverCount: meta.serverCount,
|
|
48
|
+
toolCount: usable.length,
|
|
49
|
+
quantiles: {
|
|
50
|
+
tokens: quantileTable(usable.map((t) => t.tokens)),
|
|
51
|
+
descriptionTokens: quantileTable(usable.map((t) => t.descriptionTokens)),
|
|
52
|
+
inputSchemaTokens: quantileTable(usable.map((t) => t.inputSchemaTokens)),
|
|
53
|
+
},
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
export function parseToolShapeBaseline(text) {
|
|
57
|
+
let parsed;
|
|
58
|
+
try {
|
|
59
|
+
parsed = JSON.parse(text);
|
|
60
|
+
}
|
|
61
|
+
catch {
|
|
62
|
+
return null;
|
|
63
|
+
}
|
|
64
|
+
const b = parsed;
|
|
65
|
+
if (!b || typeof b.generatedAt !== 'string' || typeof b.toolCount !== 'number')
|
|
66
|
+
return null;
|
|
67
|
+
const q = b.quantiles;
|
|
68
|
+
const table = (v) => Array.isArray(v) && v.length === 101 && v.every((x) => typeof x === 'number');
|
|
69
|
+
if (!q || !table(q.tokens) || !table(q.descriptionTokens) || !table(q.inputSchemaTokens))
|
|
70
|
+
return null;
|
|
71
|
+
return {
|
|
72
|
+
method: typeof b.method === 'string' ? b.method : TOOL_SHAPE_METHOD,
|
|
73
|
+
methodologyVersion: typeof b.methodologyVersion === 'string' ? b.methodologyVersion : 'unknown',
|
|
74
|
+
generatedAt: b.generatedAt,
|
|
75
|
+
serverCount: typeof b.serverCount === 'number' ? b.serverCount : 0,
|
|
76
|
+
toolCount: b.toolCount,
|
|
77
|
+
quantiles: { tokens: q.tokens, descriptionTokens: q.descriptionTokens, inputSchemaTokens: q.inputSchemaTokens },
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* The suggestion for one tool, or null — null is the common and correct case.
|
|
82
|
+
* Null when the measurement predates component counts, when the description
|
|
83
|
+
* sits below the threshold percentile, or when trimming toward the median
|
|
84
|
+
* would recover nothing.
|
|
85
|
+
*/
|
|
86
|
+
export function suggestFor(server, t, baseline) {
|
|
87
|
+
if (!complete(t))
|
|
88
|
+
return null;
|
|
89
|
+
const pct = percentileOf(baseline.quantiles.descriptionTokens, t.descriptionTokens);
|
|
90
|
+
if (pct < SUGGEST_DESCRIPTION_PERCENTILE)
|
|
91
|
+
return null;
|
|
92
|
+
const median = baseline.quantiles.descriptionTokens[50];
|
|
93
|
+
const approx = t.descriptionTokens - median;
|
|
94
|
+
if (approx <= 0)
|
|
95
|
+
return null;
|
|
96
|
+
return {
|
|
97
|
+
server,
|
|
98
|
+
tool: t.name,
|
|
99
|
+
tokens: t.tokens,
|
|
100
|
+
descriptionTokens: t.descriptionTokens,
|
|
101
|
+
inputSchemaTokens: t.inputSchemaTokens,
|
|
102
|
+
descriptionPercentile: pct,
|
|
103
|
+
medianDescriptionTokens: median,
|
|
104
|
+
approxRecoverableTokens: approx,
|
|
105
|
+
};
|
|
106
|
+
}
|
package/dist/sweep/regen.js
CHANGED
|
@@ -6,6 +6,7 @@ import { appendHistory } from './history.js';
|
|
|
6
6
|
import { writeServerPages } from './server-pages.js';
|
|
7
7
|
import { writeDashboard } from './dashboard.js';
|
|
8
8
|
import { applyPublishedStats } from './published-stats.js';
|
|
9
|
+
import { writeToolShapeBaseline } from './tool-shape.js';
|
|
9
10
|
const doc = parse(readFileSync('servers.yaml', 'utf8'));
|
|
10
11
|
writeLeaderboard(doc.servers);
|
|
11
12
|
// History first: the server pages read history.csv for their over-time table.
|
|
@@ -14,6 +15,10 @@ const p = writeServerPages(doc.servers);
|
|
|
14
15
|
// The dashboard reads the same results/ and history.csv as the pages do, so it
|
|
15
16
|
// belongs in the same refresh — see writeDashboard's note on why it wasn't.
|
|
16
17
|
const d = writeDashboard();
|
|
18
|
+
// The tool-shape baseline is a quantile table over every measured tool — the
|
|
19
|
+
// distribution `audit --suggest` reads percentile claims from. Derived from
|
|
20
|
+
// the same measurement files as the leaderboard, in the same refresh.
|
|
21
|
+
const ts = writeToolShapeBaseline(doc.servers);
|
|
17
22
|
// The front pages state numbers the sweep just changed; they are patched from
|
|
18
23
|
// the same results/ the leaderboard was. A missing anchor is a page regen can
|
|
19
24
|
// no longer maintain — refuse loudly rather than leave one number stale.
|
|
@@ -27,6 +32,7 @@ console.log('leaderboard:', JSON.stringify(percentiles(doc.servers)));
|
|
|
27
32
|
console.log(`history: ${h.rows} rows (${h.added >= 0 ? '+' : ''}${h.added})`);
|
|
28
33
|
console.log(`server pages: ${p.pages}`);
|
|
29
34
|
console.log(`dashboard: ${d.out} (${(d.bytes / 1024).toFixed(0)}KB)`);
|
|
35
|
+
console.log(`tool shape: ${ts.toolCount} tools across ${ts.serverCount} servers (median description ${ts.quantiles.descriptionTokens[50]})`);
|
|
30
36
|
console.log(`published stats: ${stats.changedFiles.length > 0
|
|
31
37
|
? `updated ${stats.updated.join(', ')} in ${stats.changedFiles.join(', ')}`
|
|
32
38
|
: 'pages already agree with the data'}`);
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Derive the tool-shape baseline from what is on disk and publish it beside
|
|
3
|
+
* the other generated artifacts. Re-derivable by anyone from the same
|
|
4
|
+
* measurement files; regenerated on every regen, so it moves with the data it
|
|
5
|
+
* describes and never sits stale beside a fresh sweep.
|
|
6
|
+
*/
|
|
7
|
+
import { writeFileSync } from 'node:fs';
|
|
8
|
+
import { join } from 'node:path';
|
|
9
|
+
import { METHODOLOGY_VERSION } from '../core/canonical.js';
|
|
10
|
+
import { buildToolShapeBaseline } from '../core/tool-shape.js';
|
|
11
|
+
import { loadRows } from './report.js';
|
|
12
|
+
export function writeToolShapeBaseline(entries, root = process.cwd()) {
|
|
13
|
+
const tools = [];
|
|
14
|
+
let serverCount = 0;
|
|
15
|
+
for (const r of loadRows(entries, root)) {
|
|
16
|
+
if (!r.m || (r.m.status !== 'measured' && r.m.status !== 'dynamic'))
|
|
17
|
+
continue;
|
|
18
|
+
if (r.m.tools.length === 0)
|
|
19
|
+
continue;
|
|
20
|
+
serverCount++;
|
|
21
|
+
tools.push(...r.m.tools);
|
|
22
|
+
}
|
|
23
|
+
const baseline = buildToolShapeBaseline(tools, { serverCount, methodologyVersion: METHODOLOGY_VERSION });
|
|
24
|
+
writeFileSync(join(root, 'results', 'tool-shape.json'), JSON.stringify(baseline, null, 2) + '\n');
|
|
25
|
+
return baseline;
|
|
26
|
+
}
|
package/package.json
CHANGED