mcp-context-cost 0.8.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -11
- package/dist/audit/audit.d.ts +24 -0
- package/dist/audit/audit.js +53 -1
- package/dist/audit/run.d.ts +12 -0
- package/dist/audit/run.js +26 -0
- package/dist/cli.js +31 -14
- package/dist/core/cross-check.d.ts +93 -0
- package/dist/core/cross-check.js +176 -0
- package/dist/core/tool-shape.d.ts +94 -0
- package/dist/core/tool-shape.js +106 -0
- package/dist/sweep/cross-check.d.ts +31 -0
- package/dist/sweep/cross-check.js +342 -0
- package/dist/sweep/docker.d.ts +55 -0
- package/dist/sweep/docker.js +94 -1
- package/dist/sweep/published-stats.d.ts +114 -0
- package/dist/sweep/published-stats.js +383 -0
- package/dist/sweep/regen.js +19 -0
- package/dist/sweep/report.d.ts +9 -0
- package/dist/sweep/report.js +50 -4
- package/dist/sweep/run.js +35 -6
- package/dist/sweep/sweep-all.js +41 -9
- package/dist/sweep/tool-shape.d.ts +3 -0
- package/dist/sweep/tool-shape.js +26 -0
- package/package.json +2 -1
package/README.md
CHANGED
|
@@ -183,10 +183,9 @@ INCREASE FAIL:
|
|
|
183
183
|
Add `--claude` to annotate each server with its Anthropic-request cost from the published
|
|
184
184
|
[Claude divergence](docs/METHODOLOGY.md#claude-divergence) run — an exact number when the
|
|
185
185
|
published capture hash matches what you have installed, `—` (silence, not a stale guess)
|
|
186
|
-
when it doesn't. The run holds 20 rows — the top 20 measured servers by tokens
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
twentieth blank rather than stale. Most installs will show a mix:
|
|
186
|
+
when it doesn't. The run holds 20 rows — the top 20 measured servers by tokens when it ran —
|
|
187
|
+
and [results/leaderboard.md](results/leaderboard.md) prints a claude number for the 16 that
|
|
188
|
+
still match today and silence for the rest. Most installs will show a mix:
|
|
190
189
|
|
|
191
190
|
```
|
|
192
191
|
server tools tokens share claude
|
|
@@ -194,22 +193,39 @@ twentieth blank rather than stale. Most installs will show a mix:
|
|
|
194
193
|
memory 9 2,378 4.2% —
|
|
195
194
|
```
|
|
196
195
|
|
|
196
|
+
Add `--suggest` to place each of your tools in the measured set's tool-shape distribution
|
|
197
|
+
([method](docs/METHODOLOGY.md#tool-shape)) and get advice only where the data can point at
|
|
198
|
+
something. Only descriptions draw advice — schemas are functional surface; descriptions are
|
|
199
|
+
prose every request carries — and only descriptions at or above the 90th percentile of the
|
|
200
|
+
1,150 measured tools:
|
|
201
|
+
|
|
202
|
+
```
|
|
203
|
+
suggest — descriptions at or above the 90th percentile of measured tools
|
|
204
|
+
(baseline 2026-09-03: 1,150 tools across 81 measured servers):
|
|
205
|
+
stub · wordy — 345 tokens: description 321 (p92), schema 14
|
|
206
|
+
rewriting the description toward the measured median (27) would recover ≈294 tokens on every request
|
|
207
|
+
1 of 2 tools sit inside the distribution — no advice where nothing is measurably unusual.
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
A config where nothing is out of distribution is told that in those words, and a baseline
|
|
211
|
+
that cannot be fetched is a named problem, never a silently skipped check.
|
|
212
|
+
|
|
197
213
|
Flags: `--json` (full report on stdout, progress on stderr), `--budget N`,
|
|
198
214
|
`--baseline <report.json>`, `--max-increase N`, `--context N` (default 200,000),
|
|
199
|
-
`--timeout ms`, `--concurrency N`, `--docker`, `--claude`.
|
|
215
|
+
`--timeout ms`, `--concurrency N`, `--docker`, `--claude`, `--suggest`.
|
|
200
216
|
|
|
201
217
|
## Where the numbers come from
|
|
202
218
|
|
|
203
219
|
The number `audit` gives you is the same measurement, run across a curated set of public
|
|
204
220
|
servers — which is how you can tell it is a measurement and not this tool's opinion. It also
|
|
205
|
-
shows what you are choosing between: across the
|
|
206
|
-
from
|
|
221
|
+
shows what you are choosing between: across the 81 servers measured, cost spans **1,700×**,
|
|
222
|
+
from `postgres` at 32 tokens to `github` at 54,622. The table below is a
|
|
207
223
|
sample of that range; the full range is in
|
|
208
224
|
[results/leaderboard.md](results/leaderboard.md).
|
|
209
225
|
|
|
210
226
|
| server | context cost | tools |
|
|
211
227
|
|---|---:|---:|
|
|
212
|
-
| github (official) | **54,
|
|
228
|
+
| github (official) | **54,622 tokens** | 44 |
|
|
213
229
|
| xcodebuildmcp | 26,594 | 24 |
|
|
214
230
|
| brave-search | 25,456 | 8 |
|
|
215
231
|
| notion | 17,500 | 24 |
|
|
@@ -217,7 +233,7 @@ sample of that range; the full range is in
|
|
|
217
233
|
| filesystem (reference) | 2,823 | 14 |
|
|
218
234
|
| markitdown | 64 | 1 |
|
|
219
235
|
|
|
220
|
-
*(
|
|
236
|
+
*(81 of 106 popular servers measured, each row dated by its own most recent sweep — full table in
|
|
221
237
|
[results/leaderboard.md](results/leaderboard.md); every failure is listed with its reason.
|
|
222
238
|
Each measured server also has a [detail page](https://athakur3.github.io/mcp-context-cost/servers/)
|
|
223
239
|
showing which tools its tokens are in.)*
|
|
@@ -252,7 +268,7 @@ exact launch command. Disputes reduce to a byte-level diff:
|
|
|
252
268
|
|
|
253
269
|
```bash
|
|
254
270
|
npx -y mcp-context-cost verify results/github/measurement.json
|
|
255
|
-
# OK github-mcp-server:
|
|
271
|
+
# OK github-mcp-server: 54622 tokens (o200k_base, methodology 1.0) — capture, hash, and count all agree
|
|
256
272
|
|
|
257
273
|
# or point it at a published measurement.json directly, no clone required
|
|
258
274
|
npx -y mcp-context-cost verify --remote https://raw.githubusercontent.com/athakur3/mcp-context-cost/main/results/github/measurement.json
|
|
@@ -267,6 +283,12 @@ failed, `2` usage error.
|
|
|
267
283
|
to the one that was published — which is exactly what `--claude` uses to decide whether it is
|
|
268
284
|
allowed to show you a number.
|
|
269
285
|
|
|
286
|
+
The number is also cross-checked against the other CLI that measures this,
|
|
287
|
+
[`sd2k/mcp-tokens`](https://github.com/sd2k/mcp-tokens): the leaderboard's **mcp-tokens**
|
|
288
|
+
column publishes its count beside ours wherever both tools saw the same tool set — same
|
|
289
|
+
o200k encoding, differences documented, the divergence published rather than left to be
|
|
290
|
+
discovered. Method: [CLI cross-check](docs/METHODOLOGY.md#cli-cross-check).
|
|
291
|
+
|
|
270
292
|
Full definition: [docs/METHODOLOGY.md](docs/METHODOLOGY.md) — what is counted, what the
|
|
271
293
|
number is *not*, config policy, failure taxonomy, frozen color bands, known divergences.
|
|
272
294
|
|
|
@@ -324,7 +346,10 @@ color bands are frozen against the observed distribution of the first full sweep
|
|
|
324
346
|
|
|
325
347
|
## Status
|
|
326
348
|
|
|
327
|
-
Active.
|
|
349
|
+
Active. Every row carries the date of its own most recent measurement, and what the data
|
|
350
|
+
says as a whole is written up, dated, in
|
|
351
|
+
[The State of MCP Context Cost](https://athakur3.github.io/mcp-context-cost/state-of-mcp-context-cost)
|
|
352
|
+
(September 2026). Two
|
|
328
353
|
weekly jobs re-measure the set — the `memory` reference server on Mondays, and a rotating
|
|
329
354
|
sixth of the list on Wednesdays, so every row comes round within six weeks. Read each row's
|
|
330
355
|
date as the date it means, and don't take the cadence on trust — the build history is
|
package/dist/audit/audit.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { type DivergenceRun } from '../core/divergence.js';
|
|
2
|
+
import { type ToolShapeBaseline, type ToolSuggestion } from '../core/tool-shape.js';
|
|
2
3
|
import type { Measurement, MeasurementStatus, ToolMeasurement } from '../core/types.js';
|
|
3
4
|
import type { ConfiguredServer, LoadedConfig } from './config.js';
|
|
4
5
|
import { type DeferralVerdict, type ToolSearchEnv, type ToolSearchSource } from './deferral.js';
|
|
@@ -43,6 +44,19 @@ export interface TrimAdvice {
|
|
|
43
44
|
recoverableTokens: number;
|
|
44
45
|
recoverableShare: number;
|
|
45
46
|
}
|
|
47
|
+
/**
|
|
48
|
+
* `--suggest`: this config's tools placed in the measured set's tool-shape
|
|
49
|
+
* distribution. Only descriptions draw advice (schemas are functional surface;
|
|
50
|
+
* descriptions are prose every request carries), and only descriptions the
|
|
51
|
+
* baseline puts at or above the threshold percentile — a config where nothing
|
|
52
|
+
* is measurably unusual gets that said in those words, not advice invented to
|
|
53
|
+
* have some.
|
|
54
|
+
*/
|
|
55
|
+
export interface ConfigSuggestions {
|
|
56
|
+
/** Heaviest-recovery first. */
|
|
57
|
+
outOfDistribution: ToolSuggestion[];
|
|
58
|
+
checkedTools: number;
|
|
59
|
+
}
|
|
46
60
|
export interface AuditConfigResult {
|
|
47
61
|
client: string;
|
|
48
62
|
source: string;
|
|
@@ -54,6 +68,8 @@ export interface AuditConfigResult {
|
|
|
54
68
|
skipped: AuditServerResult[];
|
|
55
69
|
heaviestTools: HeaviestTool[];
|
|
56
70
|
trimAdvice: TrimAdvice | null;
|
|
71
|
+
/** Present only when `--suggest` ran with a usable baseline. */
|
|
72
|
+
suggestions?: ConfigSuggestions;
|
|
57
73
|
/**
|
|
58
74
|
* Whether this client loads the total up front or defers it, and — when the
|
|
59
75
|
* client decides that by a threshold — which side of it this stack is on.
|
|
@@ -128,6 +144,12 @@ export interface AuditReport {
|
|
|
128
144
|
model: string;
|
|
129
145
|
measuredAt: string;
|
|
130
146
|
};
|
|
147
|
+
/** Which published tool-shape baseline `--suggest` read its percentiles from. */
|
|
148
|
+
toolShape?: {
|
|
149
|
+
generatedAt: string;
|
|
150
|
+
toolCount: number;
|
|
151
|
+
serverCount: number;
|
|
152
|
+
};
|
|
131
153
|
/** Present only when a baseline report was supplied (`--baseline`). */
|
|
132
154
|
diff?: AuditDiff;
|
|
133
155
|
/** Present only when `--max-increase` was supplied alongside a baseline. */
|
|
@@ -160,6 +182,8 @@ export declare function buildReport(configs: LoadedConfig[], measured: Map<strin
|
|
|
160
182
|
generatedAt?: string;
|
|
161
183
|
/** Published `tools-delta/v1` run to join against (`--claude`); omit to skip the join. */
|
|
162
184
|
divergence?: DivergenceRun | null;
|
|
185
|
+
/** Published `tool-shape/v1` baseline (`--suggest`); omit to skip suggestions. */
|
|
186
|
+
toolShape?: ToolShapeBaseline | null;
|
|
163
187
|
/**
|
|
164
188
|
* The audited machine's SHELL tool-search variables. Passed in rather than
|
|
165
189
|
* read here so this stays pure and a report is reproducible from its
|
package/dist/audit/audit.js
CHANGED
|
@@ -14,10 +14,21 @@
|
|
|
14
14
|
*/
|
|
15
15
|
import { METHODOLOGY_VERSION } from '../core/canonical.js';
|
|
16
16
|
import { isCurrent } from '../core/divergence.js';
|
|
17
|
+
import { SUGGEST_DESCRIPTION_PERCENTILE, suggestFor, } from '../core/tool-shape.js';
|
|
17
18
|
import { evaluateDeferral, PUBLISHED_WIRE_TO_CLIENT_RATIO, SHELL_SOURCE, } from './deferral.js';
|
|
18
19
|
import { formatDiff, formatGate } from './diff.js';
|
|
19
20
|
export const DEFAULT_CONTEXT_WINDOW = 200_000;
|
|
20
21
|
const TRIM_TOOL_COUNT = 3;
|
|
22
|
+
function buildSuggestions(pool, baseline) {
|
|
23
|
+
const outOfDistribution = [];
|
|
24
|
+
for (const { server, t } of pool) {
|
|
25
|
+
const s = suggestFor(server, t, baseline);
|
|
26
|
+
if (s)
|
|
27
|
+
outOfDistribution.push(s);
|
|
28
|
+
}
|
|
29
|
+
outOfDistribution.sort((a, b) => b.approxRecoverableTokens - a.approxRecoverableTokens);
|
|
30
|
+
return { outOfDistribution, checkedTools: pool.length };
|
|
31
|
+
}
|
|
21
32
|
function buildTrimAdvice(sortedTools, totalTokens) {
|
|
22
33
|
if (totalTokens <= 0 || sortedTools.length < 2)
|
|
23
34
|
return null;
|
|
@@ -189,6 +200,7 @@ export function buildReport(configs, measured, opts = {}) {
|
|
|
189
200
|
const ok = [];
|
|
190
201
|
const skipped = [];
|
|
191
202
|
const tools = [];
|
|
203
|
+
const shapePool = [];
|
|
192
204
|
// Counted only for servers that put a number into the total: a twin that
|
|
193
205
|
// failed to launch is already a floor, and adds nothing to a sum.
|
|
194
206
|
let sharedHere = 0;
|
|
@@ -240,8 +252,11 @@ export function buildReport(configs, measured, opts = {}) {
|
|
|
240
252
|
claudeTokens: opts.divergence ? (isCurrent(divRow, m.canonicalSha256 ?? null) ? divRow.claudeDelta : null) : undefined,
|
|
241
253
|
notes: m.status === 'dynamic' ? m.notes : undefined,
|
|
242
254
|
});
|
|
243
|
-
for (const t of m.tools)
|
|
255
|
+
for (const t of m.tools) {
|
|
244
256
|
tools.push({ server: s.name, tool: t.name, tokens: t.tokens });
|
|
257
|
+
if (opts.toolShape)
|
|
258
|
+
shapePool.push({ server: s.name, t });
|
|
259
|
+
}
|
|
245
260
|
}
|
|
246
261
|
const totalTokens = ok.reduce((a, s) => a + (s.tokens ?? 0), 0);
|
|
247
262
|
const toolCount = ok.reduce((a, s) => a + (s.toolCount ?? 0), 0);
|
|
@@ -264,6 +279,7 @@ export function buildReport(configs, measured, opts = {}) {
|
|
|
264
279
|
skipped,
|
|
265
280
|
heaviestTools: tools.slice(0, 5),
|
|
266
281
|
trimAdvice: buildTrimAdvice(tools, totalTokens),
|
|
282
|
+
suggestions: opts.toolShape ? buildSuggestions(shapePool, opts.toolShape) : undefined,
|
|
267
283
|
};
|
|
268
284
|
built.push(result);
|
|
269
285
|
shared.set(result, sharedHere);
|
|
@@ -282,6 +298,13 @@ export function buildReport(configs, measured, opts = {}) {
|
|
|
282
298
|
if (opts.divergence) {
|
|
283
299
|
report.claudeDivergence = { model: opts.divergence.model, measuredAt: opts.divergence.measuredAt };
|
|
284
300
|
}
|
|
301
|
+
if (opts.toolShape) {
|
|
302
|
+
report.toolShape = {
|
|
303
|
+
generatedAt: opts.toolShape.generatedAt,
|
|
304
|
+
toolCount: opts.toolShape.toolCount,
|
|
305
|
+
serverCount: opts.toolShape.serverCount,
|
|
306
|
+
};
|
|
307
|
+
}
|
|
285
308
|
if (typeof opts.budget === 'number') {
|
|
286
309
|
// The worst config is the gate: passing because your *lightest* client fits
|
|
287
310
|
// would be a green check on a session you don't run.
|
|
@@ -617,6 +640,35 @@ export function formatReport(report) {
|
|
|
617
640
|
`(${names}) would recover ${n(cfg.trimAdvice.recoverableTokens)} tokens ` +
|
|
618
641
|
`(${pct(cfg.trimAdvice.recoverableShare)} of this config) — if your client supports per-tool filtering.`);
|
|
619
642
|
}
|
|
643
|
+
if (cfg.suggestions) {
|
|
644
|
+
const sg = cfg.suggestions;
|
|
645
|
+
const base = report.toolShape
|
|
646
|
+
? `baseline ${report.toolShape.generatedAt}: ${n(report.toolShape.toolCount)} tools across ` +
|
|
647
|
+
`${report.toolShape.serverCount} measured servers`
|
|
648
|
+
: 'published baseline';
|
|
649
|
+
lines.push('');
|
|
650
|
+
if (sg.outOfDistribution.length === 0) {
|
|
651
|
+
lines.push(` suggest: every description in this config sits inside the measured distribution — ` +
|
|
652
|
+
`nothing the data can point at (${sg.checkedTools} tools against ${base}).`);
|
|
653
|
+
}
|
|
654
|
+
else {
|
|
655
|
+
lines.push(` suggest — descriptions at or above the ${SUGGEST_DESCRIPTION_PERCENTILE}th percentile of ` +
|
|
656
|
+
`measured tools (${base}):`);
|
|
657
|
+
const shown = sg.outOfDistribution.slice(0, 8);
|
|
658
|
+
for (const s of shown) {
|
|
659
|
+
lines.push(` ${s.server} · ${s.tool} — ${n(s.tokens)} tokens: description ${n(s.descriptionTokens)} ` +
|
|
660
|
+
`(p${s.descriptionPercentile}), schema ${n(s.inputSchemaTokens)}`);
|
|
661
|
+
lines.push(` rewriting the description toward the measured median (${n(s.medianDescriptionTokens)}) ` +
|
|
662
|
+
`would recover ≈${n(s.approxRecoverableTokens)} tokens on every request`);
|
|
663
|
+
}
|
|
664
|
+
if (sg.outOfDistribution.length > shown.length) {
|
|
665
|
+
lines.push(` …and ${sg.outOfDistribution.length - shown.length} more above the threshold.`);
|
|
666
|
+
}
|
|
667
|
+
const within = sg.checkedTools - sg.outOfDistribution.length;
|
|
668
|
+
lines.push(` ${within} of ${sg.checkedTools} tools sit inside the distribution — ` +
|
|
669
|
+
`no advice where nothing is measurably unusual.`);
|
|
670
|
+
}
|
|
671
|
+
}
|
|
620
672
|
if (cfg.skipped.length) {
|
|
621
673
|
lines.push('');
|
|
622
674
|
lines.push(' not measured');
|
package/dist/audit/run.d.ts
CHANGED
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
import type { Measurement } from '../core/types.js';
|
|
2
2
|
import { type DivergenceRun } from '../core/divergence.js';
|
|
3
|
+
import { type ToolShapeBaseline } from '../core/tool-shape.js';
|
|
3
4
|
import { type AuditReport } from './audit.js';
|
|
4
5
|
import { type ToolSearchEnv, type ToolSearchSource } from './deferral.js';
|
|
5
6
|
import { type LoadedConfig } from './config.js';
|
|
6
7
|
/** Where the published `tools-delta/v1` run lives when `--claude` doesn't override it. */
|
|
7
8
|
export declare const DEFAULT_DIVERGENCE_URL = "https://raw.githubusercontent.com/athakur3/mcp-context-cost/main/results/divergence.json";
|
|
9
|
+
/** Where the published `tool-shape/v1` baseline lives when `--suggest` doesn't override it. */
|
|
10
|
+
export declare const DEFAULT_TOOL_SHAPE_URL = "https://raw.githubusercontent.com/athakur3/mcp-context-cost/main/results/tool-shape.json";
|
|
8
11
|
export interface AuditOptions {
|
|
9
12
|
/** Explicit config path(s); when empty, every known client location is tried. */
|
|
10
13
|
configPaths?: string[];
|
|
@@ -19,6 +22,10 @@ export interface AuditOptions {
|
|
|
19
22
|
claude?: boolean;
|
|
20
23
|
/** Override the divergence.json source — mainly for tests and self-hosted mirrors. */
|
|
21
24
|
divergenceUrl?: string;
|
|
25
|
+
/** Place this config's tools in the published tool-shape distribution and advise where the data can. */
|
|
26
|
+
suggest?: boolean;
|
|
27
|
+
/** Override the tool-shape.json source — mainly for tests and self-hosted mirrors. */
|
|
28
|
+
toolShapeUrl?: string;
|
|
22
29
|
/**
|
|
23
30
|
* The tool-search variables as this process's SHELL has them. Defaults to
|
|
24
31
|
* this process's environment. Overridable so a test can state a machine
|
|
@@ -33,6 +40,11 @@ export interface AuditOptions {
|
|
|
33
40
|
settings?: ToolSearchSource[];
|
|
34
41
|
onProgress?: (name: string, done: number, total: number) => void;
|
|
35
42
|
}
|
|
43
|
+
/** Fetch and parse the published tool-shape baseline. Never throws: a failure is a report problem, not a crash. */
|
|
44
|
+
export declare function fetchToolShape(url: string): Promise<{
|
|
45
|
+
baseline: ToolShapeBaseline | null;
|
|
46
|
+
problem?: string;
|
|
47
|
+
}>;
|
|
36
48
|
/** Fetch and parse the published divergence run. Never throws: a failure is a report problem, not a crash. */
|
|
37
49
|
export declare function fetchDivergence(url: string): Promise<{
|
|
38
50
|
run: DivergenceRun | null;
|
package/dist/audit/run.js
CHANGED
|
@@ -7,11 +7,27 @@
|
|
|
7
7
|
import { homedir } from 'node:os';
|
|
8
8
|
import { measureServer } from '../sweep/run.js';
|
|
9
9
|
import { parseDivergence } from '../core/divergence.js';
|
|
10
|
+
import { parseToolShapeBaseline } from '../core/tool-shape.js';
|
|
10
11
|
import { buildReport, serverKey } from './audit.js';
|
|
11
12
|
import { toolSearchEnv } from './deferral.js';
|
|
12
13
|
import { configCandidates, loadConfigs, loadSettingsSources, settingsCandidates, } from './config.js';
|
|
13
14
|
/** Where the published `tools-delta/v1` run lives when `--claude` doesn't override it. */
|
|
14
15
|
export const DEFAULT_DIVERGENCE_URL = 'https://raw.githubusercontent.com/athakur3/mcp-context-cost/main/results/divergence.json';
|
|
16
|
+
/** Where the published `tool-shape/v1` baseline lives when `--suggest` doesn't override it. */
|
|
17
|
+
export const DEFAULT_TOOL_SHAPE_URL = 'https://raw.githubusercontent.com/athakur3/mcp-context-cost/main/results/tool-shape.json';
|
|
18
|
+
/** Fetch and parse the published tool-shape baseline. Never throws: a failure is a report problem, not a crash. */
|
|
19
|
+
export async function fetchToolShape(url) {
|
|
20
|
+
try {
|
|
21
|
+
const res = await fetch(url, { signal: AbortSignal.timeout(15_000) });
|
|
22
|
+
if (!res.ok)
|
|
23
|
+
return { baseline: null, problem: `tool shape: HTTP ${res.status} fetching ${url}` };
|
|
24
|
+
const baseline = parseToolShapeBaseline(await res.text());
|
|
25
|
+
return baseline ? { baseline } : { baseline: null, problem: `tool shape: malformed data at ${url}` };
|
|
26
|
+
}
|
|
27
|
+
catch (e) {
|
|
28
|
+
return { baseline: null, problem: `tool shape: failed to fetch ${url}: ${e.message}` };
|
|
29
|
+
}
|
|
30
|
+
}
|
|
15
31
|
/** Fetch and parse the published divergence run. Never throws: a failure is a report problem, not a crash. */
|
|
16
32
|
export async function fetchDivergence(url) {
|
|
17
33
|
try {
|
|
@@ -89,14 +105,24 @@ export async function runAudit(opts = {}) {
|
|
|
89
105
|
divergence = fetched.run;
|
|
90
106
|
divergenceProblem = fetched.problem;
|
|
91
107
|
}
|
|
108
|
+
let toolShape = null;
|
|
109
|
+
let toolShapeProblem;
|
|
110
|
+
if (opts.suggest) {
|
|
111
|
+
const fetched = await fetchToolShape(opts.toolShapeUrl ?? DEFAULT_TOOL_SHAPE_URL);
|
|
112
|
+
toolShape = fetched.baseline;
|
|
113
|
+
toolShapeProblem = fetched.problem;
|
|
114
|
+
}
|
|
92
115
|
const report = buildReport(configs, measured, {
|
|
93
116
|
contextWindow: opts.contextWindow,
|
|
94
117
|
budget: opts.budget,
|
|
95
118
|
divergence,
|
|
119
|
+
toolShape,
|
|
96
120
|
env: opts.env ?? toolSearchEnv(process.env),
|
|
97
121
|
settings: opts.settings ?? discoverSettings(opts),
|
|
98
122
|
});
|
|
99
123
|
if (divergenceProblem)
|
|
100
124
|
report.problems.push(divergenceProblem);
|
|
125
|
+
if (toolShapeProblem)
|
|
126
|
+
report.problems.push(toolShapeProblem);
|
|
101
127
|
return report;
|
|
102
128
|
}
|
package/dist/cli.js
CHANGED
|
@@ -99,8 +99,8 @@ function rejectUnknownFlags(cmd, argv, spec) {
|
|
|
99
99
|
const [, , cmd, ...rest] = process.argv;
|
|
100
100
|
if (cmd === 'audit') {
|
|
101
101
|
rejectUnknownFlags('audit', rest, {
|
|
102
|
-
value: ['config', 'budget', 'baseline', 'max-increase', 'context', 'timeout', 'concurrency', 'divergence-url'],
|
|
103
|
-
boolean: ['json', 'docker', 'claude'],
|
|
102
|
+
value: ['config', 'budget', 'baseline', 'max-increase', 'context', 'timeout', 'concurrency', 'divergence-url', 'tool-shape-url'],
|
|
103
|
+
boolean: ['json', 'docker', 'claude', 'suggest'],
|
|
104
104
|
});
|
|
105
105
|
const argOf = (name) => {
|
|
106
106
|
const i = rest.indexOf(`--${name}`);
|
|
@@ -159,18 +159,35 @@ if (cmd === 'audit') {
|
|
|
159
159
|
}
|
|
160
160
|
const { runAudit } = await import('./audit/run.js');
|
|
161
161
|
const { formatReport } = await import('./audit/audit.js');
|
|
162
|
-
const
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
162
|
+
const { DockerHarnessFault } = await import('./sweep/docker.js');
|
|
163
|
+
let report;
|
|
164
|
+
try {
|
|
165
|
+
report = await runAudit({
|
|
166
|
+
configPaths: all('config'),
|
|
167
|
+
budget,
|
|
168
|
+
contextWindow: numeric('context'),
|
|
169
|
+
timeoutMs: numeric('timeout'),
|
|
170
|
+
concurrency: numeric('concurrency'),
|
|
171
|
+
docker: rest.includes('--docker'),
|
|
172
|
+
claude: rest.includes('--claude'),
|
|
173
|
+
divergenceUrl: argOf('divergence-url'),
|
|
174
|
+
suggest: rest.includes('--suggest'),
|
|
175
|
+
toolShapeUrl: argOf('tool-shape-url'),
|
|
176
|
+
// Progress goes to stderr so `--json` stdout stays a single parseable object.
|
|
177
|
+
onProgress: json ? undefined : (name, done, total) => process.stderr.write(` [${done}/${total}] ${name}\n`),
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
catch (e) {
|
|
181
|
+
// Docker failing as docker means every measurement through it would be a
|
|
182
|
+
// statement about this machine, so the audit refuses whole rather than
|
|
183
|
+
// reporting each server as broken.
|
|
184
|
+
if (e instanceof DockerHarnessFault) {
|
|
185
|
+
console.error(`audit --docker cannot answer for this machine: ${e.message}`);
|
|
186
|
+
console.error('Fix Docker here, or run without --docker.');
|
|
187
|
+
process.exit(1);
|
|
188
|
+
}
|
|
189
|
+
throw e;
|
|
190
|
+
}
|
|
174
191
|
if (report.configs.length === 0) {
|
|
175
192
|
const where = report.problems.length ? `\n${report.problems.map((p) => ` ${p}`).join('\n')}` : '';
|
|
176
193
|
const empty = report.emptyConfigs ?? [];
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import type { Measurement } from './types.js';
|
|
2
|
+
/** Method identifier, versioned independently of METHODOLOGY_VERSION. */
|
|
3
|
+
export declare const CROSS_CHECK_METHOD = "cli-cross-check/v1";
|
|
4
|
+
/** The CLI being cross-checked against, and the release this run pins. */
|
|
5
|
+
export declare const CROSS_CHECK_CLI = "sd2k/mcp-tokens";
|
|
6
|
+
export declare const CROSS_CHECK_CLI_VERSION = "v0.2.5";
|
|
7
|
+
/**
|
|
8
|
+
* The exact analyze invocation, recorded in every run. `--model gpt-4o` is
|
|
9
|
+
* load-bearing: tiktoken-rs maps it to o200k_base, and without a model the CLI
|
|
10
|
+
* falls back to cl100k_base — a systematic difference that would swamp the one
|
|
11
|
+
* being measured (spec/upstream-notes.md, finding 1).
|
|
12
|
+
*/
|
|
13
|
+
export declare const CROSS_CHECK_CLI_ARGS: readonly ['analyze', '--provider', 'tiktoken', '--model', 'gpt-4o', '--format', 'json'];
|
|
14
|
+
export interface CrossCheckRow {
|
|
15
|
+
/** o200k count of our fresh canonical capture from this run — the headline. */
|
|
16
|
+
ourTokens: number;
|
|
17
|
+
/**
|
|
18
|
+
* o200k count of the same capture's name/description/input_schema
|
|
19
|
+
* projection — what the CLI's structs actually model, and therefore the
|
|
20
|
+
* number its count is compared against.
|
|
21
|
+
*/
|
|
22
|
+
ourMappedTokens: number;
|
|
23
|
+
/** The CLI's `.tools.total` for the same server, launched by the CLI itself. */
|
|
24
|
+
cliTokens: number;
|
|
25
|
+
ourToolCount: number;
|
|
26
|
+
cliToolCount: number;
|
|
27
|
+
/** Both tools saw the same tool names (order-insensitive) — the comparison is between like and like. */
|
|
28
|
+
toolSetMatches: boolean;
|
|
29
|
+
/**
|
|
30
|
+
* Our fresh measurement listed dynamically (tools/list differed between its
|
|
31
|
+
* own two captures). The CLI's launch is a third capture, so even with
|
|
32
|
+
* matching names its residual mixes content drift with counter disagreement
|
|
33
|
+
* — recorded, never printed.
|
|
34
|
+
*/
|
|
35
|
+
dynamic: boolean;
|
|
36
|
+
/** canonicalSha256 of our capture in this run; null when our measurement failed. */
|
|
37
|
+
capturedSha256: string | null;
|
|
38
|
+
/** Set when either side could not produce a number; nothing is published from the row. */
|
|
39
|
+
error?: string;
|
|
40
|
+
}
|
|
41
|
+
export interface CrossCheckRun {
|
|
42
|
+
method: string;
|
|
43
|
+
cli: string;
|
|
44
|
+
cliVersion: string;
|
|
45
|
+
/** The analyze invocation, verbatim, so the encoding choice is auditable. */
|
|
46
|
+
cliArgs: string[];
|
|
47
|
+
/** UTC day the run was taken (YYYY-MM-DD). */
|
|
48
|
+
measuredAt: string;
|
|
49
|
+
isolation: string;
|
|
50
|
+
servers: Record<string, CrossCheckRow>;
|
|
51
|
+
}
|
|
52
|
+
export declare function parseCrossCheck(text: string): CrossCheckRun | null;
|
|
53
|
+
/** What the CLI's JSON report contributes to a row. */
|
|
54
|
+
export interface CliReport {
|
|
55
|
+
total: number;
|
|
56
|
+
count: number;
|
|
57
|
+
names: string[];
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Read the CLI's `--format json` report. The report is that CLI's contract, not
|
|
61
|
+
* ours, so this is deliberately narrow: `.tools.total` (the number the CLI's
|
|
62
|
+
* own action publishes), `.tools.count`, and the per-tool names — and a report
|
|
63
|
+
* that does not carry a usable total is a named problem, never a zero.
|
|
64
|
+
*/
|
|
65
|
+
export declare function parseCliReport(text: string): {
|
|
66
|
+
report?: CliReport;
|
|
67
|
+
problem?: string;
|
|
68
|
+
};
|
|
69
|
+
/** Order-insensitive equality of the two tool-name lists, repeats included. */
|
|
70
|
+
export declare function sameToolSet(ours: string[], theirs: string[]): boolean;
|
|
71
|
+
/**
|
|
72
|
+
* Build a row from our fresh measurement and the CLI's report. The caller
|
|
73
|
+
* decides whether to run the CLI at all; a failed side arrives here as an
|
|
74
|
+
* `error`, and the row keeps whatever the other side established.
|
|
75
|
+
*/
|
|
76
|
+
export declare function toCrossCheckRow(m: Measurement, cli: {
|
|
77
|
+
report?: CliReport;
|
|
78
|
+
problem?: string;
|
|
79
|
+
}): CrossCheckRow;
|
|
80
|
+
/**
|
|
81
|
+
* A row is only printable while it compares like with like: no error on either
|
|
82
|
+
* side, the same tool names seen by both tools, and our capture still the one
|
|
83
|
+
* published — the exact staleness rule the claude column follows.
|
|
84
|
+
*/
|
|
85
|
+
export declare function isComparable(row: CrossCheckRow | undefined, canonicalSha256: string | null): row is CrossCheckRow;
|
|
86
|
+
/**
|
|
87
|
+
* Signed divergence of the CLI's count from our count of the projection it
|
|
88
|
+
* models, in percent. Against the mapped number and not the headline: the gap
|
|
89
|
+
* to the headline is field selection, published separately per server, and
|
|
90
|
+
* folding it in here would bury the number this column exists to check —
|
|
91
|
+
* whether two independent counters agree on the fields both count.
|
|
92
|
+
*/
|
|
93
|
+
export declare function divergencePct(row: CrossCheckRow): number | null;
|