mcp-context-cost 0.18.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +67 -114
- package/dist/audit/audit.d.ts +25 -27
- package/dist/audit/audit.js +50 -24
- package/dist/audit/config.d.ts +8 -8
- package/dist/audit/config.js +17 -8
- package/dist/audit/deferral.d.ts +12 -12
- package/dist/audit/deferral.js +27 -21
- package/dist/audit/diff.js +35 -9
- package/dist/audit/remote.d.ts +2 -2
- package/dist/audit/remote.js +12 -3
- package/dist/audit/run.d.ts +18 -18
- package/dist/audit/run.js +16 -5
- package/dist/cli.js +38 -14
- package/dist/core/adoption.js +33 -19
- package/dist/core/bands.d.ts +4 -3
- package/dist/core/bands.js +4 -3
- package/dist/core/canonical.d.ts +6 -6
- package/dist/core/capture-index.js +6 -1
- package/dist/core/cross-check.js +11 -3
- package/dist/core/format.d.ts +34 -0
- package/dist/core/format.js +37 -0
- package/dist/core/index.d.ts +39 -7
- package/dist/core/index.js +38 -7
- package/dist/core/protocol.d.ts +3 -3
- package/dist/core/protocol.js +3 -3
- package/dist/core/regression.js +7 -3
- package/dist/core/server-diff.d.ts +2 -2
- package/dist/core/server-diff.js +6 -3
- package/dist/core/session-start.d.ts +7 -50
- package/dist/core/session-start.js +7 -65
- package/dist/core/tool-shape.js +17 -5
- package/dist/core/types.d.ts +18 -17
- package/dist/sweep/client.d.ts +4 -4
- package/dist/sweep/client.js +23 -10
- package/dist/sweep/cross-check.d.ts +1 -2
- package/dist/sweep/cross-check.js +15 -15
- package/dist/sweep/dashboard.js +26 -14
- package/dist/sweep/docker.d.ts +8 -8
- package/dist/sweep/docker.js +1 -1
- package/dist/sweep/history.js +10 -7
- package/dist/sweep/pr-check.js +2 -2
- package/dist/sweep/published-stats.d.ts +8 -10
- package/dist/sweep/published-stats.js +120 -49
- package/dist/sweep/regen.js +8 -10
- package/dist/sweep/registry-scan.js +28 -6
- package/dist/sweep/regressions.js +7 -8
- package/dist/sweep/report.d.ts +1 -5
- package/dist/sweep/report.js +13 -28
- package/dist/sweep/run.d.ts +13 -13
- package/dist/sweep/run.js +5 -1
- package/dist/sweep/server-pages.js +13 -8
- package/dist/sweep/servers-schema.d.ts +1 -1
- package/dist/sweep/servers-schema.js +8 -4
- package/dist/sweep/sweep-all.js +2 -3
- package/dist/sweep/tool-shape.js +1 -1
- package/package.json +16 -3
- package/dist/sweep/session-start.d.ts +0 -3
- package/dist/sweep/session-start.js +0 -127
package/README.md
CHANGED
|
@@ -4,9 +4,9 @@
|
|
|
4
4
|
[](https://github.com/athakur3/mcp-context-cost/actions/workflows/ci.yml)
|
|
5
5
|
[](https://github.com/marketplace/actions/mcp-context-cost)
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
cost than you meant to ship:
|
|
7
|
+
Three entry points: `audit`, for the config you run (below); a GitHub Action, for the server
|
|
8
|
+
you publish; and the measurement itself, as a library, for anything else. The Action is five
|
|
9
|
+
lines in a workflow that fail a pull request adding more context cost than you meant to ship:
|
|
10
10
|
|
|
11
11
|
```yaml
|
|
12
12
|
- uses: athakur3/mcp-context-cost@v1
|
|
@@ -24,6 +24,18 @@ gate passed or not, so a later step can comment the number on the PR or publish
|
|
|
24
24
|
[how the gate decides](#defend-the-number-dont-just-display-it) ·
|
|
25
25
|
[what the number is](docs/METHODOLOGY.md).
|
|
26
26
|
|
|
27
|
+
As a library, if you want the measurement without either wrapper — the same canonical form,
|
|
28
|
+
the same tokenizer, the same badge:
|
|
29
|
+
|
|
30
|
+
```js
|
|
31
|
+
import { measureTools, canonicalString, toBadge, METHODOLOGY_VERSION } from "mcp-context-cost";
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
It is the definition rather than the harness: measurement and canonical bytes, the color
|
|
35
|
+
bands, the badge JSON, the Claude field-selection mapping, and the session-start load. It
|
|
36
|
+
does not launch servers or read your config — that is what `audit` is for. The surface is
|
|
37
|
+
pinned by a test, so a name is added or removed on purpose and says so in the changelog.
|
|
38
|
+
|
|
27
39
|
**What do the MCP servers in your config cost you before you type anything — and what did
|
|
28
40
|
that last config change add to every session you will ever run?**
|
|
29
41
|
|
|
@@ -69,16 +81,15 @@ VS Code (`.vscode/mcp.json`), Windsurf, Codex CLI (`~/.codex/config.toml`), Gemi
|
|
|
69
81
|
Goose (`~/.config/goose/config.yaml`) — or pass `--config <path>`. Servers are measured by
|
|
70
82
|
the same path as the published leaderboard (dual `tools/list` capture, `o200k_base` over
|
|
71
83
|
canonical JSON), so a server in both places gets the same number. Nothing is written to your
|
|
72
|
-
project, and
|
|
84
|
+
project, and no **value** an entry carries — an env var, a header — is ever read into the
|
|
85
|
+
output. Only their names are.
|
|
73
86
|
|
|
74
|
-
A remote entry — `url`, or the client's own spelling of it — is
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
reason. Header values an entry carries are sent and never printed — only their names are —
|
|
81
|
-
and nothing here ever opens a browser.
|
|
87
|
+
A remote entry — `url`, or the client's own spelling of it — is probed before anything is
|
|
88
|
+
launched, and reported as measured, **auth-walled** or **unreachable** depending on what it
|
|
89
|
+
answers; an auth-walled row makes the total above it a floor rather than a number. The probe,
|
|
90
|
+
what each verdict quotes, and why it exists at all are in
|
|
91
|
+
[METHODOLOGY §who pays the number](docs/METHODOLOGY.md#who-pays). Nothing here opens a
|
|
92
|
+
browser.
|
|
82
93
|
|
|
83
94
|
Totals are reported per config file, never merged: a context window belongs to one client
|
|
84
95
|
session, so summing Cursor's servers into Claude Desktop's total would describe a session
|
|
@@ -88,45 +99,18 @@ nobody runs.
|
|
|
88
99
|
|
|
89
100
|
Not every client puts every tool definition in context on every request, so the total above
|
|
90
101
|
is not automatically your bill. Which client reads the config decides it, and for Claude Code
|
|
91
|
-
so does how that client is configured **on this machine** —
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
mechanism on 2026-01-06 while its MCP configuration page, then and now, says nothing, and a
|
|
104
|
-
rule that read only that page reported an absence of a record for eight months.
|
|
105
|
-
|
|
106
|
-
**Clients with no default deferral on record** — Claude Desktop, Windsurf, Gemini CLI, Zed,
|
|
107
|
-
Kiro, Goose. The total is what every request carries, as in the example above. That sentence is
|
|
108
|
-
an absence of a record about those clients, not a measurement of them, and the report says so
|
|
109
|
-
in those words.
|
|
110
|
-
|
|
111
|
-
**Claude Code defers MCP tool definitions by default** (its **tool search**): they are not
|
|
112
|
-
in context at session start, and load when the model reaches for one. Three variables move
|
|
113
|
-
that, and `audit` reads all three — from the shell it runs in *and* from the `env` block of
|
|
114
|
-
Claude Code's own settings files (managed, `<cwd>/.claude/settings.local.json`,
|
|
115
|
-
`<cwd>/.claude/settings.json`, `~/.claude/settings.json`), because a machine that switched
|
|
116
|
-
deferral off in a settings file is not a machine running the default:
|
|
117
|
-
|
|
118
|
-
| setting | what the audit reports |
|
|
119
|
-
|---|---|
|
|
120
|
-
| nothing set (the default) | every definition deferred, at any size — no threshold applies |
|
|
121
|
-
| `ENABLE_TOOL_SEARCH=true` | same: every definition deferred |
|
|
122
|
-
| `ENABLE_TOOL_SEARCH=false` | deferral off — every request carries the full total. In a settings `env` block that is the **string** `"false"`; the JSON boolean `false` is the last row, not this one |
|
|
123
|
-
| `ENABLE_TOOL_SEARCH=auto` / `auto:N` | deferred only once definitions reach 10% / N% of the context window |
|
|
124
|
-
| `CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS` set to `1`, `true`, `yes` or `on` | tool search off — read first, because `ENABLE_TOOL_SEARCH` cannot override it |
|
|
125
|
-
| `CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS` set to `0`, `false`, `no` or `off` | it turned nothing off, so the read moves on and the rows above decide. It is a boolean flag in the client, not a marker whose presence is the signal |
|
|
126
|
-
| `ANTHROPIC_BASE_URL` off `api.anthropic.com` | falls back to loading up front — consulted only while `ENABLE_TOOL_SEARCH` is unset |
|
|
127
|
-
| anything else in `ENABLE_TOOL_SEARCH` | not a documented value, so nothing is claimed from it |
|
|
128
|
-
| any of the three set, in a settings `env` block, to something that is not a string — a JSON boolean, a number, `null` | it is set there and what it is set to is unknown, so no posture is claimed: the report says whether these tokens are deferred cannot be said from it |
|
|
129
|
-
| a server pinned `"alwaysLoad": true` in its entry | loads at session start whatever the setting says — read from the entry, named with its tokens, and left out of any threshold comparison |
|
|
102
|
+
so does how that client is configured **on this machine** — three environment variables and a
|
|
103
|
+
per-server `alwaysLoad` pin, which `audit` reads rather than assumes, from the shell it runs
|
|
104
|
+
in *and* from the `env` block of Claude Code's own settings files. No other client's posture
|
|
105
|
+
is readable from a file this opens, so for those the report gives what the vendor is on record
|
|
106
|
+
with, or says there is nothing on record, and claims neither as a measurement.
|
|
107
|
+
|
|
108
|
+
**The model is written down once, on the methodology page**, because it gets corrected often
|
|
109
|
+
and a second copy here would be a second place to correct:
|
|
110
|
+
[METHODOLOGY §who pays the number](docs/METHODOLOGY.md#who-pays) carries every variable and
|
|
111
|
+
what it resolves to, the four kinds of first-party record the rule admits, the questions the
|
|
112
|
+
report refuses to answer rather than guess, and the address and date behind every claim. What
|
|
113
|
+
follows is what that model looks like when you run it.
|
|
130
114
|
|
|
131
115
|
On a machine where none of them is set, the same stack reads:
|
|
132
116
|
|
|
@@ -147,7 +131,7 @@ On a machine where none of them is set, the same stack reads:
|
|
|
147
131
|
a tool whose _meta carries "anthropic/alwaysLoad": true, which this audit does not read from a capture
|
|
148
132
|
```
|
|
149
133
|
|
|
150
|
-
**Do not take that
|
|
134
|
+
**Do not take that model on trust — your own client will tell you.** All of it is read
|
|
151
135
|
from Anthropic's documentation, and documentation about someone else's product is exactly the
|
|
152
136
|
kind of claim this project refuses to leave unchecked elsewhere. Claude Code writes its own
|
|
153
137
|
decision to a debug log, before it sends anything, so you can check your machine rather than
|
|
@@ -164,7 +148,7 @@ Three line shapes answer three different questions:
|
|
|
164
148
|
|---|---|
|
|
165
149
|
| `[ToolSearch:optimistic] mode=…, ENABLE_TOOL_SEARCH=…, result=…` | which mode it picked at startup, and the value it read. **Optimistic is its own word for a guess** — it can be revised below |
|
|
166
150
|
| `Dynamic tool loading: 0/N deferred tools included` | the one that settles it: how many of the `N` deferrable tools went into the request. `0/N` is deferral actually happening |
|
|
167
|
-
| `[ToolSearch:optimistic] disabled: ANTHROPIC_BASE_URL=… is not a first-party Anthropic host` | the fallback
|
|
151
|
+
| `[ToolSearch:optimistic] disabled: ANTHROPIC_BASE_URL=… is not a first-party Anthropic host` | the base-URL fallback, firing, in the client's own words |
|
|
168
152
|
|
|
169
153
|
Read the *later* requests, not the first. A stdio server can finish connecting after the first
|
|
170
154
|
request has already gone, so an early low count is a race rather than a finding.
|
|
@@ -177,8 +161,8 @@ a proportionally larger one — pass `--context` to `audit` to compare against t
|
|
|
177
161
|
actually run.
|
|
178
162
|
|
|
179
163
|
None of this is free, and it is one trivial request. These are Claude Code's own debug lines
|
|
180
|
-
rather than a documented interface, so they can change; the
|
|
181
|
-
holds to a dated re-read. No other client discovered by `audit` writes anything comparable,
|
|
164
|
+
rather than a documented interface, so they can change; the methodology page is what this
|
|
165
|
+
project holds to a dated re-read. No other client discovered by `audit` writes anything comparable,
|
|
182
166
|
which is why no other row here is a measurement: three of them get their vendor's own record,
|
|
183
167
|
dated and addressed, and the rest an absence of one.
|
|
184
168
|
|
|
@@ -189,22 +173,6 @@ tool names plus the server's `instructions` — is measured per server and publi
|
|
|
189
173
|
leaderboard's `session start` column, and for at least one server in the published set it
|
|
190
174
|
costs **more** than loading the definitions would.
|
|
191
175
|
|
|
192
|
-
Three things the report will not do: it will not convert between units silently (in
|
|
193
|
-
threshold mode the stack is compared as a range, because the audit counts wire bytes and the
|
|
194
|
-
threshold is counted in what the client sends to the API — measured at 0.19×–1.93× across 86
|
|
195
|
-
servers); it will not claim a posture the machine did not state readably, which is four
|
|
196
|
-
refusals and not one — when two places set the same variable to different values, when a
|
|
197
|
-
settings file exists and cannot be read, when the place that would decide sets the variable
|
|
198
|
-
to something that is not a string, and when `ENABLE_TOOL_SEARCH` holds a value Claude Code
|
|
199
|
-
does not document; and it will not pass an absence of a record off as a measurement, or a
|
|
200
|
-
vendor's record off as one either. The first two print as unanswered questions. The third
|
|
201
|
-
prints as an answer that names itself: for the six discovered clients with no default on
|
|
202
|
-
record — `claude-desktop`, `windsurf`, `gemini`, `zed`, `kiro`, `goose` — the tokens are
|
|
203
|
-
counted as loaded up front, and the report says so in those words, "an absence of a record
|
|
204
|
-
about the client, not a measurement of it". For `cursor`, `codex` and `vscode` it prints the
|
|
205
|
-
vendor's record with its dates and everything it leaves open, and claims no side.
|
|
206
|
-
Full model, sources and dates: [METHODOLOGY §who pays the number](docs/METHODOLOGY.md#who-pays).
|
|
207
|
-
|
|
208
176
|
**In CI**, make it a gate — the bundlesize move for agents:
|
|
209
177
|
|
|
210
178
|
```bash
|
|
@@ -277,15 +245,14 @@ still match today and silence for the rest. Most installs will show a mix:
|
|
|
277
245
|
|
|
278
246
|
```
|
|
279
247
|
server tools tokens share claude
|
|
280
|
-
|
|
281
|
-
|
|
248
|
+
my-server 18 9,400 79.8% 3,120
|
|
249
|
+
my-fork 6 2,380 20.2% —
|
|
282
250
|
```
|
|
283
251
|
|
|
284
|
-
Add `--suggest` to place each of your tools in the measured set's tool-shape distribution
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
1,430 measured tools:
|
|
252
|
+
Add `--suggest` to place each of your tools in the measured set's tool-shape distribution and
|
|
253
|
+
get advice only where the data can point at something. What draws a suggestion, what never
|
|
254
|
+
does, and the percentile that decides are in
|
|
255
|
+
[METHODOLOGY §tool shape](docs/METHODOLOGY.md#tool-shape):
|
|
289
256
|
|
|
290
257
|
```
|
|
291
258
|
suggest — descriptions at or above the 90th percentile of measured tools
|
|
@@ -305,8 +272,8 @@ are not:
|
|
|
305
272
|
|
|
306
273
|
```
|
|
307
274
|
changed — published versions of your servers that have moved since
|
|
308
|
-
(index 2026-09-
|
|
309
|
-
notes (published as
|
|
275
|
+
(index 2026-09-05, 99 published captures; matched by canonical hash, never by name):
|
|
276
|
+
notes (published as my-server) — you have the capture published 2026-08-19 at 1,132 tokens;
|
|
310
277
|
the current one is 2,062 (+930, 2026-08-26)
|
|
311
278
|
updating all 1 would add 930 tokens to every request in this client.
|
|
312
279
|
```
|
|
@@ -378,9 +345,10 @@ the method is [Claude divergence](docs/METHODOLOGY.md#claude-divergence).
|
|
|
378
345
|
|
|
379
346
|
## Why trust the number?
|
|
380
347
|
|
|
381
|
-
Every published number is backed by a `measurement.json
|
|
382
|
-
|
|
383
|
-
|
|
348
|
+
Every published number is backed by a `measurement.json`. What it holds, and the five lines
|
|
349
|
+
that re-derive the number from it, are in
|
|
350
|
+
[METHODOLOGY §reproduce it](docs/METHODOLOGY.md#reproduce-it). Disputes reduce to a byte-level
|
|
351
|
+
diff, and the CLI does that for you:
|
|
384
352
|
|
|
385
353
|
```bash
|
|
386
354
|
npx -y mcp-context-cost verify results/github/measurement.json
|
|
@@ -418,7 +386,7 @@ number is *not*, config policy, failure taxonomy, frozen color bands, known dive
|
|
|
418
386
|
| `src/cli.ts` | `audit` (measure your own stack), `verify` (re-derive any published number), `measure` |
|
|
419
387
|
| `spec/fixtures/` | golden vectors shared by the TypeScript and bash implementations |
|
|
420
388
|
| `tools/` | the scripts that call a network API — the Claude divergence run, the adoption reading, the registry scan — kept out of the package so the library stays offline |
|
|
421
|
-
| `upstream/` | `badge.sh` +
|
|
389
|
+
| `upstream/` | `badge.sh` + bash tests — the self-serve badge recipe, carried here |
|
|
422
390
|
| `servers.yaml` | 107 curated candidates with live install metrics and provenance |
|
|
423
391
|
| `results/` · `badges/` | measurements, leaderboard, history series, shields endpoint JSONs |
|
|
424
392
|
| `docs/` | methodology, dashboard, and a generated page per measured server |
|
|
@@ -434,14 +402,12 @@ That prints the number and writes nothing. Published records
|
|
|
434
402
|
(`results/<name>/measurement.json`, `badges/<name>.json`, the `history.csv`
|
|
435
403
|
row) come from CI: a developer machine is a different architecture under
|
|
436
404
|
different load, and a measurement taken there describes it rather than the
|
|
437
|
-
server.
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
on that PR measures the entry read-only, and the rotation publishes it after
|
|
444
|
-
merge.
|
|
405
|
+
server. Which machine a number applies to, and the failed record that made it
|
|
406
|
+
a rule, are in
|
|
407
|
+
[METHODOLOGY §which machine a number applies to](docs/METHODOLOGY.md#which-machine-a-number-applies-to).
|
|
408
|
+
To get your server into the leaderboard, add an entry to `servers.yaml` and
|
|
409
|
+
open a pull request; the check on that PR measures the entry read-only, and the
|
|
410
|
+
rotation publishes it after merge.
|
|
445
411
|
|
|
446
412
|
For a badge on your own README, run the published CLI in your server's own CI
|
|
447
413
|
(the [gate](#defend-the-number-dont-just-display-it) below writes
|
|
@@ -469,7 +435,7 @@ cp results/my-server/measurement.json .context-cost/baseline.json
|
|
|
469
435
|
|
|
470
436
|
# on every pull request
|
|
471
437
|
npx -y mcp-context-cost measure --name my-server --command "node dist/index.js" \
|
|
472
|
-
--baseline .context-cost/baseline.json --max-increase
|
|
438
|
+
--baseline .context-cost/baseline.json --max-increase 100
|
|
473
439
|
```
|
|
474
440
|
|
|
475
441
|
```
|
|
@@ -483,26 +449,13 @@ INCREASE FAIL: +121 tokens, over the 100 allowed — this change adds that to ev
|
|
|
483
449
|
|
|
484
450
|
Both sides are single measurements carrying per-tool counts, so an established
|
|
485
451
|
change is attributed exactly: which tools arrived, which grew, and by how much.
|
|
486
|
-
And `--max-increase` fails on more than the number
|
|
487
|
-
|
|
488
|
-
improvement is the one mistake a gate like this must not make, so a change that
|
|
489
|
-
could not be established fails too.
|
|
490
|
-
|
|
491
|
-
As a GitHub Action, that whole workflow is five lines
|
|
492
|
-
([full example](examples/server-author-ci.yml)):
|
|
493
|
-
|
|
494
|
-
```yaml
|
|
495
|
-
- uses: athakur3/mcp-context-cost@v1
|
|
496
|
-
with:
|
|
497
|
-
name: my-server
|
|
498
|
-
command: node dist/index.js
|
|
499
|
-
baseline: .context-cost/baseline.json
|
|
500
|
-
max-increase: 500
|
|
501
|
-
```
|
|
452
|
+
And `--max-increase` fails on more than the number here too: a change that could
|
|
453
|
+
not be established fails the gate, for the reason the `audit` gate gives above.
|
|
502
454
|
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
455
|
+
As a GitHub Action that whole workflow is the five lines at the top of this page
|
|
456
|
+
([full example](examples/server-author-ci.yml)) — the same inputs, and the same
|
|
457
|
+
outputs written whether the gate passed or not, for a later step to comment on
|
|
458
|
+
the pull request or publish as a badge.
|
|
506
459
|
|
|
507
460
|
Point the link at the measurement behind the number — for servers in this sweep that is
|
|
508
461
|
`https://athakur3.github.io/mcp-context-cost/servers/<name>.html`; otherwise the
|
|
@@ -522,8 +475,8 @@ npx tsc --noEmit # typecheck
|
|
|
522
475
|
npm run sweep:all -- --docker # full curated sweep (Docker isolation)
|
|
523
476
|
```
|
|
524
477
|
|
|
525
|
-
Notable engineering choices: the MCP client is a deliberate
|
|
526
|
-
|
|
478
|
+
Notable engineering choices: the MCP client is a deliberate raw-wire implementation
|
|
479
|
+
rather than the SDK (schema-parsing can reorder keys, which would corrupt canonical bytes);
|
|
527
480
|
sweep servers run in credential-free Docker containers with recorded isolation; the badge
|
|
528
481
|
color bands are frozen against the observed distribution of the first full sweep.
|
|
529
482
|
|
package/dist/audit/audit.d.ts
CHANGED
|
@@ -25,23 +25,23 @@ export interface AuditServerResult {
|
|
|
25
25
|
toolCount: number | null;
|
|
26
26
|
/** Share of this config's measured total, 0–1. */
|
|
27
27
|
share: number | null;
|
|
28
|
-
command?: string;
|
|
29
|
-
url?: string;
|
|
28
|
+
command?: string | undefined;
|
|
29
|
+
url?: string | undefined;
|
|
30
30
|
/** Names only — a server's env values never enter a report. */
|
|
31
31
|
envVarNames: string[];
|
|
32
32
|
/** Names only, and only for a remote entry that carries any — values never enter a report. */
|
|
33
|
-
headerNames?: string[];
|
|
33
|
+
headerNames?: string[] | undefined;
|
|
34
34
|
/** Claude Code's `alwaysLoad: true`, read from the entry: loads at session start whatever the setting. */
|
|
35
|
-
alwaysLoad?: true;
|
|
36
|
-
canonicalSha256?: string | null;
|
|
35
|
+
alwaysLoad?: true | undefined;
|
|
36
|
+
canonicalSha256?: string | null | undefined;
|
|
37
37
|
/**
|
|
38
38
|
* Anthropic-request cost from the published Claude divergence run, only when
|
|
39
39
|
* its captured hash matches this install (`--claude`). `null` means the
|
|
40
40
|
* install doesn't match what was published — silence, not a stale guess.
|
|
41
41
|
* `undefined` means `--claude` wasn't requested at all.
|
|
42
42
|
*/
|
|
43
|
-
claudeTokens?: number | null;
|
|
44
|
-
notes?: string;
|
|
43
|
+
claudeTokens?: number | null | undefined;
|
|
44
|
+
notes?: string | undefined;
|
|
45
45
|
}
|
|
46
46
|
export interface HeaviestTool {
|
|
47
47
|
server: string;
|
|
@@ -96,9 +96,9 @@ export interface AuditConfigResult {
|
|
|
96
96
|
heaviestTools: HeaviestTool[];
|
|
97
97
|
trimAdvice: TrimAdvice | null;
|
|
98
98
|
/** Present only when `--suggest` ran with a usable baseline. */
|
|
99
|
-
suggestions?: ConfigSuggestions;
|
|
99
|
+
suggestions?: ConfigSuggestions | undefined;
|
|
100
100
|
/** Present only when `--changed` ran with a usable capture index. */
|
|
101
|
-
captureVerdicts?: ServerCaptureVerdict[];
|
|
101
|
+
captureVerdicts?: ServerCaptureVerdict[] | undefined;
|
|
102
102
|
/**
|
|
103
103
|
* Whether this client loads the total up front or defers it, and — when the
|
|
104
104
|
* client decides that by a threshold — which side of it this stack is on.
|
|
@@ -171,30 +171,30 @@ export interface AuditReport {
|
|
|
171
171
|
* against the whole stack and the verdict fails rather than passing on a
|
|
172
172
|
* total that understates by an unknown amount.
|
|
173
173
|
*/
|
|
174
|
-
unestablished?: string[];
|
|
174
|
+
unestablished?: string[] | undefined;
|
|
175
175
|
/** Present only when over budget: the arithmetic of getting back under it. */
|
|
176
|
-
fit?: BudgetFit;
|
|
176
|
+
fit?: BudgetFit | undefined;
|
|
177
177
|
};
|
|
178
178
|
/** Present only when a divergence run was supplied (`--claude`). */
|
|
179
179
|
claudeDivergence?: {
|
|
180
180
|
model: string;
|
|
181
181
|
measuredAt: string;
|
|
182
|
-
};
|
|
182
|
+
} | undefined;
|
|
183
183
|
/** Which published tool-shape baseline `--suggest` read its percentiles from. */
|
|
184
184
|
toolShape?: {
|
|
185
185
|
generatedAt: string;
|
|
186
186
|
toolCount: number;
|
|
187
187
|
serverCount: number;
|
|
188
|
-
};
|
|
188
|
+
} | undefined;
|
|
189
189
|
/** Which published capture index `--changed` joined against. */
|
|
190
190
|
captureIndex?: {
|
|
191
191
|
generatedAt: string;
|
|
192
192
|
captureCount: number;
|
|
193
|
-
};
|
|
193
|
+
} | undefined;
|
|
194
194
|
/** Present only when a baseline report was supplied (`--baseline`). */
|
|
195
|
-
diff?: AuditDiff;
|
|
195
|
+
diff?: AuditDiff | undefined;
|
|
196
196
|
/** Present only when `--max-increase` was supplied alongside a baseline. */
|
|
197
|
-
increaseGate?: IncreaseGate;
|
|
197
|
+
increaseGate?: IncreaseGate | undefined;
|
|
198
198
|
problems: string[];
|
|
199
199
|
}
|
|
200
200
|
/** Cache key for measurement reuse: the exact argv two configs would spawn. */
|
|
@@ -218,38 +218,36 @@ export declare function collapsedKeys(configs: LoadedConfig[]): Set<string>;
|
|
|
218
218
|
* launching a single server.
|
|
219
219
|
*/
|
|
220
220
|
export declare function buildReport(configs: LoadedConfig[], measured: Map<string, Measurement>, opts?: {
|
|
221
|
-
contextWindow?: number;
|
|
222
|
-
budget?: number;
|
|
223
|
-
generatedAt?: string;
|
|
221
|
+
contextWindow?: number | undefined;
|
|
222
|
+
budget?: number | undefined;
|
|
223
|
+
generatedAt?: string | undefined;
|
|
224
224
|
/** Published `tools-delta/v1` run to join against (`--claude`); omit to skip the join. */
|
|
225
|
-
divergence?: DivergenceRun | null;
|
|
225
|
+
divergence?: DivergenceRun | null | undefined;
|
|
226
226
|
/** Published `tool-shape/v1` baseline (`--suggest`); omit to skip suggestions. */
|
|
227
|
-
toolShape?: ToolShapeBaseline | null;
|
|
227
|
+
toolShape?: ToolShapeBaseline | null | undefined;
|
|
228
228
|
/** Published `capture-index/v1` (`--changed`); omit to skip the version join. */
|
|
229
|
-
captureIndex?: CaptureIndex | null;
|
|
229
|
+
captureIndex?: CaptureIndex | null | undefined;
|
|
230
230
|
/**
|
|
231
231
|
* The audited machine's SHELL tool-search variables. Passed in rather than
|
|
232
232
|
* read here so this stays pure and a report is reproducible from its
|
|
233
233
|
* inputs; `runAudit` supplies the real environment. Omitted means the shell
|
|
234
234
|
* set nothing.
|
|
235
235
|
*/
|
|
236
|
-
env?: ToolSearchEnv;
|
|
236
|
+
env?: ToolSearchEnv | undefined;
|
|
237
237
|
/**
|
|
238
238
|
* The other place those variables come from: Claude Code's own settings
|
|
239
239
|
* files, highest precedence first, as `loadSettingsSources` read them.
|
|
240
240
|
* `runAudit` supplies these. Omitted means they were not read here — which
|
|
241
241
|
* the report says, rather than reporting a default it did not establish.
|
|
242
242
|
*/
|
|
243
|
-
settings?: ToolSearchSource[];
|
|
243
|
+
settings?: ToolSearchSource[] | undefined;
|
|
244
244
|
/**
|
|
245
245
|
* What each remote endpoint said to an unauthenticated `initialize`, keyed
|
|
246
246
|
* by `serverKey`. `runAudit` supplies it from `probeRemotes`; omitted, a
|
|
247
247
|
* remote entry is reported as not probed rather than as anything else.
|
|
248
248
|
*/
|
|
249
|
-
remotes?: Map<string, RemoteProbe
|
|
249
|
+
remotes?: Map<string, RemoteProbe> | undefined;
|
|
250
250
|
}): AuditReport;
|
|
251
251
|
/** Human output. JSON output is the report object itself. */
|
|
252
252
|
export declare function formatReport(report: AuditReport): string;
|
|
253
|
-
/** Top-level tool list across every config — used by nothing yet, handy for --json consumers. */
|
|
254
|
-
export declare function allHeaviestTools(report: AuditReport, limit?: number): HeaviestTool[];
|
|
255
253
|
export type { ToolMeasurement };
|
package/dist/audit/audit.js
CHANGED
|
@@ -18,6 +18,7 @@ import { SUGGEST_DESCRIPTION_PERCENTILE, suggestFor, } from '../core/tool-shape.
|
|
|
18
18
|
import { identify } from '../core/capture-index.js';
|
|
19
19
|
import { evaluateDeferral, BAND_PRECISION, PUBLISHED_WIRE_TO_CLIENT_RATIO, SHELL_SOURCE, } from './deferral.js';
|
|
20
20
|
import { formatDiff, formatGate } from './diff.js';
|
|
21
|
+
import { signed } from '../core/format.js';
|
|
21
22
|
export const DEFAULT_CONTEXT_WINDOW = 200_000;
|
|
22
23
|
const TRIM_TOOL_COUNT = 3;
|
|
23
24
|
function buildSuggestions(pool, baseline) {
|
|
@@ -52,7 +53,7 @@ function buildTrimAdvice(sortedTools, totalTokens) {
|
|
|
52
53
|
export function planBudgetFit(config, limit) {
|
|
53
54
|
const measured = config.servers
|
|
54
55
|
.filter((srv) => typeof srv.tokens === 'number' && srv.tokens > 0)
|
|
55
|
-
.
|
|
56
|
+
.toSorted((a, b) => b.tokens - a.tokens);
|
|
56
57
|
const overBy = config.totalTokens - limit;
|
|
57
58
|
const drop = [];
|
|
58
59
|
let remaining = config.totalTokens;
|
|
@@ -87,10 +88,14 @@ function envSignature(s) {
|
|
|
87
88
|
const env = s.env ?? {};
|
|
88
89
|
const headers = s.headers ?? {};
|
|
89
90
|
return JSON.stringify([
|
|
90
|
-
Object.keys(env)
|
|
91
|
+
Object.keys(env)
|
|
92
|
+
.toSorted()
|
|
93
|
+
.map((k) => [k, env[k]]),
|
|
91
94
|
// A remote's headers decide what it serves the way env decides for a
|
|
92
95
|
// process: a bearer token selects an account, and an account its tools.
|
|
93
|
-
Object.keys(headers)
|
|
96
|
+
Object.keys(headers)
|
|
97
|
+
.toSorted()
|
|
98
|
+
.map((k) => [k, headers[k]]),
|
|
94
99
|
]);
|
|
95
100
|
}
|
|
96
101
|
/**
|
|
@@ -252,7 +257,12 @@ export function buildReport(configs, measured, opts = {}) {
|
|
|
252
257
|
if (s.transport === 'remote') {
|
|
253
258
|
const probe = opts.remotes?.get(serverKey(s));
|
|
254
259
|
if (!probe) {
|
|
255
|
-
skipped.push({
|
|
260
|
+
skipped.push({
|
|
261
|
+
...base,
|
|
262
|
+
...none,
|
|
263
|
+
status: 'unreachable',
|
|
264
|
+
notes: `${s.url ?? 'url'} — not probed`,
|
|
265
|
+
});
|
|
256
266
|
continue;
|
|
257
267
|
}
|
|
258
268
|
if (probe.kind === 'auth-walled') {
|
|
@@ -283,14 +293,26 @@ export function buildReport(configs, measured, opts = {}) {
|
|
|
283
293
|
continue;
|
|
284
294
|
}
|
|
285
295
|
if (probe.kind === 'unreachable') {
|
|
286
|
-
skipped.push({
|
|
296
|
+
skipped.push({
|
|
297
|
+
...base,
|
|
298
|
+
...none,
|
|
299
|
+
status: 'unreachable',
|
|
300
|
+
notes: `${s.url}: ${probe.detail}`,
|
|
301
|
+
});
|
|
287
302
|
continue;
|
|
288
303
|
}
|
|
289
304
|
// Open: measured through the bridge, and read below like any launch.
|
|
290
305
|
}
|
|
291
306
|
const m = measured.get(serverKey(s));
|
|
292
307
|
if (!m) {
|
|
293
|
-
skipped.push({
|
|
308
|
+
skipped.push({
|
|
309
|
+
...base,
|
|
310
|
+
status: 'startup-failure',
|
|
311
|
+
tokens: null,
|
|
312
|
+
toolCount: null,
|
|
313
|
+
share: null,
|
|
314
|
+
notes: 'not measured',
|
|
315
|
+
});
|
|
294
316
|
continue;
|
|
295
317
|
}
|
|
296
318
|
if (!measuredOk(m)) {
|
|
@@ -314,7 +336,11 @@ export function buildReport(configs, measured, opts = {}) {
|
|
|
314
336
|
toolCount: m.toolCount,
|
|
315
337
|
share: null, // filled once the total is known
|
|
316
338
|
canonicalSha256: m.canonicalSha256,
|
|
317
|
-
claudeTokens: opts.divergence
|
|
339
|
+
claudeTokens: opts.divergence
|
|
340
|
+
? isCurrent(divRow, m.canonicalSha256 ?? null)
|
|
341
|
+
? divRow.claudeDelta
|
|
342
|
+
: null
|
|
343
|
+
: undefined,
|
|
318
344
|
notes: m.status === 'dynamic' ? redact(m.notes, secrets(s)) : undefined,
|
|
319
345
|
});
|
|
320
346
|
for (const t of m.tools) {
|
|
@@ -346,7 +372,10 @@ export function buildReport(configs, measured, opts = {}) {
|
|
|
346
372
|
trimAdvice: buildTrimAdvice(tools, totalTokens),
|
|
347
373
|
suggestions: opts.toolShape ? buildSuggestions(shapePool, opts.toolShape) : undefined,
|
|
348
374
|
captureVerdicts: opts.captureIndex
|
|
349
|
-
? ok.map((s) => ({
|
|
375
|
+
? ok.map((s) => ({
|
|
376
|
+
name: s.name,
|
|
377
|
+
verdict: identify(s.canonicalSha256, opts.captureIndex),
|
|
378
|
+
}))
|
|
350
379
|
: undefined,
|
|
351
380
|
};
|
|
352
381
|
built.push(result);
|
|
@@ -364,7 +393,10 @@ export function buildReport(configs, measured, opts = {}) {
|
|
|
364
393
|
problems,
|
|
365
394
|
};
|
|
366
395
|
if (opts.divergence) {
|
|
367
|
-
report.claudeDivergence = {
|
|
396
|
+
report.claudeDivergence = {
|
|
397
|
+
model: opts.divergence.model,
|
|
398
|
+
measuredAt: opts.divergence.measuredAt,
|
|
399
|
+
};
|
|
368
400
|
}
|
|
369
401
|
if (opts.toolShape) {
|
|
370
402
|
report.toolShape = {
|
|
@@ -466,8 +498,9 @@ function postureSourceLines(d) {
|
|
|
466
498
|
: '';
|
|
467
499
|
const held = r.state === 'unreadable'
|
|
468
500
|
? 'could not be read — what it sets is unknown'
|
|
469
|
-
: [r.sets.length ? `sets ${r.sets.join(', ')}` : '', unreadableVars]
|
|
470
|
-
|
|
501
|
+
: [r.sets.length ? `sets ${r.sets.join(', ')}` : '', unreadableVars]
|
|
502
|
+
.filter(Boolean)
|
|
503
|
+
.join(', and ') || 'sets none of them';
|
|
471
504
|
// Which place the verdict came out of, said once rather than left to a
|
|
472
505
|
// reader to work out from two lists.
|
|
473
506
|
const decided = d.setting?.source === r.source ? ', which decided this' : '';
|
|
@@ -477,7 +510,7 @@ function postureSourceLines(d) {
|
|
|
477
510
|
lines.push(` ${absent} other settings file(s) it reads are not on this machine`);
|
|
478
511
|
}
|
|
479
512
|
if (!recs.some((r) => r.scope !== 'shell')) {
|
|
480
|
-
lines.push(
|
|
513
|
+
lines.push(' its settings files were NOT read here, so what they set is unknown');
|
|
481
514
|
}
|
|
482
515
|
return lines;
|
|
483
516
|
}
|
|
@@ -602,8 +635,8 @@ function deferralLines(d, skippedNames) {
|
|
|
602
635
|
// Read from the entries, so it is stated up front rather than listed among
|
|
603
636
|
// the conditions a reader has to check: whatever the setting says, these load.
|
|
604
637
|
if (d.mechanism === 'tool search' && d.alwaysLoad.servers.length) {
|
|
605
|
-
const
|
|
606
|
-
lines.push(` ${
|
|
638
|
+
const pinned = d.alwaysLoad.servers.length;
|
|
639
|
+
lines.push(` ${pinned} server${pinned === 1 ? ' is' : 's are'} pinned "alwaysLoad": true and load${pinned === 1 ? 's' : ''} at session start whatever`);
|
|
607
640
|
lines.push(` the setting says: ${d.alwaysLoad.servers.join(', ')} — ${d.alwaysLoad.tokens.toLocaleString()} wire tokens,`);
|
|
608
641
|
lines.push(' left out of any threshold comparison below.');
|
|
609
642
|
}
|
|
@@ -808,8 +841,8 @@ export function formatReport(report) {
|
|
|
808
841
|
const skippedInScope = report.configs
|
|
809
842
|
.filter((c) => c.deferral === cfg.deferral)
|
|
810
843
|
.reduce((a, c) => a + c.skipped.length, 0);
|
|
811
|
-
for (const
|
|
812
|
-
lines.push(
|
|
844
|
+
for (const deferralLine of deferralLines(cfg.deferral, skippedInScope))
|
|
845
|
+
lines.push(deferralLine);
|
|
813
846
|
}
|
|
814
847
|
if (cfg.heaviestTools.length) {
|
|
815
848
|
lines.push('');
|
|
@@ -848,7 +881,7 @@ export function formatReport(report) {
|
|
|
848
881
|
// fact worth seeing rather than one to smooth over.
|
|
849
882
|
const alias = name === v.server ? name : `${name} (published as ${v.server})`;
|
|
850
883
|
lines.push(` ${alias} — you have the capture published ${v.yourDate} at ${n(v.yourTokens)} tokens; ` +
|
|
851
|
-
`the current one is ${n(v.currentTokens)} (${
|
|
884
|
+
`the current one is ${n(v.currentTokens)} (${signed(v.deltaTokens)}, ${v.currentDate})`);
|
|
852
885
|
}
|
|
853
886
|
lines.push(` updating all ${behind.length} would ${total >= 0 ? 'add' : 'remove'} ${n(Math.abs(total))} tokens ` +
|
|
854
887
|
`${total >= 0 ? 'to' : 'from'} every request in this client.`);
|
|
@@ -971,10 +1004,3 @@ export function formatReport(report) {
|
|
|
971
1004
|
`${PUBLISHED_WIRE_TO_CLIENT_RATIO.high.toFixed(BAND_PRECISION)}× on Anthropic requests. See docs/METHODOLOGY.md §claude-divergence.`);
|
|
972
1005
|
return lines.map((l) => l.replace(/\s+$/, '')).join('\n');
|
|
973
1006
|
}
|
|
974
|
-
/** Top-level tool list across every config — used by nothing yet, handy for --json consumers. */
|
|
975
|
-
export function allHeaviestTools(report, limit = 10) {
|
|
976
|
-
return report.configs
|
|
977
|
-
.flatMap((c) => c.heaviestTools)
|
|
978
|
-
.sort((a, b) => b.tokens - a.tokens)
|
|
979
|
-
.slice(0, limit);
|
|
980
|
-
}
|