context-doctor 0.13.4 → 0.14.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -4
- package/dist/calibration.d.ts +28 -0
- package/dist/calibration.js +75 -0
- package/dist/cli.js +69 -3
- package/dist/config.d.ts +2 -0
- package/dist/config.js +2 -2
- package/dist/doctor.js +19 -0
- package/dist/experiment.d.ts +58 -0
- package/dist/experiment.js +214 -0
- package/dist/install.d.ts +10 -1
- package/dist/install.js +67 -4
- package/dist/mcp.js +1 -1
- package/dist/optimize.d.ts +9 -0
- package/dist/optimize.js +53 -7
- package/dist/parse.d.ts +7 -0
- package/dist/parse.js +5 -2
- package/dist/profile.d.ts +9 -1
- package/dist/profile.js +32 -4
- package/dist/proxy.js +39 -2
- package/dist/report.js +5 -0
- package/dist/session.d.ts +17 -0
- package/dist/session.js +35 -0
- package/dist/statusline.d.ts +44 -0
- package/dist/statusline.js +104 -0
- package/dist/timing.d.ts +10 -0
- package/dist/timing.js +42 -0
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -92,8 +92,9 @@ Practical upshot: a developer who only wants cheaper, faster API calls never tou
|
|
|
92
92
|
| `context-doctor install` / `uninstall` | Wire (or remove) everything: MCP for Claude Desktop/Code/Cursor, the Agent Skill, the every-prompt hook |
|
|
93
93
|
| `context-doctor analyze <file>` | Profile a conversation: token breakdown, findings, cost + latency estimates. `--fail-over-budget` exits 1 on a breach, for CI |
|
|
94
94
|
| `context-doctor optimize <file>` | Apply the safe fixes; add `--strategy trim-tool-calls` for big inline file writes, `--strategy prune-history` for consented lossy compaction |
|
|
95
|
-
| `context-doctor session [file]` | Profile a Claude Code session: live context, findings, **measured tokens and prompt-cache economics
|
|
95
|
+
| `context-doctor session [file]` | Profile a Claude Code session: live context, findings, **measured tokens and prompt-cache economics**, and **where the wall clock went** per tool (from transcript timestamps, permission waits included and said so). Also reads ChatGPT data exports (`conversations.json`) |
|
|
96
96
|
| `context-doctor init [preset]` | Write a `.contextdoctorrc` from a preset (`chat`, `agent`, `batch`) — a budget you can adopt in one command and tune later |
|
|
97
|
+
| `context-doctor experiment --task "…"` | Run one task twice from the same commit, in a fresh session and forked from an `--existing` one, same model and tools; compare bill, cache split, wall clock, and whether `--check` passed. The only command here that spends money, so it caps spend per arm and refuses a dirty tree |
|
|
97
98
|
| `context-doctor diff <before> <after>` | Compare two profiles: what moved by category, which findings were resolved or introduced, and what it saves in money and latency |
|
|
98
99
|
| `context-doctor accuracy` | How much of what you are billed for is visible in your transcript — the fixed harness baseline and the per-turn injected content neither you nor the profiler can see |
|
|
99
100
|
| `context-doctor cursor [--list]` | Profile a chat from Cursor's local history (both storage formats) |
|
|
@@ -102,6 +103,7 @@ Practical upshot: a developer who only wants cheaper, faster API calls never tou
|
|
|
102
103
|
| `context-doctor watch [file]` | Live monitor of a growing session/agent trace: token/cost line per change, findings as they appear |
|
|
103
104
|
| `context-doctor doctor` | Self-check the whole installation — one pasteable ✓/✗ diagnosis with fixes |
|
|
104
105
|
| `context-doctor dashboard` | Local savings dashboard on 127.0.0.1: tokens saved per day, sessions by context in use vs recoverable, budget status |
|
|
106
|
+
| `context-doctor statusline` | Claude Code status bar: live context vs window, cache share, cost. Wired by `install --statusline`; never overwrites a statusLine you already have |
|
|
105
107
|
| `context-doctor hook` | The every-prompt Claude Code hook (registered by `install`; you never run this yourself). Warning threshold tunable via `CONTEXT_DOCTOR_WARN_TOKENS` (default 80000) |
|
|
106
108
|
| `context-doctor-mcp` | The MCP server itself — stdio by default (what the installer wires); `--http [--port 8808] [--host H]` serves streamable HTTP at `/mcp` for URL-based clients like ChatGPT developer-mode connectors |
|
|
107
109
|
|
|
@@ -168,7 +170,7 @@ The proxy dedupes repeated content, trims stale tool results, and strips base64
|
|
|
168
170
|
[context-doctor] POST /v1/messages → 200 in 842ms | optimized 7.3k → 518 tokens (2 changes) | session total: 6.9k tokens ≈ $0.021 saved
|
|
169
171
|
```
|
|
170
172
|
|
|
171
|
-
`GET http://localhost:8787/stats` returns cumulative savings (requests, tokens, estimated USD), **exact upstream usage** read from every response (JSON and SSE), and **prompt-cache advisories** — the proxy watches your real traffic and flags big stable prefixes missing `cache_control` or prefix churn that silently re-bills the cache. Per-model behavior via `--config`:
|
|
173
|
+
`GET http://localhost:8787/stats` returns cumulative savings (requests, tokens, estimated USD), **exact upstream usage** read from every response (JSON and SSE), and **prompt-cache advisories** — the proxy watches your real traffic and flags big stable prefixes missing `cache_control` or prefix churn that silently re-bills the cache. Per-model behavior via `--config`: The advisor now says *where*: for a large system/tools prefix it names the block to mark (the last tool definition, or the last system block), and when the older messages were byte-identical to the previous request it names the exact message to put `cache_control` on, with the token count that run is re-billing each turn. Anyone who already placed breakpoints is left alone.
|
|
172
174
|
|
|
173
175
|
```json
|
|
174
176
|
{ "routes": [{ "modelPrefix": "gpt", "strategies": ["strip-base64"], "keepRecent": 4 }] }
|
|
@@ -246,6 +248,57 @@ const { conversation, tokensBefore, tokensAfter } = optimizeConversation(chatJso
|
|
|
246
248
|
});
|
|
247
249
|
```
|
|
248
250
|
|
|
251
|
+
## Context health in Claude Code's status bar
|
|
252
|
+
|
|
253
|
+
```bash
|
|
254
|
+
context-doctor install --statusline
|
|
255
|
+
```
|
|
256
|
+
|
|
257
|
+
Claude Code shows the first line a `statusLine` command prints, on every refresh, while you type. With this on, that line is the number that matters:
|
|
258
|
+
|
|
259
|
+
```
|
|
260
|
+
ctx 801k/1.0M ▮▮▮▮▮▮▮▮░░ 80% ⚠ · cache 100% · $12.34
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
Live context against the model's window, a warning mark from 70%, the share served from cache, and the session's cost. It reads the size from the status payload when Claude Code provides it, and otherwise from the last 256KB of the transcript (about 1ms on a 20MB file; 80ms end to end including Node startup). It is opt-in and polite: there is only one status line, so it never overwrites one you already have, and `uninstall` removes only its own. Any failure prints nothing rather than an error.
|
|
264
|
+
|
|
265
|
+
This is the "editor status bar" roadmap item, delivered for the editor most users of this tool are actually in. A VS Code / Cursor extension for the same number remains future work.
|
|
266
|
+
|
|
267
|
+
## Does a smaller context actually help? Measure it
|
|
268
|
+
|
|
269
|
+
Everything else in this tool measures what is *in* the context. None of it can tell you whether the task succeeded, so a smaller transcript can be a cheaper failure. `experiment` is the honest test:
|
|
270
|
+
|
|
271
|
+
```bash
|
|
272
|
+
context-doctor experiment \
|
|
273
|
+
--task "Add a null check to parseHeader in src/parse.ts and make the tests pass" \
|
|
274
|
+
--check "npm test" \
|
|
275
|
+
--existing 9c6f9dc9-457b-4d09-bf6d-a499c2f2f919 \
|
|
276
|
+
--model sonnet --budget 1
|
|
277
|
+
```
|
|
278
|
+
|
|
279
|
+
It runs the task in a **fresh** session, resets the tree to the starting commit, then runs it again **forked from your existing session** (`--resume … --fork-session`, so your real session is never touched), same model, same tools. For each arm it records what you were billed (input, cache read, cache write, output), cost, wall clock, turns, and whether your `--check` command passed, then puts them side by side:
|
|
280
|
+
|
|
281
|
+
```
|
|
282
|
+
fresh existing
|
|
283
|
+
──────────────────────────────────────────────────
|
|
284
|
+
billed input 20k 65k
|
|
285
|
+
of which cache read 0 60k
|
|
286
|
+
of which cache write 8.0k 4.0k
|
|
287
|
+
cost $0.110 $0.420
|
|
288
|
+
wall clock 4s 9s
|
|
289
|
+
check PASS FAIL (1)
|
|
290
|
+
|
|
291
|
+
Verdict: fresh was cheaper AND passed; existing failed the check. Clear win for fresh.
|
|
292
|
+
```
|
|
293
|
+
|
|
294
|
+
The verdict line is the point: cheaper only counts if it also passed. Because this spends your Claude budget it caps spend per arm (`--budget`, default $1), refuses to start on a dirty tree (both arms must begin from one commit, and the tree is reset between them), and refuses to run from inside a Claude Code session, where the CLI cannot start. `--dry-run` shows the exact commands first.
|
|
295
|
+
|
|
296
|
+
## Exact counts, and what they teach the estimator
|
|
297
|
+
|
|
298
|
+
The default token count is a chars-per-token heuristic so everything runs with no key and no tokenizer. Its error is content-dependent, and there is no honest way to fix that from transcripts alone (the billed number includes content the transcript never sees). `analyze --exact` fetches a true count for the exact bytes just estimated (Anthropic's count-tokens API with `ANTHROPIC_API_KEY`; tiktoken for GPT if installed) and prints the drift.
|
|
299
|
+
|
|
300
|
+
Since 0.13.9 it also **remembers the comparison**, per model family, on this machine, and later estimates for that family are scaled by it. Nothing about this is silent: the profile header says `estimates calibrated +12% from 3 exact count(s) you ran on this machine`. No exact count ever run means no calibration and unchanged numbers; out-of-range samples are ignored; `CONTEXT_DOCTOR_NO_CALIBRATION=1` returns to the raw heuristic.
|
|
301
|
+
|
|
249
302
|
## What it detects
|
|
250
303
|
|
|
251
304
|
- **Oversized tool results** — the #1 context killer in agent loops
|
|
@@ -254,7 +307,7 @@ const { conversation, tokensBefore, tokensAfter } = optimizeConversation(chatJso
|
|
|
254
307
|
- **Near-duplicates** — the same doc re-pasted with different surrounding words (shingle similarity, ≥60%)
|
|
255
308
|
- **Repeated file reads** — the same file pulled in three or more times, every copy still in context. Counts shell reads too (`cat`, `head`, `tail`, `less`), which is where most of them hide in agent sessions
|
|
256
309
|
- **Retained error output** — stack traces and failed commands kept verbatim long after the fix landed
|
|
257
|
-
- **Repeated identical tool calls**
|
|
310
|
+
- **Repeated identical tool calls**, split into the two things they can mean: a **retry** (the same call after a failure, where the fix is in the error text, and three or more is a loop) and a **re-read** (the same call after a success, where the model forgot it already had the answer). Across 42 local sessions that was 15 retries against 151 re-reads, so the old combined advice was wrong for most of them
|
|
258
311
|
- **Base64 / binary blobs** in text content — checked by character distribution, not just alphabet, so hex digests and long identifiers are not mistaken for encoded binary
|
|
259
312
|
- **Long history** past the point where models track the middle
|
|
260
313
|
- **Cache-hostile ordering** — volatile content before stable content breaks prompt caching (Anthropic `cache_control`, OpenAI automatic prefix caching)
|
|
@@ -270,10 +323,12 @@ const { conversation, tokensBefore, tokensAfter } = optimizeConversation(chatJso
|
|
|
270
323
|
| `trim-tool-calls` — shrink the arguments of calls that already ran | Mostly no | opt-in |
|
|
271
324
|
| `prune-history` — collapse old turns into a stub for summarization | Yes | opt-in |
|
|
272
325
|
|
|
273
|
-
`trim-tool-calls` is the big one for agent sessions. Writing a file through a tool call puts the entire file in context permanently, so in file-heavy work the calls outweigh every tool result combined — on a real 278k-token session, the default set reached 248k and adding `trim-tool-calls` reached 102k. It is opt-in because it edits what the model itself wrote.
|
|
326
|
+
`trim-tool-calls` is the big one for agent sessions. Writing a file through a tool call puts the entire file in context permanently, so in file-heavy work the calls outweigh every tool result combined — on a real 278k-token session, the default set reached 248k and adding `trim-tool-calls` reached 102k. It is opt-in because it edits what the model itself wrote, and that is measured, not cautious: across 42 real sessions, of 73 large writes 18 were later edited and **16 of those edits had no read in between**, meaning the model built the edit from its own earlier `Write` input, at a median distance of 43 messages. Trimming those would turn each into a failed edit plus a recovery read. In the offline optimizer the rest of the conversation is known, so exactly those writes are left intact and the 63% never touched again are trimmed; in the live proxy the future is not known, which is why it stays off there unless you turn it on.
|
|
274
327
|
|
|
275
328
|
**Optimization is cache-aware.** Prompt caches match a byte-identical prefix, so editing a message in the middle invalidates everything after it — and the naive "trim everything older than the last N messages" boundary moves every single turn. On a 25-turn agent conversation that invalidated the cached prefix on 22 of 24 turns, paying the 1.25x cache-write price on the whole prefix to save a few hundred tokens. The trim boundary is quantized so it holds still between steps (8 of 24 on the same fixture), while still reaching 15 of 20 tool results.
|
|
276
329
|
|
|
330
|
+
The step size is a trade-off, not a formula, and it is yours to set: `"trimBoundaryStep": 20` in `.contextdoctorrc` (default 10). Measured on a growing agent session at 400 turns: a step of 10 invalidated the cache on 21% of turns with ~2 stale results waiting on average; 20 gave 12% and ~4; 40 gave 10% and ~9. Adaptive steps were worse everywhere, because a step that changes size moves the boundary by itself. Heavy API users who lean on caching want a bigger step; interactive users who want stale output gone promptly want a smaller one.
|
|
331
|
+
|
|
277
332
|
Everything the optimizer does is inspectable: it prints exactly which messages changed and how many tokens each change saved.
|
|
278
333
|
|
|
279
334
|
**Summarization without an API key:** when `prune-history` runs through the MCP tools, context-doctor hands a digest of the pruned turns back to the model that called it (the Claude/GPT already running in your app) and asks *it* to write the replacement summary — LLM-quality compaction, zero extra cost, no keys.
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Heuristic calibration from the user's own exact counts.
|
|
3
|
+
*
|
|
4
|
+
* The chars-per-token heuristic is what lets everything run with no key and
|
|
5
|
+
* no tokenizer, and its error is content-dependent: dense JSON tokenizes very
|
|
6
|
+
* differently from prose. There is no honest way to fix that from transcripts
|
|
7
|
+
* alone (the billed number includes content the transcript never sees). But
|
|
8
|
+
* when someone runs `analyze --exact`, they fetch a true count for the exact
|
|
9
|
+
* bytes the heuristic just estimated — the one clean comparison available.
|
|
10
|
+
*
|
|
11
|
+
* So that comparison is remembered, per model family, on this machine, and
|
|
12
|
+
* applied to later estimates for the same family. Nothing is applied silently:
|
|
13
|
+
* the profile carries the factor and the sample count, and the report prints
|
|
14
|
+
* them. No exact count ever run means no calibration, and the numbers are
|
|
15
|
+
* exactly what they were before.
|
|
16
|
+
*/
|
|
17
|
+
export interface Calibration {
|
|
18
|
+
/** Multiply heuristic estimates by this. 1 means uncalibrated. */
|
|
19
|
+
factor: number;
|
|
20
|
+
samples: number;
|
|
21
|
+
}
|
|
22
|
+
export declare function calibrationPath(): string;
|
|
23
|
+
/** "claude", "gpt", "gemini", … — coarse on purpose; tokenizers are per vendor. */
|
|
24
|
+
export declare function modelFamily(model?: string): string;
|
|
25
|
+
/** Remember one exact-vs-heuristic observation. Best-effort; never throws. */
|
|
26
|
+
export declare function recordCalibration(model: string | undefined, exactTokens: number, heuristicTokens: number): void;
|
|
27
|
+
/** The factor to apply for a model, or 1 with zero samples when there is none. */
|
|
28
|
+
export declare function calibrationFor(model?: string): Calibration;
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Heuristic calibration from the user's own exact counts.
|
|
3
|
+
*
|
|
4
|
+
* The chars-per-token heuristic is what lets everything run with no key and
|
|
5
|
+
* no tokenizer, and its error is content-dependent: dense JSON tokenizes very
|
|
6
|
+
* differently from prose. There is no honest way to fix that from transcripts
|
|
7
|
+
* alone (the billed number includes content the transcript never sees). But
|
|
8
|
+
* when someone runs `analyze --exact`, they fetch a true count for the exact
|
|
9
|
+
* bytes the heuristic just estimated — the one clean comparison available.
|
|
10
|
+
*
|
|
11
|
+
* So that comparison is remembered, per model family, on this machine, and
|
|
12
|
+
* applied to later estimates for the same family. Nothing is applied silently:
|
|
13
|
+
* the profile carries the factor and the sample count, and the report prints
|
|
14
|
+
* them. No exact count ever run means no calibration, and the numbers are
|
|
15
|
+
* exactly what they were before.
|
|
16
|
+
*/
|
|
17
|
+
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
18
|
+
import { dirname, join } from "node:path";
|
|
19
|
+
import { statePath } from "./ledger.js";
|
|
20
|
+
/** Anything outside this is a bad sample, not a calibration. */
|
|
21
|
+
const MIN_FACTOR = 0.5;
|
|
22
|
+
const MAX_FACTOR = 2.0;
|
|
23
|
+
export function calibrationPath() {
|
|
24
|
+
return join(dirname(statePath()), ".context-doctor-calibration.json");
|
|
25
|
+
}
|
|
26
|
+
/** "claude", "gpt", "gemini", … — coarse on purpose; tokenizers are per vendor. */
|
|
27
|
+
export function modelFamily(model) {
|
|
28
|
+
const m = (model ?? "").toLowerCase();
|
|
29
|
+
if (m.includes("claude"))
|
|
30
|
+
return "claude";
|
|
31
|
+
if (/gpt|^o\d/.test(m))
|
|
32
|
+
return "gpt";
|
|
33
|
+
if (m.includes("gemini"))
|
|
34
|
+
return "gemini";
|
|
35
|
+
return m ? m.split(/[-_/]/)[0] : "unknown";
|
|
36
|
+
}
|
|
37
|
+
function readAll() {
|
|
38
|
+
try {
|
|
39
|
+
return JSON.parse(readFileSync(calibrationPath(), "utf8"));
|
|
40
|
+
}
|
|
41
|
+
catch {
|
|
42
|
+
return {};
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
/** Remember one exact-vs-heuristic observation. Best-effort; never throws. */
|
|
46
|
+
export function recordCalibration(model, exactTokens, heuristicTokens) {
|
|
47
|
+
if (!(exactTokens > 0) || !(heuristicTokens > 0))
|
|
48
|
+
return;
|
|
49
|
+
const ratio = exactTokens / heuristicTokens;
|
|
50
|
+
if (ratio < MIN_FACTOR || ratio > MAX_FACTOR)
|
|
51
|
+
return; // a broken sample must not poison the file
|
|
52
|
+
try {
|
|
53
|
+
const all = readAll();
|
|
54
|
+
const key = modelFamily(model);
|
|
55
|
+
const rec = all[key] ?? { exactSum: 0, heuristicSum: 0, samples: 0 };
|
|
56
|
+
all[key] = { exactSum: rec.exactSum + exactTokens, heuristicSum: rec.heuristicSum + heuristicTokens, samples: rec.samples + 1 };
|
|
57
|
+
mkdirSync(dirname(calibrationPath()), { recursive: true });
|
|
58
|
+
writeFileSync(calibrationPath(), JSON.stringify(all, null, 2));
|
|
59
|
+
}
|
|
60
|
+
catch {
|
|
61
|
+
/* calibration is a refinement; failing to save it must not fail the command */
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
/** The factor to apply for a model, or 1 with zero samples when there is none. */
|
|
65
|
+
export function calibrationFor(model) {
|
|
66
|
+
if (process.env.CONTEXT_DOCTOR_NO_CALIBRATION)
|
|
67
|
+
return { factor: 1, samples: 0 };
|
|
68
|
+
const rec = readAll()[modelFamily(model)];
|
|
69
|
+
if (!rec || rec.samples < 1 || rec.heuristicSum <= 0)
|
|
70
|
+
return { factor: 1, samples: 0 };
|
|
71
|
+
const factor = rec.exactSum / rec.heuristicSum;
|
|
72
|
+
if (!Number.isFinite(factor) || factor < MIN_FACTOR || factor > MAX_FACTOR)
|
|
73
|
+
return { factor: 1, samples: 0 };
|
|
74
|
+
return { factor, samples: rec.samples };
|
|
75
|
+
}
|
package/dist/cli.js
CHANGED
|
@@ -24,13 +24,17 @@ import { recordLedger } from "./ledger.js";
|
|
|
24
24
|
import { runDoctor } from "./doctor.js";
|
|
25
25
|
import { measureAccuracy, renderAccuracy } from "./accuracy.js";
|
|
26
26
|
import { renderDiff } from "./diff.js";
|
|
27
|
+
import { renderExperiment, runExperiment } from "./experiment.js";
|
|
28
|
+
import { runStatusLine } from "./statusline.js";
|
|
27
29
|
import { findPreset, PRESETS, RC_FILENAME } from "./config.js";
|
|
28
30
|
import { runWatch } from "./watch.js";
|
|
29
31
|
import { exactTokenCount } from "./exact.js";
|
|
32
|
+
import { modelFamily, recordCalibration } from "./calibration.js";
|
|
30
33
|
import { checkBudget, loadConfig } from "./config.js";
|
|
31
34
|
import { startDashboard } from "./dashboard.js";
|
|
32
35
|
import { listCursorChats, parseCursorChat } from "./cursor.js";
|
|
33
36
|
import { analyzeCacheUsage, renderCacheReport } from "./cache.js";
|
|
37
|
+
import { renderToolTimings } from "./timing.js";
|
|
34
38
|
const HELP = `context-doctor — profile and optimize LLM context windows
|
|
35
39
|
|
|
36
40
|
Usage:
|
|
@@ -44,6 +48,9 @@ Usage:
|
|
|
44
48
|
context-doctor session [file] Profile a Claude Code session transcript or a
|
|
45
49
|
ChatGPT export (default: most recent; --list to browse)
|
|
46
50
|
context-doctor cursor [--list] Profile a Cursor chat from its local history
|
|
51
|
+
context-doctor statusline Claude Code status bar line: live context size, cache
|
|
52
|
+
share, cost (wired by \`install --statusline\`; reads
|
|
53
|
+
the status JSON on stdin)
|
|
47
54
|
context-doctor hook Claude Code UserPromptSubmit hook (installed
|
|
48
55
|
automatically by \`install\`; reads hook JSON on stdin)
|
|
49
56
|
context-doctor report Impact report: exact proxy savings, hook activity,
|
|
@@ -54,6 +61,9 @@ Usage:
|
|
|
54
61
|
default 8790) — charts from your own machine only
|
|
55
62
|
context-doctor init [preset] Write a .contextdoctorrc from a preset
|
|
56
63
|
(chat, agent, batch; no argument lists them)
|
|
64
|
+
context-doctor experiment --task "<t>" Run one task twice from the same commit, in a fresh
|
|
65
|
+
session and forked from an --existing one; compare
|
|
66
|
+
bill, cache, time, and whether --check passed
|
|
57
67
|
context-doctor diff <before> <after> Compare two profiles: what moved, which findings
|
|
58
68
|
were resolved, and what it saves
|
|
59
69
|
context-doctor accuracy Measure the token heuristic against the API's own
|
|
@@ -87,6 +97,13 @@ Options:
|
|
|
87
97
|
--keep-recent <n> (optimize) Messages at the tail to leave untouched (default 6)
|
|
88
98
|
--max-tool-tokens <n> (optimize) Token budget for trimmed tool results (default 300)
|
|
89
99
|
--limit <n> (accuracy) Sessions to sample (default 20)
|
|
100
|
+
--check <cmd> (experiment) Command whose exit code is the pass/fail for each arm
|
|
101
|
+
--existing <id> (experiment) Session id to fork the second arm from (never mutated)
|
|
102
|
+
--budget <usd> (experiment) Spend cap per arm (default 1)
|
|
103
|
+
--dry-run (experiment) Print the claude commands and stop
|
|
104
|
+
--allow-dirty (experiment) Skip the clean-tree check (uncommitted changes will be lost)
|
|
105
|
+
--statusline (install) Also set Claude Code's statusLine to context-doctor (never
|
|
106
|
+
overwrites a statusLine you already have)
|
|
90
107
|
--port <n> (proxy) Port to listen on (default 8787)
|
|
91
108
|
--host <addr> (proxy) Bind address (default 127.0.0.1; use 0.0.0.0 to expose)
|
|
92
109
|
--config <file> (proxy) Per-route overrides: {"routes":[{"modelPrefix":"gpt","strategies":[...],
|
|
@@ -103,7 +120,7 @@ Examples:
|
|
|
103
120
|
export OPENAI_BASE_URL=http://localhost:8787/v1
|
|
104
121
|
`;
|
|
105
122
|
function parseArgs(argv) {
|
|
106
|
-
const args = { json: false, strategies: [], list: false, exact: false, redact: false, failOverBudget: false };
|
|
123
|
+
const args = { json: false, strategies: [], list: false, exact: false, redact: false, failOverBudget: false, dryRun: false, allowDirty: false, statusLine: false };
|
|
107
124
|
const positional = [];
|
|
108
125
|
for (let i = 0; i < argv.length; i++) {
|
|
109
126
|
const a = argv[i];
|
|
@@ -151,6 +168,27 @@ function parseArgs(argv) {
|
|
|
151
168
|
case "--limit":
|
|
152
169
|
args.limit = Number(argv[++i]);
|
|
153
170
|
break;
|
|
171
|
+
case "--task":
|
|
172
|
+
args.task = argv[++i];
|
|
173
|
+
break;
|
|
174
|
+
case "--check":
|
|
175
|
+
args.check = argv[++i];
|
|
176
|
+
break;
|
|
177
|
+
case "--existing":
|
|
178
|
+
args.existing = argv[++i];
|
|
179
|
+
break;
|
|
180
|
+
case "--budget":
|
|
181
|
+
args.budgetUsd = Number(argv[++i]);
|
|
182
|
+
break;
|
|
183
|
+
case "--dry-run":
|
|
184
|
+
args.dryRun = true;
|
|
185
|
+
break;
|
|
186
|
+
case "--allow-dirty":
|
|
187
|
+
args.allowDirty = true;
|
|
188
|
+
break;
|
|
189
|
+
case "--statusline":
|
|
190
|
+
args.statusLine = true;
|
|
191
|
+
break;
|
|
154
192
|
case "--host":
|
|
155
193
|
args.host = argv[++i];
|
|
156
194
|
break;
|
|
@@ -243,6 +281,22 @@ function main() {
|
|
|
243
281
|
console.log(" Gate a pull request on it with: context-doctor analyze <file> --fail-over-budget");
|
|
244
282
|
return;
|
|
245
283
|
}
|
|
284
|
+
if (args.command === "statusline") {
|
|
285
|
+
void runStatusLine();
|
|
286
|
+
return;
|
|
287
|
+
}
|
|
288
|
+
if (args.command === "experiment") {
|
|
289
|
+
if (!args.task) {
|
|
290
|
+
console.error('Usage: context-doctor experiment --task "<what to do>" [--check "<cmd>"] [--existing <session-id>] [--model m] [--budget usd] [--dry-run]');
|
|
291
|
+
process.exit(1);
|
|
292
|
+
}
|
|
293
|
+
const opts = { task: args.task, check: args.check, existing: args.existing, model: args.model, budgetUsd: args.budgetUsd, dryRun: args.dryRun, allowDirty: args.allowDirty };
|
|
294
|
+
const result = runExperiment(opts);
|
|
295
|
+
console.log(args.json ? JSON.stringify(result, null, 2) : renderExperiment(result, opts));
|
|
296
|
+
if (result.refused)
|
|
297
|
+
process.exitCode = 1;
|
|
298
|
+
return;
|
|
299
|
+
}
|
|
246
300
|
if (args.command === "diff") {
|
|
247
301
|
const [before, after] = args.positionals ?? [];
|
|
248
302
|
if (!before || !after) {
|
|
@@ -333,7 +387,7 @@ function main() {
|
|
|
333
387
|
}
|
|
334
388
|
const profile = profileConversation(parseConversation(parsed.conversationJson), args.model ?? parsed.model);
|
|
335
389
|
if (args.json) {
|
|
336
|
-
console.log(JSON.stringify({ session: { path: parsed.path, title: parsed.title }, profile }, null, 2));
|
|
390
|
+
console.log(JSON.stringify({ session: { path: parsed.path, title: parsed.title, toolTimings: parsed.toolTimings ?? [] }, profile }, null, 2));
|
|
337
391
|
}
|
|
338
392
|
else {
|
|
339
393
|
console.log(`Session: ${parsed.title ?? "(untitled)"}\nFile: ${parsed.path}`);
|
|
@@ -354,6 +408,11 @@ function main() {
|
|
|
354
408
|
console.log("");
|
|
355
409
|
console.log(cache);
|
|
356
410
|
}
|
|
411
|
+
const timing = renderToolTimings(parsed.toolTimings ?? []);
|
|
412
|
+
if (timing) {
|
|
413
|
+
console.log("");
|
|
414
|
+
console.log(timing);
|
|
415
|
+
}
|
|
357
416
|
applyBudgetGate(printBudgetStatus(profile, loadConfig(process.cwd(), (m) => console.error(`context-doctor: ${m}`))), args.failOverBudget);
|
|
358
417
|
}
|
|
359
418
|
return;
|
|
@@ -361,7 +420,7 @@ function main() {
|
|
|
361
420
|
if (args.command === "install") {
|
|
362
421
|
// Partial success is still installed, but not silent: any failed target
|
|
363
422
|
// makes the exit code non-zero so automation can react.
|
|
364
|
-
if (runInstall().failures.length > 0)
|
|
423
|
+
if (runInstall({ statusLine: args.statusLine }).failures.length > 0)
|
|
365
424
|
process.exitCode = 1;
|
|
366
425
|
return;
|
|
367
426
|
}
|
|
@@ -390,6 +449,7 @@ function main() {
|
|
|
390
449
|
strategies: args.strategies.length > 0 ? args.strategies : loadedRc.config.strategies,
|
|
391
450
|
keepRecent: args.keepRecent ?? loadedRc.config.keepRecent,
|
|
392
451
|
maxToolResultTokens: args.maxToolTokens ?? loadedRc.config.maxToolResultTokens,
|
|
452
|
+
trimBoundaryStep: loadedRc.config.trimBoundaryStep,
|
|
393
453
|
});
|
|
394
454
|
return; // server keeps the process alive
|
|
395
455
|
}
|
|
@@ -418,6 +478,11 @@ function main() {
|
|
|
418
478
|
if (exact.tokens !== undefined) {
|
|
419
479
|
const drift = profile.totalTokens > 0 ? Math.round(((exact.tokens - profile.totalTokens) / exact.tokens) * 100) : 0;
|
|
420
480
|
console.log(`\nExact input tokens: ${exact.tokens} (${exact.source}) — heuristic was off by ${drift}%`);
|
|
481
|
+
// Remember the comparison so the next estimate for this model family
|
|
482
|
+
// starts from the user's own ground truth instead of a constant.
|
|
483
|
+
const raw = profile.calibration ? profile.totalTokens / profile.calibration.factor : profile.totalTokens;
|
|
484
|
+
recordCalibration(args.model ?? profile.model, exact.tokens, raw);
|
|
485
|
+
console.log(`Remembered: future ${modelFamily(args.model ?? profile.model)} estimates on this machine are calibrated from this (CONTEXT_DOCTOR_NO_CALIBRATION=1 to disable).`);
|
|
421
486
|
}
|
|
422
487
|
else {
|
|
423
488
|
console.log(`\nExact count unavailable: ${exact.note}`);
|
|
@@ -434,6 +499,7 @@ function main() {
|
|
|
434
499
|
strategies: args.strategies.length > 0 ? args.strategies : loaded.config.strategies,
|
|
435
500
|
keepRecent: args.keepRecent ?? loaded.config.keepRecent,
|
|
436
501
|
maxToolResultTokens: args.maxToolTokens ?? loaded.config.maxToolResultTokens,
|
|
502
|
+
trimBoundaryStep: loaded.config.trimBoundaryStep,
|
|
437
503
|
});
|
|
438
504
|
}
|
|
439
505
|
catch (e) {
|
package/dist/config.d.ts
CHANGED
|
@@ -25,6 +25,8 @@ export interface ContextDoctorConfig {
|
|
|
25
25
|
strategies?: StrategyId[];
|
|
26
26
|
keepRecent?: number;
|
|
27
27
|
maxToolResultTokens?: number;
|
|
28
|
+
/** Messages the trim boundary moves at a time; see OptimizeOptions. */
|
|
29
|
+
trimBoundaryStep?: number;
|
|
28
30
|
/** Proxy per-model overrides, same shape as `proxy --config`. */
|
|
29
31
|
routes?: Array<{
|
|
30
32
|
modelPrefix: string;
|
package/dist/config.js
CHANGED
|
@@ -38,7 +38,7 @@ function candidatePaths(startDir) {
|
|
|
38
38
|
*/
|
|
39
39
|
/** Strategy ids the optimizer actually implements. */
|
|
40
40
|
const KNOWN_STRATEGIES = new Set(["dedupe", "trim-tool-results", "trim-tool-calls", "strip-base64", "prune-history"]);
|
|
41
|
-
const KNOWN_KEYS = new Set(["budget", "strategies", "keepRecent", "maxToolResultTokens", "routes", "model"]);
|
|
41
|
+
const KNOWN_KEYS = new Set(["budget", "strategies", "keepRecent", "maxToolResultTokens", "trimBoundaryStep", "routes", "model"]);
|
|
42
42
|
const KNOWN_BUDGET_KEYS = new Set(["maxTokens", "maxCostPerMessageUsd", "maxWindowPct"]);
|
|
43
43
|
/**
|
|
44
44
|
* Report anything in an rc file that will be silently ignored.
|
|
@@ -92,7 +92,7 @@ export function validateConfig(config, path) {
|
|
|
92
92
|
}
|
|
93
93
|
}
|
|
94
94
|
}
|
|
95
|
-
for (const key of ["keepRecent", "maxToolResultTokens"]) {
|
|
95
|
+
for (const key of ["keepRecent", "maxToolResultTokens", "trimBoundaryStep"]) {
|
|
96
96
|
const value = c[key];
|
|
97
97
|
if (value === undefined)
|
|
98
98
|
continue;
|
package/dist/doctor.js
CHANGED
|
@@ -135,6 +135,25 @@ export async function runDoctor() {
|
|
|
135
135
|
else {
|
|
136
136
|
checks.push({ label: "Every-prompt hook", status: "skip", detail: "Claude Code not detected" });
|
|
137
137
|
}
|
|
138
|
+
// Status line (opt-in, so absence is a note, not a failure)
|
|
139
|
+
if (existsSync(settingsPath)) {
|
|
140
|
+
try {
|
|
141
|
+
const settings = JSON.parse(readFileSync(settingsPath, "utf8"));
|
|
142
|
+
const cmd = settings.statusLine?.command;
|
|
143
|
+
if (cmd && /context-doctor|cli\.js"?\s+statusline/.test(cmd)) {
|
|
144
|
+
const missing = hookBinaryMissing(cmd);
|
|
145
|
+
checks.push(missing
|
|
146
|
+
? { label: "Status line", status: "fail", detail: `configured, but ${missing} no longer exists — re-run: context-doctor install --statusline` }
|
|
147
|
+
: { label: "Status line", status: "ok", detail: "live context in Claude Code's status bar" });
|
|
148
|
+
}
|
|
149
|
+
else {
|
|
150
|
+
checks.push({ label: "Status line", status: "skip", detail: cmd ? "you have your own statusLine (left alone)" : "not enabled (optional: context-doctor install --statusline)" });
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
catch {
|
|
154
|
+
/* settings.json unreadable is already reported by the hook check */
|
|
155
|
+
}
|
|
156
|
+
}
|
|
138
157
|
// Skill
|
|
139
158
|
const skillPath = join(homedir(), ".claude", "skills", "context-doctor", "SKILL.md");
|
|
140
159
|
checks.push(existsSync(skillPath)
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `context-doctor experiment` — the fresh-vs-existing session harness.
|
|
3
|
+
*
|
|
4
|
+
* Everything else in this tool measures what is IN the context. None of it can
|
|
5
|
+
* say whether the task succeeded, so a smaller transcript can be a cheaper
|
|
6
|
+
* failure. This runs the same task twice against the same starting commit —
|
|
7
|
+
* once in a fresh session, once forked from an existing one — with the same
|
|
8
|
+
* model and tools, records what each was billed and how long it took, runs the
|
|
9
|
+
* same check command against each result, and puts the two side by side.
|
|
10
|
+
*
|
|
11
|
+
* It spends the user's Claude budget, so it refuses to start on a dirty tree,
|
|
12
|
+
* caps spend per arm, forks the existing session rather than mutating it, and
|
|
13
|
+
* resets the tree between arms only because it verified the tree was clean.
|
|
14
|
+
*/
|
|
15
|
+
export interface ExperimentOptions {
|
|
16
|
+
task: string;
|
|
17
|
+
/** Shell command whose exit code decides pass/fail, e.g. "npm test". */
|
|
18
|
+
check?: string;
|
|
19
|
+
/** Session id to fork the "existing" arm from. Omit to run the fresh arm only. */
|
|
20
|
+
existing?: string;
|
|
21
|
+
model?: string;
|
|
22
|
+
/** Spend cap per arm, USD. */
|
|
23
|
+
budgetUsd?: number;
|
|
24
|
+
cwd?: string;
|
|
25
|
+
/** Print the commands and stop. */
|
|
26
|
+
dryRun?: boolean;
|
|
27
|
+
/** Skip the clean-tree requirement (you accept the reset that follows). */
|
|
28
|
+
allowDirty?: boolean;
|
|
29
|
+
}
|
|
30
|
+
export interface ArmResult {
|
|
31
|
+
arm: "fresh" | "existing";
|
|
32
|
+
sessionId?: string;
|
|
33
|
+
costUsd: number;
|
|
34
|
+
durationMs: number;
|
|
35
|
+
turns: number;
|
|
36
|
+
inputTokens: number;
|
|
37
|
+
cacheRead: number;
|
|
38
|
+
cacheWrite: number;
|
|
39
|
+
outputTokens: number;
|
|
40
|
+
/** Billed input the transcript reports on the last request, when found. */
|
|
41
|
+
liveContextTokens?: number;
|
|
42
|
+
check?: {
|
|
43
|
+
command: string;
|
|
44
|
+
passed: boolean;
|
|
45
|
+
exitCode: number;
|
|
46
|
+
};
|
|
47
|
+
diffStat?: string;
|
|
48
|
+
error?: string;
|
|
49
|
+
}
|
|
50
|
+
/** Build the argv for one arm — exported so the dry run and the tests see exactly what runs. */
|
|
51
|
+
export declare function claudeArgs(opts: ExperimentOptions, arm: "fresh" | "existing"): string[];
|
|
52
|
+
export declare function runExperiment(opts: ExperimentOptions): {
|
|
53
|
+
arms: ArmResult[];
|
|
54
|
+
startCommit?: string;
|
|
55
|
+
refused?: string;
|
|
56
|
+
treeClean?: boolean;
|
|
57
|
+
};
|
|
58
|
+
export declare function renderExperiment(result: ReturnType<typeof runExperiment>, opts: ExperimentOptions): string;
|