context-doctor 0.17.0 → 0.19.0

package/dist/profile.js

@@ -146,11 +146,18 @@ function filesReadBy(toolName, toolCallText) {
     return [...paths];
 }
 export function profileConversation(conv, model) {
+    // An explicit model wins; otherwise the request's own model field decides
+    // which tokenizer ratios apply.
+    model = model ?? conv.model;
+    // With no model at all, Anthropic's request format still says whose
+    // tokenizer counts it. Only the ratios use this; pricing and the window
+    // stay unknown rather than guessed.
+    const tokenizerModel = model ?? (conv.sourceFormat === "anthropic" ? "claude" : undefined);
     // Learned from the user's own exact counts, if they ever fetched any.
-    const calibration = calibrationFor(model);
+    const calibration = calibrationFor(tokenizerModel);
     const perMessage = conv.messages.map((m) => ({
         msg: m,
-        tokens: Math.round(estimateTokens(m.text) * calibration.factor) + MESSAGE_OVERHEAD_TOKENS,
+        tokens: Math.round(estimateTokens(m.text, tokenizerModel) * calibration.factor) + MESSAGE_OVERHEAD_TOKENS,
     }));
     const totalTokens = perMessage.reduce((sum, p) => sum + p.tokens, 0);
     const categories = {

package/dist/proxy.d.ts

@@ -23,9 +23,24 @@ export interface ProxyOptions extends OptimizeOptions {
      * the user explicitly opts in (e.g. --host 0.0.0.0 inside a container).
      */
     host?: string;
+    /**
+     * When set, every request except /health must arrive under the path prefix
+     * `/t/<token>/`, which is stripped before routing. This is what makes the
+     * proxy safe to put on a public URL (a tunnel) for apps whose servers call
+     * the base URL, such as Cursor with your own OpenAI key: those apps can set a
+     * URL but not a header, so the secret rides in the path. Compared with
+     * constant time; a wrong or missing prefix gets 401 and no upstream call.
+     */
+    token?: string;
     anthropicUpstream?: string;
     openaiUpstream?: string;
 }
+/**
+ * Remove a leading `/t/<token>` from a request path, or return undefined when
+ * the prefix is absent or the token differs. The comparison is constant time
+ * so the token cannot be guessed a character at a time.
+ */
+export declare function stripToken(url: string, token: string): string | undefined;
 export interface ProxyStats {
     startedAt: string;
     requests: number;

package/dist/proxy.js

@@ -12,13 +12,31 @@
  * Streaming responses are piped through unchanged.
  */
 import http from "node:http";
+import { timingSafeEqual } from "node:crypto";
 import { optimizeConversation } from "./optimize.js";
-import { formatTokens } from "./tokens.js";
+import { formatTokens, CHARS_PER_TOKEN, providerFor } from "./tokens.js";
 import { formatUsd, inputCostUsd, pricingFor } from "./pricing.js";
 import { recordLedger } from "./ledger.js";
 /** Connection-level headers that must not be forwarded. */
 const SKIP_REQUEST_HEADERS = new Set(["host", "content-length", "connection", "transfer-encoding", "accept-encoding", "expect"]);
 const SKIP_RESPONSE_HEADERS = new Set(["content-length", "content-encoding", "transfer-encoding", "connection"]);
+/**
+ * Remove a leading `/t/<token>` from a request path, or return undefined when
+ * the prefix is absent or the token differs. The comparison is constant time
+ * so the token cannot be guessed a character at a time.
+ */
+export function stripToken(url, token) {
+    const prefix = "/t/";
+    if (!url.startsWith(prefix))
+        return undefined;
+    const end = url.indexOf("/", prefix.length);
+    const candidate = end === -1 ? url.slice(prefix.length) : url.slice(prefix.length, end);
+    const a = Buffer.from(candidate), b = Buffer.from(token);
+    if (a.length !== b.length || !timingSafeEqual(a, b))
+        return undefined;
+    const rest = end === -1 ? "/" : url.slice(end);
+    return rest;
+}
 function upstreamFor(url, opts) {
     if (url.startsWith("/v1/messages"))
         return opts.anthropicUpstream ?? "https://api.anthropic.com";
@@ -75,13 +93,23 @@ export function startProxy(opts = {}) {
         console.error(`[context-doctor] cache advisor: ${msg}`);
     };
     const server = http.createServer(async (req, res) => {
-        const url = req.url ?? "/";
+        let url = req.url ?? "/";
         try {
             if (url === "/health") {
                 res.setHeader("content-type", "application/json");
                 res.end(JSON.stringify({ ok: true, service: "context-doctor-proxy" }));
                 return;
             }
+            if (opts.token) {
+                const stripped = stripToken(url, opts.token);
+                if (stripped === undefined) {
+                    res.statusCode = 401;
+                    res.setHeader("content-type", "application/json");
+                    res.end(JSON.stringify({ error: "context-doctor proxy: this proxy requires its token in the path: /t/<token>/v1/..." }));
+                    return;
+                }
+                url = stripped;
+            }
             if (url === "/stats") {
                 res.setHeader("content-type", "application/json");
                 res.end(JSON.stringify({ ...stats, estUsdSaved: Number(stats.estUsdSaved.toFixed(4)) }, null, 2));
@@ -127,7 +155,9 @@ export function startProxy(opts = {}) {
                         if (url.startsWith("/v1/messages") && requestModel) {
                             const stablePrefix = JSON.stringify(parsedBody.tools ?? null) + JSON.stringify(parsedBody.system ?? null);
                             const hasBreakpoint = body.includes("cache_control");
-                            if (stablePrefix.length > 4000 && !hasBreakpoint) {
+                            const stablePrefixTokens = Math.round(stablePrefix.length / CHARS_PER_TOKEN[providerFor(requestModel)].code);
+                            // Anthropic will not cache a prefix under ~1024 tokens; below that the advice is useless.
+                            if (stablePrefixTokens >= 1024 && !hasBreakpoint) {
                                 // Say WHERE, not just that. A breakpoint caches everything up
                                 // to and including the block it sits on, so it belongs on the
                                 // LAST stable block: the final tool definition if there are
@@ -135,7 +165,7 @@ export function startProxy(opts = {}) {
                                 const where = Array.isArray(parsedBody.tools) && parsedBody.tools.length > 0
                                     ? `the last entry in "tools" (tools come before system in the cached prefix)`
                                     : `the last block of "system"`;
-                                advise(`~${Math.round(stablePrefix.length / 4)}+ tokens of stable system/tools on ${requestModel} without cache_control. ` +
+                                advise(`~${stablePrefixTokens}+ tokens of stable system/tools on ${requestModel} without cache_control. ` +
                                     `Add {"cache_control":{"type":"ephemeral"}} to ${where}; everything before it then bills at ~10% on every call`);
                             }
                             const fp = fnv1a(stablePrefix);
@@ -160,7 +190,7 @@ export function startProxy(opts = {}) {
                                     stable++;
                                 if (stable >= 2) {
                                     const stableChars = msgs.slice(0, stable).reduce((n, m) => n + JSON.stringify(m).length, 0);
-                                    const stableTokens = Math.round(stableChars / 4);
+                                    const stableTokens = Math.round(stableChars / CHARS_PER_TOKEN[providerFor(requestModel)].code);
                                     // Anthropic will not cache a prefix under ~1024 tokens (2048 on Haiku).
                                     if (stableTokens >= 1024) {
                                         advise(`messages #0-#${stable - 1} (~${stableTokens} tokens) were identical to the previous ${requestModel} request and carry no cache_control. ` +
@@ -278,11 +308,18 @@ export function startProxy(opts = {}) {
     server.on("close", checkpoint);
     const host = opts.host ?? "127.0.0.1";
     server.listen(port, host, () => {
+        // Print the token as <token>, never the value: this log is what people paste into bug reports.
+        const prefix = opts.token ? "/t/<token>" : "";
         console.error(`context-doctor proxy listening on http://${host}:${port}`);
-        console.error(`  Anthropic apps/SDKs: export ANTHROPIC_BASE_URL=http://localhost:${port}`);
-        console.error(`  OpenAI apps/SDKs:    export OPENAI_BASE_URL=http://localhost:${port}/v1`);
+        console.error(`  Anthropic apps/SDKs: export ANTHROPIC_BASE_URL=http://localhost:${port}${prefix}`);
+        console.error(`  OpenAI apps/SDKs:    export OPENAI_BASE_URL=http://localhost:${port}${prefix}/v1`);
         console.error(`  Every request's context is optimized in flight; savings are logged here.`);
-        console.error(`  Cumulative savings: http://localhost:${port}/stats`);
+        console.error(`  Cumulative savings: http://localhost:${port}${prefix}/stats`);
+        if (opts.token) {
+            console.error(`  Token required: every path except /health must start with /t/<token>/.`);
+            console.error(`  Cursor with your own OpenAI key: expose this port on HTTPS (a tunnel), then Settings > Models > OpenAI API Key >`);
+            console.error(`  "Override OpenAI Base URL" = https://<your-host>/t/<token>/v1. Cursor's servers call that URL, so 127.0.0.1 will not work there.`);
+        }
     });
     return server;
 }

package/dist/sketch.d.ts

@@ -6,8 +6,10 @@
  * JSON means re-typing 50k+ tokens as a tool argument. No model does that, and
  * it would double the context it is meant to measure. A sketch is ~100 output
  * tokens: turn count plus the handful of blocks that matter (pastes, tool
- * results, images, repeats). The estimate is coarse (±30%) and says so, but it
- * turns "call profile_context" from an impossible instruction into a cheap one.
+ * results, images, repeats). Measured error: -20% to +9% on the conversation
+ * total from turn count alone, ±25% on a code block sized by lines, ±15% on
+ * one sized by chars. Coarse, stated, and enough to find what to drop; it turns
+ * "call profile_context" from an impossible instruction into a cheap one.
  */
 export type SketchKind = "paste" | "code" | "tool_result" | "image" | "base64" | "text";
 export interface SketchBlock {
@@ -52,7 +54,9 @@ export interface SketchProfile {
     perTurnUsd?: number;
     perTurnCachedUsd?: number;
 }
-export declare function blockTokens(b: SketchBlock): number;
+export declare function blockTokens(b: SketchBlock, model?: string): number;
+/** Tokens for one plain exchange under this model's tokenizer. */
+export declare function exchangeTokens(model?: string): number;
 export declare function profileSketch(sketch: ConversationSketch): SketchProfile;
 export declare function renderSketchProfile(p: SketchProfile): string;
 /** Profile a sketch, log it to the ledger like a hook check, and render. */

package/dist/sketch.js

@@ -6,51 +6,72 @@
  * JSON means re-typing 50k+ tokens as a tool argument. No model does that, and
  * it would double the context it is meant to measure. A sketch is ~100 output
  * tokens: turn count plus the handful of blocks that matter (pastes, tool
- * results, images, repeats). The estimate is coarse (±30%) and says so, but it
- * turns "call profile_context" from an impossible instruction into a cheap one.
+ * results, images, repeats). Measured error: -20% to +9% on the conversation
+ * total from turn count alone, ±25% on a code block sized by lines, ±15% on
+ * one sized by chars. Coarse, stated, and enough to find what to drop; it turns
+ * "call profile_context" from an impossible instruction into a cheap one.
  */
-import { contextWindowFor } from "./tokens.js";
-import { formatTokens } from "./tokens.js";
+import { CHARS_PER_TOKEN, contextWindowFor, formatTokens, providerFor } from "./tokens.js";
 import { formatUsd, inputCostUsd, pricingFor } from "./pricing.js";
 import { recordLedger } from "./ledger.js";
-// A plain chat turn without attachments: a short user message and a normal
-// assistant reply. Measured across Claude Code transcripts the median user turn
-// is ~120 tokens and the median assistant turn ~450; chat apps run similar.
-const BASELINE_TOKENS_PER_TURN = 570;
-// Tokens per unit when the model reports size in lines/words/chars. Code and
-// tool output are denser per line than prose; words are ~1.35 tokens each.
-const TOKENS_PER_LINE = {
-    code: 12, tool_result: 12, paste: 14, text: 14, base64: 40, image: 0,
+// Every size here is in CHARACTERS, measured on real data, and converted to
+// tokens with the model's own ratio (tokens.ts), so a Claude chat and a GPT
+// chat of the same text get different, correct counts.
+//
+// A plain exchange: the user's message plus the assistant's reply. Median over
+// 1,283 exchanges in 59 Claude Code sessions (p25 1,177, p75 3,008). Sizing a
+// 30+ exchange chat from its turn count alone landed within -20% to +9% of the
+// real visible total (p10 to p90, 12 sessions): individual turns vary a lot,
+// long chats average it out.
+const CHARS_PER_EXCHANGE = 2060;
+// Mean chars per non-empty line. Code: median over 719 source files (a line
+// count for code is within about ±25%). Logs and tool output: median over
+// 2,095 tool results, but they range 38 to 100 chars a line, so the schema
+// asks for chars or tokens on those, and lines are the fallback.
+const CHARS_PER_LINE = {
+    code: 42, tool_result: 56, paste: 56, text: 80, base64: 76, image: 0,
 };
-const TOKENS_PER_WORD = 1.35;
-const CHARS_PER_TOKEN = 4;
-// When a block carries no size at all. Images are billed at a near-fixed rate.
-const DEFAULT_TOKENS = {
-    image: 1500, paste: 800, code: 800, tool_result: 800, base64: 4000, text: 300,
+// Prose, including the space after each word (6.3 measured on assistant text).
+const CHARS_PER_WORD = 6.3;
+// When a block carries no size at all.
+const DEFAULT_CHARS = {
+    paste: 2400, code: 2400, tool_result: 2400, text: 900, base64: 12000, image: 0,
 };
+// Images are billed by pixels, not bytes: ~1,600 tokens for a full-size
+// screenshot on Claude, fewer on smaller images and on GPT. One figure is enough
+// for a sketch.
+const IMAGE_TOKENS = 1500;
 const LARGE_BLOCK_TOKENS = 2000;
 const LONG_HISTORY_TURNS = 30;
 const HANDOFF_SUMMARY_TOKENS = 300;
 const RECENT_TURNS_KEPT = 6;
-export function blockTokens(b) {
+/** Code, tool output and base64 tokenize at the code ratio; pastes and text at the prose ratio. */
+function ratioFor(kind, model) {
+    const r = CHARS_PER_TOKEN[providerFor(model)];
+    return kind === "code" || kind === "tool_result" || kind === "base64" ? r.code : r.prose;
+}
+export function blockTokens(b, model) {
     if (b.kind === "image")
-        return b.approx_tokens ?? DEFAULT_TOKENS.image;
+        return b.approx_tokens && b.approx_tokens > 0 ? Math.round(b.approx_tokens) : IMAGE_TOKENS;
     if (b.approx_tokens && b.approx_tokens > 0)
         return Math.round(b.approx_tokens);
-    if (b.approx_lines && b.approx_lines > 0)
-        return Math.round(b.approx_lines * TOKENS_PER_LINE[b.kind]);
-    if (b.approx_words && b.approx_words > 0)
-        return Math.round(b.approx_words * TOKENS_PER_WORD);
-    if (b.approx_chars && b.approx_chars > 0)
-        return Math.round(b.approx_chars / CHARS_PER_TOKEN);
-    return DEFAULT_TOKENS[b.kind];
+    const chars = b.approx_chars && b.approx_chars > 0 ? b.approx_chars
+        : b.approx_lines && b.approx_lines > 0 ? b.approx_lines * CHARS_PER_LINE[b.kind]
+            : b.approx_words && b.approx_words > 0 ? b.approx_words * CHARS_PER_WORD
+                : DEFAULT_CHARS[b.kind];
+    return Math.round(chars / ratioFor(b.kind, model));
+}
+/** Tokens for one plain exchange under this model's tokenizer. */
+export function exchangeTokens(model) {
+    return Math.round(CHARS_PER_EXCHANGE / CHARS_PER_TOKEN[providerFor(model)].prose);
 }
 export function profileSketch(sketch) {
     const turns = Math.max(0, Math.floor(sketch.turns || 0));
     const blocks = Array.isArray(sketch.blocks) ? sketch.blocks : [];
-    const baselineTokens = turns * BASELINE_TOKENS_PER_TURN;
+    const perExchange = exchangeTokens(sketch.model);
+    const baselineTokens = turns * perExchange;
     // A repeated block costs its size every time it appears.
-    const sized = blocks.map((b) => ({ block: b, tokens: blockTokens(b), copies: Math.max(1, Math.floor(b.repeated ?? 1)) }));
+    const sized = blocks.map((b) => ({ block: b, tokens: blockTokens(b, sketch.model), copies: Math.max(1, Math.floor(b.repeated ?? 1)) }));
     const blockTotal = sized.reduce((n, s) => n + s.tokens * s.copies, 0);
     const totalTokens = baselineTokens + blockTotal;
     const findings = [];
@@ -101,7 +122,7 @@ export function profileSketch(sketch) {
         });
     }
     if (turns >= LONG_HISTORY_TURNS) {
-        const afterHandoff = RECENT_TURNS_KEPT * BASELINE_TOKENS_PER_TURN + HANDOFF_SUMMARY_TOKENS;
+        const afterHandoff = RECENT_TURNS_KEPT * perExchange + HANDOFF_SUMMARY_TOKENS;
         findings.push({
             id: "long_history",
             severity: totalTokens > 100_000 ? "high" : "warn",
@@ -140,7 +161,7 @@ export function renderSketchProfile(p) {
     const lines = [];
     const window = p.usagePct !== undefined ? ` (~${p.usagePct}% of ${formatTokens(p.contextWindow)})` : "";
     lines.push(`Context estimate from sketch: ~${formatTokens(p.totalTokens)} tokens${window}, ${p.turns} turns.`);
-    lines.push(`  ${formatTokens(p.baselineTokens)} plain conversation + ${formatTokens(p.blockTokens)} in pastes, tool output and images. Estimate, ±30%.`);
+    lines.push(`  ${formatTokens(p.baselineTokens)} plain conversation + ${formatTokens(p.blockTokens)} in pastes, tool output and images. Estimate, usually within ±20%.`);
     if (p.perTurnUsd !== undefined) {
         lines.push(`  Re-read on every turn: ${formatUsd(p.perTurnUsd)} at list price, ${formatUsd(p.perTurnCachedUsd)} when cached. On a subscription this is what spends the usage limit.`);
     }

package/dist/tokenizer-measure.d.ts

@@ -0,0 +1,42 @@
+/**
+ * Measure a model's real chars-per-token from Claude Code transcripts, with no
+ * API key and no tokenizer: the API's own counts are already in the file.
+ *
+ * Two independent measurements, so one can check the other:
+ *
+ *  - PROSE. An assistant reply with no thinking block is billed as exactly
+ *    `output_tokens`, and all of it is visible text in the transcript. Visible
+ *    chars / output_tokens is the tokenizer's ratio on the model's own prose.
+ *
+ *  - BLOCKS (code, tool output, pastes). Between two consecutive API calls in
+ *    one session the prompt is the old prompt plus what was appended: the
+ *    previous reply (all of `output_tokens`, thinking included, since a tool
+ *    loop re-sends it) and the new user-side content. When that content is a
+ *    single large block, (growth − previous output_tokens) is its exact size.
+ *
+ * The injected reminders the harness adds make the block figure slightly
+ * pessimistic (a few dozen tokens on blocks of thousands), which is why only
+ * blocks over 6k chars count.
+ */
+export interface RatioStats {
+    samples: number;
+    median: number;
+    p10: number;
+    p90: number;
+}
+export interface ModelRatios {
+    model: string;
+    prose?: RatioStats;
+    blocks?: RatioStats;
+    /** What the estimator uses for this model: prose, code. */
+    assumed: {
+        prose: number;
+        code: number;
+    };
+}
+export interface TokenizerReport {
+    sessionsScanned: number;
+    models: ModelRatios[];
+}
+export declare function measureTokenizer(limit?: number, paths?: string[]): TokenizerReport;
+export declare function renderTokenizer(report: TokenizerReport): string;

package/dist/tokenizer-measure.js

@@ -0,0 +1,172 @@
+/**
+ * Measure a model's real chars-per-token from Claude Code transcripts, with no
+ * API key and no tokenizer: the API's own counts are already in the file.
+ *
+ * Two independent measurements, so one can check the other:
+ *
+ *  - PROSE. An assistant reply with no thinking block is billed as exactly
+ *    `output_tokens`, and all of it is visible text in the transcript. Visible
+ *    chars / output_tokens is the tokenizer's ratio on the model's own prose.
+ *
+ *  - BLOCKS (code, tool output, pastes). Between two consecutive API calls in
+ *    one session the prompt is the old prompt plus what was appended: the
+ *    previous reply (all of `output_tokens`, thinking included, since a tool
+ *    loop re-sends it) and the new user-side content. When that content is a
+ *    single large block, (growth − previous output_tokens) is its exact size.
+ *
+ * The injected reminders the harness adds make the block figure slightly
+ * pessimistic (a few dozen tokens on blocks of thousands), which is why only
+ * blocks over 6k chars count.
+ */
+import { forEachLine, listSessions } from "./session.js";
+import { CHARS_PER_TOKEN, providerFor } from "./tokens.js";
+const MIN_PROSE_CHARS = 1500;
+const MIN_BLOCK_CHARS = 6000;
+function stats(values) {
+    if (values.length === 0)
+        return undefined;
+    const v = [...values].sort((a, b) => a - b);
+    const at = (q) => v[Math.min(v.length - 1, Math.floor(q * (v.length - 1)))];
+    const mid = Math.floor(v.length / 2);
+    const median = v.length % 2 ? v[mid] : (v[mid - 1] + v[mid]) / 2;
+    return { samples: v.length, median, p10: at(0.1), p90: at(0.9) };
+}
+function promptTotal(u) {
+    const n = (k) => (typeof u[k] === "number" ? u[k] : 0);
+    return n("input_tokens") + n("cache_read_input_tokens") + n("cache_creation_input_tokens");
+}
+function userText(content) {
+    if (typeof content === "string")
+        return content;
+    if (!Array.isArray(content))
+        return "";
+    let out = "";
+    for (const p of content) {
+        if (p?.type === "text" && typeof p.text === "string")
+            out += p.text;
+        else if (p?.type === "tool_result") {
+            const c = p.content;
+            if (typeof c === "string")
+                out += c;
+            else if (Array.isArray(c))
+                for (const x of c)
+                    if (typeof x?.text === "string")
+                        out += x.text;
+        }
+    }
+    return out;
+}
+/** Scan one transcript into per-request records. Never throws on bad lines. */
+function readRequests(path) {
+    const reqs = [];
+    let cur;
+    forEachLine(path, (line) => {
+        let e;
+        try {
+            e = JSON.parse(line);
+        }
+        catch {
+            return;
+        }
+        if (!e || e.isSidechain)
+            return;
+        if (e.type === "assistant" && e.message && typeof e.message === "object") {
+            const m = e.message;
+            const u = (m.usage ?? {});
+            if (!cur || cur.id !== m.id) {
+                cur = { id: String(m.id), model: m.model, prompt: promptTotal(u), output: 0, types: new Set(), textChars: 0, after: [] };
+                reqs.push(cur);
+            }
+            if (typeof u.output_tokens === "number")
+                cur.output = Math.max(cur.output, u.output_tokens);
+            for (const p of Array.isArray(m.content) ? m.content : []) {
+                cur.types.add(String(p?.type));
+                if (p?.type === "text" && typeof p.text === "string")
+                    cur.textChars += p.text.length;
+            }
+        }
+        else if (e.type === "user" && cur) {
+            const meta = Boolean(e.isMeta || e.isCompactSummary);
+            const text = userText(e.message?.content);
+            cur.after.push({ chars: text.length, meta });
+        }
+    });
+    return reqs;
+}
+export function measureTokenizer(limit = 60, paths) {
+    const prose = new Map();
+    const blocks = new Map();
+    const push = (m, k, v) => (m.get(k) ?? m.set(k, []).get(k)).push(v);
+    let scanned = 0;
+    const targets = paths ?? listSessions(limit).map((s) => s.path);
+    for (const path of targets) {
+        let reqs;
+        try {
+            reqs = readRequests(path);
+        }
+        catch {
+            continue;
+        }
+        if (reqs.length === 0)
+            continue;
+        scanned++;
+        for (const r of reqs) {
+            if (!r.model || r.model.startsWith("<"))
+                continue;
+            if (r.types.size === 1 && r.types.has("text") && r.textChars >= MIN_PROSE_CHARS && r.output > 0) {
+                push(prose, r.model, r.textChars / r.output);
+            }
+        }
+        for (let i = 0; i + 1 < reqs.length; i++) {
+            const a = reqs[i], b = reqs[i + 1];
+            if (!a.model || a.model !== b.model)
+                continue;
+            const real = a.after.filter((x) => !x.meta);
+            if (a.after.some((x) => x.meta) || real.length !== 1 || real[0].chars < MIN_BLOCK_CHARS)
+                continue;
+            const exact = b.prompt - a.prompt - a.output;
+            if (exact <= 0)
+                continue; // a compaction or cache reset, not growth
+            push(blocks, a.model, real[0].chars / exact);
+        }
+    }
+    const models = [...new Set([...prose.keys(), ...blocks.keys()])]
+        .map((model) => ({
+        model,
+        prose: stats(prose.get(model) ?? []),
+        blocks: stats(blocks.get(model) ?? []),
+        assumed: CHARS_PER_TOKEN[providerFor(model)],
+    }))
+        .sort((x, y) => ((y.prose?.samples ?? 0) + (y.blocks?.samples ?? 0)) - ((x.prose?.samples ?? 0) + (x.blocks?.samples ?? 0)));
+    return { sessionsScanned: scanned, models };
+}
+export function renderTokenizer(report) {
+    const lines = [];
+    lines.push("Tokenizer check — chars per token, measured from the API's own counts");
+    lines.push("─".repeat(56));
+    if (report.models.length === 0) {
+        lines.push("No usable samples (needs Claude Code sessions with usage recorded).");
+        return lines.join("\n");
+    }
+    const f = (n) => n.toFixed(2);
+    const err = (assumed, real) => {
+        // Estimated tokens / real tokens − 1, from chars/token on each side.
+        const pct = Math.round((real / assumed - 1) * 100);
+        return pct === 0 ? "exact" : `estimates ${pct > 0 ? "+" : ""}${pct}%`;
+    };
+    const row = (label, st, unit, assumed) => `  ${label.padEnd(7)} ${f(st.median)}  ${`(p10 ${f(st.p10)}, p90 ${f(st.p90)}, ${st.samples} ${unit})`.padEnd(34)} ` +
+        `estimator ${f(assumed)} → ${err(assumed, st.median)}`;
+    for (const m of report.models) {
+        lines.push(m.model);
+        if (m.prose) {
+            lines.push(row("prose", m.prose, "replies", m.assumed.prose));
+        }
+        if (m.blocks) {
+            lines.push(row("blocks", m.blocks, "blocks", m.assumed.code));
+        }
+    }
+    lines.push("");
+    lines.push("\"estimates +x%\" means the estimator reports x% more tokens than the API bills for");
+    lines.push("the same text (negative: fewer). Blocks are mostly code and tool output.");
+    return lines.join("\n");
+}

package/dist/tokens.d.ts

@@ -11,7 +11,37 @@
 export type Provider = "anthropic" | "openai" | "google" | "generic";
 export declare function contextWindowFor(model?: string): number | undefined;
 export declare function providerFor(model?: string): Provider;
-export declare function estimateTokens(text: string): number;
+/**
+ * Characters per token, by provider and by content type.
+ *
+ * Anthropic: measured 2026-09 against the API's own counts in 59 Claude Code
+ * sessions (Opus 4.7 to 5, Fable 5.x), two independent ways that agree.
+ * Prose: 504 assistant replies with no thinking block, visible text divided by
+ * the exact output_tokens: median 2.75 (p10 2.4, p90 3.0). Code and tool
+ * output: 474 single appended blocks over 6k chars, sized by the exact growth
+ * of the billed prompt between consecutive calls: median 2.4 (p10 2.1, p90 2.8).
+ * The ratios this tool used before (4.0 / 3.2) undercounted current Claude
+ * models by about 1.45x on prose and 1.33x on code.
+ *
+ * OpenAI, Google and unknown models keep 4.0 / 3.2, the usual figures for
+ * o200k-class tokenizers on English and code. They are not re-measured here:
+ * Codex rollouts truncate tool output before the model sees it, so the same
+ * delta method does not isolate a block. `analyze --exact` calibrates any
+ * provider from its own tokenizer on your machine.
+ */
+export declare const CHARS_PER_TOKEN: Record<Provider, {
+    prose: number;
+    code: number;
+}>;
+/** True when text is dense with code/JSON symbols and tokenizes more finely. */
+export declare function isCodeLike(text: string): boolean;
+/** Chars per token to use for this text under this model's tokenizer. */
+export declare function charsPerTokenFor(text: string, model?: string): number;
+/**
+ * Estimated tokens for `text`. Pass the model when you know it: Claude's
+ * tokenizer produces ~40% more tokens than the provider-neutral default.
+ */
+export declare function estimateTokens(text: string, model?: string): number;
 /** Per-message structural overhead (role markers, delimiters) is roughly constant. */
 export declare const MESSAGE_OVERHEAD_TOKENS = 4;
 export declare function formatTokens(n: number): string;

package/dist/tokens.js

@@ -49,17 +49,51 @@ function symbolDensity(text) {
     const symbols = text.match(/[{}[\]()<>;:=_\/\\|"'`#$%&*+^~-]/g);
     return (symbols?.length ?? 0) / text.length;
 }
-export function estimateTokens(text) {
+/**
+ * Characters per token, by provider and by content type.
+ *
+ * Anthropic: measured 2026-09 against the API's own counts in 59 Claude Code
+ * sessions (Opus 4.7 to 5, Fable 5.x), two independent ways that agree.
+ * Prose: 504 assistant replies with no thinking block, visible text divided by
+ * the exact output_tokens: median 2.75 (p10 2.4, p90 3.0). Code and tool
+ * output: 474 single appended blocks over 6k chars, sized by the exact growth
+ * of the billed prompt between consecutive calls: median 2.4 (p10 2.1, p90 2.8).
+ * The ratios this tool used before (4.0 / 3.2) undercounted current Claude
+ * models by about 1.45x on prose and 1.33x on code.
+ *
+ * OpenAI, Google and unknown models keep 4.0 / 3.2, the usual figures for
+ * o200k-class tokenizers on English and code. They are not re-measured here:
+ * Codex rollouts truncate tool output before the model sees it, so the same
+ * delta method does not isolate a block. `analyze --exact` calibrates any
+ * provider from its own tokenizer on your machine.
+ */
+export const CHARS_PER_TOKEN = {
+    anthropic: { prose: 2.75, code: 2.4 },
+    openai: { prose: 4.0, code: 3.2 },
+    google: { prose: 4.0, code: 3.2 },
+    generic: { prose: 4.0, code: 3.2 },
+};
+/** True when text is dense with code/JSON symbols and tokenizes more finely. */
+export function isCodeLike(text) {
+    return symbolDensity(text) > 0.08;
+}
+/** Chars per token to use for this text under this model's tokenizer. */
+export function charsPerTokenFor(text, model) {
+    const ratios = CHARS_PER_TOKEN[providerFor(model)];
+    return isCodeLike(text) ? ratios.code : ratios.prose;
+}
+/**
+ * Estimated tokens for `text`. Pass the model when you know it: Claude's
+ * tokenizer produces ~40% more tokens than the provider-neutral default.
+ */
+export function estimateTokens(text, model) {
     // Public API: callers outside this package pass whatever they have, and a
     // TypeError from a token estimator is never the useful answer.
     if (typeof text !== "string")
         text = String(text ?? "");
     if (!text)
         return 0;
-    // Denser tokenization for code/JSON-like content, lighter for plain prose.
-    const density = symbolDensity(text);
-    const charsPerToken = density > 0.08 ? 3.2 : 4.0;
-    return Math.ceil(text.length / charsPerToken);
+    return Math.ceil(text.length / charsPerTokenFor(text, model));
 }
 /** Per-message structural overhead (role markers, delimiters) is roughly constant. */
 export const MESSAGE_OVERHEAD_TOKENS = 4;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "context-doctor",
-  "version": "0.17.0",
+  "version": "0.19.0",
   "description": "Profile and optimize LLM context windows. See what's eating your tokens and fix it — works with Claude Code, Claude Desktop, Cursor, Codex (OpenAI), and any MCP-capable AI app.",
   "keywords": [
     "claude",