context-doctor 0.6.0 → 0.7.0

package/README.md

@@ -163,7 +163,11 @@ The proxy dedupes repeated content, trims stale tool results, and strips base64
 [context-doctor] POST /v1/messages → 200 in 842ms | optimized 7.3k → 518 tokens (2 changes) | session total: 6.9k tokens ≈ $0.021 saved
 ```
 
-`GET http://localhost:8787/stats` returns cumulative savings (requests, tokens, estimated USD) since launch.
+`GET http://localhost:8787/stats` returns cumulative savings (requests, tokens, estimated USD), **exact upstream usage** read from every response (JSON and SSE), and **prompt-cache advisories** — the proxy watches your real traffic and flags big stable prefixes missing `cache_control` or prefix churn that silently re-bills the cache. Per-model behavior via `--config`:
+
+```json
+{ "routes": [{ "modelPrefix": "gpt", "strategies": ["strip-base64"], "keepRecent": 4 }] }
+```
 
 Because prompt caching matches byte-identical prefixes, deterministic strategies are chosen so repeated requests stay stable — but if you rely on aggressive cache prefixes, start with `--strategy strip-base64 --strategy dedupe` and add more as you verify.
 

package/dist/cli.js

@@ -61,6 +61,8 @@ Options:
   --max-tool-tokens <n>   (optimize) Token budget for trimmed tool results (default 300)
   --port <n>              (proxy) Port to listen on (default 8787)
   --host <addr>           (proxy) Bind address (default 127.0.0.1; use 0.0.0.0 to expose)
+  --config <file>         (proxy) Per-route overrides: {"routes":[{"modelPrefix":"gpt","strategies":[...],
+                          "keepRecent":n,"maxToolResultTokens":n}]} — first prefix match wins
   --upstream-anthropic <url>  (proxy) Override Anthropic upstream (testing)
   --upstream-openai <url>     (proxy) Override OpenAI upstream (testing)
   -h, --help              Show this help
@@ -115,6 +117,9 @@ function parseArgs(argv) {
             case "--host":
                 args.host = argv[++i];
                 break;
+            case "--config":
+                args.config = argv[++i];
+                break;
             case "--upstream-anthropic":
                 args.upstreamAnthropic = argv[++i];
                 break;
@@ -195,7 +200,18 @@ function main() {
         return;
     }
     if (args.command === "proxy") {
+        let routes;
+        if (args.config) {
+            try {
+                routes = JSON.parse(readFileSync(args.config, "utf8")).routes;
+            }
+            catch (e) {
+                console.error(`Could not read --config ${args.config}: ${e.message}`);
+                process.exit(1);
+            }
+        }
         startProxy({
+            routes: routes,
             port: args.port,
             host: args.host,
             anthropicUpstream: args.upstreamAnthropic,

package/dist/mcp.js

@@ -37,7 +37,7 @@ const STRATEGY_IDS = ["dedupe", "trim-tool-results", "strip-base64", "prune-hist
  * recommended pattern.
  */
 function createServer() {
-    const server = new McpServer({ name: "context-doctor", version: "0.6.0" }, { instructions: SERVER_INSTRUCTIONS });
+    const server = new McpServer({ name: "context-doctor", version: "0.7.0" }, { instructions: SERVER_INSTRUCTIONS });
     server.tool("profile_context", "Profile an LLM conversation or prompt: token breakdown by category, largest messages, and actionable findings about wasted context (duplicates, oversized tool results, base64 blobs, cache-unfriendly ordering). Accepts OpenAI/Anthropic conversation JSON or raw text. Call this immediately whenever the user asks about token usage, context size, LLM cost, or latency — and proactively offer it once a conversation grows long or accumulates large pasted content.", {
         conversation: z.string().describe("Conversation JSON (OpenAI or Anthropic format, or bare message array) or raw prompt text"),
         model: z.string().optional().describe("Target model name for context-window math, e.g. claude-sonnet-5 or gpt-4o"),

package/dist/proxy.d.ts

@@ -14,6 +14,8 @@
 import http from "node:http";
 import { OptimizeOptions } from "./optimize.js";
 export interface ProxyOptions extends OptimizeOptions {
+    /** Per-model-prefix overrides; first match wins, global options otherwise. */
+    routes?: RouteConfig[];
     port?: number;
     /**
      * Bind address. Defaults to 127.0.0.1 — the proxy relays authenticated
@@ -33,5 +35,18 @@ export interface ProxyStats {
     tokensSaved: number;
     /** USD saved on input tokens, when the request's model has a known price. */
     estUsdSaved: number;
+    /** Exact usage reported by upstream responses (both providers, incl. SSE). */
+    upstreamInputTokens: number;
+    upstreamOutputTokens: number;
+    /** Prompt-cache advisories observed on live traffic (unique, capped). */
+    advice: string[];
+}
+/** Per-model-prefix strategy overrides for the proxy (`--config`). */
+export interface RouteConfig {
+    /** Applies when the request body's model starts with this prefix. */
+    modelPrefix: string;
+    strategies?: OptimizeOptions["strategies"];
+    keepRecent?: number;
+    maxToolResultTokens?: number;
 }
 export declare function startProxy(opts?: ProxyOptions): http.Server;

package/dist/proxy.js

@@ -26,6 +26,29 @@ function upstreamFor(url, opts) {
     }
     return undefined;
 }
+/** Pull exact usage out of a response body — JSON or SSE, either provider. */
+function extractUsage(text) {
+    const last = (re) => {
+        let m;
+        let v = -1;
+        while ((m = re.exec(text)) !== null)
+            v = Number(m[1]);
+        return v;
+    };
+    const input = Math.max(last(/"input_tokens"\s*:\s*(\d+)/g), last(/"prompt_tokens"\s*:\s*(\d+)/g));
+    const output = Math.max(last(/"output_tokens"\s*:\s*(\d+)/g), last(/"completion_tokens"\s*:\s*(\d+)/g));
+    if (input < 0 && output < 0)
+        return null;
+    return { input: Math.max(input, 0), output: Math.max(output, 0) };
+}
+function fnv1a(s) {
+    let h = 0x811c9dc5;
+    for (let i = 0; i < s.length; i++) {
+        h ^= s.charCodeAt(i);
+        h = Math.imul(h, 0x01000193);
+    }
+    return h >>> 0;
+}
 export function startProxy(opts = {}) {
     const port = opts.port ?? 8787;
     const stats = {
@@ -36,6 +59,17 @@ export function startProxy(opts = {}) {
         tokensAfter: 0,
         tokensSaved: 0,
         estUsdSaved: 0,
+        upstreamInputTokens: 0,
+        upstreamOutputTokens: 0,
+        advice: [],
+    };
+    /** Last stable-prefix fingerprint per model, for cache-invalidation advice. */
+    const prefixFingerprints = new Map();
+    const advise = (msg) => {
+        if (stats.advice.includes(msg) || stats.advice.length >= 10)
+            return;
+        stats.advice.push(msg);
+        console.error(`[context-doctor] cache advisor: ${msg}`);
     };
     const server = http.createServer(async (req, res) => {
         const url = req.url ?? "/";
@@ -70,7 +104,39 @@ export function startProxy(opts = {}) {
             let note = "passthrough";
             if (req.method === "POST" && body && !isMeasurement) {
                 try {
-                    const result = optimizeConversation(body, opts);
+                    // Per-route overrides: first modelPrefix match wins.
+                    let effective = opts;
+                    let requestModel;
+                    try {
+                        const parsedBody = JSON.parse(body);
+                        requestModel = parsedBody.model;
+                        const route = requestModel ? opts.routes?.find((r) => requestModel.startsWith(r.modelPrefix)) : undefined;
+                        if (route) {
+                            effective = {
+                                strategies: route.strategies ?? opts.strategies,
+                                keepRecent: route.keepRecent ?? opts.keepRecent,
+                                maxToolResultTokens: route.maxToolResultTokens ?? opts.maxToolResultTokens,
+                            };
+                        }
+                        // Prompt-cache advisor (Anthropic requests): the proxy sees real
+                        // sequences, so cache-hostile patterns are observable facts here.
+                        if (url.startsWith("/v1/messages") && requestModel) {
+                            const stablePrefix = JSON.stringify(parsedBody.tools ?? null) + JSON.stringify(parsedBody.system ?? null);
+                            if (stablePrefix.length > 4000 && !body.includes("cache_control")) {
+                                advise(`~${Math.round(stablePrefix.length / 4)}+ tokens of stable system/tools on ${requestModel} without cache_control — adding a breakpoint would cut those to ~10% cost per call`);
+                            }
+                            const fp = fnv1a(stablePrefix);
+                            const prev = prefixFingerprints.get(requestModel);
+                            if (prev !== undefined && prev !== fp) {
+                                advise(`system/tools prefix changed between ${requestModel} requests — every change re-bills the whole cached prefix; keep it byte-stable`);
+                            }
+                            prefixFingerprints.set(requestModel, fp);
+                        }
+                    }
+                    catch {
+                        /* body isn't JSON — global opts apply */
+                    }
+                    const result = optimizeConversation(body, effective);
                     const saved = result.tokensBefore - result.tokensAfter;
                     stats.tokensBefore += result.tokensBefore;
                     stats.tokensAfter += result.tokensAfter;
@@ -109,13 +175,26 @@ export function startProxy(opts = {}) {
                     res.setHeader(key, value);
             });
             if (upstream.body) {
-                // Pipe through chunk-by-chunk so SSE streaming works unchanged.
+                // Pipe through chunk-by-chunk so SSE streaming works unchanged, while
+                // accumulating a bounded copy to read exact usage after the fact.
+                const USAGE_SCAN_CAP = 2 * 1024 * 1024;
+                let scanBuf = "";
+                const decoder = new TextDecoder();
                 const reader = upstream.body.getReader();
                 for (;;) {
                     const { done, value } = await reader.read();
                     if (done)
                         break;
                     res.write(value);
+                    if (scanBuf.length < USAGE_SCAN_CAP)
+                        scanBuf += decoder.decode(value, { stream: true });
+                }
+                if (upstream.ok && !isMeasurement) {
+                    const usage = extractUsage(scanBuf);
+                    if (usage) {
+                        stats.upstreamInputTokens += usage.input;
+                        stats.upstreamOutputTokens += usage.output;
+                    }
                 }
             }
             res.end();

package/dist/test/proxy.test.js

@@ -32,6 +32,7 @@ const upstream = http.createServer((req, res) => {
         receivedApiKey = req.headers["x-api-key"];
         res.writeHead(200, { "content-type": "text/event-stream" });
         res.write("event: message_start\ndata: {}\n\n");
+        res.write('event: message_delta\ndata: {"usage":{"input_tokens":120,"output_tokens":45}}\n\n');
         res.write("event: message_stop\ndata: {}\n\n");
         res.end();
     });
@@ -69,6 +70,49 @@ test("/stats reports cumulative savings with dollar estimate", async () => {
     assert.ok(stats.tokensSaved > 1000, `saved tokens tracked (${stats.tokensSaved})`);
     assert.ok(stats.estUsdSaved > 0, "dollar savings estimated from the request's model");
 });
+test("response usage is captured from the SSE stream; cache advisor fires on prefix churn", async () => {
+    // Second request with a DIFFERENT system prompt on the same model → advisory.
+    const churned = JSON.parse(payload);
+    churned.system = "You are helpful. TODAY IS A NEW DAY."; // classic cache-buster
+    churned.tools = [{ name: "t", description: "x".repeat(5000), input_schema: { type: "object" } }];
+    const first = JSON.parse(payload);
+    first.tools = churned.tools;
+    for (const body of [first, churned]) {
+        await fetch(`http://localhost:${proxyPort}/v1/messages`, {
+            method: "POST",
+            headers: { "content-type": "application/json", "x-api-key": "sk-test-not-real" },
+            body: JSON.stringify(body),
+        });
+    }
+    const stats = (await (await fetch(`http://localhost:${proxyPort}/stats`)).json());
+    // Mock upstream reports usage in its SSE close event (added below).
+    assert.ok(stats.upstreamInputTokens >= 100, `usage input captured: ${stats.upstreamInputTokens}`);
+    assert.ok(stats.upstreamOutputTokens >= 40, `usage output captured: ${stats.upstreamOutputTokens}`);
+    assert.ok(stats.advice.some((a) => a.includes("prefix changed")), `prefix-churn advisory expected, got: ${JSON.stringify(stats.advice)}`);
+    assert.ok(stats.advice.some((a) => a.includes("cache_control")), "missing-cache_control advisory expected");
+});
+test("per-route config: empty strategy list disables optimization for matching models", async () => {
+    const { startProxy } = await import("../proxy.js");
+    const routed = startProxy({
+        port: 0,
+        anthropicUpstream: `http://localhost:${upstreamPort}`,
+        routes: [{ modelPrefix: "claude-sonnet", strategies: [] }],
+    });
+    await new Promise((r) => routed.once("listening", () => r()));
+    const routedPort = routed.address().port;
+    try {
+        const sent = payload;
+        await fetch(`http://localhost:${routedPort}/v1/messages`, {
+            method: "POST",
+            headers: { "content-type": "application/json" },
+            body: sent,
+        });
+        assert.equal(received.length, sent.length, "route with no strategies must pass body through unmodified");
+    }
+    finally {
+        routed.close();
+    }
+});
 test("unsupported paths get a clear 404, health stays up", async () => {
     const notFound = await fetch(`http://localhost:${proxyPort}/v1/nope`, { method: "POST", body: "{}" });
     assert.equal(notFound.status, 404);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "context-doctor",
-  "version": "0.6.0",
+  "version": "0.7.0",
   "description": "Profile and optimize LLM context windows. See what's eating your tokens and fix it — works with Claude, GPT, Gemini, and any MCP-capable AI app.",
   "keywords": [
     "llm",