context-doctor 0.25.0 → 0.27.0

package/dist/mcp.js

@@ -11,6 +11,7 @@
  *   profile_context   — analyze a conversation/prompt, report token breakdown + findings
  *   optimize_context  — apply safe strategies, return the slimmed conversation
  *   context_best_practices — curated checklist for a given provider/use case
+ *   pack_context      — only the chunks of big files a question needs, within a token budget
  */
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
@@ -22,6 +23,7 @@ import { renderProfile } from "./report.js";
 import { formatTokens } from "./tokens.js";
 import { recordLedger } from "./ledger.js";
 import { runSketch } from "./sketch.js";
+import { packContextSemantic, readSources, renderPack } from "./pack.js";
 /**
  * Server instructions are injected by MCP clients (Claude Desktop, Cursor, …)
  * into the system context of EVERY conversation where this server is enabled.
@@ -38,18 +40,19 @@ import { runSketch } from "./sketch.js";
 // thresholds keep it from firing on every turn, which would cost more context
 // than it saves.
 const SERVER_INSTRUCTIONS = `Context hygiene rules (always on):
-1. Summarize any paste or tool result over ~2k tokens into the points you will use, then work from the summary; never carry it verbatim.
-2. Reference earlier content by name; never re-quote it. Never inline base64.
-3. When the conversation passes ~30 turns, or holds 3+ large pastes, or the user asks about tokens, cost, speed or limits: call profile_context BEFORE answering and act on its top finding. In a chat app pass a \`sketch\` (turn count + the large/repeated blocks, ~120 tokens), not the conversation. Do not estimate token counts yourself.
-4. If optimize_context returns a pruned-turns digest, you write the ≤150-token replacement summary.`;
+1. Summarize any paste or tool result over ~2k tokens into the points you need; never carry it verbatim.
+2. Refer to earlier content by name; never re-quote it or inline base64.
+3. Past ~30 turns or 3+ large pastes, or when asked about tokens, cost, speed or limits: call profile_context BEFORE answering and act on its top finding. In a chat app pass a \`sketch\` (turns + large/repeated blocks), not the conversation. Do not estimate token counts yourself.
+4. If optimize_context returns a pruned-turns digest, write its ≤150-token summary.
+5. To answer from a large file or folder you won't edit, call pack_context with the question; don't read it whole.`;
 const STRATEGY_IDS = ["dedupe", "trim-tool-results", "trim-tool-calls", "strip-base64", "prune-history"];
 /**
  * Build a fully-configured server instance. A factory (not a singleton) so the
  * stateless HTTP mode can hand every request its own server, per the MCP SDK's
  * recommended pattern.
  */
-function createServer() {
-    const server = new McpServer({ name: "context-doctor", version: "0.25.0" }, { instructions: SERVER_INSTRUCTIONS });
+function createServer({ fileAccess = true } = {}) {
+    const server = new McpServer({ name: "context-doctor", version: "0.27.0" }, { instructions: SERVER_INSTRUCTIONS });
     server.tool("profile_context", "Profile an LLM conversation or prompt: token breakdown, largest blocks, and actionable findings about wasted context (duplicates, oversized pastes or tool results, base64 blobs, long history). Two inputs, pass ONE: `conversation` (full OpenAI/Anthropic JSON or raw text, for agents, files and proxies) or `sketch` (for chat apps such as Claude Desktop or ChatGPT where you cannot export the conversation: the turn count plus the few blocks that matter, ~120 tokens to write). Call it whenever the user asks about token usage, context size, cost, speed or limits, and on your own once the conversation passes ~30 turns or holds 3+ large pastes. Act on the top finding in your reply.", {
         conversation: z.string().optional().describe("Conversation JSON (OpenAI or Anthropic format, or bare message array) or raw prompt text. Omit in chat apps and pass `sketch`."),
         sketch: z.object({
@@ -68,7 +71,9 @@ function createServer() {
             })).max(500).describe("Only the blocks over ~500 tokens, repeated, or images. Plain turns need not be listed."),
         }).optional().describe("Coarse description of the conversation for chat apps. One size hint per block: lines for code, words for prose, chars (or tokens) for logs and tool output, which vary most per line."),
         model: z.string().optional().describe("Target model name for context-window math, e.g. claude-sonnet-5 or gpt-4o"),
-    }, async ({ conversation, sketch, model }) => {
+    }, 
+    // Pure: reads its input and returns text. Clients (Codex, ChatGPT) may skip approval for read-only tools.
+    { title: "Profile context", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, async ({ conversation, sketch, model }) => {
         if (sketch) {
             return { content: [{ type: "text", text: runSketch({ ...sketch, model: sketch.model ?? model }) }] };
         }
@@ -81,13 +86,15 @@ function createServer() {
         const profile = profileConversation(parseConversation(conversation), model);
         return { content: [{ type: "text", text: renderProfile(profile) }] };
     });
-    server.tool("optimize_context", "Rewrite a conversation to reclaim tokens using deterministic strategies: dedupe repeated content, trim stale tool results, strip base64 blobs, optionally prune old history. Returns the slimmed conversation JSON plus a savings summary. No LLM calls — safe and inspectable. Call this after profile_context finds recoverable waste and the user wants it fixed; add the prune-history strategy only with the user's consent, then write the replacement summary yourself as the result instructs.", {
+    server.tool("optimize_context", "Rewrite a conversation to reclaim tokens with deterministic strategies (dedupe, trim stale tool results, strip base64; prune-history is optional and lossy). Returns the slimmed JSON and a savings summary; no LLM calls. Use after profile_context finds recoverable waste the user wants fixed. Add prune-history only with the user's consent, then write the replacement summary as the result instructs.", {
         conversation: z.string().describe("Conversation JSON (OpenAI or Anthropic format, or bare message array)"),
         strategies: z.array(z.enum(STRATEGY_IDS)).optional()
             .describe("Strategies to apply. Default: dedupe, trim-tool-results, strip-base64. Add trim-tool-calls to shrink big inline file writes, or prune-history for lossy compaction of old turns."),
         keep_recent: z.number().int().nonnegative().optional().describe("Messages at the tail to leave untouched (default 6; 0 = none)"),
         max_tool_result_tokens: z.number().int().positive().optional().describe("Token budget for trimmed tool results (default 300)"),
-    }, async ({ conversation, strategies, keep_recent, max_tool_result_tokens }) => {
+    }, 
+    // Pure: reads its input and returns text. Clients (Codex, ChatGPT) may skip approval for read-only tools.
+    { title: "Optimize context", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, async ({ conversation, strategies, keep_recent, max_tool_result_tokens }) => {
         const result = optimizeConversation(conversation, {
             strategies: strategies,
             keepRecent: keep_recent,
@@ -129,6 +136,31 @@ function createServer() {
         }
         return { content };
     });
+    server.tool("pack_context", "Read only the parts of large files a question needs: chunks files or folders along their structure, ranks the chunks against `query` and returns the best that fit `max_tokens`, each with an id and line range. No query: an outline; pick with `ids`. Use instead of reading a big doc, log or folder whole when not editing it. Text, code, PDF, Word, PowerPoint, ODT.", {
+        paths: z.array(z.string()).max(50).optional().describe("Files or folders (absolute, or relative to the server's cwd)"),
+        text: z.string().optional().describe("Raw text to pack instead of files"),
+        query: z.string().optional().describe("The question; omit for an outline"),
+        max_tokens: z.number().int().positive().max(200_000).optional().describe("Budget, default 4000"),
+        ids: z.array(z.string()).max(200).optional().describe("Chunk ids from an outline or result, e.g. docs/guide.md#4"),
+        model: z.string().optional().describe("Model the budget is counted for (default Claude)"),
+        semantic: z.boolean().optional().describe("Also rank by meaning (local Ollama embeddings)"),
+    }, 
+    // Reads files, never writes. Clients (Codex, ChatGPT) may skip approval for read-only tools.
+    { title: "Pack context", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, async ({ paths, text, query, max_tokens, ids, model, semantic }) => {
+        const err = (t) => ({ content: [{ type: "text", text: t }], isError: true });
+        if (paths?.length && !fileAccess) {
+            return err("This server runs over HTTP and does not read files. Pass the document as `text`.");
+        }
+        const { sources, skipped } = paths?.length ? readSources(paths) : { sources: [], skipped: [] };
+        if (text)
+            sources.push({ name: "text", text });
+        if (sources.length === 0) {
+            return err(skipped.length ? `Nothing readable: ${skipped.join("; ")}` : "Pass `paths` (files or folders) or `text`.");
+        }
+        const result = await packContextSemantic(sources, { query, budget: max_tokens, ids, model, semantic });
+        const note = skipped.length ? `\nSkipped: ${skipped.slice(0, 10).join("; ")}${skipped.length > 10 ? ` (+${skipped.length - 10} more)` : ""}` : "";
+        return { content: [{ type: "text", text: renderPack(result) + note }] };
+    });
     // A prompt shows up in Claude Desktop's "+" menu, so a user can run a checkup
     // with one click instead of typing a request. It is the only proactive
     // surface MCP offers besides `instructions`.
@@ -148,7 +180,9 @@ function createServer() {
     }));
     server.tool("context_best_practices", "Get a curated checklist of context-management best practices, optionally specialized for a provider (anthropic, openai).", {
         provider: z.enum(["general", "anthropic", "openai"]).optional().describe("Provider to specialize tips for (default: general)"),
-    }, async ({ provider }) => {
+    }, 
+    // Pure: reads its input and returns text. Clients (Codex, ChatGPT) may skip approval for read-only tools.
+    { title: "Context best practices", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, async ({ provider }) => {
         const tips = [...BEST_PRACTICES.general, ...(provider && provider !== "general" ? BEST_PRACTICES[provider] : [])];
         return { content: [{ type: "text", text: tips.map((t, i) => `${i + 1}. ${t}`).join("\n") }] };
     });
@@ -239,7 +273,7 @@ if (argv.includes("--http")) {
                 setHeader(req, "accept", "application/json, text/event-stream");
             }
             // Fresh server + transport per request (stateless — nothing shared).
-            const server = createServer();
+            const server = createServer({ fileAccess: false });
             const transport = new StreamableHTTPServerTransport({
                 sessionIdGenerator: undefined,
                 // A client that never asked for a stream gets plain JSON back.

package/dist/mcpschema.d.ts

@@ -0,0 +1,51 @@
+/**
+ * The MCP schema tax: what each MCP server's tool definitions add to every
+ * request, and whether the server is used at all.
+ *
+ * Tool definitions (name, description, JSON schema) ride in the request's
+ * `tools` array, so they are part of the fixed overhead `overhead` measures;
+ * transcripts never contain them. Two sources:
+ *  - usage, from transcripts: every MCP call is a tool_use named
+ *    `mcp__<server>__<tool>`, which also reveals servers the app provides
+ *    (desktop browser, connectors) that no config file lists;
+ *  - size, by asking each configured server for its tool list, exactly as
+ *    Claude Code does at startup. That launches the servers, so it runs only
+ *    when asked (`overhead --mcp`).
+ *
+ * Claude Code's tool search defers MCP definitions until the model looks them
+ * up; with it on, an unused server costs about its tool names, not its
+ * schemas. Usage of ToolSearch in the transcripts says whether that is so.
+ */
+export interface McpServerConfig {
+    name: string;
+    scope: "user" | "project" | ".mcp.json";
+    command?: string;
+    args?: string[];
+    env?: Record<string, string>;
+    url?: string;
+    type?: string;
+}
+export interface McpServerSize {
+    name: string;
+    tools?: number;
+    /** Tokens of the full definitions: what an up-front load costs on every request. */
+    schemaTokens?: number;
+    /** Tokens of the tool names alone: roughly what a deferred load costs. */
+    nameTokens?: number;
+    error?: string;
+}
+export interface McpUsage {
+    /** Calls per server name (as it appears in tool names), main chain and subagents. */
+    calls: Map<string, number>;
+    sessions: number;
+    /** Sessions in which the model used ToolSearch, i.e. MCP definitions were deferred. */
+    toolSearchSessions: number;
+}
+/** Claude Code's spelling of a server name inside tool names. */
+export declare function toolPrefixName(server: string): string;
+/** MCP servers Claude Code starts in `cwd`, with their launch configs. */
+export declare function claudeMcpConfigs(cwd?: string, home?: string): McpServerConfig[];
+/** MCP calls per server over the last `days`, from Claude Code transcripts (a cheap line scan, no JSON parse per line). */
+export declare function mcpUsage(days?: number, paths?: string[]): McpUsage;
+/** Connect to one server the way a client does and size its tool list. */
+export declare function measureServer(cfg: McpServerConfig, timeoutMs?: number): Promise<McpServerSize>;

package/dist/mcpschema.js

@@ -0,0 +1,125 @@
+/**
+ * The MCP schema tax: what each MCP server's tool definitions add to every
+ * request, and whether the server is used at all.
+ *
+ * Tool definitions (name, description, JSON schema) ride in the request's
+ * `tools` array, so they are part of the fixed overhead `overhead` measures;
+ * transcripts never contain them. Two sources:
+ *  - usage, from transcripts: every MCP call is a tool_use named
+ *    `mcp__<server>__<tool>`, which also reveals servers the app provides
+ *    (desktop browser, connectors) that no config file lists;
+ *  - size, by asking each configured server for its tool list, exactly as
+ *    Claude Code does at startup. That launches the servers, so it runs only
+ *    when asked (`overhead --mcp`).
+ *
+ * Claude Code's tool search defers MCP definitions until the model looks them
+ * up; with it on, an unused server costs about its tool names, not its
+ * schemas. Usage of ToolSearch in the transcripts says whether that is so.
+ */
+import { readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join, resolve } from "node:path";
+import { forEachLine, listSessions } from "./session.js";
+import { estimateTokens } from "./tokens.js";
+const CLAUDE = "claude-opus-5";
+/** Claude Code's spelling of a server name inside tool names. */
+export function toolPrefixName(server) {
+    return server.replace(/[^A-Za-z0-9_-]/g, "_");
+}
+/** MCP servers Claude Code starts in `cwd`, with their launch configs. */
+export function claudeMcpConfigs(cwd = process.cwd(), home = homedir()) {
+    const out = new Map();
+    const addAll = (servers, scope) => {
+        for (const [name, c] of Object.entries(servers ?? {})) {
+            if (c && typeof c === "object")
+                out.set(name, { name, scope, command: c.command, args: c.args, env: c.env, url: c.url, type: c.type });
+        }
+    };
+    try {
+        const cfg = JSON.parse(readFileSync(join(home, ".claude.json"), "utf8"));
+        addAll(cfg.mcpServers, "user");
+        addAll(cfg.projects?.[resolve(cwd)]?.mcpServers, "project");
+    }
+    catch {
+        /* absent */
+    }
+    try {
+        addAll(JSON.parse(readFileSync(join(cwd, ".mcp.json"), "utf8")).mcpServers, ".mcp.json");
+    }
+    catch {
+        /* absent */
+    }
+    return [...out.values()].sort((a, b) => a.name.localeCompare(b.name));
+}
+/** MCP calls per server over the last `days`, from Claude Code transcripts (a cheap line scan, no JSON parse per line). */
+export function mcpUsage(days = 30, paths) {
+    const since = Date.now() - days * 86_400_000;
+    const files = paths ?? listSessions(10_000).filter((s) => s.path.includes(".claude") && s.modifiedAt.getTime() >= since).map((s) => s.path);
+    const calls = new Map();
+    let toolSearchSessions = 0;
+    for (const path of files) {
+        let searched = false;
+        forEachLine(path, (line) => {
+            if (!line.includes('"tool_use"'))
+                return;
+            for (const m of line.matchAll(/"name":"mcp__([^"]+?)__[^"]+"/g))
+                calls.set(m[1], (calls.get(m[1]) ?? 0) + 1);
+            if (!searched && line.includes('"name":"ToolSearch"'))
+                searched = true;
+        });
+        if (searched)
+            toolSearchSessions++;
+    }
+    return { calls, sessions: files.length, toolSearchSessions };
+}
+/** Connect to one server the way a client does and size its tool list. */
+export async function measureServer(cfg, timeoutMs = 15_000) {
+    const { Client } = await import("@modelcontextprotocol/sdk/client/index.js");
+    const client = new Client({ name: "context-doctor-overhead", version: "1" });
+    let transport;
+    if (cfg.command) {
+        const { StdioClientTransport } = await import("@modelcontextprotocol/sdk/client/stdio.js");
+        transport = new StdioClientTransport({
+            command: cfg.command,
+            args: cfg.args ?? [],
+            env: { ...process.env, ...(cfg.env ?? {}) },
+            stderr: "ignore",
+        });
+    }
+    else if (cfg.url && cfg.type !== "sse") {
+        const { StreamableHTTPClientTransport } = await import("@modelcontextprotocol/sdk/client/streamableHttp.js");
+        transport = new StreamableHTTPClientTransport(new URL(cfg.url));
+    }
+    else {
+        return { name: cfg.name, error: cfg.type === "sse" ? "SSE server: not measured" : "no command or url" };
+    }
+    let timer;
+    const work = (async () => {
+        await client.connect(transport);
+        const tools = [];
+        let cursor;
+        do {
+            const page = await client.listTools(cursor ? { cursor } : {});
+            tools.push(...page.tools);
+            cursor = page.nextCursor;
+        } while (cursor && tools.length < 5000);
+        const prefix = `mcp__${toolPrefixName(cfg.name)}__`;
+        const schemaTokens = tools.reduce((s, t) => s + estimateTokens(JSON.stringify({ name: prefix + t.name, description: t.description ?? "", input_schema: t.inputSchema ?? {} }), CLAUDE), 0);
+        const nameTokens = tools.reduce((s, t) => s + estimateTokens(prefix + t.name + "\n", CLAUDE), 0);
+        return { name: cfg.name, tools: tools.length, schemaTokens, nameTokens };
+    })();
+    const timeout = new Promise((res) => {
+        timer = setTimeout(() => res({ name: cfg.name, error: `no answer in ${timeoutMs / 1000}s` }), timeoutMs);
+    });
+    try {
+        return await Promise.race([work, timeout]);
+    }
+    catch (e) {
+        const msg = String(e?.message ?? e).split("\n")[0];
+        return { name: cfg.name, error: /401|403|unauthori[sz]ed/i.test(msg) ? "needs sign-in (not measured)" : msg.slice(0, 100) };
+    }
+    finally {
+        clearTimeout(timer);
+        await client.close().catch(() => { });
+    }
+}

package/dist/overhead.d.ts

@@ -0,0 +1,76 @@
+/**
+ * `context-doctor overhead`: what every request pays before you type.
+ *
+ * Each request an agent sends re-sends the same front matter: the harness's
+ * system prompt, tool and MCP schemas, skills, and the memory files the user
+ * wrote (CLAUDE.md and its imports, rules, auto memory, AGENTS.md, Cursor
+ * rules, GEMINI.md). It is cached, so each read is cheap, but it is read on
+ * every request of every session, and at full write price after every cold
+ * start. Nobody sees it: transcripts do not contain it.
+ *
+ * Two measurements, both local:
+ *  - the fixed overhead itself, from the first request of each Claude Code
+ *    session (the API's own count, minus the first user message);
+ *  - the memory files that load in a directory, sized with Claude's ratios and
+ *    priced per month from the user's own request and cold-start counts.
+ */
+import { type McpServerConfig, type McpServerSize, type McpUsage } from "./mcpschema.js";
+export type Agent = "Claude Code" | "Codex" | "Cursor" | "Gemini CLI";
+export interface MemoryFile {
+    agent: Agent;
+    path: string;
+    tokens: number;
+    /** Why it loads: "project", "user", "import", "rules", "auto memory". */
+    via: string;
+}
+export interface OverheadFinding {
+    severity: "warn" | "info";
+    message: string;
+    suggestion: string;
+}
+export interface Baseline {
+    sessions: number;
+    /** Median tokens of the first request, minus the first user message. */
+    median: number;
+    p90: number;
+    requests: number;
+    coldStarts: number;
+    days: number;
+    model?: string;
+    /** USD a month for each 1,000 tokens of fixed overhead, at this usage. */
+    usdPerKPerMonth?: number;
+}
+export interface OverheadReport {
+    cwd: string;
+    baseline?: Baseline;
+    files: MemoryFile[];
+    findings: OverheadFinding[];
+    mcp: McpSection;
+}
+export interface McpSection {
+    configs: McpServerConfig[];
+    usage: McpUsage;
+    /** Present when the servers were launched and asked for their tools (`--mcp`). */
+    sizes?: McpServerSize[];
+}
+/** Every memory file the four agents load for a session started in `cwd`. */
+export declare function findMemoryFiles(cwd?: string, home?: string): MemoryFile[];
+/**
+ * Fixed overhead per session, measured: the first main-chain request's input
+ * (input + cache read + cache write) minus the first user message, plus the
+ * request and cold-start counts that turn tokens into a monthly bill.
+ */
+export declare function measureBaseline(days?: number, paths?: string[]): Baseline | undefined;
+/** What is worth changing in the memory files. */
+export declare function overheadFindings(files: MemoryFile[], baseline?: Baseline): OverheadFinding[];
+export declare function overheadReport(opts?: {
+    cwd?: string;
+    home?: string;
+    days?: number;
+    paths?: string[];
+}): OverheadReport;
+/** Launch each configured server and size its tool definitions, then add the findings they support. */
+export declare function measureMcpSizes(r: OverheadReport): Promise<void>;
+/** Configured servers nobody called: their definitions ride on every request for nothing. */
+export declare function mcpFindings(m: McpSection, baseline?: Baseline): OverheadFinding[];
+export declare function renderOverhead(r: OverheadReport, home?: string): string;