context-doctor 0.5.0 → 0.6.0

package/README.md

@@ -92,7 +92,7 @@ Practical upshot: a developer who only wants cheaper, faster API calls never tou
 | `context-doctor install` / `uninstall` | Wire (or remove) everything: MCP for Claude Desktop/Code/Cursor, the Agent Skill, the every-prompt hook |
 | `context-doctor analyze <file>` | Profile a conversation: token breakdown, findings, cost + latency estimates |
 | `context-doctor optimize <file>` | Apply the safe fixes; `--strategy prune-history` for consented lossy compaction |
-| `context-doctor session [file]` | Profile a Claude Code session transcript (defaults to your most recent; `--list` to browse) |
+| `context-doctor session [file]` | Profile a Claude Code session transcript (defaults to your most recent; `--list` to browse) — also reads ChatGPT data exports (`conversations.json`) |
 | `context-doctor report` | Machine-wide impact report: exact proxy savings, hook activity, recoverable waste in recent sessions |
 | `context-doctor proxy` | Always-on local proxy that optimizes every Anthropic/OpenAI API request in flight (`/stats` for cumulative savings) |
 | `context-doctor watch [file]` | Live monitor of a growing session/agent trace: token/cost line per change, findings as they appear |
@@ -241,6 +241,7 @@ const { conversation, tokensBefore, tokensAfter } = optimizeConversation(chatJso
 
 - **Oversized tool results** — the #1 context killer in agent loops
 - **Duplicate content** — the same doc/result pasted twice
+- **Near-duplicates** — the same doc re-pasted with different surrounding words (shingle similarity, ≥60%)
 - **Repeated identical tool calls** — a signal your agent forgot earlier results
 - **Base64 / binary blobs** in text content
 - **Long history** past the point where models track the middle
@@ -292,7 +293,11 @@ A tool that promises speed must be near-free. Measured overhead per touchpoint:
 
 Net effect is strongly negative overhead: the tokens these touchpoints save on every subsequent call dwarf what they cost.
 
-## Why token counts are "~"
+## Why token counts are "~" (and how to make them exact)
+
+Want exact numbers? `analyze --exact` uses the **Anthropic count-tokens API** for Claude models (set `ANTHROPIC_API_KEY`; opt-in network call, key never stored) or **tiktoken** for GPT models (install it next to context-doctor) — and reports how far off the heuristic was.
+
+
 
 Exact counts require each provider's private tokenizer. `context-doctor` uses a calibrated chars-per-token heuristic (denser for code/JSON) that lands within ~10% — plenty accurate for finding what's heavy and measuring savings, and it keeps the tool fully offline with zero configuration.
 

package/dist/cli.js

@@ -22,6 +22,7 @@ import { buildImpactReport } from "./impact.js";
 import { recordLedger } from "./ledger.js";
 import { runDoctor } from "./doctor.js";
 import { runWatch } from "./watch.js";
+import { exactTokenCount } from "./exact.js";
 const HELP = `context-doctor — profile and optimize LLM context windows
 
 Usage:
@@ -49,6 +50,8 @@ message array). Use "-" to read from stdin.
 
 Options:
   --model <name>          Model name for window-size math (e.g. claude-sonnet-5, gpt-4o)
+  --exact                 (analyze) Add an exact token count: Anthropic count-tokens API for
+                          Claude models (needs ANTHROPIC_API_KEY), tiktoken for GPT (if installed)
   --json                  Machine-readable output
   --out <file>            (optimize) Write result to file instead of stdout
   --strategy <id>         (optimize) Strategy to run; repeatable.
@@ -70,7 +73,7 @@ Examples:
           export OPENAI_BASE_URL=http://localhost:8787/v1
 `;
 function parseArgs(argv) {
-    const args = { json: false, strategies: [], list: false };
+    const args = { json: false, strategies: [], list: false, exact: false };
     const positional = [];
     for (let i = 0; i < argv.length; i++) {
         const a = argv[i];
@@ -85,6 +88,9 @@ function parseArgs(argv) {
             case "--list":
                 args.list = true;
                 break;
+            case "--exact":
+                args.exact = true;
+                break;
             case "--model":
                 args.model = argv[++i];
                 break;
@@ -215,6 +221,17 @@ function main() {
     if (args.command === "analyze") {
         const profile = profileConversation(parseConversation(input), args.model);
         console.log(args.json ? JSON.stringify(profile, null, 2) : renderProfile(profile));
+        if (args.exact) {
+            void exactTokenCount(input, args.model).then((exact) => {
+                if (exact.tokens !== undefined) {
+                    const drift = profile.totalTokens > 0 ? Math.round(((exact.tokens - profile.totalTokens) / exact.tokens) * 100) : 0;
+                    console.log(`\nExact input tokens: ${exact.tokens} (${exact.source}) — heuristic was off by ${drift}%`);
+                }
+                else {
+                    console.log(`\nExact count unavailable: ${exact.note}`);
+                }
+            });
+        }
         return;
     }
     if (args.command === "optimize") {

package/dist/exact.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Optional exact token counting (`analyze --exact`). Zero-config heuristic
+ * remains the default; this upgrades the TOTAL where an exact source exists:
+ *
+ *   Claude models — Anthropic's count-tokens API when ANTHROPIC_API_KEY is
+ *   set (opt-in network call; the key is read from env, never stored).
+ *   GPT models   — tiktoken, when the user has installed it alongside us
+ *   (optional peer; we never ship the WASM weight by default).
+ *
+ * Anything else falls back to the heuristic with a note saying why.
+ */
+export interface ExactResult {
+    tokens?: number;
+    source?: string;
+    note?: string;
+}
+export declare function exactTokenCount(conversationJson: string, model?: string): Promise<ExactResult>;

package/dist/exact.js

@@ -0,0 +1,77 @@
+/**
+ * Optional exact token counting (`analyze --exact`). Zero-config heuristic
+ * remains the default; this upgrades the TOTAL where an exact source exists:
+ *
+ *   Claude models — Anthropic's count-tokens API when ANTHROPIC_API_KEY is
+ *   set (opt-in network call; the key is read from env, never stored).
+ *   GPT models   — tiktoken, when the user has installed it alongside us
+ *   (optional peer; we never ship the WASM weight by default).
+ *
+ * Anything else falls back to the heuristic with a note saying why.
+ */
+export async function exactTokenCount(conversationJson, model) {
+    let conv;
+    try {
+        conv = JSON.parse(conversationJson);
+    }
+    catch {
+        return { note: "exact counting needs a JSON conversation" };
+    }
+    const targetModel = model ?? conv.model;
+    if (!targetModel)
+        return { note: "pass --model to enable exact counting" };
+    if (!Array.isArray(conv.messages) || conv.messages.length === 0) {
+        return { note: "no messages array — exact counting skipped" };
+    }
+    if (/claude/i.test(targetModel)) {
+        const apiKey = process.env.ANTHROPIC_API_KEY;
+        if (!apiKey)
+            return { note: "set ANTHROPIC_API_KEY to get exact Claude counts (count-tokens API)" };
+        try {
+            const res = await fetch("https://api.anthropic.com/v1/messages/count_tokens", {
+                method: "POST",
+                headers: { "x-api-key": apiKey, "anthropic-version": "2023-06-01", "content-type": "application/json" },
+                body: JSON.stringify({
+                    model: targetModel,
+                    messages: conv.messages,
+                    ...(conv.system != null ? { system: conv.system } : {}),
+                }),
+                signal: AbortSignal.timeout(15_000),
+            });
+            const data = (await res.json());
+            if (!res.ok || typeof data.input_tokens !== "number") {
+                return { note: `count-tokens API: ${data.error?.message ?? `HTTP ${res.status}`} — using heuristic` };
+            }
+            return { tokens: data.input_tokens, source: "Anthropic count-tokens API" };
+        }
+        catch (e) {
+            return { note: `count-tokens API unreachable (${e.message}) — using heuristic` };
+        }
+    }
+    if (/gpt|^o\d/i.test(targetModel)) {
+        try {
+            // Optional peer — resolves only if the user installed it next to us.
+            // @ts-expect-error optional dependency without bundled types
+            const tiktoken = await import("tiktoken");
+            const enc = tiktoken.get_encoding("o200k_base");
+            try {
+                const text = conv.messages
+                    .map((m) => {
+                    const c = m.content;
+                    return typeof c === "string" ? c : JSON.stringify(c ?? "");
+                })
+                    .join("\n");
+                // +4/message structural overhead, mirroring OpenAI's chat format math.
+                const tokens = enc.encode(text).length + conv.messages.length * 4;
+                return { tokens, source: "tiktoken (o200k_base)" };
+            }
+            finally {
+                enc.free();
+            }
+        }
+        catch {
+            return { note: "install tiktoken next to context-doctor for exact GPT counts (npm i tiktoken)" };
+        }
+    }
+    return { note: `no exact tokenizer for ${targetModel} — using heuristic` };
+}

package/dist/mcp.js

@@ -37,7 +37,7 @@ const STRATEGY_IDS = ["dedupe", "trim-tool-results", "strip-base64", "prune-hist
  * recommended pattern.
  */
 function createServer() {
-    const server = new McpServer({ name: "context-doctor", version: "0.5.0" }, { instructions: SERVER_INSTRUCTIONS });
+    const server = new McpServer({ name: "context-doctor", version: "0.6.0" }, { instructions: SERVER_INSTRUCTIONS });
     server.tool("profile_context", "Profile an LLM conversation or prompt: token breakdown by category, largest messages, and actionable findings about wasted context (duplicates, oversized tool results, base64 blobs, cache-unfriendly ordering). Accepts OpenAI/Anthropic conversation JSON or raw text. Call this immediately whenever the user asks about token usage, context size, LLM cost, or latency — and proactively offer it once a conversation grows long or accumulates large pasted content.", {
         conversation: z.string().describe("Conversation JSON (OpenAI or Anthropic format, or bare message array) or raw prompt text"),
         model: z.string().optional().describe("Target model name for context-window math, e.g. claude-sonnet-5 or gpt-4o"),

package/dist/profile.d.ts

@@ -12,7 +12,7 @@ export interface MessageProfile {
     preview: string;
     toolName?: string;
 }
-export type FindingId = "large_tool_result" | "duplicate_content" | "repeated_tool_call" | "base64_blob" | "long_history" | "large_system_prompt" | "cache_ordering" | "near_window_limit";
+export type FindingId = "large_tool_result" | "duplicate_content" | "near_duplicate" | "repeated_tool_call" | "base64_blob" | "long_history" | "large_system_prompt" | "cache_ordering" | "near_window_limit";
 export interface Finding {
     id: FindingId;
     severity: "info" | "warn" | "high";

package/dist/profile.js

@@ -23,6 +23,43 @@ function contentHash(text) {
     return createHash("sha1").update(text.replace(/\s+/g, " ").trim()).digest("hex");
 }
 const BASE64_RE = /(?:data:[\w/+.-]+;base64,|[A-Za-z0-9+/]{500,}={0,2})/;
+/** FNV-1a — cheap deterministic hash for shingle sampling. */
+function fnv1a(s) {
+    let h = 0x811c9dc5;
+    for (let i = 0; i < s.length; i++) {
+        h ^= s.charCodeAt(i);
+        h = Math.imul(h, 0x01000193);
+    }
+    return h >>> 0;
+}
+/**
+ * Sampled 8-word shingle set for near-duplicate detection. Keeping only
+ * ~1/8th of shingles (by hash) shrinks sets ~8x while preserving the Jaccard
+ * estimate — pairwise comparison stays cheap even on large sessions.
+ */
+function sampledShingles(text) {
+    const words = text.toLowerCase().replace(/\s+/g, " ").trim().split(" ");
+    // Sampling is a large-text optimization only: short texts keep every
+    // shingle (sampling them starves the Jaccard estimate), long ones keep ~1/8.
+    const sample = words.length > 1500;
+    const out = new Set();
+    for (let i = 0; i + 8 <= words.length; i++) {
+        const h = fnv1a(words.slice(i, i + 8).join(" "));
+        if (!sample || h % 8 === 0)
+            out.add(h);
+    }
+    return out;
+}
+function jaccard(a, b) {
+    if (a.size === 0 || b.size === 0)
+        return 0;
+    let inter = 0;
+    const [small, large] = a.size <= b.size ? [a, b] : [b, a];
+    for (const v of small)
+        if (large.has(v))
+            inter++;
+    return inter / (a.size + b.size - inter);
+}
 export function profileConversation(conv, model) {
     const perMessage = conv.messages.map((m) => ({
         msg: m,
@@ -69,6 +106,38 @@ export function profileConversation(conv, model) {
             seen.set(h, p.msg.index);
         }
     }
+    // -- Near-duplicates: same content wrapped in different lead-ins -------------
+    // Exact hashing (above) misses "here's the doc again: <doc>"; sampled-shingle
+    // Jaccard catches it. Capped to the 150 largest 300+-char messages.
+    {
+        const candidates = perMessage
+            .filter((p) => p.msg.text.length >= 300)
+            .sort((a, b) => b.tokens - a.tokens)
+            .slice(0, 150);
+        const shingleSets = candidates.map((p) => sampledShingles(p.msg.text));
+        const exactDup = new Set(findings.filter((f) => f.id === "duplicate_content").flatMap((f) => f.messages));
+        for (let i = 0; i < candidates.length; i++) {
+            for (let j = i + 1; j < candidates.length; j++) {
+                const a = candidates[i];
+                const b = candidates[j];
+                if (exactDup.has(a.msg.index) && exactDup.has(b.msg.index))
+                    continue; // already flagged exactly
+                const sim = jaccard(shingleSets[i], shingleSets[j]);
+                if (sim >= 0.6) {
+                    const smaller = Math.min(a.tokens, b.tokens);
+                    const [first, second] = a.msg.index <= b.msg.index ? [a, b] : [b, a];
+                    findings.push({
+                        id: "near_duplicate",
+                        severity: "warn",
+                        estSavings: Math.round(smaller * sim * 0.9),
+                        message: `Messages #${first.msg.index} and #${second.msg.index} are ~${Math.round(sim * 100)}% similar (~${smaller} tokens repeated with different framing).`,
+                        suggestion: "Replace the later copy with a short reference to the first — repeats survive even when the surrounding words differ.",
+                        messages: [first.msg.index, second.msg.index],
+                    });
+                }
+            }
+        }
+    }
     // -- Repeated identical tool calls ------------------------------------------
     const callSeen = new Map();
     for (const p of perMessage) {

package/dist/session.js

@@ -39,8 +39,53 @@ export function listSessions(limit = 20) {
     }
     return sessions.sort((a, b) => b.modifiedAt.getTime() - a.modifiedAt.getTime()).slice(0, limit);
 }
+/**
+ * ChatGPT data export (chatgpt.com → Settings → Data controls → Export):
+ * conversations.json is an array of conversations, each holding a `mapping`
+ * tree of nodes. We profile the most recently updated conversation.
+ */
+function parseChatGPTExport(data, path) {
+    const conversations = data
+        .filter((c) => c && typeof c.mapping === "object")
+        .sort((a, b) => (b.update_time ?? 0) - (a.update_time ?? 0));
+    const conv = conversations[0];
+    if (!conv)
+        return { conversationJson: JSON.stringify({ messages: [] }), messageCount: 0, path };
+    const nodes = Object.values(conv.mapping)
+        .filter((n) => {
+        const m = n?.message;
+        if (!m?.author?.role || !["user", "assistant", "system"].includes(m.author.role))
+            return false;
+        const parts = m.content?.parts;
+        return Array.isArray(parts) && parts.some((p) => typeof p === "string" && p.length > 0);
+    })
+        .sort((a, b) => (a.message.create_time ?? 0) - (b.message.create_time ?? 0));
+    const messages = nodes.map((n) => ({
+        role: n.message.author.role,
+        content: n.message.content.parts.filter((p) => typeof p === "string").join("\n"),
+    }));
+    return {
+        conversationJson: JSON.stringify({ messages }),
+        title: typeof conv.title === "string" ? conv.title : undefined,
+        model: typeof conv.default_model_slug === "string" ? conv.default_model_slug : "gpt-5",
+        messageCount: messages.length,
+        path,
+    };
+}
 export function parseSessionFile(path) {
     const raw = readFileSync(path, "utf8");
+    // ChatGPT exports are one big JSON array, not JSONL.
+    if (raw.trimStart().startsWith("[")) {
+        try {
+            const data = JSON.parse(raw);
+            if (Array.isArray(data) && data.some((c) => c && typeof c.mapping === "object")) {
+                return parseChatGPTExport(data, path);
+            }
+        }
+        catch {
+            /* fall through to JSONL parsing */
+        }
+    }
     const messages = [];
     let title;
     let model;

package/dist/test/chatgpt-export.test.d.ts

@@ -0,0 +1,2 @@
+/** session: ChatGPT data-export (conversations.json) parsing. */
+export {};

package/dist/test/chatgpt-export.test.js

@@ -0,0 +1,45 @@
+/** session: ChatGPT data-export (conversations.json) parsing. */
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { parseSessionFile } from "../session.js";
+function node(id, role, text, t) {
+    return [id, { id, message: { author: { role }, content: { content_type: "text", parts: [text] }, create_time: t } }];
+}
+const older = {
+    title: "Older chat",
+    update_time: 100,
+    default_model_slug: "gpt-4o",
+    mapping: Object.fromEntries([node("a", "user", "old question", 1)]),
+};
+const newer = {
+    title: "Trip planning",
+    update_time: 200,
+    default_model_slug: "gpt-5",
+    mapping: Object.fromEntries([
+        node("r", "system", "You are helpful.", 1),
+        node("x", "user", "Plan me a trip to Japan with a detailed itinerary please.", 2),
+        node("y", "assistant", "Day 1: Tokyo. Day 2: Kyoto. Day 3: Osaka with food tour.", 3),
+        ["tool-node", { id: "tool-node", message: { author: { role: "tool" }, content: { content_type: "text", parts: ["ignored"] }, create_time: 4 } }],
+    ]),
+};
+test("parses a ChatGPT export: newest conversation, ordered messages, model detected", () => {
+    const dir = mkdtempSync(join(tmpdir(), "ctxdoc-gpt-"));
+    const file = join(dir, "conversations.json");
+    writeFileSync(file, JSON.stringify([older, newer]));
+    const parsed = parseSessionFile(file);
+    assert.equal(parsed.title, "Trip planning");
+    assert.equal(parsed.model, "gpt-5");
+    assert.equal(parsed.messageCount, 3); // tool node excluded
+    const conv = JSON.parse(parsed.conversationJson);
+    assert.equal(conv.messages[0].role, "system");
+    assert.equal(conv.messages[1].content.includes("Japan"), true);
+});
+test("JSONL transcripts still parse (no regression)", () => {
+    const dir = mkdtempSync(join(tmpdir(), "ctxdoc-jsonl-"));
+    const file = join(dir, "s.jsonl");
+    writeFileSync(file, JSON.stringify({ type: "user", message: { role: "user", content: "hi" } }) + "\n");
+    assert.equal(parseSessionFile(file).messageCount, 1);
+});

package/dist/test/smoke.test.js

@@ -88,6 +88,22 @@ test("prune-history never leaves an orphaned tool result at the head of the tail
     assert.equal(isToolResult, false, "tail must not start with an orphaned tool_result");
     assert.ok(result.applied.some((c) => c.strategy === "prune-history"), "pruning still happened");
 });
+test("near-duplicate detection catches same doc with different lead-ins", () => {
+    // Varied clauses (not a repeated sentence) — like a real document.
+    const doc = Array.from({ length: 30 }, (_, i) => `Clause ${i} of the pricing policy covers refund scenario ${i} where the customer holds receipt series ${i * 7} under regional rule ${i % 5}.`).join(" ");
+    const conv = JSON.stringify({
+        messages: [
+            { role: "user", content: "Here is our policy document for you to review:\n" + doc },
+            { role: "assistant", content: "Understood, thanks for sharing the policy." },
+            { role: "user", content: "Sharing the policy doc again with a totally different intro so exact hashing misses it:\n" + doc },
+        ],
+    });
+    const profile = profileConversation(parseConversation(conv));
+    const near = profile.findings.find((f) => f.id === "near_duplicate");
+    assert.ok(near, "near_duplicate finding expected");
+    assert.deepEqual(near.messages, [0, 2]);
+    assert.ok(near.estSavings > 50);
+});
 test("raw text input still profiles", () => {
     const profile = profileConversation(parseConversation("just some prompt text"));
     assert.equal(profile.messageCount, 1);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "context-doctor",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "description": "Profile and optimize LLM context windows. See what's eating your tokens and fix it — works with Claude, GPT, Gemini, and any MCP-capable AI app.",
   "keywords": [
     "llm",
@@ -41,7 +41,7 @@
     "build": "tsc && node -e \"const fs=require('fs');['dist/cli.js','dist/mcp.js'].forEach(f=>fs.chmodSync(f,0o755))\"",
     "prepublishOnly": "npm run build",
     "dev": "tsc --watch",
-    "test": "npm run build && node --test dist/test/smoke.test.js dist/test/proxy.test.js dist/test/hook.test.js dist/test/mcp-http.test.js dist/test/doctor.test.js dist/test/watch.test.js"
+    "test": "npm run build && node --test dist/test/smoke.test.js dist/test/proxy.test.js dist/test/hook.test.js dist/test/mcp-http.test.js dist/test/doctor.test.js dist/test/watch.test.js dist/test/chatgpt-export.test.js"
   },
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.0.0",