context-doctor 0.26.0 → 0.27.0

package/dist/cli.js

@@ -41,6 +41,10 @@ import { startDashboard } from "./dashboard.js";
 import { listCursorChats, parseCursorChat } from "./cursor.js";
 import { analyzeCacheUsage, renderCacheReport } from "./cache.js";
 import { renderToolTimings } from "./timing.js";
+import { packContextSemantic, readSources, renderPack } from "./pack.js";
+import { measureBaseline, measureMcpSizes, overheadReport, renderOverhead } from "./overhead.js";
+import { applySplit, planSplit, renderSplit } from "./split.js";
+import { ciReport, renderCiMarkdown, renderCiText } from "./ci.js";
 const HELP = `context-doctor — profile and optimize LLM context windows
 
 Usage:
@@ -92,6 +96,23 @@ Usage:
                                                 were resolved, and what it saves
   context-doctor accuracy                       Measure the token heuristic against the API's own
                                                 counts recorded in your transcripts (--limit n)
+  context-doctor overhead [--days n] [--mcp]    What every request re-reads before your message:
+                                                measured first-request size, each CLAUDE.md /
+                                                AGENTS.md / rules file priced per month, findings
+  context-doctor ci [--base <ref>] [--max-tokens n] [--max-increase n] [--markdown]
+                                                For pull requests: memory files (CLAUDE.md, AGENTS.md,
+                                                GEMINI.md, rules) at HEAD vs base; exit 1 over budget
+  context-doctor overhead split <file> [--write]
+                                                Move a memory file's big sections to a reference
+                                                file, leaving pointers (plan first; --write backs up)
+  context-doctor pack <files|dirs...> --query "<q>" [--max-tokens n]
+                                                Only the parts of big docs/code a question needs:
+                                                chunk along headings and declarations, rank, fit a
+                                                token budget (default 4000). No --query: an outline
+                                                to pick from (--ids a#2,b#5). Offline, no API key.
+                                                --semantic: also rank by meaning with a local
+                                                Ollama embedding model, when one is installed
+                                                (--embed-model <name> picks it)
   context-doctor watch [file]                   Live-monitor a growing session/agent trace: running
                                                 token/cost line per change, new findings as they appear
                                                 (--interval-ms n, default 2000)
@@ -247,12 +268,55 @@ function parseArgs(argv) {
             case "--config":
                 args.config = argv[++i];
                 break;
+            case "-q":
+            case "--query":
+                args.query = argv[++i];
+                break;
+            case "--max-tokens":
+                args.maxTokens = numArg("--max-tokens", argv[++i], { min: 1, integer: true });
+                break;
+            case "--chunk-tokens":
+                args.chunkTokens = numArg("--chunk-tokens", argv[++i], { min: 50, integer: true });
+                break;
+            case "--mcp":
+                args.mcp = true;
+                break;
+            case "--write":
+                args.write = true;
+                break;
+            case "--semantic":
+                args.semantic = args.semantic || true;
+                break;
+            case "--embed-model":
+                args.semantic = argv[++i];
+                break;
+            case "--base":
+                args.base = argv[++i];
+                break;
+            case "--markdown":
+                args.markdown = true;
+                break;
+            case "--max-increase":
+                args.maxIncrease = numArg("--max-increase", argv[++i], { min: 0, integer: true });
+                break;
+            case "--requests-per-day":
+                args.requestsPerDay = numArg("--requests-per-day", argv[++i], { min: 1 });
+                break;
+            case "--min-tokens":
+                args.minTokens = numArg("--min-tokens", argv[++i], { min: 50, integer: true });
+                break;
+            case "--ids":
+                args.ids = (argv[++i] ?? "").split(",").map((x) => x.trim()).filter(Boolean);
+                break;
             case "--upstream-anthropic":
                 args.upstreamAnthropic = argv[++i];
                 break;
             case "--upstream-openai":
                 args.upstreamOpenai = argv[++i];
                 break;
+            case "--upstream-google":
+                args.upstreamGoogle = argv[++i];
+                break;
             default: positional.push(a);
         }
     }
@@ -639,6 +703,7 @@ async function main() {
             autopilotPauseFile: args.autopilotPauseFile,
             anthropicUpstream: args.upstreamAnthropic,
             openaiUpstream: args.upstreamOpenai,
+            googleUpstream: args.upstreamGoogle,
             strategies: args.strategies.length > 0 ? args.strategies : loadedRc.config.strategies,
             keepRecent: args.keepRecent ?? loadedRc.config.keepRecent,
             maxToolResultTokens: args.maxToolTokens ?? loadedRc.config.maxToolResultTokens,
@@ -646,6 +711,58 @@ async function main() {
         });
         return; // server keeps the process alive
     }
+    if (args.command === "ci") {
+        const report = ciReport({ base: args.base, maxTokens: args.maxTokens, maxIncrease: args.maxIncrease });
+        console.log(args.json ? JSON.stringify(report, null, 2) : args.markdown ? renderCiMarkdown(report, { model: args.model, requestsPerDay: args.requestsPerDay }) : renderCiText(report));
+        if (report.breaches.length)
+            process.exitCode = 1;
+        return;
+    }
+    if (args.command === "overhead" && args.positionals?.[0] === "split") {
+        const file = args.positionals[1];
+        if (!file) {
+            console.error("usage: context-doctor overhead split <memory file> [--min-tokens 400] [--write]");
+            process.exit(1);
+        }
+        let plan;
+        try {
+            plan = planSplit(file, { minTokens: args.minTokens });
+        }
+        catch (e) {
+            console.error(`Could not read ${file}: ${e.message}`);
+            process.exit(1);
+        }
+        console.log(renderSplit(plan, measureBaseline(args.days ?? 30)?.usdPerKPerMonth));
+        if (args.write && plan.sections.some((s) => s.moved)) {
+            const { backup } = applySplit(plan);
+            console.log(`\nWritten. Backup: ${backup}\nUndo: mv "${backup}" "${file}" (and delete or trim ${plan.referencePath}).`);
+        }
+        return;
+    }
+    if (args.command === "overhead") {
+        const report = overheadReport({ days: args.days });
+        if (args.mcp)
+            await measureMcpSizes(report);
+        console.log(args.json ? JSON.stringify(report, (_k, v) => (v instanceof Map ? Object.fromEntries(v) : v), 2) : renderOverhead(report));
+        return;
+    }
+    if (args.command === "pack") {
+        const paths = args.positionals ?? [];
+        if (paths.length === 0) {
+            console.error('usage: context-doctor pack <files|dirs...> --query "<question>" [--max-tokens 4000] [--ids a#2,b#5] [--json]');
+            process.exit(1);
+        }
+        const { sources, skipped } = paths.includes("-")
+            ? { sources: [{ name: "stdin", text: readFileSync(0, "utf8") }], skipped: [] }
+            : readSources(paths);
+        for (const s of skipped)
+            console.error(`context-doctor: skipped ${s}`);
+        if (sources.length === 0)
+            process.exit(1);
+        const result = await packContextSemantic(sources, { query: args.query, budget: args.maxTokens, ids: args.ids, maxChunkTokens: args.chunkTokens, model: args.model, semantic: args.semantic });
+        console.log(args.json ? JSON.stringify({ ...result, chunks: undefined, outline: result.chunks.map(({ text, ...c }) => c) }, null, 2) : renderPack(result));
+        return;
+    }
     if (!args.command || !args.file) {
         const known = ["analyze", "optimize"];
         if (args.command && !known.includes(args.command)) {

package/dist/doctor.js

@@ -13,6 +13,7 @@ import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
 import { ledgerPath, recordLedger } from "./ledger.js";
 import { loadConfig } from "./config.js";
+import { geminiCliPresent, geminiSettingsPath } from "./install.js";
 function claudeDesktopConfigPath() {
     switch (platform()) {
         case "darwin": return join(homedir(), "Library", "Application Support", "Claude", "claude_desktop_config.json");
@@ -165,6 +166,32 @@ export async function runDoctor() {
     else {
         checks.push({ label: "Codex", status: "skip", detail: "not detected (ChatGPT's Codex agent, IDE extension or CLI)" });
     }
+    // Gemini CLI: MCP server and the BeforeAgent hook, both in ~/.gemini/settings.json.
+    if (geminiCliPresent()) {
+        try {
+            const path = geminiSettingsPath();
+            const settings = existsSync(path) ? JSON.parse(readFileSync(path, "utf8")) : {};
+            checks.push(settings.mcpServers?.["context-doctor"]
+                ? { label: "Gemini CLI MCP", status: "ok", detail: "wired in ~/.gemini/settings.json" }
+                : { label: "Gemini CLI MCP", status: "fail", detail: "Gemini CLI detected but not wired — run: context-doctor install" });
+            const entries = settings.hooks?.BeforeAgent ?? [];
+            const ours = entries.map((e) => e.hooks?.[0]?.command ?? "").find((c) => /context-doctor|cli\.js"?\s+hook/.test(c));
+            const missing = ours ? hookBinaryMissing(ours) : undefined;
+            checks.push(!ours
+                ? { label: "Gemini CLI hook", status: "fail", detail: "not registered — run: context-doctor install" }
+                : missing
+                    ? { label: "Gemini CLI hook", status: "fail", detail: `registered, but ${missing} no longer exists — re-run: context-doctor install` }
+                    : settings.hooksConfig?.enabled === false
+                        ? { label: "Gemini CLI hook", status: "fail", detail: "registered, but hooksConfig.enabled is false in ~/.gemini/settings.json" }
+                        : { label: "Gemini CLI hook", status: "ok", detail: "BeforeAgent hook in ~/.gemini/settings.json" });
+        }
+        catch (e) {
+            checks.push({ label: "Gemini CLI", status: "fail", detail: `~/.gemini/settings.json unreadable (${e.message})` });
+        }
+    }
+    else {
+        checks.push({ label: "Gemini CLI", status: "skip", detail: "not detected" });
+    }
     // Cursor: its agent gets the guidance through a native postToolUse hook
     // (the Claude Code hook it also runs cannot add context there).
     if (existsSync(join(homedir(), ".cursor"))) {

package/dist/embed.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Optional semantic ranking for `pack`, through a local Ollama.
+ *
+ * BM25 misses pure paraphrase ("how long before the cache goes cold" against
+ * `COLD_IDLE_MS = 65 * 60_000`). When Ollama runs on this machine with an
+ * embedding model, `pack --semantic` embeds the question and the chunks and
+ * merges the two rankings by reciprocal rank fusion, which needs no
+ * calibration between BM25 scores and cosine similarities. Never required:
+ * without Ollama or an embedding model, pack stays lexical and says why.
+ *
+ * Local only (OLLAMA_HOST, default 127.0.0.1:11434), no key. Chunk vectors are
+ * cached on disk by model and text hash, so packing the same folder again
+ * embeds only what changed.
+ */
+import type { Chunk } from "./pack.js";
+export interface SemanticResult {
+    used: boolean;
+    model?: string;
+    /** Why semantic ranking was not used, or what it did. */
+    note: string;
+}
+/** The embedding model to use: the one asked for, else the first installed model that looks like one. */
+export declare function findEmbeddingModel(preferred?: string): Promise<string | undefined>;
+/**
+ * Re-score `chunks` in place: score = RRF(BM25 rank) + RRF(embedding rank),
+ * k = 60. A chunk BM25 never matched can still rank by meaning alone.
+ */
+export declare function semanticRerank(chunks: Chunk[], query: string, opts?: {
+    model?: string;
+}): Promise<SemanticResult>;

package/dist/embed.js

@@ -0,0 +1,127 @@
+/**
+ * Optional semantic ranking for `pack`, through a local Ollama.
+ *
+ * BM25 misses pure paraphrase ("how long before the cache goes cold" against
+ * `COLD_IDLE_MS = 65 * 60_000`). When Ollama runs on this machine with an
+ * embedding model, `pack --semantic` embeds the question and the chunks and
+ * merges the two rankings by reciprocal rank fusion, which needs no
+ * calibration between BM25 scores and cosine similarities. Never required:
+ * without Ollama or an embedding model, pack stays lexical and says why.
+ *
+ * Local only (OLLAMA_HOST, default 127.0.0.1:11434), no key. Chunk vectors are
+ * cached on disk by model and text hash, so packing the same folder again
+ * embeds only what changed.
+ */
+import { createHash } from "node:crypto";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+const EMBED_NAME = /embed|bge|e5-|gte|minilm|nomic|mxbai|arctic|snowflake/i;
+const BATCH = 64;
+const CACHE_CAP = 50_000;
+function host() {
+    const h = process.env.OLLAMA_HOST ?? "http://127.0.0.1:11434";
+    return h.startsWith("http") ? h.replace(/\/$/, "") : `http://${h}`;
+}
+async function getJson(path, body, timeoutMs = 60_000) {
+    const r = await fetch(host() + path, {
+        method: body ? "POST" : "GET",
+        headers: body ? { "content-type": "application/json" } : undefined,
+        body: body ? JSON.stringify(body) : undefined,
+        signal: AbortSignal.timeout(timeoutMs),
+    });
+    if (!r.ok)
+        throw new Error(`${path}: HTTP ${r.status}`);
+    return r.json();
+}
+/** The embedding model to use: the one asked for, else the first installed model that looks like one. */
+export async function findEmbeddingModel(preferred) {
+    const tags = await getJson("/api/tags", undefined, 2_000);
+    const names = (tags.models ?? []).map((m) => String(m.name));
+    if (preferred)
+        return names.find((n) => n === preferred || n.startsWith(preferred + ":")) ?? preferred;
+    return names.find((n) => EMBED_NAME.test(n));
+}
+function cachePath() {
+    return join(process.env.XDG_CACHE_HOME ?? join(homedir(), ".cache"), "context-doctor", "embeddings.json");
+}
+function loadCache() {
+    try {
+        return existsSync(cachePath()) ? JSON.parse(readFileSync(cachePath(), "utf8")) : {};
+    }
+    catch {
+        return {};
+    }
+}
+function saveCache(cache) {
+    try {
+        const keys = Object.keys(cache);
+        // Oldest entries go first once the cache passes its cap (insertion order).
+        if (keys.length > CACHE_CAP)
+            for (const k of keys.slice(0, keys.length - CACHE_CAP))
+                delete cache[k];
+        mkdirSync(join(cachePath(), ".."), { recursive: true });
+        writeFileSync(cachePath(), JSON.stringify(cache));
+    }
+    catch {
+        /* a cache is an optimization */
+    }
+}
+async function embed(model, texts) {
+    const out = [];
+    for (let i = 0; i < texts.length; i += BATCH) {
+        const r = await getJson("/api/embed", { model, input: texts.slice(i, i + BATCH), truncate: true });
+        out.push(...r.embeddings);
+    }
+    return out;
+}
+function cosine(a, b) {
+    let dot = 0, na = 0, nb = 0;
+    for (let i = 0; i < a.length; i++) {
+        dot += a[i] * b[i];
+        na += a[i] * a[i];
+        nb += b[i] * b[i];
+    }
+    return na && nb ? dot / Math.sqrt(na * nb) : 0;
+}
+/**
+ * Re-score `chunks` in place: score = RRF(BM25 rank) + RRF(embedding rank),
+ * k = 60. A chunk BM25 never matched can still rank by meaning alone.
+ */
+export async function semanticRerank(chunks, query, opts = {}) {
+    let model;
+    try {
+        model = await findEmbeddingModel(opts.model);
+    }
+    catch {
+        return { used: false, note: `semantic ranking off: no Ollama at ${host()} (keyword ranking only)` };
+    }
+    if (!model)
+        return { used: false, note: "semantic ranking off: Ollama has no embedding model; run `ollama pull nomic-embed-text` (keyword ranking only)" };
+    const cache = loadCache();
+    const key = (t) => createHash("sha1").update(model + "\u0000" + t).digest("hex");
+    const texts = chunks.map((c) => (c.heading ? c.heading + "\n" : "") + c.text);
+    const missing = [...new Set(texts.filter((t) => !cache[key(t)]))];
+    let vectors;
+    let queryVec;
+    try {
+        const fresh = await embed(model, [query, ...missing]);
+        queryVec = fresh[0];
+        missing.forEach((t, i) => (cache[key(t)] = fresh[i + 1]));
+        vectors = texts.map((t) => cache[key(t)]);
+    }
+    catch (e) {
+        return { used: false, model, note: `semantic ranking off: ${model} failed (${e.message}); keyword ranking only` };
+    }
+    if (missing.length)
+        saveCache(cache);
+    const K = 60;
+    const lexRank = new Map([...chunks].filter((c) => c.score > 0).sort((a, b) => b.score - a.score).map((c, i) => [c, i]));
+    const sims = chunks.map((c, i) => ({ c, s: cosine(queryVec, vectors[i]) })).sort((a, b) => b.s - a.s);
+    const semRank = new Map(sims.map((x, i) => [x.c, i]));
+    for (const c of chunks) {
+        const l = lexRank.get(c);
+        c.score = (l === undefined ? 0 : 1 / (K + l)) + 1 / (K + semRank.get(c));
+    }
+    return { used: true, model, note: `semantic ranking: ${model}, ${missing.length} of ${texts.length} chunks embedded (rest cached)` };
+}

package/dist/extract.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Text out of PDFs and office documents, with tools the OS already has.
+ *
+ * Most big documents people paste into a chat are PDFs and Word files. pack
+ * stays dependency-free, so this shells out to what is usually installed:
+ * `pdftotext` (poppler) anywhere, PDFKit through `osascript` on macOS,
+ * `textutil` for Word/RTF/ODT on macOS, and `unzip` for the zipped XML
+ * formats (docx, pptx, odt) elsewhere. When none is available the caller
+ * gets a reason that names what to install, never a silent skip.
+ */
+export declare const EXTRACTABLE: Set<string>;
+/** The document's text, or the reason it could not be read. */
+export declare function extractText(path: string): {
+    text: string;
+} | {
+    error: string;
+};

package/dist/extract.js

@@ -0,0 +1,88 @@
+/**
+ * Text out of PDFs and office documents, with tools the OS already has.
+ *
+ * Most big documents people paste into a chat are PDFs and Word files. pack
+ * stays dependency-free, so this shells out to what is usually installed:
+ * `pdftotext` (poppler) anywhere, PDFKit through `osascript` on macOS,
+ * `textutil` for Word/RTF/ODT on macOS, and `unzip` for the zipped XML
+ * formats (docx, pptx, odt) elsewhere. When none is available the caller
+ * gets a reason that names what to install, never a silent skip.
+ */
+import { spawnSync } from "node:child_process";
+import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, extname, join } from "node:path";
+export const EXTRACTABLE = new Set([".pdf", ".docx", ".doc", ".rtf", ".odt", ".pptx"]);
+const RUN = { encoding: "utf8", timeout: 30_000, maxBuffer: 64 * 1024 * 1024 };
+function run(cmd, args) {
+    try {
+        const r = spawnSync(cmd, args, RUN);
+        return r.status === 0 && typeof r.stdout === "string" && r.stdout.trim() ? r.stdout : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+const PDFKIT_JXA = 'ObjC.import("PDFKit"); function run(a) { const d = $.PDFDocument.alloc.initWithURL($.NSURL.fileURLWithPath(a[0])); return d.isNil() ? "" : d.string.js }';
+/**
+ * pdftotext comes in two families (poppler, and Xpdf as shipped with some
+ * Windows toolchains) that disagree on `-enc` names and on writing to stdout,
+ * so try the common form, then without the encoding flag, then via a file.
+ */
+function pdftotext(path) {
+    const direct = run("pdftotext", ["-enc", "UTF-8", path, "-"]) ?? run("pdftotext", [path, "-"]);
+    if (direct)
+        return direct;
+    const out = join(mkdtempSync(join(tmpdir(), "cd-pdf-")), "out.txt");
+    try {
+        spawnSync("pdftotext", [path, out], RUN);
+        const text = existsSync(out) ? readFileSync(out, "utf8") : "";
+        return text.trim() ? text : undefined;
+    }
+    catch {
+        return undefined;
+    }
+    finally {
+        rmSync(dirname(out), { recursive: true, force: true });
+    }
+}
+/** Paragraph-ish text from zipped XML: one paragraph per closing p/slide-text tag, entities decoded. */
+function fromZippedXml(path, members) {
+    const xml = run("unzip", ["-p", path, ...members]);
+    if (!xml)
+        return undefined;
+    return xml
+        .replace(/<\/(w:p|a:p|text:p|text:h)>/g, "\n\n")
+        .replace(/<(w:tab|w:br)\/>/g, " ")
+        .replace(/<[^>]+>/g, "")
+        .replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&apos;/g, "'").replace(/&amp;/g, "&")
+        .replace(/\n{3,}/g, "\n\n")
+        .trim();
+}
+/** The document's text, or the reason it could not be read. */
+export function extractText(path) {
+    const ext = extname(path).toLowerCase();
+    const mac = process.platform === "darwin";
+    let text;
+    if (ext === ".pdf") {
+        // Form feeds separate pages; as blank lines they become paragraph boundaries.
+        text = pdftotext(path)?.replace(/\f/g, "\n\n");
+        if (!text && mac)
+            text = run("osascript", ["-l", "JavaScript", "-e", PDFKIT_JXA, path]);
+        if (!text) {
+            return { error: mac ? "no text layer (scanned PDF?) or unreadable" : "needs pdftotext (install poppler-utils), or the PDF has no text layer" };
+        }
+    }
+    else {
+        if (mac && ext !== ".pptx")
+            text = run("textutil", ["-convert", "txt", "-stdout", path]);
+        if (!text) {
+            const members = ext === ".docx" ? ["word/document.xml"] : ext === ".pptx" ? ["ppt/slides/*.xml"] : ext === ".odt" ? ["content.xml"] : [];
+            if (members.length)
+                text = fromZippedXml(path, members);
+        }
+        if (!text)
+            return { error: mac ? "unreadable document" : `needs unzip${ext === ".doc" || ext === ".rtf" ? " (and .doc/.rtf need macOS textutil; save as .docx)" : ""}` };
+    }
+    return { text };
+}

package/dist/gemini.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Gemini CLI chat recordings, read into the same shape as Claude Code and
+ * Codex transcripts so `session`, `watch` and the every-prompt hook work on
+ * them.
+ *
+ * Gemini CLI writes ~/.gemini/tmp/<project>/chats/session-*.jsonl: a metadata
+ * record, then one record per message (`user`, `gemini`, `info`, `error`).
+ * A message written again with the same id replaces the earlier copy (tool
+ * results and token counts arrive after the text), `{"$rewindTo": id}` drops
+ * that message and everything after it, and `{"$set": …}` updates metadata.
+ * Older versions wrote the whole conversation as one JSON document with a
+ * `messages` array; both are read.
+ *
+ * Each `gemini` message records the API's own usage (`tokens.input` is the
+ * request's promptTokenCount, cached part included), so the live context size
+ * is exact, as it is for Claude Code and Codex.
+ */
+import { type ParsedSession, type SessionInfo } from "./session.js";
+/** True when the file's first record looks like a Gemini CLI chat (cheap: reads 4 KB). */
+export declare function isGeminiChat(path: string): boolean;
+export declare function parseGeminiChat(path: string): ParsedSession;
+/** Gemini CLI chats on this machine: ~/.gemini/tmp/<project>/chats/session-*.json[l]. */
+export declare function listGeminiChats(home?: string): SessionInfo[];