opencode-rag-plugin 1.10.6 → 1.12.1

package/dist/eval/token-analysis.js

@@ -0,0 +1,286 @@
+/**
+ * Token usage analysis for comparing RAG-on vs RAG-off sessions.
+ *
+ * Provides functions to:
+ * - Analyze per-query token breakdown from session events
+ * - Estimate token savings from RAG (fewer read tool calls)
+ * - Generate comparison reports between RAG-on and RAG-off sessions
+ */
+import { readSessionEvents, computeSummary } from "./storage.js";
+import { countTokens } from "./token-counter.js";
+/** Average tokens consumed by a `read` tool call (typical file read). */
+const AVG_READ_TOOL_TOKENS = 1200;
+/** Average tokens consumed by a `search_semantic` tool call. */
+const AVG_SEARCH_TOOL_TOKENS = 800;
+/** Average tokens for system prompt guidance injected by RAG (~150 tokens). */
+const SYSTEM_GUIDANCE_TOKENS = 150;
+/**
+ * Analyze token usage for a single session from its JSONL event log.
+ *
+ * Reads all events, groups them by message, and computes per-query breakdowns
+ * including RAG context injection, tool calls, and response times.
+ */
+export function analyzeTokenUsage(storePath, sessionID) {
+    const events = readSessionEvents(storePath, sessionID);
+    const summary = computeSummary(events);
+    // Group events by messageID
+    const messageEvents = new Map();
+    const ragContextByMessage = new Map();
+    const toolCallsByMessage = new Map();
+    for (const ev of events) {
+        const mid = ev.messageID ?? "unknown";
+        if (ev.event === "message" && ev.role === "assistant") {
+            if (!messageEvents.has(mid))
+                messageEvents.set(mid, []);
+            messageEvents.get(mid).push(ev);
+        }
+        if (ev.event === "rag.context") {
+            ragContextByMessage.set(mid, ev);
+        }
+        if (ev.event === "tool") {
+            if (!toolCallsByMessage.has(mid))
+                toolCallsByMessage.set(mid, []);
+            toolCallsByMessage.get(mid).push(ev);
+        }
+    }
+    const breakdowns = [];
+    let totalReadToolCalls = 0;
+    let totalRagToolCalls = 0;
+    let totalRagContextTokens = 0;
+    let totalCost = 0;
+    let systemGuidanceTokens = 0;
+    // System guidance is injected when chunks are indexed (ragContextCount > 0)
+    if (summary.ragContextCount > 0) {
+        systemGuidanceTokens = SYSTEM_GUIDANCE_TOKENS * summary.messageCount;
+    }
+    for (const [mid, msgEvents] of messageEvents) {
+        const msgEvent = msgEvents[0]; // Should be exactly one per messageID
+        const ragCtx = ragContextByMessage.get(mid);
+        const tools = toolCallsByMessage.get(mid) ?? [];
+        const readCalls = tools.filter((t) => t.tool === "read").length;
+        const ragCalls = tools.filter((t) => ["search_semantic", "get_file_skeleton", "find_usages", "search_semantic"].includes(t.tool ?? "")).length;
+        const timeDiff = (msgEvent.timeCreated && msgEvent.timeCompleted)
+            ? msgEvent.timeCompleted - msgEvent.timeCreated
+            : 0;
+        totalReadToolCalls += readCalls;
+        totalRagToolCalls += ragCalls;
+        totalRagContextTokens += ragCtx?.ragContextTokens ?? 0;
+        totalCost += msgEvent.cost ?? 0;
+        breakdowns.push({
+            messageID: mid,
+            inputTokens: msgEvent.tokens?.input ?? 0,
+            outputTokens: msgEvent.tokens?.output ?? 0,
+            reasoningTokens: msgEvent.tokens?.reasoning ?? 0,
+            cacheRead: msgEvent.tokens?.cache?.read ?? 0,
+            cacheWrite: msgEvent.tokens?.cache?.write ?? 0,
+            cost: msgEvent.cost ?? 0,
+            ragContextTokens: ragCtx?.ragContextTokens ?? 0,
+            ragChunkCount: ragCtx?.ragChunkCount ?? 0,
+            ragTopScore: ragCtx?.ragTopScore ?? 0,
+            readToolCalls: readCalls,
+            ragToolCalls: ragCalls,
+            responseTimeMs: timeDiff,
+        });
+    }
+    // Aggregate execution time and models
+    const responseTimes = breakdowns.filter((b) => b.responseTimeMs > 0).map((b) => b.responseTimeMs);
+    const avgResponseTimeMs = responseTimes.length > 0
+        ? Math.round(responseTimes.reduce((a, b) => a + b, 0) / responseTimes.length)
+        : 0;
+    const totalToolTimeMs = breakdowns.reduce((sum, b) => sum + b.ragToolCalls * 40 + b.readToolCalls * 25, 0); // estimated
+    const modelSet = new Set();
+    for (const ev of events) {
+        if (ev.event === "message" && ev.modelID)
+            modelSet.add(ev.modelID);
+    }
+    const totals = {
+        inputTokens: summary.totalTokens.input,
+        outputTokens: summary.totalTokens.output,
+        reasoningTokens: summary.totalTokens.reasoning,
+        cacheRead: summary.totalTokens.cacheRead,
+        cacheWrite: summary.totalTokens.cacheWrite,
+        cost: totalCost,
+        ragContextTokens: totalRagContextTokens,
+        readToolCalls: totalReadToolCalls,
+        ragToolCalls: totalRagToolCalls,
+        systemGuidanceTokens,
+        avgResponseTimeMs,
+        totalToolTimeMs,
+        models: [...modelSet],
+    };
+    // Estimate what tokens would look like without RAG:
+    // - No RAG context tokens injected
+    // - No system guidance tokens
+    // - More read tool calls (estimated 2-3 extra per query)
+    // - More search_semantic calls would become read calls
+    const avgReadCallsPerQuery = summary.messageCount > 0 ? totalReadToolCalls / summary.messageCount : 0;
+    const estimatedExtraReadsPerQuery = Math.max(0, 2.5 - avgReadCallsPerQuery); // RAG typically reduces reads by ~2.5
+    const estimatedExtraReadTokens = estimatedExtraReadsPerQuery * AVG_READ_TOOL_TOKENS * summary.messageCount;
+    const estimatedExtraSearchTokens = totalRagToolCalls * AVG_SEARCH_TOOL_TOKENS;
+    const tokensWithoutRAG = totals.inputTokens
+        - totals.ragContextTokens
+        - totals.systemGuidanceTokens
+        + estimatedExtraReadTokens
+        + estimatedExtraSearchTokens;
+    const tokensWithRAG = totals.inputTokens;
+    const netSavings = tokensWithoutRAG - tokensWithRAG;
+    const percentSavings = tokensWithoutRAG > 0 ? (netSavings / tokensWithoutRAG) * 100 : 0;
+    return {
+        sessionID,
+        queryCount: summary.messageCount,
+        breakdowns,
+        totals,
+        estimates: {
+            tokensWithoutRAG: Math.round(tokensWithoutRAG),
+            tokensWithRAG,
+            netSavings: Math.round(netSavings),
+            percentSavings: Math.round(percentSavings * 10) / 10,
+        },
+    };
+}
+/**
+ * Compare two token analyses (RAG-on vs RAG-off) and produce a delta report.
+ */
+export function compareTokenAnalyses(ragOn, ragOff) {
+    const dInput = ragOn.totals.inputTokens - ragOff.totals.inputTokens;
+    const dOutput = ragOn.totals.outputTokens - ragOff.totals.outputTokens;
+    const dReasoning = ragOn.totals.reasoningTokens - ragOff.totals.reasoningTokens;
+    const dCache = ragOn.totals.cacheRead - ragOff.totals.cacheRead;
+    const dCost = ragOn.totals.cost - ragOff.totals.cost;
+    const dRagCtx = ragOn.totals.ragContextTokens - ragOff.totals.ragContextTokens;
+    const dReads = ragOn.totals.readToolCalls - ragOff.totals.readToolCalls;
+    const dRagTools = ragOn.totals.ragToolCalls - ragOff.totals.ragToolCalls;
+    const dSysGuidance = ragOn.totals.systemGuidanceTokens - ragOff.totals.systemGuidanceTokens;
+    const avgOnTime = ragOn.breakdowns.length > 0
+        ? ragOn.breakdowns.reduce((s, b) => s + b.responseTimeMs, 0) / ragOn.breakdowns.length
+        : 0;
+    const avgOffTime = ragOff.breakdowns.length > 0
+        ? ragOff.breakdowns.reduce((s, b) => s + b.responseTimeMs, 0) / ragOff.breakdowns.length
+        : 0;
+    const dResponseTime = avgOnTime - avgOffTime;
+    const pct = (delta, base) => base === 0 ? 0 : Math.round((delta / base) * 1000) / 10;
+    const totalOn = ragOn.totals.inputTokens + ragOn.totals.outputTokens;
+    const totalOff = ragOff.totals.inputTokens + ragOff.totals.outputTokens;
+    // Verdict
+    let verdict;
+    if (ragOn.totals.inputTokens < ragOff.totals.inputTokens) {
+        verdict = `RAG SAVES tokens: ${Math.abs(dInput)} fewer input tokens (${Math.abs(pct(dInput, ragOff.totals.inputTokens))}% reduction)`;
+    }
+    else if (ragOn.totals.inputTokens > ragOff.totals.inputTokens) {
+        verdict = `RAG COSTS tokens: ${dInput} more input tokens (${pct(dInput, ragOff.totals.inputTokens)}% increase)`;
+    }
+    else {
+        verdict = "RAG has no effect on input token usage";
+    }
+    if (dReads < 0) {
+        verdict += ` | ${Math.abs(dReads)} fewer read calls`;
+    }
+    if (dRagTools > 0) {
+        verdict += ` | ${dRagTools} additional RAG tool calls`;
+    }
+    return {
+        delta: {
+            inputTokens: dInput,
+            outputTokens: dOutput,
+            reasoningTokens: dReasoning,
+            cacheRead: dCache,
+            cost: dCost,
+            ragContextTokens: dRagCtx,
+            readToolCalls: dReads,
+            ragToolCalls: dRagTools,
+            systemGuidanceTokens: dSysGuidance,
+            responseTimeMs: Math.round(dResponseTime),
+        },
+        percentChange: {
+            inputTokens: pct(dInput, ragOff.totals.inputTokens),
+            outputTokens: pct(dOutput, ragOff.totals.outputTokens),
+            cost: pct(dCost, ragOff.totals.cost),
+            readToolCalls: pct(dReads, ragOff.totals.readToolCalls),
+            responseTimeMs: pct(dResponseTime, avgOffTime),
+        },
+        verdict,
+    };
+}
+/**
+ * Format a token comparison report as a human-readable string.
+ */
+export function formatTokenReport(ragOn, ragOff, comparison) {
+    const lines = [];
+    const sep = "─".repeat(70);
+    lines.push("");
+    lines.push(sep);
+    lines.push("  TOKEN USAGE COMPARISON: RAG ON vs RAG OFF");
+    lines.push(sep);
+    lines.push("");
+    // Summary table
+    lines.push("  Metric                          RAG ON        RAG OFF       Delta");
+    lines.push(`  ${"─".repeat(66)}`);
+    const row = (label, on, off, delta) => {
+        const onStr = typeof on === "number" ? on.toLocaleString() : on;
+        const offStr = typeof off === "number" ? off.toLocaleString() : off;
+        const deltaStr = typeof delta === "number"
+            ? (delta > 0 ? `+${delta.toLocaleString()}` : delta.toLocaleString())
+            : delta;
+        lines.push(`  ${label.padEnd(34)}${onStr.padStart(12)}${offStr.padStart(14)}${deltaStr.padStart(14)}`);
+    };
+    row("Input tokens", ragOn.totals.inputTokens, ragOff.totals.inputTokens, comparison.delta.inputTokens);
+    row("Output tokens", ragOn.totals.outputTokens, ragOff.totals.outputTokens, comparison.delta.outputTokens);
+    row("Reasoning tokens", ragOn.totals.reasoningTokens, ragOff.totals.reasoningTokens, comparison.delta.reasoningTokens);
+    row("Cache read", ragOn.totals.cacheRead, ragOff.totals.cacheRead, comparison.delta.cacheRead);
+    row("Cost ($)", `$${ragOn.totals.cost.toFixed(4)}`, `$${ragOff.totals.cost.toFixed(4)}`, `$${comparison.delta.cost.toFixed(4)}`);
+    lines.push(`  ${"─".repeat(66)}`);
+    row("RAG context tokens", ragOn.totals.ragContextTokens, ragOff.totals.ragContextTokens, comparison.delta.ragContextTokens);
+    row("System guidance tokens", ragOn.totals.systemGuidanceTokens, ragOff.totals.systemGuidanceTokens, comparison.delta.systemGuidanceTokens);
+    row("Read tool calls", ragOn.totals.readToolCalls, ragOff.totals.readToolCalls, comparison.delta.readToolCalls);
+    row("RAG tool calls", ragOn.totals.ragToolCalls, ragOff.totals.ragToolCalls, comparison.delta.ragToolCalls);
+    row("Avg response time (ms)", ragOn.totals.avgResponseTimeMs, ragOff.totals.avgResponseTimeMs, comparison.delta.responseTimeMs);
+    row("Queries processed", ragOn.queryCount, ragOff.queryCount, ragOn.queryCount - ragOff.queryCount);
+    lines.push(`  ${"─".repeat(66)}`);
+    lines.push(`  Models:          ${ragOn.totals.models.join(", ") || "unknown"}`);
+    lines.push(`                   ${ragOff.totals.models.join(", ") || "unknown"}`);
+    lines.push("");
+    lines.push(`  Verdict: ${comparison.verdict}`);
+    lines.push("");
+    // Per-query breakdown
+    lines.push("  PER-QUERY BREAKDOWN (RAG ON)");
+    lines.push(`  ${"─".repeat(66)}`);
+    lines.push("  #   Input    Output   RAG ctx   Reads  RAG tools  Score");
+    lines.push(`  ${"─".repeat(66)}`);
+    for (let i = 0; i < ragOn.breakdowns.length; i++) {
+        const b = ragOn.breakdowns[i];
+        lines.push(`  ${String(i + 1).padStart(2)}  ${String(b.inputTokens).padStart(7)}  ${String(b.outputTokens).padStart(7)}  ${String(b.ragContextTokens).padStart(7)}  ${String(b.readToolCalls).padStart(5)}  ${String(b.ragToolCalls).padStart(9)}  ${b.ragTopScore.toFixed(2)}`);
+    }
+    lines.push("");
+    lines.push(sep);
+    lines.push("");
+    return lines.join("\n");
+}
+/**
+ * Estimate the token count for given text content.
+ * Uses tiktoken BPE (cl100k_base) when available, falls back to ceil(len/4).
+ */
+export function estimateContextTokens(text) {
+    return countTokens(text);
+}
+/**
+ * Project whether RAG would save tokens for a given query profile.
+ *
+ * @param avgChunkSize - Average size of injected code chunks in characters
+ * @param avgChunksPerQuery - Average chunks injected per query
+ * @param avgReadsPerQueryWithoutRAG - Average read calls without RAG
+ * @param avgReadsPerQueryWithRAG - Average read calls with RAG
+ * @param queryCount - Number of queries in the session
+ */
+export function projectTokenSavings(params) {
+    const ragOverheadTokens = Math.ceil(params.avgChunkSize / 4) * params.avgChunksPerQuery * params.queryCount;
+    const savedReads = Math.max(0, params.avgReadsPerQueryWithoutRAG - params.avgReadsPerQueryWithRAG);
+    const savedReadTokens = savedReads * AVG_READ_TOOL_TOKENS * params.queryCount;
+    const netSavings = savedReadTokens - ragOverheadTokens;
+    return {
+        ragOverheadTokens,
+        savedReadTokens,
+        netSavings,
+        isPositive: netSavings > 0,
+    };
+}
+//# sourceMappingURL=token-analysis.js.map

package/dist/eval/token-analysis.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"token-analysis.js","sourceRoot":"","sources":["../../src/eval/token-analysis.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAEjE,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEjD,yEAAyE;AACzE,MAAM,oBAAoB,GAAG,IAAI,CAAC;AAElC,gEAAgE;AAChE,MAAM,sBAAsB,GAAG,GAAG,CAAC;AAEnC,+EAA+E;AAC/E,MAAM,sBAAsB,GAAG,GAAG,CAAC;AA6CnC;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB,CAAC,SAAiB,EAAE,SAAiB;IACpE,MAAM,MAAM,GAAG,iBAAiB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;IACvD,MAAM,OAAO,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IAEvC,4BAA4B;IAC5B,MAAM,aAAa,GAAG,IAAI,GAAG,EAA0B,CAAC;IACxD,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAAwB,CAAC;IAC5D,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAA0B,CAAC;IAE7D,KAAK,MAAM,EAAE,IAAI,MAAM,EAAE,CAAC;QACxB,MAAM,GAAG,GAAG,EAAE,CAAC,SAAS,IAAI,SAAS,CAAC;QAEtC,IAAI,EAAE,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YACtD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,aAAa,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACxD,aAAa,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACnC,CAAC;QAED,IAAI,EAAE,CAAC,KAAK,KAAK,aAAa,EAAE,CAAC;YAC/B,mBAAmB,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACnC,CAAC;QAED,IAAI,EAAE,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;YACxB,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,kBAAkB,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YAClE,kBAAkB,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAwB,EAAE,CAAC;IAC3C,IAAI,kBAAkB,GAAG,CAAC,CAAC;IAC3B,IAAI,iBAAiB,GAAG,CAAC,CAAC;IAC1B,IAAI,qBAAqB,GAAG,CAAC,CAAC;IAC9B,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,oBAAoB,GAAG,CAAC,CAAC;IAE7B,4EAA4E;IAC5E,IAAI,OAAO,CAAC,eAAe,GAAG,CAAC,EAAE,CAAC;QAChC,oBAAoB,GAAG,sBAAsB,GAAG,OAAO,CAAC,YAAY,CAAC;IACvE,CAAC;IAED,KAAK,MAAM,CAAC,GAAG,EAAE,SAAS,CAAC,IAAI,aAAa,EAAE,CAAC;QAC7C,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAE,CAAC,CAAC,sCAAsC;QACtE,MAAM,MAAM,GAAG,mBAAmB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5C,MAAM,KAAK,GAAG,kBAAkB,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QAEhD,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;QAChE,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAClC,CAAC,iBAAiB,EAAE,mBAAmB,EAAE,aAAa,EAAE,iBAAiB,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAClG,CAAC,MAAM,CAAC;QAET,MAAM,QAAQ,GAAG,CAAC,QAAQ,CAAC,WAAW,IAAI,QAAQ,CAAC,aAAa,CAAC;YAC/D,CAAC,CAAC,QAAQ,CAAC,aAAa,GAAG,QAAQ,CAAC,WAAW;YAC/C,CAAC,CAAC,CAAC,CAAC;QAEN,kBAAkB,IAAI,SAAS,CAAC;QAChC,iBAAiB,IAAI,QAAQ,CAAC;QAC9B,qBAAqB,IAAI,MAAM,EAAE,gBAAgB,IAAI,CAAC,CAAC;QACvD,SAAS,IAAI,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC;QAEhC,UAAU,CAAC,IAAI,CAAC;YACd,SAAS,EAAE,GAAG;YACd,WAAW,EAAE,QAAQ,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC;YACxC,YAAY,EAAE,QAAQ,CAAC,MAAM,EAAE,MAAM,IAAI,CAAC;YAC1C,eAAe,EAAE,QAAQ,CAAC,MAAM,EAAE,SAAS,IAAI,CAAC;YAChD,SAAS,EAAE,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,IAAI,IAAI,CAAC;YAC5C,UAAU,EAAE,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,IAAI,CAAC;YAC9C,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,CAAC;YACxB,gBAAgB,EAAE,MAAM,EAAE,gBAAgB,IAAI,CAAC;YAC/C,aAAa,EAAE,MAAM,EAAE,aAAa,IAAI,CAAC;YACzC,WAAW,EAAE,MAAM,EAAE,WAAW,IAAI,CAAC;YACrC,aAAa,EAAE,SAAS;YACxB,YAAY,EAAE,QAAQ;YACtB,cAAc,EAAE,QAAQ;SACzB,CAAC,CAAC;IACL,CAAC;IAED,sCAAsC;IACtC,MAAM,aAAa,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;IAClG,MAAM,iBAAiB,GAAG,aAAa,CAAC,MAAM,GAAG,CAAC;QAChD,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,aAAa,CAAC,MAAM,CAAC;QAC7E,CAAC,CAAC,CAAC,CAAC;IACN,MAAM,eAAe,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,YAAY,GAAG,EAAE,GAAG,CAAC,CAAC,aAAa,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,YAAY;IACxH,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,KAAK,MAAM,EAAE,IAAI,MAAM,EAAE,CAAC;QACxB,IAAI,EAAE,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,CAAC,OAAO;YAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;IACrE,CAAC;IAED,MAAM,MAAM,GAAG;QACb,WAAW,EAAE,OAAO,CAAC,WAAW,CAAC,KAAK;QACtC,YAAY,EAAE,OAAO,CAAC,WAAW,CAAC,MAAM;QACxC,eAAe,EAAE,OAAO,CAAC,WAAW,CAAC,SAAS;QAC9C,SAAS,EAAE,OAAO,CAAC,WAAW,CAAC,SAAS;QACxC,UAAU,EAAE,OAAO,CAAC,WAAW,CAAC,UAAU;QAC1C,IAAI,EAAE,SAAS;QACf,gBAAgB,EAAE,qBAAqB;QACvC,aAAa,EAAE,kBAAkB;QACjC,YAAY,EAAE,iBAAiB;QAC/B,oBAAoB;QACpB,iBAAiB;QACjB,eAAe;QACf,MAAM,EAAE,CAAC,GAAG,QAAQ,CAAC;KACtB,CAAC;IAEF,oDAAoD;IACpD,mCAAmC;IACnC,8BAA8B;IAC9B,yDAAyD;IACzD,uDAAuD;IACvD,MAAM,oBAAoB,GAAG,OAAO,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,kBAAkB,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;IACtG,MAAM,2BAA2B,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,oBAAoB,CAAC,CAAC,CAAC,sCAAsC;IACnH,MAAM,wBAAwB,GAAG,2BAA2B,GAAG,oBAAoB,GAAG,OAAO,CAAC,YAAY,CAAC;IAC3G,MAAM,0BAA0B,GAAG,iBAAiB,GAAG,sBAAsB,CAAC;IAE9E,MAAM,gBAAgB,GAAG,MAAM,CAAC,WAAW;UACvC,MAAM,CAAC,gBAAgB;UACvB,MAAM,CAAC,oBAAoB;UAC3B,wBAAwB;UACxB,0BAA0B,CAAC;IAE/B,MAAM,aAAa,GAAG,MAAM,CAAC,WAAW,CAAC;IAEzC,MAAM,UAAU,GAAG,gBAAgB,GAAG,aAAa,CAAC;IACpD,MAAM,cAAc,GAAG,gBAAgB,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,gBAAgB,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAExF,OAAO;QACL,SAAS;QACT,UAAU,EAAE,OAAO,CAAC,YAAY;QAChC,UAAU;QACV,MAAM;QACN,SAAS,EAAE;YACT,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC;YAC9C,aAAa;YACb,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC;YAClC,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,EAAE,CAAC,GAAG,EAAE;SACrD;KACF,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAClC,KAAoB,EACpB,MAAqB;IAuBrB,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC;IACpE,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC;IACvE,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC;IAChF,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC;IAChE,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;IACrD,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,gBAAgB,GAAG,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC;IAC/E,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC;IACxE,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC;IACzE,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,oBAAoB,GAAG,MAAM,CAAC,MAAM,CAAC,oBAAoB,CAAC;IAE5F,MAAM,SAAS,GAAG,KAAK,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC;QAC3C,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,UAAU,CAAC,MAAM;QACtF,CAAC,CAAC,CAAC,CAAC;IACN,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC;QAC7C,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM;QACxF,CAAC,CAAC,CAAC,CAAC;IACN,MAAM,aAAa,GAAG,SAAS,GAAG,UAAU,CAAC;IAE7C,MAAM,GAAG,GAAG,CAAC,KAAa,EAAE,IAAY,EAAE,EAAE,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAErG,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,YAAY,CAAC;IACrE,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC;IAExE,UAAU;IACV,IAAI,OAAe,CAAC;IACpB,IAAI,KAAK,CAAC,MAAM,CAAC,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QACzD,OAAO,GAAG,qBAAqB,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,wBAAwB,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,cAAc,CAAC;IACxI,CAAC;SAAM,IAAI,KAAK,CAAC,MAAM,CAAC,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAChE,OAAO,GAAG,qBAAqB,MAAM,uBAAuB,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,aAAa,CAAC;IAClH,CAAC;SAAM,CAAC;QACN,OAAO,GAAG,wCAAwC,CAAC;IACrD,CAAC;IAED,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QACf,OAAO,IAAI,MAAM,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,mBAAmB,CAAC;IACvD,CAAC;IACD,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QAClB,OAAO,IAAI,MAAM,SAAS,4BAA4B,CAAC;IACzD,CAAC;IAED,OAAO;QACL,KAAK,EAAE;YACL,WAAW,EAAE,MAAM;YACnB,YAAY,EAAE,OAAO;YACrB,eAAe,EAAE,UAAU;YAC3B,SAAS,EAAE,MAAM;YACjB,IAAI,EAAE,KAAK;YACX,gBAAgB,EAAE,OAAO;YACzB,aAAa,EAAE,MAAM;YACrB,YAAY,EAAE,SAAS;YACvB,oBAAoB,EAAE,YAAY;YAClC,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC;SAC1C;QACD,aAAa,EAAE;YACb,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC;YACnD,YAAY,EAAE,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC;YACtD,IAAI,EAAE,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;YACpC,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC;YACvD,cAAc,EAAE,GAAG,CAAC,aAAa,EAAE,UAAU,CAAC;SAC/C;QACD,OAAO;KACR,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAC/B,KAAoB,EACpB,MAAqB,EACrB,UAAmD;IAEnD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAE3B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAChB,KAAK,CAAC,IAAI,CAAC,6CAA6C,CAAC,CAAC;IAC1D,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAChB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,gBAAgB;IAChB,KAAK,CAAC,IAAI,CAAC,qEAAqE,CAAC,CAAC;IAClF,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IAElC,MAAM,GAAG,GAAG,CAAC,KAAa,EAAE,EAAmB,EAAE,GAAoB,EAAE,KAAsB,EAAE,EAAE;QAC/F,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAChE,MAAM,MAAM,GAAG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;QACpE,MAAM,QAAQ,GAAG,OAAO,KAAK,KAAK,QAAQ;YACxC,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;YACrE,CAAC,CAAC,KAAK,CAAC;QACV,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IACzG,CAAC,CAAC;IAEF,GAAG,CAAC,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,UAAU,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IACvG,GAAG,CAAC,eAAe,EAAE,KAAK,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,MAAM,CAAC,YAAY,EAAE,UAAU,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;IAC3G,GAAG,CAAC,kBAAkB,EAAE,KAAK,CAAC,MAAM,CAAC,eAAe,EAAE,MAAM,CAAC,MAAM,CAAC,eAAe,EAAE,UAAU,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;IACvH,GAAG,CAAC,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,UAAU,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IAC/F,GAAG,CAAC,UAAU,EAAE,IAAI,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACjI,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IAClC,GAAG,CAAC,oBAAoB,EAAE,KAAK,CAAC,MAAM,CAAC,gBAAgB,EAAE,MAAM,CAAC,MAAM,CAAC,gBAAgB,EAAE,UAAU,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;IAC5H,GAAG,CAAC,wBAAwB,EAAE,KAAK,CAAC,MAAM,CAAC,oBAAoB,EAAE,MAAM,CAAC,MAAM,CAAC,oBAAoB,EAAE,UAAU,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;IAC5I,GAAG,CAAC,iBAAiB,EAAE,KAAK,CAAC,MAAM,CAAC,aAAa,EAAE,MAAM,CAAC,MAAM,CAAC,aAAa,EAAE,UAAU,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IAChH,GAAG,CAAC,gBAAgB,EAAE,KAAK,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,MAAM,CAAC,YAAY,EAAE,UAAU,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;IAC5G,GAAG,CAAC,wBAAwB,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,EAAE,MAAM,CAAC,MAAM,CAAC,iBAAiB,EAAE,UAAU,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;IAChI,GAAG,CAAC,mBAAmB,EAAE,KAAK,CAAC,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;IAEpG,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IAClC,KAAK,CAAC,IAAI,CAAC,sBAAsB,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,SAAS,EAAE,CAAC,CAAC;IAChF,KAAK,CAAC,IAAI,CAAC,sBAAsB,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,SAAS,EAAE,CAAC,CAAC;IAEjF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,cAAc,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC;IAC/C,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,sBAAsB;IACtB,KAAK,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;IAC7C,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IAClC,KAAK,CAAC,IAAI,CAAC,2DAA2D,CAAC,CAAC;IACxE,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IAElC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACjD,MAAM,CAAC,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,CAAE,CAAC;QAC/B,KAAK,CAAC,IAAI,CACR,KAAK,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CACrQ,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAChB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAAY;IAChD,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC;AAC3B,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,mBAAmB,CAAC,MAMnC;IAMC,MAAM,iBAAiB,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,iBAAiB,GAAG,MAAM,CAAC,UAAU,CAAC;IAC5G,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,0BAA0B,GAAG,MAAM,CAAC,uBAAuB,CAAC,CAAC;IACnG,MAAM,eAAe,GAAG,UAAU,GAAG,oBAAoB,GAAG,MAAM,CAAC,UAAU,CAAC;IAC9E,MAAM,UAAU,GAAG,eAAe,GAAG,iBAAiB,CAAC;IAEvD,OAAO;QACL,iBAAiB;QACjB,eAAe;QACf,UAAU;QACV,UAAU,EAAE,UAAU,GAAG,CAAC;KAC3B,CAAC;AACJ,CAAC"}

package/dist/eval/token-counter.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Token counting with tiktoken BPE + fallback heuristic.
+ *
+ * Uses js-tiktoken (cl100k_base encoding) for BPE-accurate counts.
+ * Falls back to ceil(text.length / 4) when tiktoken is unavailable.
+ *
+ * cl100k_base is the encoding used by GPT-4, GPT-4o, Claude, and most
+ * modern LLMs. It is the closest universal approximation available locally.
+ */
+import type { SearchResult } from "../core/interfaces.js";
+/**
+ * Count the number of tokens in a text string.
+ *
+ * Uses tiktoken BPE (cl100k_base) when available, falls back to
+ * the ceil(characters / 4) heuristic otherwise.
+ */
+export declare function countTokens(text: string): number;
+/**
+ * Count tokens for multiple texts individually.
+ * Returns an array of token counts, one per input text.
+ */
+export declare function countTokensBatch(texts: string[]): number[];
+/**
+ * Sum token counts for multiple texts.
+ */
+export declare function sumTokens(texts: string[]): number;
+/**
+ * Estimate the token overhead of auto-injected RAG context.
+ *
+ * Counts each chunk's content + description + formatting overhead individually,
+ * providing a more accurate estimate than counting the fully-assembled string.
+ *
+ * @param chunks - Search results to count
+ * @param worktree - Working tree for path relativity (unused, kept for interface compat)
+ * @param estimateFn - Fallback estimator (used if tiktoken unavailable)
+ */
+export declare function estimateContextTokensFormatted(chunks: SearchResult[], worktree: string): number;
+/**
+ * Returns the tokenizer method used ("tiktoken" or "heuristic").
+ */
+export declare function tokenizerMethod(): "tiktoken" | "heuristic";

package/dist/eval/token-counter.js

@@ -0,0 +1,102 @@
+/**
+ * Token counting with tiktoken BPE + fallback heuristic.
+ *
+ * Uses js-tiktoken (cl100k_base encoding) for BPE-accurate counts.
+ * Falls back to ceil(text.length / 4) when tiktoken is unavailable.
+ *
+ * cl100k_base is the encoding used by GPT-4, GPT-4o, Claude, and most
+ * modern LLMs. It is the closest universal approximation available locally.
+ */
+let cachedEncoder = null;
+let loadFailed = false;
+function getEncoder() {
+    if (loadFailed)
+        return null;
+    if (cachedEncoder)
+        return cachedEncoder;
+    try {
+        // Dynamic import — CJS module in an ESM project
+        const mod = require("js-tiktoken");
+        cachedEncoder = mod.getEncoding("cl100k_base");
+        return cachedEncoder;
+    }
+    catch {
+        loadFailed = true;
+        return null;
+    }
+}
+function fallbackEstimate(text) {
+    return Math.ceil(text.length / 4);
+}
+/**
+ * Count the number of tokens in a text string.
+ *
+ * Uses tiktoken BPE (cl100k_base) when available, falls back to
+ * the ceil(characters / 4) heuristic otherwise.
+ */
+export function countTokens(text) {
+    if (text.length === 0)
+        return 0;
+    const encoder = getEncoder();
+    if (encoder) {
+        try {
+            return encoder.encode(text, [], []).length;
+        }
+        catch {
+            // Encoding failed — fall through to heuristic
+        }
+    }
+    return fallbackEstimate(text);
+}
+/**
+ * Count tokens for multiple texts individually.
+ * Returns an array of token counts, one per input text.
+ */
+export function countTokensBatch(texts) {
+    return texts.map(countTokens);
+}
+/**
+ * Sum token counts for multiple texts.
+ */
+export function sumTokens(texts) {
+    let total = 0;
+    for (const text of texts) {
+        total += countTokens(text);
+    }
+    return total;
+}
+/**
+ * Estimate the token overhead of auto-injected RAG context.
+ *
+ * Counts each chunk's content + description + formatting overhead individually,
+ * providing a more accurate estimate than counting the fully-assembled string.
+ *
+ * @param chunks - Search results to count
+ * @param worktree - Working tree for path relativity (unused, kept for interface compat)
+ * @param estimateFn - Fallback estimator (used if tiktoken unavailable)
+ */
+export function estimateContextTokensFormatted(chunks, worktree) {
+    if (chunks.length === 0)
+        return 0;
+    let tokens = 0;
+    for (const r of chunks) {
+        // Content tokens
+        tokens += countTokens(r.chunk.content);
+        // Description tokens
+        if (r.chunk.description) {
+            tokens += countTokens(r.chunk.description);
+        }
+        // Formatting overhead: file path, line range, language, score, backticks, newlines
+        tokens += 12;
+    }
+    // Header overhead: "Auto-retrieved code context" + metadata line + separators
+    tokens += 30;
+    return tokens;
+}
+/**
+ * Returns the tokenizer method used ("tiktoken" or "heuristic").
+ */
+export function tokenizerMethod() {
+    return getEncoder() ? "tiktoken" : "heuristic";
+}
+//# sourceMappingURL=token-counter.js.map

package/dist/eval/token-counter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"token-counter.js","sourceRoot":"","sources":["../../src/eval/token-counter.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAQH,IAAI,aAAa,GAA2B,IAAI,CAAC;AACjD,IAAI,UAAU,GAAG,KAAK,CAAC;AAEvB,SAAS,UAAU;IACjB,IAAI,UAAU;QAAE,OAAO,IAAI,CAAC;IAC5B,IAAI,aAAa;QAAE,OAAO,aAAa,CAAC;IAExC,IAAI,CAAC;QACH,gDAAgD;QAChD,MAAM,GAAG,GAAG,OAAO,CAAC,aAAa,CAAmD,CAAC;QACrF,aAAa,GAAG,GAAG,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC;QAC/C,OAAO,aAAa,CAAC;IACvB,CAAC;IAAC,MAAM,CAAC;QACP,UAAU,GAAG,IAAI,CAAC;QAClB,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAY;IACpC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACpC,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,WAAW,CAAC,IAAY;IACtC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAEhC,MAAM,OAAO,GAAG,UAAU,EAAE,CAAC;IAC7B,IAAI,OAAO,EAAE,CAAC;QACZ,IAAI,CAAC;YACH,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC;QAC7C,CAAC;QAAC,MAAM,CAAC;YACP,8CAA8C;QAChD,CAAC;IACH,CAAC;IAED,OAAO,gBAAgB,CAAC,IAAI,CAAC,CAAC;AAChC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAe;IAC9C,OAAO,KAAK,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;AAChC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,SAAS,CAAC,KAAe;IACvC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,IAAI,WAAW,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,8BAA8B,CAC5C,MAAsB,EACtB,QAAgB;IAEhB,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAElC,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,iBAAiB;QACjB,MAAM,IAAI,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACvC,qBAAqB;QACrB,IAAI,CAAC,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;YACxB,MAAM,IAAI,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QAC7C,CAAC;QACD,mFAAmF;QACnF,MAAM,IAAI,EAAE,CAAC;IACf,CAAC;IAED,8EAA8E;IAC9E,MAAM,IAAI,EAAE,CAAC;IAEb,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe;IAC7B,OAAO,UAAU,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,WAAW,CAAC;AACjD,CAAC"}

package/dist/index.d.ts

@@ -6,7 +6,8 @@ export { retrieve } from "./retriever/retriever.js";
 export { loadConfig, DEFAULT_CONFIG } from "./core/config.js";
 export { createBackgroundIndexer } from "./watcher.js";
 export { createWatchIgnore } from "./indexer.js";
-export type { RagConfig, DescriptionConfig } from "./core/config.js";
+export { ImageChunker, createImageVisionProvider, getMimeType, SUPPORTED_IMAGE_EXTENSIONS } from "./chunker/image.js";
+export type { RagConfig, DescriptionConfig, ImageDescriptionConfig } from "./core/config.js";
 export type { Chunk, SearchResult, Chunker, DescriptionProvider, EmbeddingProvider, VectorStore } from "./core/interfaces.js";
 export { search, indexWorkspace, getContext, validateConfig, scanWorkspace, getIndexStatusSummary } from "./api.js";
 export type { SearchOptions, IndexOptions, ContextResult, ConfigValidationResult, WorkspaceFile, IndexRunStats } from "./api.js";

package/dist/index.js

@@ -7,6 +7,7 @@ export { retrieve } from "./retriever/retriever.js";
 export { loadConfig, DEFAULT_CONFIG } from "./core/config.js";
 export { createBackgroundIndexer } from "./watcher.js";
 export { createWatchIgnore } from "./indexer.js";
+export { ImageChunker, createImageVisionProvider, getMimeType, SUPPORTED_IMAGE_EXTENSIONS } from "./chunker/image.js";
 // High-level convenience API
 export { search, indexWorkspace, getContext, validateConfig, scanWorkspace, getIndexStatusSummary } from "./api.js";
 // Plugin is only importable inside OpenCode's runtime

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,4BAA4B;AAC5B,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC9E,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnE,OAAO,EAAE,yBAAyB,EAAE,MAAM,wBAAwB,CAAC;AACnE,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AACxD,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAC;AACvD,OAAO,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AAIjD,6BAA6B;AAC7B,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,UAAU,EAAE,cAAc,EAAE,aAAa,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAGpH,sDAAsD;AACtD,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,MAAM,CAAC,MAAM,MAAM,GAAG,SAAS,CAAC;AAChC,MAAM,CAAC,MAAM,EAAE,GAAG,qBAAqB,CAAC;AACxC,eAAe,EAAE,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,4BAA4B;AAC5B,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC9E,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnE,OAAO,EAAE,yBAAyB,EAAE,MAAM,wBAAwB,CAAC;AACnE,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AACxD,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAC;AACvD,OAAO,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AACjD,OAAO,EAAE,YAAY,EAAE,yBAAyB,EAAE,WAAW,EAAE,0BAA0B,EAAE,MAAM,oBAAoB,CAAC;AAItH,6BAA6B;AAC7B,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,UAAU,EAAE,cAAc,EAAE,aAAa,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAGpH,sDAAsD;AACtD,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,MAAM,CAAC,MAAM,MAAM,GAAG,SAAS,CAAC;AAChC,MAAM,CAAC,MAAM,EAAE,GAAG,qBAAqB,CAAC;AACxC,eAAe,EAAE,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC"}

package/dist/indexer.d.ts

@@ -52,7 +52,7 @@ export interface WatchPassScheduler {
     close(): void;
 }
 export declare function walkFiles(dir: string, extensions: Set<string>, excludeDirs: Set<string>): Promise<string[]>;
-export declare function scanWorkspace(cwd: string, config: RagConfig): Promise<WorkspaceFile[]>;
+export declare function scanWorkspace(cwd: string, config: RagConfig, logger?: Logger): Promise<WorkspaceFile[]>;
 export declare function runIndexPass(options: RunIndexPassOptions): Promise<IndexRunStats>;
 export declare function getIndexStatusSummary(cwd: string, storePath: string, config: RagConfig, store: VectorStore): Promise<IndexStatusSummary>;
 export declare function createWatchPassScheduler(runPass: () => Promise<void>, onError: (error: unknown) => void, debounceMs?: number): WatchPassScheduler;

package/dist/indexer.js

@@ -3,9 +3,11 @@ import path from "node:path";
 import pLimit from "p-limit";
 import { chunkFile } from "./chunker/factory.js";
 import { extractPdfText } from "./chunker/pdf.js";
+import { uuid } from "./chunker/uuid.js";
 import { extractDocxText } from "./chunker/docx.js";
 import { extractDocText } from "./chunker/doc.js";
 import { extractExcelText } from "./chunker/excel.js";
+import { createImageVisionProvider, getMimeType, SUPPORTED_IMAGE_EXTENSIONS } from "./chunker/image.js";
 import { computeFileHash, loadManifest, manifestPathFor, normalizeFilePath, saveManifest, } from "./core/manifest.js";
 import { embedBatch } from "./embedder/factory.js";
 function createLogger(logger) {
@@ -36,8 +38,28 @@ export async function walkFiles(dir, extensions, excludeDirs) {
     }
     return results;
 }
-export async function scanWorkspace(cwd, config) {
-    const files = await walkFiles(cwd, new Set(config.indexing.includeExtensions), new Set(config.indexing.excludeDirs));
+function isImageFile(fp, extensions) {
+    const lower = fp.toLowerCase();
+    for (const ext of extensions) {
+        if (lower.endsWith(ext.toLowerCase()))
+            return true;
+    }
+    return false;
+}
+function imageExtensionSet(extensions) {
+    return new Set(extensions.map((e) => e.toLowerCase()));
+}
+export async function scanWorkspace(cwd, config, logger) {
+    const extensions = new Set(config.indexing.includeExtensions);
+    let imageVisionProvider = null;
+    const imageCfg = config.imageDescription;
+    if (imageCfg?.enabled) {
+        for (const ext of SUPPORTED_IMAGE_EXTENSIONS) {
+            extensions.add(ext.toLowerCase());
+        }
+        imageVisionProvider = createImageVisionProvider(imageCfg);
+    }
+    const files = await walkFiles(cwd, extensions, new Set(config.indexing.excludeDirs));
     const isPdf = (fp) => fp.toLowerCase().endsWith(".pdf");
     const isDocx = (fp) => fp.toLowerCase().endsWith(".docx");
     const isDoc = (fp) => fp.toLowerCase().endsWith(".doc");
@@ -82,6 +104,20 @@ export async function scanWorkspace(cwd, config) {
                 content = "";
             }
         }
+        else if (imageVisionProvider && isImageFile(filePath, SUPPORTED_IMAGE_EXTENSIONS)) {
+            const buffer = await fs.readFile(filePath);
+            try {
+                const ext = path.extname(filePath).toLowerCase();
+                const mimeType = getMimeType(ext);
+                const b64 = buffer.toString("base64");
+                const prompt = imageCfg.prompt;
+                content = await imageVisionProvider.describeImage(b64, mimeType, prompt);
+            }
+            catch (err) {
+                logger?.warn(`  ${filePath} (vision failed: ${err.message})`);
+                content = "";
+            }
+        }
         else {
             content = await fs.readFile(filePath, "utf-8");
         }
@@ -116,7 +152,7 @@ function createIndexStats(totalFiles, manifestStatus) {
 }
 export async function runIndexPass(options) {
     const logger = createLogger(options.logger);
-    const workspaceFiles = await scanWorkspace(options.cwd, options.config);
+    const workspaceFiles = await scanWorkspace(options.cwd, options.config, logger);
     const loadResult = await loadManifest(options.storePath);
     const manifest = loadResult.manifest;
     let manifestStatus = loadResult.status;
@@ -180,10 +216,26 @@ export async function runIndexPass(options) {
             options.keywordIndex?.removeByFilePath(file.normalizedPath);
             isModified = true;
         }
-        const chunks = await chunkFile(file.filePath, file.content, options.config.chunking?.nodeTypes).catch((err) => {
-            logger.warn(`  ${fileLabel} (chunking failed: ${err.message})`);
-            return null;
-        });
+        let chunks;
+        // Images: 1 image → 1 chunk (description as content, no paragraph splitting)
+        if (isImageFile(file.filePath, SUPPORTED_IMAGE_EXTENSIONS) && file.content.trim().length > 0) {
+            chunks = [{
+                    id: uuid(),
+                    content: file.content,
+                    metadata: {
+                        filePath: file.filePath,
+                        startLine: 1,
+                        endLine: 1,
+                        language: "image",
+                    },
+                }];
+        }
+        else {
+            chunks = await chunkFile(file.filePath, file.content, options.config.chunking?.nodeTypes).catch((err) => {
+                logger.warn(`  ${fileLabel} (chunking failed: ${err.message})`);
+                return null;
+            });
+        }
         if (chunks === null || chunks.length === 0) {
             if (chunks === null) {
                 // Chunking failed — keep previous index state if file was indexed before
@@ -198,6 +250,7 @@ export async function runIndexPass(options) {
         options.keywordIndex?.addChunks(chunks);
         try {
             const docPrefix = options.config.embedding.documentPrefix ?? "";
+            const relPath = path.relative(options.cwd, file.filePath).replace(/\\/g, "/");
             const textToEmbed = [];
             if (options.descriptionProvider) {
                 let descriptionMap = null;
@@ -213,25 +266,24 @@ export async function runIndexPass(options) {
                     const batchDesc = descriptionMap?.get(chunk.id);
                     if (batchDesc && batchDesc.trim().length > 0) {
                         chunk.description = batchDesc;
-                        textToEmbed.push(docPrefix + chunk.description + "\n\n" + chunk.content);
+                        textToEmbed.push(docPrefix + relPath + "\n\n" + chunk.description + "\n\n" + chunk.content);
                     }
                     else {
                         try {
                             chunk.description = await options.descriptionProvider.generateDescription(chunk);
-                            textToEmbed.push(docPrefix + chunk.description + "\n\n" + chunk.content);
+                            textToEmbed.push(docPrefix + relPath + "\n\n" + chunk.description + "\n\n" + chunk.content);
                         }
                         catch (err) {
                             logger.warn(`Description generation failed for ${chunk.id}, falling back to content: ${err.message}`);
-                            textToEmbed.push(docPrefix + chunk.content);
+                            textToEmbed.push(docPrefix + relPath + "\n\n" + chunk.content);
                         }
                     }
                 }
             }
             else {
                 for (const chunk of chunks) {
-                    const relPath = path.relative(options.cwd, chunk.metadata.filePath).replace(/\\/g, "/");
-                    chunk.description = `${relPath}, lines ${chunk.metadata.startLine}-${chunk.metadata.endLine}`;
-                    textToEmbed.push(docPrefix + chunk.description + "\n\n" + chunk.content);
+                    chunk.description = `lines ${chunk.metadata.startLine}-${chunk.metadata.endLine}, ${chunk.metadata.language}`;
+                    textToEmbed.push(docPrefix + relPath + "\n\n" + chunk.description + "\n\n" + chunk.content);
                 }
             }
             const embeddings = await embedBatch(options.embedder, textToEmbed, options.config.indexing.embedBatchSize, "document");