opencode-rag-plugin 1.19.3 → 1.19.5

package/dist/plugin.js

@@ -4,6 +4,7 @@
  * automatic context injection and documentation mode.
  */
 import { tool } from "@opencode-ai/plugin/tool";
+import { CODE_SEARCH_FILTER } from "./core/interfaces.js";
 import { loadConfig, findConfigFile, DEFAULT_CONFIG, resolveLogConfig, persistProbedDimension } from "./core/config.js";
 import { createEmbedder } from "./embedder/factory.js";
 import { createDescriptionProvider } from "./describer/factory.js";
@@ -17,7 +18,7 @@ import { createBackgroundIndexer } from "./watcher.js";
 import { createRagReadTool } from "./opencode/create-read-tool.js";
 import { createFileSkeletonTool, createFindUsagesTool, createDescribeImageTool, createRecallQuirksTool, createAddQuirkTool, createUpdateQuirkTool, createDeleteQuirkTool, } from "./opencode/tools.js";
 import { resolveApiKey } from "./core/resolve-api-key.js";
-import { consumePendingRagInjection } from "./core/rag-injection-flag.js";
+import { consumePendingRagInjection, peekPendingRagInjection } from "./core/rag-injection-flag.js";
 import { loadDocProgress, markSubdirectoryDocumented } from "./core/doc-progress.js";
 import { loadManifest } from "./core/manifest.js";
 import { createSessionLogger } from "./eval/session-logger.js";
@@ -30,17 +31,16 @@ import { autoCaptureQuirks } from "./quirks/auto-capture.js";
 import { buildSystemGuidanceLines } from "./opencode/system-guidance.js";
 import { existsSync, readFileSync, readdirSync, unlinkSync } from "node:fs";
 import path from "node:path";
-import { fileURLToPath } from "node:url";
-import { spawn, execSync } from "node:child_process";
-import { tmpdir } from "node:os";
 /** Maximum entries per session map before oldest entries are evicted. */
 const MAX_SESSION_MAP_SIZE = 50;
+/** Maximum tool results kept per session (for quirk extraction). */
+const MAX_SESSION_TOOL_RESULTS = 20;
 /** Cache of loaded RAG configurations keyed by workspace directory. */
 const configCache = new Map();
 /** Active background indexer instances keyed by workspace directory. */
 const backgroundIndexers = new Map();
-/** Active MCP server instances keyed by workspace directory. */
-const mcpServers = new Map();
+/** Active vector stores keyed by workspace directory — closed on reload/shutdown. */
+const ragStores = new Map();
 /** Pending update notifications keyed by workspace directory. */
 const pendingUpdateInfo = new Map();
 /** Workspace directories that have already been prompted about an update this session. */
@@ -49,7 +49,7 @@ const notifiedUpdateDirs = new Set();
 const pendingRestartNotice = new Map();
 /** Guard flag to prevent re-entrant shutdown. */
 let shutdownRegistered = false;
-/** Close all active background indexers and MCP servers, then destroy idle sockets. */
+/** Close all active background indexers and vector stores, then destroy idle sockets. */
 async function shutdownPluginResources() {
     for (const [dir, indexer] of backgroundIndexers) {
         try {
@@ -58,12 +58,12 @@ async function shutdownPluginResources() {
         catch { /* best-effort */ }
         backgroundIndexers.delete(dir);
     }
-    for (const [dir, server] of mcpServers) {
+    for (const [dir, store] of ragStores) {
         try {
-            await server.close();
+            await store.close();
         }
         catch { /* best-effort */ }
-        mcpServers.delete(dir);
+        ragStores.delete(dir);
     }
     configCache.clear();
     pendingUpdateInfo.clear();
@@ -765,7 +765,10 @@ export function createRagHooks(options) {
             try {
                 const results = sessionToolResults.get(input.sessionID) ?? [];
                 results.push({ tool: input.tool, output: output.output ?? "" });
-                boundedSet(sessionToolResults, input.sessionID, results, MAX_SESSION_MAP_SIZE);
+                // Cap the per-session list — long sessions accumulate full tool
+                // outputs; only the recent entries matter for quirk extraction.
+                const capped = results.length > MAX_SESSION_TOOL_RESULTS ? results.slice(-MAX_SESSION_TOOL_RESULTS) : results;
+                boundedSet(sessionToolResults, input.sessionID, capped, MAX_SESSION_MAP_SIZE);
             }
             catch {
                 // Non-critical — must never throw
@@ -824,9 +827,17 @@ export function createRagHooks(options) {
                         if (query.length > 0) {
                             const quirkDeps = { embedder, store, keywordIndex: keywordIndex, cfg: getEffectiveCfg(), storePath: options.storePath };
                             const budgetMs = memCfg.autoInjectLatencyBudgetMs ?? 2000;
+                            let timer;
                             let quirkResults = await Promise.race([
-                                recallQuirks(quirkDeps, query, { topK: memCfg.autoInjectTopK ?? 2, minScore: memCfg.autoInjectMinScore }),
-                                new Promise((resolve) => setTimeout(() => resolve([]), budgetMs)),
+                                recallQuirks(quirkDeps, query, { topK: memCfg.autoInjectTopK ?? 2, minScore: memCfg.autoInjectMinScore })
+                                    .then((r) => {
+                                    if (timer)
+                                        clearTimeout(timer);
+                                    return r;
+                                }),
+                                new Promise((resolve) => {
+                                    timer = setTimeout(() => resolve([]), budgetMs);
+                                }),
                             ]);
                             // Dedup: skip quirks already injected into this session
                             const injectedSet = getOrCreateSessionSet(sessionInjectedQuirks, sessionId, MAX_SESSION_MAP_SIZE);
@@ -893,8 +904,8 @@ export function createRagHooks(options) {
                         }
                     }
                 }
-                sessionPrevUserReq.set(input.sessionID, text);
-                sessionToolResults.set(input.sessionID, []);
+                boundedSet(sessionPrevUserReq, input.sessionID, text, MAX_SESSION_MAP_SIZE);
+                boundedSet(sessionToolResults, input.sessionID, [], MAX_SESSION_MAP_SIZE);
                 boundedSet(sessionLastMessage, input.sessionID, text, MAX_SESSION_MAP_SIZE);
                 // Handle /doc slash command
                 if (text.startsWith("/doc")) {
@@ -1122,9 +1133,17 @@ export function createRagHooks(options) {
                             // Use the stricter recallMinScore for user-prompt injection
                             // (system prompt injection uses the lower autoInjectMinScore)
                             const minScore = quirkMemoryCfg.recallMinScore ?? 0.72;
+                            let timer;
                             let quirkResults = await Promise.race([
-                                recallQuirks(quirkDeps, quirkQuery, { topK: quirkMemoryCfg.autoInjectTopK ?? 2, minScore }),
-                                new Promise((resolve) => setTimeout(() => resolve([]), budgetMs)),
+                                recallQuirks(quirkDeps, quirkQuery, { topK: quirkMemoryCfg.autoInjectTopK ?? 2, minScore })
+                                    .then((r) => {
+                                    if (timer)
+                                        clearTimeout(timer);
+                                    return r;
+                                }),
+                                new Promise((resolve) => {
+                                    timer = setTimeout(() => resolve([]), budgetMs);
+                                }),
                             ]);
                             // Dedup: skip quirks already injected into this session
                             const injectedSet = getOrCreateSessionSet(sessionInjectedQuirks, input.sessionID, MAX_SESSION_MAP_SIZE);
@@ -1167,15 +1186,19 @@ export function createRagHooks(options) {
                     }
                 }
                 // Handle hotkey-triggered RAG injection (Ctrl+Enter / Ctrl+Alt+Enter)
-                const pendingInjection = consumePendingRagInjection(options.storePath);
+                // Peek FIRST and only consume once we know the index is usable —
+                // otherwise the flag is dropped on an empty index and the hotkey
+                // press is silently lost.
+                const pendingInjection = peekPendingRagInjection(options.storePath);
                 if (pendingInjection) {
+                    const count = await store.count();
+                    if (count === 0)
+                        return;
+                    consumePendingRagInjection(options.storePath);
                     appendDebugLog(options.logFilePath, {
                         scope: "chat.message",
                         message: `pending injection: ${pendingInjection}`,
                     });
-                    const count = await store.count();
-                    if (count === 0)
-                        return;
                     const effectiveCfg = getEffectiveCfg();
                     const hybridCfg = effectiveCfg.retrieval.hybridSearch;
                     const retrievalStart = Date.now();
@@ -1189,6 +1212,8 @@ export function createRagHooks(options) {
                         keywordIndex,
                         keywordWeight: hybridCfg?.keywordWeight,
                         queryPrefix: effectiveCfg.embedding.queryPrefix,
+                        // Never surface quirk chunks in hotkey file lists / chunk injections.
+                        filter: CODE_SEARCH_FILTER,
                     });
                     const retrievalTimeMs = Date.now() - retrievalStart;
                     if (results.length > 0) {
@@ -1271,122 +1296,12 @@ async function loadKeywordIndex(storePath, logFilePath, logLevel) {
         return new KeywordIndex(storePath);
     }
 }
-let _cachedNodeExecutable;
-/**
- * Resolve the path to the Node.js executable, caching the result.
- * Checks `process.execPath` first, then falls back to `where`/`which`.
- */
-function resolveNodeExecutable() {
-    if (_cachedNodeExecutable !== undefined)
-        return _cachedNodeExecutable;
-    const execPath = process.execPath;
-    const basename = path.basename(execPath).toLowerCase();
-    if (basename === "node" || basename === "node.exe") {
-        _cachedNodeExecutable = execPath;
-        return _cachedNodeExecutable;
-    }
-    const tryCmd = (cmd) => {
-        try {
-            const result = execSync(cmd, { encoding: "utf-8", timeout: 5000, stdio: ["pipe", "pipe", "pipe"] }).trim();
-            return result.split("\n")[0]?.trim() || null;
-        }
-        catch {
-            return null;
-        }
-    };
-    const found = process.platform === "win32"
-        ? tryCmd("where node")
-        : tryCmd("which node");
-    _cachedNodeExecutable = found;
-    return _cachedNodeExecutable;
-}
-/**
- * Resolve the MCP CLI entry point script path.
- * Checks `dist/cli.js` first, then falls back to `src/cli.ts`.
- */
-function resolveMcpCliEntry() {
-    const selfDir = path.dirname(fileURLToPath(import.meta.url));
-    const packageRoot = path.resolve(selfDir, "..");
-    const distCli = path.join(packageRoot, "dist", "cli.js");
-    if (existsSync(distCli))
-        return distCli;
-    const srcCli = path.join(packageRoot, "src", "cli.ts");
-    if (existsSync(srcCli))
-        return srcCli;
-    return null;
-}
-/**
- * Start an MCP server process for the given workspace directory.
- * Spawns the CLI in MCP mode as a child process and wires stdout/stderr
- * to the debug log. Returns a handle for graceful shutdown.
- */
-function startMcpServerProcess(cwd, logFilePath, logLevel) {
-    const cliEntry = resolveMcpCliEntry();
-    if (!cliEntry) {
-        appendDebugLog(logFilePath, {
-            scope: "mcp",
-            message: "Could not resolve MCP CLI entry point; skipping autostart",
-        }, logLevel);
-        return { close: async () => { } };
-    }
-    const nodeExec = resolveNodeExecutable();
-    if (!nodeExec) {
-        appendDebugLog(logFilePath, {
-            scope: "mcp",
-            message: "Could not resolve Node.js executable; skipping MCP autostart",
-        }, logLevel);
-        return { close: async () => { } };
-    }
-    const args = cliEntry.endsWith(".ts")
-        ? ["--import", "tsx", cliEntry, "mcp"]
-        : [cliEntry, "mcp"];
-    appendDebugLog(logFilePath, {
-        scope: "mcp",
-        message: `Starting MCP server: ${nodeExec} ${args.join(" ")}`,
-    }, logLevel);
-    const child = spawn(nodeExec, args, {
-        cwd,
-        stdio: "pipe",
-        detached: false,
-    });
-    child.stdout?.on("data", (chunk) => {
-        appendDebugLog(logFilePath, {
-            scope: "mcp",
-            message: `stdout: ${chunk.toString().trim()}`,
-        }, logLevel);
-    });
-    child.stderr?.on("data", (chunk) => {
-        appendDebugLog(logFilePath, {
-            scope: "mcp",
-            message: `stderr: ${chunk.toString().trim()}`,
-        }, logLevel);
-    });
-    child.on("error", (err) => {
-        appendDebugLog(logFilePath, {
-            scope: "mcp",
-            message: "MCP server process error",
-            error: err,
-        }, logLevel);
-    });
-    child.on("exit", (code, signal) => {
-        appendDebugLog(logFilePath, {
-            scope: "mcp",
-            message: `MCP server exited (code=${code}, signal=${signal})`,
-        });
-        mcpServers.delete(cwd);
-    });
-    return {
-        close: async () => {
-            if (child.exitCode !== null || child.killed)
-                return;
-            appendDebugLog(logFilePath, {
-                scope: "mcp",
-                message: "Shutting down MCP server",
-            });
-            child.kill("SIGTERM");
-        },
-    };
-}
+// NOTE: There is deliberately NO autostart of an MCP child process here.
+// A stdio MCP server can only be reached by the client that spawns it —
+// a plugin-spawned child with an unwired stdin would sit idle forever,
+// holding its own LanceDB connection (and locking the store directory on
+// Windows). External MCP clients must run `opencode-rag mcp` themselves;
+// the plugin's agent tools run in-process regardless.
 /**
  * OpenCodeRAG plugin factory — invoked by OpenCode's plugin system for each workspace.
  *
@@ -1441,20 +1356,22 @@ export const ragPlugin = async (input, _options) => {
         }
         backgroundIndexers.delete(input.directory);
     }
-    // Close existing MCP server for this directory if one exists (e.g. on plugin reload)
-    const existingMcp = mcpServers.get(input.directory);
-    if (existingMcp) {
+    // Close the previous plugin instance's vector store — without this every
+    // reload leaks a LanceDB connection (and on Windows holds the store
+    // directory open, breaking the atomic swap on `index --force`).
+    const existingStore = ragStores.get(input.directory);
+    if (existingStore) {
         try {
-            await existingMcp.close();
+            await existingStore.close();
         }
         catch (err) {
             appendDebugLog(logFilePath, {
                 scope: "plugin",
-                message: "Failed to close existing MCP server",
+                message: "Failed to close existing vector store",
                 error: err,
             }, logLevel);
         }
-        mcpServers.delete(input.directory);
+        ragStores.delete(input.directory);
     }
     // Clean up stale config cache and pending update info for this directory
     configCache.delete(input.directory);
@@ -1505,6 +1422,7 @@ export const ragPlugin = async (input, _options) => {
         }
     }
     const store = createVectorStore(effectiveCfg, storePath, vectorDimension);
+    ragStores.set(input.directory, store);
     // Load or create keyword index for hybrid search
     const keywordIndex = await loadKeywordIndex(storePath, logFilePath, logLevel);
     // Create description provider (enabled by default)
@@ -1552,15 +1470,14 @@ export const ragPlugin = async (input, _options) => {
             catch { /* ignore */ }
         }
     }
-    // Auto-start standalone MCP server if enabled (default: off). Agent tools
-    // (search_semantic, get_file_skeleton, find_usages, describe_image) and the
-    // chat.message hook run in-process regardless. Enable only when an external
-    // MCP client connects to `opencode-rag mcp`.
-    const mcpCfg = effectiveCfg.mcp ?? { enabled: false };
-    const isTempDir = path.resolve(input.directory).startsWith(tmpdir());
-    if (mcpCfg.enabled && !isTempDir) {
-        const mcpInstance = startMcpServerProcess(input.directory, logFilePath, logLevel);
-        mcpServers.set(input.directory, mcpInstance);
+    // NOTE: no MCP child-process autostart — a stdio MCP server is only
+    // reachable by the client that spawns it. External MCP clients run
+    // `opencode-rag mcp` themselves; agent tools run in-process regardless.
+    if (effectiveCfg.mcp?.enabled) {
+        appendDebugLog(logFilePath, {
+            scope: "mcp",
+            message: "mcp.enabled is set, but the plugin does not autostart a child server — run `opencode-rag mcp` from your MCP client",
+        }, logLevel);
     }
     // Auto-update check (non-blocking, best-effort). On by default; can be
     // disabled via `autoUpdate.enabled: false` in opencode-rag.json. When a newer

package/dist/quirks/auto-capture.js

@@ -21,6 +21,7 @@ function buildExchangeText(exchanges) {
     }
     return lines.join("\n");
 }
+const VALID_QUIRK_TYPES = new Set(["gotcha", "preference", "decision", "environment-constraint"]);
 function parseExtractionOutput(text) {
     const results = [];
     for (const line of text.split("\n")) {
@@ -34,6 +35,10 @@ function parseExtractionOutput(text) {
         const content = trimmed.slice(pipeIdx + 1).trim();
         if (!quirkType || !content || content.length > 200)
             continue;
+        // An LLM emitting a garbage type would otherwise pollute recall filters —
+        // skip unknown types (or default them to "gotcha").
+        if (!VALID_QUIRK_TYPES.has(quirkType))
+            continue;
         results.push({ quirkType, content });
     }
     return results;

package/dist/quirks/quirk-store.d.ts

@@ -44,7 +44,7 @@ export declare function lexicalSimilarity(a: string, b: string): number;
  * Count of meaningful word tokens shared between two texts (Jaccard numerator).
  *
  * Tokens are whitespace/punctuation-split, lowercased, and filtered to those
- * with length ≥ `minTokenLen` (default 3 — skips short filler like "the").
+ * with length ≥ `minTokenLen` (default 3 — skips short filler like "the").
  *
  * Used by the quirk auto-inject gate: candidate quirks that share no tokens
  * with the user's *current* message (i.e. they matched only against the prior

package/dist/quirks/quirk-store.js

@@ -1,4 +1,4 @@
-import { appendFileSync, existsSync, readFileSync, writeFileSync, mkdirSync } from "node:fs";
+import { appendFileSync, existsSync, readFileSync, writeFileSync, mkdirSync, renameSync } from "node:fs";
 import { randomUUID } from "node:crypto";
 import path from "node:path";
 import { retrieve } from "../retriever/retriever.js";
@@ -11,15 +11,33 @@ function jsonlPath(storePath) {
 function isMemoryStore(storePath) {
     return storePath.startsWith("memory:");
 }
-/** In-memory backup for memory:// stores. */
-const memQuirks = new Map();
+/** In-memory backup for memory:// stores, keyed by store path. */
+const memQuirksByStore = new Map();
+function memQuirksFor(storePath) {
+    let store = memQuirksByStore.get(storePath);
+    if (!store) {
+        store = new Map();
+        memQuirksByStore.set(storePath, store);
+    }
+    return store;
+}
 function readJsonl(filePath) {
     if (!existsSync(filePath))
         return [];
     const raw = readFileSync(filePath, "utf-8").trim();
     if (!raw)
         return [];
-    return raw.split("\n").map((l) => JSON.parse(l));
+    const quirks = [];
+    for (const line of raw.split("\n")) {
+        try {
+            quirks.push(JSON.parse(line));
+        }
+        catch {
+            // Skip corrupt lines — a single bad line must never break
+            // every quirk operation (readJsonl feeds get/update/remove/list).
+        }
+    }
+    return quirks;
 }
 function appendJsonl(filePath, q) {
     const dir = path.dirname(filePath);
@@ -33,7 +51,24 @@ function rewriteJsonl(filePath, quirks) {
     if (!existsSync(dir)) {
         mkdirSync(dir, { recursive: true });
     }
-    writeFileSync(filePath, quirks.map((q) => JSON.stringify(q)).join("\n") + "\n", "utf-8");
+    // Atomic write: tmp file + rename, so a crash or concurrent process
+    // can never leave a truncated/empty quirks.jsonl behind.
+    const tmpPath = `${filePath}.tmp`;
+    writeFileSync(tmpPath, quirks.map((q) => JSON.stringify(q)).join("\n") + "\n", "utf-8");
+    try {
+        renameSync(tmpPath, filePath);
+    }
+    catch {
+        // Windows: rename can fail with EPERM when another process holds the file;
+        // fall back to a direct write (best-effort) rather than losing data.
+        writeFileSync(filePath, quirks.map((q) => JSON.stringify(q)).join("\n") + "\n", "utf-8");
+        try {
+            renameSync(tmpPath, `${filePath}.bak`);
+        }
+        catch {
+            // ignore
+        }
+    }
 }
 function nowISO() {
     return new Date().toISOString();
@@ -62,7 +97,7 @@ export async function getQuirk(deps, id) {
         }
         return undefined;
     }
-    return memQuirks.get(id);
+    return memQuirksFor(deps.storePath).get(id);
 }
 /**
  * Update an existing quirk by ID. Fields in `patch` override the stored values.
@@ -100,8 +135,9 @@ export async function updateQuirk(deps, id, patch) {
         sourceRef,
     };
     const filePath = QUIRK_FILE_PREFIX + id;
-    await deps.store.deleteByFilePath(filePath);
-    deps.keywordIndex.removeByFilePath(filePath);
+    // Embed FIRST — if embedding fails, the old index entries are untouched
+    // (previously the delete happened before the embed, permanently removing
+    // the quirk from search while the JSONL still listed it).
     const prefix = deps.cfg.embedding.documentPrefix ?? "";
     const chunkContent = prefix + content;
     const embeddings = await deps.embedder.embed([chunkContent], "document");
@@ -109,6 +145,8 @@ export async function updateQuirk(deps, id, patch) {
     if (!embedding || embedding.length === 0) {
         throw new Error("Embedding returned empty vector for quirk content");
     }
+    await deps.store.deleteByFilePath(filePath);
+    deps.keywordIndex.removeByFilePath(filePath);
     const chunk = {
         id,
         content,
@@ -134,7 +172,7 @@ export async function updateQuirk(deps, id, patch) {
         rewriteJsonl(jp, all);
     }
     else {
-        memQuirks.set(id, updated);
+        memQuirksFor(deps.storePath).set(id, updated);
     }
     return updated;
 }
@@ -186,7 +224,7 @@ export async function addQuirk(deps, input) {
         appendJsonl(jsonlPath(deps.storePath), quirk);
     }
     else {
-        memQuirks.set(id, quirk);
+        memQuirksFor(deps.storePath).set(id, quirk);
     }
     return quirk;
 }
@@ -205,7 +243,7 @@ export async function removeQuirk(deps, id) {
         rewriteJsonl(jp, all);
     }
     else {
-        memQuirks.delete(id);
+        memQuirksFor(deps.storePath).delete(id);
     }
 }
 /** List all quirks sorted by lastObserved descending. */
@@ -238,7 +276,7 @@ export async function listQuirks(deps) {
         result.sort((a, b) => b.lastObserved.localeCompare(a.lastObserved));
         return result;
     }
-    const all = [...memQuirks.values()];
+    const all = [...memQuirksFor(deps.storePath).values()];
     all.sort((a, b) => b.lastObserved.localeCompare(a.lastObserved));
     return all;
 }
@@ -246,10 +284,11 @@ export async function listQuirks(deps) {
 export async function recallQuirks(deps, query, options) {
     const topK = options?.topK ?? 10;
     const minConfidence = deps.cfg.memory?.minConfidence ?? 0.5;
+    // NOTE: do NOT put quirkType into `filter.languages` — quirk chunks store
+    // the type in metadata.quirkType, not metadata.language ("quirk").
+    // A languages filter would exclude every quirk chunk and return zero hits.
+    // Type filtering happens in the post-filter below instead.
     const filter = { kinds: ["quirk"] };
-    if (options?.quirkType) {
-        filter.languages = [options.quirkType];
-    }
     const recallMinScore = options?.minScore ?? deps.cfg.memory?.recallMinScore ?? 0.72;
     const raw = await retrieve(query, deps.embedder, deps.store, {
         topK: topK * 3,
@@ -304,7 +343,7 @@ export async function lintQuirks(deps) {
         for (let j = i + 1; j < quirks.length; j++) {
             const sim = lexicalSimilarity(quirks[i].content, quirks[j].content);
             if (sim > 0.85) {
-                issues.push(`Near-duplicate (${(sim * 100).toFixed(0)}% similar): "${quirks[i].content}" ↔ "${quirks[j].content}"`);
+                issues.push(`Near-duplicate (${(sim * 100).toFixed(0)}% similar): "${quirks[i].content}" ↔ "${quirks[j].content}"`);
             }
         }
     }
@@ -322,7 +361,7 @@ export function lexicalSimilarity(a, b) {
  * Count of meaningful word tokens shared between two texts (Jaccard numerator).
  *
  * Tokens are whitespace/punctuation-split, lowercased, and filtered to those
- * with length ≥ `minTokenLen` (default 3 — skips short filler like "the").
+ * with length ≥ `minTokenLen` (default 3 — skips short filler like "the").
  *
  * Used by the quirk auto-inject gate: candidate quirks that share no tokens
  * with the user's *current* message (i.e. they matched only against the prior

package/dist/retriever/context-optimizer.js

@@ -26,10 +26,20 @@ function toOptimized(r) {
 }
 /**
  * Compute Jaccard similarity between two strings based on their token sets.
+ * Accepts a per-call token cache so repeated comparisons (the dedup loop is
+ * O(n²) over same-file pairs) don't re-tokenize the same content every time.
  */
-function jaccardSimilarity(a, b) {
-    const tokensA = new Set(tokenize(a));
-    const tokensB = new Set(tokenize(b));
+function jaccardSimilarity(a, b, tokenCache) {
+    let tokensA = tokenCache.get(a);
+    if (!tokensA) {
+        tokensA = new Set(tokenize(a));
+        tokenCache.set(a, tokensA);
+    }
+    let tokensB = tokenCache.get(b);
+    if (!tokensB) {
+        tokensB = new Set(tokenize(b));
+        tokenCache.set(b, tokensB);
+    }
     if (tokensA.size === 0 && tokensB.size === 0)
         return 1;
     let intersection = 0;
@@ -99,12 +109,16 @@ function dedupeSimilar(results, threshold) {
     if (results.length <= 1)
         return results;
     const kept = [...results];
+    // Token cache shared across all pair comparisons of this group — the
+    // dedup loop is O(n²) over pairs and was re-tokenizing full chunk
+    // contents on every comparison.
+    const tokenCache = new Map();
     let changed = true;
     while (changed) {
         changed = false;
         for (let i = 0; i < kept.length; i++) {
             for (let j = i + 1; j < kept.length; j++) {
-                const sim = jaccardSimilarity(kept[i].chunk.content, kept[j].chunk.content);
+                const sim = jaccardSimilarity(kept[i].chunk.content, kept[j].chunk.content, tokenCache);
                 if (sim > threshold) {
                     const [keepIdx, removeIdx] = kept[i].score >= kept[j].score ? [i, j] : [j, i];
                     const removedId = kept[removeIdx].chunk.id;

package/dist/retriever/keyword-index.d.ts

@@ -18,6 +18,8 @@ export declare class KeywordIndex {
     private invertedIndex;
     private chunkMap;
     private readonly storePath?;
+    /** filePath (normalized) → chunk IDs, for O(chunks-in-file) removal. */
+    private fileToIds;
     constructor(storePath?: string);
     addChunks(chunks: Chunk[]): void;
     getMatchedTerms(query: string, chunkId: string): string[];