opencode-rag-plugin 1.19.3 → 1.19.5

package/dist/retriever/keyword-index.js

@@ -1,3 +1,4 @@
+import { normalizeFilePath } from "../core/manifest.js";
 const INDEX_VERSION = 2;
 /** Suffix-stripping stemmer. Only stems words >= 6 characters to reduce false positives. */
 function stem(word) {
@@ -71,6 +72,8 @@ export class KeywordIndex {
     invertedIndex = new Map();
     chunkMap = new Map();
     storePath;
+    /** filePath (normalized) → chunk IDs, for O(chunks-in-file) removal. */
+    fileToIds = new Map();
     constructor(storePath) {
         this.storePath = storePath;
     }
@@ -78,6 +81,13 @@ export class KeywordIndex {
         for (const chunk of chunks) {
             const id = chunk.id;
             this.chunkMap.set(id, chunk);
+            const fileKey = normalizeFilePath(chunk.metadata.filePath);
+            let ids = this.fileToIds.get(fileKey);
+            if (!ids) {
+                ids = new Set();
+                this.fileToIds.set(fileKey, ids);
+            }
+            ids.add(id);
             const tokens = tokenize(chunk.content);
             for (const token of tokens) {
                 let docs = this.invertedIndex.get(token);
@@ -101,10 +111,21 @@ export class KeywordIndex {
         return matched;
     }
     removeByFilePath(filePath) {
-        const idsToRemove = [];
-        for (const [id, chunk] of this.chunkMap) {
-            if (chunk.metadata.filePath === filePath) {
-                idsToRemove.push(id);
+        const fileKey = normalizeFilePath(filePath);
+        // Fast path: per-file id index. Falls back to a full chunkMap scan when
+        // the path is not tracked (entries loaded from an older persisted index).
+        let idsToRemove;
+        const tracked = this.fileToIds.get(fileKey);
+        if (tracked) {
+            idsToRemove = [...tracked];
+            this.fileToIds.delete(fileKey);
+        }
+        else {
+            idsToRemove = [];
+            for (const [id, chunk] of this.chunkMap) {
+                if (normalizeFilePath(chunk.metadata.filePath) === fileKey) {
+                    idsToRemove.push(id);
+                }
             }
         }
         for (const id of idsToRemove) {
@@ -155,10 +176,12 @@ export class KeywordIndex {
     close() {
         this.invertedIndex.clear();
         this.chunkMap.clear();
+        this.fileToIds.clear();
     }
     clear() {
         this.invertedIndex.clear();
         this.chunkMap.clear();
+        this.fileToIds.clear();
     }
     count() {
         return this.chunkMap.size;
@@ -241,6 +264,17 @@ export class KeywordIndex {
                 },
             });
         }
+        // Rebuild the per-file id index from the loaded chunk map so removals
+        // work even for entries persisted by older versions.
+        for (const [id, chunk] of index.chunkMap) {
+            const fileKey = normalizeFilePath(chunk.metadata.filePath);
+            let ids = index.fileToIds.get(fileKey);
+            if (!ids) {
+                ids = new Set();
+                index.fileToIds.set(fileKey, ids);
+            }
+            ids.add(id);
+        }
         return index;
     }
     static async clearFile(storePath) {

package/dist/retriever/retriever.js

@@ -96,7 +96,12 @@ export async function retrieve(query, embedder, store, options = {}) {
             .slice(0, topK);
         return combinedResults;
     }
-    catch {
+    catch (err) {
+        // Never silently mask retrieval failures as "no results" — an embedder
+        // outage or store bug must be visible, not indistinguishable from an
+        // empty index.
+        const message = err instanceof Error ? err.message : String(err);
+        console.warn(`[retriever] retrieve() failed (returning []): ${message}`);
         return [];
     }
 }

package/dist/tui.js

@@ -3,7 +3,7 @@
  * settings dialog for editing config values, and model selection picker.
  */
 import { createElement, insert, setProp } from "@opentui/solid";
-import { readFileSync, existsSync, writeFileSync } from "node:fs";
+import { readFileSync, existsSync, writeFileSync, unlinkSync } from "node:fs";
 import { dirname, join, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 import { loadRuntimeOverrides, saveRuntimeOverride } from "./core/runtime-overrides.js";
@@ -277,7 +277,10 @@ function resolveProviderBaseUrl(provider) {
  */
 function saveConfigValue(configPath, path, value) {
     try {
-        const data = JSON.parse(readFileSync(configPath, "utf-8"));
+        let raw = readFileSync(configPath, "utf-8");
+        if (raw.charCodeAt(0) === 0xfeff)
+            raw = raw.slice(1);
+        const data = JSON.parse(raw);
         let target = data;
         for (let i = 0; i < path.length - 1; i++) {
             const key = path[i];
@@ -292,6 +295,26 @@ function saveConfigValue(configPath, path, value) {
     catch {
     }
 }
+/**
+ * Mirror the watcher status file that the server plugin maintains, so the
+ * sidebar reflects a watcher toggle from the settings dialog immediately:
+ * - enabled  → write the same initial status the background indexer writes on startup
+ * - disabled → remove the file (same cleanup the plugin does when auto-index is off)
+ */
+function syncWatcherStatusFile(storePath, enabled) {
+    const statusPath = join(storePath, "watcher-status.json");
+    try {
+        if (enabled) {
+            writeFileSync(statusPath, JSON.stringify({ running: false }, null, 2), "utf-8");
+        }
+        else if (existsSync(statusPath)) {
+            unlinkSync(statusPath);
+        }
+    }
+    catch {
+        // best-effort mirror; the server plugin reconciles on its next load
+    }
+}
 /**
  * Save a model selection to both runtime overrides and the config file.
  * Resolves the RAG provider name and API base URL automatically.
@@ -320,8 +343,9 @@ function saveModelSelection(storePath, configPath, selectionValue, path, provide
     }
     const apiKey = provider?.options?.apiKey ?? "";
     if (apiKey) {
+        // Persist the key only in the store-dir overrides, never in the workspace
+        // config file — `opencode-rag.json` is routinely committed to git.
         saveRuntimeOverride(storePath, [section, "apiKey"], apiKey);
-        saveConfigValue(configPath, [section, "apiKey"], apiKey);
     }
     return selectionValue;
 }
@@ -689,7 +713,12 @@ async function openSettingsDialog(api) {
                     if (entry.path[0] === "indexing" || entry.path[0] === "chunking") {
                         api.ui.toast({ variant: "warning", title: "Settings", message: "Chunking changed. Re-index required." });
                     }
+                    if (entry.path.join(".") === "openCode.autoIndex.enabled") {
+                        syncWatcherStatusFile(storePath, newVal);
+                        api.ui.toast({ variant: "warning", title: "Settings", message: "Watcher changes take effect after an OpenCode restart" });
+                    }
                     entry.currentValue = newVal;
+                    api.onSettingsChanged?.();
                     showSettingMenu(cat);
                 }
                 else if (entry.type === "number") {
@@ -711,6 +740,7 @@ async function openSettingsDialog(api) {
                                     api.ui.toast({ variant: "warning", title: "Settings", message: "Chunking changed. Re-index required." });
                                 }
                                 entry.currentValue = num;
+                                api.onSettingsChanged?.();
                             }
                             showSettingMenu(cat);
                         },
@@ -743,6 +773,7 @@ async function openSettingsDialog(api) {
                                     api.ui.toast({ variant: "warning", title: "Settings", message: "Chunking changed. Re-index required." });
                                 }
                                 entry.currentValue = parsed;
+                                api.onSettingsChanged?.();
                             }
                             catch {
                                 api.ui.toast({ variant: "error", title: "Settings", message: "Invalid JSON" });
@@ -768,6 +799,7 @@ async function openSettingsDialog(api) {
                                 message: `${entry.label}: ${input}`,
                             });
                             entry.currentValue = input;
+                            api.onSettingsChanged?.();
                             showSettingMenu(cat);
                         },
                         onCancel: () => {
@@ -799,6 +831,7 @@ async function openSettingsDialog(api) {
                                 saveConfigValue(configPath, entry.path, input);
                                 entry.currentValue = input;
                             }
+                            api.onSettingsChanged?.();
                             showSettingMenu(cat);
                         },
                         onCancel: () => showSettingMenu(cat),
@@ -821,6 +854,7 @@ async function openSettingsDialog(api) {
                             message: "Embedding changed. Re-index may be required. Restart OpenCode for changes.",
                         });
                     }
+                    api.onSettingsChanged?.();
                 }
                 showSettingMenu(cat);
             },
@@ -840,7 +874,7 @@ const plugin = {
         const version = meta.version ?? getVersion();
         let cachedStatus = DEFAULT_STATUS;
         let lastRefresh = 0;
-        const REFRESH_INTERVAL_MS = Number(process.env.OPENCODE_RAG_TUI_REFRESH_MS) || 3600000;
+        const REFRESH_INTERVAL_MS = Number(process.env.OPENCODE_RAG_TUI_REFRESH_MS) || 30000;
         // Load tui config for keybinding display
         let tuiConfig;
         const worktree = api.state.path.worktree;
@@ -948,6 +982,9 @@ const plugin = {
                             openSettingsDialog({
                                 ui: api.ui,
                                 state: api.state,
+                                onSettingsChanged: () => {
+                                    refreshStatus();
+                                },
                             });
                             return undefined;
                         },

package/dist/vectorstore/lancedb.d.ts

@@ -120,6 +120,24 @@ export declare class LanceDbStore implements VectorStore {
      * @returns An array of chunk summaries.
      */
     getChunks(offset: number, limit: number): Promise<ChunkSummary[]>;
+    /**
+     * Retrieve a paginated list of chunks without embeddings, with optional
+     * language and file-path filters pushed into the SQL WHERE clause.
+     *
+     * @param offset - Number of rows to skip (for pagination).
+     * @param limit - Maximum number of rows to return.
+     * @param lang - Optional exact language filter.
+     * @param filePrefix - Optional filePath prefix filter.
+     * @returns The page of chunks plus the total count of matching rows.
+     */
+    getChunksFiltered(offset: number, limit: number, lang?: string, filePrefix?: string): Promise<{
+        chunks: ChunkSummary[];
+        total: number;
+    }>;
+    /** Look up a single chunk by its ID. Returns undefined when not found. */
+    getChunkById(id: string): Promise<ChunkSummary | undefined>;
+    /** Fetch chunks by their IDs (for the compare view). Bounded to 100 ids. */
+    getChunksByIds(ids: string[]): Promise<ChunkSummary[]>;
     /**
      * Fetch chunks with their embedding vectors included (for embedding projection).
      * @param limit - Maximum number of rows to return.
@@ -169,7 +187,13 @@ export declare class LanceDbStore implements VectorStore {
      * @param newPath - Optional new filesystem path for the LanceDB database.
      */
     reopen(newPath?: string): Promise<void>;
-    /** Close the database connection and release resources. */
+    /**
+     * Close the database connection and release resources.
+     *
+     * The native LanceDB `close()` can hang indefinitely on Windows, so the
+     * whole close is raced against a timeout — callers (CLI commands, plugin
+     * reload, web server) must never block forever on shutdown.
+     */
     close(): Promise<void>;
     /**
      * Back up the chunks.lance directory before a destructive operation.

package/dist/vectorstore/lancedb.js

@@ -396,6 +396,12 @@ export class LanceDbStore {
     }
     async searchWithFilter(embedding, topK, filter) {
         try {
+            // Guard against dimension mismatch BEFORE the native call — LanceDB
+            // throws a cryptic error that used to be swallowed into "no results".
+            if (embedding.length !== this.vectorDimension) {
+                console.warn(`[lancedb] searchWithFilter: query embedding dimension ${embedding.length} != store dimension ${this.vectorDimension} — returning empty`);
+                return [];
+            }
             return await this.searchInternal(embedding, topK, filter);
         }
         catch (err) {
@@ -404,7 +410,12 @@ export class LanceDbStore {
                 if (repaired) {
                     return this.searchInternal(embedding, topK, filter);
                 }
+                console.warn(`[lancedb] searchWithFilter: repair failed: ${err.message}`);
+                return [];
             }
+            // Never silently mask non-corruption failures as "no results" —
+            // that hides provider outages and store bugs from every caller.
+            console.warn(`[lancedb] searchWithFilter failed (returning []): ${err.message}`);
             return [];
         }
     }
@@ -534,6 +545,104 @@ export class LanceDbStore {
             }));
         });
     }
+    /**
+     * Retrieve a paginated list of chunks without embeddings, with optional
+     * language and file-path filters pushed into the SQL WHERE clause.
+     *
+     * @param offset - Number of rows to skip (for pagination).
+     * @param limit - Maximum number of rows to return.
+     * @param lang - Optional exact language filter.
+     * @param filePrefix - Optional filePath prefix filter.
+     * @returns The page of chunks plus the total count of matching rows.
+     */
+    async getChunksFiltered(offset, limit, lang, filePrefix) {
+        const conditions = [];
+        if (lang)
+            conditions.push(`language = '${lang.replace(/'/g, "''")}'`);
+        if (filePrefix)
+            conditions.push(`filePath LIKE '${filePrefix.replace(/'/g, "''")}%'`);
+        const where = conditions.length > 0 ? conditions.join(" AND ") : undefined;
+        return this.withCorruptionRecovery(async () => {
+            const table = await this.getTable();
+            let query = table.query().select(QUERY_COLUMNS);
+            let countQuery = table.query().select(["id"]);
+            if (where) {
+                query = query.where(where);
+                countQuery = countQuery.where(where);
+            }
+            const [rows, countRows] = await Promise.all([
+                query.offset(Math.max(0, offset)).limit(Math.max(1, Math.min(limit, 1000))).toArray(),
+                countQuery.toArray(),
+            ]);
+            return {
+                chunks: rows.map((row) => ({
+                    id: row.id,
+                    filePath: row.filePath,
+                    language: row.language,
+                    startLine: row.startLine,
+                    endLine: row.endLine,
+                    content: row.content,
+                    description: row.description ?? "",
+                    kind: row.kind ?? "",
+                    quirkType: row.quirkType ?? "",
+                    tags: row.tags ?? "",
+                })),
+                total: countRows.length,
+            };
+        });
+    }
+    /** Look up a single chunk by its ID. Returns undefined when not found. */
+    async getChunkById(id) {
+        return this.withCorruptionRecovery(async () => {
+            const table = await this.getTable();
+            const rows = await table.query()
+                .select(QUERY_COLUMNS)
+                .where(`id = '${id.replace(/'/g, "''")}'`)
+                .limit(1)
+                .toArray();
+            const row = rows[0];
+            if (!row)
+                return undefined;
+            return {
+                id: row.id,
+                filePath: row.filePath,
+                language: row.language,
+                startLine: row.startLine,
+                endLine: row.endLine,
+                content: row.content,
+                description: row.description ?? "",
+                kind: row.kind ?? "",
+                quirkType: row.quirkType ?? "",
+                tags: row.tags ?? "",
+            };
+        });
+    }
+    /** Fetch chunks by their IDs (for the compare view). Bounded to 100 ids. */
+    async getChunksByIds(ids) {
+        const bounded = ids.slice(0, 100);
+        if (bounded.length === 0)
+            return [];
+        return this.withCorruptionRecovery(async () => {
+            const table = await this.getTable();
+            const idList = bounded.map((id) => `'${id.replace(/'/g, "''")}'`).join(", ");
+            const rows = await table.query()
+                .select(QUERY_COLUMNS)
+                .where(`id IN (${idList})`)
+                .toArray();
+            return rows.map((row) => ({
+                id: row.id,
+                filePath: row.filePath,
+                language: row.language,
+                startLine: row.startLine,
+                endLine: row.endLine,
+                content: row.content,
+                description: row.description ?? "",
+                kind: row.kind ?? "",
+                quirkType: row.quirkType ?? "",
+                tags: row.tags ?? "",
+            }));
+        });
+    }
     /**
      * Fetch chunks with their embedding vectors included (for embedding projection).
      * @param limit - Maximum number of rows to return.
@@ -614,8 +723,11 @@ export class LanceDbStore {
                 return 0;
             const table = await this.getTable();
             const COUNT_TIMEOUT_MS = 30_000;
+            // Attach a no-op catch to the race loser so a late rejection (after the
+            // timeout resolved `null`) cannot become an unhandled rejection.
+            const countPromise = table.countRows().catch(() => 0);
             const result = await Promise.race([
-                table.countRows(),
+                countPromise,
                 new Promise((resolve) => setTimeout(() => resolve(null), COUNT_TIMEOUT_MS)),
             ]);
             if (result === null) {
@@ -644,6 +756,21 @@ export class LanceDbStore {
                 // GC deleted fragments that the current version still referenced.
                 const threshold = new Date(Date.now() - 60 * 60 * 1000);
                 await table.optimize({ cleanupOlderThan: threshold, deleteUnverified: false });
+                // Build the ANN vector index — without it every vectorSearch() is a
+                // brute-force O(N) flat scan (slow at 50k+ chunks).
+                try {
+                    const count = await table.countRows().catch(() => 0);
+                    const numPartitions = count > 0 ? Math.max(16, Math.min(256, Math.floor(count / 256))) : 16;
+                    // The index metric MUST match the search metric ("cosine", see
+                    // searchInternal).  The ivfFlat default is "l2", which makes LanceDB
+                    // ignore the index and fall back to brute-force on every query.
+                    await table.createIndex("embedding", {
+                        config: lancedb.Index.ivfFlat({ numPartitions, distanceType: "cosine" }),
+                    });
+                }
+                catch {
+                    // Index creation is best-effort — queries still work, just slower.
+                }
             }
             catch {
                 // Optimize is best-effort �?" must not break indexing.
@@ -684,20 +811,39 @@ export class LanceDbStore {
      * @param newPath - Optional new filesystem path for the LanceDB database.
      */
     async reopen(newPath) {
-        await this.table?.close();
-        await this.db?.close();
-        this.table = null;
-        this.db = null;
-        this.tableInit = null;
+        await this.close();
         if (newPath)
             this.dbPath = newPath;
     }
-    /** Close the database connection and release resources. */
+    /**
+     * Close the database connection and release resources.
+     *
+     * The native LanceDB `close()` can hang indefinitely on Windows, so the
+     * whole close is raced against a timeout — callers (CLI commands, plugin
+     * reload, web server) must never block forever on shutdown.
+     */
     async close() {
-        await this.table?.close();
-        this.table = null;
-        await this.db?.close();
-        this.db = null;
+        await Promise.race([
+            (async () => {
+                try {
+                    await this.table?.close();
+                }
+                catch {
+                    // best-effort — the connection may already be gone
+                }
+                this.table = null;
+                try {
+                    await this.db?.close();
+                }
+                catch {
+                    // best-effort — the connection may already be gone
+                }
+                this.db = null;
+            })(),
+            new Promise((resolve) => {
+                setTimeout(resolve, 5000).unref();
+            }),
+        ]);
     }
     /**
      * Back up the chunks.lance directory before a destructive operation.

package/dist/vectorstore/memory.js

@@ -13,7 +13,11 @@ export class InMemoryVectorStore {
             .filter((c) => matchesFilter(c, filter))
             .map((chunk) => {
             const sim = cosineSimilarity(embedding, chunk.embedding);
-            return { chunk, score: sim };
+            // Normalize cosine [-1,1] into [0,1] so scores are comparable with
+            // the LanceDB store's clamped `1 - distance/2` domain (minScore and
+            // displayed scores mean the same thing across both stores).
+            const normalized = Math.min(1, Math.max(0, (sim + 1) / 2));
+            return { chunk, score: normalized };
         })
             .sort((a, b) => b.score - a.score)
             .slice(0, topK);

package/dist/watcher.js

@@ -37,7 +37,7 @@ export function createBackgroundIndexer(options) {
     const runPass = async (filterPaths) => {
         updateStatus({ running: true, lastRunAt: Date.now() });
         try {
-            await runIndexPass({
+            const stats = await runIndexPass({
                 cwd,
                 storePath,
                 config,
@@ -54,6 +54,20 @@ export function createBackgroundIndexer(options) {
                     debug: (message) => appendDebugLog(logFilePath, { scope: "autoIndex", message: `DEBUG: ${message}`, severity: "debug" }, logLevel),
                 },
             });
+            // A lock-skipped pass did NO work — retry shortly so the workspace
+            // does not stay unindexed until the next file event.
+            if (stats.skipped) {
+                appendDebugLog(logFilePath, {
+                    scope: "autoIndex",
+                    message: "Index pass skipped (another pass holds the lock) — retrying in 30s",
+                }, logLevel);
+                if (!ac.signal.aborted) {
+                    setTimeout(() => {
+                        if (!ac.signal.aborted)
+                            scheduler.notifyChange(filterPaths);
+                    }, 30_000).unref();
+                }
+            }
             updateStatus({ running: false, lastRunAt: Date.now() });
         }
         catch (err) {
@@ -106,21 +120,33 @@ export function createBackgroundIndexer(options) {
             error,
         }, logLevel);
     });
-    // Periodic timer: only needed for git backend (chokidar gets real FS events)
+    // Periodic timer: only needed for git backend (chokidar gets real FS events).
+    // Note: with git mode BOTH backends run — the scheduler coalesces redundant
+    // passes into one, so this only adds a safety net for missed events.
     const watcherBackend = autoIndexCfg.watcher ?? "chokidar";
     const periodicTimer = watcherBackend === "git"
         ? setInterval(() => {
             scheduler.notifyChange();
         }, autoIndexCfg.intervalMs)
         : undefined;
+    // Never keep the process alive just for the periodic scan
+    periodicTimer?.unref();
     return {
         async close() {
             if (periodicTimer)
                 clearInterval(periodicTimer);
             ac.abort();
             scheduler.close();
-            await scheduler.waitForIdle();
-            await watcher.close();
+            // chokidar close() can hang on some Windows setups — guard it like
+            // the CLI does.
+            await Promise.race([
+                scheduler.waitForIdle(),
+                new Promise((resolve) => setTimeout(resolve, 5000).unref()),
+            ]);
+            await Promise.race([
+                watcher.close(),
+                new Promise((resolve) => setTimeout(resolve, 5000).unref()),
+            ]);
             const statusPath = path.join(storePath, "watcher-status.json");
             if (existsSync(statusPath)) {
                 try {

package/dist/web/api.d.ts

@@ -5,7 +5,7 @@ import type { IncomingMessage, ServerResponse } from "node:http";
 import { LanceDbStore } from "../vectorstore/lancedb.js";
 import { KeywordIndex } from "../retriever/keyword-index.js";
 import type { RagConfig } from "../core/config.js";
-import type { EmbeddingProvider } from "../core/interfaces.js";
+import { type EmbeddingProvider } from "../core/interfaces.js";
 /** Internal shape for a JSON API response: an HTTP status code and a serialisable body. */
 interface ApiResponse {
     status: number;
@@ -28,7 +28,7 @@ interface ApiResponse {
  * @param cfg          - Active RAG configuration (used by quirk endpoints).
  * @returns An async handler that returns `true` when a route matched or `false` otherwise.
  */
-export declare function createApiHandler(store: LanceDbStore, keywordIndex: KeywordIndex, storePath: string, cwd?: string, cfg?: RagConfig, getEmbedder?: () => Promise<EmbeddingProvider>): (req: IncomingMessage, res: ServerResponse) => Promise<boolean>;
+export declare function createApiHandler(store: LanceDbStore, keywordIndex: KeywordIndex, storePath: string, cwd?: string, cfg?: RagConfig, getEmbedder?: () => Promise<EmbeddingProvider>, token?: string): (req: IncomingMessage, res: ServerResponse) => Promise<boolean>;
 /**
  * Perform token-usage analysis for a single evaluation session.
  *
@@ -52,4 +52,8 @@ export declare function handleEvalTokenCompare(storePath: string, params: URLSea
  * `avgReadsPerQueryWithRAG`, `queryCount`.
  */
 export declare function handleEvalProjectSavings(body: unknown): ApiResponse;
+/** Thrown when the request body exceeds {@link MAX_BODY_BYTES}; mapped to a 413 response. */
+export declare class BodyTooLargeError extends Error {
+    constructor();
+}
 export {};