opencode-rag-plugin 1.19.4 → 1.19.8

package/dist/vectorstore/lancedb.js

@@ -99,6 +99,14 @@ export class LanceDbStore {
     table = null;
     tableInit = null;
     writeLock = Promise.resolve(void 0);
+    /**
+     * Memoized once-per-process index-metric repair. Stores built by versions
+     * before the cosine search switch carry an IVF index trained with the
+     * default L2 metric, which makes every cosine query log
+     * "Requested metric Cosine is incompatible with index metric L2" and fall
+     * back to brute-force. This repairs that stale index on first search.
+     */
+    indexRepairPromise = null;
     /**
      * Execute an async function under an exclusive write lock.
      *
@@ -294,32 +302,65 @@ export class LanceDbStore {
     }
     /**
      * Store chunks in the LanceDB table. New rows are inserted first, then
-     * any old rows at the same (filePath, startLine) with different IDs are
-     * removed. This ensures no data is lost if the process aborts between
-     * insert and cleanup. Automatically attempts repair on corruption errors.
+     * old rows for the same file that are not part of this write are removed
+     * in a single delete per file. This ensures no data is lost if the process
+     * aborts between insert and cleanup. Automatically attempts repair on
+     * corruption errors.
+     *
+     * When `options.dedup` is `false` the cleanup step is skipped entirely —
+     * a pure append. Use this when writing into a store that provably has no
+     * prior rows for these files (e.g. a freshly-created rebuild store).
+     *
      * @param chunks - The chunks to add.
+     * @param options - Optional write options (`dedup`, default `true`).
      */
-    async addChunks(chunks) {
+    async addChunks(chunks, options) {
         if (chunks.length === 0)
             return;
+        const dedup = options?.dedup ?? true;
         await this.withWriteLock(async () => {
             try {
-                await this.addChunksInternal(chunks);
+                await this.addChunksInternal(chunks, dedup);
             }
             catch (err) {
                 if (isCorruptionError(err) && await this.tryRepair()) {
-                    await this.addChunksInternal(chunks);
+                    await this.addChunksInternal(chunks, dedup);
                     return;
                 }
                 throw err;
             }
         });
     }
-    async addChunksInternal(chunks) {
-        const table = await this.getTable();
-        const rows = chunks
-            .filter((c) => c.embedding && c.embedding.length > 0)
-            .map((c) => ({
+    /**
+     * Store chunks for many files in a single transaction: one `table.add`
+     * across all items, then one `table.delete` per item that needs dedup.
+     * This collapses what used to be a per-file add + per-startLine deletes
+     * (K+2 LanceDB versions per file) into ~1 + M versions per batch.
+     *
+     * @param items - Per-file chunk payloads with their dedup flags.
+     */
+    async addChunksBulk(items) {
+        const active = items.filter((item) => item.chunks.length > 0);
+        if (active.length === 0)
+            return;
+        await this.withWriteLock(async () => {
+            try {
+                await this.addChunksBulkInternal(active);
+            }
+            catch (err) {
+                if (isCorruptionError(err) && await this.tryRepair()) {
+                    await this.addChunksBulkInternal(active);
+                    return;
+                }
+                throw err;
+            }
+        });
+    }
+    /** Map a chunk to its internal row shape, or null if it has no embedding. */
+    chunkToRow(c) {
+        if (!c.embedding || c.embedding.length === 0)
+            return null;
+        return {
             id: c.id,
             content: c.content,
             description: c.description ?? "",
@@ -331,56 +372,75 @@ export class LanceDbStore {
             kind: c.metadata.kind ?? "",
             quirkType: c.metadata.quirkType ?? "",
             tags: c.metadata.tags ? JSON.stringify(c.metadata.tags) : "",
-        }));
-        if (rows.length === 0)
-            return;
-        // Build a map: (filePath, startLine) → set of new IDs for dedup after insert
-        const newIdsByLine = new Map();
+        };
+    }
+    /** Group new rows by file path for dedup deletes. */
+    rowsByFilePath(rows) {
+        const byFile = new Map();
         for (const row of rows) {
-            const key = `${row.filePath}:${row.startLine}`;
-            const ids = newIdsByLine.get(key);
+            const ids = byFile.get(row.filePath);
             if (ids) {
-                ids.add(row.id);
+                ids.push(row.id);
             }
             else {
-                newIdsByLine.set(key, new Set([row.id]));
+                byFile.set(row.filePath, [row.id]);
             }
         }
+        return byFile;
+    }
+    async addChunksInternal(chunks, dedup = true) {
+        const table = await this.getTable();
+        const rows = chunks
+            .map((c) => this.chunkToRow(c))
+            .filter((r) => r !== null);
+        if (rows.length === 0)
+            return;
         // INSERT FIRST: data is safely stored before any delete
         await table.add(rows);
-        // THEN DEDUP: remove old rows at the same (filePath, startLine) positions,
-        // but preserve the newly inserted rows by filtering out their IDs.
-        // IMPORTANT: use a single NOT IN clause per position so that when multiple
-        // new IDs share the same startLine they don't delete each other.
-        for (const [key, newIds] of newIdsByLine) {
-            const colonIdx = key.lastIndexOf(":");
-            const filePath = key.slice(0, colonIdx);
-            const startLine = parseInt(key.slice(colonIdx + 1), 10);
+        if (!dedup)
+            return;
+        // THEN DEDUP: one delete per file removes prior-revision rows at the
+        // same (filePath, startLine) positions AND stale startLines, while the
+        // NOT IN clause preserves the newly inserted rows (so multiple new IDs
+        // sharing a startLine never delete each other).  Insert-first ordering
+        // keeps an abort between insert and delete from losing data.
+        const byFile = this.rowsByFilePath(rows);
+        for (const [filePath, ids] of byFile) {
             const escapedPath = filePath.replace(/'/g, "''");
-            const idList = [...newIds]
-                .map((id) => `'${id.replace(/'/g, "''")}'`)
-                .join(", ");
-            await table.delete(`filePath = '${escapedPath}' AND startLine = ${startLine} AND id NOT IN (${idList})`);
-        }
-        // FINALLY: remove stale chunks for the same file that belong to a
-        // previous revision (different startLines).  Exclude the new inserts
-        // so an abort never orphans data.
-        const filePathsDone = new Set();
-        for (const [key] of newIdsByLine) {
-            const colonIdx = key.lastIndexOf(":");
-            const filePath = key.slice(0, colonIdx);
-            if (filePathsDone.has(filePath))
+            const idList = ids.map((id) => `'${id.replace(/'/g, "''")}'`).join(", ");
+            await table.delete(`filePath = '${escapedPath}' AND id NOT IN (${idList})`);
+        }
+    }
+    async addChunksBulkInternal(items) {
+        const table = await this.getTable();
+        const allRows = [];
+        const dedupByFile = new Map();
+        for (const item of items) {
+            const rows = item.chunks
+                .map((c) => this.chunkToRow(c))
+                .filter((r) => r !== null);
+            if (rows.length === 0)
                 continue;
-            filePathsDone.add(filePath);
-            // Collect all new IDs inserted for this file
-            const fileNewIds = [];
-            for (const [k, ids] of newIdsByLine) {
-                if (k.startsWith(filePath + ":")) {
-                    fileNewIds.push(...ids);
+            allRows.push(...rows);
+            if (item.dedup) {
+                for (const [filePath, ids] of this.rowsByFilePath(rows)) {
+                    const existing = dedupByFile.get(filePath);
+                    if (existing) {
+                        existing.push(...ids);
+                    }
+                    else {
+                        dedupByFile.set(filePath, [...ids]);
+                    }
                 }
             }
+        }
+        if (allRows.length === 0)
+            return;
+        // INSERT FIRST (single add for the whole batch), then per-file dedup
+        await table.add(allRows);
+        for (const [filePath, ids] of dedupByFile) {
             const escapedPath = filePath.replace(/'/g, "''");
-            const idList = fileNewIds.map((id) => `'${id.replace(/'/g, "''")}'`).join(", ");
+            const idList = ids.map((id) => `'${id.replace(/'/g, "''")}'`).join(", ");
             await table.delete(`filePath = '${escapedPath}' AND id NOT IN (${idList})`);
         }
     }
@@ -396,6 +456,12 @@ export class LanceDbStore {
     }
     async searchWithFilter(embedding, topK, filter) {
         try {
+            // Guard against dimension mismatch BEFORE the native call — LanceDB
+            // throws a cryptic error that used to be swallowed into "no results".
+            if (embedding.length !== this.vectorDimension) {
+                console.warn(`[lancedb] searchWithFilter: query embedding dimension ${embedding.length} != store dimension ${this.vectorDimension} — returning empty`);
+                return [];
+            }
             return await this.searchInternal(embedding, topK, filter);
         }
         catch (err) {
@@ -404,7 +470,12 @@ export class LanceDbStore {
                 if (repaired) {
                     return this.searchInternal(embedding, topK, filter);
                 }
+                console.warn(`[lancedb] searchWithFilter: repair failed: ${err.message}`);
+                return [];
             }
+            // Never silently mask non-corruption failures as "no results" —
+            // that hides provider outages and store bugs from every caller.
+            console.warn(`[lancedb] searchWithFilter failed (returning []): ${err.message}`);
             return [];
         }
     }
@@ -534,6 +605,104 @@ export class LanceDbStore {
             }));
         });
     }
+    /**
+     * Retrieve a paginated list of chunks without embeddings, with optional
+     * language and file-path filters pushed into the SQL WHERE clause.
+     *
+     * @param offset - Number of rows to skip (for pagination).
+     * @param limit - Maximum number of rows to return.
+     * @param lang - Optional exact language filter.
+     * @param filePrefix - Optional filePath prefix filter.
+     * @returns The page of chunks plus the total count of matching rows.
+     */
+    async getChunksFiltered(offset, limit, lang, filePrefix) {
+        const conditions = [];
+        if (lang)
+            conditions.push(`language = '${lang.replace(/'/g, "''")}'`);
+        if (filePrefix)
+            conditions.push(`filePath LIKE '${filePrefix.replace(/'/g, "''")}%'`);
+        const where = conditions.length > 0 ? conditions.join(" AND ") : undefined;
+        return this.withCorruptionRecovery(async () => {
+            const table = await this.getTable();
+            let query = table.query().select(QUERY_COLUMNS);
+            let countQuery = table.query().select(["id"]);
+            if (where) {
+                query = query.where(where);
+                countQuery = countQuery.where(where);
+            }
+            const [rows, countRows] = await Promise.all([
+                query.offset(Math.max(0, offset)).limit(Math.max(1, Math.min(limit, 1000))).toArray(),
+                countQuery.toArray(),
+            ]);
+            return {
+                chunks: rows.map((row) => ({
+                    id: row.id,
+                    filePath: row.filePath,
+                    language: row.language,
+                    startLine: row.startLine,
+                    endLine: row.endLine,
+                    content: row.content,
+                    description: row.description ?? "",
+                    kind: row.kind ?? "",
+                    quirkType: row.quirkType ?? "",
+                    tags: row.tags ?? "",
+                })),
+                total: countRows.length,
+            };
+        });
+    }
+    /** Look up a single chunk by its ID. Returns undefined when not found. */
+    async getChunkById(id) {
+        return this.withCorruptionRecovery(async () => {
+            const table = await this.getTable();
+            const rows = await table.query()
+                .select(QUERY_COLUMNS)
+                .where(`id = '${id.replace(/'/g, "''")}'`)
+                .limit(1)
+                .toArray();
+            const row = rows[0];
+            if (!row)
+                return undefined;
+            return {
+                id: row.id,
+                filePath: row.filePath,
+                language: row.language,
+                startLine: row.startLine,
+                endLine: row.endLine,
+                content: row.content,
+                description: row.description ?? "",
+                kind: row.kind ?? "",
+                quirkType: row.quirkType ?? "",
+                tags: row.tags ?? "",
+            };
+        });
+    }
+    /** Fetch chunks by their IDs (for the compare view). Bounded to 100 ids. */
+    async getChunksByIds(ids) {
+        const bounded = ids.slice(0, 100);
+        if (bounded.length === 0)
+            return [];
+        return this.withCorruptionRecovery(async () => {
+            const table = await this.getTable();
+            const idList = bounded.map((id) => `'${id.replace(/'/g, "''")}'`).join(", ");
+            const rows = await table.query()
+                .select(QUERY_COLUMNS)
+                .where(`id IN (${idList})`)
+                .toArray();
+            return rows.map((row) => ({
+                id: row.id,
+                filePath: row.filePath,
+                language: row.language,
+                startLine: row.startLine,
+                endLine: row.endLine,
+                content: row.content,
+                description: row.description ?? "",
+                kind: row.kind ?? "",
+                quirkType: row.quirkType ?? "",
+                tags: row.tags ?? "",
+            }));
+        });
+    }
     /**
      * Fetch chunks with their embedding vectors included (for embedding projection).
      * @param limit - Maximum number of rows to return.
@@ -575,6 +744,10 @@ export class LanceDbStore {
         const count = await table.countRows();
         if (count === 0)
             return [];
+        // One-time per-process repair: drop any stale L2 index so the cosine query
+        // below doesn't log "Requested metric Cosine is incompatible" and degrade
+        // to brute-force. Await it so the repair completes before this search.
+        await this.withWriteLock(() => this.ensureCosineIndex());
         const whereClause = buildWhereClause(filter);
         let results;
         try {
@@ -614,8 +787,11 @@ export class LanceDbStore {
                 return 0;
             const table = await this.getTable();
             const COUNT_TIMEOUT_MS = 30_000;
+            // Attach a no-op catch to the race loser so a late rejection (after the
+            // timeout resolved `null`) cannot become an unhandled rejection.
+            const countPromise = table.countRows().catch(() => 0);
             const result = await Promise.race([
-                table.countRows(),
+                countPromise,
                 new Promise((resolve) => setTimeout(() => resolve(null), COUNT_TIMEOUT_MS)),
             ]);
             if (result === null) {
@@ -628,25 +804,104 @@ export class LanceDbStore {
             return 0;
         }
     }
+    /**
+     * Ensure the ANN index on the `embedding` column uses the cosine metric,
+     * matching the `distanceType` requested by searchInternal.
+     *
+     * Older stores built the IVF index with the ivfFlat default (L2), so every
+     * cosine query hit "Requested metric Cosine is incompatible with index
+     * metric L2" and silently fell back to brute-force O(N) scans. This lazily
+     * replaces such an index with a cosine one — once per process.
+     *
+     * Callers must hold the write lock (searchInternal wraps the call in
+     * withWriteLock; optimize runs under it already).
+     */
+    ensureCosineIndex() {
+        if (!this.indexRepairPromise) {
+            this.indexRepairPromise = this.repairIndexMetricOnce();
+        }
+        return this.indexRepairPromise;
+    }
+    /**
+     * Perform a single index-metric repair pass. Skips stores that have no
+     * index and fewer than 1000 rows (brute-force is optimal there). Uses a
+     * single `createIndex` with `replace: true` — a dropIndex + createIndex
+     * sequence races in LanceDB ("Retryable commit conflict") and leaves the
+     * stale index in place. On failure the memo is cleared so the next
+     * search/optimize retries.
+     */
+    async repairIndexMetricOnce() {
+        try {
+            const table = await this.getTable();
+            const count = await table.countRows().catch(() => 0);
+            const indices = await table.listIndices();
+            const vecIndex = indices.find((i) => i.columns.includes("embedding"));
+            const idxName = vecIndex?.name ?? "embedding_idx";
+            const stats = await table.indexStats(idxName);
+            if (stats && stats.distanceType === "cosine")
+                return; // already healthy
+            if (!stats && count < 1000)
+                return; // tiny store, no index — leave brute-force
+            const numPartitions = Math.max(16, Math.min(256, Math.floor(count / 256)));
+            for (let attempt = 1; attempt <= 3; attempt++) {
+                try {
+                    await table.createIndex("embedding", {
+                        config: lancedb.Index.ivfFlat({ numPartitions, distanceType: "cosine" }),
+                        replace: true,
+                        name: idxName,
+                        // Wait for the commit so the repair is deterministic and cannot
+                        // race a background index-creation from another process.
+                        waitTimeoutSeconds: 120,
+                    });
+                    return;
+                }
+                catch (err) {
+                    const retriable = err instanceof Error &&
+                        (err.message.includes("Retryable commit conflict") || err.message.includes("Please retry"));
+                    if (!retriable || attempt === 3)
+                        throw err;
+                    await new Promise((resolve) => setTimeout(resolve, 150 * attempt));
+                }
+            }
+        }
+        catch (err) {
+            this.indexRepairPromise = null; // allow retry on a later search/optimize
+            console.warn(`[lancedb] index metric repair failed (queries continue in brute-force): ` +
+                `${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
     /**
      * Compact fragments and prune old version manifests to prevent the
-     * version-manifest accumulation that causes countRows() to hang.
-     * Should be called at the end of a successful index pass.
+     * version-manifest accumulation that causes countRows() to hang and the
+     * store phase to slow down as the index grows. Should be called at the
+     * end of a successful index pass, and periodically during long passes.
+     *
+     * @param options - `aggressive: true` prunes every version but the current
+     *   one with `deleteUnverified`. Only safe for a private store that no other
+     *   process reads (e.g. a temporary rebuild store). For the shared store,
+     *   versions newer than 1 hour are retained so in-flight queries (Web UI,
+     *   background auto-index) can finish before their data files are reclaimed.
      */
-    async optimize() {
+    async optimize(options) {
         await this.withWriteLock(async () => {
             try {
                 const table = await this.getTable();
-                // Clean up versions older than 1 hour �?" not "right now" �?" so in-flight
-                // queries (e.g. Web UI search, background auto-index) can finish before
-                // their data files are reclaimed.  Using new Date() here caused data-file
-                // race conditions where a reader got "Not found: �?� .lance" because the
-                // GC deleted fragments that the current version still referenced.
-                const threshold = new Date(Date.now() - 60 * 60 * 1000);
-                await table.optimize({ cleanupOlderThan: threshold, deleteUnverified: false });
+                if (options?.aggressive) {
+                    await table.optimize({ cleanupOlderThan: new Date(), deleteUnverified: true });
+                }
+                else {
+                    const threshold = new Date(Date.now() - 60 * 60 * 1000);
+                    await table.optimize({ cleanupOlderThan: threshold, deleteUnverified: false });
+                }
+                // Build the ANN vector index — without it every vectorSearch() is a
+                // brute-force O(N) flat scan (slow at 50k+ chunks). Skips early when
+                // an index with the correct cosine metric already exists.
+                await this.ensureCosineIndex();
             }
-            catch {
-                // Optimize is best-effort �?" must not break indexing.
+            catch (err) {
+                // Optimize is best-effort — must not break indexing, but surface the
+                // failure instead of swallowing it silently.
+                console.warn(`[lancedb] optimize failed: ${err instanceof Error ? err.message : String(err)}`);
             }
         });
     }
@@ -684,20 +939,39 @@ export class LanceDbStore {
      * @param newPath - Optional new filesystem path for the LanceDB database.
      */
     async reopen(newPath) {
-        await this.table?.close();
-        await this.db?.close();
-        this.table = null;
-        this.db = null;
-        this.tableInit = null;
+        await this.close();
         if (newPath)
             this.dbPath = newPath;
     }
-    /** Close the database connection and release resources. */
+    /**
+     * Close the database connection and release resources.
+     *
+     * The native LanceDB `close()` can hang indefinitely on Windows, so the
+     * whole close is raced against a timeout — callers (CLI commands, plugin
+     * reload, web server) must never block forever on shutdown.
+     */
     async close() {
-        await this.table?.close();
-        this.table = null;
-        await this.db?.close();
-        this.db = null;
+        await Promise.race([
+            (async () => {
+                try {
+                    await this.table?.close();
+                }
+                catch {
+                    // best-effort — the connection may already be gone
+                }
+                this.table = null;
+                try {
+                    await this.db?.close();
+                }
+                catch {
+                    // best-effort — the connection may already be gone
+                }
+                this.db = null;
+            })(),
+            new Promise((resolve) => {
+                setTimeout(resolve, 5000).unref();
+            }),
+        ]);
     }
     /**
      * Back up the chunks.lance directory before a destructive operation.

package/dist/vectorstore/memory.d.ts

@@ -1,11 +1,14 @@
 /**
  * @fileoverview Ephemeral in-memory vector store using cosine similarity search.
  */
-import type { VectorStore, Chunk, ChunkSummary, FileSummary, SearchResult, MetadataFilter } from "../core/interfaces.js";
+import type { VectorStore, Chunk, ChunkSummary, FileSummary, SearchResult, MetadataFilter, BulkChunkWrite } from "../core/interfaces.js";
 /** Ephemeral in-memory vector store using cosine similarity search. */
 export declare class InMemoryVectorStore implements VectorStore {
     private chunks;
-    addChunks(chunks: Chunk[]): Promise<void>;
+    addChunks(chunks: Chunk[], options?: {
+        dedup?: boolean;
+    }): Promise<void>;
+    addChunksBulk(items: BulkChunkWrite[]): Promise<void>;
     search(embedding: number[], topK: number): Promise<SearchResult[]>;
     searchWithFilter(embedding: number[], topK: number, filter?: MetadataFilter): Promise<SearchResult[]>;
     count(): Promise<number>;
@@ -22,5 +25,7 @@ export declare class InMemoryVectorStore implements VectorStore {
     /** Release any held resources. No-op for the in-memory store. */
     close(): Promise<void>;
     /** No-op for the in-memory store. */
-    optimize(): Promise<void>;
+    optimize(_options?: {
+        aggressive?: boolean;
+    }): Promise<void>;
 }

package/dist/vectorstore/memory.js

@@ -1,8 +1,27 @@
 /** Ephemeral in-memory vector store using cosine similarity search. */
 export class InMemoryVectorStore {
     chunks = [];
-    async addChunks(chunks) {
-        this.chunks.push(...chunks.filter((c) => c.embedding && c.embedding.length > 0));
+    async addChunks(chunks, options) {
+        const valid = chunks.filter((c) => c.embedding && c.embedding.length > 0);
+        if (valid.length === 0)
+            return;
+        if (options?.dedup === false) {
+            this.chunks.push(...valid);
+            return;
+        }
+        // Mirror LanceDB dedup semantics: keep existing rows for touched files
+        // that are part of this write; drop prior-revision rows that aren't.
+        const newIds = new Set(valid.map((c) => c.id));
+        const newPaths = new Set(valid.map((c) => c.metadata.filePath));
+        this.chunks = [
+            ...this.chunks.filter((c) => !newPaths.has(c.metadata.filePath) || newIds.has(c.id)),
+            ...valid,
+        ];
+    }
+    async addChunksBulk(items) {
+        for (const item of items) {
+            await this.addChunks(item.chunks, { dedup: item.dedup });
+        }
     }
     async search(embedding, topK) {
         return this.searchWithFilter(embedding, topK);
@@ -13,7 +32,11 @@ export class InMemoryVectorStore {
             .filter((c) => matchesFilter(c, filter))
             .map((chunk) => {
             const sim = cosineSimilarity(embedding, chunk.embedding);
-            return { chunk, score: sim };
+            // Normalize cosine [-1,1] into [0,1] so scores are comparable with
+            // the LanceDB store's clamped `1 - distance/2` domain (minScore and
+            // displayed scores mean the same thing across both stores).
+            const normalized = Math.min(1, Math.max(0, (sim + 1) / 2));
+            return { chunk, score: normalized };
         })
             .sort((a, b) => b.score - a.score)
             .slice(0, topK);
@@ -74,7 +97,7 @@ export class InMemoryVectorStore {
     async close() {
     }
     /** No-op for the in-memory store. */
-    async optimize() {
+    async optimize(_options) {
     }
 }
 /**

package/dist/watcher.d.ts

@@ -40,6 +40,14 @@ export type WatcherStatus = {
     /** Timestamp (ms since epoch) of the last completed run, or undefined. */
     lastRunAt: number | undefined;
 };
+/**
+ * Atomically claim the watcher lock for a workspace. Returns true only if
+ * this process is now the active watcher. A stale lock (dead PID) or an
+ * unreadable/corrupt lock file is cleared and re-claimed.
+ */
+export declare function tryAcquireWatcherLock(storePath: string): boolean;
+/** Release the watcher lock — only if this process owns it. */
+export declare function releaseWatcherLock(storePath: string): void;
 /**
  * Create a background file watcher that automatically re-indexes the
  * workspace when files change. Uses chokidar for file system events and