opencode-rag-plugin 1.19.5 → 1.20.0

package/dist/vectorstore/lancedb.js

@@ -99,6 +99,14 @@ export class LanceDbStore {
     table = null;
     tableInit = null;
     writeLock = Promise.resolve(void 0);
+    /**
+     * Memoized once-per-process index-metric repair. Stores built by versions
+     * before the cosine search switch carry an IVF index trained with the
+     * default L2 metric, which makes every cosine query log
+     * "Requested metric Cosine is incompatible with index metric L2" and fall
+     * back to brute-force. This repairs that stale index on first search.
+     */
+    indexRepairPromise = null;
     /**
      * Execute an async function under an exclusive write lock.
      *
@@ -294,32 +302,65 @@ export class LanceDbStore {
     }
     /**
      * Store chunks in the LanceDB table. New rows are inserted first, then
-     * any old rows at the same (filePath, startLine) with different IDs are
-     * removed. This ensures no data is lost if the process aborts between
-     * insert and cleanup. Automatically attempts repair on corruption errors.
+     * old rows for the same file that are not part of this write are removed
+     * in a single delete per file. This ensures no data is lost if the process
+     * aborts between insert and cleanup. Automatically attempts repair on
+     * corruption errors.
+     *
+     * When `options.dedup` is `false` the cleanup step is skipped entirely —
+     * a pure append. Use this when writing into a store that provably has no
+     * prior rows for these files (e.g. a freshly-created rebuild store).
+     *
      * @param chunks - The chunks to add.
+     * @param options - Optional write options (`dedup`, default `true`).
      */
-    async addChunks(chunks) {
+    async addChunks(chunks, options) {
         if (chunks.length === 0)
             return;
+        const dedup = options?.dedup ?? true;
         await this.withWriteLock(async () => {
             try {
-                await this.addChunksInternal(chunks);
+                await this.addChunksInternal(chunks, dedup);
             }
             catch (err) {
                 if (isCorruptionError(err) && await this.tryRepair()) {
-                    await this.addChunksInternal(chunks);
+                    await this.addChunksInternal(chunks, dedup);
                     return;
                 }
                 throw err;
             }
         });
     }
-    async addChunksInternal(chunks) {
-        const table = await this.getTable();
-        const rows = chunks
-            .filter((c) => c.embedding && c.embedding.length > 0)
-            .map((c) => ({
+    /**
+     * Store chunks for many files in a single transaction: one `table.add`
+     * across all items, then one `table.delete` per item that needs dedup.
+     * This collapses what used to be a per-file add + per-startLine deletes
+     * (K+2 LanceDB versions per file) into ~1 + M versions per batch.
+     *
+     * @param items - Per-file chunk payloads with their dedup flags.
+     */
+    async addChunksBulk(items) {
+        const active = items.filter((item) => item.chunks.length > 0);
+        if (active.length === 0)
+            return;
+        await this.withWriteLock(async () => {
+            try {
+                await this.addChunksBulkInternal(active);
+            }
+            catch (err) {
+                if (isCorruptionError(err) && await this.tryRepair()) {
+                    await this.addChunksBulkInternal(active);
+                    return;
+                }
+                throw err;
+            }
+        });
+    }
+    /** Map a chunk to its internal row shape, or null if it has no embedding. */
+    chunkToRow(c) {
+        if (!c.embedding || c.embedding.length === 0)
+            return null;
+        return {
             id: c.id,
             content: c.content,
             description: c.description ?? "",
@@ -331,56 +372,75 @@ export class LanceDbStore {
             kind: c.metadata.kind ?? "",
             quirkType: c.metadata.quirkType ?? "",
             tags: c.metadata.tags ? JSON.stringify(c.metadata.tags) : "",
-        }));
-        if (rows.length === 0)
-            return;
-        // Build a map: (filePath, startLine) → set of new IDs for dedup after insert
-        const newIdsByLine = new Map();
+        };
+    }
+    /** Group new rows by file path for dedup deletes. */
+    rowsByFilePath(rows) {
+        const byFile = new Map();
         for (const row of rows) {
-            const key = `${row.filePath}:${row.startLine}`;
-            const ids = newIdsByLine.get(key);
+            const ids = byFile.get(row.filePath);
             if (ids) {
-                ids.add(row.id);
+                ids.push(row.id);
             }
             else {
-                newIdsByLine.set(key, new Set([row.id]));
+                byFile.set(row.filePath, [row.id]);
             }
         }
+        return byFile;
+    }
+    async addChunksInternal(chunks, dedup = true) {
+        const table = await this.getTable();
+        const rows = chunks
+            .map((c) => this.chunkToRow(c))
+            .filter((r) => r !== null);
+        if (rows.length === 0)
+            return;
         // INSERT FIRST: data is safely stored before any delete
         await table.add(rows);
-        // THEN DEDUP: remove old rows at the same (filePath, startLine) positions,
-        // but preserve the newly inserted rows by filtering out their IDs.
-        // IMPORTANT: use a single NOT IN clause per position so that when multiple
-        // new IDs share the same startLine they don't delete each other.
-        for (const [key, newIds] of newIdsByLine) {
-            const colonIdx = key.lastIndexOf(":");
-            const filePath = key.slice(0, colonIdx);
-            const startLine = parseInt(key.slice(colonIdx + 1), 10);
+        if (!dedup)
+            return;
+        // THEN DEDUP: one delete per file removes prior-revision rows at the
+        // same (filePath, startLine) positions AND stale startLines, while the
+        // NOT IN clause preserves the newly inserted rows (so multiple new IDs
+        // sharing a startLine never delete each other).  Insert-first ordering
+        // keeps an abort between insert and delete from losing data.
+        const byFile = this.rowsByFilePath(rows);
+        for (const [filePath, ids] of byFile) {
             const escapedPath = filePath.replace(/'/g, "''");
-            const idList = [...newIds]
-                .map((id) => `'${id.replace(/'/g, "''")}'`)
-                .join(", ");
-            await table.delete(`filePath = '${escapedPath}' AND startLine = ${startLine} AND id NOT IN (${idList})`);
+            const idList = ids.map((id) => `'${id.replace(/'/g, "''")}'`).join(", ");
+            await table.delete(`filePath = '${escapedPath}' AND id NOT IN (${idList})`);
         }
-        // FINALLY: remove stale chunks for the same file that belong to a
-        // previous revision (different startLines).  Exclude the new inserts
-        // so an abort never orphans data.
-        const filePathsDone = new Set();
-        for (const [key] of newIdsByLine) {
-            const colonIdx = key.lastIndexOf(":");
-            const filePath = key.slice(0, colonIdx);
-            if (filePathsDone.has(filePath))
+    }
+    async addChunksBulkInternal(items) {
+        const table = await this.getTable();
+        const allRows = [];
+        const dedupByFile = new Map();
+        for (const item of items) {
+            const rows = item.chunks
+                .map((c) => this.chunkToRow(c))
+                .filter((r) => r !== null);
+            if (rows.length === 0)
                 continue;
-            filePathsDone.add(filePath);
-            // Collect all new IDs inserted for this file
-            const fileNewIds = [];
-            for (const [k, ids] of newIdsByLine) {
-                if (k.startsWith(filePath + ":")) {
-                    fileNewIds.push(...ids);
+            allRows.push(...rows);
+            if (item.dedup) {
+                for (const [filePath, ids] of this.rowsByFilePath(rows)) {
+                    const existing = dedupByFile.get(filePath);
+                    if (existing) {
+                        existing.push(...ids);
+                    }
+                    else {
+                        dedupByFile.set(filePath, [...ids]);
+                    }
                 }
             }
+        }
+        if (allRows.length === 0)
+            return;
+        // INSERT FIRST (single add for the whole batch), then per-file dedup
+        await table.add(allRows);
+        for (const [filePath, ids] of dedupByFile) {
             const escapedPath = filePath.replace(/'/g, "''");
-            const idList = fileNewIds.map((id) => `'${id.replace(/'/g, "''")}'`).join(", ");
+            const idList = ids.map((id) => `'${id.replace(/'/g, "''")}'`).join(", ");
             await table.delete(`filePath = '${escapedPath}' AND id NOT IN (${idList})`);
         }
     }
@@ -684,6 +744,10 @@ export class LanceDbStore {
         const count = await table.countRows();
         if (count === 0)
             return [];
+        // One-time per-process repair: drop any stale L2 index so the cosine query
+        // below doesn't log "Requested metric Cosine is incompatible" and degrade
+        // to brute-force. Await it so the repair completes before this search.
+        await this.withWriteLock(() => this.ensureCosineIndex());
         const whereClause = buildWhereClause(filter);
         let results;
         try {
@@ -741,39 +805,103 @@ export class LanceDbStore {
         }
     }
     /**
-     * Compact fragments and prune old version manifests to prevent the
-     * version-manifest accumulation that causes countRows() to hang.
-     * Should be called at the end of a successful index pass.
+     * Ensure the ANN index on the `embedding` column uses the cosine metric,
+     * matching the `distanceType` requested by searchInternal.
+     *
+     * Older stores built the IVF index with the ivfFlat default (L2), so every
+     * cosine query hit "Requested metric Cosine is incompatible with index
+     * metric L2" and silently fell back to brute-force O(N) scans. This lazily
+     * replaces such an index with a cosine one — once per process.
+     *
+     * Callers must hold the write lock (searchInternal wraps the call in
+     * withWriteLock; optimize runs under it already).
      */
-    async optimize() {
-        await this.withWriteLock(async () => {
-            try {
-                const table = await this.getTable();
-                // Clean up versions older than 1 hour �?" not "right now" �?" so in-flight
-                // queries (e.g. Web UI search, background auto-index) can finish before
-                // their data files are reclaimed.  Using new Date() here caused data-file
-                // race conditions where a reader got "Not found: �?� .lance" because the
-                // GC deleted fragments that the current version still referenced.
-                const threshold = new Date(Date.now() - 60 * 60 * 1000);
-                await table.optimize({ cleanupOlderThan: threshold, deleteUnverified: false });
-                // Build the ANN vector index — without it every vectorSearch() is a
-                // brute-force O(N) flat scan (slow at 50k+ chunks).
+    ensureCosineIndex() {
+        if (!this.indexRepairPromise) {
+            this.indexRepairPromise = this.repairIndexMetricOnce();
+        }
+        return this.indexRepairPromise;
+    }
+    /**
+     * Perform a single index-metric repair pass. Skips stores that have no
+     * index and fewer than 1000 rows (brute-force is optimal there). Uses a
+     * single `createIndex` with `replace: true` — a dropIndex + createIndex
+     * sequence races in LanceDB ("Retryable commit conflict") and leaves the
+     * stale index in place. On failure the memo is cleared so the next
+     * search/optimize retries.
+     */
+    async repairIndexMetricOnce() {
+        try {
+            const table = await this.getTable();
+            const count = await table.countRows().catch(() => 0);
+            const indices = await table.listIndices();
+            const vecIndex = indices.find((i) => i.columns.includes("embedding"));
+            const idxName = vecIndex?.name ?? "embedding_idx";
+            const stats = await table.indexStats(idxName);
+            if (stats && stats.distanceType === "cosine")
+                return; // already healthy
+            if (!stats && count < 1000)
+                return; // tiny store, no index — leave brute-force
+            const numPartitions = Math.max(16, Math.min(256, Math.floor(count / 256)));
+            for (let attempt = 1; attempt <= 3; attempt++) {
                 try {
-                    const count = await table.countRows().catch(() => 0);
-                    const numPartitions = count > 0 ? Math.max(16, Math.min(256, Math.floor(count / 256))) : 16;
-                    // The index metric MUST match the search metric ("cosine", see
-                    // searchInternal).  The ivfFlat default is "l2", which makes LanceDB
-                    // ignore the index and fall back to brute-force on every query.
                     await table.createIndex("embedding", {
                         config: lancedb.Index.ivfFlat({ numPartitions, distanceType: "cosine" }),
+                        replace: true,
+                        name: idxName,
+                        // Wait for the commit so the repair is deterministic and cannot
+                        // race a background index-creation from another process.
+                        waitTimeoutSeconds: 120,
                     });
+                    return;
                 }
-                catch {
-                    // Index creation is best-effort — queries still work, just slower.
+                catch (err) {
+                    const retriable = err instanceof Error &&
+                        (err.message.includes("Retryable commit conflict") || err.message.includes("Please retry"));
+                    if (!retriable || attempt === 3)
+                        throw err;
+                    await new Promise((resolve) => setTimeout(resolve, 150 * attempt));
                 }
             }
-            catch {
-                // Optimize is best-effort �?" must not break indexing.
+        }
+        catch (err) {
+            this.indexRepairPromise = null; // allow retry on a later search/optimize
+            console.warn(`[lancedb] index metric repair failed (queries continue in brute-force): ` +
+                `${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    /**
+     * Compact fragments and prune old version manifests to prevent the
+     * version-manifest accumulation that causes countRows() to hang and the
+     * store phase to slow down as the index grows. Should be called at the
+     * end of a successful index pass, and periodically during long passes.
+     *
+     * @param options - `aggressive: true` prunes every version but the current
+     *   one with `deleteUnverified`. Only safe for a private store that no other
+     *   process reads (e.g. a temporary rebuild store). For the shared store,
+     *   versions newer than 1 hour are retained so in-flight queries (Web UI,
+     *   background auto-index) can finish before their data files are reclaimed.
+     */
+    async optimize(options) {
+        await this.withWriteLock(async () => {
+            try {
+                const table = await this.getTable();
+                if (options?.aggressive) {
+                    await table.optimize({ cleanupOlderThan: new Date(), deleteUnverified: true });
+                }
+                else {
+                    const threshold = new Date(Date.now() - 60 * 60 * 1000);
+                    await table.optimize({ cleanupOlderThan: threshold, deleteUnverified: false });
+                }
+                // Build the ANN vector index — without it every vectorSearch() is a
+                // brute-force O(N) flat scan (slow at 50k+ chunks). Skips early when
+                // an index with the correct cosine metric already exists.
+                await this.ensureCosineIndex();
+            }
+            catch (err) {
+                // Optimize is best-effort — must not break indexing, but surface the
+                // failure instead of swallowing it silently.
+                console.warn(`[lancedb] optimize failed: ${err instanceof Error ? err.message : String(err)}`);
             }
         });
     }

package/dist/vectorstore/memory.d.ts

@@ -1,11 +1,14 @@
 /**
  * @fileoverview Ephemeral in-memory vector store using cosine similarity search.
  */
-import type { VectorStore, Chunk, ChunkSummary, FileSummary, SearchResult, MetadataFilter } from "../core/interfaces.js";
+import type { VectorStore, Chunk, ChunkSummary, FileSummary, SearchResult, MetadataFilter, BulkChunkWrite } from "../core/interfaces.js";
 /** Ephemeral in-memory vector store using cosine similarity search. */
 export declare class InMemoryVectorStore implements VectorStore {
     private chunks;
-    addChunks(chunks: Chunk[]): Promise<void>;
+    addChunks(chunks: Chunk[], options?: {
+        dedup?: boolean;
+    }): Promise<void>;
+    addChunksBulk(items: BulkChunkWrite[]): Promise<void>;
     search(embedding: number[], topK: number): Promise<SearchResult[]>;
     searchWithFilter(embedding: number[], topK: number, filter?: MetadataFilter): Promise<SearchResult[]>;
     count(): Promise<number>;
@@ -22,5 +25,7 @@ export declare class InMemoryVectorStore implements VectorStore {
     /** Release any held resources. No-op for the in-memory store. */
     close(): Promise<void>;
     /** No-op for the in-memory store. */
-    optimize(): Promise<void>;
+    optimize(_options?: {
+        aggressive?: boolean;
+    }): Promise<void>;
 }

package/dist/vectorstore/memory.js

@@ -1,8 +1,27 @@
 /** Ephemeral in-memory vector store using cosine similarity search. */
 export class InMemoryVectorStore {
     chunks = [];
-    async addChunks(chunks) {
-        this.chunks.push(...chunks.filter((c) => c.embedding && c.embedding.length > 0));
+    async addChunks(chunks, options) {
+        const valid = chunks.filter((c) => c.embedding && c.embedding.length > 0);
+        if (valid.length === 0)
+            return;
+        if (options?.dedup === false) {
+            this.chunks.push(...valid);
+            return;
+        }
+        // Mirror LanceDB dedup semantics: keep existing rows for touched files
+        // that are part of this write; drop prior-revision rows that aren't.
+        const newIds = new Set(valid.map((c) => c.id));
+        const newPaths = new Set(valid.map((c) => c.metadata.filePath));
+        this.chunks = [
+            ...this.chunks.filter((c) => !newPaths.has(c.metadata.filePath) || newIds.has(c.id)),
+            ...valid,
+        ];
+    }
+    async addChunksBulk(items) {
+        for (const item of items) {
+            await this.addChunks(item.chunks, { dedup: item.dedup });
+        }
     }
     async search(embedding, topK) {
         return this.searchWithFilter(embedding, topK);
@@ -78,7 +97,7 @@ export class InMemoryVectorStore {
     async close() {
     }
     /** No-op for the in-memory store. */
-    async optimize() {
+    async optimize(_options) {
     }
 }
 /**

package/dist/watcher.d.ts

@@ -40,6 +40,14 @@ export type WatcherStatus = {
     /** Timestamp (ms since epoch) of the last completed run, or undefined. */
     lastRunAt: number | undefined;
 };
+/**
+ * Atomically claim the watcher lock for a workspace. Returns true only if
+ * this process is now the active watcher. A stale lock (dead PID) or an
+ * unreadable/corrupt lock file is cleared and re-claimed.
+ */
+export declare function tryAcquireWatcherLock(storePath: string): boolean;
+/** Release the watcher lock — only if this process owns it. */
+export declare function releaseWatcherLock(storePath: string): void;
 /**
  * Create a background file watcher that automatically re-indexes the
  * workspace when files change. Uses chokidar for file system events and