opencode-rag-plugin 1.19.5 → 1.19.8

package/dist/indexer/pipeline.js

@@ -382,19 +382,176 @@ async function runIndexPassInner(options, logger) {
             options.logger?.warn?.(`Failed to save manifest: ${err.message}`);
         }));
     }
+    /**
+     * Store one window's prepared files into the vector store and update the
+     * manifest. Runs as a standalone async step so the caller can overlap it
+     * with the next window's prepare/describe/embed phases (the store phase is
+     * I/O-bound; the embed phase is GPU/API-bound).
+     *
+     * All chunks in the window are written in ONE bulk transaction (a single
+     * table.add plus one delete per modified file) instead of per-file adds
+     * plus per-startLine deletes.  Per-file transactions caused LanceDB
+     * version-manifest accumulation (K+2 versions per file) that made the
+     * store phase degrade quadratically as the index grew.
+     */
+    async function storeWindow(prepared, windowEarlyResults) {
+        const filesToStore = prepared.filter((p) => !windowEarlyResults.has(prepared.indexOf(p)) && p.chunks && (p.textToEmbed?.length ?? 0) > 0).length;
+        const storePhaseStart = Date.now();
+        logger.info(`Store phase: storing ${filesToStore} file(s) into vector database...`);
+        let storedFiles = 0;
+        // Compute per-file store payloads first (pure computation, no I/O).
+        const storePayloads = [];
+        for (const prep of prepared) {
+            if (aborted())
+                break;
+            if (windowEarlyResults.has(prepared.indexOf(prep)))
+                continue;
+            if (!prep.chunks || prep.textToEmbed?.length === 0)
+                continue;
+            const validChunks = (prep.chunks ?? []).filter((c) => c.embedding && c.embedding.length > 0);
+            // Partial embed failure: some chunks have no vector. Do NOT record the
+            // file as complete — keep the previous manifest entry (or none, for
+            // new files) so the next pass retries the missing chunks.
+            const allEmbedded = validChunks.length === (prep.chunks?.length ?? 0);
+            if (!allEmbedded && (prep.chunks?.length ?? 0) > 0) {
+                options.logger?.warn?.(`  ${prep.fileLabel}: ${validChunks.length}/${prep.chunks?.length} chunks embedded — marking for retry on next pass`);
+            }
+            storePayloads.push({ prep, validChunks, allEmbedded });
+        }
+        // One bulk write per window. Dedup (per-modified-file deletes) is skipped
+        // when the store provably has no prior rows: during a full rebuild the
+        // temp store starts empty, and on a first-time index the store counted 0
+        // chunks at the start of the pass. Nothing can collide there, so the
+        // delete transactions (which would each scan the table) are pure waste.
+        const skipDedup = tempStorePath !== undefined || existingCount === 0;
+        if (storePayloads.length > 0) {
+            const bulkItems = storePayloads.map(({ validChunks }) => ({
+                chunks: validChunks,
+                dedup: !skipDedup,
+            }));
+            if (effectiveStore.addChunksBulk) {
+                await effectiveStore.addChunksBulk(bulkItems);
+            }
+            else {
+                // Fallback for stores without bulk-write support
+                for (const item of bulkItems) {
+                    await effectiveStore.addChunks(item.chunks, { dedup: item.dedup });
+                }
+            }
+        }
+        // Manifest updates + progress reporting (per file, in-memory only).
+        const storeResults = [];
+        for (let fi = 0; fi < prepared.length; fi++) {
+            if (aborted()) {
+                storeResults.push({ normalizedPath: prepared[fi].normalizedPath, skipped: true });
+                break;
+            }
+            // Return early results from phase 1
+            const earlyResult = windowEarlyResults.get(fi);
+            if (earlyResult) {
+                storeResults.push(earlyResult);
+                continue;
+            }
+            const prep = prepared[fi];
+            // No-embed path (shouldn't reach here but guard anyway)
+            if (!prep.chunks || prep.textToEmbed?.length === 0) {
+                options.progress?.finishFile(prep.fileLabel);
+                storeResults.push({
+                    normalizedPath: prep.normalizedPath, hash: prep.hash, chunkCount: 0,
+                    fileLabel: prep.fileLabel,
+                    isNew: false, isModified: false, isUnchanged: false, isEmpty: false,
+                    isTooSmall: false, isRemoved: true, hadChunks: false,
+                    descriptionFailed: prep.descriptionFailed,
+                });
+                continue;
+            }
+            const payload = storePayloads.find((p) => p.prep === prep);
+            const validChunks = payload?.validChunks ?? [];
+            const allEmbedded = payload?.allEmbedded ?? false;
+            const result = {
+                normalizedPath: prep.normalizedPath,
+                hash: prep.hash,
+                chunkCount: validChunks.length,
+                fileLabel: prep.fileLabel,
+                isNew: !prep.isModified,
+                isModified: prep.isModified,
+                isUnchanged: false,
+                isEmpty: false,
+                isTooSmall: false,
+                isRemoved: (prep.chunks?.length ?? 0) > 0 && validChunks.length === 0 ? false : validChunks.length === 0,
+                hadChunks: (prep.chunks?.length ?? 0) > 0,
+                descriptionFailed: prep.descriptionFailed,
+                descHash: prep.descHash,
+            };
+            // Update manifest — only when ALL chunks were embedded. A partial
+            // write must not bump the hash/chunkCount, otherwise the missing
+            // chunks would never be retried (hash match on the next pass).
+            if (result.chunkCount > 0 && !result.isRemoved && allEmbedded) {
+                const meta = fileMeta.get(result.normalizedPath);
+                const entry = {
+                    hash: result.hash,
+                    chunkCount: result.chunkCount,
+                    indexedAt: Date.now(),
+                    mtime: meta?.mtime,
+                    size: meta?.size,
+                    descriptionFailed: result.descriptionFailed,
+                };
+                if (result.descHash) {
+                    entry.descHash = result.descHash;
+                }
+                manifest.files[result.normalizedPath] = entry;
+                enqueueManifestSave();
+            }
+            else if (result.isRemoved) {
+                delete manifest.files[result.normalizedPath];
+                enqueueManifestSave();
+            }
+            options.progress?.finishFile(prep.fileLabel);
+            storedFiles++;
+            logChunkProgress("Storing", prep.fileLabel, storedFiles, filesToStore, storedFiles, filesToStore);
+            storeResults.push(result);
+        }
+        const workerResults = storeResults;
+        for (const r of workerResults) {
+            if (r.skipped) {
+                abortedInWindow = true;
+                break;
+            }
+            allFinalResults.push(r);
+        }
+        aggregateStats(stats, allFinalResults.slice(-workerResults.length));
+        const storePhaseSec = ((Date.now() - storePhaseStart) / 1000).toFixed(1);
+        logger.info(`Store phase: ${storedFiles} files stored in ${storePhaseSec}s (running total ${stats.totalChunks} chunks)`);
+        // Free this window's chunk payloads so GC can reclaim them before the
+        // next window is prepared.
+        for (const prep of prepared) {
+            prep.chunks = undefined;
+            prep.textToEmbed = undefined;
+        }
+    }
     // ── Windowed pipeline ────────────────────────────────────────────────────
     // The prepare→describe→embed→store chain used to materialize ALL chunks,
     // embed texts, and vectors of the whole workspace simultaneously (hundreds
     // of MB on large repos). Processing bounded windows of files keeps peak
     // memory proportional to the window size instead of the workspace size.
+    //
+    // The store phase is launched as a non-awaited promise per window so it
+    // overlaps the next window's prepare/describe/embed (the store is I/O-bound,
+    // the embed is GPU/API-bound). Stores themselves stay serialized: the next
+    // window's store only starts after the previous one resolves.
     const WINDOW_SIZE = Math.max(50, (options.config.indexing.concurrency ?? 4) * 10);
     const allFinalResults = [];
     let abortedInWindow = false;
+    const earlyWorkerResults = new Map();
+    let prevStore = null;
+    let windowCount = 0;
+    const optimizeInterval = options.config.indexing.optimizeIntervalWindows ?? 8;
     for (let windowStart = 0; windowStart < workspaceFiles.length; windowStart += WINDOW_SIZE) {
         if (aborted()) {
             abortedInWindow = true;
             break;
         }
+        windowCount++;
         const windowFiles = workspaceFiles.slice(windowStart, windowStart + WINDOW_SIZE);
         const prepared = await prepareWindow(windowFiles);
         if (windowStart === 0) {
@@ -590,7 +747,7 @@ async function runIndexPassInner(options, logger) {
         // ── Phase 1: Collect embed queue + handle early results ────────────────
         const embedQueue = [];
         let totalEmbedChunks = 0;
-        const earlyWorkerResults = new Map();
+        earlyWorkerResults.clear();
         for (let fi = 0; fi < prepared.length; fi++) {
             const prep = prepared[fi];
             if (prep.earlyResult) {
@@ -662,106 +819,33 @@ async function runIndexPassInner(options, logger) {
                 prep.chunks[chunkIdx].embedding = emb;
             }
         }
-        // ── Phase 3: Store + manifest update per file (parallel) ──────────────
-        const filesToStore = prepared.filter((p) => !earlyWorkerResults.has(prepared.indexOf(p)) && p.chunks && (p.textToEmbed?.length ?? 0) > 0).length;
-        const storePhaseStart = Date.now();
-        logger.info(`Store phase: storing ${filesToStore} file(s) into vector database...`);
-        let storedFiles = 0;
-        const storeLimit = pLimit(options.config.indexing.concurrency);
-        const storeResults = await Promise.all(prepared.map((prep, fi) => storeLimit(async () => {
-            if (aborted()) {
-                return { normalizedPath: prep.normalizedPath, skipped: true };
-            }
-            // Return early results from phase 1
-            const earlyResult = earlyWorkerResults.get(fi);
-            if (earlyResult)
-                return earlyResult;
-            // No-embed path (shouldn't reach here but guard anyway)
-            if (!prep.chunks || prep.textToEmbed?.length === 0) {
-                options.progress?.finishFile(prep.fileLabel);
-                return {
-                    normalizedPath: prep.normalizedPath, hash: prep.hash, chunkCount: 0,
-                    fileLabel: prep.fileLabel,
-                    isNew: false, isModified: false, isUnchanged: false, isEmpty: false,
-                    isTooSmall: false, isRemoved: true, hadChunks: false,
-                    descriptionFailed: prep.descriptionFailed,
-                };
-            }
-            // Store chunks with pre-attached embeddings
-            const validChunks = (prep.chunks ?? []).filter((c) => c.embedding && c.embedding.length > 0);
-            // Partial embed failure: some chunks have no vector. Do NOT record the
-            // file as complete — keep the previous manifest entry (or none, for
-            // new files) so the next pass retries the missing chunks.
-            const allEmbedded = validChunks.length === (prep.chunks?.length ?? 0);
-            if (!allEmbedded && (prep.chunks?.length ?? 0) > 0) {
-                options.logger?.warn?.(`  ${prep.fileLabel}: ${validChunks.length}/${prep.chunks?.length} chunks embedded — marking for retry on next pass`);
-            }
-            if (validChunks.length > 0) {
-                await effectiveStore.addChunks(validChunks);
-            }
-            const result = {
-                normalizedPath: prep.normalizedPath,
-                hash: prep.hash,
-                chunkCount: validChunks.length,
-                fileLabel: prep.fileLabel,
-                isNew: !prep.isModified,
-                isModified: prep.isModified,
-                isUnchanged: false,
-                isEmpty: false,
-                isTooSmall: false,
-                isRemoved: (prep.chunks?.length ?? 0) > 0 && validChunks.length === 0 ? false : validChunks.length === 0,
-                hadChunks: (prep.chunks?.length ?? 0) > 0,
-                descriptionFailed: prep.descriptionFailed,
-                descHash: prep.descHash,
-            };
-            // Update manifest — only when ALL chunks were embedded. A partial
-            // write must not bump the hash/chunkCount, otherwise the missing
-            // chunks would never be retried (hash match on the next pass).
-            if (result.chunkCount > 0 && !result.isRemoved && allEmbedded) {
-                const meta = fileMeta.get(result.normalizedPath);
-                const entry = {
-                    hash: result.hash,
-                    chunkCount: result.chunkCount,
-                    indexedAt: Date.now(),
-                    mtime: meta?.mtime,
-                    size: meta?.size,
-                    descriptionFailed: result.descriptionFailed,
-                };
-                if (result.descHash) {
-                    entry.descHash = result.descHash;
-                }
-                manifest.files[result.normalizedPath] = entry;
-                enqueueManifestSave();
-            }
-            else if (result.isRemoved) {
-                delete manifest.files[result.normalizedPath];
-                enqueueManifestSave();
-            }
-            options.progress?.finishFile(prep.fileLabel);
-            storedFiles++;
-            logChunkProgress("Storing", prep.fileLabel, storedFiles, filesToStore, storedFiles, filesToStore);
-            return result;
-        })));
-        const workerResults = storeResults;
-        for (const r of workerResults) {
-            if (r.skipped) {
-                abortedInWindow = true;
-                break;
-            }
-            allFinalResults.push(r);
-        }
-        aggregateStats(stats, allFinalResults.slice(-workerResults.length));
-        const storePhaseSec = ((Date.now() - storePhaseStart) / 1000).toFixed(1);
-        logger.info(`Store phase: ${storedFiles} files stored in ${storePhaseSec}s (running total ${stats.totalChunks} chunks)`);
-        // Free this window's chunk payloads so GC can reclaim them before the
-        // next window is prepared.
-        for (const prep of prepared) {
-            prep.chunks = undefined;
-            prep.textToEmbed = undefined;
+        // ── Phase 3: Store + manifest update per file ─────────────────────────
+        // The store phase is I/O-bound, so it is launched without awaiting and
+        // overlaps the next window's prepare/describe/embed (GPU/API-bound)
+        // phases. Stores stay serialized: the previous window's store is drained
+        // before the next one starts.
+        if (prevStore)
+            await prevStore;
+        // Snapshot the current window's early results BEFORE the next window's
+        // embed phase clears/repopulates the shared map — storeWindow runs
+        // asynchronously (overlapped with the next window's prepare/embed), so a
+        // shared mutable map would make its per-file loop read the WRONG window's
+        // entries and skip every file ("Store phase: storing N → 0 files stored").
+        prevStore = storeWindow(prepared, new Map(earlyWorkerResults));
+        // Periodically compact fragments and prune old versions so the store
+        // phase doesn't slow down as the index grows during long runs.
+        if (optimizeInterval > 0 && windowCount % optimizeInterval === 0) {
+            await prevStore;
+            prevStore = null;
+            logger.info("Optimizing vector store (mid-run compaction, pruning old versions)...");
+            await effectiveStore.optimize?.(tempStorePath ? { aggressive: true } : undefined);
         }
         if (abortedInWindow)
             break;
     } // end windowed pipeline loop
+    // Drain the final window's store before finishing.
+    if (prevStore)
+        await prevStore;
     const finalResults = allFinalResults;
     // Drain any in-flight manifest saves so all file entries are durable
     await manifestSaveChain;
@@ -822,12 +906,16 @@ async function runIndexPassInner(options, logger) {
     await saveManifest(options.storePath, manifest);
     await options.keywordIndex?.save(options.storePath);
     // Compact fragments and prune old version manifests so countRows() can't
-    // hang on accumulated versions from many add/delete cycles.
+    // hang on accumulated versions from many add/delete cycles. After a temp
+    // store rebuild, optimize the reopened real store handle (the temp handle
+    // was closed and its directory moved); use aggressive pruning since the
+    // swapped-in store is private to this process.
     if (!aborted()) {
         logger.info("Optimizing vector store (compacting fragments, pruning old versions)...");
         const optimizeStart = Date.now();
         try {
-            await effectiveStore.optimize?.();
+            const optimizeTarget = tempStorePath ? options.store : effectiveStore;
+            await optimizeTarget.optimize?.(tempStorePath ? { aggressive: true } : undefined);
             logger.info(`Vector store optimized in ${((Date.now() - optimizeStart) / 1000).toFixed(1)}s`);
         }
         catch (err) {

package/dist/vectorstore/lancedb.d.ts

@@ -32,6 +32,16 @@ export declare function isTransientConflictError(err: unknown): boolean;
  * @param realPath - Path to the current store (destination, will be replaced).
  */
 export declare function swapStoreDirectories(tempPath: string, realPath: string): Promise<void>;
+/**
+ * A single file's chunk payload for a bulk store write.
+ * `dedup: true` removes prior-revision rows for the same file path that are
+ * not part of this write; `dedup: false` appends only (safe when writing into
+ * a freshly-created store where no rows can collide).
+ */
+export interface BulkChunkWrite {
+    chunks: Chunk[];
+    dedup: boolean;
+}
 /**
  * A LanceDB-backed vector store with persistent on-disk storage, vector search,
  * and chunk metadata queries. Supports automatic corruption recovery by falling
@@ -44,6 +54,14 @@ export declare class LanceDbStore implements VectorStore {
     private table;
     private tableInit;
     private writeLock;
+    /**
+     * Memoized once-per-process index-metric repair. Stores built by versions
+     * before the cosine search switch carry an IVF index trained with the
+     * default L2 metric, which makes every cosine query log
+     * "Requested metric Cosine is incompatible with index metric L2" and fall
+     * back to brute-force. This repairs that stale index on first search.
+     */
+    private indexRepairPromise;
     /**
      * Execute an async function under an exclusive write lock.
      *
@@ -81,13 +99,36 @@ export declare class LanceDbStore implements VectorStore {
     private ensureColumnsNullable;
     /**
      * Store chunks in the LanceDB table. New rows are inserted first, then
-     * any old rows at the same (filePath, startLine) with different IDs are
-     * removed. This ensures no data is lost if the process aborts between
-     * insert and cleanup. Automatically attempts repair on corruption errors.
+     * old rows for the same file that are not part of this write are removed
+     * in a single delete per file. This ensures no data is lost if the process
+     * aborts between insert and cleanup. Automatically attempts repair on
+     * corruption errors.
+     *
+     * When `options.dedup` is `false` the cleanup step is skipped entirely —
+     * a pure append. Use this when writing into a store that provably has no
+     * prior rows for these files (e.g. a freshly-created rebuild store).
+     *
      * @param chunks - The chunks to add.
+     * @param options - Optional write options (`dedup`, default `true`).
      */
-    addChunks(chunks: Chunk[]): Promise<void>;
+    addChunks(chunks: Chunk[], options?: {
+        dedup?: boolean;
+    }): Promise<void>;
+    /**
+     * Store chunks for many files in a single transaction: one `table.add`
+     * across all items, then one `table.delete` per item that needs dedup.
+     * This collapses what used to be a per-file add + per-startLine deletes
+     * (K+2 LanceDB versions per file) into ~1 + M versions per batch.
+     *
+     * @param items - Per-file chunk payloads with their dedup flags.
+     */
+    addChunksBulk(items: BulkChunkWrite[]): Promise<void>;
+    /** Map a chunk to its internal row shape, or null if it has no embedding. */
+    private chunkToRow;
+    /** Group new rows by file path for dedup deletes. */
+    private rowsByFilePath;
     private addChunksInternal;
+    private addChunksBulkInternal;
     /**
      * Perform ANN (approximate nearest neighbor) search using LanceDB's native vector index.
      * Returns results scored as cosine similarity (0-1). Falls back to repair on corruption.
@@ -168,12 +209,43 @@ export declare class LanceDbStore implements VectorStore {
      * @returns The chunk count, or 0 if the table does not exist or times out.
      */
     count(): Promise<number>;
+    /**
+     * Ensure the ANN index on the `embedding` column uses the cosine metric,
+     * matching the `distanceType` requested by searchInternal.
+     *
+     * Older stores built the IVF index with the ivfFlat default (L2), so every
+     * cosine query hit "Requested metric Cosine is incompatible with index
+     * metric L2" and silently fell back to brute-force O(N) scans. This lazily
+     * replaces such an index with a cosine one — once per process.
+     *
+     * Callers must hold the write lock (searchInternal wraps the call in
+     * withWriteLock; optimize runs under it already).
+     */
+    private ensureCosineIndex;
+    /**
+     * Perform a single index-metric repair pass. Skips stores that have no
+     * index and fewer than 1000 rows (brute-force is optimal there). Uses a
+     * single `createIndex` with `replace: true` — a dropIndex + createIndex
+     * sequence races in LanceDB ("Retryable commit conflict") and leaves the
+     * stale index in place. On failure the memo is cleared so the next
+     * search/optimize retries.
+     */
+    private repairIndexMetricOnce;
     /**
      * Compact fragments and prune old version manifests to prevent the
-     * version-manifest accumulation that causes countRows() to hang.
-     * Should be called at the end of a successful index pass.
+     * version-manifest accumulation that causes countRows() to hang and the
+     * store phase to slow down as the index grows. Should be called at the
+     * end of a successful index pass, and periodically during long passes.
+     *
+     * @param options - `aggressive: true` prunes every version but the current
+     *   one with `deleteUnverified`. Only safe for a private store that no other
+     *   process reads (e.g. a temporary rebuild store). For the shared store,
+     *   versions newer than 1 hour are retained so in-flight queries (Web UI,
+     *   background auto-index) can finish before their data files are reclaimed.
      */
-    optimize(): Promise<void>;
+    optimize(options?: {
+        aggressive?: boolean;
+    }): Promise<void>;
     /**
      * Return all unique file paths currently stored in the index.
      * @returns An array of normalized file paths.