opencode-rag-plugin 1.19.4 → 1.19.8

package/dist/indexer/pipeline.js

@@ -66,9 +66,18 @@ export async function runIndexPass(options) {
         if (lock.pid && !isPidAlive(lock.pid)) {
             logger.debug(`Stale lock from dead process ${lock.pid} — continuing`);
         }
+        else if (lock.pid) {
+            // A LIVE process owns the lock — always skip, regardless of age.
+            // (Long re-indexes routinely exceed LOCK_MAX_AGE_MS; allowing a second
+            // process through the age check defeats the lock entirely and causes
+            // concurrent LanceDB writes.)
+            logger.warn(`Another index pass is running (PID ${lock.pid}). Skipping.`);
+            return { ...createIndexStats(0, "missing"), skipped: true };
+        }
         else if (lock.startedAt && age < LOCK_MAX_AGE_MS) {
-            logger.warn(`Another index pass is running (PID ${lock.pid ?? "unknown"}). Skipping.`);
-            return createIndexStats(0, "missing");
+            // No readable PID but a fresh lock — treat as active.
+            logger.warn(`Another index pass is running (no PID). Skipping.`);
+            return { ...createIndexStats(0, "missing"), skipped: true };
         }
     }
     catch {
@@ -77,8 +86,10 @@ export async function runIndexPass(options) {
     try {
         await fs.writeFile(lockPath, JSON.stringify({ pid: process.pid, startedAt: Date.now() }), "utf-8");
     }
-    catch {
-        // Best-effort lock
+    catch (err) {
+        // Best-effort lock — but surface it: without the lock file, the
+        // concurrent-pass protection above silently stops working.
+        logger.debug(`Could not write index lock file: ${err.message} — concurrent-pass protection disabled`);
     }
     try {
         return await runIndexPassInner(options, logger);
@@ -159,7 +170,24 @@ async function runIndexPassInner(options, logger) {
                     changedSet.add(f);
                 for (const f of untracked)
                     changedSet.add(f);
-                filterPaths = Array.from(changedSet);
+                // Git paths are relative to the REPO ROOT. When the workspace is a
+                // subdirectory of the repo (monorepos), convert them to workspace-
+                // relative paths before passing them to the scan — resolving repo-root
+                // paths against `cwd` would silently miss every change.
+                const toCwdRelative = (p) => {
+                    const abs = path.resolve(repoRoot, p);
+                    const rel = path.relative(options.cwd, abs);
+                    if (rel.startsWith("..") || path.isAbsolute(rel))
+                        return null;
+                    return rel;
+                };
+                const cwdRelative = [];
+                for (const p of changedSet) {
+                    const rel = toCwdRelative(p);
+                    if (rel !== null)
+                        cwdRelative.push(rel);
+                }
+                filterPaths = cwdRelative;
                 gitDeletedPaths = diffResult.deletedFiles;
                 logger.debug(`Git incremental: ${filterPaths.length} changed/untracked, ${gitDeletedPaths.length} deleted since ${manifest.lastGitCommit.slice(0, 8)}`);
             }
@@ -212,7 +240,6 @@ async function runIndexPassInner(options, logger) {
     let effectiveStore = options.store;
     let tempStorePath;
     if (options.force || (manifestStatus !== "ok" && existingCount > 0)) {
-        options.keywordIndex?.clear();
         for (const key of Object.keys(manifest.files)) {
             delete manifest.files[key];
         }
@@ -233,6 +260,10 @@ async function runIndexPassInner(options, logger) {
             catch { /* may not exist */ }
             effectiveStore = createVectorStore(options.config, tempStorePath, options.dimension);
             logger.debug(`Rebuilding index in temporary store at ${tempStorePath}`);
+            // Only clear the shared in-memory keyword index once the rebuild is
+            // guaranteed to proceed — the guard below may still bail out, and
+            // clearing it there would degrade live hybrid search for nothing.
+            options.keywordIndex?.clear();
         }
         else if (existingCount > 0) {
             // NEVER destroy existing data when we can't do an atomic rebuild.
@@ -244,6 +275,7 @@ async function runIndexPassInner(options, logger) {
         }
         else {
             // No existing data — safe to proceed with in-place indexing (no clear needed)
+            options.keywordIndex?.clear();
             logger.debug("No existing data; indexing from scratch.");
         }
     }
@@ -305,28 +337,30 @@ async function runIndexPassInner(options, logger) {
     let chunkedDone = 0;
     const chunkProgressInterval = Math.max(1, Math.floor(activeCount / 20));
     const chunkPhaseStart = Date.now();
-    const prepared = await Promise.all(workspaceFiles.map((file) => limit(async () => {
-        const fileLabel = path.relative(options.cwd, file.normalizedPath).replace(/\\/g, "/");
-        const isActive = !file.isEmpty && !file.isTooSmall &&
-            (!manifest.files[file.normalizedPath] || manifest.files[file.normalizedPath].hash !== file.hash);
-        if (isActive) {
-            options.progress?.startFile(fileLabel);
-        }
-        const prep = await prepareFile(file, options.cwd, manifest.files[file.normalizedPath], options.config, options.keywordIndex, options.descriptionProvider, logger, deferDescriptions, descHash);
-        if (isActive) {
-            chunkedDone++;
-            if (chunkedDone % chunkProgressInterval === 0 || chunkedDone === activeCount) {
-                const pct = ((chunkedDone / activeCount) * 100).toFixed(1);
-                logger.info(`  Chunking: ${chunkedDone}/${activeCount} files (${pct}%)`);
-            }
-        }
-        if (prep.earlyResult && isActive) {
-            options.progress?.finishFile(fileLabel);
-        }
-        return prep;
-    })));
-    const chunkPhaseSec = ((Date.now() - chunkPhaseStart) / 1000).toFixed(1);
-    logger.info(`Chunking complete: ${activeCount} files in ${chunkPhaseSec}s`);
+    /** Chunk one window of files (used by the windowed pipeline loop below). */
+    async function prepareWindow(windowFiles) {
+        const prepared = await Promise.all(windowFiles.map((file) => limit(async () => {
+            const fileLabel = path.relative(options.cwd, file.normalizedPath).replace(/\\/g, "/");
+            const isActive = !file.isEmpty && !file.isTooSmall &&
+                (!manifest.files[file.normalizedPath] || manifest.files[file.normalizedPath].hash !== file.hash);
+            if (isActive) {
+                options.progress?.startFile(fileLabel);
+            }
+            const prep = await prepareFile(file, options.cwd, manifest.files[file.normalizedPath], options.config, options.keywordIndex, options.descriptionProvider, logger, deferDescriptions, descHash);
+            if (isActive) {
+                chunkedDone++;
+                if (chunkedDone % chunkProgressInterval === 0 || chunkedDone === activeCount) {
+                    const pct = ((chunkedDone / activeCount) * 100).toFixed(1);
+                    logger.info(`  Chunking: ${chunkedDone}/${activeCount} files (${pct}%)`);
+                }
+            }
+            if (prep.earlyResult && isActive) {
+                options.progress?.finishFile(fileLabel);
+            }
+            return prep;
+        })));
+        return prepared;
+    }
     const aborted = () => options.abortSignal?.aborted ?? false;
     // Shared progress logger: "stage <file> (chunk i/n) — X/total remaining (P%)".
     const logChunkProgress = (stage, fileLabel, index, count, completed, total) => {
@@ -334,356 +368,485 @@ async function runIndexPassInner(options, logger) {
         const pct = total > 0 ? ((remaining / total) * 100).toFixed(1) : "0.0";
         logger.info(`${stage} ${fileLabel} (chunk ${index}/${count}) — ${remaining}/${total} remaining (${pct}%)`);
     };
-    if (deferDescriptions) {
-        const deferredPreps = prepared.filter((p) => p.chunks && p.chunks.length > 0 && p.relPath !== undefined);
-        if (deferredPreps.length > 0) {
-            const allChunks = [];
-            const oversizedChunks = [];
-            const maxContentChars = options.config.description?.maxContentChars;
-            logger.info(`Description phase: ${deferredPreps.length} files with chunks to describe`);
-            for (const prep of deferredPreps) {
-                for (const chunk of prep.chunks) {
-                    if (chunk.metadata.contentType !== "image") {
-                        if (maxContentChars && chunk.content.length > maxContentChars) {
-                            oversizedChunks.push(chunk);
-                        }
-                        else {
-                            allChunks.push(chunk);
-                        }
-                    }
-                }
+    // File metadata look-up for manifest entries
+    const fileMeta = new Map(workspaceFiles.map((f) => [f.normalizedPath, { mtime: f.mtime, size: f.size }]));
+    // Serialised manifest-save queue — prevents concurrent write races and acts
+    // as a checkpoint for Ctrl+C resilience.  Each worker appends to this chain
+    // after a successful store, so previously completed files are never lost.
+    // During a temp-store rebuild, saves go to the temp path so the real
+    // manifest stays consistent with the real store if the process is aborted.
+    const manifestTargetPath = () => tempStorePath ?? options.storePath;
+    let manifestSaveChain = Promise.resolve(undefined);
+    function enqueueManifestSave() {
+        manifestSaveChain = manifestSaveChain.then(() => saveManifest(manifestTargetPath(), manifest).catch((err) => {
+            options.logger?.warn?.(`Failed to save manifest: ${err.message}`);
+        }));
+    }
+    /**
+     * Store one window's prepared files into the vector store and update the
+     * manifest. Runs as a standalone async step so the caller can overlap it
+     * with the next window's prepare/describe/embed phases (the store phase is
+     * I/O-bound; the embed phase is GPU/API-bound).
+     *
+     * All chunks in the window are written in ONE bulk transaction (a single
+     * table.add plus one delete per modified file) instead of per-file adds
+     * plus per-startLine deletes.  Per-file transactions caused LanceDB
+     * version-manifest accumulation (K+2 versions per file) that made the
+     * store phase degrade quadratically as the index grew.
+     */
+    async function storeWindow(prepared, windowEarlyResults) {
+        const filesToStore = prepared.filter((p) => !windowEarlyResults.has(prepared.indexOf(p)) && p.chunks && (p.textToEmbed?.length ?? 0) > 0).length;
+        const storePhaseStart = Date.now();
+        logger.info(`Store phase: storing ${filesToStore} file(s) into vector database...`);
+        let storedFiles = 0;
+        // Compute per-file store payloads first (pure computation, no I/O).
+        const storePayloads = [];
+        for (const prep of prepared) {
+            if (aborted())
+                break;
+            if (windowEarlyResults.has(prepared.indexOf(prep)))
+                continue;
+            if (!prep.chunks || prep.textToEmbed?.length === 0)
+                continue;
+            const validChunks = (prep.chunks ?? []).filter((c) => c.embedding && c.embedding.length > 0);
+            // Partial embed failure: some chunks have no vector. Do NOT record the
+            // file as complete — keep the previous manifest entry (or none, for
+            // new files) so the next pass retries the missing chunks.
+            const allEmbedded = validChunks.length === (prep.chunks?.length ?? 0);
+            if (!allEmbedded && (prep.chunks?.length ?? 0) > 0) {
+                options.logger?.warn?.(`  ${prep.fileLabel}: ${validChunks.length}/${prep.chunks?.length} chunks embedded — marking for retry on next pass`);
             }
-            for (const chunk of oversizedChunks) {
-                chunk.description = buildFallbackDescription(chunk);
+            storePayloads.push({ prep, validChunks, allEmbedded });
+        }
+        // One bulk write per window. Dedup (per-modified-file deletes) is skipped
+        // when the store provably has no prior rows: during a full rebuild the
+        // temp store starts empty, and on a first-time index the store counted 0
+        // chunks at the start of the pass. Nothing can collide there, so the
+        // delete transactions (which would each scan the table) are pure waste.
+        const skipDedup = tempStorePath !== undefined || existingCount === 0;
+        if (storePayloads.length > 0) {
+            const bulkItems = storePayloads.map(({ validChunks }) => ({
+                chunks: validChunks,
+                dedup: !skipDedup,
+            }));
+            if (effectiveStore.addChunksBulk) {
+                await effectiveStore.addChunksBulk(bulkItems);
             }
-            // Per-file chunk index + label for clear progress reporting, plus a shared
-            // progress logger emitting "stage <file> (chunk i/n) — X/total remaining (P%)".
-            const chunkToFileLabel = new Map();
-            for (const prep of deferredPreps) {
-                for (const chunk of prep.chunks ?? []) {
-                    chunkToFileLabel.set(chunk.id, prep.fileLabel);
+            else {
+                // Fallback for stores without bulk-write support
+                for (const item of bulkItems) {
+                    await effectiveStore.addChunks(item.chunks, { dedup: item.dedup });
                 }
             }
-            const chunkMeta = new Map();
-            {
-                const perFileCount = new Map();
-                for (const chunk of allChunks) {
-                    perFileCount.set(chunk.metadata.filePath, (perFileCount.get(chunk.metadata.filePath) ?? 0) + 1);
-                }
-                const perFileSeen = new Map();
-                for (const chunk of allChunks) {
-                    const seen = perFileSeen.get(chunk.metadata.filePath) ?? 0;
-                    perFileSeen.set(chunk.metadata.filePath, seen + 1);
-                    chunkMeta.set(chunk.id, {
-                        fileLabel: chunkToFileLabel.get(chunk.id) ?? chunk.metadata.filePath,
-                        index: seen + 1,
-                        count: perFileCount.get(chunk.metadata.filePath) ?? 1,
-                    });
+        }
+        // Manifest updates + progress reporting (per file, in-memory only).
+        const storeResults = [];
+        for (let fi = 0; fi < prepared.length; fi++) {
+            if (aborted()) {
+                storeResults.push({ normalizedPath: prepared[fi].normalizedPath, skipped: true });
+                break;
+            }
+            // Return early results from phase 1
+            const earlyResult = windowEarlyResults.get(fi);
+            if (earlyResult) {
+                storeResults.push(earlyResult);
+                continue;
+            }
+            const prep = prepared[fi];
+            // No-embed path (shouldn't reach here but guard anyway)
+            if (!prep.chunks || prep.textToEmbed?.length === 0) {
+                options.progress?.finishFile(prep.fileLabel);
+                storeResults.push({
+                    normalizedPath: prep.normalizedPath, hash: prep.hash, chunkCount: 0,
+                    fileLabel: prep.fileLabel,
+                    isNew: false, isModified: false, isUnchanged: false, isEmpty: false,
+                    isTooSmall: false, isRemoved: true, hadChunks: false,
+                    descriptionFailed: prep.descriptionFailed,
+                });
+                continue;
+            }
+            const payload = storePayloads.find((p) => p.prep === prep);
+            const validChunks = payload?.validChunks ?? [];
+            const allEmbedded = payload?.allEmbedded ?? false;
+            const result = {
+                normalizedPath: prep.normalizedPath,
+                hash: prep.hash,
+                chunkCount: validChunks.length,
+                fileLabel: prep.fileLabel,
+                isNew: !prep.isModified,
+                isModified: prep.isModified,
+                isUnchanged: false,
+                isEmpty: false,
+                isTooSmall: false,
+                isRemoved: (prep.chunks?.length ?? 0) > 0 && validChunks.length === 0 ? false : validChunks.length === 0,
+                hadChunks: (prep.chunks?.length ?? 0) > 0,
+                descriptionFailed: prep.descriptionFailed,
+                descHash: prep.descHash,
+            };
+            // Update manifest — only when ALL chunks were embedded. A partial
+            // write must not bump the hash/chunkCount, otherwise the missing
+            // chunks would never be retried (hash match on the next pass).
+            if (result.chunkCount > 0 && !result.isRemoved && allEmbedded) {
+                const meta = fileMeta.get(result.normalizedPath);
+                const entry = {
+                    hash: result.hash,
+                    chunkCount: result.chunkCount,
+                    indexedAt: Date.now(),
+                    mtime: meta?.mtime,
+                    size: meta?.size,
+                    descriptionFailed: result.descriptionFailed,
+                };
+                if (result.descHash) {
+                    entry.descHash = result.descHash;
                 }
+                manifest.files[result.normalizedPath] = entry;
+                enqueueManifestSave();
             }
-            // Advance progress to Description stage before descriptions start
-            for (const prep of deferredPreps) {
-                options.progress?.finishStage(prep.fileLabel);
-            }
-            if (allChunks.length > 0 && descHash) {
-                // Check description cache first — reuse cached descriptions for unchanged chunks
-                const cacheHits = [];
-                const cacheMisses = [];
-                for (const chunk of allChunks) {
-                    const cacheKey = DescriptionCache.codeKey(chunk.content, descHash);
-                    const cached = descCache.get(cacheKey);
-                    if (cached) {
-                        cacheHits.push({ chunk, desc: cached });
-                    }
-                    else {
-                        cacheMisses.push(chunk);
+            else if (result.isRemoved) {
+                delete manifest.files[result.normalizedPath];
+                enqueueManifestSave();
+            }
+            options.progress?.finishFile(prep.fileLabel);
+            storedFiles++;
+            logChunkProgress("Storing", prep.fileLabel, storedFiles, filesToStore, storedFiles, filesToStore);
+            storeResults.push(result);
+        }
+        const workerResults = storeResults;
+        for (const r of workerResults) {
+            if (r.skipped) {
+                abortedInWindow = true;
+                break;
+            }
+            allFinalResults.push(r);
+        }
+        aggregateStats(stats, allFinalResults.slice(-workerResults.length));
+        const storePhaseSec = ((Date.now() - storePhaseStart) / 1000).toFixed(1);
+        logger.info(`Store phase: ${storedFiles} files stored in ${storePhaseSec}s (running total ${stats.totalChunks} chunks)`);
+        // Free this window's chunk payloads so GC can reclaim them before the
+        // next window is prepared.
+        for (const prep of prepared) {
+            prep.chunks = undefined;
+            prep.textToEmbed = undefined;
+        }
+    }
+    // ── Windowed pipeline ────────────────────────────────────────────────────
+    // The prepare→describe→embed→store chain used to materialize ALL chunks,
+    // embed texts, and vectors of the whole workspace simultaneously (hundreds
+    // of MB on large repos). Processing bounded windows of files keeps peak
+    // memory proportional to the window size instead of the workspace size.
+    //
+    // The store phase is launched as a non-awaited promise per window so it
+    // overlaps the next window's prepare/describe/embed (the store is I/O-bound,
+    // the embed is GPU/API-bound). Stores themselves stay serialized: the next
+    // window's store only starts after the previous one resolves.
+    const WINDOW_SIZE = Math.max(50, (options.config.indexing.concurrency ?? 4) * 10);
+    const allFinalResults = [];
+    let abortedInWindow = false;
+    const earlyWorkerResults = new Map();
+    let prevStore = null;
+    let windowCount = 0;
+    const optimizeInterval = options.config.indexing.optimizeIntervalWindows ?? 8;
+    for (let windowStart = 0; windowStart < workspaceFiles.length; windowStart += WINDOW_SIZE) {
+        if (aborted()) {
+            abortedInWindow = true;
+            break;
+        }
+        windowCount++;
+        const windowFiles = workspaceFiles.slice(windowStart, windowStart + WINDOW_SIZE);
+        const prepared = await prepareWindow(windowFiles);
+        if (windowStart === 0) {
+            const chunkPhaseSec = ((Date.now() - chunkPhaseStart) / 1000).toFixed(1);
+            logger.info(`Chunking complete: ${activeCount} files in ${chunkPhaseSec}s`);
+        }
+        if (deferDescriptions) {
+            const deferredPreps = prepared.filter((p) => p.chunks && p.chunks.length > 0 && p.relPath !== undefined);
+            if (deferredPreps.length > 0) {
+                const allChunks = [];
+                const oversizedChunks = [];
+                const maxContentChars = options.config.description?.maxContentChars;
+                logger.info(`Description phase: ${deferredPreps.length} files with chunks to describe`);
+                for (const prep of deferredPreps) {
+                    for (const chunk of prep.chunks) {
+                        if (chunk.metadata.contentType !== "image") {
+                            if (maxContentChars && chunk.content.length > maxContentChars) {
+                                oversizedChunks.push(chunk);
+                            }
+                            else {
+                                allChunks.push(chunk);
+                            }
+                        }
                     }
                 }
-                if (cacheHits.length > 0) {
-                    logger.debug(`  Using ${cacheHits.length} cached descriptions`);
-                    for (const { chunk, desc } of cacheHits) {
-                        chunk.description = desc;
-                    }
+                for (const chunk of oversizedChunks) {
+                    chunk.description = buildFallbackDescription(chunk);
                 }
-                // Build chunkId → prep map for tracking description failures per-file
-                const chunkToPrep = new Map();
+                // Per-file chunk index + label for clear progress reporting, plus a shared
+                // progress logger emitting "stage <file> (chunk i/n) — X/total remaining (P%)".
+                const chunkToFileLabel = new Map();
                 for (const prep of deferredPreps) {
                     for (const chunk of prep.chunks ?? []) {
-                        chunkToPrep.set(chunk.id, prep);
+                        chunkToFileLabel.set(chunk.id, prep.fileLabel);
                     }
                 }
-                // Process cache misses in parallel waves with throttled cache saves.
-                // Parallel sub-batches reduce serial LLM turnaround time.
-                // Cache saves are throttled to avoid O(n^2) full-JSON serialization I/O.
-                if (cacheMisses.length > 0) {
-                    const totalMisses = cacheMisses.length;
-                    const descConcurrency = options.config.indexing.descriptionConcurrency ?? 4;
-                    const SUB_BATCH = 10;
-                    let describedDone = 0;
-                    let describedSinceLastSave = 0;
-                    const SAVE_INTERVAL = 200;
-                    const descLimit = pLimit(descConcurrency);
-                    const subBatchTasks = [];
-                    for (let i = 0; i < totalMisses; i += SUB_BATCH) {
-                        subBatchTasks.push({ i, chunks: cacheMisses.slice(i, i + SUB_BATCH) });
+                const chunkMeta = new Map();
+                {
+                    const perFileCount = new Map();
+                    for (const chunk of allChunks) {
+                        perFileCount.set(chunk.metadata.filePath, (perFileCount.get(chunk.metadata.filePath) ?? 0) + 1);
+                    }
+                    const perFileSeen = new Map();
+                    for (const chunk of allChunks) {
+                        const seen = perFileSeen.get(chunk.metadata.filePath) ?? 0;
+                        perFileSeen.set(chunk.metadata.filePath, seen + 1);
+                        chunkMeta.set(chunk.id, {
+                            fileLabel: chunkToFileLabel.get(chunk.id) ?? chunk.metadata.filePath,
+                            index: seen + 1,
+                            count: perFileCount.get(chunk.metadata.filePath) ?? 1,
+                        });
+                    }
+                }
+                // Advance progress to Description stage before descriptions start
+                for (const prep of deferredPreps) {
+                    options.progress?.finishStage(prep.fileLabel);
+                }
+                if (allChunks.length > 0 && descHash) {
+                    // Check description cache first — reuse cached descriptions for unchanged chunks.
+                    // The key includes the chunk's file context: identical content in two
+                    // files must not reuse a contextually wrong description.
+                    const chunkContext = (c) => `${c.metadata.filePath}:${c.metadata.startLine}-${c.metadata.endLine}`;
+                    const cacheHits = [];
+                    const cacheMisses = [];
+                    for (const chunk of allChunks) {
+                        const cacheKey = DescriptionCache.codeKey(chunk.content, descHash, chunkContext(chunk));
+                        const cached = descCache.get(cacheKey);
+                        if (cached) {
+                            cacheHits.push({ chunk, desc: cached });
+                        }
+                        else {
+                            cacheMisses.push(chunk);
+                        }
                     }
-                    await Promise.all(subBatchTasks.map(({ i, chunks: subBatch }) => descLimit(async () => {
-                        if (aborted())
-                            return;
-                        try {
-                            const batchResult = await options.descriptionProvider.generateBatchDescriptions(subBatch, logger, {
-                                total: totalMisses,
-                                onProgress: (chunk) => {
-                                    describedDone++;
-                                    const meta = chunkMeta.get(chunk.id);
-                                    if (meta)
-                                        logChunkProgress("Describing", meta.fileLabel, meta.index, meta.count, describedDone, totalMisses);
-                                },
-                            });
-                            const newCacheEntries = [];
-                            for (const chunk of subBatch) {
-                                const desc = batchResult.get(chunk.id);
-                                if (desc && desc.trim().length > 0) {
-                                    chunk.description = desc;
-                                    newCacheEntries.push([DescriptionCache.codeKey(chunk.content, descHash), desc]);
+                    if (cacheHits.length > 0) {
+                        logger.debug(`  Using ${cacheHits.length} cached descriptions`);
+                        for (const { chunk, desc } of cacheHits) {
+                            chunk.description = desc;
+                        }
+                    }
+                    // Build chunkId → prep map for tracking description failures per-file
+                    const chunkToPrep = new Map();
+                    for (const prep of deferredPreps) {
+                        for (const chunk of prep.chunks ?? []) {
+                            chunkToPrep.set(chunk.id, prep);
+                        }
+                    }
+                    // Process cache misses in parallel waves with throttled cache saves.
+                    // Parallel sub-batches reduce serial LLM turnaround time.
+                    // Cache saves are throttled to avoid O(n^2) full-JSON serialization I/O.
+                    if (cacheMisses.length > 0) {
+                        const totalMisses = cacheMisses.length;
+                        const descConcurrency = options.config.indexing.descriptionConcurrency ?? 4;
+                        const SUB_BATCH = 10;
+                        let describedDone = 0;
+                        let describedSinceLastSave = 0;
+                        const SAVE_INTERVAL = 200;
+                        const descLimit = pLimit(descConcurrency);
+                        const subBatchTasks = [];
+                        for (let i = 0; i < totalMisses; i += SUB_BATCH) {
+                            subBatchTasks.push({ i, chunks: cacheMisses.slice(i, i + SUB_BATCH) });
+                        }
+                        await Promise.all(subBatchTasks.map(({ i, chunks: subBatch }) => descLimit(async () => {
+                            if (aborted())
+                                return;
+                            try {
+                                const batchResult = await options.descriptionProvider.generateBatchDescriptions(subBatch, logger, {
+                                    total: totalMisses,
+                                    onProgress: (chunk) => {
+                                        describedDone++;
+                                        const meta = chunkMeta.get(chunk.id);
+                                        if (meta)
+                                            logChunkProgress("Describing", meta.fileLabel, meta.index, meta.count, describedDone, totalMisses);
+                                    },
+                                });
+                                const newCacheEntries = [];
+                                for (const chunk of subBatch) {
+                                    const desc = batchResult.get(chunk.id);
+                                    if (desc && desc.trim().length > 0) {
+                                        chunk.description = desc;
+                                        newCacheEntries.push([DescriptionCache.codeKey(chunk.content, descHash, chunkContext(chunk)), desc]);
+                                    }
+                                }
+                                if (newCacheEntries.length > 0) {
+                                    descCache.setMany(newCacheEntries);
+                                    describedSinceLastSave += newCacheEntries.length;
+                                    if (describedSinceLastSave >= SAVE_INTERVAL) {
+                                        await descCache.save();
+                                        describedSinceLastSave = 0;
+                                    }
                                 }
                             }
-                            if (newCacheEntries.length > 0) {
-                                descCache.setMany(newCacheEntries);
-                                describedSinceLastSave += newCacheEntries.length;
-                                if (describedSinceLastSave >= SAVE_INTERVAL) {
-                                    await descCache.save();
-                                    describedSinceLastSave = 0;
+                            catch (err) {
+                                logger.warn(`  Description sub-batch failed (${i}-${i + subBatch.length}): ${err.message}`);
+                                const failedPreps = new Set();
+                                for (const chunk of subBatch) {
+                                    const prep = chunkToPrep.get(chunk.id);
+                                    if (prep)
+                                        failedPreps.add(prep);
+                                }
+                                for (const prep of failedPreps) {
+                                    prep.descriptionFailed = true;
                                 }
                             }
+                        })));
+                        if (describedSinceLastSave > 0) {
+                            await descCache.save();
                         }
-                        catch (err) {
-                            logger.warn(`  Description sub-batch failed (${i}-${i + subBatch.length}): ${err.message}`);
-                            const failedPreps = new Set();
-                            for (const chunk of subBatch) {
-                                const prep = chunkToPrep.get(chunk.id);
-                                if (prep)
-                                    failedPreps.add(prep);
+                        logger.debug(`Descriptions generated for ${totalMisses} chunks (concurrency: ${descConcurrency})`);
+                    }
+                }
+                else if (allChunks.length > 0) {
+                    // No descHash available (no description provider) — still generate descriptions
+                    let describedDone = 0;
+                    try {
+                        const batchResult = await options.descriptionProvider.generateBatchDescriptions(allChunks, logger, {
+                            total: allChunks.length,
+                            onProgress: (chunk) => {
+                                describedDone++;
+                                const meta = chunkMeta.get(chunk.id);
+                                if (meta)
+                                    logChunkProgress("Describing", meta.fileLabel, meta.index, meta.count, describedDone, allChunks.length);
+                            },
+                        });
+                        for (const chunk of allChunks) {
+                            const desc = batchResult.get(chunk.id);
+                            if (desc && desc.trim().length > 0) {
+                                chunk.description = desc;
                             }
-                            for (const prep of failedPreps) {
+                        }
+                    }
+                    catch (err) {
+                        logger.warn(`  Global description generation failed: ${err.message}`);
+                        for (const prep of deferredPreps) {
+                            if (prep.chunks.some((c) => c.metadata.contentType !== "image")) {
                                 prep.descriptionFailed = true;
                             }
                         }
-                    })));
-                    if (describedSinceLastSave > 0) {
-                        await descCache.save();
                     }
-                    logger.debug(`Descriptions generated for ${totalMisses} chunks (concurrency: ${descConcurrency})`);
                 }
-            }
-            else if (allChunks.length > 0) {
-                // No descHash available (no description provider) — still generate descriptions
-                let describedDone = 0;
-                try {
-                    const batchResult = await options.descriptionProvider.generateBatchDescriptions(allChunks, logger, {
-                        total: allChunks.length,
-                        onProgress: (chunk) => {
-                            describedDone++;
-                            const meta = chunkMeta.get(chunk.id);
-                            if (meta)
-                                logChunkProgress("Describing", meta.fileLabel, meta.index, meta.count, describedDone, allChunks.length);
-                        },
-                    });
-                    for (const chunk of allChunks) {
-                        const desc = batchResult.get(chunk.id);
-                        if (desc && desc.trim().length > 0) {
-                            chunk.description = desc;
-                        }
-                    }
+                for (const prep of deferredPreps) {
+                    prep.textToEmbed = buildTextsToEmbed(prep.chunks, prep.relPath, prep.metaHeader ?? "", prep.docPrefix ?? "", prep.isImageFile ?? false);
                 }
-                catch (err) {
-                    logger.warn(`  Global description generation failed: ${err.message}`);
-                    for (const prep of deferredPreps) {
-                        if (prep.chunks.some((c) => c.metadata.contentType !== "image")) {
-                            prep.descriptionFailed = true;
-                        }
-                    }
+                const totalDescribedChunks = deferredPreps.reduce((s, p) => s + (p.chunks?.length ?? 0), 0);
+                logger.info(`Description phase complete: ${totalDescribedChunks} chunks across ${deferredPreps.length} files`);
+            }
+        }
+        // Cross-file embedding batch: collect this window's texts into a single queue,
+        // embed in one batched call (with concurrency), then distribute back.
+        const isOllama = options.embedder.name === "ollama";
+        const defaultBatchSize = options.config.indexing.embedBatchSize;
+        const defaultConcurrency = options.config.indexing.embedConcurrency ?? 1;
+        // ── Phase 1: Collect embed queue + handle early results ────────────────
+        const embedQueue = [];
+        let totalEmbedChunks = 0;
+        earlyWorkerResults.clear();
+        for (let fi = 0; fi < prepared.length; fi++) {
+            const prep = prepared[fi];
+            if (prep.earlyResult) {
+                if (prep.earlyResult.isRemoved) {
+                    await effectiveStore.deleteByFilePath(prep.normalizedPath);
+                    options.keywordIndex?.removeByFilePath(prep.normalizedPath);
+                    delete manifest.files[prep.normalizedPath];
+                    enqueueManifestSave();
                 }
+                earlyWorkerResults.set(fi, prep.earlyResult);
+                continue;
             }
-            for (const prep of deferredPreps) {
-                prep.textToEmbed = buildTextsToEmbed(prep.chunks, prep.relPath, prep.metaHeader ?? "", prep.docPrefix ?? "", prep.isImageFile ?? false);
+            if (!prep.chunks || !prep.textToEmbed || prep.textToEmbed.length === 0) {
+                options.progress?.finishFile(prep.fileLabel);
+                earlyWorkerResults.set(fi, {
+                    normalizedPath: prep.normalizedPath, hash: prep.hash, chunkCount: 0,
+                    fileLabel: prep.fileLabel,
+                    isNew: false, isModified: false, isUnchanged: false, isEmpty: false,
+                    isTooSmall: false, isRemoved: true, hadChunks: false,
+                    descriptionFailed: prep.descriptionFailed,
+                });
+                continue;
             }
-            const totalDescribedChunks = deferredPreps.reduce((s, p) => s + (p.chunks?.length ?? 0), 0);
-            logger.info(`Description phase complete: ${totalDescribedChunks} chunks across ${deferredPreps.length} files`);
-        }
-    }
-    // Cross-file embedding batch: collect all texts into a single queue,
-    // embed in one batched call (with concurrency), then distribute back.
-    const isOllama = options.embedder.name === "ollama";
-    const defaultBatchSize = options.config.indexing.embedBatchSize;
-    const defaultConcurrency = options.config.indexing.embedConcurrency ?? 1;
-    // File metadata look-up for manifest entries
-    const fileMeta = new Map(workspaceFiles.map((f) => [f.normalizedPath, { mtime: f.mtime, size: f.size }]));
-    // Serialised manifest-save queue — prevents concurrent write races and acts
-    // as a checkpoint for Ctrl+C resilience.  Each worker appends to this chain
-    // after a successful store, so previously completed files are never lost.
-    // During a temp-store rebuild, saves go to the temp path so the real
-    // manifest stays consistent with the real store if the process is aborted.
-    const manifestTargetPath = () => tempStorePath ?? options.storePath;
-    let manifestSaveChain = Promise.resolve(undefined);
-    function enqueueManifestSave() {
-        manifestSaveChain = manifestSaveChain.then(() => saveManifest(manifestTargetPath(), manifest).catch((err) => {
-            options.logger?.warn?.(`Failed to save manifest: ${err.message}`);
-        }));
-    }
-    // ── Phase 1: Collect embed queue + handle early results ────────────────
-    const embedQueue = [];
-    let totalEmbedChunks = 0;
-    const earlyWorkerResults = new Map();
-    for (let fi = 0; fi < prepared.length; fi++) {
-        const prep = prepared[fi];
-        if (prep.earlyResult) {
-            if (prep.earlyResult.isRemoved) {
-                await effectiveStore.deleteByFilePath(prep.normalizedPath);
-                options.keywordIndex?.removeByFilePath(prep.normalizedPath);
-                delete manifest.files[prep.normalizedPath];
-                enqueueManifestSave();
+            options.progress?.finishStage(prep.fileLabel);
+            for (let ci = 0; ci < prep.textToEmbed.length; ci++) {
+                embedQueue.push({ fileIdx: fi, chunkIdx: ci, text: prep.textToEmbed[ci] });
             }
-            earlyWorkerResults.set(fi, prep.earlyResult);
-            continue;
-        }
-        if (!prep.chunks || !prep.textToEmbed || prep.textToEmbed.length === 0) {
-            options.progress?.finishFile(prep.fileLabel);
-            earlyWorkerResults.set(fi, {
-                normalizedPath: prep.normalizedPath, hash: prep.hash, chunkCount: 0,
-                fileLabel: prep.fileLabel,
-                isNew: false, isModified: false, isUnchanged: false, isEmpty: false,
-                isTooSmall: false, isRemoved: true, hadChunks: false,
-                descriptionFailed: prep.descriptionFailed,
-            });
-            continue;
-        }
-        options.progress?.finishStage(prep.fileLabel);
-        for (let ci = 0; ci < prep.textToEmbed.length; ci++) {
-            embedQueue.push({ fileIdx: fi, chunkIdx: ci, text: prep.textToEmbed[ci] });
-        }
-        totalEmbedChunks += prep.textToEmbed.length;
-    }
-    // ── Phase 2: Embed all texts in a single batched call ──────────────────
-    const batchSize = isOllama
-        ? Math.min(options.config.indexing.ollamaMaxBatchSize ?? 500, defaultBatchSize)
-        : defaultBatchSize;
-    let embeddedDone = 0;
-    const allTexts = embedQueue.map(item => item.text);
-    let allEmbeddings = [];
-    const embedPhaseStart = Date.now();
-    if (allTexts.length > 0) {
-        logger.info(`Embedding phase: ${allTexts.length} texts in batches of ${batchSize}...`);
-        try {
-            allEmbeddings = await embedBatch(options.embedder, allTexts, batchSize, "document", defaultConcurrency, (completed, total) => {
-                embeddedDone = completed;
-                logChunkProgress("Embedding", "", completed, total, embeddedDone, totalEmbedChunks);
-            });
-            logger.info(`Embedding complete: ${allTexts.length} texts in ${((Date.now() - embedPhaseStart) / 1000).toFixed(1)}s`);
+            totalEmbedChunks += prep.textToEmbed.length;
         }
-        catch (err) {
-            logger.warn(`  Global embedding failed: ${err.message}`);
-            for (const { fileIdx } of embedQueue) {
-                options.progress?.failFile(prepared[fileIdx].fileLabel);
-                earlyWorkerResults.set(fileIdx, {
-                    normalizedPath: prepared[fileIdx].normalizedPath,
-                    hash: prepared[fileIdx].hash,
-                    chunkCount: 0, fileLabel: prepared[fileIdx].fileLabel,
-                    isNew: false, isModified: false, isUnchanged: false, isEmpty: false,
-                    isTooSmall: false, isRemoved: false, hadChunks: false,
-                    descriptionFailed: prepared[fileIdx].descriptionFailed,
+        // ── Phase 2: Embed all texts in a single batched call ──────────────────
+        const batchSize = isOllama
+            ? Math.min(options.config.indexing.ollamaMaxBatchSize ?? 500, defaultBatchSize)
+            : defaultBatchSize;
+        let embeddedDone = 0;
+        const allTexts = embedQueue.map(item => item.text);
+        let allEmbeddings = [];
+        const embedPhaseStart = Date.now();
+        if (allTexts.length > 0) {
+            logger.info(`Embedding phase: ${allTexts.length} texts in batches of ${batchSize}...`);
+            try {
+                allEmbeddings = await embedBatch(options.embedder, allTexts, batchSize, "document", defaultConcurrency, (completed, total) => {
+                    embeddedDone = completed;
+                    logChunkProgress("Embedding", "", completed, total, embeddedDone, totalEmbedChunks);
                 });
+                logger.info(`Embedding complete: ${allTexts.length} texts in ${((Date.now() - embedPhaseStart) / 1000).toFixed(1)}s`);
+            }
+            catch (err) {
+                logger.warn(`  Global embedding failed: ${err.message}`);
+                for (const { fileIdx } of embedQueue) {
+                    options.progress?.failFile(prepared[fileIdx].fileLabel);
+                    earlyWorkerResults.set(fileIdx, {
+                        normalizedPath: prepared[fileIdx].normalizedPath,
+                        hash: prepared[fileIdx].hash,
+                        chunkCount: 0, fileLabel: prepared[fileIdx].fileLabel,
+                        isNew: false, isModified: false, isUnchanged: false, isEmpty: false,
+                        isTooSmall: false, isRemoved: false, hadChunks: false,
+                        descriptionFailed: prepared[fileIdx].descriptionFailed,
+                    });
+                }
+                embedQueue.length = 0; // prevent double-processing in store phase
             }
-            embedQueue.length = 0; // prevent double-processing in store phase
         }
-    }
-    // ── Distribute embeddings back to per-file chunks ─────────────────────
-    for (let i = 0; i < embedQueue.length; i++) {
-        const { fileIdx, chunkIdx } = embedQueue[i];
-        const emb = allEmbeddings[i];
-        const prep = prepared[fileIdx];
-        if (prep.chunks && prep.chunks[chunkIdx] && Array.isArray(emb) && emb.length > 0 && typeof emb[0] === "number") {
-            prep.chunks[chunkIdx].embedding = emb;
+        // ── Distribute embeddings back to per-file chunks ─────────────────────
+        for (let i = 0; i < embedQueue.length; i++) {
+            const { fileIdx, chunkIdx } = embedQueue[i];
+            const emb = allEmbeddings[i];
+            const prep = prepared[fileIdx];
+            if (prep.chunks && prep.chunks[chunkIdx] && Array.isArray(emb) && emb.length > 0 && typeof emb[0] === "number") {
+                prep.chunks[chunkIdx].embedding = emb;
+            }
         }
-    }
-    // ── Phase 3: Store + manifest update per file (parallel) ──────────────
-    const filesToStore = prepared.filter((p) => !earlyWorkerResults.has(prepared.indexOf(p)) && p.chunks && (p.textToEmbed?.length ?? 0) > 0).length;
-    const storePhaseStart = Date.now();
-    logger.info(`Store phase: storing ${filesToStore} file(s) into vector database...`);
-    let storedFiles = 0;
-    const storeLimit = pLimit(options.config.indexing.concurrency);
-    const storeResults = await Promise.all(prepared.map((prep, fi) => storeLimit(async () => {
-        if (aborted()) {
-            return { normalizedPath: prep.normalizedPath, skipped: true };
-        }
-        // Return early results from phase 1
-        const earlyResult = earlyWorkerResults.get(fi);
-        if (earlyResult)
-            return earlyResult;
-        // No-embed path (shouldn't reach here but guard anyway)
-        if (!prep.chunks || prep.textToEmbed?.length === 0) {
-            options.progress?.finishFile(prep.fileLabel);
-            return {
-                normalizedPath: prep.normalizedPath, hash: prep.hash, chunkCount: 0,
-                fileLabel: prep.fileLabel,
-                isNew: false, isModified: false, isUnchanged: false, isEmpty: false,
-                isTooSmall: false, isRemoved: true, hadChunks: false,
-                descriptionFailed: prep.descriptionFailed,
-            };
+        // ── Phase 3: Store + manifest update per file ─────────────────────────
+        // The store phase is I/O-bound, so it is launched without awaiting and
+        // overlaps the next window's prepare/describe/embed (GPU/API-bound)
+        // phases. Stores stay serialized: the previous window's store is drained
+        // before the next one starts.
+        if (prevStore)
+            await prevStore;
+        // Snapshot the current window's early results BEFORE the next window's
+        // embed phase clears/repopulates the shared map — storeWindow runs
+        // asynchronously (overlapped with the next window's prepare/embed), so a
+        // shared mutable map would make its per-file loop read the WRONG window's
+        // entries and skip every file ("Store phase: storing N → 0 files stored").
+        prevStore = storeWindow(prepared, new Map(earlyWorkerResults));
+        // Periodically compact fragments and prune old versions so the store
+        // phase doesn't slow down as the index grows during long runs.
+        if (optimizeInterval > 0 && windowCount % optimizeInterval === 0) {
+            await prevStore;
+            prevStore = null;
+            logger.info("Optimizing vector store (mid-run compaction, pruning old versions)...");
+            await effectiveStore.optimize?.(tempStorePath ? { aggressive: true } : undefined);
         }
-        // Store chunks with pre-attached embeddings
-        const validChunks = (prep.chunks ?? []).filter((c) => c.embedding && c.embedding.length > 0);
-        if (validChunks.length > 0) {
-            await effectiveStore.addChunks(validChunks);
-        }
-        const result = {
-            normalizedPath: prep.normalizedPath,
-            hash: prep.hash,
-            chunkCount: validChunks.length,
-            fileLabel: prep.fileLabel,
-            isNew: !prep.isModified,
-            isModified: prep.isModified,
-            isUnchanged: false,
-            isEmpty: false,
-            isTooSmall: false,
-            isRemoved: (prep.chunks?.length ?? 0) > 0 && validChunks.length === 0 ? false : validChunks.length === 0,
-            hadChunks: (prep.chunks?.length ?? 0) > 0,
-            descriptionFailed: prep.descriptionFailed,
-            descHash: prep.descHash,
-        };
-        // Update manifest
-        if (result.chunkCount > 0 && !result.isRemoved) {
-            const meta = fileMeta.get(result.normalizedPath);
-            const entry = {
-                hash: result.hash,
-                chunkCount: result.chunkCount,
-                indexedAt: Date.now(),
-                mtime: meta?.mtime,
-                size: meta?.size,
-                descriptionFailed: result.descriptionFailed,
-            };
-            if (result.descHash) {
-                entry.descHash = result.descHash;
-            }
-            manifest.files[result.normalizedPath] = entry;
-            enqueueManifestSave();
-        }
-        else if (result.isRemoved) {
-            delete manifest.files[result.normalizedPath];
-            enqueueManifestSave();
-        }
-        options.progress?.finishFile(prep.fileLabel);
-        storedFiles++;
-        logChunkProgress("Storing", prep.fileLabel, storedFiles, filesToStore, storedFiles, filesToStore);
-        return result;
-    })));
-    const workerResults = storeResults;
-    const finalResults = [];
-    for (const r of workerResults) {
-        if (r.skipped)
+        if (abortedInWindow)
             break;
-        finalResults.push(r);
-    }
+    } // end windowed pipeline loop
+    // Drain the final window's store before finishing.
+    if (prevStore)
+        await prevStore;
+    const finalResults = allFinalResults;
     // Drain any in-flight manifest saves so all file entries are durable
     await manifestSaveChain;
     // Update mtime/size for unchanged files (speeds up the next scan)
@@ -694,9 +857,7 @@ async function runIndexPassInner(options, logger) {
             entry.size = size;
         }
     }
-    const storePhaseSec = ((Date.now() - storePhaseStart) / 1000).toFixed(1);
-    logger.info(`Store phase complete: ${storedFiles} files stored in ${storePhaseSec}s (${stats.totalChunks} total chunks)`);
-    aggregateStats(stats, finalResults);
+    logger.info(`Index pass complete: ${stats.totalChunks} total chunks (${finalResults.length} files processed)`);
     // Update timestamps; advance lastGitCommit ONLY on a complete pass
     manifest.lastIndexedAt = Date.now();
     if (!aborted()) {
@@ -745,12 +906,16 @@ async function runIndexPassInner(options, logger) {
     await saveManifest(options.storePath, manifest);
     await options.keywordIndex?.save(options.storePath);
     // Compact fragments and prune old version manifests so countRows() can't
-    // hang on accumulated versions from many add/delete cycles.
+    // hang on accumulated versions from many add/delete cycles. After a temp
+    // store rebuild, optimize the reopened real store handle (the temp handle
+    // was closed and its directory moved); use aggressive pruning since the
+    // swapped-in store is private to this process.
     if (!aborted()) {
         logger.info("Optimizing vector store (compacting fragments, pruning old versions)...");
         const optimizeStart = Date.now();
         try {
-            await effectiveStore.optimize?.();
+            const optimizeTarget = tempStorePath ? options.store : effectiveStore;
+            await optimizeTarget.optimize?.(tempStorePath ? { aggressive: true } : undefined);
             logger.info(`Vector store optimized in ${((Date.now() - optimizeStart) / 1000).toFixed(1)}s`);
         }
         catch (err) {