opencode-rag-plugin 1.15.1 → 1.17.2

package/dist/indexer/pipeline.js

@@ -5,12 +5,13 @@ import fs from "node:fs/promises";
 import path from "node:path";
 import pLimit from "p-limit";
 import { scanWorkspaceFiles } from "../content/reader.js";
-import { loadManifest, saveManifest, normalizeFilePath } from "../core/manifest.js";
+import { loadManifest, saveManifest, normalizeFilePath, computeDescriptionConfigHash } from "../core/manifest.js";
+import { DescriptionCache } from "../core/desc-cache.js";
 import { embedBatch } from "../embedder/factory.js";
 import { createVectorStore } from "../vectorstore/factory.js";
 import { swapStoreDirectories } from "../vectorstore/lancedb.js";
 import { createIndexStats } from "./stats.js";
-import { prepareFile, buildTextsToEmbed, storeFileChunks } from "./worker.js";
+import { prepareFile, buildTextsToEmbed } from "./worker.js";
 import { buildFallbackDescription } from "./description-stage.js";
 import { getCurrentCommit, getChangedFilesSince, getUntrackedFiles, getRepoRoot } from "./git-diff.js";
 export { createWatchPassScheduler, createWatchIgnore } from "./watch.js";
@@ -99,6 +100,11 @@ async function runIndexPassInner(options, logger) {
         manifestStatus = "missing";
         logger.debug("Force mode: ignoring manifest");
     }
+    // Compute a hash of the current description config — used to skip re-description
+    // when neither the file content nor the description config has changed.
+    const descHash = options.descriptionProvider
+        ? computeDescriptionConfigHash(options.config)
+        : undefined;
     // Clear files that had description failures so they are fully re-indexed
     const descFailedPaths = Object.keys(manifest.files).filter((p) => manifest.files[p]?.descriptionFailed);
     if (descFailedPaths.length > 0) {
@@ -107,6 +113,39 @@ async function runIndexPassInner(options, logger) {
             delete manifest.files[p];
         }
     }
+    // Persistent description cache — survives aborted runs so that descriptions
+    // generated before an abort are reused on the next run.
+    const descCachePath = options.storePath;
+    const descCache = new DescriptionCache(descCachePath);
+    await descCache.load();
+    // Resume-on-interrupt: detect manifest entries that are inconsistent with
+    // the vector store (e.g. indexing was aborted before all chunks were stored,
+    // or a full rebuild was aborted leaving a stale manifest pointing to old data).
+    // These entries are removed so they will be re-processed in this pass.
+    if (!options.force && manifestStatus === "ok" && Object.keys(manifest.files).length > 0) {
+        try {
+            const storedPaths = await options.store.getFilePaths();
+            const storedSet = new Set(storedPaths);
+            let removedCount = 0;
+            for (const [p, entry] of Object.entries(manifest.files)) {
+                // Path missing from store entirely (aborted before store finished)
+                if (entry.chunkCount > 0 && !storedSet.has(p)) {
+                    delete manifest.files[p];
+                    removedCount++;
+                }
+            }
+            if (removedCount > 0) {
+                logger.info(`  ${removedCount} file(s) in manifest but missing from store — re-indexing (resume after interruption)`);
+            }
+        }
+        catch {
+            logger.debug("  Could not check store paths for resume — proceeding without");
+        }
+    }
+    // Clear files where the stored descHash doesn't match current descHash and the
+    // file content IS different — these would be caught by hash comparison below but
+    // this pre-clear ensures the description cache is consulted during re-description.
+    // (Files with same hash but different descHash already fall through in prepareFile.)
     let filterPaths;
     let gitDeletedPaths = [];
     if (!options.force && manifestStatus === "ok" && manifest.lastGitCommit) {
@@ -127,7 +166,7 @@ async function runIndexPassInner(options, logger) {
         }
     }
     const scanStart = Date.now();
-    const workspaceFiles = await scanWorkspaceFiles(options.cwd, options.config, logger, options.force ? undefined : manifest, filterPaths);
+    const workspaceFiles = await scanWorkspaceFiles(options.cwd, options.config, logger, options.force ? undefined : manifest, filterPaths, options.imageVisionProvider, descCache);
     const scanSec = ((Date.now() - scanStart) / 1000).toFixed(1);
     logger.info(`Workspace scan complete: ${workspaceFiles.length} files in ${scanSec}s`);
     const existingCount = await options.store.count();
@@ -171,10 +210,17 @@ async function runIndexPassInner(options, logger) {
             effectiveStore = createVectorStore(options.config, tempStorePath, options.dimension);
             logger.debug(`Rebuilding index in temporary store at ${tempStorePath}`);
         }
+        else if (existingCount > 0) {
+            // NEVER destroy existing data when we can't do an atomic rebuild.
+            // Abort and ask the user to run 'opencode-rag index --force' manually.
+            logger.warn("Cannot rebuild safely without embedding dimension — aborting to protect existing data. " +
+                "Run 'opencode-rag index --force' manually to rebuild.");
+            // Restore manifest entries we just deleted so the next pass can retry incrementally
+            return createIndexStats(workspaceFiles.length, manifestStatus);
+        }
         else {
-            // Fallback — in-memory store or no dimension: clear in place.
-            logger.warn("No embedding dimension available; falling back to in-place clear.");
-            await options.store.clear();
+            // No existing data — safe to proceed with in-place indexing (no clear needed)
+            logger.debug("No existing data; indexing from scratch.");
         }
     }
     const stats = createIndexStats(workspaceFiles.length, manifestStatus);
@@ -237,12 +283,19 @@ async function runIndexPassInner(options, logger) {
         if (isActive) {
             options.progress?.startFile(fileLabel);
         }
-        const prep = await prepareFile(file, options.cwd, manifest.files[file.normalizedPath], options.config, options.keywordIndex, options.descriptionProvider, logger, deferDescriptions);
+        const prep = await prepareFile(file, options.cwd, manifest.files[file.normalizedPath], options.config, options.keywordIndex, options.descriptionProvider, logger, deferDescriptions, descHash);
         if (prep.earlyResult && isActive) {
             options.progress?.finishFile(fileLabel);
         }
         return prep;
     })));
+    const aborted = () => options.abortSignal?.aborted ?? false;
+    // Shared progress logger: "stage <file> (chunk i/n) — X/total remaining (P%)".
+    const logChunkProgress = (stage, fileLabel, index, count, completed, total) => {
+        const remaining = total - completed;
+        const pct = total > 0 ? ((remaining / total) * 100).toFixed(1) : "0.0";
+        logger.info(`${stage} ${fileLabel} (chunk ${index}/${count}) — ${remaining}/${total} remaining (${pct}%)`);
+    };
     if (deferDescriptions) {
         const deferredPreps = prepared.filter((p) => p.chunks && p.chunks.length > 0 && p.relPath !== undefined);
         if (deferredPreps.length > 0) {
@@ -264,13 +317,139 @@ async function runIndexPassInner(options, logger) {
             for (const chunk of oversizedChunks) {
                 chunk.description = buildFallbackDescription(chunk);
             }
+            // Per-file chunk index + label for clear progress reporting, plus a shared
+            // progress logger emitting "stage <file> (chunk i/n) — X/total remaining (P%)".
+            const chunkToFileLabel = new Map();
+            for (const prep of deferredPreps) {
+                for (const chunk of prep.chunks ?? []) {
+                    chunkToFileLabel.set(chunk.id, prep.fileLabel);
+                }
+            }
+            const chunkMeta = new Map();
+            {
+                const perFileCount = new Map();
+                for (const chunk of allChunks) {
+                    perFileCount.set(chunk.metadata.filePath, (perFileCount.get(chunk.metadata.filePath) ?? 0) + 1);
+                }
+                const perFileSeen = new Map();
+                for (const chunk of allChunks) {
+                    const seen = perFileSeen.get(chunk.metadata.filePath) ?? 0;
+                    perFileSeen.set(chunk.metadata.filePath, seen + 1);
+                    chunkMeta.set(chunk.id, {
+                        fileLabel: chunkToFileLabel.get(chunk.id) ?? chunk.metadata.filePath,
+                        index: seen + 1,
+                        count: perFileCount.get(chunk.metadata.filePath) ?? 1,
+                    });
+                }
+            }
             // Advance progress to Description stage before descriptions start
             for (const prep of deferredPreps) {
                 options.progress?.finishStage(prep.fileLabel);
             }
-            if (allChunks.length > 0) {
+            if (allChunks.length > 0 && descHash) {
+                // Check description cache first — reuse cached descriptions for unchanged chunks
+                const cacheHits = [];
+                const cacheMisses = [];
+                for (const chunk of allChunks) {
+                    const cacheKey = DescriptionCache.codeKey(chunk.content, descHash);
+                    const cached = descCache.get(cacheKey);
+                    if (cached) {
+                        cacheHits.push({ chunk, desc: cached });
+                    }
+                    else {
+                        cacheMisses.push(chunk);
+                    }
+                }
+                if (cacheHits.length > 0) {
+                    logger.debug(`  Using ${cacheHits.length} cached descriptions`);
+                    for (const { chunk, desc } of cacheHits) {
+                        chunk.description = desc;
+                    }
+                }
+                // Build chunkId → prep map for tracking description failures per-file
+                const chunkToPrep = new Map();
+                for (const prep of deferredPreps) {
+                    for (const chunk of prep.chunks ?? []) {
+                        chunkToPrep.set(chunk.id, prep);
+                    }
+                }
+                // Process cache misses in parallel waves with throttled cache saves.
+                // Parallel sub-batches reduce serial LLM turnaround time.
+                // Cache saves are throttled to avoid O(n^2) full-JSON serialization I/O.
+                if (cacheMisses.length > 0) {
+                    const totalMisses = cacheMisses.length;
+                    const descConcurrency = options.config.indexing.descriptionConcurrency ?? 4;
+                    const SUB_BATCH = 10;
+                    let describedDone = 0;
+                    let describedSinceLastSave = 0;
+                    const SAVE_INTERVAL = 200;
+                    const descLimit = pLimit(descConcurrency);
+                    const subBatchTasks = [];
+                    for (let i = 0; i < totalMisses; i += SUB_BATCH) {
+                        subBatchTasks.push({ i, chunks: cacheMisses.slice(i, i + SUB_BATCH) });
+                    }
+                    await Promise.all(subBatchTasks.map(({ i, chunks: subBatch }) => descLimit(async () => {
+                        if (aborted())
+                            return;
+                        try {
+                            const batchResult = await options.descriptionProvider.generateBatchDescriptions(subBatch, logger, {
+                                total: totalMisses,
+                                onProgress: (chunk) => {
+                                    describedDone++;
+                                    const meta = chunkMeta.get(chunk.id);
+                                    if (meta)
+                                        logChunkProgress("Describing", meta.fileLabel, meta.index, meta.count, describedDone, totalMisses);
+                                },
+                            });
+                            const newCacheEntries = [];
+                            for (const chunk of subBatch) {
+                                const desc = batchResult.get(chunk.id);
+                                if (desc && desc.trim().length > 0) {
+                                    chunk.description = desc;
+                                    newCacheEntries.push([DescriptionCache.codeKey(chunk.content, descHash), desc]);
+                                }
+                            }
+                            if (newCacheEntries.length > 0) {
+                                descCache.setMany(newCacheEntries);
+                                describedSinceLastSave += newCacheEntries.length;
+                                if (describedSinceLastSave >= SAVE_INTERVAL) {
+                                    await descCache.save();
+                                    describedSinceLastSave = 0;
+                                }
+                            }
+                        }
+                        catch (err) {
+                            logger.warn(`  Description sub-batch failed (${i}-${i + subBatch.length}): ${err.message}`);
+                            const failedPreps = new Set();
+                            for (const chunk of subBatch) {
+                                const prep = chunkToPrep.get(chunk.id);
+                                if (prep)
+                                    failedPreps.add(prep);
+                            }
+                            for (const prep of failedPreps) {
+                                prep.descriptionFailed = true;
+                            }
+                        }
+                    })));
+                    if (describedSinceLastSave > 0) {
+                        await descCache.save();
+                    }
+                    logger.debug(`Descriptions generated for ${totalMisses} chunks (concurrency: ${descConcurrency})`);
+                }
+            }
+            else if (allChunks.length > 0) {
+                // No descHash available (no description provider) — still generate descriptions
+                let describedDone = 0;
                 try {
-                    const batchResult = await options.descriptionProvider.generateBatchDescriptions(allChunks, logger);
+                    const batchResult = await options.descriptionProvider.generateBatchDescriptions(allChunks, logger, {
+                        total: allChunks.length,
+                        onProgress: (chunk) => {
+                            describedDone++;
+                            const meta = chunkMeta.get(chunk.id);
+                            if (meta)
+                                logChunkProgress("Describing", meta.fileLabel, meta.index, meta.count, describedDone, allChunks.length);
+                        },
+                    });
                     for (const chunk of allChunks) {
                         const desc = batchResult.get(chunk.id);
                         if (desc && desc.trim().length > 0) {
@@ -292,10 +471,9 @@ async function runIndexPassInner(options, logger) {
             }
         }
     }
-    // Deletions for modified/re-removed files happen per-worker, right before
-    // embedding, so that an abort mid-embed doesn't orphan old entries.
+    // Cross-file embedding batch: collect all texts into a single queue,
+    // embed in one batched call (with concurrency), then distribute back.
     const isOllama = options.embedder.name === "ollama";
-    const ollamaMaxBatch = options.config.indexing.ollamaMaxBatchSize ?? 4000;
     const defaultBatchSize = options.config.indexing.embedBatchSize;
     const defaultConcurrency = options.config.indexing.embedConcurrency ?? 1;
     // File metadata look-up for manifest entries
@@ -303,48 +481,102 @@ async function runIndexPassInner(options, logger) {
     // Serialised manifest-save queue — prevents concurrent write races and acts
     // as a checkpoint for Ctrl+C resilience.  Each worker appends to this chain
     // after a successful store, so previously completed files are never lost.
+    // During a temp-store rebuild, saves go to the temp path so the real
+    // manifest stays consistent with the real store if the process is aborted.
+    const manifestTargetPath = () => tempStorePath ?? options.storePath;
     let manifestSaveChain = Promise.resolve(undefined);
     function enqueueManifestSave() {
-        manifestSaveChain = manifestSaveChain.then(() => saveManifest(options.storePath, manifest));
-    }
-    const aborted = () => options.abortSignal?.aborted ?? false;
-    const embedStoreLimit = pLimit(options.config.indexing.concurrency);
-    const workerResults = await Promise.all(prepared.map((prep) => embedStoreLimit(async () => {
-        if (aborted()) {
-            return { normalizedPath: prep.normalizedPath, skipped: true };
-        }
+        manifestSaveChain = manifestSaveChain.then(() => saveManifest(manifestTargetPath(), manifest).catch((err) => {
+            options.logger?.warn?.(`Failed to save manifest: ${err.message}`);
+        }));
+    }
+    // ── Phase 1: Collect embed queue + handle early results ────────────────
+    const embedQueue = [];
+    let totalEmbedChunks = 0;
+    const earlyWorkerResults = new Map();
+    for (let fi = 0; fi < prepared.length; fi++) {
+        const prep = prepared[fi];
         if (prep.earlyResult) {
-            // Remove stale manifest/store entries for files that no longer produce chunks
             if (prep.earlyResult.isRemoved) {
                 await effectiveStore.deleteByFilePath(prep.normalizedPath);
                 options.keywordIndex?.removeByFilePath(prep.normalizedPath);
                 delete manifest.files[prep.normalizedPath];
                 enqueueManifestSave();
             }
-            return prep.earlyResult;
+            earlyWorkerResults.set(fi, prep.earlyResult);
+            continue;
         }
         if (!prep.chunks || !prep.textToEmbed || prep.textToEmbed.length === 0) {
             options.progress?.finishFile(prep.fileLabel);
-            return {
+            earlyWorkerResults.set(fi, {
                 normalizedPath: prep.normalizedPath, hash: prep.hash, chunkCount: 0,
                 fileLabel: prep.fileLabel,
                 isNew: false, isModified: false, isUnchanged: false, isEmpty: false,
                 isTooSmall: false, isRemoved: true, hadChunks: false,
                 descriptionFailed: prep.descriptionFailed,
-            };
+            });
+            continue;
         }
-        const batchSize = isOllama
-            ? Math.min(prep.textToEmbed.length, ollamaMaxBatch)
-            : defaultBatchSize;
-        const concurrency = isOllama ? 1 : defaultConcurrency;
         options.progress?.finishStage(prep.fileLabel);
-        let embeddings;
+        for (let ci = 0; ci < prep.textToEmbed.length; ci++) {
+            embedQueue.push({ fileIdx: fi, chunkIdx: ci, text: prep.textToEmbed[ci] });
+        }
+        totalEmbedChunks += prep.textToEmbed.length;
+    }
+    // ── Phase 2: Embed all texts in a single batched call ──────────────────
+    const batchSize = isOllama
+        ? Math.min(options.config.indexing.ollamaMaxBatchSize ?? 500, defaultBatchSize)
+        : defaultBatchSize;
+    let embeddedDone = 0;
+    const allTexts = embedQueue.map(item => item.text);
+    let allEmbeddings = [];
+    if (allTexts.length > 0) {
         try {
-            embeddings = await embedBatch(options.embedder, prep.textToEmbed, batchSize, "document", concurrency);
+            allEmbeddings = await embedBatch(options.embedder, allTexts, batchSize, "document", defaultConcurrency, (completed, total) => {
+                embeddedDone = completed;
+                logChunkProgress("Embedding", "", completed, total, embeddedDone, totalEmbedChunks);
+            });
         }
         catch (err) {
-            logger.warn(`  ${prep.fileLabel} (embedding failed: ${err.message})`);
-            options.progress?.failFile(prep.fileLabel);
+            logger.warn(`  Global embedding failed: ${err.message}`);
+            for (const { fileIdx } of embedQueue) {
+                options.progress?.failFile(prepared[fileIdx].fileLabel);
+                earlyWorkerResults.set(fileIdx, {
+                    normalizedPath: prepared[fileIdx].normalizedPath,
+                    hash: prepared[fileIdx].hash,
+                    chunkCount: 0, fileLabel: prepared[fileIdx].fileLabel,
+                    isNew: false, isModified: false, isUnchanged: false, isEmpty: false,
+                    isTooSmall: false, isRemoved: true, hadChunks: false,
+                    descriptionFailed: prepared[fileIdx].descriptionFailed,
+                });
+            }
+            embedQueue.length = 0; // prevent double-processing in store phase
+        }
+    }
+    // ── Distribute embeddings back to per-file chunks ─────────────────────
+    for (let i = 0; i < embedQueue.length; i++) {
+        const { fileIdx, chunkIdx } = embedQueue[i];
+        const emb = allEmbeddings[i];
+        const prep = prepared[fileIdx];
+        if (prep.chunks && prep.chunks[chunkIdx] && Array.isArray(emb) && emb.length > 0 && typeof emb[0] === "number") {
+            prep.chunks[chunkIdx].embedding = emb;
+        }
+    }
+    // ── Phase 3: Store + manifest update per file (parallel) ──────────────
+    const filesToStore = prepared.filter((p) => !earlyWorkerResults.has(prepared.indexOf(p)) && p.chunks && (p.textToEmbed?.length ?? 0) > 0).length;
+    let storedFiles = 0;
+    const storeLimit = pLimit(options.config.indexing.concurrency);
+    const storeResults = await Promise.all(prepared.map((prep, fi) => storeLimit(async () => {
+        if (aborted()) {
+            return { normalizedPath: prep.normalizedPath, skipped: true };
+        }
+        // Return early results from phase 1
+        const earlyResult = earlyWorkerResults.get(fi);
+        if (earlyResult)
+            return earlyResult;
+        // No-embed path (shouldn't reach here but guard anyway)
+        if (!prep.chunks || prep.textToEmbed?.length === 0) {
+            options.progress?.finishFile(prep.fileLabel);
             return {
                 normalizedPath: prep.normalizedPath, hash: prep.hash, chunkCount: 0,
                 fileLabel: prep.fileLabel,
@@ -353,12 +585,30 @@ async function runIndexPassInner(options, logger) {
                 descriptionFailed: prep.descriptionFailed,
             };
         }
-        // ── Store (new data first; orphan cleanup is handled inside the store) ──
-        const result = await storeFileChunks(prep, embeddings, effectiveStore, logger);
-        // ── Update manifest in-memory and enqueue an atomic save ──
+        // Store chunks with pre-attached embeddings
+        const validChunks = (prep.chunks ?? []).filter((c) => c.embedding && c.embedding.length > 0);
+        if (validChunks.length > 0) {
+            await effectiveStore.addChunks(validChunks);
+        }
+        const result = {
+            normalizedPath: prep.normalizedPath,
+            hash: prep.hash,
+            chunkCount: prep.chunks?.length ?? 0,
+            fileLabel: prep.fileLabel,
+            isNew: !prep.isModified,
+            isModified: prep.isModified,
+            isUnchanged: false,
+            isEmpty: false,
+            isTooSmall: false,
+            isRemoved: validChunks.length === 0,
+            hadChunks: (prep.chunks?.length ?? 0) > 0,
+            descriptionFailed: prep.descriptionFailed,
+            descHash: prep.descHash,
+        };
+        // Update manifest
         if (result.chunkCount > 0 && !result.isRemoved) {
             const meta = fileMeta.get(result.normalizedPath);
-            manifest.files[result.normalizedPath] = {
+            const entry = {
                 hash: result.hash,
                 chunkCount: result.chunkCount,
                 indexedAt: Date.now(),
@@ -366,6 +616,10 @@ async function runIndexPassInner(options, logger) {
                 size: meta?.size,
                 descriptionFailed: result.descriptionFailed,
             };
+            if (result.descHash) {
+                entry.descHash = result.descHash;
+            }
+            manifest.files[result.normalizedPath] = entry;
             enqueueManifestSave();
         }
         else if (result.isRemoved) {
@@ -373,8 +627,11 @@ async function runIndexPassInner(options, logger) {
             enqueueManifestSave();
         }
         options.progress?.finishFile(prep.fileLabel);
+        storedFiles++;
+        logChunkProgress("Storing", prep.fileLabel, storedFiles, filesToStore, storedFiles, filesToStore);
         return result;
     })));
+    const workerResults = storeResults;
     const finalResults = [];
     for (const r of workerResults) {
         if (r.skipped)
@@ -406,9 +663,7 @@ async function runIndexPassInner(options, logger) {
                 // Swap the newly-built temp directory into the real path
                 await swapStoreDirectories(tempStorePath, options.storePath);
                 // Re-open the original store handle so callers can search the new data
-                if (typeof options.store.reopen === "function") {
-                    await options.store.reopen(options.storePath);
-                }
+                await options.store.reopen?.(options.storePath);
                 logger.debug(`Promoted temporary store ${tempStorePath} → ${options.storePath}`);
             }
             catch (err) {
@@ -421,19 +676,28 @@ async function runIndexPassInner(options, logger) {
             }
         }
         else {
-            // Aborted — discard temp, keep original data intact
+            // Aborted — discard temp, keep original data intact.
+            // Do NOT save the manifest to the real path — the in-memory manifest
+            // was cleared at rebuild start and only partially rebuilt.  The old
+            // manifest on disk (at the real path) is still consistent with the
+            // old store data.
             effectiveStore.close().catch(() => { });
             try {
                 await fs.rm(tempStorePath, { recursive: true, force: true });
             }
             catch { }
             logger.debug("Index pass cancelled; discarded temporary store.");
+            // Skip final manifest/keyword save since we want the old state preserved
+            await descCache.save();
+            return stats;
         }
     }
     // Save manifest and keyword index (always to the real store path — after
     // a successful swap this points to the new data; after an abort it's the old).
     await saveManifest(options.storePath, manifest);
     await options.keywordIndex?.save(options.storePath);
+    // Persist description cache
+    await descCache.save();
     // Count from the store — after a successful swap, the original handle has
     // been reopened pointing to the new directory.
     try {

package/dist/indexer/watch.js

@@ -66,9 +66,7 @@ export function createWatchPassScheduler(runPass, onError, debounceMs = 300) {
                 rerunRequested = false;
                 schedule();
             }
-            else {
-                resolveWaiters();
-            }
+            resolveWaiters();
         }
     }
     return {

package/dist/indexer/worker.d.ts

@@ -26,6 +26,8 @@ export interface WorkerResult {
     hadChunks: boolean;
     /** Whether description generation failed for this file. */
     descriptionFailed?: boolean;
+    /** Hash of the description config used when generating descriptions for this file. */
+    descHash?: string;
 }
 /** Intermediate result after chunking a file but before embedding/storing. */
 export interface PreparedFile {
@@ -53,6 +55,8 @@ export interface PreparedFile {
     docPrefix?: string;
     /** Whether the source file is an image. */
     isImageFile?: boolean;
+    /** Hash of the description config used when generating descriptions. */
+    descHash?: string;
 }
 /**
  * Build the list of text strings that will be sent to the embedding provider.
@@ -76,6 +80,7 @@ interface ManifestFile {
     hash: string;
     chunkCount: number;
     indexedAt?: number;
+    descHash?: string;
 }
 /**
  * Prepare a single workspace file for indexing: chunk the content, build
@@ -92,6 +97,10 @@ interface ManifestFile {
  * @param deferDescriptions   - When true, skip description generation and build
  *                              fallback descriptions instead; descriptions are
  *                              expected to be generated in a later global pass.
+ * @param descHash            - Hash of the current description config. When the file
+ *                              is unchanged and `previous.descHash === descHash`,
+ *                              descriptions are skipped entirely (they already exist
+ *                              in the vector store from a prior run).
  * @returns A prepared file descriptor ready for embedding and storage.
  */
 export declare function prepareFile(file: WorkspaceFile, cwd: string, previous: ManifestFile | undefined, config: {
@@ -104,7 +113,10 @@ export declare function prepareFile(file: WorkspaceFile, cwd: string, previous:
     description?: {
         maxContentChars?: number;
     };
-}, keywordIndex: KeywordIndex | undefined, descriptionProvider: DescriptionProvider | undefined, logger: Logger, deferDescriptions?: boolean): Promise<PreparedFile>;
+    indexing?: {
+        maxSvgSizeBytes?: number;
+    };
+}, keywordIndex: KeywordIndex | undefined, descriptionProvider: DescriptionProvider | undefined, logger: Logger, deferDescriptions?: boolean, descHash?: string): Promise<PreparedFile>;
 /**
  * Store the chunks of a prepared file into the vector store, attaching their
  * computed embeddings.
@@ -139,6 +151,7 @@ export declare function processFile(file: WorkspaceFile, cwd: string, previous:
     indexing: {
         embedBatchSize: number;
         embedConcurrency?: number;
+        maxSvgSizeBytes?: number;
     };
     chunking?: {
         nodeTypes?: Record<string, string[]>;
@@ -146,5 +159,5 @@ export declare function processFile(file: WorkspaceFile, cwd: string, previous:
     description?: {
         maxContentChars?: number;
     };
-}, store: VectorStore, keywordIndex: KeywordIndex | undefined, embedder: EmbeddingProvider, descriptionProvider: DescriptionProvider | undefined, logger: Logger): Promise<WorkerResult>;
+}, store: VectorStore, keywordIndex: KeywordIndex | undefined, embedder: EmbeddingProvider, descriptionProvider: DescriptionProvider | undefined, logger: Logger, descHash?: string): Promise<WorkerResult>;
 export {};

package/dist/indexer/worker.js

@@ -52,9 +52,13 @@ export function buildTextsToEmbed(chunks, relPath, metaHeader, docPrefix, isImag
  * @param deferDescriptions   - When true, skip description generation and build
  *                              fallback descriptions instead; descriptions are
  *                              expected to be generated in a later global pass.
+ * @param descHash            - Hash of the current description config. When the file
+ *                              is unchanged and `previous.descHash === descHash`,
+ *                              descriptions are skipped entirely (they already exist
+ *                              in the vector store from a prior run).
  * @returns A prepared file descriptor ready for embedding and storage.
  */
-export async function prepareFile(file, cwd, previous, config, keywordIndex, descriptionProvider, logger, deferDescriptions) {
+export async function prepareFile(file, cwd, previous, config, keywordIndex, descriptionProvider, logger, deferDescriptions, descHash) {
     const fileLabel = path.relative(cwd, file.filePath).replace(/\\/g, "/");
     if (file.isEmpty) {
         return {
@@ -77,14 +81,18 @@ export async function prepareFile(file, cwd, previous, config, keywordIndex, des
         };
     }
     if (previous && previous.hash === file.hash) {
-        return {
-            normalizedPath: file.normalizedPath, hash: file.hash, fileLabel,
-            isModified: false,
-            earlyResult: {
-                normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: 0, fileLabel,
-                isNew: false, isModified: false, isUnchanged: true, isEmpty: false, isTooSmall: false, isRemoved: false, hadChunks: false,
-            },
-        };
+        if (!descHash || previous.descHash === descHash) {
+            return {
+                normalizedPath: file.normalizedPath, hash: file.hash, fileLabel,
+                isModified: false,
+                earlyResult: {
+                    normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: 0, fileLabel,
+                    isNew: false, isModified: false, isUnchanged: true, isEmpty: false, isTooSmall: false, isRemoved: false, hadChunks: false,
+                },
+            };
+        }
+        // File content is unchanged but description config changed — fall through to re-describe
+        logger.debug(`  ${fileLabel} (unchanged but descHash differs — re-describing)`);
     }
     const isModified = !!previous;
     let chunks;
@@ -102,7 +110,9 @@ export async function prepareFile(file, cwd, previous, config, keywordIndex, des
             }];
     }
     else {
-        chunks = await chunkFile(file.filePath, file.content, config.chunking?.nodeTypes).catch((err) => {
+        chunks = await chunkFile(file.filePath, file.content, config.chunking?.nodeTypes, {
+            maxSvgSizeBytes: config.indexing?.maxSvgSizeBytes,
+        }).catch((err) => {
             logger.warn(`  ${fileLabel} (chunking failed: ${err.message})`);
             return null;
         });
@@ -152,6 +162,7 @@ export async function prepareFile(file, cwd, previous, config, keywordIndex, des
             metaHeader,
             docPrefix,
             isImageFile: isImage,
+            descHash,
         };
     }
     let descriptionFailed = false;
@@ -188,6 +199,7 @@ export async function prepareFile(file, cwd, previous, config, keywordIndex, des
         chunks,
         textToEmbed,
         descriptionFailed,
+        descHash,
     };
 }
 /**
@@ -235,6 +247,7 @@ export async function storeFileChunks(prep, embeddings, store, _logger) {
         isRemoved: false,
         hadChunks: prep.chunks.length > 0,
         descriptionFailed: prep.descriptionFailed,
+        descHash: prep.descHash,
     };
 }
 /**
@@ -253,8 +266,8 @@ export async function storeFileChunks(prep, embeddings, store, _logger) {
  * @param logger              - Logger for diagnostic messages.
  * @returns A worker result describing the outcome.
  */
-export async function processFile(file, cwd, previous, config, store, keywordIndex, embedder, descriptionProvider, logger) {
-    const prep = await prepareFile(file, cwd, previous, config, keywordIndex, descriptionProvider, logger);
+export async function processFile(file, cwd, previous, config, store, keywordIndex, embedder, descriptionProvider, logger, descHash) {
+    const prep = await prepareFile(file, cwd, previous, config, keywordIndex, descriptionProvider, logger, false, descHash);
     if (prep.earlyResult) {
         if ((prep.earlyResult.isEmpty || prep.earlyResult.isTooSmall) && prep.earlyResult.isRemoved) {
             await store.deleteByFilePath(prep.normalizedPath);