opencode-rag-plugin 1.12.17 → 1.14.6

package/dist/indexer/embed-stage.js.map

@@ -1 +1 @@
-{"version":3,"file":"embed-stage.js","sourceRoot":"","sources":["../../src/indexer/embed-stage.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,SAAS,CAAC;AAE7B,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AASpD,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,EAChC,MAAM,EACN,QAAQ,EACR,SAAS,EACT,WAAW,GACQ;IACnB,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEnC,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC;IAElC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAC1B,KAAK,CAAC,KAAK,IAAI,EAAE;QACf,IAAI,CAAC,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrD,OAAO,KAAK,CAAC;QACf,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC,CAAC,CACH,CACF,CAAC;IAEF,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,MAAM,YAAY,GAAa,EAAE,CAAC;IAElC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;QACzB,IAAI,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS;QAC5D,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACjC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;QAEhG,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,MAAM,QAAQ,GAAG,YAAY,CAAC,CAAC,CAAE,CAAC;YAClC,MAAM,GAAG,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;YAC1B,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC,KAAK,QAAQ,EAAE,CAAC;gBACvE,MAAM,CAAC,QAAQ,CAAE,CAAC,SAAS,GAAG,GAAe,CAAC;YAChD,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
+{"version":3,"file":"embed-stage.js","sourceRoot":"","sources":["../../src/indexer/embed-stage.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AASpD,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,EAChC,MAAM,EACN,QAAQ,EACR,SAAS,EACT,WAAW,GACQ;IACnB,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEnC,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,MAAM,YAAY,GAAa,EAAE,CAAC;IAElC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;QACzB,IAAI,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS;QAC5D,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACjC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;QAEhG,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,MAAM,QAAQ,GAAG,YAAY,CAAC,CAAC,CAAE,CAAC;YAClC,MAAM,GAAG,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;YAC1B,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC,KAAK,QAAQ,EAAE,CAAC;gBACvE,MAAM,CAAC,QAAQ,CAAE,CAAC,SAAS,GAAG,GAAe,CAAC;YAChD,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/indexer/pipeline.d.ts

@@ -1,23 +1,47 @@
 import type { RagConfig } from "../core/config.js";
-import type { DescriptionProvider, EmbeddingProvider, KeywordIndex, VectorStore } from "../core/interfaces.js";
+import type { DescriptionProvider, EmbeddingProvider, IndexProgress, KeywordIndex, VectorStore } from "../core/interfaces.js";
 import { type IndexRunStats, type IndexStatusSummary } from "./stats.js";
 export type { WatchPassScheduler } from "./watch.js";
 export { createWatchPassScheduler, createWatchIgnore } from "./watch.js";
+/** Options for configuring a single index pass. */
 export interface RunIndexPassOptions {
+    /** Workspace root directory. */
     cwd: string;
+    /** Path to the vector store data directory. */
     storePath: string;
+    /** Full RAG configuration for the workspace. */
     config: RagConfig;
+    /** Vector store instance for persisting chunks. */
     store: VectorStore;
+    /** Embedding provider for generating vector representations. */
     embedder: EmbeddingProvider;
+    /** When true, ignore the existing manifest and re-index everything. */
     force?: boolean;
+    /** Optional partial logger (missing methods default to no-ops). */
     logger?: Partial<Logger>;
+    /** Optional keyword index to populate during the pass. */
     keywordIndex?: KeywordIndex;
+    /** Optional provider for AI-generated chunk descriptions. */
     descriptionProvider?: DescriptionProvider;
+    /** Optional progress tracker for reporting file-level progress. */
+    progress?: IndexProgress;
+    /** Optional abort signal – when fired, the pass finishes the current file then stops. */
+    abortSignal?: AbortSignal;
+    /** Embedding vector dimension — needed when creating a temporary store for atomic rebuilds. */
+    dimension?: number;
 }
-interface Logger {
+export interface Logger {
     info(message: string): void;
     warn(message: string): void;
     debug(message: string): void;
 }
+/**
+ * Execute a full index pass: scan workspace, prepare files, generate embeddings
+ * (and optionally AI descriptions), and store results into the vector store.
+ * Uses a lock file to prevent concurrent passes.
+ *
+ * @param options - Configuration for the index pass.
+ * @returns Aggregate statistics for the pass.
+ */
 export declare function runIndexPass(options: RunIndexPassOptions): Promise<IndexRunStats>;
-export declare function getIndexStatusSummary(cwd: string, storePath: string, config: RagConfig, store: VectorStore): Promise<IndexStatusSummary>;
+export declare function getIndexStatusSummary(cwd: string, storePath: string, config: RagConfig, store: VectorStore, skipScan?: boolean): Promise<IndexStatusSummary>;

package/dist/indexer/pipeline.js

@@ -1,10 +1,13 @@
+import fs from "node:fs/promises";
 import path from "node:path";
 import pLimit from "p-limit";
 import { scanWorkspaceFiles } from "../content/reader.js";
 import { loadManifest, saveManifest, normalizeFilePath } from "../core/manifest.js";
 import { embedBatch } from "../embedder/factory.js";
+import { createVectorStore } from "../vectorstore/factory.js";
+import { swapStoreDirectories } from "../vectorstore/lancedb.js";
 import { createIndexStats } from "./stats.js";
-import { prepareFile } from "./worker.js";
+import { prepareFile, buildTextsToEmbed, storeFileChunks } from "./worker.js";
 import { getCurrentCommit, getChangedFilesSince, getUntrackedFiles, getRepoRoot } from "./git-diff.js";
 export { createWatchPassScheduler, createWatchIgnore } from "./watch.js";
 function createLogger(logger) {
@@ -29,17 +32,77 @@ function tryUpdateLastGitCommit(cwd, manifest) {
         return false;
     }
 }
+const LOCK_FILE = "index.lock";
+const LOCK_MAX_AGE_MS = 5 * 60 * 1000;
+function isPidAlive(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Execute a full index pass: scan workspace, prepare files, generate embeddings
+ * (and optionally AI descriptions), and store results into the vector store.
+ * Uses a lock file to prevent concurrent passes.
+ *
+ * @param options - Configuration for the index pass.
+ * @returns Aggregate statistics for the pass.
+ */
 export async function runIndexPass(options) {
     const logger = createLogger(options.logger);
+    const lockPath = path.join(options.storePath, LOCK_FILE);
+    try {
+        const raw = await fs.readFile(lockPath, "utf-8");
+        const lock = JSON.parse(raw);
+        const age = lock.startedAt ? Date.now() - lock.startedAt : Infinity;
+        if (lock.pid && !isPidAlive(lock.pid)) {
+            logger.debug(`Stale lock from dead process ${lock.pid} — continuing`);
+        }
+        else if (lock.startedAt && age < LOCK_MAX_AGE_MS) {
+            logger.warn(`Another index pass is running (PID ${lock.pid ?? "unknown"}). Skipping.`);
+            return createIndexStats(0, "missing");
+        }
+    }
+    catch {
+        // No lock file — proceed
+    }
+    try {
+        await fs.writeFile(lockPath, JSON.stringify({ pid: process.pid, startedAt: Date.now() }), "utf-8");
+    }
+    catch {
+        // Best-effort lock
+    }
+    try {
+        return await runIndexPassInner(options, logger);
+    }
+    finally {
+        try {
+            await fs.unlink(lockPath);
+        }
+        catch { }
+    }
+}
+async function runIndexPassInner(options, logger) {
     const loadResult = await loadManifest(options.storePath);
     const manifest = loadResult.manifest;
     let manifestStatus = loadResult.status;
     let rebuildPerformed = false;
-    logger.debug(`Manifest loaded: ${manifestStatus}, ${Object.keys(manifest.files).length} entries`);
+    logger.info(`Manifest loaded: ${manifestStatus}, ${Object.keys(manifest.files).length} entries`);
     if (options.force) {
         manifestStatus = "missing";
         logger.debug("Force mode: ignoring manifest");
     }
+    // Clear files that had description failures so they are fully re-indexed
+    const descFailedPaths = Object.keys(manifest.files).filter((p) => manifest.files[p]?.descriptionFailed);
+    if (descFailedPaths.length > 0) {
+        logger.info(`  ${descFailedPaths.length} file(s) marked as description-failed — re-indexing`);
+        for (const p of descFailedPaths) {
+            delete manifest.files[p];
+        }
+    }
     let filterPaths;
     let gitDeletedPaths = [];
     if (!options.force && manifestStatus === "ok" && manifest.lastGitCommit) {
@@ -59,8 +122,10 @@ export async function runIndexPass(options) {
             }
         }
     }
+    const scanStart = Date.now();
     const workspaceFiles = await scanWorkspaceFiles(options.cwd, options.config, logger, options.force ? undefined : manifest, filterPaths);
-    logger.debug(`Workspace scan complete: ${workspaceFiles.length} files`);
+    const scanSec = ((Date.now() - scanStart) / 1000).toFixed(1);
+    logger.info(`Workspace scan complete: ${workspaceFiles.length} files in ${scanSec}s`);
     const existingCount = await options.store.count();
     // Detect data loss: if the store has far fewer chunks than the manifest expects,
     // treat it as a corrupt store (e.g. schema migration dropped the old table).
@@ -76,8 +141,10 @@ export async function runIndexPass(options) {
             manifestStatus = "missing";
         }
     }
+    // Effective store used throughout the pass — may be a temp store for atomic rebuild.
+    let effectiveStore = options.store;
+    let tempStorePath;
     if (options.force || (manifestStatus !== "ok" && existingCount > 0)) {
-        await options.store.clear();
         options.keywordIndex?.clear();
         for (const key of Object.keys(manifest.files)) {
             delete manifest.files[key];
@@ -88,6 +155,23 @@ export async function runIndexPass(options) {
             logger.warn("Manifest missing or corrupt; rebuilding full index.");
         }
         manifestStatus = options.force ? "missing" : manifestStatus;
+        // Build the new index into a temporary store first, then atomically
+        // swap on completion.  Original data stays untouched if the process
+        // is aborted (Ctrl+C, crash) before the swap completes.
+        if (options.dimension) {
+            tempStorePath = options.storePath + "_tmp";
+            try {
+                await fs.rm(tempStorePath, { recursive: true, force: true });
+            }
+            catch { /* may not exist */ }
+            effectiveStore = createVectorStore(options.config, tempStorePath, options.dimension);
+            logger.debug(`Rebuilding index in temporary store at ${tempStorePath}`);
+        }
+        else {
+            // Fallback — in-memory store or no dimension: clear in place.
+            logger.warn("No embedding dimension available; falling back to in-place clear.");
+            await options.store.clear();
+        }
     }
     const stats = createIndexStats(workspaceFiles.length, manifestStatus);
     stats.rebuildPerformed = rebuildPerformed;
@@ -103,7 +187,8 @@ export async function runIndexPass(options) {
     const currentPaths = new Set(workspaceFiles.map((file) => file.normalizedPath));
     let stalePaths;
     if (filterPaths) {
-        stalePaths = gitDeletedPaths.map((p) => normalizeFilePath(path.resolve(options.cwd, p)));
+        const repoRoot = getRepoRoot(options.cwd) ?? options.cwd;
+        stalePaths = gitDeletedPaths.map((p) => normalizeFilePath(path.resolve(repoRoot, p)));
     }
     else {
         stalePaths = Object.keys(manifest.files).filter((p) => !currentPaths.has(p));
@@ -112,7 +197,7 @@ export async function runIndexPass(options) {
         logger.debug(`Removing ${stalePaths.length} stale files from index...`);
         const deleteLimit = pLimit(options.config.indexing.concurrency);
         await Promise.all(stalePaths.map((p) => deleteLimit(async () => {
-            await options.store.deleteByFilePath(p);
+            await effectiveStore.deleteByFilePath(p);
             options.keywordIndex?.removeByFilePath(p);
             delete manifest.files[p];
             stats.deletedFiles++;
@@ -136,143 +221,236 @@ export async function runIndexPass(options) {
         }
     }
     if (stats.deletedFiles > 0) {
-        logger.info(`Removed ${stats.deletedFiles} deleted files from index.`);
+        logger.debug(`Removed ${stats.deletedFiles} deleted files from index.`);
     }
-    logger.info(`Processing ${workspaceFiles.length} files (${newCount} new, ${modifiedCount} modified, ${unchangedCount} unchanged)...`);
-    // ── Phase 1: prepare all files (chunk + description + keyword + textToEmbed) ──
+    logger.debug(`Processing ${workspaceFiles.length} files (${newCount} new, ${modifiedCount} modified, ${unchangedCount} unchanged)...`);
+    // ── Phase 1: prepare all files (chunk + keyword, defer descriptions if provider present) ──
     const limit = pLimit(options.config.indexing.concurrency);
-    let completed = 0;
-    let started = 0;
-    const total = workspaceFiles.length;
+    const deferDescriptions = !!options.descriptionProvider;
     const prepared = await Promise.all(workspaceFiles.map((file) => limit(async () => {
-        const fileLabel = file.normalizedPath;
-        started++;
-        logger.info(`  Preparing file: ${started} - ${fileLabel}`);
-        const prep = await prepareFile(file, options.cwd, manifest.files[file.normalizedPath], options.config, options.keywordIndex, options.descriptionProvider, logger);
-        completed++;
-        if (total > 20 && completed % 50 === 0) {
-            logger.info(`  Preparing file ${completed}/${total}...`);
+        const fileLabel = path.relative(options.cwd, file.normalizedPath).replace(/\\/g, "/");
+        const isActive = !file.isEmpty && !file.isTooSmall &&
+            (!manifest.files[file.normalizedPath] || manifest.files[file.normalizedPath].hash !== file.hash);
+        if (isActive) {
+            options.progress?.startFile(fileLabel);
         }
-        return prep;
-    })));
-    // ── Handle deletions from store for files being re-processed ──
-    const deletionLimit = pLimit(options.config.indexing.concurrency);
-    await Promise.all(prepared.map((prep) => deletionLimit(async () => {
-        if (prep.earlyResult?.isRemoved) {
-            await options.store.deleteByFilePath(prep.normalizedPath);
-            options.keywordIndex?.removeByFilePath(prep.normalizedPath);
-            return;
-        }
-        if (prep.isModified && prep.chunks && prep.chunks.length > 0) {
-            await options.store.deleteByFilePath(prep.normalizedPath);
-            options.keywordIndex?.removeByFilePath(prep.normalizedPath);
+        const prep = await prepareFile(file, options.cwd, manifest.files[file.normalizedPath], options.config, options.keywordIndex, options.descriptionProvider, logger, deferDescriptions);
+        if (prep.earlyResult && isActive) {
+            options.progress?.finishFile(fileLabel);
         }
+        return prep;
     })));
-    // ── Phase 2: global embedding batch ──
-    const allTextToEmbed = [];
-    const textPrepIndex = [];
-    for (let i = 0; i < prepared.length; i++) {
-        const prep = prepared[i];
-        if (prep.chunks && prep.textToEmbed && prep.textToEmbed.length > 0) {
-            for (const text of prep.textToEmbed) {
-                allTextToEmbed.push(text);
-                textPrepIndex.push(i);
+    // ── Phase 1.5: global description generation (single pool, no nested concurrency) ──
+    if (deferDescriptions) {
+        const deferredPreps = prepared.filter((p) => p.chunks && p.chunks.length > 0 && p.relPath !== undefined);
+        if (deferredPreps.length > 0) {
+            const allChunks = [];
+            for (const prep of deferredPreps) {
+                for (const chunk of prep.chunks) {
+                    if (chunk.metadata.contentType !== "image") {
+                        allChunks.push(chunk);
+                    }
+                }
+            }
+            // Advance progress to Description stage before descriptions start
+            for (const prep of deferredPreps) {
+                options.progress?.finishStage(prep.fileLabel);
+            }
+            if (allChunks.length > 0) {
+                try {
+                    const batchResult = await options.descriptionProvider.generateBatchDescriptions(allChunks, (msg) => logger.debug(msg));
+                    for (const chunk of allChunks) {
+                        const desc = batchResult.get(chunk.id);
+                        if (desc && desc.trim().length > 0) {
+                            chunk.description = desc;
+                        }
+                    }
+                }
+                catch (err) {
+                    logger.warn(`  Global description generation failed: ${err.message}`);
+                    for (const prep of deferredPreps) {
+                        if (prep.chunks.some((c) => c.metadata.contentType !== "image")) {
+                            prep.descriptionFailed = true;
+                        }
+                    }
+                }
+            }
+            // Phase 1.6: build textToEmbed with descriptions injected
+            for (const prep of deferredPreps) {
+                prep.textToEmbed = buildTextsToEmbed(prep.chunks, prep.relPath, prep.metaHeader ?? "", prep.docPrefix ?? "", prep.isImageFile ?? false);
             }
         }
     }
-    let allEmbeddings = [];
-    if (allTextToEmbed.length > 0) {
-        const embedConcurrency = options.config.indexing.embedConcurrency ?? 1;
-        logger.info(`  Embedding ${allTextToEmbed.length} chunks in global batch (concurrency: ${embedConcurrency})...`);
-        allEmbeddings = await embedBatch(options.embedder, allTextToEmbed, options.config.indexing.embedBatchSize, "document", embedConcurrency);
-        logger.debug(`  Embedding batch complete: ${allEmbeddings.length} vectors`);
+    // ── Phase 2+3: per-file delete, embed + store (in parallel) ──
+    // Deletions for modified/re-removed files happen per-worker, right before
+    // embedding, so that an abort mid-embed doesn't orphan old entries.
+    const isOllama = options.embedder.name === "ollama";
+    const ollamaMaxBatch = options.config.indexing.ollamaMaxBatchSize ?? 4000;
+    const defaultBatchSize = options.config.indexing.embedBatchSize;
+    const defaultConcurrency = options.config.indexing.embedConcurrency ?? 1;
+    // File metadata look-up for manifest entries
+    const fileMeta = new Map(workspaceFiles.map((f) => [f.normalizedPath, { mtime: f.mtime, size: f.size }]));
+    // Serialised manifest-save queue — prevents concurrent write races and acts
+    // as a checkpoint for Ctrl+C resilience.  Each worker appends to this chain
+    // after a successful store, so previously completed files are never lost.
+    let manifestSaveChain = Promise.resolve(undefined);
+    function enqueueManifestSave() {
+        manifestSaveChain = manifestSaveChain.then(() => saveManifest(options.storePath, manifest));
     }
-    // ── Distribute embeddings back to prepared files ──
-    const embedByPrep = new Map();
-    for (let i = 0; i < textPrepIndex.length; i++) {
-        const prepIdx = textPrepIndex[i];
-        const emb = allEmbeddings[i];
-        if (!embedByPrep.has(prepIdx)) {
-            embedByPrep.set(prepIdx, []);
-        }
-        if (emb) {
-            embedByPrep.get(prepIdx).push(emb);
+    const aborted = () => options.abortSignal?.aborted ?? false;
+    const embedStoreLimit = pLimit(options.config.indexing.concurrency);
+    const workerResults = await Promise.all(prepared.map((prep) => embedStoreLimit(async () => {
+        // ── Check abort signal before any work ──
+        if (aborted()) {
+            return { normalizedPath: prep.normalizedPath, skipped: true };
         }
-    }
-    // ── Phase 3: store all files with their embeddings (batched) ──
-    logger.debug(`Phase 3: storing chunks for ${prepared.length} files...`);
-    const allValidChunks = [];
-    const workerResults = prepared.map((prep, idx) => {
+        // ── Early results (empty / too-small / unchanged / chunking failure) ──
         if (prep.earlyResult) {
+            // Remove stale manifest/store entries for files that no longer produce chunks
+            if (prep.earlyResult.isRemoved) {
+                await effectiveStore.deleteByFilePath(prep.normalizedPath);
+                options.keywordIndex?.removeByFilePath(prep.normalizedPath);
+                delete manifest.files[prep.normalizedPath];
+                enqueueManifestSave();
+            }
             return prep.earlyResult;
         }
-        if (!prep.chunks || !prep.textToEmbed) {
+        if (!prep.chunks || !prep.textToEmbed || prep.textToEmbed.length === 0) {
+            options.progress?.finishFile(prep.fileLabel);
             return {
-                normalizedPath: prep.normalizedPath, hash: prep.hash, chunkCount: 0, fileLabel: prep.fileLabel,
-                isNew: false, isModified: false, isUnchanged: false, isEmpty: false, isTooSmall: false, isRemoved: true, hadChunks: false,
+                normalizedPath: prep.normalizedPath, hash: prep.hash, chunkCount: 0,
+                fileLabel: prep.fileLabel,
+                isNew: false, isModified: false, isUnchanged: false, isEmpty: false,
+                isTooSmall: false, isRemoved: true, hadChunks: false,
+                descriptionFailed: prep.descriptionFailed,
             };
         }
-        const embeddings = embedByPrep.get(idx) ?? [];
-        for (let i = 0; i < prep.chunks.length; i++) {
-            const emb = embeddings[i];
-            if (Array.isArray(emb) && emb.length > 0 && typeof emb[0] === "number") {
-                prep.chunks[i].embedding = emb;
+        // ── Embed ──
+        const batchSize = isOllama
+            ? Math.min(prep.textToEmbed.length, ollamaMaxBatch)
+            : defaultBatchSize;
+        const concurrency = isOllama ? 1 : defaultConcurrency;
+        options.progress?.finishStage(prep.fileLabel);
+        let embeddings;
+        try {
+            embeddings = await embedBatch(options.embedder, prep.textToEmbed, batchSize, "document", concurrency);
+        }
+        catch (err) {
+            logger.warn(`  ${prep.fileLabel} (embedding failed: ${err.message})`);
+            options.progress?.failFile(prep.fileLabel);
+            return {
+                normalizedPath: prep.normalizedPath, hash: prep.hash, chunkCount: 0,
+                fileLabel: prep.fileLabel,
+                isNew: false, isModified: false, isUnchanged: false, isEmpty: false,
+                isTooSmall: false, isRemoved: true, hadChunks: false,
+                descriptionFailed: prep.descriptionFailed,
+            };
+        }
+        // ── Store (new data first; orphan cleanup is handled inside the store) ──
+        const result = await storeFileChunks(prep, embeddings, effectiveStore, logger);
+        // ── Update manifest in-memory and enqueue an atomic save ──
+        if (result.chunkCount > 0 && !result.isRemoved) {
+            const meta = fileMeta.get(result.normalizedPath);
+            manifest.files[result.normalizedPath] = {
+                hash: result.hash,
+                chunkCount: result.chunkCount,
+                indexedAt: Date.now(),
+                mtime: meta?.mtime,
+                size: meta?.size,
+                descriptionFailed: result.descriptionFailed,
+            };
+            enqueueManifestSave();
+        }
+        else if (result.isRemoved) {
+            delete manifest.files[result.normalizedPath];
+            enqueueManifestSave();
+        }
+        options.progress?.finishFile(prep.fileLabel);
+        return result;
+    })));
+    // Strip skipped sentinels
+    const finalResults = [];
+    for (const r of workerResults) {
+        if (r.skipped)
+            break;
+        finalResults.push(r);
+    }
+    // Drain any in-flight manifest saves so all file entries are durable
+    await manifestSaveChain;
+    // Update mtime/size for unchanged files (speeds up the next scan)
+    for (const { normalizedPath, mtime, size } of workspaceFiles) {
+        const entry = manifest.files[normalizedPath];
+        if (entry && (mtime !== undefined || size !== undefined)) {
+            entry.mtime = mtime;
+            entry.size = size;
+        }
+    }
+    // ── Aggregate stats ──
+    aggregateStats(stats, finalResults);
+    // Update timestamps; advance lastGitCommit ONLY on a complete pass
+    manifest.lastIndexedAt = Date.now();
+    if (!aborted()) {
+        tryUpdateLastGitCommit(options.cwd, manifest);
+    }
+    // ── Atomically promote temp store if a full rebuild was performed ──
+    if (tempStorePath) {
+        if (!aborted()) {
+            try {
+                await effectiveStore.close();
+                await options.store.close();
+                // Swap the newly-built temp directory into the real path
+                await swapStoreDirectories(tempStorePath, options.storePath);
+                // Re-open the original store handle so callers can search the new data
+                if (typeof options.store.reopen === "function") {
+                    await options.store.reopen(options.storePath);
+                }
+                logger.debug(`Promoted temporary store ${tempStorePath} → ${options.storePath}`);
             }
-            else {
-                prep.chunks[i].embedding = undefined;
+            catch (err) {
+                logger.warn(`Could not promote temporary store: ${err.message}. ` +
+                    `Original data preserved at ${options.storePath}`);
+                try {
+                    await fs.rm(tempStorePath, { recursive: true, force: true });
+                }
+                catch { }
             }
         }
-        const validChunks = prep.chunks.filter((c) => c.embedding && c.embedding.length > 0);
-        allValidChunks.push(...validChunks);
-        logger.info(`  ${prep.fileLabel} (${prep.chunks.length} chunks${prep.isModified ? ", modified" : ", new"})`);
-        return {
-            normalizedPath: prep.normalizedPath,
-            hash: prep.hash,
-            chunkCount: prep.chunks.length,
-            fileLabel: prep.fileLabel,
-            isNew: !prep.isModified,
-            isModified: prep.isModified,
-            isUnchanged: false,
-            isEmpty: false,
-            isTooSmall: false,
-            isRemoved: false,
-            hadChunks: prep.chunks.length > 0,
-        };
-    });
-    if (allValidChunks.length > 0) {
-        const STORE_BATCH = 1000;
-        for (let i = 0; i < allValidChunks.length; i += STORE_BATCH) {
-            const batch = allValidChunks.slice(i, i + STORE_BATCH);
-            await options.store.addChunks(batch);
+        else {
+            // Aborted — discard temp, keep original data intact
+            effectiveStore.close().catch(() => { });
+            try {
+                await fs.rm(tempStorePath, { recursive: true, force: true });
+            }
+            catch { }
+            logger.debug("Index pass cancelled; discarded temporary store.");
         }
-        logger.debug(`  Stored ${allValidChunks.length} chunks in batched writes`);
     }
-    // ── Aggregate stats and update manifest ──
-    aggregateStats(stats, workerResults, manifest, workspaceFiles);
-    manifest.lastIndexedAt = Date.now();
-    tryUpdateLastGitCommit(options.cwd, manifest);
+    // Save manifest and keyword index (always to the real store path — after
+    // a successful swap this points to the new data; after an abort it's the old).
     await saveManifest(options.storePath, manifest);
     await options.keywordIndex?.save(options.storePath);
-    stats.finalCount = await options.store.count();
+    // Count from the store — after a successful swap, the original handle has
+    // been reopened pointing to the new directory.
+    try {
+        stats.finalCount = await options.store.count();
+    }
+    catch {
+        stats.finalCount = (tempStorePath ? 0 : stats.totalChunks);
+    }
     return stats;
 }
-function aggregateStats(stats, results, manifest, workspaceFiles) {
-    const fileMeta = new Map(workspaceFiles.map((f) => [f.normalizedPath, { mtime: f.mtime, size: f.size }]));
+function aggregateStats(stats, results) {
     for (const result of results) {
         if (result.isEmpty) {
             stats.skippedEmptyFiles++;
-            if (result.isRemoved) {
-                delete manifest.files[result.normalizedPath];
+            if (result.isRemoved)
                 stats.removedFiles++;
-            }
             continue;
         }
         if (result.isTooSmall) {
             stats.skippedSmallFiles++;
-            if (result.isRemoved) {
-                delete manifest.files[result.normalizedPath];
+            if (result.isRemoved)
                 stats.removedFiles++;
-            }
             continue;
         }
         if (result.isUnchanged) {
@@ -280,7 +458,6 @@ function aggregateStats(stats, results, manifest, workspaceFiles) {
             continue;
         }
         if (result.isRemoved) {
-            delete manifest.files[result.normalizedPath];
             stats.removedFiles++;
             continue;
         }
@@ -291,65 +468,59 @@ function aggregateStats(stats, results, manifest, workspaceFiles) {
             stats.newFiles++;
         }
         if (result.chunkCount > 0) {
-            const meta = fileMeta.get(result.normalizedPath);
-            manifest.files[result.normalizedPath] = {
-                hash: result.hash,
-                chunkCount: result.chunkCount,
-                indexedAt: Date.now(),
-                mtime: meta?.mtime,
-                size: meta?.size,
-            };
+            if (result.descriptionFailed) {
+                stats.descriptionFailedFiles++;
+            }
             stats.totalChunks += result.chunkCount;
             stats.batchesFlushed++;
         }
     }
-    for (const { normalizedPath, mtime, size } of workspaceFiles) {
-        const entry = manifest.files[normalizedPath];
-        if (entry && (mtime !== undefined || size !== undefined)) {
-            entry.mtime = mtime;
-            entry.size = size;
-        }
-    }
 }
-export async function getIndexStatusSummary(cwd, storePath, config, store) {
+export async function getIndexStatusSummary(cwd, storePath, config, store, skipScan) {
     const loadResult = await loadManifest(storePath);
     const manifest = loadResult.manifest;
-    const workspaceFiles = await scanWorkspaceFiles(cwd, config, undefined, manifest);
     const storeCount = await store.count();
     if (loadResult.status !== "ok") {
         return {
             manifestStatus: loadResult.status,
             manifestEntries: 0,
             upToDateFiles: 0,
-            pendingFiles: workspaceFiles.length,
+            pendingFiles: 0,
             rebuildRequired: storeCount > 0,
             storeChunkCount: storeCount,
             manifestExpectedChunks: 0,
         };
     }
-    const currentPaths = new Set(workspaceFiles.map((file) => file.normalizedPath));
     let upToDateFiles = 0;
     let pendingFiles = 0;
-    for (const file of workspaceFiles) {
-        const previous = manifest.files[file.normalizedPath];
-        if (file.isEmpty || file.isTooSmall) {
-            if (previous)
+    let manifestExpectedChunks = 0;
+    if (!skipScan) {
+        const workspaceFiles = await scanWorkspaceFiles(cwd, config, undefined, manifest);
+        const currentPaths = new Set(workspaceFiles.map((file) => file.normalizedPath));
+        for (const file of workspaceFiles) {
+            const previous = manifest.files[file.normalizedPath];
+            if (file.isEmpty || file.isTooSmall) {
+                if (previous)
+                    pendingFiles++;
+                continue;
+            }
+            if (previous && previous.hash === file.hash) {
+                upToDateFiles++;
+            }
+            else {
                 pendingFiles++;
-            continue;
-        }
-        if (previous && previous.hash === file.hash) {
-            upToDateFiles++;
+            }
         }
-        else {
-            pendingFiles++;
+        for (const indexedPath of Object.keys(manifest.files)) {
+            if (!currentPaths.has(indexedPath)) {
+                pendingFiles++;
+            }
         }
+        manifestExpectedChunks = Object.values(manifest.files).reduce((sum, entry) => sum + entry.chunkCount, 0);
     }
-    for (const indexedPath of Object.keys(manifest.files)) {
-        if (!currentPaths.has(indexedPath)) {
-            pendingFiles++;
-        }
+    else {
+        manifestExpectedChunks = Object.values(manifest.files).reduce((sum, entry) => sum + entry.chunkCount, 0);
     }
-    const manifestExpectedChunks = Object.values(manifest.files).reduce((sum, entry) => sum + entry.chunkCount, 0);
     return {
         manifestStatus: loadResult.status,
         manifestEntries: Object.keys(manifest.files).length,