akm-cli 0.9.0-beta.6 → 0.9.0-rc.0

package/dist/indexer/indexer.js

@@ -10,8 +10,10 @@ import { getDbPath } from "../core/paths.js";
 import { isVerbose, warn, warnVerbose } from "../core/warn.js";
 import { resolveIndexPassLLM } from "../llm/index-passes.js";
 import { takeWorkflowDocument } from "../workflows/runtime/document-cache.js";
-import { clearStaleCacheEntries, closeDatabase, deleteEntriesByDir, deleteEntriesByIds, deleteEntriesByStashDir, deleteIndexDirStatesByStashDir, getAllEntriesForEmbedding, getEmbeddableEntryCount, getEmbeddingCount, getEntriesByDir, getEntryCount, getIndexDirState, getMeta, isVecAvailable, openDatabase, openExistingDatabase, rebuildFts, relinkUsageEvents, setMeta, upsertEmbedding, upsertEntry, upsertIndexDirState, upsertUtilityScore, upsertWorkflowDocument, warnIfVecMissing, } from "./db/db.js";
+import { clearStaleCacheEntries, closeDatabase, deleteEntriesByDir, deleteEntriesByIds, deleteEntriesByStashDir, deleteIndexDirStatesByStashDir, getAllEntriesForEmbedding, getEmbeddableEntryCount, getEmbeddingCount, getEntryCount, getMeta, isVecAvailable, openExistingDatabase, openIndexDatabase, purgeEmbeddings, rebuildFts, relinkUsageEvents, setMeta, upsertEmbedding, upsertEntry, upsertIndexDirState, upsertUtilityScore, upsertWorkflowDocument, warnIfVecMissing, } from "./db/db.js";
 import { deleteStoredGraph } from "./db/graph-db.js";
+import { withIndexWriterLease } from "./index-writer-lock.js";
+import { canUseIncrementalSkip, computeDirFingerprint, getCachedZeroRowDirState, getDirIndexState, inferZeroRowReason, } from "./passes/dir-staleness.js";
 import { applyCuratedFrontmatter, applyWikiFrontmatter, generateMetadataFlat, isEnrichmentComplete, isWorkflowSkipWarning, loadStashFile, shouldIndexStashFile, } from "./passes/metadata.js";
 import { buildSearchText } from "./search/search-fields.js";
 import { classifySemanticFailure, clearSemanticStatus, deriveSemanticProviderFingerprint, writeSemanticStatus, } from "./search/semantic-status.js";
@@ -116,7 +118,7 @@ async function runWalkPhase(ctx) {
     ctx.timing.tWalkEnd = Date.now();
     throwIfAborted(signal);
     // LLM enrichment for directories that need it
-    await enhanceDirsWithLlm(db, config, dirsNeedingLlm, onProgress, signal, true, reEnrich);
+    await enhanceDirsWithLlm(db, config, dirsNeedingLlm, onProgress, signal, reEnrich);
     onProgress({
         phase: "llm",
         message: resolveIndexPassLLM("enrichment", config)
@@ -141,6 +143,7 @@ async function runEmbeddingPhase(ctx) {
  */
 async function runFinalizePhase(ctx) {
     const { db, config, sources, sourceDirs, isIncremental, stashDir, signal, onProgress } = ctx;
+    ctx.timing.tFinalizeStart = Date.now();
     // Rebuild FTS after all inserts. Use incremental mode when this whole
     // index run is incremental — only entries touched by `upsertEntry`
     // since the last rebuild are re-indexed.
@@ -151,10 +154,13 @@ async function runFinalizePhase(ctx) {
     });
     ctx.timing.tFtsEnd = Date.now();
     // Re-link detached usage_events and recompute utility scores.
+    onProgress({ phase: "finalize", message: "Relinking usage events." });
     relinkUsageEvents(db);
+    onProgress({ phase: "finalize", message: "Recomputing utility scores." });
     recomputeUtilityScores(db);
     // Purge LLM cache entries for assets that no longer exist in the index.
     try {
+        onProgress({ phase: "finalize", message: "Clearing stale LLM cache entries." });
         clearStaleCacheEntries(db);
     }
     catch {
@@ -162,6 +168,7 @@ async function runFinalizePhase(ctx) {
     }
     // Regenerate each wiki's index.md from its pages' frontmatter. Best-effort.
     try {
+        onProgress({ phase: "finalize", message: "Regenerating wiki indexes." });
         const { regenerateAllWikiIndexes } = await import("../wiki/wiki.js");
         regenerateAllWikiIndexes(stashDir);
     }
@@ -178,6 +185,7 @@ async function runFinalizePhase(ctx) {
     warnIfVecMissing(db);
     const totalEntries = getEntryCount(db);
     const semanticEntryCount = getEmbeddableEntryCount(db);
+    onProgress({ phase: "finalize", message: "Verifying semantic search state." });
     const verification = verifyIndexState(db, config, semanticEntryCount, embeddingResult);
     if (config.semanticSearchMode === "off") {
         clearSemanticStatus();
@@ -195,8 +203,9 @@ async function runFinalizePhase(ctx) {
     }
     onProgress({ phase: "verify", message: verification.message });
     // Store verification result and totalEntries on ctx for the caller to use
-    ctx._verification = verification;
-    ctx._totalEntries = totalEntries;
+    ctx.verification = verification;
+    ctx.totalEntries = totalEntries;
+    ctx.timing.tFinalizeEnd = Date.now();
     // suppress unused warning — sources was previously used inline
     void sources;
 }
@@ -224,129 +233,186 @@ function runCleanPass(db, dryRun) {
     };
 }
 // ── Indexer ──────────────────────────────────────────────────────────────────
+// ── Test seam ────────────────────────────────────────────────────────────────
+// Swap-and-restore override. Inert in production; only tests call the setter.
+let akmIndexOverride;
+/** TEST-ONLY. Swap the implementation of `akmIndex`; pass undefined to restore. */
+export function _setAkmIndexForTests(fake) {
+    akmIndexOverride = fake;
+}
 export async function akmIndex(options) {
-    const stashDir = options?.stashDir || resolveStashDir();
-    const onProgress = options?.onProgress ?? (() => { });
-    const signal = options?.signal;
-    const reEnrich = options?.reEnrich === true;
-    const full = options?.full === true;
-    const clean = options?.clean === true;
-    const dryRun = options?.dryRun === true;
-    // Load config and resolve all stash sources
-    const { loadConfig } = await import("../core/config/config.js");
-    const config = loadConfig();
-    // One-time, read-only guard: warn if the writable stash still holds an
-    // un-migrated `vaults/` directory. In 0.9.0 the indexer skips `vaults/`
-    // entirely, so an unmigrated vault's `.env` data would silently never be
-    // indexed. Non-destructive — only stats, never reads/writes/deletes.
-    const { warnOnUnmigratedVaults } = await import("./usage/unmigrated-vaults-guard.js");
-    warnOnUnmigratedVaults(stashDir);
-    // Ensure git stash caches are extracted before resolving stash dirs,
-    // so their content directories exist on disk for the walker to discover.
-    const { ensureSourceCaches, resolveSourceEntries } = await import("./search/search-source.js");
-    await ensureSourceCaches(config, { force: full });
-    const allSourceEntries = resolveSourceEntries(stashDir, config);
-    const allSourceDirs = allSourceEntries.map((s) => s.path);
-    const t0 = Date.now();
-    // Open database — pass embedding dimension from config if available
-    const dbPath = getDbPath();
-    const embeddingDim = config.embedding?.dimension;
-    const db = openDatabase(dbPath, embeddingDim ? { embeddingDim } : undefined);
-    try {
-        // Determine incremental vs full mode
-        const prevStashDir = getMeta(db, "stashDir");
-        const prevBuiltAt = getMeta(db, "builtAt");
-        const isIncremental = !full && prevStashDir === stashDir && !!prevBuiltAt;
-        const builtAtMs = isIncremental && prevBuiltAt ? new Date(prevBuiltAt).getTime() : 0;
-        // Assemble the run context
-        const ctx = {
-            db,
-            config,
-            sources: allSourceEntries,
-            sourceDirs: allSourceDirs,
-            full,
-            reEnrich,
-            stashDir,
-            onProgress,
-            signal,
-            timing: {
-                t0,
-                tWalkStart: t0,
-                tWalkEnd: t0,
-                tLlmEnd: t0,
-                tFtsEnd: t0,
-                tEmbedEnd: t0,
-            },
-            isIncremental,
-            builtAtMs,
-            hadRemovedSources: false,
-            scannedDirs: 0,
-            skippedDirs: 0,
-            generatedCount: 0,
-            walkWarnings: [],
-            dirsNeedingLlm: [],
-            embeddingResult: null,
-            graphExtractionResult: null,
-        };
+    if (akmIndexOverride)
+        return akmIndexOverride(options);
+    return akmIndexReal(options);
+}
+async function akmIndexReal(options) {
+    const requestedAt = Date.now();
+    let acquiredAt = requestedAt;
+    return withIndexWriterLease({
+        purpose: "akm-index",
+        signal: options?.signal,
+        onWait: ({ waitedMs }) => {
+            options?.onProgress?.({
+                phase: "preflight",
+                message: `Waiting for index writer lease (${Math.round(waitedMs / 1000)}s elapsed).`,
+            });
+        },
+        onAcquired: ({ waitedMs }) => {
+            acquiredAt = requestedAt + waitedMs;
+        },
+    }, async () => {
+        const stashDir = options?.stashDir || resolveStashDir();
+        const onProgress = options?.onProgress ?? (() => { });
+        const signal = options?.signal;
+        const reEnrich = options?.reEnrich === true;
+        const full = options?.full === true;
+        const clean = options?.clean === true;
+        const dryRun = options?.dryRun === true;
+        // Load config and resolve all stash sources
+        const { loadConfig } = await import("../core/config/config.js");
+        const config = loadConfig();
+        // One-time, read-only guard: warn if the writable stash still holds an
+        // un-migrated `vaults/` directory. In 0.9.0 the indexer skips `vaults/`
+        // entirely, so an unmigrated vault's `.env` data would silently never be
+        // indexed. Non-destructive — only stats, never reads/writes/deletes.
+        const { warnOnUnmigratedVaults } = await import("./usage/unmigrated-vaults-guard.js");
+        warnOnUnmigratedVaults(stashDir);
+        // Ensure git stash caches are extracted before resolving stash dirs,
+        // so their content directories exist on disk for the walker to discover.
+        const sourceCacheStart = Date.now();
+        onProgress({ phase: "preflight", message: "Hydrating source caches." });
+        const { ensureSourceCaches, resolveSourceEntries } = await import("./search/search-source.js");
+        await ensureSourceCaches(config, { force: full });
+        const sourceCacheEnd = Date.now();
+        const allSourceEntries = resolveSourceEntries(stashDir, config);
+        const allSourceDirs = allSourceEntries.map((s) => s.path);
         onProgress({
-            phase: "summary",
-            message: buildIndexSummaryMessage({
-                mode: isIncremental ? "incremental" : "full",
-                sourcesCount: allSourceDirs.length,
-                semanticSearchMode: config.semanticSearchMode,
-                embeddingProvider: getEmbeddingProvider(config.embedding),
-                llmEnabled: !!resolveIndexPassLLM("enrichment", config),
-                vecAvailable: isVecAvailable(db),
-            }),
+            phase: "preflight",
+            message: `Resolved ${allSourceDirs.length} stash source${allSourceDirs.length === 1 ? "" : "s"}.`,
         });
-        // ── Phase sequence ───────────────────────────────────────────────────────
-        await runSourceCachePhase(ctx);
-        await runWalkPhase(ctx);
-        await runEmbeddingPhase(ctx);
-        await runFinalizePhase(ctx);
-        // ────────────────────────────────────────────────────────────────────────
-        const { _verification: verification, _totalEntries: totalEntries } = ctx;
-        const { timing } = ctx;
-        // ── Clean pass ───────────────────────────────────────────────────────────
-        // After the normal index completes, remove entries whose source files no
-        // longer exist on disk. Remote entries (empty file_path) are skipped.
-        let cleanResult;
-        if (clean) {
-            cleanResult = runCleanPass(db, dryRun);
+        const t0 = Date.now();
+        // Open database — pass embedding dimension from config if available
+        const dbPath = getDbPath();
+        const embeddingDim = config.embedding?.dimension;
+        const db = openIndexDatabase(dbPath, embeddingDim ? { embeddingDim } : undefined);
+        try {
+            // Determine incremental vs full mode
+            const prevStashDir = getMeta(db, "stashDir");
+            const prevBuiltAt = getMeta(db, "builtAt");
+            const isIncremental = !full && prevStashDir === stashDir && !!prevBuiltAt;
+            const builtAtMs = isIncremental && prevBuiltAt ? new Date(prevBuiltAt).getTime() : 0;
+            // Assemble the run context
+            const ctx = {
+                db,
+                config,
+                sources: allSourceEntries,
+                sourceDirs: allSourceDirs,
+                full,
+                reEnrich,
+                stashDir,
+                onProgress,
+                signal,
+                timing: {
+                    t0,
+                    tWalkStart: t0,
+                    tWalkEnd: t0,
+                    tLlmEnd: t0,
+                    tFtsEnd: t0,
+                    tEmbedEnd: t0,
+                    tFinalizeStart: t0,
+                    tFinalizeEnd: t0,
+                },
+                isIncremental,
+                builtAtMs,
+                hadRemovedSources: false,
+                scannedDirs: 0,
+                skippedDirs: 0,
+                generatedCount: 0,
+                walkWarnings: [],
+                dirsNeedingLlm: [],
+                embeddingResult: null,
+            };
+            onProgress({
+                phase: "summary",
+                message: buildIndexSummaryMessage({
+                    mode: isIncremental ? "incremental" : "full",
+                    sourcesCount: allSourceDirs.length,
+                    semanticSearchMode: config.semanticSearchMode,
+                    embeddingProvider: getEmbeddingProvider(config.embedding),
+                    llmEnabled: !!resolveIndexPassLLM("enrichment", config),
+                    vecAvailable: isVecAvailable(db),
+                }),
+            });
+            // ── Phase sequence ───────────────────────────────────────────────────────
+            await runSourceCachePhase(ctx);
+            await runWalkPhase(ctx);
+            await runEmbeddingPhase(ctx);
+            await runFinalizePhase(ctx);
+            // ────────────────────────────────────────────────────────────────────────
+            // runFinalizePhase always populates these before returning.
+            const verification = ctx.verification;
+            const totalEntries = ctx.totalEntries;
+            const { timing } = ctx;
+            // ── Clean pass ───────────────────────────────────────────────────────────
+            // After the normal index completes, remove entries whose source files no
+            // longer exist on disk. Remote entries (empty file_path) are skipped.
+            let cleanResult;
+            const cleanStart = Date.now();
+            if (clean) {
+                onProgress({
+                    phase: "finalize",
+                    message: dryRun ? "Scanning for stale index entries (dry run)." : "Removing stale index entries.",
+                });
+                cleanResult = runCleanPass(db, dryRun);
+            }
+            const cleanEnd = Date.now();
+            // ────────────────────────────────────────────────────────────────────────
+            return {
+                stashDir,
+                totalEntries,
+                generatedMetadata: ctx.generatedCount,
+                indexPath: dbPath,
+                mode: isIncremental ? "incremental" : "full",
+                directoriesScanned: ctx.scannedDirs,
+                directoriesSkipped: ctx.skippedDirs,
+                ...(ctx.walkWarnings.length > 0 ? { warnings: ctx.walkWarnings } : {}),
+                verification,
+                timing: {
+                    totalMs: Date.now() - timing.t0,
+                    walkMs: timing.tWalkEnd - timing.tWalkStart,
+                    llmMs: timing.tLlmEnd - timing.tWalkEnd,
+                    embedMs: timing.tEmbedEnd - timing.tLlmEnd,
+                    ftsMs: timing.tFtsEnd - timing.tEmbedEnd,
+                    finalizeMs: timing.tFinalizeEnd - timing.tFinalizeStart,
+                    cleanMs: clean ? cleanEnd - cleanStart : 0,
+                    preflightMs: timing.t0 - requestedAt,
+                    leaseWaitMs: acquiredAt - requestedAt,
+                    sourceCacheMs: sourceCacheEnd - sourceCacheStart,
+                    endToEndMs: Date.now() - requestedAt,
+                },
+                ...(cleanResult !== undefined ? { clean: cleanResult } : {}),
+            };
         }
-        // ────────────────────────────────────────────────────────────────────────
-        return {
-            stashDir,
-            totalEntries,
-            generatedMetadata: ctx.generatedCount,
-            indexPath: dbPath,
-            mode: isIncremental ? "incremental" : "full",
-            directoriesScanned: ctx.scannedDirs,
-            directoriesSkipped: ctx.skippedDirs,
-            ...(ctx.walkWarnings.length > 0 ? { warnings: ctx.walkWarnings } : {}),
-            verification,
-            timing: {
-                totalMs: Date.now() - timing.t0,
-                walkMs: timing.tWalkEnd - timing.tWalkStart,
-                llmMs: timing.tLlmEnd - timing.tWalkEnd,
-                embedMs: timing.tEmbedEnd - timing.tLlmEnd,
-                ftsMs: timing.tFtsEnd - timing.tEmbedEnd,
-            },
-            ...(cleanResult !== undefined ? { clean: cleanResult } : {}),
-        };
-    }
-    finally {
-        closeDatabase(db);
-    }
+        finally {
+            closeDatabase(db);
+        }
+    });
 }
-// ── Extracted helpers for indexing ────────────────────────────────────────────
-async function indexEntries(db, allSourceEntries, isIncremental, builtAtMs, hadRemovedSources, doFullDelete = false, onProgress) {
+/**
+ * Phase 1 (async): walk every source directory and pre-generate all metadata
+ * outside any transaction, producing the per-directory scan records that
+ * {@link persistDirRecords} later writes.
+ *
+ * generateMetadataFlat is async (uses dynamic import for the matcher/renderer
+ * registry), so it cannot run inside a db.transaction() callback — hence the
+ * split into a pure-ish scan pass and a synchronous persist pass.
+ */
+async function scanSourceDirs(db, allSourceEntries, isIncremental, builtAtMs, hadRemovedSources, onProgress) {
     let scannedDirs = 0;
     let skippedDirs = 0;
     let generatedCount = 0;
     const warnings = [];
     const seenPaths = new Set();
-    const dirsNeedingLlm = [];
     const dirRecords = [];
     let processedDirs = 0;
     let priorDirsChanged = hadRemovedSources;
@@ -366,6 +432,53 @@ async function indexEntries(db, allSourceEntries, isIncremental, builtAtMs, hadR
         reportScanProgress(`${kind === "scan" ? "Rescanning" : "Skipping"} ${path.relative(currentStashDir, dirPath) || "."} ` +
             `from ${currentStashDir}: ${reason.kind}${detail}${rowInfo}`);
     };
+    // Duplicate-directory guard shared by the wiki-root and normal branches. A
+    // dir may surface via multiple stash roots; only the first occurrence is
+    // indexed, and the later ones are recorded as skips. Returns true when the
+    // dir was already seen (caller should skip further processing).
+    const markSeenOrSkipDuplicate = (dirPath, currentStashDir, files) => {
+        const resolved = path.resolve(dirPath);
+        if (seenPaths.has(resolved)) {
+            const reason = { kind: "duplicate-dir" };
+            dirRecords.push({ dirPath, currentStashDir, files, stash: null, skip: true, reason });
+            reportDirDecision("skip", dirPath, currentStashDir, reason);
+            return true;
+        }
+        seenPaths.add(resolved);
+        return false;
+    };
+    // Incremental freshness gate shared by both branches: consult the persisted
+    // dir state and record either a skip (unchanged + eligible for incremental
+    // skip) or a scan record carrying the candidate stash.
+    const recordFreshnessDecision = (dirPath, currentStashDir, stateFiles, stash) => {
+        const previousState = getDirIndexState(db, dirPath, stateFiles, builtAtMs);
+        if (isIncremental && !previousState.stale && canUseIncrementalSkip(previousState, priorDirsChanged)) {
+            skippedDirs++;
+            dirRecords.push({
+                dirPath,
+                currentStashDir,
+                files: stateFiles,
+                stash: null,
+                skip: true,
+                reason: previousState.reason,
+            });
+            reportDirDecision("skip", dirPath, currentStashDir, previousState.reason, previousState.persistedRowCount);
+            return;
+        }
+        scannedDirs++;
+        priorDirsChanged = true;
+        const reason = isIncremental ? previousState.reason : { kind: "full-rebuild" };
+        dirRecords.push({
+            dirPath,
+            currentStashDir,
+            files: stateFiles,
+            stash,
+            skip: false,
+            reason,
+            persistedRowCount: previousState.persistedRowCount,
+        });
+        reportDirDecision("scan", dirPath, currentStashDir, reason, previousState.persistedRowCount);
+    };
     for (const sourceAdded of allSourceEntries) {
         const currentStashDir = sourceAdded.path;
         const fileContexts = walkStashFlat(currentStashDir);
@@ -403,33 +516,9 @@ async function indexEntries(db, allSourceEntries, isIncremental, builtAtMs, hadR
                 }
             }
             for (const [dirPath, { files, entries }] of wikiDirGroups) {
-                if (seenPaths.has(path.resolve(dirPath))) {
-                    const reason = { kind: "duplicate-dir" };
-                    dirRecords.push({ dirPath, currentStashDir, files, stash: null, skip: true, reason });
-                    reportDirDecision("skip", dirPath, currentStashDir, reason);
+                if (markSeenOrSkipDuplicate(dirPath, currentStashDir, files))
                     continue;
-                }
-                seenPaths.add(path.resolve(dirPath));
-                const previousState = getDirIndexState(db, dirPath, files, builtAtMs);
-                if (isIncremental && !previousState.stale && canUseIncrementalSkip(previousState, priorDirsChanged)) {
-                    skippedDirs++;
-                    dirRecords.push({ dirPath, currentStashDir, files, stash: null, skip: true, reason: previousState.reason });
-                    reportDirDecision("skip", dirPath, currentStashDir, previousState.reason, previousState.persistedRowCount);
-                    continue;
-                }
-                scannedDirs++;
-                priorDirsChanged = true;
-                const reason = isIncremental ? previousState.reason : { kind: "full-rebuild" };
-                dirRecords.push({
-                    dirPath,
-                    currentStashDir,
-                    files,
-                    stash: { entries },
-                    skip: false,
-                    reason,
-                    persistedRowCount: previousState.persistedRowCount,
-                });
-                reportDirDecision("scan", dirPath, currentStashDir, reason, previousState.persistedRowCount);
+                recordFreshnessDecision(dirPath, currentStashDir, files, { entries });
             }
             continue;
         }
@@ -444,13 +533,8 @@ async function indexEntries(db, allSourceEntries, isIncremental, builtAtMs, hadR
         }
         for (const [dirPath, files] of dirGroups) {
             const indexableFiles = files.filter((file) => shouldIndexStashFile(currentStashDir, file));
-            if (seenPaths.has(path.resolve(dirPath))) {
-                const reason = { kind: "duplicate-dir" };
-                dirRecords.push({ dirPath, currentStashDir, files: indexableFiles, stash: null, skip: true, reason });
-                reportDirDecision("skip", dirPath, currentStashDir, reason);
+            if (markSeenOrSkipDuplicate(dirPath, currentStashDir, indexableFiles))
                 continue;
-            }
-            seenPaths.add(path.resolve(dirPath));
             if (indexableFiles.length === 0) {
                 skippedDirs++;
                 const reason = { kind: "no-indexable-files" };
@@ -480,37 +564,17 @@ async function indexEntries(db, allSourceEntries, isIncremental, builtAtMs, hadR
             if (generated.entries.length > 0) {
                 generatedCount += generated.entries.length;
             }
-            const previousState = getDirIndexState(db, dirPath, staleFiles, builtAtMs);
-            if (isIncremental && !previousState.stale && canUseIncrementalSkip(previousState, priorDirsChanged)) {
-                skippedDirs++;
-                dirRecords.push({
-                    dirPath,
-                    currentStashDir,
-                    files: staleFiles,
-                    stash: null,
-                    skip: true,
-                    reason: previousState.reason,
-                });
-                reportDirDecision("skip", dirPath, currentStashDir, previousState.reason, previousState.persistedRowCount);
-                continue;
-            }
-            scannedDirs++;
-            priorDirsChanged = true;
-            const reason = isIncremental ? previousState.reason : { kind: "full-rebuild" };
-            dirRecords.push({
-                dirPath,
-                currentStashDir,
-                files: staleFiles,
-                stash,
-                skip: false,
-                reason,
-                persistedRowCount: previousState.persistedRowCount,
-            });
-            reportDirDecision("scan", dirPath, currentStashDir, reason, previousState.persistedRowCount);
+            recordFreshnessDecision(dirPath, currentStashDir, staleFiles, stash);
         }
     }
-    // Phase 2 (sync): write all pre-generated metadata inside a single transaction.
-    //
+    return { dirRecords, scannedDirs, skippedDirs, generatedCount, warnings };
+}
+/**
+ * Phase 2 (sync): write all pre-generated scan records inside a single
+ * transaction, returning the directories that still need LLM enrichment.
+ */
+function persistDirRecords(db, dirRecords, doFullDelete, warnings) {
+    const dirsNeedingLlm = [];
     // Cross-stash dedup: track indexed assets by content identity
     // (type + filename + description) so the same asset from a lower-priority
     // stash root is skipped when a higher-priority root already covers it.
@@ -632,101 +696,17 @@ async function indexEntries(db, allSourceEntries, isIncremental, builtAtMs, hadR
         }
     });
     insertTransaction();
-    return { scannedDirs, skippedDirs, generatedCount, warnings, dirsNeedingLlm };
-}
-function getDirIndexState(db, dirPath, files, builtAtMs) {
-    const prevEntries = getEntriesByDir(db, dirPath);
-    const fingerprint = computeDirFingerprint(dirPath, files);
-    if (prevEntries.length > 0) {
-        const staleReason = getDirStaleReason(dirPath, files, prevEntries, builtAtMs);
-        if (!staleReason) {
-            return { stale: false, reason: { kind: "unchanged" }, persistedRowCount: prevEntries.length };
-        }
-        return { stale: true, reason: staleReason, persistedRowCount: prevEntries.length };
-    }
-    const cachedState = getIndexDirState(db, dirPath);
-    if (cachedState &&
-        cachedState.fileSetHash === fingerprint.fileSetHash &&
-        cachedState.fileMtimeMaxMs === fingerprint.fileMtimeMaxMs) {
-        return {
-            stale: false,
-            reason: { kind: "cached-zero-row-state", detail: cachedState.reason },
-            persistedRowCount: 0,
-        };
-    }
-    return {
-        stale: true,
-        reason: { kind: "no-previous-rows", detail: cachedState ? `cached=${cachedState.reason}` : undefined },
-        persistedRowCount: 0,
-    };
-}
-function getCachedZeroRowDirState(db, dirPath, files, builtAtMs, priorDirsChanged) {
-    const state = getDirIndexState(db, dirPath, files, builtAtMs);
-    if (state.stale || state.reason.kind !== "cached-zero-row-state")
-        return undefined;
-    if (!canUseIncrementalSkip(state, priorDirsChanged))
-        return undefined;
-    return state;
-}
-function canUseIncrementalSkip(state, priorDirsChanged) {
-    return !(priorDirsChanged &&
-        state.reason.kind === "cached-zero-row-state" &&
-        state.reason.detail === "deduped-zero-row");
-}
-function computeDirFingerprint(_dirPath, files) {
-    const normalizedFiles = [...new Set(files.map((file) => path.basename(file)))].sort();
-    let fileMtimeMaxMs = 0;
-    for (const file of files) {
-        try {
-            fileMtimeMaxMs = Math.max(fileMtimeMaxMs, fs.statSync(file).mtimeMs);
-        }
-        catch {
-            fileMtimeMaxMs = Number.POSITIVE_INFINITY;
-            break;
-        }
-    }
-    return {
-        fileSetHash: normalizedFiles.join("\0"),
-        fileMtimeMaxMs,
-    };
-}
-function getDirStaleReason(_dirPath, currentFiles, previousEntries, builtAtMs) {
-    const prevFileNames = new Set(previousEntries
-        .map((ie) => {
-        const fromPath = path.basename(ie.filePath);
-        return fromPath || ie.entry.filename;
-    })
-        .filter((e) => !!e));
-    const currFileNames = new Set(currentFiles.map((f) => path.basename(f)));
-    if (prevFileNames.size !== currFileNames.size) {
-        return { kind: "file-set-changed", detail: `${prevFileNames.size} -> ${currFileNames.size} files` };
-    }
-    for (const name of currFileNames) {
-        if (!prevFileNames.has(name))
-            return { kind: "file-set-changed", detail: name };
-    }
-    for (const file of currentFiles) {
-        try {
-            if (fs.statSync(file).mtimeMs > builtAtMs)
-                return { kind: "mtime-changed", detail: path.basename(file) };
-        }
-        catch {
-            return { kind: "missing-file", detail: path.basename(file) };
-        }
-    }
-    return undefined;
+    return { dirsNeedingLlm };
 }
-function inferZeroRowReason(stash, priorReason, warnings, dirPath, dedupedRows) {
-    if (dedupedRows > 0)
-        return "deduped-zero-row";
-    const workflowNoise = warnings.some((warning) => warning.startsWith("Skipped workflow ") && warning.includes(dirPath));
-    if (workflowNoise)
-        return "workflow-noise";
-    if (!stash || stash.entries.length === 0)
-        return "empty-generated-set";
-    return `zero-row:${priorReason?.kind ?? "unknown"}`;
+async function indexEntries(db, allSourceEntries, isIncremental, builtAtMs, hadRemovedSources, doFullDelete = false, onProgress) {
+    // Phase 1 (async): walk directories and pre-generate all metadata outside the
+    // transaction.
+    const { dirRecords, scannedDirs, skippedDirs, generatedCount, warnings } = await scanSourceDirs(db, allSourceEntries, isIncremental, builtAtMs, hadRemovedSources, onProgress);
+    // Phase 2 (sync): write all pre-generated metadata inside a single transaction.
+    const { dirsNeedingLlm } = persistDirRecords(db, dirRecords, doFullDelete, warnings);
+    return { scannedDirs, skippedDirs, generatedCount, warnings, dirsNeedingLlm };
 }
-async function enhanceDirsWithLlm(db, config, dirsNeedingLlm, onProgress, signal, _enrich = false, reEnrich = false) {
+async function enhanceDirsWithLlm(db, config, dirsNeedingLlm, onProgress, signal, reEnrich = false) {
     // Resolve per-pass LLM config via the unified shim. Returns undefined when
     // either no `akm.llm` is configured or the user opted this pass out via
     // `index.enrichment.llm = false`. (#208)
@@ -896,21 +876,9 @@ async function generateEmbeddingsForDb(db, config, onProgress, signal) {
     const currentFingerprint = deriveSemanticProviderFingerprint(config.embedding);
     const storedFingerprint = getMeta(db, "embeddingFingerprint");
     if (storedFingerprint && storedFingerprint !== currentFingerprint) {
-        try {
-            db.exec("DELETE FROM embeddings");
-        }
-        catch {
-            /* ignore */
-        }
-        if (isVecAvailable(db)) {
-            try {
-                db.exec("DELETE FROM entries_vec");
-            }
-            catch {
-                /* ignore */
-            }
-        }
-        setMeta(db, "hasEmbeddings", "0");
+        // Model/provider changed → stored vectors are incompatible. Clear them
+        // (same dimension, so keep the vec table); re-embedded by this index run.
+        purgeEmbeddings(db);
     }
     try {
         const { embedBatch } = await import("../llm/embedder.js");
@@ -940,31 +908,44 @@ async function generateEmbeddingsForDb(db, config, onProgress, signal) {
                 warnVerbose(`[embed] ${ref} (${chars} chars, est. ${tokens} tokens) → batch ${batchNum}/${totalBatches}`);
             }
         }
-        const embeddings = await embedBatch(texts, config.embedding, signal);
-        throwIfAborted(signal);
-        // Wrap all embedding upserts in a single transaction so partial
-        // state is rolled back on failure rather than leaving the table half-filled.
-        let storedCount = 0;
-        let skippedCount = 0;
-        db.transaction(() => {
-            for (let i = 0; i < allEntries.length; i++) {
-                if (upsertEmbedding(db, allEntries[i].id, embeddings[i])) {
-                    storedCount++;
-                }
-                else {
-                    skippedCount++;
+        let heartbeatTimer;
+        try {
+            heartbeatTimer = setInterval(() => {
+                onProgress({
+                    phase: "embeddings",
+                    message: `Still generating embeddings for ${allEntries.length} entr${allEntries.length === 1 ? "y" : "ies"}; waiting on embedding provider.`,
+                });
+            }, 15000);
+            const embeddings = await embedBatch(texts, config.embedding, signal);
+            throwIfAborted(signal);
+            // Wrap all embedding upserts in a single transaction so partial
+            // state is rolled back on failure rather than leaving the table half-filled.
+            let storedCount = 0;
+            let skippedCount = 0;
+            db.transaction(() => {
+                for (let i = 0; i < allEntries.length; i++) {
+                    if (upsertEmbedding(db, allEntries[i].id, embeddings[i])) {
+                        storedCount++;
+                    }
+                    else {
+                        skippedCount++;
+                    }
                 }
+            })();
+            if (skippedCount > 0) {
+                warn(`[embed] ${skippedCount} embedding${skippedCount === 1 ? "" : "s"} skipped (entry deleted between queue and write)`);
             }
-        })();
-        if (skippedCount > 0) {
-            warn(`[embed] ${skippedCount} embedding${skippedCount === 1 ? "" : "s"} skipped (entry deleted between queue and write)`);
+            onProgress({
+                phase: "embeddings",
+                message: `Stored ${storedCount} embedding${storedCount === 1 ? "" : "s"}.`,
+            });
+            setMeta(db, "embeddingFingerprint", currentFingerprint);
+            return { success: true };
+        }
+        finally {
+            if (heartbeatTimer)
+                clearInterval(heartbeatTimer);
         }
-        onProgress({
-            phase: "embeddings",
-            message: `Stored ${storedCount} embedding${storedCount === 1 ? "" : "s"}.`,
-        });
-        setMeta(db, "embeddingFingerprint", currentFingerprint);
-        return { success: true };
     }
     catch (error) {
         const message = error instanceof Error ? error.message : String(error);
@@ -1075,7 +1056,7 @@ function resolveIndexedFiles(dirPath, files, stash) {
     for (const entry of stash.entries) {
         const entryPath = entry.filename
             ? path.join(dirPath, entry.filename)
-            : matchEntryToFile(entry.name, fileBasenameMap, files);
+            : matchEntryToFile(entry.name, fileBasenameMap);
         if (entryPath)
             resolved.add(entryPath);
     }
@@ -1194,7 +1175,7 @@ export function buildFileBasenameMap(files) {
  *      try matching the last segment
  *   3. No implicit file fallback: ambiguous legacy entries are skipped
  */
-export function matchEntryToFile(entryName, fileMap, _files) {
+export function matchEntryToFile(entryName, fileMap) {
     // Exact match on entry name
     const exact = fileMap.get(entryName);
     if (exact)