@ulpi/cli 0.1.5 → 0.1.6

package/dist/{chunk-6OURRFP7.js → chunk-IV6MWETF.js}

@@ -1,16 +1,13 @@
+import {
+  extractTagsFromTree
+} from "./chunk-AWOSRA5F.js";
 import {
   commitInWorktree,
   copyAndStage,
   historyBranchExists,
   withWorktree,
   writeAndStage
-} from "./chunk-L3PWNHSA.js";
-import {
-  CodemapConfigSchema
-} from "./chunk-74WVVWJ4.js";
-import {
-  extractTagsFromTree
-} from "./chunk-DOIKS6C5.js";
+} from "./chunk-3WB5CXH4.js";
 import {
   LOGS_DIR,
   codemapBranchDir,
@@ -23,7 +20,10 @@ import {
   getCodemapBranch,
   getCurrentBranch,
   projectCodemapDir
-} from "./chunk-DDRLI6JU.js";
+} from "./chunk-C7CLUQI6.js";
+import {
+  CodemapConfigSchema
+} from "./chunk-YOKL7RB5.js";
 
 // ../../packages/codemap-engine/dist/index.js
 import * as fs from "fs";
@@ -47,6 +47,174 @@ import * as path7 from "path";
 import { connect } from "@lancedb/lancedb";
 import * as fs9 from "fs";
 import * as path11 from "path";
+
+// ../../packages/embed-engine/dist/index.js
+async function createEmbedder(config) {
+  switch (config.provider) {
+    case "openai": {
+      const { OpenAIEmbedder: OpenAIEmbedder2 } = await import("./openai-E7G2YAHU-T3HMBPH7.js");
+      return new OpenAIEmbedder2(config.model, config.dimensions);
+    }
+    case "ollama": {
+      const { OllamaEmbedder: OllamaEmbedder2 } = await import("./ollama-3XCUZMZT-4JMH6B7P.js");
+      return new OllamaEmbedder2(config.model, config.dimensions);
+    }
+    case "ulpi": {
+      const { UlpiEmbedder: UlpiEmbedder2 } = await import("./ulpi-5EN6JCAS-LFE3WSL4.js");
+      return new UlpiEmbedder2(config.model, config.dimensions, config.baseUrl);
+    }
+    default:
+      throw new Error(`Unknown embedding provider: ${config.provider}`);
+  }
+}
+async function embedOne(embedder, text) {
+  const [vector] = await embedder.embed([text]);
+  return vector;
+}
+async function embedAll(embedder, texts, options) {
+  if (texts.length === 0) return [];
+  const {
+    onProgress,
+    onBatch,
+    chunkSize = 100,
+    timeoutMs = 3e5,
+    pollIntervalMs = 2e3,
+    batchSubmitSize = 1e3
+  } = options ?? {};
+  if (embedder.supportsBatch && embedder.submitBatch && embedder.pollBatch) {
+    return embedAllViaBatch(embedder, texts, {
+      onProgress,
+      onBatch,
+      timeoutMs,
+      pollIntervalMs,
+      batchSubmitSize
+    });
+  }
+  return embedAllViaSync(embedder, texts, {
+    onProgress,
+    onBatch,
+    chunkSize
+  });
+}
+async function embedAllViaBatch(embedder, texts, opts) {
+  const { onProgress, onBatch, timeoutMs, pollIntervalMs, batchSubmitSize } = opts;
+  const total = texts.length;
+  const collected = onBatch ? [] : new Array(total);
+  let globalProcessed = 0;
+  const MAX_POLL_ERRORS = 5;
+  for (let offset = 0; offset < total; offset += batchSubmitSize) {
+    const end = Math.min(offset + batchSubmitSize, total);
+    const batchTexts = texts.slice(offset, end);
+    const batchNum = Math.floor(offset / batchSubmitSize) + 1;
+    const totalBatches = Math.ceil(total / batchSubmitSize);
+    onProgress?.({
+      current: globalProcessed,
+      total,
+      message: `Submitting batch ${batchNum}/${totalBatches} (${batchTexts.length} texts)...`
+    });
+    const batchId = await embedder.submitBatch(batchTexts);
+    console.error(
+      `[embed-engine] Batch ${batchNum}/${totalBatches} submitted (${batchId.slice(0, 8)}...), starting poll loop...`
+    );
+    onProgress?.({
+      current: globalProcessed,
+      total,
+      message: `Batch ${batchNum}/${totalBatches} (${batchId.slice(0, 8)}...) polling...`
+    });
+    const deadline = timeoutMs > 0 ? Date.now() + timeoutMs : 0;
+    let watermark = 0;
+    let pollCount = 0;
+    let consecutiveErrors = 0;
+    while (deadline === 0 || Date.now() < deadline) {
+      pollCount++;
+      let status;
+      try {
+        status = await embedder.pollBatch(batchId);
+        consecutiveErrors = 0;
+      } catch (err) {
+        consecutiveErrors++;
+        const msg = err instanceof Error ? err.message : String(err);
+        console.error(
+          `[embed-engine] Poll error #${consecutiveErrors} for batch ${batchId.slice(0, 8)}...: ${msg}`
+        );
+        if (consecutiveErrors >= MAX_POLL_ERRORS) {
+          throw new Error(
+            `Batch polling failed after ${MAX_POLL_ERRORS} consecutive errors (batch ${batchId}). Last error: ${msg}`
+          );
+        }
+        await new Promise((resolve2) => setTimeout(resolve2, pollIntervalMs * 2));
+        continue;
+      }
+      if (status.results && status.results.length > watermark) {
+        const newStart = watermark;
+        const newCount = status.results.length - watermark;
+        const newEmbeddings = status.results.slice(newStart, newStart + newCount);
+        if (onBatch) {
+          await onBatch(newEmbeddings, offset + newStart, newCount);
+        } else {
+          for (let i = 0; i < newCount; i++) {
+            collected[offset + newStart + i] = newEmbeddings[i];
+          }
+        }
+        watermark = status.results.length;
+        console.error(
+          `[embed-engine] Batch ${batchId.slice(0, 8)}... delivered ${watermark}/${batchTexts.length} embeddings`
+        );
+      }
+      if (status.status === "failed") {
+        throw new Error(
+          `Batch embedding failed (batch ${batchId}). ${globalProcessed + watermark} of ${total} texts were processed before failure.`
+        );
+      }
+      const currentProcessed = globalProcessed + watermark;
+      onProgress?.({
+        current: currentProcessed,
+        total,
+        message: `Embedded ${currentProcessed}/${total}`
+      });
+      if (status.status === "completed") {
+        console.error(
+          `[embed-engine] Batch ${batchId.slice(0, 8)}... completed after ${pollCount} polls, ${watermark} embeddings`
+        );
+        break;
+      }
+      await new Promise((resolve2) => setTimeout(resolve2, pollIntervalMs));
+    }
+    if (watermark < batchTexts.length && deadline > 0 && Date.now() >= deadline) {
+      throw new Error(
+        `Batch embedding timed out after ${Math.round(timeoutMs / 1e3)}s (batch ${batchId}). ${globalProcessed + watermark} of ${total} texts were processed.`
+      );
+    }
+    globalProcessed += batchTexts.length;
+  }
+  console.error(
+    `[embed-engine] All ${Math.ceil(total / batchSubmitSize)} batches complete, ${globalProcessed} embeddings total`
+  );
+  return onBatch ? [] : collected;
+}
+async function embedAllViaSync(embedder, texts, opts) {
+  const { onProgress, onBatch, chunkSize } = opts;
+  const total = texts.length;
+  const collected = [];
+  for (let i = 0; i < total; i += chunkSize) {
+    const chunk = texts.slice(i, i + chunkSize);
+    const embeddings = await embedder.embed(chunk);
+    if (onBatch) {
+      await onBatch(embeddings, i, embeddings.length);
+    } else {
+      collected.push(...embeddings);
+    }
+    const processed = Math.min(i + chunkSize, total);
+    onProgress?.({
+      current: processed,
+      total,
+      message: `Embedded ${processed}/${total}`
+    });
+  }
+  return onBatch ? [] : collected;
+}
+
+// ../../packages/codemap-engine/dist/index.js
 import * as fs6 from "fs";
 import * as path8 from "path";
 import * as fs7 from "fs";
@@ -931,24 +1099,6 @@ function diffManifest(oldManifest, newFiles) {
   }
   return { added, updated, removed };
 }
-async function createEmbedder(config) {
-  switch (config.provider) {
-    case "openai": {
-      const { OpenAIEmbedder: OpenAIEmbedder2 } = await import("./openai-E7G2YAHU-IG33BFYF.js");
-      return new OpenAIEmbedder2(config.model, config.dimensions);
-    }
-    case "ollama": {
-      const { OllamaEmbedder: OllamaEmbedder2 } = await import("./ollama-3XCUZMZT-FYKHW4TZ.js");
-      return new OllamaEmbedder2(config.model, config.dimensions);
-    }
-    case "ulpi": {
-      const { UlpiEmbedder } = await import("./ulpi-RMMCUAGP-EWYUE7RU.js");
-      return new UlpiEmbedder(config.model, config.dimensions, config.baseUrl);
-    }
-    default:
-      throw new Error(`Unknown embedding provider: ${config.provider}`);
-  }
-}
 var CodemapStore = class {
   db = null;
   table = null;
@@ -1002,14 +1152,44 @@ var CodemapStore = class {
       await this.table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute(records);
     }
   }
+  /**
+   * Append items to the store without merge/upsert semantics.
+   * Uses simple `add()` which only creates data fragments (no `_deletions/` files).
+   * Use this for bulk initial indexing where no duplicate IDs exist.
+   */
+  async appendItems(items) {
+    if (items.length === 0) return;
+    const records = items.map((item) => ({
+      id: item.id,
+      vector: item.vector,
+      filePath: item.metadata.filePath,
+      startLine: item.metadata.startLine,
+      endLine: item.metadata.endLine,
+      snippet: item.metadata.snippet
+    }));
+    if (!this.table) {
+      this.table = await this.db.createTable("chunks", records);
+    } else {
+      await this.table.add(records);
+    }
+  }
   /**
    * Remove items by their chunk IDs.
    * Uses SQL IN clause with properly escaped string literals.
+   * Silently ignores errors from missing data files (stale fragment references).
    */
   async removeItems(chunkIds) {
     if (!this.table || chunkIds.length === 0) return;
     const quoted = chunkIds.map((id) => "'" + id.replace(/'/g, "''") + "'").join(", ");
-    await this.table.delete("id IN (" + quoted + ")");
+    try {
+      await this.table.delete("id IN (" + quoted + ")");
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      if (/Not found.*\.lance/i.test(msg) || /LanceError/i.test(msg)) {
+        return;
+      }
+      throw err;
+    }
   }
   /**
    * Query the store for similar vectors.
@@ -1034,6 +1214,26 @@ var CodemapStore = class {
     if (!this.table) return 0;
     return this.table.countRows();
   }
+  /**
+   * Close the LanceDB connection and table, releasing underlying native resources.
+   * After calling close(), the store cannot be used — create a new instance instead.
+   */
+  close() {
+    if (this.table) {
+      try {
+        this.table.close();
+      } catch {
+      }
+      this.table = null;
+    }
+    if (this.db) {
+      try {
+        this.db.close();
+      } catch {
+      }
+      this.db = null;
+    }
+  }
   /**
    * Get the index directory size in bytes.
    * Walks recursively since LanceDB stores data in subdirectories with fragment files.
@@ -1663,7 +1863,7 @@ async function runPipelineInner(projectDir, onProgress, branch) {
   let depgraphResult;
   if (allTags.size > 0) {
     try {
-      const { buildReferenceGraph, computePageRank, savePageRank, computeMetrics, saveMetrics, saveGraph } = await import("./dist-WAMAQVPK.js");
+      const { buildReferenceGraph, computePageRank, savePageRank, computeMetrics, saveMetrics, saveGraph } = await import("./dist-4U5L2X2C.js");
       onProgress?.({
         phase: "graph",
         current: 0,
@@ -1735,6 +1935,7 @@ async function runPipelineInner(projectDir, onProgress, branch) {
   };
   writeStats(projectDir, resolvedBranch, stats);
   if (embeddingError) {
+    store.close();
     onProgress?.({
       phase: "finalizing",
       current: 3,
@@ -1749,6 +1950,7 @@ async function runPipelineInner(projectDir, onProgress, branch) {
     total: 3,
     message: "Done"
   });
+  store.close();
   return {
     totalFiles: scannedFiles.length,
     totalChunks: chunks.length,
@@ -1819,7 +2021,7 @@ async function rebuildDepgraphInner(projectDir, onProgress, branch) {
     log("warn", "No tags extracted \u2014 cannot build depgraph");
     return { nodeCount: 0, edgeCount: 0, definitionCount: 0, referenceCount: 0, cycleCount: 0, durationMs: Date.now() - startTime, taggedFiles: 0, totalFiles: scannedFiles.length };
   }
-  const { buildReferenceGraph, computePageRank, savePageRank, computeMetrics, saveMetrics, saveGraph } = await import("./dist-WAMAQVPK.js");
+  const { buildReferenceGraph, computePageRank, savePageRank, computeMetrics, saveMetrics, saveGraph } = await import("./dist-4U5L2X2C.js");
   onProgress?.({ phase: "graph", current: 0, total: 3, message: "Building dependency graph..." });
   const graph = buildReferenceGraph(allTags, filePaths);
   for (const sf of scannedFiles) {
@@ -1854,121 +2056,39 @@ var STORE_BATCH_SIZE = 5e3;
 async function embedAndStore(embedder, texts, chunks, store, batchConfig, onProgress) {
   if (texts.length === 0) return;
   const total = texts.length;
-  if (embedder.supportsBatch && embedder.submitBatch && embedder.pollBatch) {
-    await embedAndStoreBatch(embedder, texts, chunks, store, batchConfig, onProgress);
-    return;
-  }
   onProgress?.({
     phase: "embedding",
     current: 0,
     total,
     message: "Creating embeddings..."
   });
-  const allEmbeddings = [];
-  for (let i = 0; i < texts.length; i += batchConfig.size) {
-    const batch = texts.slice(i, i + batchConfig.size);
-    const embeddings = await embedder.embed(batch);
-    allEmbeddings.push(...embeddings);
-    const processed = Math.min(i + batchConfig.size, total);
-    onProgress?.({
-      phase: "embedding",
-      current: processed,
-      total,
-      message: `Embedded ${processed}/${total} chunks`
-    });
-  }
-  onProgress?.({
-    phase: "storing",
-    current: 0,
-    total,
-    message: "Storing vectors..."
-  });
-  for (let i = 0; i < allEmbeddings.length; i += STORE_BATCH_SIZE) {
-    const items = buildVectorItems(chunks, allEmbeddings, i, Math.min(i + STORE_BATCH_SIZE, allEmbeddings.length));
-    await store.upsertItems(items);
-    const stored = Math.min(i + STORE_BATCH_SIZE, allEmbeddings.length);
-    onProgress?.({
-      phase: "storing",
-      current: stored,
-      total,
-      message: `Stored ${stored}/${total} vectors`
-    });
-  }
-}
-async function embedAndStoreBatch(embedder, texts, chunks, store, batchConfig, onProgress) {
-  const total = texts.length;
-  onProgress?.({
-    phase: "embedding",
-    current: 0,
-    total,
-    message: `Submitting batch of ${total} texts...`
-  });
-  const batchId = await embedder.submitBatch(texts);
-  log("info", `Batch submitted: id=${batchId}, texts=${total}`);
-  onProgress?.({
-    phase: "embedding",
-    current: 0,
-    total,
-    message: `Batch ${batchId.slice(0, 8)}... submitted, polling...`
-  });
-  await pollAndStoreProgressively(embedder, batchId, chunks, store, batchConfig, total, onProgress);
-}
-async function pollAndStoreProgressively(embedder, batchId, chunks, store, batchConfig, total, onProgress) {
-  const deadline = batchConfig.timeoutMs > 0 ? Date.now() + batchConfig.timeoutMs : 0;
-  let pollCount = 0;
-  let watermark = 0;
-  while (deadline === 0 || Date.now() < deadline) {
-    const status = await embedder.pollBatch(batchId);
-    pollCount++;
-    if (pollCount <= 3 || pollCount % 10 === 0) {
-      log("info", `Poll #${pollCount} batch=${batchId.slice(0, 8)}: status=${status.status}, processed=${status.processedInputs}/${status.totalInputs}, watermark=${watermark}`);
-    }
-    if (status.results && status.results.length > watermark) {
-      const newCount = status.results.length - watermark;
-      try {
-        for (let i = watermark; i < status.results.length; i += STORE_BATCH_SIZE) {
-          const end = Math.min(i + STORE_BATCH_SIZE, status.results.length);
-          const items = buildVectorItems(chunks, status.results, i, end);
-          await store.upsertItems(items);
-        }
-        const prevWatermark = watermark;
-        watermark = status.results.length;
-        log("info", `Stored ${newCount} new embeddings (${prevWatermark}\u2192${watermark}/${total})`);
-      } catch (storeErr) {
-        log("error", `Store failed at watermark=${watermark}: ${storeErr instanceof Error ? storeErr.message : String(storeErr)}`);
-      }
-    }
-    if (status.status === "failed") {
-      log("error", `Batch ${batchId.slice(0, 8)} failed (watermark=${watermark}/${total} stored)`);
-      throw new Error(`Batch embedding failed (batch ${batchId}). ${watermark} of ${total} chunks were stored before failure.`);
-    }
-    onProgress?.({
-      phase: "embedding",
-      current: watermark,
-      total,
-      message: `Embedded & stored ${watermark}/${total} chunks`
-    });
-    if (status.status === "completed") {
-      log("info", `Batch ${batchId.slice(0, 8)} completed: ${watermark}/${total} stored`);
+  await embedAll(embedder, texts, {
+    onProgress: (info) => {
       onProgress?.({
         phase: "embedding",
-        current: total,
-        total,
-        message: `Embedded & stored ${total}/${total} chunks`
+        current: info.current,
+        total: info.total,
+        message: info.message
       });
-      return;
-    }
-    await new Promise((resolve2) => setTimeout(resolve2, batchConfig.pollIntervalMs));
-  }
-  log("error", `Batch ${batchId} timed out after ${Math.round(batchConfig.timeoutMs / 1e3)}s (${pollCount} polls, watermark=${watermark}/${total})`);
-  throw new Error(
-    `Batch embedding timed out after ${Math.round(batchConfig.timeoutMs / 1e3)}s (batch ${batchId}). ${watermark} of ${total} chunks were stored.`
-  );
+    },
+    onBatch: async (embeddings, startIndex, count) => {
+      for (let i = 0; i < count; i += STORE_BATCH_SIZE) {
+        const batchEnd = Math.min(i + STORE_BATCH_SIZE, count);
+        const items = buildVectorItemsFromSlice(chunks, embeddings, startIndex, i, batchEnd);
+        await store.appendItems(items);
+      }
+      log("info", `Stored ${count} embeddings at offset ${startIndex}`);
+    },
+    chunkSize: batchConfig.size,
+    timeoutMs: batchConfig.timeoutMs,
+    pollIntervalMs: batchConfig.pollIntervalMs,
+    batchSubmitSize: 1e3
+  });
 }
-function buildVectorItems(chunks, embeddings, from, to) {
+function buildVectorItemsFromSlice(chunks, embeddings, chunkOffset, embFrom, embTo) {
   const items = [];
-  for (let i = from; i < to; i++) {
-    const chunk = chunks[i];
+  for (let i = embFrom; i < embTo; i++) {
+    const chunk = chunks[chunkOffset + i];
     items.push({
       id: chunk.id,
       vector: embeddings[i],
@@ -2072,7 +2192,12 @@ async function searchCode(projectDir, query, options = {}) {
   const resolvedBranch = options.branch ?? getCurrentBranch(projectDir);
   const config = loadCodemapConfig(projectDir);
   const embedder = await createEmbedder(config.embedding);
-  const [queryVector] = await embedder.embed([query]);
+  const embedPromise = embedOne(embedder, query);
+  const timeoutMs = config.searchTimeoutMs;
+  const timeoutPromise = new Promise(
+    (_, reject) => setTimeout(() => reject(new Error(`Embedding timed out after ${timeoutMs}ms`)), timeoutMs)
+  );
+  const queryVector = await Promise.race([embedPromise, timeoutPromise]);
   const store = new CodemapStore(projectDir, resolvedBranch);
   await store.initialize();
   const hybridConfig = config.hybrid;
@@ -2090,7 +2215,7 @@ async function searchCode(projectDir, query, options = {}) {
     let graphRanks;
     if (weights.graphRank && weights.graphRank > 0) {
       try {
-        const { loadPageRankMap } = await import("./dist-WAMAQVPK.js");
+        const { loadPageRankMap } = await import("./dist-4U5L2X2C.js");
         graphRanks = loadPageRankMap(projectDir, resolvedBranch) ?? void 0;
       } catch {
       }
@@ -2330,7 +2455,7 @@ async function processIncremental(projectDir, events, config, store, embedder, b
           snippet: chunk.content.slice(0, 200)
         }
       }));
-      await store.upsertItems(vectorItems);
+      await store.appendItems(vectorItems);
       if (bm25Index) {
         const bm25Docs = chunks.map((c) => ({
           id: c.id,
@@ -2427,6 +2552,19 @@ var IGNORED_DIRS = /* @__PURE__ */ new Set([
   "__pycache__",
   ".tsbuildinfo"
 ]);
+var LANCE_CORRUPTION_PATTERNS = [
+  /Not found:.*\.lance/i,
+  /LanceError/i,
+  /No such file or directory.*lance/i,
+  /corrupted/i,
+  /Invalid IPC message/i,
+  /Panic in async function/i,
+  /thread '.*' panicked/i
+];
+function isIndexCorruptionError(err) {
+  const message = err instanceof Error ? err.message : String(err);
+  return LANCE_CORRUPTION_PATTERNS.some((pattern) => pattern.test(message));
+}
 var MAX_RECENT_EVENTS = 20;
 var CodemapWatcher = class {
   projectDir;
@@ -2441,6 +2579,7 @@ var CodemapWatcher = class {
   _recentEvents = [];
   _totalEventsProcessed = 0;
   _lastFlushAt = null;
+  _store = null;
   constructor(options) {
     this.projectDir = options.projectDir;
     this.branch = options.branch ?? getCurrentBranch(options.projectDir);
@@ -2471,12 +2610,48 @@ var CodemapWatcher = class {
   async start() {
     if (this.running) return;
     this.running = true;
-    const config = loadCodemapConfig(this.projectDir);
-    const embedder = await createEmbedder(config.embedding);
-    const store = new CodemapStore(this.projectDir, this.branch);
+    let config = loadCodemapConfig(this.projectDir);
+    let embedder = await createEmbedder(config.embedding);
+    let store = new CodemapStore(this.projectDir, this.branch);
     await store.initialize();
-    const ignorePatterns = loadCodemapIgnore(this.projectDir);
-    const denyPatterns = [...config.deny, ...ignorePatterns];
+    this._store = store;
+    try {
+      await store.getItemCount();
+    } catch (err) {
+      if (isIndexCorruptionError(err)) {
+        console.warn(
+          `[codemap-watcher] LanceDB index corrupted on startup, rebuilding...`
+        );
+        store.close();
+        const lanceDir = codemapLanceDir(this.projectDir, this.branch);
+        fs12.rmSync(lanceDir, { recursive: true, force: true });
+        await runInitPipeline(this.projectDir, void 0, this.branch);
+        store = new CodemapStore(this.projectDir, this.branch);
+        await store.initialize();
+        this._store = store;
+        config = loadCodemapConfig(this.projectDir);
+        embedder = await createEmbedder(config.embedding);
+        console.log(`[codemap-watcher] Index rebuilt successfully`);
+      } else {
+        throw err;
+      }
+    }
+    const RECOVERY_COOLDOWN_MS = 5 * 60 * 1e3;
+    let lastRecoveryAttempt = 0;
+    let recovering = false;
+    let cachedDenyPatterns = null;
+    let denyPatternsReadAt = 0;
+    const DENY_CACHE_TTL_MS = 5e3;
+    const getDenyPatterns = () => {
+      const now = Date.now();
+      if (cachedDenyPatterns && now - denyPatternsReadAt < DENY_CACHE_TTL_MS) {
+        return cachedDenyPatterns;
+      }
+      const ignorePatterns = loadCodemapIgnore(this.projectDir);
+      cachedDenyPatterns = [...config.deny, ...ignorePatterns];
+      denyPatternsReadAt = now;
+      return cachedDenyPatterns;
+    };
     const processBatch = async (initialEvents) => {
       this.processing = true;
       let events = initialEvents;
@@ -2486,7 +2661,37 @@ var CodemapWatcher = class {
           this._totalEventsProcessed += events.length;
           this._lastFlushAt = (/* @__PURE__ */ new Date()).toISOString();
         } catch (err) {
-          console.error(`[codemap-watcher] Incremental update error: ${err instanceof Error ? err.message : String(err)}`);
+          const msg = err instanceof Error ? err.message : String(err);
+          if (isIndexCorruptionError(err) && !recovering && Date.now() - lastRecoveryAttempt > RECOVERY_COOLDOWN_MS) {
+            recovering = true;
+            lastRecoveryAttempt = Date.now();
+            console.warn(
+              `[codemap-watcher] Detected corrupted LanceDB index, auto-recovering via full re-index...`
+            );
+            try {
+              store.close();
+              this._store = null;
+              const lanceDir = codemapLanceDir(this.projectDir, this.branch);
+              fs12.rmSync(lanceDir, { recursive: true, force: true });
+              await runInitPipeline(this.projectDir, void 0, this.branch);
+              store = new CodemapStore(this.projectDir, this.branch);
+              await store.initialize();
+              this._store = store;
+              const freshConfig = loadCodemapConfig(this.projectDir);
+              embedder = await createEmbedder(freshConfig.embedding);
+              this.pendingEvents = [];
+              console.log(`[codemap-watcher] Index recovered successfully via full re-index`);
+            } catch (recoveryErr) {
+              console.error(
+                `[codemap-watcher] Auto-recovery failed: ${recoveryErr instanceof Error ? recoveryErr.message : String(recoveryErr)}`
+              );
+            } finally {
+              recovering = false;
+            }
+            break;
+          } else if (!recovering) {
+            console.error(`[codemap-watcher] Incremental update error: ${msg}`);
+          }
         }
         events = this.drainPendingEvents();
       }
@@ -2507,7 +2712,7 @@ var CodemapWatcher = class {
       }
       const ext = path14.extname(relPath).toLowerCase();
       if (BINARY_EXTENSIONS2.has(ext)) return false;
-      if (matchesDenyPattern(relPath, denyPatterns)) return false;
+      if (matchesDenyPattern(relPath, getDenyPatterns())) return false;
       return true;
     };
     try {
@@ -2576,15 +2781,18 @@ var CodemapWatcher = class {
     const deduped = /* @__PURE__ */ new Map();
     for (const e of [...queued, ...pending]) deduped.set(e.filePath, e);
     const remaining = Array.from(deduped.values());
-    if (remaining.length > 0) {
+    if (remaining.length > 0 && this._store) {
       try {
         const config = loadCodemapConfig(this.projectDir);
         const embedder = await createEmbedder(config.embedding);
-        const store = new CodemapStore(this.projectDir, this.branch);
-        await processIncremental(this.projectDir, remaining, config, store, embedder, this.branch);
+        await processIncremental(this.projectDir, remaining, config, this._store, embedder, this.branch);
       } catch {
       }
     }
+    if (this._store) {
+      this._store.close();
+      this._store = null;
+    }
     this.queue.destroy();
     this.removePidFile();
   }
@@ -2681,16 +2889,9 @@ async function exportIndex(projectDir, branch) {
     }
     const metadataDir = codemapMetadataDir(projectDir, resolvedBranch);
     if (fs13.existsSync(metadataDir)) {
-      const metadataFiles = fs13.readdirSync(metadataDir);
-      for (const file of metadataFiles) {
-        const sourcePath = path15.join(metadataDir, file);
-        const stat = fs13.statSync(sourcePath);
-        if (stat.isFile()) {
-          copyAndStage(worktreeDir, `index/metadata/${file}`, sourcePath);
-          totalSizeBytes += stat.size;
-          filesExported++;
-        }
-      }
+      const result = copyMetadataRecursive(metadataDir, worktreeDir, "index/metadata");
+      filesExported += result.fileCount;
+      totalSizeBytes += result.totalBytes;
     }
     const exportMeta = {
       exportedAt: (/* @__PURE__ */ new Date()).toISOString(),
@@ -2776,6 +2977,26 @@ function copyDirRecursive(srcDir, worktreeDir, prefix) {
   }
   return { fileCount, totalBytes, warnings };
 }
+function copyMetadataRecursive(srcDir, worktreeDir, prefix) {
+  let fileCount = 0;
+  let totalBytes = 0;
+  const entries = fs13.readdirSync(srcDir);
+  for (const entry of entries) {
+    const srcPath = path15.join(srcDir, entry);
+    const destRelative = `${prefix}/${entry}`;
+    const stat = fs13.statSync(srcPath);
+    if (stat.isDirectory()) {
+      const sub = copyMetadataRecursive(srcPath, worktreeDir, destRelative);
+      fileCount += sub.fileCount;
+      totalBytes += sub.totalBytes;
+    } else if (stat.isFile()) {
+      copyAndStage(worktreeDir, destRelative, srcPath);
+      totalBytes += stat.size;
+      fileCount++;
+    }
+  }
+  return { fileCount, totalBytes };
+}
 async function importIndex(projectDir, branch) {
   const resolvedBranch = branch ?? getCurrentBranch(projectDir);
   const branchName = getCodemapBranch(resolvedBranch);
@@ -2827,17 +3048,9 @@ async function importIndex(projectDir, branch) {
       const metadataSrcDir = path16.join(worktreeDir, "index", "metadata");
       if (fs14.existsSync(metadataSrcDir)) {
         const metadataDestDir = path16.join(tmpDir, "index", "metadata");
-        fs14.mkdirSync(metadataDestDir, { recursive: true });
-        const files = fs14.readdirSync(metadataSrcDir);
-        for (const file of files) {
-          const src = path16.join(metadataSrcDir, file);
-          const stat = fs14.statSync(src);
-          if (stat.isFile()) {
-            fs14.copyFileSync(src, path16.join(metadataDestDir, file));
-            totalSizeBytes += stat.size;
-            filesImported++;
-          }
-        }
+        const result = copyDirRecursiveImport(metadataSrcDir, metadataDestDir);
+        filesImported += result.fileCount;
+        totalSizeBytes += result.totalBytes;
       }
       if (fs14.existsSync(codemapDir)) {
         fs14.rmSync(codemapDir, { recursive: true });
@@ -3113,6 +3326,9 @@ async function runEvalHarness(projectDir, datasetPath) {
 }
 
 export {
+  createEmbedder,
+  embedOne,
+  embedAll,
   DEFAULT_CODEMAP_CONFIG,
   loadCodemapConfig,
   saveCodemapConfig,
@@ -3135,7 +3351,6 @@ export {
   loadManifest,
   saveManifest,
   diffManifest,
-  createEmbedder,
   CodemapStore,
   saveSchema,
   loadSchema,