@comfanion/usethis_search 4.4.0 → 4.5.1

package/vectorizer/index.ts

@@ -21,6 +21,7 @@ import { UsageTracker } from "./usage-tracker.ts";
 import { ChunkStore } from "./chunk-store.ts";
 import { decomposeQuery, rrfMerge, DEFAULT_DECOMPOSER_CONFIG } from "./query-decomposer.ts";
 import type { DecomposerConfig } from "./query-decomposer.ts";
+import { glob } from "glob";
 
 // Suppress transformers.js logs unless DEBUG is set
 const DEBUG = process.env.DEBUG?.includes("vectorizer") || process.env.DEBUG === "*";
@@ -106,8 +107,8 @@ const DEFAULT_WORKSPACE_CONFIG = {
   maxFiles: 30,                // Max number of files in workspace
   attachTopN: 5,               // Top N search results to attach with full content
   attachRelatedPerFile: 3,     // Max graph relations per main file
-  minScoreMain: 0.65,          // Min score for main files
-  minScoreRelated: 0.5,        // Min score for graph relations
+  minScoreMain: 0.65,          // Min score for main files (override in vectorizer.yaml)
+  minScoreRelated: 0.5,        // Min score for graph relations (override in vectorizer.yaml)
   persistContent: false,       // Save full content in snapshots (debug mode)
   autoPruneSearch: true,       // Replace old search outputs with compact summaries
   substituteToolOutputs: true, // Replace tool outputs when files in workspace
@@ -498,6 +499,54 @@ function clearQueryCache() {
   }
 }
 
+// ── Shared ONNX model singleton ─────────────────────────────────────────────
+// Model lives for the entire session — not tied to indexer pool TTL.
+// Indexer eviction clears caches/DB but model stays loaded for fast search.
+let _sharedModel: any = null
+let _sharedModelPromise: Promise<any> | null = null
+
+async function getSharedModel(retries = 3): Promise<any> {
+  if (_sharedModel) return _sharedModel
+
+  // Prevent concurrent loads — reuse in-flight promise
+  if (_sharedModelPromise) return _sharedModelPromise
+
+  _sharedModelPromise = (async () => {
+    let lastError: Error | null = null
+    for (let attempt = 1; attempt <= retries; attempt++) {
+      try {
+        if (DEBUG) console.log(`[vectorizer] Loading embedding model: ${EMBEDDING_MODEL}... (attempt ${attempt}/${retries})`)
+        _sharedModel = await pipeline("feature-extraction", EMBEDDING_MODEL, {
+          progress_callback: DEBUG ? undefined : null,
+        })
+        if (DEBUG) console.log(`[vectorizer] Model loaded: ${EMBEDDING_MODEL}`)
+        _sharedModelPromise = null
+        return _sharedModel
+      } catch (error) {
+        lastError = error as Error
+        if (attempt < retries) {
+          const delay = attempt * 2000
+          if (DEBUG) console.log(`[vectorizer] Model load attempt ${attempt} failed: ${lastError.message}. Retrying in ${delay}ms...`)
+          await new Promise(r => setTimeout(r, delay))
+        }
+      }
+    }
+    _sharedModelPromise = null
+    throw new Error(`Model loading failed after ${retries} attempts: ${lastError?.message || "unknown"}`)
+  })()
+
+  return _sharedModelPromise
+}
+
+/** Dispose shared model — call only on process exit or explicit cleanup. */
+async function disposeSharedModel(): Promise<void> {
+  if (_sharedModel) {
+    try { await _sharedModel.dispose() } catch { /* best effort */ }
+    _sharedModel = null
+  }
+  _sharedModelPromise = null
+}
+
 class CodebaseIndexer {
   constructor(projectRoot, indexName = "code") {
     this.root = projectRoot;
@@ -566,21 +615,14 @@ class CodebaseIndexer {
 
   async loadModel() {
     if (!this.model) {
-      try {
-        if (DEBUG) console.log(`[vectorizer] Loading embedding model: ${EMBEDDING_MODEL}...`);
-        this.model = await pipeline("feature-extraction", EMBEDDING_MODEL, {
-          progress_callback: DEBUG ? undefined : null,
-        });
-        if (DEBUG) console.log(`[vectorizer] Model loaded: ${EMBEDDING_MODEL}`);
-      } catch (error) {
-        this.model = null;
-        throw new Error(`Model loading failed: ${error.message || error}`);
-      }
+      this.model = await getSharedModel();
     }
     return this.model;
   }
 
   async unloadModel() {
+    // Drop reference to shared model — do NOT dispose it.
+    // Model singleton lives for the entire session.
     this.model = null;
     // Release BM25 data held in memory
     if (this.bm25) {
@@ -609,6 +651,14 @@ class CodebaseIndexer {
       try { await this.usageTracker.save(); } catch { /* best effort */ }
       this.usageTracker = null;
     }
+    // Close LanceDB connection (releases file handles)
+    if (this.db) {
+      try {
+        // LanceDB v0.x doesn't expose close() — drop reference to release
+        // Arrow/IPC file handles are released when Connection is GC'd
+        this.db = null;
+      } catch { /* best effort */ }
+    }
     this._chunkCache = null;
     clearQueryCache();
     if (global.gc) global.gc();
@@ -716,6 +766,8 @@ class CodebaseIndexer {
       return null; // unchanged
     }
 
+    if (DEBUG) console.log(`[vectorizer] prepareFile: ${relPath} [read]`);
+
     // Extract metadata
     const fileMeta = await extractFileMetadata(filePath, content);
     const archived = this.isArchived(relPath, content);
@@ -723,9 +775,13 @@ class CodebaseIndexer {
     // Clean content before chunking
     const cleaned = cleanContent(content, fileMeta.file_type, CLEANING_CONFIG);
 
+    if (DEBUG) console.log(`[vectorizer] prepareFile: ${relPath} [chunking]`);
+
     // Semantic chunking (async for LSP-based chunking)
     const chunks = await chunkContent(cleaned, fileMeta.file_type, fileMeta.language, CHUNKING_CONFIG, filePath, this.root);
 
+    if (DEBUG) console.log(`[vectorizer] prepareFile: ${relPath} [${chunks.length} chunks, building graph]`);
+
     // Assign chunk IDs
     const chunksWithIds = this.graphBuilder
       ? this.graphBuilder.assignChunkIds(relPath, chunks)
@@ -753,23 +809,39 @@ class CodebaseIndexer {
     }
 
     // Return prepared rows (without vector — Phase 2 fills it)
-    const rows = chunksWithIds.map((chunk, i) => ({
-      chunk_id: chunk.chunk_id,
-      file: relPath,
-      chunk_index: i,
-      content: chunk.content,
-      archived,
-      file_type: fileMeta.file_type,
-      language: fileMeta.language,
-      last_modified: fileMeta.last_modified,
-      file_size: fileMeta.file_size,
-      heading_context: chunk.heading_context || "",
-      function_name: chunk.function_name || "",
-      class_name: chunk.class_name || "",
-      tags: (fileMeta.tags || []).join(","),
-      start_line: chunk.start_line ?? -1,
-      end_line: chunk.end_line ?? -1,
-    }));
+    const rows = chunksWithIds.map((chunk, i) => {
+      // Build metadata prefix for embedding enrichment.
+      // This helps the embedding model associate function/class names with the code,
+      // improving search relevance for queries like "ensureBudget" or "WorkspaceCache".
+      // The prefix is stored in a separate field — original content stays clean.
+      const metaParts: string[] = []
+      if (relPath) metaParts.push(`File: ${relPath}`)
+      if (chunk.class_name) metaParts.push(`Class: ${chunk.class_name}`)
+      if (chunk.function_name) metaParts.push(`Method: ${chunk.function_name}`)
+      if (chunk.heading_context) metaParts.push(`Section: ${chunk.heading_context}`)
+      const metaPrefix = metaParts.length > 0 ? `// ${metaParts.join(" | ")}\n` : ""
+
+      return {
+        chunk_id: chunk.chunk_id,
+        file: relPath,
+        chunk_index: i,
+        content: chunk.content,
+        // Enriched content for embedding — metadata prefix + original content.
+        // Phase 2 embeds this instead of raw content.
+        content_for_embedding: metaPrefix + chunk.content,
+        archived,
+        file_type: fileMeta.file_type,
+        language: fileMeta.language,
+        last_modified: fileMeta.last_modified,
+        file_size: fileMeta.file_size,
+        heading_context: chunk.heading_context || "",
+        function_name: chunk.function_name || "",
+        class_name: chunk.class_name || "",
+        tags: (fileMeta.tags || []).join(","),
+        start_line: chunk.start_line ?? -1,
+        end_line: chunk.end_line ?? -1,
+      }
+    });
 
     // Store chunks in ChunkStore (Phase 1 — BM25 available immediately)
     if (this.chunkStore) {
@@ -781,6 +853,7 @@ class CodebaseIndexer {
       }
     }
 
+    if (DEBUG) console.log(`[vectorizer] prepareFile: ${relPath} [done, ${rows.length} rows]`);
     return { relPath, hash, rows };
   }
 
@@ -805,19 +878,24 @@ class CodebaseIndexer {
 
     // Batch embed
     const allData = [];
+    let embedErrors = 0;
     for (let i = 0; i < allRows.length; i += batchSize) {
       const batch = allRows.slice(i, i + batchSize);
-      const texts = batch.map(r => r.content);
-
-      // Embed batch — @xenova/transformers processes array inputs efficiently
-      const embeddings = [];
-      for (const text of texts) {
-        const result = await model(text, { pooling: "mean", normalize: true });
-        embeddings.push(Array.from(result.data));
-      }
 
-      for (let j = 0; j < batch.length; j++) {
-        allData.push({ ...batch[j], vector: embeddings[j] });
+      for (const row of batch) {
+        try {
+          // Use enriched content (with metadata prefix) for embedding,
+          // but store original content in LanceDB for display.
+          const textToEmbed = row.content_for_embedding || row.content;
+          const result = await model(textToEmbed, { pooling: "mean", normalize: true });
+          // Don't store content_for_embedding in LanceDB — it's only for embedding
+          const { content_for_embedding, ...rowWithoutEmbeddingText } = row;
+          allData.push({ ...rowWithoutEmbeddingText, vector: Array.from(result.data) });
+        } catch (e) {
+          embedErrors++;
+          if (DEBUG) console.log(`[vectorizer] Embed failed for ${row.chunk_id || row.file}: ${(e as Error).message}`);
+          // Skip this chunk — don't let one bad chunk kill the entire index
+        }
       }
 
       if (onProgress) {
@@ -825,14 +903,25 @@ class CodebaseIndexer {
       }
     }
 
+    if (embedErrors > 0 && DEBUG) {
+      console.log(`[vectorizer] ${embedErrors} chunks failed to embed (skipped)`);
+    }
+
     // Bulk store in LanceDB
+    if (allData.length === 0) return 0;
+
     const tableName = "chunks";
-    const tables = await this.db.tableNames();
-    if (tables.includes(tableName)) {
-      const table = await this.db.openTable(tableName);
-      await table.add(allData);
-    } else {
-      await this.db.createTable(tableName, allData);
+    try {
+      const tables = await this.db.tableNames();
+      if (tables.includes(tableName)) {
+        const table = await this.db.openTable(tableName);
+        await table.add(allData);
+      } else {
+        await this.db.createTable(tableName, allData);
+      }
+    } catch (e) {
+      if (DEBUG) console.log(`[vectorizer] LanceDB store failed: ${(e as Error).message}`);
+      throw e; // Re-throw — caller (indexAll) will catch and log
     }
 
     // Update hashes + mark vectorized in ChunkStore
@@ -876,8 +965,8 @@ class CodebaseIndexer {
     // Clean content before chunking
     const cleaned = cleanContent(content, fileMeta.file_type, CLEANING_CONFIG);
 
-    // Semantic chunking
-    const chunks = chunkContent(cleaned, fileMeta.file_type, fileMeta.language, CHUNKING_CONFIG);
+    // Semantic chunking (await for LSP-based chunking when filePath is provided)
+    const chunks = await chunkContent(cleaned, fileMeta.file_type, fileMeta.language, CHUNKING_CONFIG, filePath, this.root);
 
     // v3: Assign chunk IDs for graph tracking (works without graph — just adds IDs)
     const chunksWithIds = this.graphBuilder
@@ -916,12 +1005,22 @@ class CodebaseIndexer {
 
     const data = [];
     for (let i = 0; i < chunksWithIds.length; i++) {
-      const embedding = await this.embed(chunksWithIds[i].content);
+      const chunk = chunksWithIds[i];
+      // Build metadata prefix for embedding enrichment (same as prepareFile)
+      const metaParts: string[] = [];
+      if (relPath) metaParts.push(`File: ${relPath}`);
+      if (chunk.class_name) metaParts.push(`Class: ${chunk.class_name}`);
+      if (chunk.function_name) metaParts.push(`Method: ${chunk.function_name}`);
+      if (chunk.heading_context) metaParts.push(`Section: ${chunk.heading_context}`);
+      const metaPrefix = metaParts.length > 0 ? `// ${metaParts.join(" | ")}\n` : "";
+      const textToEmbed = metaPrefix + chunk.content;
+
+      const embedding = await this.embed(textToEmbed);
       data.push({
-        chunk_id: chunksWithIds[i].chunk_id,
+        chunk_id: chunk.chunk_id,
         file: relPath,
         chunk_index: i,
-        content: chunksWithIds[i].content,
+        content: chunk.content,
         vector: embedding,
         archived: archived,
         // v2 metadata
@@ -929,13 +1028,13 @@ class CodebaseIndexer {
         language: fileMeta.language,
         last_modified: fileMeta.last_modified,
         file_size: fileMeta.file_size,
-        heading_context: chunksWithIds[i].heading_context || "",
-        function_name: chunksWithIds[i].function_name || "",
-        class_name: chunksWithIds[i].class_name || "",
+        heading_context: chunk.heading_context || "",
+        function_name: chunk.function_name || "",
+        class_name: chunk.class_name || "",
         tags: (fileMeta.tags || []).join(","),
         // Line numbers for "from-to" extraction (default to -1 when unknown)
-        start_line: chunksWithIds[i].start_line ?? -1,
-        end_line: chunksWithIds[i].end_line ?? -1,
+        start_line: chunk.start_line ?? -1,
+        end_line: chunk.end_line ?? -1,
       });
     }
 
@@ -1110,7 +1209,13 @@ class CodebaseIndexer {
           content: neighborChunk.content,
           relation: edge.predicate,
           score,
-          via: edge.source
+          via: edge.source,
+          start_line: neighborChunk.start_line,
+          end_line: neighborChunk.end_line,
+          chunk_index: neighborChunk.chunk_index,
+          language: neighborChunk.language,
+          function_name: neighborChunk.function_name,
+          class_name: neighborChunk.class_name,
         });
       }
 
@@ -1374,69 +1479,99 @@ class CodebaseIndexer {
   }
 
   async findChunkById(chunkId) {
-    // Lazy-build an in-memory Map keyed by chunk_id on first call.
-    // The cache lives until unloadModel() clears it.
+    // LRU cache with bounded size — avoids loading all 100K rows into memory.
+    // Point lookups: ChunkStore (SQLite) first, LanceDB fallback for vectors.
     if (!this._chunkCache) {
       this._chunkCache = new Map();
+    }
 
-      // Primary: LanceDB (has vectors for cosine similarity in graph expansion)
-      const tableName = "chunks";
-      const tables = await this.db.tableNames();
-      if (tables.includes(tableName)) {
-        try {
+    // Check LRU cache first
+    const cached = this._chunkCache.get(chunkId);
+    if (cached) {
+      // Move to end (most recently used) — delete & re-insert
+      this._chunkCache.delete(chunkId);
+      this._chunkCache.set(chunkId, cached);
+      return cached;
+    }
+
+    // Point lookup: ChunkStore (SQLite — fast, no vectors)
+    let chunk = null;
+    if (this.chunkStore) {
+      try {
+        chunk = this.chunkStore.getChunkById(chunkId);
+      } catch {
+        // best effort
+      }
+    }
+
+    // If we need vectors (for cosine similarity in graph expansion),
+    // try LanceDB point lookup. Only if ChunkStore had no result or we need vectors.
+    if (!chunk) {
+      try {
+        const tableName = "chunks";
+        const tables = await this.db.tableNames();
+        if (tables.includes(tableName)) {
           const table = await this.db.openTable(tableName);
-          const rows = await table.filter("true").limit(100000).execute();
-          for (const row of rows) {
-            if (row.chunk_id) {
-              this._chunkCache.set(row.chunk_id, row);
-            }
+          const rows = await table.filter(`chunk_id = '${chunkId.replace(/'/g, "''")}'`).limit(1).execute();
+          if (rows.length > 0) {
+            chunk = rows[0];
           }
-        } catch (e) {
-          if (DEBUG) console.log("[vectorizer] Chunk cache from LanceDB failed:", e.message);
         }
+      } catch (e) {
+        if (DEBUG) console.log("[vectorizer] LanceDB point lookup failed:", e.message);
       }
+    }
 
-      // Fallback: ChunkStore (no vectors, but has content for BM25-only mode)
-      if (this._chunkCache.size === 0 && this.chunkStore) {
-        try {
-          const allChunks = this.chunkStore.getAllChunks();
-          for (const chunk of allChunks) {
-            if (chunk.chunk_id) {
-              this._chunkCache.set(chunk.chunk_id, chunk);
-            }
-          }
-          if (DEBUG && allChunks.length > 0) {
-            console.log(`[vectorizer] Chunk cache from ChunkStore (${allChunks.length} chunks, no vectors)`);
-          }
-        } catch (e) {
-          if (DEBUG) console.log("[vectorizer] Chunk cache from ChunkStore failed:", e.message);
+    if (chunk) {
+      // LRU eviction: cap at 500 entries (each ~2KB without vector, ~4KB with)
+      const MAX_CHUNK_CACHE = 500;
+      if (this._chunkCache.size >= MAX_CHUNK_CACHE) {
+        const oldest = this._chunkCache.keys().next().value;
+        if (oldest !== undefined) {
+          this._chunkCache.delete(oldest);
         }
       }
+      this._chunkCache.set(chunkId, chunk);
     }
-    return this._chunkCache.get(chunkId) || null;
+
+    return chunk || null;
   }
 
   /**
    * Find all chunks belonging to a specific file path.
+   * Uses ChunkStore (SQLite) for efficient file-level queries — no full cache load.
    * @param {string} filePath - Relative file path (e.g. "src/auth.ts")
    * @returns {Promise<Array>} Array of chunks from this file
    */
   async findChunksByPath(filePath) {
-    // Ensure chunk cache is loaded
-    await this.findChunkById("__force_cache_load__");
-    
-    if (!this._chunkCache) return [];
-    
-    const chunks = [];
-    for (const chunk of this._chunkCache.values()) {
-      if (chunk.file === filePath) {
-        chunks.push(chunk);
+    // Primary: ChunkStore has an index on file column — fast
+    if (this.chunkStore) {
+      try {
+        const chunks = this.chunkStore.getChunksByFile(filePath);
+        if (chunks.length > 0) {
+          chunks.sort((a, b) => (a.chunk_index || 0) - (b.chunk_index || 0));
+          return chunks;
+        }
+      } catch {
+        // fallback below
       }
     }
-    
-    // Sort by chunk_index
-    chunks.sort((a, b) => (a.chunk_index || 0) - (b.chunk_index || 0));
-    return chunks;
+
+    // Fallback: LanceDB filter by file
+    try {
+      const tableName = "chunks";
+      const tables = await this.db.tableNames();
+      if (tables.includes(tableName)) {
+        const table = await this.db.openTable(tableName);
+        const rows = await table.filter(`file = '${filePath.replace(/'/g, "''")}'`).limit(1000).execute();
+        rows.sort((a, b) => (a.chunk_index || 0) - (b.chunk_index || 0));
+        return rows;
+      }
+    } catch (e) {
+      if (DEBUG) console.log("[vectorizer] findChunksByPath LanceDB failed:", e.message);
+    }
+
+    return [];
   }
 
   cosineSimilarity(vecA, vecB) {
@@ -1448,11 +1583,11 @@ class CodebaseIndexer {
       normA += vecA[i] * vecA[i];
       normB += vecB[i] * vecB[i];
     }
-    return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
+    const denom = Math.sqrt(normA) * Math.sqrt(normB);
+    return denom === 0 ? 0 : dotProduct / denom;
   }
 
   async checkHealth(extraIgnore = []) {
-    const { glob } = await import("glob");
     const preset = INDEX_PRESETS[this.indexName] || DEFAULT_PRESETS.code;
 
     const ignore = [
@@ -1544,7 +1679,6 @@ class CodebaseIndexer {
   }
 
   async indexAll(onProgress = null, extraIgnore = []) {
-    const { glob } = await import("glob");
     const preset = INDEX_PRESETS[this.indexName] || DEFAULT_PRESETS.code;
 
     const ignore = [
@@ -1611,22 +1745,33 @@ class CodebaseIndexer {
     const total = files.length;
     const CONCURRENCY = 5;
 
+    // Helper: write to indexer.log (always, not just DEBUG)
+    const logPath = path.join(this.root, ".opencode", "indexer.log");
+    const fsSync = await import("fs");
+    const logToFile = (msg: string) => {
+      const ts = new Date().toISOString().slice(11, 19);
+      try { fsSync.appendFileSync(logPath, `${ts} ${msg}\n`); } catch { /* non-fatal */ }
+    };
+
     // ══════════════════════════════════════════════════════════════════════════
-    // Phase 1: Prepare files in parallel (chunk + graph, no embedding)
+    // Phase 1: Prepare files sequentially (chunk + graph, no embedding)
     // ══════════════════════════════════════════════════════════════════════════
+    logToFile(`Phase 1: preparing ${total} files (concurrency=${CONCURRENCY})`);
+
     const preparedFiles = [];
     let prepared = 0;
     let skipped = 0;
+    let errors = 0;
 
-    // Process in batches of CONCURRENCY
     for (let i = 0; i < files.length; i += CONCURRENCY) {
       const batch = files.slice(i, i + CONCURRENCY);
       const promises = batch.map(async (relPath) => {
         const filePath = path.join(this.root, relPath);
         try {
-          const result = await this.prepareFile(filePath);
-          return result;
-        } catch {
+          return await this.prepareFile(filePath);
+        } catch (e) {
+          logToFile(`ERROR prepare ${relPath}: ${(e as Error).message}`);
+          errors++;
           return null;
         }
       });
@@ -1643,7 +1788,7 @@ class CodebaseIndexer {
       }
     }
 
-    if (DEBUG) console.log(`[vectorizer] Phase 1 done: ${prepared} files prepared, ${skipped} skipped`);
+    logToFile(`Phase 1 done: ${prepared} prepared, ${skipped} skipped, ${errors} errors`);
 
     // ══════════════════════════════════════════════════════════════════════════
     // Phase 2: Batch embed + store (sequential, batch forward pass)
@@ -1651,19 +1796,24 @@ class CodebaseIndexer {
     let chunksEmbedded = 0;
     if (preparedFiles.length > 0) {
       const totalChunks = preparedFiles.reduce((sum, pf) => sum + pf.rows.length, 0);
-      if (DEBUG) console.log(`[vectorizer] Phase 2: embedding ${totalChunks} chunks from ${preparedFiles.length} files`);
-
-      chunksEmbedded = await this.embedAndStore(preparedFiles, 32, (done, embedTotal, phase) => {
-        if (onProgress) onProgress(done, embedTotal, `embedding`, done, "embed");
-      });
+      logToFile(`Phase 2: embedding ${totalChunks} chunks from ${preparedFiles.length} files`);
 
-      if (DEBUG) console.log(`[vectorizer] Phase 2 done: ${chunksEmbedded} chunks embedded and stored`);
+      try {
+        chunksEmbedded = await this.embedAndStore(preparedFiles, 32, (done, embedTotal, phase) => {
+          if (onProgress) onProgress(done, embedTotal, `embedding`, done, "embed");
+        });
+        logToFile(`Phase 2 done: ${chunksEmbedded} chunks embedded and stored`);
+      } catch (e) {
+        logToFile(`Phase 2 FAILED: ${(e as Error).message}\n${(e as Error).stack || ""}`);
+        throw e;
+      }
+    } else {
+      logToFile(`Phase 2 skipped: no prepared files`);
     }
 
-    const indexed = prepared; // file count for backward compat
+    const indexed = prepared;
 
     // FR-005: Build semantic similarity edges as post-pass
-    // Disabled by default (O(n²) — slow on large repos). Enable via graph.semantic_edges: true
     let semanticEdges = 0;
     if (chunksEmbedded > 0 && this.graphBuilder && this.graphDB && GRAPH_CONFIG.semantic_edges) {
       try {
@@ -1676,30 +1826,25 @@ class CodebaseIndexer {
             .filter(r => r.chunk_id && r.vector)
             .map(r => ({ chunk_id: r.chunk_id, vector: Array.from(r.vector), file: r.file }));
 
-          // Skip if too many chunks — O(n²) becomes prohibitive
           const maxChunks = GRAPH_CONFIG.semantic_edges_max_chunks ?? 500;
           if (chunkData.length > maxChunks) {
-            if (DEBUG) console.log(`[vectorizer] Skipping semantic edges: ${chunkData.length} chunks > max ${maxChunks}`);
+            logToFile(`Semantic edges skipped: ${chunkData.length} chunks > max ${maxChunks}`);
           } else {
             semanticEdges = await this.graphBuilder.buildSemanticEdges(chunkData, 0.8, 3);
-            if (DEBUG) console.log(`[vectorizer] Built ${semanticEdges} semantic similarity edges`);
+            logToFile(`Semantic edges: ${semanticEdges} built`);
           }
         }
       } catch (e) {
-        if (DEBUG) console.log(`[vectorizer] Semantic edge building failed:`, e.message);
-        // non-fatal — explicit edges still work
+        logToFile(`Semantic edges FAILED: ${(e as Error).message}`);
       }
     }
 
-    // Cleanup: shutdown LSP to prevent memory leaks after bulk indexing
+    // Cleanup LSP
     if (this.graphBuilder) {
-      try {
-        await this.graphBuilder.cleanup();
-      } catch {
-        // Best effort — continue even if cleanup fails
-      }
+      try { await this.graphBuilder.cleanup(); } catch { /* best effort */ }
     }
 
+    logToFile(`indexAll complete: ${indexed} indexed, ${chunksEmbedded} embedded, ${semanticEdges} semantic edges`);
     return { indexed, skipped, total, semanticEdges };
   }
 
@@ -1763,9 +1908,13 @@ class CodebaseIndexer {
         if (entry.isDirectory() && entry.name !== "lancedb") {
           try {
             const indexer = await new CodebaseIndexer(this.root, entry.name).init();
-            const stat = await indexer.getStats();
-            if (stat.fileCount > 0 || stat.chunkCount > 0) {
-              stats.push(stat);
+            try {
+              const stat = await indexer.getStats();
+              if (stat.fileCount > 0 || stat.chunkCount > 0) {
+                stats.push(stat);
+              }
+            } finally {
+              await indexer.unloadModel();
             }
           } catch {}
         }
@@ -1775,20 +1924,38 @@ class CodebaseIndexer {
   }
 
   async clear() {
+    // Close open DB handles before deleting their files
+    if (this.chunkStore) { try { this.chunkStore.close(); } catch { /* best effort */ } this.chunkStore = null; }
+    if (this.graphBuilder) { try { await this.graphBuilder.cleanup(); } catch { /* best effort */ } this.graphBuilder = null; }
+    if (this.graphDB) { try { await this.graphDB.close(); } catch { /* best effort */ } this.graphDB = null; }
+    if (this.usageTracker) { try { await this.usageTracker.save(); } catch { /* best effort */ } this.usageTracker = null; }
+    this.db = null;
+
     await fs.rm(this.cacheDir, { recursive: true, force: true });
     this.hashes = {};
     if (this.bm25) { this.bm25.clear(); this.bm25 = null; }
     this._bm25Rows = null;
+    this._chunkCache = null;
     this.metrics = null;
+    this.model = null;
     await this.init();
   }
 
   async clearAll() {
+    // Close open DB handles before deleting their files
+    if (this.chunkStore) { try { this.chunkStore.close(); } catch { /* best effort */ } this.chunkStore = null; }
+    if (this.graphBuilder) { try { await this.graphBuilder.cleanup(); } catch { /* best effort */ } this.graphBuilder = null; }
+    if (this.graphDB) { try { await this.graphDB.close(); } catch { /* best effort */ } this.graphDB = null; }
+    if (this.usageTracker) { try { await this.usageTracker.save(); } catch { /* best effort */ } this.usageTracker = null; }
+    this.db = null;
+
     await fs.rm(this.baseDir, { recursive: true, force: true });
     this.hashes = {};
     if (this.bm25) { this.bm25.clear(); this.bm25 = null; }
     this._bm25Rows = null;
+    this._chunkCache = null;
     this.metrics = null;
+    this.model = null;
     clearQueryCache();
     await this.init();
   }
@@ -1832,7 +1999,9 @@ function getWorkspaceConfig() {
 // ── Singleton indexer pool ──────────────────────────────────────────────────
 // Prevents LevelDB lock conflicts when parallel searches hit the same index.
 // Each unique (projectRoot, indexName) gets one shared CodebaseIndexer.
-const _indexerPool = new Map<string, { indexer: CodebaseIndexer; refCount: number; initPromise: Promise<CodebaseIndexer> }>();
+// TTL eviction: idle indexers are unloaded after POOL_IDLE_TTL_MS to free memory.
+const POOL_IDLE_TTL_MS = 5 * 60 * 1000; // 5 minutes idle → unload model + caches
+const _indexerPool = new Map<string, { indexer: CodebaseIndexer; refCount: number; initPromise: Promise<CodebaseIndexer>; idleTimer?: ReturnType<typeof setTimeout> }>();
 
 /**
  * Get or create a shared CodebaseIndexer for the given project + index.
@@ -1851,6 +2020,11 @@ async function getIndexer(projectRoot: string, indexName: string = "code"): Prom
   const existing = _indexerPool.get(key);
   if (existing) {
     existing.refCount++;
+    // Cancel pending idle eviction — someone is using it again
+    if (existing.idleTimer) {
+      clearTimeout(existing.idleTimer);
+      existing.idleTimer = undefined;
+    }
     return existing.initPromise;
   }
 
@@ -1862,15 +2036,35 @@ async function getIndexer(projectRoot: string, indexName: string = "code"): Prom
 
 /**
  * Release a reference to a shared indexer. When refCount reaches 0,
- * the indexer is kept alive (for future reuse) but model memory is freed.
- * Call destroyIndexer() to fully close and remove from pool.
+ * starts a TTL timer. If no one reuses within POOL_IDLE_TTL_MS,
+ * the indexer is fully unloaded (model disposed, caches freed, DB closed).
  */
 function releaseIndexer(projectRoot: string, indexName: string = "code") {
   const key = `${projectRoot}::${indexName}`;
   const entry = _indexerPool.get(key);
   if (!entry) return;
   entry.refCount = Math.max(0, entry.refCount - 1);
-  // Keep in pool — don't unload. Next search reuses the same instance.
+
+  if (entry.refCount === 0) {
+    // Start idle eviction timer — free memory if not reused soon
+    if (entry.idleTimer) clearTimeout(entry.idleTimer);
+    entry.idleTimer = setTimeout(async () => {
+      // Double-check: still idle?
+      const current = _indexerPool.get(key);
+      if (!current || current.refCount > 0) return;
+      _indexerPool.delete(key);
+      try {
+        const indexer = await current.initPromise;
+        await indexer.unloadModel();
+      } catch {
+        // best effort
+      }
+    }, POOL_IDLE_TTL_MS);
+    // Don't keep process alive for idle eviction
+    if (entry.idleTimer && typeof entry.idleTimer === "object" && "unref" in entry.idleTimer) {
+      (entry.idleTimer as NodeJS.Timeout).unref();
+    }
+  }
 }
 
 /**
@@ -1881,6 +2075,7 @@ async function destroyIndexer(projectRoot: string, indexName: string = "code") {
   const key = `${projectRoot}::${indexName}`;
   const entry = _indexerPool.get(key);
   if (!entry) return;
+  if (entry.idleTimer) clearTimeout(entry.idleTimer);
   _indexerPool.delete(key);
   try {
     const indexer = await entry.initPromise;
@@ -1894,4 +2089,4 @@ function getDecomposerConfig() {
   return DECOMPOSER_CONFIG;
 }
 
-export { CodebaseIndexer, INDEX_PRESETS, getEmbeddingModel, getSearchConfig, getWorkspaceConfig, getDecomposerConfig, getIndexer, releaseIndexer, destroyIndexer };
+export { CodebaseIndexer, INDEX_PRESETS, getEmbeddingModel, getSearchConfig, getWorkspaceConfig, getDecomposerConfig, getIndexer, releaseIndexer, destroyIndexer, disposeSharedModel };