@softerist/heuristic-mcp 2.1.47 → 3.0.0

package/features/index-codebase.js

@@ -1,77 +1,86 @@
-import { fdir } from "fdir";
-import fs from "fs/promises";
-import chokidar from "chokidar";
-import path from "path";
-import os from "os";
-import { Worker } from "worker_threads";
-import { fileURLToPath } from "url";
-import { smartChunk, hashContent } from "../lib/utils.js";
-import { extractCallData } from "../lib/call-graph.js";
-
-function escapeRegExp(value) {
-  return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+import { fdir } from 'fdir';
+import fs from 'fs/promises';
+import chokidar from 'chokidar';
+import path from 'path';
+import os from 'os';
+import { Worker } from 'worker_threads';
+import { spawn } from 'child_process';
+import { setTimeout as delay } from 'timers/promises';
+import { fileURLToPath } from 'url';
+import { smartChunk, hashContent } from '../lib/utils.js';
+import { extractCallData } from '../lib/call-graph.js';
+
+import ignore from 'ignore';
+
+function toFloat32Array(vector) {
+  // Always create a copy to ensure we have a unique buffer
+  // and avoid issues with reusable WASM memory views
+  return new Float32Array(vector);
+}
+
+function isTestEnv() {
+  return process.env.VITEST === 'true' || process.env.NODE_ENV === 'test';
+}
+
+function normalizePath(value) {
+  if (typeof value !== 'string') return '';
+  return value.split(path.sep).join('/');
 }
 
 function globToRegExp(pattern) {
-  let regex = "^";
-  for (let i = 0; i < pattern.length; ) {
+  let regex = '^';
+  for (let i = 0; i < pattern.length; i += 1) {
     const char = pattern[i];
-    if (char === "*") {
-      if (pattern[i + 1] === "*") {
-        if (pattern[i + 2] === "/") {
-          regex += "(?:.*/)?";
-          i += 3;
-        } else {
-          regex += ".*";
+    if (char === '*') {
+      if (pattern[i + 1] === '*') {
+        if (pattern[i + 2] === '/') {
+          regex += '(?:.*/)?';
           i += 2;
+        } else {
+          regex += '.*';
+          i += 1;
         }
       } else {
-        regex += "[^/]*";
-        i += 1;
+        regex += '[^/]*';
       }
-      continue;
-    }
-    if (char === "?") {
-      regex += "[^/]";
-      i += 1;
-      continue;
+    } else if (char === '?') {
+      regex += '[^/]';
+    } else if ('\\.[]{}()+-^$|'.includes(char)) {
+      regex += `\\${char}`;
+    } else {
+      regex += char;
     }
-    regex += escapeRegExp(char);
-    i += 1;
   }
-  regex += "$";
+  regex += '$';
   return new RegExp(regex);
 }
 
-function normalizePath(filePath) {
-  return filePath.split(path.sep).join("/");
-}
-
 function buildExcludeMatchers(patterns) {
-  return [...new Set(patterns)]
-    .filter(Boolean)
-    .map(pattern => ({
-      matchBase: !pattern.includes("/"),
-      regex: globToRegExp(pattern)
-    }));
+  if (!Array.isArray(patterns)) return [];
+  return patterns
+    .filter((pattern) => typeof pattern === 'string' && pattern.length > 0)
+    .map((pattern) => {
+      const normalized = pattern.replace(/\\/g, '/');
+      const matchBase = !normalized.includes('/');
+      return {
+        pattern: normalized,
+        matchBase,
+        regex: globToRegExp(normalized),
+      };
+    });
 }
 
 function matchesExcludePatterns(filePath, matchers) {
-  if (matchers.length === 0) return false;
+  if (!filePath || matchers.length === 0) return false;
   const normalized = normalizePath(filePath);
-  const basename = path.posix.basename(normalized);
-
+  const base = path.posix.basename(normalized);
   for (const matcher of matchers) {
-    const target = matcher.matchBase ? basename : normalized;
-    if (matcher.regex.test(target)) {
-      return true;
-    }
+    const target = matcher.matchBase ? base : normalized;
+    if (matcher.regex.test(target)) return true;
   }
   return false;
 }
 
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
-
 export class CodebaseIndexer {
   constructor(embedder, cache, config, server = null) {
     this.embedder = embedder;
@@ -82,54 +91,175 @@ export class CodebaseIndexer {
     this.workers = [];
     this.workerReady = [];
     this.isIndexing = false;
-    this.excludeMatchers = buildExcludeMatchers(this.config.excludePatterns || []);
+    this.processingWatchEvents = false;
+    this.pendingWatchEvents = new Map();
+    const cacheRelative = this.getCacheRelativePath();
+    const autoExclude = ['.smart-coding-cache'];
+    if (cacheRelative) {
+      autoExclude.push(cacheRelative, `${cacheRelative}/**`);
+    }
+    this.excludeMatchers = buildExcludeMatchers([
+      ...autoExclude,
+      ...(this.config.excludePatterns || []),
+    ]);
+    this.gitignore = ignore();
+    this.workerFailureCount = 0;
+    this.workersDisabledUntil = 0;
+    this.workerCircuitOpen = false;
+    this._retryTimer = null;
+    this._lastProgress = null;
+    this.currentIndexMode = null;
+  }
+
+  maybeResetWorkerCircuit() {
+    if (
+      this.workerCircuitOpen &&
+      this.workersDisabledUntil &&
+      Date.now() >= this.workersDisabledUntil
+    ) {
+      this.workerCircuitOpen = false;
+      this.workersDisabledUntil = 0;
+      this.workerFailureCount = 0;
+      if (this.config.verbose) {
+        console.info('[Indexer] Worker circuit closed; resuming worker use');
+      }
+    }
+  }
+
+  shouldUseWorkers() {
+    this.maybeResetWorkerCircuit();
+    if (this.workersDisabledUntil && Date.now() < this.workersDisabledUntil) {
+      return false;
+    }
+    if (isTestEnv()) return false;
+    return os.cpus().length > 1 && this.config.workerThreads !== 0 && !this.config.embeddingProcessPerBatch;
+  }
+
+  scheduleRetry() {
+    if (this._retryTimer || isTestEnv()) return;
+    const delayMs = Math.max(1000, this.workersDisabledUntil - Date.now());
+    if (!Number.isFinite(delayMs) || delayMs <= 0) return;
+    this._retryTimer = setTimeout(() => {
+      this._retryTimer = null;
+      if (!this.isIndexing && !this.processingWatchEvents) {
+        this.indexAll().catch(() => null);
+      }
+    }, delayMs);
+  }
+
+  recordWorkerFailure(reason) {
+    const threshold = Number.isInteger(this.config.workerFailureThreshold)
+      ? this.config.workerFailureThreshold
+      : 1;
+    const cooldownMs = Number.isInteger(this.config.workerFailureCooldownMs)
+      ? this.config.workerFailureCooldownMs
+      : 10 * 60 * 1000;
+
+    this.workerFailureCount += 1;
+    console.warn(`[Indexer] Worker failure: ${reason} (${this.workerFailureCount}/${threshold})`);
+
+    if (this.workerFailureCount >= threshold) {
+      this.workersDisabledUntil = Date.now() + cooldownMs;
+      this.workerCircuitOpen = true;
+      console.warn(
+        `[Indexer] Worker circuit open; pausing worker use for ${Math.round(cooldownMs / 1000)}s`
+      );
+      this.scheduleRetry();
+    }
   }
 
   /**
    * Initialize worker thread pool for parallel embedding
    */
   async initializeWorkers() {
-    const numWorkers = this.config.workerThreads === "auto"
-      ? Math.min(4, Math.max(1, os.cpus().length - 1)) // Cap 'auto' at 4 workers
-      : (this.config.workerThreads || 1);
+    // Check if we have any active workers
+    const activeWorkers = this.workers.filter(w => w !== null);
+    if (activeWorkers.length > 0) return;
+    
+    // If we have workers array but they are all null, reset it
+    if (this.workers.length > 0) {
+        this.workers = [];
+        this.workerReady = [];
+    }
+
+    if (this.initWorkerPromise) return this.initWorkerPromise;
+
+    this.initWorkerPromise = (async () => {
+      try {
+    let numWorkers =
+      this.config.workerThreads === 'auto'
+        ? Math.min(2, Math.max(1, os.cpus().length - 1)) // Cap 'auto' at 2 workers
+        : typeof this.config.workerThreads === 'number'
+          ? this.config.workerThreads
+          : 1;
+
+    // Resource-aware scaling: check available RAM (skip in test env to avoid mocking issues)
+    // We apply this if we have > 1 worker, regardless of whether it was 'auto' or explicit
+    if (numWorkers > 1 && !isTestEnv() && typeof os.freemem === 'function') {
+      // Jina model typically requires ~1.5GB - 2GB per worker
+      const freeMemGb = os.freemem() / 1024 / 1024 / 1024;
+      const isHeavyModel = this.config.embeddingModel.includes('jina');
+      const memPerWorker = isHeavyModel ? 2.0 : 0.8;
+
+      const memCappedWorkers = Math.max(1, Math.floor(freeMemGb / memPerWorker));
+      if (memCappedWorkers < numWorkers) {
+        if (this.config.verbose) {
+          console.info(
+            `[Indexer] Throttling workers from ${numWorkers} to ${memCappedWorkers} due to available RAM (${freeMemGb.toFixed(1)}GB)`
+          );
+        }
+        numWorkers = memCappedWorkers;
+      }
+    }
 
-    // Only use workers if we have more than 1 CPU
-    if (numWorkers <= 1) {
-      console.error("[Indexer] Single-threaded mode (1 CPU detected)");
+    // Use workers even for single worker to benefit from --expose-gc and separate heap
+    if (numWorkers < 1) {
+      console.info('[Indexer] No workers configured, using main thread (warning: higher RAM usage)');
       return;
     }
 
     if (this.config.verbose) {
-      console.error(`[Indexer] Worker config: workerThreads=${this.config.workerThreads}, resolved to ${numWorkers}`);
+      console.info(
+        `[Indexer] Worker config: workerThreads=${this.config.workerThreads}, resolved to ${numWorkers}`
+      );
     }
 
-    console.error(`[Indexer] Initializing ${numWorkers} worker threads...`);
+    // Force 1 thread per worker to prevent CPU saturation (ONNX is very aggressive)
+    const threadsPerWorker = 1;
 
-    const workerPath = path.join(__dirname, "../lib/embedding-worker.js");
+    console.info(`[Indexer] Initializing ${numWorkers} worker threads (${threadsPerWorker} threads per worker)...`);
 
     for (let i = 0; i < numWorkers; i++) {
       try {
-        const worker = new Worker(workerPath, {
+        const worker = new Worker(new URL('../lib/embedding-worker.js', import.meta.url), {
           workerData: {
+            workerId: i,
             embeddingModel: this.config.embeddingModel,
-            verbose: this.config.verbose
-          }
+            verbose: this.config.verbose,
+            numThreads: threadsPerWorker,
+          },
         });
 
         const readyPromise = new Promise((resolve, reject) => {
-          const timeout = setTimeout(() => reject(new Error("Worker init timeout")), 120000);
+          const readyTimeoutMs = isTestEnv() ? 1000 : 120000;
+          const timeout = setTimeout(
+            () => reject(new Error('Worker init timeout')),
+            readyTimeoutMs
+          );
 
-          worker.once("message", (msg) => {
+          worker.once('message', (msg) => {
             clearTimeout(timeout);
-            if (msg.type === "ready") {
+            if (msg.type === 'ready') {
               resolve(worker);
-            } else if (msg.type === "error") {
+            } else if (msg.type === 'error') {
+              console.warn(`[Indexer] Worker initialization failed: ${msg.error}`);
               reject(new Error(msg.error));
             }
           });
 
-          worker.once("error", (err) => {
+          worker.once('error', (err) => {
             clearTimeout(timeout);
+            console.warn(`[Indexer] Worker initialization failed: ${err.message}`);
             reject(err);
           });
         });
@@ -137,40 +267,143 @@ export class CodebaseIndexer {
         this.workers.push(worker);
         this.workerReady.push(readyPromise);
       } catch (err) {
-        console.error(`[Indexer] Failed to create worker ${i}: ${err.message}`);
+        console.warn(`[Indexer] Failed to create worker ${i}: ${err.message}`);
       }
     }
 
     // Wait for all workers to be ready
     try {
       await Promise.all(this.workerReady);
-      console.error(`[Indexer] ${this.workers.length} workers ready`);
+      console.info(`[Indexer] ${this.workers.length} workers ready`);
       if (this.config.verbose) {
-        console.error(`[Indexer] Each worker loaded model: ${this.config.embeddingModel}`);
+        console.info(`[Indexer] Each worker loaded model: ${this.config.embeddingModel}`);
       }
     } catch (err) {
-      console.error(`[Indexer] Worker initialization failed: ${err.message}, falling back to single-threaded`);
+      console.warn(
+        `[Indexer] Worker initialization failed: ${err.message}, falling back to single-threaded`
+      );
       await this.terminateWorkers();
     }
+      } finally {
+        this.initWorkerPromise = null;
+      }
+    })();
+    return this.initWorkerPromise;
   }
 
   /**
    * Terminate all worker threads
    */
   async terminateWorkers() {
-    const terminations = this.workers.map((worker) => {
+    const WORKER_SHUTDOWN_TIMEOUT = isTestEnv() ? 50 : 5000;
+    const terminations = this.workers
+      .filter(Boolean)
+      .map((worker) => {
       try {
-        worker.postMessage({ type: "shutdown" });
-      } catch {}
-      return worker.terminate().catch(() => null);
-    });
+        worker.postMessage({ type: 'shutdown' });
+      } catch { /* ignore */ }
+
+      let exited = false;
+      const exitPromise = new Promise((resolve) => {
+        worker.once('exit', () => {
+          exited = true;
+          resolve();
+        });
+      });
+      const timeoutPromise = delay(WORKER_SHUTDOWN_TIMEOUT);
+
+      return Promise.race([exitPromise, timeoutPromise]).then(() => {
+        if (!exited) {
+          const termination = worker.terminate?.();
+          return Promise.resolve(termination).catch(() => null);
+        }
+        return null;
+      });
+      });
     await Promise.all(terminations);
     this.workers = [];
     this.workerReady = [];
   }
 
+  async loadGitignore() {
+    if (!this.config.searchDirectory) {
+      this.gitignore = ignore();
+      return;
+    }
+    try {
+      const gitignorePath = path.join(this.config.searchDirectory, '.gitignore');
+      const content = await fs.readFile(gitignorePath, 'utf8');
+      this.gitignore = ignore().add(content);
+      if (this.config.verbose) console.info('[Indexer] Loaded .gitignore rules');
+    } catch (_e) {
+      // No .gitignore or error reading it
+      this.gitignore = ignore();
+    }
+  }
+
+  getCacheRelativePath() {
+    if (!this.config.cacheDirectory || !this.config.searchDirectory) return null;
+    const relative = path.relative(this.config.searchDirectory, this.config.cacheDirectory);
+    if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) return null;
+    return normalizePath(relative);
+  }
+
   isExcluded(filePath) {
-    return matchesExcludePatterns(filePath, this.excludeMatchers);
+    if (!filePath || typeof filePath !== 'string') {
+      return false;
+    }
+
+    let relative = filePath;
+    if (path.isAbsolute(filePath)) {
+      if (this.config.searchDirectory) {
+        relative = path.relative(this.config.searchDirectory, filePath);
+        if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) {
+          return false;
+        }
+      } else {
+        const root = path.parse(filePath).root;
+        relative = filePath.slice(root.length);
+      }
+    }
+
+    relative = normalizePath(relative);
+
+    if (matchesExcludePatterns(relative, this.excludeMatchers)) return true;
+
+    if (this.gitignore.ignores(relative)) return true;
+
+    return false;
+  }
+
+  async replaceDeadWorker(index) {
+     if (this.config.verbose) console.info(`[Indexer] Replacing dead worker at index ${index}...`);
+     
+     const newWorker = new Worker(new URL('../lib/embedding-worker.js', import.meta.url), {
+          workerData: {
+            workerId: index,
+            embeddingModel: this.config.embeddingModel,
+            verbose: this.config.verbose,
+            numThreads: 1,
+          },
+     });
+
+     // Wait for ready
+     await new Promise((resolve, reject) => {
+         const timeout = setTimeout(() => reject(new Error('Timeout')), 30000);
+         newWorker.once('message', (msg) => {
+            if (msg.type === 'ready') {
+                clearTimeout(timeout);
+                resolve();
+            }
+         });
+         newWorker.once('error', (err) => {
+             clearTimeout(timeout);
+             reject(err);
+         });
+     });
+     
+     this.workers[index] = newWorker;
+     if (this.config.verbose) console.info(`[Indexer] Worker ${index} respawned successfully`);
   }
 
   /**
@@ -179,87 +412,319 @@ export class CodebaseIndexer {
   sendProgress(progress, total, message) {
     if (this.server) {
       try {
-        this.server.sendNotification("notifications/progress", {
-          progressToken: "indexing",
+        this.server.sendNotification('notifications/progress', {
+          progressToken: 'indexing',
           progress,
           total,
-          message
+          message,
         });
-      } catch (err) {
+      } catch (_err) {
         // Silently ignore if client doesn't support progress notifications
       }
     }
+    this.writeProgressFile(progress, total, message).catch(() => null);
+  }
+
+  async writeProgressFile(progress, total, message) {
+    if (!this.config.enableCache) return;
+
+    const payload = {
+      progress,
+      total,
+      message,
+      updatedAt: new Date().toISOString(),
+      indexMode: this.currentIndexMode || null,
+      workerCircuitOpen: !!this.workerCircuitOpen,
+      workersDisabledUntil: Number.isFinite(this.workersDisabledUntil)
+        ? this.workersDisabledUntil
+        : null,
+    };
+
+    const prev = this._lastProgress;
+    if (
+      prev &&
+      prev.progress === payload.progress &&
+      prev.total === payload.total &&
+      prev.message === payload.message
+    ) {
+      return;
+    }
+
+    this._lastProgress = payload;
+    try {
+      await fs.mkdir(this.config.cacheDirectory, { recursive: true });
+      const progressPath = path.join(this.config.cacheDirectory, 'progress.json');
+      await fs.writeFile(progressPath, JSON.stringify(payload), 'utf-8');
+    } catch {
+      // ignore progress write errors
+    }
+  }
+
+  async processFilesWithWorkers(allFiles) {
+    const activeWorkers = this.workers
+      .map((worker, index) => ({ worker, index }))
+      .filter((entry) => entry.worker);
+
+    if (activeWorkers.length === 0) {
+      // Fallback: This method shouldn't be called if workers aren't available, 
+      // but if it is, we return empty and let the caller handle legacy fallback.
+      return [];
+    }
+
+    const results = [];
+    const chunkSize = Math.ceil(allFiles.length / activeWorkers.length);
+    const workerPromises = [];
+    const configuredTimeout = Number.isInteger(this.config.workerBatchTimeoutMs)
+      ? this.config.workerBatchTimeoutMs
+      : 300000;
+    const WORKER_TIMEOUT = isTestEnv() ? 1000 : configuredTimeout;
+
+    for (let i = 0; i < activeWorkers.length; i++) {
+      const { worker, index: workerIndex } = activeWorkers[i];
+      const workerFiles = allFiles.slice(i * chunkSize, (i + 1) * chunkSize);
+      if (workerFiles.length === 0) continue;
+
+      if (this.config.verbose) {
+        console.info(`[Indexer] Worker ${workerIndex}: processing ${workerFiles.length} files`);
+      }
+
+      const promise = new Promise((resolve) => {
+        const batchId = `file-batch-${i}-${Date.now()}`;
+        const batchResults = [];
+
+        const killWorker = async () => {
+          try {
+            await worker.terminate?.();
+          } catch (_err) {
+            // ignore termination errors
+          }
+          this.workers[workerIndex] = null;
+          this.replaceDeadWorker(workerIndex).catch(() => {});
+        };
+
+        const handleTimeout = () => {
+          // Terminate first to ensure no more messages arrive
+          void killWorker();
+          worker.off('message', handler);
+          worker.off('error', errorHandler);
+          console.warn(`[Indexer] Worker ${workerIndex} timed out (files)`);
+          this.recordWorkerFailure(`timeout (batch ${batchId})`);
+          resolve([]);
+        };
+
+        let timeout = setTimeout(handleTimeout, WORKER_TIMEOUT);
+
+        const finalize = (results) => {
+          clearTimeout(timeout);
+          worker.off('message', handler);
+          worker.off('error', errorHandler);
+          resolve(results);
+        };
+
+        const handler = (msg) => {
+          if (msg.batchId === batchId) {
+            if (msg.type === 'results') {
+              if (Array.isArray(msg.results)) {
+                batchResults.push(...msg.results);
+              }
+              if (msg.done) {
+                finalize(batchResults);
+              }
+            } else if (msg.type === 'error') {
+              finalize([]);
+            }
+          }
+        };
+
+        const errorHandler = (err) => {
+          console.warn(`[Indexer] Worker ${workerIndex} crashed: ${err.message}`);
+          this.recordWorkerFailure(`crash (${err.message})`);
+          void killWorker();
+          finalize([]);
+        };
+
+        worker.once('error', errorHandler);
+        worker.on('message', handler);
+        
+        try {
+          worker.postMessage({ type: 'processFiles', files: workerFiles, batchId, chunkConfig: this.config });
+        } catch (_error) {
+          finalize([]);
+        }
+      });
+
+      workerPromises.push({ promise, files: workerFiles });
+    }
+
+    const workerResults = await Promise.all(workerPromises.map((p) => p.promise));
+
+    // Identify failed files for retry
+    const failedFiles = [];
+    for (let i = 0; i < workerResults.length; i++) {
+      if (workerResults[i].length > 0) {
+        results.push(...workerResults[i]);
+      } else if (workerPromises[i].files.length > 0) {
+        failedFiles.push(...workerPromises[i].files);
+      }
+    }
+
+    // Pass failed files back to be handled by legacy path
+    if (failedFiles.length > 0) {
+      if (this.config.verbose) {
+        console.warn(`[Indexer] ${failedFiles.length} files failed in workers, falling back to main thread`);
+      }
+      // Mark these as failed in the results so the caller knows to process them manually
+      for (const f of failedFiles) {
+        results.push({ file: f.file, status: 'retry' });
+      }
+    }
+
+    return results;
   }
 
   /**
    * Process chunks using worker thread pool with timeout and error recovery
    */
   async processChunksWithWorkers(allChunks) {
-    if (this.workers.length === 0) {
+    const activeWorkers = this.workers
+      .map((worker, index) => ({ worker, index }))
+      .filter((entry) => entry.worker);
+
+    if (activeWorkers.length === 0) {
       // Fallback to single-threaded processing
       return this.processChunksSingleThreaded(allChunks);
     }
 
     const results = [];
-    const chunkSize = Math.ceil(allChunks.length / this.workers.length);
+    const allowSingleThreadFallback = this.config.allowSingleThreadFallback !== false;
+    const chunkSize = Math.ceil(allChunks.length / activeWorkers.length);
     const workerPromises = [];
-    const WORKER_TIMEOUT = 300000; // 5 minutes per batch
+    const configuredTimeout = Number.isInteger(this.config.workerBatchTimeoutMs)
+      ? this.config.workerBatchTimeoutMs
+      : 300000;
+    const WORKER_TIMEOUT = isTestEnv() ? 1000 : configuredTimeout; // 1s in tests, configurable in prod
 
     if (this.config.verbose) {
-      console.error(`[Indexer] Distributing ${allChunks.length} chunks across ${this.workers.length} workers (~${chunkSize} chunks each)`);
+      console.info(
+        `[Indexer] Distributing ${allChunks.length} chunks across ${activeWorkers.length} workers (~${chunkSize} chunks each)`
+      );
     }
 
-    for (let i = 0; i < this.workers.length; i++) {
+    for (let i = 0; i < activeWorkers.length; i++) {
+      const { worker, index: workerIndex } = activeWorkers[i];
       const workerChunks = allChunks.slice(i * chunkSize, (i + 1) * chunkSize);
       if (workerChunks.length === 0) continue;
 
       if (this.config.verbose) {
-        console.error(`[Indexer] Worker ${i}: processing ${workerChunks.length} chunks`);
+        console.info(`[Indexer] Worker ${workerIndex}: processing ${workerChunks.length} chunks`);
       }
 
-      const promise = new Promise((resolve, reject) => {
-        const worker = this.workers[i];
+      const promise = new Promise((resolve, _reject) => {
         const batchId = `batch-${i}-${Date.now()}`;
+        const batchResults = [];
 
         // Timeout handler
-        const timeout = setTimeout(() => {
-          worker.off("message", handler);
-          console.error(`[Indexer] Worker ${i} timed out, falling back to single-threaded for this batch`);
+        const killWorker = async () => {
+          try {
+            await worker.terminate?.();
+          } catch {
+            // ignore terminate errors
+          }
+          this.workers[workerIndex] = null;
+          
+          // Attempt to replace the dead worker asynchronously
+          this.replaceDeadWorker(workerIndex).catch(err => {
+             console.warn(`[Indexer] Failed to replace worker ${workerIndex}: ${err.message}`);
+          });
+        };
+
+        const handleTimeout = (label) => {
+          // Terminate first to ensure no more messages arrive
+          void killWorker();
+          worker.off('message', handler);
+          worker.off('error', errorHandler);
+          console.warn(
+            `[Indexer] Worker ${workerIndex} timed out, ${label}`
+          );
+          this.recordWorkerFailure(`timeout (batch ${batchId})`);
           // Return empty and let fallback handle it
           resolve([]);
-        }, WORKER_TIMEOUT);
+        };
+
+        let timeout = setTimeout(
+          () => handleTimeout('killing worker and falling back to single-threaded for this batch'),
+          WORKER_TIMEOUT
+        );
+
+        const resetTimeout = () => {
+          clearTimeout(timeout);
+          timeout = setTimeout(
+            () => handleTimeout('killing worker and falling back to single-threaded for this batch'),
+            WORKER_TIMEOUT
+          );
+        };
+
+        let exitHandler;
+
+        const finalize = (results) => {
+          clearTimeout(timeout);
+          worker.off('message', handler);
+          worker.off('error', errorHandler);
+          if (exitHandler) worker.off('exit', exitHandler);
+          resolve(results);
+        };
 
         const handler = (msg) => {
           if (msg.batchId === batchId) {
-            clearTimeout(timeout);
-            worker.off("message", handler);
-            if (msg.type === "results") {
-              resolve(msg.results);
-            } else if (msg.type === "error") {
-              console.error(`[Indexer] Worker ${i} error: ${msg.error}`);
-              resolve([]); // Return empty, don't reject - let fallback handle
+            resetTimeout();
+            if (msg.type === 'results') {
+              if (Array.isArray(msg.results) && msg.results.length > 0) {
+                batchResults.push(...msg.results);
+              }
+              if (msg.done === false) {
+                return;
+              }
+              finalize(batchResults);
+            } else if (msg.type === 'error') {
+              console.warn(`[Indexer] Worker ${workerIndex} error: ${msg.error}`);
+              finalize([]); // Return empty, don't reject - let fallback handle
             }
           }
         };
 
         // Handle worker crash
         const errorHandler = (err) => {
-          clearTimeout(timeout);
-          worker.off("message", handler);
-          console.error(`[Indexer] Worker ${i} crashed: ${err.message}`);
-          resolve([]); // Return empty, don't reject
+          console.warn(`[Indexer] Worker ${workerIndex} crashed: ${err.message}`);
+          this.recordWorkerFailure(`crash (${err.message})`);
+          void killWorker();
+          finalize([]); // Return empty, don't reject
         };
-        worker.once("error", errorHandler);
+        worker.once('error', errorHandler);
+
+        exitHandler = (code) => {
+          if (code !== 0) {
+            console.warn(`[Indexer] Worker ${workerIndex} exited unexpectedly with code ${code}`);
+            this.recordWorkerFailure(`exit ${code}`);
+            void killWorker();
+            finalize([]);
+          }
+        };
+        worker.once('exit', exitHandler);
 
-        worker.on("message", handler);
-        worker.postMessage({ type: "process", chunks: workerChunks, batchId });
+        worker.on('message', handler);
+        try {
+          worker.postMessage({ type: 'process', chunks: workerChunks, batchId });
+        } catch (error) {
+          console.warn(`[Indexer] Worker ${i} postMessage failed: ${error.message}`);
+          finalize([]);
+        }
       });
 
       workerPromises.push({ promise, chunks: workerChunks });
     }
 
     // Wait for all workers with error recovery
-    const workerResults = await Promise.all(workerPromises.map(p => p.promise));
+    const workerResults = await Promise.all(workerPromises.map((p) => p.promise));
 
     // Collect results and identify failed chunks that need retry
     const failedChunks = [];
@@ -273,39 +738,128 @@ export class CodebaseIndexer {
     }
 
     // Retry failed chunks with single-threaded fallback
-    if (failedChunks.length > 0) {
-      console.error(`[Indexer] Retrying ${failedChunks.length} chunks with single-threaded fallback...`);
+    if (failedChunks.length > 0 && allowSingleThreadFallback) {
+      console.warn(
+        `[Indexer] Retrying ${failedChunks.length} chunks with single-threaded fallback...`
+      );
       const retryResults = await this.processChunksSingleThreaded(failedChunks);
       results.push(...retryResults);
+    } else if (failedChunks.length > 0) {
+      console.warn(
+        `[Indexer] Skipping ${failedChunks.length} chunks (single-threaded fallback disabled)`
+      );
     }
 
     return results;
   }
 
+  async processChunksInChildProcess(chunks) {
+    const nodePath = process.execPath || 'node';
+    const scriptPath = fileURLToPath(new URL('../lib/embedding-process.js', import.meta.url));
+    const payload = {
+      embeddingModel: this.config.embeddingModel,
+      chunks,
+      numThreads: 1,
+    };
+
+    return new Promise((resolve) => {
+      const child = spawn(nodePath, [scriptPath], {
+        stdio: ['pipe', 'pipe', 'pipe'],
+      });
+
+      let stdout = '';
+      let stderr = '';
+      child.stdout.on('data', (chunk) => {
+        stdout += chunk.toString();
+      });
+      child.stderr.on('data', (chunk) => {
+        stderr += chunk.toString();
+      });
+
+      const timeoutMs = Number.isInteger(this.config.workerBatchTimeoutMs)
+        ? this.config.workerBatchTimeoutMs
+        : 120000;
+      const timeout = setTimeout(() => {
+        try {
+          child.kill('SIGKILL');
+        } catch {
+          // ignore
+        }
+        this.recordWorkerFailure('child process timeout');
+        resolve([]);
+      }, timeoutMs);
+
+      child.on('error', (err) => {
+        clearTimeout(timeout);
+        this.recordWorkerFailure(`child process error (${err.message})`);
+        resolve([]);
+      });
+
+      child.on('close', (code, signal) => {
+        clearTimeout(timeout);
+        if (code !== 0) {
+          this.recordWorkerFailure(
+            `child process exited (${code ?? 'null'}${signal ? `, signal=${signal}` : ''})`
+          );
+          if (stderr) {
+            console.warn(`[Indexer] Child process error: ${stderr.trim()}`);
+          }
+          return resolve([]);
+        }
+        try {
+          const parsed = JSON.parse(stdout);
+          resolve(parsed?.results || []);
+        } catch (err) {
+          this.recordWorkerFailure(`child process parse error (${err.message})`);
+          resolve([]);
+        }
+      });
+
+      child.stdin.end(JSON.stringify(payload));
+    });
+  }
+
   /**
    * Single-threaded chunk processing (fallback)
    */
   async processChunksSingleThreaded(chunks) {
     const results = [];
 
+    // Manual GC and yield loop to prevent CPU lockup
+    let processedSinceGc = 0;
+    
     for (const chunk of chunks) {
+      // Throttle speed (balanced) - yield to event loop but don't wait unnecessarily
+      await delay(0);
+
       try {
-        const output = await this.embedder(chunk.text, { pooling: "mean", normalize: true });
-        results.push({
-          file: chunk.file,
-          startLine: chunk.startLine,
-          endLine: chunk.endLine,
-          content: chunk.text,
-          vector: Array.from(output.data),
-          success: true
-        });
+          const output = await this.embedder(chunk.text, {
+            pooling: 'mean',
+            normalize: true,
+          });
+          results.push({
+            file: chunk.file,
+            startLine: chunk.startLine,
+            endLine: chunk.endLine,
+            content: chunk.text,
+            vector: toFloat32Array(output.data),
+            success: true,
+          });
+
+          // Periodic GC to prevent memory creep (only if flag is present)
+          processedSinceGc++;
+          // Removed manual GC call to prevent performance degradation
+          if (processedSinceGc >= 50) { 
+            processedSinceGc = 0;
+          }
+
       } catch (error) {
         results.push({
           file: chunk.file,
           startLine: chunk.startLine,
           endLine: chunk.endLine,
           error: error.message,
-          success: false
+          success: false,
         });
       }
     }
@@ -317,12 +871,12 @@ export class CodebaseIndexer {
     const fileName = path.basename(file);
     if (this.isExcluded(file)) {
       if (this.config.verbose) {
-        console.error(`[Indexer] Skipped ${fileName} (excluded by pattern)`);
+        console.info(`[Indexer] Skipped ${fileName} (excluded by pattern)`);
       }
       return 0;
     }
     if (this.config.verbose) {
-      console.error(`[Indexer] Processing: ${fileName}...`);
+      console.info(`[Indexer] Processing: ${fileName}...`);
     }
 
     try {
@@ -336,62 +890,115 @@ export class CodebaseIndexer {
 
       if (stats.size > this.config.maxFileSize) {
         if (this.config.verbose) {
-          console.error(`[Indexer] Skipped ${fileName} (too large: ${(stats.size / 1024 / 1024).toFixed(2)}MB)`);
+          console.warn(
+            `[Indexer] Skipped ${fileName} (too large: ${(stats.size / 1024 / 1024).toFixed(2)}MB)`
+          );
         }
         return 0;
       }
 
-      const content = await fs.readFile(file, "utf-8");
+      const content = await fs.readFile(file, 'utf-8');
       const hash = hashContent(content);
 
       // Skip if file hasn't changed
-      if (this.cache.getFileHash(file) === hash) {
+      const cachedHash = typeof this.cache.getFileHash === 'function' ? this.cache.getFileHash(file) : null;
+      if (cachedHash === hash) {
         if (this.config.verbose) {
-          console.error(`[Indexer] Skipped ${fileName} (unchanged)`);
+          console.info(`[Indexer] Skipped ${fileName} (unchanged)`);
         }
+        // Still update metadata (size, mtime) even if hash is same
+        this.cache.setFileHash(file, hash, stats);
         return 0;
       }
 
       if (this.config.verbose) {
-        console.error(`[Indexer] Indexing ${fileName}...`);
+        console.info(`[Indexer] Indexing ${fileName}...`);
       }
 
-      // Remove old chunks for this file
-      this.cache.removeFileFromStore(file);
+      // Extract call graph data if enabled
+      let callData = null;
+      if (this.config.callGraphEnabled) {
+        try {
+          callData = extractCallData(content, file);
+        } catch (err) {
+          if (this.config.verbose) {
+            console.warn(
+              `[Indexer] Call graph extraction failed for ${fileName}: ${err.message}`
+            );
+          }
+        }
+      }
 
-      const chunks = smartChunk(content, file, this.config);
+      const rawChunks = smartChunk(content, file, this.config);
+      const chunks = Array.isArray(rawChunks) ? rawChunks : [];
       let addedChunks = 0;
+      let successChunks = 0;
       let failedChunks = 0;
+      const newChunks = [];
 
-      for (const chunk of chunks) {
-        try {
-          const output = await this.embedder(chunk.text, { pooling: "mean", normalize: true });
+      // Use workers for watcher-triggered embedding to keep main thread responsive
+      const useWorkers = this.shouldUseWorkers();
+      if (useWorkers && this.workers.length === 0) {
+        await this.initializeWorkers();
+      }
+
+      const chunksToProcess = chunks.map((c) => ({
+        file,
+        text: c.text,
+        startLine: c.startLine,
+        endLine: c.endLine
+      }));
 
-          this.cache.addToStore({
+      let results = [];
+      if (useWorkers && this.workers.length > 0) {
+        results = await this.processChunksWithWorkers(chunksToProcess);
+      } else {
+        results = await this.processChunksSingleThreaded(chunksToProcess);
+      }
+
+      for (const result of results) {
+        if (result.success) {
+          newChunks.push({
             file,
-            startLine: chunk.startLine,
-            endLine: chunk.endLine,
-            content: chunk.text,
-            vector: Array.from(output.data)
+            startLine: result.startLine,
+            endLine: result.endLine,
+            content: result.content,
+            vector: toFloat32Array(result.vector),
           });
           addedChunks++;
-        } catch (embeddingError) {
+          successChunks++;
+        } else {
+          console.warn(`[Indexer] Failed to embed chunk in ${fileName}:`, result.error);
           failedChunks++;
-          console.error(`[Indexer] Failed to embed chunk in ${fileName}:`, embeddingError.message);
         }
       }
 
-      if (chunks.length === 0 || failedChunks === 0) {
-        this.cache.setFileHash(file, hash);
+      const totalChunks = chunks.length;
+      const allSucceeded = totalChunks === 0 || failedChunks === 0;
+
+      if (allSucceeded) {
+        this.cache.removeFileFromStore(file);
+        for (const chunk of newChunks) {
+          this.cache.addToStore(chunk);
+        }
+        this.cache.setFileHash(file, hash, stats);
+        if (this.config.callGraphEnabled && callData) {
+          this.cache.setFileCallData(file, callData);
+        }
       } else if (this.config.verbose) {
-        console.error(`[Indexer] Skipped hash update for ${fileName} (${addedChunks}/${chunks.length} chunks embedded)`);
+        console.warn(
+          `[Indexer] Skipped hash update for ${fileName} (${successChunks}/${totalChunks} chunks embedded)`
+        );
       }
+      
       if (this.config.verbose) {
-        console.error(`[Indexer] Completed ${fileName} (${addedChunks} chunks)`);
+        console.info(`[Indexer] Completed ${fileName} (${addedChunks} chunks)`);
       }
       return addedChunks;
     } catch (error) {
-      console.error(`[Indexer] Error indexing ${fileName}:`, error.message);
+      if (this.config.verbose) {
+        console.warn(`[Indexer] Error indexing ${fileName}:`, error.message);
+      }
       return 0;
     }
   }
@@ -404,40 +1011,39 @@ export class CodebaseIndexer {
     const startTime = Date.now();
 
     // Build extension filter from config
-    const extensions = new Set(this.config.fileExtensions.map(ext => `.${ext}`));
+    const extensions = new Set(this.config.fileExtensions.map((ext) => `.${ext}`));
+    const allowedFileNames = new Set(this.config.fileNames || []);
 
-    // Extract directory names from glob patterns in config.excludePatterns
-    // Patterns like "**/node_modules/**" -> "node_modules"
-    const excludeDirs = new Set();
-    for (const pattern of this.config.excludePatterns) {
-      // Extract directory names from glob patterns
-      const match = pattern.match(/\*\*\/([^/*]+)\/?\*?\*?$/);
-      if (match) {
-        excludeDirs.add(match[1]);
-      }
-      // Also handle patterns like "**/dirname/**"
-      const match2 = pattern.match(/\*\*\/([^/*]+)\/\*\*$/);
-      if (match2) {
-        excludeDirs.add(match2[1]);
-      }
-    }
-
-    // Always exclude cache directory
-    excludeDirs.add(".smart-coding-cache");
+    // Load .gitignore before discovery
+    await this.loadGitignore();
 
-    if (this.config.verbose) {
-      console.error(`[Indexer] Using ${excludeDirs.size} exclude directories from config`);
+    if (!this.config.searchDirectory) {
+      return [];
     }
 
     const api = new fdir()
       .withFullPaths()
-      .exclude((dirName) => excludeDirs.has(dirName))
-      .filter((filePath) => extensions.has(path.extname(filePath)) && !this.isExcluded(filePath))
+      .exclude((dirName, dirPath) => {
+        // Always exclude specific heavy folders immediately
+        if (dirName === 'node_modules' || dirName === '.git' || dirName === '.smart-coding-cache') return true;
+
+        // Check exclusion rules for directories
+        const fullPath = path.join(dirPath, dirName);
+        return this.isExcluded(fullPath);
+      })
+      .filter((filePath) => {
+          if (this.isExcluded(filePath)) return false;
+
+          // Check extensions/filenames
+          const base = path.basename(filePath);
+          const ext = path.extname(filePath);
+          return (extensions.has(ext) || allowedFileNames.has(base));
+      })
       .crawl(this.config.searchDirectory);
 
     const files = await api.withPromise();
 
-    console.error(`[Indexer] File discovery: ${files.length} files in ${Date.now() - startTime}ms`);
+    console.info(`[Indexer] File discovery: ${files.length} files in ${Date.now() - startTime}ms`);
     return files;
   }
 
@@ -450,13 +1056,17 @@ export class CodebaseIndexer {
     const skippedCount = { unchanged: 0, tooLarge: 0, error: 0 };
 
     // Process in parallel batches for speed
-    const BATCH_SIZE = 500;
+    // We fetch stats for 100 files at a time to keep IO efficient
+    const STAT_BATCH_SIZE = Math.min(100, this.config.batchSize || 100);
+    // Limit concurrent file reads to 50MB to prevent OOM
+    const MAX_READ_BATCH_BYTES = 50 * 1024 * 1024;
 
-    for (let i = 0; i < files.length; i += BATCH_SIZE) {
-      const batch = files.slice(i, i + BATCH_SIZE);
+    for (let i = 0; i < files.length; i += STAT_BATCH_SIZE) {
+      const batchFiles = files.slice(i, i + STAT_BATCH_SIZE);
 
-      const results = await Promise.all(
-        batch.map(async (file) => {
+      // 1. Get stats for all files in this batch parallel
+      const fileStats = await Promise.all(
+        batchFiles.map(async (file) => {
           try {
             const stats = await fs.stat(file);
 
@@ -469,112 +1079,202 @@ export class CodebaseIndexer {
               return null;
             }
 
-            const content = await fs.readFile(file, "utf-8");
-            const hash = hashContent(content);
-
-            if (this.cache.getFileHash(file) === hash) {
-              skippedCount.unchanged++;
-              return null;
-            }
-
-            return { file, content, hash };
-          } catch (error) {
+            return { file, size: stats.size, mtimeMs: stats.mtimeMs };
+          } catch (_err) {
             skippedCount.error++;
             return null;
           }
         })
       );
 
-      for (const result of results) {
-        if (result) filesToProcess.push(result);
+      // 2. Process valid files in size-constrained sub-batches
+      let currentReadBatch = [];
+      let currentReadBytes = 0;
+
+      const processReadBatch = async (batch) => {
+        const results = await Promise.all(
+          batch.map(async ({ file, size, mtimeMs }) => {
+            // Check if we have cached metadata for this file
+            const cachedHash =
+              typeof this.cache.getFileHash === 'function' ? this.cache.getFileHash(file) : null;
+            const cachedMeta = this.cache.getFileMeta ? this.cache.getFileMeta(file) : null;
+            
+            if (cachedHash && cachedMeta && 
+                Number.isFinite(cachedMeta.mtimeMs) && cachedMeta.mtimeMs === mtimeMs &&
+                Number.isFinite(cachedMeta.size) && cachedMeta.size === size) {
+              // Metadata matches exactly, skip reading/hashing
+              skippedCount.unchanged++;
+              return null;
+            }
+
+            // Suspect file: Either new, or metadata changed. 
+            // We pass it to indexAll with the cachedHash as 'expectedHash'
+            // so workers can perform the actual hashing and unchanged check.
+            return { file, hash: null, expectedHash: cachedHash, force: false, size, mtimeMs };
+          })
+        );
+
+        for (const result of results) {
+          if (result) filesToProcess.push(result);
+        }
+      };
+
+      for (const item of fileStats) {
+        if (!item) continue;
+
+        if (
+          currentReadBytes + item.size > MAX_READ_BATCH_BYTES &&
+          currentReadBatch.length > 0
+        ) {
+          await processReadBatch(currentReadBatch);
+          currentReadBatch = [];
+          currentReadBytes = 0;
+        }
+
+        currentReadBatch.push(item);
+        currentReadBytes += item.size;
+      }
+
+      if (currentReadBatch.length > 0) {
+        await processReadBatch(currentReadBatch);
+      }
+      
+      // Pre-warm HybridSearch cache if available
+      if (this.server && this.server.hybridSearch && this.server.hybridSearch.fileModTimes) {
+          for (const stat of fileStats) {
+            if (stat && stat.file && typeof stat.mtimeMs === 'number') {
+              this.server.hybridSearch.fileModTimes.set(stat.file, stat.mtimeMs);
+            }
+          }
       }
     }
 
-    console.error(`[Indexer] Pre-filter: ${filesToProcess.length} changed, ${skippedCount.unchanged} unchanged, ${skippedCount.tooLarge} too large, ${skippedCount.error} errors (${Date.now() - startTime}ms)`);
+    if (this.config.verbose) {
+       console.info(
+        `[Indexer] Pre-filter: ${filesToProcess.length} changed, ${skippedCount.unchanged} unchanged, ${skippedCount.tooLarge} too large, ${skippedCount.error} errors (${Date.now() - startTime}ms)`
+      );
+    }
+    
+
+    
     return filesToProcess;
   }
 
   async indexAll(force = false) {
-    if (this.isIndexing) {
-      console.error("[Indexer] Indexing already in progress, skipping concurrent request");
-      return { skipped: true, reason: "Indexing already in progress" };
+    if (this.isIndexing || this.processingWatchEvents) {
+      console.warn('[Indexer] Indexing already in progress, skipping concurrent request');
+      return {
+        skipped: true,
+        reason: 'Indexing already in progress or pending file updates are being applied',
+      };
     }
 
     this.isIndexing = true;
+    let memoryTimer = null;
+    const logMemory = (label) => {
+      if (!this.config.verbose) return;
+      const { rss, heapUsed, heapTotal } = process.memoryUsage();
+      const toMb = (value) => `${(value / 1024 / 1024).toFixed(1)}MB`;
+      console.info(
+        `[Indexer] Memory ${label}: rss=${toMb(rss)} heap=${toMb(heapUsed)}/${toMb(heapTotal)}`,
+      );
+    };
 
     try {
+      logMemory('start');
+      if (this.config.verbose) {
+        memoryTimer = setInterval(() => logMemory('periodic'), 15000);
+      }
+
       if (force) {
-        console.error("[Indexer] Force reindex requested: clearing cache");
-        this.cache.setVectorStore([]);
-        this.cache.fileHashes = new Map();
-        await this.cache.clearCallGraphData({ removeFile: true });
+        console.info('[Indexer] Force reindex requested: clearing cache');
+        await this.cache.reset();
+      } else {
+        if (typeof this.cache.ensureLoaded === 'function') {
+          await this.cache.ensureLoaded();
+        }
       }
 
       const totalStartTime = Date.now();
-    console.error(`[Indexer] Starting optimized indexing in ${this.config.searchDirectory}...`);
-
-    // Step 1: Fast file discovery with fdir
-    const files = await this.discoverFiles();
-
-    if (files.length === 0) {
-      console.error("[Indexer] No files found to index");
-      this.sendProgress(100, 100, "No files found to index");
-      return { skipped: false, filesProcessed: 0, chunksCreated: 0, message: "No files found to index" };
-    }
+      const indexStartedAt = new Date(totalStartTime).toISOString();
+      let indexMode = force
+        ? 'full'
+        : this.cache.getVectorStore().length === 0
+          ? 'initial'
+          : 'incremental';
+      this.currentIndexMode = indexMode;
+      this.sendProgress(0, 100, 'Indexing started');
+        console.info(`[Indexer] Starting optimized indexing in ${this.config.searchDirectory}...`);
+
+      // Step 1: Fast file discovery with fdir
+      const files = await this.discoverFiles();
+
+      if (files.length === 0) {
+        console.info('[Indexer] No files found to index');
+        this.sendProgress(100, 100, 'No files found to index');
+        return {
+          skipped: false,
+          filesProcessed: 0,
+          chunksCreated: 0,
+          message: 'No files found to index',
+        };
+      }
 
-    // Send progress: discovery complete
-    this.sendProgress(5, 100, `Discovered ${files.length} files`);
+      // Send progress: discovery complete
+      this.sendProgress(5, 100, `Discovered ${files.length} files`);
 
-    const currentFilesSet = new Set(files);
+      const currentFilesSet = new Set(files);
 
-    // Step 1.5: Prune deleted or excluded files from cache
-    if (!force) {
-      const cachedFiles = Array.from(this.cache.fileHashes.keys());
-      let prunedCount = 0;
+      // Step 1.5: Prune deleted or excluded files from cache
+      if (!force) {
+        const cachedFiles =
+          typeof this.cache.getFileHashKeys === 'function' ? this.cache.getFileHashKeys() : [];
+        let prunedCount = 0;
 
-      for (const cachedFile of cachedFiles) {
-        if (!currentFilesSet.has(cachedFile)) {
-          this.cache.removeFileFromStore(cachedFile);
-          this.cache.deleteFileHash(cachedFile);
-          prunedCount++;
+        for (const cachedFile of cachedFiles) {
+          if (!currentFilesSet.has(cachedFile)) {
+            this.cache.removeFileFromStore(cachedFile);
+            this.cache.deleteFileHash(cachedFile);
+            prunedCount++;
+          }
         }
-      }
 
-      if (prunedCount > 0) {
-        if (this.config.verbose) {
-          console.error(`[Indexer] Pruned ${prunedCount} deleted/excluded files from index`);
+        if (prunedCount > 0) {
+          if (this.config.verbose) {
+            console.info(`[Indexer] Pruned ${prunedCount} deleted/excluded files from index`);
+          }
+          // If we pruned files, we should save these changes even if no other files changed
         }
-        // If we pruned files, we should save these changes even if no other files changed
-      }
 
-      const prunedCallGraph = this.cache.pruneCallGraphData(currentFilesSet);
-      if (prunedCallGraph > 0 && this.config.verbose) {
-        console.error(`[Indexer] Pruned ${prunedCallGraph} call-graph entries`);
+        const prunedCallGraph = this.cache.pruneCallGraphData(currentFilesSet);
+        if (prunedCallGraph > 0 && this.config.verbose) {
+          console.info(`[Indexer] Pruned ${prunedCallGraph} call-graph entries`);
+        }
       }
-    }
-
-    // Step 2: Pre-filter unchanged files (early hash check)
-    const filesToProcess = await this.preFilterFiles(files);
-    const filesToProcessSet = new Set(filesToProcess.map(entry => entry.file));
 
-    if (filesToProcess.length === 0) {
-      console.error("[Indexer] All files unchanged, nothing to index");
+      // Step 2: Pre-filter unchanged files (early hash check)
+      const filesToProcess = await this.preFilterFiles(files);
+      const filesToProcessSet = new Set(filesToProcess.map((entry) => entry.file));
+      const filesToProcessByFile = new Map(filesToProcess.map((entry) => [entry.file, entry]));
 
-      // If we have no call graph data but we have cached files, we should try to rebuild it
+      // Re-index files missing call graph data (if enabled)
       if (this.config.callGraphEnabled && this.cache.getVectorStore().length > 0) {
-        // Check for files that are in cache but missing from call graph data
-        const cachedFiles = new Set(this.cache.getVectorStore().map(c => c.file));
-        const callDataFiles = new Set(this.cache.fileCallData.keys());
+        const cachedFiles = new Set(this.cache.getVectorStore().map((c) => c.file));
+        const callDataFiles = new Set(this.cache.getFileCallDataKeys());
 
         const missingCallData = [];
         for (const file of cachedFiles) {
           if (!callDataFiles.has(file) && currentFilesSet.has(file)) {
             missingCallData.push(file);
+            const existing = filesToProcessByFile.get(file);
+            if (existing) existing.force = true;
           }
         }
 
         if (missingCallData.length > 0) {
-          console.error(`[Indexer] Found ${missingCallData.length} files missing call graph data, re-indexing...`);
+          console.info(
+            `[Indexer] Found ${missingCallData.length} files missing call graph data, re-indexing...`
+          );
           const BATCH_SIZE = 100;
           for (let i = 0; i < missingCallData.length; i += BATCH_SIZE) {
             const batch = missingCallData.slice(i, i + BATCH_SIZE);
@@ -582,11 +1282,14 @@ export class CodebaseIndexer {
               batch.map(async (file) => {
                 try {
                   const stats = await fs.stat(file);
+                  if (!stats || typeof stats.isDirectory !== 'function') {
+                    return null;
+                  }
                   if (stats.isDirectory()) return null;
                   if (stats.size > this.config.maxFileSize) return null;
-                  const content = await fs.readFile(file, "utf-8");
+                  const content = await fs.readFile(file, 'utf-8');
                   const hash = hashContent(content);
-                  return { file, content, hash };
+                  return { file, hash, force: true, size: stats.size, mtimeMs: stats.mtimeMs };
                 } catch {
                   return null;
                 }
@@ -595,198 +1298,535 @@ export class CodebaseIndexer {
 
             for (const result of results) {
               if (!result) continue;
-              if (filesToProcessSet.has(result.file)) continue;
-              filesToProcess.push(result);
-              filesToProcessSet.add(result.file);
+              if (!filesToProcessSet.has(result.file)) {
+                filesToProcess.push(result);
+                filesToProcessSet.add(result.file);
+              }
             }
           }
         }
       }
 
-      // If still empty after checking for missing call data, then we are truly done
+      indexMode = force
+        ? 'full'
+        : this.cache.getVectorStore().length === 0
+          ? 'initial'
+          : filesToProcess.length === files.length
+            ? 'full'
+            : 'incremental';
+      this.currentIndexMode = indexMode;
+
       if (filesToProcess.length === 0) {
-        this.sendProgress(100, 100, "All files up to date");
+        console.info('[Indexer] All files unchanged, nothing to index');
+        this.sendProgress(100, 100, 'All files up to date');
         await this.cache.save();
         const vectorStore = this.cache.getVectorStore();
         return {
           skipped: false,
           filesProcessed: 0,
           chunksCreated: 0,
-          totalFiles: new Set(vectorStore.map(v => v.file)).size,
+          totalFiles: new Set(vectorStore.map((v) => v.file)).size,
           totalChunks: vectorStore.length,
-          message: "All files up to date"
+          message: 'All files up to date',
         };
       }
-    }
 
-    // Send progress: filtering complete
-    this.sendProgress(10, 100, `Processing ${filesToProcess.length} changed files`);
+      // Send progress: filtering complete
+      console.info(`[Indexer] Processing ${filesToProcess.length} changed files`);
+      this.sendProgress(10, 100, `Processing ${filesToProcess.length} changed files`);
+
+      // Step 3: Determine batch size based on project size
+      // Adaptive batch size: use larger batches for larger projects to reduce overhead
+      let adaptiveBatchSize = 10;
+      if (files.length > 500) adaptiveBatchSize = 50;
+      if (files.length > 1000) adaptiveBatchSize = 100;
+      if (files.length > 5000) adaptiveBatchSize = 500;
+
+      if (this.config.verbose) {
+        console.info(
+          `[Indexer] Processing ${filesToProcess.length} files (batch size: ${adaptiveBatchSize})`
+        );
+      }
 
-    // Step 3: Determine batch size based on project size
-    const adaptiveBatchSize = files.length > 10000 ? 500 :
-                              files.length > 1000 ? 200 :
-                              this.config.batchSize || 100;
+      // Step 4: Initialize worker threads (skip if explicitly disabled)
+      const allowSingleThreadFallback =
+        this.config.allowSingleThreadFallback !== false ||
+        this.config.workerThreads === 0 ||
+        isTestEnv();
+      const useWorkers = this.shouldUseWorkers();
+
+      if (useWorkers) {
+        await this.initializeWorkers();
+        if (this.config.verbose && this.workers.length > 0) {
+          console.info(`[Indexer] Multi-threaded mode: ${this.workers.length} workers active`);
+        }
+      } else if (this.config.verbose) {
+        const until = this.workersDisabledUntil - Date.now();
+        if (this.workersDisabledUntil && until > 0) {
+          console.info(
+            `[Indexer] Workers disabled for ${Math.round(until / 1000)}s; single-threaded fallback ${allowSingleThreadFallback ? 'enabled' : 'disabled'}`
+          );
+        } else {
+          console.info(`[Indexer] Single-threaded mode (single-core system)`);
+        }
+      }
 
-    console.error(`[Indexer] Processing ${filesToProcess.length} files (batch size: ${adaptiveBatchSize})`);
+      const resolvedWorkerThreads = useWorkers ? this.workers.length : 0;
 
-    // Step 4: Initialize worker threads (always use when multi-core available)
-    const useWorkers = os.cpus().length > 1;
+      let totalChunks = 0;
+      let processedFiles = 0;
 
-    if (useWorkers) {
-      await this.initializeWorkers();
-      console.error(`[Indexer] Multi-threaded mode: ${this.workers.length} workers active`);
-    } else {
-      console.error(`[Indexer] Single-threaded mode (single-core system)`);
-    }
+      console.info(
+        `[Indexer] Embedding pass started: ${filesToProcess.length} files using ${this.config.embeddingModel}`
+      );
 
-    let totalChunks = 0;
-    let processedFiles = 0;
+      // Step 5: Process files in adaptive batches
+      for (let i = 0; i < filesToProcess.length; i += adaptiveBatchSize) {
+        const batch = filesToProcess.slice(i, i + adaptiveBatchSize);
 
-    // Step 5: Process files in adaptive batches
-    for (let i = 0; i < filesToProcess.length; i += adaptiveBatchSize) {
-      const batch = filesToProcess.slice(i, i + adaptiveBatchSize);
+        const allChunks = [];
+        const fileStats = new Map();
+        const newChunksByFile = new Map();
+        const callDataByFile = new Map();
+        const filesForWorkers = [];
 
-      // Generate all chunks for this batch
-      const allChunks = [];
-      const fileStats = new Map();
+        // Memory safeguard
+        const mem = process.memoryUsage();
+        if (mem.rss > 2048 * 1024 * 1024) {
+           if (global.gc) global.gc();
+        }
 
-      for (const { file, content, hash } of batch) {
-        // Remove old chunks for this file
-        this.cache.removeFileFromStore(file);
+        const useWorkersForBatch = useWorkers && this.workers.length > 0 && !this.config.embeddingProcessPerBatch;
+
+        for (const item of batch) {
+          const { file, force, content: presetContent, hash: presetHash, expectedHash: presetExpectedHash, size: presetSize, mtimeMs: presetMtimeMs } = item;
+          let content = presetContent;
+          let liveHash = presetHash;
+          let size = presetSize;
+          let mtimeMs = presetMtimeMs;
+          const expectedHash =
+            presetExpectedHash ||
+            (typeof this.cache.getFileHash === 'function' ? this.cache.getFileHash(file) : null);
+
+          if (useWorkersForBatch && (content === undefined || content === null)) {
+            // Speed optimization: Offload reading and hashing to workers.
+            // Main thread skips I/O entirely for this file.
+            filesForWorkers.push({ file, content: null, force, expectedHash });
+            // Initialize stats placeholder (will be updated with worker results)
+            fileStats.set(file, { hash: null, totalChunks: 0, successChunks: 0, size, mtimeMs });
+            continue;
+          }
 
-        // Extract call graph data if enabled
-        if (this.config.callGraphEnabled) {
-          try {
-            const callData = extractCallData(content, file);
-            this.cache.setFileCallData(file, callData);
-          } catch (err) {
-            if (this.config.verbose) {
-              console.error(`[Indexer] Call graph extraction failed for ${path.basename(file)}: ${err.message}`);
+          // Read content if not provided (Legacy Path or workers disabled)
+          if (content === undefined || content === null) {
+            let stats = null;
+            try {
+              stats = await fs.stat(file);
+            } catch (err) {
+              if (this.config.verbose) {
+                console.warn(`[Indexer] Failed to stat ${path.basename(file)}: ${err.message}`);
+              }
+              continue;
+            }
+            if (!stats || typeof stats.isDirectory !== 'function') {
+              if (this.config.verbose) {
+                console.warn(`[Indexer] Invalid stat result for ${path.basename(file)}`);
+              }
+              continue;
+            }
+            if (stats.isDirectory()) continue;
+            if (stats.size > this.config.maxFileSize) {
+              if (this.config.verbose) {
+                console.warn(
+                  `[Indexer] Skipped ${path.basename(file)} (too large: ${(stats.size / 1024 / 1024).toFixed(2)}MB)`
+                );
+              }
+              continue;
+            }
+            try {
+              content = await fs.readFile(file, 'utf-8');
+            } catch (err) {
+              if (this.config.verbose) {
+                console.warn(`[Indexer] Failed to read ${path.basename(file)}: ${err.message}`);
+              }
+              continue;
+            }
+            liveHash = hashContent(content);
+            size = stats.size;
+            mtimeMs = stats.mtimeMs;
+          } else {
+            if (typeof content !== 'string') content = String(content);
+            if (!liveHash) liveHash = hashContent(content);
+            if (!Number.isFinite(size)) {
+                // Use character length as approximation to avoid blocking Buffer.byteLength on large strings
+                size = content.length;
+            }
+            if (size > this.config.maxFileSize) {
+              if (this.config.verbose) {
+                console.warn(
+                  `[Indexer] Skipped ${path.basename(file)} (too large: ${(size / 1024 / 1024).toFixed(2)}MB)`
+                );
+              }
+              continue;
             }
           }
-        }
 
-        const chunks = smartChunk(content, file, this.config);
-        fileStats.set(file, { hash, totalChunks: 0, successChunks: 0 });
+          const cachedFileHash =
+            typeof this.cache.getFileHash === 'function' ? this.cache.getFileHash(file) : null;
+          if (!force && liveHash && cachedFileHash === liveHash) {
+            if (this.config.verbose) console.info(`[Indexer] Skipped ${path.basename(file)} (unchanged)`);
+            this.cache.setFileHash(file, liveHash, { size, mtimeMs });
+            continue;
+          }
 
-        for (const chunk of chunks) {
-          allChunks.push({
-            file,
-            text: chunk.text,
-            startLine: chunk.startLine,
-            endLine: chunk.endLine
-          });
-          const stats = fileStats.get(file);
-          if (stats) {
-            stats.totalChunks++;
+          if (useWorkersForBatch) {
+            filesForWorkers.push({ file, content, force, expectedHash: liveHash });
+            // Initialize stats placeholder (will be updated with worker results)
+            fileStats.set(file, { hash: liveHash, totalChunks: 0, successChunks: 0, size, mtimeMs });
+            continue;
+          }
+
+          // Legacy / Fallback path: Chunk on main thread
+          if (this.config.callGraphEnabled) {
+            try {
+              const callData = extractCallData(content, file);
+              callDataByFile.set(file, callData);
+            } catch (err) {
+              if (this.config.verbose) {
+                console.warn(
+                  `[Indexer] Call graph extraction failed for ${path.basename(file)}: ${err.message}`
+                );
+              }
+            }
+          }
+
+          const rawChunks = smartChunk(content, file, this.config);
+          const chunks = Array.isArray(rawChunks) ? rawChunks : [];
+          fileStats.set(file, { hash: liveHash, totalChunks: chunks.length, successChunks: 0, size, mtimeMs });
+
+          for (const chunk of chunks) {
+            allChunks.push({
+              file,
+              text: chunk.text,
+              startLine: chunk.startLine,
+              endLine: chunk.endLine,
+            });
           }
         }
-      }
 
-      // Process chunks (with workers if available, otherwise single-threaded)
-      let results;
-      if (useWorkers && this.workers.length > 0) {
-        results = await this.processChunksWithWorkers(allChunks);
-      } else {
-        results = await this.processChunksSingleThreaded(allChunks);
-      }
+        // Process files with workers (New Path)
+        if (filesForWorkers.length > 0) {
+          const results = await this.processFilesWithWorkers(filesForWorkers);
+          
+          for (const res of results) {
+            const stats = fileStats.get(res.file);
+            if (res.status === 'indexed' && stats) {
+              stats.totalChunks = res.results.length;
+              stats.successChunks = res.results.length;
+              if (res.hash) stats.hash = res.hash; // Update with new hash from worker
+              if (res.callData) callDataByFile.set(res.file, res.callData);
+              
+              const chunks = res.results.map(r => ({
+                file: res.file,
+                startLine: r.startLine,
+                endLine: r.endLine,
+                content: r.text,
+                vector: toFloat32Array(r.vectorBuffer),
+              }));
+              newChunksByFile.set(res.file, chunks);
+            } else if (res.status === 'unchanged' && stats) {
+              // Worker found file hash matches old hash
+              stats.totalChunks = 0; // Signal skip commit
+              stats.successChunks = 0;
+              stats.hash = res.hash;
+              this.cache.setFileHash(res.file, res.hash, { size: res.size, mtimeMs: res.mtimeMs });
+              if (res.callData && this.config.callGraphEnabled) {
+                this.cache.setFileCallData(res.file, res.callData);
+              }
+            } else if ((res.status === 'retry' || res.status === 'error') && stats) {
+              // Worker failed, fallback to local chunking + single threaded
+              const original = filesForWorkers.find(f => f.file === res.file);
+              if (original) {
+                if (this.config.verbose) console.info(`[Indexer] Fallback for ${path.basename(res.file)}`);
+                
+                if (this.config.callGraphEnabled) {
+                  try {
+                    callDataByFile.set(res.file, extractCallData(original.content, res.file));
+                  } catch (err) {
+                    if (this.config.verbose) {
+                      console.warn(
+                        `[Indexer] Call graph extraction failed for ${path.basename(res.file)}: ${err.message}`
+                      );
+                    }
+                  }
+                }
+                const fallbackChunks = smartChunk(original.content, res.file, this.config);
+                const chunks = Array.isArray(fallbackChunks) ? fallbackChunks : [];
+                stats.totalChunks = chunks.length;
+                for (const chunk of chunks) {
+                  allChunks.push({
+                    file: res.file,
+                    text: chunk.text,
+                    startLine: chunk.startLine,
+                    endLine: chunk.endLine,
+                  });
+                }
+              }
+            }
+          }
+        }
 
-      // Store successful results
-      for (const result of results) {
-        const stats = fileStats.get(result.file);
-        if (result.success) {
-          this.cache.addToStore({
-            file: result.file,
-            startLine: result.startLine,
-            endLine: result.endLine,
-            content: result.content,
-            vector: result.vector
-          });
-          totalChunks++;
-          if (stats) {
-            stats.successChunks++;
+        // Process chunks (Legacy Path & Fallbacks)
+        if (allChunks.length > 0) {
+          const chunksToProcess = allChunks.slice();
+          let results = [];
+          if (this.config.embeddingProcessPerBatch) {
+            results = await this.processChunksInChildProcess(chunksToProcess);
+          } else {
+            // If we are here, either workers are disabled/full or these are retry chunks
+            // Use single threaded fallback if not using child process
+            results = await this.processChunksSingleThreaded(chunksToProcess);
+          }
+
+          for (const result of results) {
+            const stats = fileStats.get(result.file);
+            if (result.success && stats) {
+              const items = newChunksByFile.get(result.file) || [];
+              items.push({
+                file: result.file,
+                startLine: result.startLine,
+                endLine: result.endLine,
+                content: result.content,
+                vector: toFloat32Array(result.vector),
+              });
+              newChunksByFile.set(result.file, items);
+              stats.successChunks++;
+            }
+          }
+        }
+
+        // Commit changes to cache
+        for (const [file, stats] of fileStats) {
+          if (stats.totalChunks > 0 && stats.successChunks === stats.totalChunks) {
+            this.cache.removeFileFromStore(file);
+            const newChunks = newChunksByFile.get(file) || [];
+            for (const chunk of newChunks) {
+              this.cache.addToStore(chunk);
+              totalChunks++;
+            }
+            this.cache.setFileHash(file, stats.hash, { size: stats.size, mtimeMs: stats.mtimeMs });
+            const callData = callDataByFile.get(file);
+            if (callData && this.config.callGraphEnabled) {
+              this.cache.setFileCallData(file, callData);
+            }
+          } else if (stats.totalChunks === 0) {
+             // File had no chunks (empty or comments only), just mark as indexed
+             this.cache.setFileHash(file, stats.hash, { size: stats.size, mtimeMs: stats.mtimeMs });
+             const callData = callDataByFile.get(file);
+             if (callData && this.config.callGraphEnabled) {
+               this.cache.setFileCallData(file, callData);
+             }
+          } else if (this.config.verbose) {
+            console.warn(
+              `[Indexer] Skipped hash update for ${path.basename(file)} (${stats.successChunks}/${stats.totalChunks} chunks embedded)`
+            );
           }
         }
+
+        if (global.gc) global.gc();
+
+        processedFiles += batch.length;
+
+        // Progress indicator
+        if (
+          processedFiles % (adaptiveBatchSize * 2) === 0 ||
+          processedFiles === filesToProcess.length
+        ) {
+          const elapsed = ((Date.now() - totalStartTime) / 1000).toFixed(1);
+          const rate = (processedFiles / parseFloat(elapsed)).toFixed(1);
+          console.info(
+            `[Indexer] Progress: ${processedFiles}/${filesToProcess.length} files (${rate} files/sec, ${elapsed}s elapsed)`
+          );
+          const progressPercent = Math.floor(10 + (processedFiles / filesToProcess.length) * 85);
+          this.sendProgress(
+            progressPercent,
+            100,
+            `Indexed ${processedFiles}/${filesToProcess.length} files (${rate}/sec)`
+          );
+        }
+
+        // Batch-level memory cleanup to reduce peak usage
+        allChunks.length = 0;
+        filesForWorkers.length = 0;
+        fileStats.clear();
+        newChunksByFile.clear();
+        callDataByFile.clear();
+        await delay(0);
+      }
+
+      // Cleanup workers
+      if (this.workers.length > 0) {
+        await this.terminateWorkers();
       }
+      if (global.gc) global.gc();
+
+      const totalDurationMs = Date.now() - totalStartTime;
+      const totalTime = (totalDurationMs / 1000).toFixed(1);
+      console.info(
+        `[Indexer] Embedding pass complete: ${totalChunks} chunks from ${filesToProcess.length} files in ${totalTime}s`
+      );
+
+      // Send completion progress
+      this.sendProgress(
+        100,
+        100,
+        `Complete: ${totalChunks} chunks from ${filesToProcess.length} files in ${totalTime}s`
+      );
+
+      this.cache.setLastIndexDuration?.(totalDurationMs);
+      this.cache.setLastIndexStats?.({
+        lastIndexStartedAt: indexStartedAt,
+        lastIndexEndedAt: new Date().toISOString(),
+        lastDiscoveredFiles: files.length,
+        lastFilesProcessed: filesToProcess.length,
+        lastIndexMode: indexMode,
+        lastBatchSize: adaptiveBatchSize,
+        lastWorkerThreads: resolvedWorkerThreads,
+        lastEmbeddingProcessPerBatch: this.config.embeddingProcessPerBatch,
+      });
+      await this.cache.save();
 
-      // Update file hashes
-      for (const [file, stats] of fileStats) {
-        if (stats.totalChunks === 0 || stats.successChunks === stats.totalChunks) {
-          this.cache.setFileHash(file, stats.hash);
-        } else if (this.config.verbose) {
-          console.error(`[Indexer] Skipped hash update for ${path.basename(file)} (${stats.successChunks}/${stats.totalChunks} chunks embedded)`);
+      if (this.config.clearCacheAfterIndex) {
+        console.info(
+          '[Indexer] clearCacheAfterIndex enabled; in-memory vectors will be reloaded on next query'
+        );
+        await this.cache.dropInMemoryVectors();
+        if (this.config.verbose) {
+          console.info('[Cache] Cleared in-memory vectors after indexing');
         }
       }
 
-      processedFiles += batch.length;
+      // Rebuild call graph in background
+      if (this.config.callGraphEnabled) {
+        this.cache.rebuildCallGraph();
+      }
 
-      // Progress indicator every batch
-      if (processedFiles % (adaptiveBatchSize * 2) === 0 || processedFiles === filesToProcess.length) {
-        const elapsed = ((Date.now() - totalStartTime) / 1000).toFixed(1);
-        const rate = (processedFiles / parseFloat(elapsed)).toFixed(0);
-        console.error(`[Indexer] Progress: ${processedFiles}/${filesToProcess.length} files (${rate} files/sec)`);
+      if (!this.config.clearCacheAfterIndex) {
+        void this.cache.ensureAnnIndex().catch((error) => {
+          if (this.config.verbose) {
+            console.warn(`[ANN] Background ANN build failed: ${error.message}`);
+          }
+        });
+      }
 
-        // Send MCP progress notification (10-95% range for batch processing)
-        const progressPercent = Math.floor(10 + (processedFiles / filesToProcess.length) * 85);
-        this.sendProgress(progressPercent, 100, `Indexed ${processedFiles}/${filesToProcess.length} files (${rate}/sec)`);
+      const vectorStore = this.cache.getVectorStore();
+      return {
+        skipped: false,
+        filesProcessed: filesToProcess.length,
+        chunksCreated: totalChunks,
+        totalFiles: new Set(vectorStore.map((v) => v.file)).size,
+        totalChunks: vectorStore.length,
+        duration: totalTime,
+        message: `Indexed ${filesToProcess.length} files (${totalChunks} chunks) in ${totalTime}s`,
+      };
+    } finally {
+      if (memoryTimer) {
+        clearInterval(memoryTimer);
       }
+      logMemory('end');
+      this.isIndexing = false;
+      try {
+        await this.processPendingWatchEvents();
+      } catch (error) {
+      console.warn(`[Indexer] Failed to apply queued file updates: ${error.message}`);
     }
+  }
+  }
 
-    // Cleanup workers
-    if (useWorkers) {
-      await this.terminateWorkers();
+  enqueueWatchEvent(type, filePath) {
+    // If it's a delete, it always wins
+    if (type === 'unlink') {
+      this.pendingWatchEvents.set(filePath, 'unlink');
+      return;
     }
 
-    const totalTime = ((Date.now() - totalStartTime) / 1000).toFixed(1);
-    console.error(`[Indexer] Complete: ${totalChunks} chunks from ${filesToProcess.length} files in ${totalTime}s`);
+    // If we're adding/changing, it overwrites a potential unlink (file came back)
+    this.pendingWatchEvents.set(filePath, type);
+  }
 
-    // Send completion progress
-    this.sendProgress(100, 100, `Complete: ${totalChunks} chunks from ${filesToProcess.length} files in ${totalTime}s`);
+  async processPendingWatchEvents() {
+    if (this.processingWatchEvents || this.pendingWatchEvents.size === 0) {
+      return;
+    }
 
-    await this.cache.save();
+    this.processingWatchEvents = true;
+    try {
+      while (this.pendingWatchEvents.size > 0) {
+        const pending = Array.from(this.pendingWatchEvents.entries());
+        this.pendingWatchEvents.clear();
 
-    // Rebuild call graph in background
-    if (this.config.callGraphEnabled) {
-      this.cache.rebuildCallGraph();
-    }
+        for (const [filePath, type] of pending) {
+          if (this.server && this.server.hybridSearch) {
+            this.server.hybridSearch.clearFileModTime(filePath);
+          }
 
-    void this.cache.ensureAnnIndex().catch((error) => {
-      if (this.config.verbose) {
-        console.error(`[ANN] Background ANN build failed: ${error.message}`);
-      }
-    });
+          if (type === 'unlink') {
+            await this.cache.removeFileFromStore(filePath);
+            this.cache.deleteFileHash(filePath);
+          } else {
+            await this.indexFile(filePath);
+          }
+        }
 
-    const vectorStore = this.cache.getVectorStore();
-    return {
-      skipped: false,
-      filesProcessed: filesToProcess.length,
-      chunksCreated: totalChunks,
-      totalFiles: new Set(vectorStore.map(v => v.file)).size,
-      totalChunks: vectorStore.length,
-      duration: totalTime,
-      message: `Indexed ${filesToProcess.length} files (${totalChunks} chunks) in ${totalTime}s`
-    };
+        await this.cache.save();
+      }
     } finally {
-      this.isIndexing = false;
+      this.processingWatchEvents = false;
     }
   }
 
-  setupFileWatcher() {
+  async setupFileWatcher() {
     if (!this.config.watchFiles) return;
 
-    const pattern = this.config.fileExtensions.map(ext => `**/*.${ext}`);
+    // Close existing watcher if active to prevent leaks
+    if (this.watcher) {
+      await this.watcher.close();
+      this.watcher = null;
+    }
+
+    await this.loadGitignore();
+
+    const pattern = [
+      ...this.config.fileExtensions.map((ext) => `**/*.${ext}`),
+      ...(this.config.fileNames || []).map((name) => `**/${name}`),
+    ];
+
+    const ignored = (filePath) => {
+      const fullPath = path.isAbsolute(filePath)
+        ? filePath
+        : path.join(this.config.searchDirectory, filePath);
+      return this.isExcluded(fullPath);
+    };
 
     this.watcher = chokidar.watch(pattern, {
       cwd: this.config.searchDirectory,
-      ignored: this.config.excludePatterns,
+      ignored,
       persistent: true,
-      ignoreInitial: true
+      ignoreInitial: true,
     });
 
     this.watcher
-      .on("add", async (filePath) => {
+      .on('add', async (filePath) => {
         const fullPath = path.join(this.config.searchDirectory, filePath);
-        console.error(`[Indexer] New file detected: ${filePath}`);
+        console.info(`[Indexer] New file detected: ${filePath}`);
+
+        if (this.isIndexing || this.processingWatchEvents) {
+          if (this.config.verbose) {
+            console.info(`[Indexer] Queued add event during indexing: ${filePath}`);
+          }
+          this.enqueueWatchEvent('add', fullPath);
+          return;
+        }
 
         // Invalidate recency cache
         if (this.server && this.server.hybridSearch) {
@@ -796,9 +1836,17 @@ export class CodebaseIndexer {
         await this.indexFile(fullPath);
         await this.cache.save();
       })
-      .on("change", async (filePath) => {
+      .on('change', async (filePath) => {
         const fullPath = path.join(this.config.searchDirectory, filePath);
-        console.error(`[Indexer] File changed: ${filePath}`);
+        console.info(`[Indexer] File changed: ${filePath}`);
+
+        if (this.isIndexing || this.processingWatchEvents) {
+          if (this.config.verbose) {
+            console.info(`[Indexer] Queued change event during indexing: ${filePath}`);
+          }
+          this.enqueueWatchEvent('change', fullPath);
+          return;
+        }
 
         // Invalidate recency cache
         if (this.server && this.server.hybridSearch) {
@@ -808,46 +1856,55 @@ export class CodebaseIndexer {
         await this.indexFile(fullPath);
         await this.cache.save();
       })
-      .on("unlink", (filePath) => {
+      .on('unlink', async (filePath) => {
         const fullPath = path.join(this.config.searchDirectory, filePath);
-        console.error(`[Indexer] File deleted: ${filePath}`);
+        console.info(`[Indexer] File deleted: ${filePath}`);
+
+        if (this.isIndexing || this.processingWatchEvents) {
+          if (this.config.verbose) {
+            console.info(`[Indexer] Queued delete event during indexing: ${filePath}`);
+          }
+          this.enqueueWatchEvent('unlink', fullPath);
+          return;
+        }
 
         // Invalidate recency cache
         if (this.server && this.server.hybridSearch) {
           this.server.hybridSearch.clearFileModTime(fullPath);
         }
 
-        this.cache.removeFileFromStore(fullPath);
+        await this.cache.removeFileFromStore(fullPath);
         this.cache.deleteFileHash(fullPath);
-        this.cache.save();
+        await this.cache.save();
       });
 
-    console.error("[Indexer] File watcher enabled for incremental indexing");
+    console.info('[Indexer] File watcher enabled for incremental indexing');
   }
 }
 
 // MCP Tool definition for this feature
 export function getToolDefinition() {
   return {
-    name: "b_index_codebase",
-    description: "Manually trigger a full reindex of the codebase. This will scan all files and update the embeddings cache. Useful after large code changes or if the index seems out of date.",
+    name: 'b_index_codebase',
+    description:
+      'Manually trigger a full reindex of the codebase. This will scan all files and update the embeddings cache. Useful after large code changes or if the index seems out of date.',
     inputSchema: {
-      type: "object",
+      type: 'object',
       properties: {
         force: {
-          type: "boolean",
+          type: 'boolean',
           description: "Force reindex even if files haven't changed",
-          default: false
-        }
-      }
+          default: false,
+        },
+      },
     },
     annotations: {
-      title: "Reindex Codebase",
+      title: 'Reindex Codebase',
       readOnlyHint: false,
       destructiveHint: false,
       idempotentHint: true,
-      openWorldHint: false
-    }
+      openWorldHint: false,
+    },
   };
 }
 
@@ -859,10 +1916,12 @@ export async function handleToolCall(request, indexer) {
   // Handle case when indexing was skipped due to concurrent request
   if (result?.skipped) {
     return {
-      content: [{
-        type: "text",
-        text: `Indexing skipped: ${result.reason}\n\nPlease wait for the current indexing operation to complete before requesting another reindex.`
-      }]
+      content: [
+        {
+          type: 'text',
+          text: `Indexing skipped: ${result.reason}\n\nPlease wait for the current indexing operation to complete before requesting another reindex.`,
+        },
+      ],
     };
   }
 
@@ -870,9 +1929,9 @@ export async function handleToolCall(request, indexer) {
   const vectorStore = indexer.cache.getVectorStore();
   const stats = {
     totalChunks: result?.totalChunks ?? vectorStore.length,
-    totalFiles: result?.totalFiles ?? new Set(vectorStore.map(v => v.file)).size,
+    totalFiles: result?.totalFiles ?? new Set(vectorStore.map((v) => v.file)).size,
     filesProcessed: result?.filesProcessed ?? 0,
-    chunksCreated: result?.chunksCreated ?? 0
+    chunksCreated: result?.chunksCreated ?? 0,
   };
 
   let message = result?.message
@@ -886,9 +1945,11 @@ export async function handleToolCall(request, indexer) {
   }
 
   return {
-    content: [{
-      type: "text",
-      text: message
-    }]
+    content: [
+      {
+        type: 'text',
+        text: message,
+      },
+    ],
   };
 }