codesift-mcp 0.1.0 → 0.2.1

package/dist/tools/index-tools.js

@@ -1,81 +1,82 @@
-import { readdir, readFile, stat, unlink, rm, mkdir as mkdirAsync } from "node:fs/promises";
+import { readFile, stat, unlink, rm, mkdir as mkdirAsync } from "node:fs/promises";
 import { join, relative, extname, resolve, basename } from "node:path";
 import { execFileSync } from "node:child_process";
+import { createHash } from "node:crypto";
 import { parseFile } from "../parser/parser-manager.js";
-import { extractSymbols, extractMarkdownSymbols, extractPrismaSymbols } from "../parser/symbol-extractor.js";
+import { extractSymbols, extractMarkdownSymbols, extractPrismaSymbols, extractAstroSymbols, extractConversationSymbols } from "../parser/symbol-extractor.js";
 import { getLanguageForExtension } from "../parser/parser-manager.js";
-import { saveIndex, loadIndex, getIndexPath, saveIncremental } from "../storage/index-store.js";
-import { registerRepo, listRepos as listRegistryRepos, getRepo, removeRepo, getRepoName } from "../storage/registry.js";
+import { saveIndex, loadIndex, getIndexPath, saveIncremental, removeFileFromIndex } from "../storage/index-store.js";
+import { registerRepo, listRepos as listRegistryRepos, getRepo, removeRepo, getRepoName, updateRepoMeta } from "../storage/registry.js";
 import { startWatcher, stopWatcher } from "../storage/watcher.js";
 import { buildBM25Index } from "../search/bm25.js";
 import { buildSymbolText, createEmbeddingProvider } from "../search/semantic.js";
 import { loadEmbeddings, saveEmbeddings, saveEmbeddingMeta, getEmbeddingPath, getEmbeddingMetaPath, batchEmbed } from "../storage/embedding-store.js";
 import { saveChunks, saveChunkEmbeddings, loadChunkEmbeddings, getChunkPath, getChunkEmbeddingPath } from "../storage/chunk-store.js";
-import { chunkFile } from "../search/chunker.js";
+import { chunkFile, chunkBySymbols } from "../search/chunker.js";
 import { loadConfig } from "../config.js";
 import { validateGitUrl, validateGitRef } from "../utils/git-validation.js";
-// Ignore patterns for directory walking (same as watcher)
-const IGNORE_DIRS = new Set([
-    "node_modules", ".git", "dist", "build", "coverage",
-    ".codesift", ".next", "__pycache__", ".pytest_cache",
-    ".venv", "venv", ".tox", ".mypy_cache", ".turbo",
-    "generated", "audit-results", ".backup", "jscpd-report",
-]);
-const MAX_FILE_SIZE = 1_000_000; // 1MB — skip giant files
+import { walkDirectory } from "../utils/walk.js";
+import { onFileChanged as scanOnChanged, onFileDeleted as scanOnDeleted, scanFileForSecrets } from "./secret-tools.js";
+import { getGraphPath } from "../storage/graph-store.js";
 const PARSE_CONCURRENCY = 8;
+const CHUNK_EMBEDDING_BATCH_SIZE = 96;
+const GIT_CLONE_TIMEOUT_MS = 120_000;
+const GIT_CHECKOUT_TIMEOUT_MS = 30_000;
+const GIT_PULL_TIMEOUT_MS = 60_000;
 // Active watchers and in-memory indexes keyed by repo name
 const activeWatchers = new Map();
 const bm25Indexes = new Map();
+const codeIndexes = new Map();
 const embeddingCaches = new Map();
 /**
- * Walk a directory tree, collecting files that can be parsed.
- * Respects .gitignore patterns and skips known non-source directories.
+ * Parse a single file and extract its symbols + metadata.
+ * Returns null if the file cannot be parsed.
  */
-async function walkDirectory(rootPath, includePaths) {
-    const files = [];
-    async function walk(dirPath) {
-        let entries;
-        try {
-            entries = await readdir(dirPath, { withFileTypes: true });
+async function parseOneFile(filePath, repoRoot, repoName) {
+    try {
+        const stat = await import("node:fs/promises").then((fs) => fs.stat(filePath));
+        const source = await readFile(filePath, "utf-8");
+        const relPath = relative(repoRoot, filePath);
+        const ext = extname(filePath);
+        const baseName = filePath.split("/").pop() ?? "";
+        const language = getLanguageForExtension(ext)
+            ?? (baseName.startsWith(".env") ? "config" : "unknown");
+        let symbols;
+        if (language === "markdown") {
+            symbols = extractMarkdownSymbols(source, relPath, repoName);
         }
-        catch {
-            return; // permission denied, etc.
+        else if (language === "prisma") {
+            symbols = extractPrismaSymbols(source, relPath, repoName);
         }
-        for (const entry of entries) {
-            const fullPath = join(dirPath, entry.name);
-            if (entry.isDirectory()) {
-                if (IGNORE_DIRS.has(entry.name) || entry.name.startsWith(".")) {
-                    continue;
-                }
-                await walk(fullPath);
-            }
-            else if (entry.isFile()) {
-                const ext = extname(entry.name);
-                const language = getLanguageForExtension(ext);
-                if (!language)
-                    continue;
-                // Filter by include paths if specified
-                if (includePaths && includePaths.length > 0) {
-                    const relPath = relative(rootPath, fullPath);
-                    const matches = includePaths.some((p) => relPath.startsWith(p));
-                    if (!matches)
-                        continue;
-                }
-                // Skip files that are too large
-                try {
-                    const fileStat = await stat(fullPath);
-                    if (fileStat.size > MAX_FILE_SIZE)
-                        continue;
-                }
-                catch {
-                    continue;
-                }
-                files.push(fullPath);
-            }
+        else if (language === "astro") {
+            symbols = extractAstroSymbols(source, relPath, repoName);
+        }
+        else if (language === "conversation") {
+            symbols = extractConversationSymbols(source, relPath, repoName);
+        }
+        else if (language === "config") {
+            symbols = [];
         }
+        else {
+            const tree = await parseFile(filePath, source);
+            if (!tree)
+                return null;
+            symbols = extractSymbols(tree, relPath, source, repoName, language);
+        }
+        const entry = {
+            path: relPath,
+            language,
+            symbol_count: symbols.length,
+            last_modified: Date.now(),
+            mtime_ms: Math.round(stat.mtimeMs),
+        };
+        return { symbols, entry };
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        console.warn(`[codesift] Failed to parse ${relative(repoRoot, filePath)}: ${message}`);
+        return null;
     }
-    await walk(rootPath);
-    return files;
 }
 /**
  * Parse files in parallel batches.
@@ -83,41 +84,9 @@ async function walkDirectory(rootPath, includePaths) {
 async function parseFiles(files, repoRoot, repoName) {
     const allSymbols = [];
     const fileEntries = [];
-    // Process in batches for controlled concurrency
     for (let i = 0; i < files.length; i += PARSE_CONCURRENCY) {
         const batch = files.slice(i, i + PARSE_CONCURRENCY);
-        const results = await Promise.all(batch.map(async (filePath) => {
-            try {
-                const source = await readFile(filePath, "utf-8");
-                const relPath = relative(repoRoot, filePath);
-                const ext = extname(filePath);
-                const language = getLanguageForExtension(ext) ?? "unknown";
-                let symbols;
-                // Markdown and Prisma use custom parsers (no tree-sitter grammar)
-                if (language === "markdown") {
-                    symbols = extractMarkdownSymbols(source, relPath, repoName);
-                }
-                else if (language === "prisma") {
-                    symbols = extractPrismaSymbols(source, relPath, repoName);
-                }
-                else {
-                    const tree = await parseFile(filePath, source);
-                    if (!tree)
-                        return null;
-                    symbols = extractSymbols(tree, relPath, source, repoName, language);
-                }
-                const entry = {
-                    path: relPath,
-                    language,
-                    symbol_count: symbols.length,
-                    last_modified: Date.now(),
-                };
-                return { symbols, entry };
-            }
-            catch {
-                return null;
-            }
-        }));
+        const results = await Promise.all(batch.map((filePath) => parseOneFile(filePath, repoRoot, repoName)));
         for (const result of results) {
             if (result) {
                 allSymbols.push(...result.symbols);
@@ -127,35 +96,241 @@ async function parseFiles(files, repoRoot, repoName) {
     }
     return { symbols: allSymbols, fileEntries };
 }
+// ---------------------------------------------------------------------------
+// Dirty propagation — mark caller files stale when a callee signature changes
+// ---------------------------------------------------------------------------
+/**
+ * Compute a hash of a symbol's public interface (name + kind + signature).
+ * Body changes don't trigger propagation — only signature changes.
+ */
+function computeSignatureHash(sym) {
+    const key = `${sym.name}|${sym.kind}|${sym.signature ?? ""}`;
+    return createHash("sha256").update(key).digest("hex").slice(0, 16);
+}
+/**
+ * Detect signature changes and mark caller files as stale.
+ * Returns the set of files marked stale.
+ */
+function propagateDirtySignatures(oldSymbols, newSymbols, fileEntries) {
+    // Build old signature hashes
+    const oldHashes = new Map();
+    for (const sym of oldSymbols) {
+        oldHashes.set(sym.id, computeSignatureHash(sym));
+    }
+    // Find symbols with changed signatures
+    const changedSymbolFiles = new Set();
+    for (const sym of newSymbols) {
+        const oldHash = oldHashes.get(sym.id);
+        if (oldHash && oldHash !== computeSignatureHash(sym)) {
+            changedSymbolFiles.add(sym.file);
+        }
+    }
+    if (changedSymbolFiles.size === 0)
+        return new Set();
+    // Find files that import from changed files (1 level of callers)
+    // Use a simple heuristic: check if any symbol source mentions a changed file's name
+    const changedBasenames = new Set();
+    for (const f of changedSymbolFiles) {
+        const base = f.split("/").pop()?.replace(/\.\w+$/, "");
+        if (base)
+            changedBasenames.add(base);
+    }
+    const staleFiles = new Set();
+    for (const sym of newSymbols) {
+        if (changedSymbolFiles.has(sym.file))
+            continue; // Don't mark the changed file itself
+        if (!sym.source)
+            continue;
+        for (const base of changedBasenames) {
+            if (sym.source.includes(base)) {
+                staleFiles.add(sym.file);
+                break;
+            }
+        }
+    }
+    // Mark stale in file entries (clear mtime so next index re-parses them)
+    for (const entry of fileEntries) {
+        if (staleFiles.has(entry.path)) {
+            entry.stale = true;
+            delete entry.mtime_ms; // Force re-parse on next indexFolder
+        }
+    }
+    return staleFiles;
+}
+/**
+ * Embed symbols using the configured embedding provider.
+ * Non-fatal — BM25 search still works if embedding fails.
+ */
+export async function embedSymbols(symbols, indexPath, repoName, config) {
+    if (!config.embeddingProvider)
+        return;
+    const embeddingPath = getEmbeddingPath(indexPath);
+    const metaPath = getEmbeddingMetaPath(indexPath);
+    try {
+        const provider = createEmbeddingProvider(config.embeddingProvider, config);
+        const symbolTexts = new Map(symbols.map((s) => [s.id, buildSymbolText(s)]));
+        const existing = await loadEmbeddings(embeddingPath);
+        const embeddings = await batchEmbed(symbolTexts, existing, provider.embed.bind(provider), config.embeddingBatchSize, repoName);
+        await saveEmbeddings(embeddingPath, embeddings);
+        await saveEmbeddingMeta(metaPath, {
+            model: provider.model,
+            provider: config.embeddingProvider,
+            dimensions: provider.dimensions,
+            symbol_count: embeddings.size,
+            updated_at: Date.now(),
+        });
+        embeddingCaches.set(repoName, embeddings);
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        console.error(`[codesift] Embedding failed for ${repoName}: ${message}`);
+    }
+}
+/**
+ * Read files in parallel batches and split each into chunks.
+ */
+async function readAndChunkFiles(fileEntries, rootPath, repoName, symbols) {
+    const allChunks = [];
+    for (let i = 0; i < fileEntries.length; i += PARSE_CONCURRENCY) {
+        const batch = fileEntries.slice(i, i + PARSE_CONCURRENCY);
+        const batchResults = await Promise.all(batch.map(async (entry) => {
+            const fullPath = join(rootPath, entry.path);
+            try {
+                const content = await readFile(fullPath, "utf-8");
+                if (symbols) {
+                    const fileSymbols = symbols
+                        .filter((s) => s.file === entry.path)
+                        .map((s) => ({ name: s.name, start_line: s.start_line, end_line: s.end_line }));
+                    return chunkBySymbols(entry.path, content, repoName, fileSymbols);
+                }
+                return chunkFile(entry.path, content, repoName);
+            }
+            catch (err) {
+                const message = err instanceof Error ? err.message : String(err);
+                console.warn(`[codesift] Failed to read ${entry.path} for chunking: ${message}`);
+                return [];
+            }
+        }));
+        for (const chunks of batchResults) {
+            allChunks.push(...chunks);
+        }
+    }
+    return allChunks;
+}
+/**
+ * Embed file chunks using the configured embedding provider.
+ * Non-fatal — symbol-level and BM25 search still work if this fails.
+ */
+async function embedChunks(fileEntries, rootPath, repoName, indexPath, config, symbols) {
+    if (!config.embeddingProvider)
+        return;
+    const chunkPath = getChunkPath(indexPath);
+    const chunkEmbeddingPath = getChunkEmbeddingPath(indexPath);
+    try {
+        const provider = createEmbeddingProvider(config.embeddingProvider, config);
+        const existingChunkEmbeddings = await loadChunkEmbeddings(chunkEmbeddingPath) ?? new Map();
+        const allChunks = await readAndChunkFiles(fileEntries, rootPath, repoName, symbols);
+        if (allChunks.length > 0) {
+            const chunkTexts = new Map(allChunks.map((c) => [c.id, c.text]));
+            const chunkEmbeddings = await batchEmbed(chunkTexts, existingChunkEmbeddings, provider.embed.bind(provider), CHUNK_EMBEDDING_BATCH_SIZE, `${repoName}:chunks`);
+            await saveChunks(chunkPath, allChunks);
+            await saveChunkEmbeddings(chunkEmbeddingPath, chunkEmbeddings);
+        }
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        console.error(`[codesift] Chunk embedding failed for ${repoName}: ${message}`);
+    }
+}
 export async function indexFolder(folderPath, options) {
+    if (!folderPath || typeof folderPath !== "string") {
+        throw new Error("folderPath is required and must be a non-empty string");
+    }
     const config = loadConfig();
     const startTime = Date.now();
     const rootPath = resolve(folderPath);
     const repoName = getRepoName(rootPath);
     const indexPath = getIndexPath(config.dataDir, rootPath);
-    // Check for incremental update
-    if (options?.incremental) {
-        const existing = await loadIndex(indexPath);
-        if (existing) {
-            // For now, incremental just returns existing stats.
-            // Full incremental support comes via file watcher.
-            return {
-                repo: repoName,
-                root: rootPath,
-                file_count: existing.file_count,
-                symbol_count: existing.symbol_count,
-                duration_ms: Date.now() - startTime,
-            };
+    // Walk directory and collect parseable files
+    const files = await walkDirectory(rootPath, {
+        includePaths: options?.include_paths,
+        fileFilter: (ext, name) => !!getLanguageForExtension(ext) || (name?.startsWith(".env") ?? false),
+    });
+    // mtime-based incremental: skip files unchanged since last index
+    const existing = await loadIndex(indexPath);
+    const mtimeMap = new Map();
+    if (existing) {
+        for (const f of existing.files) {
+            if (f.mtime_ms)
+                mtimeMap.set(f.path, f.mtime_ms);
         }
     }
-    // Walk directory and collect parseable files
-    const files = await walkDirectory(rootPath, options?.include_paths);
-    // Parse all files and extract symbols
-    const { symbols, fileEntries } = await parseFiles(files, rootPath, repoName);
-    // Build and cache BM25 index
+    const filesToParse = [];
+    const keptSymbols = [];
+    const keptEntries = [];
+    if (mtimeMap.size > 0) {
+        const { stat } = await import("node:fs/promises");
+        for (const filePath of files) {
+            const relPath = relative(rootPath, filePath);
+            const prevMtime = mtimeMap.get(relPath);
+            if (prevMtime !== undefined) {
+                const fileEntry = existing.files.find((f) => f.path === relPath);
+                // Force re-parse if file is marked stale (callee signature changed)
+                if (fileEntry?.stale) {
+                    filesToParse.push(filePath);
+                    continue;
+                }
+                try {
+                    const st = await stat(filePath);
+                    if (Math.round(st.mtimeMs) === prevMtime) {
+                        // File unchanged — keep existing symbols
+                        const fileSymbols = existing.symbols.filter((s) => s.file === relPath);
+                        if (fileEntry) {
+                            keptSymbols.push(...fileSymbols);
+                            keptEntries.push(fileEntry);
+                            continue;
+                        }
+                    }
+                }
+                catch { /* file may have been deleted — reparse */ }
+            }
+            filesToParse.push(filePath);
+        }
+    }
+    else {
+        filesToParse.push(...files);
+    }
+    // Parse only changed/new files
+    const { symbols: parsedSymbols, fileEntries: parsedEntries } = await parseFiles(filesToParse, rootPath, repoName);
+    const symbols = [...keptSymbols, ...parsedSymbols];
+    const fileEntries = [...keptEntries, ...parsedEntries];
+    // Dirty propagation: detect signature changes and mark caller files stale
+    if (existing && filesToParse.length > 0 && filesToParse.length < files.length) {
+        const staleFiles = propagateDirtySignatures(existing.symbols, symbols, fileEntries);
+        if (staleFiles.size > 0) {
+            console.error(`[codesift] Dirty propagation: ${staleFiles.size} caller files marked stale`);
+        }
+    }
+    // Build and cache BM25 index; invalidate code index cache
     const bm25 = buildBM25Index(symbols);
     bm25Indexes.set(repoName, bm25);
-    // Build code index
+    codeIndexes.delete(repoName);
+    // Sanity check: don't overwrite a complete index with a partial one
+    // (WASM crash or walk failure can produce truncated results)
+    const DROP_THRESHOLD = 0.5; // Reject if new index has <50% of old file count
+    if (existing && fileEntries.length < existing.file_count * DROP_THRESHOLD && existing.file_count > 50) {
+        console.error(`[codesift] SANITY CHECK FAILED for ${repoName}: ` +
+            `new index has ${fileEntries.length} files vs ${existing.file_count} previously. ` +
+            `Keeping old index. Use invalidate_cache + index_folder to force reindex.`);
+        return {
+            repo: repoName,
+            root: rootPath,
+            file_count: existing.file_count,
+            symbol_count: existing.symbol_count,
+            duration_ms: Date.now() - startTime,
+        };
+    }
+    // Build and save code index
     const codeIndex = {
         repo: repoName,
         root: rootPath,
@@ -166,67 +341,15 @@ export async function indexFolder(folderPath, options) {
         symbol_count: symbols.length,
         file_count: fileEntries.length,
     };
-    // Save index to disk
     await saveIndex(indexPath, codeIndex);
-    // Embed symbols if an embedding provider is configured (non-fatal if it fails)
-    if (config.embeddingProvider) {
-        const embeddingPath = getEmbeddingPath(indexPath);
-        const metaPath = getEmbeddingMetaPath(indexPath);
-        try {
-            const provider = createEmbeddingProvider(config.embeddingProvider, config);
-            const symbolTexts = new Map(symbols.map((s) => [s.id, buildSymbolText(s)]));
-            const existing = await loadEmbeddings(embeddingPath);
-            const embeddings = await batchEmbed(symbolTexts, existing, provider.embed.bind(provider), config.embeddingBatchSize);
-            await saveEmbeddings(embeddingPath, embeddings);
-            await saveEmbeddingMeta(metaPath, {
-                model: provider.model,
-                provider: config.embeddingProvider,
-                dimensions: provider.dimensions,
-                symbol_count: embeddings.size,
-                updated_at: Date.now(),
-            });
-            embeddingCaches.set(repoName, embeddings);
-        }
-        catch (err) {
-            const message = err instanceof Error ? err.message : String(err);
-            console.error(`[codesift] Embedding failed for ${repoName}: ${message}`);
-            // Non-fatal — BM25 search still works
-        }
-    }
-    // Embed file chunks if an embedding provider is configured (non-fatal if it fails)
-    if (config.embeddingProvider) {
-        const chunkPath = getChunkPath(indexPath);
-        const chunkEmbeddingPath = getChunkEmbeddingPath(indexPath);
-        try {
-            const provider = createEmbeddingProvider(config.embeddingProvider, config);
-            // Build chunks for all indexed files
-            const allChunks = [];
-            for (const entry of fileEntries) {
-                const fullPath = join(rootPath, entry.path);
-                try {
-                    const content = await readFile(fullPath, "utf-8");
-                    const fileChunks = chunkFile(entry.path, content, repoName);
-                    allChunks.push(...fileChunks);
-                }
-                catch {
-                    // Skip unreadable files
-                }
-            }
-            if (allChunks.length > 0) {
-                // Load existing chunk embeddings to avoid re-embedding unchanged chunks
-                const existingChunkEmbeddings = await loadChunkEmbeddings(chunkEmbeddingPath) ?? new Map();
-                const chunkTexts = new Map(allChunks.map((c) => [c.id, c.text]));
-                const chunkEmbeddings = await batchEmbed(chunkTexts, existingChunkEmbeddings, provider.embed.bind(provider), 96);
-                await saveChunks(chunkPath, allChunks);
-                await saveChunkEmbeddings(chunkEmbeddingPath, chunkEmbeddings);
-            }
-        }
-        catch (err) {
-            const message = err instanceof Error ? err.message : String(err);
-            console.error(`[codesift] Chunk embedding failed for ${repoName}: ${message}`);
-            // Non-fatal — symbol-level and BM25 search still work
-        }
-    }
+    // Embed symbols and chunks in background (non-fatal, don't block MCP response)
+    // Large repos (71K symbols) can take minutes — fire-and-forget to prevent timeout
+    embedSymbols(symbols, indexPath, repoName, config)
+        .then(() => embedChunks(fileEntries, rootPath, repoName, indexPath, config, symbols))
+        .catch((err) => {
+        const msg = err instanceof Error ? err.message : String(err);
+        console.error(`[codesift] Background embedding failed for ${repoName}: ${msg}`);
+    });
     // Register in the global registry
     const meta = {
         name: repoName,
@@ -237,20 +360,19 @@ export async function indexFolder(folderPath, options) {
         updated_at: Date.now(),
     };
     await registerRepo(config.registryPath, meta);
+    // Capture git HEAD for auto-refresh tracking
+    try {
+        const head = execFileSync("git", ["rev-parse", "HEAD"], {
+            cwd: rootPath, encoding: "utf-8", timeout: 5000,
+        }).trim();
+        await updateRepoMeta(config.registryPath, repoName, { last_git_commit: head });
+    }
+    catch {
+        // Not a git repo — skip
+    }
     // Start file watcher for incremental updates (unless disabled)
-    const shouldWatch = options?.watch !== false;
-    if (shouldWatch) {
-        const existingWatcher = activeWatchers.get(repoName);
-        if (existingWatcher) {
-            await stopWatcher(existingWatcher);
-        }
-        const watcher = startWatcher(rootPath, (changedFile) => {
-            handleFileChange(rootPath, repoName, indexPath, changedFile).catch((err) => {
-                const message = err instanceof Error ? err.message : String(err);
-                console.error(`[codesift] Watcher error for ${changedFile}: ${message}`);
-            });
-        });
-        activeWatchers.set(repoName, watcher);
+    if (options?.watch !== false) {
+        await setupWatcher(rootPath, repoName, indexPath);
     }
     return {
         repo: repoName,
@@ -294,7 +416,7 @@ export async function indexRepo(url, options) {
         if (options?.branch)
             args.push("--branch", options.branch);
         args.push("--", url, cloneTarget);
-        execFileSync("git", args, { stdio: "pipe", timeout: 120_000 });
+        execFileSync("git", args, { stdio: "pipe", timeout: GIT_CLONE_TIMEOUT_MS });
     }
     else {
         // Pull latest changes
@@ -302,22 +424,24 @@ export async function indexRepo(url, options) {
             if (options?.branch) {
                 execFileSync("git", ["-C", cloneTarget, "checkout", options.branch], {
                     stdio: "pipe",
-                    timeout: 30_000,
+                    timeout: GIT_CHECKOUT_TIMEOUT_MS,
                 });
             }
             execFileSync("git", ["-C", cloneTarget, "pull", "--ff-only"], {
                 stdio: "pipe",
-                timeout: 60_000,
+                timeout: GIT_PULL_TIMEOUT_MS,
             });
         }
-        catch {
+        catch (err) {
             // Pull may fail if detached HEAD — force fresh clone
+            const message = err instanceof Error ? err.message : String(err);
+            console.warn(`[codesift] Git pull failed for ${urlBasename}, re-cloning: ${message}`);
             await rm(cloneTarget, { recursive: true, force: true });
             const args = ["clone", "--depth", "1"];
             if (options?.branch)
                 args.push("--branch", options.branch);
             args.push("--", url, cloneTarget);
-            execFileSync("git", args, { stdio: "pipe", timeout: 120_000 });
+            execFileSync("git", args, { stdio: "pipe", timeout: GIT_CLONE_TIMEOUT_MS });
         }
     }
     // Index the cloned repo (no watcher for remote repos)
@@ -326,50 +450,72 @@ export async function indexRepo(url, options) {
         watch: false,
     });
 }
+/**
+ * Replace or create a file watcher for incremental index updates.
+ */
+async function setupWatcher(rootPath, repoName, indexPath) {
+    const existingWatcher = activeWatchers.get(repoName);
+    if (existingWatcher) {
+        await stopWatcher(existingWatcher);
+    }
+    const watcher = startWatcher(rootPath, (changedFile) => {
+        handleFileChange(rootPath, repoName, indexPath, changedFile).catch((err) => {
+            const message = err instanceof Error ? err.message : String(err);
+            console.error(`[codesift] Watcher error for ${changedFile}: ${message}`);
+        });
+    }, (deletedFile) => {
+        handleFileDelete(repoName, indexPath, deletedFile).catch((err) => {
+            const message = err instanceof Error ? err.message : String(err);
+            console.error(`[codesift] Watcher delete error for ${deletedFile}: ${message}`);
+        });
+    });
+    activeWatchers.set(repoName, watcher);
+}
 /**
  * Handle a file change event from the watcher.
  * Re-parses the changed file and updates the index incrementally.
  */
 async function handleFileChange(repoRoot, repoName, indexPath, relativeFile) {
     const fullPath = join(repoRoot, relativeFile);
-    try {
-        const source = await readFile(fullPath, "utf-8");
-        const ext = extname(relativeFile);
-        const language = getLanguageForExtension(ext) ?? "unknown";
-        let symbols;
-        // Markdown and Prisma use custom parsers (no tree-sitter grammar)
-        if (language === "markdown") {
-            symbols = extractMarkdownSymbols(source, relativeFile, repoName);
-        }
-        else if (language === "prisma") {
-            symbols = extractPrismaSymbols(source, relativeFile, repoName);
-        }
-        else {
-            const tree = await parseFile(fullPath, source);
-            if (!tree)
-                return;
-            symbols = extractSymbols(tree, relativeFile, source, repoName, language);
+    // Invalidate cached findings so the next scan sees the updated file contents.
+    scanOnChanged(repoName, relativeFile);
+    const result = await parseOneFile(fullPath, repoRoot, repoName);
+    if (!result)
+        return;
+    await saveIncremental(indexPath, relativeFile, result.symbols, result.entry);
+    if (loadConfig().secretScanEnabled) {
+        try {
+            await scanFileForSecrets(fullPath, relativeFile, repoName, result.symbols);
         }
-        const fileEntry = {
-            path: relativeFile,
-            language,
-            symbol_count: symbols.length,
-            last_modified: Date.now(),
-        };
-        await saveIncremental(indexPath, relativeFile, symbols, fileEntry);
-        // Rebuild in-memory BM25 index
-        const index = await loadIndex(indexPath);
-        if (index) {
-            bm25Indexes.set(repoName, buildBM25Index(index.symbols));
+        catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            console.warn(`[codesift] Secret scan failed for ${relativeFile}: ${message}`);
         }
     }
-    catch {
-        // File may have been deleted between event and read — ignore
-    }
+    // Invalidate caches — lazy rebuild on next query via getBM25Index()
+    bm25Indexes.delete(repoName);
+    codeIndexes.delete(repoName);
+    embeddingCaches.delete(repoName);
 }
-export async function listAllRepos() {
+/**
+ * Handle a file deletion event from the watcher.
+ * Removes all symbols for the deleted file from the index.
+ */
+async function handleFileDelete(repoName, indexPath, relativeFile) {
+    await removeFileFromIndex(indexPath, relativeFile);
+    // Invalidate caches — lazy rebuild on next query via getBM25Index()
+    bm25Indexes.delete(repoName);
+    codeIndexes.delete(repoName);
+    embeddingCaches.delete(repoName);
+    scanOnDeleted(repoName, relativeFile);
+}
+export async function listAllRepos(options) {
     const config = loadConfig();
-    return listRegistryRepos(config.registryPath);
+    const repos = await listRegistryRepos(config.registryPath);
+    if (options?.compact === false)
+        return repos;
+    // Default: ultra-compact — just repo names (agents only need the identifier)
+    return repos.map((r) => r.name);
 }
 export async function invalidateCache(repoName) {
     const config = loadConfig();
@@ -384,13 +530,15 @@ export async function invalidateCache(repoName) {
     }
     // Remove in-memory caches
     bm25Indexes.delete(repoName);
+    codeIndexes.delete(repoName);
     embeddingCaches.delete(repoName);
     // Delete index file + embedding files + chunk files
     const embeddingPath = getEmbeddingPath(meta.index_path);
     const embeddingMetaPath = getEmbeddingMetaPath(meta.index_path);
     const chunkPath = getChunkPath(meta.index_path);
     const chunkEmbeddingPath = getChunkEmbeddingPath(meta.index_path);
-    for (const fp of [meta.index_path, embeddingPath, embeddingMetaPath, chunkPath, chunkEmbeddingPath]) {
+    const graphStorePath = getGraphPath(meta.index_path);
+    for (const fp of [meta.index_path, embeddingPath, embeddingMetaPath, chunkPath, chunkEmbeddingPath, graphStorePath]) {
         try {
             await unlink(fp);
         }
@@ -400,11 +548,159 @@ export async function invalidateCache(repoName) {
     await removeRepo(config.registryPath, repoName);
     return true;
 }
+/**
+ * Re-index a single file instantly. Finds the repo by matching the file
+ * path against indexed repo roots. Updates symbols, BM25 index, and
+ * invalidates embedding cache — no full repo walk needed.
+ */
+export async function indexFile(filePath) {
+    const absPath = resolve(filePath);
+    const config = loadConfig();
+    const repos = await listRegistryRepos(config.registryPath);
+    // Find the most specific repo root that contains this file
+    const matchingRepo = repos
+        .filter((r) => absPath.startsWith(r.root + "/") || absPath === r.root)
+        .sort((a, b) => b.root.length - a.root.length)[0];
+    if (!matchingRepo) {
+        throw new Error(`No indexed repo contains "${absPath}". Run index_folder first.`);
+    }
+    const startTime = Date.now();
+    const relPath = relative(matchingRepo.root, absPath);
+    // mtime check — skip if unchanged
+    const existing = await loadIndex(matchingRepo.index_path);
+    if (existing) {
+        const prevEntry = existing.files.find((f) => f.path === relPath);
+        if (prevEntry?.mtime_ms) {
+            const st = await stat(absPath);
+            if (Math.round(st.mtimeMs) === prevEntry.mtime_ms) {
+                return {
+                    repo: matchingRepo.name,
+                    file: relPath,
+                    symbol_count: prevEntry.symbol_count,
+                    duration_ms: Date.now() - startTime,
+                    skipped: true,
+                };
+            }
+        }
+    }
+    const result = await parseOneFile(absPath, matchingRepo.root, matchingRepo.name);
+    if (!result) {
+        throw new Error(`Failed to parse "${relPath}"`);
+    }
+    await saveIncremental(matchingRepo.index_path, relPath, result.symbols, result.entry);
+    let secretFindingsCount = 0;
+    if (config.secretScanEnabled) {
+        try {
+            secretFindingsCount = (await scanFileForSecrets(absPath, relPath, matchingRepo.name, result.symbols)).length;
+        }
+        catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            console.warn(`[codesift] Secret scan failed for ${relPath}: ${message}`);
+        }
+    }
+    // Invalidate caches — lazy rebuild on next query via getBM25Index()
+    bm25Indexes.delete(matchingRepo.name);
+    codeIndexes.delete(matchingRepo.name);
+    embeddingCaches.delete(matchingRepo.name);
+    let secretsWarning;
+    if (secretFindingsCount > 0) {
+        secretsWarning = `\u26A0 ${secretFindingsCount} potential secret(s) detected`;
+    }
+    return {
+        repo: matchingRepo.name,
+        file: relPath,
+        symbol_count: result.symbols.length,
+        duration_ms: Date.now() - startTime,
+        ...(secretsWarning ? { secrets_warning: secretsWarning } : {}),
+    };
+}
+// ---------------------------------------------------------------------------
+// Git-based auto-refresh — transparent freshness check before index access
+// ---------------------------------------------------------------------------
+const freshnessChecked = new Map();
+const FRESHNESS_INTERVAL_MS = 60_000;
+const MAX_DIFF_FILES = 50;
+/**
+ * Ensure the index for a repo is fresh relative to git HEAD.
+ * Throttled to once per minute per repo. Reindexes changed files if HEAD moved.
+ * No-op for non-git repos.
+ */
+export async function ensureIndexFresh(repoName) {
+    const lastCheck = freshnessChecked.get(repoName);
+    if (lastCheck && Date.now() - lastCheck < FRESHNESS_INTERVAL_MS) {
+        return { status: "fresh" };
+    }
+    const config = loadConfig();
+    const meta = await getRepo(config.registryPath, repoName);
+    if (!meta)
+        return { status: "skipped" };
+    let currentCommit;
+    try {
+        currentCommit = execFileSync("git", ["rev-parse", "HEAD"], {
+            cwd: meta.root, encoding: "utf-8", timeout: 5000,
+        }).trim();
+    }
+    catch {
+        freshnessChecked.set(repoName, Date.now());
+        return { status: "skipped" };
+    }
+    if (meta.last_git_commit === currentCommit) {
+        freshnessChecked.set(repoName, Date.now());
+        return { status: "fresh" };
+    }
+    // HEAD moved — find changed files
+    let changedFiles = [];
+    if (meta.last_git_commit) {
+        try {
+            const diff = execFileSync("git", [
+                "diff", "--name-only", "--diff-filter=ACMR",
+                `${meta.last_git_commit}..${currentCommit}`,
+            ], {
+                cwd: meta.root, encoding: "utf-8", timeout: 10_000,
+            });
+            changedFiles = diff.trim().split("\n").filter(Boolean);
+        }
+        catch {
+            // Stored commit gone (rebase/squash) — will do full incremental
+            changedFiles = [];
+        }
+    }
+    if (changedFiles.length > 0 && changedFiles.length <= MAX_DIFF_FILES) {
+        for (const file of changedFiles) {
+            try {
+                await indexFile(join(meta.root, file));
+            }
+            catch {
+                // File deleted or unparseable — skip
+            }
+        }
+    }
+    else if (changedFiles.length > MAX_DIFF_FILES || !meta.last_git_commit) {
+        await indexFolder(meta.root, { incremental: true, watch: false });
+    }
+    await updateRepoMeta(config.registryPath, repoName, {
+        last_git_commit: currentCommit,
+        updated_at: Date.now(),
+    });
+    bm25Indexes.delete(repoName);
+    codeIndexes.delete(repoName);
+    embeddingCaches.delete(repoName);
+    freshnessChecked.set(repoName, Date.now());
+    return { status: "refreshed", files_updated: changedFiles.length };
+}
+/** Reset freshness throttle cache. Exported for testing. */
+export function resetFreshnessCache() {
+    freshnessChecked.clear();
+}
+// ---------------------------------------------------------------------------
+// Index access — with auto-refresh
+// ---------------------------------------------------------------------------
 /**
  * Get the in-memory BM25 index for a repo.
- * Loads from disk if not cached.
+ * Loads from disk if not cached. Auto-refreshes if git HEAD moved.
  */
 export async function getBM25Index(repoName) {
+    await ensureIndexFresh(repoName);
     const cached = bm25Indexes.get(repoName);
     if (cached)
         return cached;
@@ -421,13 +717,25 @@ export async function getBM25Index(repoName) {
 }
 /**
  * Get the code index for a repo from disk.
+ * Starts watcher if not running (lazy start after server restart).
+ */
+/**
+ * Get the code index for a repo from disk. Auto-refreshes if git HEAD moved.
  */
 export async function getCodeIndex(repoName) {
+    await ensureIndexFresh(repoName);
+    const cached = codeIndexes.get(repoName);
+    if (cached)
+        return cached;
     const config = loadConfig();
     const meta = await getRepo(config.registryPath, repoName);
     if (!meta)
         return null;
-    return loadIndex(meta.index_path);
+    const index = await loadIndex(meta.index_path);
+    if (!index)
+        return null;
+    codeIndexes.set(repoName, index);
+    return index;
 }
 /**
  * Get the in-memory embedding cache for a repo.