codesift-mcp 0.9.5 → 0.10.0

package/dist/tools/index-tools/file-indexer.js

@@ -0,0 +1,223 @@
+import { readFile, stat } from "node:fs/promises";
+import { join, resolve, relative, basename } from "node:path";
+import { execFileSync } from "node:child_process";
+import { createHash } from "node:crypto";
+import { clearTsconfigCache } from "../../utils/tsconfig-paths.js";
+import { getRepo, listRepos as listRegistryRepos, updateRepoMeta, } from "../../storage/registry.js";
+import { loadIndex, saveIncremental } from "../../storage/index-store.js";
+import { loadConfig } from "../../config.js";
+import { scanFileForSecrets } from "../secret-scan-shared.js";
+import { parseOneFile } from "./parse.js";
+import { indexFolder } from "./folder-indexer.js";
+import { bm25Indexes, codeIndexes, embeddingCaches } from "./state.js";
+/**
+ * In-process record of the last indexed state per absolute file path.
+ *
+ * Telemetry (30d, 2026-06): 750 consecutive duplicate index_file calls at
+ * avg 3.7s each (~47 min of agent wall-clock). Two causes: (1) duplicate
+ * hook registrations firing index_file twice per edit, and (2) a race where
+ * call N+1's on-disk mtime pre-check read the index before call N's
+ * serialized saveIncremental landed, forcing a full re-parse + full-index
+ * save. This map short-circuits both in-process in ~1ms (mtime first, then
+ * content hash for touch/no-op rewrites) without loading the on-disk index.
+ */
+const lastIndexedState = new Map();
+/** Test hook — clear the in-process last-indexed state. */
+export function clearLastIndexedStateForTesting() {
+    lastIndexedState.clear();
+}
+/**
+ * Re-index a single file instantly. Finds the repo by matching the file
+ * path against indexed repo roots. Updates symbols, BM25 index, and
+ * invalidates embedding cache — no full repo walk needed.
+ */
+export async function indexFile(filePath) {
+    const absPath = resolve(filePath);
+    const config = loadConfig();
+    const repos = await listRegistryRepos(config.registryPath);
+    // Find the most specific repo root that contains this file
+    const matchingRepo = repos
+        .filter((r) => absPath.startsWith(r.root + "/") || absPath === r.root)
+        .sort((a, b) => b.root.length - a.root.length)[0];
+    if (!matchingRepo) {
+        throw new Error(`No indexed repo contains "${absPath}". Run index_folder first.`);
+    }
+    const startTime = Date.now();
+    const relPath = relative(matchingRepo.root, absPath);
+    // If the changed file is a TS/JS config that drives path resolution, drop
+    // caches so incremental indexing picks up new `paths` / `extends`.
+    {
+        const cfg = basename(absPath).toLowerCase();
+        if ((cfg.startsWith("tsconfig") || cfg.startsWith("jsconfig")) &&
+            cfg.endsWith(".json")) {
+            clearTsconfigCache();
+        }
+    }
+    // In-process short-circuit: mtime, then content hash. Both avoid loading
+    // the on-disk index entirely (the expensive part on large repos).
+    const st = await stat(absPath);
+    const mem = lastIndexedState.get(absPath);
+    if (mem && Math.round(st.mtimeMs) === mem.mtimeMs) {
+        return {
+            repo: matchingRepo.name,
+            file: relPath,
+            symbol_count: mem.symbolCount,
+            duration_ms: Date.now() - startTime,
+            skipped: true,
+        };
+    }
+    const content = await readFile(absPath, "utf-8").catch(() => null);
+    const contentHash = content !== null ? createHash("sha1").update(content).digest("hex") : null;
+    if (mem && contentHash !== null && contentHash === mem.contentHash) {
+        // Touched / rewritten with identical content — refresh mtime, skip work.
+        mem.mtimeMs = Math.round(st.mtimeMs);
+        return {
+            repo: matchingRepo.name,
+            file: relPath,
+            symbol_count: mem.symbolCount,
+            duration_ms: Date.now() - startTime,
+            skipped: true,
+        };
+    }
+    // On-disk mtime check — first touch of this file in this process (CLI
+    // hook invocations, fresh server). Skips files unchanged since the last
+    // full index, and seeds the in-process state for subsequent calls.
+    if (!mem) {
+        const existing = await loadIndex(matchingRepo.index_path);
+        if (existing) {
+            const prevEntry = existing.files.find((f) => f.path === relPath);
+            if (prevEntry?.mtime_ms && Math.round(st.mtimeMs) === prevEntry.mtime_ms) {
+                if (contentHash !== null) {
+                    lastIndexedState.set(absPath, {
+                        mtimeMs: Math.round(st.mtimeMs),
+                        contentHash,
+                        symbolCount: prevEntry.symbol_count,
+                    });
+                }
+                return {
+                    repo: matchingRepo.name,
+                    file: relPath,
+                    symbol_count: prevEntry.symbol_count,
+                    duration_ms: Date.now() - startTime,
+                    skipped: true,
+                };
+            }
+        }
+    }
+    const result = await parseOneFile(absPath, matchingRepo.root, matchingRepo.name);
+    if (!result) {
+        throw new Error(`Failed to parse "${relPath}"`);
+    }
+    await saveIncremental(matchingRepo.index_path, relPath, result.symbols, result.entry);
+    if (contentHash !== null) {
+        lastIndexedState.set(absPath, {
+            mtimeMs: Math.round(st.mtimeMs),
+            contentHash,
+            symbolCount: result.symbols.length,
+        });
+    }
+    let secretFindingsCount = 0;
+    if (config.secretScanEnabled) {
+        try {
+            secretFindingsCount = (await scanFileForSecrets(absPath, relPath, matchingRepo.name, result.symbols)).length;
+        }
+        catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            console.warn(`[codesift] Secret scan failed for ${relPath}: ${message}`);
+        }
+    }
+    // Invalidate caches — lazy rebuild on next query via getBM25Index()
+    bm25Indexes.delete(matchingRepo.name);
+    codeIndexes.delete(matchingRepo.name);
+    embeddingCaches.delete(matchingRepo.name);
+    let secretsWarning;
+    if (secretFindingsCount > 0) {
+        secretsWarning = `\u26A0 ${secretFindingsCount} potential secret(s) detected`;
+    }
+    return {
+        repo: matchingRepo.name,
+        file: relPath,
+        symbol_count: result.symbols.length,
+        duration_ms: Date.now() - startTime,
+        ...(secretsWarning ? { secrets_warning: secretsWarning } : {}),
+    };
+}
+// ---------------------------------------------------------------------------
+// Git-based auto-refresh — transparent freshness check before index access
+// ---------------------------------------------------------------------------
+const freshnessChecked = new Map();
+const FRESHNESS_INTERVAL_MS = 60_000;
+const MAX_DIFF_FILES = 50;
+/**
+ * Ensure the index for a repo is fresh relative to git HEAD.
+ * Throttled to once per minute per repo. Reindexes changed files if HEAD moved.
+ * No-op for non-git repos.
+ */
+export async function ensureIndexFresh(repoName) {
+    const lastCheck = freshnessChecked.get(repoName);
+    if (lastCheck && Date.now() - lastCheck < FRESHNESS_INTERVAL_MS) {
+        return { status: "fresh" };
+    }
+    const config = loadConfig();
+    const meta = await getRepo(config.registryPath, repoName);
+    if (!meta)
+        return { status: "skipped" };
+    let currentCommit;
+    try {
+        currentCommit = execFileSync("git", ["rev-parse", "HEAD"], {
+            cwd: meta.root, encoding: "utf-8", timeout: 5000,
+        }).trim();
+    }
+    catch {
+        freshnessChecked.set(repoName, Date.now());
+        return { status: "skipped" };
+    }
+    if (meta.last_git_commit === currentCommit) {
+        freshnessChecked.set(repoName, Date.now());
+        return { status: "fresh" };
+    }
+    // HEAD moved — find changed files
+    let changedFiles = [];
+    if (meta.last_git_commit) {
+        try {
+            const diff = execFileSync("git", [
+                "diff", "--name-only", "--diff-filter=ACMR",
+                `${meta.last_git_commit}..${currentCommit}`,
+            ], {
+                cwd: meta.root, encoding: "utf-8", timeout: 10_000,
+            });
+            changedFiles = diff.trim().split("\n").filter(Boolean);
+        }
+        catch {
+            // Stored commit gone (rebase/squash) — will do full incremental
+            changedFiles = [];
+        }
+    }
+    if (changedFiles.length > 0 && changedFiles.length <= MAX_DIFF_FILES) {
+        for (const file of changedFiles) {
+            try {
+                await indexFile(join(meta.root, file));
+            }
+            catch {
+                // File deleted or unparseable — skip
+            }
+        }
+    }
+    else if (changedFiles.length > MAX_DIFF_FILES || !meta.last_git_commit) {
+        await indexFolder(meta.root, { incremental: true, watch: false });
+    }
+    await updateRepoMeta(config.registryPath, repoName, {
+        last_git_commit: currentCommit,
+        updated_at: Date.now(),
+    });
+    bm25Indexes.delete(repoName);
+    codeIndexes.delete(repoName);
+    embeddingCaches.delete(repoName);
+    freshnessChecked.set(repoName, Date.now());
+    return { status: "refreshed", files_updated: changedFiles.length };
+}
+/** Reset freshness throttle cache. Exported for testing. */
+export function resetFreshnessCache() {
+    freshnessChecked.clear();
+}
+//# sourceMappingURL=file-indexer.js.map

package/dist/tools/index-tools/file-indexer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"file-indexer.js","sourceRoot":"","sources":["../../../src/tools/index-tools/file-indexer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AACnE,OAAO,EACL,OAAO,EACP,SAAS,IAAI,iBAAiB,EAC9B,cAAc,GACf,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAEvE;;;;;;;;;;GAUG;AACH,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAyE,CAAC;AAE1G,2DAA2D;AAC3D,MAAM,UAAU,+BAA+B;IAC7C,gBAAgB,CAAC,KAAK,EAAE,CAAC;AAC3B,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,QAAgB;IAQ9C,MAAM,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IAClC,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;IAC5B,MAAM,KAAK,GAAG,MAAM,iBAAiB,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;IAE3D,2DAA2D;IAC3D,MAAM,YAAY,GAAG,KAAK;SACvB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,GAAG,GAAG,CAAC,IAAI,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC;SACrE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAEpD,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,6BAA6B,OAAO,4BAA4B,CAAC,CAAC;IACpF,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,OAAO,GAAG,QAAQ,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAErD,0EAA0E;IAC1E,mEAAmE;IACnE,CAAC;QACC,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;QAC5C,IACE,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;YAC1D,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,EACrB,CAAC;YACD,kBAAkB,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;IAED,yEAAyE;IACzE,kEAAkE;IAClE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,CAAC;IAC/B,MAAM,GAAG,GAAG,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC1C,IAAI,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC;QAClD,OAAO;YACL,IAAI,EAAE,YAAY,CAAC,IAAI;YACvB,IAAI,EAAE,OAAO;YACb,YAAY,EAAE,GAAG,CAAC,WAAW;YAC7B,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YACnC,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;IACnE,MAAM,WAAW,GAAG,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC/F,IAAI,GAAG,IAAI,WAAW,KAAK,IAAI,IAAI,WAAW,KAAK,GAAG,CAAC,WAAW,EAAE,CAAC;QACnE,yEAAyE;QACzE,GAAG,CAAC,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;QACrC,OAAO;YACL,IAAI,EAAE,YAAY,CAAC,IAAI;YACvB,IAAI,EAAE,OAAO;YACb,YAAY,EAAE,GAAG,CAAC,WAAW;YAC7B,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YACnC,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;IAED,sEAAsE;IACtE,wEAAwE;IACxE,mEAAmE;IACnE,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;QAC1D,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC;YACjE,IAAI,SAAS,EAAE,QAAQ,IAAI,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,SAAS,CAAC,QAAQ,EAAE,CAAC;gBACzE,IAAI,WAAW,KAAK,IAAI,EAAE,CAAC;oBACzB,gBAAgB,CAAC,GAAG,CAAC,OAAO,EAAE;wBAC5B,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,OAAO,CAAC;wBAC/B,WAAW;wBACX,WAAW,EAAE,SAAS,CAAC,YAAY;qBACpC,CAAC,CAAC;gBACL,CAAC;gBACD,OAAO;oBACL,IAAI,EAAE,YAAY,CAAC,IAAI;oBACvB,IAAI,EAAE,OAAO;oBACb,YAAY,EAAE,SAAS,CAAC,YAAY;oBACpC,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;oBACnC,OAAO,EAAE,IAAI;iBACd,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,OAAO,EAAE,YAAY,CAAC,IAAI,EAAE,YAAY,CAAC,IAAI,CAAC,CAAC;IACjF,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CAAC,oBAAoB,OAAO,GAAG,CAAC,CAAC;IAClD,CAAC;IAED,MAAM,eAAe,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;IAEtF,IAAI,WAAW,KAAK,IAAI,EAAE,CAAC;QACzB,gBAAgB,CAAC,GAAG,CAAC,OAAO,EAAE;YAC5B,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,OAAO,CAAC;YAC/B,WAAW;YACX,WAAW,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM;SACnC,CAAC,CAAC;IACL,CAAC;IAED,IAAI,mBAAmB,GAAG,CAAC,CAAC;IAC5B,IAAI,MAAM,CAAC,iBAAiB,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,mBAAmB,GAAG,CACpB,MAAM,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,CAC9E,CAAC,MAAM,CAAC;QACX,CAAC;QAAC,OAAO,GAAY,EAAE,CAAC;YACtB,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,OAAO,CAAC,IAAI,CAAC,qCAAqC,OAAO,KAAK,OAAO,EAAE,CAAC,CAAC;QAC3E,CAAC;IACH,CAAC;IAED,oEAAoE;IACpE,WAAW,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IACtC,WAAW,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IACtC,eAAe,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAE1C,IAAI,cAAkC,CAAC;IACvC,IAAI,mBAAmB,GAAG,CAAC,EAAE,CAAC;QAC5B,cAAc,GAAG,UAAU,mBAAmB,+BAA+B,CAAC;IAChF,CAAC;IAED,OAAO;QACL,IAAI,EAAE,YAAY,CAAC,IAAI;QACvB,IAAI,EAAE,OAAO;QACb,YAAY,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM;QACnC,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;QACnC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC/D,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,2EAA2E;AAC3E,8EAA8E;AAE9E,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAkB,CAAC;AACnD,MAAM,qBAAqB,GAAG,MAAM,CAAC;AACrC,MAAM,cAAc,GAAG,EAAE,CAAC;AAE1B;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,QAAgB;IAIrD,MAAM,SAAS,GAAG,gBAAgB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACjD,IAAI,SAAS,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,GAAG,qBAAqB,EAAE,CAAC;QAChE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IAC7B,CAAC;IAED,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;IAC5B,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;IAC1D,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IAExC,IAAI,aAAqB,CAAC;IAC1B,IAAI,CAAC;QACH,aAAa,GAAG,YAAY,CAAC,KAAK,EAAE,CAAC,WAAW,EAAE,MAAM,CAAC,EAAE;YACzD,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI;SACjD,CAAC,CAAC,IAAI,EAAE,CAAC;IACZ,CAAC;IAAC,MAAM,CAAC;QACP,gBAAgB,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAC3C,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IAC/B,CAAC;IAED,IAAI,IAAI,CAAC,eAAe,KAAK,aAAa,EAAE,CAAC;QAC3C,gBAAgB,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAC3C,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IAC7B,CAAC;IAED,kCAAkC;IAClC,IAAI,YAAY,GAAa,EAAE,CAAC;IAChC,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,YAAY,CAAC,KAAK,EAAE;gBAC/B,MAAM,EAAE,aAAa,EAAE,oBAAoB;gBAC3C,GAAG,IAAI,CAAC,eAAe,KAAK,aAAa,EAAE;aAC5C,EAAE;gBACD,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM;aACnD,CAAC,CAAC;YACH,YAAY,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACzD,CAAC;QAAC,MAAM,CAAC;YACP,gEAAgE;YAChE,YAAY,GAAG,EAAE,CAAC;QACpB,CAAC;IACH,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,YAAY,CAAC,MAAM,IAAI,cAAc,EAAE,CAAC;QACrE,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;YAChC,IAAI,CAAC;gBACH,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;YACzC,CAAC;YAAC,MAAM,CAAC;gBACP,qCAAqC;YACvC,CAAC;QACH,CAAC;IACH,CAAC;SAAM,IAAI,YAAY,CAAC,MAAM,GAAG,cAAc,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;QACzE,MAAM,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;IACpE,CAAC;IAED,MAAM,cAAc,CAAC,MAAM,CAAC,YAAY,EAAE,QAAQ,EAAE;QAClD,eAAe,EAAE,aAAa;QAC9B,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE;KACvB,CAAC,CAAC;IAEH,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC7B,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC7B,eAAe,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAEjC,gBAAgB,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAC3C,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,aAAa,EAAE,YAAY,CAAC,MAAM,EAAE,CAAC;AACrE,CAAC;AAED,4DAA4D;AAC5D,MAAM,UAAU,mBAAmB;IACjC,gBAAgB,CAAC,KAAK,EAAE,CAAC;AAC3B,CAAC"}

package/dist/tools/index-tools/folder-indexer.d.ts

@@ -0,0 +1,21 @@
+import type { IndexFolderResult } from "./types.js";
+export type { IndexFolderResult } from "./types.js";
+/** Test-only — clear short-circuit state. */
+export declare function resetIndexFolderRedundancyForTesting(): void;
+export declare function indexFolder(folderPath: string, options?: {
+    incremental?: boolean | undefined;
+    include_paths?: string[] | undefined;
+    watch?: boolean | undefined;
+    /**
+     * Cap on files indexed in a single pass. When the walker hits this, it
+     * returns partial results with a warning rather than blowing through
+     * memory. Default: DEFAULT_MAX_FILES (or CODESIFT_MAX_FILES env var).
+     */
+    max_files?: number | undefined;
+    /**
+     * Bypass the watcher-active short-circuit (see lastFullIndexAt). Used by
+     * indexRepo for fresh clones where defensive reindex is correct.
+     */
+    force?: boolean | undefined;
+}): Promise<IndexFolderResult>;
+//# sourceMappingURL=folder-indexer.d.ts.map

package/dist/tools/index-tools/folder-indexer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"folder-indexer.d.ts","sourceRoot":"","sources":["../../../src/tools/index-tools/folder-indexer.ts"],"names":[],"mappings":"AAwBA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAEpD,YAAY,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAcpD,6CAA6C;AAC7C,wBAAgB,oCAAoC,IAAI,IAAI,CAE3D;AAED,wBAAsB,WAAW,CAC/B,UAAU,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE;IACR,WAAW,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAClC,aAAa,CAAC,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;IACrC,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC5B;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B;;;OAGG;IACH,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CAC7B,GACA,OAAO,CAAC,iBAAiB,CAAC,CA+X5B"}

package/dist/tools/index-tools/folder-indexer.js

@@ -0,0 +1,399 @@
+import { readFile } from "node:fs/promises";
+import { join, relative, resolve } from "node:path";
+import { execFileSync } from "node:child_process";
+import { EXTRACTOR_VERSIONS } from "../index-shared.js";
+import { getLanguageForExtension } from "../../parser/parser-manager.js";
+import { saveIndex, loadIndex, getIndexPath } from "../../storage/index-store.js";
+import { clearTsconfigCache } from "../../utils/tsconfig-paths.js";
+import { registerRepo, listRepos as listRegistryRepos, removeRepo, getRepoName, updateRepoMeta, } from "../../storage/registry.js";
+import { buildBM25Index } from "../../search/bm25.js";
+import { loadConfig } from "../../config.js";
+import { walkDirectory } from "../../utils/walk.js";
+import { HASH_SNAPSHOT_VERSION } from "../../storage/hash-snapshot.js";
+import { activeWatchers, bm25Indexes, codeIndexes, lastFullIndexAt } from "./state.js";
+import { parseFiles, propagateDirtySignatures, embedSymbols, embedChunks } from "./parse.js";
+import { drainLegacyHashQueue, loadIndexSnapshot, saveIndexSnapshot, sha1OfFile } from "./snapshots.js";
+import { setupWatcher } from "./watcher.js";
+import { validateAndMergeFolderWalk } from "./folder-merge.js";
+const INDEX_FOLDER_REDUNDANT_WINDOW_MS = 60_000;
+const DEFAULT_MAX_FILES = 50_000;
+function getDefaultMaxFiles() {
+    const envVal = process.env.CODESIFT_MAX_FILES;
+    if (envVal) {
+        const parsed = Number.parseInt(envVal, 10);
+        if (Number.isFinite(parsed) && parsed > 0)
+            return parsed;
+    }
+    return DEFAULT_MAX_FILES;
+}
+/** Test-only — clear short-circuit state. */
+export function resetIndexFolderRedundancyForTesting() {
+    lastFullIndexAt.clear();
+}
+export async function indexFolder(folderPath, options) {
+    if (!folderPath || typeof folderPath !== "string") {
+        throw new Error("folderPath is required and must be a non-empty string");
+    }
+    const rootPath = resolve(folderPath);
+    const repoName = getRepoName(rootPath);
+    // Short-circuit: if a watcher is already keeping the index for this root
+    // live and we re-indexed recently, return a skipped status instead of
+    // walking the filesystem again.
+    if (!options?.force) {
+        const lastTs = lastFullIndexAt.get(rootPath);
+        const watcher = activeWatchers.get(repoName);
+        if (watcher && lastTs && Date.now() - lastTs < INDEX_FOLDER_REDUNDANT_WINDOW_MS) {
+            return {
+                repo: repoName,
+                root: rootPath,
+                file_count: 0,
+                symbol_count: 0,
+                duration_ms: 0,
+                status: "skipped",
+                reason: "watcher active, recent index",
+                last_indexed: new Date(lastTs).toISOString(),
+                hint: "pass force=true to override",
+            };
+        }
+    }
+    // Clear tsconfig path resolver cache so config edits between runs take effect.
+    // The two-level cache (configCache + dirToConfigCache) is module-level and
+    // would otherwise serve stale alias mappings if a user edited tsconfig.json
+    // between successive index_folder calls within the same MCP server process.
+    clearTsconfigCache();
+    const config = loadConfig();
+    const startTime = Date.now();
+    const indexPath = getIndexPath(config.dataDir, rootPath);
+    // Read .codesiftignore for user-defined exclude patterns
+    let excludePatterns;
+    try {
+        const ignoreContent = await readFile(join(rootPath, ".codesiftignore"), "utf-8");
+        excludePatterns = ignoreContent
+            .split("\n")
+            .map((line) => line.replace(/#.*$/, "").trim())
+            .filter((line) => line.length > 0);
+        if (excludePatterns.length === 0)
+            excludePatterns = undefined;
+    }
+    catch {
+        // .codesiftignore not found — proceed without patterns
+    }
+    // Walk directory and collect parseable files. maxFiles caps unbounded walks
+    // (huge monorepos, vendored data sets) before they OOM the process — see
+    // DEFAULT_MAX_FILES rationale above.
+    const maxFiles = options?.max_files ?? getDefaultMaxFiles();
+    const files = await walkDirectory(rootPath, {
+        includePaths: options?.include_paths,
+        excludePatterns,
+        maxFiles,
+        fileFilter: (ext, name) => !!getLanguageForExtension(ext) || (name?.startsWith(".env") ?? false),
+    });
+    const hitFileLimit = files.length >= maxFiles;
+    if (hitFileLimit) {
+        console.error(`[codesift] index_folder: ${rootPath} hit max_files=${maxFiles}; ` +
+            `partial index. Pass include_paths to scope the walk or raise max_files.`);
+    }
+    // mtime-based incremental: skip files unchanged since last index
+    const existing = await loadIndex(indexPath);
+    const mtimeMap = new Map();
+    if (existing) {
+        for (const f of existing.files) {
+            if (f.mtime_ms)
+                mtimeMap.set(f.path, f.mtime_ms);
+        }
+    }
+    // Load and validate the persistent hash snapshot paired with the existing index.
+    const oldSnapshot = existing
+        ? await loadIndexSnapshot(indexPath, repoName, existing.updated_at)
+        : null;
+    const filesToParse = [];
+    const keptSymbols = [];
+    const keptEntries = [];
+    // sha1 of every file in the NEW index, by relPath. Populated for reused files
+    // here (from the old snapshot when present, else hashed-now for convergence)
+    // and for parsed files after parseFiles resolves.
+    const newSnapshotFiles = {};
+    // CRITICAL-1: reused files whose sha1 must be (re)computed because the old
+    // snapshot lacks it (legacy snapshot-less index, or stale snapshot discarded
+    // above). Collected here and hashed AFTER the loop in PARSE_CONCURRENCY
+    // batches instead of one serial await per file inside the loop — on a first
+    // run after upgrade against a many-thousand-file repo the serial version cost
+    // thousands of sequential awaits. Behavior is identical, wall-clock is
+    // parallelized.
+    //
+    // mtimeMs: the mtime observed at decision time (the moment we confirmed
+    // mtime === prevMtime and placed the file in the queue). We re-stat after
+    // hashing to detect any concurrent modification that landed between the two
+    // operations. If the mtime drifted, we omit the file from newSnapshotFiles
+    // entirely — the missing sha causes the next cold run to re-parse, avoiding
+    // a snapshot that pairs new-content sha against old (reused) symbols.
+    const legacyHashQueue = [];
+    // PERF: pre-build per-file lookups ONCE before the reuse loop. Both reuse
+    // branches need (a) the existing index's symbols for a given relPath and (b)
+    // its FileEntry. Doing `existing.symbols.filter(s => s.file === relPath)` /
+    // `existing.files.find(f => f.path === relPath)` per file is O(files ×
+    // symbols) and O(files²) respectively — quadratic, and on a many-thousand
+    // file/symbol repo that dominated the reuse-heavy fast path. A single pass
+    // builds Map lookups each branch hits in O(1). Built only when there's an
+    // existing index to reuse from.
+    const symbolsByFile = new Map();
+    const fileEntryByPath = new Map();
+    if (existing) {
+        for (const sym of existing.symbols) {
+            const list = symbolsByFile.get(sym.file);
+            if (list)
+                list.push(sym);
+            else
+                symbolsByFile.set(sym.file, [sym]);
+        }
+        for (const fe of existing.files) {
+            fileEntryByPath.set(fe.path, fe);
+        }
+    }
+    if (mtimeMap.size > 0) {
+        const { stat } = await import("node:fs/promises");
+        for (const filePath of files) {
+            const relPath = relative(rootPath, filePath);
+            const prevMtime = mtimeMap.get(relPath);
+            if (prevMtime !== undefined) {
+                const fileEntry = fileEntryByPath.get(relPath);
+                // Force re-parse if file is marked stale (callee signature changed)
+                if (fileEntry?.stale) {
+                    filesToParse.push(filePath);
+                    continue;
+                }
+                try {
+                    const st = await stat(filePath);
+                    if (Math.round(st.mtimeMs) === prevMtime) {
+                        // Fast path: mtime unchanged → reuse symbols without hashing.
+                        const fileSymbols = symbolsByFile.get(relPath) ?? [];
+                        if (fileEntry) {
+                            keptSymbols.push(...fileSymbols);
+                            keptEntries.push(fileEntry);
+                            // Carry the sha1 forward: reuse from old snapshot if present,
+                            // else DEFER hashing so legacy (snapshot-less) indexes converge
+                            // to a complete snapshot after one run — without paying a serial
+                            // hash per file inside this loop.
+                            const carried = oldSnapshot?.files[relPath];
+                            if (carried !== undefined) {
+                                newSnapshotFiles[relPath] = carried;
+                            }
+                            else {
+                                legacyHashQueue.push({ relPath, filePath, mtimeMs: Math.round(st.mtimeMs) });
+                            }
+                            continue;
+                        }
+                    }
+                    else {
+                        // mtime changed — hash decides reuse vs re-parse. This catches
+                        // touch/checkout that bumped mtime without changing content.
+                        const snapSha = oldSnapshot?.files[relPath];
+                        if (snapSha !== undefined && fileEntry && !fileEntry.stale) {
+                            const currentSha = await sha1OfFile(filePath);
+                            if (currentSha !== null && currentSha === snapSha) {
+                                const fileSymbols = symbolsByFile.get(relPath) ?? [];
+                                keptSymbols.push(...fileSymbols);
+                                // FIX: the file's mtime changed but content is identical (touch /
+                                // checkout no-op rewrite). Reuse the symbols, but DON'T carry the
+                                // stale FileEntry verbatim — its mtime_ms still holds the OLD
+                                // mtime, so every future run would see mtime !== prevMtime and
+                                // re-hash this file forever, permanently degrading it off the
+                                // mtime fast path. Clone the entry with mtime_ms bumped to the
+                                // CURRENT stat's mtime so the next run takes the cheap fast path.
+                                keptEntries.push({ ...fileEntry, mtime_ms: Math.round(st.mtimeMs) });
+                                newSnapshotFiles[relPath] = currentSha;
+                                continue;
+                            }
+                        }
+                    }
+                }
+                catch { /* file may have been deleted — reparse */ }
+            }
+            filesToParse.push(filePath);
+        }
+    }
+    else {
+        filesToParse.push(...files);
+    }
+    // Drain the deferred legacy-hash queue (CRITICAL-1): files reused via the
+    // mtime fast path that had no carried sha1 (legacy snapshot-less index, or a
+    // stale snapshot discarded by the guard above). See drainLegacyHashQueue for
+    // the TOCTOU guard details — entries whose mtime drifted between decision
+    // time and hash time are omitted so the next run re-parses rather than
+    // reusing symbols against a mismatched sha.
+    if (legacyHashQueue.length > 0) {
+        const drained = await drainLegacyHashQueue(legacyHashQueue);
+        Object.assign(newSnapshotFiles, drained);
+    }
+    // Parse only changed/new files
+    const { symbols: parsedSymbols, fileEntries: parsedEntries, shas: parsedShas } = await parseFiles(filesToParse, rootPath, repoName);
+    const symbols = [...keptSymbols, ...parsedSymbols];
+    const fileEntries = [...keptEntries, ...parsedEntries];
+    // Record sha1s for the files that were actually parsed (changed/new).
+    // CRITICAL-1 (TOCTOU): these hashes come straight from parseOneFile — they
+    // are the sha1 of the EXACT source string that produced the symbols, so the
+    // snapshot can never pair old symbols with a newer file's sha. Only entries
+    // that survived parseFiles (parseOneFile returned non-null) have a sha here,
+    // keeping the snapshot in lockstep with fileEntries. The previous post-parse
+    // double-read loop is gone — one fewer full read per parsed file.
+    for (const entry of parsedEntries) {
+        const sha = parsedShas[entry.path];
+        if (sha !== undefined)
+            newSnapshotFiles[entry.path] = sha;
+    }
+    // Dirty propagation: detect signature changes and mark caller files stale
+    if (existing && filesToParse.length > 0 && filesToParse.length < files.length) {
+        const staleFiles = propagateDirtySignatures(existing.symbols, symbols, fileEntries);
+        if (staleFiles.size > 0) {
+            console.error(`[codesift] Dirty propagation: ${staleFiles.size} caller files marked stale`);
+        }
+    }
+    // Invalidate code index cache (BM25 is rebuilt below from the FINAL symbol
+    // set — possibly merged with out-of-scope existing symbols, see merge block).
+    codeIndexes.delete(repoName);
+    const mergeResult = await validateAndMergeFolderWalk({
+        existing,
+        fileEntries,
+        symbols,
+        newSnapshotFiles,
+        oldSnapshot,
+        rootPath,
+        repoName,
+        startTime,
+        maxFiles,
+        hitFileLimit,
+        includePaths: options?.include_paths,
+    });
+    if (!("mergedSymbols" in mergeResult))
+        return mergeResult;
+    const { mergedSymbols, mergedEntries, mergedSnapshotFiles } = mergeResult;
+    // Build and cache BM25 index from the FINAL (possibly merged) symbol set.
+    // Built here (not before the guard) so a rejected_partial early-return leaves
+    // the previous in-memory BM25 index intact rather than swapping in a partial.
+    const bm25 = buildBM25Index(mergedSymbols);
+    bm25Indexes.set(repoName, bm25);
+    // Resolve workspaces (Task 7) — runs before persistence so collectImportEdges
+    // and other downstream consumers see the populated `workspaces` field.
+    // Gated behind CODESIFT_DISABLE_MONOREPO=1 kill switch (spec D-FB).
+    let workspaces;
+    if (process.env.CODESIFT_DISABLE_MONOREPO !== "1") {
+        try {
+            const { resolveWorkspaces } = await import("../../storage/workspace-resolver.js");
+            const resolved = await resolveWorkspaces(rootPath);
+            if (resolved)
+                workspaces = resolved.workspaces;
+        }
+        catch {
+            // Resolver should never throw, but guard belt-and-braces — flat-repo
+            // mode is the safe fallback.
+        }
+    }
+    // Build and save code index from the FINAL (possibly merged) sets.
+    const codeIndex = {
+        repo: repoName,
+        root: rootPath,
+        symbols: mergedSymbols,
+        files: mergedEntries,
+        created_at: Date.now(),
+        updated_at: Date.now(),
+        symbol_count: mergedSymbols.length,
+        file_count: mergedEntries.length,
+        extractor_version: { ...EXTRACTOR_VERSIONS },
+        ...(workspaces ? { workspaces } : {}),
+    };
+    await saveIndex(indexPath, codeIndex);
+    // Persist the hash snapshot AFTER the index lands (mirrors registerRepo
+    // ordering) and only on the success path — the rejected_partial branch
+    // returned earlier, leaving the previous snapshot intact. Non-fatal: the
+    // snapshot is a reuse-optimization cache; a write failure just costs a full
+    // re-parse next run, so we warn and continue.
+    try {
+        const newSnapshot = {
+            version: HASH_SNAPSHOT_VERSION,
+            repo: repoName,
+            // CRITICAL-2 (created_at race): use the EXACT timestamp serialized into
+            // the index, not a fresh Date.now(). A watcher's saveIncremental that
+            // lands between saveIndex and this write would otherwise leave the
+            // snapshot OLDER than created_at, blinding the staleness guard above. By
+            // anchoring to codeIndex.updated_at, snapshot.created_at === the index's
+            // updated_at on a fresh write, so any later incremental strictly advances
+            // index.updated_at past it and the guard fires correctly.
+            created_at: codeIndex.updated_at,
+            files: mergedSnapshotFiles,
+        };
+        await saveIndexSnapshot(indexPath, newSnapshot);
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        console.warn(`[codesift] hash-snapshot save failed for ${repoName} (non-fatal): ${msg}`);
+    }
+    // Embed symbols and chunks in background (non-fatal, don't block MCP response)
+    // Large repos (71K symbols) can take minutes — fire-and-forget to prevent timeout
+    embedSymbols(mergedSymbols, indexPath, repoName, config)
+        .then(() => embedChunks(mergedEntries, rootPath, repoName, indexPath, config, mergedSymbols))
+        .catch((err) => {
+        const msg = err instanceof Error ? err.message : String(err);
+        console.error(`[codesift] Background embedding failed for ${repoName}: ${msg}`);
+    });
+    // Register in the global registry. If a stale entry exists with the same
+    // root but a different name (e.g. `local/workspace` from before the git
+    // origin auto-detect landed), drop it so `list_repos` doesn't show ghosts.
+    const existingRepos = await listRegistryRepos(config.registryPath);
+    for (const stale of existingRepos) {
+        if (stale.root === rootPath && stale.name !== repoName) {
+            await removeRepo(config.registryPath, stale.name);
+            console.error(`[codesift] Migrated registry: ${stale.name} -> ${repoName} (same root)`);
+        }
+    }
+    const meta = {
+        name: repoName,
+        root: rootPath,
+        index_path: indexPath,
+        symbol_count: mergedSymbols.length,
+        file_count: mergedEntries.length,
+        updated_at: Date.now(),
+    };
+    await registerRepo(config.registryPath, meta);
+    // Capture git HEAD for auto-refresh tracking
+    try {
+        const head = execFileSync("git", ["rev-parse", "HEAD"], {
+            cwd: rootPath, encoding: "utf-8", timeout: 5000,
+        }).trim();
+        await updateRepoMeta(config.registryPath, repoName, { last_git_commit: head });
+    }
+    catch {
+        // Not a git repo — skip
+    }
+    // Start file watcher for incremental updates (unless disabled)
+    if (options?.watch !== false) {
+        await setupWatcher(rootPath, repoName, indexPath);
+    }
+    // Auto-enable framework-specific tool bundles (NestJS, etc.)
+    // Lazy import to avoid circular dep: index-tools → register-tools → tool handlers → index-tools
+    try {
+        const { detectFrameworks } = await import("../../utils/framework-detect.js");
+        const { enableFrameworkToolBundle } = await import("../../register-tools.js");
+        const tempIndex = { root: rootPath, files: mergedEntries, symbols: mergedSymbols };
+        const frameworks = detectFrameworks(tempIndex);
+        for (const fw of frameworks) {
+            const enabled = enableFrameworkToolBundle(fw);
+            if (enabled.length > 0) {
+                console.error(`[codesift] auto-enabled ${enabled.length} ${fw} tools for ${repoName}: ${enabled.join(", ")}`);
+            }
+        }
+    }
+    catch {
+        // Non-fatal — framework auto-enable is a convenience feature
+    }
+    // Record completion timestamp so subsequent re-runs can short-circuit when
+    // the watcher is keeping the index fresh.
+    lastFullIndexAt.set(rootPath, Date.now());
+    return {
+        repo: repoName,
+        root: rootPath,
+        file_count: mergedEntries.length,
+        symbol_count: mergedSymbols.length,
+        duration_ms: Date.now() - startTime,
+    };
+}
+//# sourceMappingURL=folder-indexer.js.map