@colbymchenry/codegraph-darwin-x64 1.0.1 → 1.1.1

package/lib/dist/extraction/index.js

@@ -55,6 +55,7 @@ const crypto = __importStar(require("crypto"));
 const child_process_1 = require("child_process");
 const tree_sitter_1 = require("./tree-sitter");
 const grammars_1 = require("./grammars");
+const project_config_1 = require("../project-config");
 const directory_1 = require("../directory");
 const errors_1 = require("../errors");
 const utils_1 = require("../utils");
@@ -65,6 +66,17 @@ const frameworks_1 = require("../resolution/frameworks");
  * File reads are I/O-bound; batching overlaps I/O wait with CPU parse work.
  */
 const FILE_IO_BATCH_SIZE = 10;
+/**
+ * How many files the `sync()` reconcile processes between cooperative yields to
+ * the event loop. The reconcile runs two O(files) loops of synchronous `fs`
+ * calls (existsSync for removals, statSync for adds/mods); on a very large repo
+ * (~100k files) an un-yielded run wedges the main thread for minutes, which both
+ * trips the liveness watchdog (it SIGKILLs a process whose loop stops turning)
+ * and blocks the first MCP tool call behind the catch-up gate (issue #905).
+ * Yielding every N files keeps the socket, the watchdog heartbeat, and any
+ * concurrent read query responsive while the reconcile runs.
+ */
+const SYNC_RECONCILE_YIELD_INTERVAL = 1000;
 // PARSER_RESET_INTERVAL moved to parse-worker.ts (runs in worker thread)
 /**
  * Maximum time (ms) to wait for a single file to parse in the worker thread.
@@ -240,6 +252,33 @@ function buildDefaultIgnore(rootDir) {
 function defaultsOnlyIgnore() {
     return (0, ignore_1.default)().add(DEFAULT_IGNORE_PATTERNS);
 }
+/**
+ * Matcher for the project's `codegraph.json` `includeIgnored` patterns — the
+ * explicit opt-in to index embedded git repos living inside gitignored
+ * directories (#622, #699). Returns `null` when the project opted in nothing,
+ * which is the zero-config DEFAULT: `.gitignore` is then fully respected and a
+ * gitignored directory (even one holding nested repos) is never walked or
+ * indexed (#970, #976). Built once per scan/sync/scope operation from the scan
+ * root and threaded down — never global, so multi-project daemons stay isolated.
+ */
+function loadIncludeIgnoredMatcher(rootDir) {
+    const patterns = (0, project_config_1.loadIncludeIgnoredPatterns)(rootDir);
+    return patterns.length > 0 ? (0, ignore_1.default)().add(patterns) : null;
+}
+/**
+ * `git ls-files --directory` collapses a wholly-untracked/ignored directory into
+ * one entry — and when the command's own cwd is such a directory (the indexed
+ * root is itself a git-ignored subdir of an enclosing repo), git emits the
+ * literal `./` meaning "this entire directory". That sentinel is not a real
+ * nested path: feeding it to the `ignore` matcher throws ("path should be a
+ * `path.relative()`d string, but got "./""), which used to abort `buildScopeIgnore`
+ * and so break the MCP daemon's watcher/auto-sync on connect; and joining it back
+ * onto `repoDir` would just re-point at the cwd. Drop it wherever we consume
+ * `--directory` output. (#936)
+ */
+function isWholeCwdEntry(entry) {
+    return entry === './' || entry === '.' || entry === '';
+}
 /**
  * List the gitignored DIRECTORIES of a repo (collapsed, trailing-slash form),
  * relative to `repoDir`. These are invisible to every other `git ls-files` /
@@ -250,7 +289,7 @@ function defaultsOnlyIgnore() {
 function listIgnoredDirs(repoDir) {
     try {
         const out = (0, child_process_1.execFileSync)('git', ['ls-files', '-z', '-o', '-i', '--exclude-standard', '--directory'], { cwd: repoDir, encoding: 'utf-8', timeout: 30000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true });
-        return out.split('\0').filter((e) => e.endsWith('/'));
+        return out.split('\0').filter((e) => e.endsWith('/') && !isWholeCwdEntry(e));
     }
     catch {
         return [];
@@ -268,8 +307,10 @@ const EMBEDDED_REPO_SEARCH_ENTRIES = 2000;
  * - A `.git` **file** is a pointer (`gitdir: …`). A git **worktree** points into
  *   the host repo's own `.git/worktrees/<name>`, so it is a second working view
  *   of a repo CodeGraph already indexes — indexing it just duplicates the whole
- *   graph N times; skip it (#848). A **submodule** points into `.git/modules/`
- *   and is distinct code, so index it as before.
+ *   graph N times; skip it (#848). A **submodule worktree** points into
+ *   `.git/modules/<module>/worktrees/<name>` — same duplication, so skip it too
+ *   (#945). A **submodule** checkout points into `.git/modules/<module>` (no
+ *   `worktrees/` segment) and is distinct code, so index it as before.
  *
  * Returns `'none'` when there is no `.git` entry here.
  */
@@ -287,9 +328,12 @@ function classifyGitDir(absDir) {
         return 'none';
     try {
         const gitdir = fs.readFileSync(path.join(absDir, '.git'), 'utf8').match(/^gitdir:\s*(.+)$/m)?.[1]?.trim();
-        // A linked worktree's gitdir lives under some repo's `.git/worktrees/`.
+        // A worktree's gitdir lives under some repo's `.git/worktrees/<name>` —
+        // either the top-level repo's (`.git/worktrees/`) or, for a worktree of a
+        // submodule, that submodule's gitdir (`.git/modules/<module>/worktrees/`).
+        // The optional `modules/<module>` segment covers the submodule case (#945).
         // Match both separators so a Windows-style pointer is recognized too.
-        if (gitdir && /(^|[\\/])\.git[\\/]worktrees[\\/]/.test(gitdir))
+        if (gitdir && /(^|[\\/])\.git[\\/](modules[\\/][^\\/]+[\\/])?worktrees[\\/]/.test(gitdir))
             return 'worktree';
     }
     catch {
@@ -398,9 +442,12 @@ function buildScopeIgnore(rootDir, embeddedRoots) {
 }
 /**
  * Standalone discovery of every embedded repo root under `rootDir` (relative,
- * trailing-slashed) — both the untracked kind (#193) and the gitignored kind
- * (#514), recursively (an embedded repo can embed further repos). Returns []
- * for non-git roots: the filesystem walk handles nested repos there already.
+ * trailing-slashed) — the untracked kind (#193) always, and the gitignored kind
+ * (#514) only for directories the project opted in via `codegraph.json`
+ * `includeIgnored` (#622, #699); otherwise `.gitignore` is respected and they
+ * are not discovered (#970, #976). Recursive (an embedded repo can embed further
+ * repos). Returns [] for non-git roots: the filesystem walk handles nested repos
+ * there already.
  */
 function discoverEmbeddedRepoRoots(rootDir) {
     try {
@@ -411,18 +458,19 @@ function discoverEmbeddedRepoRoots(rootDir) {
     }
     const out = [];
     const defaults = defaultsOnlyIgnore();
+    const includeIgnored = loadIncludeIgnoredMatcher(rootDir);
     const visit = (repoAbs, prefix) => {
         const candidates = [];
         try {
             const o = (0, child_process_1.execFileSync)('git', ['ls-files', '-z', '-o', '--exclude-standard', '--directory'], { cwd: repoAbs, encoding: 'utf-8', timeout: 30000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true });
             for (const e of o.split('\0')) {
-                if (e.endsWith('/') && !defaults.ignores(e)) {
+                if (e.endsWith('/') && !isWholeCwdEntry(e) && !defaults.ignores(e)) {
                     candidates.push(...findNestedGitRepos(path.join(repoAbs, e), e));
                 }
             }
         }
         catch { /* untracked listing failed — ignored-side discovery still runs */ }
-        candidates.push(...findIgnoredEmbeddedRepos(repoAbs));
+        candidates.push(...findIgnoredEmbeddedRepos(repoAbs, includeIgnored, prefix));
         for (const rel of candidates) {
             const full = (0, utils_1.normalizePath)(prefix + rel);
             out.push(full);
@@ -433,16 +481,30 @@ function discoverEmbeddedRepoRoots(rootDir) {
     return out;
 }
 /**
- * Discover embedded repos hidden by `repoDir`'s OWN ignore rules: for each
- * gitignored directory (skipping built-in default excludes), search for nested
- * `.git` roots. Returns repo paths relative to `repoDir`, trailing-slashed.
+ * Discover embedded repos hidden by `repoDir`'s OWN gitignore rules: for each
+ * gitignored directory, search for nested `.git` roots. Returns repo paths
+ * relative to `repoDir`, trailing-slashed.
+ *
+ * OPT-IN ONLY. Walking into a gitignored directory contradicts what every other
+ * tool (and CodeGraph's own `git ls-files` foundation) does — `.gitignore`
+ * excludes. So this returns `[]` unless the project opted the directory in via
+ * `codegraph.json` `includeIgnored`; without that, a gitignored dir — including
+ * a huge reference/data dir full of nested clones — is left untouched (#970,
+ * #976). When opted in, it restores the super-repo-of-clones behavior (#622,
+ * #699). `prefix` is the scan-root-relative path of `repoDir`, so a pattern like
+ * `services/` opts that whole subtree in at any recursion depth. Built-in
+ * default excludes (`node_modules`, …) are always skipped.
  */
-function findIgnoredEmbeddedRepos(repoDir) {
+function findIgnoredEmbeddedRepos(repoDir, includeIgnored, prefix) {
+    if (!includeIgnored)
+        return [];
     const defaults = defaultsOnlyIgnore();
     const repos = [];
     for (const dir of listIgnoredDirs(repoDir)) {
         if (defaults.ignores(dir))
             continue;
+        if (!includeIgnored.ignores((0, utils_1.normalizePath)(prefix + dir)))
+            continue;
         repos.push(...findNestedGitRepos(path.join(repoDir, dir), dir));
     }
     return repos;
@@ -458,12 +520,15 @@ function findIgnoredEmbeddedRepos(repoDir) {
  * skips them entirely, and untracked output reports them only as an opaque
  * "subdir/" entry (trailing slash) rather than expanding their files. Each
  * embedded repo is its own git boundary, so we re-run `git ls-files` inside it.
- * (See issue #193.) GITIGNORED embedded repos are invisible even to that —
- * they're discovered separately via `findIgnoredEmbeddedRepos` (#514); every
- * embedded repo root (however found) is recorded in `embeddedRoots` so callers
- * can exempt its files from the parent's own gitignore rules.
+ * (See issue #193.) GITIGNORED embedded repos are invisible even to that; they
+ * are discovered separately via `findIgnoredEmbeddedRepos` (#514) but ONLY for
+ * directories the project opted in through `codegraph.json` `includeIgnored`
+ * (`includeIgnored` here, threaded from the scan root) — by default `.gitignore`
+ * is respected and they stay out (#970, #976). Every embedded repo root (however
+ * found) is recorded in `embeddedRoots` so callers can exempt its files from the
+ * parent's own gitignore rules.
  */
-function collectGitFiles(repoDir, prefix, files, embeddedRoots) {
+function collectGitFiles(repoDir, prefix, files, embeddedRoots, includeIgnored = null) {
     const gitOpts = { cwd: repoDir, encoding: 'utf-8', timeout: 30000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true };
     // Tracked files. --recurse-submodules pulls in files from active submodules,
     // which the index would otherwise represent only as a commit pointer.
@@ -497,18 +562,20 @@ function collectGitFiles(repoDir, prefix, files, embeddedRoots) {
             // it's a duplicate working view of an already-indexed repo (#848).
             if (classifyGitDir(childDir) === 'embedded' && !defaultsOnlyIgnore().ignores(rel)) {
                 embeddedRoots?.add((0, utils_1.normalizePath)(prefix + rel));
-                collectGitFiles(childDir, prefix + rel, files, embeddedRoots);
+                collectGitFiles(childDir, prefix + rel, files, embeddedRoots, includeIgnored);
             }
             continue;
         }
         files.add((0, utils_1.normalizePath)(prefix + rel));
     }
     // Embedded repos hidden by THIS repo's ignore rules (`/packages/` in a
-    // super-repo .gitignore) never appear in any listing above — discover and
-    // recurse into them too. (#514)
-    for (const rel of findIgnoredEmbeddedRepos(repoDir)) {
+    // super-repo .gitignore) never appear in any listing above. By default they
+    // stay hidden — `.gitignore` is respected (#970, #976). They are recursed into
+    // only when the project opted the directory in via `codegraph.json`
+    // `includeIgnored` (#622, #699), which `findIgnoredEmbeddedRepos` enforces.
+    for (const rel of findIgnoredEmbeddedRepos(repoDir, includeIgnored, prefix)) {
         embeddedRoots?.add((0, utils_1.normalizePath)(prefix + rel));
-        collectGitFiles(path.join(repoDir, rel), prefix + rel, files, embeddedRoots);
+        collectGitFiles(path.join(repoDir, rel), prefix + rel, files, embeddedRoots, includeIgnored);
     }
 }
 /**
@@ -536,7 +603,7 @@ function getGitVisibleFiles(rootDir) {
         }
         const files = new Set();
         const embeddedRoots = new Set();
-        collectGitFiles(rootDir, '', files, embeddedRoots);
+        collectGitFiles(rootDir, '', files, embeddedRoots, loadIncludeIgnoredMatcher(rootDir));
         // Apply built-in default ignores uniformly — to tracked files too, since
         // committing a dependency/build dir doesn't make it project code. A
         // `.gitignore` negation (e.g. `!vendor/`) is the explicit opt-in. (issue #407)
@@ -554,26 +621,42 @@ function getGitVisibleFiles(rootDir) {
  * Use `git status` to detect changed files instead of scanning every file.
  * Returns null on failure so callers fall back to full scan.
  *
- * Recurses into embedded repos — both the untracked kind (#193: the parent's
- * status collapses them to an opaque `?? subdir/` entry) and the gitignored
- * kind (#514: they never appear in the parent's status at all) — running
- * `git status` inside each, so changes in a multi-repo workspace sync without
- * a full rescan. Deleting an ENTIRE embedded repo dir is the one case this
- * cannot see (the child status that would report the deletions is gone with
- * it); a full `codegraph index` reconciles that.
+ * Recurses into embedded repos — the untracked kind (#193: the parent's status
+ * collapses them to an opaque `?? subdir/` entry) always, and the gitignored
+ * kind (#514: they never appear in the parent's status at all) only for
+ * directories opted in via `codegraph.json` `includeIgnored` (#622, #699) —
+ * running `git status` inside each, so changes in a multi-repo workspace sync
+ * without a full rescan. By default a gitignored dir is left alone, matching the
+ * full-index scan (#970, #976). Deleting an ENTIRE embedded repo dir is the one
+ * case this cannot see (the child status that would report the deletions is gone
+ * with it); a full `codegraph index` reconciles that.
  */
 function getGitChangedFiles(rootDir) {
     try {
         const changes = { modified: [], added: [], deleted: [] };
-        collectGitStatus(rootDir, '', changes);
+        // Custom extension → language overrides from the project's codegraph.json,
+        // so change detection sees the same custom-extension files the full index does.
+        const overrides = (0, project_config_1.loadExtensionOverrides)(rootDir);
+        collectGitStatus(rootDir, '', changes, overrides, loadIncludeIgnoredMatcher(rootDir));
         return changes;
     }
     catch {
         return null;
     }
 }
-function collectGitStatus(repoDir, prefix, out) {
+function collectGitStatus(repoDir, prefix, out, overrides, includeIgnored = null) {
     const output = (0, child_process_1.execFileSync)('git', ['status', '--porcelain', '--no-renames'], { cwd: repoDir, encoding: 'utf-8', timeout: 10000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true });
+    // This repo's own ignore rules — built-in defaults (#407) plus its .gitignore.
+    // Change detection must exclude the SAME files the full index does, but git
+    // status hides neither: it ignores nothing for *tracked* paths, and the
+    // built-in defaults aren't gitignore at all. Without this filter a committed
+    // vendor/ dir, or a tracked file under a .gitignored dir, surfaces here as a
+    // change — so `codegraph status` (which reads getChangedFiles) reports a
+    // pending edit the full index never tracks and `sync` never clears. Matching
+    // repo-relative `rel` at each recursion level mirrors getGitVisibleFiles'
+    // ScopeIgnore: every embedded repo is judged by ITS OWN rules, never the
+    // parent's. (#766)
+    const ig = buildDefaultIgnore(repoDir);
     const untrackedDirs = [];
     for (const line of output.split('\n')) {
         if (line.length < 4)
@@ -587,29 +670,38 @@ function collectGitStatus(repoDir, prefix, out) {
             continue;
         }
         const filePath = (0, utils_1.normalizePath)(prefix + rel);
-        // Skip non-source files (git status already omits .gitignored paths).
-        if (!(0, grammars_1.isSourceFile)(filePath))
+        if (!(0, grammars_1.isSourceFile)(filePath, overrides))
+            continue;
+        if (statusCode.includes('D')) {
+            // Deletions stay unfiltered: getChangedFiles acts on one only when the
+            // path is already tracked in the DB, where removal is always correct — and
+            // that lets a newly-excluded dir's stale rows clean themselves up. (#766)
+            out.deleted.push(filePath);
+            continue;
+        }
+        // Added (`??`) / modified files inside an excluded dir must not enter the
+        // index — match against the repo-relative path, same as the full scan. (#766)
+        if (ig.ignores(rel))
             continue;
         if (statusCode === '??') {
             out.added.push(filePath);
         }
-        else if (statusCode.includes('D')) {
-            out.deleted.push(filePath);
-        }
         else {
             // M, MM, AM, A (staged), etc. — treat as modified
             out.modified.push(filePath);
         }
     }
     // Recurse embedded repos found under untracked dirs (at the dir itself or
-    // nested deeper) and under this repo's gitignored dirs.
+    // nested deeper). Gitignored dirs are walked only for the directories the
+    // project opted in via `includeIgnored`; by default `.gitignore` is respected
+    // and they are left alone (#970, #976), mirroring the full-index scan.
     for (const rel of untrackedDirs) {
         for (const repoRel of findNestedGitRepos(path.join(repoDir, rel), rel)) {
-            collectGitStatus(path.join(repoDir, repoRel), prefix + repoRel, out);
+            collectGitStatus(path.join(repoDir, repoRel), prefix + repoRel, out, overrides, includeIgnored);
         }
     }
-    for (const rel of findIgnoredEmbeddedRepos(repoDir)) {
-        collectGitStatus(path.join(repoDir, rel), prefix + rel, out);
+    for (const rel of findIgnoredEmbeddedRepos(repoDir, includeIgnored, prefix)) {
+        collectGitStatus(path.join(repoDir, rel), prefix + rel, out, overrides, includeIgnored);
     }
 }
 /**
@@ -620,13 +712,15 @@ function collectGitStatus(repoDir, prefix, out) {
  * projects, falls back to a filesystem walk that parses .gitignore itself.
  */
 function scanDirectory(rootDir, onProgress) {
+    // Custom extension → language overrides from the project's codegraph.json.
+    const overrides = (0, project_config_1.loadExtensionOverrides)(rootDir);
     // Fast path: use git to get all visible files (respects .gitignore everywhere)
     const gitFiles = getGitVisibleFiles(rootDir);
     if (gitFiles) {
         const files = [];
         let count = 0;
         for (const filePath of gitFiles) {
-            if ((0, grammars_1.isSourceFile)(filePath)) {
+            if ((0, grammars_1.isSourceFile)(filePath, overrides)) {
                 files.push(filePath);
                 count++;
                 onProgress?.(count, filePath);
@@ -642,12 +736,14 @@ function scanDirectory(rootDir, onProgress) {
  * allowing worker threads to receive and render progress messages.
  */
 async function scanDirectoryAsync(rootDir, onProgress) {
+    // Custom extension → language overrides from the project's codegraph.json.
+    const overrides = (0, project_config_1.loadExtensionOverrides)(rootDir);
     const gitFiles = getGitVisibleFiles(rootDir);
     if (gitFiles) {
         const files = [];
         let count = 0;
         for (const filePath of gitFiles) {
-            if ((0, grammars_1.isSourceFile)(filePath)) {
+            if ((0, grammars_1.isSourceFile)(filePath, overrides)) {
                 files.push(filePath);
                 count++;
                 onProgress?.(count, filePath);
@@ -668,6 +764,8 @@ function scanDirectoryWalk(rootDir, onProgress) {
     const files = [];
     let count = 0;
     const visitedDirs = new Set();
+    // Custom extension → language overrides from the project's codegraph.json.
+    const overrides = (0, project_config_1.loadExtensionOverrides)(rootDir);
     const loadIgnore = (dir) => {
         const giPath = path.join(dir, '.gitignore');
         if (!fs.existsSync(giPath))
@@ -734,7 +832,7 @@ function scanDirectoryWalk(rootDir, onProgress) {
                         }
                     }
                     else if (stat.isFile()) {
-                        if (!isIgnored(fullPath, false, active) && (0, grammars_1.isSourceFile)(relativePath)) {
+                        if (!isIgnored(fullPath, false, active) && (0, grammars_1.isSourceFile)(relativePath, overrides)) {
                             files.push(relativePath);
                             count++;
                             onProgress?.(count, relativePath);
@@ -752,7 +850,7 @@ function scanDirectoryWalk(rootDir, onProgress) {
                 }
             }
             else if (entry.isFile()) {
-                if (!isIgnored(fullPath, false, active) && (0, grammars_1.isSourceFile)(relativePath)) {
+                if (!isIgnored(fullPath, false, active) && (0, grammars_1.isSourceFile)(relativePath, overrides)) {
                     files.push(relativePath);
                     count++;
                     onProgress?.(count, relativePath);
@@ -866,6 +964,10 @@ class ExtractionOrchestrator {
         let filesErrored = 0;
         let totalNodes = 0;
         let totalEdges = 0;
+        // Custom extension → language overrides from the project's codegraph.json.
+        // Threaded into language detection so custom-extension files load the right
+        // grammar and store under the mapped language.
+        const overrides = (0, project_config_1.loadExtensionOverrides)(this.rootDir);
         const log = verbose
             ? (msg) => { console.log(`[worker] ${msg}`); }
             : (_msg) => { };
@@ -915,7 +1017,7 @@ class ExtractionOrchestrator {
         });
         await new Promise(resolve => setImmediate(resolve));
         // Detect needed languages and load grammars in the parse worker
-        const neededLanguages = [...new Set(files.map((f) => (0, grammars_1.detectLanguage)(f)))];
+        const neededLanguages = [...new Set(files.map((f) => (0, grammars_1.detectLanguage)(f, undefined, overrides)))];
         // .h files default to 'c' but may be C++ — ensure cpp grammar is loaded when c is needed
         if (neededLanguages.includes('c') && !neededLanguages.includes('cpp')) {
             neededLanguages.push('cpp');
@@ -1014,9 +1116,13 @@ class ExtractionOrchestrator {
             w.terminate().catch(() => { });
         }
         async function requestParse(filePath, content) {
+            // Resolve the language on the main thread (where the project's
+            // codegraph.json overrides are loaded) and hand it to the worker, so the
+            // worker never needs the override map itself.
+            const language = (0, grammars_1.detectLanguage)(filePath, content, overrides);
             if (!WorkerClass) {
                 // In-process fallback
-                return (0, tree_sitter_1.extractFromSource)(filePath, content, (0, grammars_1.detectLanguage)(filePath, content), frameworkNames);
+                return (0, tree_sitter_1.extractFromSource)(filePath, content, language, frameworkNames);
             }
             // Recycle the worker before the next parse if we've hit the threshold.
             // This destroys the WASM linear memory (which can grow but never shrink)
@@ -1041,7 +1147,7 @@ class ExtractionOrchestrator {
                     worker.terminate().catch(() => { });
                 }, timeoutMs);
                 pendingParses.set(id, { resolve, reject, timer });
-                worker.postMessage({ type: 'parse', id, filePath, content, frameworkNames });
+                worker.postMessage({ type: 'parse', id, filePath, content, frameworkNames, language });
             });
         }
         for (let i = 0; i < files.length; i += FILE_IO_BATCH_SIZE) {
@@ -1063,7 +1169,10 @@ class ExtractionOrchestrator {
             // Read files in parallel (with path validation before any I/O)
             const fileContents = await Promise.all(batch.map(async (fp) => {
                 try {
-                    const fullPath = (0, utils_1.validatePathWithinRoot)(this.rootDir, fp);
+                    // Indexing read: follow in-root symlinks the directory walk already
+                    // descended into (the `../` guard still applies) so files reached
+                    // via an in-root symlink-to-outside still index (#935).
+                    const fullPath = (0, utils_1.validatePathWithinRoot)(this.rootDir, fp, { allowSymlinkEscape: true });
                     if (!fullPath) {
                         (0, errors_1.logWarn)('Path traversal blocked in batch reader', { filePath: fp });
                         return { filePath: fp, content: null, stats: null, error: new Error('Path traversal blocked') };
@@ -1147,7 +1256,7 @@ class ExtractionOrchestrator {
                 processed++;
                 // Store in database on main thread (SQLite is not thread-safe)
                 if (result.nodes.length > 0 || result.errors.length === 0) {
-                    const language = (0, grammars_1.detectLanguage)(filePath, content);
+                    const language = (0, grammars_1.detectLanguage)(filePath, content, overrides);
                     this.storeExtractionResult(filePath, content, language, stats, result);
                 }
                 if (result.errors.length > 0) {
@@ -1169,7 +1278,7 @@ class ExtractionOrchestrator {
                     // Files with no symbols but no errors (yaml, twig, properties) are
                     // tracked at the file level — count them as indexed so the CLI
                     // doesn't misleadingly report "No files found to index".
-                    const lang = (0, grammars_1.detectLanguage)(filePath, content);
+                    const lang = (0, grammars_1.detectLanguage)(filePath, content, overrides);
                     if ((0, grammars_1.isFileLevelOnlyLanguage)(lang)) {
                         filesIndexed++;
                     }
@@ -1222,7 +1331,7 @@ class ExtractionOrchestrator {
                     continue;
                 }
                 if (result.nodes.length > 0 || result.errors.length === 0) {
-                    const language = (0, grammars_1.detectLanguage)(filePath, content);
+                    const language = (0, grammars_1.detectLanguage)(filePath, content, overrides);
                     const stats = await fsp.stat(path.join(this.rootDir, filePath));
                     this.storeExtractionResult(filePath, content, language, stats, result);
                     const idx = errors.indexOf(errEntry);
@@ -1270,7 +1379,7 @@ class ExtractionOrchestrator {
                         continue;
                     }
                     if (result.nodes.length > 0 || result.errors.length === 0) {
-                        const language = (0, grammars_1.detectLanguage)(filePath, fullContent);
+                        const language = (0, grammars_1.detectLanguage)(filePath, fullContent, overrides);
                         const stats = await fsp.stat(path.join(this.rootDir, filePath));
                         this.storeExtractionResult(filePath, fullContent, language, stats, result);
                         const idx = errors.indexOf(errEntry);
@@ -1350,7 +1459,8 @@ class ExtractionOrchestrator {
      * Index a single file
      */
     async indexFile(relativePath) {
-        const fullPath = (0, utils_1.validatePathWithinRoot)(this.rootDir, relativePath);
+        // Indexing read: follow in-root symlinks (the `../` guard still applies), #935.
+        const fullPath = (0, utils_1.validatePathWithinRoot)(this.rootDir, relativePath, { allowSymlinkEscape: true });
         if (!fullPath) {
             return {
                 nodes: [],
@@ -1390,8 +1500,8 @@ class ExtractionOrchestrator {
      * Used by the parallel batch reader to avoid redundant file I/O.
      */
     async indexFileWithContent(relativePath, content, stats) {
-        // Prevent path traversal
-        const fullPath = (0, utils_1.validatePathWithinRoot)(this.rootDir, relativePath);
+        // Prevent `../` traversal; follow in-root symlinks like the directory walk (#935).
+        const fullPath = (0, utils_1.validatePathWithinRoot)(this.rootDir, relativePath, { allowSymlinkEscape: true });
         if (!fullPath) {
             (0, errors_1.logWarn)('Path traversal blocked in indexFileWithContent', { relativePath });
             return {
@@ -1419,8 +1529,8 @@ class ExtractionOrchestrator {
                 durationMs: 0,
             };
         }
-        // Detect language
-        const language = (0, grammars_1.detectLanguage)(relativePath, content);
+        // Detect language (honoring the project's codegraph.json extension overrides)
+        const language = (0, grammars_1.detectLanguage)(relativePath, content, (0, project_config_1.loadExtensionOverrides)(this.rootDir));
         if (!(0, grammars_1.isLanguageSupported)(language)) {
             return {
                 nodes: [],
@@ -1451,6 +1561,25 @@ class ExtractionOrchestrator {
         if (existingFile && existingFile.contentHash === contentHash) {
             return; // No changes
         }
+        // Snapshot incoming cross-file edges BEFORE deleting this file's nodes.
+        // `deleteFile` cascades to delete every edge whose source OR target is a
+        // node in this file (edges.FK ... ON DELETE CASCADE). Edges whose SOURCE is
+        // in this file are re-emitted by the extractor below, but edges whose SOURCE
+        // is in a *different* (unchanged) file are not — they would be silently
+        // dropped, which is issue #899: re-indexing a callee file severs `calls`/
+        // `references` edges from callers that import it via module-attribute
+        // access (`pkg.mod.fn(...)`).
+        //
+        // We snapshot the edge plus the target node's (name, kind) so we can
+        // re-resolve to the re-indexed target's NEW id. Node ids are
+        // `sha256(filePath:kind:name:line)`, so any line shift in the callee file
+        // (e.g. a docstring-only edit above the symbol) changes every target id and
+        // a naive re-insert by old id would silently drop every edge. Matching by
+        // (filePath, kind, name) is stable across line shifts; if the symbol was
+        // renamed/removed, no match is found and the edge stays dropped (correct).
+        const crossFileIncomingEdges = existingFile
+            ? this.queries.getCrossFileIncomingEdgesWithTarget(filePath)
+            : [];
         // Delete existing data for this file
         if (existingFile) {
             this.queries.deleteFile(filePath);
@@ -1471,6 +1600,31 @@ class ExtractionOrchestrator {
                 this.queries.insertEdges(validEdges);
             }
         }
+        // Re-insert cross-file incoming edges snapshotted before the delete,
+        // re-resolving each edge's target to the re-indexed node's new id by
+        // (filePath, kind, name). Node ids include the source line, so any line
+        // shift in the callee file (e.g. a docstring-only edit above the symbol)
+        // changes every target id and a naive re-insert by old id would drop them
+        // all. `insertEdges` still filters to endpoints that exist, so edges whose
+        // caller (source) was deleted, or whose callee (target) was renamed/removed
+        // during the re-index (no match in `newTargetIds`), are dropped. This
+        // closes the #899 edge-drop on `sync`.
+        if (crossFileIncomingEdges.length > 0) {
+            const newNodesByKindName = new Map();
+            for (const n of validNodes) {
+                newNodesByKindName.set(`${n.kind}\0${n.name}`, n.id);
+            }
+            const reinserted = [];
+            for (const e of crossFileIncomingEdges) {
+                const newTargetId = newNodesByKindName.get(`${e.targetKind}\0${e.targetName}`);
+                if (newTargetId) {
+                    reinserted.push({ source: e.source, target: newTargetId, kind: e.kind, metadata: e.metadata, line: e.line, column: e.column, provenance: e.provenance });
+                }
+            }
+            if (reinserted.length > 0) {
+                this.queries.insertEdges(reinserted);
+            }
+        }
         // Insert unresolved references in batch with denormalized filePath/language
         if (result.unresolvedReferences.length > 0) {
             const insertedIds = new Set(validNodes.map((n) => n.id));
@@ -1530,7 +1684,7 @@ class ExtractionOrchestrator {
         // whether or not the project uses git, and crucially also catches committed
         // changes from `git pull`/`checkout`/`merge`/`rebase` — which `git status`
         // cannot see, because the working tree is clean afterward.
-        const currentFiles = scanDirectory(this.rootDir);
+        const currentFiles = await scanDirectoryAsync(this.rootDir);
         filesChecked = currentFiles.length;
         const currentSet = new Set(currentFiles);
         const trackedFiles = this.queries.getAllFiles();
@@ -1541,14 +1695,26 @@ class ExtractionOrchestrator {
         // Removals: tracked in the DB but no longer a present source file. Check the
         // filesystem directly — `scanDirectory` (via `git ls-files`) still lists a
         // file deleted from disk but not yet staged, so set membership alone misses it.
+        // `reconcileChecks` drives the cooperative yield shared with the adds/mods loop
+        // below (see SYNC_RECONCILE_YIELD_INTERVAL / issue #905).
+        let reconcileChecks = 0;
         for (const tracked of trackedFiles) {
             if (!currentSet.has(tracked.path) || !fs.existsSync(path.join(this.rootDir, tracked.path))) {
                 this.queries.deleteFile(tracked.path);
                 filesRemoved++;
             }
+            if (++reconcileChecks % SYNC_RECONCILE_YIELD_INTERVAL === 0) {
+                await new Promise((resolve) => setImmediate(resolve));
+            }
         }
         // Adds / modifications.
         for (const filePath of currentFiles) {
+            // Same cooperative yield as the removals loop — this is the other O(files)
+            // synchronous-stat loop that wedges the main thread on a large repo (#905).
+            // Yield at the top of the body so the `continue` fast-paths below still hit it.
+            if (++reconcileChecks % SYNC_RECONCILE_YIELD_INTERVAL === 0) {
+                await new Promise((resolve) => setImmediate(resolve));
+            }
             const fullPath = path.join(this.rootDir, filePath);
             const tracked = trackedMap.get(filePath);
             // Cheap pre-filter: an already-indexed file whose size AND mtime both match
@@ -1591,7 +1757,8 @@ class ExtractionOrchestrator {
         }
         // Load only grammars needed for changed files
         if (filesToIndex.length > 0) {
-            const neededLanguages = [...new Set(filesToIndex.map((f) => (0, grammars_1.detectLanguage)(f)))];
+            const overrides = (0, project_config_1.loadExtensionOverrides)(this.rootDir);
+            const neededLanguages = [...new Set(filesToIndex.map((f) => (0, grammars_1.detectLanguage)(f, undefined, overrides)))];
             // .h files default to 'c' but may be C++ — ensure cpp grammar is loaded
             if (neededLanguages.includes('c') && !neededLanguages.includes('cpp')) {
                 neededLanguages.push('cpp');