homegraph 0.9.10 → 1.1.1

package/dist/extraction/index.js

@@ -41,9 +41,11 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.loadAllGrammars = exports.loadGrammarsForLanguages = exports.initGrammars = exports.getSupportedLanguages = exports.isGrammarLoaded = exports.isLanguageSupported = exports.isSourceFile = exports.detectLanguage = exports.extractFromSource = exports.ExtractionOrchestrator = void 0;
+exports.loadAllGrammars = exports.loadGrammarsForLanguages = exports.initGrammars = exports.getSupportedLanguages = exports.isGrammarLoaded = exports.isLanguageSupported = exports.isSourceFile = exports.detectLanguage = exports.extractFromSource = exports.ExtractionOrchestrator = exports.ScopeIgnore = void 0;
 exports.hashContent = hashContent;
 exports.buildDefaultIgnore = buildDefaultIgnore;
+exports.buildScopeIgnore = buildScopeIgnore;
+exports.discoverEmbeddedRepoRoots = discoverEmbeddedRepoRoots;
 exports.scanDirectory = scanDirectory;
 exports.scanDirectoryAsync = scanDirectoryAsync;
 const fs = __importStar(require("fs"));
@@ -53,6 +55,8 @@ const crypto = __importStar(require("crypto"));
 const child_process_1 = require("child_process");
 const tree_sitter_1 = require("./tree-sitter");
 const grammars_1 = require("./grammars");
+const project_config_1 = require("../project-config");
+const directory_1 = require("../directory");
 const errors_1 = require("../errors");
 const utils_1 = require("../utils");
 const ignore_1 = __importDefault(require("ignore"));
@@ -62,6 +66,17 @@ const frameworks_1 = require("../resolution/frameworks");
  * File reads are I/O-bound; batching overlaps I/O wait with CPU parse work.
  */
 const FILE_IO_BATCH_SIZE = 10;
+/**
+ * How many files the `sync()` reconcile processes between cooperative yields to
+ * the event loop. The reconcile runs two O(files) loops of synchronous `fs`
+ * calls (existsSync for removals, statSync for adds/mods); on a very large repo
+ * (~100k files) an un-yielded run wedges the main thread for minutes, which both
+ * trips the liveness watchdog (it SIGKILLs a process whose loop stops turning)
+ * and blocks the first MCP tool call behind the catch-up gate (issue #905).
+ * Yielding every N files keeps the socket, the watchdog heartbeat, and any
+ * concurrent read query responsive while the reconcile runs.
+ */
+const SYNC_RECONCILE_YIELD_INTERVAL = 1000;
 // PARSER_RESET_INTERVAL moved to parse-worker.ts (runs in worker thread)
 /**
  * Maximum time (ms) to wait for a single file to parse in the worker thread.
@@ -91,7 +106,7 @@ function hashContent(content) {
 const MAX_FILE_SIZE = 1024 * 1024;
 /**
  * Directory names that are dependency, build, cache, or tooling output across the
- * languages/frameworks CodeGraph supports — curated from the canonical
+ * languages/frameworks HomeGraph supports — curated from the canonical
  * github/gitignore templates. Excluded by default so the graph reflects your code,
  * not third-party noise, without requiring a `.gitignore` (issue #407). The
  * exclusion applies uniformly (git or not, tracked or not); the only opt-in is an
@@ -100,7 +115,7 @@ const MAX_FILE_SIZE = 1024 * 1024;
  * `Library`) are deliberately NOT listed, to avoid ever hiding real source.
  *
  * Only dirs that actually contain *indexable source* (or are enormous) earn a slot
- * — IDE/state dirs like `.idea`/`.vs` are omitted because CodeGraph indexes only
+ * — IDE/state dirs like `.idea`/`.vs` are omitted because HomeGraph indexes only
  * recognized source extensions, so they produce no symbols regardless.
  */
 const DEFAULT_IGNORE_DIRS = new Set([
@@ -147,6 +162,74 @@ const DEFAULT_IGNORE_PATTERNS = [
     'cmake-build-*/', // CLion / CMake build trees
     'bazel-*/', // Bazel output symlink trees
 ];
+/** True if `buf` decodes as strict UTF-8 (no invalid byte sequences). */
+function isValidUtf8(buf) {
+    try {
+        new TextDecoder('utf-8', { fatal: true }).decode(buf);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Read a `.gitignore` and return patterns safe to hand to the `ignore` matcher —
+ * never throwing, even when the file isn't real gitignore text. Two failure
+ * modes, both seen in the wild (issue #682):
+ *
+ *  - The file isn't valid UTF-8 — e.g. transparently encrypted in place by
+ *    corporate DLP / endpoint-security software, leaving a UTF-16 header plus
+ *    ciphertext. None of it is meaningful patterns, so the whole file is skipped.
+ *  - The file is text but a single line can't be compiled to a regex by the
+ *    `ignore` library — `\\[` and friends throw "Unterminated character class".
+ *    Crucially the throw is LAZY (at match time, not `.add()`), so it would
+ *    otherwise escape mid-scan. That one pattern is dropped; the rest are kept.
+ *
+ * Either way a warning that NAMES the file is logged (the reporter couldn't tell
+ * which `.gitignore` was at fault) and indexing continues instead of aborting.
+ * Returns '' when there's nothing usable.
+ */
+function readGitignorePatterns(giPath) {
+    let buf;
+    try {
+        buf = fs.readFileSync(giPath);
+    }
+    catch {
+        return ''; // unreadable (permissions / race) — treat as absent
+    }
+    // A NUL byte never appears in real gitignore text, and a fatal UTF-8 decode
+    // catches the rest. Such a file isn't ignore patterns at all.
+    if (buf.includes(0) || !isValidUtf8(buf)) {
+        (0, errors_1.logWarn)('Ignoring a .gitignore that is not valid UTF-8 text — it may have been encrypted ' +
+            'in place by endpoint-security software. Indexing continues without it.', { file: giPath });
+        return '';
+    }
+    const content = buf.toString('utf-8');
+    // Fast path: one `.ignores()` call forces the library to compile EVERY rule,
+    // so if it doesn't throw, the whole file is safe to use verbatim.
+    try {
+        (0, ignore_1.default)().add(content).ignores('.homegraph-probe');
+        return content;
+    }
+    catch {
+        // Fall through: a line is uncompilable — keep the good ones, drop the bad.
+    }
+    const kept = [];
+    let dropped = 0;
+    for (const line of content.split(/\r?\n/)) {
+        try {
+            (0, ignore_1.default)().add(line).ignores('.homegraph-probe');
+            kept.push(line);
+        }
+        catch {
+            dropped++;
+        }
+    }
+    if (dropped > 0) {
+        (0, errors_1.logWarn)(`Skipped ${dropped} unparseable pattern(s) in a .gitignore; the rest are applied.`, { file: giPath });
+    }
+    return kept.join('\n');
+}
 /**
  * An `ignore` matcher seeded with the built-in defaults, merged with the project's
  * root .gitignore so a negation there (e.g. `!vendor/`) overrides a default. Shared
@@ -156,15 +239,275 @@ const DEFAULT_IGNORE_PATTERNS = [
  */
 function buildDefaultIgnore(rootDir) {
     const ig = (0, ignore_1.default)().add(DEFAULT_IGNORE_PATTERNS);
+    const rootGitignore = path.join(rootDir, '.gitignore');
+    if (fs.existsSync(rootGitignore))
+        ig.add(readGitignorePatterns(rootGitignore));
+    return ig;
+}
+/**
+ * Defaults-only ignore matcher (no root `.gitignore` merged). Used wherever the
+ * parent repo's own ignore rules must NOT apply — inside embedded child repos,
+ * whose gitignore semantics their own `git ls-files` already enforced (#514).
+ */
+function defaultsOnlyIgnore() {
+    return (0, ignore_1.default)().add(DEFAULT_IGNORE_PATTERNS);
+}
+/**
+ * Matcher for the project's `homegraph.json` `includeIgnored` patterns — the
+ * explicit opt-in to index embedded git repos living inside gitignored
+ * directories (#622, #699). Returns `null` when the project opted in nothing,
+ * which is the zero-config DEFAULT: `.gitignore` is then fully respected and a
+ * gitignored directory (even one holding nested repos) is never walked or
+ * indexed (#970, #976). Built once per scan/sync/scope operation from the scan
+ * root and threaded down — never global, so multi-project daemons stay isolated.
+ */
+function loadIncludeIgnoredMatcher(rootDir) {
+    const patterns = (0, project_config_1.loadIncludeIgnoredPatterns)(rootDir);
+    return patterns.length > 0 ? (0, ignore_1.default)().add(patterns) : null;
+}
+/**
+ * `git ls-files --directory` collapses a wholly-untracked/ignored directory into
+ * one entry — and when the command's own cwd is such a directory (the indexed
+ * root is itself a git-ignored subdir of an enclosing repo), git emits the
+ * literal `./` meaning "this entire directory". That sentinel is not a real
+ * nested path: feeding it to the `ignore` matcher throws ("path should be a
+ * `path.relative()`d string, but got "./""), which used to abort `buildScopeIgnore`
+ * and so break the MCP daemon's watcher/auto-sync on connect; and joining it back
+ * onto `repoDir` would just re-point at the cwd. Drop it wherever we consume
+ * `--directory` output. (#936)
+ */
+function isWholeCwdEntry(entry) {
+    return entry === './' || entry === '.' || entry === '';
+}
+/**
+ * List the gitignored DIRECTORIES of a repo (collapsed, trailing-slash form),
+ * relative to `repoDir`. These are invisible to every other `git ls-files` /
+ * `git status` mode — and in a multi-repo workspace they are exactly where the
+ * nested project repos live (a super-repo `.gitignore`s its child repos to keep
+ * `git status` quiet; that does not make them third-party code). (#514)
+ */
+function listIgnoredDirs(repoDir) {
     try {
-        const rootGitignore = path.join(rootDir, '.gitignore');
-        if (fs.existsSync(rootGitignore))
-            ig.add(fs.readFileSync(rootGitignore, 'utf-8'));
+        const out = (0, child_process_1.execFileSync)('git', ['ls-files', '-z', '-o', '-i', '--exclude-standard', '--directory'], { cwd: repoDir, encoding: 'utf-8', timeout: 30000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true });
+        return out.split('\0').filter((e) => e.endsWith('/') && !isWholeCwdEntry(e));
     }
     catch {
-        // Unreadable root .gitignore — the built-in defaults still apply.
+        return [];
     }
-    return ig;
+}
+/** Max directory depth searched below an ignored dir for nested `.git` roots. */
+const EMBEDDED_REPO_SEARCH_DEPTH = 4;
+/** Max directories examined per search — a huge ignored data dir must never stall a scan/sync. */
+const EMBEDDED_REPO_SEARCH_ENTRIES = 2000;
+/**
+ * Classify a directory's `.git` entry for embedded-repo discovery.
+ *
+ * - A `.git` **directory** is an embedded clone — distinct first-party code a
+ *   super-repo merely hides from git; index it (#193, #514).
+ * - A `.git` **file** is a pointer (`gitdir: …`). A git **worktree** points into
+ *   the host repo's own `.git/worktrees/<name>`, so it is a second working view
+ *   of a repo HomeGraph already indexes — indexing it just duplicates the whole
+ *   graph N times; skip it (#848). A **submodule worktree** points into
+ *   `.git/modules/<module>/worktrees/<name>` — same duplication, so skip it too
+ *   (#945). A **submodule** checkout points into `.git/modules/<module>` (no
+ *   `worktrees/` segment) and is distinct code, so index it as before.
+ *
+ * Returns `'none'` when there is no `.git` entry here.
+ */
+function classifyGitDir(absDir) {
+    let st;
+    try {
+        st = fs.statSync(path.join(absDir, '.git'));
+    }
+    catch {
+        return 'none';
+    }
+    if (st.isDirectory())
+        return 'embedded';
+    if (!st.isFile())
+        return 'none';
+    try {
+        const gitdir = fs.readFileSync(path.join(absDir, '.git'), 'utf8').match(/^gitdir:\s*(.+)$/m)?.[1]?.trim();
+        // A worktree's gitdir lives under some repo's `.git/worktrees/<name>` —
+        // either the top-level repo's (`.git/worktrees/`) or, for a worktree of a
+        // submodule, that submodule's gitdir (`.git/modules/<module>/worktrees/`).
+        // The optional `modules/<module>` segment covers the submodule case (#945).
+        // Match both separators so a Windows-style pointer is recognized too.
+        if (gitdir && /(^|[\\/])\.git[\\/](modules[\\/][^\\/]+[\\/])?worktrees[\\/]/.test(gitdir))
+            return 'worktree';
+    }
+    catch {
+        // Unreadable `.git` pointer — fall back to the prior "index it" behavior.
+    }
+    return 'embedded';
+}
+/**
+ * Find git repositories nested under `absDir` (inclusive), shallow bounded BFS.
+ * Stops descending at each repo root found — contents belong to that repo's own
+ * enumeration. Skips default-ignored dirs (`node_modules` can contain `.git`
+ * from npm git-dependencies — that never makes it project code) and HomeGraph
+ * data dirs. Depth- and entry-capped so a huge ignored tree can't stall the scan.
+ */
+function findNestedGitRepos(absDir, relPrefix) {
+    const found = [];
+    const defaults = defaultsOnlyIgnore();
+    const queue = [
+        { abs: absDir, rel: relPrefix, depth: 0 },
+    ];
+    let examined = 0;
+    while (queue.length > 0) {
+        const { abs, rel, depth } = queue.shift();
+        if (++examined > EMBEDDED_REPO_SEARCH_ENTRIES) {
+            (0, errors_1.logDebug)('Embedded-repo search entry cap hit — deeper repos (if any) not discovered', { under: relPrefix });
+            break;
+        }
+        const cls = classifyGitDir(abs);
+        if (cls === 'worktree') {
+            continue; // a git worktree duplicates an already-indexed repo (#848) — skip
+        }
+        if (cls === 'embedded') {
+            found.push(rel);
+            continue; // its own git handles everything below
+        }
+        if (depth >= EMBEDDED_REPO_SEARCH_DEPTH)
+            continue;
+        let entries;
+        try {
+            entries = fs.readdirSync(abs, { withFileTypes: true });
+        }
+        catch {
+            continue;
+        }
+        for (const entry of entries) {
+            if (!entry.isDirectory())
+                continue;
+            if (entry.name === '.git' || (0, directory_1.isHomeGraphDataDir)(entry.name))
+                continue;
+            const childRel = rel + entry.name + '/';
+            if (defaults.ignores(childRel))
+                continue;
+            queue.push({ abs: path.join(abs, entry.name), rel: childRel, depth: depth + 1 });
+        }
+    }
+    return found;
+}
+/**
+ * Workspace-scope ignore matcher. Ordinary paths get the root's matcher
+ * (built-in defaults + root `.gitignore`); paths inside an EMBEDDED repo get
+ * that repo's own matcher (defaults + its root `.gitignore`) — the parent's
+ * `.gitignore` hides a child repo from git, not from the index (#514). A
+ * directory path (trailing slash) that is an ANCESTOR of an embedded root is
+ * never ignored, so directory-pruning callers (the Linux per-directory
+ * watcher) still descend to reach the embedded repos.
+ *
+ * Single source of truth for indexer and watcher scope — they must not diverge.
+ */
+class ScopeIgnore {
+    rootMatcher;
+    embedded;
+    defaults = defaultsOnlyIgnore();
+    constructor(rootMatcher, embedded) {
+        this.rootMatcher = rootMatcher;
+        // Longest root first so paths in nested embedded repos hit the innermost matcher.
+        this.embedded = [...embedded].sort((a, b) => b.root.length - a.root.length);
+    }
+    ignores(rel) {
+        for (const { root, matcher } of this.embedded) {
+            if (rel.startsWith(root)) {
+                const inner = rel.slice(root.length);
+                if (inner === '')
+                    return false;
+                // Built-in defaults apply to the FULL path uniformly (#407) — an
+                // embedded repo inside node_modules (an npm git-dependency) must stay
+                // excluded even though its own rules wouldn't ignore its files.
+                return this.defaults.ignores(rel) || matcher.ignores(inner);
+            }
+        }
+        // Never prune a directory that leads to an embedded repo.
+        if (rel.endsWith('/') && this.embedded.some(({ root }) => root.startsWith(rel))) {
+            return false;
+        }
+        return this.rootMatcher.ignores(rel);
+    }
+}
+exports.ScopeIgnore = ScopeIgnore;
+/**
+ * Build the workspace-scope matcher. When the caller already knows the
+ * embedded roots (the scanner discovers them during collection), pass them to
+ * skip rediscovery; otherwise they're discovered here (the watcher path).
+ */
+function buildScopeIgnore(rootDir, embeddedRoots) {
+    const roots = embeddedRoots ? [...embeddedRoots] : discoverEmbeddedRepoRoots(rootDir);
+    return new ScopeIgnore(buildDefaultIgnore(rootDir), roots.map((root) => ({ root, matcher: buildDefaultIgnore(path.join(rootDir, root)) })));
+}
+/**
+ * Standalone discovery of every embedded repo root under `rootDir` (relative,
+ * trailing-slashed) — the untracked kind (#193) always, and the gitignored kind
+ * (#514) only for directories the project opted in via `homegraph.json`
+ * `includeIgnored` (#622, #699); otherwise `.gitignore` is respected and they
+ * are not discovered (#970, #976). Recursive (an embedded repo can embed further
+ * repos). Returns [] for non-git roots: the filesystem walk handles nested repos
+ * there already.
+ */
+function discoverEmbeddedRepoRoots(rootDir) {
+    try {
+        (0, child_process_1.execFileSync)('git', ['rev-parse', '--git-dir'], { cwd: rootDir, encoding: 'utf-8', timeout: 5000, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true });
+    }
+    catch {
+        return [];
+    }
+    const out = [];
+    const defaults = defaultsOnlyIgnore();
+    const includeIgnored = loadIncludeIgnoredMatcher(rootDir);
+    const visit = (repoAbs, prefix) => {
+        const candidates = [];
+        try {
+            const o = (0, child_process_1.execFileSync)('git', ['ls-files', '-z', '-o', '--exclude-standard', '--directory'], { cwd: repoAbs, encoding: 'utf-8', timeout: 30000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true });
+            for (const e of o.split('\0')) {
+                if (e.endsWith('/') && !isWholeCwdEntry(e) && !defaults.ignores(e)) {
+                    candidates.push(...findNestedGitRepos(path.join(repoAbs, e), e));
+                }
+            }
+        }
+        catch { /* untracked listing failed — ignored-side discovery still runs */ }
+        candidates.push(...findIgnoredEmbeddedRepos(repoAbs, includeIgnored, prefix));
+        for (const rel of candidates) {
+            const full = (0, utils_1.normalizePath)(prefix + rel);
+            out.push(full);
+            visit(path.join(repoAbs, rel), full);
+        }
+    };
+    visit(rootDir, '');
+    return out;
+}
+/**
+ * Discover embedded repos hidden by `repoDir`'s OWN gitignore rules: for each
+ * gitignored directory, search for nested `.git` roots. Returns repo paths
+ * relative to `repoDir`, trailing-slashed.
+ *
+ * OPT-IN ONLY. Walking into a gitignored directory contradicts what every other
+ * tool (and HomeGraph's own `git ls-files` foundation) does — `.gitignore`
+ * excludes. So this returns `[]` unless the project opted the directory in via
+ * `homegraph.json` `includeIgnored`; without that, a gitignored dir — including
+ * a huge reference/data dir full of nested clones — is left untouched (#970,
+ * #976). When opted in, it restores the super-repo-of-clones behavior (#622,
+ * #699). `prefix` is the scan-root-relative path of `repoDir`, so a pattern like
+ * `services/` opts that whole subtree in at any recursion depth. Built-in
+ * default excludes (`node_modules`, …) are always skipped.
+ */
+function findIgnoredEmbeddedRepos(repoDir, includeIgnored, prefix) {
+    if (!includeIgnored)
+        return [];
+    const defaults = defaultsOnlyIgnore();
+    const repos = [];
+    for (const dir of listIgnoredDirs(repoDir)) {
+        if (defaults.ignores(dir))
+            continue;
+        if (!includeIgnored.ignores((0, utils_1.normalizePath)(prefix + dir)))
+            continue;
+        repos.push(...findNestedGitRepos(path.join(repoDir, dir), dir));
+    }
+    return repos;
 }
 /**
  * Collect git-visible files (tracked + untracked, .gitignore-respected) from the
@@ -177,9 +520,15 @@ function buildDefaultIgnore(rootDir) {
  * skips them entirely, and untracked output reports them only as an opaque
  * "subdir/" entry (trailing slash) rather than expanding their files. Each
  * embedded repo is its own git boundary, so we re-run `git ls-files` inside it.
- * (See issue #193.)
+ * (See issue #193.) GITIGNORED embedded repos are invisible even to that; they
+ * are discovered separately via `findIgnoredEmbeddedRepos` (#514) but ONLY for
+ * directories the project opted in through `homegraph.json` `includeIgnored`
+ * (`includeIgnored` here, threaded from the scan root) — by default `.gitignore`
+ * is respected and they stay out (#970, #976). Every embedded repo root (however
+ * found) is recorded in `embeddedRoots` so callers can exempt its files from the
+ * parent's own gitignore rules.
  */
-function collectGitFiles(repoDir, prefix, files) {
+function collectGitFiles(repoDir, prefix, files, embeddedRoots, includeIgnored = null) {
     const gitOpts = { cwd: repoDir, encoding: 'utf-8', timeout: 30000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true };
     // Tracked files. --recurse-submodules pulls in files from active submodules,
     // which the index would otherwise represent only as a commit pointer.
@@ -205,15 +554,29 @@ function collectGitFiles(repoDir, prefix, files) {
         if (rel.endsWith('/')) {
             // git only emits a trailing-slash directory entry for an embedded repo.
             // Guard with a .git check anyway, and skip anything else exactly as git
-            // itself skips it (we never descend into a non-repo opaque dir).
+            // itself skips it (we never descend into a non-repo opaque dir). Never
+            // descend into default-ignored locations — an embedded repo inside
+            // node_modules is an npm git-dependency, not project code.
             const childDir = path.join(repoDir, rel);
-            if (fs.existsSync(path.join(childDir, '.git'))) {
-                collectGitFiles(childDir, prefix + rel, files);
+            // A git worktree surfaces here as an opaque untracked dir too — skip it,
+            // it's a duplicate working view of an already-indexed repo (#848).
+            if (classifyGitDir(childDir) === 'embedded' && !defaultsOnlyIgnore().ignores(rel)) {
+                embeddedRoots?.add((0, utils_1.normalizePath)(prefix + rel));
+                collectGitFiles(childDir, prefix + rel, files, embeddedRoots, includeIgnored);
             }
             continue;
         }
         files.add((0, utils_1.normalizePath)(prefix + rel));
     }
+    // Embedded repos hidden by THIS repo's ignore rules (`/packages/` in a
+    // super-repo .gitignore) never appear in any listing above. By default they
+    // stay hidden — `.gitignore` is respected (#970, #976). They are recursed into
+    // only when the project opted the directory in via `homegraph.json`
+    // `includeIgnored` (#622, #699), which `findIgnoredEmbeddedRepos` enforces.
+    for (const rel of findIgnoredEmbeddedRepos(repoDir, includeIgnored, prefix)) {
+        embeddedRoots?.add((0, utils_1.normalizePath)(prefix + rel));
+        collectGitFiles(path.join(repoDir, rel), prefix + rel, files, embeddedRoots, includeIgnored);
+    }
 }
 /**
  * Get all files visible to git (tracked + untracked but not ignored).
@@ -239,11 +602,15 @@ function getGitVisibleFiles(rootDir) {
             }
         }
         const files = new Set();
-        collectGitFiles(rootDir, '', files);
+        const embeddedRoots = new Set();
+        collectGitFiles(rootDir, '', files, embeddedRoots, loadIncludeIgnoredMatcher(rootDir));
         // Apply built-in default ignores uniformly — to tracked files too, since
         // committing a dependency/build dir doesn't make it project code. A
         // `.gitignore` negation (e.g. `!vendor/`) is the explicit opt-in. (issue #407)
-        const ig = buildDefaultIgnore(rootDir);
+        // Files inside an EMBEDDED repo are matched against that repo's own rules,
+        // not the parent's: the parent's .gitignore hides the child repo from git,
+        // not from the index. (#514)
+        const ig = buildScopeIgnore(rootDir, embeddedRoots);
         return new Set([...files].filter((f) => !ig.ignores(f)));
     }
     catch {
@@ -253,38 +620,90 @@ function getGitVisibleFiles(rootDir) {
 /**
  * Use `git status` to detect changed files instead of scanning every file.
  * Returns null on failure so callers fall back to full scan.
+ *
+ * Recurses into embedded repos — the untracked kind (#193: the parent's status
+ * collapses them to an opaque `?? subdir/` entry) always, and the gitignored
+ * kind (#514: they never appear in the parent's status at all) only for
+ * directories opted in via `homegraph.json` `includeIgnored` (#622, #699) —
+ * running `git status` inside each, so changes in a multi-repo workspace sync
+ * without a full rescan. By default a gitignored dir is left alone, matching the
+ * full-index scan (#970, #976). Deleting an ENTIRE embedded repo dir is the one
+ * case this cannot see (the child status that would report the deletions is gone
+ * with it); a full `homegraph index` reconciles that.
  */
 function getGitChangedFiles(rootDir) {
     try {
-        const output = (0, child_process_1.execFileSync)('git', ['status', '--porcelain', '--no-renames'], { cwd: rootDir, encoding: 'utf-8', timeout: 10000, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true });
-        const modified = [];
-        const added = [];
-        const deleted = [];
-        for (const line of output.split('\n')) {
-            if (line.length < 4)
-                continue; // Minimum: "XY file"
-            const statusCode = line.substring(0, 2);
-            const filePath = (0, utils_1.normalizePath)(line.substring(3));
-            // Skip non-source files (git status already omits .gitignored paths).
-            if (!(0, grammars_1.isSourceFile)(filePath))
-                continue;
-            if (statusCode === '??') {
-                added.push(filePath);
-            }
-            else if (statusCode.includes('D')) {
-                deleted.push(filePath);
-            }
-            else {
-                // M, MM, AM, A (staged), etc. — treat as modified
-                modified.push(filePath);
-            }
-        }
-        return { modified, added, deleted };
+        const changes = { modified: [], added: [], deleted: [] };
+        // Custom extension → language overrides from the project's homegraph.json,
+        // so change detection sees the same custom-extension files the full index does.
+        const overrides = (0, project_config_1.loadExtensionOverrides)(rootDir);
+        collectGitStatus(rootDir, '', changes, overrides, loadIncludeIgnoredMatcher(rootDir));
+        return changes;
     }
     catch {
         return null;
     }
 }
+function collectGitStatus(repoDir, prefix, out, overrides, includeIgnored = null) {
+    const output = (0, child_process_1.execFileSync)('git', ['status', '--porcelain', '--no-renames'], { cwd: repoDir, encoding: 'utf-8', timeout: 10000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true });
+    // This repo's own ignore rules — built-in defaults (#407) plus its .gitignore.
+    // Change detection must exclude the SAME files the full index does, but git
+    // status hides neither: it ignores nothing for *tracked* paths, and the
+    // built-in defaults aren't gitignore at all. Without this filter a committed
+    // vendor/ dir, or a tracked file under a .gitignored dir, surfaces here as a
+    // change — so `homegraph status` (which reads getChangedFiles) reports a
+    // pending edit the full index never tracks and `sync` never clears. Matching
+    // repo-relative `rel` at each recursion level mirrors getGitVisibleFiles'
+    // ScopeIgnore: every embedded repo is judged by ITS OWN rules, never the
+    // parent's. (#766)
+    const ig = buildDefaultIgnore(repoDir);
+    const untrackedDirs = [];
+    for (const line of output.split('\n')) {
+        if (line.length < 4)
+            continue; // Minimum: "XY file"
+        const statusCode = line.substring(0, 2);
+        const rel = (0, utils_1.normalizePath)(line.substring(3));
+        // Untracked directory entries (trailing slash) may hide an embedded repo —
+        // collect for the recursion below instead of treating as a file.
+        if (statusCode === '??' && rel.endsWith('/')) {
+            untrackedDirs.push(rel);
+            continue;
+        }
+        const filePath = (0, utils_1.normalizePath)(prefix + rel);
+        if (!(0, grammars_1.isSourceFile)(filePath, overrides))
+            continue;
+        if (statusCode.includes('D')) {
+            // Deletions stay unfiltered: getChangedFiles acts on one only when the
+            // path is already tracked in the DB, where removal is always correct — and
+            // that lets a newly-excluded dir's stale rows clean themselves up. (#766)
+            out.deleted.push(filePath);
+            continue;
+        }
+        // Added (`??`) / modified files inside an excluded dir must not enter the
+        // index — match against the repo-relative path, same as the full scan. (#766)
+        if (ig.ignores(rel))
+            continue;
+        if (statusCode === '??') {
+            out.added.push(filePath);
+        }
+        else {
+            // M, MM, AM, A (staged), etc. — treat as modified
+            out.modified.push(filePath);
+        }
+    }
+    // Recurse embedded repos found under untracked dirs (at the dir itself or
+    // nested deeper). Gitignored dirs are walked only for the directories the
+    // project opted in via `includeIgnored`; by default `.gitignore` is respected
+    // and they are left alone (#970, #976), mirroring the full-index scan.
+    for (const rel of untrackedDirs) {
+        for (const repoRel of findNestedGitRepos(path.join(repoDir, rel), rel)) {
+            collectGitStatus(path.join(repoDir, repoRel), prefix + repoRel, out, overrides, includeIgnored);
+        }
+    }
+    for (const rel of findIgnoredEmbeddedRepos(repoDir, includeIgnored, prefix)) {
+        collectGitStatus(path.join(repoDir, rel), prefix + rel, out, overrides, includeIgnored);
+    }
+}
 /**
  * Recursively scan a directory for source files.
  *
@@ -293,13 +712,15 @@ function getGitChangedFiles(rootDir) {
  * projects, falls back to a filesystem walk that parses .gitignore itself.
  */
 function scanDirectory(rootDir, onProgress) {
+    // Custom extension → language overrides from the project's homegraph.json.
+    const overrides = (0, project_config_1.loadExtensionOverrides)(rootDir);
     // Fast path: use git to get all visible files (respects .gitignore everywhere)
     const gitFiles = getGitVisibleFiles(rootDir);
     if (gitFiles) {
         const files = [];
         let count = 0;
         for (const filePath of gitFiles) {
-            if ((0, grammars_1.isSourceFile)(filePath)) {
+            if ((0, grammars_1.isSourceFile)(filePath, overrides)) {
                 files.push(filePath);
                 count++;
                 onProgress?.(count, filePath);
@@ -315,12 +736,14 @@ function scanDirectory(rootDir, onProgress) {
  * allowing worker threads to receive and render progress messages.
  */
 async function scanDirectoryAsync(rootDir, onProgress) {
+    // Custom extension → language overrides from the project's homegraph.json.
+    const overrides = (0, project_config_1.loadExtensionOverrides)(rootDir);
     const gitFiles = getGitVisibleFiles(rootDir);
     if (gitFiles) {
         const files = [];
         let count = 0;
         for (const filePath of gitFiles) {
-            if ((0, grammars_1.isSourceFile)(filePath)) {
+            if ((0, grammars_1.isSourceFile)(filePath, overrides)) {
                 files.push(filePath);
                 count++;
                 onProgress?.(count, filePath);
@@ -341,17 +764,17 @@ function scanDirectoryWalk(rootDir, onProgress) {
     const files = [];
     let count = 0;
     const visitedDirs = new Set();
+    // Custom extension → language overrides from the project's homegraph.json.
+    const overrides = (0, project_config_1.loadExtensionOverrides)(rootDir);
     const loadIgnore = (dir) => {
-        try {
-            const giPath = path.join(dir, '.gitignore');
-            if (fs.existsSync(giPath)) {
-                return { dir, ig: (0, ignore_1.default)().add(fs.readFileSync(giPath, 'utf-8')) };
-            }
-        }
-        catch {
-            // Unreadable .gitignore — treat as absent.
-        }
-        return null;
+        const giPath = path.join(dir, '.gitignore');
+        if (!fs.existsSync(giPath))
+            return null;
+        // readGitignorePatterns is defensive: a non-UTF-8 (DLP-encrypted) or
+        // uncompilable .gitignore is skipped/filtered with a warning, never thrown
+        // (issue #682) — so the per-file `.ignores()` calls below can't crash.
+        const patterns = readGitignorePatterns(giPath);
+        return patterns ? { dir, ig: (0, ignore_1.default)().add(patterns) } : null;
     };
     const isIgnored = (fullPath, isDir, matchers) => {
         for (const { dir, ig } of matchers) {
@@ -393,8 +816,9 @@ function scanDirectoryWalk(rootDir, onProgress) {
             return;
         }
         for (const entry of entries) {
-            // Never descend into git internals or our own data directory.
-            if (entry.name === '.git' || entry.name === '.homegraph')
+            // Never descend into git internals or any HomeGraph data directory
+            // (the active one or a sibling another environment created — #636).
+            if (entry.name === '.git' || (0, directory_1.isHomeGraphDataDir)(entry.name))
                 continue;
             const fullPath = path.join(dir, entry.name);
             const relativePath = (0, utils_1.normalizePath)(path.relative(rootDir, fullPath));
@@ -408,7 +832,7 @@ function scanDirectoryWalk(rootDir, onProgress) {
                         }
                     }
                     else if (stat.isFile()) {
-                        if (!isIgnored(fullPath, false, active) && (0, grammars_1.isSourceFile)(relativePath)) {
+                        if (!isIgnored(fullPath, false, active) && (0, grammars_1.isSourceFile)(relativePath, overrides)) {
                             files.push(relativePath);
                             count++;
                             onProgress?.(count, relativePath);
@@ -426,7 +850,7 @@ function scanDirectoryWalk(rootDir, onProgress) {
                 }
             }
             else if (entry.isFile()) {
-                if (!isIgnored(fullPath, false, active) && (0, grammars_1.isSourceFile)(relativePath)) {
+                if (!isIgnored(fullPath, false, active) && (0, grammars_1.isSourceFile)(relativePath, overrides)) {
                     files.push(relativePath);
                     count++;
                     onProgress?.(count, relativePath);
@@ -540,6 +964,10 @@ class ExtractionOrchestrator {
         let filesErrored = 0;
         let totalNodes = 0;
         let totalEdges = 0;
+        // Custom extension → language overrides from the project's homegraph.json.
+        // Threaded into language detection so custom-extension files load the right
+        // grammar and store under the mapped language.
+        const overrides = (0, project_config_1.loadExtensionOverrides)(this.rootDir);
         const log = verbose
             ? (msg) => { console.log(`[worker] ${msg}`); }
             : (_msg) => { };
@@ -589,7 +1017,7 @@ class ExtractionOrchestrator {
         });
         await new Promise(resolve => setImmediate(resolve));
         // Detect needed languages and load grammars in the parse worker
-        const neededLanguages = [...new Set(files.map((f) => (0, grammars_1.detectLanguage)(f)))];
+        const neededLanguages = [...new Set(files.map((f) => (0, grammars_1.detectLanguage)(f, undefined, overrides)))];
         // .h files default to 'c' but may be C++ — ensure cpp grammar is loaded when c is needed
         if (neededLanguages.includes('c') && !neededLanguages.includes('cpp')) {
             neededLanguages.push('cpp');
@@ -688,7 +1116,10 @@ class ExtractionOrchestrator {
             w.terminate().catch(() => { });
         }
         async function requestParse(filePath, content) {
-            const language = (0, grammars_1.detectLanguage)(filePath, content);
+            // Resolve the language on the main thread (where the project's
+            // homegraph.json overrides are loaded) and hand it to the worker, so the
+            // worker never needs the override map itself.
+            const language = (0, grammars_1.detectLanguage)(filePath, content, overrides);
             if (!WorkerClass || (0, grammars_1.requiresInProcessExtraction)(language)) {
                 // In-process fallback (also required for languages whose extractors
                 // cannot run in a worker thread).
@@ -717,7 +1148,7 @@ class ExtractionOrchestrator {
                     worker.terminate().catch(() => { });
                 }, timeoutMs);
                 pendingParses.set(id, { resolve, reject, timer });
-                worker.postMessage({ type: 'parse', id, filePath, content, frameworkNames });
+                worker.postMessage({ type: 'parse', id, filePath, content, frameworkNames, language });
             });
         }
         for (let i = 0; i < files.length; i += FILE_IO_BATCH_SIZE) {
@@ -739,7 +1170,10 @@ class ExtractionOrchestrator {
             // Read files in parallel (with path validation before any I/O)
             const fileContents = await Promise.all(batch.map(async (fp) => {
                 try {
-                    const fullPath = (0, utils_1.validatePathWithinRoot)(this.rootDir, fp);
+                    // Indexing read: follow in-root symlinks the directory walk already
+                    // descended into (the `../` guard still applies) so files reached
+                    // via an in-root symlink-to-outside still index (#935).
+                    const fullPath = (0, utils_1.validatePathWithinRoot)(this.rootDir, fp, { allowSymlinkEscape: true });
                     if (!fullPath) {
                         (0, errors_1.logWarn)('Path traversal blocked in batch reader', { filePath: fp });
                         return { filePath: fp, content: null, stats: null, error: new Error('Path traversal blocked') };
@@ -823,7 +1257,7 @@ class ExtractionOrchestrator {
                 processed++;
                 // Store in database on main thread (SQLite is not thread-safe)
                 if (result.nodes.length > 0 || result.errors.length === 0) {
-                    const language = (0, grammars_1.detectLanguage)(filePath, content);
+                    const language = (0, grammars_1.detectLanguage)(filePath, content, overrides);
                     this.storeExtractionResult(filePath, content, language, stats, result);
                 }
                 if (result.errors.length > 0) {
@@ -845,7 +1279,7 @@ class ExtractionOrchestrator {
                     // Files with no symbols but no errors (yaml, twig, properties) are
                     // tracked at the file level — count them as indexed so the CLI
                     // doesn't misleadingly report "No files found to index".
-                    const lang = (0, grammars_1.detectLanguage)(filePath, content);
+                    const lang = (0, grammars_1.detectLanguage)(filePath, content, overrides);
                     if ((0, grammars_1.isFileLevelOnlyLanguage)(lang)) {
                         filesIndexed++;
                     }
@@ -898,7 +1332,7 @@ class ExtractionOrchestrator {
                     continue;
                 }
                 if (result.nodes.length > 0 || result.errors.length === 0) {
-                    const language = (0, grammars_1.detectLanguage)(filePath, content);
+                    const language = (0, grammars_1.detectLanguage)(filePath, content, overrides);
                     const stats = await fsp.stat(path.join(this.rootDir, filePath));
                     this.storeExtractionResult(filePath, content, language, stats, result);
                     const idx = errors.indexOf(errEntry);
@@ -946,7 +1380,7 @@ class ExtractionOrchestrator {
                         continue;
                     }
                     if (result.nodes.length > 0 || result.errors.length === 0) {
-                        const language = (0, grammars_1.detectLanguage)(filePath, fullContent);
+                        const language = (0, grammars_1.detectLanguage)(filePath, fullContent, overrides);
                         const stats = await fsp.stat(path.join(this.rootDir, filePath));
                         this.storeExtractionResult(filePath, fullContent, language, stats, result);
                         const idx = errors.indexOf(errEntry);
@@ -1026,7 +1460,8 @@ class ExtractionOrchestrator {
      * Index a single file
      */
     async indexFile(relativePath) {
-        const fullPath = (0, utils_1.validatePathWithinRoot)(this.rootDir, relativePath);
+        // Indexing read: follow in-root symlinks (the `../` guard still applies), #935.
+        const fullPath = (0, utils_1.validatePathWithinRoot)(this.rootDir, relativePath, { allowSymlinkEscape: true });
         if (!fullPath) {
             return {
                 nodes: [],
@@ -1066,8 +1501,8 @@ class ExtractionOrchestrator {
      * Used by the parallel batch reader to avoid redundant file I/O.
      */
     async indexFileWithContent(relativePath, content, stats) {
-        // Prevent path traversal
-        const fullPath = (0, utils_1.validatePathWithinRoot)(this.rootDir, relativePath);
+        // Prevent `../` traversal; follow in-root symlinks like the directory walk (#935).
+        const fullPath = (0, utils_1.validatePathWithinRoot)(this.rootDir, relativePath, { allowSymlinkEscape: true });
         if (!fullPath) {
             (0, errors_1.logWarn)('Path traversal blocked in indexFileWithContent', { relativePath });
             return {
@@ -1095,8 +1530,8 @@ class ExtractionOrchestrator {
                 durationMs: 0,
             };
         }
-        // Detect language
-        const language = (0, grammars_1.detectLanguage)(relativePath, content);
+        // Detect language (honoring the project's homegraph.json extension overrides)
+        const language = (0, grammars_1.detectLanguage)(relativePath, content, (0, project_config_1.loadExtensionOverrides)(this.rootDir));
         if (!(0, grammars_1.isLanguageSupported)(language)) {
             return {
                 nodes: [],
@@ -1127,6 +1562,25 @@ class ExtractionOrchestrator {
         if (existingFile && existingFile.contentHash === contentHash) {
             return; // No changes
         }
+        // Snapshot incoming cross-file edges BEFORE deleting this file's nodes.
+        // `deleteFile` cascades to delete every edge whose source OR target is a
+        // node in this file (edges.FK ... ON DELETE CASCADE). Edges whose SOURCE is
+        // in this file are re-emitted by the extractor below, but edges whose SOURCE
+        // is in a *different* (unchanged) file are not — they would be silently
+        // dropped, which is issue #899: re-indexing a callee file severs `calls`/
+        // `references` edges from callers that import it via module-attribute
+        // access (`pkg.mod.fn(...)`).
+        //
+        // We snapshot the edge plus the target node's (name, kind) so we can
+        // re-resolve to the re-indexed target's NEW id. Node ids are
+        // `sha256(filePath:kind:name:line)`, so any line shift in the callee file
+        // (e.g. a docstring-only edit above the symbol) changes every target id and
+        // a naive re-insert by old id would silently drop every edge. Matching by
+        // (filePath, kind, name) is stable across line shifts; if the symbol was
+        // renamed/removed, no match is found and the edge stays dropped (correct).
+        const crossFileIncomingEdges = existingFile
+            ? this.queries.getCrossFileIncomingEdgesWithTarget(filePath)
+            : [];
         // Delete existing data for this file
         if (existingFile) {
             this.queries.deleteFile(filePath);
@@ -1147,6 +1601,31 @@ class ExtractionOrchestrator {
                 this.queries.insertEdges(validEdges);
             }
         }
+        // Re-insert cross-file incoming edges snapshotted before the delete,
+        // re-resolving each edge's target to the re-indexed node's new id by
+        // (filePath, kind, name). Node ids include the source line, so any line
+        // shift in the callee file (e.g. a docstring-only edit above the symbol)
+        // changes every target id and a naive re-insert by old id would drop them
+        // all. `insertEdges` still filters to endpoints that exist, so edges whose
+        // caller (source) was deleted, or whose callee (target) was renamed/removed
+        // during the re-index (no match in `newTargetIds`), are dropped. This
+        // closes the #899 edge-drop on `sync`.
+        if (crossFileIncomingEdges.length > 0) {
+            const newNodesByKindName = new Map();
+            for (const n of validNodes) {
+                newNodesByKindName.set(`${n.kind}\0${n.name}`, n.id);
+            }
+            const reinserted = [];
+            for (const e of crossFileIncomingEdges) {
+                const newTargetId = newNodesByKindName.get(`${e.targetKind}\0${e.targetName}`);
+                if (newTargetId) {
+                    reinserted.push({ source: e.source, target: newTargetId, kind: e.kind, metadata: e.metadata, line: e.line, column: e.column, provenance: e.provenance });
+                }
+            }
+            if (reinserted.length > 0) {
+                this.queries.insertEdges(reinserted);
+            }
+        }
         // Insert unresolved references in batch with denormalized filePath/language
         if (result.unresolvedReferences.length > 0) {
             const insertedIds = new Set(validNodes.map((n) => n.id));
@@ -1206,7 +1685,7 @@ class ExtractionOrchestrator {
         // whether or not the project uses git, and crucially also catches committed
         // changes from `git pull`/`checkout`/`merge`/`rebase` — which `git status`
         // cannot see, because the working tree is clean afterward.
-        const currentFiles = scanDirectory(this.rootDir);
+        const currentFiles = await scanDirectoryAsync(this.rootDir);
         filesChecked = currentFiles.length;
         const currentSet = new Set(currentFiles);
         const trackedFiles = this.queries.getAllFiles();
@@ -1217,14 +1696,26 @@ class ExtractionOrchestrator {
         // Removals: tracked in the DB but no longer a present source file. Check the
         // filesystem directly — `scanDirectory` (via `git ls-files`) still lists a
         // file deleted from disk but not yet staged, so set membership alone misses it.
+        // `reconcileChecks` drives the cooperative yield shared with the adds/mods loop
+        // below (see SYNC_RECONCILE_YIELD_INTERVAL / issue #905).
+        let reconcileChecks = 0;
         for (const tracked of trackedFiles) {
             if (!currentSet.has(tracked.path) || !fs.existsSync(path.join(this.rootDir, tracked.path))) {
                 this.queries.deleteFile(tracked.path);
                 filesRemoved++;
             }
+            if (++reconcileChecks % SYNC_RECONCILE_YIELD_INTERVAL === 0) {
+                await new Promise((resolve) => setImmediate(resolve));
+            }
         }
         // Adds / modifications.
         for (const filePath of currentFiles) {
+            // Same cooperative yield as the removals loop — this is the other O(files)
+            // synchronous-stat loop that wedges the main thread on a large repo (#905).
+            // Yield at the top of the body so the `continue` fast-paths below still hit it.
+            if (++reconcileChecks % SYNC_RECONCILE_YIELD_INTERVAL === 0) {
+                await new Promise((resolve) => setImmediate(resolve));
+            }
             const fullPath = path.join(this.rootDir, filePath);
             const tracked = trackedMap.get(filePath);
             // Cheap pre-filter: an already-indexed file whose size AND mtime both match
@@ -1267,7 +1758,8 @@ class ExtractionOrchestrator {
         }
         // Load only grammars needed for changed files
         if (filesToIndex.length > 0) {
-            const neededLanguages = [...new Set(filesToIndex.map((f) => (0, grammars_1.detectLanguage)(f)))];
+            const overrides = (0, project_config_1.loadExtensionOverrides)(this.rootDir);
+            const neededLanguages = [...new Set(filesToIndex.map((f) => (0, grammars_1.detectLanguage)(f, undefined, overrides)))];
             // .h files default to 'c' but may be C++ — ensure cpp grammar is loaded
             if (neededLanguages.includes('c') && !neededLanguages.includes('cpp')) {
                 neededLanguages.push('cpp');