@quolu/lattice 0.1.0

package/sensor/dist/extraction/index.js

@@ -0,0 +1,2563 @@
+"use strict";
+/**
+ * Extraction Orchestrator
+ *
+ * Coordinates file scanning, parsing, and database storage.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.loadAllGrammars = exports.loadGrammarsForLanguages = exports.initGrammars = exports.getSupportedLanguages = exports.isGrammarLoaded = exports.isLanguageSupported = exports.isSourceFile = exports.detectLanguage = exports.extractFromSource = exports.ExtractionOrchestrator = exports.ScopeIgnore = void 0;
+exports.hashContent = hashContent;
+exports.buildDefaultIgnore = buildDefaultIgnore;
+exports.buildScopeIgnore = buildScopeIgnore;
+exports.discoverEmbeddedRepoRoots = discoverEmbeddedRepoRoots;
+exports.findUnindexedIgnoredRepos = findUnindexedIgnoredRepos;
+exports.scanDirectory = scanDirectory;
+exports.scanDirectoryAsync = scanDirectoryAsync;
+const fs = __importStar(require("fs"));
+const fsp = __importStar(require("fs/promises"));
+const path = __importStar(require("path"));
+const os = __importStar(require("os"));
+const crypto = __importStar(require("crypto"));
+const child_process_1 = require("child_process");
+const tree_sitter_1 = require("./tree-sitter");
+const parse_pool_1 = require("./parse-pool");
+const store_writer_1 = require("./store-writer");
+const kernel_1 = require("./kernel");
+const grammars_1 = require("./grammars");
+const project_config_1 = require("../project-config");
+const directory_1 = require("../directory");
+const errors_1 = require("../errors");
+const utils_1 = require("../utils");
+const ignore_1 = __importDefault(require("ignore"));
+const frameworks_1 = require("../resolution/frameworks");
+const cooperative_yield_1 = require("../resolution/cooperative-yield");
+/**
+ * Number of files to read in parallel during indexing.
+ * File reads are I/O-bound; batching overlaps I/O wait with CPU parse work.
+ */
+const FILE_IO_BATCH_SIZE = 10;
+/**
+ * How many files the `sync()` reconcile processes between cooperative yields to
+ * the event loop. The reconcile runs two O(files) loops of synchronous `fs`
+ * calls (existsSync for removals, statSync for adds/mods); on a very large repo
+ * (~100k files) an un-yielded run wedges the main thread for minutes, which both
+ * trips the liveness watchdog (it SIGKILLs a process whose loop stops turning)
+ * and blocks the first MCP tool call behind the catch-up gate (issue #905).
+ * Yielding every N files keeps the socket, the watchdog heartbeat, and any
+ * concurrent read query responsive while the reconcile runs.
+ */
+const SYNC_RECONCILE_YIELD_INTERVAL = 1000;
+// PARSER_RESET_INTERVAL moved to parse-worker.ts (runs in worker thread)
+/**
+ * Maximum time (ms) to wait for a single file to parse in the worker thread.
+ * If tree-sitter hangs or WASM runs out of memory, this prevents the entire
+ * indexing run from freezing. The worker is restarted after a (hard) timeout.
+ * Env-overridable via CODEGRAPH_PARSE_TIMEOUT_MS for slow storage (#1231).
+ */
+const PARSE_TIMEOUT_MS = (0, parse_pool_1.resolveParseTimeoutMs)(process.env.CODEGRAPH_PARSE_TIMEOUT_MS);
+/**
+ * Number of files to parse before recycling the worker thread.
+ * WASM linear memory can grow but NEVER shrink (WebAssembly spec limitation).
+ * The only way to reclaim tree-sitter's WASM heap is to destroy the entire
+ * V8 isolate by terminating the worker thread and spawning a fresh one.
+ * This interval balances memory usage against the cost of reloading grammars.
+ */
+const WORKER_RECYCLE_INTERVAL = 250;
+/**
+ * Calculate SHA256 hash of file contents
+ */
+function hashContent(content) {
+    return crypto.createHash('sha256').update(content).digest('hex');
+}
+/**
+ * Skip files larger than this (bytes). Generated bundles, minified JS, and
+ * vendored blobs blow the WASM heap and the worker-recycle budget for no useful
+ * symbols. 1 MB covers essentially all hand-written source.
+ */
+const MAX_FILE_SIZE = 1024 * 1024;
+/**
+ * Directory names that are dependency, build, cache, or tooling output across the
+ * languages/frameworks CodeGraph supports — curated from the canonical
+ * github/gitignore templates. Excluded by default so the graph reflects your code,
+ * not third-party noise, without requiring a `.gitignore` (issue #407). The
+ * exclusion applies uniformly (git or not, tracked or not); the only opt-in is an
+ * explicit `.gitignore` negation (e.g. `!vendor/`). First-party-prone or generic
+ * names (`packages`, `lib`, `app`, `bin`, `src`, `deps`, `env`, `tmp`, `storage`,
+ * `Library`) are deliberately NOT listed, to avoid ever hiding real source.
+ *
+ * Only dirs that actually contain *indexable source* (or are enormous) earn a slot
+ * — IDE/state dirs like `.idea`/`.vs` are omitted because CodeGraph indexes only
+ * recognized source extensions, so they produce no symbols regardless.
+ */
+const DEFAULT_IGNORE_DIRS = new Set([
+    // JS / TS — dependency directories
+    'node_modules', 'bower_components', 'jspm_packages', 'web_modules',
+    '.yarn', '.pnpm-store',
+    // JS / TS — framework & bundler build / cache / deploy output
+    '.next', '.nuxt', '.svelte-kit', '.turbo', '.vite', '.parcel-cache', '.angular',
+    '.docusaurus', 'storybook-static', '.vinxi', '.nitro', 'out-tsc',
+    '.vercel', '.netlify', '.wrangler',
+    // Build output (common across ecosystems)
+    'dist', 'build', 'out', '.output',
+    // Test / coverage
+    'coverage', '.nyc_output',
+    // Python
+    '__pycache__', '__pypackages__', '.venv', 'venv', '.pixi', '.pdm-build',
+    '.mypy_cache', '.pytest_cache', '.ruff_cache', '.tox', '.nox', '.hypothesis',
+    '.ipynb_checkpoints', '.eggs',
+    // Rust / JVM (Maven, Gradle, Scala)
+    'target', '.gradle',
+    // .NET
+    'obj',
+    // Vendored deps (Go, PHP/Composer, Ruby/Bundler)
+    'vendor',
+    // Swift / iOS
+    '.build', 'Pods', 'Carthage', 'DerivedData', '.swiftpm',
+    // Dart / Flutter
+    '.dart_tool', '.pub-cache',
+    // Native (Android NDK, C/C++ deps)
+    '.cxx', '.externalNativeBuild', 'vcpkg_installed',
+    // Scala tooling
+    '.bloop', '.metals',
+    // Lua / Luau (LuaRocks)
+    'lua_modules', '.luarocks',
+    // Delphi / RAD Studio IDE backups (duplicate .pas source — would double-count)
+    '__history', '__recovery',
+    // Generic cache
+    '.cache',
+]);
+/**
+ * Android resource directory types. A `res/` tree holds ONLY non-code resources —
+ * layouts, drawables, value bags (strings/colors/styles), menus, navigation
+ * graphs — split into one typed subdirectory per kind, optionally density/locale/
+ * version-qualified (`values-es`, `drawable-hdpi`, `layout-v21`, …). None of it
+ * yields an extractable code symbol, yet on an Android app it DOMINATES the tree
+ * (one report: 26k XML files = 97% of the project, 0 symbols), bloating the DB,
+ * slowing indexing, and skewing both the file count and `codegraph_explore`
+ * results (#1047). So these are excluded by default. The structure is
+ * self-identifying — a non-Android project has no `res/layout/` etc., so it's
+ * untouched — and the only XML that DOES produce symbols (MyBatis mappers) lives
+ * under `src/main/resources/`, never `res/`, so nothing useful is dropped.
+ * `res/raw/` is deliberately NOT here: it holds arbitrary bundled assets that can
+ * be code-ish (a `.sql` schema, a `.js`), so we leave it indexed. Override any of
+ * these with a `.gitignore` negation (e.g. `!res/values/`).
+ */
+const ANDROID_RES_TYPES = [
+    'anim', 'animator', 'color', 'drawable', 'font', 'layout',
+    'menu', 'mipmap', 'navigation', 'transition', 'values', 'xml',
+];
+/** Gitignore-style patterns for the `ignore` matcher: the dirs above plus a few globs. */
+const DEFAULT_IGNORE_PATTERNS = [
+    ...Array.from(DEFAULT_IGNORE_DIRS, (d) => `${d}/`),
+    '*.egg-info/', // Python packaging metadata
+    'cmake-build-*/', // CLion / CMake build trees
+    'bazel-*/', // Bazel output symlink trees
+    // Android resource dirs at any depth, with their qualifier variants (#1047).
+    ...ANDROID_RES_TYPES.map((t) => `**/res/${t}*/`),
+];
+/** True if `buf` decodes as strict UTF-8 (no invalid byte sequences). */
+function isValidUtf8(buf) {
+    try {
+        new TextDecoder('utf-8', { fatal: true }).decode(buf);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Read a `.gitignore` and return patterns safe to hand to the `ignore` matcher —
+ * never throwing, even when the file isn't real gitignore text. Two failure
+ * modes, both seen in the wild (issue #682):
+ *
+ *  - The file isn't valid UTF-8 — e.g. transparently encrypted in place by
+ *    corporate DLP / endpoint-security software, leaving a UTF-16 header plus
+ *    ciphertext. None of it is meaningful patterns, so the whole file is skipped.
+ *  - The file is text but a single line can't be compiled to a regex by the
+ *    `ignore` library — `\\[` and friends throw "Unterminated character class".
+ *    Crucially the throw is LAZY (at match time, not `.add()`), so it would
+ *    otherwise escape mid-scan. That one pattern is dropped; the rest are kept.
+ *
+ * Either way a warning that NAMES the file is logged (the reporter couldn't tell
+ * which `.gitignore` was at fault) and indexing continues instead of aborting.
+ * Returns '' when there's nothing usable.
+ */
+function readGitignorePatterns(giPath) {
+    let buf;
+    try {
+        buf = fs.readFileSync(giPath);
+    }
+    catch {
+        return ''; // unreadable (permissions / race) — treat as absent
+    }
+    // A NUL byte never appears in real gitignore text, and a fatal UTF-8 decode
+    // catches the rest. Such a file isn't ignore patterns at all.
+    if (buf.includes(0) || !isValidUtf8(buf)) {
+        (0, errors_1.logWarn)('Ignoring a .gitignore that is not valid UTF-8 text — it may have been encrypted ' +
+            'in place by endpoint-security software. Indexing continues without it.', { file: giPath });
+        return '';
+    }
+    const content = buf.toString('utf-8');
+    // Fast path: one `.ignores()` call forces the library to compile EVERY rule,
+    // so if it doesn't throw, the whole file is safe to use verbatim.
+    try {
+        (0, ignore_1.default)().add(content).ignores('.codegraph-probe');
+        return content;
+    }
+    catch {
+        // Fall through: a line is uncompilable — keep the good ones, drop the bad.
+    }
+    const kept = [];
+    let dropped = 0;
+    for (const line of content.split(/\r?\n/)) {
+        try {
+            (0, ignore_1.default)().add(line).ignores('.codegraph-probe');
+            kept.push(line);
+        }
+        catch {
+            dropped++;
+        }
+    }
+    if (dropped > 0) {
+        (0, errors_1.logWarn)(`Skipped ${dropped} unparseable pattern(s) in a .gitignore; the rest are applied.`, { file: giPath });
+    }
+    return kept.join('\n');
+}
+/**
+ * An `ignore` matcher seeded with the built-in defaults, merged with the project's
+ * root .gitignore so a negation there (e.g. `!vendor/`) overrides a default. Shared
+ * by both enumeration paths so behavior is identical with or without git — and so
+ * the defaults apply to tracked files too (committing a dependency dir doesn't make
+ * it project code; the explicit `.gitignore` negation is the only opt-in).
+ */
+function buildDefaultIgnore(rootDir) {
+    const ig = (0, ignore_1.default)().add(DEFAULT_IGNORE_PATTERNS);
+    const rootGitignore = path.join(rootDir, '.gitignore');
+    if (fs.existsSync(rootGitignore))
+        ig.add(readGitignorePatterns(rootGitignore));
+    return ig;
+}
+/**
+ * Defaults-only ignore matcher (no root `.gitignore` merged). Used wherever the
+ * parent repo's own ignore rules must NOT apply — inside embedded child repos,
+ * whose gitignore semantics their own `git ls-files` already enforced (#514).
+ */
+function defaultsOnlyIgnore() {
+    return (0, ignore_1.default)().add(DEFAULT_IGNORE_PATTERNS);
+}
+/**
+ * Matcher for the project's `codegraph.json` `includeIgnored` patterns — the
+ * explicit opt-in to index embedded git repos living inside gitignored
+ * directories (#622, #699). Returns `null` when the project opted in nothing,
+ * which is the zero-config DEFAULT: `.gitignore` is then fully respected and a
+ * gitignored directory (even one holding nested repos) is never walked or
+ * indexed (#970, #976). Built once per scan/sync/scope operation from the scan
+ * root and threaded down — never global, so multi-project daemons stay isolated.
+ */
+function loadIncludeIgnoredMatcher(rootDir) {
+    const patterns = (0, project_config_1.loadIncludeIgnoredPatterns)(rootDir);
+    return patterns.length > 0 ? (0, ignore_1.default)().add(patterns) : null;
+}
+/**
+ * Matcher for the project's `codegraph.json` `exclude` patterns — paths to keep
+ * OUT of the index even when git-tracked, which `.gitignore` cannot do (#999).
+ * The escape hatch for a committed vendor/theme/SDK directory. Returns `null`
+ * when nothing is excluded (the zero-config default → no overhead). Matched
+ * against project-root-relative paths, so it applies uniformly across the whole
+ * workspace, including inside embedded repos (excluding `static/` means gone
+ * everywhere). Built once per scan/sync/scope operation from the scan root.
+ */
+function loadExcludeMatcher(rootDir) {
+    const patterns = (0, project_config_1.loadExcludePatterns)(rootDir);
+    return patterns.length > 0 ? (0, ignore_1.default)().add(patterns) : null;
+}
+/**
+ * Matcher for the project's `codegraph.json` `include` patterns — first-party
+ * source to force INTO the index even when `.gitignore` drops it (the general
+ * whitelist `includeIgnored` never was — that one only revives *embedded git
+ * repos*). The case it exists for: a project under a second VCS (SVN/Perforce)
+ * `.gitignore`s its own real source so it stays out of Git, yet we still want it
+ * indexed. Returns `null` when nothing is force-included (the zero-config
+ * default → no overhead, no extra walk). Built once per scan/sync/scope
+ * operation from the scan root.
+ */
+function loadIncludeMatcher(rootDir) {
+    const patterns = (0, project_config_1.loadIncludePatterns)(rootDir);
+    return patterns.length > 0 ? (0, ignore_1.default)().add(patterns) : null;
+}
+/** Glob metacharacters that end the static (literal) prefix of an `include` pattern. */
+const GLOB_META = /[*?[\]{}!]/;
+/**
+ * The static directory prefix of each `include` pattern — the literal leading
+ * path up to the first glob segment — trailing-slashed, used to (a) walk only
+ * the opted-in subtrees in `collectIncludedFiles` and (b) let `ScopeIgnore` keep
+ * the watcher descending toward them. `Tools/` stays `Tools/`; a recursive
+ * `Tools/**` glob yields `Tools/`; `src/local/file.ts` yields `src/local/` (the
+ * file's dir); a pattern that starts with a glob (like a leading `**`) yields
+ * `''`, meaning "no static root — walk the whole tree". Duplicates and roots
+ * nested under a broader root are collapsed so each subtree is walked once.
+ */
+function includeStaticRoots(patterns) {
+    const roots = new Set();
+    for (const pattern of patterns) {
+        let p = pattern.replace(/^\/+/, '');
+        const trailingSlash = p.endsWith('/');
+        if (trailingSlash)
+            p = p.slice(0, -1);
+        const segs = p.split('/').filter(Boolean);
+        const lead = [];
+        for (const s of segs) {
+            if (GLOB_META.test(s))
+                break;
+            lead.push(s);
+        }
+        const hadWildcard = lead.length < segs.length;
+        // A wholly-literal pattern with no trailing slash names a file (or a dir we
+        // can't tell apart) — drop its last segment so we walk the containing dir
+        // and let the matcher pick the file. A trailing slash or a glob means the
+        // remaining `lead` is already the directory to walk.
+        if (!hadWildcard && !trailingSlash && lead.length > 0)
+            lead.pop();
+        if (lead.length === 0) {
+            roots.clear();
+            roots.add('');
+            return ['']; // a top-level glob forces a whole-tree walk; nothing narrower matters
+        }
+        roots.add(lead.join('/') + '/');
+    }
+    // Collapse roots nested under a broader one (e.g. drop `a/b/` if `a/` is present).
+    const all = [...roots];
+    return all.filter((r) => !all.some((other) => other !== r && r.startsWith(other)));
+}
+/**
+ * Actively discover the source files an `include` whitelist forces in. `git
+ * ls-files` never lists gitignored files, so a filtered filesystem walk of just
+ * the opted-in subtrees (`includeStaticRoots`) is the only way to find them.
+ * Returns project-root-relative, normalized source-file paths.
+ *
+ * A file is collected when it MATCHES `include`, is NOT hit by `exclude` (an
+ * explicit exclude always wins), is a recognized source file, and does not live
+ * under a built-in default-ignored dir (`node_modules`, `dist`, …), `.git`, or
+ * CodeGraph's data dir — those are never resurfaced, mirroring `ScopeIgnore`.
+ * `.gitignore` is deliberately NOT consulted: overriding it is the whole point.
+ */
+function collectIncludedFiles(rootDir, include, exclude, roots, overrides) {
+    const out = new Set();
+    const defaults = defaultsOnlyIgnore();
+    const visited = new Set();
+    const consider = (abs, rel, isDir) => {
+        if (isDir) {
+            if (defaults.ignores(rel + '/'))
+                return; // never node_modules/dist/… via include
+            // An explicit `exclude` always wins over `include`; prune the whole subtree
+            // here so a large excluded dir (a committed frontend's own vendored deps,
+            // build output, …) is never walked — the per-file guard below still catches
+            // anything a directory pattern doesn't, so this is a pure efficiency win.
+            if (exclude && exclude.ignores(rel + '/'))
+                return;
+            walk(abs);
+        }
+        else {
+            if (defaults.ignores(rel))
+                return;
+            if (!include.ignores(rel))
+                return;
+            if (exclude && exclude.ignores(rel))
+                return;
+            if (!(0, grammars_1.isSourceFile)(rel, overrides))
+                return;
+            out.add(rel);
+        }
+    };
+    function walk(absDir) {
+        let realDir;
+        try {
+            realDir = fs.realpathSync(absDir);
+        }
+        catch {
+            return;
+        }
+        if (visited.has(realDir))
+            return; // symlink-cycle guard
+        visited.add(realDir);
+        let entries;
+        try {
+            entries = fs.readdirSync(absDir, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        for (const entry of entries) {
+            if (entry.name === '.git' || (0, directory_1.isCodeGraphDataDir)(entry.name))
+                continue;
+            const abs = path.join(absDir, entry.name);
+            const rel = (0, utils_1.normalizePath)(path.relative(rootDir, abs));
+            if (!rel || rel.startsWith('..'))
+                continue;
+            if (entry.isSymbolicLink()) {
+                try {
+                    const st = fs.statSync(fs.realpathSync(abs));
+                    consider(abs, rel, st.isDirectory());
+                }
+                catch {
+                    // broken symlink — skip
+                }
+                continue;
+            }
+            consider(abs, rel, entry.isDirectory());
+        }
+    }
+    for (const root of roots) {
+        walk(root === '' ? rootDir : path.join(rootDir, root));
+    }
+    return out;
+}
+/**
+ * The included source files (`codegraph.json` `include`) for a scan root, or an
+ * empty set when nothing is force-included. Centralizes loading the matcher,
+ * roots, exclude, and overrides so both enumeration paths (git and filesystem
+ * walk) add the same files.
+ */
+function collectIncludedFilesForRoot(rootDir) {
+    const include = loadIncludeMatcher(rootDir);
+    if (!include)
+        return new Set();
+    const roots = includeStaticRoots((0, project_config_1.loadIncludePatterns)(rootDir));
+    return collectIncludedFiles(rootDir, include, loadExcludeMatcher(rootDir), roots, (0, project_config_1.loadExtensionOverrides)(rootDir));
+}
+/**
+ * `git ls-files --directory` collapses a wholly-untracked/ignored directory into
+ * one entry — and when the command's own cwd is such a directory (the indexed
+ * root is itself a git-ignored subdir of an enclosing repo), git emits the
+ * literal `./` meaning "this entire directory". That sentinel is not a real
+ * nested path: feeding it to the `ignore` matcher throws ("path should be a
+ * `path.relative()`d string, but got "./""), which used to abort `buildScopeIgnore`
+ * and so break the MCP daemon's watcher/auto-sync on connect; and joining it back
+ * onto `repoDir` would just re-point at the cwd. Drop it wherever we consume
+ * `--directory` output. (#936)
+ */
+function isWholeCwdEntry(entry) {
+    return entry === './' || entry === '.' || entry === '';
+}
+/**
+ * List the gitignored DIRECTORIES of a repo (collapsed, trailing-slash form),
+ * relative to `repoDir`. These are invisible to every other `git ls-files` /
+ * `git status` mode — and in a multi-repo workspace they are exactly where the
+ * nested project repos live (a super-repo `.gitignore`s its child repos to keep
+ * `git status` quiet; that does not make them third-party code). (#514)
+ */
+function listIgnoredDirs(repoDir) {
+    try {
+        const out = (0, child_process_1.execFileSync)('git', ['ls-files', '-z', '-o', '-i', '--exclude-standard', '--directory'], { cwd: repoDir, encoding: 'utf-8', timeout: 30000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true });
+        return out.split('\0').filter((e) => e.endsWith('/') && !isWholeCwdEntry(e));
+    }
+    catch {
+        return [];
+    }
+}
+/** Max directory depth searched below an ignored dir for nested `.git` roots. */
+const EMBEDDED_REPO_SEARCH_DEPTH = 4;
+/** Max directories examined per search — a huge ignored data dir must never stall a scan/sync. */
+const EMBEDDED_REPO_SEARCH_ENTRIES = 2000;
+/**
+ * Classify a directory's `.git` entry for embedded-repo discovery.
+ *
+ * - A `.git` **directory** is an embedded clone — distinct first-party code a
+ *   super-repo merely hides from git; index it (#193, #514).
+ * - A `.git` **file** is a pointer (`gitdir: …`). A git **worktree** points into
+ *   the host repo's own `.git/worktrees/<name>`, so it is a second working view
+ *   of a repo CodeGraph already indexes — indexing it just duplicates the whole
+ *   graph N times; skip it (#848). A **submodule worktree** points into
+ *   `.git/modules/<module>/worktrees/<name>` — same duplication, so skip it too
+ *   (#945). A **submodule** checkout points into `.git/modules/<module>` (no
+ *   `worktrees/` segment) and is distinct code, so index it as before.
+ *
+ * Returns `'none'` when there is no `.git` entry here.
+ */
+function classifyGitDir(absDir) {
+    let st;
+    try {
+        st = fs.statSync(path.join(absDir, '.git'));
+    }
+    catch {
+        return 'none';
+    }
+    if (st.isDirectory())
+        return 'embedded';
+    if (!st.isFile())
+        return 'none';
+    try {
+        const gitdir = fs.readFileSync(path.join(absDir, '.git'), 'utf8').match(/^gitdir:\s*(.+)$/m)?.[1]?.trim();
+        // A worktree's gitdir lives under some repo's `.git/worktrees/<name>` —
+        // either the top-level repo's (`.git/worktrees/`) or, for a worktree of a
+        // submodule, that submodule's gitdir (`.git/modules/<module>/worktrees/`).
+        // The optional `modules/<module>` segment covers the submodule case (#945).
+        // Match both separators so a Windows-style pointer is recognized too.
+        if (gitdir && /(^|[\\/])\.git[\\/](modules[\\/][^\\/]+[\\/])?worktrees[\\/]/.test(gitdir))
+            return 'worktree';
+    }
+    catch {
+        // Unreadable `.git` pointer — fall back to the prior "index it" behavior.
+    }
+    return 'embedded';
+}
+/**
+ * Find git repositories nested under `absDir` (inclusive), shallow bounded BFS.
+ * Stops descending at each repo root found — contents belong to that repo's own
+ * enumeration. Skips default-ignored dirs (`node_modules` can contain `.git`
+ * from npm git-dependencies — that never makes it project code) and CodeGraph
+ * data dirs. Depth- and entry-capped so a huge ignored tree can't stall the scan.
+ */
+function findNestedGitRepos(absDir, relPrefix) {
+    const found = [];
+    const defaults = defaultsOnlyIgnore();
+    const queue = [
+        { abs: absDir, rel: relPrefix, depth: 0 },
+    ];
+    let examined = 0;
+    while (queue.length > 0) {
+        const { abs, rel, depth } = queue.shift();
+        if (++examined > EMBEDDED_REPO_SEARCH_ENTRIES) {
+            (0, errors_1.logDebug)('Embedded-repo search entry cap hit — deeper repos (if any) not discovered', { under: relPrefix });
+            break;
+        }
+        const cls = classifyGitDir(abs);
+        if (cls === 'worktree') {
+            continue; // a git worktree duplicates an already-indexed repo (#848) — skip
+        }
+        if (cls === 'embedded') {
+            found.push(rel);
+            continue; // its own git handles everything below
+        }
+        if (depth >= EMBEDDED_REPO_SEARCH_DEPTH)
+            continue;
+        let entries;
+        try {
+            entries = fs.readdirSync(abs, { withFileTypes: true });
+        }
+        catch {
+            continue;
+        }
+        for (const entry of entries) {
+            if (!entry.isDirectory())
+                continue;
+            if (entry.name === '.git' || (0, directory_1.isCodeGraphDataDir)(entry.name))
+                continue;
+            const childRel = rel + entry.name + '/';
+            if (defaults.ignores(childRel))
+                continue;
+            queue.push({ abs: path.join(abs, entry.name), rel: childRel, depth: depth + 1 });
+        }
+    }
+    return found;
+}
+/**
+ * Workspace-scope ignore matcher. Ordinary paths get the root's matcher
+ * (built-in defaults + root `.gitignore`); paths inside an EMBEDDED repo get
+ * that repo's own matcher (defaults + its root `.gitignore`) — the parent's
+ * `.gitignore` hides a child repo from git, not from the index (#514). A
+ * directory path (trailing slash) that is an ANCESTOR of an embedded root is
+ * never ignored, so directory-pruning callers (the Linux per-directory
+ * watcher) still descend to reach the embedded repos.
+ *
+ * Single source of truth for indexer and watcher scope — they must not diverge.
+ */
+class ScopeIgnore {
+    rootMatcher;
+    exclude;
+    include;
+    includeRoots;
+    embedded;
+    defaults = defaultsOnlyIgnore();
+    constructor(rootMatcher, embedded, 
+    /**
+     * Project `codegraph.json` `exclude` patterns (#999), matched against the
+     * full root-relative path. Wins over everything else — an explicit user
+     * exclude applies even to tracked files and even inside embedded repos.
+     */
+    exclude = null, 
+    /**
+     * Project `codegraph.json` `include` patterns — first-party source forced
+     * INTO the index despite `.gitignore`. When a path matches, it is NOT
+     * ignored (so the watcher watches it), overriding `.gitignore`/`rootMatcher`
+     * — but never `exclude` (checked first) and never a built-in default-ignored
+     * dir. `includeRoots` are the static prefixes so a gitignored ANCESTOR
+     * directory of an included subtree still isn't pruned by the directory
+     * walker/watcher.
+     */
+    include = null, includeRoots = []) {
+        this.rootMatcher = rootMatcher;
+        this.exclude = exclude;
+        this.include = include;
+        this.includeRoots = includeRoots;
+        // Longest root first so paths in nested embedded repos hit the innermost matcher.
+        this.embedded = [...embedded].sort((a, b) => b.root.length - a.root.length);
+    }
+    ignores(rel) {
+        // User `exclude` (#999) is checked first and against the full root-relative
+        // path: it must drop git-TRACKED paths (which `.gitignore` can't) and apply
+        // everywhere, including ancestors of embedded repos.
+        if (this.exclude && this.exclude.ignores(rel))
+            return true;
+        // User `include`: force first-party source in despite `.gitignore`. Never
+        // resurfaces a built-in default-ignored dir (node_modules/dist/…), so an
+        // include pattern can't accidentally pull in dependency/build trees.
+        if (this.include && !this.defaults.ignores(rel)) {
+            if (rel.endsWith('/')) {
+                // A directory on (or leading to) an included subtree must stay walkable
+                // so the watcher/walker descends to reach the forced-in files.
+                if (this.includeRoots.some((r) => r.startsWith(rel) || rel.startsWith(r)))
+                    return false;
+            }
+            else if (this.include.ignores(rel)) {
+                return false;
+            }
+        }
+        for (const { root, matcher } of this.embedded) {
+            if (rel.startsWith(root)) {
+                const inner = rel.slice(root.length);
+                if (inner === '')
+                    return false;
+                // Built-in defaults apply to the FULL path uniformly (#407) — an
+                // embedded repo inside node_modules (an npm git-dependency) must stay
+                // excluded even though its own rules wouldn't ignore its files.
+                return this.defaults.ignores(rel) || matcher.ignores(inner);
+            }
+        }
+        // Never prune a directory that leads to an embedded repo.
+        if (rel.endsWith('/') && this.embedded.some(({ root }) => root.startsWith(rel))) {
+            return false;
+        }
+        return this.rootMatcher.ignores(rel);
+    }
+}
+exports.ScopeIgnore = ScopeIgnore;
+/**
+ * Build the workspace-scope matcher. When the caller already knows the
+ * embedded roots (the scanner discovers them during collection), pass them to
+ * skip rediscovery; otherwise they're discovered here (the watcher path).
+ */
+function buildScopeIgnore(rootDir, embeddedRoots) {
+    const roots = embeddedRoots ? [...embeddedRoots] : discoverEmbeddedRepoRoots(rootDir);
+    const include = loadIncludeMatcher(rootDir);
+    return new ScopeIgnore(buildDefaultIgnore(rootDir), roots.map((root) => ({ root, matcher: buildDefaultIgnore(path.join(rootDir, root)) })), loadExcludeMatcher(rootDir), include, include ? includeStaticRoots((0, project_config_1.loadIncludePatterns)(rootDir)) : []);
+}
+/**
+ * Whether an embedded repo found as a tracked gitlink (mode 160000, #1031/#1033)
+ * must be SKIPPED rather than indexed. A gitlink is tracked, so `.gitignore`
+ * can't untrack it — but the discovery passes for it must still honor the same
+ * scope rules as every other path, or a gitignored reference/data dir full of
+ * `git add`ed clones gets pulled into the index against the user's stated intent
+ * (#1065). Two reasons to skip:
+ *   1. It sits in a built-in default-ignored location — an npm git-dependency
+ *      under `node_modules` is never project code; not even an explicit opt-in
+ *      revives it (matches `findIgnoredEmbeddedRepos`).
+ *   2. The parent repo's own `.gitignore` covers its path and the project did
+ *      NOT opt that path in via `codegraph.json` `includeIgnored`. The gitignore
+ *      rule is the user's stated intent to keep that path out of scope, exactly
+ *      as for an UNtracked embedded repo — respect it by default, opt back in
+ *      with `includeIgnored` (#514, #970, #976).
+ * `relDir` is repoDir-relative (trailing-slashed); `prefix` is repoDir's
+ * scan-root-relative path so the `includeIgnored` pattern is matched on the full
+ * scan-root-relative path. `defaults` is `defaultsOnlyIgnore()` and `repoIgnore`
+ * is `buildDefaultIgnore(repoDir)` (defaults + the repo's own `.gitignore`),
+ * both passed in so they're built once per repo level rather than per gitlink.
+ */
+function gitlinkEmbeddedRepoSkipped(relDir, prefix, defaults, repoIgnore, includeIgnored) {
+    if (defaults.ignores(relDir))
+        return true; // default-ignored — never index, opt-in can't revive
+    if (!repoIgnore.ignores(relDir))
+        return false; // not ignored at all — index as before (#1031/#1033)
+    // Gitignored by the repo's own rules — skip unless the project opted it in.
+    return !includeIgnored?.ignores((0, utils_1.normalizePath)(prefix + relDir));
+}
+/**
+ * Standalone discovery of every embedded repo root under `rootDir` (relative,
+ * trailing-slashed) — the untracked kind (#193) always, and the gitignored kind
+ * (#514) only for directories the project opted in via `codegraph.json`
+ * `includeIgnored` (#622, #699); otherwise `.gitignore` is respected and they
+ * are not discovered (#970, #976). Recursive (an embedded repo can embed further
+ * repos). Returns [] for non-git roots: the filesystem walk handles nested repos
+ * there already.
+ */
+function discoverEmbeddedRepoRoots(rootDir) {
+    try {
+        (0, child_process_1.execFileSync)('git', ['rev-parse', '--git-dir'], { cwd: rootDir, encoding: 'utf-8', timeout: 5000, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true });
+    }
+    catch {
+        return [];
+    }
+    const out = [];
+    const defaults = defaultsOnlyIgnore();
+    const includeIgnored = loadIncludeIgnoredMatcher(rootDir);
+    const visit = (repoAbs, prefix) => {
+        const candidates = [];
+        try {
+            const o = (0, child_process_1.execFileSync)('git', ['ls-files', '-z', '-o', '--exclude-standard', '--directory'], { cwd: repoAbs, encoding: 'utf-8', timeout: 30000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true });
+            for (const e of o.split('\0')) {
+                if (e.endsWith('/') && !isWholeCwdEntry(e) && !defaults.ignores(e)) {
+                    candidates.push(...findNestedGitRepos(path.join(repoAbs, e), e));
+                }
+            }
+        }
+        catch { /* untracked listing failed — ignored-side discovery still runs */ }
+        // Unexpanded gitlinks (mode 160000) with a real checkout on disk — embedded
+        // repos `git add`ed without `.gitmodules`, or submodules not active here. The
+        // untracked listing above can't see them (they're tracked), so find them the
+        // same way collectGitFiles does, keeping watcher scope == indexer scope.
+        // (#1031, #1033)
+        try {
+            const staged = (0, child_process_1.execFileSync)('git', ['ls-files', '-z', '-s', '--recurse-submodules'], { cwd: repoAbs, encoding: 'utf-8', timeout: 30000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true });
+            const repoIgnore = buildDefaultIgnore(repoAbs);
+            for (const entry of staged.split('\0')) {
+                if (!entry || entry.slice(0, 6) !== '160000')
+                    continue;
+                const tab = entry.indexOf('\t');
+                if (tab === -1)
+                    continue;
+                const rel = entry.slice(tab + 1);
+                const relDir = rel.endsWith('/') ? rel : rel + '/';
+                // A gitlink under a gitignored path is respected (not indexed) unless the
+                // project opted it in — same rule as the untracked-ignored kind (#1065).
+                if (gitlinkEmbeddedRepoSkipped(relDir, prefix, defaults, repoIgnore, includeIgnored))
+                    continue;
+                if (classifyGitDir(path.join(repoAbs, rel)) === 'embedded')
+                    candidates.push(relDir);
+            }
+        }
+        catch { /* staged listing failed — other discovery still runs */ }
+        candidates.push(...findIgnoredEmbeddedRepos(repoAbs, includeIgnored, prefix));
+        for (const rel of candidates) {
+            const full = (0, utils_1.normalizePath)(prefix + rel);
+            out.push(full);
+            visit(path.join(repoAbs, rel), full);
+        }
+    };
+    visit(rootDir, '');
+    return out;
+}
+/**
+ * Cap on how many skipped gitignored repos the CLI hint enumerates — a huge
+ * gitignored data dir full of clones must never turn the hint scan into a long
+ * walk. Enough to make the point; the caller says "+N more" past this.
+ */
+const UNINDEXED_IGNORED_REPO_HINT_CAP = 100;
+/**
+ * The INVERSE of the gitignored side of {@link discoverEmbeddedRepoRoots}:
+ * nested git repositories under a gitignored directory that the project has NOT
+ * opted into via `codegraph.json` `includeIgnored`. These are real repos the
+ * default `init`/`index` deliberately skips because `.gitignore` excludes them
+ * (#970, #976) — most visibly the "super-repo `.gitignore`s its child repos"
+ * layout (#1156), where `init` at the parent correctly indexes ~nothing while
+ * `init` inside each child works. The CLI uses this to turn that silent empty
+ * index into an actionable hint: it names the skipped repos and offers to opt
+ * them in. Paths are `rootDir`-relative and trailing-slashed (valid
+ * `includeIgnored` patterns as-is). Returns `[]` for a non-git root (a
+ * filesystem walk already descends into nested repos there), skips built-in
+ * default-ignored dirs (`node_modules`, …), and is bounded so it never stalls
+ * on a giant ignored tree.
+ */
+function findUnindexedIgnoredRepos(rootDir) {
+    try {
+        (0, child_process_1.execFileSync)('git', ['rev-parse', '--git-dir'], { cwd: rootDir, encoding: 'utf-8', timeout: 5000, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true });
+    }
+    catch {
+        return [];
+    }
+    const defaults = defaultsOnlyIgnore();
+    const includeIgnored = loadIncludeIgnoredMatcher(rootDir);
+    const repos = [];
+    for (const dir of listIgnoredDirs(rootDir)) {
+        if (defaults.ignores(dir))
+            continue; // node_modules etc. — never project code
+        if (includeIgnored?.ignores((0, utils_1.normalizePath)(dir)))
+            continue; // already opted in — nothing to nag about
+        for (const repo of findNestedGitRepos(path.join(rootDir, dir), dir)) {
+            // Per-repo opt-in check, mirroring findIgnoredEmbeddedRepos: a child
+            // pattern (`repos/a/`) doesn't match the parent dir above but DOES
+            // cover this repo — it's indexed, so don't nag about it (#1295).
+            if (includeIgnored?.ignores((0, utils_1.normalizePath)(repo)))
+                continue;
+            repos.push(repo);
+            if (repos.length >= UNINDEXED_IGNORED_REPO_HINT_CAP)
+                return repos;
+        }
+    }
+    return repos;
+}
+/**
+ * Discover embedded repos hidden by `repoDir`'s OWN gitignore rules: for each
+ * gitignored directory, search for nested `.git` roots. Returns repo paths
+ * relative to `repoDir`, trailing-slashed.
+ *
+ * OPT-IN ONLY. Walking into a gitignored directory contradicts what every other
+ * tool (and CodeGraph's own `git ls-files` foundation) does — `.gitignore`
+ * excludes. So this returns `[]` unless the project opted the directory in via
+ * `codegraph.json` `includeIgnored`; without that, a gitignored dir — including
+ * a huge reference/data dir full of nested clones — is left untouched (#970,
+ * #976). When opted in, it restores the super-repo-of-clones behavior (#622,
+ * #699). `prefix` is the scan-root-relative path of `repoDir`, so a pattern like
+ * `services/` opts that whole subtree in at any recursion depth. Built-in
+ * default excludes (`node_modules`, …) are always skipped.
+ */
+function findIgnoredEmbeddedRepos(repoDir, includeIgnored, prefix) {
+    if (!includeIgnored)
+        return [];
+    const defaults = defaultsOnlyIgnore();
+    const repos = [];
+    for (const dir of listIgnoredDirs(repoDir)) {
+        if (defaults.ignores(dir))
+            continue;
+        const nested = findNestedGitRepos(path.join(repoDir, dir), dir);
+        if (includeIgnored.ignores((0, utils_1.normalizePath)(prefix + dir))) {
+            // The whole ignored dir is opted in — every nested repo under it counts.
+            repos.push(...nested);
+        }
+        else {
+            // A single gitignore rule often covers the PARENT of the opted-in
+            // repos: `.gitignore: /repos/` lists `repos/` as ONE ignored entry,
+            // while `includeIgnored: ["repos/a/"]` (the CLI hint's own suggested
+            // spelling) names the child — which never matches the parent path, so
+            // the opt-in silently did nothing (#1295). Match each nested repo
+            // root individually so both spellings work. The walk is bounded
+            // (depth/entry caps in findNestedGitRepos) and only runs when
+            // includeIgnored is configured at all.
+            repos.push(...nested.filter((r) => includeIgnored.ignores((0, utils_1.normalizePath)(prefix + r))));
+        }
+    }
+    return repos;
+}
+/**
+ * Collect git-visible files (tracked + untracked, .gitignore-respected) from the
+ * git repository rooted at `repoDir`, adding each to `files` with `prefix`
+ * prepended so paths stay relative to the original scan root.
+ *
+ * Recurses into embedded git repositories — nested repos that are NOT submodules
+ * (independent clones living inside the workspace, common in CMake "super-repo"
+ * layouts). The parent repo's `git ls-files` cannot see into them: tracked output
+ * skips them entirely, and untracked output reports them only as an opaque
+ * "subdir/" entry (trailing slash) rather than expanding their files. Each
+ * embedded repo is its own git boundary, so we re-run `git ls-files` inside it.
+ * (See issue #193.) GITIGNORED embedded repos are invisible even to that; they
+ * are discovered separately via `findIgnoredEmbeddedRepos` (#514) but ONLY for
+ * directories the project opted in through `codegraph.json` `includeIgnored`
+ * (`includeIgnored` here, threaded from the scan root) — by default `.gitignore`
+ * is respected and they stay out (#970, #976). Every embedded repo root (however
+ * found) is recorded in `embeddedRoots` so callers can exempt its files from the
+ * parent's own gitignore rules.
+ */
+function collectGitFiles(repoDir, prefix, files, embeddedRoots, includeIgnored = null) {
+    const gitOpts = { cwd: repoDir, encoding: 'utf-8', timeout: 30000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true };
+    // Tracked files. --recurse-submodules pulls in files from active submodules,
+    // which the index would otherwise represent only as a commit pointer.
+    // Without this, monorepos using submodules index 0 files. (See issue #147.)
+    // Note: --recurse-submodules only supports -c/--cached and --stage modes — it
+    // can't be combined with -o, so untracked files are gathered separately below.
+    //
+    // We use --stage (-s) rather than -c so each entry carries its file mode. That
+    // lets us spot gitlink entries (mode 160000) that --recurse-submodules did NOT
+    // expand: a nested repo `git add`ed without a `.gitmodules` entry, or a
+    // submodule that isn't active/initialized in this checkout. Such a gitlink
+    // falls through every pass — it's tracked, so the untracked `-o` listing below
+    // never reports it, and --recurse-submodules only expands ACTIVE submodules —
+    // so its source would be silently skipped, leaving only the super-repo's own
+    // files indexed. We collect those gitlinks here and recurse into them below.
+    // (An active submodule is expanded inline by --recurse-submodules and so never
+    // surfaces as a 160000 entry — only the unhandled gitlinks do.) (#1031, #1033)
+    //
+    // -z gives NUL-separated, unquoted output so non-ASCII (e.g. CJK) paths
+    // survive verbatim. Without it git octal-escapes and double-quotes such paths
+    // (the core.quotepath default), and the quoted form never matches a real file
+    // on disk → those files are silently dropped from the index. (#541) With -s the
+    // path follows a TAB after the `<mode> <object> <stage>` prefix.
+    const gitlinkRels = [];
+    const tracked = (0, child_process_1.execFileSync)('git', ['ls-files', '-z', '-s', '--recurse-submodules'], gitOpts);
+    for (const entry of tracked.split('\0')) {
+        if (!entry)
+            continue;
+        const tab = entry.indexOf('\t');
+        if (tab === -1)
+            continue; // --stage always emits "<mode> <object> <stage>\t<path>"
+        const rel = entry.slice(tab + 1);
+        if (entry.slice(0, 6) === '160000') {
+            gitlinkRels.push(rel); // an unexpanded gitlink — recursed into below, not a source file itself
+            continue;
+        }
+        files.add((0, utils_1.normalizePath)(prefix + rel));
+    }
+    // Untracked files (submodules manage their own untracked state). Embedded git
+    // repos surface here as a single "subdir/" entry that git refuses to descend
+    // into — recurse into those as their own repos so their source gets indexed.
+    const untracked = (0, child_process_1.execFileSync)('git', ['ls-files', '-z', '-o', '--exclude-standard'], gitOpts);
+    for (const rel of untracked.split('\0')) {
+        if (!rel)
+            continue;
+        if (rel.endsWith('/')) {
+            // git only emits a trailing-slash directory entry for an embedded repo.
+            // Guard with a .git check anyway, and skip anything else exactly as git
+            // itself skips it (we never descend into a non-repo opaque dir). Never
+            // descend into default-ignored locations — an embedded repo inside
+            // node_modules is an npm git-dependency, not project code.
+            const childDir = path.join(repoDir, rel);
+            // A git worktree surfaces here as an opaque untracked dir too — skip it,
+            // it's a duplicate working view of an already-indexed repo (#848).
+            if (classifyGitDir(childDir) === 'embedded' && !defaultsOnlyIgnore().ignores(rel)) {
+                embeddedRoots?.add((0, utils_1.normalizePath)(prefix + rel));
+                collectGitFiles(childDir, prefix + rel, files, embeddedRoots, includeIgnored);
+            }
+            continue;
+        }
+        files.add((0, utils_1.normalizePath)(prefix + rel));
+    }
+    // Gitlink entries (mode 160000) that --recurse-submodules left unexpanded —
+    // an embedded repo `git add`ed without `.gitmodules`, or a submodule not
+    // active/initialized in this checkout. When such a gitlink has a real working
+    // tree on disk it is distinct first-party code we must index as its own
+    // embedded repo: the tracked pass skipped its contents and the untracked pass
+    // never sees it (it's tracked, not "other"). A gitlink with no checkout on disk
+    // (an uninitialized submodule — empty dir, no `.git`) has nothing to index and
+    // is left alone, as is a submodule worktree (a duplicate view, #945). (#1031, #1033)
+    if (gitlinkRels.length > 0) {
+        const defaults = defaultsOnlyIgnore();
+        const repoIgnore = buildDefaultIgnore(repoDir);
+        for (const rel of gitlinkRels) {
+            const relDir = rel.endsWith('/') ? rel : rel + '/';
+            // A gitlink under a gitignored path is respected (not indexed) unless the
+            // project opted it in via `includeIgnored` — keep tracked gitlinks under
+            // the same scope rule as the untracked-ignored kind below (#1065).
+            if (gitlinkEmbeddedRepoSkipped(relDir, prefix, defaults, repoIgnore, includeIgnored))
+                continue;
+            const childDir = path.join(repoDir, rel);
+            // 'embedded' = a real .git checkout on disk; 'worktree' and 'none' are skipped.
+            if (classifyGitDir(childDir) !== 'embedded')
+                continue;
+            embeddedRoots?.add((0, utils_1.normalizePath)(prefix + relDir));
+            collectGitFiles(childDir, prefix + relDir, files, embeddedRoots, includeIgnored);
+        }
+    }
+    // Embedded repos hidden by THIS repo's ignore rules (`/packages/` in a
+    // super-repo .gitignore) never appear in any listing above. By default they
+    // stay hidden — `.gitignore` is respected (#970, #976). They are recursed into
+    // only when the project opted the directory in via `codegraph.json`
+    // `includeIgnored` (#622, #699), which `findIgnoredEmbeddedRepos` enforces.
+    for (const rel of findIgnoredEmbeddedRepos(repoDir, includeIgnored, prefix)) {
+        embeddedRoots?.add((0, utils_1.normalizePath)(prefix + rel));
+        collectGitFiles(path.join(repoDir, rel), prefix + rel, files, embeddedRoots, includeIgnored);
+    }
+}
+/**
+ * Get all files visible to git (tracked + untracked but not ignored).
+ * Respects .gitignore at all levels (root, subdirectories) and descends into
+ * embedded (nested, non-submodule) git repos. Returns null on failure
+ * (non-git project) so callers can fall back to a filesystem walk.
+ */
+function getGitVisibleFiles(rootDir) {
+    try {
+        // Check if the project directory is gitignored by a parent repo.
+        // When rootDir lives inside a parent git repo that ignores it,
+        // `git ls-files` returns nothing — fall back to filesystem walk.
+        const gitRoot = (0, child_process_1.execFileSync)('git', ['rev-parse', '--show-toplevel'], { cwd: rootDir, encoding: 'utf-8', timeout: 5000, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true }).trim();
+        if (path.resolve(gitRoot) !== path.resolve(rootDir)) {
+            try {
+                // git check-ignore exits 0 if the path IS ignored, 1 if not
+                (0, child_process_1.execFileSync)('git', ['check-ignore', '-q', path.resolve(rootDir)], { cwd: rootDir, encoding: 'utf-8', timeout: 5000, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true });
+                // Directory is gitignored by parent repo — fall back to filesystem walk
+                return null;
+            }
+            catch {
+                // Not ignored — safe to use git ls-files
+            }
+        }
+        const files = new Set();
+        const embeddedRoots = new Set();
+        collectGitFiles(rootDir, '', files, embeddedRoots, loadIncludeIgnoredMatcher(rootDir));
+        // Apply built-in default ignores uniformly — to tracked files too, since
+        // committing a dependency/build dir doesn't make it project code. A
+        // `.gitignore` negation (e.g. `!vendor/`) is the explicit opt-in. (issue #407)
+        // Files inside an EMBEDDED repo are matched against that repo's own rules,
+        // not the parent's: the parent's .gitignore hides the child repo from git,
+        // not from the index. (#514)
+        const ig = buildScopeIgnore(rootDir, embeddedRoots);
+        const visible = new Set([...files].filter((f) => !ig.ignores(f)));
+        // Force-include first-party source the project whitelisted in
+        // `codegraph.json` `include`. These are gitignored, so `git ls-files` never
+        // listed them above — discover them directly off disk and add them. (The
+        // common SVN+Git dual-VCS case: source committed to SVN, gitignored out of
+        // Git, but still wanted in the graph.)
+        for (const f of collectIncludedFilesForRoot(rootDir))
+            visible.add(f);
+        return visible;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Use `git status` to detect changed files instead of scanning every file.
+ * Returns null on failure so callers fall back to full scan.
+ *
+ * Recurses into embedded repos — the untracked kind (#193: the parent's status
+ * collapses them to an opaque `?? subdir/` entry) always, and the gitignored
+ * kind (#514: they never appear in the parent's status at all) only for
+ * directories opted in via `codegraph.json` `includeIgnored` (#622, #699) —
+ * running `git status` inside each, so changes in a multi-repo workspace sync
+ * without a full rescan. By default a gitignored dir is left alone, matching the
+ * full-index scan (#970, #976). Deleting an ENTIRE embedded repo dir is the one
+ * case this cannot see (the child status that would report the deletions is gone
+ * with it); a full `codegraph index` reconciles that.
+ */
+function getGitChangedFiles(rootDir) {
+    try {
+        const changes = { modified: [], added: [], deleted: [] };
+        // Custom extension → language overrides from the project's codegraph.json,
+        // so change detection sees the same custom-extension files the full index does.
+        const overrides = (0, project_config_1.loadExtensionOverrides)(rootDir);
+        collectGitStatus(rootDir, '', changes, overrides, loadIncludeIgnoredMatcher(rootDir), loadExcludeMatcher(rootDir));
+        return changes;
+    }
+    catch {
+        return null;
+    }
+}
+function collectGitStatus(repoDir, prefix, out, overrides, includeIgnored = null, exclude = null) {
+    const output = (0, child_process_1.execFileSync)('git', ['status', '--porcelain', '--no-renames'], { cwd: repoDir, encoding: 'utf-8', timeout: 10000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true });
+    // This repo's own ignore rules — built-in defaults (#407) plus its .gitignore.
+    // Change detection must exclude the SAME files the full index does, but git
+    // status hides neither: it ignores nothing for *tracked* paths, and the
+    // built-in defaults aren't gitignore at all. Without this filter a committed
+    // vendor/ dir, or a tracked file under a .gitignored dir, surfaces here as a
+    // change — so `codegraph status` (which reads getChangedFiles) reports a
+    // pending edit the full index never tracks and `sync` never clears. Matching
+    // repo-relative `rel` at each recursion level mirrors getGitVisibleFiles'
+    // ScopeIgnore: every embedded repo is judged by ITS OWN rules, never the
+    // parent's. (#766)
+    const ig = buildDefaultIgnore(repoDir);
+    const untrackedDirs = [];
+    for (const line of output.split('\n')) {
+        if (line.length < 4)
+            continue; // Minimum: "XY file"
+        const statusCode = line.substring(0, 2);
+        const rel = (0, utils_1.normalizePath)(line.substring(3));
+        // Untracked directory entries (trailing slash) may hide an embedded repo —
+        // collect for the recursion below instead of treating as a file.
+        if (statusCode === '??' && rel.endsWith('/')) {
+            untrackedDirs.push(rel);
+            continue;
+        }
+        const filePath = (0, utils_1.normalizePath)(prefix + rel);
+        if (!(0, grammars_1.isSourceFile)(filePath, overrides))
+            continue;
+        if (statusCode.includes('D')) {
+            // Deletions stay unfiltered: getChangedFiles acts on one only when the
+            // path is already tracked in the DB, where removal is always correct — and
+            // that lets a newly-excluded dir's stale rows clean themselves up. (#766)
+            out.deleted.push(filePath);
+            continue;
+        }
+        // Added (`??`) / modified files inside an excluded dir must not enter the
+        // index — match against the repo-relative path, same as the full scan. (#766)
+        if (ig.ignores(rel))
+            continue;
+        // User `codegraph.json` `exclude` (#999) is project-root-relative, so it's
+        // matched against the full path — sync must not re-add a tracked file the
+        // full index now keeps out. Deletions above stay unfiltered so a file that
+        // WAS indexed before an exclude was added still cleans itself out.
+        if (exclude && exclude.ignores(filePath))
+            continue;
+        if (statusCode === '??') {
+            out.added.push(filePath);
+        }
+        else {
+            // M, MM, AM, A (staged), etc. — treat as modified
+            out.modified.push(filePath);
+        }
+    }
+    // Recurse embedded repos found under untracked dirs (at the dir itself or
+    // nested deeper). Gitignored dirs are walked only for the directories the
+    // project opted in via `includeIgnored`; by default `.gitignore` is respected
+    // and they are left alone (#970, #976), mirroring the full-index scan.
+    for (const rel of untrackedDirs) {
+        for (const repoRel of findNestedGitRepos(path.join(repoDir, rel), rel)) {
+            collectGitStatus(path.join(repoDir, repoRel), prefix + repoRel, out, overrides, includeIgnored, exclude);
+        }
+    }
+    for (const rel of findIgnoredEmbeddedRepos(repoDir, includeIgnored, prefix)) {
+        collectGitStatus(path.join(repoDir, rel), prefix + rel, out, overrides, includeIgnored, exclude);
+    }
+}
+/**
+ * Recursively scan a directory for source files.
+ *
+ * In git repos, uses `git ls-files` (inherently respects .gitignore at all
+ * levels), then keeps files with a supported source extension. For non-git
+ * projects, falls back to a filesystem walk that parses .gitignore itself.
+ */
+function scanDirectory(rootDir, onProgress) {
+    // Custom extension → language overrides from the project's codegraph.json.
+    const overrides = (0, project_config_1.loadExtensionOverrides)(rootDir);
+    // Fast path: use git to get all visible files (respects .gitignore everywhere)
+    const gitFiles = getGitVisibleFiles(rootDir);
+    if (gitFiles) {
+        const files = [];
+        let count = 0;
+        for (const filePath of gitFiles) {
+            if ((0, grammars_1.isSourceFile)(filePath, overrides)) {
+                files.push(filePath);
+                count++;
+                onProgress?.(count, filePath);
+            }
+        }
+        return files;
+    }
+    // Fallback: walk filesystem for non-git projects
+    return scanDirectoryWalk(rootDir, onProgress);
+}
+/**
+ * Async variant of scanDirectory that yields to the event loop periodically,
+ * allowing worker threads to receive and render progress messages.
+ */
+async function scanDirectoryAsync(rootDir, onProgress) {
+    // Custom extension → language overrides from the project's codegraph.json.
+    const overrides = (0, project_config_1.loadExtensionOverrides)(rootDir);
+    const gitFiles = getGitVisibleFiles(rootDir);
+    if (gitFiles) {
+        const files = [];
+        let count = 0;
+        for (const filePath of gitFiles) {
+            if ((0, grammars_1.isSourceFile)(filePath, overrides)) {
+                files.push(filePath);
+                count++;
+                onProgress?.(count, filePath);
+                // Yield every 100 files so worker threads can render progress
+                if (count % 100 === 0) {
+                    await new Promise(r => setImmediate(r));
+                }
+            }
+        }
+        return files;
+    }
+    return scanDirectoryWalk(rootDir, onProgress);
+}
+/**
+ * Filesystem walk fallback for non-git projects.
+ */
+function scanDirectoryWalk(rootDir, onProgress) {
+    const files = [];
+    let count = 0;
+    const visitedDirs = new Set();
+    // Custom extension → language overrides from the project's codegraph.json.
+    const overrides = (0, project_config_1.loadExtensionOverrides)(rootDir);
+    const loadIgnore = (dir) => {
+        const giPath = path.join(dir, '.gitignore');
+        if (!fs.existsSync(giPath))
+            return null;
+        // readGitignorePatterns is defensive: a non-UTF-8 (DLP-encrypted) or
+        // uncompilable .gitignore is skipped/filtered with a warning, never thrown
+        // (issue #682) — so the per-file `.ignores()` calls below can't crash.
+        const patterns = readGitignorePatterns(giPath);
+        return patterns ? { dir, ig: (0, ignore_1.default)().add(patterns) } : null;
+    };
+    const isIgnored = (fullPath, isDir, matchers) => {
+        for (const { dir, ig } of matchers) {
+            let rel = (0, utils_1.normalizePath)(path.relative(dir, fullPath));
+            if (!rel || rel.startsWith('..'))
+                continue; // not under this matcher's dir
+            if (isDir)
+                rel += '/'; // dir-only rules (e.g. `build/`) only match with the slash
+            if (ig.ignores(rel))
+                return true;
+        }
+        return false;
+    };
+    function walk(dir, matchers) {
+        let realDir;
+        try {
+            realDir = fs.realpathSync(dir);
+        }
+        catch {
+            (0, errors_1.logDebug)('Skipping unresolvable directory', { dir });
+            return;
+        }
+        if (visitedDirs.has(realDir)) {
+            (0, errors_1.logDebug)('Skipping already-visited directory (symlink cycle)', { dir, realDir });
+            return;
+        }
+        visitedDirs.add(realDir);
+        // This directory's own .gitignore (if present) applies to everything below it.
+        // The root's .gitignore is already merged into the seeded base matcher (so a
+        // negation there can override a built-in default), so skip it here.
+        const own = dir === rootDir ? null : loadIgnore(dir);
+        const active = own ? [...matchers, own] : matchers;
+        let entries;
+        try {
+            entries = fs.readdirSync(dir, { withFileTypes: true });
+        }
+        catch (error) {
+            (0, errors_1.logDebug)('Skipping unreadable directory', { dir, error: String(error) });
+            return;
+        }
+        for (const entry of entries) {
+            // Never descend into git internals or any CodeGraph data directory
+            // (the active one or a sibling another environment created — #636).
+            if (entry.name === '.git' || (0, directory_1.isCodeGraphDataDir)(entry.name))
+                continue;
+            const fullPath = path.join(dir, entry.name);
+            const relativePath = (0, utils_1.normalizePath)(path.relative(rootDir, fullPath));
+            if (entry.isSymbolicLink()) {
+                try {
+                    const realTarget = fs.realpathSync(fullPath);
+                    const stat = fs.statSync(realTarget);
+                    if (stat.isDirectory()) {
+                        if (!isIgnored(fullPath, true, active)) {
+                            walk(fullPath, active);
+                        }
+                    }
+                    else if (stat.isFile()) {
+                        if (!isIgnored(fullPath, false, active) && (0, grammars_1.isSourceFile)(relativePath, overrides)) {
+                            files.push(relativePath);
+                            count++;
+                            onProgress?.(count, relativePath);
+                        }
+                    }
+                }
+                catch {
+                    (0, errors_1.logDebug)('Skipping broken symlink', { path: fullPath });
+                }
+                continue;
+            }
+            if (entry.isDirectory()) {
+                if (!isIgnored(fullPath, true, active)) {
+                    walk(fullPath, active);
+                }
+            }
+            else if (entry.isFile()) {
+                if (!isIgnored(fullPath, false, active) && (0, grammars_1.isSourceFile)(relativePath, overrides)) {
+                    files.push(relativePath);
+                    count++;
+                    onProgress?.(count, relativePath);
+                }
+            }
+        }
+    }
+    // Seed a base matcher with the built-in default ignores (merged with the root
+    // .gitignore so a negation can override). Nested .gitignores still layer per-dir.
+    const baseMatchers = [{ dir: rootDir, ig: buildDefaultIgnore(rootDir) }];
+    // Project `codegraph.json` `exclude` patterns (#999), rooted at the project so
+    // `isIgnored` matches them against root-relative paths — same coverage the
+    // git path gets via ScopeIgnore, for non-git projects.
+    const exclude = loadExcludeMatcher(rootDir);
+    if (exclude)
+        baseMatchers.push({ dir: rootDir, ig: exclude });
+    walk(rootDir, baseMatchers);
+    // Force-include first-party source whitelisted in `codegraph.json` `include`
+    // — the walk above honours `.gitignore`, so anything gitignored was dropped;
+    // add it back here (deduped). Mirrors the git path's union.
+    const included = collectIncludedFilesForRoot(rootDir);
+    if (included.size > 0) {
+        const seen = new Set(files);
+        for (const f of included) {
+            if (!seen.has(f)) {
+                files.push(f);
+                seen.add(f);
+            }
+        }
+    }
+    return files;
+}
+/**
+ * Resurrect a resolution edge that is about to be dropped (its target symbol
+ * was removed, renamed, or its whole file deleted) as the ORIGINAL unresolved
+ * reference that created it, read from the refName/refKind stamp
+ * `createEdges` writes into edge metadata. Inserted as status='pending', the
+ * ref is consumed by the same sync's resolution sweep: it rebinds to an
+ * alternative definition if one exists, or parks as status='failed' where the
+ * #1240 retry finds it if the symbol later reappears.
+ *
+ * Returns null — drop silently, the pre-#1240 behavior — for edges without a
+ * refName stamp (created before the stamp existed, or synthesized): rebuilding
+ * a ref from the target's plain node name would strip the receiver/qualifier
+ * context the original text carried (`h.greet` → `greet`) and could rebind
+ * somewhere a full re-index never would. Silent beats wrong.
+ */
+function resurrectRefFromDroppedEdge(e) {
+    const refName = e.metadata?.refName;
+    if (typeof refName !== 'string' || refName.length === 0)
+        return null;
+    const refKind = typeof e.metadata?.refKind === 'string' ? e.metadata.refKind : e.kind;
+    return {
+        fromNodeId: e.source,
+        referenceName: refName,
+        referenceKind: refKind,
+        line: e.line ?? 0,
+        column: e.column ?? 0,
+        filePath: e.sourceFilePath,
+        language: e.sourceLanguage,
+    };
+}
+/**
+ * Extraction orchestrator
+ */
+class ExtractionOrchestrator {
+    rootDir;
+    queries;
+    /**
+     * Names of frameworks detected for this project, populated by indexAll().
+     * Passed to extractFromSource so framework-specific extractors (route nodes,
+     * middleware, etc.) run after the tree-sitter pass. Cleared if detection
+     * hasn't run yet so single-file re-index paths can detect on the spot.
+     */
+    detectedFrameworkNames = null;
+    constructor(rootDir, queries) {
+        this.rootDir = rootDir;
+        this.queries = queries;
+    }
+    /**
+     * Build a filesystem-backed ResolutionContext sufficient for framework
+     * detection. Graph-query methods (getNodesByName etc.) return empty because
+     * the DB hasn't been populated yet, but detect() only uses readFile,
+     * fileExists, and getAllFiles, so that's fine.
+     */
+    buildDetectionContext(files) {
+        const rootDir = this.rootDir;
+        return {
+            getNodesInFile: () => [],
+            getNodesByName: () => [],
+            getNodesByQualifiedName: () => [],
+            getNodesByKind: () => [],
+            getNodesByLowerName: () => [],
+            getImportMappings: () => [],
+            getAllFiles: () => files,
+            getProjectRoot: () => rootDir,
+            fileExists: (relativePath) => {
+                const full = (0, utils_1.validatePathWithinRoot)(rootDir, relativePath);
+                if (!full)
+                    return false;
+                try {
+                    return fs.existsSync(full);
+                }
+                catch {
+                    return false;
+                }
+            },
+            readFile: (relativePath) => {
+                const full = (0, utils_1.validatePathWithinRoot)(rootDir, relativePath);
+                if (!full)
+                    return null;
+                try {
+                    return fs.readFileSync(full, 'utf-8');
+                }
+                catch {
+                    return null;
+                }
+            },
+            // Monorepo support — needed by framework detect()s that probe
+            // subpackage manifests (e.g. fabric-view looking at
+            // packages/<sub>/package.json when the root manifest is just a
+            // workspace declaration). Matches the resolver-context shape.
+            listDirectories: (relativePath) => {
+                const target = relativePath === '.' || relativePath === ''
+                    ? rootDir
+                    : path.join(rootDir, relativePath);
+                try {
+                    return fs
+                        .readdirSync(target, { withFileTypes: true })
+                        .filter((entry) => entry.isDirectory())
+                        .map((entry) => entry.name);
+                }
+                catch {
+                    return [];
+                }
+            },
+        };
+    }
+    /**
+     * Detect frameworks on demand using the current scanned files (or a fresh
+     * scan if none are provided). Cached on the orchestrator so repeat calls
+     * inside a single run don't re-scan.
+     */
+    ensureDetectedFrameworks(files) {
+        if (this.detectedFrameworkNames !== null)
+            return this.detectedFrameworkNames;
+        const fileList = files ?? scanDirectory(this.rootDir);
+        const context = this.buildDetectionContext(fileList);
+        this.detectedFrameworkNames = (0, frameworks_1.detectFrameworks)(context).map((r) => r.name);
+        return this.detectedFrameworkNames;
+    }
+    /**
+     * Index all files in the project
+     */
+    async indexAll(onProgress, signal, verbose, 
+    // Writer-side backstop for deferred WAL checkpointing (#1231): returns
+    // null in the normal case, or a promise to await (at this safe,
+    // between-transactions boundary) when the WAL has outrun the off-thread
+    // checkpointer past its hard cap. See db/wal-valve.ts.
+    walBackpressure, 
+    // Fresh-DB store offload (perf): when set, per-file store bundles are
+    // applied by a dedicated writer thread instead of the main thread. Only
+    // passed for a COMPLETELY fresh database, where the main thread performs
+    // no reads/writes during the parse loop, so one writer applying bundles
+    // in file order preserves the #1015 determinism exactly.
+    storeWriterOpts) {
+        const tGrammar = Date.now();
+        await (0, grammars_1.initGrammars)();
+        if (process.env.CODEGRAPH_SYNTH_TIMINGS)
+            console.error(`[phase-timing] grammar-init: ${Date.now() - tGrammar}ms`);
+        const startTime = Date.now();
+        const errors = [];
+        let filesIndexed = 0;
+        let filesSkipped = 0;
+        let filesErrored = 0;
+        let totalNodes = 0;
+        let totalEdges = 0;
+        // Custom extension → language overrides from the project's codegraph.json.
+        // Threaded into language detection so custom-extension files load the right
+        // grammar and store under the mapped language.
+        const overrides = (0, project_config_1.loadExtensionOverrides)(this.rootDir);
+        const log = verbose
+            ? (msg) => { console.log(`[worker] ${msg}`); }
+            : (_msg) => { };
+        // Phase 1: Scan for files
+        onProgress?.({
+            phase: 'scanning',
+            current: 0,
+            total: 0,
+        });
+        // Phase attribution to stderr (same opt-in as the synthesis timings):
+        // early-run 5-10s single stalls were observed on 95k-file repos but never
+        // attributed — these labels settle scan vs framework-detect vs grammars.
+        const tScan = Date.now();
+        const files = await scanDirectoryAsync(this.rootDir, (current, file) => {
+            onProgress?.({
+                phase: 'scanning',
+                current,
+                total: 0,
+                currentFile: file,
+            });
+        });
+        if (process.env.CODEGRAPH_SYNTH_TIMINGS)
+            console.error(`[phase-timing] scan: ${Date.now() - tScan}ms (${files.length} files)`);
+        // Detect frameworks once per indexAll run using the scanned file list.
+        // Names are passed to each parse call so framework-specific extractors
+        // (route nodes, middleware, etc.) run after the tree-sitter pass.
+        // Framework detection is reset each run so adding e.g. requirements.txt
+        // between runs is picked up without restarting the process.
+        this.detectedFrameworkNames = null;
+        const tFw = Date.now();
+        const frameworkNames = this.ensureDetectedFrameworks(files);
+        if (process.env.CODEGRAPH_SYNTH_TIMINGS)
+            console.error(`[phase-timing] framework-detect: ${Date.now() - tFw}ms`);
+        if (signal?.aborted) {
+            return {
+                success: false,
+                filesIndexed: 0,
+                filesSkipped: 0,
+                filesErrored: 0,
+                nodesCreated: 0,
+                edgesCreated: 0,
+                errors: [{ message: 'Aborted', severity: 'error' }],
+                durationMs: Date.now() - startTime,
+            };
+        }
+        // Phase 2: Parse files in a worker thread (keeps main thread unblocked for UI)
+        const total = files.length;
+        let processed = 0;
+        // Emit parsing phase immediately so the progress bar appears during worker setup.
+        // The yield lets the shimmer worker flush the phase transition to stdout before
+        // the main thread starts synchronous grammar detection work.
+        onProgress?.({
+            phase: 'parsing',
+            current: 0,
+            total,
+        });
+        await new Promise(resolve => setImmediate(resolve));
+        // Detect needed languages and load grammars in the parse worker
+        const neededLanguages = [...new Set(files.map((f) => (0, grammars_1.detectLanguage)(f, undefined, overrides)))];
+        // .h files default to 'c' but may be C++ — ensure cpp grammar is loaded when c is needed
+        if (neededLanguages.includes('c') && !neededLanguages.includes('cpp')) {
+            neededLanguages.push('cpp');
+        }
+        // Parse files on a pool of worker threads (keeps the main thread free for UI
+        // and uses every core). Falls back to in-process parsing when the compiled
+        // worker is unavailable (e.g. running from source in tests).
+        const parseWorkerPath = path.join(__dirname, 'parse-worker.js');
+        const useWorker = fs.existsSync(parseWorkerPath);
+        let pool = null;
+        if (useWorker) {
+            // CODEGRAPH_PARSE_WORKERS: explicit worker count; 1 = the old single-worker
+            // behaviour (the conservative rollback). Unset → clamp(cores-1, 1, 8).
+            const poolSize = (0, parse_pool_1.resolveParsePoolSize)(process.env.CODEGRAPH_PARSE_WORKERS, os.cpus().length);
+            // Read each needed grammar's WASM ONCE here and hand the bytes to every
+            // worker, so spawns/respawns load grammars from memory instead of
+            // re-reading them from disk (#1231: on an HDD, respawn re-reads amplify
+            // the very I/O contention that caused the respawn).
+            const grammarBuffers = await (0, grammars_1.readGrammarWasmBytes)(neededLanguages);
+            pool = new parse_pool_1.ParseWorkerPool({
+                languages: neededLanguages,
+                size: poolSize,
+                workerScriptPath: parseWorkerPath,
+                recycleInterval: WORKER_RECYCLE_INTERVAL,
+                parseTimeoutMs: PARSE_TIMEOUT_MS,
+                log,
+                grammarBuffers,
+            });
+            log(`Parse worker pool: ${poolSize} worker(s)`);
+            // Bulk index: every core will be needed — spawn the whole pool now so
+            // worker boot overlaps the first read batches instead of trickling in
+            // behind queue-pressure growth.
+            pool.prewarm();
+        }
+        else {
+            // In-process fallback: load grammars locally and parse on the main thread.
+            await (0, grammars_1.loadGrammarsForLanguages)(neededLanguages);
+        }
+        // Dedicated store writer thread (fresh DB only — see the parameter doc).
+        // Same availability rule as the parse pool: needs the compiled worker
+        // (absent when running from source in tests → main-thread fallback).
+        const storeWorkerPath = path.join(__dirname, 'store-worker.js');
+        let storeWriter = null;
+        if (storeWriterOpts &&
+            process.env.CODEGRAPH_NO_STORE_WORKER !== '1' &&
+            fs.existsSync(storeWorkerPath)) {
+            // Deliberately NOT awaiting ready(): worker_threads delivers messages in
+            // order, so bundles posted while the worker is still booting queue
+            // behind 'open'. A boot failure surfaces at the first drain() — same
+            // propagation point as a store error.
+            storeWriter = new store_writer_1.StoreWriter(storeWorkerPath, storeWriterOpts.dbPath, storeWriterOpts.fastInit);
+            log('Store writer thread active');
+        }
+        /** Queue-depth bound for un-acked bundles (bundles hold whole node/edge arrays). */
+        const STORE_WRITER_WINDOW = 64;
+        /**
+         * Parse one file: on the pool when available (the promise REJECTS on a worker
+         * crash/timeout — the caller records it and the retry pass re-attempts), or
+         * in-process synchronously as the no-worker fallback. The language is resolved
+         * here on the main thread, where the codegraph.json overrides are loaded.
+         */
+        const parseFile = (filePath, content) => {
+            const language = (0, grammars_1.detectLanguage)(filePath, content, overrides);
+            if (!pool)
+                return Promise.resolve((0, tree_sitter_1.extractFromSource)(filePath, content, language, frameworkNames));
+            return pool.requestParse({ filePath, content, language, frameworkNames });
+        };
+        // --- Bounded rolling-window dispatch, ordered commit ---
+        // Reads stay batched/parallel; parses run concurrently across the pool; the
+        // SQLite store stays on the main thread (it isn't thread-safe). Crucially we
+        // COMMIT results in original file order, not parse-completion order: the
+        // resolution phase (run after indexing) resolves an ambiguous reference to one
+        // of several same-named candidates by the nodes' DB insertion order, so a
+        // stable commit order keeps the resulting graph deterministic — byte-identical
+        // to the single-worker path — instead of drifting with parse timing. The
+        // `completed` buffer holds at most ~windowSize out-of-order results, so memory
+        // stays bounded.
+        const windowSize = pool ? Math.max(4, pool.size * 2) : 1;
+        const inFlight = new Set();
+        const completed = new Map();
+        let nextSeq = 0; // file-order sequence assigned at dispatch
+        let nextToStore = 0; // cursor: next sequence to commit
+        let aborted = false;
+        // Yielder for the in-order commit path: a single giant generated file's
+        // store is otherwise one unyielding multi-second transaction span on the
+        // main thread (5–14s single stalls measured on llvm-project), starving
+        // the #850 watchdog heartbeat on slow hardware.
+        const commitYield = (0, cooperative_yield_1.createYielder)();
+        const storeResult = async (filePath, content, stats, result) => {
+            processed++;
+            // WAL hard-cap backstop: between files (never mid-transaction), pause
+            // the store until the off-thread checkpoint catches up. Resolves to
+            // null in the normal case — a single size check, no cost.
+            const bp = walBackpressure?.();
+            if (bp)
+                await bp;
+            // Kernel deferred-decode results carry table sizes in kernelCounts
+            // (their object arrays are empty — decode happens at the store).
+            const nodeCount = result.kernelCounts?.nodes ?? result.nodes.length;
+            const edgeCount = result.kernelCounts?.edges ?? result.edges.length;
+            // Store: on the writer thread when active (fresh DB — bundles applied
+            // in the same file order this chain dispatches them), else on the main
+            // thread (SQLite connections are per-thread).
+            if (nodeCount > 0 || result.errors.length === 0) {
+                const language = (0, grammars_1.detectLanguage)(filePath, content, overrides);
+                if (storeWriter) {
+                    if (result.kernelBuffers) {
+                        // Buffers go to the writer as-is; the worker decodes + finalizes.
+                        // The main thread's only per-file work stays O(1) + the content hash.
+                        storeWriter.send({
+                            kernel: true,
+                            filePath,
+                            language,
+                            buffers: result.kernelBuffers,
+                            file: this.buildFileRecord(filePath, content, language, stats, nodeCount, result.errors),
+                        });
+                    }
+                    else {
+                        storeWriter.send(this.buildFreshStoreBundle(filePath, content, language, stats, result));
+                    }
+                    await storeWriter.waitBelow(STORE_WRITER_WINDOW);
+                }
+                else {
+                    const materialized = (0, kernel_1.materializeKernelResult)(result, filePath, language);
+                    await this.storeExtractionResult(filePath, content, language, stats, materialized, commitYield);
+                }
+            }
+            if (result.errors.length > 0) {
+                for (const err of result.errors) {
+                    if (!err.filePath)
+                        err.filePath = filePath;
+                }
+                errors.push(...result.errors);
+            }
+            if (nodeCount > 0) {
+                filesIndexed++;
+                totalNodes += nodeCount;
+                totalEdges += edgeCount;
+            }
+            else if (result.errors.some((e) => e.severity === 'error')) {
+                filesErrored++;
+            }
+            else {
+                // Files with no symbols but no errors (yaml, twig, properties) are
+                // tracked at the file level — count them as indexed so the CLI doesn't
+                // misleadingly report "No files found to index".
+                const lang = (0, grammars_1.detectLanguage)(filePath, content, overrides);
+                if ((0, grammars_1.isFileLevelOnlyLanguage)(lang)) {
+                    filesIndexed++;
+                }
+                else {
+                    filesSkipped++;
+                }
+            }
+            onProgress?.({ phase: 'parsing', current: processed, total, currentFile: filePath });
+        };
+        const recordParseFailure = (filePath, err) => {
+            processed++;
+            filesErrored++;
+            errors.push({
+                message: err instanceof Error ? err.message : String(err),
+                filePath,
+                severity: 'error',
+                code: 'parse_error',
+            });
+            onProgress?.({ phase: 'parsing', current: processed, total });
+        };
+        // Commit buffered parses to the DB in file order, advancing the cursor over
+        // contiguous completed results. Runs after each parse settles (and once more
+        // after the drain). storeResult is now async (it yields between chunked
+        // inserts), so commits are SERIALIZED on a promise chain — concurrent parse
+        // completions append to the chain instead of interleaving mid-store, which
+        // preserves both the file-order commit invariant (#1015: resolution
+        // disambiguates same-named candidates by insertion order) and the
+        // single-writer discipline for SQLite. Errors are recorded and re-thrown
+        // at the drain, matching the old synchronous propagation.
+        let flushChain = Promise.resolve();
+        let flushError = null;
+        const flushOrdered = () => {
+            flushChain = flushChain.then(async () => {
+                if (aborted || flushError)
+                    return;
+                try {
+                    while (completed.has(nextToStore)) {
+                        const item = completed.get(nextToStore);
+                        completed.delete(nextToStore);
+                        nextToStore++;
+                        if (item.ok)
+                            await storeResult(item.filePath, item.content, item.stats, item.result);
+                        else
+                            recordParseFailure(item.filePath, item.err);
+                    }
+                }
+                catch (err) {
+                    flushError = err;
+                }
+            });
+            return flushChain;
+        };
+        // Dispatch one file's parse (parses run concurrently across the pool), tagged
+        // with its file-order sequence so flushOrdered commits results in order. The
+        // backpressure below bounds how far parsing runs ahead of the in-order commit.
+        const feed = async (filePath, content, stats) => {
+            const seq = nextSeq++;
+            const p = (async () => {
+                try {
+                    const result = await parseFile(filePath, content);
+                    completed.set(seq, { ok: true, filePath, content, stats, result });
+                }
+                catch (parseErr) {
+                    completed.set(seq, { ok: false, filePath, err: parseErr });
+                }
+                flushOrdered();
+            })();
+            const tracked = p.finally(() => { inFlight.delete(tracked); });
+            inFlight.add(tracked);
+            // Backpressure on the dispatched-but-not-yet-committed count (in-flight +
+            // buffered), not just in-flight: a slow file sitting at the commit cursor
+            // lets later parses finish and buffer, which would otherwise grow without
+            // bound. Wait for parses to settle (each may advance the cursor) until the
+            // window has room. When nothing is in flight but the window is still full,
+            // the async commit chain is what's behind — await it so the cursor
+            // advances (buffered items hold whole file contents, so this bound is
+            // load-bearing for memory).
+            while (nextSeq - nextToStore >= windowSize) {
+                if (inFlight.size > 0)
+                    await Promise.race(inFlight);
+                else
+                    await flushOrdered();
+            }
+        };
+        const tParseLoop = Date.now();
+        for (let i = 0; i < files.length; i += FILE_IO_BATCH_SIZE) {
+            if (signal?.aborted) {
+                aborted = true;
+                break;
+            }
+            const batch = files.slice(i, i + FILE_IO_BATCH_SIZE);
+            // Read files in parallel (with path validation before any I/O)
+            const fileContents = await Promise.all(batch.map(async (fp) => {
+                try {
+                    // Indexing read: follow in-root symlinks the directory walk already
+                    // descended into (the `../` guard still applies) so files reached
+                    // via an in-root symlink-to-outside still index (#935).
+                    const fullPath = (0, utils_1.validatePathWithinRoot)(this.rootDir, fp, { allowSymlinkEscape: true });
+                    if (!fullPath) {
+                        (0, errors_1.logWarn)('Path traversal blocked in batch reader', { filePath: fp });
+                        return { filePath: fp, content: null, stats: null, error: new Error('Path traversal blocked') };
+                    }
+                    const content = await fsp.readFile(fullPath, 'utf-8');
+                    const stats = await fsp.stat(fullPath);
+                    return { filePath: fp, content, stats, error: null };
+                }
+                catch (err) {
+                    return { filePath: fp, content: null, stats: null, error: err };
+                }
+            }));
+            // Dispatch each readable file into the bounded parse window; the window
+            // stores results on the main thread as they arrive.
+            for (const { filePath, content, stats, error } of fileContents) {
+                if (signal?.aborted) {
+                    aborted = true;
+                    break;
+                }
+                if (error || content === null || stats === null) {
+                    processed++;
+                    filesErrored++;
+                    errors.push({
+                        message: `Failed to read file: ${error instanceof Error ? error.message : String(error)}`,
+                        filePath,
+                        severity: 'error',
+                        code: 'read_error',
+                    });
+                    onProgress?.({ phase: 'parsing', current: processed, total });
+                    continue;
+                }
+                // Honour MAX_FILE_SIZE. Without this check, vendored generated
+                // headers, minified bundles, and other multi-MB files get indexed,
+                // wasting WASM heap and the worker recycle budget on inputs with no
+                // useful symbols. The single-file extractFile path already enforces
+                // this; the bulk path used to silently skip the check.
+                if (stats.size > MAX_FILE_SIZE) {
+                    processed++;
+                    filesSkipped++;
+                    errors.push({
+                        message: `File exceeds max size (${stats.size} > ${MAX_FILE_SIZE})`,
+                        filePath,
+                        severity: 'warning',
+                        code: 'size_exceeded',
+                    });
+                    onProgress?.({ phase: 'parsing', current: processed, total });
+                    continue;
+                }
+                // Parse on the pool (main thread stays unblocked). Errors/timeouts are
+                // handled inside feed() → recordParseFailure, feeding the retry pass.
+                await feed(filePath, content, stats);
+            }
+            if (aborted)
+                break;
+        }
+        // Drain parses still in flight (skip on abort — we tear down below instead),
+        // then commit any results the cursor hasn't reached yet.
+        if (!aborted) {
+            await Promise.all(inFlight);
+            await flushOrdered();
+            if (flushError) {
+                if (storeWriter)
+                    await storeWriter.close();
+                throw flushError;
+            }
+            // All bundles are posted; wait for the writer to apply them, then close
+            // its connection BEFORE any main-thread DB work below (retry pass,
+            // resolution) so exactly one connection writes at a time.
+            if (storeWriter) {
+                try {
+                    await storeWriter.drain();
+                }
+                finally {
+                    await storeWriter.close();
+                    storeWriter = null;
+                }
+            }
+        }
+        if (process.env.CODEGRAPH_SYNTH_TIMINGS)
+            console.error(`[phase-timing] parse-loop: ${Date.now() - tParseLoop}ms`);
+        if (signal?.aborted || aborted) {
+            if (storeWriter)
+                await storeWriter.close();
+            if (pool)
+                await pool.destroy();
+            return {
+                success: false,
+                filesIndexed,
+                filesSkipped,
+                filesErrored,
+                filesDiscovered: total,
+                nodesCreated: totalNodes,
+                edgesCreated: totalEdges,
+                errors: [{ message: 'Aborted', severity: 'error' }, ...errors],
+                durationMs: Date.now() - startTime,
+            };
+        }
+        // Report 100% so the progress bar doesn't hang at 99%
+        onProgress?.({
+            phase: 'parsing',
+            current: total,
+            total,
+        });
+        // Yield so the shimmer worker's buffered stdout writes can flush.
+        // Worker thread stdout is proxied through the main thread's event loop,
+        // so synchronous work here blocks the animation from rendering.
+        await new Promise(resolve => setImmediate(resolve));
+        // Retry pass: files that failed due to WASM memory corruption may succeed
+        // on a fresh worker with a clean heap. Recycle before each attempt so
+        // every file gets the absolute cleanest WASM state possible. Timeouts are
+        // retried too (#1231): most are main-thread-stall artifacts, not slow
+        // parses, and this pass parses one file at a time with the store strictly
+        // after each parse resolves, so the stall window can't recur here.
+        const retryableErrors = errors.filter((e) => e.code === 'parse_error' && e.filePath &&
+            (e.message.includes('Worker exited') ||
+                e.message.includes('memory access out of bounds') ||
+                e.message.includes('timed out')));
+        if (retryableErrors.length > 0 && pool) {
+            log(`Retrying ${retryableErrors.length} files that failed due to WASM memory errors or timeouts...`);
+            // Fresh WASM heaps for the retry phase. A retry that still crashes its
+            // worker makes the pool respawn it, so later retries keep landing on clean
+            // workers too.
+            pool.recycleAll();
+            const stillFailing = [];
+            for (const errEntry of retryableErrors) {
+                const filePath = errEntry.filePath;
+                if (signal?.aborted)
+                    break;
+                let content;
+                try {
+                    const fullPath = (0, utils_1.validatePathWithinRoot)(this.rootDir, filePath);
+                    if (!fullPath)
+                        continue;
+                    content = await fsp.readFile(fullPath, 'utf-8');
+                }
+                catch {
+                    continue;
+                }
+                let result;
+                try {
+                    result = await parseFile(filePath, content);
+                }
+                catch {
+                    stillFailing.push(errEntry);
+                    continue;
+                }
+                if (result.nodes.length > 0 || result.errors.length === 0) {
+                    const language = (0, grammars_1.detectLanguage)(filePath, content, overrides);
+                    const stats = await fsp.stat(path.join(this.rootDir, filePath));
+                    await this.storeExtractionResult(filePath, content, language, stats, result, commitYield);
+                    const idx = errors.indexOf(errEntry);
+                    if (idx >= 0)
+                        errors.splice(idx, 1);
+                    filesErrored--;
+                    filesIndexed++;
+                    totalNodes += result.nodes.length;
+                    totalEdges += result.edges.length;
+                    log(`Retry OK: ${filePath} (${result.nodes.length} nodes)`);
+                }
+            }
+            // Last resort: for files that still crash on a clean worker, strip
+            // comment-only lines to reduce WASM memory pressure. Many compiler
+            // test files are 90%+ comments (CHECK directives) that don't contribute
+            // code nodes but consume parser memory.
+            if (stillFailing.length > 0) {
+                log(`${stillFailing.length} files still failing — retrying with comments stripped...`);
+                pool.recycleAll();
+                for (const errEntry of stillFailing) {
+                    const filePath = errEntry.filePath;
+                    if (signal?.aborted)
+                        break;
+                    let fullContent;
+                    try {
+                        const fullPath = (0, utils_1.validatePathWithinRoot)(this.rootDir, filePath);
+                        if (!fullPath)
+                            continue;
+                        fullContent = await fsp.readFile(fullPath, 'utf-8');
+                    }
+                    catch {
+                        continue;
+                    }
+                    // Strip lines that are entirely comments (preserving line numbers
+                    // by replacing with empty lines so node positions stay correct)
+                    const stripped = fullContent
+                        .split('\n')
+                        .map(line => /^\s*\/\//.test(line) ? '' : line)
+                        .join('\n');
+                    let result;
+                    try {
+                        result = await parseFile(filePath, stripped);
+                    }
+                    catch {
+                        continue;
+                    }
+                    if (result.nodes.length > 0 || result.errors.length === 0) {
+                        const language = (0, grammars_1.detectLanguage)(filePath, fullContent, overrides);
+                        const stats = await fsp.stat(path.join(this.rootDir, filePath));
+                        await this.storeExtractionResult(filePath, fullContent, language, stats, result, commitYield);
+                        const idx = errors.indexOf(errEntry);
+                        if (idx >= 0)
+                            errors.splice(idx, 1);
+                        filesErrored--;
+                        filesIndexed++;
+                        totalNodes += result.nodes.length;
+                        totalEdges += result.edges.length;
+                        log(`Retry (stripped) OK: ${filePath} (${result.nodes.length} nodes)`);
+                    }
+                }
+            }
+        }
+        // Shut down the parse worker pool.
+        if (pool)
+            await pool.destroy();
+        return {
+            success: filesIndexed > 0 || errors.filter((e) => e.severity === 'error').length === 0,
+            filesIndexed,
+            filesSkipped,
+            filesErrored,
+            filesDiscovered: total,
+            nodesCreated: totalNodes,
+            edgesCreated: totalEdges,
+            errors,
+            durationMs: Date.now() - startTime,
+        };
+    }
+    /**
+     * Index specific files
+     */
+    async indexFiles(filePaths) {
+        const startTime = Date.now();
+        const errors = [];
+        let filesIndexed = 0;
+        let filesSkipped = 0;
+        let filesErrored = 0;
+        let totalNodes = 0;
+        let totalEdges = 0;
+        for (const filePath of filePaths) {
+            const result = await this.indexFile(filePath);
+            if (result.errors.length > 0) {
+                errors.push(...result.errors);
+            }
+            if (result.nodes.length > 0) {
+                filesIndexed++;
+                totalNodes += result.nodes.length;
+                totalEdges += result.edges.length;
+            }
+            else if (result.errors.some((e) => e.severity === 'error')) {
+                filesErrored++;
+            }
+            else {
+                const tracked = this.queries.getFileByPath(filePath);
+                if (tracked && (0, grammars_1.isFileLevelOnlyLanguage)(tracked.language)) {
+                    filesIndexed++;
+                }
+                else {
+                    filesSkipped++;
+                }
+            }
+        }
+        return {
+            success: filesIndexed > 0 || errors.filter((e) => e.severity === 'error').length === 0,
+            filesIndexed,
+            filesSkipped,
+            filesErrored,
+            nodesCreated: totalNodes,
+            edgesCreated: totalEdges,
+            errors,
+            durationMs: Date.now() - startTime,
+        };
+    }
+    /**
+     * Index a single file
+     */
+    async indexFile(relativePath) {
+        // Indexing read: follow in-root symlinks (the `../` guard still applies), #935.
+        const fullPath = (0, utils_1.validatePathWithinRoot)(this.rootDir, relativePath, { allowSymlinkEscape: true });
+        if (!fullPath) {
+            return {
+                nodes: [],
+                edges: [],
+                unresolvedReferences: [],
+                errors: [{ message: `Path traversal blocked: ${relativePath}`, filePath: relativePath, severity: 'error', code: 'path_traversal' }],
+                durationMs: 0,
+            };
+        }
+        // Read file content and stats
+        let content;
+        let stats;
+        try {
+            stats = await fsp.stat(fullPath);
+            content = await fsp.readFile(fullPath, 'utf-8');
+        }
+        catch (error) {
+            return {
+                nodes: [],
+                edges: [],
+                unresolvedReferences: [],
+                errors: [
+                    {
+                        message: `Failed to read file: ${error instanceof Error ? error.message : String(error)}`,
+                        filePath: relativePath,
+                        severity: 'error',
+                        code: 'read_error',
+                    },
+                ],
+                durationMs: 0,
+            };
+        }
+        return this.indexFileWithContent(relativePath, content, stats);
+    }
+    /**
+     * Index a single file with pre-read content and stats.
+     * Used by the parallel batch reader to avoid redundant file I/O.
+     */
+    async indexFileWithContent(relativePath, content, stats) {
+        // Prevent `../` traversal; follow in-root symlinks like the directory walk (#935).
+        const fullPath = (0, utils_1.validatePathWithinRoot)(this.rootDir, relativePath, { allowSymlinkEscape: true });
+        if (!fullPath) {
+            (0, errors_1.logWarn)('Path traversal blocked in indexFileWithContent', { relativePath });
+            return {
+                nodes: [],
+                edges: [],
+                unresolvedReferences: [],
+                errors: [{ message: 'Path traversal blocked', filePath: relativePath, severity: 'error', code: 'path_traversal' }],
+                durationMs: 0,
+            };
+        }
+        // Check file size
+        if (stats.size > MAX_FILE_SIZE) {
+            return {
+                nodes: [],
+                edges: [],
+                unresolvedReferences: [],
+                errors: [
+                    {
+                        message: `File exceeds max size (${stats.size} > ${MAX_FILE_SIZE})`,
+                        filePath: relativePath,
+                        severity: 'warning',
+                        code: 'size_exceeded',
+                    },
+                ],
+                durationMs: 0,
+            };
+        }
+        // Detect language (honoring the project's codegraph.json extension overrides)
+        const language = (0, grammars_1.detectLanguage)(relativePath, content, (0, project_config_1.loadExtensionOverrides)(this.rootDir));
+        if (!(0, grammars_1.isLanguageSupported)(language)) {
+            return {
+                nodes: [],
+                edges: [],
+                unresolvedReferences: [],
+                errors: [],
+                durationMs: 0,
+            };
+        }
+        // Extract from source. Use cached framework names if indexAll has run,
+        // otherwise detect on the spot so single-file re-index paths still emit
+        // route nodes / middleware / etc.
+        const frameworkNames = this.ensureDetectedFrameworks();
+        const result = (0, tree_sitter_1.extractFromSource)(relativePath, content, language, frameworkNames);
+        // Store in database
+        if (result.nodes.length > 0 || result.errors.length === 0) {
+            await this.storeExtractionResult(relativePath, content, language, stats, result, (0, cooperative_yield_1.createYielder)());
+        }
+        return result;
+    }
+    /**
+     * Store extraction result in database
+     */
+    async storeExtractionResult(filePath, content, language, stats, result, onYield) {
+        // Bulk inserts run in bounded sub-transactions with a yield between, so a
+        // giant generated file (tens of thousands of symbols) can't block the
+        // event loop — and the #850 watchdog heartbeat — for the whole store.
+        // The file was NEVER one atomic transaction (each insert call has its
+        // own), and the files-table record still lands last, so crash recovery
+        // is unchanged: a partially-stored file has no record and re-indexes.
+        const STORE_CHUNK = 2000;
+        const contentHash = hashContent(content);
+        // Check if file already exists and hasn't changed
+        const existingFile = this.queries.getFileByPath(filePath);
+        if (existingFile && existingFile.contentHash === contentHash) {
+            return; // No changes
+        }
+        // Snapshot incoming cross-file edges BEFORE deleting this file's nodes.
+        // `deleteFile` cascades to delete every edge whose source OR target is a
+        // node in this file (edges.FK ... ON DELETE CASCADE). Edges whose SOURCE is
+        // in this file are re-emitted by the extractor below, but edges whose SOURCE
+        // is in a *different* (unchanged) file are not — they would be silently
+        // dropped, which is issue #899: re-indexing a callee file severs `calls`/
+        // `references` edges from callers that import it via module-attribute
+        // access (`pkg.mod.fn(...)`).
+        //
+        // We snapshot the edge plus the target node's (name, kind) so we can
+        // re-resolve to the re-indexed target's NEW id. Node ids are
+        // `sha256(filePath:kind:name:line)`, so any line shift in the callee file
+        // (e.g. a docstring-only edit above the symbol) changes every target id and
+        // a naive re-insert by old id would silently drop every edge. Matching by
+        // (filePath, kind, name) is stable across line shifts; if the symbol was
+        // renamed/removed, no match is found and the edge stays dropped (correct).
+        const crossFileIncomingEdges = existingFile
+            ? this.queries.getCrossFileIncomingEdgesWithTarget(filePath)
+            : [];
+        // Delete existing data for this file
+        if (existingFile) {
+            this.queries.deleteFile(filePath);
+        }
+        // Filter out nodes with missing required fields before insertion.
+        // This prevents FK violations when edges reference nodes that would
+        // be silently skipped by insertNode() (see issue #42).
+        const validNodes = result.nodes.filter((n) => n.id && n.kind && n.name && n.filePath && n.language);
+        const insertedIds = new Set(validNodes.map((n) => n.id));
+        const validEdges = result.edges.filter((e) => insertedIds.has(e.source) && insertedIds.has(e.target));
+        const validRefs = result.unresolvedReferences
+            .filter((ref) => insertedIds.has(ref.fromNodeId))
+            .map((ref) => ({
+            ...ref,
+            filePath: ref.filePath ?? filePath,
+            language: ref.language ?? language,
+        }));
+        // Fast path for the common case (everything fits one chunk): the whole
+        // file — nodes, edges, refs, file record — lands in ONE transaction with
+        // no event-loop yields in between. Giant generated files keep the chunked
+        // + yielding path below so the #850 watchdog heartbeat stays serviced.
+        const fitsOneChunk = validNodes.length <= STORE_CHUNK &&
+            validEdges.length <= STORE_CHUNK &&
+            validRefs.length <= STORE_CHUNK;
+        if (fitsOneChunk) {
+            // Snapshot/re-resolution of cross-file incoming edges (below) still runs
+            // for the sync path; on a fresh bulk index crossFileIncomingEdges is [].
+            this.queries.storeFileBundle({
+                nodes: validNodes,
+                edges: validEdges,
+                refs: validRefs,
+                file: {
+                    path: filePath,
+                    contentHash,
+                    language,
+                    size: stats.size,
+                    modifiedAt: stats.mtimeMs,
+                    indexedAt: Date.now(),
+                    nodeCount: result.nodes.length,
+                    errors: result.errors.length > 0 ? result.errors : undefined,
+                },
+            });
+            if (crossFileIncomingEdges.length > 0) {
+                this.reattachCrossFileEdges(crossFileIncomingEdges, validNodes);
+            }
+            return;
+        }
+        // Insert nodes (chunked — see STORE_CHUNK above)
+        for (let i = 0; i < validNodes.length; i += STORE_CHUNK) {
+            this.queries.insertNodes(validNodes.slice(i, i + STORE_CHUNK));
+            await onYield?.();
+        }
+        // Filter edges to only reference nodes that were actually inserted
+        if (validEdges.length > 0) {
+            for (let i = 0; i < validEdges.length; i += STORE_CHUNK) {
+                this.queries.insertEdges(validEdges.slice(i, i + STORE_CHUNK));
+                await onYield?.();
+            }
+        }
+        // Re-insert cross-file incoming edges snapshotted before the delete,
+        // re-resolving each edge's target to the re-indexed node's new id by
+        // (filePath, kind, name). Node ids include the source line, so any line
+        // shift in the callee file (e.g. a docstring-only edit above the symbol)
+        // changes every target id and a naive re-insert by old id would drop them
+        // all. `insertEdges` still filters to endpoints that exist. This closes
+        // the #899 edge-drop on `sync`.
+        //
+        // Edges whose callee (target) was renamed/removed during the re-index (no
+        // match in `newNodesByKindName`) are not silently dropped anymore: each is
+        // resurrected as its ORIGINAL unresolved ref (stamped on the edge as
+        // metadata.refName/refKind at creation) so the same sync's resolution
+        // sweep can rebind it to an alternative definition elsewhere, or park it
+        // as status='failed' to be retried when the symbol reappears — the
+        // removal-side counterpart of #1240. Edges without refName (built before
+        // the stamp existed, or synthesized) still drop silently: reconstructing
+        // a ref from the target's plain name would strip receiver/qualifier
+        // context and risk a rebind a full re-index would never make.
+        if (crossFileIncomingEdges.length > 0) {
+            this.reattachCrossFileEdges(crossFileIncomingEdges, validNodes);
+        }
+        // Insert unresolved references in batch with denormalized filePath/language
+        for (let i = 0; i < validRefs.length; i += STORE_CHUNK) {
+            this.queries.insertUnresolvedRefsBatch(validRefs.slice(i, i + STORE_CHUNK));
+            await onYield?.();
+        }
+        // Insert file record
+        const fileRecord = {
+            path: filePath,
+            contentHash,
+            language,
+            size: stats.size,
+            modifiedAt: stats.mtimeMs,
+            indexedAt: Date.now(),
+            nodeCount: result.nodes.length,
+            errors: result.errors.length > 0 ? result.errors : undefined,
+        };
+        this.queries.upsertFile(fileRecord);
+    }
+    /**
+     * Build one file's store bundle for the FRESH-DB path: no existing-file
+     * check, no cross-file edge snapshot (both are re-index concerns — a fresh
+     * database has neither). Filters mirror storeExtractionResult exactly.
+     */
+    /** The FileRecord for a fresh-index store (nodeCount is the PRE-filter count). */
+    buildFileRecord(filePath, content, language, stats, nodeCount, resultErrors) {
+        return {
+            path: filePath,
+            contentHash: hashContent(content),
+            language,
+            size: stats.size,
+            modifiedAt: stats.mtimeMs,
+            indexedAt: Date.now(),
+            nodeCount,
+            errors: resultErrors.length > 0 ? resultErrors : undefined,
+        };
+    }
+    buildFreshStoreBundle(filePath, content, language, stats, result) {
+        return (0, store_writer_1.finalizeStoreBundle)(result, filePath, language, this.buildFileRecord(filePath, content, language, stats, result.nodes.length, result.errors));
+    }
+    /**
+     * Re-attach cross-file incoming edges snapshotted before a re-index delete
+     * (#899): re-resolve each edge's target to the re-indexed node's new id by
+     * (kind, name); targets that vanished are resurrected as their original
+     * unresolved ref (#1240's removal-side counterpart) when the edge carries
+     * its refName stamp.
+     */
+    reattachCrossFileEdges(crossFileIncomingEdges, validNodes) {
+        const newNodesByKindName = new Map();
+        for (const n of validNodes) {
+            newNodesByKindName.set(`${n.kind}\0${n.name}`, n.id);
+        }
+        const reinserted = [];
+        const resurrected = [];
+        for (const e of crossFileIncomingEdges) {
+            const newTargetId = newNodesByKindName.get(`${e.targetKind}\0${e.targetName}`);
+            if (newTargetId) {
+                reinserted.push({ source: e.source, target: newTargetId, kind: e.kind, metadata: e.metadata, line: e.line, column: e.column, provenance: e.provenance });
+            }
+            else {
+                const ref = resurrectRefFromDroppedEdge(e);
+                if (ref)
+                    resurrected.push(ref);
+            }
+        }
+        if (reinserted.length > 0) {
+            this.queries.insertEdges(reinserted);
+        }
+        if (resurrected.length > 0) {
+            this.queries.insertUnresolvedRefsBatch(resurrected);
+        }
+    }
+    /**
+     * Sync the index with the current file state.
+     *
+     * Change detection is filesystem-based, never git: a (size, mtime) stat
+     * pre-filter skips unchanged files, then a content-hash compare confirms real
+     * changes. This works in non-git projects and catches committed changes from
+     * `git pull`/`checkout`/`merge`/`rebase` that `git status` cannot see.
+     */
+    async sync(onProgress) {
+        await (0, grammars_1.initGrammars)(); // Initialize WASM runtime (grammars loaded lazily below)
+        const startTime = Date.now();
+        let filesChecked = 0;
+        let filesAdded = 0;
+        let filesModified = 0;
+        let filesRemoved = 0;
+        let nodesUpdated = 0;
+        const changedFilePaths = [];
+        onProgress?.({
+            phase: 'scanning',
+            current: 0,
+            total: 0,
+        });
+        const filesToIndex = [];
+        // === Filesystem reconcile (git-independent) ===
+        // The source of truth for "what changed" is the filesystem vs the indexed
+        // state — never git. We enumerate the current source files and reconcile
+        // each against the DB. A cheap (size, mtime) stat pre-filter skips unchanged
+        // files without reading or hashing them, so the expensive read+hash+parse
+        // only runs for files that actually changed. This catches edits/adds/deletes
+        // whether or not the project uses git, and crucially also catches committed
+        // changes from `git pull`/`checkout`/`merge`/`rebase` — which `git status`
+        // cannot see, because the working tree is clean afterward.
+        const tSyncScan = Date.now();
+        const currentFiles = await scanDirectoryAsync(this.rootDir);
+        if (process.env.CODEGRAPH_SYNTH_TIMINGS)
+            console.error(`[phase-timing] sync-scan: ${Date.now() - tSyncScan}ms (${currentFiles.length} files)`);
+        filesChecked = currentFiles.length;
+        const currentSet = new Set(currentFiles);
+        const tTracked = Date.now();
+        const trackedFiles = this.queries.getAllFiles();
+        if (process.env.CODEGRAPH_SYNTH_TIMINGS)
+            console.error(`[phase-timing] sync-tracked-load: ${Date.now() - tTracked}ms (${trackedFiles.length} tracked)`);
+        const trackedMap = new Map();
+        for (const f of trackedFiles) {
+            trackedMap.set(f.path, f);
+        }
+        // Removals: tracked in the DB but no longer a present source file. Check the
+        // filesystem directly — `scanDirectory` (via `git ls-files`) still lists a
+        // file deleted from disk but not yet staged, so set membership alone misses it.
+        // `reconcileChecks` drives the cooperative yield shared with the adds/mods loop
+        // below (see SYNC_RECONCILE_YIELD_INTERVAL / issue #905).
+        let reconcileChecks = 0;
+        for (const tracked of trackedFiles) {
+            if (!currentSet.has(tracked.path) || !fs.existsSync(path.join(this.rootDir, tracked.path))) {
+                // Before the cascade deletes them, resurrect incoming cross-file
+                // resolution edges as their original refs (#1240 removal case): the
+                // callers live in files this sync will NOT revisit, so this is their
+                // only chance to rebind to an alternative definition — or to park as
+                // failed until the symbol reappears somewhere. (A deleted file whose
+                // CALLERS are also being deleted is fine: their nodes cascade later
+                // in this loop and take the resurrected rows with them.)
+                const incoming = this.queries.getCrossFileIncomingEdgesWithTarget(tracked.path);
+                if (incoming.length > 0) {
+                    const resurrected = incoming
+                        .map((e) => resurrectRefFromDroppedEdge(e))
+                        .filter((r) => r !== null);
+                    if (resurrected.length > 0) {
+                        this.queries.insertUnresolvedRefsBatch(resurrected);
+                    }
+                }
+                this.queries.deleteFile(tracked.path);
+                filesRemoved++;
+            }
+            if (++reconcileChecks % SYNC_RECONCILE_YIELD_INTERVAL === 0) {
+                await new Promise((resolve) => setImmediate(resolve));
+            }
+        }
+        // Adds / modifications.
+        for (const filePath of currentFiles) {
+            // Same cooperative yield as the removals loop — this is the other O(files)
+            // synchronous-stat loop that wedges the main thread on a large repo (#905).
+            // Yield at the top of the body so the `continue` fast-paths below still hit it.
+            if (++reconcileChecks % SYNC_RECONCILE_YIELD_INTERVAL === 0) {
+                await new Promise((resolve) => setImmediate(resolve));
+            }
+            const fullPath = path.join(this.rootDir, filePath);
+            const tracked = trackedMap.get(filePath);
+            // Cheap pre-filter: an already-indexed file whose size AND mtime both match
+            // the DB is unchanged — skip it without reading or hashing. (A content
+            // change that preserves both exactly is the blind spot every mtime-based
+            // incremental tool accepts; `index --force` is the escape hatch. Git bumps
+            // mtime on every file it writes during checkout/merge, so pulls are caught.)
+            if (tracked) {
+                try {
+                    const stat = fs.statSync(fullPath);
+                    if (stat.size === tracked.size && Math.floor(stat.mtimeMs) === Math.floor(tracked.modifiedAt)) {
+                        continue;
+                    }
+                }
+                catch (error) {
+                    (0, errors_1.logDebug)('Skipping unstattable file during sync', { filePath, error: String(error) });
+                    continue;
+                }
+            }
+            // New, or size/mtime changed — read + hash to confirm a real content change.
+            let content;
+            try {
+                content = fs.readFileSync(fullPath, 'utf-8');
+            }
+            catch (error) {
+                (0, errors_1.logDebug)('Skipping unreadable file during sync', { filePath, error: String(error) });
+                continue;
+            }
+            const contentHash = hashContent(content);
+            if (!tracked) {
+                filesToIndex.push(filePath);
+                changedFilePaths.push(filePath);
+                filesAdded++;
+            }
+            else if (tracked.contentHash !== contentHash) {
+                filesToIndex.push(filePath);
+                changedFilePaths.push(filePath);
+                filesModified++;
+            }
+        }
+        // Load only grammars needed for changed files
+        if (filesToIndex.length > 0) {
+            const overrides = (0, project_config_1.loadExtensionOverrides)(this.rootDir);
+            const neededLanguages = [...new Set(filesToIndex.map((f) => (0, grammars_1.detectLanguage)(f, undefined, overrides)))];
+            // .h files default to 'c' but may be C++ — ensure cpp grammar is loaded
+            if (neededLanguages.includes('c') && !neededLanguages.includes('cpp')) {
+                neededLanguages.push('cpp');
+            }
+            await (0, grammars_1.loadGrammarsForLanguages)(neededLanguages);
+        }
+        // Index changed files
+        const total = filesToIndex.length;
+        for (let i = 0; i < filesToIndex.length; i++) {
+            const filePath = filesToIndex[i];
+            onProgress?.({
+                phase: 'parsing',
+                current: i + 1,
+                total,
+                currentFile: filePath,
+            });
+            const result = await this.indexFile(filePath);
+            nodesUpdated += result.nodes.length;
+        }
+        return {
+            filesChecked,
+            filesAdded,
+            filesModified,
+            filesRemoved,
+            nodesUpdated,
+            durationMs: Date.now() - startTime,
+            changedFilePaths: changedFilePaths.length > 0 ? changedFilePaths : undefined,
+        };
+    }
+    /**
+     * Get files that have changed since last index.
+     * Uses git status as a fast path when available, falling back to full scan.
+     */
+    getChangedFiles() {
+        const gitChanges = getGitChangedFiles(this.rootDir);
+        if (gitChanges) {
+            // === Git fast path ===
+            const added = [];
+            const modified = [];
+            const removed = [];
+            // Deleted files — only report if tracked in DB
+            for (const filePath of gitChanges.deleted) {
+                const tracked = this.queries.getFileByPath(filePath);
+                if (tracked) {
+                    removed.push(filePath);
+                }
+            }
+            // Modified + added files — read + hash, compare with DB. Untracked (`??`)
+            // files stay untracked in git even after indexing, so they must be
+            // hash-compared like modified files instead of always counting as added —
+            // otherwise status reports them as pending forever. (See issue #206.)
+            for (const filePath of [...gitChanges.modified, ...gitChanges.added]) {
+                const fullPath = path.join(this.rootDir, filePath);
+                let content;
+                try {
+                    content = fs.readFileSync(fullPath, 'utf-8');
+                }
+                catch (error) {
+                    (0, errors_1.logDebug)('Skipping unreadable file while detecting changes', { filePath, error: String(error) });
+                    continue;
+                }
+                const contentHash = hashContent(content);
+                const tracked = this.queries.getFileByPath(filePath);
+                if (!tracked) {
+                    added.push(filePath);
+                }
+                else if (tracked.contentHash !== contentHash) {
+                    modified.push(filePath);
+                }
+            }
+            return { added, modified, removed };
+        }
+        // === Fallback: full scan (non-git project or git failure) ===
+        const currentFiles = new Set(scanDirectory(this.rootDir));
+        const trackedFiles = this.queries.getAllFiles();
+        // Build Map for O(1) lookups
+        const trackedMap = new Map();
+        for (const f of trackedFiles) {
+            trackedMap.set(f.path, f);
+        }
+        const added = [];
+        const modified = [];
+        const removed = [];
+        // Find removed files
+        for (const tracked of trackedFiles) {
+            if (!currentFiles.has(tracked.path)) {
+                removed.push(tracked.path);
+            }
+        }
+        // Find added and modified files
+        for (const filePath of currentFiles) {
+            const fullPath = path.join(this.rootDir, filePath);
+            let content;
+            try {
+                content = fs.readFileSync(fullPath, 'utf-8');
+            }
+            catch (error) {
+                (0, errors_1.logDebug)('Skipping unreadable file while detecting changes', { filePath, error: String(error) });
+                continue;
+            }
+            const contentHash = hashContent(content);
+            const tracked = trackedMap.get(filePath);
+            if (!tracked) {
+                added.push(filePath);
+            }
+            else if (tracked.contentHash !== contentHash) {
+                modified.push(filePath);
+            }
+        }
+        return { added, modified, removed };
+    }
+}
+exports.ExtractionOrchestrator = ExtractionOrchestrator;
+// Re-export useful types and functions
+var tree_sitter_2 = require("./tree-sitter");
+Object.defineProperty(exports, "extractFromSource", { enumerable: true, get: function () { return tree_sitter_2.extractFromSource; } });
+var grammars_2 = require("./grammars");
+Object.defineProperty(exports, "detectLanguage", { enumerable: true, get: function () { return grammars_2.detectLanguage; } });
+Object.defineProperty(exports, "isSourceFile", { enumerable: true, get: function () { return grammars_2.isSourceFile; } });
+Object.defineProperty(exports, "isLanguageSupported", { enumerable: true, get: function () { return grammars_2.isLanguageSupported; } });
+Object.defineProperty(exports, "isGrammarLoaded", { enumerable: true, get: function () { return grammars_2.isGrammarLoaded; } });
+Object.defineProperty(exports, "getSupportedLanguages", { enumerable: true, get: function () { return grammars_2.getSupportedLanguages; } });
+Object.defineProperty(exports, "initGrammars", { enumerable: true, get: function () { return grammars_2.initGrammars; } });
+Object.defineProperty(exports, "loadGrammarsForLanguages", { enumerable: true, get: function () { return grammars_2.loadGrammarsForLanguages; } });
+Object.defineProperty(exports, "loadAllGrammars", { enumerable: true, get: function () { return grammars_2.loadAllGrammars; } });
+//# sourceMappingURL=index.js.map