homegraph 1.2.0 → 1.5.0

package/dist/extraction/index.js

@@ -41,11 +41,11 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.loadAllGrammars = exports.loadGrammarsForLanguages = exports.initGrammars = exports.getSupportedLanguages = exports.isGrammarLoaded = exports.isLanguageSupported = exports.isSourceFile = exports.detectLanguage = exports.extractFromSource = exports.ExtractionOrchestrator = exports.ScopeIgnore = void 0;
+exports.loadAllGrammars = exports.loadGrammarsForLanguages = exports.initGrammars = exports.getSupportedLanguages = exports.isGrammarLoaded = exports.isLanguageSupported = exports.isSourceFile = exports.detectLanguage = exports.extractFromSource = exports.ExtractionOrchestrator = exports.ScopeIgnore = exports.buildDefaultIgnore = void 0;
 exports.hashContent = hashContent;
-exports.buildDefaultIgnore = buildDefaultIgnore;
 exports.buildScopeIgnore = buildScopeIgnore;
 exports.discoverEmbeddedRepoRoots = discoverEmbeddedRepoRoots;
+exports.findUnindexedIgnoredRepos = findUnindexedIgnoredRepos;
 exports.scanDirectory = scanDirectory;
 exports.scanDirectoryAsync = scanDirectoryAsync;
 const fs = __importStar(require("fs"));
@@ -56,12 +56,17 @@ const crypto = __importStar(require("crypto"));
 const child_process_1 = require("child_process");
 const tree_sitter_1 = require("./tree-sitter");
 const parse_pool_1 = require("./parse-pool");
+const store_writer_1 = require("./store-writer");
+const cooperative_yield_1 = require("../resolution/cooperative-yield");
 const grammars_1 = require("./grammars");
 const project_config_1 = require("../project-config");
 const directory_1 = require("../directory");
 const errors_1 = require("../errors");
 const utils_1 = require("../utils");
 const ignore_1 = __importDefault(require("ignore"));
+const default_ignore_1 = require("./default-ignore");
+var default_ignore_2 = require("./default-ignore");
+Object.defineProperty(exports, "buildDefaultIgnore", { enumerable: true, get: function () { return default_ignore_2.buildDefaultIgnore; } });
 const frameworks_1 = require("../resolution/frameworks");
 const context_1 = require("./context");
 const arkts_1 = require("./languages/arkts");
@@ -108,176 +113,6 @@ function hashContent(content) {
  * symbols. 1 MB covers essentially all hand-written source.
  */
 const MAX_FILE_SIZE = 1024 * 1024;
-/**
- * Directory names that are dependency, build, cache, or tooling output across the
- * languages/frameworks HomeGraph supports — curated from the canonical
- * github/gitignore templates. Excluded by default so the graph reflects your code,
- * not third-party noise, without requiring a `.gitignore` (issue #407). The
- * exclusion applies uniformly (git or not, tracked or not); the only opt-in is an
- * explicit `.gitignore` negation (e.g. `!vendor/`). First-party-prone or generic
- * names (`packages`, `lib`, `app`, `bin`, `src`, `deps`, `env`, `tmp`, `storage`,
- * `Library`) are deliberately NOT listed, to avoid ever hiding real source.
- *
- * Only dirs that actually contain *indexable source* (or are enormous) earn a slot
- * — IDE/state dirs like `.idea`/`.vs` are omitted because HomeGraph indexes only
- * recognized source extensions, so they produce no symbols regardless.
- */
-const DEFAULT_IGNORE_DIRS = new Set([
-    // JS / TS — dependency directories
-    'node_modules', 'bower_components', 'jspm_packages', 'web_modules',
-    '.yarn', '.pnpm-store',
-    // JS / TS — framework & bundler build / cache / deploy output
-    '.next', '.nuxt', '.svelte-kit', '.turbo', '.vite', '.parcel-cache', '.angular',
-    '.docusaurus', 'storybook-static', '.vinxi', '.nitro', 'out-tsc',
-    '.vercel', '.netlify', '.wrangler',
-    // Build output (common across ecosystems)
-    'dist', 'build', 'out', '.output',
-    // Test / coverage
-    'coverage', '.nyc_output',
-    // Python
-    '__pycache__', '__pypackages__', '.venv', 'venv', '.pixi', '.pdm-build',
-    '.mypy_cache', '.pytest_cache', '.ruff_cache', '.tox', '.nox', '.hypothesis',
-    '.ipynb_checkpoints', '.eggs',
-    // Rust / JVM (Maven, Gradle, Scala)
-    'target', '.gradle',
-    // .NET
-    'obj',
-    // Vendored deps (Go, PHP/Composer, Ruby/Bundler)
-    'vendor',
-    // Swift / iOS
-    '.build', 'Pods', 'Carthage', 'DerivedData', '.swiftpm',
-    // Dart / Flutter
-    '.dart_tool', '.pub-cache',
-    // Native (Android NDK, C/C++ deps)
-    '.cxx', '.externalNativeBuild', 'vcpkg_installed',
-    // Scala tooling
-    '.bloop', '.metals',
-    // Lua / Luau (LuaRocks)
-    'lua_modules', '.luarocks',
-    // Delphi / RAD Studio IDE backups (duplicate .pas source — would double-count)
-    '__history', '__recovery',
-    // Generic cache
-    '.cache',
-]);
-/**
- * Android resource directory types. A `res/` tree holds ONLY non-code resources —
- * layouts, drawables, value bags (strings/colors/styles), menus, navigation
- * graphs — split into one typed subdirectory per kind, optionally density/locale/
- * version-qualified (`values-es`, `drawable-hdpi`, `layout-v21`, …). None of it
- * yields an extractable code symbol, yet on an Android app it DOMINATES the tree
- * (one report: 26k XML files = 97% of the project, 0 symbols), bloating the DB,
- * slowing indexing, and skewing both the file count and `codegraph_explore`
- * results (#1047). So these are excluded by default. The structure is
- * self-identifying — a non-Android project has no `res/layout/` etc., so it's
- * untouched — and the only XML that DOES produce symbols (MyBatis mappers) lives
- * under `src/main/resources/`, never `res/`, so nothing useful is dropped.
- * `res/raw/` is deliberately NOT here: it holds arbitrary bundled assets that can
- * be code-ish (a `.sql` schema, a `.js`), so we leave it indexed. Override any of
- * these with a `.gitignore` negation (e.g. `!res/values/`).
- */
-const ANDROID_RES_TYPES = [
-    'anim', 'animator', 'color', 'drawable', 'font', 'layout',
-    'menu', 'mipmap', 'navigation', 'transition', 'values', 'xml',
-];
-/** Gitignore-style patterns for the `ignore` matcher: the dirs above plus a few globs. */
-const DEFAULT_IGNORE_PATTERNS = [
-    ...Array.from(DEFAULT_IGNORE_DIRS, (d) => `${d}/`),
-    '*.egg-info/', // Python packaging metadata
-    'cmake-build-*/', // CLion / CMake build trees
-    'bazel-*/', // Bazel output symlink trees
-    // Android resource dirs at any depth, with their qualifier variants (#1047).
-    ...ANDROID_RES_TYPES.map((t) => `**/res/${t}*/`),
-];
-/** True if `buf` decodes as strict UTF-8 (no invalid byte sequences). */
-function isValidUtf8(buf) {
-    try {
-        new TextDecoder('utf-8', { fatal: true }).decode(buf);
-        return true;
-    }
-    catch {
-        return false;
-    }
-}
-/**
- * Read a `.gitignore` and return patterns safe to hand to the `ignore` matcher —
- * never throwing, even when the file isn't real gitignore text. Two failure
- * modes, both seen in the wild (issue #682):
- *
- *  - The file isn't valid UTF-8 — e.g. transparently encrypted in place by
- *    corporate DLP / endpoint-security software, leaving a UTF-16 header plus
- *    ciphertext. None of it is meaningful patterns, so the whole file is skipped.
- *  - The file is text but a single line can't be compiled to a regex by the
- *    `ignore` library — `\\[` and friends throw "Unterminated character class".
- *    Crucially the throw is LAZY (at match time, not `.add()`), so it would
- *    otherwise escape mid-scan. That one pattern is dropped; the rest are kept.
- *
- * Either way a warning that NAMES the file is logged (the reporter couldn't tell
- * which `.gitignore` was at fault) and indexing continues instead of aborting.
- * Returns '' when there's nothing usable.
- */
-function readGitignorePatterns(giPath) {
-    let buf;
-    try {
-        buf = fs.readFileSync(giPath);
-    }
-    catch {
-        return ''; // unreadable (permissions / race) — treat as absent
-    }
-    // A NUL byte never appears in real gitignore text, and a fatal UTF-8 decode
-    // catches the rest. Such a file isn't ignore patterns at all.
-    if (buf.includes(0) || !isValidUtf8(buf)) {
-        (0, errors_1.logWarn)('Ignoring a .gitignore that is not valid UTF-8 text — it may have been encrypted ' +
-            'in place by endpoint-security software. Indexing continues without it.', { file: giPath });
-        return '';
-    }
-    const content = buf.toString('utf-8');
-    // Fast path: one `.ignores()` call forces the library to compile EVERY rule,
-    // so if it doesn't throw, the whole file is safe to use verbatim.
-    try {
-        (0, ignore_1.default)().add(content).ignores('.homegraph-probe');
-        return content;
-    }
-    catch {
-        // Fall through: a line is uncompilable — keep the good ones, drop the bad.
-    }
-    const kept = [];
-    let dropped = 0;
-    for (const line of content.split(/\r?\n/)) {
-        try {
-            (0, ignore_1.default)().add(line).ignores('.homegraph-probe');
-            kept.push(line);
-        }
-        catch {
-            dropped++;
-        }
-    }
-    if (dropped > 0) {
-        (0, errors_1.logWarn)(`Skipped ${dropped} unparseable pattern(s) in a .gitignore; the rest are applied.`, { file: giPath });
-    }
-    return kept.join('\n');
-}
-/**
- * An `ignore` matcher seeded with the built-in defaults, merged with the project's
- * root .gitignore so a negation there (e.g. `!vendor/`) overrides a default. Shared
- * by both enumeration paths so behavior is identical with or without git — and so
- * the defaults apply to tracked files too (committing a dependency dir doesn't make
- * it project code; the explicit `.gitignore` negation is the only opt-in).
- */
-function buildDefaultIgnore(rootDir) {
-    const ig = (0, ignore_1.default)().add(DEFAULT_IGNORE_PATTERNS);
-    const rootGitignore = path.join(rootDir, '.gitignore');
-    if (fs.existsSync(rootGitignore))
-        ig.add(readGitignorePatterns(rootGitignore));
-    return ig;
-}
-/**
- * Defaults-only ignore matcher (no root `.gitignore` merged). Used wherever the
- * parent repo's own ignore rules must NOT apply — inside embedded child repos,
- * whose gitignore semantics their own `git ls-files` already enforced (#514).
- */
-function defaultsOnlyIgnore() {
-    return (0, ignore_1.default)().add(DEFAULT_IGNORE_PATTERNS);
-}
 /**
  * Matcher for the project's `homegraph.json` `includeIgnored` patterns — the
  * explicit opt-in to index embedded git repos living inside gitignored
@@ -304,6 +139,160 @@ function loadExcludeMatcher(rootDir) {
     const patterns = (0, project_config_1.loadExcludePatterns)(rootDir);
     return patterns.length > 0 ? (0, ignore_1.default)().add(patterns) : null;
 }
+/**
+ * Matcher for the project's `homegraph.json` `include` patterns — first-party
+ * source to force INTO the index even when `.gitignore` drops it (the general
+ * whitelist `includeIgnored` never was — that one only revives *embedded git
+ * repos*). The case it exists for: a project under a second VCS (SVN/Perforce)
+ * `.gitignore`s its own real source so it stays out of Git, yet we still want it
+ * indexed. Returns `null` when nothing is force-included (the zero-config
+ * default → no overhead, no extra walk). Built once per scan/sync/scope
+ * operation from the scan root.
+ */
+function loadIncludeMatcher(rootDir) {
+    const patterns = (0, project_config_1.loadIncludePatterns)(rootDir);
+    return patterns.length > 0 ? (0, ignore_1.default)().add(patterns) : null;
+}
+/** Glob metacharacters that end the static (literal) prefix of an `include` pattern. */
+const GLOB_META = /[*?[\]{}!]/;
+/**
+ * The static directory prefix of each `include` pattern — the literal leading
+ * path up to the first glob segment — trailing-slashed, used to (a) walk only
+ * the opted-in subtrees in `collectIncludedFiles` and (b) let `ScopeIgnore` keep
+ * the watcher descending toward them. `Tools/` stays `Tools/`; a recursive
+ * `Tools/**` glob yields `Tools/`; `src/local/file.ts` yields `src/local/` (the
+ * file's dir); a pattern that starts with a glob (like a leading `**`) yields
+ * `''`, meaning "no static root — walk the whole tree". Duplicates and roots
+ * nested under a broader root are collapsed so each subtree is walked once.
+ */
+function includeStaticRoots(patterns) {
+    const roots = new Set();
+    for (const pattern of patterns) {
+        let p = pattern.replace(/^\/+/, '');
+        const trailingSlash = p.endsWith('/');
+        if (trailingSlash)
+            p = p.slice(0, -1);
+        const segs = p.split('/').filter(Boolean);
+        const lead = [];
+        for (const s of segs) {
+            if (GLOB_META.test(s))
+                break;
+            lead.push(s);
+        }
+        const hadWildcard = lead.length < segs.length;
+        // A wholly-literal pattern with no trailing slash names a file (or a dir we
+        // can't tell apart) — drop its last segment so we walk the containing dir
+        // and let the matcher pick the file. A trailing slash or a glob means the
+        // remaining `lead` is already the directory to walk.
+        if (!hadWildcard && !trailingSlash && lead.length > 0)
+            lead.pop();
+        if (lead.length === 0) {
+            roots.clear();
+            roots.add('');
+            return ['']; // a top-level glob forces a whole-tree walk; nothing narrower matters
+        }
+        roots.add(lead.join('/') + '/');
+    }
+    // Collapse roots nested under a broader one (e.g. drop `a/b/` if `a/` is present).
+    const all = [...roots];
+    return all.filter((r) => !all.some((other) => other !== r && r.startsWith(other)));
+}
+/**
+ * Actively discover the source files an `include` whitelist forces in. `git
+ * ls-files` never lists gitignored files, so a filtered filesystem walk of just
+ * the opted-in subtrees (`includeStaticRoots`) is the only way to find them.
+ * Returns project-root-relative, normalized source-file paths.
+ *
+ * A file is collected when it MATCHES `include`, is NOT hit by `exclude` (an
+ * explicit exclude always wins), is a recognized source file, and does not live
+ * under a built-in default-ignored dir (`node_modules`, `dist`, …), `.git`, or
+ * HomeGraph's data dir — those are never resurfaced, mirroring `ScopeIgnore`.
+ * `.gitignore` is deliberately NOT consulted: overriding it is the whole point.
+ */
+function collectIncludedFiles(rootDir, include, exclude, roots, overrides) {
+    const out = new Set();
+    const defaults = (0, default_ignore_1.defaultsOnlyIgnore)();
+    const visited = new Set();
+    const consider = (abs, rel, isDir) => {
+        if (isDir) {
+            if (defaults.ignores(rel + '/'))
+                return; // never node_modules/dist/… via include
+            // An explicit `exclude` always wins over `include`; prune the whole subtree
+            // here so a large excluded dir (a committed frontend's own vendored deps,
+            // build output, …) is never walked — the per-file guard below still catches
+            // anything a directory pattern doesn't, so this is a pure efficiency win.
+            if (exclude && exclude.ignores(rel + '/'))
+                return;
+            walk(abs);
+        }
+        else {
+            if (defaults.ignores(rel))
+                return;
+            if (!include.ignores(rel))
+                return;
+            if (exclude && exclude.ignores(rel))
+                return;
+            if (!(0, grammars_1.isSourceFile)(rel, overrides))
+                return;
+            out.add(rel);
+        }
+    };
+    function walk(absDir) {
+        let realDir;
+        try {
+            realDir = fs.realpathSync(absDir);
+        }
+        catch {
+            return;
+        }
+        if (visited.has(realDir))
+            return; // symlink-cycle guard
+        visited.add(realDir);
+        let entries;
+        try {
+            entries = fs.readdirSync(absDir, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        for (const entry of entries) {
+            if (entry.name === '.git' || (0, directory_1.isHomeGraphDataDir)(entry.name))
+                continue;
+            const abs = path.join(absDir, entry.name);
+            const rel = (0, utils_1.normalizePath)(path.relative(rootDir, abs));
+            if (!rel || rel.startsWith('..'))
+                continue;
+            if (entry.isSymbolicLink()) {
+                try {
+                    const st = fs.statSync(fs.realpathSync(abs));
+                    consider(abs, rel, st.isDirectory());
+                }
+                catch {
+                    // broken symlink — skip
+                }
+                continue;
+            }
+            consider(abs, rel, entry.isDirectory());
+        }
+    }
+    for (const root of roots) {
+        walk(root === '' ? rootDir : path.join(rootDir, root));
+    }
+    return out;
+}
+/**
+ * The included source files (`homegraph.json` `include`) for a scan root, or an
+ * empty set when nothing is force-included. Centralizes loading the matcher,
+ * roots, exclude, and overrides so both enumeration paths (git and filesystem
+ * walk) add the same files.
+ */
+function collectIncludedFilesForRoot(rootDir) {
+    const include = loadIncludeMatcher(rootDir);
+    if (!include)
+        return new Set();
+    const roots = includeStaticRoots((0, project_config_1.loadIncludePatterns)(rootDir));
+    return collectIncludedFiles(rootDir, include, loadExcludeMatcher(rootDir), roots, (0, project_config_1.loadExtensionOverrides)(rootDir));
+}
 /**
  * `git ls-files --directory` collapses a wholly-untracked/ignored directory into
  * one entry — and when the command's own cwd is such a directory (the indexed
@@ -389,7 +378,7 @@ function classifyGitDir(absDir) {
  */
 function findNestedGitRepos(absDir, relPrefix) {
     const found = [];
-    const defaults = defaultsOnlyIgnore();
+    const defaults = (0, default_ignore_1.defaultsOnlyIgnore)();
     const queue = [
         { abs: absDir, rel: relPrefix, depth: 0 },
     ];
@@ -444,17 +433,31 @@ function findNestedGitRepos(absDir, relPrefix) {
 class ScopeIgnore {
     rootMatcher;
     exclude;
+    include;
+    includeRoots;
     embedded;
-    defaults = defaultsOnlyIgnore();
+    defaults = (0, default_ignore_1.defaultsOnlyIgnore)();
     constructor(rootMatcher, embedded, 
     /**
-     * Project `codegraph.json` `exclude` patterns (#999), matched against the
+     * Project `homegraph.json` `exclude` patterns (#999), matched against the
      * full root-relative path. Wins over everything else — an explicit user
      * exclude applies even to tracked files and even inside embedded repos.
      */
-    exclude = null) {
+    exclude = null, 
+    /**
+     * Project `homegraph.json` `include` patterns — first-party source forced
+     * INTO the index despite `.gitignore`. When a path matches, it is NOT
+     * ignored (so the watcher watches it), overriding `.gitignore`/`rootMatcher`
+     * — but never `exclude` (checked first) and never a built-in default-ignored
+     * dir. `includeRoots` are the static prefixes so a gitignored ANCESTOR
+     * directory of an included subtree still isn't pruned by the directory
+     * walker/watcher.
+     */
+    include = null, includeRoots = []) {
         this.rootMatcher = rootMatcher;
         this.exclude = exclude;
+        this.include = include;
+        this.includeRoots = includeRoots;
         // Longest root first so paths in nested embedded repos hit the innermost matcher.
         this.embedded = [...embedded].sort((a, b) => b.root.length - a.root.length);
     }
@@ -464,6 +467,20 @@ class ScopeIgnore {
         // everywhere, including ancestors of embedded repos.
         if (this.exclude && this.exclude.ignores(rel))
             return true;
+        // User `include`: force first-party source in despite `.gitignore`. Never
+        // resurfaces a built-in default-ignored dir (node_modules/dist/…), so an
+        // include pattern can't accidentally pull in dependency/build trees.
+        if (this.include && !this.defaults.ignores(rel)) {
+            if (rel.endsWith('/')) {
+                // A directory on (or leading to) an included subtree must stay walkable
+                // so the watcher/walker descends to reach the forced-in files.
+                if (this.includeRoots.some((r) => r.startsWith(rel) || rel.startsWith(r)))
+                    return false;
+            }
+            else if (this.include.ignores(rel)) {
+                return false;
+            }
+        }
         for (const { root, matcher } of this.embedded) {
             if (rel.startsWith(root)) {
                 const inner = rel.slice(root.length);
@@ -490,7 +507,8 @@ exports.ScopeIgnore = ScopeIgnore;
  */
 function buildScopeIgnore(rootDir, embeddedRoots) {
     const roots = embeddedRoots ? [...embeddedRoots] : discoverEmbeddedRepoRoots(rootDir);
-    return new ScopeIgnore(buildDefaultIgnore(rootDir), roots.map((root) => ({ root, matcher: buildDefaultIgnore(path.join(rootDir, root)) })), loadExcludeMatcher(rootDir));
+    const include = loadIncludeMatcher(rootDir);
+    return new ScopeIgnore((0, default_ignore_1.buildDefaultIgnore)(rootDir), roots.map((root) => ({ root, matcher: (0, default_ignore_1.buildDefaultIgnore)(path.join(rootDir, root)) })), loadExcludeMatcher(rootDir), include, include ? includeStaticRoots((0, project_config_1.loadIncludePatterns)(rootDir)) : []);
 }
 /**
  * Whether an embedded repo found as a tracked gitlink (mode 160000, #1031/#1033)
@@ -503,7 +521,7 @@ function buildScopeIgnore(rootDir, embeddedRoots) {
  *      under `node_modules` is never project code; not even an explicit opt-in
  *      revives it (matches `findIgnoredEmbeddedRepos`).
  *   2. The parent repo's own `.gitignore` covers its path and the project did
- *      NOT opt that path in via `codegraph.json` `includeIgnored`. The gitignore
+ *      NOT opt that path in via `homegraph.json` `includeIgnored`. The gitignore
  *      rule is the user's stated intent to keep that path out of scope, exactly
  *      as for an UNtracked embedded repo — respect it by default, opt back in
  *      with `includeIgnored` (#514, #970, #976).
@@ -538,7 +556,7 @@ function discoverEmbeddedRepoRoots(rootDir) {
         return [];
     }
     const out = [];
-    const defaults = defaultsOnlyIgnore();
+    const defaults = (0, default_ignore_1.defaultsOnlyIgnore)();
     const includeIgnored = loadIncludeIgnoredMatcher(rootDir);
     const visit = (repoAbs, prefix) => {
         const candidates = [];
@@ -558,7 +576,7 @@ function discoverEmbeddedRepoRoots(rootDir) {
         // (#1031, #1033)
         try {
             const gitOpts = { cwd: repoAbs, encoding: 'utf-8', timeout: 30000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true };
-            const repoIgnore = buildDefaultIgnore(repoAbs);
+            const repoIgnore = (0, default_ignore_1.buildDefaultIgnore)(repoAbs);
             for (const rel of listGitlinkRels(gitOpts)) {
                 const relDir = rel.endsWith('/') ? rel : rel + '/';
                 // A gitlink under a gitignored path is respected (not indexed) unless the
@@ -580,6 +598,50 @@ function discoverEmbeddedRepoRoots(rootDir) {
     visit(rootDir, '');
     return out;
 }
+/**
+ * Cap on how many skipped gitignored repos the CLI hint enumerates — a huge
+ * gitignored data dir full of clones must never turn the hint scan into a long
+ * walk. Enough to make the point; the caller says "+N more" past this.
+ */
+const UNINDEXED_IGNORED_REPO_HINT_CAP = 100;
+/**
+ * The INVERSE of the gitignored side of {@link discoverEmbeddedRepoRoots}:
+ * nested git repositories under a gitignored directory that the project has NOT
+ * opted into via `homegraph.json` `includeIgnored`. These are real repos the
+ * default `init`/`index` deliberately skips because `.gitignore` excludes them
+ * (#970, #976) — most visibly the "super-repo `.gitignore`s its child repos"
+ * layout (#1156), where `init` at the parent correctly indexes ~nothing while
+ * `init` inside each child works. The CLI uses this to turn that silent empty
+ * index into an actionable hint: it names the skipped repos and offers to opt
+ * them in. Paths are `rootDir`-relative and trailing-slashed (valid
+ * `includeIgnored` patterns as-is). Returns `[]` for a non-git root (a
+ * filesystem walk already descends into nested repos there), skips built-in
+ * default-ignored dirs (`node_modules`, …), and is bounded so it never stalls
+ * on a giant ignored tree.
+ */
+function findUnindexedIgnoredRepos(rootDir) {
+    try {
+        (0, child_process_1.execFileSync)('git', ['rev-parse', '--git-dir'], { cwd: rootDir, encoding: 'utf-8', timeout: 5000, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true });
+    }
+    catch {
+        return [];
+    }
+    const defaults = (0, default_ignore_1.defaultsOnlyIgnore)();
+    const includeIgnored = loadIncludeIgnoredMatcher(rootDir);
+    const repos = [];
+    for (const dir of listIgnoredDirs(rootDir)) {
+        if (defaults.ignores(dir))
+            continue; // node_modules etc. — never project code
+        if (includeIgnored?.ignores((0, utils_1.normalizePath)(dir)))
+            continue; // already opted in — nothing to nag about
+        for (const repo of findNestedGitRepos(path.join(rootDir, dir), dir)) {
+            repos.push(repo);
+            if (repos.length >= UNINDEXED_IGNORED_REPO_HINT_CAP)
+                return repos;
+        }
+    }
+    return repos;
+}
 /**
  * Discover embedded repos hidden by `repoDir`'s OWN gitignore rules: for each
  * gitignored directory, search for nested `.git` roots. Returns repo paths
@@ -598,7 +660,7 @@ function discoverEmbeddedRepoRoots(rootDir) {
 function findIgnoredEmbeddedRepos(repoDir, includeIgnored, prefix) {
     if (!includeIgnored)
         return [];
-    const defaults = defaultsOnlyIgnore();
+    const defaults = (0, default_ignore_1.defaultsOnlyIgnore)();
     const repos = [];
     for (const dir of listIgnoredDirs(repoDir)) {
         if (defaults.ignores(dir))
@@ -710,7 +772,7 @@ function collectGitFiles(repoDir, prefix, files, embeddedRoots, includeIgnored =
             const childDir = path.join(repoDir, rel);
             // A git worktree surfaces here as an opaque untracked dir too — skip it,
             // it's a duplicate working view of an already-indexed repo (#848).
-            if (classifyGitDir(childDir) === 'embedded' && !defaultsOnlyIgnore().ignores(rel)) {
+            if (classifyGitDir(childDir) === 'embedded' && !(0, default_ignore_1.defaultsOnlyIgnore)().ignores(rel)) {
                 embeddedRoots?.add((0, utils_1.normalizePath)(prefix + rel));
                 collectGitFiles(childDir, prefix + rel, files, embeddedRoots, includeIgnored);
             }
@@ -727,8 +789,8 @@ function collectGitFiles(repoDir, prefix, files, embeddedRoots, includeIgnored =
     // (an uninitialized submodule — empty dir, no `.git`) has nothing to index and
     // is left alone, as is a submodule worktree (a duplicate view, #945). (#1031, #1033)
     if (gitlinkRels.length > 0) {
-        const defaults = defaultsOnlyIgnore();
-        const repoIgnore = buildDefaultIgnore(repoDir);
+        const defaults = (0, default_ignore_1.defaultsOnlyIgnore)();
+        const repoIgnore = (0, default_ignore_1.buildDefaultIgnore)(repoDir);
         for (const rel of gitlinkRels) {
             const relDir = rel.endsWith('/') ? rel : rel + '/';
             // A gitlink under a gitignored path is respected (not indexed) unless the
@@ -787,7 +849,11 @@ function getGitVisibleFiles(rootDir) {
         // not the parent's: the parent's .gitignore hides the child repo from git,
         // not from the index. (#514)
         const ig = buildScopeIgnore(rootDir, embeddedRoots);
-        return new Set([...files].filter((f) => !ig.ignores(f)));
+        const visible = new Set([...files].filter((f) => !ig.ignores(f)));
+        // Force-include first-party source whitelisted in `homegraph.json` `include`.
+        for (const f of collectIncludedFilesForRoot(rootDir))
+            visible.add(f);
+        return visible;
     }
     catch {
         return null;
@@ -832,7 +898,7 @@ function collectGitStatus(repoDir, prefix, out, overrides, includeIgnored = null
     // repo-relative `rel` at each recursion level mirrors getGitVisibleFiles'
     // ScopeIgnore: every embedded repo is judged by ITS OWN rules, never the
     // parent's. (#766)
-    const ig = buildDefaultIgnore(repoDir);
+    const ig = (0, default_ignore_1.buildDefaultIgnore)(repoDir);
     const untrackedDirs = [];
     for (const line of output.split('\n')) {
         if (line.length < 4)
@@ -955,7 +1021,7 @@ function scanDirectoryWalk(rootDir, onProgress) {
         // readGitignorePatterns is defensive: a non-UTF-8 (DLP-encrypted) or
         // uncompilable .gitignore is skipped/filtered with a warning, never thrown
         // (issue #682) — so the per-file `.ignores()` calls below can't crash.
-        const patterns = readGitignorePatterns(giPath);
+        const patterns = (0, default_ignore_1.readGitignorePatterns)(giPath);
         return patterns ? { dir, ig: (0, ignore_1.default)().add(patterns) } : null;
     };
     const isIgnored = (fullPath, isDir, matchers) => {
@@ -1042,7 +1108,7 @@ function scanDirectoryWalk(rootDir, onProgress) {
     }
     // Seed a base matcher with the built-in default ignores (merged with the root
     // .gitignore so a negation can override). Nested .gitignores still layer per-dir.
-    const baseMatchers = [{ dir: rootDir, ig: buildDefaultIgnore(rootDir) }];
+    const baseMatchers = [{ dir: rootDir, ig: (0, default_ignore_1.buildDefaultIgnore)(rootDir) }];
     // Project `codegraph.json` `exclude` patterns (#999), rooted at the project so
     // `isIgnored` matches them against root-relative paths — same coverage the
     // git path gets via ScopeIgnore, for non-git projects.
@@ -1050,8 +1116,52 @@ function scanDirectoryWalk(rootDir, onProgress) {
     if (exclude)
         baseMatchers.push({ dir: rootDir, ig: exclude });
     walk(rootDir, baseMatchers);
+    // Force-include first-party source whitelisted in `homegraph.json` `include`.
+    const included = collectIncludedFilesForRoot(rootDir);
+    if (included.size > 0) {
+        const seen = new Set(files);
+        for (const f of included) {
+            if (!seen.has(f)) {
+                files.push(f);
+                seen.add(f);
+            }
+        }
+    }
     return files;
 }
+/**
+ * Extraction orchestrator
+ */
+/**
+ * Resurrect a resolution edge that is about to be dropped (its target symbol
+ * was removed, renamed, or its whole file deleted) as the ORIGINAL unresolved
+ * reference that created it, read from the refName/refKind stamp
+ * `createEdges` writes into edge metadata. Inserted as status='pending', the
+ * ref is consumed by the same sync's resolution sweep: it rebinds to an
+ * alternative definition if one exists, or parks as status='failed' where the
+ * #1240 retry finds it if the symbol later reappears.
+ *
+ * Returns null — drop silently, the pre-#1240 behavior — for edges without a
+ * refName stamp (created before the stamp existed, or synthesized): rebuilding
+ * a ref from the target's plain node name would strip the receiver/qualifier
+ * context the original text carried (`h.greet` → `greet`) and could rebind
+ * somewhere a full re-index never would. Silent beats wrong.
+ */
+function resurrectRefFromDroppedEdge(e) {
+    const refName = e.metadata?.refName;
+    if (typeof refName !== 'string' || refName.length === 0)
+        return null;
+    const refKind = typeof e.metadata?.refKind === 'string' ? e.metadata.refKind : e.kind;
+    return {
+        fromNodeId: e.source,
+        referenceName: refName,
+        referenceKind: refKind,
+        line: e.line ?? 0,
+        column: e.column ?? 0,
+        filePath: e.sourceFilePath,
+        language: e.sourceLanguage,
+    };
+}
 /**
  * Extraction orchestrator
  */
@@ -1144,7 +1254,7 @@ class ExtractionOrchestrator {
     /**
      * Index all files in the project
      */
-    async indexAll(onProgress, signal, verbose) {
+    async indexAll(onProgress, signal, verbose, walBackpressure, storeWriterOpts) {
         await (0, grammars_1.initGrammars)();
         const startTime = Date.now();
         const errors = [];
@@ -1246,6 +1356,15 @@ class ExtractionOrchestrator {
                 // In-process fallback: load grammars locally and parse on the main thread.
                 await (0, grammars_1.loadGrammarsForLanguages)(neededLanguages);
             }
+            const storeWorkerPath = path.join(__dirname, 'store-worker.js');
+            let storeWriter = null;
+            if (storeWriterOpts &&
+                process.env.HOMEGRAPH_NO_STORE_WORKER !== '1' &&
+                fs.existsSync(storeWorkerPath)) {
+                storeWriter = new store_writer_1.StoreWriter(storeWorkerPath, storeWriterOpts.dbPath, storeWriterOpts.fastInit);
+                log('Store writer thread active');
+            }
+            const STORE_WRITER_WINDOW = 64;
             /**
              * Parse one file: on the pool when available (the promise REJECTS on a worker
              * crash/timeout — the caller records it and the retry pass re-attempts), or
@@ -1275,12 +1394,21 @@ class ExtractionOrchestrator {
             let nextSeq = 0; // file-order sequence assigned at dispatch
             let nextToStore = 0; // cursor: next sequence to commit
             let aborted = false;
-            const storeResult = (filePath, content, stats, result) => {
+            const commitYield = (0, cooperative_yield_1.createYielder)();
+            const storeResult = async (filePath, content, stats, result) => {
                 processed++;
-                // Store in database on main thread (SQLite is not thread-safe)
+                const bp = walBackpressure?.();
+                if (bp)
+                    await bp;
                 if (result.nodes.length > 0 || result.errors.length === 0) {
                     const language = (0, grammars_1.detectLanguage)(filePath, content, overrides);
-                    this.storeExtractionResult(filePath, content, language, stats, result);
+                    if (storeWriter) {
+                        storeWriter.send(this.buildFreshStoreBundle(filePath, content, language, stats, result));
+                        await storeWriter.waitBelow(STORE_WRITER_WINDOW);
+                    }
+                    else {
+                        await this.storeExtractionResult(filePath, content, language, stats, result, commitYield);
+                    }
                 }
                 if (result.errors.length > 0) {
                     for (const err of result.errors) {
@@ -1298,9 +1426,6 @@ class ExtractionOrchestrator {
                     filesErrored++;
                 }
                 else {
-                    // Files with no symbols but no errors (yaml, twig, properties) are
-                    // tracked at the file level — count them as indexed so the CLI doesn't
-                    // misleadingly report "No files found to index".
                     const lang = (0, grammars_1.detectLanguage)(filePath, content, overrides);
                     if ((0, grammars_1.isFileLevelOnlyLanguage)(lang)) {
                         filesIndexed++;
@@ -1322,22 +1447,28 @@ class ExtractionOrchestrator {
                 });
                 onProgress?.({ phase: 'parsing', current: processed, total });
             };
-            // Commit buffered parses to the DB in file order, advancing the cursor over
-            // contiguous completed results. Runs after each parse settles (and once more
-            // after the drain). storeResult / recordParseFailure run here single-threaded,
-            // so shared counters and SQLite writes never race despite parallel parsing.
+            let flushChain = Promise.resolve();
+            let flushError = null;
             const flushOrdered = () => {
-                if (aborted)
-                    return;
-                while (completed.has(nextToStore)) {
-                    const item = completed.get(nextToStore);
-                    completed.delete(nextToStore);
-                    nextToStore++;
-                    if (item.ok)
-                        storeResult(item.filePath, item.content, item.stats, item.result);
-                    else
-                        recordParseFailure(item.filePath, item.err);
-                }
+                flushChain = flushChain.then(async () => {
+                    if (aborted || flushError)
+                        return;
+                    try {
+                        while (completed.has(nextToStore)) {
+                            const item = completed.get(nextToStore);
+                            completed.delete(nextToStore);
+                            nextToStore++;
+                            if (item.ok)
+                                await storeResult(item.filePath, item.content, item.stats, item.result);
+                            else
+                                recordParseFailure(item.filePath, item.err);
+                        }
+                    }
+                    catch (err) {
+                        flushError = err;
+                    }
+                });
+                return flushChain;
             };
             // Dispatch one file's parse (parses run concurrently across the pool), tagged
             // with its file-order sequence so flushOrdered commits results in order. The
@@ -1360,10 +1491,15 @@ class ExtractionOrchestrator {
                 // buffered), not just in-flight: a slow file sitting at the commit cursor
                 // lets later parses finish and buffer, which would otherwise grow without
                 // bound. Wait for parses to settle (each may advance the cursor) until the
-                // window has room. `inFlight.size > 0` guards against an empty race — the
-                // cursor file is always still in flight when the window is full.
-                while (nextSeq - nextToStore >= windowSize && inFlight.size > 0) {
-                    await Promise.race(inFlight);
+                // window has room. When nothing is in flight but the window is still full,
+                // the async commit chain is what's behind — await it so the cursor
+                // advances (buffered items hold whole file contents, so this bound is
+                // load-bearing for memory).
+                while (nextSeq - nextToStore >= windowSize) {
+                    if (inFlight.size > 0)
+                        await Promise.race(inFlight);
+                    else
+                        await flushOrdered();
                 }
             };
             const firstEtsFile = files.find((f) => (0, grammars_1.detectLanguage)(f, undefined, overrides) === 'arkts');
@@ -1468,9 +1604,25 @@ class ExtractionOrchestrator {
             // then commit any results the cursor hasn't reached yet.
             if (!aborted) {
                 await Promise.all(inFlight);
-                flushOrdered();
+                await flushOrdered();
+                if (flushError) {
+                    if (storeWriter)
+                        await storeWriter.close();
+                    throw flushError;
+                }
+                if (storeWriter) {
+                    try {
+                        await storeWriter.drain();
+                    }
+                    finally {
+                        await storeWriter.close();
+                        storeWriter = null;
+                    }
+                }
             }
             if (signal?.aborted || aborted) {
+                if (storeWriter)
+                    await storeWriter.close();
                 if (pool)
                     await pool.destroy();
                 return {
@@ -1478,6 +1630,7 @@ class ExtractionOrchestrator {
                     filesIndexed,
                     filesSkipped,
                     filesErrored,
+                    filesDiscovered: total,
                     nodesCreated: totalNodes,
                     edgesCreated: totalEdges,
                     errors: [{ message: 'Aborted', severity: 'error' }, ...errors],
@@ -1531,7 +1684,7 @@ class ExtractionOrchestrator {
                     if (result.nodes.length > 0 || result.errors.length === 0) {
                         const language = (0, grammars_1.detectLanguage)(filePath, content, overrides);
                         const stats = await fsp.stat(path.join(this.rootDir, filePath));
-                        this.storeExtractionResult(filePath, content, language, stats, result);
+                        await this.storeExtractionResult(filePath, content, language, stats, result, (0, cooperative_yield_1.createYielder)());
                         const idx = errors.indexOf(errEntry);
                         if (idx >= 0)
                             errors.splice(idx, 1);
@@ -1579,7 +1732,7 @@ class ExtractionOrchestrator {
                         if (result.nodes.length > 0 || result.errors.length === 0) {
                             const language = (0, grammars_1.detectLanguage)(filePath, fullContent, overrides);
                             const stats = await fsp.stat(path.join(this.rootDir, filePath));
-                            this.storeExtractionResult(filePath, fullContent, language, stats, result);
+                            await this.storeExtractionResult(filePath, fullContent, language, stats, result, (0, cooperative_yield_1.createYielder)());
                             const idx = errors.indexOf(errEntry);
                             if (idx >= 0)
                                 errors.splice(idx, 1);
@@ -1600,6 +1753,7 @@ class ExtractionOrchestrator {
                 filesIndexed,
                 filesSkipped,
                 filesErrored,
+                filesDiscovered: total,
                 nodesCreated: totalNodes,
                 edgesCreated: totalEdges,
                 errors,
@@ -1756,99 +1910,77 @@ class ExtractionOrchestrator {
         const result = (0, tree_sitter_1.extractFromSource)(relativePath, content, language, frameworkNames);
         // Store in database
         if (result.nodes.length > 0 || result.errors.length === 0) {
-            this.storeExtractionResult(relativePath, content, language, stats, result);
+            await this.storeExtractionResult(relativePath, content, language, stats, result, (0, cooperative_yield_1.createYielder)());
         }
         return result;
     }
     /**
      * Store extraction result in database
      */
-    storeExtractionResult(filePath, content, language, stats, result) {
+    async storeExtractionResult(filePath, content, language, stats, result, onYield) {
+        const STORE_CHUNK = 2000;
         const contentHash = hashContent(content);
-        // Check if file already exists and hasn't changed
         const existingFile = this.queries.getFileByPath(filePath);
         if (existingFile && existingFile.contentHash === contentHash) {
-            return; // No changes
-        }
-        // Snapshot incoming cross-file edges BEFORE deleting this file's nodes.
-        // `deleteFile` cascades to delete every edge whose source OR target is a
-        // node in this file (edges.FK ... ON DELETE CASCADE). Edges whose SOURCE is
-        // in this file are re-emitted by the extractor below, but edges whose SOURCE
-        // is in a *different* (unchanged) file are not — they would be silently
-        // dropped, which is issue #899: re-indexing a callee file severs `calls`/
-        // `references` edges from callers that import it via module-attribute
-        // access (`pkg.mod.fn(...)`).
-        //
-        // We snapshot the edge plus the target node's (name, kind) so we can
-        // re-resolve to the re-indexed target's NEW id. Node ids are
-        // `sha256(filePath:kind:name:line)`, so any line shift in the callee file
-        // (e.g. a docstring-only edit above the symbol) changes every target id and
-        // a naive re-insert by old id would silently drop every edge. Matching by
-        // (filePath, kind, name) is stable across line shifts; if the symbol was
-        // renamed/removed, no match is found and the edge stays dropped (correct).
+            return;
+        }
         const crossFileIncomingEdges = existingFile
             ? this.queries.getCrossFileIncomingEdgesWithTarget(filePath)
             : [];
-        // Delete existing data for this file
         if (existingFile) {
             this.queries.deleteFile(filePath);
         }
-        // Filter out nodes with missing required fields before insertion.
-        // This prevents FK violations when edges reference nodes that would
-        // be silently skipped by insertNode() (see issue #42).
         const validNodes = result.nodes.filter((n) => n.id && n.kind && n.name && n.filePath && n.language);
-        // Insert nodes
-        if (validNodes.length > 0) {
-            this.queries.insertNodes(validNodes);
-        }
-        // Filter edges to only reference nodes that were actually inserted
-        if (result.edges.length > 0) {
-            const insertedIds = new Set(validNodes.map((n) => n.id));
-            const validEdges = result.edges.filter((e) => insertedIds.has(e.source) && insertedIds.has(e.target));
-            if (validEdges.length > 0) {
-                this.queries.insertEdges(validEdges);
+        const insertedIds = new Set(validNodes.map((n) => n.id));
+        const validEdges = result.edges.filter((e) => insertedIds.has(e.source) && insertedIds.has(e.target));
+        const validRefs = result.unresolvedReferences
+            .filter((ref) => insertedIds.has(ref.fromNodeId))
+            .map((ref) => ({
+            ...ref,
+            filePath: ref.filePath ?? filePath,
+            language: ref.language ?? language,
+        }));
+        const fitsOneChunk = validNodes.length <= STORE_CHUNK &&
+            validEdges.length <= STORE_CHUNK &&
+            validRefs.length <= STORE_CHUNK;
+        if (fitsOneChunk) {
+            this.queries.storeFileBundle({
+                nodes: validNodes,
+                edges: validEdges,
+                refs: validRefs,
+                file: {
+                    path: filePath,
+                    contentHash,
+                    language,
+                    size: stats.size,
+                    modifiedAt: stats.mtimeMs,
+                    indexedAt: Date.now(),
+                    nodeCount: result.nodes.length,
+                    errors: result.errors.length > 0 ? result.errors : undefined,
+                },
+            });
+            if (crossFileIncomingEdges.length > 0) {
+                this.reattachCrossFileEdges(crossFileIncomingEdges, validNodes);
             }
+            return;
         }
-        // Re-insert cross-file incoming edges snapshotted before the delete,
-        // re-resolving each edge's target to the re-indexed node's new id by
-        // (filePath, kind, name). Node ids include the source line, so any line
-        // shift in the callee file (e.g. a docstring-only edit above the symbol)
-        // changes every target id and a naive re-insert by old id would drop them
-        // all. `insertEdges` still filters to endpoints that exist, so edges whose
-        // caller (source) was deleted, or whose callee (target) was renamed/removed
-        // during the re-index (no match in `newTargetIds`), are dropped. This
-        // closes the #899 edge-drop on `sync`.
-        if (crossFileIncomingEdges.length > 0) {
-            const newNodesByKindName = new Map();
-            for (const n of validNodes) {
-                newNodesByKindName.set(`${n.kind}\0${n.name}`, n.id);
-            }
-            const reinserted = [];
-            for (const e of crossFileIncomingEdges) {
-                const newTargetId = newNodesByKindName.get(`${e.targetKind}\0${e.targetName}`);
-                if (newTargetId) {
-                    reinserted.push({ source: e.source, target: newTargetId, kind: e.kind, metadata: e.metadata, line: e.line, column: e.column, provenance: e.provenance });
-                }
-            }
-            if (reinserted.length > 0) {
-                this.queries.insertEdges(reinserted);
-            }
+        for (let i = 0; i < validNodes.length; i += STORE_CHUNK) {
+            this.queries.insertNodes(validNodes.slice(i, i + STORE_CHUNK));
+            await onYield?.();
         }
-        // Insert unresolved references in batch with denormalized filePath/language
-        if (result.unresolvedReferences.length > 0) {
-            const insertedIds = new Set(validNodes.map((n) => n.id));
-            const refsWithContext = result.unresolvedReferences
-                .filter((ref) => insertedIds.has(ref.fromNodeId))
-                .map((ref) => ({
-                ...ref,
-                filePath: ref.filePath ?? filePath,
-                language: ref.language ?? language,
-            }));
-            if (refsWithContext.length > 0) {
-                this.queries.insertUnresolvedRefsBatch(refsWithContext);
+        if (validEdges.length > 0) {
+            for (let i = 0; i < validEdges.length; i += STORE_CHUNK) {
+                this.queries.insertEdges(validEdges.slice(i, i + STORE_CHUNK));
+                await onYield?.();
             }
         }
-        // Insert file record
+        if (crossFileIncomingEdges.length > 0) {
+            this.reattachCrossFileEdges(crossFileIncomingEdges, validNodes);
+        }
+        for (let i = 0; i < validRefs.length; i += STORE_CHUNK) {
+            this.queries.insertUnresolvedRefsBatch(validRefs.slice(i, i + STORE_CHUNK));
+            await onYield?.();
+        }
         const fileRecord = {
             path: filePath,
             contentHash,
@@ -1861,6 +1993,46 @@ class ExtractionOrchestrator {
         };
         this.queries.upsertFile(fileRecord);
     }
+    buildFileRecord(filePath, content, language, stats, nodeCount, resultErrors) {
+        return {
+            path: filePath,
+            contentHash: hashContent(content),
+            language,
+            size: stats.size,
+            modifiedAt: stats.mtimeMs,
+            indexedAt: Date.now(),
+            nodeCount,
+            errors: resultErrors.length > 0 ? resultErrors : undefined,
+        };
+    }
+    buildFreshStoreBundle(filePath, content, language, stats, result) {
+        return (0, store_writer_1.finalizeStoreBundle)(result, filePath, language, this.buildFileRecord(filePath, content, language, stats, result.nodes.length, result.errors));
+    }
+    reattachCrossFileEdges(crossFileIncomingEdges, validNodes) {
+        const newNodesByKindName = new Map();
+        for (const n of validNodes) {
+            newNodesByKindName.set(`${n.kind}\0${n.name}`, n.id);
+        }
+        const reinserted = [];
+        const resurrected = [];
+        for (const e of crossFileIncomingEdges) {
+            const newTargetId = newNodesByKindName.get(`${e.targetKind}\0${e.targetName}`);
+            if (newTargetId) {
+                reinserted.push({ source: e.source, target: newTargetId, kind: e.kind, metadata: e.metadata, line: e.line, column: e.column, provenance: e.provenance });
+            }
+            else {
+                const ref = resurrectRefFromDroppedEdge(e);
+                if (ref)
+                    resurrected.push(ref);
+            }
+        }
+        if (reinserted.length > 0) {
+            this.queries.insertEdges(reinserted);
+        }
+        if (resurrected.length > 0) {
+            this.queries.insertUnresolvedRefsBatch(resurrected);
+        }
+    }
     /**
      * Sync the index with the current file state.
      *
@@ -1869,7 +2041,7 @@ class ExtractionOrchestrator {
      * changes. This works in non-git projects and catches committed changes from
      * `git pull`/`checkout`/`merge`/`rebase` that `git status` cannot see.
      */
-    async sync(onProgress) {
+    async sync(onProgress, scopedPaths) {
         await (0, grammars_1.initGrammars)(); // Initialize WASM runtime (grammars loaded lazily below)
         const startTime = Date.now();
         let filesChecked = 0;
@@ -1884,31 +2056,48 @@ class ExtractionOrchestrator {
             total: 0,
         });
         const filesToIndex = [];
-        // === Filesystem reconcile (git-independent) ===
-        // The source of truth for "what changed" is the filesystem vs the indexed
-        // state — never git. We enumerate the current source files and reconcile
-        // each against the DB. A cheap (size, mtime) stat pre-filter skips unchanged
-        // files without reading or hashing them, so the expensive read+hash+parse
-        // only runs for files that actually changed. This catches edits/adds/deletes
-        // whether or not the project uses git, and crucially also catches committed
-        // changes from `git pull`/`checkout`/`merge`/`rebase` — which `git status`
-        // cannot see, because the working tree is clean afterward.
-        const currentFiles = await scanDirectoryAsync(this.rootDir);
-        filesChecked = currentFiles.length;
+        let currentFiles;
+        let trackedFiles;
+        if (scopedPaths && scopedPaths.length > 0) {
+            const unique = [...new Set(scopedPaths)];
+            currentFiles = unique.filter((p) => fs.existsSync(path.join(this.rootDir, p)));
+            trackedFiles = [];
+            for (const p of unique) {
+                const rec = this.queries.getFileByPath(p);
+                if (rec)
+                    trackedFiles.push(rec);
+            }
+            filesChecked = unique.length;
+        }
+        else {
+            currentFiles = await scanDirectoryAsync(this.rootDir);
+            filesChecked = currentFiles.length;
+            trackedFiles = this.queries.getAllFiles();
+        }
         const currentSet = new Set(currentFiles);
-        const trackedFiles = this.queries.getAllFiles();
         const trackedMap = new Map();
         for (const f of trackedFiles) {
             trackedMap.set(f.path, f);
         }
-        // Removals: tracked in the DB but no longer a present source file. Check the
-        // filesystem directly — `scanDirectory` (via `git ls-files`) still lists a
-        // file deleted from disk but not yet staged, so set membership alone misses it.
-        // `reconcileChecks` drives the cooperative yield shared with the adds/mods loop
-        // below (see SYNC_RECONCILE_YIELD_INTERVAL / issue #905).
+        const removedArkAnalyzerFiles = [];
         let reconcileChecks = 0;
         for (const tracked of trackedFiles) {
             if (!currentSet.has(tracked.path) || !fs.existsSync(path.join(this.rootDir, tracked.path))) {
+                if (tracked.path.startsWith('@')) {
+                    continue;
+                }
+                const incoming = this.queries.getCrossFileIncomingEdgesWithTarget(tracked.path);
+                if (incoming.length > 0) {
+                    const resurrected = incoming
+                        .map((e) => resurrectRefFromDroppedEdge(e))
+                        .filter((r) => r !== null);
+                    if (resurrected.length > 0) {
+                        this.queries.insertUnresolvedRefsBatch(resurrected);
+                    }
+                }
+                if ((0, arkts_1.isArkAnalyzerSourcePath)(tracked.path)) {
+                    removedArkAnalyzerFiles.push(tracked.path);
+                }
                 this.queries.deleteFile(tracked.path);
                 filesRemoved++;
             }
@@ -1918,10 +2107,12 @@ class ExtractionOrchestrator {
         }
         // Adds / modifications.
         for (const filePath of currentFiles) {
-            // Same cooperative yield as the removals loop — this is the other O(files)
-            // synchronous-stat loop that wedges the main thread on a large repo (#905).
-            // Yield at the top of the body so the `continue` fast-paths below still hit it.
             if (++reconcileChecks % SYNC_RECONCILE_YIELD_INTERVAL === 0) {
+                onProgress?.({
+                    phase: 'scanning',
+                    current: reconcileChecks,
+                    total: currentFiles.length,
+                });
                 await new Promise((resolve) => setImmediate(resolve));
             }
             const fullPath = path.join(this.rootDir, filePath);
@@ -1978,7 +2169,19 @@ class ExtractionOrchestrator {
         const total = filesToIndex.length;
         const overrides = (0, project_config_1.loadExtensionOverrides)(this.rootDir);
         const needsArkTS = filesToIndex.some((f) => (0, grammars_1.detectLanguage)(f, undefined, overrides) === 'arkts');
-        if (needsArkTS) {
+        const arktsChangedForBatch = [
+            ...removedArkAnalyzerFiles,
+            ...filesToIndex.filter((f) => (0, arkts_1.isArkAnalyzerSourcePath)(f) ||
+                f.endsWith('module.json5') ||
+                f.endsWith('oh-package.json5') ||
+                f === 'build-profile.json5' ||
+                f.endsWith('/build-profile.json5')),
+        ];
+        // Removals alone can still dirty a Harmony module (RTA for remaining files).
+        const needsArkTSBatch = needsArkTS ||
+            removedArkAnalyzerFiles.length > 0 ||
+            arktsChangedForBatch.some((f) => f === 'build-profile.json5' || f.endsWith('/build-profile.json5'));
+        if (needsArkTSBatch && arktsChangedForBatch.length > 0) {
             (0, context_1.setArkTSBatchProgressCallback)((batchProgress) => {
                 onProgress?.({
                     phase: 'arkts-batch',
@@ -1988,8 +2191,12 @@ class ExtractionOrchestrator {
                     subphase: batchProgress.subphase,
                 });
             });
-            const firstEts = filesToIndex.find((f) => (0, grammars_1.detectLanguage)(f, undefined, overrides) === 'arkts');
-            await (0, arkts_1.primeArkTSBatch)(this.rootDir, this.queries, firstEts);
+            const firstEts = filesToIndex.find((f) => (0, grammars_1.detectLanguage)(f, undefined, overrides) === 'arkts') ??
+                arktsChangedForBatch.find((f) => (0, arkts_1.isArkAnalyzerSourcePath)(f)) ??
+                arktsChangedForBatch[0];
+            await (0, arkts_1.primeArkTSBatch)(this.rootDir, this.queries, firstEts, {
+                changedFiles: arktsChangedForBatch,
+            });
         }
         try {
             for (let i = 0; i < filesToIndex.length; i++) {
@@ -2014,7 +2221,7 @@ class ExtractionOrchestrator {
             }
         }
         finally {
-            if (needsArkTS) {
+            if (needsArkTSBatch) {
                 (0, context_1.setArkTSBatchProgressCallback)(null);
             }
         }