homegraph 1.1.3 → 1.4.1

package/dist/extraction/index.js

@@ -46,14 +46,17 @@ exports.hashContent = hashContent;
 exports.buildDefaultIgnore = buildDefaultIgnore;
 exports.buildScopeIgnore = buildScopeIgnore;
 exports.discoverEmbeddedRepoRoots = discoverEmbeddedRepoRoots;
+exports.findUnindexedIgnoredRepos = findUnindexedIgnoredRepos;
 exports.scanDirectory = scanDirectory;
 exports.scanDirectoryAsync = scanDirectoryAsync;
 const fs = __importStar(require("fs"));
 const fsp = __importStar(require("fs/promises"));
 const path = __importStar(require("path"));
+const os = __importStar(require("os"));
 const crypto = __importStar(require("crypto"));
 const child_process_1 = require("child_process");
 const tree_sitter_1 = require("./tree-sitter");
+const parse_pool_1 = require("./parse-pool");
 const grammars_1 = require("./grammars");
 const project_config_1 = require("../project-config");
 const directory_1 = require("../directory");
@@ -61,6 +64,8 @@ const errors_1 = require("../errors");
 const utils_1 = require("../utils");
 const ignore_1 = __importDefault(require("ignore"));
 const frameworks_1 = require("../resolution/frameworks");
+const context_1 = require("./context");
+const arkts_1 = require("./languages/arkts");
 /**
  * Number of files to read in parallel during indexing.
  * File reads are I/O-bound; batching overlaps I/O wait with CPU parse work.
@@ -155,12 +160,34 @@ const DEFAULT_IGNORE_DIRS = new Set([
     // Generic cache
     '.cache',
 ]);
+/**
+ * Android resource directory types. A `res/` tree holds ONLY non-code resources —
+ * layouts, drawables, value bags (strings/colors/styles), menus, navigation
+ * graphs — split into one typed subdirectory per kind, optionally density/locale/
+ * version-qualified (`values-es`, `drawable-hdpi`, `layout-v21`, …). None of it
+ * yields an extractable code symbol, yet on an Android app it DOMINATES the tree
+ * (one report: 26k XML files = 97% of the project, 0 symbols), bloating the DB,
+ * slowing indexing, and skewing both the file count and `codegraph_explore`
+ * results (#1047). So these are excluded by default. The structure is
+ * self-identifying — a non-Android project has no `res/layout/` etc., so it's
+ * untouched — and the only XML that DOES produce symbols (MyBatis mappers) lives
+ * under `src/main/resources/`, never `res/`, so nothing useful is dropped.
+ * `res/raw/` is deliberately NOT here: it holds arbitrary bundled assets that can
+ * be code-ish (a `.sql` schema, a `.js`), so we leave it indexed. Override any of
+ * these with a `.gitignore` negation (e.g. `!res/values/`).
+ */
+const ANDROID_RES_TYPES = [
+    'anim', 'animator', 'color', 'drawable', 'font', 'layout',
+    'menu', 'mipmap', 'navigation', 'transition', 'values', 'xml',
+];
 /** Gitignore-style patterns for the `ignore` matcher: the dirs above plus a few globs. */
 const DEFAULT_IGNORE_PATTERNS = [
     ...Array.from(DEFAULT_IGNORE_DIRS, (d) => `${d}/`),
     '*.egg-info/', // Python packaging metadata
     'cmake-build-*/', // CLion / CMake build trees
     'bazel-*/', // Bazel output symlink trees
+    // Android resource dirs at any depth, with their qualifier variants (#1047).
+    ...ANDROID_RES_TYPES.map((t) => `**/res/${t}*/`),
 ];
 /** True if `buf` decodes as strict UTF-8 (no invalid byte sequences). */
 function isValidUtf8(buf) {
@@ -265,6 +292,173 @@ function loadIncludeIgnoredMatcher(rootDir) {
     const patterns = (0, project_config_1.loadIncludeIgnoredPatterns)(rootDir);
     return patterns.length > 0 ? (0, ignore_1.default)().add(patterns) : null;
 }
+/**
+ * Matcher for the project's `codegraph.json` `exclude` patterns — paths to keep
+ * OUT of the index even when git-tracked, which `.gitignore` cannot do (#999).
+ * The escape hatch for a committed vendor/theme/SDK directory. Returns `null`
+ * when nothing is excluded (the zero-config default → no overhead). Matched
+ * against project-root-relative paths, so it applies uniformly across the whole
+ * workspace, including inside embedded repos (excluding `static/` means gone
+ * everywhere). Built once per scan/sync/scope operation from the scan root.
+ */
+function loadExcludeMatcher(rootDir) {
+    const patterns = (0, project_config_1.loadExcludePatterns)(rootDir);
+    return patterns.length > 0 ? (0, ignore_1.default)().add(patterns) : null;
+}
+/**
+ * Matcher for the project's `homegraph.json` `include` patterns — first-party
+ * source to force INTO the index even when `.gitignore` drops it (the general
+ * whitelist `includeIgnored` never was — that one only revives *embedded git
+ * repos*). The case it exists for: a project under a second VCS (SVN/Perforce)
+ * `.gitignore`s its own real source so it stays out of Git, yet we still want it
+ * indexed. Returns `null` when nothing is force-included (the zero-config
+ * default → no overhead, no extra walk). Built once per scan/sync/scope
+ * operation from the scan root.
+ */
+function loadIncludeMatcher(rootDir) {
+    const patterns = (0, project_config_1.loadIncludePatterns)(rootDir);
+    return patterns.length > 0 ? (0, ignore_1.default)().add(patterns) : null;
+}
+/** Glob metacharacters that end the static (literal) prefix of an `include` pattern. */
+const GLOB_META = /[*?[\]{}!]/;
+/**
+ * The static directory prefix of each `include` pattern — the literal leading
+ * path up to the first glob segment — trailing-slashed, used to (a) walk only
+ * the opted-in subtrees in `collectIncludedFiles` and (b) let `ScopeIgnore` keep
+ * the watcher descending toward them. `Tools/` stays `Tools/`; a recursive
+ * `Tools/**` glob yields `Tools/`; `src/local/file.ts` yields `src/local/` (the
+ * file's dir); a pattern that starts with a glob (like a leading `**`) yields
+ * `''`, meaning "no static root — walk the whole tree". Duplicates and roots
+ * nested under a broader root are collapsed so each subtree is walked once.
+ */
+function includeStaticRoots(patterns) {
+    const roots = new Set();
+    for (const pattern of patterns) {
+        let p = pattern.replace(/^\/+/, '');
+        const trailingSlash = p.endsWith('/');
+        if (trailingSlash)
+            p = p.slice(0, -1);
+        const segs = p.split('/').filter(Boolean);
+        const lead = [];
+        for (const s of segs) {
+            if (GLOB_META.test(s))
+                break;
+            lead.push(s);
+        }
+        const hadWildcard = lead.length < segs.length;
+        // A wholly-literal pattern with no trailing slash names a file (or a dir we
+        // can't tell apart) — drop its last segment so we walk the containing dir
+        // and let the matcher pick the file. A trailing slash or a glob means the
+        // remaining `lead` is already the directory to walk.
+        if (!hadWildcard && !trailingSlash && lead.length > 0)
+            lead.pop();
+        if (lead.length === 0) {
+            roots.clear();
+            roots.add('');
+            return ['']; // a top-level glob forces a whole-tree walk; nothing narrower matters
+        }
+        roots.add(lead.join('/') + '/');
+    }
+    // Collapse roots nested under a broader one (e.g. drop `a/b/` if `a/` is present).
+    const all = [...roots];
+    return all.filter((r) => !all.some((other) => other !== r && r.startsWith(other)));
+}
+/**
+ * Actively discover the source files an `include` whitelist forces in. `git
+ * ls-files` never lists gitignored files, so a filtered filesystem walk of just
+ * the opted-in subtrees (`includeStaticRoots`) is the only way to find them.
+ * Returns project-root-relative, normalized source-file paths.
+ *
+ * A file is collected when it MATCHES `include`, is NOT hit by `exclude` (an
+ * explicit exclude always wins), is a recognized source file, and does not live
+ * under a built-in default-ignored dir (`node_modules`, `dist`, …), `.git`, or
+ * HomeGraph's data dir — those are never resurfaced, mirroring `ScopeIgnore`.
+ * `.gitignore` is deliberately NOT consulted: overriding it is the whole point.
+ */
+function collectIncludedFiles(rootDir, include, exclude, roots, overrides) {
+    const out = new Set();
+    const defaults = defaultsOnlyIgnore();
+    const visited = new Set();
+    const consider = (abs, rel, isDir) => {
+        if (isDir) {
+            if (defaults.ignores(rel + '/'))
+                return; // never node_modules/dist/… via include
+            // An explicit `exclude` always wins over `include`; prune the whole subtree
+            // here so a large excluded dir (a committed frontend's own vendored deps,
+            // build output, …) is never walked — the per-file guard below still catches
+            // anything a directory pattern doesn't, so this is a pure efficiency win.
+            if (exclude && exclude.ignores(rel + '/'))
+                return;
+            walk(abs);
+        }
+        else {
+            if (defaults.ignores(rel))
+                return;
+            if (!include.ignores(rel))
+                return;
+            if (exclude && exclude.ignores(rel))
+                return;
+            if (!(0, grammars_1.isSourceFile)(rel, overrides))
+                return;
+            out.add(rel);
+        }
+    };
+    function walk(absDir) {
+        let realDir;
+        try {
+            realDir = fs.realpathSync(absDir);
+        }
+        catch {
+            return;
+        }
+        if (visited.has(realDir))
+            return; // symlink-cycle guard
+        visited.add(realDir);
+        let entries;
+        try {
+            entries = fs.readdirSync(absDir, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        for (const entry of entries) {
+            if (entry.name === '.git' || (0, directory_1.isHomeGraphDataDir)(entry.name))
+                continue;
+            const abs = path.join(absDir, entry.name);
+            const rel = (0, utils_1.normalizePath)(path.relative(rootDir, abs));
+            if (!rel || rel.startsWith('..'))
+                continue;
+            if (entry.isSymbolicLink()) {
+                try {
+                    const st = fs.statSync(fs.realpathSync(abs));
+                    consider(abs, rel, st.isDirectory());
+                }
+                catch {
+                    // broken symlink — skip
+                }
+                continue;
+            }
+            consider(abs, rel, entry.isDirectory());
+        }
+    }
+    for (const root of roots) {
+        walk(root === '' ? rootDir : path.join(rootDir, root));
+    }
+    return out;
+}
+/**
+ * The included source files (`homegraph.json` `include`) for a scan root, or an
+ * empty set when nothing is force-included. Centralizes loading the matcher,
+ * roots, exclude, and overrides so both enumeration paths (git and filesystem
+ * walk) add the same files.
+ */
+function collectIncludedFilesForRoot(rootDir) {
+    const include = loadIncludeMatcher(rootDir);
+    if (!include)
+        return new Set();
+    const roots = includeStaticRoots((0, project_config_1.loadIncludePatterns)(rootDir));
+    return collectIncludedFiles(rootDir, include, loadExcludeMatcher(rootDir), roots, (0, project_config_1.loadExtensionOverrides)(rootDir));
+}
 /**
  * `git ls-files --directory` collapses a wholly-untracked/ignored directory into
  * one entry — and when the command's own cwd is such a directory (the indexed
@@ -404,14 +598,55 @@ function findNestedGitRepos(absDir, relPrefix) {
  */
 class ScopeIgnore {
     rootMatcher;
+    exclude;
+    include;
+    includeRoots;
     embedded;
     defaults = defaultsOnlyIgnore();
-    constructor(rootMatcher, embedded) {
+    constructor(rootMatcher, embedded, 
+    /**
+     * Project `homegraph.json` `exclude` patterns (#999), matched against the
+     * full root-relative path. Wins over everything else — an explicit user
+     * exclude applies even to tracked files and even inside embedded repos.
+     */
+    exclude = null, 
+    /**
+     * Project `homegraph.json` `include` patterns — first-party source forced
+     * INTO the index despite `.gitignore`. When a path matches, it is NOT
+     * ignored (so the watcher watches it), overriding `.gitignore`/`rootMatcher`
+     * — but never `exclude` (checked first) and never a built-in default-ignored
+     * dir. `includeRoots` are the static prefixes so a gitignored ANCESTOR
+     * directory of an included subtree still isn't pruned by the directory
+     * walker/watcher.
+     */
+    include = null, includeRoots = []) {
         this.rootMatcher = rootMatcher;
+        this.exclude = exclude;
+        this.include = include;
+        this.includeRoots = includeRoots;
         // Longest root first so paths in nested embedded repos hit the innermost matcher.
         this.embedded = [...embedded].sort((a, b) => b.root.length - a.root.length);
     }
     ignores(rel) {
+        // User `exclude` (#999) is checked first and against the full root-relative
+        // path: it must drop git-TRACKED paths (which `.gitignore` can't) and apply
+        // everywhere, including ancestors of embedded repos.
+        if (this.exclude && this.exclude.ignores(rel))
+            return true;
+        // User `include`: force first-party source in despite `.gitignore`. Never
+        // resurfaces a built-in default-ignored dir (node_modules/dist/…), so an
+        // include pattern can't accidentally pull in dependency/build trees.
+        if (this.include && !this.defaults.ignores(rel)) {
+            if (rel.endsWith('/')) {
+                // A directory on (or leading to) an included subtree must stay walkable
+                // so the watcher/walker descends to reach the forced-in files.
+                if (this.includeRoots.some((r) => r.startsWith(rel) || rel.startsWith(r)))
+                    return false;
+            }
+            else if (this.include.ignores(rel)) {
+                return false;
+            }
+        }
         for (const { root, matcher } of this.embedded) {
             if (rel.startsWith(root)) {
                 const inner = rel.slice(root.length);
@@ -438,7 +673,37 @@ exports.ScopeIgnore = ScopeIgnore;
  */
 function buildScopeIgnore(rootDir, embeddedRoots) {
     const roots = embeddedRoots ? [...embeddedRoots] : discoverEmbeddedRepoRoots(rootDir);
-    return new ScopeIgnore(buildDefaultIgnore(rootDir), roots.map((root) => ({ root, matcher: buildDefaultIgnore(path.join(rootDir, root)) })));
+    const include = loadIncludeMatcher(rootDir);
+    return new ScopeIgnore(buildDefaultIgnore(rootDir), roots.map((root) => ({ root, matcher: buildDefaultIgnore(path.join(rootDir, root)) })), loadExcludeMatcher(rootDir), include, include ? includeStaticRoots((0, project_config_1.loadIncludePatterns)(rootDir)) : []);
+}
+/**
+ * Whether an embedded repo found as a tracked gitlink (mode 160000, #1031/#1033)
+ * must be SKIPPED rather than indexed. A gitlink is tracked, so `.gitignore`
+ * can't untrack it — but the discovery passes for it must still honor the same
+ * scope rules as every other path, or a gitignored reference/data dir full of
+ * `git add`ed clones gets pulled into the index against the user's stated intent
+ * (#1065). Two reasons to skip:
+ *   1. It sits in a built-in default-ignored location — an npm git-dependency
+ *      under `node_modules` is never project code; not even an explicit opt-in
+ *      revives it (matches `findIgnoredEmbeddedRepos`).
+ *   2. The parent repo's own `.gitignore` covers its path and the project did
+ *      NOT opt that path in via `homegraph.json` `includeIgnored`. The gitignore
+ *      rule is the user's stated intent to keep that path out of scope, exactly
+ *      as for an UNtracked embedded repo — respect it by default, opt back in
+ *      with `includeIgnored` (#514, #970, #976).
+ * `relDir` is repoDir-relative (trailing-slashed); `prefix` is repoDir's
+ * scan-root-relative path so the `includeIgnored` pattern is matched on the full
+ * scan-root-relative path. `defaults` is `defaultsOnlyIgnore()` and `repoIgnore`
+ * is `buildDefaultIgnore(repoDir)` (defaults + the repo's own `.gitignore`),
+ * both passed in so they're built once per repo level rather than per gitlink.
+ */
+function gitlinkEmbeddedRepoSkipped(relDir, prefix, defaults, repoIgnore, includeIgnored) {
+    if (defaults.ignores(relDir))
+        return true; // default-ignored — never index, opt-in can't revive
+    if (!repoIgnore.ignores(relDir))
+        return false; // not ignored at all — index as before (#1031/#1033)
+    // Gitignored by the repo's own rules — skip unless the project opted it in.
+    return !includeIgnored?.ignores((0, utils_1.normalizePath)(prefix + relDir));
 }
 /**
  * Standalone discovery of every embedded repo root under `rootDir` (relative,
@@ -470,6 +735,25 @@ function discoverEmbeddedRepoRoots(rootDir) {
             }
         }
         catch { /* untracked listing failed — ignored-side discovery still runs */ }
+        // Unexpanded gitlinks (mode 160000) with a real checkout on disk — embedded
+        // repos `git add`ed without `.gitmodules`, or submodules not active here. The
+        // untracked listing above can't see them (they're tracked), so find them the
+        // same way collectGitFiles does, keeping watcher scope == indexer scope.
+        // (#1031, #1033)
+        try {
+            const gitOpts = { cwd: repoAbs, encoding: 'utf-8', timeout: 30000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true };
+            const repoIgnore = buildDefaultIgnore(repoAbs);
+            for (const rel of listGitlinkRels(gitOpts)) {
+                const relDir = rel.endsWith('/') ? rel : rel + '/';
+                // A gitlink under a gitignored path is respected (not indexed) unless the
+                // project opted it in — same rule as the untracked-ignored kind (#1065).
+                if (gitlinkEmbeddedRepoSkipped(relDir, prefix, defaults, repoIgnore, includeIgnored))
+                    continue;
+                if (classifyGitDir(path.join(repoAbs, rel)) === 'embedded')
+                    candidates.push(relDir);
+            }
+        }
+        catch { /* staged listing failed — other discovery still runs */ }
         candidates.push(...findIgnoredEmbeddedRepos(repoAbs, includeIgnored, prefix));
         for (const rel of candidates) {
             const full = (0, utils_1.normalizePath)(prefix + rel);
@@ -480,6 +764,50 @@ function discoverEmbeddedRepoRoots(rootDir) {
     visit(rootDir, '');
     return out;
 }
+/**
+ * Cap on how many skipped gitignored repos the CLI hint enumerates — a huge
+ * gitignored data dir full of clones must never turn the hint scan into a long
+ * walk. Enough to make the point; the caller says "+N more" past this.
+ */
+const UNINDEXED_IGNORED_REPO_HINT_CAP = 100;
+/**
+ * The INVERSE of the gitignored side of {@link discoverEmbeddedRepoRoots}:
+ * nested git repositories under a gitignored directory that the project has NOT
+ * opted into via `homegraph.json` `includeIgnored`. These are real repos the
+ * default `init`/`index` deliberately skips because `.gitignore` excludes them
+ * (#970, #976) — most visibly the "super-repo `.gitignore`s its child repos"
+ * layout (#1156), where `init` at the parent correctly indexes ~nothing while
+ * `init` inside each child works. The CLI uses this to turn that silent empty
+ * index into an actionable hint: it names the skipped repos and offers to opt
+ * them in. Paths are `rootDir`-relative and trailing-slashed (valid
+ * `includeIgnored` patterns as-is). Returns `[]` for a non-git root (a
+ * filesystem walk already descends into nested repos there), skips built-in
+ * default-ignored dirs (`node_modules`, …), and is bounded so it never stalls
+ * on a giant ignored tree.
+ */
+function findUnindexedIgnoredRepos(rootDir) {
+    try {
+        (0, child_process_1.execFileSync)('git', ['rev-parse', '--git-dir'], { cwd: rootDir, encoding: 'utf-8', timeout: 5000, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true });
+    }
+    catch {
+        return [];
+    }
+    const defaults = defaultsOnlyIgnore();
+    const includeIgnored = loadIncludeIgnoredMatcher(rootDir);
+    const repos = [];
+    for (const dir of listIgnoredDirs(rootDir)) {
+        if (defaults.ignores(dir))
+            continue; // node_modules etc. — never project code
+        if (includeIgnored?.ignores((0, utils_1.normalizePath)(dir)))
+            continue; // already opted in — nothing to nag about
+        for (const repo of findNestedGitRepos(path.join(rootDir, dir), dir)) {
+            repos.push(repo);
+            if (repos.length >= UNINDEXED_IGNORED_REPO_HINT_CAP)
+                return repos;
+        }
+    }
+    return repos;
+}
 /**
  * Discover embedded repos hidden by `repoDir`'s OWN gitignore rules: for each
  * gitignored directory, search for nested `.git` roots. Returns repo paths
@@ -510,24 +838,65 @@ function findIgnoredEmbeddedRepos(repoDir, includeIgnored, prefix) {
     return repos;
 }
 /**
- * Collect git-visible files (tracked + untracked, .gitignore-respected) from the
- * git repository rooted at `repoDir`, adding each to `files` with `prefix`
- * prepended so paths stay relative to the original scan root.
+ * Collect tracked paths and unexpanded gitlinks (mode 160000) from `git ls-files`.
  *
- * Recurses into embedded git repositories — nested repos that are NOT submodules
- * (independent clones living inside the workspace, common in CMake "super-repo"
- * layouts). The parent repo's `git ls-files` cannot see into them: tracked output
- * skips them entirely, and untracked output reports them only as an opaque
- * "subdir/" entry (trailing slash) rather than expanding their files. Each
- * embedded repo is its own git boundary, so we re-run `git ls-files` inside it.
- * (See issue #193.) GITIGNORED embedded repos are invisible even to that; they
- * are discovered separately via `findIgnoredEmbeddedRepos` (#514) but ONLY for
- * directories the project opted in through `homegraph.json` `includeIgnored`
- * (`includeIgnored` here, threaded from the scan root) — by default `.gitignore`
- * is respected and they stay out (#970, #976). Every embedded repo root (however
- * found) is recorded in `embeddedRoots` so callers can exempt its files from the
- * parent's own gitignore rules.
+ * Prefer one `-s --recurse-submodules` pass (git ≥2.38). Older git rejects that
+ * combo ("unsupported mode"), so fall back to `-c --recurse-submodules` for
+ * expanded submodule files plus a plain `-s` pass for gitlink detection (#1031).
  */
+function listTrackedGitFiles(gitOpts, prefix, files) {
+    const gitlinkRels = [];
+    const ingestStaged = (tracked) => {
+        for (const entry of tracked.split('\0')) {
+            if (!entry)
+                continue;
+            const tab = entry.indexOf('\t');
+            if (tab === -1)
+                continue; // --stage always emits "<mode> <object> <stage>\t<path>"
+            const rel = entry.slice(tab + 1);
+            if (entry.slice(0, 6) === '160000') {
+                gitlinkRels.push(rel); // an unexpanded gitlink — recursed into below, not a source file itself
+                continue;
+            }
+            files.add((0, utils_1.normalizePath)(prefix + rel));
+        }
+    };
+    try {
+        ingestStaged((0, child_process_1.execFileSync)('git', ['ls-files', '-z', '-s', '--recurse-submodules'], gitOpts));
+        return gitlinkRels;
+    }
+    catch {
+        // Older git: submodule expansion and gitlink mode detection need two passes.
+        const cached = (0, child_process_1.execFileSync)('git', ['ls-files', '-z', '-c', '--recurse-submodules'], gitOpts);
+        for (const rel of cached.split('\0')) {
+            if (rel)
+                files.add((0, utils_1.normalizePath)(prefix + rel));
+        }
+        ingestStaged((0, child_process_1.execFileSync)('git', ['ls-files', '-z', '-s'], gitOpts));
+        return gitlinkRels;
+    }
+}
+/** Unexpanded gitlink paths (mode 160000) from `git ls-files --stage`. */
+function listGitlinkRels(gitOpts) {
+    const rels = [];
+    const ingest = (staged) => {
+        for (const entry of staged.split('\0')) {
+            if (!entry || entry.slice(0, 6) !== '160000')
+                continue;
+            const tab = entry.indexOf('\t');
+            if (tab === -1)
+                continue;
+            rels.push(entry.slice(tab + 1));
+        }
+    };
+    try {
+        ingest((0, child_process_1.execFileSync)('git', ['ls-files', '-z', '-s', '--recurse-submodules'], gitOpts));
+    }
+    catch {
+        ingest((0, child_process_1.execFileSync)('git', ['ls-files', '-z', '-s'], gitOpts));
+    }
+    return rels;
+}
 function collectGitFiles(repoDir, prefix, files, embeddedRoots, includeIgnored = null) {
     const gitOpts = { cwd: repoDir, encoding: 'utf-8', timeout: 30000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true };
     // Tracked files. --recurse-submodules pulls in files from active submodules,
@@ -535,15 +904,24 @@ function collectGitFiles(repoDir, prefix, files, embeddedRoots, includeIgnored =
     // Without this, monorepos using submodules index 0 files. (See issue #147.)
     // Note: --recurse-submodules only supports -c/--cached and --stage modes — it
     // can't be combined with -o, so untracked files are gathered separately below.
+    //
+    // We use --stage (-s) rather than -c so each entry carries its file mode. That
+    // lets us spot gitlink entries (mode 160000) that --recurse-submodules did NOT
+    // expand: a nested repo `git add`ed without a `.gitmodules` entry, or a
+    // submodule that isn't active/initialized in this checkout. Such a gitlink
+    // falls through every pass — it's tracked, so the untracked `-o` listing below
+    // never reports it, and --recurse-submodules only expands ACTIVE submodules —
+    // so its source would be silently skipped, leaving only the super-repo's own
+    // files indexed. We collect those gitlinks here and recurse into them below.
+    // (An active submodule is expanded inline by --recurse-submodules and so never
+    // surfaces as a 160000 entry — only the unhandled gitlinks do.) (#1031, #1033)
+    //
     // -z gives NUL-separated, unquoted output so non-ASCII (e.g. CJK) paths
     // survive verbatim. Without it git octal-escapes and double-quotes such paths
     // (the core.quotepath default), and the quoted form never matches a real file
-    // on disk → those files are silently dropped from the index. (#541)
-    const tracked = (0, child_process_1.execFileSync)('git', ['ls-files', '-z', '-c', '--recurse-submodules'], gitOpts);
-    for (const rel of tracked.split('\0')) {
-        if (rel)
-            files.add((0, utils_1.normalizePath)(prefix + rel));
-    }
+    // on disk → those files are silently dropped from the index. (#541) With -s the
+    // path follows a TAB after the `<mode> <object> <stage>` prefix.
+    const gitlinkRels = listTrackedGitFiles(gitOpts, prefix, files);
     // Untracked files (submodules manage their own untracked state). Embedded git
     // repos surface here as a single "subdir/" entry that git refuses to descend
     // into — recurse into those as their own repos so their source gets indexed.
@@ -568,6 +946,32 @@ function collectGitFiles(repoDir, prefix, files, embeddedRoots, includeIgnored =
         }
         files.add((0, utils_1.normalizePath)(prefix + rel));
     }
+    // Gitlink entries (mode 160000) that --recurse-submodules left unexpanded —
+    // an embedded repo `git add`ed without `.gitmodules`, or a submodule not
+    // active/initialized in this checkout. When such a gitlink has a real working
+    // tree on disk it is distinct first-party code we must index as its own
+    // embedded repo: the tracked pass skipped its contents and the untracked pass
+    // never sees it (it's tracked, not "other"). A gitlink with no checkout on disk
+    // (an uninitialized submodule — empty dir, no `.git`) has nothing to index and
+    // is left alone, as is a submodule worktree (a duplicate view, #945). (#1031, #1033)
+    if (gitlinkRels.length > 0) {
+        const defaults = defaultsOnlyIgnore();
+        const repoIgnore = buildDefaultIgnore(repoDir);
+        for (const rel of gitlinkRels) {
+            const relDir = rel.endsWith('/') ? rel : rel + '/';
+            // A gitlink under a gitignored path is respected (not indexed) unless the
+            // project opted it in via `includeIgnored` — keep tracked gitlinks under
+            // the same scope rule as the untracked-ignored kind below (#1065).
+            if (gitlinkEmbeddedRepoSkipped(relDir, prefix, defaults, repoIgnore, includeIgnored))
+                continue;
+            const childDir = path.join(repoDir, rel);
+            // 'embedded' = a real .git checkout on disk; 'worktree' and 'none' are skipped.
+            if (classifyGitDir(childDir) !== 'embedded')
+                continue;
+            embeddedRoots?.add((0, utils_1.normalizePath)(prefix + relDir));
+            collectGitFiles(childDir, prefix + relDir, files, embeddedRoots, includeIgnored);
+        }
+    }
     // Embedded repos hidden by THIS repo's ignore rules (`/packages/` in a
     // super-repo .gitignore) never appear in any listing above. By default they
     // stay hidden — `.gitignore` is respected (#970, #976). They are recursed into
@@ -611,7 +1015,11 @@ function getGitVisibleFiles(rootDir) {
         // not the parent's: the parent's .gitignore hides the child repo from git,
         // not from the index. (#514)
         const ig = buildScopeIgnore(rootDir, embeddedRoots);
-        return new Set([...files].filter((f) => !ig.ignores(f)));
+        const visible = new Set([...files].filter((f) => !ig.ignores(f)));
+        // Force-include first-party source whitelisted in `homegraph.json` `include`.
+        for (const f of collectIncludedFilesForRoot(rootDir))
+            visible.add(f);
+        return visible;
     }
     catch {
         return null;
@@ -637,14 +1045,14 @@ function getGitChangedFiles(rootDir) {
         // Custom extension → language overrides from the project's homegraph.json,
         // so change detection sees the same custom-extension files the full index does.
         const overrides = (0, project_config_1.loadExtensionOverrides)(rootDir);
-        collectGitStatus(rootDir, '', changes, overrides, loadIncludeIgnoredMatcher(rootDir));
+        collectGitStatus(rootDir, '', changes, overrides, loadIncludeIgnoredMatcher(rootDir), loadExcludeMatcher(rootDir));
         return changes;
     }
     catch {
         return null;
     }
 }
-function collectGitStatus(repoDir, prefix, out, overrides, includeIgnored = null) {
+function collectGitStatus(repoDir, prefix, out, overrides, includeIgnored = null, exclude = null) {
     const output = (0, child_process_1.execFileSync)('git', ['status', '--porcelain', '--no-renames'], { cwd: repoDir, encoding: 'utf-8', timeout: 10000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true });
     // This repo's own ignore rules — built-in defaults (#407) plus its .gitignore.
     // Change detection must exclude the SAME files the full index does, but git
@@ -683,6 +1091,12 @@ function collectGitStatus(repoDir, prefix, out, overrides, includeIgnored = null
         // index — match against the repo-relative path, same as the full scan. (#766)
         if (ig.ignores(rel))
             continue;
+        // User `codegraph.json` `exclude` (#999) is project-root-relative, so it's
+        // matched against the full path — sync must not re-add a tracked file the
+        // full index now keeps out. Deletions above stay unfiltered so a file that
+        // WAS indexed before an exclude was added still cleans itself out.
+        if (exclude && exclude.ignores(filePath))
+            continue;
         if (statusCode === '??') {
             out.added.push(filePath);
         }
@@ -697,11 +1111,11 @@ function collectGitStatus(repoDir, prefix, out, overrides, includeIgnored = null
     // and they are left alone (#970, #976), mirroring the full-index scan.
     for (const rel of untrackedDirs) {
         for (const repoRel of findNestedGitRepos(path.join(repoDir, rel), rel)) {
-            collectGitStatus(path.join(repoDir, repoRel), prefix + repoRel, out, overrides, includeIgnored);
+            collectGitStatus(path.join(repoDir, repoRel), prefix + repoRel, out, overrides, includeIgnored, exclude);
         }
     }
     for (const rel of findIgnoredEmbeddedRepos(repoDir, includeIgnored, prefix)) {
-        collectGitStatus(path.join(repoDir, rel), prefix + rel, out, overrides, includeIgnored);
+        collectGitStatus(path.join(repoDir, rel), prefix + rel, out, overrides, includeIgnored, exclude);
     }
 }
 /**
@@ -860,9 +1274,60 @@ function scanDirectoryWalk(rootDir, onProgress) {
     }
     // Seed a base matcher with the built-in default ignores (merged with the root
     // .gitignore so a negation can override). Nested .gitignores still layer per-dir.
-    walk(rootDir, [{ dir: rootDir, ig: buildDefaultIgnore(rootDir) }]);
+    const baseMatchers = [{ dir: rootDir, ig: buildDefaultIgnore(rootDir) }];
+    // Project `codegraph.json` `exclude` patterns (#999), rooted at the project so
+    // `isIgnored` matches them against root-relative paths — same coverage the
+    // git path gets via ScopeIgnore, for non-git projects.
+    const exclude = loadExcludeMatcher(rootDir);
+    if (exclude)
+        baseMatchers.push({ dir: rootDir, ig: exclude });
+    walk(rootDir, baseMatchers);
+    // Force-include first-party source whitelisted in `homegraph.json` `include`.
+    const included = collectIncludedFilesForRoot(rootDir);
+    if (included.size > 0) {
+        const seen = new Set(files);
+        for (const f of included) {
+            if (!seen.has(f)) {
+                files.push(f);
+                seen.add(f);
+            }
+        }
+    }
     return files;
 }
+/**
+ * Extraction orchestrator
+ */
+/**
+ * Resurrect a resolution edge that is about to be dropped (its target symbol
+ * was removed, renamed, or its whole file deleted) as the ORIGINAL unresolved
+ * reference that created it, read from the refName/refKind stamp
+ * `createEdges` writes into edge metadata. Inserted as status='pending', the
+ * ref is consumed by the same sync's resolution sweep: it rebinds to an
+ * alternative definition if one exists, or parks as status='failed' where the
+ * #1240 retry finds it if the symbol later reappears.
+ *
+ * Returns null — drop silently, the pre-#1240 behavior — for edges without a
+ * refName stamp (created before the stamp existed, or synthesized): rebuilding
+ * a ref from the target's plain node name would strip the receiver/qualifier
+ * context the original text carried (`h.greet` → `greet`) and could rebind
+ * somewhere a full re-index never would. Silent beats wrong.
+ */
+function resurrectRefFromDroppedEdge(e) {
+    const refName = e.metadata?.refName;
+    if (typeof refName !== 'string' || refName.length === 0)
+        return null;
+    const refKind = typeof e.metadata?.refKind === 'string' ? e.metadata.refKind : e.kind;
+    return {
+        fromNodeId: e.source,
+        referenceName: refName,
+        referenceKind: refKind,
+        line: e.line ?? 0,
+        column: e.column ?? 0,
+        filePath: e.sourceFilePath,
+        language: e.sourceLanguage,
+    };
+}
 /**
  * Extraction orchestrator
  */
@@ -992,268 +1457,101 @@ class ExtractionOrchestrator {
         // between runs is picked up without restarting the process.
         this.detectedFrameworkNames = null;
         const frameworkNames = this.ensureDetectedFrameworks(files);
-        if (signal?.aborted) {
-            return {
-                success: false,
-                filesIndexed: 0,
-                filesSkipped: 0,
-                filesErrored: 0,
-                nodesCreated: 0,
-                edgesCreated: 0,
-                errors: [{ message: 'Aborted', severity: 'error' }],
-                durationMs: Date.now() - startTime,
-            };
-        }
-        // Phase 2: Parse files in a worker thread (keeps main thread unblocked for UI)
-        const total = files.length;
-        let processed = 0;
-        // Emit parsing phase immediately so the progress bar appears during worker setup.
-        // The yield lets the shimmer worker flush the phase transition to stdout before
-        // the main thread starts synchronous grammar detection work.
-        onProgress?.({
-            phase: 'parsing',
-            current: 0,
-            total,
-        });
-        await new Promise(resolve => setImmediate(resolve));
-        // Detect needed languages and load grammars in the parse worker
-        const neededLanguages = [...new Set(files.map((f) => (0, grammars_1.detectLanguage)(f, undefined, overrides)))];
-        // .h files default to 'c' but may be C++ — ensure cpp grammar is loaded when c is needed
-        if (neededLanguages.includes('c') && !neededLanguages.includes('cpp')) {
-            neededLanguages.push('cpp');
-        }
-        // Try to use a worker thread for parsing (keeps main thread unblocked for UI).
-        // Falls back to in-process parsing if the compiled worker is unavailable (e.g. tests).
-        const parseWorkerPath = path.join(__dirname, 'parse-worker.js');
-        const useWorker = fs.existsSync(parseWorkerPath);
-        let WorkerClass = null;
-        if (useWorker) {
-            const { Worker } = await Promise.resolve().then(() => __importStar(require('worker_threads')));
-            WorkerClass = Worker;
-        }
-        else {
-            // In-process fallback: load grammars locally
-            await (0, grammars_1.loadGrammarsForLanguages)(neededLanguages);
-        }
-        // --- Worker lifecycle management ---
-        // The worker can crash (OOM in WASM) or hang on pathological files.
-        // We track pending parse promises and handle both cases:
-        //   - Timeout: terminate + restart the worker, reject the timed-out request
-        //   - Crash: reject all pending promises, restart for remaining files
-        let parseWorker = null;
-        let nextId = 0;
-        let workerParseCount = 0;
-        const pendingParses = new Map();
-        function rejectAllPending(reason) {
-            for (const [id, pending] of pendingParses) {
-                clearTimeout(pending.timer);
-                pendingParses.delete(id);
-                pending.reject(new Error(reason));
-            }
-        }
-        function attachWorkerHandlers(w) {
-            w.on('message', (msg) => {
-                if (msg.type === 'parse-result' && msg.id !== undefined) {
-                    const pending = pendingParses.get(msg.id);
-                    if (pending) {
-                        clearTimeout(pending.timer);
-                        pendingParses.delete(msg.id);
-                        pending.resolve(msg.result);
-                    }
-                }
-            });
-            w.on('error', (err) => {
-                (0, errors_1.logWarn)('Parse worker error', { error: err.message });
-                rejectAllPending(`Worker error: ${err.message}`);
-            });
-            w.on('exit', (code) => {
-                if (code !== 0 && pendingParses.size > 0) {
-                    (0, errors_1.logWarn)('Parse worker exited unexpectedly', { code });
-                    rejectAllPending(`Worker exited with code ${code}`);
-                }
-                // Clear reference so we know to respawn, reset count so
-                // the fresh worker gets a full cycle before recycling.
-                if (parseWorker === w) {
-                    parseWorker = null;
-                    workerParseCount = 0;
-                }
-            });
-        }
-        async function ensureWorker() {
-            if (parseWorker)
-                return parseWorker;
-            log('Spawning new parse worker...');
-            parseWorker = new WorkerClass(parseWorkerPath);
-            attachWorkerHandlers(parseWorker);
-            // Load grammars in the new worker
-            await new Promise((resolve, reject) => {
-                parseWorker.once('message', (msg) => {
-                    if (msg.type === 'grammars-loaded')
-                        resolve();
-                    else
-                        reject(new Error(`Unexpected message: ${msg.type}`));
-                });
-                parseWorker.postMessage({ type: 'load-grammars', languages: neededLanguages });
-            });
-            return parseWorker;
-        }
-        if (WorkerClass) {
-            await ensureWorker();
-        }
-        /**
-         * Recycle the worker thread to reclaim WASM memory.
-         * Terminates the current worker and clears the reference so
-         * ensureWorker() will spawn a fresh one on the next call.
-         */
-        function recycleWorker() {
-            if (!parseWorker)
-                return;
-            log(`Recycling worker after ${workerParseCount} parses (heap: ${Math.round(process.memoryUsage().rss / 1024 / 1024)}MB RSS)`);
-            const w = parseWorker;
-            parseWorker = null;
-            workerParseCount = 0;
-            // Fire-and-forget: worker.terminate() can hang if WASM is stuck
-            w.terminate().catch(() => { });
-        }
-        async function requestParse(filePath, content) {
-            // Resolve the language on the main thread (where the project's
-            // homegraph.json overrides are loaded) and hand it to the worker, so the
-            // worker never needs the override map itself.
-            const language = (0, grammars_1.detectLanguage)(filePath, content, overrides);
-            if (!WorkerClass || (0, grammars_1.requiresInProcessExtraction)(language)) {
-                // In-process fallback (also required for languages whose extractors
-                // cannot run in a worker thread).
-                return (0, tree_sitter_1.extractFromSource)(filePath, content, language, frameworkNames);
-            }
-            // Recycle the worker before the next parse if we've hit the threshold.
-            // This destroys the WASM linear memory (which can grow but never shrink)
-            // and starts a fresh worker with a clean heap.
-            if (workerParseCount >= WORKER_RECYCLE_INTERVAL) {
-                await recycleWorker();
-            }
-            const worker = await ensureWorker();
-            const id = nextId++;
-            workerParseCount++;
-            // Scale timeout for large files: base 10s + 10s per 100KB
-            const timeoutMs = PARSE_TIMEOUT_MS + Math.floor(content.length / 100_000) * 10_000;
-            return new Promise((resolve, reject) => {
-                const timer = setTimeout(() => {
-                    pendingParses.delete(id);
-                    log(`TIMEOUT: ${filePath} exceeded ${timeoutMs}ms — killing worker`);
-                    // Reject FIRST — worker.terminate() can hang if WASM is stuck
-                    parseWorker = null;
-                    workerParseCount = 0;
-                    reject(new Error(`Parse timed out after ${timeoutMs}ms`));
-                    // Fire-and-forget: kill the stuck worker in the background
-                    worker.terminate().catch(() => { });
-                }, timeoutMs);
-                pendingParses.set(id, { resolve, reject, timer });
-                worker.postMessage({ type: 'parse', id, filePath, content, frameworkNames, language });
+        (0, arkts_1.resetArkTSBatch)();
+        (0, context_1.setArkTSBatchProgressCallback)((batchProgress) => {
+            onProgress?.({
+                phase: 'arkts-batch',
+                current: batchProgress.current,
+                total: batchProgress.total,
+                currentFile: batchProgress.currentFile,
+                subphase: batchProgress.subphase,
             });
-        }
-        for (let i = 0; i < files.length; i += FILE_IO_BATCH_SIZE) {
+        });
+        try {
             if (signal?.aborted) {
-                if (parseWorker)
-                    parseWorker.terminate().catch(() => { });
                 return {
                     success: false,
-                    filesIndexed,
-                    filesSkipped,
-                    filesErrored,
-                    nodesCreated: totalNodes,
-                    edgesCreated: totalEdges,
-                    errors: [{ message: 'Aborted', severity: 'error' }, ...errors],
+                    filesIndexed: 0,
+                    filesSkipped: 0,
+                    filesErrored: 0,
+                    nodesCreated: 0,
+                    edgesCreated: 0,
+                    errors: [{ message: 'Aborted', severity: 'error' }],
                     durationMs: Date.now() - startTime,
                 };
             }
-            const batch = files.slice(i, i + FILE_IO_BATCH_SIZE);
-            // Read files in parallel (with path validation before any I/O)
-            const fileContents = await Promise.all(batch.map(async (fp) => {
-                try {
-                    // Indexing read: follow in-root symlinks the directory walk already
-                    // descended into (the `../` guard still applies) so files reached
-                    // via an in-root symlink-to-outside still index (#935).
-                    const fullPath = (0, utils_1.validatePathWithinRoot)(this.rootDir, fp, { allowSymlinkEscape: true });
-                    if (!fullPath) {
-                        (0, errors_1.logWarn)('Path traversal blocked in batch reader', { filePath: fp });
-                        return { filePath: fp, content: null, stats: null, error: new Error('Path traversal blocked') };
-                    }
-                    const content = await fsp.readFile(fullPath, 'utf-8');
-                    const stats = await fsp.stat(fullPath);
-                    return { filePath: fp, content, stats, error: null };
-                }
-                catch (err) {
-                    return { filePath: fp, content: null, stats: null, error: err };
-                }
-            }));
-            // Send to worker for parsing, store results on main thread
-            for (const { filePath, content, stats, error } of fileContents) {
-                if (signal?.aborted) {
-                    if (parseWorker)
-                        parseWorker.terminate().catch(() => { });
-                    return {
-                        success: false,
-                        filesIndexed,
-                        filesSkipped,
-                        filesErrored,
-                        nodesCreated: totalNodes,
-                        edgesCreated: totalEdges,
-                        errors: [{ message: 'Aborted', severity: 'error' }, ...errors],
-                        durationMs: Date.now() - startTime,
-                    };
-                }
-                // Report progress before parsing (show current file being worked on)
-                onProgress?.({
-                    phase: 'parsing',
-                    current: processed,
-                    total,
-                    currentFile: filePath,
+            // Phase 2: Parse files in a worker thread (keeps main thread unblocked for UI)
+            const total = files.length;
+            let processed = 0;
+            // Emit parsing phase immediately so the progress bar appears during worker setup.
+            // The yield lets the shimmer worker flush the phase transition to stdout before
+            // the main thread starts synchronous grammar detection work.
+            onProgress?.({
+                phase: 'parsing',
+                current: 0,
+                total,
+            });
+            await new Promise(resolve => setImmediate(resolve));
+            // Detect needed languages and load grammars in the parse worker
+            const neededLanguages = [...new Set(files.map((f) => (0, grammars_1.detectLanguage)(f, undefined, overrides)))];
+            // .h files default to 'c' but may be C++ — ensure cpp grammar is loaded when c is needed
+            if (neededLanguages.includes('c') && !neededLanguages.includes('cpp')) {
+                neededLanguages.push('cpp');
+            }
+            // Parse files on a pool of worker threads (keeps the main thread free for UI
+            // and uses every core). Falls back to in-process parsing when the compiled
+            // worker is unavailable (e.g. running from source in tests).
+            const parseWorkerPath = path.join(__dirname, 'parse-worker.js');
+            const useWorker = fs.existsSync(parseWorkerPath);
+            let pool = null;
+            if (useWorker) {
+                // CODEGRAPH_PARSE_WORKERS: explicit worker count; 1 = the old single-worker
+                // behaviour (the conservative rollback). Unset → clamp(cores-1, 1, 8).
+                const poolSize = (0, parse_pool_1.resolveParsePoolSize)(process.env.CODEGRAPH_PARSE_WORKERS, os.cpus().length);
+                pool = new parse_pool_1.ParseWorkerPool({
+                    languages: neededLanguages,
+                    size: poolSize,
+                    workerScriptPath: parseWorkerPath,
+                    recycleInterval: WORKER_RECYCLE_INTERVAL,
+                    parseTimeoutMs: PARSE_TIMEOUT_MS,
+                    log,
                 });
-                if (error || content === null || stats === null) {
-                    processed++;
-                    filesErrored++;
-                    errors.push({
-                        message: `Failed to read file: ${error instanceof Error ? error.message : String(error)}`,
-                        filePath,
-                        severity: 'error',
-                        code: 'read_error',
-                    });
-                    continue;
-                }
-                // Honour MAX_FILE_SIZE. Without this check, vendored generated
-                // headers, minified bundles, and other multi-MB files get indexed,
-                // wasting WASM heap and the worker recycle budget on inputs with no
-                // useful symbols. The single-file extractFile path already enforces
-                // this; the bulk path used to silently skip the check.
-                if (stats.size > MAX_FILE_SIZE) {
-                    processed++;
-                    filesSkipped++;
-                    errors.push({
-                        message: `File exceeds max size (${stats.size} > ${MAX_FILE_SIZE})`,
-                        filePath,
-                        severity: 'warning',
-                        code: 'size_exceeded',
-                    });
-                    onProgress?.({ phase: 'parsing', current: processed, total });
-                    continue;
-                }
-                // Parse in worker thread (main thread stays unblocked).
-                // Wrapped in try/catch to handle worker timeouts and crashes gracefully.
-                let result;
-                try {
-                    result = await requestParse(filePath, content);
-                }
-                catch (parseErr) {
-                    processed++;
-                    filesErrored++;
-                    errors.push({
-                        message: parseErr instanceof Error ? parseErr.message : String(parseErr),
-                        filePath,
-                        severity: 'error',
-                        code: 'parse_error',
-                    });
-                    continue;
+                log(`Parse worker pool: ${poolSize} worker(s)`);
+            }
+            else {
+                // In-process fallback: load grammars locally and parse on the main thread.
+                await (0, grammars_1.loadGrammarsForLanguages)(neededLanguages);
+            }
+            /**
+             * Parse one file: on the pool when available (the promise REJECTS on a worker
+             * crash/timeout — the caller records it and the retry pass re-attempts), or
+             * in-process synchronously as the no-worker fallback. The language is resolved
+             * here on the main thread, where the codegraph.json overrides are loaded.
+             */
+            const parseFile = (filePath, content) => {
+                const language = (0, grammars_1.detectLanguage)(filePath, content, overrides);
+                if (!pool || (0, grammars_1.requiresInProcessExtraction)(language)) {
+                    return Promise.resolve((0, tree_sitter_1.extractFromSource)(filePath, content, language, frameworkNames));
                 }
+                return pool.requestParse({ filePath, content, language, frameworkNames });
+            };
+            // --- Bounded rolling-window dispatch, ordered commit ---
+            // Reads stay batched/parallel; parses run concurrently across the pool; the
+            // SQLite store stays on the main thread (it isn't thread-safe). Crucially we
+            // COMMIT results in original file order, not parse-completion order: the
+            // resolution phase (run after indexing) resolves an ambiguous reference to one
+            // of several same-named candidates by the nodes' DB insertion order, so a
+            // stable commit order keeps the resulting graph deterministic — byte-identical
+            // to the single-worker path — instead of drifting with parse timing. The
+            // `completed` buffer holds at most ~windowSize out-of-order results, so memory
+            // stays bounded.
+            const windowSize = pool ? Math.max(4, pool.size * 2) : 1;
+            const inFlight = new Set();
+            const completed = new Map();
+            let nextSeq = 0; // file-order sequence assigned at dispatch
+            let nextToStore = 0; // cursor: next sequence to commit
+            let aborted = false;
+            const storeResult = (filePath, content, stats, result) => {
                 processed++;
                 // Store in database on main thread (SQLite is not thread-safe)
                 if (result.nodes.length > 0 || result.errors.length === 0) {
@@ -1277,8 +1575,8 @@ class ExtractionOrchestrator {
                 }
                 else {
                     // Files with no symbols but no errors (yaml, twig, properties) are
-                    // tracked at the file level — count them as indexed so the CLI
-                    // doesn't misleadingly report "No files found to index".
+                    // tracked at the file level — count them as indexed so the CLI doesn't
+                    // misleadingly report "No files found to index".
                     const lang = (0, grammars_1.detectLanguage)(filePath, content, overrides);
                     if ((0, grammars_1.isFileLevelOnlyLanguage)(lang)) {
                         filesIndexed++;
@@ -1287,102 +1585,230 @@ class ExtractionOrchestrator {
                         filesSkipped++;
                     }
                 }
-            }
-        }
-        // Report 100% so the progress bar doesn't hang at 99%
-        onProgress?.({
-            phase: 'parsing',
-            current: total,
-            total,
-        });
-        // Yield so the shimmer worker's buffered stdout writes can flush.
-        // Worker thread stdout is proxied through the main thread's event loop,
-        // so synchronous work here blocks the animation from rendering.
-        await new Promise(resolve => setImmediate(resolve));
-        // Retry pass: files that failed due to WASM memory corruption may succeed
-        // on a fresh worker with a clean heap. Recycle before each attempt so
-        // every file gets the absolute cleanest WASM state possible.
-        const retryableErrors = errors.filter((e) => e.code === 'parse_error' && e.filePath &&
-            (e.message.includes('Worker exited') || e.message.includes('memory access out of bounds')));
-        if (retryableErrors.length > 0 && WorkerClass) {
-            log(`Retrying ${retryableErrors.length} files that failed due to WASM memory errors...`);
-            const stillFailing = [];
-            for (const errEntry of retryableErrors) {
-                const filePath = errEntry.filePath;
-                if (signal?.aborted)
-                    break;
-                // Fresh worker for every retry — maximum WASM headroom
-                recycleWorker();
-                let content;
-                try {
-                    const fullPath = (0, utils_1.validatePathWithinRoot)(this.rootDir, filePath);
-                    if (!fullPath)
-                        continue;
-                    content = await fsp.readFile(fullPath, 'utf-8');
-                }
-                catch {
-                    continue;
+                onProgress?.({ phase: 'parsing', current: processed, total, currentFile: filePath });
+            };
+            const recordParseFailure = (filePath, err) => {
+                processed++;
+                filesErrored++;
+                errors.push({
+                    message: err instanceof Error ? err.message : String(err),
+                    filePath,
+                    severity: 'error',
+                    code: 'parse_error',
+                });
+                onProgress?.({ phase: 'parsing', current: processed, total });
+            };
+            // Commit buffered parses to the DB in file order, advancing the cursor over
+            // contiguous completed results. Runs after each parse settles (and once more
+            // after the drain). storeResult / recordParseFailure run here single-threaded,
+            // so shared counters and SQLite writes never race despite parallel parsing.
+            const flushOrdered = () => {
+                if (aborted)
+                    return;
+                while (completed.has(nextToStore)) {
+                    const item = completed.get(nextToStore);
+                    completed.delete(nextToStore);
+                    nextToStore++;
+                    if (item.ok)
+                        storeResult(item.filePath, item.content, item.stats, item.result);
+                    else
+                        recordParseFailure(item.filePath, item.err);
                 }
-                let result;
-                try {
-                    result = await requestParse(filePath, content);
+            };
+            // Dispatch one file's parse (parses run concurrently across the pool), tagged
+            // with its file-order sequence so flushOrdered commits results in order. The
+            // backpressure below bounds how far parsing runs ahead of the in-order commit.
+            const feed = async (filePath, content, stats) => {
+                const seq = nextSeq++;
+                const p = (async () => {
+                    try {
+                        const result = await parseFile(filePath, content);
+                        completed.set(seq, { ok: true, filePath, content, stats, result });
+                    }
+                    catch (parseErr) {
+                        completed.set(seq, { ok: false, filePath, err: parseErr });
+                    }
+                    flushOrdered();
+                })();
+                const tracked = p.finally(() => { inFlight.delete(tracked); });
+                inFlight.add(tracked);
+                // Backpressure on the dispatched-but-not-yet-committed count (in-flight +
+                // buffered), not just in-flight: a slow file sitting at the commit cursor
+                // lets later parses finish and buffer, which would otherwise grow without
+                // bound. Wait for parses to settle (each may advance the cursor) until the
+                // window has room. `inFlight.size > 0` guards against an empty race — the
+                // cursor file is always still in flight when the window is full.
+                while (nextSeq - nextToStore >= windowSize && inFlight.size > 0) {
+                    await Promise.race(inFlight);
                 }
-                catch {
-                    stillFailing.push(errEntry);
-                    continue;
+            };
+            const firstEtsFile = files.find((f) => (0, grammars_1.detectLanguage)(f, undefined, overrides) === 'arkts');
+            if (firstEtsFile) {
+                await (0, arkts_1.primeArkTSBatch)(this.rootDir, this.queries, firstEtsFile);
+                errors.push(...(0, arkts_1.drainArkTSIndexNotices)());
+            }
+            for (let i = 0; i < files.length; i += FILE_IO_BATCH_SIZE) {
+                if (signal?.aborted) {
+                    aborted = true;
+                    break;
                 }
-                if (result.nodes.length > 0 || result.errors.length === 0) {
-                    const language = (0, grammars_1.detectLanguage)(filePath, content, overrides);
-                    const stats = await fsp.stat(path.join(this.rootDir, filePath));
-                    this.storeExtractionResult(filePath, content, language, stats, result);
-                    const idx = errors.indexOf(errEntry);
-                    if (idx >= 0)
-                        errors.splice(idx, 1);
-                    filesErrored--;
-                    filesIndexed++;
-                    totalNodes += result.nodes.length;
-                    totalEdges += result.edges.length;
-                    log(`Retry OK: ${filePath} (${result.nodes.length} nodes)`);
+                const batch = files.slice(i, i + FILE_IO_BATCH_SIZE);
+                // Read files in parallel (with path validation before any I/O)
+                const fileContents = await Promise.all(batch.map(async (fp) => {
+                    if ((0, arkts_1.isArkTSBatchPersisted)(fp)) {
+                        return {
+                            filePath: fp,
+                            content: null,
+                            stats: null,
+                            error: null,
+                            arktsBatchSkipped: true,
+                        };
+                    }
+                    try {
+                        // Indexing read: follow in-root symlinks the directory walk already
+                        // descended into (the `../` guard still applies) so files reached
+                        // via an in-root symlink-to-outside still index (#935).
+                        const fullPath = (0, utils_1.validatePathWithinRoot)(this.rootDir, fp, { allowSymlinkEscape: true });
+                        if (!fullPath) {
+                            (0, errors_1.logWarn)('Path traversal blocked in batch reader', { filePath: fp });
+                            return { filePath: fp, content: null, stats: null, error: new Error('Path traversal blocked') };
+                        }
+                        const content = await fsp.readFile(fullPath, 'utf-8');
+                        const stats = await fsp.stat(fullPath);
+                        return { filePath: fp, content, stats, error: null };
+                    }
+                    catch (err) {
+                        return { filePath: fp, content: null, stats: null, error: err };
+                    }
+                }));
+                // Dispatch each readable file into the bounded parse window; the window
+                // stores results on the main thread as they arrive.
+                for (const fileEntry of fileContents) {
+                    const { filePath, content, stats, error } = fileEntry;
+                    const arktsBatchSkipped = 'arktsBatchSkipped' in fileEntry && fileEntry.arktsBatchSkipped;
+                    if (signal?.aborted) {
+                        aborted = true;
+                        break;
+                    }
+                    if ((0, arkts_1.isArkTSBatchPersisted)(filePath) || arktsBatchSkipped) {
+                        processed++;
+                        filesIndexed++;
+                        onProgress?.({ phase: 'parsing', current: processed, total, currentFile: filePath });
+                        continue;
+                    }
+                    if (!(0, context_1.isArktsBatchRunning)()) {
+                        onProgress?.({
+                            phase: 'parsing',
+                            current: processed,
+                            total,
+                            currentFile: filePath,
+                        });
+                    }
+                    if (error || content === null || stats === null) {
+                        processed++;
+                        filesErrored++;
+                        errors.push({
+                            message: `Failed to read file: ${error instanceof Error ? error.message : String(error)}`,
+                            filePath,
+                            severity: 'error',
+                            code: 'read_error',
+                        });
+                        onProgress?.({ phase: 'parsing', current: processed, total });
+                        continue;
+                    }
+                    // Honour MAX_FILE_SIZE. Without this check, vendored generated
+                    // headers, minified bundles, and other multi-MB files get indexed,
+                    // wasting WASM heap and the worker recycle budget on inputs with no
+                    // useful symbols. The single-file extractFile path already enforces
+                    // this; the bulk path used to silently skip the check.
+                    if (stats.size > MAX_FILE_SIZE) {
+                        processed++;
+                        filesSkipped++;
+                        errors.push({
+                            message: `File exceeds max size (${stats.size} > ${MAX_FILE_SIZE})`,
+                            filePath,
+                            severity: 'warning',
+                            code: 'size_exceeded',
+                        });
+                        onProgress?.({ phase: 'parsing', current: processed, total });
+                        continue;
+                    }
+                    // Parse on the pool (main thread stays unblocked). Errors/timeouts are
+                    // handled inside feed() → recordParseFailure, feeding the retry pass.
+                    await feed(filePath, content, stats);
                 }
+                if (aborted)
+                    break;
+            }
+            // Drain parses still in flight (skip on abort — we tear down below instead),
+            // then commit any results the cursor hasn't reached yet.
+            if (!aborted) {
+                await Promise.all(inFlight);
+                flushOrdered();
+            }
+            if (signal?.aborted || aborted) {
+                if (pool)
+                    await pool.destroy();
+                return {
+                    success: false,
+                    filesIndexed,
+                    filesSkipped,
+                    filesErrored,
+                    filesDiscovered: total,
+                    nodesCreated: totalNodes,
+                    edgesCreated: totalEdges,
+                    errors: [{ message: 'Aborted', severity: 'error' }, ...errors],
+                    durationMs: Date.now() - startTime,
+                };
             }
-            // Last resort: for files that still crash on a clean worker, strip
-            // comment-only lines to reduce WASM memory pressure. Many compiler
-            // test files are 90%+ comments (CHECK directives) that don't contribute
-            // code nodes but consume parser memory.
-            if (stillFailing.length > 0) {
-                log(`${stillFailing.length} files still failing — retrying with comments stripped...`);
-                for (const errEntry of stillFailing) {
+            // Report 100% so the progress bar doesn't hang at 99%
+            onProgress?.({
+                phase: 'parsing',
+                current: total,
+                total,
+            });
+            // Yield so the shimmer worker's buffered stdout writes can flush.
+            // Worker thread stdout is proxied through the main thread's event loop,
+            // so synchronous work here blocks the animation from rendering.
+            await new Promise(resolve => setImmediate(resolve));
+            // Retry pass: files that failed due to WASM memory corruption may succeed
+            // on a fresh worker with a clean heap. Recycle before each attempt so
+            // every file gets the absolute cleanest WASM state possible.
+            const retryableErrors = errors.filter((e) => e.code === 'parse_error' && e.filePath &&
+                (e.message.includes('Worker exited') || e.message.includes('memory access out of bounds')));
+            if (retryableErrors.length > 0 && pool) {
+                log(`Retrying ${retryableErrors.length} files that failed due to WASM memory errors...`);
+                // Fresh WASM heaps for the retry phase. A retry that still crashes its
+                // worker makes the pool respawn it, so later retries keep landing on clean
+                // workers too.
+                pool.recycleAll();
+                const stillFailing = [];
+                for (const errEntry of retryableErrors) {
                     const filePath = errEntry.filePath;
                     if (signal?.aborted)
                         break;
-                    recycleWorker();
-                    let fullContent;
+                    let content;
                     try {
                         const fullPath = (0, utils_1.validatePathWithinRoot)(this.rootDir, filePath);
                         if (!fullPath)
                             continue;
-                        fullContent = await fsp.readFile(fullPath, 'utf-8');
+                        content = await fsp.readFile(fullPath, 'utf-8');
                     }
                     catch {
                         continue;
                     }
-                    // Strip lines that are entirely comments (preserving line numbers
-                    // by replacing with empty lines so node positions stay correct)
-                    const stripped = fullContent
-                        .split('\n')
-                        .map(line => /^\s*\/\//.test(line) ? '' : line)
-                        .join('\n');
                     let result;
                     try {
-                        result = await requestParse(filePath, stripped);
+                        result = await parseFile(filePath, content);
                     }
                     catch {
+                        stillFailing.push(errEntry);
                         continue;
                     }
                     if (result.nodes.length > 0 || result.errors.length === 0) {
-                        const language = (0, grammars_1.detectLanguage)(filePath, fullContent, overrides);
+                        const language = (0, grammars_1.detectLanguage)(filePath, content, overrides);
                         const stats = await fsp.stat(path.join(this.rootDir, filePath));
-                        this.storeExtractionResult(filePath, fullContent, language, stats, result);
+                        this.storeExtractionResult(filePath, content, language, stats, result);
                         const idx = errors.indexOf(errEntry);
                         if (idx >= 0)
                             errors.splice(idx, 1);
@@ -1390,26 +1816,77 @@ class ExtractionOrchestrator {
                         filesIndexed++;
                         totalNodes += result.nodes.length;
                         totalEdges += result.edges.length;
-                        log(`Retry (stripped) OK: ${filePath} (${result.nodes.length} nodes)`);
+                        log(`Retry OK: ${filePath} (${result.nodes.length} nodes)`);
+                    }
+                }
+                // Last resort: for files that still crash on a clean worker, strip
+                // comment-only lines to reduce WASM memory pressure. Many compiler
+                // test files are 90%+ comments (CHECK directives) that don't contribute
+                // code nodes but consume parser memory.
+                if (stillFailing.length > 0) {
+                    log(`${stillFailing.length} files still failing — retrying with comments stripped...`);
+                    pool.recycleAll();
+                    for (const errEntry of stillFailing) {
+                        const filePath = errEntry.filePath;
+                        if (signal?.aborted)
+                            break;
+                        let fullContent;
+                        try {
+                            const fullPath = (0, utils_1.validatePathWithinRoot)(this.rootDir, filePath);
+                            if (!fullPath)
+                                continue;
+                            fullContent = await fsp.readFile(fullPath, 'utf-8');
+                        }
+                        catch {
+                            continue;
+                        }
+                        // Strip lines that are entirely comments (preserving line numbers
+                        // by replacing with empty lines so node positions stay correct)
+                        const stripped = fullContent
+                            .split('\n')
+                            .map(line => /^\s*\/\//.test(line) ? '' : line)
+                            .join('\n');
+                        let result;
+                        try {
+                            result = await parseFile(filePath, stripped);
+                        }
+                        catch {
+                            continue;
+                        }
+                        if (result.nodes.length > 0 || result.errors.length === 0) {
+                            const language = (0, grammars_1.detectLanguage)(filePath, fullContent, overrides);
+                            const stats = await fsp.stat(path.join(this.rootDir, filePath));
+                            this.storeExtractionResult(filePath, fullContent, language, stats, result);
+                            const idx = errors.indexOf(errEntry);
+                            if (idx >= 0)
+                                errors.splice(idx, 1);
+                            filesErrored--;
+                            filesIndexed++;
+                            totalNodes += result.nodes.length;
+                            totalEdges += result.edges.length;
+                            log(`Retry (stripped) OK: ${filePath} (${result.nodes.length} nodes)`);
+                        }
                     }
                 }
             }
+            // Shut down the parse worker pool.
+            if (pool)
+                await pool.destroy();
+            return {
+                success: filesIndexed > 0 || errors.filter((e) => e.severity === 'error').length === 0,
+                filesIndexed,
+                filesSkipped,
+                filesErrored,
+                filesDiscovered: total,
+                nodesCreated: totalNodes,
+                edgesCreated: totalEdges,
+                errors,
+                durationMs: Date.now() - startTime,
+            };
         }
-        // Shut down parse worker and clear any pending timers
-        rejectAllPending('Indexing complete');
-        if (parseWorker) {
-            parseWorker.terminate().catch(() => { });
+        finally {
+            (0, context_1.setArkTSBatchProgressCallback)(null);
         }
-        return {
-            success: filesIndexed > 0 || errors.filter((e) => e.severity === 'error').length === 0,
-            filesIndexed,
-            filesSkipped,
-            filesErrored,
-            nodesCreated: totalNodes,
-            edgesCreated: totalEdges,
-            errors,
-            durationMs: Date.now() - startTime,
-        };
     }
     /**
      * Index specific files
@@ -1532,6 +2009,15 @@ class ExtractionOrchestrator {
         }
         // Detect language (honoring the project's homegraph.json extension overrides)
         const language = (0, grammars_1.detectLanguage)(relativePath, content, (0, project_config_1.loadExtensionOverrides)(this.rootDir));
+        if ((0, arkts_1.isArkTSBatchPersisted)(relativePath)) {
+            return {
+                nodes: [],
+                edges: [],
+                unresolvedReferences: [],
+                errors: [],
+                durationMs: 0,
+            };
+        }
         if (!(0, grammars_1.isLanguageSupported)(language)) {
             return {
                 nodes: [],
@@ -1616,15 +2102,24 @@ class ExtractionOrchestrator {
                 newNodesByKindName.set(`${n.kind}\0${n.name}`, n.id);
             }
             const reinserted = [];
+            const resurrected = [];
             for (const e of crossFileIncomingEdges) {
                 const newTargetId = newNodesByKindName.get(`${e.targetKind}\0${e.targetName}`);
                 if (newTargetId) {
                     reinserted.push({ source: e.source, target: newTargetId, kind: e.kind, metadata: e.metadata, line: e.line, column: e.column, provenance: e.provenance });
                 }
+                else {
+                    const ref = resurrectRefFromDroppedEdge(e);
+                    if (ref)
+                        resurrected.push(ref);
+                }
             }
             if (reinserted.length > 0) {
                 this.queries.insertEdges(reinserted);
             }
+            if (resurrected.length > 0) {
+                this.queries.insertUnresolvedRefsBatch(resurrected);
+            }
         }
         // Insert unresolved references in batch with denormalized filePath/language
         if (result.unresolvedReferences.length > 0) {
@@ -1701,6 +2196,17 @@ class ExtractionOrchestrator {
         let reconcileChecks = 0;
         for (const tracked of trackedFiles) {
             if (!currentSet.has(tracked.path) || !fs.existsSync(path.join(this.rootDir, tracked.path))) {
+                // Before the cascade deletes them, resurrect incoming cross-file
+                // resolution edges as their original refs (#1240 removal case).
+                const incoming = this.queries.getCrossFileIncomingEdgesWithTarget(tracked.path);
+                if (incoming.length > 0) {
+                    const resurrected = incoming
+                        .map((e) => resurrectRefFromDroppedEdge(e))
+                        .filter((r) => r !== null);
+                    if (resurrected.length > 0) {
+                        this.queries.insertUnresolvedRefsBatch(resurrected);
+                    }
+                }
                 this.queries.deleteFile(tracked.path);
                 filesRemoved++;
             }
@@ -1714,6 +2220,11 @@ class ExtractionOrchestrator {
             // synchronous-stat loop that wedges the main thread on a large repo (#905).
             // Yield at the top of the body so the `continue` fast-paths below still hit it.
             if (++reconcileChecks % SYNC_RECONCILE_YIELD_INTERVAL === 0) {
+                onProgress?.({
+                    phase: 'scanning',
+                    current: reconcileChecks,
+                    total: currentFiles.length,
+                });
                 await new Promise((resolve) => setImmediate(resolve));
             }
             const fullPath = path.join(this.rootDir, filePath);
@@ -1768,16 +2279,47 @@ class ExtractionOrchestrator {
         }
         // Index changed files
         const total = filesToIndex.length;
-        for (let i = 0; i < filesToIndex.length; i++) {
-            const filePath = filesToIndex[i];
-            onProgress?.({
-                phase: 'parsing',
-                current: i + 1,
-                total,
-                currentFile: filePath,
+        const overrides = (0, project_config_1.loadExtensionOverrides)(this.rootDir);
+        const needsArkTS = filesToIndex.some((f) => (0, grammars_1.detectLanguage)(f, undefined, overrides) === 'arkts');
+        if (needsArkTS) {
+            (0, context_1.setArkTSBatchProgressCallback)((batchProgress) => {
+                onProgress?.({
+                    phase: 'arkts-batch',
+                    current: batchProgress.current,
+                    total: batchProgress.total,
+                    currentFile: batchProgress.currentFile,
+                    subphase: batchProgress.subphase,
+                });
             });
-            const result = await this.indexFile(filePath);
-            nodesUpdated += result.nodes.length;
+            const firstEts = filesToIndex.find((f) => (0, grammars_1.detectLanguage)(f, undefined, overrides) === 'arkts');
+            await (0, arkts_1.primeArkTSBatch)(this.rootDir, this.queries, firstEts);
+        }
+        try {
+            for (let i = 0; i < filesToIndex.length; i++) {
+                const filePath = filesToIndex[i];
+                if ((0, arkts_1.isArkTSBatchPersisted)(filePath)) {
+                    onProgress?.({
+                        phase: 'parsing',
+                        current: i + 1,
+                        total,
+                        currentFile: filePath,
+                    });
+                    continue;
+                }
+                onProgress?.({
+                    phase: 'parsing',
+                    current: i + 1,
+                    total,
+                    currentFile: filePath,
+                });
+                const result = await this.indexFile(filePath);
+                nodesUpdated += result.nodes.length;
+            }
+        }
+        finally {
+            if (needsArkTS) {
+                (0, context_1.setArkTSBatchProgressCallback)(null);
+            }
         }
         return {
             filesChecked,