homegraph 1.4.1 → 1.5.0

package/dist/extraction/index.js

@@ -41,9 +41,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.loadAllGrammars = exports.loadGrammarsForLanguages = exports.initGrammars = exports.getSupportedLanguages = exports.isGrammarLoaded = exports.isLanguageSupported = exports.isSourceFile = exports.detectLanguage = exports.extractFromSource = exports.ExtractionOrchestrator = exports.ScopeIgnore = void 0;
+exports.loadAllGrammars = exports.loadGrammarsForLanguages = exports.initGrammars = exports.getSupportedLanguages = exports.isGrammarLoaded = exports.isLanguageSupported = exports.isSourceFile = exports.detectLanguage = exports.extractFromSource = exports.ExtractionOrchestrator = exports.ScopeIgnore = exports.buildDefaultIgnore = void 0;
 exports.hashContent = hashContent;
-exports.buildDefaultIgnore = buildDefaultIgnore;
 exports.buildScopeIgnore = buildScopeIgnore;
 exports.discoverEmbeddedRepoRoots = discoverEmbeddedRepoRoots;
 exports.findUnindexedIgnoredRepos = findUnindexedIgnoredRepos;
@@ -57,12 +56,17 @@ const crypto = __importStar(require("crypto"));
 const child_process_1 = require("child_process");
 const tree_sitter_1 = require("./tree-sitter");
 const parse_pool_1 = require("./parse-pool");
+const store_writer_1 = require("./store-writer");
+const cooperative_yield_1 = require("../resolution/cooperative-yield");
 const grammars_1 = require("./grammars");
 const project_config_1 = require("../project-config");
 const directory_1 = require("../directory");
 const errors_1 = require("../errors");
 const utils_1 = require("../utils");
 const ignore_1 = __importDefault(require("ignore"));
+const default_ignore_1 = require("./default-ignore");
+var default_ignore_2 = require("./default-ignore");
+Object.defineProperty(exports, "buildDefaultIgnore", { enumerable: true, get: function () { return default_ignore_2.buildDefaultIgnore; } });
 const frameworks_1 = require("../resolution/frameworks");
 const context_1 = require("./context");
 const arkts_1 = require("./languages/arkts");
@@ -109,176 +113,6 @@ function hashContent(content) {
  * symbols. 1 MB covers essentially all hand-written source.
  */
 const MAX_FILE_SIZE = 1024 * 1024;
-/**
- * Directory names that are dependency, build, cache, or tooling output across the
- * languages/frameworks HomeGraph supports — curated from the canonical
- * github/gitignore templates. Excluded by default so the graph reflects your code,
- * not third-party noise, without requiring a `.gitignore` (issue #407). The
- * exclusion applies uniformly (git or not, tracked or not); the only opt-in is an
- * explicit `.gitignore` negation (e.g. `!vendor/`). First-party-prone or generic
- * names (`packages`, `lib`, `app`, `bin`, `src`, `deps`, `env`, `tmp`, `storage`,
- * `Library`) are deliberately NOT listed, to avoid ever hiding real source.
- *
- * Only dirs that actually contain *indexable source* (or are enormous) earn a slot
- * — IDE/state dirs like `.idea`/`.vs` are omitted because HomeGraph indexes only
- * recognized source extensions, so they produce no symbols regardless.
- */
-const DEFAULT_IGNORE_DIRS = new Set([
-    // JS / TS — dependency directories
-    'node_modules', 'bower_components', 'jspm_packages', 'web_modules',
-    '.yarn', '.pnpm-store',
-    // JS / TS — framework & bundler build / cache / deploy output
-    '.next', '.nuxt', '.svelte-kit', '.turbo', '.vite', '.parcel-cache', '.angular',
-    '.docusaurus', 'storybook-static', '.vinxi', '.nitro', 'out-tsc',
-    '.vercel', '.netlify', '.wrangler',
-    // Build output (common across ecosystems)
-    'dist', 'build', 'out', '.output',
-    // Test / coverage
-    'coverage', '.nyc_output',
-    // Python
-    '__pycache__', '__pypackages__', '.venv', 'venv', '.pixi', '.pdm-build',
-    '.mypy_cache', '.pytest_cache', '.ruff_cache', '.tox', '.nox', '.hypothesis',
-    '.ipynb_checkpoints', '.eggs',
-    // Rust / JVM (Maven, Gradle, Scala)
-    'target', '.gradle',
-    // .NET
-    'obj',
-    // Vendored deps (Go, PHP/Composer, Ruby/Bundler)
-    'vendor',
-    // Swift / iOS
-    '.build', 'Pods', 'Carthage', 'DerivedData', '.swiftpm',
-    // Dart / Flutter
-    '.dart_tool', '.pub-cache',
-    // Native (Android NDK, C/C++ deps)
-    '.cxx', '.externalNativeBuild', 'vcpkg_installed',
-    // Scala tooling
-    '.bloop', '.metals',
-    // Lua / Luau (LuaRocks)
-    'lua_modules', '.luarocks',
-    // Delphi / RAD Studio IDE backups (duplicate .pas source — would double-count)
-    '__history', '__recovery',
-    // Generic cache
-    '.cache',
-]);
-/**
- * Android resource directory types. A `res/` tree holds ONLY non-code resources —
- * layouts, drawables, value bags (strings/colors/styles), menus, navigation
- * graphs — split into one typed subdirectory per kind, optionally density/locale/
- * version-qualified (`values-es`, `drawable-hdpi`, `layout-v21`, …). None of it
- * yields an extractable code symbol, yet on an Android app it DOMINATES the tree
- * (one report: 26k XML files = 97% of the project, 0 symbols), bloating the DB,
- * slowing indexing, and skewing both the file count and `codegraph_explore`
- * results (#1047). So these are excluded by default. The structure is
- * self-identifying — a non-Android project has no `res/layout/` etc., so it's
- * untouched — and the only XML that DOES produce symbols (MyBatis mappers) lives
- * under `src/main/resources/`, never `res/`, so nothing useful is dropped.
- * `res/raw/` is deliberately NOT here: it holds arbitrary bundled assets that can
- * be code-ish (a `.sql` schema, a `.js`), so we leave it indexed. Override any of
- * these with a `.gitignore` negation (e.g. `!res/values/`).
- */
-const ANDROID_RES_TYPES = [
-    'anim', 'animator', 'color', 'drawable', 'font', 'layout',
-    'menu', 'mipmap', 'navigation', 'transition', 'values', 'xml',
-];
-/** Gitignore-style patterns for the `ignore` matcher: the dirs above plus a few globs. */
-const DEFAULT_IGNORE_PATTERNS = [
-    ...Array.from(DEFAULT_IGNORE_DIRS, (d) => `${d}/`),
-    '*.egg-info/', // Python packaging metadata
-    'cmake-build-*/', // CLion / CMake build trees
-    'bazel-*/', // Bazel output symlink trees
-    // Android resource dirs at any depth, with their qualifier variants (#1047).
-    ...ANDROID_RES_TYPES.map((t) => `**/res/${t}*/`),
-];
-/** True if `buf` decodes as strict UTF-8 (no invalid byte sequences). */
-function isValidUtf8(buf) {
-    try {
-        new TextDecoder('utf-8', { fatal: true }).decode(buf);
-        return true;
-    }
-    catch {
-        return false;
-    }
-}
-/**
- * Read a `.gitignore` and return patterns safe to hand to the `ignore` matcher —
- * never throwing, even when the file isn't real gitignore text. Two failure
- * modes, both seen in the wild (issue #682):
- *
- *  - The file isn't valid UTF-8 — e.g. transparently encrypted in place by
- *    corporate DLP / endpoint-security software, leaving a UTF-16 header plus
- *    ciphertext. None of it is meaningful patterns, so the whole file is skipped.
- *  - The file is text but a single line can't be compiled to a regex by the
- *    `ignore` library — `\\[` and friends throw "Unterminated character class".
- *    Crucially the throw is LAZY (at match time, not `.add()`), so it would
- *    otherwise escape mid-scan. That one pattern is dropped; the rest are kept.
- *
- * Either way a warning that NAMES the file is logged (the reporter couldn't tell
- * which `.gitignore` was at fault) and indexing continues instead of aborting.
- * Returns '' when there's nothing usable.
- */
-function readGitignorePatterns(giPath) {
-    let buf;
-    try {
-        buf = fs.readFileSync(giPath);
-    }
-    catch {
-        return ''; // unreadable (permissions / race) — treat as absent
-    }
-    // A NUL byte never appears in real gitignore text, and a fatal UTF-8 decode
-    // catches the rest. Such a file isn't ignore patterns at all.
-    if (buf.includes(0) || !isValidUtf8(buf)) {
-        (0, errors_1.logWarn)('Ignoring a .gitignore that is not valid UTF-8 text — it may have been encrypted ' +
-            'in place by endpoint-security software. Indexing continues without it.', { file: giPath });
-        return '';
-    }
-    const content = buf.toString('utf-8');
-    // Fast path: one `.ignores()` call forces the library to compile EVERY rule,
-    // so if it doesn't throw, the whole file is safe to use verbatim.
-    try {
-        (0, ignore_1.default)().add(content).ignores('.homegraph-probe');
-        return content;
-    }
-    catch {
-        // Fall through: a line is uncompilable — keep the good ones, drop the bad.
-    }
-    const kept = [];
-    let dropped = 0;
-    for (const line of content.split(/\r?\n/)) {
-        try {
-            (0, ignore_1.default)().add(line).ignores('.homegraph-probe');
-            kept.push(line);
-        }
-        catch {
-            dropped++;
-        }
-    }
-    if (dropped > 0) {
-        (0, errors_1.logWarn)(`Skipped ${dropped} unparseable pattern(s) in a .gitignore; the rest are applied.`, { file: giPath });
-    }
-    return kept.join('\n');
-}
-/**
- * An `ignore` matcher seeded with the built-in defaults, merged with the project's
- * root .gitignore so a negation there (e.g. `!vendor/`) overrides a default. Shared
- * by both enumeration paths so behavior is identical with or without git — and so
- * the defaults apply to tracked files too (committing a dependency dir doesn't make
- * it project code; the explicit `.gitignore` negation is the only opt-in).
- */
-function buildDefaultIgnore(rootDir) {
-    const ig = (0, ignore_1.default)().add(DEFAULT_IGNORE_PATTERNS);
-    const rootGitignore = path.join(rootDir, '.gitignore');
-    if (fs.existsSync(rootGitignore))
-        ig.add(readGitignorePatterns(rootGitignore));
-    return ig;
-}
-/**
- * Defaults-only ignore matcher (no root `.gitignore` merged). Used wherever the
- * parent repo's own ignore rules must NOT apply — inside embedded child repos,
- * whose gitignore semantics their own `git ls-files` already enforced (#514).
- */
-function defaultsOnlyIgnore() {
-    return (0, ignore_1.default)().add(DEFAULT_IGNORE_PATTERNS);
-}
 /**
  * Matcher for the project's `homegraph.json` `includeIgnored` patterns — the
  * explicit opt-in to index embedded git repos living inside gitignored
@@ -377,7 +211,7 @@ function includeStaticRoots(patterns) {
  */
 function collectIncludedFiles(rootDir, include, exclude, roots, overrides) {
     const out = new Set();
-    const defaults = defaultsOnlyIgnore();
+    const defaults = (0, default_ignore_1.defaultsOnlyIgnore)();
     const visited = new Set();
     const consider = (abs, rel, isDir) => {
         if (isDir) {
@@ -544,7 +378,7 @@ function classifyGitDir(absDir) {
  */
 function findNestedGitRepos(absDir, relPrefix) {
     const found = [];
-    const defaults = defaultsOnlyIgnore();
+    const defaults = (0, default_ignore_1.defaultsOnlyIgnore)();
     const queue = [
         { abs: absDir, rel: relPrefix, depth: 0 },
     ];
@@ -602,7 +436,7 @@ class ScopeIgnore {
     include;
     includeRoots;
     embedded;
-    defaults = defaultsOnlyIgnore();
+    defaults = (0, default_ignore_1.defaultsOnlyIgnore)();
     constructor(rootMatcher, embedded, 
     /**
      * Project `homegraph.json` `exclude` patterns (#999), matched against the
@@ -674,7 +508,7 @@ exports.ScopeIgnore = ScopeIgnore;
 function buildScopeIgnore(rootDir, embeddedRoots) {
     const roots = embeddedRoots ? [...embeddedRoots] : discoverEmbeddedRepoRoots(rootDir);
     const include = loadIncludeMatcher(rootDir);
-    return new ScopeIgnore(buildDefaultIgnore(rootDir), roots.map((root) => ({ root, matcher: buildDefaultIgnore(path.join(rootDir, root)) })), loadExcludeMatcher(rootDir), include, include ? includeStaticRoots((0, project_config_1.loadIncludePatterns)(rootDir)) : []);
+    return new ScopeIgnore((0, default_ignore_1.buildDefaultIgnore)(rootDir), roots.map((root) => ({ root, matcher: (0, default_ignore_1.buildDefaultIgnore)(path.join(rootDir, root)) })), loadExcludeMatcher(rootDir), include, include ? includeStaticRoots((0, project_config_1.loadIncludePatterns)(rootDir)) : []);
 }
 /**
  * Whether an embedded repo found as a tracked gitlink (mode 160000, #1031/#1033)
@@ -722,7 +556,7 @@ function discoverEmbeddedRepoRoots(rootDir) {
         return [];
     }
     const out = [];
-    const defaults = defaultsOnlyIgnore();
+    const defaults = (0, default_ignore_1.defaultsOnlyIgnore)();
     const includeIgnored = loadIncludeIgnoredMatcher(rootDir);
     const visit = (repoAbs, prefix) => {
         const candidates = [];
@@ -742,7 +576,7 @@ function discoverEmbeddedRepoRoots(rootDir) {
         // (#1031, #1033)
         try {
             const gitOpts = { cwd: repoAbs, encoding: 'utf-8', timeout: 30000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true };
-            const repoIgnore = buildDefaultIgnore(repoAbs);
+            const repoIgnore = (0, default_ignore_1.buildDefaultIgnore)(repoAbs);
             for (const rel of listGitlinkRels(gitOpts)) {
                 const relDir = rel.endsWith('/') ? rel : rel + '/';
                 // A gitlink under a gitignored path is respected (not indexed) unless the
@@ -792,7 +626,7 @@ function findUnindexedIgnoredRepos(rootDir) {
     catch {
         return [];
     }
-    const defaults = defaultsOnlyIgnore();
+    const defaults = (0, default_ignore_1.defaultsOnlyIgnore)();
     const includeIgnored = loadIncludeIgnoredMatcher(rootDir);
     const repos = [];
     for (const dir of listIgnoredDirs(rootDir)) {
@@ -826,7 +660,7 @@ function findUnindexedIgnoredRepos(rootDir) {
 function findIgnoredEmbeddedRepos(repoDir, includeIgnored, prefix) {
     if (!includeIgnored)
         return [];
-    const defaults = defaultsOnlyIgnore();
+    const defaults = (0, default_ignore_1.defaultsOnlyIgnore)();
     const repos = [];
     for (const dir of listIgnoredDirs(repoDir)) {
         if (defaults.ignores(dir))
@@ -938,7 +772,7 @@ function collectGitFiles(repoDir, prefix, files, embeddedRoots, includeIgnored =
             const childDir = path.join(repoDir, rel);
             // A git worktree surfaces here as an opaque untracked dir too — skip it,
             // it's a duplicate working view of an already-indexed repo (#848).
-            if (classifyGitDir(childDir) === 'embedded' && !defaultsOnlyIgnore().ignores(rel)) {
+            if (classifyGitDir(childDir) === 'embedded' && !(0, default_ignore_1.defaultsOnlyIgnore)().ignores(rel)) {
                 embeddedRoots?.add((0, utils_1.normalizePath)(prefix + rel));
                 collectGitFiles(childDir, prefix + rel, files, embeddedRoots, includeIgnored);
             }
@@ -955,8 +789,8 @@ function collectGitFiles(repoDir, prefix, files, embeddedRoots, includeIgnored =
     // (an uninitialized submodule — empty dir, no `.git`) has nothing to index and
     // is left alone, as is a submodule worktree (a duplicate view, #945). (#1031, #1033)
     if (gitlinkRels.length > 0) {
-        const defaults = defaultsOnlyIgnore();
-        const repoIgnore = buildDefaultIgnore(repoDir);
+        const defaults = (0, default_ignore_1.defaultsOnlyIgnore)();
+        const repoIgnore = (0, default_ignore_1.buildDefaultIgnore)(repoDir);
         for (const rel of gitlinkRels) {
             const relDir = rel.endsWith('/') ? rel : rel + '/';
             // A gitlink under a gitignored path is respected (not indexed) unless the
@@ -1064,7 +898,7 @@ function collectGitStatus(repoDir, prefix, out, overrides, includeIgnored = null
     // repo-relative `rel` at each recursion level mirrors getGitVisibleFiles'
     // ScopeIgnore: every embedded repo is judged by ITS OWN rules, never the
     // parent's. (#766)
-    const ig = buildDefaultIgnore(repoDir);
+    const ig = (0, default_ignore_1.buildDefaultIgnore)(repoDir);
     const untrackedDirs = [];
     for (const line of output.split('\n')) {
         if (line.length < 4)
@@ -1187,7 +1021,7 @@ function scanDirectoryWalk(rootDir, onProgress) {
         // readGitignorePatterns is defensive: a non-UTF-8 (DLP-encrypted) or
         // uncompilable .gitignore is skipped/filtered with a warning, never thrown
         // (issue #682) — so the per-file `.ignores()` calls below can't crash.
-        const patterns = readGitignorePatterns(giPath);
+        const patterns = (0, default_ignore_1.readGitignorePatterns)(giPath);
         return patterns ? { dir, ig: (0, ignore_1.default)().add(patterns) } : null;
     };
     const isIgnored = (fullPath, isDir, matchers) => {
@@ -1274,7 +1108,7 @@ function scanDirectoryWalk(rootDir, onProgress) {
     }
     // Seed a base matcher with the built-in default ignores (merged with the root
     // .gitignore so a negation can override). Nested .gitignores still layer per-dir.
-    const baseMatchers = [{ dir: rootDir, ig: buildDefaultIgnore(rootDir) }];
+    const baseMatchers = [{ dir: rootDir, ig: (0, default_ignore_1.buildDefaultIgnore)(rootDir) }];
     // Project `codegraph.json` `exclude` patterns (#999), rooted at the project so
     // `isIgnored` matches them against root-relative paths — same coverage the
     // git path gets via ScopeIgnore, for non-git projects.
@@ -1420,7 +1254,7 @@ class ExtractionOrchestrator {
     /**
      * Index all files in the project
      */
-    async indexAll(onProgress, signal, verbose) {
+    async indexAll(onProgress, signal, verbose, walBackpressure, storeWriterOpts) {
         await (0, grammars_1.initGrammars)();
         const startTime = Date.now();
         const errors = [];
@@ -1522,6 +1356,15 @@ class ExtractionOrchestrator {
                 // In-process fallback: load grammars locally and parse on the main thread.
                 await (0, grammars_1.loadGrammarsForLanguages)(neededLanguages);
             }
+            const storeWorkerPath = path.join(__dirname, 'store-worker.js');
+            let storeWriter = null;
+            if (storeWriterOpts &&
+                process.env.HOMEGRAPH_NO_STORE_WORKER !== '1' &&
+                fs.existsSync(storeWorkerPath)) {
+                storeWriter = new store_writer_1.StoreWriter(storeWorkerPath, storeWriterOpts.dbPath, storeWriterOpts.fastInit);
+                log('Store writer thread active');
+            }
+            const STORE_WRITER_WINDOW = 64;
             /**
              * Parse one file: on the pool when available (the promise REJECTS on a worker
              * crash/timeout — the caller records it and the retry pass re-attempts), or
@@ -1551,12 +1394,21 @@ class ExtractionOrchestrator {
             let nextSeq = 0; // file-order sequence assigned at dispatch
             let nextToStore = 0; // cursor: next sequence to commit
             let aborted = false;
-            const storeResult = (filePath, content, stats, result) => {
+            const commitYield = (0, cooperative_yield_1.createYielder)();
+            const storeResult = async (filePath, content, stats, result) => {
                 processed++;
-                // Store in database on main thread (SQLite is not thread-safe)
+                const bp = walBackpressure?.();
+                if (bp)
+                    await bp;
                 if (result.nodes.length > 0 || result.errors.length === 0) {
                     const language = (0, grammars_1.detectLanguage)(filePath, content, overrides);
-                    this.storeExtractionResult(filePath, content, language, stats, result);
+                    if (storeWriter) {
+                        storeWriter.send(this.buildFreshStoreBundle(filePath, content, language, stats, result));
+                        await storeWriter.waitBelow(STORE_WRITER_WINDOW);
+                    }
+                    else {
+                        await this.storeExtractionResult(filePath, content, language, stats, result, commitYield);
+                    }
                 }
                 if (result.errors.length > 0) {
                     for (const err of result.errors) {
@@ -1574,9 +1426,6 @@ class ExtractionOrchestrator {
                     filesErrored++;
                 }
                 else {
-                    // Files with no symbols but no errors (yaml, twig, properties) are
-                    // tracked at the file level — count them as indexed so the CLI doesn't
-                    // misleadingly report "No files found to index".
                     const lang = (0, grammars_1.detectLanguage)(filePath, content, overrides);
                     if ((0, grammars_1.isFileLevelOnlyLanguage)(lang)) {
                         filesIndexed++;
@@ -1598,22 +1447,28 @@ class ExtractionOrchestrator {
                 });
                 onProgress?.({ phase: 'parsing', current: processed, total });
             };
-            // Commit buffered parses to the DB in file order, advancing the cursor over
-            // contiguous completed results. Runs after each parse settles (and once more
-            // after the drain). storeResult / recordParseFailure run here single-threaded,
-            // so shared counters and SQLite writes never race despite parallel parsing.
+            let flushChain = Promise.resolve();
+            let flushError = null;
             const flushOrdered = () => {
-                if (aborted)
-                    return;
-                while (completed.has(nextToStore)) {
-                    const item = completed.get(nextToStore);
-                    completed.delete(nextToStore);
-                    nextToStore++;
-                    if (item.ok)
-                        storeResult(item.filePath, item.content, item.stats, item.result);
-                    else
-                        recordParseFailure(item.filePath, item.err);
-                }
+                flushChain = flushChain.then(async () => {
+                    if (aborted || flushError)
+                        return;
+                    try {
+                        while (completed.has(nextToStore)) {
+                            const item = completed.get(nextToStore);
+                            completed.delete(nextToStore);
+                            nextToStore++;
+                            if (item.ok)
+                                await storeResult(item.filePath, item.content, item.stats, item.result);
+                            else
+                                recordParseFailure(item.filePath, item.err);
+                        }
+                    }
+                    catch (err) {
+                        flushError = err;
+                    }
+                });
+                return flushChain;
             };
             // Dispatch one file's parse (parses run concurrently across the pool), tagged
             // with its file-order sequence so flushOrdered commits results in order. The
@@ -1636,10 +1491,15 @@ class ExtractionOrchestrator {
                 // buffered), not just in-flight: a slow file sitting at the commit cursor
                 // lets later parses finish and buffer, which would otherwise grow without
                 // bound. Wait for parses to settle (each may advance the cursor) until the
-                // window has room. `inFlight.size > 0` guards against an empty race — the
-                // cursor file is always still in flight when the window is full.
-                while (nextSeq - nextToStore >= windowSize && inFlight.size > 0) {
-                    await Promise.race(inFlight);
+                // window has room. When nothing is in flight but the window is still full,
+                // the async commit chain is what's behind — await it so the cursor
+                // advances (buffered items hold whole file contents, so this bound is
+                // load-bearing for memory).
+                while (nextSeq - nextToStore >= windowSize) {
+                    if (inFlight.size > 0)
+                        await Promise.race(inFlight);
+                    else
+                        await flushOrdered();
                 }
             };
             const firstEtsFile = files.find((f) => (0, grammars_1.detectLanguage)(f, undefined, overrides) === 'arkts');
@@ -1744,9 +1604,25 @@ class ExtractionOrchestrator {
             // then commit any results the cursor hasn't reached yet.
             if (!aborted) {
                 await Promise.all(inFlight);
-                flushOrdered();
+                await flushOrdered();
+                if (flushError) {
+                    if (storeWriter)
+                        await storeWriter.close();
+                    throw flushError;
+                }
+                if (storeWriter) {
+                    try {
+                        await storeWriter.drain();
+                    }
+                    finally {
+                        await storeWriter.close();
+                        storeWriter = null;
+                    }
+                }
             }
             if (signal?.aborted || aborted) {
+                if (storeWriter)
+                    await storeWriter.close();
                 if (pool)
                     await pool.destroy();
                 return {
@@ -1808,7 +1684,7 @@ class ExtractionOrchestrator {
                     if (result.nodes.length > 0 || result.errors.length === 0) {
                         const language = (0, grammars_1.detectLanguage)(filePath, content, overrides);
                         const stats = await fsp.stat(path.join(this.rootDir, filePath));
-                        this.storeExtractionResult(filePath, content, language, stats, result);
+                        await this.storeExtractionResult(filePath, content, language, stats, result, (0, cooperative_yield_1.createYielder)());
                         const idx = errors.indexOf(errEntry);
                         if (idx >= 0)
                             errors.splice(idx, 1);
@@ -1856,7 +1732,7 @@ class ExtractionOrchestrator {
                         if (result.nodes.length > 0 || result.errors.length === 0) {
                             const language = (0, grammars_1.detectLanguage)(filePath, fullContent, overrides);
                             const stats = await fsp.stat(path.join(this.rootDir, filePath));
-                            this.storeExtractionResult(filePath, fullContent, language, stats, result);
+                            await this.storeExtractionResult(filePath, fullContent, language, stats, result, (0, cooperative_yield_1.createYielder)());
                             const idx = errors.indexOf(errEntry);
                             if (idx >= 0)
                                 errors.splice(idx, 1);
@@ -2034,108 +1910,77 @@ class ExtractionOrchestrator {
         const result = (0, tree_sitter_1.extractFromSource)(relativePath, content, language, frameworkNames);
         // Store in database
         if (result.nodes.length > 0 || result.errors.length === 0) {
-            this.storeExtractionResult(relativePath, content, language, stats, result);
+            await this.storeExtractionResult(relativePath, content, language, stats, result, (0, cooperative_yield_1.createYielder)());
         }
         return result;
     }
     /**
      * Store extraction result in database
      */
-    storeExtractionResult(filePath, content, language, stats, result) {
+    async storeExtractionResult(filePath, content, language, stats, result, onYield) {
+        const STORE_CHUNK = 2000;
         const contentHash = hashContent(content);
-        // Check if file already exists and hasn't changed
         const existingFile = this.queries.getFileByPath(filePath);
         if (existingFile && existingFile.contentHash === contentHash) {
-            return; // No changes
-        }
-        // Snapshot incoming cross-file edges BEFORE deleting this file's nodes.
-        // `deleteFile` cascades to delete every edge whose source OR target is a
-        // node in this file (edges.FK ... ON DELETE CASCADE). Edges whose SOURCE is
-        // in this file are re-emitted by the extractor below, but edges whose SOURCE
-        // is in a *different* (unchanged) file are not — they would be silently
-        // dropped, which is issue #899: re-indexing a callee file severs `calls`/
-        // `references` edges from callers that import it via module-attribute
-        // access (`pkg.mod.fn(...)`).
-        //
-        // We snapshot the edge plus the target node's (name, kind) so we can
-        // re-resolve to the re-indexed target's NEW id. Node ids are
-        // `sha256(filePath:kind:name:line)`, so any line shift in the callee file
-        // (e.g. a docstring-only edit above the symbol) changes every target id and
-        // a naive re-insert by old id would silently drop every edge. Matching by
-        // (filePath, kind, name) is stable across line shifts; if the symbol was
-        // renamed/removed, no match is found and the edge stays dropped (correct).
+            return;
+        }
         const crossFileIncomingEdges = existingFile
             ? this.queries.getCrossFileIncomingEdgesWithTarget(filePath)
             : [];
-        // Delete existing data for this file
         if (existingFile) {
             this.queries.deleteFile(filePath);
         }
-        // Filter out nodes with missing required fields before insertion.
-        // This prevents FK violations when edges reference nodes that would
-        // be silently skipped by insertNode() (see issue #42).
         const validNodes = result.nodes.filter((n) => n.id && n.kind && n.name && n.filePath && n.language);
-        // Insert nodes
-        if (validNodes.length > 0) {
-            this.queries.insertNodes(validNodes);
-        }
-        // Filter edges to only reference nodes that were actually inserted
-        if (result.edges.length > 0) {
-            const insertedIds = new Set(validNodes.map((n) => n.id));
-            const validEdges = result.edges.filter((e) => insertedIds.has(e.source) && insertedIds.has(e.target));
-            if (validEdges.length > 0) {
-                this.queries.insertEdges(validEdges);
+        const insertedIds = new Set(validNodes.map((n) => n.id));
+        const validEdges = result.edges.filter((e) => insertedIds.has(e.source) && insertedIds.has(e.target));
+        const validRefs = result.unresolvedReferences
+            .filter((ref) => insertedIds.has(ref.fromNodeId))
+            .map((ref) => ({
+            ...ref,
+            filePath: ref.filePath ?? filePath,
+            language: ref.language ?? language,
+        }));
+        const fitsOneChunk = validNodes.length <= STORE_CHUNK &&
+            validEdges.length <= STORE_CHUNK &&
+            validRefs.length <= STORE_CHUNK;
+        if (fitsOneChunk) {
+            this.queries.storeFileBundle({
+                nodes: validNodes,
+                edges: validEdges,
+                refs: validRefs,
+                file: {
+                    path: filePath,
+                    contentHash,
+                    language,
+                    size: stats.size,
+                    modifiedAt: stats.mtimeMs,
+                    indexedAt: Date.now(),
+                    nodeCount: result.nodes.length,
+                    errors: result.errors.length > 0 ? result.errors : undefined,
+                },
+            });
+            if (crossFileIncomingEdges.length > 0) {
+                this.reattachCrossFileEdges(crossFileIncomingEdges, validNodes);
+            }
+            return;
+        }
+        for (let i = 0; i < validNodes.length; i += STORE_CHUNK) {
+            this.queries.insertNodes(validNodes.slice(i, i + STORE_CHUNK));
+            await onYield?.();
+        }
+        if (validEdges.length > 0) {
+            for (let i = 0; i < validEdges.length; i += STORE_CHUNK) {
+                this.queries.insertEdges(validEdges.slice(i, i + STORE_CHUNK));
+                await onYield?.();
             }
         }
-        // Re-insert cross-file incoming edges snapshotted before the delete,
-        // re-resolving each edge's target to the re-indexed node's new id by
-        // (filePath, kind, name). Node ids include the source line, so any line
-        // shift in the callee file (e.g. a docstring-only edit above the symbol)
-        // changes every target id and a naive re-insert by old id would drop them
-        // all. `insertEdges` still filters to endpoints that exist, so edges whose
-        // caller (source) was deleted, or whose callee (target) was renamed/removed
-        // during the re-index (no match in `newTargetIds`), are dropped. This
-        // closes the #899 edge-drop on `sync`.
         if (crossFileIncomingEdges.length > 0) {
-            const newNodesByKindName = new Map();
-            for (const n of validNodes) {
-                newNodesByKindName.set(`${n.kind}\0${n.name}`, n.id);
-            }
-            const reinserted = [];
-            const resurrected = [];
-            for (const e of crossFileIncomingEdges) {
-                const newTargetId = newNodesByKindName.get(`${e.targetKind}\0${e.targetName}`);
-                if (newTargetId) {
-                    reinserted.push({ source: e.source, target: newTargetId, kind: e.kind, metadata: e.metadata, line: e.line, column: e.column, provenance: e.provenance });
-                }
-                else {
-                    const ref = resurrectRefFromDroppedEdge(e);
-                    if (ref)
-                        resurrected.push(ref);
-                }
-            }
-            if (reinserted.length > 0) {
-                this.queries.insertEdges(reinserted);
-            }
-            if (resurrected.length > 0) {
-                this.queries.insertUnresolvedRefsBatch(resurrected);
-            }
+            this.reattachCrossFileEdges(crossFileIncomingEdges, validNodes);
         }
-        // Insert unresolved references in batch with denormalized filePath/language
-        if (result.unresolvedReferences.length > 0) {
-            const insertedIds = new Set(validNodes.map((n) => n.id));
-            const refsWithContext = result.unresolvedReferences
-                .filter((ref) => insertedIds.has(ref.fromNodeId))
-                .map((ref) => ({
-                ...ref,
-                filePath: ref.filePath ?? filePath,
-                language: ref.language ?? language,
-            }));
-            if (refsWithContext.length > 0) {
-                this.queries.insertUnresolvedRefsBatch(refsWithContext);
-            }
+        for (let i = 0; i < validRefs.length; i += STORE_CHUNK) {
+            this.queries.insertUnresolvedRefsBatch(validRefs.slice(i, i + STORE_CHUNK));
+            await onYield?.();
         }
-        // Insert file record
         const fileRecord = {
             path: filePath,
             contentHash,
@@ -2148,6 +1993,46 @@ class ExtractionOrchestrator {
         };
         this.queries.upsertFile(fileRecord);
     }
+    buildFileRecord(filePath, content, language, stats, nodeCount, resultErrors) {
+        return {
+            path: filePath,
+            contentHash: hashContent(content),
+            language,
+            size: stats.size,
+            modifiedAt: stats.mtimeMs,
+            indexedAt: Date.now(),
+            nodeCount,
+            errors: resultErrors.length > 0 ? resultErrors : undefined,
+        };
+    }
+    buildFreshStoreBundle(filePath, content, language, stats, result) {
+        return (0, store_writer_1.finalizeStoreBundle)(result, filePath, language, this.buildFileRecord(filePath, content, language, stats, result.nodes.length, result.errors));
+    }
+    reattachCrossFileEdges(crossFileIncomingEdges, validNodes) {
+        const newNodesByKindName = new Map();
+        for (const n of validNodes) {
+            newNodesByKindName.set(`${n.kind}\0${n.name}`, n.id);
+        }
+        const reinserted = [];
+        const resurrected = [];
+        for (const e of crossFileIncomingEdges) {
+            const newTargetId = newNodesByKindName.get(`${e.targetKind}\0${e.targetName}`);
+            if (newTargetId) {
+                reinserted.push({ source: e.source, target: newTargetId, kind: e.kind, metadata: e.metadata, line: e.line, column: e.column, provenance: e.provenance });
+            }
+            else {
+                const ref = resurrectRefFromDroppedEdge(e);
+                if (ref)
+                    resurrected.push(ref);
+            }
+        }
+        if (reinserted.length > 0) {
+            this.queries.insertEdges(reinserted);
+        }
+        if (resurrected.length > 0) {
+            this.queries.insertUnresolvedRefsBatch(resurrected);
+        }
+    }
     /**
      * Sync the index with the current file state.
      *
@@ -2156,7 +2041,7 @@ class ExtractionOrchestrator {
      * changes. This works in non-git projects and catches committed changes from
      * `git pull`/`checkout`/`merge`/`rebase` that `git status` cannot see.
      */
-    async sync(onProgress) {
+    async sync(onProgress, scopedPaths) {
         await (0, grammars_1.initGrammars)(); // Initialize WASM runtime (grammars loaded lazily below)
         const startTime = Date.now();
         let filesChecked = 0;
@@ -2171,33 +2056,36 @@ class ExtractionOrchestrator {
             total: 0,
         });
         const filesToIndex = [];
-        // === Filesystem reconcile (git-independent) ===
-        // The source of truth for "what changed" is the filesystem vs the indexed
-        // state — never git. We enumerate the current source files and reconcile
-        // each against the DB. A cheap (size, mtime) stat pre-filter skips unchanged
-        // files without reading or hashing them, so the expensive read+hash+parse
-        // only runs for files that actually changed. This catches edits/adds/deletes
-        // whether or not the project uses git, and crucially also catches committed
-        // changes from `git pull`/`checkout`/`merge`/`rebase` — which `git status`
-        // cannot see, because the working tree is clean afterward.
-        const currentFiles = await scanDirectoryAsync(this.rootDir);
-        filesChecked = currentFiles.length;
+        let currentFiles;
+        let trackedFiles;
+        if (scopedPaths && scopedPaths.length > 0) {
+            const unique = [...new Set(scopedPaths)];
+            currentFiles = unique.filter((p) => fs.existsSync(path.join(this.rootDir, p)));
+            trackedFiles = [];
+            for (const p of unique) {
+                const rec = this.queries.getFileByPath(p);
+                if (rec)
+                    trackedFiles.push(rec);
+            }
+            filesChecked = unique.length;
+        }
+        else {
+            currentFiles = await scanDirectoryAsync(this.rootDir);
+            filesChecked = currentFiles.length;
+            trackedFiles = this.queries.getAllFiles();
+        }
         const currentSet = new Set(currentFiles);
-        const trackedFiles = this.queries.getAllFiles();
         const trackedMap = new Map();
         for (const f of trackedFiles) {
             trackedMap.set(f.path, f);
         }
-        // Removals: tracked in the DB but no longer a present source file. Check the
-        // filesystem directly — `scanDirectory` (via `git ls-files`) still lists a
-        // file deleted from disk but not yet staged, so set membership alone misses it.
-        // `reconcileChecks` drives the cooperative yield shared with the adds/mods loop
-        // below (see SYNC_RECONCILE_YIELD_INTERVAL / issue #905).
+        const removedArkAnalyzerFiles = [];
         let reconcileChecks = 0;
         for (const tracked of trackedFiles) {
             if (!currentSet.has(tracked.path) || !fs.existsSync(path.join(this.rootDir, tracked.path))) {
-                // Before the cascade deletes them, resurrect incoming cross-file
-                // resolution edges as their original refs (#1240 removal case).
+                if (tracked.path.startsWith('@')) {
+                    continue;
+                }
                 const incoming = this.queries.getCrossFileIncomingEdgesWithTarget(tracked.path);
                 if (incoming.length > 0) {
                     const resurrected = incoming
@@ -2207,6 +2095,9 @@ class ExtractionOrchestrator {
                         this.queries.insertUnresolvedRefsBatch(resurrected);
                     }
                 }
+                if ((0, arkts_1.isArkAnalyzerSourcePath)(tracked.path)) {
+                    removedArkAnalyzerFiles.push(tracked.path);
+                }
                 this.queries.deleteFile(tracked.path);
                 filesRemoved++;
             }
@@ -2216,9 +2107,6 @@ class ExtractionOrchestrator {
         }
         // Adds / modifications.
         for (const filePath of currentFiles) {
-            // Same cooperative yield as the removals loop — this is the other O(files)
-            // synchronous-stat loop that wedges the main thread on a large repo (#905).
-            // Yield at the top of the body so the `continue` fast-paths below still hit it.
             if (++reconcileChecks % SYNC_RECONCILE_YIELD_INTERVAL === 0) {
                 onProgress?.({
                     phase: 'scanning',
@@ -2281,7 +2169,19 @@ class ExtractionOrchestrator {
         const total = filesToIndex.length;
         const overrides = (0, project_config_1.loadExtensionOverrides)(this.rootDir);
         const needsArkTS = filesToIndex.some((f) => (0, grammars_1.detectLanguage)(f, undefined, overrides) === 'arkts');
-        if (needsArkTS) {
+        const arktsChangedForBatch = [
+            ...removedArkAnalyzerFiles,
+            ...filesToIndex.filter((f) => (0, arkts_1.isArkAnalyzerSourcePath)(f) ||
+                f.endsWith('module.json5') ||
+                f.endsWith('oh-package.json5') ||
+                f === 'build-profile.json5' ||
+                f.endsWith('/build-profile.json5')),
+        ];
+        // Removals alone can still dirty a Harmony module (RTA for remaining files).
+        const needsArkTSBatch = needsArkTS ||
+            removedArkAnalyzerFiles.length > 0 ||
+            arktsChangedForBatch.some((f) => f === 'build-profile.json5' || f.endsWith('/build-profile.json5'));
+        if (needsArkTSBatch && arktsChangedForBatch.length > 0) {
             (0, context_1.setArkTSBatchProgressCallback)((batchProgress) => {
                 onProgress?.({
                     phase: 'arkts-batch',
@@ -2291,8 +2191,12 @@ class ExtractionOrchestrator {
                     subphase: batchProgress.subphase,
                 });
             });
-            const firstEts = filesToIndex.find((f) => (0, grammars_1.detectLanguage)(f, undefined, overrides) === 'arkts');
-            await (0, arkts_1.primeArkTSBatch)(this.rootDir, this.queries, firstEts);
+            const firstEts = filesToIndex.find((f) => (0, grammars_1.detectLanguage)(f, undefined, overrides) === 'arkts') ??
+                arktsChangedForBatch.find((f) => (0, arkts_1.isArkAnalyzerSourcePath)(f)) ??
+                arktsChangedForBatch[0];
+            await (0, arkts_1.primeArkTSBatch)(this.rootDir, this.queries, firstEts, {
+                changedFiles: arktsChangedForBatch,
+            });
         }
         try {
             for (let i = 0; i < filesToIndex.length; i++) {
@@ -2317,7 +2221,7 @@ class ExtractionOrchestrator {
             }
         }
         finally {
-            if (needsArkTS) {
+            if (needsArkTSBatch) {
                 (0, context_1.setArkTSBatchProgressCallback)(null);
             }
         }