@colbymchenry/codegraph-win32-x64 1.2.0 → 1.3.1

package/lib/dist/extraction/index.js

@@ -46,6 +46,7 @@ exports.hashContent = hashContent;
 exports.buildDefaultIgnore = buildDefaultIgnore;
 exports.buildScopeIgnore = buildScopeIgnore;
 exports.discoverEmbeddedRepoRoots = discoverEmbeddedRepoRoots;
+exports.findUnindexedIgnoredRepos = findUnindexedIgnoredRepos;
 exports.scanDirectory = scanDirectory;
 exports.scanDirectoryAsync = scanDirectoryAsync;
 const fs = __importStar(require("fs"));
@@ -63,6 +64,7 @@ const errors_1 = require("../errors");
 const utils_1 = require("../utils");
 const ignore_1 = __importDefault(require("ignore"));
 const frameworks_1 = require("../resolution/frameworks");
+const cooperative_yield_1 = require("../resolution/cooperative-yield");
 /**
  * Number of files to read in parallel during indexing.
  * File reads are I/O-bound; batching overlaps I/O wait with CPU parse work.
@@ -302,6 +304,160 @@ function loadExcludeMatcher(rootDir) {
     const patterns = (0, project_config_1.loadExcludePatterns)(rootDir);
     return patterns.length > 0 ? (0, ignore_1.default)().add(patterns) : null;
 }
+/**
+ * Matcher for the project's `codegraph.json` `include` patterns — first-party
+ * source to force INTO the index even when `.gitignore` drops it (the general
+ * whitelist `includeIgnored` never was — that one only revives *embedded git
+ * repos*). The case it exists for: a project under a second VCS (SVN/Perforce)
+ * `.gitignore`s its own real source so it stays out of Git, yet we still want it
+ * indexed. Returns `null` when nothing is force-included (the zero-config
+ * default → no overhead, no extra walk). Built once per scan/sync/scope
+ * operation from the scan root.
+ */
+function loadIncludeMatcher(rootDir) {
+    const patterns = (0, project_config_1.loadIncludePatterns)(rootDir);
+    return patterns.length > 0 ? (0, ignore_1.default)().add(patterns) : null;
+}
+/** Glob metacharacters that end the static (literal) prefix of an `include` pattern. */
+const GLOB_META = /[*?[\]{}!]/;
+/**
+ * The static directory prefix of each `include` pattern — the literal leading
+ * path up to the first glob segment — trailing-slashed, used to (a) walk only
+ * the opted-in subtrees in `collectIncludedFiles` and (b) let `ScopeIgnore` keep
+ * the watcher descending toward them. `Tools/` stays `Tools/`; a recursive
+ * `Tools/**` glob yields `Tools/`; `src/local/file.ts` yields `src/local/` (the
+ * file's dir); a pattern that starts with a glob (like a leading `**`) yields
+ * `''`, meaning "no static root — walk the whole tree". Duplicates and roots
+ * nested under a broader root are collapsed so each subtree is walked once.
+ */
+function includeStaticRoots(patterns) {
+    const roots = new Set();
+    for (const pattern of patterns) {
+        let p = pattern.replace(/^\/+/, '');
+        const trailingSlash = p.endsWith('/');
+        if (trailingSlash)
+            p = p.slice(0, -1);
+        const segs = p.split('/').filter(Boolean);
+        const lead = [];
+        for (const s of segs) {
+            if (GLOB_META.test(s))
+                break;
+            lead.push(s);
+        }
+        const hadWildcard = lead.length < segs.length;
+        // A wholly-literal pattern with no trailing slash names a file (or a dir we
+        // can't tell apart) — drop its last segment so we walk the containing dir
+        // and let the matcher pick the file. A trailing slash or a glob means the
+        // remaining `lead` is already the directory to walk.
+        if (!hadWildcard && !trailingSlash && lead.length > 0)
+            lead.pop();
+        if (lead.length === 0) {
+            roots.clear();
+            roots.add('');
+            return ['']; // a top-level glob forces a whole-tree walk; nothing narrower matters
+        }
+        roots.add(lead.join('/') + '/');
+    }
+    // Collapse roots nested under a broader one (e.g. drop `a/b/` if `a/` is present).
+    const all = [...roots];
+    return all.filter((r) => !all.some((other) => other !== r && r.startsWith(other)));
+}
+/**
+ * Actively discover the source files an `include` whitelist forces in. `git
+ * ls-files` never lists gitignored files, so a filtered filesystem walk of just
+ * the opted-in subtrees (`includeStaticRoots`) is the only way to find them.
+ * Returns project-root-relative, normalized source-file paths.
+ *
+ * A file is collected when it MATCHES `include`, is NOT hit by `exclude` (an
+ * explicit exclude always wins), is a recognized source file, and does not live
+ * under a built-in default-ignored dir (`node_modules`, `dist`, …), `.git`, or
+ * CodeGraph's data dir — those are never resurfaced, mirroring `ScopeIgnore`.
+ * `.gitignore` is deliberately NOT consulted: overriding it is the whole point.
+ */
+function collectIncludedFiles(rootDir, include, exclude, roots, overrides) {
+    const out = new Set();
+    const defaults = defaultsOnlyIgnore();
+    const visited = new Set();
+    const consider = (abs, rel, isDir) => {
+        if (isDir) {
+            if (defaults.ignores(rel + '/'))
+                return; // never node_modules/dist/… via include
+            // An explicit `exclude` always wins over `include`; prune the whole subtree
+            // here so a large excluded dir (a committed frontend's own vendored deps,
+            // build output, …) is never walked — the per-file guard below still catches
+            // anything a directory pattern doesn't, so this is a pure efficiency win.
+            if (exclude && exclude.ignores(rel + '/'))
+                return;
+            walk(abs);
+        }
+        else {
+            if (defaults.ignores(rel))
+                return;
+            if (!include.ignores(rel))
+                return;
+            if (exclude && exclude.ignores(rel))
+                return;
+            if (!(0, grammars_1.isSourceFile)(rel, overrides))
+                return;
+            out.add(rel);
+        }
+    };
+    function walk(absDir) {
+        let realDir;
+        try {
+            realDir = fs.realpathSync(absDir);
+        }
+        catch {
+            return;
+        }
+        if (visited.has(realDir))
+            return; // symlink-cycle guard
+        visited.add(realDir);
+        let entries;
+        try {
+            entries = fs.readdirSync(absDir, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        for (const entry of entries) {
+            if (entry.name === '.git' || (0, directory_1.isCodeGraphDataDir)(entry.name))
+                continue;
+            const abs = path.join(absDir, entry.name);
+            const rel = (0, utils_1.normalizePath)(path.relative(rootDir, abs));
+            if (!rel || rel.startsWith('..'))
+                continue;
+            if (entry.isSymbolicLink()) {
+                try {
+                    const st = fs.statSync(fs.realpathSync(abs));
+                    consider(abs, rel, st.isDirectory());
+                }
+                catch {
+                    // broken symlink — skip
+                }
+                continue;
+            }
+            consider(abs, rel, entry.isDirectory());
+        }
+    }
+    for (const root of roots) {
+        walk(root === '' ? rootDir : path.join(rootDir, root));
+    }
+    return out;
+}
+/**
+ * The included source files (`codegraph.json` `include`) for a scan root, or an
+ * empty set when nothing is force-included. Centralizes loading the matcher,
+ * roots, exclude, and overrides so both enumeration paths (git and filesystem
+ * walk) add the same files.
+ */
+function collectIncludedFilesForRoot(rootDir) {
+    const include = loadIncludeMatcher(rootDir);
+    if (!include)
+        return new Set();
+    const roots = includeStaticRoots((0, project_config_1.loadIncludePatterns)(rootDir));
+    return collectIncludedFiles(rootDir, include, loadExcludeMatcher(rootDir), roots, (0, project_config_1.loadExtensionOverrides)(rootDir));
+}
 /**
  * `git ls-files --directory` collapses a wholly-untracked/ignored directory into
  * one entry — and when the command's own cwd is such a directory (the indexed
@@ -442,6 +598,8 @@ function findNestedGitRepos(absDir, relPrefix) {
 class ScopeIgnore {
     rootMatcher;
     exclude;
+    include;
+    includeRoots;
     embedded;
     defaults = defaultsOnlyIgnore();
     constructor(rootMatcher, embedded, 
@@ -450,9 +608,21 @@ class ScopeIgnore {
      * full root-relative path. Wins over everything else — an explicit user
      * exclude applies even to tracked files and even inside embedded repos.
      */
-    exclude = null) {
+    exclude = null, 
+    /**
+     * Project `codegraph.json` `include` patterns — first-party source forced
+     * INTO the index despite `.gitignore`. When a path matches, it is NOT
+     * ignored (so the watcher watches it), overriding `.gitignore`/`rootMatcher`
+     * — but never `exclude` (checked first) and never a built-in default-ignored
+     * dir. `includeRoots` are the static prefixes so a gitignored ANCESTOR
+     * directory of an included subtree still isn't pruned by the directory
+     * walker/watcher.
+     */
+    include = null, includeRoots = []) {
         this.rootMatcher = rootMatcher;
         this.exclude = exclude;
+        this.include = include;
+        this.includeRoots = includeRoots;
         // Longest root first so paths in nested embedded repos hit the innermost matcher.
         this.embedded = [...embedded].sort((a, b) => b.root.length - a.root.length);
     }
@@ -462,6 +632,20 @@ class ScopeIgnore {
         // everywhere, including ancestors of embedded repos.
         if (this.exclude && this.exclude.ignores(rel))
             return true;
+        // User `include`: force first-party source in despite `.gitignore`. Never
+        // resurfaces a built-in default-ignored dir (node_modules/dist/…), so an
+        // include pattern can't accidentally pull in dependency/build trees.
+        if (this.include && !this.defaults.ignores(rel)) {
+            if (rel.endsWith('/')) {
+                // A directory on (or leading to) an included subtree must stay walkable
+                // so the watcher/walker descends to reach the forced-in files.
+                if (this.includeRoots.some((r) => r.startsWith(rel) || rel.startsWith(r)))
+                    return false;
+            }
+            else if (this.include.ignores(rel)) {
+                return false;
+            }
+        }
         for (const { root, matcher } of this.embedded) {
             if (rel.startsWith(root)) {
                 const inner = rel.slice(root.length);
@@ -488,7 +672,8 @@ exports.ScopeIgnore = ScopeIgnore;
  */
 function buildScopeIgnore(rootDir, embeddedRoots) {
     const roots = embeddedRoots ? [...embeddedRoots] : discoverEmbeddedRepoRoots(rootDir);
-    return new ScopeIgnore(buildDefaultIgnore(rootDir), roots.map((root) => ({ root, matcher: buildDefaultIgnore(path.join(rootDir, root)) })), loadExcludeMatcher(rootDir));
+    const include = loadIncludeMatcher(rootDir);
+    return new ScopeIgnore(buildDefaultIgnore(rootDir), roots.map((root) => ({ root, matcher: buildDefaultIgnore(path.join(rootDir, root)) })), loadExcludeMatcher(rootDir), include, include ? includeStaticRoots((0, project_config_1.loadIncludePatterns)(rootDir)) : []);
 }
 /**
  * Whether an embedded repo found as a tracked gitlink (mode 160000, #1031/#1033)
@@ -584,6 +769,50 @@ function discoverEmbeddedRepoRoots(rootDir) {
     visit(rootDir, '');
     return out;
 }
+/**
+ * Cap on how many skipped gitignored repos the CLI hint enumerates — a huge
+ * gitignored data dir full of clones must never turn the hint scan into a long
+ * walk. Enough to make the point; the caller says "+N more" past this.
+ */
+const UNINDEXED_IGNORED_REPO_HINT_CAP = 100;
+/**
+ * The INVERSE of the gitignored side of {@link discoverEmbeddedRepoRoots}:
+ * nested git repositories under a gitignored directory that the project has NOT
+ * opted into via `codegraph.json` `includeIgnored`. These are real repos the
+ * default `init`/`index` deliberately skips because `.gitignore` excludes them
+ * (#970, #976) — most visibly the "super-repo `.gitignore`s its child repos"
+ * layout (#1156), where `init` at the parent correctly indexes ~nothing while
+ * `init` inside each child works. The CLI uses this to turn that silent empty
+ * index into an actionable hint: it names the skipped repos and offers to opt
+ * them in. Paths are `rootDir`-relative and trailing-slashed (valid
+ * `includeIgnored` patterns as-is). Returns `[]` for a non-git root (a
+ * filesystem walk already descends into nested repos there), skips built-in
+ * default-ignored dirs (`node_modules`, …), and is bounded so it never stalls
+ * on a giant ignored tree.
+ */
+function findUnindexedIgnoredRepos(rootDir) {
+    try {
+        (0, child_process_1.execFileSync)('git', ['rev-parse', '--git-dir'], { cwd: rootDir, encoding: 'utf-8', timeout: 5000, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true });
+    }
+    catch {
+        return [];
+    }
+    const defaults = defaultsOnlyIgnore();
+    const includeIgnored = loadIncludeIgnoredMatcher(rootDir);
+    const repos = [];
+    for (const dir of listIgnoredDirs(rootDir)) {
+        if (defaults.ignores(dir))
+            continue; // node_modules etc. — never project code
+        if (includeIgnored?.ignores((0, utils_1.normalizePath)(dir)))
+            continue; // already opted in — nothing to nag about
+        for (const repo of findNestedGitRepos(path.join(rootDir, dir), dir)) {
+            repos.push(repo);
+            if (repos.length >= UNINDEXED_IGNORED_REPO_HINT_CAP)
+                return repos;
+        }
+    }
+    return repos;
+}
 /**
  * Discover embedded repos hidden by `repoDir`'s OWN gitignore rules: for each
  * gitignored directory, search for nested `.git` roots. Returns repo paths
@@ -764,7 +993,15 @@ function getGitVisibleFiles(rootDir) {
         // not the parent's: the parent's .gitignore hides the child repo from git,
         // not from the index. (#514)
         const ig = buildScopeIgnore(rootDir, embeddedRoots);
-        return new Set([...files].filter((f) => !ig.ignores(f)));
+        const visible = new Set([...files].filter((f) => !ig.ignores(f)));
+        // Force-include first-party source the project whitelisted in
+        // `codegraph.json` `include`. These are gitignored, so `git ls-files` never
+        // listed them above — discover them directly off disk and add them. (The
+        // common SVN+Git dual-VCS case: source committed to SVN, gitignored out of
+        // Git, but still wanted in the graph.)
+        for (const f of collectIncludedFilesForRoot(rootDir))
+            visible.add(f);
+        return visible;
     }
     catch {
         return null;
@@ -1027,6 +1264,19 @@ function scanDirectoryWalk(rootDir, onProgress) {
     if (exclude)
         baseMatchers.push({ dir: rootDir, ig: exclude });
     walk(rootDir, baseMatchers);
+    // Force-include first-party source whitelisted in `codegraph.json` `include`
+    // — the walk above honours `.gitignore`, so anything gitignored was dropped;
+    // add it back here (deduped). Mirrors the git path's union.
+    const included = collectIncludedFilesForRoot(rootDir);
+    if (included.size > 0) {
+        const seen = new Set(files);
+        for (const f of included) {
+            if (!seen.has(f)) {
+                files.push(f);
+                seen.add(f);
+            }
+        }
+    }
     return files;
 }
 /**
@@ -1143,6 +1393,10 @@ class ExtractionOrchestrator {
             current: 0,
             total: 0,
         });
+        // Phase attribution to stderr (same opt-in as the synthesis timings):
+        // early-run 5-10s single stalls were observed on 95k-file repos but never
+        // attributed — these labels settle scan vs framework-detect vs grammars.
+        const tScan = Date.now();
         const files = await scanDirectoryAsync(this.rootDir, (current, file) => {
             onProgress?.({
                 phase: 'scanning',
@@ -1151,13 +1405,18 @@ class ExtractionOrchestrator {
                 currentFile: file,
             });
         });
+        if (process.env.CODEGRAPH_SYNTH_TIMINGS)
+            console.error(`[phase-timing] scan: ${Date.now() - tScan}ms (${files.length} files)`);
         // Detect frameworks once per indexAll run using the scanned file list.
         // Names are passed to each parse call so framework-specific extractors
         // (route nodes, middleware, etc.) run after the tree-sitter pass.
         // Framework detection is reset each run so adding e.g. requirements.txt
         // between runs is picked up without restarting the process.
         this.detectedFrameworkNames = null;
+        const tFw = Date.now();
         const frameworkNames = this.ensureDetectedFrameworks(files);
+        if (process.env.CODEGRAPH_SYNTH_TIMINGS)
+            console.error(`[phase-timing] framework-detect: ${Date.now() - tFw}ms`);
         if (signal?.aborted) {
             return {
                 success: false,
@@ -1240,12 +1499,17 @@ class ExtractionOrchestrator {
         let nextSeq = 0; // file-order sequence assigned at dispatch
         let nextToStore = 0; // cursor: next sequence to commit
         let aborted = false;
-        const storeResult = (filePath, content, stats, result) => {
+        // Yielder for the in-order commit path: a single giant generated file's
+        // store is otherwise one unyielding multi-second transaction span on the
+        // main thread (5–14s single stalls measured on llvm-project), starving
+        // the #850 watchdog heartbeat on slow hardware.
+        const commitYield = (0, cooperative_yield_1.createYielder)();
+        const storeResult = async (filePath, content, stats, result) => {
             processed++;
             // Store in database on main thread (SQLite is not thread-safe)
             if (result.nodes.length > 0 || result.errors.length === 0) {
                 const language = (0, grammars_1.detectLanguage)(filePath, content, overrides);
-                this.storeExtractionResult(filePath, content, language, stats, result);
+                await this.storeExtractionResult(filePath, content, language, stats, result, commitYield);
             }
             if (result.errors.length > 0) {
                 for (const err of result.errors) {
@@ -1289,20 +1553,35 @@ class ExtractionOrchestrator {
         };
         // Commit buffered parses to the DB in file order, advancing the cursor over
         // contiguous completed results. Runs after each parse settles (and once more
-        // after the drain). storeResult / recordParseFailure run here single-threaded,
-        // so shared counters and SQLite writes never race despite parallel parsing.
+        // after the drain). storeResult is now async (it yields between chunked
+        // inserts), so commits are SERIALIZED on a promise chain — concurrent parse
+        // completions append to the chain instead of interleaving mid-store, which
+        // preserves both the file-order commit invariant (#1015: resolution
+        // disambiguates same-named candidates by insertion order) and the
+        // single-writer discipline for SQLite. Errors are recorded and re-thrown
+        // at the drain, matching the old synchronous propagation.
+        let flushChain = Promise.resolve();
+        let flushError = null;
         const flushOrdered = () => {
-            if (aborted)
-                return;
-            while (completed.has(nextToStore)) {
-                const item = completed.get(nextToStore);
-                completed.delete(nextToStore);
-                nextToStore++;
-                if (item.ok)
-                    storeResult(item.filePath, item.content, item.stats, item.result);
-                else
-                    recordParseFailure(item.filePath, item.err);
-            }
+            flushChain = flushChain.then(async () => {
+                if (aborted || flushError)
+                    return;
+                try {
+                    while (completed.has(nextToStore)) {
+                        const item = completed.get(nextToStore);
+                        completed.delete(nextToStore);
+                        nextToStore++;
+                        if (item.ok)
+                            await storeResult(item.filePath, item.content, item.stats, item.result);
+                        else
+                            recordParseFailure(item.filePath, item.err);
+                    }
+                }
+                catch (err) {
+                    flushError = err;
+                }
+            });
+            return flushChain;
         };
         // Dispatch one file's parse (parses run concurrently across the pool), tagged
         // with its file-order sequence so flushOrdered commits results in order. The
@@ -1325,10 +1604,15 @@ class ExtractionOrchestrator {
             // buffered), not just in-flight: a slow file sitting at the commit cursor
             // lets later parses finish and buffer, which would otherwise grow without
             // bound. Wait for parses to settle (each may advance the cursor) until the
-            // window has room. `inFlight.size > 0` guards against an empty race — the
-            // cursor file is always still in flight when the window is full.
-            while (nextSeq - nextToStore >= windowSize && inFlight.size > 0) {
-                await Promise.race(inFlight);
+            // window has room. When nothing is in flight but the window is still full,
+            // the async commit chain is what's behind — await it so the cursor
+            // advances (buffered items hold whole file contents, so this bound is
+            // load-bearing for memory).
+            while (nextSeq - nextToStore >= windowSize) {
+                if (inFlight.size > 0)
+                    await Promise.race(inFlight);
+                else
+                    await flushOrdered();
             }
         };
         for (let i = 0; i < files.length; i += FILE_IO_BATCH_SIZE) {
@@ -1403,7 +1687,9 @@ class ExtractionOrchestrator {
         // then commit any results the cursor hasn't reached yet.
         if (!aborted) {
             await Promise.all(inFlight);
-            flushOrdered();
+            await flushOrdered();
+            if (flushError)
+                throw flushError;
         }
         if (signal?.aborted || aborted) {
             if (pool)
@@ -1413,6 +1699,7 @@ class ExtractionOrchestrator {
                 filesIndexed,
                 filesSkipped,
                 filesErrored,
+                filesDiscovered: total,
                 nodesCreated: totalNodes,
                 edgesCreated: totalEdges,
                 errors: [{ message: 'Aborted', severity: 'error' }, ...errors],
@@ -1466,7 +1753,7 @@ class ExtractionOrchestrator {
                 if (result.nodes.length > 0 || result.errors.length === 0) {
                     const language = (0, grammars_1.detectLanguage)(filePath, content, overrides);
                     const stats = await fsp.stat(path.join(this.rootDir, filePath));
-                    this.storeExtractionResult(filePath, content, language, stats, result);
+                    await this.storeExtractionResult(filePath, content, language, stats, result, commitYield);
                     const idx = errors.indexOf(errEntry);
                     if (idx >= 0)
                         errors.splice(idx, 1);
@@ -1514,7 +1801,7 @@ class ExtractionOrchestrator {
                     if (result.nodes.length > 0 || result.errors.length === 0) {
                         const language = (0, grammars_1.detectLanguage)(filePath, fullContent, overrides);
                         const stats = await fsp.stat(path.join(this.rootDir, filePath));
-                        this.storeExtractionResult(filePath, fullContent, language, stats, result);
+                        await this.storeExtractionResult(filePath, fullContent, language, stats, result, commitYield);
                         const idx = errors.indexOf(errEntry);
                         if (idx >= 0)
                             errors.splice(idx, 1);
@@ -1535,6 +1822,7 @@ class ExtractionOrchestrator {
             filesIndexed,
             filesSkipped,
             filesErrored,
+            filesDiscovered: total,
             nodesCreated: totalNodes,
             edgesCreated: totalEdges,
             errors,
@@ -1678,14 +1966,21 @@ class ExtractionOrchestrator {
         const result = (0, tree_sitter_1.extractFromSource)(relativePath, content, language, frameworkNames);
         // Store in database
         if (result.nodes.length > 0 || result.errors.length === 0) {
-            this.storeExtractionResult(relativePath, content, language, stats, result);
+            await this.storeExtractionResult(relativePath, content, language, stats, result, (0, cooperative_yield_1.createYielder)());
         }
         return result;
     }
     /**
      * Store extraction result in database
      */
-    storeExtractionResult(filePath, content, language, stats, result) {
+    async storeExtractionResult(filePath, content, language, stats, result, onYield) {
+        // Bulk inserts run in bounded sub-transactions with a yield between, so a
+        // giant generated file (tens of thousands of symbols) can't block the
+        // event loop — and the #850 watchdog heartbeat — for the whole store.
+        // The file was NEVER one atomic transaction (each insert call has its
+        // own), and the files-table record still lands last, so crash recovery
+        // is unchanged: a partially-stored file has no record and re-indexes.
+        const STORE_CHUNK = 2000;
         const contentHash = hashContent(content);
         // Check if file already exists and hasn't changed
         const existingFile = this.queries.getFileByPath(filePath);
@@ -1719,16 +2014,18 @@ class ExtractionOrchestrator {
         // This prevents FK violations when edges reference nodes that would
         // be silently skipped by insertNode() (see issue #42).
         const validNodes = result.nodes.filter((n) => n.id && n.kind && n.name && n.filePath && n.language);
-        // Insert nodes
-        if (validNodes.length > 0) {
-            this.queries.insertNodes(validNodes);
+        // Insert nodes (chunked — see STORE_CHUNK above)
+        for (let i = 0; i < validNodes.length; i += STORE_CHUNK) {
+            this.queries.insertNodes(validNodes.slice(i, i + STORE_CHUNK));
+            await onYield?.();
         }
         // Filter edges to only reference nodes that were actually inserted
         if (result.edges.length > 0) {
             const insertedIds = new Set(validNodes.map((n) => n.id));
             const validEdges = result.edges.filter((e) => insertedIds.has(e.source) && insertedIds.has(e.target));
-            if (validEdges.length > 0) {
-                this.queries.insertEdges(validEdges);
+            for (let i = 0; i < validEdges.length; i += STORE_CHUNK) {
+                this.queries.insertEdges(validEdges.slice(i, i + STORE_CHUNK));
+                await onYield?.();
             }
         }
         // Re-insert cross-file incoming edges snapshotted before the delete,
@@ -1766,8 +2063,9 @@ class ExtractionOrchestrator {
                 filePath: ref.filePath ?? filePath,
                 language: ref.language ?? language,
             }));
-            if (refsWithContext.length > 0) {
-                this.queries.insertUnresolvedRefsBatch(refsWithContext);
+            for (let i = 0; i < refsWithContext.length; i += STORE_CHUNK) {
+                this.queries.insertUnresolvedRefsBatch(refsWithContext.slice(i, i + STORE_CHUNK));
+                await onYield?.();
             }
         }
         // Insert file record
@@ -1815,10 +2113,16 @@ class ExtractionOrchestrator {
         // whether or not the project uses git, and crucially also catches committed
         // changes from `git pull`/`checkout`/`merge`/`rebase` — which `git status`
         // cannot see, because the working tree is clean afterward.
+        const tSyncScan = Date.now();
         const currentFiles = await scanDirectoryAsync(this.rootDir);
+        if (process.env.CODEGRAPH_SYNTH_TIMINGS)
+            console.error(`[phase-timing] sync-scan: ${Date.now() - tSyncScan}ms (${currentFiles.length} files)`);
         filesChecked = currentFiles.length;
         const currentSet = new Set(currentFiles);
+        const tTracked = Date.now();
         const trackedFiles = this.queries.getAllFiles();
+        if (process.env.CODEGRAPH_SYNTH_TIMINGS)
+            console.error(`[phase-timing] sync-tracked-load: ${Date.now() - tTracked}ms (${trackedFiles.length} tracked)`);
         const trackedMap = new Map();
         for (const f of trackedFiles) {
             trackedMap.set(f.path, f);