@colbymchenry/codegraph-darwin-x64 1.1.1 → 1.1.3

package/lib/dist/extraction/index.js

@@ -51,9 +51,11 @@ exports.scanDirectoryAsync = scanDirectoryAsync;
 const fs = __importStar(require("fs"));
 const fsp = __importStar(require("fs/promises"));
 const path = __importStar(require("path"));
+const os = __importStar(require("os"));
 const crypto = __importStar(require("crypto"));
 const child_process_1 = require("child_process");
 const tree_sitter_1 = require("./tree-sitter");
+const parse_pool_1 = require("./parse-pool");
 const grammars_1 = require("./grammars");
 const project_config_1 = require("../project-config");
 const directory_1 = require("../directory");
@@ -155,12 +157,34 @@ const DEFAULT_IGNORE_DIRS = new Set([
     // Generic cache
     '.cache',
 ]);
+/**
+ * Android resource directory types. A `res/` tree holds ONLY non-code resources —
+ * layouts, drawables, value bags (strings/colors/styles), menus, navigation
+ * graphs — split into one typed subdirectory per kind, optionally density/locale/
+ * version-qualified (`values-es`, `drawable-hdpi`, `layout-v21`, …). None of it
+ * yields an extractable code symbol, yet on an Android app it DOMINATES the tree
+ * (one report: 26k XML files = 97% of the project, 0 symbols), bloating the DB,
+ * slowing indexing, and skewing both the file count and `codegraph_explore`
+ * results (#1047). So these are excluded by default. The structure is
+ * self-identifying — a non-Android project has no `res/layout/` etc., so it's
+ * untouched — and the only XML that DOES produce symbols (MyBatis mappers) lives
+ * under `src/main/resources/`, never `res/`, so nothing useful is dropped.
+ * `res/raw/` is deliberately NOT here: it holds arbitrary bundled assets that can
+ * be code-ish (a `.sql` schema, a `.js`), so we leave it indexed. Override any of
+ * these with a `.gitignore` negation (e.g. `!res/values/`).
+ */
+const ANDROID_RES_TYPES = [
+    'anim', 'animator', 'color', 'drawable', 'font', 'layout',
+    'menu', 'mipmap', 'navigation', 'transition', 'values', 'xml',
+];
 /** Gitignore-style patterns for the `ignore` matcher: the dirs above plus a few globs. */
 const DEFAULT_IGNORE_PATTERNS = [
     ...Array.from(DEFAULT_IGNORE_DIRS, (d) => `${d}/`),
     '*.egg-info/', // Python packaging metadata
     'cmake-build-*/', // CLion / CMake build trees
     'bazel-*/', // Bazel output symlink trees
+    // Android resource dirs at any depth, with their qualifier variants (#1047).
+    ...ANDROID_RES_TYPES.map((t) => `**/res/${t}*/`),
 ];
 /** True if `buf` decodes as strict UTF-8 (no invalid byte sequences). */
 function isValidUtf8(buf) {
@@ -265,6 +289,19 @@ function loadIncludeIgnoredMatcher(rootDir) {
     const patterns = (0, project_config_1.loadIncludeIgnoredPatterns)(rootDir);
     return patterns.length > 0 ? (0, ignore_1.default)().add(patterns) : null;
 }
+/**
+ * Matcher for the project's `codegraph.json` `exclude` patterns — paths to keep
+ * OUT of the index even when git-tracked, which `.gitignore` cannot do (#999).
+ * The escape hatch for a committed vendor/theme/SDK directory. Returns `null`
+ * when nothing is excluded (the zero-config default → no overhead). Matched
+ * against project-root-relative paths, so it applies uniformly across the whole
+ * workspace, including inside embedded repos (excluding `static/` means gone
+ * everywhere). Built once per scan/sync/scope operation from the scan root.
+ */
+function loadExcludeMatcher(rootDir) {
+    const patterns = (0, project_config_1.loadExcludePatterns)(rootDir);
+    return patterns.length > 0 ? (0, ignore_1.default)().add(patterns) : null;
+}
 /**
  * `git ls-files --directory` collapses a wholly-untracked/ignored directory into
  * one entry — and when the command's own cwd is such a directory (the indexed
@@ -404,14 +441,27 @@ function findNestedGitRepos(absDir, relPrefix) {
  */
 class ScopeIgnore {
     rootMatcher;
+    exclude;
     embedded;
     defaults = defaultsOnlyIgnore();
-    constructor(rootMatcher, embedded) {
+    constructor(rootMatcher, embedded, 
+    /**
+     * Project `codegraph.json` `exclude` patterns (#999), matched against the
+     * full root-relative path. Wins over everything else — an explicit user
+     * exclude applies even to tracked files and even inside embedded repos.
+     */
+    exclude = null) {
         this.rootMatcher = rootMatcher;
+        this.exclude = exclude;
         // Longest root first so paths in nested embedded repos hit the innermost matcher.
         this.embedded = [...embedded].sort((a, b) => b.root.length - a.root.length);
     }
     ignores(rel) {
+        // User `exclude` (#999) is checked first and against the full root-relative
+        // path: it must drop git-TRACKED paths (which `.gitignore` can't) and apply
+        // everywhere, including ancestors of embedded repos.
+        if (this.exclude && this.exclude.ignores(rel))
+            return true;
         for (const { root, matcher } of this.embedded) {
             if (rel.startsWith(root)) {
                 const inner = rel.slice(root.length);
@@ -438,7 +488,7 @@ exports.ScopeIgnore = ScopeIgnore;
  */
 function buildScopeIgnore(rootDir, embeddedRoots) {
     const roots = embeddedRoots ? [...embeddedRoots] : discoverEmbeddedRepoRoots(rootDir);
-    return new ScopeIgnore(buildDefaultIgnore(rootDir), roots.map((root) => ({ root, matcher: buildDefaultIgnore(path.join(rootDir, root)) })));
+    return new ScopeIgnore(buildDefaultIgnore(rootDir), roots.map((root) => ({ root, matcher: buildDefaultIgnore(path.join(rootDir, root)) })), loadExcludeMatcher(rootDir));
 }
 /**
  * Standalone discovery of every embedded repo root under `rootDir` (relative,
@@ -470,6 +520,28 @@ function discoverEmbeddedRepoRoots(rootDir) {
             }
         }
         catch { /* untracked listing failed — ignored-side discovery still runs */ }
+        // Unexpanded gitlinks (mode 160000) with a real checkout on disk — embedded
+        // repos `git add`ed without `.gitmodules`, or submodules not active here. The
+        // untracked listing above can't see them (they're tracked), so find them the
+        // same way collectGitFiles does, keeping watcher scope == indexer scope.
+        // (#1031, #1033)
+        try {
+            const staged = (0, child_process_1.execFileSync)('git', ['ls-files', '-z', '-s', '--recurse-submodules'], { cwd: repoAbs, encoding: 'utf-8', timeout: 30000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true });
+            for (const entry of staged.split('\0')) {
+                if (!entry || entry.slice(0, 6) !== '160000')
+                    continue;
+                const tab = entry.indexOf('\t');
+                if (tab === -1)
+                    continue;
+                const rel = entry.slice(tab + 1);
+                const relDir = rel.endsWith('/') ? rel : rel + '/';
+                if (defaults.ignores(relDir))
+                    continue;
+                if (classifyGitDir(path.join(repoAbs, rel)) === 'embedded')
+                    candidates.push(relDir);
+            }
+        }
+        catch { /* staged listing failed — other discovery still runs */ }
         candidates.push(...findIgnoredEmbeddedRepos(repoAbs, includeIgnored, prefix));
         for (const rel of candidates) {
             const full = (0, utils_1.normalizePath)(prefix + rel);
@@ -535,14 +607,37 @@ function collectGitFiles(repoDir, prefix, files, embeddedRoots, includeIgnored =
     // Without this, monorepos using submodules index 0 files. (See issue #147.)
     // Note: --recurse-submodules only supports -c/--cached and --stage modes — it
     // can't be combined with -o, so untracked files are gathered separately below.
+    //
+    // We use --stage (-s) rather than -c so each entry carries its file mode. That
+    // lets us spot gitlink entries (mode 160000) that --recurse-submodules did NOT
+    // expand: a nested repo `git add`ed without a `.gitmodules` entry, or a
+    // submodule that isn't active/initialized in this checkout. Such a gitlink
+    // falls through every pass — it's tracked, so the untracked `-o` listing below
+    // never reports it, and --recurse-submodules only expands ACTIVE submodules —
+    // so its source would be silently skipped, leaving only the super-repo's own
+    // files indexed. We collect those gitlinks here and recurse into them below.
+    // (An active submodule is expanded inline by --recurse-submodules and so never
+    // surfaces as a 160000 entry — only the unhandled gitlinks do.) (#1031, #1033)
+    //
     // -z gives NUL-separated, unquoted output so non-ASCII (e.g. CJK) paths
     // survive verbatim. Without it git octal-escapes and double-quotes such paths
     // (the core.quotepath default), and the quoted form never matches a real file
-    // on disk → those files are silently dropped from the index. (#541)
-    const tracked = (0, child_process_1.execFileSync)('git', ['ls-files', '-z', '-c', '--recurse-submodules'], gitOpts);
-    for (const rel of tracked.split('\0')) {
-        if (rel)
-            files.add((0, utils_1.normalizePath)(prefix + rel));
+    // on disk → those files are silently dropped from the index. (#541) With -s the
+    // path follows a TAB after the `<mode> <object> <stage>` prefix.
+    const gitlinkRels = [];
+    const tracked = (0, child_process_1.execFileSync)('git', ['ls-files', '-z', '-s', '--recurse-submodules'], gitOpts);
+    for (const entry of tracked.split('\0')) {
+        if (!entry)
+            continue;
+        const tab = entry.indexOf('\t');
+        if (tab === -1)
+            continue; // --stage always emits "<mode> <object> <stage>\t<path>"
+        const rel = entry.slice(tab + 1);
+        if (entry.slice(0, 6) === '160000') {
+            gitlinkRels.push(rel); // an unexpanded gitlink — recursed into below, not a source file itself
+            continue;
+        }
+        files.add((0, utils_1.normalizePath)(prefix + rel));
     }
     // Untracked files (submodules manage their own untracked state). Embedded git
     // repos surface here as a single "subdir/" entry that git refuses to descend
@@ -568,6 +663,25 @@ function collectGitFiles(repoDir, prefix, files, embeddedRoots, includeIgnored =
         }
         files.add((0, utils_1.normalizePath)(prefix + rel));
     }
+    // Gitlink entries (mode 160000) that --recurse-submodules left unexpanded —
+    // an embedded repo `git add`ed without `.gitmodules`, or a submodule not
+    // active/initialized in this checkout. When such a gitlink has a real working
+    // tree on disk it is distinct first-party code we must index as its own
+    // embedded repo: the tracked pass skipped its contents and the untracked pass
+    // never sees it (it's tracked, not "other"). A gitlink with no checkout on disk
+    // (an uninitialized submodule — empty dir, no `.git`) has nothing to index and
+    // is left alone, as is a submodule worktree (a duplicate view, #945). (#1031, #1033)
+    for (const rel of gitlinkRels) {
+        const relDir = rel.endsWith('/') ? rel : rel + '/';
+        if (defaultsOnlyIgnore().ignores(relDir))
+            continue;
+        const childDir = path.join(repoDir, rel);
+        // 'embedded' = a real .git checkout on disk; 'worktree' and 'none' are skipped.
+        if (classifyGitDir(childDir) !== 'embedded')
+            continue;
+        embeddedRoots?.add((0, utils_1.normalizePath)(prefix + relDir));
+        collectGitFiles(childDir, prefix + relDir, files, embeddedRoots, includeIgnored);
+    }
     // Embedded repos hidden by THIS repo's ignore rules (`/packages/` in a
     // super-repo .gitignore) never appear in any listing above. By default they
     // stay hidden — `.gitignore` is respected (#970, #976). They are recursed into
@@ -637,14 +751,14 @@ function getGitChangedFiles(rootDir) {
         // Custom extension → language overrides from the project's codegraph.json,
         // so change detection sees the same custom-extension files the full index does.
         const overrides = (0, project_config_1.loadExtensionOverrides)(rootDir);
-        collectGitStatus(rootDir, '', changes, overrides, loadIncludeIgnoredMatcher(rootDir));
+        collectGitStatus(rootDir, '', changes, overrides, loadIncludeIgnoredMatcher(rootDir), loadExcludeMatcher(rootDir));
         return changes;
     }
     catch {
         return null;
     }
 }
-function collectGitStatus(repoDir, prefix, out, overrides, includeIgnored = null) {
+function collectGitStatus(repoDir, prefix, out, overrides, includeIgnored = null, exclude = null) {
     const output = (0, child_process_1.execFileSync)('git', ['status', '--porcelain', '--no-renames'], { cwd: repoDir, encoding: 'utf-8', timeout: 10000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true });
     // This repo's own ignore rules — built-in defaults (#407) plus its .gitignore.
     // Change detection must exclude the SAME files the full index does, but git
@@ -683,6 +797,12 @@ function collectGitStatus(repoDir, prefix, out, overrides, includeIgnored = null
         // index — match against the repo-relative path, same as the full scan. (#766)
         if (ig.ignores(rel))
             continue;
+        // User `codegraph.json` `exclude` (#999) is project-root-relative, so it's
+        // matched against the full path — sync must not re-add a tracked file the
+        // full index now keeps out. Deletions above stay unfiltered so a file that
+        // WAS indexed before an exclude was added still cleans itself out.
+        if (exclude && exclude.ignores(filePath))
+            continue;
         if (statusCode === '??') {
             out.added.push(filePath);
         }
@@ -697,11 +817,11 @@ function collectGitStatus(repoDir, prefix, out, overrides, includeIgnored = null
     // and they are left alone (#970, #976), mirroring the full-index scan.
     for (const rel of untrackedDirs) {
         for (const repoRel of findNestedGitRepos(path.join(repoDir, rel), rel)) {
-            collectGitStatus(path.join(repoDir, repoRel), prefix + repoRel, out, overrides, includeIgnored);
+            collectGitStatus(path.join(repoDir, repoRel), prefix + repoRel, out, overrides, includeIgnored, exclude);
         }
     }
     for (const rel of findIgnoredEmbeddedRepos(repoDir, includeIgnored, prefix)) {
-        collectGitStatus(path.join(repoDir, rel), prefix + rel, out, overrides, includeIgnored);
+        collectGitStatus(path.join(repoDir, rel), prefix + rel, out, overrides, includeIgnored, exclude);
     }
 }
 /**
@@ -860,7 +980,14 @@ function scanDirectoryWalk(rootDir, onProgress) {
     }
     // Seed a base matcher with the built-in default ignores (merged with the root
     // .gitignore so a negation can override). Nested .gitignores still layer per-dir.
-    walk(rootDir, [{ dir: rootDir, ig: buildDefaultIgnore(rootDir) }]);
+    const baseMatchers = [{ dir: rootDir, ig: buildDefaultIgnore(rootDir) }];
+    // Project `codegraph.json` `exclude` patterns (#999), rooted at the project so
+    // `isIgnored` matches them against root-relative paths — same coverage the
+    // git path gets via ScopeIgnore, for non-git projects.
+    const exclude = loadExcludeMatcher(rootDir);
+    if (exclude)
+        baseMatchers.push({ dir: rootDir, ig: exclude });
+    walk(rootDir, baseMatchers);
     return files;
 }
 /**
@@ -1022,148 +1149,153 @@ class ExtractionOrchestrator {
         if (neededLanguages.includes('c') && !neededLanguages.includes('cpp')) {
             neededLanguages.push('cpp');
         }
-        // Try to use a worker thread for parsing (keeps main thread unblocked for UI).
-        // Falls back to in-process parsing if the compiled worker is unavailable (e.g. tests).
+        // Parse files on a pool of worker threads (keeps the main thread free for UI
+        // and uses every core). Falls back to in-process parsing when the compiled
+        // worker is unavailable (e.g. running from source in tests).
         const parseWorkerPath = path.join(__dirname, 'parse-worker.js');
         const useWorker = fs.existsSync(parseWorkerPath);
-        let WorkerClass = null;
+        let pool = null;
         if (useWorker) {
-            const { Worker } = await Promise.resolve().then(() => __importStar(require('worker_threads')));
-            WorkerClass = Worker;
+            // CODEGRAPH_PARSE_WORKERS: explicit worker count; 1 = the old single-worker
+            // behaviour (the conservative rollback). Unset → clamp(cores-1, 1, 8).
+            const poolSize = (0, parse_pool_1.resolveParsePoolSize)(process.env.CODEGRAPH_PARSE_WORKERS, os.cpus().length);
+            pool = new parse_pool_1.ParseWorkerPool({
+                languages: neededLanguages,
+                size: poolSize,
+                workerScriptPath: parseWorkerPath,
+                recycleInterval: WORKER_RECYCLE_INTERVAL,
+                parseTimeoutMs: PARSE_TIMEOUT_MS,
+                log,
+            });
+            log(`Parse worker pool: ${poolSize} worker(s)`);
         }
         else {
-            // In-process fallback: load grammars locally
+            // In-process fallback: load grammars locally and parse on the main thread.
             await (0, grammars_1.loadGrammarsForLanguages)(neededLanguages);
         }
-        // --- Worker lifecycle management ---
-        // The worker can crash (OOM in WASM) or hang on pathological files.
-        // We track pending parse promises and handle both cases:
-        //   - Timeout: terminate + restart the worker, reject the timed-out request
-        //   - Crash: reject all pending promises, restart for remaining files
-        let parseWorker = null;
-        let nextId = 0;
-        let workerParseCount = 0;
-        const pendingParses = new Map();
-        function rejectAllPending(reason) {
-            for (const [id, pending] of pendingParses) {
-                clearTimeout(pending.timer);
-                pendingParses.delete(id);
-                pending.reject(new Error(reason));
+        /**
+         * Parse one file: on the pool when available (the promise REJECTS on a worker
+         * crash/timeout — the caller records it and the retry pass re-attempts), or
+         * in-process synchronously as the no-worker fallback. The language is resolved
+         * here on the main thread, where the codegraph.json overrides are loaded.
+         */
+        const parseFile = (filePath, content) => {
+            const language = (0, grammars_1.detectLanguage)(filePath, content, overrides);
+            if (!pool)
+                return Promise.resolve((0, tree_sitter_1.extractFromSource)(filePath, content, language, frameworkNames));
+            return pool.requestParse({ filePath, content, language, frameworkNames });
+        };
+        // --- Bounded rolling-window dispatch, ordered commit ---
+        // Reads stay batched/parallel; parses run concurrently across the pool; the
+        // SQLite store stays on the main thread (it isn't thread-safe). Crucially we
+        // COMMIT results in original file order, not parse-completion order: the
+        // resolution phase (run after indexing) resolves an ambiguous reference to one
+        // of several same-named candidates by the nodes' DB insertion order, so a
+        // stable commit order keeps the resulting graph deterministic — byte-identical
+        // to the single-worker path — instead of drifting with parse timing. The
+        // `completed` buffer holds at most ~windowSize out-of-order results, so memory
+        // stays bounded.
+        const windowSize = pool ? Math.max(4, pool.size * 2) : 1;
+        const inFlight = new Set();
+        const completed = new Map();
+        let nextSeq = 0; // file-order sequence assigned at dispatch
+        let nextToStore = 0; // cursor: next sequence to commit
+        let aborted = false;
+        const storeResult = (filePath, content, stats, result) => {
+            processed++;
+            // Store in database on main thread (SQLite is not thread-safe)
+            if (result.nodes.length > 0 || result.errors.length === 0) {
+                const language = (0, grammars_1.detectLanguage)(filePath, content, overrides);
+                this.storeExtractionResult(filePath, content, language, stats, result);
             }
-        }
-        function attachWorkerHandlers(w) {
-            w.on('message', (msg) => {
-                if (msg.type === 'parse-result' && msg.id !== undefined) {
-                    const pending = pendingParses.get(msg.id);
-                    if (pending) {
-                        clearTimeout(pending.timer);
-                        pendingParses.delete(msg.id);
-                        pending.resolve(msg.result);
-                    }
+            if (result.errors.length > 0) {
+                for (const err of result.errors) {
+                    if (!err.filePath)
+                        err.filePath = filePath;
                 }
-            });
-            w.on('error', (err) => {
-                (0, errors_1.logWarn)('Parse worker error', { error: err.message });
-                rejectAllPending(`Worker error: ${err.message}`);
-            });
-            w.on('exit', (code) => {
-                if (code !== 0 && pendingParses.size > 0) {
-                    (0, errors_1.logWarn)('Parse worker exited unexpectedly', { code });
-                    rejectAllPending(`Worker exited with code ${code}`);
+                errors.push(...result.errors);
+            }
+            if (result.nodes.length > 0) {
+                filesIndexed++;
+                totalNodes += result.nodes.length;
+                totalEdges += result.edges.length;
+            }
+            else if (result.errors.some((e) => e.severity === 'error')) {
+                filesErrored++;
+            }
+            else {
+                // Files with no symbols but no errors (yaml, twig, properties) are
+                // tracked at the file level — count them as indexed so the CLI doesn't
+                // misleadingly report "No files found to index".
+                const lang = (0, grammars_1.detectLanguage)(filePath, content, overrides);
+                if ((0, grammars_1.isFileLevelOnlyLanguage)(lang)) {
+                    filesIndexed++;
                 }
-                // Clear reference so we know to respawn, reset count so
-                // the fresh worker gets a full cycle before recycling.
-                if (parseWorker === w) {
-                    parseWorker = null;
-                    workerParseCount = 0;
+                else {
+                    filesSkipped++;
                 }
+            }
+            onProgress?.({ phase: 'parsing', current: processed, total, currentFile: filePath });
+        };
+        const recordParseFailure = (filePath, err) => {
+            processed++;
+            filesErrored++;
+            errors.push({
+                message: err instanceof Error ? err.message : String(err),
+                filePath,
+                severity: 'error',
+                code: 'parse_error',
             });
-        }
-        async function ensureWorker() {
-            if (parseWorker)
-                return parseWorker;
-            log('Spawning new parse worker...');
-            parseWorker = new WorkerClass(parseWorkerPath);
-            attachWorkerHandlers(parseWorker);
-            // Load grammars in the new worker
-            await new Promise((resolve, reject) => {
-                parseWorker.once('message', (msg) => {
-                    if (msg.type === 'grammars-loaded')
-                        resolve();
-                    else
-                        reject(new Error(`Unexpected message: ${msg.type}`));
-                });
-                parseWorker.postMessage({ type: 'load-grammars', languages: neededLanguages });
-            });
-            return parseWorker;
-        }
-        if (WorkerClass) {
-            await ensureWorker();
-        }
-        /**
-         * Recycle the worker thread to reclaim WASM memory.
-         * Terminates the current worker and clears the reference so
-         * ensureWorker() will spawn a fresh one on the next call.
-         */
-        function recycleWorker() {
-            if (!parseWorker)
+            onProgress?.({ phase: 'parsing', current: processed, total });
+        };
+        // Commit buffered parses to the DB in file order, advancing the cursor over
+        // contiguous completed results. Runs after each parse settles (and once more
+        // after the drain). storeResult / recordParseFailure run here single-threaded,
+        // so shared counters and SQLite writes never race despite parallel parsing.
+        const flushOrdered = () => {
+            if (aborted)
                 return;
-            log(`Recycling worker after ${workerParseCount} parses (heap: ${Math.round(process.memoryUsage().rss / 1024 / 1024)}MB RSS)`);
-            const w = parseWorker;
-            parseWorker = null;
-            workerParseCount = 0;
-            // Fire-and-forget: worker.terminate() can hang if WASM is stuck
-            w.terminate().catch(() => { });
-        }
-        async function requestParse(filePath, content) {
-            // Resolve the language on the main thread (where the project's
-            // codegraph.json overrides are loaded) and hand it to the worker, so the
-            // worker never needs the override map itself.
-            const language = (0, grammars_1.detectLanguage)(filePath, content, overrides);
-            if (!WorkerClass) {
-                // In-process fallback
-                return (0, tree_sitter_1.extractFromSource)(filePath, content, language, frameworkNames);
+            while (completed.has(nextToStore)) {
+                const item = completed.get(nextToStore);
+                completed.delete(nextToStore);
+                nextToStore++;
+                if (item.ok)
+                    storeResult(item.filePath, item.content, item.stats, item.result);
+                else
+                    recordParseFailure(item.filePath, item.err);
             }
-            // Recycle the worker before the next parse if we've hit the threshold.
-            // This destroys the WASM linear memory (which can grow but never shrink)
-            // and starts a fresh worker with a clean heap.
-            if (workerParseCount >= WORKER_RECYCLE_INTERVAL) {
-                await recycleWorker();
+        };
+        // Dispatch one file's parse (parses run concurrently across the pool), tagged
+        // with its file-order sequence so flushOrdered commits results in order. The
+        // backpressure below bounds how far parsing runs ahead of the in-order commit.
+        const feed = async (filePath, content, stats) => {
+            const seq = nextSeq++;
+            const p = (async () => {
+                try {
+                    const result = await parseFile(filePath, content);
+                    completed.set(seq, { ok: true, filePath, content, stats, result });
+                }
+                catch (parseErr) {
+                    completed.set(seq, { ok: false, filePath, err: parseErr });
+                }
+                flushOrdered();
+            })();
+            const tracked = p.finally(() => { inFlight.delete(tracked); });
+            inFlight.add(tracked);
+            // Backpressure on the dispatched-but-not-yet-committed count (in-flight +
+            // buffered), not just in-flight: a slow file sitting at the commit cursor
+            // lets later parses finish and buffer, which would otherwise grow without
+            // bound. Wait for parses to settle (each may advance the cursor) until the
+            // window has room. `inFlight.size > 0` guards against an empty race — the
+            // cursor file is always still in flight when the window is full.
+            while (nextSeq - nextToStore >= windowSize && inFlight.size > 0) {
+                await Promise.race(inFlight);
             }
-            const worker = await ensureWorker();
-            const id = nextId++;
-            workerParseCount++;
-            // Scale timeout for large files: base 10s + 10s per 100KB
-            const timeoutMs = PARSE_TIMEOUT_MS + Math.floor(content.length / 100_000) * 10_000;
-            return new Promise((resolve, reject) => {
-                const timer = setTimeout(() => {
-                    pendingParses.delete(id);
-                    log(`TIMEOUT: ${filePath} exceeded ${timeoutMs}ms — killing worker`);
-                    // Reject FIRST — worker.terminate() can hang if WASM is stuck
-                    parseWorker = null;
-                    workerParseCount = 0;
-                    reject(new Error(`Parse timed out after ${timeoutMs}ms`));
-                    // Fire-and-forget: kill the stuck worker in the background
-                    worker.terminate().catch(() => { });
-                }, timeoutMs);
-                pendingParses.set(id, { resolve, reject, timer });
-                worker.postMessage({ type: 'parse', id, filePath, content, frameworkNames, language });
-            });
-        }
+        };
         for (let i = 0; i < files.length; i += FILE_IO_BATCH_SIZE) {
             if (signal?.aborted) {
-                if (parseWorker)
-                    parseWorker.terminate().catch(() => { });
-                return {
-                    success: false,
-                    filesIndexed,
-                    filesSkipped,
-                    filesErrored,
-                    nodesCreated: totalNodes,
-                    edgesCreated: totalEdges,
-                    errors: [{ message: 'Aborted', severity: 'error' }, ...errors],
-                    durationMs: Date.now() - startTime,
-                };
+                aborted = true;
+                break;
             }
             const batch = files.slice(i, i + FILE_IO_BATCH_SIZE);
             // Read files in parallel (with path validation before any I/O)
@@ -1185,29 +1317,13 @@ class ExtractionOrchestrator {
                     return { filePath: fp, content: null, stats: null, error: err };
                 }
             }));
-            // Send to worker for parsing, store results on main thread
+            // Dispatch each readable file into the bounded parse window; the window
+            // stores results on the main thread as they arrive.
             for (const { filePath, content, stats, error } of fileContents) {
                 if (signal?.aborted) {
-                    if (parseWorker)
-                        parseWorker.terminate().catch(() => { });
-                    return {
-                        success: false,
-                        filesIndexed,
-                        filesSkipped,
-                        filesErrored,
-                        nodesCreated: totalNodes,
-                        edgesCreated: totalEdges,
-                        errors: [{ message: 'Aborted', severity: 'error' }, ...errors],
-                        durationMs: Date.now() - startTime,
-                    };
+                    aborted = true;
+                    break;
                 }
-                // Report progress before parsing (show current file being worked on)
-                onProgress?.({
-                    phase: 'parsing',
-                    current: processed,
-                    total,
-                    currentFile: filePath,
-                });
                 if (error || content === null || stats === null) {
                     processed++;
                     filesErrored++;
@@ -1217,6 +1333,7 @@ class ExtractionOrchestrator {
                         severity: 'error',
                         code: 'read_error',
                     });
+                    onProgress?.({ phase: 'parsing', current: processed, total });
                     continue;
                 }
                 // Honour MAX_FILE_SIZE. Without this check, vendored generated
@@ -1236,57 +1353,32 @@ class ExtractionOrchestrator {
                     onProgress?.({ phase: 'parsing', current: processed, total });
                     continue;
                 }
-                // Parse in worker thread (main thread stays unblocked).
-                // Wrapped in try/catch to handle worker timeouts and crashes gracefully.
-                let result;
-                try {
-                    result = await requestParse(filePath, content);
-                }
-                catch (parseErr) {
-                    processed++;
-                    filesErrored++;
-                    errors.push({
-                        message: parseErr instanceof Error ? parseErr.message : String(parseErr),
-                        filePath,
-                        severity: 'error',
-                        code: 'parse_error',
-                    });
-                    continue;
-                }
-                processed++;
-                // Store in database on main thread (SQLite is not thread-safe)
-                if (result.nodes.length > 0 || result.errors.length === 0) {
-                    const language = (0, grammars_1.detectLanguage)(filePath, content, overrides);
-                    this.storeExtractionResult(filePath, content, language, stats, result);
-                }
-                if (result.errors.length > 0) {
-                    for (const err of result.errors) {
-                        if (!err.filePath)
-                            err.filePath = filePath;
-                    }
-                    errors.push(...result.errors);
-                }
-                if (result.nodes.length > 0) {
-                    filesIndexed++;
-                    totalNodes += result.nodes.length;
-                    totalEdges += result.edges.length;
-                }
-                else if (result.errors.some((e) => e.severity === 'error')) {
-                    filesErrored++;
-                }
-                else {
-                    // Files with no symbols but no errors (yaml, twig, properties) are
-                    // tracked at the file level — count them as indexed so the CLI
-                    // doesn't misleadingly report "No files found to index".
-                    const lang = (0, grammars_1.detectLanguage)(filePath, content, overrides);
-                    if ((0, grammars_1.isFileLevelOnlyLanguage)(lang)) {
-                        filesIndexed++;
-                    }
-                    else {
-                        filesSkipped++;
-                    }
-                }
+                // Parse on the pool (main thread stays unblocked). Errors/timeouts are
+                // handled inside feed() → recordParseFailure, feeding the retry pass.
+                await feed(filePath, content, stats);
             }
+            if (aborted)
+                break;
+        }
+        // Drain parses still in flight (skip on abort — we tear down below instead),
+        // then commit any results the cursor hasn't reached yet.
+        if (!aborted) {
+            await Promise.all(inFlight);
+            flushOrdered();
+        }
+        if (signal?.aborted || aborted) {
+            if (pool)
+                await pool.destroy();
+            return {
+                success: false,
+                filesIndexed,
+                filesSkipped,
+                filesErrored,
+                nodesCreated: totalNodes,
+                edgesCreated: totalEdges,
+                errors: [{ message: 'Aborted', severity: 'error' }, ...errors],
+                durationMs: Date.now() - startTime,
+            };
         }
         // Report 100% so the progress bar doesn't hang at 99%
         onProgress?.({
@@ -1303,15 +1395,17 @@ class ExtractionOrchestrator {
         // every file gets the absolute cleanest WASM state possible.
         const retryableErrors = errors.filter((e) => e.code === 'parse_error' && e.filePath &&
             (e.message.includes('Worker exited') || e.message.includes('memory access out of bounds')));
-        if (retryableErrors.length > 0 && WorkerClass) {
+        if (retryableErrors.length > 0 && pool) {
             log(`Retrying ${retryableErrors.length} files that failed due to WASM memory errors...`);
+            // Fresh WASM heaps for the retry phase. A retry that still crashes its
+            // worker makes the pool respawn it, so later retries keep landing on clean
+            // workers too.
+            pool.recycleAll();
             const stillFailing = [];
             for (const errEntry of retryableErrors) {
                 const filePath = errEntry.filePath;
                 if (signal?.aborted)
                     break;
-                // Fresh worker for every retry — maximum WASM headroom
-                recycleWorker();
                 let content;
                 try {
                     const fullPath = (0, utils_1.validatePathWithinRoot)(this.rootDir, filePath);
@@ -1324,7 +1418,7 @@ class ExtractionOrchestrator {
                 }
                 let result;
                 try {
-                    result = await requestParse(filePath, content);
+                    result = await parseFile(filePath, content);
                 }
                 catch {
                     stillFailing.push(errEntry);
@@ -1350,11 +1444,11 @@ class ExtractionOrchestrator {
             // code nodes but consume parser memory.
             if (stillFailing.length > 0) {
                 log(`${stillFailing.length} files still failing — retrying with comments stripped...`);
+                pool.recycleAll();
                 for (const errEntry of stillFailing) {
                     const filePath = errEntry.filePath;
                     if (signal?.aborted)
                         break;
-                    recycleWorker();
                     let fullContent;
                     try {
                         const fullPath = (0, utils_1.validatePathWithinRoot)(this.rootDir, filePath);
@@ -1373,7 +1467,7 @@ class ExtractionOrchestrator {
                         .join('\n');
                     let result;
                     try {
-                        result = await requestParse(filePath, stripped);
+                        result = await parseFile(filePath, stripped);
                     }
                     catch {
                         continue;
@@ -1394,11 +1488,9 @@ class ExtractionOrchestrator {
                 }
             }
         }
-        // Shut down parse worker and clear any pending timers
-        rejectAllPending('Indexing complete');
-        if (parseWorker) {
-            parseWorker.terminate().catch(() => { });
-        }
+        // Shut down the parse worker pool.
+        if (pool)
+            await pool.destroy();
         return {
             success: filesIndexed > 0 || errors.filter((e) => e.severity === 'error').length === 0,
             filesIndexed,