@colbymchenry/codegraph-darwin-x64 1.1.0 → 1.1.2

package/lib/dist/extraction/index.js

@@ -51,9 +51,11 @@ exports.scanDirectoryAsync = scanDirectoryAsync;
 const fs = __importStar(require("fs"));
 const fsp = __importStar(require("fs/promises"));
 const path = __importStar(require("path"));
+const os = __importStar(require("os"));
 const crypto = __importStar(require("crypto"));
 const child_process_1 = require("child_process");
 const tree_sitter_1 = require("./tree-sitter");
+const parse_pool_1 = require("./parse-pool");
 const grammars_1 = require("./grammars");
 const project_config_1 = require("../project-config");
 const directory_1 = require("../directory");
@@ -252,6 +254,32 @@ function buildDefaultIgnore(rootDir) {
 function defaultsOnlyIgnore() {
     return (0, ignore_1.default)().add(DEFAULT_IGNORE_PATTERNS);
 }
+/**
+ * Matcher for the project's `codegraph.json` `includeIgnored` patterns — the
+ * explicit opt-in to index embedded git repos living inside gitignored
+ * directories (#622, #699). Returns `null` when the project opted in nothing,
+ * which is the zero-config DEFAULT: `.gitignore` is then fully respected and a
+ * gitignored directory (even one holding nested repos) is never walked or
+ * indexed (#970, #976). Built once per scan/sync/scope operation from the scan
+ * root and threaded down — never global, so multi-project daemons stay isolated.
+ */
+function loadIncludeIgnoredMatcher(rootDir) {
+    const patterns = (0, project_config_1.loadIncludeIgnoredPatterns)(rootDir);
+    return patterns.length > 0 ? (0, ignore_1.default)().add(patterns) : null;
+}
+/**
+ * Matcher for the project's `codegraph.json` `exclude` patterns — paths to keep
+ * OUT of the index even when git-tracked, which `.gitignore` cannot do (#999).
+ * The escape hatch for a committed vendor/theme/SDK directory. Returns `null`
+ * when nothing is excluded (the zero-config default → no overhead). Matched
+ * against project-root-relative paths, so it applies uniformly across the whole
+ * workspace, including inside embedded repos (excluding `static/` means gone
+ * everywhere). Built once per scan/sync/scope operation from the scan root.
+ */
+function loadExcludeMatcher(rootDir) {
+    const patterns = (0, project_config_1.loadExcludePatterns)(rootDir);
+    return patterns.length > 0 ? (0, ignore_1.default)().add(patterns) : null;
+}
 /**
  * `git ls-files --directory` collapses a wholly-untracked/ignored directory into
  * one entry — and when the command's own cwd is such a directory (the indexed
@@ -391,14 +419,27 @@ function findNestedGitRepos(absDir, relPrefix) {
  */
 class ScopeIgnore {
     rootMatcher;
+    exclude;
     embedded;
     defaults = defaultsOnlyIgnore();
-    constructor(rootMatcher, embedded) {
+    constructor(rootMatcher, embedded, 
+    /**
+     * Project `codegraph.json` `exclude` patterns (#999), matched against the
+     * full root-relative path. Wins over everything else — an explicit user
+     * exclude applies even to tracked files and even inside embedded repos.
+     */
+    exclude = null) {
         this.rootMatcher = rootMatcher;
+        this.exclude = exclude;
         // Longest root first so paths in nested embedded repos hit the innermost matcher.
         this.embedded = [...embedded].sort((a, b) => b.root.length - a.root.length);
     }
     ignores(rel) {
+        // User `exclude` (#999) is checked first and against the full root-relative
+        // path: it must drop git-TRACKED paths (which `.gitignore` can't) and apply
+        // everywhere, including ancestors of embedded repos.
+        if (this.exclude && this.exclude.ignores(rel))
+            return true;
         for (const { root, matcher } of this.embedded) {
             if (rel.startsWith(root)) {
                 const inner = rel.slice(root.length);
@@ -425,13 +466,16 @@ exports.ScopeIgnore = ScopeIgnore;
  */
 function buildScopeIgnore(rootDir, embeddedRoots) {
     const roots = embeddedRoots ? [...embeddedRoots] : discoverEmbeddedRepoRoots(rootDir);
-    return new ScopeIgnore(buildDefaultIgnore(rootDir), roots.map((root) => ({ root, matcher: buildDefaultIgnore(path.join(rootDir, root)) })));
+    return new ScopeIgnore(buildDefaultIgnore(rootDir), roots.map((root) => ({ root, matcher: buildDefaultIgnore(path.join(rootDir, root)) })), loadExcludeMatcher(rootDir));
 }
 /**
  * Standalone discovery of every embedded repo root under `rootDir` (relative,
- * trailing-slashed) — both the untracked kind (#193) and the gitignored kind
- * (#514), recursively (an embedded repo can embed further repos). Returns []
- * for non-git roots: the filesystem walk handles nested repos there already.
+ * trailing-slashed) — the untracked kind (#193) always, and the gitignored kind
+ * (#514) only for directories the project opted in via `codegraph.json`
+ * `includeIgnored` (#622, #699); otherwise `.gitignore` is respected and they
+ * are not discovered (#970, #976). Recursive (an embedded repo can embed further
+ * repos). Returns [] for non-git roots: the filesystem walk handles nested repos
+ * there already.
  */
 function discoverEmbeddedRepoRoots(rootDir) {
     try {
@@ -442,6 +486,7 @@ function discoverEmbeddedRepoRoots(rootDir) {
     }
     const out = [];
     const defaults = defaultsOnlyIgnore();
+    const includeIgnored = loadIncludeIgnoredMatcher(rootDir);
     const visit = (repoAbs, prefix) => {
         const candidates = [];
         try {
@@ -453,7 +498,7 @@ function discoverEmbeddedRepoRoots(rootDir) {
             }
         }
         catch { /* untracked listing failed — ignored-side discovery still runs */ }
-        candidates.push(...findIgnoredEmbeddedRepos(repoAbs));
+        candidates.push(...findIgnoredEmbeddedRepos(repoAbs, includeIgnored, prefix));
         for (const rel of candidates) {
             const full = (0, utils_1.normalizePath)(prefix + rel);
             out.push(full);
@@ -464,16 +509,30 @@ function discoverEmbeddedRepoRoots(rootDir) {
     return out;
 }
 /**
- * Discover embedded repos hidden by `repoDir`'s OWN ignore rules: for each
- * gitignored directory (skipping built-in default excludes), search for nested
- * `.git` roots. Returns repo paths relative to `repoDir`, trailing-slashed.
+ * Discover embedded repos hidden by `repoDir`'s OWN gitignore rules: for each
+ * gitignored directory, search for nested `.git` roots. Returns repo paths
+ * relative to `repoDir`, trailing-slashed.
+ *
+ * OPT-IN ONLY. Walking into a gitignored directory contradicts what every other
+ * tool (and CodeGraph's own `git ls-files` foundation) does — `.gitignore`
+ * excludes. So this returns `[]` unless the project opted the directory in via
+ * `codegraph.json` `includeIgnored`; without that, a gitignored dir — including
+ * a huge reference/data dir full of nested clones — is left untouched (#970,
+ * #976). When opted in, it restores the super-repo-of-clones behavior (#622,
+ * #699). `prefix` is the scan-root-relative path of `repoDir`, so a pattern like
+ * `services/` opts that whole subtree in at any recursion depth. Built-in
+ * default excludes (`node_modules`, …) are always skipped.
  */
-function findIgnoredEmbeddedRepos(repoDir) {
+function findIgnoredEmbeddedRepos(repoDir, includeIgnored, prefix) {
+    if (!includeIgnored)
+        return [];
     const defaults = defaultsOnlyIgnore();
     const repos = [];
     for (const dir of listIgnoredDirs(repoDir)) {
         if (defaults.ignores(dir))
             continue;
+        if (!includeIgnored.ignores((0, utils_1.normalizePath)(prefix + dir)))
+            continue;
         repos.push(...findNestedGitRepos(path.join(repoDir, dir), dir));
     }
     return repos;
@@ -489,12 +548,15 @@ function findIgnoredEmbeddedRepos(repoDir) {
  * skips them entirely, and untracked output reports them only as an opaque
  * "subdir/" entry (trailing slash) rather than expanding their files. Each
  * embedded repo is its own git boundary, so we re-run `git ls-files` inside it.
- * (See issue #193.) GITIGNORED embedded repos are invisible even to that —
- * they're discovered separately via `findIgnoredEmbeddedRepos` (#514); every
- * embedded repo root (however found) is recorded in `embeddedRoots` so callers
- * can exempt its files from the parent's own gitignore rules.
+ * (See issue #193.) GITIGNORED embedded repos are invisible even to that; they
+ * are discovered separately via `findIgnoredEmbeddedRepos` (#514) but ONLY for
+ * directories the project opted in through `codegraph.json` `includeIgnored`
+ * (`includeIgnored` here, threaded from the scan root) — by default `.gitignore`
+ * is respected and they stay out (#970, #976). Every embedded repo root (however
+ * found) is recorded in `embeddedRoots` so callers can exempt its files from the
+ * parent's own gitignore rules.
  */
-function collectGitFiles(repoDir, prefix, files, embeddedRoots) {
+function collectGitFiles(repoDir, prefix, files, embeddedRoots, includeIgnored = null) {
     const gitOpts = { cwd: repoDir, encoding: 'utf-8', timeout: 30000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true };
     // Tracked files. --recurse-submodules pulls in files from active submodules,
     // which the index would otherwise represent only as a commit pointer.
@@ -528,18 +590,20 @@ function collectGitFiles(repoDir, prefix, files, embeddedRoots) {
             // it's a duplicate working view of an already-indexed repo (#848).
             if (classifyGitDir(childDir) === 'embedded' && !defaultsOnlyIgnore().ignores(rel)) {
                 embeddedRoots?.add((0, utils_1.normalizePath)(prefix + rel));
-                collectGitFiles(childDir, prefix + rel, files, embeddedRoots);
+                collectGitFiles(childDir, prefix + rel, files, embeddedRoots, includeIgnored);
             }
             continue;
         }
         files.add((0, utils_1.normalizePath)(prefix + rel));
     }
     // Embedded repos hidden by THIS repo's ignore rules (`/packages/` in a
-    // super-repo .gitignore) never appear in any listing above — discover and
-    // recurse into them too. (#514)
-    for (const rel of findIgnoredEmbeddedRepos(repoDir)) {
+    // super-repo .gitignore) never appear in any listing above. By default they
+    // stay hidden — `.gitignore` is respected (#970, #976). They are recursed into
+    // only when the project opted the directory in via `codegraph.json`
+    // `includeIgnored` (#622, #699), which `findIgnoredEmbeddedRepos` enforces.
+    for (const rel of findIgnoredEmbeddedRepos(repoDir, includeIgnored, prefix)) {
         embeddedRoots?.add((0, utils_1.normalizePath)(prefix + rel));
-        collectGitFiles(path.join(repoDir, rel), prefix + rel, files, embeddedRoots);
+        collectGitFiles(path.join(repoDir, rel), prefix + rel, files, embeddedRoots, includeIgnored);
     }
 }
 /**
@@ -567,7 +631,7 @@ function getGitVisibleFiles(rootDir) {
         }
         const files = new Set();
         const embeddedRoots = new Set();
-        collectGitFiles(rootDir, '', files, embeddedRoots);
+        collectGitFiles(rootDir, '', files, embeddedRoots, loadIncludeIgnoredMatcher(rootDir));
         // Apply built-in default ignores uniformly — to tracked files too, since
         // committing a dependency/build dir doesn't make it project code. A
         // `.gitignore` negation (e.g. `!vendor/`) is the explicit opt-in. (issue #407)
@@ -585,13 +649,15 @@ function getGitVisibleFiles(rootDir) {
  * Use `git status` to detect changed files instead of scanning every file.
  * Returns null on failure so callers fall back to full scan.
  *
- * Recurses into embedded repos — both the untracked kind (#193: the parent's
- * status collapses them to an opaque `?? subdir/` entry) and the gitignored
- * kind (#514: they never appear in the parent's status at all) — running
- * `git status` inside each, so changes in a multi-repo workspace sync without
- * a full rescan. Deleting an ENTIRE embedded repo dir is the one case this
- * cannot see (the child status that would report the deletions is gone with
- * it); a full `codegraph index` reconciles that.
+ * Recurses into embedded repos — the untracked kind (#193: the parent's status
+ * collapses them to an opaque `?? subdir/` entry) always, and the gitignored
+ * kind (#514: they never appear in the parent's status at all) only for
+ * directories opted in via `codegraph.json` `includeIgnored` (#622, #699) —
+ * running `git status` inside each, so changes in a multi-repo workspace sync
+ * without a full rescan. By default a gitignored dir is left alone, matching the
+ * full-index scan (#970, #976). Deleting an ENTIRE embedded repo dir is the one
+ * case this cannot see (the child status that would report the deletions is gone
+ * with it); a full `codegraph index` reconciles that.
  */
 function getGitChangedFiles(rootDir) {
     try {
@@ -599,14 +665,14 @@ function getGitChangedFiles(rootDir) {
         // Custom extension → language overrides from the project's codegraph.json,
         // so change detection sees the same custom-extension files the full index does.
         const overrides = (0, project_config_1.loadExtensionOverrides)(rootDir);
-        collectGitStatus(rootDir, '', changes, overrides);
+        collectGitStatus(rootDir, '', changes, overrides, loadIncludeIgnoredMatcher(rootDir), loadExcludeMatcher(rootDir));
         return changes;
     }
     catch {
         return null;
     }
 }
-function collectGitStatus(repoDir, prefix, out, overrides) {
+function collectGitStatus(repoDir, prefix, out, overrides, includeIgnored = null, exclude = null) {
     const output = (0, child_process_1.execFileSync)('git', ['status', '--porcelain', '--no-renames'], { cwd: repoDir, encoding: 'utf-8', timeout: 10000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true });
     // This repo's own ignore rules — built-in defaults (#407) plus its .gitignore.
     // Change detection must exclude the SAME files the full index does, but git
@@ -645,6 +711,12 @@ function collectGitStatus(repoDir, prefix, out, overrides) {
         // index — match against the repo-relative path, same as the full scan. (#766)
         if (ig.ignores(rel))
             continue;
+        // User `codegraph.json` `exclude` (#999) is project-root-relative, so it's
+        // matched against the full path — sync must not re-add a tracked file the
+        // full index now keeps out. Deletions above stay unfiltered so a file that
+        // WAS indexed before an exclude was added still cleans itself out.
+        if (exclude && exclude.ignores(filePath))
+            continue;
         if (statusCode === '??') {
             out.added.push(filePath);
         }
@@ -654,14 +726,16 @@ function collectGitStatus(repoDir, prefix, out, overrides) {
         }
     }
     // Recurse embedded repos found under untracked dirs (at the dir itself or
-    // nested deeper) and under this repo's gitignored dirs.
+    // nested deeper). Gitignored dirs are walked only for the directories the
+    // project opted in via `includeIgnored`; by default `.gitignore` is respected
+    // and they are left alone (#970, #976), mirroring the full-index scan.
     for (const rel of untrackedDirs) {
         for (const repoRel of findNestedGitRepos(path.join(repoDir, rel), rel)) {
-            collectGitStatus(path.join(repoDir, repoRel), prefix + repoRel, out, overrides);
+            collectGitStatus(path.join(repoDir, repoRel), prefix + repoRel, out, overrides, includeIgnored, exclude);
         }
     }
-    for (const rel of findIgnoredEmbeddedRepos(repoDir)) {
-        collectGitStatus(path.join(repoDir, rel), prefix + rel, out, overrides);
+    for (const rel of findIgnoredEmbeddedRepos(repoDir, includeIgnored, prefix)) {
+        collectGitStatus(path.join(repoDir, rel), prefix + rel, out, overrides, includeIgnored, exclude);
     }
 }
 /**
@@ -820,7 +894,14 @@ function scanDirectoryWalk(rootDir, onProgress) {
     }
     // Seed a base matcher with the built-in default ignores (merged with the root
     // .gitignore so a negation can override). Nested .gitignores still layer per-dir.
-    walk(rootDir, [{ dir: rootDir, ig: buildDefaultIgnore(rootDir) }]);
+    const baseMatchers = [{ dir: rootDir, ig: buildDefaultIgnore(rootDir) }];
+    // Project `codegraph.json` `exclude` patterns (#999), rooted at the project so
+    // `isIgnored` matches them against root-relative paths — same coverage the
+    // git path gets via ScopeIgnore, for non-git projects.
+    const exclude = loadExcludeMatcher(rootDir);
+    if (exclude)
+        baseMatchers.push({ dir: rootDir, ig: exclude });
+    walk(rootDir, baseMatchers);
     return files;
 }
 /**
@@ -982,148 +1063,153 @@ class ExtractionOrchestrator {
         if (neededLanguages.includes('c') && !neededLanguages.includes('cpp')) {
             neededLanguages.push('cpp');
         }
-        // Try to use a worker thread for parsing (keeps main thread unblocked for UI).
-        // Falls back to in-process parsing if the compiled worker is unavailable (e.g. tests).
+        // Parse files on a pool of worker threads (keeps the main thread free for UI
+        // and uses every core). Falls back to in-process parsing when the compiled
+        // worker is unavailable (e.g. running from source in tests).
         const parseWorkerPath = path.join(__dirname, 'parse-worker.js');
         const useWorker = fs.existsSync(parseWorkerPath);
-        let WorkerClass = null;
+        let pool = null;
         if (useWorker) {
-            const { Worker } = await Promise.resolve().then(() => __importStar(require('worker_threads')));
-            WorkerClass = Worker;
+            // CODEGRAPH_PARSE_WORKERS: explicit worker count; 1 = the old single-worker
+            // behaviour (the conservative rollback). Unset → clamp(cores-1, 1, 8).
+            const poolSize = (0, parse_pool_1.resolveParsePoolSize)(process.env.CODEGRAPH_PARSE_WORKERS, os.cpus().length);
+            pool = new parse_pool_1.ParseWorkerPool({
+                languages: neededLanguages,
+                size: poolSize,
+                workerScriptPath: parseWorkerPath,
+                recycleInterval: WORKER_RECYCLE_INTERVAL,
+                parseTimeoutMs: PARSE_TIMEOUT_MS,
+                log,
+            });
+            log(`Parse worker pool: ${poolSize} worker(s)`);
         }
         else {
-            // In-process fallback: load grammars locally
+            // In-process fallback: load grammars locally and parse on the main thread.
             await (0, grammars_1.loadGrammarsForLanguages)(neededLanguages);
         }
-        // --- Worker lifecycle management ---
-        // The worker can crash (OOM in WASM) or hang on pathological files.
-        // We track pending parse promises and handle both cases:
-        //   - Timeout: terminate + restart the worker, reject the timed-out request
-        //   - Crash: reject all pending promises, restart for remaining files
-        let parseWorker = null;
-        let nextId = 0;
-        let workerParseCount = 0;
-        const pendingParses = new Map();
-        function rejectAllPending(reason) {
-            for (const [id, pending] of pendingParses) {
-                clearTimeout(pending.timer);
-                pendingParses.delete(id);
-                pending.reject(new Error(reason));
+        /**
+         * Parse one file: on the pool when available (the promise REJECTS on a worker
+         * crash/timeout — the caller records it and the retry pass re-attempts), or
+         * in-process synchronously as the no-worker fallback. The language is resolved
+         * here on the main thread, where the codegraph.json overrides are loaded.
+         */
+        const parseFile = (filePath, content) => {
+            const language = (0, grammars_1.detectLanguage)(filePath, content, overrides);
+            if (!pool)
+                return Promise.resolve((0, tree_sitter_1.extractFromSource)(filePath, content, language, frameworkNames));
+            return pool.requestParse({ filePath, content, language, frameworkNames });
+        };
+        // --- Bounded rolling-window dispatch, ordered commit ---
+        // Reads stay batched/parallel; parses run concurrently across the pool; the
+        // SQLite store stays on the main thread (it isn't thread-safe). Crucially we
+        // COMMIT results in original file order, not parse-completion order: the
+        // resolution phase (run after indexing) resolves an ambiguous reference to one
+        // of several same-named candidates by the nodes' DB insertion order, so a
+        // stable commit order keeps the resulting graph deterministic — byte-identical
+        // to the single-worker path — instead of drifting with parse timing. The
+        // `completed` buffer holds at most ~windowSize out-of-order results, so memory
+        // stays bounded.
+        const windowSize = pool ? Math.max(4, pool.size * 2) : 1;
+        const inFlight = new Set();
+        const completed = new Map();
+        let nextSeq = 0; // file-order sequence assigned at dispatch
+        let nextToStore = 0; // cursor: next sequence to commit
+        let aborted = false;
+        const storeResult = (filePath, content, stats, result) => {
+            processed++;
+            // Store in database on main thread (SQLite is not thread-safe)
+            if (result.nodes.length > 0 || result.errors.length === 0) {
+                const language = (0, grammars_1.detectLanguage)(filePath, content, overrides);
+                this.storeExtractionResult(filePath, content, language, stats, result);
             }
-        }
-        function attachWorkerHandlers(w) {
-            w.on('message', (msg) => {
-                if (msg.type === 'parse-result' && msg.id !== undefined) {
-                    const pending = pendingParses.get(msg.id);
-                    if (pending) {
-                        clearTimeout(pending.timer);
-                        pendingParses.delete(msg.id);
-                        pending.resolve(msg.result);
-                    }
+            if (result.errors.length > 0) {
+                for (const err of result.errors) {
+                    if (!err.filePath)
+                        err.filePath = filePath;
                 }
-            });
-            w.on('error', (err) => {
-                (0, errors_1.logWarn)('Parse worker error', { error: err.message });
-                rejectAllPending(`Worker error: ${err.message}`);
-            });
-            w.on('exit', (code) => {
-                if (code !== 0 && pendingParses.size > 0) {
-                    (0, errors_1.logWarn)('Parse worker exited unexpectedly', { code });
-                    rejectAllPending(`Worker exited with code ${code}`);
+                errors.push(...result.errors);
+            }
+            if (result.nodes.length > 0) {
+                filesIndexed++;
+                totalNodes += result.nodes.length;
+                totalEdges += result.edges.length;
+            }
+            else if (result.errors.some((e) => e.severity === 'error')) {
+                filesErrored++;
+            }
+            else {
+                // Files with no symbols but no errors (yaml, twig, properties) are
+                // tracked at the file level — count them as indexed so the CLI doesn't
+                // misleadingly report "No files found to index".
+                const lang = (0, grammars_1.detectLanguage)(filePath, content, overrides);
+                if ((0, grammars_1.isFileLevelOnlyLanguage)(lang)) {
+                    filesIndexed++;
                 }
-                // Clear reference so we know to respawn, reset count so
-                // the fresh worker gets a full cycle before recycling.
-                if (parseWorker === w) {
-                    parseWorker = null;
-                    workerParseCount = 0;
+                else {
+                    filesSkipped++;
                 }
+            }
+            onProgress?.({ phase: 'parsing', current: processed, total, currentFile: filePath });
+        };
+        const recordParseFailure = (filePath, err) => {
+            processed++;
+            filesErrored++;
+            errors.push({
+                message: err instanceof Error ? err.message : String(err),
+                filePath,
+                severity: 'error',
+                code: 'parse_error',
             });
-        }
-        async function ensureWorker() {
-            if (parseWorker)
-                return parseWorker;
-            log('Spawning new parse worker...');
-            parseWorker = new WorkerClass(parseWorkerPath);
-            attachWorkerHandlers(parseWorker);
-            // Load grammars in the new worker
-            await new Promise((resolve, reject) => {
-                parseWorker.once('message', (msg) => {
-                    if (msg.type === 'grammars-loaded')
-                        resolve();
-                    else
-                        reject(new Error(`Unexpected message: ${msg.type}`));
-                });
-                parseWorker.postMessage({ type: 'load-grammars', languages: neededLanguages });
-            });
-            return parseWorker;
-        }
-        if (WorkerClass) {
-            await ensureWorker();
-        }
-        /**
-         * Recycle the worker thread to reclaim WASM memory.
-         * Terminates the current worker and clears the reference so
-         * ensureWorker() will spawn a fresh one on the next call.
-         */
-        function recycleWorker() {
-            if (!parseWorker)
+            onProgress?.({ phase: 'parsing', current: processed, total });
+        };
+        // Commit buffered parses to the DB in file order, advancing the cursor over
+        // contiguous completed results. Runs after each parse settles (and once more
+        // after the drain). storeResult / recordParseFailure run here single-threaded,
+        // so shared counters and SQLite writes never race despite parallel parsing.
+        const flushOrdered = () => {
+            if (aborted)
                 return;
-            log(`Recycling worker after ${workerParseCount} parses (heap: ${Math.round(process.memoryUsage().rss / 1024 / 1024)}MB RSS)`);
-            const w = parseWorker;
-            parseWorker = null;
-            workerParseCount = 0;
-            // Fire-and-forget: worker.terminate() can hang if WASM is stuck
-            w.terminate().catch(() => { });
-        }
-        async function requestParse(filePath, content) {
-            // Resolve the language on the main thread (where the project's
-            // codegraph.json overrides are loaded) and hand it to the worker, so the
-            // worker never needs the override map itself.
-            const language = (0, grammars_1.detectLanguage)(filePath, content, overrides);
-            if (!WorkerClass) {
-                // In-process fallback
-                return (0, tree_sitter_1.extractFromSource)(filePath, content, language, frameworkNames);
+            while (completed.has(nextToStore)) {
+                const item = completed.get(nextToStore);
+                completed.delete(nextToStore);
+                nextToStore++;
+                if (item.ok)
+                    storeResult(item.filePath, item.content, item.stats, item.result);
+                else
+                    recordParseFailure(item.filePath, item.err);
             }
-            // Recycle the worker before the next parse if we've hit the threshold.
-            // This destroys the WASM linear memory (which can grow but never shrink)
-            // and starts a fresh worker with a clean heap.
-            if (workerParseCount >= WORKER_RECYCLE_INTERVAL) {
-                await recycleWorker();
+        };
+        // Dispatch one file's parse (parses run concurrently across the pool), tagged
+        // with its file-order sequence so flushOrdered commits results in order. The
+        // backpressure below bounds how far parsing runs ahead of the in-order commit.
+        const feed = async (filePath, content, stats) => {
+            const seq = nextSeq++;
+            const p = (async () => {
+                try {
+                    const result = await parseFile(filePath, content);
+                    completed.set(seq, { ok: true, filePath, content, stats, result });
+                }
+                catch (parseErr) {
+                    completed.set(seq, { ok: false, filePath, err: parseErr });
+                }
+                flushOrdered();
+            })();
+            const tracked = p.finally(() => { inFlight.delete(tracked); });
+            inFlight.add(tracked);
+            // Backpressure on the dispatched-but-not-yet-committed count (in-flight +
+            // buffered), not just in-flight: a slow file sitting at the commit cursor
+            // lets later parses finish and buffer, which would otherwise grow without
+            // bound. Wait for parses to settle (each may advance the cursor) until the
+            // window has room. `inFlight.size > 0` guards against an empty race — the
+            // cursor file is always still in flight when the window is full.
+            while (nextSeq - nextToStore >= windowSize && inFlight.size > 0) {
+                await Promise.race(inFlight);
             }
-            const worker = await ensureWorker();
-            const id = nextId++;
-            workerParseCount++;
-            // Scale timeout for large files: base 10s + 10s per 100KB
-            const timeoutMs = PARSE_TIMEOUT_MS + Math.floor(content.length / 100_000) * 10_000;
-            return new Promise((resolve, reject) => {
-                const timer = setTimeout(() => {
-                    pendingParses.delete(id);
-                    log(`TIMEOUT: ${filePath} exceeded ${timeoutMs}ms — killing worker`);
-                    // Reject FIRST — worker.terminate() can hang if WASM is stuck
-                    parseWorker = null;
-                    workerParseCount = 0;
-                    reject(new Error(`Parse timed out after ${timeoutMs}ms`));
-                    // Fire-and-forget: kill the stuck worker in the background
-                    worker.terminate().catch(() => { });
-                }, timeoutMs);
-                pendingParses.set(id, { resolve, reject, timer });
-                worker.postMessage({ type: 'parse', id, filePath, content, frameworkNames, language });
-            });
-        }
+        };
         for (let i = 0; i < files.length; i += FILE_IO_BATCH_SIZE) {
             if (signal?.aborted) {
-                if (parseWorker)
-                    parseWorker.terminate().catch(() => { });
-                return {
-                    success: false,
-                    filesIndexed,
-                    filesSkipped,
-                    filesErrored,
-                    nodesCreated: totalNodes,
-                    edgesCreated: totalEdges,
-                    errors: [{ message: 'Aborted', severity: 'error' }, ...errors],
-                    durationMs: Date.now() - startTime,
-                };
+                aborted = true;
+                break;
             }
             const batch = files.slice(i, i + FILE_IO_BATCH_SIZE);
             // Read files in parallel (with path validation before any I/O)
@@ -1145,29 +1231,13 @@ class ExtractionOrchestrator {
                     return { filePath: fp, content: null, stats: null, error: err };
                 }
             }));
-            // Send to worker for parsing, store results on main thread
+            // Dispatch each readable file into the bounded parse window; the window
+            // stores results on the main thread as they arrive.
             for (const { filePath, content, stats, error } of fileContents) {
                 if (signal?.aborted) {
-                    if (parseWorker)
-                        parseWorker.terminate().catch(() => { });
-                    return {
-                        success: false,
-                        filesIndexed,
-                        filesSkipped,
-                        filesErrored,
-                        nodesCreated: totalNodes,
-                        edgesCreated: totalEdges,
-                        errors: [{ message: 'Aborted', severity: 'error' }, ...errors],
-                        durationMs: Date.now() - startTime,
-                    };
+                    aborted = true;
+                    break;
                 }
-                // Report progress before parsing (show current file being worked on)
-                onProgress?.({
-                    phase: 'parsing',
-                    current: processed,
-                    total,
-                    currentFile: filePath,
-                });
                 if (error || content === null || stats === null) {
                     processed++;
                     filesErrored++;
@@ -1177,6 +1247,7 @@ class ExtractionOrchestrator {
                         severity: 'error',
                         code: 'read_error',
                     });
+                    onProgress?.({ phase: 'parsing', current: processed, total });
                     continue;
                 }
                 // Honour MAX_FILE_SIZE. Without this check, vendored generated
@@ -1196,57 +1267,32 @@ class ExtractionOrchestrator {
                     onProgress?.({ phase: 'parsing', current: processed, total });
                     continue;
                 }
-                // Parse in worker thread (main thread stays unblocked).
-                // Wrapped in try/catch to handle worker timeouts and crashes gracefully.
-                let result;
-                try {
-                    result = await requestParse(filePath, content);
-                }
-                catch (parseErr) {
-                    processed++;
-                    filesErrored++;
-                    errors.push({
-                        message: parseErr instanceof Error ? parseErr.message : String(parseErr),
-                        filePath,
-                        severity: 'error',
-                        code: 'parse_error',
-                    });
-                    continue;
-                }
-                processed++;
-                // Store in database on main thread (SQLite is not thread-safe)
-                if (result.nodes.length > 0 || result.errors.length === 0) {
-                    const language = (0, grammars_1.detectLanguage)(filePath, content, overrides);
-                    this.storeExtractionResult(filePath, content, language, stats, result);
-                }
-                if (result.errors.length > 0) {
-                    for (const err of result.errors) {
-                        if (!err.filePath)
-                            err.filePath = filePath;
-                    }
-                    errors.push(...result.errors);
-                }
-                if (result.nodes.length > 0) {
-                    filesIndexed++;
-                    totalNodes += result.nodes.length;
-                    totalEdges += result.edges.length;
-                }
-                else if (result.errors.some((e) => e.severity === 'error')) {
-                    filesErrored++;
-                }
-                else {
-                    // Files with no symbols but no errors (yaml, twig, properties) are
-                    // tracked at the file level — count them as indexed so the CLI
-                    // doesn't misleadingly report "No files found to index".
-                    const lang = (0, grammars_1.detectLanguage)(filePath, content, overrides);
-                    if ((0, grammars_1.isFileLevelOnlyLanguage)(lang)) {
-                        filesIndexed++;
-                    }
-                    else {
-                        filesSkipped++;
-                    }
-                }
+                // Parse on the pool (main thread stays unblocked). Errors/timeouts are
+                // handled inside feed() → recordParseFailure, feeding the retry pass.
+                await feed(filePath, content, stats);
             }
+            if (aborted)
+                break;
+        }
+        // Drain parses still in flight (skip on abort — we tear down below instead),
+        // then commit any results the cursor hasn't reached yet.
+        if (!aborted) {
+            await Promise.all(inFlight);
+            flushOrdered();
+        }
+        if (signal?.aborted || aborted) {
+            if (pool)
+                await pool.destroy();
+            return {
+                success: false,
+                filesIndexed,
+                filesSkipped,
+                filesErrored,
+                nodesCreated: totalNodes,
+                edgesCreated: totalEdges,
+                errors: [{ message: 'Aborted', severity: 'error' }, ...errors],
+                durationMs: Date.now() - startTime,
+            };
         }
         // Report 100% so the progress bar doesn't hang at 99%
         onProgress?.({
@@ -1263,15 +1309,17 @@ class ExtractionOrchestrator {
         // every file gets the absolute cleanest WASM state possible.
         const retryableErrors = errors.filter((e) => e.code === 'parse_error' && e.filePath &&
             (e.message.includes('Worker exited') || e.message.includes('memory access out of bounds')));
-        if (retryableErrors.length > 0 && WorkerClass) {
+        if (retryableErrors.length > 0 && pool) {
             log(`Retrying ${retryableErrors.length} files that failed due to WASM memory errors...`);
+            // Fresh WASM heaps for the retry phase. A retry that still crashes its
+            // worker makes the pool respawn it, so later retries keep landing on clean
+            // workers too.
+            pool.recycleAll();
             const stillFailing = [];
             for (const errEntry of retryableErrors) {
                 const filePath = errEntry.filePath;
                 if (signal?.aborted)
                     break;
-                // Fresh worker for every retry — maximum WASM headroom
-                recycleWorker();
                 let content;
                 try {
                     const fullPath = (0, utils_1.validatePathWithinRoot)(this.rootDir, filePath);
@@ -1284,7 +1332,7 @@ class ExtractionOrchestrator {
                 }
                 let result;
                 try {
-                    result = await requestParse(filePath, content);
+                    result = await parseFile(filePath, content);
                 }
                 catch {
                     stillFailing.push(errEntry);
@@ -1310,11 +1358,11 @@ class ExtractionOrchestrator {
             // code nodes but consume parser memory.
             if (stillFailing.length > 0) {
                 log(`${stillFailing.length} files still failing — retrying with comments stripped...`);
+                pool.recycleAll();
                 for (const errEntry of stillFailing) {
                     const filePath = errEntry.filePath;
                     if (signal?.aborted)
                         break;
-                    recycleWorker();
                     let fullContent;
                     try {
                         const fullPath = (0, utils_1.validatePathWithinRoot)(this.rootDir, filePath);
@@ -1333,7 +1381,7 @@ class ExtractionOrchestrator {
                         .join('\n');
                     let result;
                     try {
-                        result = await requestParse(filePath, stripped);
+                        result = await parseFile(filePath, stripped);
                     }
                     catch {
                         continue;
@@ -1354,11 +1402,9 @@ class ExtractionOrchestrator {
                 }
             }
         }
-        // Shut down parse worker and clear any pending timers
-        rejectAllPending('Indexing complete');
-        if (parseWorker) {
-            parseWorker.terminate().catch(() => { });
-        }
+        // Shut down the parse worker pool.
+        if (pool)
+            await pool.destroy();
         return {
             success: filesIndexed > 0 || errors.filter((e) => e.severity === 'error').length === 0,
             filesIndexed,