@colbymchenry/codegraph 0.6.8 → 0.7.3

package/dist/extraction/index.js

@@ -45,6 +45,7 @@ exports.loadAllGrammars = exports.loadGrammarsForLanguages = exports.initGrammar
 exports.hashContent = hashContent;
 exports.shouldIncludeFile = shouldIncludeFile;
 exports.scanDirectory = scanDirectory;
+exports.scanDirectoryAsync = scanDirectoryAsync;
 const fs = __importStar(require("fs"));
 const fsp = __importStar(require("fs/promises"));
 const path = __importStar(require("path"));
@@ -60,6 +61,21 @@ const picomatch_1 = __importDefault(require("picomatch"));
  * File reads are I/O-bound; batching overlaps I/O wait with CPU parse work.
  */
 const FILE_IO_BATCH_SIZE = 10;
+// PARSER_RESET_INTERVAL moved to parse-worker.ts (runs in worker thread)
+/**
+ * Maximum time (ms) to wait for a single file to parse in the worker thread.
+ * If tree-sitter hangs or WASM runs out of memory, this prevents the entire
+ * indexing run from freezing. The worker is restarted after a timeout.
+ */
+const PARSE_TIMEOUT_MS = 10_000;
+/**
+ * Number of files to parse before recycling the worker thread.
+ * WASM linear memory can grow but NEVER shrink (WebAssembly spec limitation).
+ * The only way to reclaim tree-sitter's WASM heap is to destroy the entire
+ * V8 isolate by terminating the worker thread and spawning a fresh one.
+ * This interval balances memory usage against the cost of reloading grammars.
+ */
+const WORKER_RECYCLE_INTERVAL = 250;
 /**
  * Calculate SHA256 hash of file contents
  */
@@ -98,6 +114,21 @@ function shouldIncludeFile(filePath, config) {
  */
 function getGitVisibleFiles(rootDir) {
     try {
+        // Check if the project directory is gitignored by a parent repo.
+        // When rootDir lives inside a parent git repo that ignores it,
+        // `git ls-files` returns nothing — fall back to filesystem walk.
+        const gitRoot = (0, child_process_1.execFileSync)('git', ['rev-parse', '--show-toplevel'], { cwd: rootDir, encoding: 'utf-8', timeout: 5000, stdio: ['pipe', 'pipe', 'pipe'] }).trim();
+        if (path.resolve(gitRoot) !== path.resolve(rootDir)) {
+            try {
+                // git check-ignore exits 0 if the path IS ignored, 1 if not
+                (0, child_process_1.execFileSync)('git', ['check-ignore', '-q', path.resolve(rootDir)], { cwd: rootDir, encoding: 'utf-8', timeout: 5000, stdio: ['pipe', 'pipe', 'pipe'] });
+                // Directory is gitignored by parent repo — fall back to filesystem walk
+                return null;
+            }
+            catch {
+                // Not ignored — safe to use git ls-files
+            }
+        }
         // -c = cached (tracked), -o = others (untracked), --exclude-standard = respect .gitignore
         const output = (0, child_process_1.execFileSync)('git', ['ls-files', '-co', '--exclude-standard'], { cwd: rootDir, encoding: 'utf-8', timeout: 30000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'] });
         const files = new Set();
@@ -177,6 +208,30 @@ function scanDirectory(rootDir, config, onProgress) {
     // Fallback: walk filesystem for non-git projects
     return scanDirectoryWalk(rootDir, config, onProgress);
 }
+/**
+ * Async variant of scanDirectory that yields to the event loop periodically,
+ * allowing worker threads to receive and render progress messages.
+ */
+async function scanDirectoryAsync(rootDir, config, onProgress) {
+    const gitFiles = getGitVisibleFiles(rootDir);
+    if (gitFiles) {
+        const files = [];
+        let count = 0;
+        for (const filePath of gitFiles) {
+            if (shouldIncludeFile(filePath, config)) {
+                files.push(filePath);
+                count++;
+                onProgress?.(count, filePath);
+                // Yield every 100 files so worker threads can render progress
+                if (count % 100 === 0) {
+                    await new Promise(r => setImmediate(r));
+                }
+            }
+        }
+        return files;
+    }
+    return scanDirectoryWalk(rootDir, config, onProgress);
+}
 /**
  * Filesystem walk fallback for non-git projects.
  */
@@ -285,21 +340,25 @@ class ExtractionOrchestrator {
     /**
      * Index all files in the project
      */
-    async indexAll(onProgress, signal) {
+    async indexAll(onProgress, signal, verbose) {
         await (0, grammars_1.initGrammars)();
         const startTime = Date.now();
         const errors = [];
         let filesIndexed = 0;
         let filesSkipped = 0;
+        let filesErrored = 0;
         let totalNodes = 0;
         let totalEdges = 0;
+        const log = verbose
+            ? (msg) => { console.log(`[worker] ${msg}`); }
+            : (_msg) => { };
         // Phase 1: Scan for files
         onProgress?.({
             phase: 'scanning',
             current: 0,
             total: 0,
         });
-        const files = scanDirectory(this.rootDir, this.config, (current, file) => {
+        const files = await scanDirectoryAsync(this.rootDir, this.config, (current, file) => {
             onProgress?.({
                 phase: 'scanning',
                 current,
@@ -312,26 +371,164 @@ class ExtractionOrchestrator {
                 success: false,
                 filesIndexed: 0,
                 filesSkipped: 0,
+                filesErrored: 0,
                 nodesCreated: 0,
                 edgesCreated: 0,
                 errors: [{ message: 'Aborted', severity: 'error' }],
                 durationMs: Date.now() - startTime,
             };
         }
-        // Load only the grammars needed for languages actually present in the project.
-        // This avoids compiling all 16+ WASM grammar modules upfront, which can cause
-        // V8 WASM Zone OOM on large codebases (see issue #54).
-        const neededLanguages = [...new Set(files.map((f) => (0, grammars_1.detectLanguage)(f)))];
-        await (0, grammars_1.loadGrammarsForLanguages)(neededLanguages);
-        // Phase 2: Parse files (read in parallel batches, parse/store sequentially)
+        // Phase 2: Parse files in a worker thread (keeps main thread unblocked for UI)
         const total = files.length;
         let processed = 0;
+        // Emit parsing phase immediately so the progress bar appears during worker setup.
+        // The yield lets the shimmer worker flush the phase transition to stdout before
+        // the main thread starts synchronous grammar detection work.
+        onProgress?.({
+            phase: 'parsing',
+            current: 0,
+            total,
+        });
+        await new Promise(resolve => setImmediate(resolve));
+        // Detect needed languages and load grammars in the parse worker
+        const neededLanguages = [...new Set(files.map((f) => (0, grammars_1.detectLanguage)(f)))];
+        // .h files default to 'c' but may be C++ — ensure cpp grammar is loaded when c is needed
+        if (neededLanguages.includes('c') && !neededLanguages.includes('cpp')) {
+            neededLanguages.push('cpp');
+        }
+        // Try to use a worker thread for parsing (keeps main thread unblocked for UI).
+        // Falls back to in-process parsing if the compiled worker is unavailable (e.g. tests).
+        const parseWorkerPath = path.join(__dirname, 'parse-worker.js');
+        const useWorker = fs.existsSync(parseWorkerPath);
+        let WorkerClass = null;
+        if (useWorker) {
+            const { Worker } = await Promise.resolve().then(() => __importStar(require('worker_threads')));
+            WorkerClass = Worker;
+        }
+        else {
+            // In-process fallback: load grammars locally
+            await (0, grammars_1.loadGrammarsForLanguages)(neededLanguages);
+        }
+        // --- Worker lifecycle management ---
+        // The worker can crash (OOM in WASM) or hang on pathological files.
+        // We track pending parse promises and handle both cases:
+        //   - Timeout: terminate + restart the worker, reject the timed-out request
+        //   - Crash: reject all pending promises, restart for remaining files
+        let parseWorker = null;
+        let nextId = 0;
+        let workerParseCount = 0;
+        const pendingParses = new Map();
+        function rejectAllPending(reason) {
+            for (const [id, pending] of pendingParses) {
+                clearTimeout(pending.timer);
+                pendingParses.delete(id);
+                pending.reject(new Error(reason));
+            }
+        }
+        function attachWorkerHandlers(w) {
+            w.on('message', (msg) => {
+                if (msg.type === 'parse-result' && msg.id !== undefined) {
+                    const pending = pendingParses.get(msg.id);
+                    if (pending) {
+                        clearTimeout(pending.timer);
+                        pendingParses.delete(msg.id);
+                        pending.resolve(msg.result);
+                    }
+                }
+            });
+            w.on('error', (err) => {
+                (0, errors_1.logWarn)('Parse worker error', { error: err.message });
+                rejectAllPending(`Worker error: ${err.message}`);
+            });
+            w.on('exit', (code) => {
+                if (code !== 0 && pendingParses.size > 0) {
+                    (0, errors_1.logWarn)('Parse worker exited unexpectedly', { code });
+                    rejectAllPending(`Worker exited with code ${code}`);
+                }
+                // Clear reference so we know to respawn, reset count so
+                // the fresh worker gets a full cycle before recycling.
+                if (parseWorker === w) {
+                    parseWorker = null;
+                    workerParseCount = 0;
+                }
+            });
+        }
+        async function ensureWorker() {
+            if (parseWorker)
+                return parseWorker;
+            log('Spawning new parse worker...');
+            parseWorker = new WorkerClass(parseWorkerPath);
+            attachWorkerHandlers(parseWorker);
+            // Load grammars in the new worker
+            await new Promise((resolve, reject) => {
+                parseWorker.once('message', (msg) => {
+                    if (msg.type === 'grammars-loaded')
+                        resolve();
+                    else
+                        reject(new Error(`Unexpected message: ${msg.type}`));
+                });
+                parseWorker.postMessage({ type: 'load-grammars', languages: neededLanguages });
+            });
+            return parseWorker;
+        }
+        if (WorkerClass) {
+            await ensureWorker();
+        }
+        /**
+         * Recycle the worker thread to reclaim WASM memory.
+         * Terminates the current worker and clears the reference so
+         * ensureWorker() will spawn a fresh one on the next call.
+         */
+        function recycleWorker() {
+            if (!parseWorker)
+                return;
+            log(`Recycling worker after ${workerParseCount} parses (heap: ${Math.round(process.memoryUsage().rss / 1024 / 1024)}MB RSS)`);
+            const w = parseWorker;
+            parseWorker = null;
+            workerParseCount = 0;
+            // Fire-and-forget: worker.terminate() can hang if WASM is stuck
+            w.terminate().catch(() => { });
+        }
+        async function requestParse(filePath, content) {
+            if (!WorkerClass) {
+                // In-process fallback
+                return (0, tree_sitter_1.extractFromSource)(filePath, content, (0, grammars_1.detectLanguage)(filePath, content));
+            }
+            // Recycle the worker before the next parse if we've hit the threshold.
+            // This destroys the WASM linear memory (which can grow but never shrink)
+            // and starts a fresh worker with a clean heap.
+            if (workerParseCount >= WORKER_RECYCLE_INTERVAL) {
+                await recycleWorker();
+            }
+            const worker = await ensureWorker();
+            const id = nextId++;
+            workerParseCount++;
+            // Scale timeout for large files: base 10s + 10s per 100KB
+            const timeoutMs = PARSE_TIMEOUT_MS + Math.floor(content.length / 100_000) * 10_000;
+            return new Promise((resolve, reject) => {
+                const timer = setTimeout(() => {
+                    pendingParses.delete(id);
+                    log(`TIMEOUT: ${filePath} exceeded ${timeoutMs}ms — killing worker`);
+                    // Reject FIRST — worker.terminate() can hang if WASM is stuck
+                    parseWorker = null;
+                    workerParseCount = 0;
+                    reject(new Error(`Parse timed out after ${timeoutMs}ms`));
+                    // Fire-and-forget: kill the stuck worker in the background
+                    worker.terminate().catch(() => { });
+                }, timeoutMs);
+                pendingParses.set(id, { resolve, reject, timer });
+                worker.postMessage({ type: 'parse', id, filePath, content });
+            });
+        }
         for (let i = 0; i < files.length; i += FILE_IO_BATCH_SIZE) {
             if (signal?.aborted) {
+                if (parseWorker)
+                    parseWorker.terminate().catch(() => { });
                 return {
                     success: false,
                     filesIndexed,
                     filesSkipped,
+                    filesErrored,
                     nodesCreated: totalNodes,
                     edgesCreated: totalEdges,
                     errors: [{ message: 'Aborted', severity: 'error' }, ...errors],
@@ -355,20 +552,23 @@ class ExtractionOrchestrator {
                     return { filePath: fp, content: null, stats: null, error: err };
                 }
             }));
-            // Parse and store sequentially
+            // Send to worker for parsing, store results on main thread
             for (const { filePath, content, stats, error } of fileContents) {
                 if (signal?.aborted) {
+                    if (parseWorker)
+                        parseWorker.terminate().catch(() => { });
                     return {
                         success: false,
                         filesIndexed,
                         filesSkipped,
+                        filesErrored,
                         nodesCreated: totalNodes,
                         edgesCreated: totalEdges,
                         errors: [{ message: 'Aborted', severity: 'error' }, ...errors],
                         durationMs: Date.now() - startTime,
                     };
                 }
-                processed++;
+                // Report progress before parsing (show current file being worked on)
                 onProgress?.({
                     phase: 'parsing',
                     current: processed,
@@ -376,14 +576,44 @@ class ExtractionOrchestrator {
                     currentFile: filePath,
                 });
                 if (error || content === null || stats === null) {
+                    processed++;
+                    filesErrored++;
                     errors.push({
                         message: `Failed to read file: ${error instanceof Error ? error.message : String(error)}`,
+                        filePath,
+                        severity: 'error',
+                        code: 'read_error',
+                    });
+                    continue;
+                }
+                // Parse in worker thread (main thread stays unblocked).
+                // Wrapped in try/catch to handle worker timeouts and crashes gracefully.
+                let result;
+                try {
+                    result = await requestParse(filePath, content);
+                }
+                catch (parseErr) {
+                    processed++;
+                    filesErrored++;
+                    errors.push({
+                        message: parseErr instanceof Error ? parseErr.message : String(parseErr),
+                        filePath,
                         severity: 'error',
+                        code: 'parse_error',
                     });
                     continue;
                 }
-                const result = await this.indexFileWithContent(filePath, content, stats);
+                processed++;
+                // Store in database on main thread (SQLite is not thread-safe)
+                if (result.nodes.length > 0 || result.errors.length === 0) {
+                    const language = (0, grammars_1.detectLanguage)(filePath, content);
+                    this.storeExtractionResult(filePath, content, language, stats, result);
+                }
                 if (result.errors.length > 0) {
+                    for (const err of result.errors) {
+                        if (!err.filePath)
+                            err.filePath = filePath;
+                    }
                     errors.push(...result.errors);
                 }
                 if (result.nodes.length > 0) {
@@ -391,22 +621,130 @@ class ExtractionOrchestrator {
                     totalNodes += result.nodes.length;
                     totalEdges += result.edges.length;
                 }
-                else if (result.errors.length === 0) {
+                else if (result.errors.some((e) => e.severity === 'error')) {
+                    filesErrored++;
+                }
+                else {
                     filesSkipped++;
                 }
             }
         }
-        // Phase 3: Resolve references
+        // Report 100% so the progress bar doesn't hang at 99%
         onProgress?.({
-            phase: 'resolving',
-            current: 0,
-            total: 1,
+            phase: 'parsing',
+            current: total,
+            total,
         });
-        // TODO: Implement reference resolution in Phase 3
+        // Yield so the shimmer worker's buffered stdout writes can flush.
+        // Worker thread stdout is proxied through the main thread's event loop,
+        // so synchronous work here blocks the animation from rendering.
+        await new Promise(resolve => setImmediate(resolve));
+        // Retry pass: files that failed due to WASM memory corruption may succeed
+        // on a fresh worker with a clean heap. Recycle before each attempt so
+        // every file gets the absolute cleanest WASM state possible.
+        const retryableErrors = errors.filter((e) => e.code === 'parse_error' && e.filePath &&
+            (e.message.includes('Worker exited') || e.message.includes('memory access out of bounds')));
+        if (retryableErrors.length > 0 && WorkerClass) {
+            log(`Retrying ${retryableErrors.length} files that failed due to WASM memory errors...`);
+            const stillFailing = [];
+            for (const errEntry of retryableErrors) {
+                const filePath = errEntry.filePath;
+                if (signal?.aborted)
+                    break;
+                // Fresh worker for every retry — maximum WASM headroom
+                recycleWorker();
+                let content;
+                try {
+                    const fullPath = (0, utils_1.validatePathWithinRoot)(this.rootDir, filePath);
+                    if (!fullPath)
+                        continue;
+                    content = await fsp.readFile(fullPath, 'utf-8');
+                }
+                catch {
+                    continue;
+                }
+                let result;
+                try {
+                    result = await requestParse(filePath, content);
+                }
+                catch {
+                    stillFailing.push(errEntry);
+                    continue;
+                }
+                if (result.nodes.length > 0 || result.errors.length === 0) {
+                    const language = (0, grammars_1.detectLanguage)(filePath, content);
+                    const stats = await fsp.stat(path.join(this.rootDir, filePath));
+                    this.storeExtractionResult(filePath, content, language, stats, result);
+                    const idx = errors.indexOf(errEntry);
+                    if (idx >= 0)
+                        errors.splice(idx, 1);
+                    filesErrored--;
+                    filesIndexed++;
+                    totalNodes += result.nodes.length;
+                    totalEdges += result.edges.length;
+                    log(`Retry OK: ${filePath} (${result.nodes.length} nodes)`);
+                }
+            }
+            // Last resort: for files that still crash on a clean worker, strip
+            // comment-only lines to reduce WASM memory pressure. Many compiler
+            // test files are 90%+ comments (CHECK directives) that don't contribute
+            // code nodes but consume parser memory.
+            if (stillFailing.length > 0) {
+                log(`${stillFailing.length} files still failing — retrying with comments stripped...`);
+                for (const errEntry of stillFailing) {
+                    const filePath = errEntry.filePath;
+                    if (signal?.aborted)
+                        break;
+                    recycleWorker();
+                    let fullContent;
+                    try {
+                        const fullPath = (0, utils_1.validatePathWithinRoot)(this.rootDir, filePath);
+                        if (!fullPath)
+                            continue;
+                        fullContent = await fsp.readFile(fullPath, 'utf-8');
+                    }
+                    catch {
+                        continue;
+                    }
+                    // Strip lines that are entirely comments (preserving line numbers
+                    // by replacing with empty lines so node positions stay correct)
+                    const stripped = fullContent
+                        .split('\n')
+                        .map(line => /^\s*\/\//.test(line) ? '' : line)
+                        .join('\n');
+                    let result;
+                    try {
+                        result = await requestParse(filePath, stripped);
+                    }
+                    catch {
+                        continue;
+                    }
+                    if (result.nodes.length > 0 || result.errors.length === 0) {
+                        const language = (0, grammars_1.detectLanguage)(filePath, fullContent);
+                        const stats = await fsp.stat(path.join(this.rootDir, filePath));
+                        this.storeExtractionResult(filePath, fullContent, language, stats, result);
+                        const idx = errors.indexOf(errEntry);
+                        if (idx >= 0)
+                            errors.splice(idx, 1);
+                        filesErrored--;
+                        filesIndexed++;
+                        totalNodes += result.nodes.length;
+                        totalEdges += result.edges.length;
+                        log(`Retry (stripped) OK: ${filePath} (${result.nodes.length} nodes)`);
+                    }
+                }
+            }
+        }
+        // Shut down parse worker and clear any pending timers
+        rejectAllPending('Indexing complete');
+        if (parseWorker) {
+            parseWorker.terminate().catch(() => { });
+        }
         return {
-            success: errors.filter((e) => e.severity === 'error').length === 0,
+            success: filesIndexed > 0 || errors.filter((e) => e.severity === 'error').length === 0,
             filesIndexed,
             filesSkipped,
+            filesErrored,
             nodesCreated: totalNodes,
             edgesCreated: totalEdges,
             errors,
@@ -421,6 +759,7 @@ class ExtractionOrchestrator {
         const errors = [];
         let filesIndexed = 0;
         let filesSkipped = 0;
+        let filesErrored = 0;
         let totalNodes = 0;
         let totalEdges = 0;
         for (const filePath of filePaths) {
@@ -433,14 +772,18 @@ class ExtractionOrchestrator {
                 totalNodes += result.nodes.length;
                 totalEdges += result.edges.length;
             }
+            else if (result.errors.some((e) => e.severity === 'error')) {
+                filesErrored++;
+            }
             else {
                 filesSkipped++;
             }
         }
         return {
-            success: errors.filter((e) => e.severity === 'error').length === 0,
+            success: filesIndexed > 0 || errors.filter((e) => e.severity === 'error').length === 0,
             filesIndexed,
             filesSkipped,
+            filesErrored,
             nodesCreated: totalNodes,
             edgesCreated: totalEdges,
             errors,
@@ -457,7 +800,7 @@ class ExtractionOrchestrator {
                 nodes: [],
                 edges: [],
                 unresolvedReferences: [],
-                errors: [{ message: `Path traversal blocked: ${relativePath}`, severity: 'error' }],
+                errors: [{ message: `Path traversal blocked: ${relativePath}`, filePath: relativePath, severity: 'error', code: 'path_traversal' }],
                 durationMs: 0,
             };
         }
@@ -476,7 +819,9 @@ class ExtractionOrchestrator {
                 errors: [
                     {
                         message: `Failed to read file: ${error instanceof Error ? error.message : String(error)}`,
+                        filePath: relativePath,
                         severity: 'error',
+                        code: 'read_error',
                     },
                 ],
                 durationMs: 0,
@@ -497,7 +842,7 @@ class ExtractionOrchestrator {
                 nodes: [],
                 edges: [],
                 unresolvedReferences: [],
-                errors: [{ message: 'Path traversal blocked', severity: 'error' }],
+                errors: [{ message: 'Path traversal blocked', filePath: relativePath, severity: 'error', code: 'path_traversal' }],
                 durationMs: 0,
             };
         }
@@ -510,14 +855,16 @@ class ExtractionOrchestrator {
                 errors: [
                     {
                         message: `File exceeds max size (${stats.size} > ${this.config.maxFileSize})`,
+                        filePath: relativePath,
                         severity: 'warning',
+                        code: 'size_exceeded',
                     },
                 ],
                 durationMs: 0,
             };
         }
         // Detect language
-        const language = (0, grammars_1.detectLanguage)(relativePath);
+        const language = (0, grammars_1.detectLanguage)(relativePath, content);
         if (!(0, grammars_1.isLanguageSupported)(language)) {
             return {
                 nodes: [],
@@ -700,6 +1047,10 @@ class ExtractionOrchestrator {
         // Load only grammars needed for changed files
         if (filesToIndex.length > 0) {
             const neededLanguages = [...new Set(filesToIndex.map((f) => (0, grammars_1.detectLanguage)(f)))];
+            // .h files default to 'c' but may be C++ — ensure cpp grammar is loaded
+            if (neededLanguages.includes('c') && !neededLanguages.includes('cpp')) {
+                neededLanguages.push('cpp');
+            }
             await (0, grammars_1.loadGrammarsForLanguages)(neededLanguages);
         }
         // Index changed files