@vizalice/codegraph 1.5.0 → 1.6.1

package/dist/extraction/index.js

@@ -47,6 +47,7 @@ exports.buildDefaultIgnore = buildDefaultIgnore;
 exports.buildScopeIgnore = buildScopeIgnore;
 exports.discoverEmbeddedRepoRoots = discoverEmbeddedRepoRoots;
 exports.findUnindexedIgnoredRepos = findUnindexedIgnoredRepos;
+exports.getGitChangedFiles = getGitChangedFiles;
 exports.scanDirectory = scanDirectory;
 exports.scanDirectoryAsync = scanDirectoryAsync;
 const fs = __importStar(require("fs"));
@@ -1057,7 +1058,14 @@ function getGitChangedFiles(rootDir) {
     }
 }
 function collectGitStatus(repoDir, prefix, out, overrides, includeIgnored = null, exclude = null) {
-    const output = (0, child_process_1.execFileSync)('git', ['status', '--porcelain', '--no-renames'], { cwd: repoDir, encoding: 'utf-8', timeout: 10000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true });
+    const output = (0, child_process_1.execFileSync)('git', 
+    // `-uall` lists individual untracked files instead of collapsing an
+    // entirely-untracked directory into one `?? dir/` entry, which would
+    // otherwise be dropped here (only embedded git repos are recursed into
+    // below). Nested untracked git repos still collapse to `?? repo/` even
+    // with `-uall` — git never crosses a repo boundary — so the recursion
+    // still handles them. (#1213)
+    ['status', '--porcelain', '--no-renames', '-uall'], { cwd: repoDir, encoding: 'utf-8', timeout: 10000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true });
     // This repo's own ignore rules — built-in defaults (#407) plus its .gitignore.
     // Change detection must exclude the SAME files the full index does, but git
     // status hides neither: it ignores nothing for *tracked* paths, and the
@@ -1344,10 +1352,47 @@ class ExtractionOrchestrator {
      * hasn't run yet so single-file re-index paths can detect on the spot.
      */
     detectedFrameworkNames = null;
+    /**
+     * Scope matcher for SCOPED syncs, memoized on the mtimes of the two root
+     * files it is derived from (`codegraph.json`, `.gitignore`). See
+     * {@link scopedSyncMatcher}.
+     */
+    scopedMatcher = null;
     constructor(rootDir, queries) {
         this.rootDir = rootDir;
         this.queries = queries;
     }
+    /**
+     * The scope matcher a scoped sync applies to the paths it was handed — the
+     * same `buildScopeIgnore` the full scan uses, so an explicitly-passed path
+     * that is OUT of scope (a user `exclude` in `codegraph.json`, a `.gitignore`
+     * rule, a built-in default) is treated exactly as the full walk would treat
+     * it: absent, hence removed if tracked, never parsed (#1590).
+     *
+     * Memoized on the root config + root `.gitignore` mtimes: building the
+     * matcher runs embedded-repo discovery (`git ls-files`), which would defeat
+     * the scoped path's whole point (skipping O(repo) work) if paid per sync.
+     * Two `stat`s per sync while nothing changed. An embedded repo created
+     * between config edits joins the scoped matcher on the next full sync, the
+     * same lifecycle the watcher's own matcher already has.
+     */
+    scopedSyncMatcher() {
+        const key = [project_config_1.PROJECT_CONFIG_FILENAME, '.gitignore']
+            .map((name) => {
+            try {
+                return String(fs.statSync(path.join(this.rootDir, name)).mtimeMs);
+            }
+            catch {
+                return '-';
+            }
+        })
+            .join('|');
+        if (this.scopedMatcher && this.scopedMatcher.key === key)
+            return this.scopedMatcher.matcher;
+        const matcher = buildScopeIgnore(this.rootDir);
+        this.scopedMatcher = { key, matcher };
+        return matcher;
+    }
     /**
      * Build a filesystem-backed ResolutionContext sufficient for framework
      * detection. Graph-query methods (getNodesByName etc.) return empty because
@@ -1473,6 +1518,10 @@ class ExtractionOrchestrator {
         });
         if (process.env.CODEGRAPH_SYNTH_TIMINGS)
             console.error(`[phase-timing] scan: ${Date.now() - tScan}ms (${files.length} files)`);
+        // A re-index over an existing DB skips unchanged-hash files at the store,
+        // which would preserve wiped zero-node rows (#1541) — drop them first so
+        // this run stores their files fresh. No-op on a fresh DB.
+        this.healZeroNodeRows();
         // Detect frameworks once per indexAll run using the scanned file list.
         // Names are passed to each parse call so framework-specific extractors
         // (route nodes, middleware, etc.) run after the tree-sitter pass.
@@ -1619,29 +1668,27 @@ class ExtractionOrchestrator {
             // Store: on the writer thread when active (fresh DB — bundles applied
             // in the same file order this chain dispatches them), else on the main
             // thread (SQLite connections are per-thread).
-            if (nodeCount > 0 || result.errors.length === 0) {
-                const language = (0, grammars_1.detectLanguage)(filePath, content, overrides);
-                if (storeWriter) {
-                    if (result.kernelBuffers) {
-                        // Buffers go to the writer as-is; the worker decodes + finalizes.
-                        // The main thread's only per-file work stays O(1) + the content hash.
-                        storeWriter.send({
-                            kernel: true,
-                            filePath,
-                            language,
-                            buffers: result.kernelBuffers,
-                            file: this.buildFileRecord(filePath, content, language, stats, nodeCount, result.errors),
-                        });
-                    }
-                    else {
-                        storeWriter.send(this.buildFreshStoreBundle(filePath, content, language, stats, result));
-                    }
-                    await storeWriter.waitBelow(STORE_WRITER_WINDOW);
+            const language = (0, grammars_1.detectLanguage)(filePath, content, overrides);
+            if (storeWriter) {
+                if (result.kernelBuffers) {
+                    // Buffers go to the writer as-is; the worker decodes + finalizes.
+                    // The main thread's only per-file work stays O(1) + the content hash.
+                    storeWriter.send({
+                        kernel: true,
+                        filePath,
+                        language,
+                        buffers: result.kernelBuffers,
+                        file: this.buildFileRecord(filePath, content, language, stats, nodeCount, result.errors),
+                    });
                 }
                 else {
-                    const materialized = (0, kernel_1.materializeKernelResult)(result, filePath, language);
-                    await this.storeExtractionResult(filePath, content, language, stats, materialized, commitYield);
+                    storeWriter.send(this.buildFreshStoreBundle(filePath, content, language, stats, result));
                 }
+                await storeWriter.waitBelow(STORE_WRITER_WINDOW);
+            }
+            else {
+                const materialized = (0, kernel_1.materializeKernelResult)(result, filePath, language);
+                await this.storeExtractionResult(filePath, content, language, stats, materialized, commitYield);
             }
             if (result.errors.length > 0) {
                 for (const err of result.errors) {
@@ -1672,16 +1719,19 @@ class ExtractionOrchestrator {
             }
             onProgress?.({ phase: 'parsing', current: processed, total, currentFile: filePath });
         };
-        const recordParseFailure = (filePath, err) => {
-            processed++;
-            filesErrored++;
-            errors.push({
-                message: err instanceof Error ? err.message : String(err),
-                filePath,
-                severity: 'error',
-                code: 'parse_error',
+        const recordParseFailure = async (filePath, content, stats, err) => {
+            await storeResult(filePath, content, stats, {
+                nodes: [],
+                edges: [],
+                unresolvedReferences: [],
+                errors: [{
+                        message: err instanceof Error ? err.message : String(err),
+                        filePath,
+                        severity: 'error',
+                        code: 'parse_error',
+                    }],
+                durationMs: 0,
             });
-            onProgress?.({ phase: 'parsing', current: processed, total });
         };
         // Commit buffered parses to the DB in file order, advancing the cursor over
         // contiguous completed results. Runs after each parse settles (and once more
@@ -1706,7 +1756,7 @@ class ExtractionOrchestrator {
                         if (item.ok)
                             await storeResult(item.filePath, item.content, item.stats, item.result);
                         else
-                            recordParseFailure(item.filePath, item.err);
+                            await recordParseFailure(item.filePath, item.content, item.stats, item.err);
                     }
                 }
                 catch (err) {
@@ -1726,7 +1776,7 @@ class ExtractionOrchestrator {
                     completed.set(seq, { ok: true, filePath, content, stats, result });
                 }
                 catch (parseErr) {
-                    completed.set(seq, { ok: false, filePath, err: parseErr });
+                    completed.set(seq, { ok: false, filePath, content, stats, err: parseErr });
                 }
                 flushOrdered();
             })();
@@ -1798,15 +1848,18 @@ class ExtractionOrchestrator {
                 // useful symbols. The single-file extractFile path already enforces
                 // this; the bulk path used to silently skip the check.
                 if (stats.size > MAX_FILE_SIZE) {
-                    processed++;
-                    filesSkipped++;
-                    errors.push({
-                        message: `File exceeds max size (${stats.size} > ${MAX_FILE_SIZE})`,
-                        filePath,
-                        severity: 'warning',
-                        code: 'size_exceeded',
+                    await storeResult(filePath, content, stats, {
+                        nodes: [],
+                        edges: [],
+                        unresolvedReferences: [],
+                        errors: [{
+                                message: `File exceeds max size (${stats.size} > ${MAX_FILE_SIZE})`,
+                                filePath,
+                                severity: 'warning',
+                                code: 'size_exceeded',
+                            }],
+                        durationMs: 0,
                     });
-                    onProgress?.({ phase: 'parsing', current: processed, total });
                     continue;
                 }
                 // Parse on the pool (main thread stays unblocked). Errors/timeouts are
@@ -1907,8 +1960,15 @@ class ExtractionOrchestrator {
                     stillFailing.push(errEntry);
                     continue;
                 }
+                // The pool hands kernel results back as an undecoded buffer transport
+                // (`nodes`/`edges` EMPTY, tables in kernelBuffers). The main loop
+                // decodes or forwards to the store worker; this path stores directly,
+                // so decode here — otherwise a kernel-language retry passes the gate
+                // below via `errors.length === 0`, stores nothing, and the file is
+                // permanently recorded as "(0 symbols)" with the error erased (#1541).
+                const language = (0, grammars_1.detectLanguage)(filePath, content, overrides);
+                result = (0, kernel_1.materializeKernelResult)(result, filePath, language);
                 if (result.nodes.length > 0 || result.errors.length === 0) {
-                    const language = (0, grammars_1.detectLanguage)(filePath, content, overrides);
                     const stats = await fsp.stat(path.join(this.rootDir, filePath));
                     await this.storeExtractionResult(filePath, content, language, stats, result, commitYield);
                     const idx = errors.indexOf(errEntry);
@@ -1955,13 +2015,20 @@ class ExtractionOrchestrator {
                     catch {
                         continue;
                     }
+                    // Same undecoded-transport hazard as the first retry pass (#1541).
+                    const language = (0, grammars_1.detectLanguage)(filePath, fullContent, overrides);
+                    result = (0, kernel_1.materializeKernelResult)(result, filePath, language);
                     if (result.nodes.length > 0 || result.errors.length === 0) {
-                        const language = (0, grammars_1.detectLanguage)(filePath, fullContent, overrides);
                         const stats = await fsp.stat(path.join(this.rootDir, filePath));
                         await this.storeExtractionResult(filePath, fullContent, language, stats, result, commitYield);
-                        const idx = errors.indexOf(errEntry);
-                        if (idx >= 0)
-                            errors.splice(idx, 1);
+                        // Salvaged from comment-stripped source: keep a visible trace in
+                        // the summary instead of erasing the failure outright — the
+                        // stored result may be missing whatever the failing parse choked
+                        // on, and a silently "clean" file here is how an index quietly
+                        // disagrees with a later per-file sync of the same bytes (#1565).
+                        errEntry.severity = 'warning';
+                        errEntry.code = 'salvaged_stripped';
+                        errEntry.message = `Indexed from comment-stripped source after repeated parse failures (symbols may be incomplete until the file is re-indexed): ${errEntry.message}`;
                         filesErrored--;
                         filesIndexed++;
                         totalNodes += result.nodes.length;
@@ -2088,9 +2155,10 @@ class ExtractionOrchestrator {
                 durationMs: 0,
             };
         }
+        const language = (0, grammars_1.detectLanguage)(relativePath, content, (0, project_config_1.loadExtensionOverrides)(this.rootDir));
         // Check file size
         if (stats.size > MAX_FILE_SIZE) {
-            return {
+            const result = {
                 nodes: [],
                 edges: [],
                 unresolvedReferences: [],
@@ -2104,9 +2172,10 @@ class ExtractionOrchestrator {
                 ],
                 durationMs: 0,
             };
+            await this.storeExtractionResult(relativePath, content, language, stats, result, (0, cooperative_yield_1.createYielder)());
+            return result;
         }
         // Detect language (honoring the project's codegraph.json extension overrides)
-        const language = (0, grammars_1.detectLanguage)(relativePath, content, (0, project_config_1.loadExtensionOverrides)(this.rootDir));
         if (!(0, grammars_1.isLanguageSupported)(language)) {
             return {
                 nodes: [],
@@ -2122,15 +2191,43 @@ class ExtractionOrchestrator {
         const frameworkNames = this.ensureDetectedFrameworks();
         const result = (0, tree_sitter_1.extractFromSource)(relativePath, content, language, frameworkNames);
         // Store in database
-        if (result.nodes.length > 0 || result.errors.length === 0) {
-            await this.storeExtractionResult(relativePath, content, language, stats, result, (0, cooperative_yield_1.createYielder)());
-        }
+        await this.storeExtractionResult(relativePath, content, language, stats, result, (0, cooperative_yield_1.createYielder)());
         return result;
     }
     /**
      * Store extraction result in database
      */
+    /**
+     * Delete file rows recorded with ZERO nodes so their files re-index.
+     *
+     * No extraction path stores an empty, error-free result for a
+     * symbol-bearing language — even an empty file keeps its file node — so a
+     * zero-node row is a wiped one (#1541: an interrupted parse's retry stored
+     * an undecoded kernel transport). The wiped row's content hash matches the
+     * on-disk bytes, so every hash-based reconcile skips the file forever;
+     * deleting the row lets the normal add path repair it. File-level-only
+     * languages (yaml, twig, properties) are left alone. Deleting a zero-node
+     * row cascades nothing: it has no nodes, so no edges or refs either.
+     */
+    healZeroNodeRows() {
+        for (const f of this.queries.getAllFiles()) {
+            // A zero-node row WITH recorded errors is a deliberate skip marker
+            // (#1557: oversized / repeatedly-unparseable files are persisted with
+            // their reason so syncs stop retrying them) — leave those alone. The
+            // #1541 wipe rows are the error-FREE zero-node rows.
+            if (f.nodeCount === 0 &&
+                !(0, grammars_1.isFileLevelOnlyLanguage)(f.language) &&
+                (f.errors === undefined || f.errors.length === 0)) {
+                this.queries.deleteFile(f.path);
+            }
+        }
+    }
     async storeExtractionResult(filePath, content, language, stats, result, onYield) {
+        // A kernel result can arrive as an undecoded buffer transport (empty
+        // node/edge arrays, tables riding in kernelBuffers). Decode it before
+        // storing — persisting the transport as-is records the file as having no
+        // symbols at all (#1541). No-op for already-decoded results.
+        result = (0, kernel_1.materializeKernelResult)(result, filePath, language);
         // Bulk inserts run in bounded sub-transactions with a yield between, so a
         // giant generated file (tens of thousands of symbols) can't block the
         // event loop — and the #850 watchdog heartbeat — for the whole store.
@@ -2139,10 +2236,19 @@ class ExtractionOrchestrator {
         // is unchanged: a partially-stored file has no record and re-indexes.
         const STORE_CHUNK = 2000;
         const contentHash = hashContent(content);
-        // Check if file already exists and hasn't changed
+        // Check if file already exists and hasn't changed. A skip/failure MARKER
+        // row (zero nodes + recorded errors, #1557) never blocks a store carrying
+        // real content: markers are written BEFORE the retry pass under the same
+        // content hash, so treating them as "no changes" would silently discard a
+        // successful retry's symbols — a permanent empty file presented as
+        // recovered (the #1541 wipe, reintroduced through the marker path).
         const existingFile = this.queries.getFileByPath(filePath);
         if (existingFile && existingFile.contentHash === contentHash) {
-            return; // No changes
+            const existingIsMarker = existingFile.nodeCount === 0 && (existingFile.errors?.length ?? 0) > 0;
+            const incomingHasContent = result.nodes.length > 0;
+            if (!existingIsMarker || !incomingHasContent) {
+                return; // No changes
+            }
         }
         // Re-decided on every re-index of a changed file, so a banner added (or
         // removed) by an edit is reflected on the next sync (#1500). Computed after
@@ -2439,7 +2545,20 @@ class ExtractionOrchestrator {
             // reads `filesChecked === 0 && durationMs === 0` as the
             // lock-unavailable signature (#449).
             const unique = [...new Set(scopedPaths)];
-            currentFiles = unique.filter((p) => fs.existsSync(path.join(this.rootDir, p)));
+            // A scoped path is "present" only if it exists AND is in scope — the
+            // same two gates the full walk applies (source extension, scope
+            // matcher). Without the scope gate a caller's stale view of scope
+            // leaked straight into the index: the watcher re-parsed a file the
+            // user had just excluded in `codegraph.json` while `codegraph sync`
+            // removed it (#1590). Out-of-scope paths fall out of `currentFiles`,
+            // so a tracked one takes the removal branch below, exactly as a full
+            // sync would treat it. (`include`-forced paths pass: ScopeIgnore
+            // applies the include precedence itself.)
+            const scope = this.scopedSyncMatcher();
+            const overrides = (0, project_config_1.loadExtensionOverrides)(this.rootDir);
+            currentFiles = unique.filter((p) => (0, grammars_1.isSourceFile)(p, overrides) &&
+                !scope.ignores(p) &&
+                fs.existsSync(path.join(this.rootDir, p)));
             trackedFiles = [];
             for (const p of unique) {
                 const rec = this.queries.getFileByPath(p);
@@ -2455,6 +2574,9 @@ class ExtractionOrchestrator {
             if (process.env.CODEGRAPH_SYNTH_TIMINGS)
                 console.error(`[phase-timing] sync-scan: ${Date.now() - tSyncScan}ms (${currentFiles.length} files)`);
             filesChecked = currentFiles.length;
+            // Full reconcile only (scoped syncs must not touch rows outside their
+            // scope): drop zero-node rows so the wiped files re-index as adds below.
+            this.healZeroNodeRows();
             const tTracked = Date.now();
             trackedFiles = this.queries.getAllFiles();
             if (process.env.CODEGRAPH_SYNTH_TIMINGS)