@stupidloud/codegraph 0.7.20 → 0.9.5

package/dist/extraction/index.js

@@ -41,9 +41,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.loadAllGrammars = exports.loadGrammarsForLanguages = exports.initGrammars = exports.getSupportedLanguages = exports.isGrammarLoaded = exports.isLanguageSupported = exports.detectLanguage = exports.extractFromSource = exports.ExtractionOrchestrator = void 0;
+exports.loadAllGrammars = exports.loadGrammarsForLanguages = exports.initGrammars = exports.getSupportedLanguages = exports.isGrammarLoaded = exports.isLanguageSupported = exports.isSourceFile = exports.detectLanguage = exports.extractFromSource = exports.ExtractionOrchestrator = void 0;
 exports.hashContent = hashContent;
-exports.shouldIncludeFile = shouldIncludeFile;
 exports.scanDirectory = scanDirectory;
 exports.scanDirectoryAsync = scanDirectoryAsync;
 const fs = __importStar(require("fs"));
@@ -55,7 +54,7 @@ const tree_sitter_1 = require("./tree-sitter");
 const grammars_1 = require("./grammars");
 const errors_1 = require("../errors");
 const utils_1 = require("../utils");
-const picomatch_1 = __importDefault(require("picomatch"));
+const ignore_1 = __importDefault(require("ignore"));
 const frameworks_1 = require("../resolution/frameworks");
 /**
  * Number of files to read in parallel during indexing.
@@ -84,34 +83,64 @@ function hashContent(content) {
     return crypto.createHash('sha256').update(content).digest('hex');
 }
 /**
- * Check if a path matches any glob pattern (simplified)
+ * Skip files larger than this (bytes). Generated bundles, minified JS, and
+ * vendored blobs blow the WASM heap and the worker-recycle budget for no useful
+ * symbols. 1 MB covers essentially all hand-written source.
  */
-function matchesGlob(filePath, pattern) {
-    filePath = (0, utils_1.normalizePath)(filePath);
-    return picomatch_1.default.isMatch(filePath, pattern, { dot: true });
-}
+const MAX_FILE_SIZE = 1024 * 1024;
 /**
- * Check if a file should be included based on config
+ * Collect git-visible files (tracked + untracked, .gitignore-respected) from the
+ * git repository rooted at `repoDir`, adding each to `files` with `prefix`
+ * prepended so paths stay relative to the original scan root.
+ *
+ * Recurses into embedded git repositories — nested repos that are NOT submodules
+ * (independent clones living inside the workspace, common in CMake "super-repo"
+ * layouts). The parent repo's `git ls-files` cannot see into them: tracked output
+ * skips them entirely, and untracked output reports them only as an opaque
+ * "subdir/" entry (trailing slash) rather than expanding their files. Each
+ * embedded repo is its own git boundary, so we re-run `git ls-files` inside it.
+ * (See issue #193.)
  */
-function shouldIncludeFile(filePath, config) {
-    // Check exclude patterns first
-    for (const pattern of config.exclude) {
-        if (matchesGlob(filePath, pattern)) {
-            return false;
+function collectGitFiles(repoDir, prefix, files) {
+    const gitOpts = { cwd: repoDir, encoding: 'utf-8', timeout: 30000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'] };
+    // Tracked files. --recurse-submodules pulls in files from active submodules,
+    // which the index would otherwise represent only as a commit pointer.
+    // Without this, monorepos using submodules index 0 files. (See issue #147.)
+    // Note: --recurse-submodules only supports -c/--cached and --stage modes — it
+    // can't be combined with -o, so untracked files are gathered separately below.
+    const tracked = (0, child_process_1.execFileSync)('git', ['ls-files', '-c', '--recurse-submodules'], gitOpts);
+    for (const line of tracked.split('\n')) {
+        const trimmed = line.trim();
+        if (trimmed) {
+            files.add((0, utils_1.normalizePath)(prefix + trimmed));
         }
     }
-    // Check include patterns
-    for (const pattern of config.include) {
-        if (matchesGlob(filePath, pattern)) {
-            return true;
+    // Untracked files (submodules manage their own untracked state). Embedded git
+    // repos surface here as a single "subdir/" entry that git refuses to descend
+    // into — recurse into those as their own repos so their source gets indexed.
+    const untracked = (0, child_process_1.execFileSync)('git', ['ls-files', '-o', '--exclude-standard'], gitOpts);
+    for (const line of untracked.split('\n')) {
+        const trimmed = line.trim();
+        if (!trimmed)
+            continue;
+        if (trimmed.endsWith('/')) {
+            // git only emits a trailing-slash directory entry for an embedded repo.
+            // Guard with a .git check anyway, and skip anything else exactly as git
+            // itself skips it (we never descend into a non-repo opaque dir).
+            const childDir = path.join(repoDir, trimmed);
+            if (fs.existsSync(path.join(childDir, '.git'))) {
+                collectGitFiles(childDir, prefix + trimmed, files);
+            }
+            continue;
         }
+        files.add((0, utils_1.normalizePath)(prefix + trimmed));
     }
-    return false;
 }
 /**
  * Get all files visible to git (tracked + untracked but not ignored).
- * Respects .gitignore at all levels (root, subdirectories).
- * Returns null on failure (non-git project) so callers can fall back.
+ * Respects .gitignore at all levels (root, subdirectories) and descends into
+ * embedded (nested, non-submodule) git repos. Returns null on failure
+ * (non-git project) so callers can fall back to a filesystem walk.
  */
 function getGitVisibleFiles(rootDir) {
     try {
@@ -131,27 +160,7 @@ function getGitVisibleFiles(rootDir) {
             }
         }
         const files = new Set();
-        const gitOpts = { cwd: rootDir, encoding: 'utf-8', timeout: 30000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'] };
-        // Tracked files. --recurse-submodules pulls in files from active submodules,
-        // which the main repo's index would otherwise represent only as a commit pointer.
-        // Without this, monorepos using submodules index 0 files. (See issue #147.)
-        // Note: --recurse-submodules only supports -c/--cached and --stage modes — it
-        // can't be combined with -o, so untracked files are gathered separately below.
-        const tracked = (0, child_process_1.execFileSync)('git', ['ls-files', '-c', '--recurse-submodules'], gitOpts);
-        for (const line of tracked.split('\n')) {
-            const trimmed = line.trim();
-            if (trimmed) {
-                files.add((0, utils_1.normalizePath)(trimmed));
-            }
-        }
-        // Untracked files in the main repo (submodules manage their own untracked state).
-        const untracked = (0, child_process_1.execFileSync)('git', ['ls-files', '-o', '--exclude-standard'], gitOpts);
-        for (const line of untracked.split('\n')) {
-            const trimmed = line.trim();
-            if (trimmed) {
-                files.add((0, utils_1.normalizePath)(trimmed));
-            }
-        }
+        collectGitFiles(rootDir, '', files);
         return files;
     }
     catch {
@@ -162,7 +171,7 @@ function getGitVisibleFiles(rootDir) {
  * Use `git status` to detect changed files instead of scanning every file.
  * Returns null on failure so callers fall back to full scan.
  */
-function getGitChangedFiles(rootDir, config) {
+function getGitChangedFiles(rootDir) {
     try {
         const output = (0, child_process_1.execFileSync)('git', ['status', '--porcelain', '--no-renames'], { cwd: rootDir, encoding: 'utf-8', timeout: 10000, stdio: ['pipe', 'pipe', 'pipe'] });
         const modified = [];
@@ -173,8 +182,8 @@ function getGitChangedFiles(rootDir, config) {
                 continue; // Minimum: "XY file"
             const statusCode = line.substring(0, 2);
             const filePath = (0, utils_1.normalizePath)(line.substring(3));
-            // Skip files that don't match include/exclude config
-            if (!shouldIncludeFile(filePath, config))
+            // Skip non-source files (git status already omits .gitignored paths).
+            if (!(0, grammars_1.isSourceFile)(filePath))
                 continue;
             if (statusCode === '??') {
                 added.push(filePath);
@@ -194,24 +203,20 @@ function getGitChangedFiles(rootDir, config) {
     }
 }
 /**
- * Marker file name that indicates a directory (and all children) should be skipped
- */
-const CODEGRAPH_IGNORE_MARKER = '.codegraphignore';
-/**
- * Recursively scan directory for source files.
+ * Recursively scan a directory for source files.
  *
- * In git repos, uses `git ls-files` to get the file list (inherently
- * respects .gitignore at all levels), then filters by config include patterns.
- * Falls back to filesystem walk for non-git projects.
+ * In git repos, uses `git ls-files` (inherently respects .gitignore at all
+ * levels), then keeps files with a supported source extension. For non-git
+ * projects, falls back to a filesystem walk that parses .gitignore itself.
  */
-function scanDirectory(rootDir, config, onProgress) {
+function scanDirectory(rootDir, onProgress) {
     // Fast path: use git to get all visible files (respects .gitignore everywhere)
     const gitFiles = getGitVisibleFiles(rootDir);
     if (gitFiles) {
         const files = [];
         let count = 0;
         for (const filePath of gitFiles) {
-            if (shouldIncludeFile(filePath, config)) {
+            if ((0, grammars_1.isSourceFile)(filePath)) {
                 files.push(filePath);
                 count++;
                 onProgress?.(count, filePath);
@@ -220,19 +225,19 @@ function scanDirectory(rootDir, config, onProgress) {
         return files;
     }
     // Fallback: walk filesystem for non-git projects
-    return scanDirectoryWalk(rootDir, config, onProgress);
+    return scanDirectoryWalk(rootDir, onProgress);
 }
 /**
  * Async variant of scanDirectory that yields to the event loop periodically,
  * allowing worker threads to receive and render progress messages.
  */
-async function scanDirectoryAsync(rootDir, config, onProgress) {
+async function scanDirectoryAsync(rootDir, onProgress) {
     const gitFiles = getGitVisibleFiles(rootDir);
     if (gitFiles) {
         const files = [];
         let count = 0;
         for (const filePath of gitFiles) {
-            if (shouldIncludeFile(filePath, config)) {
+            if ((0, grammars_1.isSourceFile)(filePath)) {
                 files.push(filePath);
                 count++;
                 onProgress?.(count, filePath);
@@ -244,16 +249,40 @@ async function scanDirectoryAsync(rootDir, config, onProgress) {
         }
         return files;
     }
-    return scanDirectoryWalk(rootDir, config, onProgress);
+    return scanDirectoryWalk(rootDir, onProgress);
 }
 /**
  * Filesystem walk fallback for non-git projects.
  */
-function scanDirectoryWalk(rootDir, config, onProgress) {
+function scanDirectoryWalk(rootDir, onProgress) {
     const files = [];
     let count = 0;
     const visitedDirs = new Set();
-    function walk(dir) {
+    const loadIgnore = (dir) => {
+        try {
+            const giPath = path.join(dir, '.gitignore');
+            if (fs.existsSync(giPath)) {
+                return { dir, ig: (0, ignore_1.default)().add(fs.readFileSync(giPath, 'utf-8')) };
+            }
+        }
+        catch {
+            // Unreadable .gitignore — treat as absent.
+        }
+        return null;
+    };
+    const isIgnored = (fullPath, isDir, matchers) => {
+        for (const { dir, ig } of matchers) {
+            let rel = (0, utils_1.normalizePath)(path.relative(dir, fullPath));
+            if (!rel || rel.startsWith('..'))
+                continue; // not under this matcher's dir
+            if (isDir)
+                rel += '/'; // dir-only rules (e.g. `build/`) only match with the slash
+            if (ig.ignores(rel))
+                return true;
+        }
+        return false;
+    };
+    function walk(dir, matchers) {
         let realDir;
         try {
             realDir = fs.realpathSync(dir);
@@ -267,12 +296,9 @@ function scanDirectoryWalk(rootDir, config, onProgress) {
             return;
         }
         visitedDirs.add(realDir);
-        // Check for .codegraphignore marker file
-        const ignoreMarker = path.join(dir, CODEGRAPH_IGNORE_MARKER);
-        if (fs.existsSync(ignoreMarker)) {
-            (0, errors_1.logDebug)('Skipping directory due to .codegraphignore marker', { dir });
-            return;
-        }
+        // This directory's own .gitignore (if present) applies to everything below it.
+        const own = loadIgnore(dir);
+        const active = own ? [...matchers, own] : matchers;
         let entries;
         try {
             entries = fs.readdirSync(dir, { withFileTypes: true });
@@ -282,6 +308,9 @@ function scanDirectoryWalk(rootDir, config, onProgress) {
             return;
         }
         for (const entry of entries) {
+            // Never descend into git internals or our own data directory.
+            if (entry.name === '.git' || entry.name === '.codegraph')
+                continue;
             const fullPath = path.join(dir, entry.name);
             const relativePath = (0, utils_1.normalizePath)(path.relative(rootDir, fullPath));
             if (entry.isSymbolicLink()) {
@@ -289,20 +318,12 @@ function scanDirectoryWalk(rootDir, config, onProgress) {
                     const realTarget = fs.realpathSync(fullPath);
                     const stat = fs.statSync(realTarget);
                     if (stat.isDirectory()) {
-                        const dirPattern = relativePath + '/';
-                        let excluded = false;
-                        for (const pattern of config.exclude) {
-                            if (matchesGlob(dirPattern, pattern) || matchesGlob(relativePath, pattern)) {
-                                excluded = true;
-                                break;
-                            }
-                        }
-                        if (!excluded) {
-                            walk(fullPath);
+                        if (!isIgnored(fullPath, true, active)) {
+                            walk(fullPath, active);
                         }
                     }
                     else if (stat.isFile()) {
-                        if (shouldIncludeFile(relativePath, config)) {
+                        if (!isIgnored(fullPath, false, active) && (0, grammars_1.isSourceFile)(relativePath)) {
                             files.push(relativePath);
                             count++;
                             onProgress?.(count, relativePath);
@@ -315,20 +336,12 @@ function scanDirectoryWalk(rootDir, config, onProgress) {
                 continue;
             }
             if (entry.isDirectory()) {
-                const dirPattern = relativePath + '/';
-                let excluded = false;
-                for (const pattern of config.exclude) {
-                    if (matchesGlob(dirPattern, pattern) || matchesGlob(relativePath, pattern)) {
-                        excluded = true;
-                        break;
-                    }
-                }
-                if (!excluded) {
-                    walk(fullPath);
+                if (!isIgnored(fullPath, true, active)) {
+                    walk(fullPath, active);
                 }
             }
             else if (entry.isFile()) {
-                if (shouldIncludeFile(relativePath, config)) {
+                if (!isIgnored(fullPath, false, active) && (0, grammars_1.isSourceFile)(relativePath)) {
                     files.push(relativePath);
                     count++;
                     onProgress?.(count, relativePath);
@@ -336,7 +349,7 @@ function scanDirectoryWalk(rootDir, config, onProgress) {
             }
         }
     }
-    walk(rootDir);
+    walk(rootDir, []);
     return files;
 }
 /**
@@ -344,7 +357,6 @@ function scanDirectoryWalk(rootDir, config, onProgress) {
  */
 class ExtractionOrchestrator {
     rootDir;
-    config;
     queries;
     /**
      * Names of frameworks detected for this project, populated by indexAll().
@@ -353,9 +365,8 @@ class ExtractionOrchestrator {
      * hasn't run yet so single-file re-index paths can detect on the spot.
      */
     detectedFrameworkNames = null;
-    constructor(rootDir, config, queries) {
+    constructor(rootDir, queries) {
         this.rootDir = rootDir;
-        this.config = config;
         this.queries = queries;
     }
     /**
@@ -407,7 +418,7 @@ class ExtractionOrchestrator {
     ensureDetectedFrameworks(files) {
         if (this.detectedFrameworkNames !== null)
             return this.detectedFrameworkNames;
-        const fileList = files ?? scanDirectory(this.rootDir, this.config);
+        const fileList = files ?? scanDirectory(this.rootDir);
         const context = this.buildDetectionContext(fileList);
         this.detectedFrameworkNames = (0, frameworks_1.detectFrameworks)(context).map((r) => r.name);
         return this.detectedFrameworkNames;
@@ -433,7 +444,7 @@ class ExtractionOrchestrator {
             current: 0,
             total: 0,
         });
-        const files = await scanDirectoryAsync(this.rootDir, this.config, (current, file) => {
+        const files = await scanDirectoryAsync(this.rootDir, (current, file) => {
             onProgress?.({
                 phase: 'scanning',
                 current,
@@ -668,18 +679,16 @@ class ExtractionOrchestrator {
                     });
                     continue;
                 }
-                // Honour config.maxFileSize. Without this check, vendored
-                // generated headers, minified bundles, and other multi-MB
-                // files get indexed despite the user setting a size cap —
-                // wasting WASM heap and the worker recycle budget on inputs
-                // the user explicitly opted out of. The single-file extractFile
-                // path already enforces this; the bulk path used to silently
-                // skip the check.
-                if (stats.size > this.config.maxFileSize) {
+                // Honour MAX_FILE_SIZE. Without this check, vendored generated
+                // headers, minified bundles, and other multi-MB files get indexed,
+                // wasting WASM heap and the worker recycle budget on inputs with no
+                // useful symbols. The single-file extractFile path already enforces
+                // this; the bulk path used to silently skip the check.
+                if (stats.size > MAX_FILE_SIZE) {
                     processed++;
                     filesSkipped++;
                     errors.push({
-                        message: `File exceeds max size (${stats.size} > ${this.config.maxFileSize})`,
+                        message: `File exceeds max size (${stats.size} > ${MAX_FILE_SIZE})`,
                         filePath,
                         severity: 'warning',
                         code: 'size_exceeded',
@@ -948,14 +957,14 @@ class ExtractionOrchestrator {
             };
         }
         // Check file size
-        if (stats.size > this.config.maxFileSize) {
+        if (stats.size > MAX_FILE_SIZE) {
             return {
                 nodes: [],
                 edges: [],
                 unresolvedReferences: [],
                 errors: [
                     {
-                        message: `File exceeds max size (${stats.size} > ${this.config.maxFileSize})`,
+                        message: `File exceeds max size (${stats.size} > ${MAX_FILE_SIZE})`,
                         filePath: relativePath,
                         severity: 'warning',
                         code: 'size_exceeded',
@@ -1062,7 +1071,7 @@ class ExtractionOrchestrator {
             total: 0,
         });
         const filesToIndex = [];
-        const gitChanges = getGitChangedFiles(this.rootDir, this.config);
+        const gitChanges = getGitChangedFiles(this.rootDir);
         if (gitChanges) {
             // === Git fast path ===
             // Only inspect the files git reports as changed instead of scanning everything.
@@ -1075,8 +1084,12 @@ class ExtractionOrchestrator {
                     filesRemoved++;
                 }
             }
-            // Handle modified files — read + hash only these files
-            for (const filePath of gitChanges.modified) {
+            // Handle modified + added files — read + hash only these. Untracked
+            // (`??`) files stay untracked in git even after we index them, so they
+            // can't be trusted as "new": re-hash and compare against the DB exactly
+            // like modified files. Otherwise every sync re-indexes them and status
+            // reports them as pending forever. (See issue #206.)
+            for (const filePath of [...gitChanges.modified, ...gitChanges.added]) {
                 const fullPath = path.join(this.rootDir, filePath);
                 let content;
                 try {
@@ -1099,16 +1112,10 @@ class ExtractionOrchestrator {
                     filesModified++;
                 }
             }
-            // Handle added (untracked) files
-            for (const filePath of gitChanges.added) {
-                filesToIndex.push(filePath);
-                changedFilePaths.push(filePath);
-                filesAdded++;
-            }
         }
         else {
             // === Fallback: full scan (non-git project or git failure) ===
-            const currentFiles = new Set(scanDirectory(this.rootDir, this.config));
+            const currentFiles = new Set(scanDirectory(this.rootDir));
             filesChecked = currentFiles.size;
             // Build Map for O(1) lookups instead of .find() per file
             const trackedFiles = this.queries.getAllFiles();
@@ -1185,7 +1192,7 @@ class ExtractionOrchestrator {
      * Uses git status as a fast path when available, falling back to full scan.
      */
     getChangedFiles() {
-        const gitChanges = getGitChangedFiles(this.rootDir, this.config);
+        const gitChanges = getGitChangedFiles(this.rootDir);
         if (gitChanges) {
             // === Git fast path ===
             const added = [];
@@ -1198,8 +1205,11 @@ class ExtractionOrchestrator {
                     removed.push(filePath);
                 }
             }
-            // Modified files — read + hash only these, compare with DB
-            for (const filePath of gitChanges.modified) {
+            // Modified + added files — read + hash, compare with DB. Untracked (`??`)
+            // files stay untracked in git even after indexing, so they must be
+            // hash-compared like modified files instead of always counting as added —
+            // otherwise status reports them as pending forever. (See issue #206.)
+            for (const filePath of [...gitChanges.modified, ...gitChanges.added]) {
                 const fullPath = path.join(this.rootDir, filePath);
                 let content;
                 try {
@@ -1218,14 +1228,10 @@ class ExtractionOrchestrator {
                     modified.push(filePath);
                 }
             }
-            // Added (untracked) files
-            for (const filePath of gitChanges.added) {
-                added.push(filePath);
-            }
             return { added, modified, removed };
         }
         // === Fallback: full scan (non-git project or git failure) ===
-        const currentFiles = new Set(scanDirectory(this.rootDir, this.config));
+        const currentFiles = new Set(scanDirectory(this.rootDir));
         const trackedFiles = this.queries.getAllFiles();
         // Build Map for O(1) lookups
         const trackedMap = new Map();
@@ -1270,6 +1276,7 @@ var tree_sitter_2 = require("./tree-sitter");
 Object.defineProperty(exports, "extractFromSource", { enumerable: true, get: function () { return tree_sitter_2.extractFromSource; } });
 var grammars_2 = require("./grammars");
 Object.defineProperty(exports, "detectLanguage", { enumerable: true, get: function () { return grammars_2.detectLanguage; } });
+Object.defineProperty(exports, "isSourceFile", { enumerable: true, get: function () { return grammars_2.isSourceFile; } });
 Object.defineProperty(exports, "isLanguageSupported", { enumerable: true, get: function () { return grammars_2.isLanguageSupported; } });
 Object.defineProperty(exports, "isGrammarLoaded", { enumerable: true, get: function () { return grammars_2.isGrammarLoaded; } });
 Object.defineProperty(exports, "getSupportedLanguages", { enumerable: true, get: function () { return grammars_2.getSupportedLanguages; } });