@stupidloud/codegraph 0.8.1 → 0.9.9

package/dist/extraction/index.js

@@ -41,9 +41,9 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.loadAllGrammars = exports.loadGrammarsForLanguages = exports.initGrammars = exports.getSupportedLanguages = exports.isGrammarLoaded = exports.isLanguageSupported = exports.detectLanguage = exports.extractFromSource = exports.ExtractionOrchestrator = void 0;
+exports.loadAllGrammars = exports.loadGrammarsForLanguages = exports.initGrammars = exports.getSupportedLanguages = exports.isGrammarLoaded = exports.isLanguageSupported = exports.isSourceFile = exports.detectLanguage = exports.extractFromSource = exports.ExtractionOrchestrator = void 0;
 exports.hashContent = hashContent;
-exports.shouldIncludeFile = shouldIncludeFile;
+exports.buildDefaultIgnore = buildDefaultIgnore;
 exports.scanDirectory = scanDirectory;
 exports.scanDirectoryAsync = scanDirectoryAsync;
 const fs = __importStar(require("fs"));
@@ -55,7 +55,7 @@ const tree_sitter_1 = require("./tree-sitter");
 const grammars_1 = require("./grammars");
 const errors_1 = require("../errors");
 const utils_1 = require("../utils");
-const picomatch_1 = __importDefault(require("picomatch"));
+const ignore_1 = __importDefault(require("ignore"));
 const frameworks_1 = require("../resolution/frameworks");
 /**
  * Number of files to read in parallel during indexing.
@@ -84,29 +84,87 @@ function hashContent(content) {
     return crypto.createHash('sha256').update(content).digest('hex');
 }
 /**
- * Check if a path matches any glob pattern (simplified)
+ * Skip files larger than this (bytes). Generated bundles, minified JS, and
+ * vendored blobs blow the WASM heap and the worker-recycle budget for no useful
+ * symbols. 1 MB covers essentially all hand-written source.
  */
-function matchesGlob(filePath, pattern) {
-    filePath = (0, utils_1.normalizePath)(filePath);
-    return picomatch_1.default.isMatch(filePath, pattern, { dot: true });
-}
+const MAX_FILE_SIZE = 1024 * 1024;
 /**
- * Check if a file should be included based on config
+ * Directory names that are dependency, build, cache, or tooling output across the
+ * languages/frameworks CodeGraph supports — curated from the canonical
+ * github/gitignore templates. Excluded by default so the graph reflects your code,
+ * not third-party noise, without requiring a `.gitignore` (issue #407). The
+ * exclusion applies uniformly (git or not, tracked or not); the only opt-in is an
+ * explicit `.gitignore` negation (e.g. `!vendor/`). First-party-prone or generic
+ * names (`packages`, `lib`, `app`, `bin`, `src`, `deps`, `env`, `tmp`, `storage`,
+ * `Library`) are deliberately NOT listed, to avoid ever hiding real source.
+ *
+ * Only dirs that actually contain *indexable source* (or are enormous) earn a slot
+ * — IDE/state dirs like `.idea`/`.vs` are omitted because CodeGraph indexes only
+ * recognized source extensions, so they produce no symbols regardless.
  */
-function shouldIncludeFile(filePath, config) {
-    // Check exclude patterns first
-    for (const pattern of config.exclude) {
-        if (matchesGlob(filePath, pattern)) {
-            return false;
-        }
+const DEFAULT_IGNORE_DIRS = new Set([
+    // JS / TS — dependency directories
+    'node_modules', 'bower_components', 'jspm_packages', 'web_modules',
+    '.yarn', '.pnpm-store',
+    // JS / TS — framework & bundler build / cache / deploy output
+    '.next', '.nuxt', '.svelte-kit', '.turbo', '.vite', '.parcel-cache', '.angular',
+    '.docusaurus', 'storybook-static', '.vinxi', '.nitro', 'out-tsc',
+    '.vercel', '.netlify', '.wrangler',
+    // Build output (common across ecosystems)
+    'dist', 'build', 'out', '.output',
+    // Test / coverage
+    'coverage', '.nyc_output',
+    // Python
+    '__pycache__', '__pypackages__', '.venv', 'venv', '.pixi', '.pdm-build',
+    '.mypy_cache', '.pytest_cache', '.ruff_cache', '.tox', '.nox', '.hypothesis',
+    '.ipynb_checkpoints', '.eggs',
+    // Rust / JVM (Maven, Gradle, Scala)
+    'target', '.gradle',
+    // .NET
+    'obj',
+    // Vendored deps (Go, PHP/Composer, Ruby/Bundler)
+    'vendor',
+    // Swift / iOS
+    '.build', 'Pods', 'Carthage', 'DerivedData', '.swiftpm',
+    // Dart / Flutter
+    '.dart_tool', '.pub-cache',
+    // Native (Android NDK, C/C++ deps)
+    '.cxx', '.externalNativeBuild', 'vcpkg_installed',
+    // Scala tooling
+    '.bloop', '.metals',
+    // Lua / Luau (LuaRocks)
+    'lua_modules', '.luarocks',
+    // Delphi / RAD Studio IDE backups (duplicate .pas source — would double-count)
+    '__history', '__recovery',
+    // Generic cache
+    '.cache',
+]);
+/** Gitignore-style patterns for the `ignore` matcher: the dirs above plus a few globs. */
+const DEFAULT_IGNORE_PATTERNS = [
+    ...Array.from(DEFAULT_IGNORE_DIRS, (d) => `${d}/`),
+    '*.egg-info/', // Python packaging metadata
+    'cmake-build-*/', // CLion / CMake build trees
+    'bazel-*/', // Bazel output symlink trees
+];
+/**
+ * An `ignore` matcher seeded with the built-in defaults, merged with the project's
+ * root .gitignore so a negation there (e.g. `!vendor/`) overrides a default. Shared
+ * by both enumeration paths so behavior is identical with or without git — and so
+ * the defaults apply to tracked files too (committing a dependency dir doesn't make
+ * it project code; the explicit `.gitignore` negation is the only opt-in).
+ */
+function buildDefaultIgnore(rootDir) {
+    const ig = (0, ignore_1.default)().add(DEFAULT_IGNORE_PATTERNS);
+    try {
+        const rootGitignore = path.join(rootDir, '.gitignore');
+        if (fs.existsSync(rootGitignore))
+            ig.add(fs.readFileSync(rootGitignore, 'utf-8'));
     }
-    // Check include patterns
-    for (const pattern of config.include) {
-        if (matchesGlob(filePath, pattern)) {
-            return true;
-        }
+    catch {
+        // Unreadable root .gitignore — the built-in defaults still apply.
     }
-    return false;
+    return ig;
 }
 /**
  * Collect git-visible files (tracked + untracked, .gitignore-respected) from the
@@ -122,38 +180,39 @@ function shouldIncludeFile(filePath, config) {
  * (See issue #193.)
  */
 function collectGitFiles(repoDir, prefix, files) {
-    const gitOpts = { cwd: repoDir, encoding: 'utf-8', timeout: 30000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'] };
+    const gitOpts = { cwd: repoDir, encoding: 'utf-8', timeout: 30000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true };
     // Tracked files. --recurse-submodules pulls in files from active submodules,
     // which the index would otherwise represent only as a commit pointer.
     // Without this, monorepos using submodules index 0 files. (See issue #147.)
     // Note: --recurse-submodules only supports -c/--cached and --stage modes — it
     // can't be combined with -o, so untracked files are gathered separately below.
-    const tracked = (0, child_process_1.execFileSync)('git', ['ls-files', '-c', '--recurse-submodules'], gitOpts);
-    for (const line of tracked.split('\n')) {
-        const trimmed = line.trim();
-        if (trimmed) {
-            files.add((0, utils_1.normalizePath)(prefix + trimmed));
-        }
+    // -z gives NUL-separated, unquoted output so non-ASCII (e.g. CJK) paths
+    // survive verbatim. Without it git octal-escapes and double-quotes such paths
+    // (the core.quotepath default), and the quoted form never matches a real file
+    // on disk → those files are silently dropped from the index. (#541)
+    const tracked = (0, child_process_1.execFileSync)('git', ['ls-files', '-z', '-c', '--recurse-submodules'], gitOpts);
+    for (const rel of tracked.split('\0')) {
+        if (rel)
+            files.add((0, utils_1.normalizePath)(prefix + rel));
     }
     // Untracked files (submodules manage their own untracked state). Embedded git
     // repos surface here as a single "subdir/" entry that git refuses to descend
     // into — recurse into those as their own repos so their source gets indexed.
-    const untracked = (0, child_process_1.execFileSync)('git', ['ls-files', '-o', '--exclude-standard'], gitOpts);
-    for (const line of untracked.split('\n')) {
-        const trimmed = line.trim();
-        if (!trimmed)
+    const untracked = (0, child_process_1.execFileSync)('git', ['ls-files', '-z', '-o', '--exclude-standard'], gitOpts);
+    for (const rel of untracked.split('\0')) {
+        if (!rel)
             continue;
-        if (trimmed.endsWith('/')) {
+        if (rel.endsWith('/')) {
             // git only emits a trailing-slash directory entry for an embedded repo.
             // Guard with a .git check anyway, and skip anything else exactly as git
             // itself skips it (we never descend into a non-repo opaque dir).
-            const childDir = path.join(repoDir, trimmed);
+            const childDir = path.join(repoDir, rel);
             if (fs.existsSync(path.join(childDir, '.git'))) {
-                collectGitFiles(childDir, prefix + trimmed, files);
+                collectGitFiles(childDir, prefix + rel, files);
             }
             continue;
         }
-        files.add((0, utils_1.normalizePath)(prefix + trimmed));
+        files.add((0, utils_1.normalizePath)(prefix + rel));
     }
 }
 /**
@@ -167,11 +226,11 @@ function getGitVisibleFiles(rootDir) {
         // Check if the project directory is gitignored by a parent repo.
         // When rootDir lives inside a parent git repo that ignores it,
         // `git ls-files` returns nothing — fall back to filesystem walk.
-        const gitRoot = (0, child_process_1.execFileSync)('git', ['rev-parse', '--show-toplevel'], { cwd: rootDir, encoding: 'utf-8', timeout: 5000, stdio: ['pipe', 'pipe', 'pipe'] }).trim();
+        const gitRoot = (0, child_process_1.execFileSync)('git', ['rev-parse', '--show-toplevel'], { cwd: rootDir, encoding: 'utf-8', timeout: 5000, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true }).trim();
         if (path.resolve(gitRoot) !== path.resolve(rootDir)) {
             try {
                 // git check-ignore exits 0 if the path IS ignored, 1 if not
-                (0, child_process_1.execFileSync)('git', ['check-ignore', '-q', path.resolve(rootDir)], { cwd: rootDir, encoding: 'utf-8', timeout: 5000, stdio: ['pipe', 'pipe', 'pipe'] });
+                (0, child_process_1.execFileSync)('git', ['check-ignore', '-q', path.resolve(rootDir)], { cwd: rootDir, encoding: 'utf-8', timeout: 5000, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true });
                 // Directory is gitignored by parent repo — fall back to filesystem walk
                 return null;
             }
@@ -181,7 +240,11 @@ function getGitVisibleFiles(rootDir) {
         }
         const files = new Set();
         collectGitFiles(rootDir, '', files);
-        return files;
+        // Apply built-in default ignores uniformly — to tracked files too, since
+        // committing a dependency/build dir doesn't make it project code. A
+        // `.gitignore` negation (e.g. `!vendor/`) is the explicit opt-in. (issue #407)
+        const ig = buildDefaultIgnore(rootDir);
+        return new Set([...files].filter((f) => !ig.ignores(f)));
     }
     catch {
         return null;
@@ -191,9 +254,9 @@ function getGitVisibleFiles(rootDir) {
  * Use `git status` to detect changed files instead of scanning every file.
  * Returns null on failure so callers fall back to full scan.
  */
-function getGitChangedFiles(rootDir, config) {
+function getGitChangedFiles(rootDir) {
     try {
-        const output = (0, child_process_1.execFileSync)('git', ['status', '--porcelain', '--no-renames'], { cwd: rootDir, encoding: 'utf-8', timeout: 10000, stdio: ['pipe', 'pipe', 'pipe'] });
+        const output = (0, child_process_1.execFileSync)('git', ['status', '--porcelain', '--no-renames'], { cwd: rootDir, encoding: 'utf-8', timeout: 10000, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true });
         const modified = [];
         const added = [];
         const deleted = [];
@@ -202,8 +265,8 @@ function getGitChangedFiles(rootDir, config) {
                 continue; // Minimum: "XY file"
             const statusCode = line.substring(0, 2);
             const filePath = (0, utils_1.normalizePath)(line.substring(3));
-            // Skip files that don't match include/exclude config
-            if (!shouldIncludeFile(filePath, config))
+            // Skip non-source files (git status already omits .gitignored paths).
+            if (!(0, grammars_1.isSourceFile)(filePath))
                 continue;
             if (statusCode === '??') {
                 added.push(filePath);
@@ -223,24 +286,20 @@ function getGitChangedFiles(rootDir, config) {
     }
 }
 /**
- * Marker file name that indicates a directory (and all children) should be skipped
- */
-const CODEGRAPH_IGNORE_MARKER = '.codegraphignore';
-/**
- * Recursively scan directory for source files.
+ * Recursively scan a directory for source files.
  *
- * In git repos, uses `git ls-files` to get the file list (inherently
- * respects .gitignore at all levels), then filters by config include patterns.
- * Falls back to filesystem walk for non-git projects.
+ * In git repos, uses `git ls-files` (inherently respects .gitignore at all
+ * levels), then keeps files with a supported source extension. For non-git
+ * projects, falls back to a filesystem walk that parses .gitignore itself.
  */
-function scanDirectory(rootDir, config, onProgress) {
+function scanDirectory(rootDir, onProgress) {
     // Fast path: use git to get all visible files (respects .gitignore everywhere)
     const gitFiles = getGitVisibleFiles(rootDir);
     if (gitFiles) {
         const files = [];
         let count = 0;
         for (const filePath of gitFiles) {
-            if (shouldIncludeFile(filePath, config)) {
+            if ((0, grammars_1.isSourceFile)(filePath)) {
                 files.push(filePath);
                 count++;
                 onProgress?.(count, filePath);
@@ -249,19 +308,19 @@ function scanDirectory(rootDir, config, onProgress) {
         return files;
     }
     // Fallback: walk filesystem for non-git projects
-    return scanDirectoryWalk(rootDir, config, onProgress);
+    return scanDirectoryWalk(rootDir, onProgress);
 }
 /**
  * Async variant of scanDirectory that yields to the event loop periodically,
  * allowing worker threads to receive and render progress messages.
  */
-async function scanDirectoryAsync(rootDir, config, onProgress) {
+async function scanDirectoryAsync(rootDir, onProgress) {
     const gitFiles = getGitVisibleFiles(rootDir);
     if (gitFiles) {
         const files = [];
         let count = 0;
         for (const filePath of gitFiles) {
-            if (shouldIncludeFile(filePath, config)) {
+            if ((0, grammars_1.isSourceFile)(filePath)) {
                 files.push(filePath);
                 count++;
                 onProgress?.(count, filePath);
@@ -273,16 +332,40 @@ async function scanDirectoryAsync(rootDir, config, onProgress) {
         }
         return files;
     }
-    return scanDirectoryWalk(rootDir, config, onProgress);
+    return scanDirectoryWalk(rootDir, onProgress);
 }
 /**
  * Filesystem walk fallback for non-git projects.
  */
-function scanDirectoryWalk(rootDir, config, onProgress) {
+function scanDirectoryWalk(rootDir, onProgress) {
     const files = [];
     let count = 0;
     const visitedDirs = new Set();
-    function walk(dir) {
+    const loadIgnore = (dir) => {
+        try {
+            const giPath = path.join(dir, '.gitignore');
+            if (fs.existsSync(giPath)) {
+                return { dir, ig: (0, ignore_1.default)().add(fs.readFileSync(giPath, 'utf-8')) };
+            }
+        }
+        catch {
+            // Unreadable .gitignore — treat as absent.
+        }
+        return null;
+    };
+    const isIgnored = (fullPath, isDir, matchers) => {
+        for (const { dir, ig } of matchers) {
+            let rel = (0, utils_1.normalizePath)(path.relative(dir, fullPath));
+            if (!rel || rel.startsWith('..'))
+                continue; // not under this matcher's dir
+            if (isDir)
+                rel += '/'; // dir-only rules (e.g. `build/`) only match with the slash
+            if (ig.ignores(rel))
+                return true;
+        }
+        return false;
+    };
+    function walk(dir, matchers) {
         let realDir;
         try {
             realDir = fs.realpathSync(dir);
@@ -296,12 +379,11 @@ function scanDirectoryWalk(rootDir, config, onProgress) {
             return;
         }
         visitedDirs.add(realDir);
-        // Check for .codegraphignore marker file
-        const ignoreMarker = path.join(dir, CODEGRAPH_IGNORE_MARKER);
-        if (fs.existsSync(ignoreMarker)) {
-            (0, errors_1.logDebug)('Skipping directory due to .codegraphignore marker', { dir });
-            return;
-        }
+        // This directory's own .gitignore (if present) applies to everything below it.
+        // The root's .gitignore is already merged into the seeded base matcher (so a
+        // negation there can override a built-in default), so skip it here.
+        const own = dir === rootDir ? null : loadIgnore(dir);
+        const active = own ? [...matchers, own] : matchers;
         let entries;
         try {
             entries = fs.readdirSync(dir, { withFileTypes: true });
@@ -311,6 +393,9 @@ function scanDirectoryWalk(rootDir, config, onProgress) {
             return;
         }
         for (const entry of entries) {
+            // Never descend into git internals or our own data directory.
+            if (entry.name === '.git' || entry.name === '.codegraph')
+                continue;
             const fullPath = path.join(dir, entry.name);
             const relativePath = (0, utils_1.normalizePath)(path.relative(rootDir, fullPath));
             if (entry.isSymbolicLink()) {
@@ -318,20 +403,12 @@ function scanDirectoryWalk(rootDir, config, onProgress) {
                     const realTarget = fs.realpathSync(fullPath);
                     const stat = fs.statSync(realTarget);
                     if (stat.isDirectory()) {
-                        const dirPattern = relativePath + '/';
-                        let excluded = false;
-                        for (const pattern of config.exclude) {
-                            if (matchesGlob(dirPattern, pattern) || matchesGlob(relativePath, pattern)) {
-                                excluded = true;
-                                break;
-                            }
-                        }
-                        if (!excluded) {
-                            walk(fullPath);
+                        if (!isIgnored(fullPath, true, active)) {
+                            walk(fullPath, active);
                         }
                     }
                     else if (stat.isFile()) {
-                        if (shouldIncludeFile(relativePath, config)) {
+                        if (!isIgnored(fullPath, false, active) && (0, grammars_1.isSourceFile)(relativePath)) {
                             files.push(relativePath);
                             count++;
                             onProgress?.(count, relativePath);
@@ -344,20 +421,12 @@ function scanDirectoryWalk(rootDir, config, onProgress) {
                 continue;
             }
             if (entry.isDirectory()) {
-                const dirPattern = relativePath + '/';
-                let excluded = false;
-                for (const pattern of config.exclude) {
-                    if (matchesGlob(dirPattern, pattern) || matchesGlob(relativePath, pattern)) {
-                        excluded = true;
-                        break;
-                    }
-                }
-                if (!excluded) {
-                    walk(fullPath);
+                if (!isIgnored(fullPath, true, active)) {
+                    walk(fullPath, active);
                 }
             }
             else if (entry.isFile()) {
-                if (shouldIncludeFile(relativePath, config)) {
+                if (!isIgnored(fullPath, false, active) && (0, grammars_1.isSourceFile)(relativePath)) {
                     files.push(relativePath);
                     count++;
                     onProgress?.(count, relativePath);
@@ -365,7 +434,9 @@ function scanDirectoryWalk(rootDir, config, onProgress) {
             }
         }
     }
-    walk(rootDir);
+    // Seed a base matcher with the built-in default ignores (merged with the root
+    // .gitignore so a negation can override). Nested .gitignores still layer per-dir.
+    walk(rootDir, [{ dir: rootDir, ig: buildDefaultIgnore(rootDir) }]);
     return files;
 }
 /**
@@ -373,7 +444,6 @@ function scanDirectoryWalk(rootDir, config, onProgress) {
  */
 class ExtractionOrchestrator {
     rootDir;
-    config;
     queries;
     /**
      * Names of frameworks detected for this project, populated by indexAll().
@@ -382,9 +452,8 @@ class ExtractionOrchestrator {
      * hasn't run yet so single-file re-index paths can detect on the spot.
      */
     detectedFrameworkNames = null;
-    constructor(rootDir, config, queries) {
+    constructor(rootDir, queries) {
         this.rootDir = rootDir;
-        this.config = config;
         this.queries = queries;
     }
     /**
@@ -426,6 +495,24 @@ class ExtractionOrchestrator {
                     return null;
                 }
             },
+            // Monorepo support — needed by framework detect()s that probe
+            // subpackage manifests (e.g. fabric-view looking at
+            // packages/<sub>/package.json when the root manifest is just a
+            // workspace declaration). Matches the resolver-context shape.
+            listDirectories: (relativePath) => {
+                const target = relativePath === '.' || relativePath === ''
+                    ? rootDir
+                    : path.join(rootDir, relativePath);
+                try {
+                    return fs
+                        .readdirSync(target, { withFileTypes: true })
+                        .filter((entry) => entry.isDirectory())
+                        .map((entry) => entry.name);
+                }
+                catch {
+                    return [];
+                }
+            },
         };
     }
     /**
@@ -436,7 +523,7 @@ class ExtractionOrchestrator {
     ensureDetectedFrameworks(files) {
         if (this.detectedFrameworkNames !== null)
             return this.detectedFrameworkNames;
-        const fileList = files ?? scanDirectory(this.rootDir, this.config);
+        const fileList = files ?? scanDirectory(this.rootDir);
         const context = this.buildDetectionContext(fileList);
         this.detectedFrameworkNames = (0, frameworks_1.detectFrameworks)(context).map((r) => r.name);
         return this.detectedFrameworkNames;
@@ -462,7 +549,7 @@ class ExtractionOrchestrator {
             current: 0,
             total: 0,
         });
-        const files = await scanDirectoryAsync(this.rootDir, this.config, (current, file) => {
+        const files = await scanDirectoryAsync(this.rootDir, (current, file) => {
             onProgress?.({
                 phase: 'scanning',
                 current,
@@ -697,18 +784,16 @@ class ExtractionOrchestrator {
                     });
                     continue;
                 }
-                // Honour config.maxFileSize. Without this check, vendored
-                // generated headers, minified bundles, and other multi-MB
-                // files get indexed despite the user setting a size cap —
-                // wasting WASM heap and the worker recycle budget on inputs
-                // the user explicitly opted out of. The single-file extractFile
-                // path already enforces this; the bulk path used to silently
-                // skip the check.
-                if (stats.size > this.config.maxFileSize) {
+                // Honour MAX_FILE_SIZE. Without this check, vendored generated
+                // headers, minified bundles, and other multi-MB files get indexed,
+                // wasting WASM heap and the worker recycle budget on inputs with no
+                // useful symbols. The single-file extractFile path already enforces
+                // this; the bulk path used to silently skip the check.
+                if (stats.size > MAX_FILE_SIZE) {
                     processed++;
                     filesSkipped++;
                     errors.push({
-                        message: `File exceeds max size (${stats.size} > ${this.config.maxFileSize})`,
+                        message: `File exceeds max size (${stats.size} > ${MAX_FILE_SIZE})`,
                         filePath,
                         severity: 'warning',
                         code: 'size_exceeded',
@@ -755,7 +840,16 @@ class ExtractionOrchestrator {
                     filesErrored++;
                 }
                 else {
-                    filesSkipped++;
+                    // Files with no symbols but no errors (yaml, twig, properties) are
+                    // tracked at the file level — count them as indexed so the CLI
+                    // doesn't misleadingly report "No files found to index".
+                    const lang = (0, grammars_1.detectLanguage)(filePath, content);
+                    if ((0, grammars_1.isFileLevelOnlyLanguage)(lang)) {
+                        filesIndexed++;
+                    }
+                    else {
+                        filesSkipped++;
+                    }
                 }
             }
         }
@@ -906,7 +1000,13 @@ class ExtractionOrchestrator {
                 filesErrored++;
             }
             else {
-                filesSkipped++;
+                const tracked = this.queries.getFileByPath(filePath);
+                if (tracked && (0, grammars_1.isFileLevelOnlyLanguage)(tracked.language)) {
+                    filesIndexed++;
+                }
+                else {
+                    filesSkipped++;
+                }
             }
         }
         return {
@@ -977,14 +1077,14 @@ class ExtractionOrchestrator {
             };
         }
         // Check file size
-        if (stats.size > this.config.maxFileSize) {
+        if (stats.size > MAX_FILE_SIZE) {
             return {
                 nodes: [],
                 edges: [],
                 unresolvedReferences: [],
                 errors: [
                     {
-                        message: `File exceeds max size (${stats.size} > ${this.config.maxFileSize})`,
+                        message: `File exceeds max size (${stats.size} > ${MAX_FILE_SIZE})`,
                         filePath: relativePath,
                         severity: 'warning',
                         code: 'size_exceeded',
@@ -1073,8 +1173,12 @@ class ExtractionOrchestrator {
         this.queries.upsertFile(fileRecord);
     }
     /**
-     * Sync with current file state.
-     * Uses git status as a fast path when available, falling back to full scan.
+     * Sync the index with the current file state.
+     *
+     * Change detection is filesystem-based, never git: a (size, mtime) stat
+     * pre-filter skips unchanged files, then a content-hash compare confirms real
+     * changes. This works in non-git projects and catches committed changes from
+     * `git pull`/`checkout`/`merge`/`rebase` that `git status` cannot see.
      */
     async sync(onProgress) {
         await (0, grammars_1.initGrammars)(); // Initialize WASM runtime (grammars loaded lazily below)
@@ -1091,88 +1195,72 @@ class ExtractionOrchestrator {
             total: 0,
         });
         const filesToIndex = [];
-        const gitChanges = getGitChangedFiles(this.rootDir, this.config);
-        if (gitChanges) {
-            // === Git fast path ===
-            // Only inspect the files git reports as changed instead of scanning everything.
-            filesChecked = gitChanges.modified.length + gitChanges.added.length + gitChanges.deleted.length;
-            // Handle deleted files
-            for (const filePath of gitChanges.deleted) {
-                const tracked = this.queries.getFileByPath(filePath);
-                if (tracked) {
-                    this.queries.deleteFile(filePath);
-                    filesRemoved++;
-                }
+        // === Filesystem reconcile (git-independent) ===
+        // The source of truth for "what changed" is the filesystem vs the indexed
+        // state — never git. We enumerate the current source files and reconcile
+        // each against the DB. A cheap (size, mtime) stat pre-filter skips unchanged
+        // files without reading or hashing them, so the expensive read+hash+parse
+        // only runs for files that actually changed. This catches edits/adds/deletes
+        // whether or not the project uses git, and crucially also catches committed
+        // changes from `git pull`/`checkout`/`merge`/`rebase` — which `git status`
+        // cannot see, because the working tree is clean afterward.
+        const currentFiles = scanDirectory(this.rootDir);
+        filesChecked = currentFiles.length;
+        const currentSet = new Set(currentFiles);
+        const trackedFiles = this.queries.getAllFiles();
+        const trackedMap = new Map();
+        for (const f of trackedFiles) {
+            trackedMap.set(f.path, f);
+        }
+        // Removals: tracked in the DB but no longer a present source file. Check the
+        // filesystem directly — `scanDirectory` (via `git ls-files`) still lists a
+        // file deleted from disk but not yet staged, so set membership alone misses it.
+        for (const tracked of trackedFiles) {
+            if (!currentSet.has(tracked.path) || !fs.existsSync(path.join(this.rootDir, tracked.path))) {
+                this.queries.deleteFile(tracked.path);
+                filesRemoved++;
             }
-            // Handle modified + added files — read + hash only these. Untracked
-            // (`??`) files stay untracked in git even after we index them, so they
-            // can't be trusted as "new": re-hash and compare against the DB exactly
-            // like modified files. Otherwise every sync re-indexes them and status
-            // reports them as pending forever. (See issue #206.)
-            for (const filePath of [...gitChanges.modified, ...gitChanges.added]) {
-                const fullPath = path.join(this.rootDir, filePath);
-                let content;
+        }
+        // Adds / modifications.
+        for (const filePath of currentFiles) {
+            const fullPath = path.join(this.rootDir, filePath);
+            const tracked = trackedMap.get(filePath);
+            // Cheap pre-filter: an already-indexed file whose size AND mtime both match
+            // the DB is unchanged — skip it without reading or hashing. (A content
+            // change that preserves both exactly is the blind spot every mtime-based
+            // incremental tool accepts; `index --force` is the escape hatch. Git bumps
+            // mtime on every file it writes during checkout/merge, so pulls are caught.)
+            if (tracked) {
                 try {
-                    content = fs.readFileSync(fullPath, 'utf-8');
+                    const stat = fs.statSync(fullPath);
+                    if (stat.size === tracked.size && Math.floor(stat.mtimeMs) === Math.floor(tracked.modifiedAt)) {
+                        continue;
+                    }
                 }
                 catch (error) {
-                    (0, errors_1.logDebug)('Skipping unreadable file during sync', { filePath, error: String(error) });
+                    (0, errors_1.logDebug)('Skipping unstattable file during sync', { filePath, error: String(error) });
                     continue;
                 }
-                const contentHash = hashContent(content);
-                const tracked = this.queries.getFileByPath(filePath);
-                if (!tracked) {
-                    filesToIndex.push(filePath);
-                    changedFilePaths.push(filePath);
-                    filesAdded++;
-                }
-                else if (tracked.contentHash !== contentHash) {
-                    filesToIndex.push(filePath);
-                    changedFilePaths.push(filePath);
-                    filesModified++;
-                }
             }
-        }
-        else {
-            // === Fallback: full scan (non-git project or git failure) ===
-            const currentFiles = new Set(scanDirectory(this.rootDir, this.config));
-            filesChecked = currentFiles.size;
-            // Build Map for O(1) lookups instead of .find() per file
-            const trackedFiles = this.queries.getAllFiles();
-            const trackedMap = new Map();
-            for (const f of trackedFiles) {
-                trackedMap.set(f.path, f);
+            // New, or size/mtime changed — read + hash to confirm a real content change.
+            let content;
+            try {
+                content = fs.readFileSync(fullPath, 'utf-8');
             }
-            // Find files to remove (in DB but not on disk)
-            for (const tracked of trackedFiles) {
-                if (!currentFiles.has(tracked.path)) {
-                    this.queries.deleteFile(tracked.path);
-                    filesRemoved++;
-                }
+            catch (error) {
+                (0, errors_1.logDebug)('Skipping unreadable file during sync', { filePath, error: String(error) });
+                continue;
             }
-            // Find files to add or update
-            for (const filePath of currentFiles) {
-                const fullPath = path.join(this.rootDir, filePath);
-                let content;
-                try {
-                    content = fs.readFileSync(fullPath, 'utf-8');
-                }
-                catch (error) {
-                    (0, errors_1.logDebug)('Skipping unreadable file during sync', { filePath, error: String(error) });
-                    continue;
-                }
-                const contentHash = hashContent(content);
-                const tracked = trackedMap.get(filePath);
-                if (!tracked) {
-                    filesToIndex.push(filePath);
-                    changedFilePaths.push(filePath);
-                    filesAdded++;
-                }
-                else if (tracked.contentHash !== contentHash) {
-                    filesToIndex.push(filePath);
-                    changedFilePaths.push(filePath);
-                    filesModified++;
-                }
+            const contentHash = hashContent(content);
+            if (!tracked) {
+                filesToIndex.push(filePath);
+                changedFilePaths.push(filePath);
+                filesAdded++;
+            }
+            else if (tracked.contentHash !== contentHash) {
+                filesToIndex.push(filePath);
+                changedFilePaths.push(filePath);
+                filesModified++;
             }
         }
         // Load only grammars needed for changed files
@@ -1212,7 +1300,7 @@ class ExtractionOrchestrator {
      * Uses git status as a fast path when available, falling back to full scan.
      */
     getChangedFiles() {
-        const gitChanges = getGitChangedFiles(this.rootDir, this.config);
+        const gitChanges = getGitChangedFiles(this.rootDir);
         if (gitChanges) {
             // === Git fast path ===
             const added = [];
@@ -1251,7 +1339,7 @@ class ExtractionOrchestrator {
             return { added, modified, removed };
         }
         // === Fallback: full scan (non-git project or git failure) ===
-        const currentFiles = new Set(scanDirectory(this.rootDir, this.config));
+        const currentFiles = new Set(scanDirectory(this.rootDir));
         const trackedFiles = this.queries.getAllFiles();
         // Build Map for O(1) lookups
         const trackedMap = new Map();
@@ -1296,6 +1384,7 @@ var tree_sitter_2 = require("./tree-sitter");
 Object.defineProperty(exports, "extractFromSource", { enumerable: true, get: function () { return tree_sitter_2.extractFromSource; } });
 var grammars_2 = require("./grammars");
 Object.defineProperty(exports, "detectLanguage", { enumerable: true, get: function () { return grammars_2.detectLanguage; } });
+Object.defineProperty(exports, "isSourceFile", { enumerable: true, get: function () { return grammars_2.isSourceFile; } });
 Object.defineProperty(exports, "isLanguageSupported", { enumerable: true, get: function () { return grammars_2.isLanguageSupported; } });
 Object.defineProperty(exports, "isGrammarLoaded", { enumerable: true, get: function () { return grammars_2.isGrammarLoaded; } });
 Object.defineProperty(exports, "getSupportedLanguages", { enumerable: true, get: function () { return grammars_2.getSupportedLanguages; } });