@quolu/lattice 0.70.0 → 0.71.0

package/sensor/dist/extraction/index.js

@@ -41,12 +41,16 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.loadAllGrammars = exports.loadGrammarsForLanguages = exports.initGrammars = exports.getSupportedLanguages = exports.isGrammarLoaded = exports.isLanguageSupported = exports.isSourceFile = exports.detectLanguage = exports.extractFromSource = exports.ExtractionOrchestrator = exports.ScopeIgnore = void 0;
+exports.loadAllGrammars = exports.loadGrammarsForLanguages = exports.initGrammars = exports.getSupportedLanguages = exports.isGrammarLoaded = exports.isLanguageSupported = exports.isSourceFile = exports.detectLanguage = exports.extractFromSource = exports.ExtractionOrchestrator = exports.INDEXED_AT_COMMIT_KEY = exports.ScopeIgnore = exports.oversizeStamp = void 0;
 exports.hashContent = hashContent;
 exports.buildDefaultIgnore = buildDefaultIgnore;
+exports.preloadLanguagesForFiles = preloadLanguagesForFiles;
 exports.buildScopeIgnore = buildScopeIgnore;
 exports.discoverEmbeddedRepoRoots = discoverEmbeddedRepoRoots;
 exports.findUnindexedIgnoredRepos = findUnindexedIgnoredRepos;
+exports.getGitChangedFiles = getGitChangedFiles;
+exports.getGitHeadSha = getGitHeadSha;
+exports.canTrustGitFastPath = canTrustGitFastPath;
 exports.scanDirectory = scanDirectory;
 exports.scanDirectoryAsync = scanDirectoryAsync;
 const fs = __importStar(require("fs"));
@@ -60,6 +64,7 @@ const tree_sitter_1 = require("./tree-sitter");
 const parse_pool_1 = require("./parse-pool");
 const store_writer_1 = require("./store-writer");
 const kernel_1 = require("./kernel");
+const generated_detection_1 = require("./generated-detection");
 const grammars_1 = require("./grammars");
 const project_config_1 = require("../project-config");
 const directory_1 = require("../directory");
@@ -68,6 +73,8 @@ const utils_1 = require("../utils");
 const ignore_1 = __importDefault(require("ignore"));
 const frameworks_1 = require("../resolution/frameworks");
 const cooperative_yield_1 = require("../resolution/cooperative-yield");
+const file_limits_1 = require("../file-limits");
+Object.defineProperty(exports, "oversizeStamp", { enumerable: true, get: function () { return file_limits_1.oversizeStamp; } });
 /**
  * Number of files to read in parallel during indexing.
  * File reads are I/O-bound; batching overlaps I/O wait with CPU parse work.
@@ -107,11 +114,26 @@ function hashContent(content) {
     return crypto.createHash('sha256').update(content).digest('hex');
 }
 /**
- * Skip files larger than this (bytes). Generated bundles, minified JS, and
- * vendored blobs blow the WASM heap and the worker-recycle budget for no useful
- * symbols. 1 MB covers essentially all hand-written source.
+ * What change detection hashes for a file: its text when it is under the size
+ * limit, the size stamp when it is over — an oversize file is never decoded.
+ * `null` when the bytes show a `.ts` file is an MPEG transport stream, not
+ * TypeScript (#1910): decided from the bytes already read, so it costs no I/O
+ * and runs only for a file that is new or changed — never once per `.ts` file
+ * at discovery, which on a slow disk is a random read per file per scan.
  */
-const MAX_FILE_SIZE = 1024 * 1024;
+function readSourceOrStamp(fullPath) {
+    const { stats, bytes } = (0, file_limits_1.readBoundedSourceSync)(fullPath);
+    if (bytes === null)
+        return (0, file_limits_1.oversizeStamp)(stats.size);
+    return isMpegTsBytes(fullPath, bytes) ? null : bytes.toString('utf8');
+}
+/** Whether these bytes, read from `filePath`, are a `.ts` MPEG transport stream (#1910). */
+function isMpegTsBytes(filePath, bytes) {
+    if (!(0, grammars_1.hasMpegTsExtension)(filePath) || !(0, grammars_1.isMpegTransportStream)(bytes.subarray(0, grammars_1.MPEG_TS_SNIFF_BYTES)))
+        return false;
+    (0, errors_1.logDebug)('Skipping MPEG transport stream named .ts — not TypeScript', { filePath });
+    return true;
+}
 /**
  * Directory names that are dependency, build, cache, or tooling output across the
  * languages/frameworks LatticeSensor supports — curated from the canonical
@@ -191,6 +213,13 @@ const DEFAULT_IGNORE_PATTERNS = [
     'bazel-*/', // Bazel output symlink trees
     // Android resource dirs at any depth, with their qualifier variants (#1047).
     ...ANDROID_RES_TYPES.map((t) => `**/res/${t}*/`),
+    // `build` is also a legal JVM package segment. Keep it under conventional
+    // source roots (any source set: main, test, androidTest, ...) while continuing
+    // to exclude module/build output (#1642). Unignore only the directory, not its
+    // subtree: other defaults still apply.
+    '!**/src/*/java/**/build/',
+    '!**/src/*/kotlin/**/build/',
+    '!**/src/*/scala/**/build/',
 ];
 /** True if `buf` decodes as strict UTF-8 (no invalid byte sequences). */
 function isValidUtf8(buf) {
@@ -260,18 +289,119 @@ function readGitignorePatterns(giPath) {
     }
     return kept.join('\n');
 }
+/**
+ * Resolve the repository GIT_DIR for `repoRoot` (a `.git` directory, or the
+ * target of a `.git` file pointer). Null when this isn't a git checkout.
+ */
+function resolveGitDir(repoRoot) {
+    const gitPath = path.join(repoRoot, '.git');
+    let st;
+    try {
+        st = fs.statSync(gitPath);
+    }
+    catch {
+        return null;
+    }
+    if (st.isDirectory())
+        return gitPath;
+    if (!st.isFile())
+        return null;
+    try {
+        const raw = fs.readFileSync(gitPath, 'utf8').match(/^gitdir:\s*(.+)$/m)?.[1]?.trim();
+        if (!raw)
+            return null;
+        return path.isAbsolute(raw) ? path.normalize(raw) : path.resolve(repoRoot, raw);
+    }
+    catch {
+        return null;
+    }
+}
+/** Expand a leading `~/` the way git does for `core.excludesFile`. */
+function expandUserPath(p) {
+    if (p === '~')
+        return os.homedir();
+    if (p.startsWith('~/'))
+        return path.join(os.homedir(), p.slice(2));
+    return p;
+}
+/**
+ * Root-relative exclude patterns from git sources that are NOT the root
+ * `.gitignore`: `.git/info/exclude` and `core.excludesFile`. Same semantics as
+ * the root `.gitignore`, so they merge into {@link buildDefaultIgnore}. Without
+ * these, the watcher / FS-walk scope silently diverged from
+ * `git ls-files --exclude-standard` (#1728).
+ */
+function readGitExcludeExtraPatterns(rootDir) {
+    const chunks = [];
+    const gitDir = resolveGitDir(rootDir);
+    if (gitDir) {
+        const excludePath = path.join(gitDir, 'info', 'exclude');
+        if (fs.existsSync(excludePath)) {
+            const patterns = readGitignorePatterns(excludePath);
+            if (patterns)
+                chunks.push(patterns);
+        }
+    }
+    try {
+        const configured = (0, child_process_1.execFileSync)('git', ['-C', rootDir, 'config', '--get', 'core.excludesFile'], { encoding: 'utf8', timeout: 5_000, stdio: ['ignore', 'pipe', 'ignore'] }).trim();
+        if (configured) {
+            const abs = expandUserPath(configured);
+            if (fs.existsSync(abs)) {
+                const patterns = readGitignorePatterns(abs);
+                if (patterns)
+                    chunks.push(patterns);
+            }
+        }
+    }
+    catch {
+        // No git, unset, or timeout — leave extras empty.
+    }
+    return chunks.join('\n');
+}
+/**
+ * Directories `git ls-files -o -i --exclude-standard --directory` reports as
+ * ignored-untracked. Seeded into {@link ScopeIgnore} so nested `.gitignore`
+ * effects (and any exclude-standard rule the flat matcher might miss) prune the
+ * watcher the same way the indexer skips them (#1728).
+ */
+function listGitIgnoredDirectories(rootDir) {
+    try {
+        const out = (0, child_process_1.execFileSync)('git', ['-C', rootDir, 'ls-files', '-z', '-o', '-i', '--exclude-standard', '--directory'], {
+            encoding: 'utf8',
+            timeout: 60_000,
+            maxBuffer: 50 * 1024 * 1024,
+            stdio: ['ignore', 'pipe', 'ignore'],
+        });
+        const dirs = [];
+        for (const entry of out.split('\0')) {
+            if (!entry)
+                continue;
+            dirs.push(entry.endsWith('/') ? entry : `${entry}/`);
+        }
+        return dirs;
+    }
+    catch {
+        return [];
+    }
+}
 /**
  * An `ignore` matcher seeded with the built-in defaults, merged with the project's
- * root .gitignore so a negation there (e.g. `!vendor/`) overrides a default. Shared
- * by both enumeration paths so behavior is identical with or without git — and so
- * the defaults apply to tracked files too (committing a dependency dir doesn't make
- * it project code; the explicit `.gitignore` negation is the only opt-in).
+ * root .gitignore so a negation there (e.g. `!vendor/`) overrides a default, plus
+ * git's other root-relative exclude files (`.git/info/exclude`, `core.excludesFile`)
+ * so watcher / FS-walk scope matches `git ls-files --exclude-standard` (#1728).
+ * Shared by both enumeration paths so behavior is identical with or without git —
+ * and so the defaults apply to tracked files too (committing a dependency dir
+ * doesn't make it project code; the explicit `.gitignore` negation is the only
+ * opt-in).
  */
 function buildDefaultIgnore(rootDir) {
     const ig = (0, ignore_1.default)().add(DEFAULT_IGNORE_PATTERNS);
     const rootGitignore = path.join(rootDir, '.gitignore');
     if (fs.existsSync(rootGitignore))
         ig.add(readGitignorePatterns(rootGitignore));
+    const extra = readGitExcludeExtraPatterns(rootDir);
+    if (extra)
+        ig.add(extra);
     return ig;
 }
 /**
@@ -590,15 +720,38 @@ function findNestedGitRepos(absDir, relPrefix) {
 }
 /**
  * Workspace-scope ignore matcher. Ordinary paths get the root's matcher
- * (built-in defaults + root `.gitignore`); paths inside an EMBEDDED repo get
- * that repo's own matcher (defaults + its root `.gitignore`) — the parent's
- * `.gitignore` hides a child repo from git, not from the index (#514). A
- * directory path (trailing slash) that is an ANCESTOR of an embedded root is
- * never ignored, so directory-pruning callers (the Linux per-directory
- * watcher) still descend to reach the embedded repos.
+ * (built-in defaults + root `.gitignore` + `.git/info/exclude` +
+ * `core.excludesFile`, plus directories `git ls-files --exclude-standard`
+ * reports as ignored); paths inside an EMBEDDED repo get that repo's own
+ * matcher — the parent's `.gitignore` hides a child repo from git, not from
+ * the index (#514). A directory path (trailing slash) that is an ANCESTOR of
+ * an embedded root is never ignored, so directory-pruning callers (the Linux
+ * per-directory watcher) still descend to reach the embedded repos.
  *
- * Single source of truth for indexer and watcher scope — they must not diverge.
+ * Shared by the indexer (scoped sync / skip checks) and the watcher so their
+ * scope cannot diverge from each other or from `git ls-files --exclude-standard`
+ * (#1728).
  */
+/**
+ * The grammars to preload for a file set.
+ *
+ * Path-only detection calls every `.h` file C, but parse-time detection reads
+ * the source and can reclassify it as C++ or Objective-C (`detectLanguage`
+ * with a `source` argument). Workers only ever get the grammars named here, so
+ * a header that turns out to be Objective-C in a project with no `.m` file
+ * found no parser and failed with `Failed to get parser for language: objc`
+ * (#1628). C++ was already covered; Objective-C was not.
+ */
+function preloadLanguagesForFiles(files, overrides) {
+    const languages = [...new Set(files.map((f) => (0, grammars_1.detectLanguage)(f, undefined, overrides)))];
+    if (languages.includes('c')) {
+        for (const ambiguous of ['cpp', 'objc']) {
+            if (!languages.includes(ambiguous))
+                languages.push(ambiguous);
+        }
+    }
+    return languages;
+}
 class ScopeIgnore {
     rootMatcher;
     exclude;
@@ -677,7 +830,14 @@ exports.ScopeIgnore = ScopeIgnore;
 function buildScopeIgnore(rootDir, embeddedRoots) {
     const roots = embeddedRoots ? [...embeddedRoots] : discoverEmbeddedRepoRoots(rootDir);
     const include = loadIncludeMatcher(rootDir);
-    return new ScopeIgnore(buildDefaultIgnore(rootDir), roots.map((root) => ({ root, matcher: buildDefaultIgnore(path.join(rootDir, root)) })), loadExcludeMatcher(rootDir), include, include ? includeStaticRoots((0, project_config_1.loadIncludePatterns)(rootDir)) : []);
+    // Root matcher already has defaults + root `.gitignore` + info/exclude +
+    // core.excludesFile. Seed ignored-untracked directories from git so nested
+    // `.gitignore` effects prune the watcher identically to the indexer (#1728).
+    const rootMatcher = buildDefaultIgnore(rootDir);
+    for (const dir of listGitIgnoredDirectories(rootDir)) {
+        rootMatcher.add(dir);
+    }
+    return new ScopeIgnore(rootMatcher, roots.map((root) => ({ root, matcher: buildDefaultIgnore(path.join(rootDir, root)) })), loadExcludeMatcher(rootDir), include, include ? includeStaticRoots((0, project_config_1.loadIncludePatterns)(rootDir)) : []);
 }
 /**
  * Whether an embedded repo found as a tracked gitlink (mode 160000, #1031/#1033)
@@ -744,7 +904,7 @@ function discoverEmbeddedRepoRoots(rootDir) {
         // same way collectGitFiles does, keeping watcher scope == indexer scope.
         // (#1031, #1033)
         try {
-            const staged = (0, child_process_1.execFileSync)('git', ['ls-files', '-z', '-s', '--recurse-submodules'], { cwd: repoAbs, encoding: 'utf-8', timeout: 30000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true });
+            const staged = lsFilesStaged({ cwd: repoAbs, encoding: 'utf-8', timeout: 30000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true });
             const repoIgnore = buildDefaultIgnore(repoAbs);
             for (const entry of staged.split('\0')) {
                 if (!entry || entry.slice(0, 6) !== '160000')
@@ -864,6 +1024,30 @@ function findIgnoredEmbeddedRepos(repoDir, includeIgnored, prefix) {
     }
     return repos;
 }
+/**
+ * `git ls-files -z -s`, expanding submodules where git allows it.
+ *
+ * `--recurse-submodules` could not be combined with `-s` before git 2.36:
+ * `builtin/ls-files.c` listed `show_stage` among the modes that die, and the
+ * check is unconditional — it does not look at whether the repo actually has
+ * submodules, so every call fails on older git. Ubuntu 22.04 LTS (2.34.1) and
+ * Debian 11 (2.30.2) are both below that line.
+ *
+ * Letting the throw escape cost far more than submodule expansion: it unwound
+ * the whole git-visible pass, so `includeIgnored`, gitlink recursion and the
+ * `lattice-sensor.json` include allowlist silently stopped applying and files went
+ * missing from the index with no error (#1549). Retry without the flag instead
+ * — `-s` is the part that matters here, since gitlink detection reads the mode
+ * bits, and embedded repos are reached through the gitlink recursion anyway.
+ */
+function lsFilesStaged(gitOpts) {
+    try {
+        return (0, child_process_1.execFileSync)('git', ['ls-files', '-z', '-s', '--recurse-submodules'], gitOpts);
+    }
+    catch {
+        return (0, child_process_1.execFileSync)('git', ['ls-files', '-z', '-s'], gitOpts);
+    }
+}
 /**
  * Collect git-visible files (tracked + untracked, .gitignore-respected) from the
  * git repository rooted at `repoDir`, adding each to `files` with `prefix`
@@ -908,7 +1092,12 @@ function collectGitFiles(repoDir, prefix, files, embeddedRoots, includeIgnored =
     // on disk → those files are silently dropped from the index. (#541) With -s the
     // path follows a TAB after the `<mode> <object> <stage>` prefix.
     const gitlinkRels = [];
-    const tracked = (0, child_process_1.execFileSync)('git', ['ls-files', '-z', '-s', '--recurse-submodules'], gitOpts);
+    // Lattice owns `.lattice/` (its ToDo store, run worktrees, adapter registry):
+    // the FS walks and the watcher never descend into it (isLatticeStateDir), and
+    // git's listing must agree — an untracked script written there by a run is not
+    // project source, and counting it made status report the index stale mid-run.
+    const inLatticeState = (rel) => rel.split('/').some((part) => (0, directory_1.isLatticeStateDir)(part));
+    const tracked = lsFilesStaged(gitOpts);
     for (const entry of tracked.split('\0')) {
         if (!entry)
             continue;
@@ -916,6 +1105,8 @@ function collectGitFiles(repoDir, prefix, files, embeddedRoots, includeIgnored =
         if (tab === -1)
             continue; // --stage always emits "<mode> <object> <stage>\t<path>"
         const rel = entry.slice(tab + 1);
+        if (inLatticeState(rel))
+            continue;
         if (entry.slice(0, 6) === '160000') {
             gitlinkRels.push(rel); // an unexpanded gitlink — recursed into below, not a source file itself
             continue;
@@ -929,6 +1120,8 @@ function collectGitFiles(repoDir, prefix, files, embeddedRoots, includeIgnored =
     for (const rel of untracked.split('\0')) {
         if (!rel)
             continue;
+        if (inLatticeState(rel))
+            continue;
         if (rel.endsWith('/')) {
             // git only emits a trailing-slash directory entry for an embedded repo.
             // Guard with a .git check anyway, and skip anything else exactly as git
@@ -999,6 +1192,10 @@ function getGitVisibleFiles(rootDir) {
                 // git check-ignore exits 0 if the path IS ignored, 1 if not
                 (0, child_process_1.execFileSync)('git', ['check-ignore', '-q', path.resolve(rootDir)], { cwd: rootDir, encoding: 'utf-8', timeout: 5000, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true });
                 // Directory is gitignored by parent repo — fall back to filesystem walk
+                (0, errors_1.logDebug)('project root is gitignored by a parent repo — falling back to filesystem walk', {
+                    rootDir,
+                    gitRoot,
+                });
                 return null;
             }
             catch {
@@ -1025,7 +1222,20 @@ function getGitVisibleFiles(rootDir) {
             visible.add(f);
         return visible;
     }
-    catch {
+    catch (error) {
+        // Any failure here (git missing, a `git rev-parse`/`ls-files` timeout or
+        // buffer overrun under load, an unreadable repo, unsupported flag combo on
+        // older git, etc.) silently sent every caller to `scanDirectoryWalk` with
+        // zero signal that the fast git-delegated path was skipped — making reports
+        // like #1567 (nested-`.gitignore`-excluded `node_modules` walked into)
+        // hard to triage, since both ignore implementations look correct in
+        // isolation but there was no way to tell which one ran. Log it under the
+        // existing LATTICE_SENSOR_DEBUG gate so a future report can confirm or rule out
+        // the fallback in one step.
+        (0, errors_1.logDebug)('git-based file listing unavailable — falling back to filesystem walk', {
+            rootDir,
+            error: error instanceof Error ? error.message : String(error),
+        });
         return null;
     }
 }
@@ -1043,21 +1253,102 @@ function getGitVisibleFiles(rootDir) {
  * case this cannot see (the child status that would report the deletions is gone
  * with it); a full `lattice sensor index` reconciles that.
  */
-function getGitChangedFiles(rootDir) {
+function getGitChangedFiles(rootDir, sinceCommit) {
     try {
+        // `git status` only ever describes the WORKING TREE, so a change that has
+        // been committed leaves no entry and never enters the candidate set — the
+        // hash comparison in getChangedFiles is correct but is never reached for
+        // it, and `pendingChanges` reads 0 while the index is genuinely behind
+        // (#1829). `sinceCommit` — the commit the index was last brought up to
+        // date at — adds the other half: what has been committed since. Callers
+        // that hold no such stamp still get exactly what they always did, the
+        // working-tree changes.
         const changes = { modified: [], added: [], deleted: [] };
         // Custom extension → language overrides from the project's lattice-sensor.json,
         // so change detection sees the same custom-extension files the full index does.
         const overrides = (0, project_config_1.loadExtensionOverrides)(rootDir);
-        collectGitStatus(rootDir, '', changes, overrides, loadIncludeIgnoredMatcher(rootDir), loadExcludeMatcher(rootDir));
+        collectGitStatus(rootDir, '', changes, overrides, loadIncludeIgnoredMatcher(rootDir), loadExcludeMatcher(rootDir), sinceCommit ?? undefined);
         return changes;
     }
     catch {
         return null;
     }
 }
-function collectGitStatus(repoDir, prefix, out, overrides, includeIgnored = null, exclude = null) {
-    const output = (0, child_process_1.execFileSync)('git', ['status', '--porcelain', '--no-renames'], { cwd: repoDir, encoding: 'utf-8', timeout: 10000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true });
+/**
+ * Metadata key: the commit the index was last brought up to date at. Written by
+ * a full index AND by every successful sync — unlike the extraction stamp, which
+ * a sync must not advance because it only touches a subset of files. This one is
+ * about the tree; failed file paths remain explicit retry candidates. (#1829)
+ */
+exports.INDEXED_AT_COMMIT_KEY = 'indexed_at_commit';
+/** HEAD's commit sha, or null in a non-git repo or one with no commits yet. */
+function getGitHeadSha(rootDir) {
+    try {
+        return (0, child_process_1.execFileSync)('git', ['rev-parse', 'HEAD'], {
+            cwd: rootDir, encoding: 'utf-8', timeout: 5000, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true,
+        }).trim() || null;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * NUL-delimited status/path pairs for every path committed
+ * between `sinceCommit` and HEAD. Empty when the stamp IS HEAD, which is the
+ * common case — one cheap git call on the hot path.
+ */
+function gitCommittedChangesSince(repoDir, sinceCommit) {
+    // NUL framing preserves Unicode, quotes, tabs and newlines in Git paths.
+    // Let command failures reach getGitChangedFiles: [] would falsely mean clean.
+    const out = (0, child_process_1.execFileSync)('git', ['diff', '--relative', '--name-status', '--no-renames', '-z', sinceCommit, 'HEAD', '--', '.'], {
+        cwd: repoDir, encoding: 'utf-8', timeout: 10000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true,
+    });
+    return out.split('\0');
+}
+/**
+ * Can an INDEX trust the git fast path, given the commit it was built at?
+ *
+ * False means "fall back to the full scan" — the expensive path that compares
+ * every file on disk against the DB, and the only correct read when git cannot
+ * say what happened between the stamp and now:
+ *
+ *  - stamp present but unknown to this repo (rebase, gc, shallow clone, a stamp
+ *    from a different checkout) — history moved under the index.
+ *  - stamp absent while the repo HAS commits — an index built before stamping
+ *    existed. One full scan; the next sync stamps it and the fast path returns.
+ *
+ * A repo with NO commits keeps the fast path with or without a stamp: every
+ * file is untracked, so `git status` already sees all of them. Callers with no
+ * index behind them (the exported `getGitChangedFiles`) never ask this — a
+ * working-tree diff is the whole of what they wanted. (#1829)
+ */
+function canTrustGitFastPath(rootDir, sinceCommit) {
+    const head = getGitHeadSha(rootDir);
+    if (head == null)
+        return true; // no commits (or not a git repo — caller handles that)
+    if (!sinceCommit)
+        return false;
+    if (sinceCommit === head)
+        return true;
+    try {
+        (0, child_process_1.execFileSync)('git', ['cat-file', '-e', `${sinceCommit}^{commit}`], {
+            cwd: rootDir, encoding: 'utf-8', timeout: 5000, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true,
+        });
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+function collectGitStatus(repoDir, prefix, out, overrides, includeIgnored = null, exclude = null, sinceCommit) {
+    const output = (0, child_process_1.execFileSync)('git', 
+    // `-uall` lists individual untracked files instead of collapsing an
+    // entirely-untracked directory into one `?? dir/` entry, which would
+    // otherwise be dropped here (only embedded git repos are recursed into
+    // below). Nested untracked git repos still collapse to `?? repo/` even
+    // with `-uall` — git never crosses a repo boundary — so the recursion
+    // still handles them. (#1213)
+    ['status', '--porcelain', '--no-renames', '-z', '-uall'], { cwd: repoDir, encoding: 'utf-8', timeout: 10000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true });
     // This repo's own ignore rules — built-in defaults (#407) plus its .gitignore.
     // Change detection must exclude the SAME files the full index does, but git
     // status hides neither: it ignores nothing for *tracked* paths, and the
@@ -1069,45 +1360,68 @@ function collectGitStatus(repoDir, prefix, out, overrides, includeIgnored = null
     // ScopeIgnore: every embedded repo is judged by ITS OWN rules, never the
     // parent's. (#766)
     const ig = buildDefaultIgnore(repoDir);
-    const untrackedDirs = [];
-    for (const line of output.split('\n')) {
-        if (line.length < 4)
-            continue; // Minimum: "XY file"
-        const statusCode = line.substring(0, 2);
-        const rel = (0, utils_1.normalizePath)(line.substring(3));
-        // Untracked directory entries (trailing slash) may hide an embedded repo —
-        // collect for the recursion below instead of treating as a file.
-        if (statusCode === '??' && rel.endsWith('/')) {
-            untrackedDirs.push(rel);
-            continue;
-        }
+    // One classifier for both candidate sources below, so a committed change is
+    // filtered by exactly the rules a working-tree change is (#766, #999, #1829).
+    const classify = (statusCode, rel) => {
         const filePath = (0, utils_1.normalizePath)(prefix + rel);
         if (!(0, grammars_1.isSourceFile)(filePath, overrides))
-            continue;
+            return;
         if (statusCode.includes('D')) {
             // Deletions stay unfiltered: getChangedFiles acts on one only when the
             // path is already tracked in the DB, where removal is always correct — and
             // that lets a newly-excluded dir's stale rows clean themselves up. (#766)
             out.deleted.push(filePath);
-            continue;
+            return;
         }
+        // Lattice owns `.lattice/`: the full scan and git listing skip it, so a
+        // change there is never pending (a run's untracked scripts would otherwise
+        // read as an added file and mark the index stale).
+        if (filePath.split('/').some((part) => (0, directory_1.isLatticeStateDir)(part)))
+            return;
         // Added (`??`) / modified files inside an excluded dir must not enter the
         // index — match against the repo-relative path, same as the full scan. (#766)
         if (ig.ignores(rel))
-            continue;
+            return;
         // User `lattice-sensor.json` `exclude` (#999) is project-root-relative, so it's
         // matched against the full path — sync must not re-add a tracked file the
         // full index now keeps out. Deletions above stay unfiltered so a file that
         // WAS indexed before an exclude was added still cleans itself out.
         if (exclude && exclude.ignores(filePath))
-            continue;
+            return;
         if (statusCode === '??') {
             out.added.push(filePath);
         }
         else {
-            // M, MM, AM, A (staged), etc. — treat as modified
+            // M, MM, AM, A (staged), etc. — treat as modified. getChangedFiles
+            // re-decides added-vs-modified from the DB, so a committed `A` that the
+            // index never saw still lands in `added`.
             out.modified.push(filePath);
         }
+    };
+    const untrackedDirs = [];
+    for (const line of output.split('\0')) {
+        if (line.length < 4)
+            continue; // Minimum: "XY file"
+        const statusCode = line.substring(0, 2);
+        const rel = (0, utils_1.normalizePath)(line.substring(3));
+        // Untracked directory entries (trailing slash) may hide an embedded repo —
+        // collect for the recursion below instead of treating as a file.
+        if (statusCode === '??' && rel.endsWith('/')) {
+            untrackedDirs.push(rel);
+            continue;
+        }
+        classify(statusCode, rel);
+    }
+    // Committed but unindexed: everything between the commit this index was last
+    // brought up to date at and HEAD. `git status` cannot see these — committing
+    // is precisely what removes a file from its output — so without this pass a
+    // `git commit` makes a real pending change read as zero (#1829). The stamp
+    // belongs to the ROOT repo, so the embedded-repo recursion below passes none.
+    if (sinceCommit) {
+        const fields = gitCommittedChangesSince(repoDir, sinceCommit);
+        for (let i = 0; i + 1 < fields.length; i += 2) {
+            classify(`${fields[i].charAt(0)} `, (0, utils_1.normalizePath)(fields[i + 1]));
+        }
     }
     // Recurse embedded repos found under untracked dirs (at the dir itself or
     // nested deeper). Gitignored dirs are walked only for the directories the
@@ -1149,11 +1463,16 @@ function scanDirectory(rootDir, onProgress) {
     // Fallback: walk filesystem for non-git projects
     return scanDirectoryWalk(rootDir, onProgress);
 }
-/**
- * Async variant of scanDirectory that yields to the event loop periodically,
- * allowing worker threads to receive and render progress messages.
- */
-async function scanDirectoryAsync(rootDir, onProgress) {
+/** Record one file the scan declined to index. */
+function tallySkip(stats, rel) {
+    if (!stats)
+        return;
+    const ext = path.extname(rel).toLowerCase();
+    if (!ext)
+        return;
+    stats.unsupportedByExtension.set(ext, (stats.unsupportedByExtension.get(ext) ?? 0) + 1);
+}
+async function scanDirectoryAsync(rootDir, onProgress, stats) {
     // Custom extension → language overrides from the project's lattice-sensor.json.
     const overrides = (0, project_config_1.loadExtensionOverrides)(rootDir);
     const gitFiles = getGitVisibleFiles(rootDir);
@@ -1170,15 +1489,18 @@ async function scanDirectoryAsync(rootDir, onProgress) {
                     await new Promise(r => setImmediate(r));
                 }
             }
+            else {
+                tallySkip(stats, filePath);
+            }
         }
         return files;
     }
-    return scanDirectoryWalk(rootDir, onProgress);
+    return scanDirectoryWalk(rootDir, onProgress, stats);
 }
 /**
  * Filesystem walk fallback for non-git projects.
  */
-function scanDirectoryWalk(rootDir, onProgress) {
+function scanDirectoryWalk(rootDir, onProgress, stats) {
     const files = [];
     let count = 0;
     const visitedDirs = new Set();
@@ -1250,10 +1572,15 @@ function scanDirectoryWalk(rootDir, onProgress) {
                         }
                     }
                     else if (stat.isFile()) {
-                        if (!isIgnored(fullPath, false, active) && (0, grammars_1.isSourceFile)(relativePath, overrides)) {
-                            files.push(relativePath);
-                            count++;
-                            onProgress?.(count, relativePath);
+                        if (!isIgnored(fullPath, false, active)) {
+                            if ((0, grammars_1.isSourceFile)(relativePath, overrides)) {
+                                files.push(relativePath);
+                                count++;
+                                onProgress?.(count, relativePath);
+                            }
+                            else {
+                                tallySkip(stats, relativePath);
+                            }
                         }
                     }
                 }
@@ -1268,10 +1595,15 @@ function scanDirectoryWalk(rootDir, onProgress) {
                 }
             }
             else if (entry.isFile()) {
-                if (!isIgnored(fullPath, false, active) && (0, grammars_1.isSourceFile)(relativePath, overrides)) {
-                    files.push(relativePath);
-                    count++;
-                    onProgress?.(count, relativePath);
+                if (!isIgnored(fullPath, false, active)) {
+                    if ((0, grammars_1.isSourceFile)(relativePath, overrides)) {
+                        files.push(relativePath);
+                        count++;
+                        onProgress?.(count, relativePath);
+                    }
+                    else {
+                        tallySkip(stats, relativePath);
+                    }
                 }
             }
         }
@@ -1344,10 +1676,57 @@ class ExtractionOrchestrator {
      * hasn't run yet so single-file re-index paths can detect on the spot.
      */
     detectedFrameworkNames = null;
+    /**
+     * Scope matcher for SCOPED syncs, memoized on the mtimes of the two root
+     * files it is derived from (`lattice-sensor.json`, `.gitignore`). See
+     * {@link scopedSyncMatcher}.
+     */
+    scopedMatcher = null;
     constructor(rootDir, queries) {
         this.rootDir = rootDir;
         this.queries = queries;
     }
+    /**
+     * The scope matcher a scoped sync applies to the paths it was handed — the
+     * same `buildScopeIgnore` the full scan uses, so an explicitly-passed path
+     * that is OUT of scope (a user `exclude` in `lattice-sensor.json`, a `.gitignore`
+     * rule, a built-in default) is treated exactly as the full walk would treat
+     * it: absent, hence removed if tracked, never parsed (#1590).
+     *
+     * Memoized on the root config + root `.gitignore` mtimes: building the
+     * matcher runs embedded-repo discovery (`git ls-files`), which would defeat
+     * the scoped path's whole point (skipping O(repo) work) if paid per sync.
+     * Two `stat`s per sync while nothing changed. An embedded repo created
+     * between config edits joins the scoped matcher on the next full sync, the
+     * same lifecycle the watcher's own matcher already has.
+     */
+    scopedSyncMatcher() {
+        // Bust when any root-level exclude source the matcher reads may have
+        // changed. Nested `.gitignore` edits force a full watcher sync, which
+        // clears this cache (see the full-reconcile branch in sync()).
+        const gitDir = resolveGitDir(this.rootDir);
+        const key = [
+            project_config_1.PROJECT_CONFIG_FILENAME,
+            '.gitignore',
+            gitDir ? path.join(gitDir, 'info', 'exclude') : '',
+        ]
+            .map((name) => {
+            if (!name)
+                return '-';
+            try {
+                return String(fs.statSync(path.isAbsolute(name) ? name : path.join(this.rootDir, name)).mtimeMs);
+            }
+            catch {
+                return '-';
+            }
+        })
+            .join('|');
+        if (this.scopedMatcher && this.scopedMatcher.key === key)
+            return this.scopedMatcher.matcher;
+        const matcher = buildScopeIgnore(this.rootDir);
+        this.scopedMatcher = { key, matcher };
+        return matcher;
+    }
     /**
      * Build a filesystem-backed ResolutionContext sufficient for framework
      * detection. Graph-query methods (getNodesByName etc.) return empty because
@@ -1381,7 +1760,9 @@ class ExtractionOrchestrator {
                 if (!full)
                     return null;
                 try {
-                    return fs.readFileSync(full, 'utf-8');
+                    // Framework detectors scan source by name; a file over the size
+                    // limit was never indexed and must not be decoded here either (#1910).
+                    return (0, file_limits_1.readBoundedSourceSync)(full).bytes?.toString('utf8') ?? null;
                 }
                 catch {
                     return null;
@@ -1463,6 +1844,7 @@ class ExtractionOrchestrator {
         // early-run 5-10s single stalls were observed on 95k-file repos but never
         // attributed — these labels settle scan vs framework-detect vs grammars.
         const tScan = Date.now();
+        const skipStats = { unsupportedByExtension: new Map() };
         const files = await scanDirectoryAsync(this.rootDir, (current, file) => {
             onProgress?.({
                 phase: 'scanning',
@@ -1470,9 +1852,26 @@ class ExtractionOrchestrator {
                 total: 0,
                 currentFile: file,
             });
-        });
+        }, skipStats);
         if (process.env.LATTICE_SENSOR_SYNTH_TIMINGS)
             console.error(`[phase-timing] scan: ${Date.now() - tScan}ms (${files.length} files)`);
+        /** Only meaningful when nothing was indexable — see IndexResult (#1502). */
+        const skipSummary = () => {
+            let total = 0;
+            for (const n of skipStats.unsupportedByExtension.values())
+                total += n;
+            if (total === 0)
+                return {};
+            const top = [...skipStats.unsupportedByExtension.entries()]
+                .map(([ext, count]) => ({ ext, count }))
+                .sort((a, b) => b.count - a.count || a.ext.localeCompare(b.ext))
+                .slice(0, 5);
+            return { filesSkippedUnsupported: total, topUnsupportedExtensions: top };
+        };
+        // A re-index over an existing DB skips unchanged-hash files at the store,
+        // which would preserve wiped zero-node rows (#1541) — drop them first so
+        // this run stores their files fresh. No-op on a fresh DB.
+        this.healZeroNodeRows();
         // Detect frameworks once per indexAll run using the scanned file list.
         // Names are passed to each parse call so framework-specific extractors
         // (route nodes, middleware, etc.) run after the tree-sitter pass.
@@ -1508,11 +1907,7 @@ class ExtractionOrchestrator {
         });
         await new Promise(resolve => setImmediate(resolve));
         // Detect needed languages and load grammars in the parse worker
-        const neededLanguages = [...new Set(files.map((f) => (0, grammars_1.detectLanguage)(f, undefined, overrides)))];
-        // .h files default to 'c' but may be C++ — ensure cpp grammar is loaded when c is needed
-        if (neededLanguages.includes('c') && !neededLanguages.includes('cpp')) {
-            neededLanguages.push('cpp');
-        }
+        const neededLanguages = preloadLanguagesForFiles(files, overrides);
         // Parse files on a pool of worker threads (keeps the main thread free for UI
         // and uses every core). Falls back to in-process parsing when the compiled
         // worker is unavailable (e.g. running from source in tests).
@@ -1619,29 +2014,27 @@ class ExtractionOrchestrator {
             // Store: on the writer thread when active (fresh DB — bundles applied
             // in the same file order this chain dispatches them), else on the main
             // thread (SQLite connections are per-thread).
-            if (nodeCount > 0 || result.errors.length === 0) {
-                const language = (0, grammars_1.detectLanguage)(filePath, content, overrides);
-                if (storeWriter) {
-                    if (result.kernelBuffers) {
-                        // Buffers go to the writer as-is; the worker decodes + finalizes.
-                        // The main thread's only per-file work stays O(1) + the content hash.
-                        storeWriter.send({
-                            kernel: true,
-                            filePath,
-                            language,
-                            buffers: result.kernelBuffers,
-                            file: this.buildFileRecord(filePath, content, language, stats, nodeCount, result.errors),
-                        });
-                    }
-                    else {
-                        storeWriter.send(this.buildFreshStoreBundle(filePath, content, language, stats, result));
-                    }
-                    await storeWriter.waitBelow(STORE_WRITER_WINDOW);
+            const language = (0, grammars_1.detectLanguage)(filePath, content, overrides);
+            if (storeWriter) {
+                if (result.kernelBuffers) {
+                    // Buffers go to the writer as-is; the worker decodes + finalizes.
+                    // The main thread's only per-file work stays O(1) + the content hash.
+                    storeWriter.send({
+                        kernel: true,
+                        filePath,
+                        language,
+                        buffers: result.kernelBuffers,
+                        file: this.buildFileRecord(filePath, content, language, stats, nodeCount, result.errors),
+                    });
                 }
                 else {
-                    const materialized = (0, kernel_1.materializeKernelResult)(result, filePath, language);
-                    await this.storeExtractionResult(filePath, content, language, stats, materialized, commitYield);
+                    storeWriter.send(this.buildFreshStoreBundle(filePath, content, language, stats, result));
                 }
+                await storeWriter.waitBelow(STORE_WRITER_WINDOW);
+            }
+            else {
+                const materialized = (0, kernel_1.materializeKernelResult)(result, filePath, language);
+                await this.storeExtractionResult(filePath, content, language, stats, materialized, commitYield);
             }
             if (result.errors.length > 0) {
                 for (const err of result.errors) {
@@ -1672,16 +2065,19 @@ class ExtractionOrchestrator {
             }
             onProgress?.({ phase: 'parsing', current: processed, total, currentFile: filePath });
         };
-        const recordParseFailure = (filePath, err) => {
-            processed++;
-            filesErrored++;
-            errors.push({
-                message: err instanceof Error ? err.message : String(err),
-                filePath,
-                severity: 'error',
-                code: 'parse_error',
+        const recordParseFailure = async (filePath, content, stats, err) => {
+            await storeResult(filePath, content, stats, {
+                nodes: [],
+                edges: [],
+                unresolvedReferences: [],
+                errors: [{
+                        message: err instanceof Error ? err.message : String(err),
+                        filePath,
+                        severity: 'error',
+                        code: 'parse_error',
+                    }],
+                durationMs: 0,
             });
-            onProgress?.({ phase: 'parsing', current: processed, total });
         };
         // Commit buffered parses to the DB in file order, advancing the cursor over
         // contiguous completed results. Runs after each parse settles (and once more
@@ -1706,7 +2102,7 @@ class ExtractionOrchestrator {
                         if (item.ok)
                             await storeResult(item.filePath, item.content, item.stats, item.result);
                         else
-                            recordParseFailure(item.filePath, item.err);
+                            await recordParseFailure(item.filePath, item.content, item.stats, item.err);
                     }
                 }
                 catch (err) {
@@ -1726,7 +2122,7 @@ class ExtractionOrchestrator {
                     completed.set(seq, { ok: true, filePath, content, stats, result });
                 }
                 catch (parseErr) {
-                    completed.set(seq, { ok: false, filePath, err: parseErr });
+                    completed.set(seq, { ok: false, filePath, content, stats, err: parseErr });
                 }
                 flushOrdered();
             })();
@@ -1739,8 +2135,11 @@ class ExtractionOrchestrator {
             // window has room. When nothing is in flight but the window is still full,
             // the async commit chain is what's behind — await it so the cursor
             // advances (buffered items hold whole file contents, so this bound is
-            // load-bearing for memory).
-            while (nextSeq - nextToStore >= windowSize) {
+            // load-bearing for memory). Once a store has failed the cursor never
+            // moves again — flushOrdered returns at once — so stop waiting and let
+            // the drain below rethrow; waiting would spin on microtasks forever,
+            // pinning a core and starving every timer in the process.
+            while (nextSeq - nextToStore >= windowSize && !flushError && !aborted) {
                 if (inFlight.size > 0)
                     await Promise.race(inFlight);
                 else
@@ -1765,8 +2164,20 @@ class ExtractionOrchestrator {
                         (0, errors_1.logWarn)('Path traversal blocked in batch reader', { filePath: fp });
                         return { filePath: fp, content: null, stats: null, error: new Error('Path traversal blocked') };
                     }
-                    const content = await fsp.readFile(fullPath, 'utf-8');
-                    const stats = await fsp.stat(fullPath);
+                    // Stat first: a file over the size limit is stored as skipped
+                    // without ever being read or decoded (#1910), so ten oversize
+                    // fixtures in one I/O batch no longer cost their size in RSS.
+                    const { stats, bytes } = await (0, file_limits_1.readBoundedSource)(fullPath);
+                    if (bytes === null) {
+                        return { filePath: fp, content: (0, file_limits_1.oversizeStamp)(stats.size), stats, error: null };
+                    }
+                    // Read bytes, not text: a `.ts` that is really an MPEG transport
+                    // stream (#1910) is recognised from its head here, at no extra I/O,
+                    // and never decoded or parsed.
+                    if (isMpegTsBytes(fp, bytes)) {
+                        return { filePath: fp, content: null, stats: null, error: null, skipped: true };
+                    }
+                    const content = bytes.toString('utf-8');
                     return { filePath: fp, content, stats, error: null };
                 }
                 catch (err) {
@@ -1775,11 +2186,17 @@ class ExtractionOrchestrator {
             }));
             // Dispatch each readable file into the bounded parse window; the window
             // stores results on the main thread as they arrive.
-            for (const { filePath, content, stats, error } of fileContents) {
+            for (const { filePath, content, stats, error, skipped } of fileContents) {
                 if (signal?.aborted) {
                     aborted = true;
                     break;
                 }
+                if (skipped) {
+                    // Not a source file after all — counted as done, stored as nothing.
+                    processed++;
+                    onProgress?.({ phase: 'parsing', current: processed, total });
+                    continue;
+                }
                 if (error || content === null || stats === null) {
                     processed++;
                     filesErrored++;
@@ -1792,21 +2209,24 @@ class ExtractionOrchestrator {
                     onProgress?.({ phase: 'parsing', current: processed, total });
                     continue;
                 }
-                // Honour MAX_FILE_SIZE. Without this check, vendored generated
+                // Honour MAX_SOURCE_FILE_SIZE_BYTES. Without this check, vendored generated
                 // headers, minified bundles, and other multi-MB files get indexed,
                 // wasting WASM heap and the worker recycle budget on inputs with no
                 // useful symbols. The single-file extractFile path already enforces
                 // this; the bulk path used to silently skip the check.
-                if (stats.size > MAX_FILE_SIZE) {
-                    processed++;
-                    filesSkipped++;
-                    errors.push({
-                        message: `File exceeds max size (${stats.size} > ${MAX_FILE_SIZE})`,
-                        filePath,
-                        severity: 'warning',
-                        code: 'size_exceeded',
+                if (stats.size > file_limits_1.MAX_SOURCE_FILE_SIZE_BYTES) {
+                    await storeResult(filePath, content, stats, {
+                        nodes: [],
+                        edges: [],
+                        unresolvedReferences: [],
+                        errors: [{
+                                message: `File exceeds max size (${stats.size} > ${file_limits_1.MAX_SOURCE_FILE_SIZE_BYTES})`,
+                                filePath,
+                                severity: 'warning',
+                                code: 'size_exceeded',
+                            }],
+                        durationMs: 0,
                     });
-                    onProgress?.({ phase: 'parsing', current: processed, total });
                     continue;
                 }
                 // Parse on the pool (main thread stays unblocked). Errors/timeouts are
@@ -1824,6 +2244,9 @@ class ExtractionOrchestrator {
             if (flushError) {
                 if (storeWriter)
                     await storeWriter.close();
+                // Its worker threads would otherwise outlive the failed index.
+                if (pool)
+                    await pool.destroy();
                 throw flushError;
             }
             // All bundles are posted; wait for the writer to apply them, then close
@@ -1852,6 +2275,7 @@ class ExtractionOrchestrator {
                 filesSkipped,
                 filesErrored,
                 filesDiscovered: total,
+                ...skipSummary(),
                 nodesCreated: totalNodes,
                 edgesCreated: totalEdges,
                 errors: [{ message: 'Aborted', severity: 'error' }, ...errors],
@@ -1894,7 +2318,11 @@ class ExtractionOrchestrator {
                     const fullPath = (0, utils_1.validatePathWithinRoot)(this.rootDir, filePath);
                     if (!fullPath)
                         continue;
-                    content = await fsp.readFile(fullPath, 'utf-8');
+                    // Bounded like the first read: the file may have grown since (#1910).
+                    const bytes = (await (0, file_limits_1.readBoundedSource)(fullPath)).bytes;
+                    if (bytes === null)
+                        continue;
+                    content = bytes.toString('utf8');
                 }
                 catch {
                     continue;
@@ -1907,8 +2335,15 @@ class ExtractionOrchestrator {
                     stillFailing.push(errEntry);
                     continue;
                 }
+                // The pool hands kernel results back as an undecoded buffer transport
+                // (`nodes`/`edges` EMPTY, tables in kernelBuffers). The main loop
+                // decodes or forwards to the store worker; this path stores directly,
+                // so decode here — otherwise a kernel-language retry passes the gate
+                // below via `errors.length === 0`, stores nothing, and the file is
+                // permanently recorded as "(0 symbols)" with the error erased (#1541).
+                const language = (0, grammars_1.detectLanguage)(filePath, content, overrides);
+                result = (0, kernel_1.materializeKernelResult)(result, filePath, language);
                 if (result.nodes.length > 0 || result.errors.length === 0) {
-                    const language = (0, grammars_1.detectLanguage)(filePath, content, overrides);
                     const stats = await fsp.stat(path.join(this.rootDir, filePath));
                     await this.storeExtractionResult(filePath, content, language, stats, result, commitYield);
                     const idx = errors.indexOf(errEntry);
@@ -1937,7 +2372,10 @@ class ExtractionOrchestrator {
                         const fullPath = (0, utils_1.validatePathWithinRoot)(this.rootDir, filePath);
                         if (!fullPath)
                             continue;
-                        fullContent = await fsp.readFile(fullPath, 'utf-8');
+                        const bytes = (await (0, file_limits_1.readBoundedSource)(fullPath)).bytes;
+                        if (bytes === null)
+                            continue;
+                        fullContent = bytes.toString('utf8');
                     }
                     catch {
                         continue;
@@ -1955,13 +2393,20 @@ class ExtractionOrchestrator {
                     catch {
                         continue;
                     }
+                    // Same undecoded-transport hazard as the first retry pass (#1541).
+                    const language = (0, grammars_1.detectLanguage)(filePath, fullContent, overrides);
+                    result = (0, kernel_1.materializeKernelResult)(result, filePath, language);
                     if (result.nodes.length > 0 || result.errors.length === 0) {
-                        const language = (0, grammars_1.detectLanguage)(filePath, fullContent, overrides);
                         const stats = await fsp.stat(path.join(this.rootDir, filePath));
                         await this.storeExtractionResult(filePath, fullContent, language, stats, result, commitYield);
-                        const idx = errors.indexOf(errEntry);
-                        if (idx >= 0)
-                            errors.splice(idx, 1);
+                        // Salvaged from comment-stripped source: keep a visible trace in
+                        // the summary instead of erasing the failure outright — the
+                        // stored result may be missing whatever the failing parse choked
+                        // on, and a silently "clean" file here is how an index quietly
+                        // disagrees with a later per-file sync of the same bytes (#1565).
+                        errEntry.severity = 'warning';
+                        errEntry.code = 'salvaged_stripped';
+                        errEntry.message = `Indexed from comment-stripped source after repeated parse failures (symbols may be incomplete until the file is re-indexed): ${errEntry.message}`;
                         filesErrored--;
                         filesIndexed++;
                         totalNodes += result.nodes.length;
@@ -1980,6 +2425,7 @@ class ExtractionOrchestrator {
             filesSkipped,
             filesErrored,
             filesDiscovered: total,
+            ...skipSummary(),
             nodesCreated: totalNodes,
             edgesCreated: totalEdges,
             errors,
@@ -2050,8 +2496,14 @@ class ExtractionOrchestrator {
         let content;
         let stats;
         try {
-            stats = await fsp.stat(fullPath);
-            content = await fsp.readFile(fullPath, 'utf-8');
+            // An oversize file is stored as skipped; its bytes are never needed (#1910).
+            const read = await (0, file_limits_1.readBoundedSource)(fullPath);
+            stats = read.stats;
+            // An MPEG transport stream named `.ts` is not TypeScript (#1910).
+            if (read.bytes !== null && isMpegTsBytes(relativePath, read.bytes)) {
+                return { nodes: [], edges: [], unresolvedReferences: [], errors: [], durationMs: 0 };
+            }
+            content = read.bytes === null ? (0, file_limits_1.oversizeStamp)(stats.size) : read.bytes.toString('utf8');
         }
         catch (error) {
             return {
@@ -2088,15 +2540,16 @@ class ExtractionOrchestrator {
                 durationMs: 0,
             };
         }
+        const language = (0, grammars_1.detectLanguage)(relativePath, content, (0, project_config_1.loadExtensionOverrides)(this.rootDir));
         // Check file size
-        if (stats.size > MAX_FILE_SIZE) {
-            return {
+        if (stats.size > file_limits_1.MAX_SOURCE_FILE_SIZE_BYTES) {
+            const result = {
                 nodes: [],
                 edges: [],
                 unresolvedReferences: [],
                 errors: [
                     {
-                        message: `File exceeds max size (${stats.size} > ${MAX_FILE_SIZE})`,
+                        message: `File exceeds max size (${stats.size} > ${file_limits_1.MAX_SOURCE_FILE_SIZE_BYTES})`,
                         filePath: relativePath,
                         severity: 'warning',
                         code: 'size_exceeded',
@@ -2104,9 +2557,10 @@ class ExtractionOrchestrator {
                 ],
                 durationMs: 0,
             };
+            await this.storeExtractionResult(relativePath, content, language, stats, result, (0, cooperative_yield_1.createYielder)());
+            return result;
         }
         // Detect language (honoring the project's lattice-sensor.json extension overrides)
-        const language = (0, grammars_1.detectLanguage)(relativePath, content, (0, project_config_1.loadExtensionOverrides)(this.rootDir));
         if (!(0, grammars_1.isLanguageSupported)(language)) {
             return {
                 nodes: [],
@@ -2122,15 +2576,43 @@ class ExtractionOrchestrator {
         const frameworkNames = this.ensureDetectedFrameworks();
         const result = (0, tree_sitter_1.extractFromSource)(relativePath, content, language, frameworkNames);
         // Store in database
-        if (result.nodes.length > 0 || result.errors.length === 0) {
-            await this.storeExtractionResult(relativePath, content, language, stats, result, (0, cooperative_yield_1.createYielder)());
-        }
+        await this.storeExtractionResult(relativePath, content, language, stats, result, (0, cooperative_yield_1.createYielder)());
         return result;
     }
     /**
      * Store extraction result in database
      */
+    /**
+     * Delete file rows recorded with ZERO nodes so their files re-index.
+     *
+     * No extraction path stores an empty, error-free result for a
+     * symbol-bearing language — even an empty file keeps its file node — so a
+     * zero-node row is a wiped one (#1541: an interrupted parse's retry stored
+     * an undecoded kernel transport). The wiped row's content hash matches the
+     * on-disk bytes, so every hash-based reconcile skips the file forever;
+     * deleting the row lets the normal add path repair it. File-level-only
+     * languages (yaml, twig, properties) are left alone. Deleting a zero-node
+     * row cascades nothing: it has no nodes, so no edges or refs either.
+     */
+    healZeroNodeRows() {
+        for (const f of this.queries.getAllFiles()) {
+            // A zero-node row WITH recorded errors is a deliberate skip marker
+            // (#1557: oversized / repeatedly-unparseable files are persisted with
+            // their reason so syncs stop retrying them) — leave those alone. The
+            // #1541 wipe rows are the error-FREE zero-node rows.
+            if (f.nodeCount === 0 &&
+                !(0, grammars_1.isFileLevelOnlyLanguage)(f.language) &&
+                (f.errors === undefined || f.errors.length === 0)) {
+                this.queries.deleteFile(f.path);
+            }
+        }
+    }
     async storeExtractionResult(filePath, content, language, stats, result, onYield) {
+        // A kernel result can arrive as an undecoded buffer transport (empty
+        // node/edge arrays, tables riding in kernelBuffers). Decode it before
+        // storing — persisting the transport as-is records the file as having no
+        // symbols at all (#1541). No-op for already-decoded results.
+        result = (0, kernel_1.materializeKernelResult)(result, filePath, language);
         // Bulk inserts run in bounded sub-transactions with a yield between, so a
         // giant generated file (tens of thousands of symbols) can't block the
         // event loop — and the #850 watchdog heartbeat — for the whole store.
@@ -2139,7 +2621,12 @@ class ExtractionOrchestrator {
         // is unchanged: a partially-stored file has no record and re-indexes.
         const STORE_CHUNK = 2000;
         const contentHash = hashContent(content);
-        // Check if file already exists and hasn't changed
+        // Check if file already exists and hasn't changed. A skip/failure MARKER
+        // row (zero nodes + recorded errors, #1557) never blocks a store carrying
+        // real content: markers are written BEFORE the retry pass under the same
+        // content hash, so treating them as "no changes" would silently discard a
+        // successful retry's symbols — a permanent empty file presented as
+        // recovered (the #1541 wipe, reintroduced through the marker path).
         const existingFile = this.queries.getFileByPath(filePath);
         if (existingFile && existingFile.contentHash === contentHash
             && existingFile.extractionVersion >= extraction_version_1.EXTRACTION_VERSION) {
@@ -2147,8 +2634,16 @@ class ExtractionOrchestrator {
             // one. `>=` not `===`: rewriting a newer engine's row with this engine's
             // output is a downgrade, and two engines sharing an index would overwrite
             // each other forever.
-            return;
+            const existingIsMarker = existingFile.nodeCount === 0 && (existingFile.errors?.length ?? 0) > 0;
+            const incomingHasContent = result.nodes.length > 0;
+            if (!existingIsMarker || !incomingHasContent) {
+                return; // No changes
+            }
         }
+        // Re-decided on every re-index of a changed file, so a banner added (or
+        // removed) by an edit is reflected on the next sync (#1500). Computed after
+        // the unchanged-file early return so untouched files pay nothing.
+        const generated = (0, generated_detection_1.detectGeneratedFile)(filePath, content);
         // Snapshot incoming cross-file edges BEFORE deleting this file's nodes.
         // `deleteFile` cascades to delete every edge whose source OR target is a
         // node in this file (edges.FK ... ON DELETE CASCADE). Edges whose SOURCE is
@@ -2209,6 +2704,7 @@ class ExtractionOrchestrator {
                     nodeCount: result.nodes.length,
                     extractionVersion: extraction_version_1.EXTRACTION_VERSION,
                     errors: result.errors.length > 0 ? result.errors : undefined,
+                    generated,
                 },
             });
             if (crossFileIncomingEdges.length > 0) {
@@ -2265,6 +2761,7 @@ class ExtractionOrchestrator {
             nodeCount: result.nodes.length,
             extractionVersion: extraction_version_1.EXTRACTION_VERSION,
             errors: result.errors.length > 0 ? result.errors : undefined,
+            generated,
         };
         this.queries.upsertFile(fileRecord);
     }
@@ -2285,6 +2782,10 @@ class ExtractionOrchestrator {
             nodeCount,
             extractionVersion: extraction_version_1.EXTRACTION_VERSION,
             errors: resultErrors.length > 0 ? resultErrors : undefined,
+            // Decided here, once, while the content is already in memory — never at
+            // query time (#1500). The header scan short-circuits on a single
+            // substring test for ~every hand-written file.
+            generated: (0, generated_detection_1.detectGeneratedFile)(filePath, content),
         };
     }
     buildFreshStoreBundle(filePath, content, language, stats, result) {
@@ -2322,6 +2823,64 @@ class ExtractionOrchestrator {
             this.queries.insertUnresolvedRefsBatch(resurrected);
         }
     }
+    /**
+     * Re-open, for re-resolution, every resolution edge whose answer this sync
+     * may have changed — the fix for index drift (CG-33).
+     *
+     * Incremental sync re-resolves only the references IN the changed files, but
+     * resolution's answer is a function of the WHOLE graph: a reference binds to
+     * one of the same-named definitions project-wide, so adding or removing a
+     * definition of `pct` can change which `pct` every other file's `pct(...)`
+     * should bind to. Those other files are never revisited, and their references
+     * resolved successfully once and were deleted from `unresolved_refs`, so
+     * nothing existed to revisit them with — the index kept an answer that was
+     * correct against an older graph. Measured on lattice sensor's own long-lived
+     * index: 4.3% of distinct edges differed from a clean rebuild, in BOTH
+     * directions, overwhelmingly `calls`. See docs/benchmarks/index-drift-cg33.md.
+     *
+     * This deletes each affected edge and re-inserts it as the reference that
+     * created it (the refName/refKind stamp), status='pending', for the sync's
+     * resolution sweep to bind against the post-sync graph — the same input a
+     * full rebuild resolves from, which is what makes the two converge.
+     *
+     * Deliberately conservative in three ways, because a wrong deletion is a
+     * permanent edge loss while a missed rebind is only residual drift:
+     * - an edge with no refName stamp (synthesized, or built by an engine older
+     *   than the stamp) is left ALONE rather than reconstructed from the target's
+     *   plain name, same rule as `resurrectRefFromDroppedEdge`;
+     * - edges whose source is in a file this sync already re-extracted are
+     *   skipped — their references were re-resolved from scratch moments ago;
+     * - very common names are skipped by the per-name ceiling in
+     *   `getResolutionEdgesByTargetName`.
+     *
+     * Returns the number of references resurrected.
+     */
+    resurrectStaleResolutionEdges(definitionDelta, changedFilePaths) {
+        if (definitionDelta.length === 0)
+            return 0;
+        const alreadyFresh = new Set(changedFilePaths);
+        const candidates = this.queries.getResolutionEdgesByTargetName(definitionDelta);
+        const edgeIds = [];
+        const refs = [];
+        for (const e of candidates) {
+            if (alreadyFresh.has(e.sourceFilePath))
+                continue;
+            const ref = resurrectRefFromDroppedEdge(e);
+            if (!ref)
+                continue; // no stamp — never delete what we cannot restore
+            edgeIds.push(e.edgeId);
+            refs.push(ref);
+        }
+        if (refs.length === 0)
+            return 0;
+        // Delete first. The sweep re-inserts whichever edge resolution now picks,
+        // and `insertEdges` is INSERT OR IGNORE against idx_edges_identity — so a
+        // rebind to the same target is a clean no-op, but leaving the old row in
+        // place for a rebind ELSEWHERE would keep both, turning drift into
+        // duplication.
+        this.queries.replaceResolutionEdgesWithUnresolvedRefs(edgeIds, refs);
+        return refs.length;
+    }
     /**
      * Sync the index with the current file state.
      *
@@ -2340,7 +2899,15 @@ class ExtractionOrchestrator {
      * set is not exactly known (directory removals, event overflow): the full
      * scan-diff remains the ground truth those cases need (#1285).
      */
-    scopedPaths) {
+    scopedPaths, 
+    /**
+     * Writer-side WAL pressure valve (#1539). Called after every changed file
+     * is stored, when no extraction transaction is open, so a checkpoint can
+     * safely catch up before the next file grows the WAL further.
+     */
+    backpressure, 
+    /** Inspect changed inputs before deletion/re-extraction cascades their edges. */
+    onFileChange) {
         await (0, grammars_1.initGrammars)(); // Initialize WASM runtime (grammars loaded lazily below)
         const startTime = Date.now();
         let filesChecked = 0;
@@ -2350,12 +2917,17 @@ class ExtractionOrchestrator {
         let extractionHealed = 0;
         let nodesUpdated = 0;
         const changedFilePaths = [];
+        // `file\0name` definition pairs for the files this sync touches, sampled
+        // BEFORE their nodes are replaced/deleted. Compared against the post-store
+        // pairs below to derive `definitionDelta` (CG-33).
+        const pairsBefore = new Set();
         onProgress?.({
             phase: 'scanning',
             current: 0,
             total: 0,
         });
         const filesToIndex = [];
+        const failedFilePaths = [];
         // === Filesystem reconcile (git-independent) ===
         // The source of truth for "what changed" is the filesystem vs the indexed
         // state — never git. We enumerate the current source files and reconcile
@@ -2375,7 +2947,20 @@ class ExtractionOrchestrator {
             // reads `filesChecked === 0 && durationMs === 0` as the
             // lock-unavailable signature (#449).
             const unique = [...new Set(scopedPaths)];
-            currentFiles = unique.filter((p) => fs.existsSync(path.join(this.rootDir, p)));
+            // A scoped path is "present" only if it exists AND is in scope — the
+            // same two gates the full walk applies (source extension, scope
+            // matcher). Without the scope gate a caller's stale view of scope
+            // leaked straight into the index: the watcher re-parsed a file the
+            // user had just excluded in `lattice-sensor.json` while `lattice sensor sync`
+            // removed it (#1590). Out-of-scope paths fall out of `currentFiles`,
+            // so a tracked one takes the removal branch below, exactly as a full
+            // sync would treat it. (`include`-forced paths pass: ScopeIgnore
+            // applies the include precedence itself.)
+            const scope = this.scopedSyncMatcher();
+            const overrides = (0, project_config_1.loadExtensionOverrides)(this.rootDir);
+            currentFiles = unique.filter((p) => (0, grammars_1.isSourceFile)(p, overrides) &&
+                !scope.ignores(p) &&
+                fs.existsSync(path.join(this.rootDir, p)));
             trackedFiles = [];
             for (const p of unique) {
                 const rec = this.queries.getFileByPath(p);
@@ -2387,10 +2972,17 @@ class ExtractionOrchestrator {
                 console.error(`[phase-timing] sync-scoped: ${Date.now() - tSyncScan}ms (${unique.length} paths, ${trackedFiles.length} tracked)`);
         }
         else {
+            // Full reconcile: drop the memoized scope matcher so a nested
+            // `.gitignore` / exclude-standard change that forced this full sync is
+            // visible to the next scoped sync (#1728).
+            this.scopedMatcher = null;
             currentFiles = await scanDirectoryAsync(this.rootDir);
             if (process.env.LATTICE_SENSOR_SYNTH_TIMINGS)
                 console.error(`[phase-timing] sync-scan: ${Date.now() - tSyncScan}ms (${currentFiles.length} files)`);
             filesChecked = currentFiles.length;
+            // Full reconcile only (scoped syncs must not touch rows outside their
+            // scope): drop zero-node rows so the wiped files re-index as adds below.
+            this.healZeroNodeRows();
             const tTracked = Date.now();
             trackedFiles = this.queries.getAllFiles();
             if (process.env.LATTICE_SENSOR_SYNTH_TIMINGS)
@@ -2407,26 +2999,34 @@ class ExtractionOrchestrator {
         // `reconcileChecks` drives the cooperative yield shared with the adds/mods loop
         // below (see SYNC_RECONCILE_YIELD_INTERVAL / issue #905).
         let reconcileChecks = 0;
+        const removeTracked = (tracked) => {
+            // Before the cascade deletes them, resurrect incoming cross-file
+            // resolution edges as their original refs (#1240 removal case): the
+            // callers live in files this sync will NOT revisit, so this is their
+            // only chance to rebind to an alternative definition — or to park as
+            // failed until the symbol reappears somewhere. (A deleted file whose
+            // CALLERS are also being deleted is fine: their nodes cascade later
+            // in this loop and take the resurrected rows with them.)
+            // Every name this file defined is about to stop existing here, which
+            // narrows the candidate set for that name repo-wide (CG-33).
+            for (const pair of this.queries.getNodeNamePairsByFiles([tracked.path]))
+                pairsBefore.add(pair);
+            const incoming = this.queries.getCrossFileIncomingEdgesWithTarget(tracked.path);
+            if (incoming.length > 0) {
+                const resurrected = incoming
+                    .map((e) => resurrectRefFromDroppedEdge(e))
+                    .filter((r) => r !== null);
+                if (resurrected.length > 0) {
+                    this.queries.insertUnresolvedRefsBatch(resurrected);
+                }
+            }
+            onFileChange?.(tracked.path);
+            this.queries.deleteFile(tracked.path);
+            filesRemoved++;
+        };
         for (const tracked of trackedFiles) {
             if (!currentSet.has(tracked.path) || !fs.existsSync(path.join(this.rootDir, tracked.path))) {
-                // Before the cascade deletes them, resurrect incoming cross-file
-                // resolution edges as their original refs (#1240 removal case): the
-                // callers live in files this sync will NOT revisit, so this is their
-                // only chance to rebind to an alternative definition — or to park as
-                // failed until the symbol reappears somewhere. (A deleted file whose
-                // CALLERS are also being deleted is fine: their nodes cascade later
-                // in this loop and take the resurrected rows with them.)
-                const incoming = this.queries.getCrossFileIncomingEdgesWithTarget(tracked.path);
-                if (incoming.length > 0) {
-                    const resurrected = incoming
-                        .map((e) => resurrectRefFromDroppedEdge(e))
-                        .filter((r) => r !== null);
-                    if (resurrected.length > 0) {
-                        this.queries.insertUnresolvedRefsBatch(resurrected);
-                    }
-                }
-                this.queries.deleteFile(tracked.path);
-                filesRemoved++;
+                removeTracked(tracked);
             }
             if (++reconcileChecks % SYNC_RECONCILE_YIELD_INTERVAL === 0) {
                 await new Promise((resolve) => setImmediate(resolve));
@@ -2467,39 +3067,54 @@ class ExtractionOrchestrator {
                 }
                 catch (error) {
                     (0, errors_1.logDebug)('Skipping unstattable file during sync', { filePath, error: String(error) });
+                    failedFilePaths.push(filePath);
                     continue;
                 }
             }
             // New, or size/mtime changed — read + hash to confirm a real content change.
+            // (An oversize file hashes as its size stamp, unread — #1910.)
             let content;
             try {
-                content = fs.readFileSync(fullPath, 'utf-8');
+                content = readSourceOrStamp(fullPath);
             }
             catch (error) {
                 (0, errors_1.logDebug)('Skipping unreadable file during sync', { filePath, error: String(error) });
+                failedFilePaths.push(filePath);
+                continue;
+            }
+            // Not source after all — an MPEG transport stream named `.ts` (#1910):
+            // a new one is ignored, a tracked file that became one is removed.
+            if (content === null) {
+                if (tracked)
+                    removeTracked(tracked);
                 continue;
             }
             const contentHash = hashContent(content);
             if (!tracked) {
+                onFileChange?.(filePath, content);
                 filesToIndex.push(filePath);
                 changedFilePaths.push(filePath);
                 filesAdded++;
             }
             else if (tracked.contentHash !== contentHash) {
+                onFileChange?.(filePath, content);
                 filesToIndex.push(filePath);
                 changedFilePaths.push(filePath);
                 filesModified++;
             }
         }
+        // Sampled here — after the add/modify classification, before any file is
+        // re-extracted — because `storeExtractionResult` deletes a file's nodes
+        // before inserting the new ones, so this is the last point the pre-edit
+        // definition set is readable (CG-33).
+        if (filesToIndex.length > 0) {
+            for (const pair of this.queries.getNodeNamePairsByFiles(filesToIndex))
+                pairsBefore.add(pair);
+        }
         // Load only grammars needed for changed files
         if (filesToIndex.length > 0) {
             const overrides = (0, project_config_1.loadExtensionOverrides)(this.rootDir);
-            const neededLanguages = [...new Set(filesToIndex.map((f) => (0, grammars_1.detectLanguage)(f, undefined, overrides)))];
-            // .h files default to 'c' but may be C++ — ensure cpp grammar is loaded
-            if (neededLanguages.includes('c') && !neededLanguages.includes('cpp')) {
-                neededLanguages.push('cpp');
-            }
-            await (0, grammars_1.loadGrammarsForLanguages)(neededLanguages);
+            await (0, grammars_1.loadGrammarsForLanguages)(preloadLanguagesForFiles(filesToIndex, overrides));
         }
         // Index changed files
         const total = filesToIndex.length;
@@ -2512,7 +3127,12 @@ class ExtractionOrchestrator {
                 currentFile: filePath,
             });
             const result = await this.indexFile(filePath);
+            if (result.errors.some(e => e.severity === 'error'))
+                failedFilePaths.push(filePath);
             nodesUpdated += result.nodes.length;
+            const pause = backpressure?.();
+            if (pause)
+                await pause;
         }
         // The sync pass above re-extracted every stale-stamp file it saw. When no
         // stale rows remain, the whole index reflects current extraction semantics
@@ -2529,6 +3149,28 @@ class ExtractionOrchestrator {
             }
             catch { /* metadata is advisory — never fail a sync over it */ }
         }
+        // Names whose definition set this sync changed: a `file\0name` pair present
+        // before but not after (removed/renamed away) or after but not before
+        // (added). A pair on both sides is untouched as far as resolution's
+        // candidate set is concerned — only its node id moved, which
+        // reattachCrossFileEdges already follows — so an edit that only changes
+        // bodies yields an empty delta and no downstream rebind work (CG-33).
+        //
+        // Compared per FILE, not as one name set over the whole batch: a commit
+        // that adds `collect` to a new file while an unrelated changed file already
+        // defined `collect` must still flag the name, and a bare name set cancels
+        // exactly that case out. That miss left the largest residual class in the
+        // first measurement of this fix.
+        const pairsAfter = this.queries.getNodeNamePairsByFiles(filesToIndex);
+        const deltaNames = new Set();
+        const nameOf = (pair) => pair.slice(pair.indexOf('\0') + 1);
+        for (const pair of pairsBefore)
+            if (!pairsAfter.has(pair))
+                deltaNames.add(nameOf(pair));
+        for (const pair of pairsAfter)
+            if (!pairsBefore.has(pair))
+                deltaNames.add(nameOf(pair));
+        const definitionDelta = [...deltaNames];
         return {
             filesChecked,
             filesAdded,
@@ -2538,48 +3180,112 @@ class ExtractionOrchestrator {
             nodesUpdated,
             durationMs: Date.now() - startTime,
             changedFilePaths: changedFilePaths.length > 0 ? changedFilePaths : undefined,
+            ...(failedFilePaths.length > 0 ? { failedFilePaths } : {}),
+            definitionDelta: definitionDelta.length > 0 ? definitionDelta : undefined,
         };
     }
+    indexedDirtyPaths(stamp) {
+        try {
+            const state = JSON.parse(this.queries.getMetadata('indexed_dirty_paths') ?? 'null');
+            if (!state || state.commit !== (stamp ?? '') || !Array.isArray(state.paths))
+                return null;
+            if (!state.paths.every((p) => typeof p === 'string' && p.length > 0 &&
+                !path.isAbsolute(p) && !p.split('/').includes('..')))
+                return null;
+            return state.paths;
+        }
+        catch {
+            return null;
+        }
+    }
+    /** Capture before file reads. In-flight/failed full writes must not claim freshness. */
+    beginGitIndexState(full) {
+        const head = getGitHeadSha(this.rootDir) ?? '';
+        const stamp = this.queries.getMetadata(exports.INDEXED_AT_COMMIT_KEY) ?? '';
+        const prior = this.indexedDirtyPaths(stamp);
+        const status = getGitChangedFiles(this.rootDir);
+        const dirty = status ? [...new Set([
+                ...(full ? [] : prior ?? []), ...status.added, ...status.modified, ...status.deleted,
+            ])] : null;
+        if (full || prior === null || dirty === null) {
+            this.queries.setMetadata(exports.INDEXED_AT_COMMIT_KEY, '');
+            this.queries.setMetadata('indexed_dirty_paths', '');
+        }
+        else {
+            // Scoped writes leave the commit alone and retain dirty paths before the
+            // first write, so a crash cannot forget an indexed uncommitted edit.
+            this.queries.setMetadata('indexed_dirty_paths', JSON.stringify({ commit: stamp, paths: dirty }));
+        }
+        return { head, stamp, dirty };
+    }
+    finishGitIndexState(snapshot, full, retries = []) {
+        const after = getGitChangedFiles(this.rootDir);
+        if (!snapshot.dirty || !after)
+            return;
+        const commit = full ? snapshot.head : snapshot.stamp;
+        const paths = [...new Set([...snapshot.dirty, ...after.added, ...after.modified, ...after.deleted, ...retries])].sort();
+        // The embedded commit makes a torn pair fail closed: readers reject a dirty
+        // set that doesn't match the separately stored commit. Write the set first.
+        this.queries.setMetadata('indexed_dirty_paths', JSON.stringify({ commit, paths }));
+        if (full)
+            this.queries.setMetadata(exports.INDEXED_AT_COMMIT_KEY, commit);
+    }
     /**
      * Get files that have changed since last index.
      * Uses git status as a fast path when available, falling back to full scan.
      */
     getChangedFiles() {
-        const gitChanges = getGitChangedFiles(this.rootDir);
+        // The commit this index was last brought up to date at. Absent on an index
+        // built before stamping existed — getGitChangedFiles then declines the fast
+        // path and the full scan below answers correctly, once, until a sync or a
+        // full index writes the stamp. (#1829)
+        let sinceCommit = null;
+        try {
+            sinceCommit = this.queries.getMetadata(exports.INDEXED_AT_COMMIT_KEY) ?? null;
+        }
+        catch { /* advisory */ }
+        const dirtyPaths = this.indexedDirtyPaths(sinceCommit);
+        const gitChanges = dirtyPaths !== null && canTrustGitFastPath(this.rootDir, sinceCommit)
+            ? getGitChangedFiles(this.rootDir, sinceCommit)
+            : null;
         if (gitChanges) {
             // === Git fast path ===
             const added = [];
             const modified = [];
             const removed = [];
-            // Deleted files — only report if tracked in DB
-            for (const filePath of gitChanges.deleted) {
+            // Git supplies candidates, never the verdict. A committed deletion may
+            // have been recreated locally; a previously indexed dirty path may have
+            // vanished from git status after restore. Classify current disk vs DB once.
+            const candidates = new Set([...gitChanges.deleted, ...gitChanges.modified, ...gitChanges.added, ...dirtyPaths]);
+            const scope = this.scopedSyncMatcher();
+            const overrides = (0, project_config_1.loadExtensionOverrides)(this.rootDir);
+            for (const filePath of candidates) {
                 const tracked = this.queries.getFileByPath(filePath);
-                if (tracked) {
-                    removed.push(filePath);
-                }
-            }
-            // Modified + added files — read + hash, compare with DB. Untracked (`??`)
-            // files stay untracked in git even after indexing, so they must be
-            // hash-compared like modified files instead of always counting as added —
-            // otherwise status reports them as pending forever. (See issue #206.)
-            for (const filePath of [...gitChanges.modified, ...gitChanges.added]) {
                 const fullPath = path.join(this.rootDir, filePath);
+                if (!(0, grammars_1.isSourceFile)(filePath, overrides) || scope.ignores(filePath) || !fs.existsSync(fullPath)) {
+                    if (tracked)
+                        removed.push(filePath);
+                    continue;
+                }
                 let content;
                 try {
-                    content = fs.readFileSync(fullPath, 'utf-8');
+                    content = readSourceOrStamp(fullPath);
                 }
                 catch (error) {
                     (0, errors_1.logDebug)('Skipping unreadable file while detecting changes', { filePath, error: String(error) });
                     continue;
                 }
-                const contentHash = hashContent(content);
-                const tracked = this.queries.getFileByPath(filePath);
-                if (!tracked) {
-                    added.push(filePath);
+                // A `.ts` that is an MPEG transport stream is not source (#1910), so an
+                // untracked clip is never pending and a tracked file that became one is gone.
+                if (content === null) {
+                    if (tracked)
+                        removed.push(filePath);
+                    continue;
                 }
-                else if (tracked.contentHash !== contentHash) {
+                if (!tracked)
+                    added.push(filePath);
+                else if (tracked.contentHash !== hashContent(content))
                     modified.push(filePath);
-                }
             }
             // Rows written by a different extractor are pending even though git sees
             // no change — the observer moved, not the file. Without this, the git
@@ -2613,14 +3319,20 @@ class ExtractionOrchestrator {
             const fullPath = path.join(this.rootDir, filePath);
             let content;
             try {
-                content = fs.readFileSync(fullPath, 'utf-8');
+                content = readSourceOrStamp(fullPath);
             }
             catch (error) {
                 (0, errors_1.logDebug)('Skipping unreadable file while detecting changes', { filePath, error: String(error) });
                 continue;
             }
-            const contentHash = hashContent(content);
             const tracked = trackedMap.get(filePath);
+            // An MPEG transport stream named `.ts` (#1910) is not source.
+            if (content === null) {
+                if (tracked)
+                    removed.push(filePath);
+                continue;
+            }
+            const contentHash = hashContent(content);
             if (!tracked) {
                 added.push(filePath);
             }