@mnemonik/shared 6.48.0 → 6.51.0

package/dist/codeScanner.js

@@ -5,9 +5,11 @@ import { readdir, readFile, stat, lstat, realpath } from 'fs/promises';
 import { join, relative, extname, sep } from 'path';
 import { createHash } from 'crypto';
 import ignore from 'ignore';
-import { debug as logDebug, warn as logWarn } from './logger.js';
+import { debug as logDebug, info as logInfo, warn as logWarn } from './logger.js';
 import { withTimeout } from './asyncUtils.js';
 import { scrubSecrets } from './secretPatterns.js';
+import { MAX_AST_PARSE_BYTES, chunkWithAst } from './ast/astChunker.js';
+import { astArtifactReport, resolveAstLanguage } from './ast/grammars.js';
 /**
  * File operation timeout (5 seconds) to prevent hanging on slow/unresponsive filesystems
  */
@@ -150,6 +152,9 @@ const BUILT_IN_IGNORE_FILE_PATTERNS = [
  *   real, common SOURCE directory name (gRPC, auth libraries) and
  *   `shouldIgnore` cannot tell a file from a directory at its call sites —
  *   an unqualified rule would silently delete that subtree from the index.
+ * - `.tfvars` is excluded whole rather than per-basename: the credential file
+ *   is conventionally `terraform.tfvars` but the name is free-form, and the
+ *   variables that are NOT secret are visible in `.tf` anyway.
  */
 const SECRET_FILE_PLACEHOLDER_BASENAMES = new Set(['.env.example']);
 const SECRET_FILE_BASENAME_PATTERNS = [
@@ -158,6 +163,12 @@ const SECRET_FILE_BASENAME_PATTERNS = [
     /^\.(?:npmrc|netrc|pgpass)$/i, // registry / ftp / postgres password files
     /^kubeconfig$/i,
     /\.(?:pem|key|p12|pfx|keystore|jks|kubeconfig)$/i,
+    // Terraform variable files: `terraform.tfvars` is the conventional home for
+    // provider credentials, and unlike the entries above nothing else was
+    // keeping it out — it is excluded here rather than merely left off
+    // `DEFAULT_INCLUDE_EXTENSIONS`, so a future widening of the allowlist cannot
+    // quietly opt a project into embedding its own cloud keys.
+    /\.tfvars(\.json)?$/i,
 ];
 /**
  * True when `relPath` names a file whose contents are credentials. Applied
@@ -184,38 +195,264 @@ export function isSecretFile(relPath) {
  * list (e.g. `secrets/`). Matched hierarchically via the `ignore` package.
  */
 const IGNORE_FILE_NAMES = ['.gitignore', '.mnemonikignore'];
+/**
+ * Every file type the scanner will chunk, by extension. Matched
+ * case-INSENSITIVELY (see `isIncludedExtension`), so lowercase spellings here
+ * also cover `.R`, `.SQL`, `.PS1` and the uppercase `.C`/`.H` of older trees.
+ *
+ * Exported because this list IS the product's language coverage: a project
+ * written in something absent from it indexes zero code, and `code_search`
+ * reports that as "no indexed matches" rather than "I do not read this
+ * language" — invisible to the developer and to us. Coverage is a separate
+ * dial from parse quality: a language with no structured extractor falls
+ * through to `chunkRaw`, which is what Go, Java and C already get in
+ * production, and crude chunks beat no chunks by an enormous margin.
+ *
+ * Data formats (`.json`, `.yaml`, `.toml`, `.lock`) are deliberately absent:
+ * the ones that carry meaning are already collected verbatim by
+ * `AUTHORITY_FILE_MATCHERS`, and blanket-indexing the extension would pull in
+ * lockfiles and generated output. `.tfvars` is absent for a different reason —
+ * `terraform.tfvars` is a conventional home for provider credentials and
+ * `isSecretFile` now excludes it outright.
+ */
+export const DEFAULT_INCLUDE_EXTENSIONS = [
+    '.ts',
+    '.tsx',
+    '.js',
+    '.jsx',
+    '.py',
+    '.rs',
+    '.go',
+    '.java',
+    '.c',
+    '.cpp',
+    '.h',
+    '.cs',
+    '.rb',
+    '.php',
+    '.swift',
+    '.kt',
+    // Shell and Lua are the working languages of whole real projects —
+    // deployment tooling, container entrypoints, imapfilter/nginx/redis
+    // configuration. Omitting them meant such a project indexed ZERO code and
+    // code_search could not answer anything about it, while reporting that as
+    // "no indexed matches" rather than as missing coverage.
+    '.sh',
+    '.bash',
+    '.zsh',
+    '.lua',
+    '.md',
+    // ── Variants of languages already on the list. `.mjs`/`.cjs`/`.mts`/`.cts`/
+    // `.pyi` inherit their language string from a listed extension, so their
+    // absence was oversight rather than policy. The C++ spellings are a choice,
+    // not an inheritance: `.h` stays 'c' (C headers dominate, and reading a C++
+    // header as C is the safer default), while the unambiguously-C++ spellings
+    // resolve to 'cpp'.
+    '.mjs',
+    '.cjs',
+    '.mts',
+    '.cts',
+    '.pyi',
+    '.hpp',
+    '.hh',
+    '.hxx',
+    '.cc',
+    '.cxx',
+    '.kts',
+    // ── Languages the scanner could not read at all.
+    '.dart',
+    '.m',
+    '.mm',
+    '.scala',
+    '.sc',
+    '.ex',
+    '.exs',
+    '.erl',
+    '.hrl',
+    '.hs',
+    '.jl',
+    '.ml',
+    '.mli',
+    '.clj',
+    '.cljs',
+    '.cljc',
+    '.groovy',
+    '.gradle',
+    '.ps1',
+    '.psm1',
+    '.pl',
+    '.pm',
+    '.r',
+    '.sol',
+    '.zig',
+    '.vue',
+    '.svelte',
+    // ── Infrastructure and schema DSLs, where real logic lives and where "how
+    // is this deployed?" and "what does this table hold?" go unanswered today.
+    '.tf',
+    '.sql',
+    '.proto',
+    '.graphql',
+    '.gql',
+    '.cmake',
+    '.nix',
+    '.bzl',
+];
+/**
+ * Extension → language string, the value carried on every chunk and on the
+ * wire (`/scan/push` accepts any non-empty string up to 50 chars).
+ *
+ * A string no grammar claims is correct and expected: the AST layer resolves
+ * such a language to `null` and the heuristic chunker takes over. Keep this in
+ * sync with `DEFAULT_INCLUDE_EXTENSIONS` — an allowlisted extension that lands
+ * on 'unknown' still gets chunked, but nothing downstream can reason about it.
+ *
+ * Two extensions are genuinely ambiguous and are resolved rather than fudged:
+ * `.m` is Objective-C here, not MATLAB, because a repo carrying `.m` alongside
+ * `.h`/`.mm` is overwhelmingly an Apple-platform project; `.pl` is Perl, not
+ * Prolog, on the same frequency argument.
+ */
+const EXTENSION_LANGUAGES = {
+    '.ts': 'typescript',
+    '.tsx': 'typescript',
+    '.mts': 'typescript',
+    '.cts': 'typescript',
+    '.js': 'javascript',
+    '.jsx': 'javascript',
+    '.mjs': 'javascript',
+    '.cjs': 'javascript',
+    '.py': 'python',
+    '.pyi': 'python',
+    '.rs': 'rust',
+    '.go': 'go',
+    '.java': 'java',
+    '.c': 'c',
+    '.h': 'c',
+    '.cpp': 'cpp',
+    '.cc': 'cpp',
+    '.cxx': 'cpp',
+    '.hpp': 'cpp',
+    '.hh': 'cpp',
+    '.hxx': 'cpp',
+    '.cs': 'csharp',
+    '.rb': 'ruby',
+    '.php': 'php',
+    '.swift': 'swift',
+    '.kt': 'kotlin',
+    '.kts': 'kotlin',
+    '.sh': 'shell',
+    '.bash': 'shell',
+    '.zsh': 'shell',
+    '.lua': 'lua',
+    '.md': 'markdown',
+    '.dart': 'dart',
+    '.m': 'objc',
+    '.mm': 'objc',
+    '.scala': 'scala',
+    '.sc': 'scala',
+    '.ex': 'elixir',
+    '.exs': 'elixir',
+    '.erl': 'erlang',
+    '.hrl': 'erlang',
+    '.hs': 'haskell',
+    '.jl': 'julia',
+    '.ml': 'ocaml',
+    '.mli': 'ocaml',
+    '.clj': 'clojure',
+    '.cljs': 'clojure',
+    '.cljc': 'clojure',
+    '.groovy': 'groovy',
+    '.gradle': 'groovy',
+    '.ps1': 'powershell',
+    '.psm1': 'powershell',
+    '.pl': 'perl',
+    '.pm': 'perl',
+    '.r': 'r',
+    '.sol': 'solidity',
+    '.zig': 'zig',
+    '.vue': 'vue',
+    '.svelte': 'svelte',
+    '.tf': 'terraform',
+    '.sql': 'sql',
+    '.proto': 'protobuf',
+    '.graphql': 'graphql',
+    '.gql': 'graphql',
+    '.cmake': 'cmake',
+    '.nix': 'nix',
+    '.bzl': 'starlark',
+};
+/**
+ * Language string for a file path or a bare extension, `'unknown'` when the
+ * extension is unmapped. A free function rather than a method because callers
+ * that never scan anything (AST grammar resolution, server-side symbol
+ * preference) need the same answer without constructing a scanner.
+ */
+export function languageForExtension(filePathOrExt) {
+    // `extname` FIRST. A dotfile that carries a real extension ('.eslintrc.js',
+    // '.mocharc.cjs', '.prettierrc.ts') starts with '.' and contains no '/', so
+    // a bare-extension-first reading swallowed the whole name and answered
+    // 'unknown' — while the same file spelled 'src/.eslintrc.js' answered
+    // 'javascript'. Those extensions are allowlisted, so the files are indexed
+    // and reach the server with exactly the root-relative spelling that failed.
+    const fromPath = EXTENSION_LANGUAGES[extname(filePathOrExt).toLowerCase()];
+    if (fromPath !== undefined)
+        return fromPath;
+    // Fallback: the argument IS the extension ('.dart'), for which `extname`
+    // returns ''. Every key contains a leading dot and no separator, so a real
+    // path can never collide here.
+    return EXTENSION_LANGUAGES[filePathOrExt.toLowerCase()] ?? 'unknown';
+}
+/**
+ * THE definition of "this path is a SQL migration the schema_columns authority
+ * collects verbatim". One predicate, referenced by both halves of the deal —
+ * `AUTHORITY_FILE_MATCHERS` (collect it) and `isAuthorityOnlyPath` (therefore
+ * do not chunk it) — because two hand-written regexes drifted once already and
+ * the failure is silent in both directions.
+ *
+ * Root-anchored and CASE-SENSITIVE on purpose: it mirrors the server-side
+ * extractor, which does `listFiles('migrations/')` (LIKE 'migrations/%') then
+ * `endsWith('.sql')`, both case-sensitive. `Migrations/001.sql` (the .NET/EF
+ * Core spelling) and `migrations/002.SQL` are NOT collected, so they must not
+ * be suppressed from chunking either — that would index them nowhere. Same
+ * reason a nested `packages/x/migrations/y.sql` is left alone.
+ */
+const isMigrationSqlAuthorityPath = (posixRelPath) => /^migrations\/.*\.sql$/.test(posixRelPath);
+/**
+ * Predicates for paths whose verbatim content is ALREADY shipped by
+ * `collectAuthorityFiles` and that carry no additional value as embedded code
+ * chunks. Checked at the extension gate rather than in `shouldIgnore`, because
+ * `shouldIgnore` also guards the authority walk and must keep letting these
+ * through.
+ *
+ * `migrations/**.sql` is the live case: adding `.sql` to the allowlist without
+ * this exclusion would dual-collect every migration — once verbatim, once
+ * chunked and embedded. On this repo alone that is 208 files of append-only
+ * DDL (140 forward, the rest rollback/manual), embedded to answer questions
+ * the authority path already answers exactly.
+ *
+ * INVARIANT: every predicate here must also appear in
+ * `AUTHORITY_FILE_MATCHERS`, so no path can be excluded from chunking unless
+ * the authority path definitely collects it.
+ */
+const AUTHORITY_ONLY_PATH_PREDICATES = [
+    isMigrationSqlAuthorityPath,
+];
+/**
+ * True when `relPath` is collected verbatim as authority content and must not
+ * additionally be chunked. Accepts OS-native or POSIX separators.
+ */
+export function isAuthorityOnlyPath(relPath) {
+    if (!relPath)
+        return false;
+    const posix = relPath.split(sep).join('/');
+    return AUTHORITY_ONLY_PATH_PREDICATES.some((matches) => matches(posix));
+}
 const DEFAULT_OPTIONS = {
     maxChunkSize: 8000, // ~2000 tokens
     minChunkSize: 100,
     ignorePatterns: [...BUILT_IN_IGNORE_DIRS, ...BUILT_IN_IGNORE_FILE_PATTERNS],
-    includeExtensions: [
-        '.ts',
-        '.tsx',
-        '.js',
-        '.jsx',
-        '.py',
-        '.rs',
-        '.go',
-        '.java',
-        '.c',
-        '.cpp',
-        '.h',
-        '.cs',
-        '.rb',
-        '.php',
-        '.swift',
-        '.kt',
-        // Shell and Lua are the working languages of whole real projects —
-        // deployment tooling, container entrypoints, imapfilter/nginx/redis
-        // configuration. Omitting them meant such a project indexed ZERO code and
-        // code_search could not answer anything about it, while reporting that as
-        // "no indexed matches" rather than as missing coverage.
-        '.sh',
-        '.bash',
-        '.zsh',
-        '.lua',
-        '.md',
-    ],
+    includeExtensions: [...DEFAULT_INCLUDE_EXTENSIONS],
+    maxAstParseBytes: MAX_AST_PARSE_BYTES,
 };
 /**
  * Matchers for authority manifest / config / CI files whose verbatim content
@@ -237,8 +474,10 @@ export const AUTHORITY_FILE_MATCHERS = [
     // SQL migrations: the schema_columns authority extractor reads every .sql
     // under `migrations/` (listFiles('migrations/') -> LIKE 'migrations/%' then
     // .endsWith('.sql')). Without collecting these, that authority is empty and
-    // every schema_table_enumeration claim falls to unverifiable.
-    (p) => /^migrations\/.*\.sql$/.test(p),
+    // every schema_table_enumeration claim falls to unverifiable. Shared with
+    // `AUTHORITY_ONLY_PATH_PREDICATES` by reference, not by a copied regex, so
+    // collection and the chunking exclusion cannot disagree.
+    isMigrationSqlAuthorityPath,
 ];
 /**
  * Segment-anchored match for the `tests/fixtures/` ignore pattern above —
@@ -307,10 +546,90 @@ export async function isGitBoundary(dirPath) {
         return false;
     }
 }
+/**
+ * Latch for `logAstCapabilityOnce`. A promise, not a boolean: two concurrent
+ * callers must both wait on the same report rather than the second returning
+ * before the first has logged.
+ */
+let astCapabilityLog = null;
+/**
+ * `reason:grammar` pairs already warned about, for the reasons that are
+ * process-global rather than file-specific.
+ *
+ * Only `grammar_unavailable` qualifies: `loadGrammar` caches its failure, so the
+ * answer is identical for every file of that language and warning per file
+ * printed 1,037 lines in one scan of this repo. `file_too_large` and
+ * `parse_failed` are properties of one file and stay per file.
+ */
+const reportedGrammarFallbacks = new Set();
+/**
+ * Log which grammars this install ships — ONCE per process, at daemon start,
+ * never per file.
+ *
+ * A missing grammar silently degrades every file of that language to the
+ * heuristic chunker. Per-file warnings would say so 30,000 times and drown the
+ * log; saying nothing is how a half-broken install looks healthy. One startup
+ * line naming the vendored grammars, and a WARNING when an artifact is missing,
+ * is the whole contract.
+ *
+ * Deliberately a `statSync` of the artifacts (`astArtifactReport`) and not a
+ * load of them (`astCapabilityReport`). Loading all 18 to print this line costs
+ * ~690 ms and ~75 MB of RSS that is never returned — web-tree-sitter exposes no
+ * `Language.delete` — which is a permanent tax on a daemon watching a pure
+ * TypeScript repo, paid to pre-answer a question about seventeen languages it
+ * will never see. Grammars load lazily instead, on the first file of a language,
+ * and the two failure modes only a load can detect (`wasm_load_failed`,
+ * `query_compile_failed`) are warned there, once per language, by the
+ * `grammar_unavailable` branch in `chunkFile`.
+ *
+ * Cheap to call repeatedly: this latches, and the report instantiates nothing.
+ */
+export function logAstCapabilityOnce() {
+    astCapabilityLog ??= (async () => {
+        const report = astArtifactReport();
+        const detail = {
+            grammars: report.vendored.length,
+            languages: report.vendored.join(' '),
+        };
+        if (report.missing.length > 0) {
+            logWarn('AST chunking: some vendored grammar artifacts are missing', {
+                ...detail,
+                missing: report.missing.map((m) => `${m.id}(${m.reason}: ${m.detail})`).join('; '),
+            });
+        }
+        else {
+            logInfo('AST chunking ready (grammars load lazily, per language)', detail);
+        }
+    })();
+    return astCapabilityLog;
+}
 export class CodeScanner {
     options;
+    /**
+     * `includeExtensions` folded to lowercase for matching. The gate used to
+     * compare `extname()` verbatim while `detectLanguage` lowercased, so a
+     * project spelling its files the canonical way — `.R` for R, `.SQL`/`.PS1`
+     * on Windows, `.C`/`.H` in older C trees — indexed zero of them and nothing
+     * said why.
+     */
+    includeExtensionSet;
     constructor(options = {}) {
         this.options = { ...DEFAULT_OPTIONS, ...options };
+        this.includeExtensionSet = new Set(this.options.includeExtensions.map((ext) => ext.toLowerCase()));
+    }
+    /**
+     * The chunkable-file gate, shared by every walker and by the explicit
+     * file-list path so all three agree on what exists. `relPath` is the path
+     * relative to the scan root (OS-native separators accepted).
+     */
+    isChunkable(absOrRelPath, relPath) {
+        if (!this.includeExtensionSet.has(extname(absOrRelPath).toLowerCase()))
+            return false;
+        if (isAuthorityOnlyPath(relPath)) {
+            logDebug('Skipping chunking for authority-collected path', { relPath });
+            return false;
+        }
+        return true;
     }
     /**
      * Read `.gitignore` + `.mnemonikignore` in `absDir` and compile them into one
@@ -538,7 +857,7 @@ export class CodeScanner {
                 else if (stats.isFile()) {
                     if (this.ignoredByStack(relativePath, false, localStack))
                         continue;
-                    if (this.options.includeExtensions.includes(extname(fullPath))) {
+                    if (this.isChunkable(fullPath, relativePath)) {
                         out.push(relativePath);
                     }
                 }
@@ -579,8 +898,7 @@ export class CodeScanner {
                 if (rootPath && (await this.insideNestedGitBoundary(filePath, rootPath))) {
                     continue;
                 }
-                const ext = extname(filePath);
-                if (this.options.includeExtensions.includes(ext)) {
+                if (this.isChunkable(filePath, fileRel)) {
                     const fileChunks = await this.parseFile(filePath, rootPath || filePath);
                     chunks.push(...fileChunks);
                 }
@@ -666,8 +984,7 @@ export class CodeScanner {
                 else if (stats.isFile()) {
                     if (this.ignoredByStack(relativePath, false, localStack))
                         continue;
-                    const ext = extname(fullPath);
-                    if (this.options.includeExtensions.includes(ext)) {
+                    if (this.isChunkable(fullPath, relativePath)) {
                         const fileChunks = await this.parseFile(fullPath, rootPath);
                         chunks.push(...fileChunks);
                     }
@@ -755,12 +1072,130 @@ export class CodeScanner {
             if (language === 'markdown') {
                 return this.chunkMarkdown(content, relativePath, stats.size);
             }
-            const structuredChunks = this.extractStructuredChunks(content, language);
             const fileMetadata = {
                 fileName: filePath.split('/').pop() || '',
                 extension: extname(filePath),
                 size: stats.size,
             };
+            // AST first for the languages with a vendored grammar. `null` means "no
+            // grammar for this" — the honest majority of the allowlist — and the
+            // heuristic path below takes over with no log, because that is not a
+            // degradation.
+            const astLanguage = resolveAstLanguage(fileMetadata.extension, language);
+            if (astLanguage) {
+                const ast = await chunkWithAst(content, astLanguage, {
+                    maxParseBytes: this.options.maxAstParseBytes,
+                });
+                if ('chunks' in ast) {
+                    // A definition the chunker could not place reaches no chunk, so its
+                    // name reaches no index and a citation to it resolves as
+                    // `unresolved_symbol`. It cannot be emitted (two chunks over one line
+                    // range collide on the `filePath:startLine-endLine` staleness key), but
+                    // a scan that quietly loses symbols must not look like a healthy one.
+                    if (ast.droppedDefinitions > 0) {
+                        logWarn('AST chunker dropped definitions that share a line range', {
+                            filePath: relativePath,
+                            grammar: astLanguage,
+                            droppedDefinitions: ast.droppedDefinitions,
+                        });
+                    }
+                    // `errorNodes` is handed back precisely so this degradation is not
+                    // silent, and discarding it made it silent. One ERROR node can swallow
+                    // the rest of a file — `typeof import(...)` in a type argument does
+                    // exactly that to tree-sitter-typescript — after which no definition is
+                    // captured and the file arrives as one unnamed whole-file chunk. Naming
+                    // the file, the error count and how many definitions survived is what
+                    // makes that visible in a scan log.
+                    //
+                    // Deliberately NOT a fallback to the heuristic path: measured on the
+                    // real construct (tests/AgentInjectionDeliveryWiring.test.ts), the
+                    // heuristic path names nothing either (its name pattern needs a
+                    // leading `function`/`class`/`const`) AND covers less — it emitted 3
+                    // chunks starting at line 18 and dropped lines 1-17, because the
+                    // uncovered-region supplement only fires above 50 lines. Trading
+                    // total coverage for no gain is worse than one honest blob, so the
+                    // AST result stands and the log says so.
+                    if (ast.errorNodes > 0) {
+                        logWarn('AST parse errors; extents and symbol names degraded for this file', {
+                            filePath: relativePath,
+                            grammar: astLanguage,
+                            errorNodes: ast.errorNodes,
+                            definitions: ast.chunks.filter((chunk) => chunk.symbolKind).length,
+                            chunks: ast.chunks.length,
+                        });
+                    }
+                    // The wire cap (`scanChunkSchema` allows 500,000 chars) and the
+                    // embedding cap (8191 tokens per input) are both downstream of
+                    // `maxChunkSize`, and the AST chunker only bounds its uncovered spans:
+                    // a definition chunk is whatever the definition is. `boundAstChunks`
+                    // makes the cap unreachable by construction, or answers null when the
+                    // file cannot be bounded on line boundaries at all.
+                    const bounded = this.boundAstChunks(ast.chunks);
+                    if (bounded) {
+                        // Zero chunks here means the file has no non-whitespace line — the AST
+                        // chunker covers every other line by construction — so returning
+                        // nothing is the correct answer, not a lost file.
+                        return bounded.map(({ symbolName, symbolKind, signature, ...chunk }) => ({
+                            ...chunk,
+                            filePath: relativePath,
+                            language,
+                            contentHash: this.hash(chunk.content),
+                            metadata: {
+                                ...fileMetadata,
+                                ...(signature ? { signature } : {}),
+                                ...(symbolName ? { symbolName } : {}),
+                                ...(symbolKind ? { symbolKind } : {}),
+                            },
+                        }));
+                    }
+                    // One line longer than `maxChunkSize` — a generated `*_pb2.py` carries
+                    // the whole serialized descriptor on one. Splitting it would have to
+                    // cut mid-line, and two chunks over one line range collide on the
+                    // `filePath:startLine-endLine` staleness key, so the whole file goes to
+                    // the heuristic chunker, which force-splits long lines by character
+                    // count. Loud, because a file that loses its symbol names must not look
+                    // like a healthy one.
+                    logWarn('AST chunk exceeds maxChunkSize on a single line; heuristic chunking instead', {
+                        filePath: relativePath,
+                        grammar: astLanguage,
+                        maxChunkSize: this.options.maxChunkSize,
+                        longestLine: content
+                            .split('\n')
+                            .reduce((longest, line) => Math.max(longest, line.length), 0),
+                    });
+                }
+                else if (ast.unsupported === 'grammar_unavailable') {
+                    // Process-global, not per file: `loadGrammar` caches its failures, so
+                    // every file of this language degrades for the same reason. Warned per
+                    // file, this printed 1,037 identical lines in one scan of this repo —
+                    // the log flood `logAstCapabilityOnce` exists to avoid. Once per
+                    // (reason, grammar); the file-specific reasons below stay per file.
+                    const key = `${ast.unsupported}:${astLanguage}`;
+                    if (!reportedGrammarFallbacks.has(key)) {
+                        reportedGrammarFallbacks.add(key);
+                        logWarn('AST chunking unavailable for a whole language; heuristic chunking instead', {
+                            grammar: astLanguage,
+                            reason: ast.unsupported,
+                            detail: ast.detail,
+                            firstFile: relativePath,
+                            note: 'logged once per grammar per process; every file of this language degrades',
+                        });
+                    }
+                }
+                else {
+                    // Not a silent no-op: name the file AND the reason, then degrade to the
+                    // heuristic path. A degraded scan indistinguishable from a healthy one
+                    // is the defect class this routing exists to avoid — and degrading to
+                    // zero chunks would delete the file from the index instead.
+                    logWarn('AST chunking unavailable; falling back to heuristic chunking', {
+                        filePath: relativePath,
+                        grammar: astLanguage,
+                        reason: ast.unsupported,
+                        detail: ast.detail,
+                    });
+                }
+            }
+            const structuredChunks = this.extractStructuredChunks(content, language);
             if (structuredChunks.length > 0) {
                 const mapped = structuredChunks.map(({ signature, symbolName, ...chunk }) => ({
                     ...chunk,
@@ -806,35 +1241,90 @@ export class CodeScanner {
             return [];
         }
     }
+    /**
+     * Hold every AST chunk at or under `maxChunkSize`, or answer `null` when this
+     * file cannot be held there on line boundaries.
+     *
+     * The AST chunker bounds only the spans it invents (`MAX_RAW_SPAN_CHARS`); a
+     * chunk that came from a definition is exactly as big as the definition, and a
+     * span that is one enormous line is left whole on purpose. Both used to reach
+     * the wire verbatim, where `scanChunkSchema` caps content at 500,000 chars and
+     * rejects the WHOLE push if any chunk is over — so one generated file stopped
+     * all scanning for that push — and where anything past 8191 tokens is only
+     * partially embedded.
+     *
+     * The invariants, all of which the split preserves:
+     * - content stays verbatim contiguous source: `lines[startLine-1..endLine-1]`
+     *   joined by '\n', never a join of disjoint regions and never an elision;
+     * - total coverage is unchanged: the pieces tile the original range, in order,
+     *   with no gap and no overlap;
+     * - the definition's identity rides on the piece that carries its signature —
+     *   the first one — and the continuations are plain `raw` spans, because a
+     *   continuation is not the definition and must not claim its name.
+     *
+     * `null` (the caller degrades the whole file to the heuristic chunker) is
+     * reserved for the one shape a line-boundary split cannot fix: a single line
+     * longer than the cap, as every generated `*_pb2.py` has. Cutting mid-line
+     * would give two chunks the same `filePath:startLine-endLine` staleness key,
+     * which is the collision the chunker refuses everywhere else.
+     */
+    boundAstChunks(chunks) {
+        const max = this.options.maxChunkSize;
+        if (chunks.every((chunk) => chunk.content.length <= max))
+            return [...chunks];
+        const bounded = [];
+        for (const chunk of chunks) {
+            if (chunk.content.length <= max) {
+                bounded.push(chunk);
+                continue;
+            }
+            // `content` is exactly `startLine..endLine` joined by '\n', so this split
+            // recovers the file's own lines for that range.
+            const lines = chunk.content.split('\n');
+            if (lines.some((line) => line.length > max))
+                return null;
+            /** `from`/`to` are 0-based offsets into `lines`, inclusive. */
+            const emit = (from, to) => {
+                const content = lines.slice(from, to + 1).join('\n');
+                // A piece of nothing but blank lines is owed no chunk (the chunker's own
+                // rule) and `scanChunkSchema` requires content.min(1) anyway.
+                if (content.trim() === '')
+                    return;
+                const isFirst = from === 0;
+                bounded.push({
+                    content,
+                    startLine: chunk.startLine + from,
+                    endLine: chunk.startLine + to,
+                    chunkType: isFirst ? chunk.chunkType : 'raw',
+                    ...(isFirst && chunk.symbolName ? { symbolName: chunk.symbolName } : {}),
+                    ...(isFirst && chunk.symbolKind ? { symbolKind: chunk.symbolKind } : {}),
+                    ...(isFirst && chunk.signature ? { signature: chunk.signature } : {}),
+                });
+            };
+            let pieceStart = 0;
+            let pieceLength = 0;
+            for (let i = 0; i < lines.length; i++) {
+                const lineLength = (lines[i] ?? '').length;
+                // The '\n' the join will put back is part of what has to fit.
+                const withLine = i === pieceStart ? lineLength : pieceLength + 1 + lineLength;
+                if (i > pieceStart && withLine > max) {
+                    emit(pieceStart, i - 1);
+                    pieceStart = i;
+                    pieceLength = lineLength;
+                }
+                else {
+                    pieceLength = withLine;
+                }
+            }
+            emit(pieceStart, lines.length - 1);
+        }
+        return bounded;
+    }
     /**
      * Detect language from file extension
      */
     detectLanguage(filePath) {
-        const ext = extname(filePath).toLowerCase();
-        const langMap = {
-            '.ts': 'typescript',
-            '.tsx': 'typescript',
-            '.js': 'javascript',
-            '.jsx': 'javascript',
-            '.py': 'python',
-            '.rs': 'rust',
-            '.go': 'go',
-            '.java': 'java',
-            '.c': 'c',
-            '.cpp': 'cpp',
-            '.h': 'c',
-            '.cs': 'csharp',
-            '.rb': 'ruby',
-            '.php': 'php',
-            '.swift': 'swift',
-            '.kt': 'kotlin',
-            '.sh': 'shell',
-            '.bash': 'shell',
-            '.zsh': 'shell',
-            '.lua': 'lua',
-            '.md': 'markdown',
-        };
-        return langMap[ext] || 'unknown';
+        return languageForExtension(filePath);
     }
     /**
      * Chunk markdown files by headers