@mnemonik/shared 6.48.0 → 6.51.0

package/src/codeScanner.ts

@@ -6,9 +6,11 @@ import { readdir, readFile, stat, lstat, realpath } from 'fs/promises';
 import { join, relative, extname, sep } from 'path';
 import { createHash } from 'crypto';
 import ignore, { type Ignore } from 'ignore';
-import { debug as logDebug, warn as logWarn } from './logger.js';
+import { debug as logDebug, info as logInfo, warn as logWarn } from './logger.js';
 import { withTimeout } from './asyncUtils.js';
 import { scrubSecrets } from './secretPatterns.js';
+import { MAX_AST_PARSE_BYTES, chunkWithAst, type AstChunk } from './ast/astChunker.js';
+import { astArtifactReport, resolveAstLanguage } from './ast/grammars.js';
 
 /**
  * File operation timeout (5 seconds) to prevent hanging on slow/unresponsive filesystems
@@ -51,6 +53,11 @@ export interface CodeChunk {
     size: number;
     signature?: string; // Function/class signature (e.g. "function foo(bar: string): number")
     symbolName?: string; // Symbol name (e.g. "foo")
+    // The AST chunker's finer classification, verbatim from the tag capture
+    // ('method', 'interface', 'struct', ...). `chunkType` above is the coarse
+    // wire enum and cannot express it; only the AST path sets this, so its
+    // absence is exactly the signal "this chunk came from the heuristic path".
+    symbolKind?: string;
   };
 }
 
@@ -59,6 +66,13 @@ export interface ScanOptions {
   minChunkSize?: number;
   ignorePatterns?: string[];
   includeExtensions?: string[];
+  /**
+   * Byte ceiling above which a file is chunked heuristically instead of parsed.
+   * Defaults to `MAX_AST_PARSE_BYTES`; exposed so a caller (and the routing
+   * test) can drive the degradation path through production code rather than a
+   * mock.
+   */
+  maxAstParseBytes?: number;
 }
 
 /**
@@ -192,6 +206,9 @@ const BUILT_IN_IGNORE_FILE_PATTERNS: readonly string[] = [
  *   real, common SOURCE directory name (gRPC, auth libraries) and
  *   `shouldIgnore` cannot tell a file from a directory at its call sites —
  *   an unqualified rule would silently delete that subtree from the index.
+ * - `.tfvars` is excluded whole rather than per-basename: the credential file
+ *   is conventionally `terraform.tfvars` but the name is free-form, and the
+ *   variables that are NOT secret are visible in `.tf` anyway.
  */
 const SECRET_FILE_PLACEHOLDER_BASENAMES: ReadonlySet<string> = new Set(['.env.example']);
 
@@ -201,6 +218,12 @@ const SECRET_FILE_BASENAME_PATTERNS: readonly RegExp[] = [
   /^\.(?:npmrc|netrc|pgpass)$/i, // registry / ftp / postgres password files
   /^kubeconfig$/i,
   /\.(?:pem|key|p12|pfx|keystore|jks|kubeconfig)$/i,
+  // Terraform variable files: `terraform.tfvars` is the conventional home for
+  // provider credentials, and unlike the entries above nothing else was
+  // keeping it out — it is excluded here rather than merely left off
+  // `DEFAULT_INCLUDE_EXTENSIONS`, so a future widening of the allowlist cannot
+  // quietly opt a project into embedding its own cloud keys.
+  /\.tfvars(\.json)?$/i,
 ];
 
 /**
@@ -227,38 +250,272 @@ export function isSecretFile(relPath: string): boolean {
  */
 const IGNORE_FILE_NAMES = ['.gitignore', '.mnemonikignore'] as const;
 
+/**
+ * Every file type the scanner will chunk, by extension. Matched
+ * case-INSENSITIVELY (see `isIncludedExtension`), so lowercase spellings here
+ * also cover `.R`, `.SQL`, `.PS1` and the uppercase `.C`/`.H` of older trees.
+ *
+ * Exported because this list IS the product's language coverage: a project
+ * written in something absent from it indexes zero code, and `code_search`
+ * reports that as "no indexed matches" rather than "I do not read this
+ * language" — invisible to the developer and to us. Coverage is a separate
+ * dial from parse quality: a language with no structured extractor falls
+ * through to `chunkRaw`, which is what Go, Java and C already get in
+ * production, and crude chunks beat no chunks by an enormous margin.
+ *
+ * Data formats (`.json`, `.yaml`, `.toml`, `.lock`) are deliberately absent:
+ * the ones that carry meaning are already collected verbatim by
+ * `AUTHORITY_FILE_MATCHERS`, and blanket-indexing the extension would pull in
+ * lockfiles and generated output. `.tfvars` is absent for a different reason —
+ * `terraform.tfvars` is a conventional home for provider credentials and
+ * `isSecretFile` now excludes it outright.
+ */
+export const DEFAULT_INCLUDE_EXTENSIONS: readonly string[] = [
+  '.ts',
+  '.tsx',
+  '.js',
+  '.jsx',
+  '.py',
+  '.rs',
+  '.go',
+  '.java',
+  '.c',
+  '.cpp',
+  '.h',
+  '.cs',
+  '.rb',
+  '.php',
+  '.swift',
+  '.kt',
+  // Shell and Lua are the working languages of whole real projects —
+  // deployment tooling, container entrypoints, imapfilter/nginx/redis
+  // configuration. Omitting them meant such a project indexed ZERO code and
+  // code_search could not answer anything about it, while reporting that as
+  // "no indexed matches" rather than as missing coverage.
+  '.sh',
+  '.bash',
+  '.zsh',
+  '.lua',
+  '.md',
+
+  // ── Variants of languages already on the list. `.mjs`/`.cjs`/`.mts`/`.cts`/
+  // `.pyi` inherit their language string from a listed extension, so their
+  // absence was oversight rather than policy. The C++ spellings are a choice,
+  // not an inheritance: `.h` stays 'c' (C headers dominate, and reading a C++
+  // header as C is the safer default), while the unambiguously-C++ spellings
+  // resolve to 'cpp'.
+  '.mjs',
+  '.cjs',
+  '.mts',
+  '.cts',
+  '.pyi',
+  '.hpp',
+  '.hh',
+  '.hxx',
+  '.cc',
+  '.cxx',
+  '.kts',
+
+  // ── Languages the scanner could not read at all.
+  '.dart',
+  '.m',
+  '.mm',
+  '.scala',
+  '.sc',
+  '.ex',
+  '.exs',
+  '.erl',
+  '.hrl',
+  '.hs',
+  '.jl',
+  '.ml',
+  '.mli',
+  '.clj',
+  '.cljs',
+  '.cljc',
+  '.groovy',
+  '.gradle',
+  '.ps1',
+  '.psm1',
+  '.pl',
+  '.pm',
+  '.r',
+  '.sol',
+  '.zig',
+  '.vue',
+  '.svelte',
+
+  // ── Infrastructure and schema DSLs, where real logic lives and where "how
+  // is this deployed?" and "what does this table hold?" go unanswered today.
+  '.tf',
+  '.sql',
+  '.proto',
+  '.graphql',
+  '.gql',
+  '.cmake',
+  '.nix',
+  '.bzl',
+];
+
+/**
+ * Extension → language string, the value carried on every chunk and on the
+ * wire (`/scan/push` accepts any non-empty string up to 50 chars).
+ *
+ * A string no grammar claims is correct and expected: the AST layer resolves
+ * such a language to `null` and the heuristic chunker takes over. Keep this in
+ * sync with `DEFAULT_INCLUDE_EXTENSIONS` — an allowlisted extension that lands
+ * on 'unknown' still gets chunked, but nothing downstream can reason about it.
+ *
+ * Two extensions are genuinely ambiguous and are resolved rather than fudged:
+ * `.m` is Objective-C here, not MATLAB, because a repo carrying `.m` alongside
+ * `.h`/`.mm` is overwhelmingly an Apple-platform project; `.pl` is Perl, not
+ * Prolog, on the same frequency argument.
+ */
+const EXTENSION_LANGUAGES: Readonly<Record<string, string>> = {
+  '.ts': 'typescript',
+  '.tsx': 'typescript',
+  '.mts': 'typescript',
+  '.cts': 'typescript',
+  '.js': 'javascript',
+  '.jsx': 'javascript',
+  '.mjs': 'javascript',
+  '.cjs': 'javascript',
+  '.py': 'python',
+  '.pyi': 'python',
+  '.rs': 'rust',
+  '.go': 'go',
+  '.java': 'java',
+  '.c': 'c',
+  '.h': 'c',
+  '.cpp': 'cpp',
+  '.cc': 'cpp',
+  '.cxx': 'cpp',
+  '.hpp': 'cpp',
+  '.hh': 'cpp',
+  '.hxx': 'cpp',
+  '.cs': 'csharp',
+  '.rb': 'ruby',
+  '.php': 'php',
+  '.swift': 'swift',
+  '.kt': 'kotlin',
+  '.kts': 'kotlin',
+  '.sh': 'shell',
+  '.bash': 'shell',
+  '.zsh': 'shell',
+  '.lua': 'lua',
+  '.md': 'markdown',
+  '.dart': 'dart',
+  '.m': 'objc',
+  '.mm': 'objc',
+  '.scala': 'scala',
+  '.sc': 'scala',
+  '.ex': 'elixir',
+  '.exs': 'elixir',
+  '.erl': 'erlang',
+  '.hrl': 'erlang',
+  '.hs': 'haskell',
+  '.jl': 'julia',
+  '.ml': 'ocaml',
+  '.mli': 'ocaml',
+  '.clj': 'clojure',
+  '.cljs': 'clojure',
+  '.cljc': 'clojure',
+  '.groovy': 'groovy',
+  '.gradle': 'groovy',
+  '.ps1': 'powershell',
+  '.psm1': 'powershell',
+  '.pl': 'perl',
+  '.pm': 'perl',
+  '.r': 'r',
+  '.sol': 'solidity',
+  '.zig': 'zig',
+  '.vue': 'vue',
+  '.svelte': 'svelte',
+  '.tf': 'terraform',
+  '.sql': 'sql',
+  '.proto': 'protobuf',
+  '.graphql': 'graphql',
+  '.gql': 'graphql',
+  '.cmake': 'cmake',
+  '.nix': 'nix',
+  '.bzl': 'starlark',
+};
+
+/**
+ * Language string for a file path or a bare extension, `'unknown'` when the
+ * extension is unmapped. A free function rather than a method because callers
+ * that never scan anything (AST grammar resolution, server-side symbol
+ * preference) need the same answer without constructing a scanner.
+ */
+export function languageForExtension(filePathOrExt: string): string {
+  // `extname` FIRST. A dotfile that carries a real extension ('.eslintrc.js',
+  // '.mocharc.cjs', '.prettierrc.ts') starts with '.' and contains no '/', so
+  // a bare-extension-first reading swallowed the whole name and answered
+  // 'unknown' — while the same file spelled 'src/.eslintrc.js' answered
+  // 'javascript'. Those extensions are allowlisted, so the files are indexed
+  // and reach the server with exactly the root-relative spelling that failed.
+  const fromPath = EXTENSION_LANGUAGES[extname(filePathOrExt).toLowerCase()];
+  if (fromPath !== undefined) return fromPath;
+  // Fallback: the argument IS the extension ('.dart'), for which `extname`
+  // returns ''. Every key contains a leading dot and no separator, so a real
+  // path can never collide here.
+  return EXTENSION_LANGUAGES[filePathOrExt.toLowerCase()] ?? 'unknown';
+}
+
+/**
+ * THE definition of "this path is a SQL migration the schema_columns authority
+ * collects verbatim". One predicate, referenced by both halves of the deal —
+ * `AUTHORITY_FILE_MATCHERS` (collect it) and `isAuthorityOnlyPath` (therefore
+ * do not chunk it) — because two hand-written regexes drifted once already and
+ * the failure is silent in both directions.
+ *
+ * Root-anchored and CASE-SENSITIVE on purpose: it mirrors the server-side
+ * extractor, which does `listFiles('migrations/')` (LIKE 'migrations/%') then
+ * `endsWith('.sql')`, both case-sensitive. `Migrations/001.sql` (the .NET/EF
+ * Core spelling) and `migrations/002.SQL` are NOT collected, so they must not
+ * be suppressed from chunking either — that would index them nowhere. Same
+ * reason a nested `packages/x/migrations/y.sql` is left alone.
+ */
+const isMigrationSqlAuthorityPath = (posixRelPath: string): boolean =>
+  /^migrations\/.*\.sql$/.test(posixRelPath);
+
+/**
+ * Predicates for paths whose verbatim content is ALREADY shipped by
+ * `collectAuthorityFiles` and that carry no additional value as embedded code
+ * chunks. Checked at the extension gate rather than in `shouldIgnore`, because
+ * `shouldIgnore` also guards the authority walk and must keep letting these
+ * through.
+ *
+ * `migrations/**.sql` is the live case: adding `.sql` to the allowlist without
+ * this exclusion would dual-collect every migration — once verbatim, once
+ * chunked and embedded. On this repo alone that is 208 files of append-only
+ * DDL (140 forward, the rest rollback/manual), embedded to answer questions
+ * the authority path already answers exactly.
+ *
+ * INVARIANT: every predicate here must also appear in
+ * `AUTHORITY_FILE_MATCHERS`, so no path can be excluded from chunking unless
+ * the authority path definitely collects it.
+ */
+const AUTHORITY_ONLY_PATH_PREDICATES: readonly ((posixRelPath: string) => boolean)[] = [
+  isMigrationSqlAuthorityPath,
+];
+
+/**
+ * True when `relPath` is collected verbatim as authority content and must not
+ * additionally be chunked. Accepts OS-native or POSIX separators.
+ */
+export function isAuthorityOnlyPath(relPath: string): boolean {
+  if (!relPath) return false;
+  const posix = relPath.split(sep).join('/');
+  return AUTHORITY_ONLY_PATH_PREDICATES.some((matches) => matches(posix));
+}
+
 const DEFAULT_OPTIONS: Required<ScanOptions> = {
   maxChunkSize: 8000, // ~2000 tokens
   minChunkSize: 100,
   ignorePatterns: [...BUILT_IN_IGNORE_DIRS, ...BUILT_IN_IGNORE_FILE_PATTERNS],
-  includeExtensions: [
-    '.ts',
-    '.tsx',
-    '.js',
-    '.jsx',
-    '.py',
-    '.rs',
-    '.go',
-    '.java',
-    '.c',
-    '.cpp',
-    '.h',
-    '.cs',
-    '.rb',
-    '.php',
-    '.swift',
-    '.kt',
-    // Shell and Lua are the working languages of whole real projects —
-    // deployment tooling, container entrypoints, imapfilter/nginx/redis
-    // configuration. Omitting them meant such a project indexed ZERO code and
-    // code_search could not answer anything about it, while reporting that as
-    // "no indexed matches" rather than as missing coverage.
-    '.sh',
-    '.bash',
-    '.zsh',
-    '.lua',
-    '.md',
-  ],
+  includeExtensions: [...DEFAULT_INCLUDE_EXTENSIONS],
+  maxAstParseBytes: MAX_AST_PARSE_BYTES,
 };
 
 /**
@@ -281,8 +538,10 @@ export const AUTHORITY_FILE_MATCHERS: Array<(relPath: string) => boolean> = [
   // SQL migrations: the schema_columns authority extractor reads every .sql
   // under `migrations/` (listFiles('migrations/') -> LIKE 'migrations/%' then
   // .endsWith('.sql')). Without collecting these, that authority is empty and
-  // every schema_table_enumeration claim falls to unverifiable.
-  (p) => /^migrations\/.*\.sql$/.test(p),
+  // every schema_table_enumeration claim falls to unverifiable. Shared with
+  // `AUTHORITY_ONLY_PATH_PREDICATES` by reference, not by a copied regex, so
+  // collection and the chunking exclusion cannot disagree.
+  isMigrationSqlAuthorityPath,
 ];
 
 /**
@@ -369,11 +628,96 @@ interface IgnoreLayer {
 }
 type IgnoreStack = IgnoreLayer[];
 
+/**
+ * Latch for `logAstCapabilityOnce`. A promise, not a boolean: two concurrent
+ * callers must both wait on the same report rather than the second returning
+ * before the first has logged.
+ */
+let astCapabilityLog: Promise<void> | null = null;
+
+/**
+ * `reason:grammar` pairs already warned about, for the reasons that are
+ * process-global rather than file-specific.
+ *
+ * Only `grammar_unavailable` qualifies: `loadGrammar` caches its failure, so the
+ * answer is identical for every file of that language and warning per file
+ * printed 1,037 lines in one scan of this repo. `file_too_large` and
+ * `parse_failed` are properties of one file and stay per file.
+ */
+const reportedGrammarFallbacks = new Set<string>();
+
+/**
+ * Log which grammars this install ships — ONCE per process, at daemon start,
+ * never per file.
+ *
+ * A missing grammar silently degrades every file of that language to the
+ * heuristic chunker. Per-file warnings would say so 30,000 times and drown the
+ * log; saying nothing is how a half-broken install looks healthy. One startup
+ * line naming the vendored grammars, and a WARNING when an artifact is missing,
+ * is the whole contract.
+ *
+ * Deliberately a `statSync` of the artifacts (`astArtifactReport`) and not a
+ * load of them (`astCapabilityReport`). Loading all 18 to print this line costs
+ * ~690 ms and ~75 MB of RSS that is never returned — web-tree-sitter exposes no
+ * `Language.delete` — which is a permanent tax on a daemon watching a pure
+ * TypeScript repo, paid to pre-answer a question about seventeen languages it
+ * will never see. Grammars load lazily instead, on the first file of a language,
+ * and the two failure modes only a load can detect (`wasm_load_failed`,
+ * `query_compile_failed`) are warned there, once per language, by the
+ * `grammar_unavailable` branch in `chunkFile`.
+ *
+ * Cheap to call repeatedly: this latches, and the report instantiates nothing.
+ */
+export function logAstCapabilityOnce(): Promise<void> {
+  astCapabilityLog ??= (async () => {
+    const report = astArtifactReport();
+    const detail = {
+      grammars: report.vendored.length,
+      languages: report.vendored.join(' '),
+    };
+    if (report.missing.length > 0) {
+      logWarn('AST chunking: some vendored grammar artifacts are missing', {
+        ...detail,
+        missing: report.missing.map((m) => `${m.id}(${m.reason}: ${m.detail})`).join('; '),
+      });
+    } else {
+      logInfo('AST chunking ready (grammars load lazily, per language)', detail);
+    }
+  })();
+  return astCapabilityLog;
+}
+
 export class CodeScanner {
   private options: Required<ScanOptions>;
 
+  /**
+   * `includeExtensions` folded to lowercase for matching. The gate used to
+   * compare `extname()` verbatim while `detectLanguage` lowercased, so a
+   * project spelling its files the canonical way — `.R` for R, `.SQL`/`.PS1`
+   * on Windows, `.C`/`.H` in older C trees — indexed zero of them and nothing
+   * said why.
+   */
+  private readonly includeExtensionSet: ReadonlySet<string>;
+
   constructor(options: ScanOptions = {}) {
     this.options = { ...DEFAULT_OPTIONS, ...options };
+    this.includeExtensionSet = new Set(
+      this.options.includeExtensions.map((ext) => ext.toLowerCase())
+    );
+  }
+
+  /**
+   * The chunkable-file gate, shared by every walker and by the explicit
+   * file-list path so all three agree on what exists. `relPath` is the path
+   * relative to the scan root (OS-native separators accepted).
+   */
+  private isChunkable(absOrRelPath: string, relPath: string): boolean {
+    if (!this.includeExtensionSet.has(extname(absOrRelPath).toLowerCase())) return false;
+    if (isAuthorityOnlyPath(relPath)) {
+      logDebug('Skipping chunking for authority-collected path', { relPath });
+      return false;
+    }
+    return true;
   }
 
   /**
@@ -625,7 +969,7 @@ export class CodeScanner {
           await this.traversePaths(fullPath, rootPath, out, depth + 1, walk, localStack);
         } else if (stats.isFile()) {
           if (this.ignoredByStack(relativePath, false, localStack)) continue;
-          if (this.options.includeExtensions.includes(extname(fullPath))) {
+          if (this.isChunkable(fullPath, relativePath)) {
             out.push(relativePath);
           }
         }
@@ -668,8 +1012,7 @@ export class CodeScanner {
           continue;
         }
 
-        const ext = extname(filePath);
-        if (this.options.includeExtensions.includes(ext)) {
+        if (this.isChunkable(filePath, fileRel)) {
           const fileChunks = await this.parseFile(filePath, rootPath || filePath);
           chunks.push(...fileChunks);
         }
@@ -775,8 +1118,7 @@ export class CodeScanner {
           await this.traverseDirectory(fullPath, rootPath, chunks, depth + 1, walk, localStack);
         } else if (stats.isFile()) {
           if (this.ignoredByStack(relativePath, false, localStack)) continue;
-          const ext = extname(fullPath);
-          if (this.options.includeExtensions.includes(ext)) {
+          if (this.isChunkable(fullPath, relativePath)) {
             const fileChunks = await this.parseFile(fullPath, rootPath);
             chunks.push(...fileChunks);
           }
@@ -876,14 +1218,131 @@ export class CodeScanner {
         return this.chunkMarkdown(content, relativePath, stats.size);
       }
 
-      const structuredChunks = this.extractStructuredChunks(content, language);
-
       const fileMetadata = {
         fileName: filePath.split('/').pop() || '',
         extension: extname(filePath),
         size: stats.size,
       };
 
+      // AST first for the languages with a vendored grammar. `null` means "no
+      // grammar for this" — the honest majority of the allowlist — and the
+      // heuristic path below takes over with no log, because that is not a
+      // degradation.
+      const astLanguage = resolveAstLanguage(fileMetadata.extension, language);
+      if (astLanguage) {
+        const ast = await chunkWithAst(content, astLanguage, {
+          maxParseBytes: this.options.maxAstParseBytes,
+        });
+        if ('chunks' in ast) {
+          // A definition the chunker could not place reaches no chunk, so its
+          // name reaches no index and a citation to it resolves as
+          // `unresolved_symbol`. It cannot be emitted (two chunks over one line
+          // range collide on the `filePath:startLine-endLine` staleness key), but
+          // a scan that quietly loses symbols must not look like a healthy one.
+          if (ast.droppedDefinitions > 0) {
+            logWarn('AST chunker dropped definitions that share a line range', {
+              filePath: relativePath,
+              grammar: astLanguage,
+              droppedDefinitions: ast.droppedDefinitions,
+            });
+          }
+          // `errorNodes` is handed back precisely so this degradation is not
+          // silent, and discarding it made it silent. One ERROR node can swallow
+          // the rest of a file — `typeof import(...)` in a type argument does
+          // exactly that to tree-sitter-typescript — after which no definition is
+          // captured and the file arrives as one unnamed whole-file chunk. Naming
+          // the file, the error count and how many definitions survived is what
+          // makes that visible in a scan log.
+          //
+          // Deliberately NOT a fallback to the heuristic path: measured on the
+          // real construct (tests/AgentInjectionDeliveryWiring.test.ts), the
+          // heuristic path names nothing either (its name pattern needs a
+          // leading `function`/`class`/`const`) AND covers less — it emitted 3
+          // chunks starting at line 18 and dropped lines 1-17, because the
+          // uncovered-region supplement only fires above 50 lines. Trading
+          // total coverage for no gain is worse than one honest blob, so the
+          // AST result stands and the log says so.
+          if (ast.errorNodes > 0) {
+            logWarn('AST parse errors; extents and symbol names degraded for this file', {
+              filePath: relativePath,
+              grammar: astLanguage,
+              errorNodes: ast.errorNodes,
+              definitions: ast.chunks.filter((chunk) => chunk.symbolKind).length,
+              chunks: ast.chunks.length,
+            });
+          }
+          // The wire cap (`scanChunkSchema` allows 500,000 chars) and the
+          // embedding cap (8191 tokens per input) are both downstream of
+          // `maxChunkSize`, and the AST chunker only bounds its uncovered spans:
+          // a definition chunk is whatever the definition is. `boundAstChunks`
+          // makes the cap unreachable by construction, or answers null when the
+          // file cannot be bounded on line boundaries at all.
+          const bounded = this.boundAstChunks(ast.chunks);
+          if (bounded) {
+            // Zero chunks here means the file has no non-whitespace line — the AST
+            // chunker covers every other line by construction — so returning
+            // nothing is the correct answer, not a lost file.
+            return bounded.map(({ symbolName, symbolKind, signature, ...chunk }) => ({
+              ...chunk,
+              filePath: relativePath,
+              language,
+              contentHash: this.hash(chunk.content),
+              metadata: {
+                ...fileMetadata,
+                ...(signature ? { signature } : {}),
+                ...(symbolName ? { symbolName } : {}),
+                ...(symbolKind ? { symbolKind } : {}),
+              },
+            }));
+          }
+          // One line longer than `maxChunkSize` — a generated `*_pb2.py` carries
+          // the whole serialized descriptor on one. Splitting it would have to
+          // cut mid-line, and two chunks over one line range collide on the
+          // `filePath:startLine-endLine` staleness key, so the whole file goes to
+          // the heuristic chunker, which force-splits long lines by character
+          // count. Loud, because a file that loses its symbol names must not look
+          // like a healthy one.
+          logWarn('AST chunk exceeds maxChunkSize on a single line; heuristic chunking instead', {
+            filePath: relativePath,
+            grammar: astLanguage,
+            maxChunkSize: this.options.maxChunkSize,
+            longestLine: content
+              .split('\n')
+              .reduce((longest, line) => Math.max(longest, line.length), 0),
+          });
+        } else if (ast.unsupported === 'grammar_unavailable') {
+          // Process-global, not per file: `loadGrammar` caches its failures, so
+          // every file of this language degrades for the same reason. Warned per
+          // file, this printed 1,037 identical lines in one scan of this repo —
+          // the log flood `logAstCapabilityOnce` exists to avoid. Once per
+          // (reason, grammar); the file-specific reasons below stay per file.
+          const key = `${ast.unsupported}:${astLanguage}`;
+          if (!reportedGrammarFallbacks.has(key)) {
+            reportedGrammarFallbacks.add(key);
+            logWarn('AST chunking unavailable for a whole language; heuristic chunking instead', {
+              grammar: astLanguage,
+              reason: ast.unsupported,
+              detail: ast.detail,
+              firstFile: relativePath,
+              note: 'logged once per grammar per process; every file of this language degrades',
+            });
+          }
+        } else {
+          // Not a silent no-op: name the file AND the reason, then degrade to the
+          // heuristic path. A degraded scan indistinguishable from a healthy one
+          // is the defect class this routing exists to avoid — and degrading to
+          // zero chunks would delete the file from the index instead.
+          logWarn('AST chunking unavailable; falling back to heuristic chunking', {
+            filePath: relativePath,
+            grammar: astLanguage,
+            reason: ast.unsupported,
+            detail: ast.detail,
+          });
+        }
+      }
+
+      const structuredChunks = this.extractStructuredChunks(content, language);
+
       if (structuredChunks.length > 0) {
         const mapped = structuredChunks.map(({ signature, symbolName, ...chunk }) => ({
           ...chunk,
@@ -933,35 +1392,90 @@ export class CodeScanner {
     }
   }
 
+  /**
+   * Hold every AST chunk at or under `maxChunkSize`, or answer `null` when this
+   * file cannot be held there on line boundaries.
+   *
+   * The AST chunker bounds only the spans it invents (`MAX_RAW_SPAN_CHARS`); a
+   * chunk that came from a definition is exactly as big as the definition, and a
+   * span that is one enormous line is left whole on purpose. Both used to reach
+   * the wire verbatim, where `scanChunkSchema` caps content at 500,000 chars and
+   * rejects the WHOLE push if any chunk is over — so one generated file stopped
+   * all scanning for that push — and where anything past 8191 tokens is only
+   * partially embedded.
+   *
+   * The invariants, all of which the split preserves:
+   * - content stays verbatim contiguous source: `lines[startLine-1..endLine-1]`
+   *   joined by '\n', never a join of disjoint regions and never an elision;
+   * - total coverage is unchanged: the pieces tile the original range, in order,
+   *   with no gap and no overlap;
+   * - the definition's identity rides on the piece that carries its signature —
+   *   the first one — and the continuations are plain `raw` spans, because a
+   *   continuation is not the definition and must not claim its name.
+   *
+   * `null` (the caller degrades the whole file to the heuristic chunker) is
+   * reserved for the one shape a line-boundary split cannot fix: a single line
+   * longer than the cap, as every generated `*_pb2.py` has. Cutting mid-line
+   * would give two chunks the same `filePath:startLine-endLine` staleness key,
+   * which is the collision the chunker refuses everywhere else.
+   */
+  private boundAstChunks(chunks: readonly AstChunk[]): AstChunk[] | null {
+    const max = this.options.maxChunkSize;
+    if (chunks.every((chunk) => chunk.content.length <= max)) return [...chunks];
+
+    const bounded: AstChunk[] = [];
+    for (const chunk of chunks) {
+      if (chunk.content.length <= max) {
+        bounded.push(chunk);
+        continue;
+      }
+      // `content` is exactly `startLine..endLine` joined by '\n', so this split
+      // recovers the file's own lines for that range.
+      const lines = chunk.content.split('\n');
+      if (lines.some((line) => line.length > max)) return null;
+
+      /** `from`/`to` are 0-based offsets into `lines`, inclusive. */
+      const emit = (from: number, to: number): void => {
+        const content = lines.slice(from, to + 1).join('\n');
+        // A piece of nothing but blank lines is owed no chunk (the chunker's own
+        // rule) and `scanChunkSchema` requires content.min(1) anyway.
+        if (content.trim() === '') return;
+        const isFirst = from === 0;
+        bounded.push({
+          content,
+          startLine: chunk.startLine + from,
+          endLine: chunk.startLine + to,
+          chunkType: isFirst ? chunk.chunkType : 'raw',
+          ...(isFirst && chunk.symbolName ? { symbolName: chunk.symbolName } : {}),
+          ...(isFirst && chunk.symbolKind ? { symbolKind: chunk.symbolKind } : {}),
+          ...(isFirst && chunk.signature ? { signature: chunk.signature } : {}),
+        });
+      };
+
+      let pieceStart = 0;
+      let pieceLength = 0;
+      for (let i = 0; i < lines.length; i++) {
+        const lineLength = (lines[i] ?? '').length;
+        // The '\n' the join will put back is part of what has to fit.
+        const withLine = i === pieceStart ? lineLength : pieceLength + 1 + lineLength;
+        if (i > pieceStart && withLine > max) {
+          emit(pieceStart, i - 1);
+          pieceStart = i;
+          pieceLength = lineLength;
+        } else {
+          pieceLength = withLine;
+        }
+      }
+      emit(pieceStart, lines.length - 1);
+    }
+    return bounded;
+  }
+
   /**
    * Detect language from file extension
    */
   private detectLanguage(filePath: string): string {
-    const ext = extname(filePath).toLowerCase();
-    const langMap: Record<string, string> = {
-      '.ts': 'typescript',
-      '.tsx': 'typescript',
-      '.js': 'javascript',
-      '.jsx': 'javascript',
-      '.py': 'python',
-      '.rs': 'rust',
-      '.go': 'go',
-      '.java': 'java',
-      '.c': 'c',
-      '.cpp': 'cpp',
-      '.h': 'c',
-      '.cs': 'csharp',
-      '.rb': 'ruby',
-      '.php': 'php',
-      '.swift': 'swift',
-      '.kt': 'kotlin',
-      '.sh': 'shell',
-      '.bash': 'shell',
-      '.zsh': 'shell',
-      '.lua': 'lua',
-      '.md': 'markdown',
-    };
-    return langMap[ext] || 'unknown';
+    return languageForExtension(filePath);
   }
 
   /**